diff --git a/.agents/skills b/.agents/skills new file mode 120000 index 0000000000000..d17c931ce114a --- /dev/null +++ b/.agents/skills @@ -0,0 +1 @@ +../doc/src/devdocs/agents/skills \ No newline at end of file diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 0000000000000..d17c931ce114a --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../doc/src/devdocs/agents/skills \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3df89f0f4d096..12aba7063cb9c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,8 @@ CODEOWNERS @JuliaLang/github-actions /.github/workflows/rerun_failed.yml @DilumAluthge /.github/workflows/statuses.yml @DilumAluthge /.github/workflows/PrAssignee.yml @LilithHafner @DilumAluthge +/.github/workflows/backport-label-audit.yml @IanButterworth @DilumAluthge +/.github/workflows/backport-label-cleanup.yml @IanButterworth @DilumAluthge /base/special/ @oscardssmith /base/sort.jl @LilithHafner /test/sorting.jl @LilithHafner @@ -12,3 +14,5 @@ CODEOWNERS @JuliaLang/github-actions /base/binaryplatforms.jl @giordano /src/julia_gcext.h @fingolfin /test/gcext/gcext.c @fingolfin +/src/gc-* @udesou @qinsoon @d-netto +/stdlib/Markdown @fingolfin diff --git a/.github/_typos.toml b/.github/_typos.toml new file mode 100644 index 0000000000000..699862bf71341 --- /dev/null +++ b/.github/_typos.toml @@ -0,0 +1,4 @@ +# Configuration for the `typos` spell checker (.github/workflows/Typos.yml) +[default.extend-words] +# allowlist for typo check +invokee = "invokee" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2ad7fdc1efa0a..db63d07c0e1a5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,7 @@ version: 2 updates: - package-ecosystem: "github-actions" + target-branch: "master" directory: "/" schedule: interval: "monthly" @@ -8,4 +9,57 @@ updates: labels: - "dependencies" - "github-actions" - - "domain:ci" + - "ci" + # Dependabot doesn't yet support wildcards in `target-branch`, so we have to list each release branch manually. + # https://github.com/dependabot/dependabot-core/issues/6890 + # https://github.com/dependabot/dependabot-core/issues/2511 + # + # Docs for prefix: https://docs.github.com/en/code-security/reference/supply-chain-security/dependabot-options-reference#prefix + - package-ecosystem: "github-actions" + target-branch: "release-1.10" + commit-message: + prefix: "[release-1.10] " + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 100 + labels: + - "dependencies" + - "github-actions" + - "ci" + - package-ecosystem: "github-actions" + target-branch: "release-1.11" + commit-message: + prefix: "[release-1.11] " + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 100 + labels: + - "dependencies" + - "github-actions" + - "ci" + - package-ecosystem: "github-actions" + target-branch: "release-1.12" + commit-message: + prefix: "[release-1.12] " + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 100 + labels: + - "dependencies" + - "github-actions" + - "ci" + - package-ecosystem: "github-actions" + target-branch: "release-1.13" + commit-message: + prefix: "[release-1.13] " + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 100 + labels: + - "dependencies" + - "github-actions" + - "ci" diff --git a/.github/workflows/LabelCheck.yml b/.github/workflows/LabelCheck.yml index c966e478e3fe0..5937e5e28f017 100644 --- a/.github/workflows/LabelCheck.yml +++ b/.github/workflows/LabelCheck.yml @@ -5,15 +5,19 @@ permissions: on: pull_request: types: [labeled, unlabeled, opened, reopened, edited, synchronize] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: enforce-labels: name: Check for blocking labels runs-on: ubuntu-latest timeout-minutes: 2 steps: - - uses: yogevbd/enforce-label-action@a3c219da6b8fa73f6ba62b68ff09c469b3a1c024 # 2.2.2 + - uses: mheap/github-action-required-labels@23e10fde7e062233401931a0eece796cd9bf3177 # v5.6.0 with: - # REQUIRED_LABELS_ANY: "bug,enhancement,skip-changelog" - # REQUIRED_LABELS_ANY_DESCRIPTION: "Select at least one label ['bug','enhancement','skip-changelog']" - BANNED_LABELS: "needs docs,needs compat annotation,needs more info,needs nanosoldier run,needs news,needs pkgeval,needs tests,needs decision,DO NOT MERGE,status:DO NOT MERGE" - BANNED_LABELS_DESCRIPTION: "A PR should not be merged with `needs *` or `status:DO NOT MERGE` labels" + mode: exactly + count: 0 + labels: "needs docs,needs compat annotation,needs more info,needs nanosoldier run,needs news,needs pkgeval,needs tests,needs decision,DO NOT MERGE,status:DO NOT MERGE" + add_comment: false + message: "A PR should not be merged with `needs *` or `status:DO NOT MERGE` labels" diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml deleted file mode 100644 index 074e7a5528ce8..0000000000000 --- a/.github/workflows/PrAssignee.yml +++ /dev/null @@ -1,210 +0,0 @@ -name: PR Assignee -on: - # Important security note: Do NOT use `actions/checkout` - # or any other method for checking out the pull request's source code. - # This is because the pull request's source code is untrusted, but the - # GITHUB_TOKEN has write permissions (because of the `on: pull_request_target` event). - # - # Quoting from the GitHub Docs: - # > For workflows that are triggered by the pull_request_target event, the GITHUB_TOKEN is granted - # > read/write repository permission unless the permissions key is specified and the workflow can access secrets, - # > even when it is triggered from a fork. - # > - # > Although the workflow runs in the context of the base of the pull request, - # > you should make sure that you do not check out, build, or run untrusted code from the pull request with this event. - # - # Source: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target - # - # See also: https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ - pull_request_target: - types: [opened, reopened, ready_for_review] - -# Permissions for the `GITHUB_TOKEN`: -permissions: - pull-requests: write # Needed in order to assign a user as the PR assignee - -jobs: - pr-assignee: - runs-on: ubuntu-latest - if: ${{ github.event.pull_request.draft != true }} - steps: - # Important security note: As discussed above, do NOT use `actions/checkout` - # or any other method for checking out the pull request's source code. - # This is because the pull request's source code is untrusted, but the - # GITHUB_TOKEN has write permissions (because of the `on: pull_request_target` event). - - name: Add Assignee - # We pin all third-party actions to a full length commit SHA - # https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-third-party-actions - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - with: - retries: 5 # retry GitHub API requests up to 5 times, with exponential backoff - retry-exempt-status-codes: 404 - # Don't retry 404 because we will hit a 404 when the PR author is a committer. - # This 404 is normal and expected. - # Do retry 400 and other 4xx errors because github sometimes (erroneously) - # returns a 4xx error code due to server errors. - script: | - const oldPrAssignees = context.payload.pull_request.assignees - .map(obj => obj.login) - console.log('oldPrAssignees: ', oldPrAssignees); - const prAuthor = context.payload.pull_request.user.login; - - // Check if the PR is opened by a collaborator on the repo, aka someone with write (commit) permissions or higher. - const relevantPerms = [ - // 'triage', // Uncomment this line if you don't want PRs from triagers to get auto-assignees. - 'push', - 'maintain', - 'admin', - ] - const allCollaboratorsNestedPromises = relevantPerms.map( - (perm) => github.paginate( - // We use the `/repos/{owner}/{repo}/collaborators` endpoint to avoid needing org scope permissions: - '/repos/{owner}/{repo}/collaborators', - { - owner: context.repo.owner, - repo: context.repo.repo, - per_page: 100, - permission: perm, - }, - (response) => response.data.map((collaboratorInfo) => collaboratorInfo.login), - ) - ) - const allCollaboratorsNested = await Promise.all(allCollaboratorsNestedPromises); - const allCollaboratorsFlattened = allCollaboratorsNested.flat(); - - // Skip BumpStdlibs.jl PRs - allCollaboratorsFlattened.push('DilumAluthgeBot'); - // Skip Dependabot PRs - allCollaboratorsFlattened.push('dependabot'); - - const isCollaborator = allCollaboratorsFlattened.includes(prAuthor); - - console.log('prAuthor: ', prAuthor); - console.log('isCollaborator: ', isCollaborator); - - // Load the list of assignable reviewers from the JuliaLang/pr-assignment repo at: - // https://github.com/JuliaLang/pr-assignment/blob/main/users.txt - // - // NOTE to JuliaLang committers: If you want to be assigned to new PRs, please add your - // GitHub username to that file. - - // Load file contents - const { data: fileContentsObj } = await github.rest.repos.getContent({ - owner: 'JuliaLang', - repo: 'pr-assignment', - path: 'users.txt', - ref: 'main', - }); - - const fileContentsBufferObj = Buffer.from(fileContentsObj.content, "base64"); - const fileContentsText = fileContentsBufferObj.toString("utf8"); - - // Find lines that match the following regex, and extract the usernames: - const regex = /^@([a-zA-Z0-9\-]+)(\s*?)?(#[\S]*?)?$/; - const assigneeCandidates = fileContentsText - .split('\n') - .map(line => line.trim()) - .map(line => line.match(regex)) - .filter(match => match !== null) - .map(match => match[1]); - - console.log('assigneeCandidates: ', assigneeCandidates); - if (assigneeCandidates.length < 1) { - const msg = 'ERROR: Could not find any assigneeCandidates'; - console.error(msg); - throw new Error(msg); - } - - if (oldPrAssignees.length >= 1) { - console.log('Skipping this PR, because it already has at least one assignee'); - return; - } - - - const RUNNER_DEBUG_original = process.env.RUNNER_DEBUG; - console.log('RUNNER_DEBUG_original: ', RUNNER_DEBUG_original); - if (RUNNER_DEBUG_original === undefined) { - var thisIsActionsRunnerDebugMode = false; - } else { - const RUNNER_DEBUG_trimmed = RUNNER_DEBUG_original.trim().toLowerCase() - if (RUNNER_DEBUG_trimmed.length < 1) { - var thisIsActionsRunnerDebugMode = false; - } else { - var thisIsActionsRunnerDebugMode = (RUNNER_DEBUG_trimmed == 'true') || (RUNNER_DEBUG_trimmed == '1'); - } - } - console.log('thisIsActionsRunnerDebugMode: ', thisIsActionsRunnerDebugMode); - - if (isCollaborator == true) { - - if (thisIsActionsRunnerDebugMode) { - // The PR author is a committer - // But thisIsActionsRunnerDebugMode is true, so we proceed to still run the rest of the script - console.log('PR is authored by JuliaLang committer, but thisIsActionsRunnerDebugMode is true, so we will still run the rest of the script: ', prAuthor); - } else { - // The PR author is a committer, so we skip assigning them - console.log('Skipping PR authored by JuliaLang committer: ', prAuthor); - console.log('Note: If you want to run the full script (even though the PR author is a committer), simply re-run this job with Actions debug logging enabled'); - return; - } - } - - var weDidEncounterError = false; - - // Assign random committer - const selectedAssignee = assigneeCandidates[Math.floor(Math.random()*assigneeCandidates.length)] - console.log('selectedAssignee: ', selectedAssignee); - console.log(`Attempting to assign @${selectedAssignee} to this PR...`); - await github.rest.issues.addAssignees({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - assignees: selectedAssignee, - }); - - // The following is commented out because the label only makes sense in the presence of a larger state machine - // // Add the "pr review" label - // const prReviewLabel = 'status: waiting for PR reviewer'; - // console.log('Attempting to add prReviewLabel to this PR...'); - // await github.rest.issues.addLabels({ - // owner: context.repo.owner, - // repo: context.repo.repo, - // issue_number: context.payload.pull_request.number, - // labels: [prReviewLabel], - // }); - - // Now get the updated PR info, and see if we were successful: - const updatedPrData = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - }); - const newPrAssignees = updatedPrData - .data - .assignees - .map(element => element.login) - console.log('newPrAssignees: ', newPrAssignees); - if (newPrAssignees.includes(selectedAssignee)) { - console.log(`Successfully assigned @${selectedAssignee}`); - } else { - weDidEncounterError = true; - console.log(`ERROR: Failed to assign @${selectedAssignee}`); - } - // const newPrLabels = updatedPrData - // .data - // .labels - // .map(element => element.name) - // console.log('newPrLabels: ', newPrLabels); - // if (newPrLabels.includes(prReviewLabel)) { - // console.log('Successfully added prReviewLabel'); - // } else { - // weDidEncounterError = true; - // console.log('ERROR: Failed to add add prReviewLabel'); - // } - - // Exit with error if any problems were encountered earlier - if (weDidEncounterError) { - const msg = 'ERROR: Encountered at least one problem while running the script'; - console.error(msg); - throw new Error(msg); - } diff --git a/.github/workflows/Typos.yml b/.github/workflows/Typos.yml index 4dcfdcf0095b9..7d7b55a27b880 100644 --- a/.github/workflows/Typos.yml +++ b/.github/workflows/Typos.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Check spelling with typos @@ -40,12 +40,12 @@ jobs: | tar -xz -C "${{ runner.temp }}/typos" ./typos "${{ runner.temp }}/typos/typos" --version - echo -n $NEW_FILES | xargs "${{ runner.temp }}/typos/typos" --format json >> ${{ runner.temp }}/new_typos.jsonl || true + echo -n $NEW_FILES | xargs "${{ runner.temp }}/typos/typos" --config .github/_typos.toml --format json >> ${{ runner.temp }}/new_typos.jsonl || true git checkout FETCH_HEAD -- $OLD_FILES if [ -z "$OLD_FILES" ]; then touch "${{ runner.temp }}/old_typos.jsonl" # No old files, so no old typos. else - echo -n $OLD_FILES | xargs "${{ runner.temp }}/typos/typos" --format json >> ${{ runner.temp }}/old_typos.jsonl || true + echo -n $OLD_FILES | xargs "${{ runner.temp }}/typos/typos" --config .github/_typos.toml --format json >> ${{ runner.temp }}/old_typos.jsonl || true fi @@ -65,6 +65,6 @@ jobs: if len(new["typo"]) > 6: # Short typos might be false positives. Long are probably real. clean = False print("::warning file={},line={},col={}::perhaps \"{}\" should be \"{}\".".format( - new["path"], new["line_num"], new["byte_offset"], + new["path"], new.get("line_num", 1), new["byte_offset"], new["typo"], " or ".join(new["corrections"]))) sys.exit(1 if not clean else 0)' "${{ runner.temp }}/old_typos.jsonl" "${{ runner.temp }}/new_typos.jsonl" diff --git a/.github/workflows/Whitespace.yml b/.github/workflows/Whitespace.yml index 7414365322292..10c468f1fe3ad 100644 --- a/.github/workflows/Whitespace.yml +++ b/.github/workflows/Whitespace.yml @@ -15,12 +15,12 @@ jobs: timeout-minutes: 2 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1 + - uses: julia-actions/setup-julia@fa02766e078afaaf09b14210362cee14137e6a32 # v3.0.2 with: - version: '1' + version: '1.11.6' - name: Check whitespace run: | contrib/check-whitespace.jl diff --git a/.github/workflows/backport-label-audit.yml b/.github/workflows/backport-label-audit.yml new file mode 100644 index 0000000000000..4d4fbbaf7a318 --- /dev/null +++ b/.github/workflows/backport-label-audit.yml @@ -0,0 +1,29 @@ +name: Backport Label Audit +# Run this workflow manually to audit backport labels on pull requests +# and remove labels from PRs that have already been backported. +# Optionally specify a release version to limit the audit to that version + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version to audit (e.g., 1.13). Leave empty to audit all versions.' + required: false + type: string + dry_run: + description: 'Dry run (only report, do not modify)' + required: true + type: choice + options: + - 'true' + - 'false' + default: 'true' + +jobs: + audit: + uses: JuliaLang/backporter-github-actions-workflows/.github/workflows/audit.yml@f0cf777a492c5ff31196822a7d643a17ee0efbd1 # main + permissions: + pull-requests: write # necessary to be able to remove labels from PRs + with: + version: ${{ inputs.version }} + dry_run: ${{ inputs.dry_run }} diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml index 3e481f18e6f75..58730254d64e0 100644 --- a/.github/workflows/cffconvert.yml +++ b/.github/workflows/cffconvert.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.gitignore b/.gitignore index c4df2542005d4..443011f174e21 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ repo_key # Buildkite: Ignore any agent keys (public or private) we have stored agent_key* +contrib/ci-timing/Manifest.toml diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 0000000000000..dd4cd5eaa65ed --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,20 @@ +{ + "configurations": [ + { + "name": "Julia", + "includePath": [ + "${workspaceFolder}/src/**", + "${workspaceFolder}/usr/include/**" + ], + "cStandard": "c11", + "cppStandard": "c++17", + "compileCommands": [ + "${workspaceFolder}/src/compile_commands.json", + "${workspaceFolder}/src/flisp/compile_commands.json", + "${workspaceFolder}/src/support/compile_commands.json" + ] + } + ], + "version": 4, + "enableConfigurationSquiggles": true +} diff --git a/AGENTS.md b/AGENTS.md index 4dcaa2069fa1f..253a54d09c60c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,12 @@ # Information for AI agents ## Module Organization -- `base/` - Core standard library (loaded at startup) -- `stdlib/` - Standard library packages (can be loaded independently) -- `Compiler/` - Julia compiler as a separate module (can be swapped) -- `src/` - C/C++ runtime and LLVM codegen -- `cli/` - Command-line interface and loader -- `doc/` - Documentation and User Manual +- [`base/`](base/) - Core standard library (loaded at startup) +- [`stdlib/`](stdlib/) - Standard library packages (can be loaded independently) +- [`Compiler/`](Compiler/) - Julia compiler as a separate module (can be swapped) +- [`src/`](src/) - C/C++ runtime and LLVM codegen +- [`cli/`](cli/) - Command-line interface and loader +- [`doc/`](doc/) - Documentation and User Manual ## Running Julia @@ -19,7 +19,12 @@ will not be reflected, unless you use `Revise`. ## For all changes -1. Run `make fix-whitespace` before creating the PR to make sure you're not committing any whitespace errors. +1. Sign off every commit you create with an `Assisted-by: ()` trailer naming both the + tool and the model behind it, for example `Assisted-by: Claude Code (Opus 5)`. Name the model, + not just the harness - it is what tells a later reader what actually produced the work. Do not + use `Co-authored-by:` for tools, and do not list a tool as an author; this trailer replaces any + co-author trailer your harness adds by default. +2. Agents can only open PRs or post comments once the human user gives them explicit permission. ## Building Julia @@ -27,77 +32,64 @@ If you made changes to the runtime (any files in `src/`), you will need to rebui julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes depending on your changes. -After `make` run these static analysis checks: - - `make -C src clang-sa-` (replace `` with the basename of the file you modified) - - `make -C src clang-sagc-` which may require adding JL_GC_PUSH arguments, or JL_GC_PROMISE_ROOTED statements., or require fixing locks. Remember arguments are assumed rooted, so check the callers to make sure that is handled. If the value is being temporarily moved around in a struct or arraylist, `JL_GC_PROMISE_ROOTED(struct->field)` may be needed as a statement (it return void) immediately after reloading the struct before any use of struct. Put the promise as early in the code as is legal. - - `make -C src clang-tidy-` +After modifying any C/C++ file under `src/`, also run the Clang static analysis +checks — see the `c-static-analysis` skill ([`doc/src/devdocs/agents/skills/c-static-analysis/`](doc/src/devdocs/agents/skills/c-static-analysis/SKILL.md)). -## Using Revise - -If you have made changes to files included in the system image (base/ or stdlib/), -and need to run code with these changes included, you can use `Revise`. -To do so, run `using Revise; Revise.track(Base)` (or Revise.track with the stdlib you modified). -The test system supports doing this automatically (see below). - -## Specific instructions for particular changes - -### Doctests - -#### Writing doctests - -If you are asked to write new doctests, first review `doc/src/devdocs/contributing/jldoctests.md` -for best practices. - -#### Verifying doctests -If you have changed any `jldoctest` code blocks you should take -the following steps to verify your work: -- Review `doc/src/devdocs/contributing/jldoctests.md`. In particular, determine - if any of the changed doctests require filters, labels or setup code. -- Run the doctests to verify that your change works: - - To run doctest with the pre-built juliaup: `make -C doc doctest=true revise=true JULIA_EXECUTABLE=$HOME/.juliaup/bin/julia` - - To run doctest with in-trr julia (preferred): `make -C doc doctest=true revise=true`. Do not pass any other options. - - IMPORTANT: The doctests may take up to 15 minutes. Do NOT terminate the doctests before completion. Do NOT use a timeout for doctests. - - If you are ChatGPT, you may have to increase yield_timeout_ms. +### Testing LLVM-related changes -Follow these steps for EVERY change you make in a doctest. +When making changes to LLVM passes or codegen, add `LLVM_ASSERTIONS=1` to `Make.user` to enable +LLVM assertions. This helps catch IR verification errors early: -### Test changes +```bash +echo "LLVM_ASSERTIONS=1" >> Make.user +``` -If you have changed a test (e.g. `foo`), you should run `make test-revise-foo` for the -corresponding test to ensure that the test is still passing with your changes. -- If you are adding a new test, add it to an existing test file. Do not create a new test file unless explicitly instructed. -- Write one comment at the top of the test to explain what is being tested. - Otherwise keep comments minimal. -- Use the environment variable `JULIA_TEST_FAILFAST=1` to make tests fail fast. +To run LLVM pass tests: +```bash +make -C test/llvmpasses .ll +``` -### External dependencies - -When modifying external dependencies (patches in `deps/patches/` or version updates in `deps/`): +## Using Revise -1. Always test builds with `USE_BINARYBUILDER=0` to ensure source builds work correctly -2. For patches to external libraries: - - Verify the patch applies cleanly by running the extraction and patch steps - - Test the full build of the dependency: `make -C deps USE_BINARYBUILDER=0 compile-` - - Prefer using the full upstream commit in `git am` format (e.g., `git format-patch`) which includes proper commit metadata -3. When updating dependency versions, ensure all associated patches still apply +If you have made changes to files included in the system image (base/ or stdlib/), +and need to run code with these changes included, you can use `Revise`. +To do so, run `using Revise; Revise.track(Base)` (or Revise.track with the stdlib you modified). +The test system supports doing this automatically. -### External JLLs +For instance testing Base changes without rebuilding, using failfast, you can run: +``` +JULIA_TEST_FAILFAST=1 ./julia -e 'using Revise; Revise.track(Base); include("test.jl")' +``` -To update a JLL to the latest version: -- Update the version number in the appropriate jll folder -- If the dependencies in the upstream jll changed, update the Project.toml -- Run `make -f contrib/refresh_checksums.mk ` to update the checksums. This may take a few minutes. +## Writing code -### Writing code After writing code, look up the docstring for each function you used. If there are recommendations or additional considerations that apply to these functions, make sure to take them into account. -#### Specific instructions - Do not `ccall` runtime C functions directly if there are existing wrappers for the function. - Do not explicitly add a module prefix if the code you're adding is in the same module. E.g. do not use `Base.` for code in Base unless required. -## Commit message formatting +## Task-specific skills + +Detailed, situational procedures are provided as Agent Skills following the +[agentskills.io](https://agentskills.io) open standard. The canonical location is +`doc/src/devdocs/agents/skills/`; `.agents/skills/` and `.claude/skills/` are +symlinks to that directory so skills-aware agents can discover the skills +automatically. Agents that support the standard load each skill's `description` +automatically and pull in the full `SKILL.md` when relevant; otherwise read the +canonical `SKILL.md` directly. + +- [`doc/src/devdocs/agents/skills/doctests/`](doc/src/devdocs/agents/skills/doctests/SKILL.md) — writing and verifying `jldoctest` code blocks. +- [`doc/src/devdocs/agents/skills/test-changes/`](doc/src/devdocs/agents/skills/test-changes/SKILL.md) — running and updating tests after changing them. +- [`doc/src/devdocs/agents/skills/c-static-analysis/`](doc/src/devdocs/agents/skills/c-static-analysis/SKILL.md) — Clang static analysis and GC-rooting for C/C++ changes under `src/`. +- [`doc/src/devdocs/agents/skills/external-deps/`](doc/src/devdocs/agents/skills/external-deps/SKILL.md) — modifying external dependencies (`deps/`, patches) and JLLs. +- [`doc/src/devdocs/agents/skills/buildkite-logs/`](doc/src/devdocs/agents/skills/buildkite-logs/SKILL.md) — fetching and inspecting Buildkite CI logs without web sign-in. +- [`doc/src/devdocs/agents/skills/ci-timing/`](doc/src/devdocs/agents/skills/ci-timing/SKILL.md) — comparing a PR's Buildkite job durations against recent CI history. +- [`doc/src/devdocs/agents/skills/compiler-jl/`](doc/src/devdocs/agents/skills/compiler-jl/SKILL.md) — developing and testing Compiler.jl. +- [`doc/src/devdocs/agents/skills/julia-syntax-lowering/`](doc/src/devdocs/agents/skills/julia-syntax-lowering/SKILL.md) — developing and testing JuliaSyntax and JuliaLowering. + +## Commit messages and pull requests When writing commit messages, follow the format "component: Brief summary" for the title. In the body of the commit message, provide a brief prose summary @@ -108,8 +100,11 @@ If your change fixes one or more issues, use the syntax "Fixes #" at the end of When referencing external GitHub PRs or issues, use proper GitHub interlinking format (e.g., `owner/repo#123` for PRs/issues). When fixing CI failures, include the link to the specific CI failure in the commit message. - -When creating pull requests, if the pull request consists of one commit only, -use the body of the commit for the body of the pull request. If there are multiple -commits in the pull request, follow the same guidelines for the pull request -as for the commit body. +Always quote macro names in backticks in commit messages and PR titles/bodies (e.g. `` `@inbounds` ``, not @inbounds), so GitHub does not notify the unrelated user with that handle. + +When preparing a pull request for the human author to open, draft the body as follows, expecting +them to reword it: +1. If the pull request consists of one commit only, use the body of the commit for the body of the pull request. +2. If there are multiple commits in the pull request, follow the same guidelines for the pull request as for the commit body. +3. Make sure that the base commit of the pull request is recent (within the past two days) - if not rebase your changes first. +4. If a separate tool reviews the commit, this is useful to note. Encourage the human to state how carefully they read and understood the content also, for example, by drafting the text to say the human author has *not* read any of it, and expecting they will update that once they read that disclaimer. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3d863cf..0000000000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000000..c41433523e33f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,9 @@ + + +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 36ec53c6a181d..39a290c4a76cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,9 @@ Hi! If you are new to the Julia community: welcome, and thanks for trying Julia. If you are already familiar with Julia itself, this blog post by Katharine Hyatt on [Making your first Julia pull request](https://kshyatt.github.io/post/firstjuliapr/) is a great way to get started. +> [!IMPORTANT] +> If your pull request contains substantive contributions from a generative AI tool, please disclose so with details, and review all changes before opening. This also applies to other content, such as issues, discussions, and comments. + ## Learning Julia [The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike. @@ -40,6 +43,18 @@ A useful bug report filed as a GitHub issue provides information about how to re ## Submitting contributions +### Working on issues + +We rarely assign issues to contributors, especially not to new contributors. If you want to work on an issue, just go ahead and open a pull request. There's no need to ask for permission or comment that you're planning to work on it—just start working and submit your PR when ready. + +This policy exists to avoid ["cookie licking"](https://www.redhat.com/en/blog/dont-lick-cookie) (claiming an issue without following through), which can discourage others from contributing. If someone else is already working on the same issue, that's okay—multiple approaches can be valuable, and reviewers will help sort out which solution is best. + +### Reviewing pull requests + +Pull request reviews from all members of the community are welcomed and encouraged, even if you are not a committer or triager. Reading through someone else's PR can be a great way to learn the codebase, and thoughtful review with an explicit approval (or rejection) is a helpful signal to those with merge rights to accelerate a PR out of pending-review limbo. + +Similarly to other types of contribution, AI-assisted review is allowed so long as it is disclosed, but all humans remain responsible for the content they generate. Low-effort "LGTM" or unabridged walls of AI-generated text are not useful. + ### Contributor Checklist * Create a [GitHub account](https://github.com/signup/free). @@ -54,7 +69,7 @@ A useful bug report filed as a GitHub issue provides information about how to re * Review discussions on the [Julia Discourse forum](https://discourse.julialang.org). -* If your pull request contains substantial contributions from a generative AI tool, please disclose so with details, and review all changes before opening. +* If your pull request contains substantive contributions from a generative AI tool, please disclose so with details, and review all changes before opening. Open the pull request yourself once you have reviewed it, rather than having a tool open it on your behalf. * Relax and have fun! diff --git a/Compiler/.gitignore b/Compiler/.gitignore index ba39cc531edeb..fe24cf170f6f5 100644 --- a/Compiler/.gitignore +++ b/Compiler/.gitignore @@ -1 +1,2 @@ Manifest.toml +Manifest-v*.toml diff --git a/Compiler/Project.toml b/Compiler/Project.toml index 1a0cdf4abca39..3ef68ad507617 100644 --- a/Compiler/Project.toml +++ b/Compiler/Project.toml @@ -2,6 +2,9 @@ name = "Compiler" uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" version = "0.1.1" +[syntax] +julia_version = "1.14" + [compat] julia = "1.10" diff --git a/Compiler/extras/CompilerDevTools/.gitignore b/Compiler/extras/CompilerDevTools/.gitignore new file mode 100644 index 0000000000000..fe24cf170f6f5 --- /dev/null +++ b/Compiler/extras/CompilerDevTools/.gitignore @@ -0,0 +1,2 @@ +Manifest.toml +Manifest-v*.toml diff --git a/Compiler/extras/CompilerDevTools/Manifest.toml b/Compiler/extras/CompilerDevTools/Manifest.toml deleted file mode 100644 index bcc78f1ded34a..0000000000000 --- a/Compiler/extras/CompilerDevTools/Manifest.toml +++ /dev/null @@ -1,15 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.12.0-DEV" -manifest_format = "2.0" -project_hash = "84f495a1bf065c95f732a48af36dd0cd2cefb9d5" - -[[deps.Compiler]] -path = "../.." -uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" -version = "0.0.2" - -[[deps.CompilerDevTools]] -path = "." -uuid = "92b2d91f-d2bd-4c05-9214-4609ac33433f" -version = "0.0.0" diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index ddf202f378fb5..89b902e18d292 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -12,14 +12,14 @@ struct SplitCacheInterp <: Compiler.AbstractInterpreter owner::SplitCacheOwner inf_params::Compiler.InferenceParams opt_params::Compiler.OptimizationParams - inf_cache::Vector{Compiler.InferenceResult} + inf_cache::Compiler.InferenceCache codegen_cache::IdDict{CodeInstance,CodeInfo} function SplitCacheInterp(; world::UInt = Base.get_world_counter(), owner::SplitCacheOwner = SplitCacheOwner(), inf_params::Compiler.InferenceParams = Compiler.InferenceParams(), opt_params::Compiler.OptimizationParams = Compiler.OptimizationParams(), - inf_cache::Vector{Compiler.InferenceResult} = Compiler.InferenceResult[]) + inf_cache::Compiler.InferenceCache = Compiler.InferenceCache()) new(world, owner, inf_params, opt_params, inf_cache, IdDict{CodeInstance,CodeInfo}()) end end diff --git a/Compiler/extras/CompilerDevTools/test/runtests.jl b/Compiler/extras/CompilerDevTools/test/runtests.jl index 89dc4696d9e1c..791a8bb618e2b 100644 --- a/Compiler/extras/CompilerDevTools/test/runtests.jl +++ b/Compiler/extras/CompilerDevTools/test/runtests.jl @@ -2,7 +2,7 @@ using Test using Compiler: code_cache using Base: inferencebarrier using CompilerDevTools -using CompilerDevTools: lookup_method_instance, SplitCacheInterp +using CompilerDevTools: SplitCacheInterp, lookup_method_instance @testset "CompilerDevTools" begin do_work(x, y) = x + y diff --git a/Compiler/extras/CompilerDevTools/test/testpkg.jl b/Compiler/extras/CompilerDevTools/test/testpkg.jl index b31459219c8dd..fdc3f815637fa 100644 --- a/Compiler/extras/CompilerDevTools/test/testpkg.jl +++ b/Compiler/extras/CompilerDevTools/test/testpkg.jl @@ -1,6 +1,19 @@ using Pkg +function with_output_on_failure(f) + output = IOBuffer() + try + return f(output) + catch + seekstart(output) + write(stderr, read(output)) + rethrow() + end +end + Pkg.activate(dirname(@__DIR__)) do - Pkg.instantiate() - include("runtests.jl") + with_output_on_failure() do output + Pkg.instantiate(; io=output) + end + include("runtests.jl") end diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index e1c167e57ed08..1f6e8dcd0f44e 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -15,8 +15,8 @@ end # When generating an incremental precompile file, we first check whether we # already have a copy of this *exact* code in the system image. If so, we -# simply generates a pkgimage that has the dependency edges we recorded in -# the system image and simply returns that copy of the compiler. If not, +# simply generate a pkgimage that has the dependency edges we recorded in +# the system image and simply return that copy of the compiler. If not, # we proceed to load/precompile this as an ordinary package. elseif (isdefined(Base, :generating_output) && Base.generating_output(true) && Base.samefile(joinpath(Sys.BINDIR, Base.DATAROOTDIR, Base._compiler_require_dependencies[1][2]), @eval @__FILE__) && @@ -37,37 +37,44 @@ else using Core.Intrinsics, Core.IR -using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstance, MethodMatch, +using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, AnyType, MethodInstance, MethodMatch, MethodTable, MethodCache, PartialOpaque, SimpleVector, TypeofVararg, + TypeEq, _apply_iterate, apply_type, compilerbarrier, donotdelete, memoryref_isassigned, - memoryrefget, memoryrefnew, memoryrefoffset, memoryrefset!, print, println, show, svec, - typename, unsafe_write, write + memoryrefget, memoryrefnew, memoryrefoffset, memoryrefset!, memoryrefunset!, print, println, show, svec, + typename, unsafe_write, write, stdout, stderr -using Base using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer, PARTITION_KIND_GLOBAL, PARTITION_KIND_UNDEF_CONST, PARTITION_KIND_BACKDATED_CONST, PARTITION_KIND_DECLARED, PARTITION_FLAG_DEPWARN, + JL_OPTIONS_COMPILE_OFF, JL_OPTIONS_COMPILE_MIN, Base, BitVector, Bottom, Callable, DataTypeFieldDesc, - EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES, - OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME, - _array_for, _bits_findnext, _methods_by_ftype, _uniontypes, all, allocatedinline, any, - argument_datatype, binding_kind, cconvert, copy_exprargs, datatype_arrayelem, + EffectsOverride, Filter, Generator, NUM_EFFECTS_OVERRIDES, + OneTo, Ordering, RefValue, _NAMEDTUPLE_NAME, + _array_for, _bits_findnext, _defaultctors, _methods_by_ftype, _uniontypes, all, allocatedinline, any, + argument_datatypename, binding_kind, cconvert, copy_exprargs, datatype_arrayelem, datatype_fieldcount, datatype_fieldtypes, datatype_layoutsize, datatype_nfields, - datatype_pointerfree, decode_effects_override, diff_names, fieldindex, - generating_output, get_nospecializeinfer_sig, get_world_counter, has_free_typevars, - hasgenerator, hasintersect, indexed_iterate, isType, is_file_tracked, is_function_def, + datatype_pointerfree, decode_effects_override, diff_names, fieldindex, visit, + generating_output, get_nospecializeinfer_sig, get_world_counter, has_free_typevars, has_typevar, + hasgenerator, hasintersect, indexed_iterate, isType, isTypeEq, isTypeEgal, + is_file_tracked, is_function_def, is_meta_expr, is_meta_expr_head, is_nospecialized, is_nospecializeinfer, is_defined_const_binding, - is_some_const_binding, is_some_guard, is_some_imported, is_some_explicit_imported, is_some_binding_imported, is_valid_intrinsic_elptr, + is_some_const_binding, is_some_guard, is_some_global, is_some_imported, is_some_explicit_imported, is_some_binding_imported, is_valid_intrinsic_elptr, isbitsunion, isconcretedispatch, isdispatchelem, isexpr, isfieldatomic, isidentityfree, iskindtype, ismutabletypename, ismutationfree, issingletontype, isvarargtype, isvatuple, - kwerr, lookup_binding_partition, may_invoke_generator, methods, midpoint, moduleroot, - partition_restriction, quoted, rename_unionall, rewrap_unionall, specialize_method, - structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout, + PARTITION_KIND_CONST, kwerr, lookup_binding_partition, may_invoke_generator, methods, midpoint, moduleroot, + partition_owner, partition_restriction, quoted, rename_unionall, rewrap_unionall, specialize_method, + structdiff, tls_world_age, type_parameter, unconstrain_vararg_length, unionlen, uniontype_layout, uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal, - _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized, + _uncompressed_ir, datatype_min_ninitialized, partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using, - get_ci_mi + get_ci_mi, get_methodtable, morespecific, specializations, has_image_globalref, + rewrap_free_typevars, find_free_typevars, typeintersect_env, + PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED, + BINDING_FLAG_ANY_IMPLICIT_EDGES, is_some_implicit, IteratorSize, SizeUnknown, get_require_world, JLOptions, + devnull, devnull as stdin +using Base using Base.Order import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, get!, @@ -158,6 +165,12 @@ else using Base: @show end +# JuliaSyntax doesn't support syntax evolution in bare modules via Project.toml +# This surfaces only when Compiler.jl is loaded as a standalone package. +if isdefined(Base, :end_base_include) && isdefined(Base, :set_syntax_version) + Base.set_syntax_version(Compiler, Base.VersionNumber(1, 14)) +end + include("cicache.jl") include("methodtable.jl") include("effects.jl") @@ -185,9 +198,14 @@ include("typeinfer.jl") include("optimize.jl") include("bootstrap.jl") +include("precompile.jl") include("reflection_interface.jl") include("opaque_closure.jl") +baremodule ReinferUtils end +include(ReinferUtils, "reinfer.jl") +include(ReinferUtils, "bindinginvalidations.jl") + macro __SOURCE_FILE__() __source__.file === nothing && return nothing return QuoteNode(__source__.file::Symbol) @@ -214,6 +232,23 @@ else # During bootstrap, skip including these files and defer to base/show.jl to include it later end +# The Compiler sources use Julia 1.14 syntax (`typegroup` blocks; see the +# `[syntax]` section in Compiler/Project.toml). When built into the sysimg this +# module bypasses package loading, so install the module parser binding +# consulted by `Base.parser_for_module` directly; without it, `Meta.parse`-style +# reparsing of Compiler sources (e.g. by Revise) uses unversioned syntax and +# rejects `typegroup` blocks. `Base.JuliaSyntax` and `Base.VersionNumber` only +# need to exist by the time this is called, not when it is defined, so this is +# safe to define this early in bootstrap. When loaded as a package instead, +# package loading has already declared an equivalent binding from the project's +# `[syntax]` entry (and defining over it would error), so skip it then. +if !isdefined(@__MODULE__, Symbol("#_internal_julia_parse")) +function var"#_internal_julia_parse"(code, filename::String, lineno::Int, offset::Int, options::Symbol) + return Base.JuliaSyntax.core_parser_hook( + code, filename, lineno, offset, options, Base.VersionNumber(1, 14, 0)) +end +end + end # baremodule Compiler end # if isdefined(Base, :generating_output) && ... diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 3552525f70cf3..1af8c86ec3049 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -22,7 +22,7 @@ function can_propagate_conditional(@nospecialize(rt), argtypes::Vector{Any}) return false end return isa(argtypes[rt.slot], Conditional) && - is_const_bool_or_bottom(rt.thentype) && is_const_bool_or_bottom(rt.thentype) + is_const_bool_or_bottom(rt.thentype) && is_const_bool_or_bottom(rt.elsetype) end function propagate_conditional(rt::InterConditional, cond::Conditional) @@ -30,12 +30,12 @@ function propagate_conditional(rt::InterConditional, cond::Conditional) new_elsetype = rt.elsetype === Const(true) ? cond.thentype : cond.elsetype if rt.thentype == Bottom @assert rt.elsetype != Bottom - return Conditional(cond.slot, Bottom, new_elsetype) + return Conditional(cond.slot, cond.ssadef, Bottom, new_elsetype) elseif rt.elsetype == Bottom @assert rt.thentype != Bottom - return Conditional(cond.slot, new_thentype, Bottom) + return Conditional(cond.slot, cond.ssadef, new_thentype, Bottom) end - return Conditional(cond.slot, new_thentype, new_elsetype) + return Conditional(cond.slot, cond.ssadef, new_thentype, new_elsetype) end mutable struct SafeBox{T} @@ -53,6 +53,8 @@ end struct MethodMatchTarget match::MethodMatch edges::Vector{Union{Nothing,CodeInstance}} + needs_mi_edges::BitVector + call_results::Vector{Union{Nothing,InferredCallResult}} edge_idx::Int end @@ -66,6 +68,7 @@ any_ambig(info::MethodMatchInfo) = any_ambig(info.results) any_ambig(m::MethodMatches) = any_ambig(m.info) fully_covering(info::MethodMatchInfo) = info.fullmatch fully_covering(m::MethodMatches) = fully_covering(m.info) +multiple_methods(m::MethodMatches) = length(m.applicable) > 1 struct UnionSplitMethodMatches applicable::Vector{MethodMatchTarget} @@ -77,6 +80,17 @@ any_ambig(info::UnionSplitInfo) = any(any_ambig, info.split) any_ambig(m::UnionSplitMethodMatches) = any_ambig(m.info) fully_covering(info::UnionSplitInfo) = all(fully_covering, info.split) fully_covering(m::UnionSplitMethodMatches) = fully_covering(m.info) +function multiple_methods(m::UnionSplitMethodMatches) + first_method = nothing + for target in m.applicable + if first_method === nothing + first_method = target.match.method + elseif target.match.method !== first_method + return true + end + end + return false +end nmatches(info::MethodMatchInfo) = length(info.results) function nmatches(info::UnionSplitInfo) @@ -93,38 +107,51 @@ mutable struct CallInferenceState rettype exctype all_effects::Effects - const_results::Union{Nothing,Vector{Union{Nothing,ConstResult}}} # keeps the results of inference with the extended lattice elements (if happened) conditionals::Union{Nothing,Tuple{Vector{Any},Vector{Any}}} # keeps refinement information of call argument types when the return type is boolean slotrefinements::Union{Nothing,Vector{Any}} # keeps refinement information on slot types obtained from call signature # some additional fields for untyped objects (just to avoid capturing) - func - matches::Union{MethodMatches,UnionSplitMethodMatches} + const func + const matches::Union{MethodMatches,UnionSplitMethodMatches} function CallInferenceState(@nospecialize(func), matches::Union{MethodMatches,UnionSplitMethodMatches}) return new(#=inferidx=#1, #=rettype=#Bottom, #=exctype=#Bottom, #=all_effects=#EFFECTS_TOTAL, - #=const_results=#nothing, #=conditionals=#nothing, #=slotrefinements=#nothing, - func, matches) + #=conditionals=#nothing, #=slotrefinements=#nothing, func, matches) end end +widen_call_result(::AbstractInterpreter, si::StmtInfo, state::CallInferenceState, ::AbsIntState) = + call_result_unused(si) && !(state.rettype === Bottom) + function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(func), arginfo::ArgInfo, si::StmtInfo, @nospecialize(atype), - sv::AbsIntState, max_methods::Int) + vtypes::Union{VarTable,Nothing}, sv::AbsIntState, max_methods::Int) 𝕃ₚ, 𝕃ᵢ = ipo_lattice(interp), typeinf_lattice(interp) - ⊑ₚ, ⋤ₚ, ⊔ₚ, ⊔ᵢ = partialorder(𝕃ₚ), strictneqpartialorder(𝕃ₚ), join(𝕃ₚ), join(𝕃ᵢ) + ⊑ₚ, ⊔ₚ, ⊔ᵢ = partialorder(𝕃ₚ), join(𝕃ₚ), join(𝕃ᵢ) argtypes = arginfo.argtypes if si.saw_latestworld add_remark!(interp, sv, "Cannot infer call, because we previously saw :latestworld") return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) end - matches = find_method_matches(interp, argtypes, atype; max_methods) + current_world = get_world_counter() + matches = find_method_matches(interp, argtypes, atype; max_methods, fargs=arginfo.fargs) if isa(matches, FailedMethodMatch) add_remark!(interp, sv, matches.reason) return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) end (; valid_worlds, applicable) = matches - update_valid_age!(sv, valid_worlds) # need to record the negative world now, since even if we don't generate any useful information, inlining might want to add an invoke edge and it won't have this information anymore + update_valid_age!(sv, get_inference_world(interp), valid_worlds) # need to record the negative world now, since even if we don't generate any useful information, inlining might want to add an invoke edge and it won't have this information anymore + # Concrete-only functions refuse to commit (and record no backedge) when any + # applicable match has a non-concrete signature, regardless of scope. This is the + # generalization of the top-level `!isdispatchtuple` bail below to a per-function opt-in. + if is_concrete_only(func) + for i = 1:length(applicable) + if !isdispatchtuple(applicable[i].match.spec_types) + add_remark!(interp, sv, "Refusing to infer non-concrete call site for concrete-only function") + return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) + end + end + end if bail_out_toplevel_call(interp, sv) local napplicable = length(applicable) for i = 1:napplicable @@ -144,9 +171,9 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun # split the for loop off into a function, so that we can pause and restart it at will function infercalls(interp, sv) local napplicable = length(applicable) - local multiple_matches = napplicable > 1 + local multiple_matches = multiple_methods(matches) while state.inferidx <= napplicable - (; match, edges, edge_idx) = applicable[state.inferidx] + (; match, edges, needs_mi_edges, call_results, edge_idx) = applicable[state.inferidx] local method = match.method local sig = match.spec_types if bail_out_call(interp, InferenceLoopState(state.rettype, state.all_effects), sv) @@ -166,7 +193,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun #end mresult = abstract_call_method(interp, method, sig, match.sparams, multiple_matches, si, sv)::Future function handle1(interp, sv) - local (; rt, exct, effects, edge, volatile_inf_result) = mresult[] + local (; rt, exct, effects, edge, needs_mi_edge, call_result) = mresult[] this_conditional = ignorelimited(rt) this_rt = widenwrappedconditional(rt) this_exct = exct @@ -177,11 +204,10 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun this_arginfo = ArgInfo(arginfo.fargs, this_argtypes) const_call_result = abstract_call_method_with_const_args(interp, mresult[], state.func, this_arginfo, si, match, sv) - const_result = volatile_inf_result if const_call_result !== nothing this_const_conditional = ignorelimited(const_call_result.rt) this_const_rt = widenwrappedconditional(const_call_result.rt) - const_edge = nothing + const_result = nothing if this_const_rt ⊑ₚ this_rt # As long as the const-prop result we have is not *worse* than # what we found out on types, we'd like to use it. Even if the @@ -192,9 +218,9 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun # e.g. in cases when there are cycles but cached result is still accurate this_conditional = this_const_conditional this_rt = this_const_rt - (; effects, const_result, const_edge) = const_call_result + (; effects, const_result) = const_call_result elseif is_better_effects(const_call_result.effects, effects) - (; effects, const_result, const_edge) = const_call_result + (; effects, const_result) = const_call_result else add_remark!(interp, sv, "[constprop] Discarded because the result was wider than inference") end @@ -202,24 +228,18 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun # because consistent-cy does not apply to exceptions. if const_call_result.exct ⋤ this_exct this_exct = const_call_result.exct - (; const_result, const_edge) = const_call_result + (; const_result) = const_call_result else add_remark!(interp, sv, "[constprop] Discarded exception type because result was wider than inference") end - if const_edge !== nothing - edge = const_edge - update_valid_age!(sv, world_range(const_edge)) + if const_result !== nothing + update_valid_age!(sv, get_inference_world(interp), + proof_worlds(inference_proof(const_result))) + call_result = const_result end end state.all_effects = merge_effects(state.all_effects, effects) - if const_result !== nothing - local const_results = state.const_results - if const_results === nothing - const_results = state.const_results = fill!(Vector{Union{Nothing,ConstResult}}(undef, napplicable), nothing) - end - const_results[state.inferidx] = const_result - end @assert !(this_conditional isa Conditional || this_rt isa MustAlias) "invalid lattice element returned from inter-procedural context" if can_propagate_conditional(this_conditional, argtypes) # The only case where we need to keep this in rt is where @@ -245,6 +265,8 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun end end edges[edge_idx] = edge + needs_mi_edges[edge_idx] = needs_mi_edge + call_results[edge_idx] = call_result state.inferidx += 1 return true @@ -260,11 +282,6 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun seenall = state.inferidx > napplicable retinfo = state.matches.info if seenall # small optimization to skip some work that is already implied - local const_results = state.const_results - if const_results !== nothing - @assert napplicable == nmatches(retinfo) == length(const_results) - retinfo = ConstCallInfo(retinfo, const_results) - end if !fully_covering(state.matches) || any_ambig(state.matches) # Account for the fact that we may encounter a MethodError with a non-covered or ambiguous signature. state.all_effects = Effects(state.all_effects; nothrow=false) @@ -274,15 +291,14 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun if sv isa InferenceState && fargs !== nothing state.slotrefinements = collect_slot_refinements(𝕃ᵢ, applicable, argtypes, fargs, sv) end - state.rettype = from_interprocedural!(interp, state.rettype, sv, arginfo, state.conditionals) - if call_result_unused(si) && !(state.rettype === Bottom) - add_remark!(interp, sv, "Call result type was widened because the return value is unused") - # We're mainly only here because the optimizer might want this code, - # but we ourselves locally don't typically care about it locally - # (beyond checking if it always throws). - # So avoid adding an edge, since we don't want to bother attempting - # to improve our result even if it does change (to always throw), - # and avoid keeping track of a more complex result type. + state.rettype = from_interprocedural!(interp, state.rettype, sv, arginfo, state.conditionals, vtypes) + if widen_call_result(interp, si, state, sv) + add_remark!(interp, sv, "Call result type was widened") + # Encode the decision as a local `Any` in `state.rettype`, which flows into + # `ssavaluetypes[pc]` of the enclosing frame. Downstream `=== Any` gates + # (most notably the cycle backedge revisit filter in `update_cycle_worklists!`) + # then treat this call site as needing no further refinement. By default + # `Bottom` is excluded so that "always throws" remains observable. state.rettype = Any end # if from_interprocedural added any pclimitations to the set inherited from the arguments, @@ -297,31 +313,46 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun # there is unanalyzed candidate, widen type and effects to the top state.rettype = state.exctype = Any state.all_effects = Effects() - state.const_results = nothing end - # Also considering inferring the compilation signature for this method, so + # Also consider inferring the compilation signature for this method, so # it is available to the compiler in case it ends up needing it for the invoke. if (isa(sv, InferenceState) && infer_compilation_signature(interp) && (!is_removable_if_unused(state.all_effects) || !call_result_unused(si))) inferidx = SafeBox{Int}(1) function infercalls2(interp, sv) local napplicable = length(applicable) - local multiple_matches = napplicable > 1 + local multiple_matches = multiple_methods(matches) while inferidx[] <= napplicable - (; match, edges, edge_idx) = applicable[inferidx[]] + (; match, call_results, edge_idx) = applicable[inferidx[]] inferidx[] += 1 local method = match.method local sig = match.spec_types - mi = specialize_method(match; preexisting=true) - if mi === nothing || !const_prop_methodinstance_heuristic(interp, mi, arginfo, sv) + local mi = specialize_method(match; preexisting=true) + local call_result = call_results[edge_idx] + if (mi === nothing || !(call_result isa LocalInferenceResult) || + !const_prop_methodinstance_heuristic(interp, call_result.result, mi, arginfo, sv)) csig = get_compileable_sig(method, sig, match.sparams) if csig !== nothing && (!seenall || csig !== sig) # corresponds to whether the first look already looked at this, so repeating abstract_call_method is not useful #println(sig, " changed to ", csig, " for ", method) - sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), csig, method.sig)::SimpleVector - sparams = sp_[2]::SimpleVector + (_, sparams) = typeintersect_env(csig, method.sig) mresult = abstract_call_method(interp, method, csig, sparams, multiple_matches, StmtInfo(false, false), sv)::Future - isready(mresult) || return false # wait for mresult Future to resolve off the callstack before continuing + function infercalls3(interp, sv) + local edge = mresult[].edge + if edge !== nothing + local sig = match.spec_types + local mi = get_ci_mi(edge) + local vw = matches.valid_worlds + ccall(:jl_recache_method_by_type, Cvoid, (Any, Any, Any, UInt, UInt, UInt, UInt), + sig, mi, mi.specTypes, get_inference_world(interp), + first(vw), last(vw), current_world) + end + return true + end + if !isready(mresult) || !infercalls3(interp, sv) + push!(sv.tasks, infercalls3) + return false # wait for mresult Future to resolve off the callstack before continuing + end end end end @@ -341,20 +372,23 @@ end function find_method_matches(interp::AbstractInterpreter, argtypes::Vector{Any}, @nospecialize(atype); max_union_splitting::Int = InferenceParams(interp).max_union_splitting, - max_methods::Int = InferenceParams(interp).max_methods) - if is_union_split_eligible(typeinf_lattice(interp), argtypes, max_union_splitting) - return find_union_split_method_matches(interp, argtypes, atype, max_methods) + max_methods::Int = InferenceParams(interp).max_methods, + fargs::Union{Nothing,Vector{Any}}=nothing) + if is_union_split_eligible(typeinf_lattice(interp), argtypes, max_union_splitting; fargs) + return find_union_split_method_matches(interp, argtypes, max_methods; fargs) end return find_simple_method_matches(interp, atype, max_methods) end # NOTE this is valid as far as any "constant" lattice element doesn't represent `Union` type -is_union_split_eligible(𝕃::AbstractLattice, argtypes::Vector{Any}, max_union_splitting::Int) = - 1 < unionsplitcost(𝕃, argtypes) <= max_union_splitting +is_union_split_eligible(𝕃::AbstractLattice, argtypes::Vector{Any}, max_union_splitting::Int; + fargs::Union{Nothing,Vector{Any}}=nothing) = + 1 < unionsplitcost(𝕃, argtypes; fargs) <= max_union_splitting function find_union_split_method_matches(interp::AbstractInterpreter, argtypes::Vector{Any}, - @nospecialize(atype), max_methods::Int) - split_argtypes = switchtupleunion(typeinf_lattice(interp), argtypes) + max_methods::Int; + fargs::Union{Nothing,Vector{Any}}=nothing) + split_argtypes = switchtupleunion(typeinf_lattice(interp), argtypes; fargs) infos = MethodMatchInfo[] applicable = MethodMatchTarget[] applicable_argtypes = Vector{Any}[] # arrays like `argtypes`, including constants, for each match @@ -373,7 +407,8 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch) push!(infos, thisinfo) for idx = 1:length(thismatches) - push!(applicable, MethodMatchTarget(thismatches[idx], thisinfo.edges, idx)) + push!(applicable, MethodMatchTarget(thismatches[idx], thisinfo.edges, + thisinfo.needs_mi_edges, thisinfo.call_results, idx)) push!(applicable_argtypes, arg_n) end end @@ -392,7 +427,8 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a fullmatch = any(match::MethodMatch->match.fully_covers, matches) mt = Core.methodtable info = MethodMatchInfo(matches, mt, atype, fullmatch) - applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)] + applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, + info.needs_mi_edges, info.call_results, idx) for idx = 1:length(matches)] return MethodMatches(applicable, info, matches.valid_worlds) end @@ -415,18 +451,18 @@ In such cases `maybecondinfo` should be either of: - `maybecondinfo::Tuple{Vector{Any},Vector{Any}}`: precomputed argument type refinement information - method call signature tuple type When we deal with multiple `MethodMatch`es, it's better to precompute `maybecondinfo` by -`tmerge`ing argument signature type of each method call. +`tmerge`ing argument signature types of each method call. """ function from_interprocedural!(interp::AbstractInterpreter, @nospecialize(rt), sv::AbsIntState, - arginfo::ArgInfo, @nospecialize(maybecondinfo)) + arginfo::ArgInfo, @nospecialize(maybecondinfo), vtypes::Union{VarTable,Nothing}) rt = collect_limitations!(rt, sv) if isa(rt, InterMustAlias) - rt = from_intermustalias(typeinf_lattice(interp), rt, arginfo, sv) + rt = from_intermustalias(typeinf_lattice(interp), rt, arginfo, vtypes, sv) elseif is_lattice_bool(ipo_lattice(interp), rt) if maybecondinfo === nothing rt = widenconditional(rt) else - rt = from_interconditional(typeinf_lattice(interp), rt, sv, arginfo, maybecondinfo) + rt = from_interconditional(typeinf_lattice(interp), rt, sv, arginfo, maybecondinfo, vtypes) end end @assert !(rt isa InterConditional || rt isa InterMustAlias) "invalid lattice element returned from inter-procedural context" @@ -441,17 +477,17 @@ function collect_limitations!(@nospecialize(typ), sv::InferenceState) return typ end -function from_intermustalias(𝕃ᵢ::AbstractLattice, rt::InterMustAlias, arginfo::ArgInfo, sv::AbsIntState) +function from_intermustalias(𝕃ᵢ::AbstractLattice, rt::InterMustAlias, arginfo::ArgInfo, vtypes::Union{VarTable,Nothing}, sv::AbsIntState) fargs = arginfo.fargs if fargs !== nothing && 1 ≤ rt.slot ≤ length(fargs) arg = ssa_def_slot(fargs[rt.slot], sv) if isa(arg, SlotNumber) + @assert vtypes !== nothing argtyp = widenslotwrapper(arginfo.argtypes[rt.slot]) ⊑ = partialorder(𝕃ᵢ) if rt.vartyp ⊑ argtyp - return MustAlias(arg, rt.vartyp, rt.fldidx, rt.fldtyp) - else - # TODO optimize this case? + vtyp = vtypes[slot_id(arg)] + return MustAlias(arg, vtyp.ssadef, rt.vartyp, rt.fldidx, rt.fldtyp) end end end @@ -459,7 +495,7 @@ function from_intermustalias(𝕃ᵢ::AbstractLattice, rt::InterMustAlias, argin end function from_interconditional(𝕃ᵢ::AbstractLattice, @nospecialize(rt), sv::AbsIntState, - arginfo::ArgInfo, @nospecialize(maybecondinfo)) + arginfo::ArgInfo, @nospecialize(maybecondinfo), vtypes::Union{VarTable,Nothing}) has_conditional(𝕃ᵢ, sv) || return widenconditional(rt) (; fargs, argtypes) = arginfo fargs === nothing && return widenconditional(rt) @@ -541,7 +577,8 @@ function from_interconditional(𝕃ᵢ::AbstractLattice, @nospecialize(rt), sv:: if alias !== nothing return form_mustalias_conditional(alias, thentype, elsetype) end - return Conditional(slot, thentype, elsetype) # record a Conditional improvement to this slot + @assert vtypes !== nothing + return Conditional(slot, vtypes[slot].ssadef, thentype, elsetype) # record a Conditional improvement to this slot end return widenconditional(rt) end @@ -588,6 +625,34 @@ function collect_slot_refinements(𝕃ᵢ::AbstractLattice, applicable::Vector{M end slotrefinements[fidx] = sigt end + elseif argtypes[i] isa MustAlias + alias = argtypes[i]::MustAlias + argt = alias.fldtyp + if isvarargtype(argt) + argt = unwrapva(argt) + end + sigt = Bottom + for j = 1:length(applicable) + (;match) = applicable[j] + valid_as_lattice(match.spec_types, true) || continue + sigt = sigt ⊔ fieldtype(match.spec_types, i) + end + if sigt ⊏ argt # i.e. signature type is strictly more specific than the field type + newtyp = form_mustalias_refinement(alias, sigt) + if newtyp !== nothing + aidx = alias.slot + if slotrefinements === nothing + slotrefinements = fill!(Vector{Any}(undef, length(sv.slottypes)), nothing) + end + # TODO: if multiple MustAlias arguments refer to different fields + # of the same slot, we only apply the first refinement. Merging + # multiple PartialStruct refinements would require a meet operation + # on PartialStruct, which is not currently implemented. + if slotrefinements[aidx] === nothing + slotrefinements[aidx] = newtyp + end + end + end end end return slotrefinements @@ -614,7 +679,7 @@ function abstract_call_method(interp::AbstractInterpreter, # look through the parents list to see if there's a call to the same method # and from the same method. # Returns the topmost occurrence of that repeated edge. - edgecycle = edgelimited = false + edgecycle = edgelimited = edgerecursed = false topmost = nothing for sv′ in AbsIntStackUnwind(sv) @@ -622,9 +687,9 @@ function abstract_call_method(interp::AbstractInterpreter, if method === infmi.def if infmi.specTypes::Type == sig::Type # avoid widening when detecting self-recursion - # TODO: merge call cycle and return right away topmost = nothing edgecycle = true + edgerecursed = true break end topmost === nothing || continue @@ -674,8 +739,8 @@ function abstract_call_method(interp::AbstractInterpreter, # if we don't (typically) actually care about this result, # don't bother trying to examine some complex abstract signature # since it's very unlikely that we'll try to inline this, - # or want make an invoke edge to its calling convention return type. - # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases) + # or want to make an invoke edge to its calling convention return type. + # (atypically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases) return Future(MethodCallResult(Any, Any, Effects(), nothing, true, true)) end add_remark!(interp, sv, washardlimit ? RECURSION_MSG_HARDLIMIT : RECURSION_MSG) @@ -708,7 +773,7 @@ function abstract_call_method(interp::AbstractInterpreter, # if sig changed, may need to recompute the sparams environment if isa(method.sig, UnionAll) && isempty(sparams) - recomputed = ccall(:jl_type_intersection_with_env, Any, (Any, Any), sig, method.sig)::SimpleVector + (_, sparams) = typeintersect_env(sig, method.sig) #@assert recomputed[1] !== Bottom # We must not use `sig` here, since that may re-introduce structural complexity that # our limiting heuristic sought to eliminate. The alternative would be to not increment depth over covariant contexts, @@ -729,15 +794,14 @@ function abstract_call_method(interp::AbstractInterpreter, # newsig = recomputed[2] # end # sig = ? - sparams = recomputed[2]::SimpleVector end - return typeinf_edge(interp, method, sig, sparams, sv, edgecycle, edgelimited) + return typeinf_edge(interp, method, sig, sparams, sv, edgecycle, edgelimited, edgerecursed) end -function edge_matches_sv(interp::AbstractInterpreter, frame::AbsIntState, +function edge_matches_sv(interp::I, frame::AbsIntState, method::Method, @nospecialize(sig), sparams::SimpleVector, - hardlimit::Bool, sv::AbsIntState) + hardlimit::Bool, sv::AbsIntState) where {I<:AbstractInterpreter} # The `method_for_inference_heuristics` will expand the given method's generator if # necessary in order to retrieve this field from the generated `CodeInfo`, if it exists. # The other `CodeInfo`s we inspect will already have this field inflated, so we just @@ -748,8 +812,10 @@ function edge_matches_sv(interp::AbstractInterpreter, frame::AbsIntState, if callee_method2 !== inf_method2 # limit only if user token match return false end - if isa(frame, InferenceState) && cache_owner(frame.interp) !== cache_owner(interp) - # Don't assume that frames in different interpreters are the same + # Frames in one callstack share the same interpreter type (enforced by the + # `AbsIntState{I}` parameter), but distinct instances of that type may still + # have different cache owners. + if isa(frame, InferenceState) && cache_owner(frame.interp::I) !== cache_owner(interp) return false end if !hardlimit || InferenceParams(interp).ignore_recursion_hardlimit @@ -802,12 +868,6 @@ function matches_sv(parent::AbsIntState, sv::AbsIntState) method_for_inference_limit_heuristics(sv) === method_for_inference_limit_heuristics(parent)) end -function is_edge_recursed(edge::CodeInstance, caller::AbsIntState) - return any(AbsIntStackUnwind(caller)) do sv::AbsIntState - return edge.def === frame_instance(sv) - end -end - function is_method_recursed(method::Method, caller::AbsIntState) return any(AbsIntStackUnwind(caller)) do sv::AbsIntState return method === frame_instance(sv).def @@ -826,7 +886,7 @@ function is_constprop_method_recursed(method::Method, caller::AbsIntState) end end -# keeps result and context information of abstract_method_call, which will later be used for +# keeps result and context information of abstract_call_method, which will later be used for # backedge computation, and concrete evaluation or constant-propagation struct MethodCallResult rt @@ -835,11 +895,15 @@ struct MethodCallResult edge::Union{Nothing,CodeInstance} edgecycle::Bool edgelimited::Bool - volatile_inf_result::Union{Nothing,VolatileInferenceResult} + call_result::Union{Nothing,InferredCallResult} + needs_mi_edge::Bool # targetless body-derived facts require an invalidation target function MethodCallResult(@nospecialize(rt), @nospecialize(exct), effects::Effects, edge::Union{Nothing,CodeInstance}, edgecycle::Bool, edgelimited::Bool, - volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) - return new(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result) + call_result::Union{Nothing,InferredCallResult} = nothing; + needs_mi_edge::Bool = false) + @assert !needs_mi_edge || (edge === nothing && call_result === nothing) + return new(rt, exct, effects, edge, edgecycle, edgelimited, call_result, + needs_mi_edge) end end @@ -851,17 +915,31 @@ end struct ConstCallResult rt::Any exct::Any - const_result::ConstResult + const_result::InferredCallResult effects::Effects - const_edge::Union{Nothing,CodeInstance} function ConstCallResult( @nospecialize(rt), @nospecialize(exct), - const_result::ConstResult, effects::Effects, - const_edge::Union{Nothing,CodeInstance}) - return new(rt, exct, const_result, effects, const_edge) + const_result::InferredCallResult, effects::Effects) + return new(rt, exct, const_result, effects) + end + function ConstCallResult( + result::ConstCallResult; + effects::Effects = result.effects + ) + return new(result.rt, result.exct, result.const_result, effects) end end +function use_concrete_eval_result( + interp::AbstractInterpreter, concrete_eval_result::ConstCallResult + ) + # if we don't inline the result of this concrete evaluation, + # give const-prop' a chance to inline a better method body + return !may_optimize(interp) || + may_inline_concrete_result(concrete_eval_result.const_result::ConcreteResult) || + concrete_eval_result.rt === Bottom # unless this call deterministically throws and thus is non-inlineable +end + function abstract_call_method_with_const_args(interp::AbstractInterpreter, result::MethodCallResult, @nospecialize(f), arginfo::ArgInfo, si::StmtInfo, match::MethodMatch, sv::AbsIntState, invokecall::Union{Nothing,InvokeCall}=nothing) @@ -870,14 +948,16 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, end eligibility = concrete_eval_eligible(interp, f, result, arginfo, sv) concrete_eval_result = nothing + always_nothrow = false if eligibility === :concrete_eval concrete_eval_result = concrete_eval_call(interp, f, result, arginfo, sv, invokecall) - # if we don't inline the result of this concrete evaluation, - # give const-prop' a chance to inline a better method body - if !may_optimize(interp) || ( - may_inline_concrete_result(concrete_eval_result.const_result::ConcreteResult) || - concrete_eval_result.rt === Bottom) # unless this call deterministically throws and thus is non-inlineable - return concrete_eval_result + # allow external abstract interpreters to disable concrete evaluation ad-hoc + if concrete_eval_result !== nothing + if use_concrete_eval_result(interp, concrete_eval_result) + return concrete_eval_result + elseif concrete_eval_result.rt !== Bottom + always_nothrow = true + end end # TODO allow semi-concrete interp for this call? end @@ -895,7 +975,32 @@ function abstract_call_method_with_const_args(interp::AbstractInterpreter, end end # try constant prop' - return const_prop_call(interp, mi, result, arginfo, sv, concrete_eval_result) + new_result = const_prop_call(interp, mi, result, arginfo, sv, concrete_eval_result) + new_result === nothing && return nothing + if eligibility === :none + # const-prop' may have refined effects to be foldable when the original + # call was not; in that case, prefer concrete eval over the const-prop' result + proof = inference_proof(new_result.const_result) + new_eligibility = _concrete_eval_eligible( + interp, f, new_result.effects, proof, arginfo, sv) + if new_eligibility === :concrete_eval + new_concrete_eval_result = _concrete_eval_call( + interp, f, result.edge, new_result.effects, arginfo, sv, invokecall; proof) + if new_concrete_eval_result !== nothing + if use_concrete_eval_result(interp, new_concrete_eval_result) + return new_concrete_eval_result + elseif new_concrete_eval_result.rt !== Bottom + always_nothrow = true + end + end + end + end + if always_nothrow + return ConstCallResult(new_result; + effects = Effects(new_result.effects; nothrow = true)) + else + return new_result + end end function bail_out_const_call(interp::AbstractInterpreter, result::MethodCallResult, @@ -931,9 +1036,18 @@ function bail_out_const_call(interp::AbstractInterpreter, result::MethodCallResu return false end -function concrete_eval_eligible(interp::AbstractInterpreter, - @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState) - (;effects) = result +function concrete_eval_eligible( + interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, + arginfo::ArgInfo, sv::AbsIntState + ) + proof = result.call_result === nothing ? result.edge : inference_proof(result.call_result) + return _concrete_eval_eligible(interp, f, result.effects, proof, arginfo, sv) +end + +function _concrete_eval_eligible( + interp::AbstractInterpreter, @nospecialize(f), effects::Effects, + proof::Union{Nothing,InferenceProof}, arginfo::ArgInfo, sv::AbsIntState + ) if inbounds_option() === :off if !is_nothrow(effects) # Disable concrete evaluation in `--check-bounds=no` mode, @@ -941,7 +1055,7 @@ function concrete_eval_eligible(interp::AbstractInterpreter, return :none end end - if result.edge !== nothing && is_foldable(effects, #=check_rtcall=#true) + if proof !== nothing && is_foldable(effects, #=check_rtcall=#true) if f !== nothing && is_all_const_arg(arginfo, #=start=#2) if (is_nonoverlayed(interp) || is_nonoverlayed(effects) || # Even if overlay methods are involved, when `:consistent_overlay` is @@ -958,13 +1072,19 @@ function concrete_eval_eligible(interp::AbstractInterpreter, # method since currently there is no easy way to execute overlayed methods add_remark!(interp, sv, "[constprop] Concrete eval disabled for overlayed methods") end - if !any_conditional(arginfo) - if may_optimize(interp) - return :semi_concrete_eval + if may_optimize(interp) + if any_conditional(arginfo) + # N.B. semi-concrete eval uses `IRInterpretationState` which does not support + # `Conditional`, so skip it when these lattice elements are present in argtypes + add_remark!(interp, sv, "[constprop] Semi-concrete interpretation disabled due to conditional") + elseif !iszero(typename(typeof(f)).constprop_heuristic & Core.DISABLE_SEMI_CONCRETE_EVAL) + add_remark!(interp, sv, "[constprop] Semi-concrete interpretation disabled due to constprop_heuristic") else - # disable irinterp if optimization is disabled, since it requires optimized IR - add_remark!(interp, sv, "[constprop] Semi-concrete interpretation disabled for non-optimizing interpreter") + return :semi_concrete_eval end + else + # disable irinterp if optimization is disabled, since it requires optimized IR + add_remark!(interp, sv, "[constprop] Semi-concrete interpretation disabled for non-optimizing interpreter") end end return :none @@ -988,14 +1108,25 @@ collect_const_args(arginfo::ArgInfo, start::Int) = collect_const_args(arginfo.ar function collect_const_args(argtypes::Vector{Any}, start::Int) return Any[ let a = widenslotwrapper(argtypes[i]) isa(a, Const) ? a.val : - isconstType(a) ? a.parameters[1] : + isconstType(a) ? type_parameter(a) : (a::DataType).instance end for i = start:length(argtypes) ] end -function concrete_eval_call(interp::AbstractInterpreter, - @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState, - invokecall::Union{InvokeCall,Nothing}=nothing) +function concrete_eval_call( + interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, + arginfo::ArgInfo, sv::AbsIntState, invokecall::Union{InvokeCall,Nothing} = nothing + ) + proof = result.call_result === nothing ? result.edge : inference_proof(result.call_result) + return _concrete_eval_call( + interp, f, result.edge, result.effects, arginfo, sv, invokecall; proof) +end + +function _concrete_eval_call( + interp::AbstractInterpreter, @nospecialize(f), edge::Union{Nothing,CodeInstance}, effects::Effects, + arginfo::ArgInfo, ::AbsIntState, invokecall::Union{InvokeCall,Nothing} = nothing; + proof::Union{Nothing,InferenceProof} = nothing + ) args = collect_const_args(arginfo, #=start=#2) if invokecall !== nothing # this call should be `invoke`d, rewrite `args` back now @@ -1003,17 +1134,16 @@ function concrete_eval_call(interp::AbstractInterpreter, f = invoke end world = get_inference_world(interp) - edge = result.edge::CodeInstance value = try Core._call_in_world_total(world, f, args...) - catch e + catch # The evaluation threw. By :consistent-cy, we're guaranteed this would have happened at runtime. - # Howevever, at present, :consistency does not mandate the type of the exception - concrete_result = ConcreteResult(edge, result.effects) - return ConstCallResult(Bottom, Any, concrete_result, result.effects, #=const_edge=#nothing) + # However, at present, :consistency does not mandate the type of the exception + concrete_result = ConcreteResult(edge, effects; proof) + return ConstCallResult(Bottom, Any, concrete_result, effects) end - concrete_result = ConcreteResult(edge, EFFECTS_TOTAL, value) - return ConstCallResult(Const(value), Bottom, concrete_result, EFFECTS_TOTAL, #=const_edge=#nothing) + concrete_result = ConcreteResult(edge, EFFECTS_TOTAL, value; proof) + return ConstCallResult(Const(value), Bottom, concrete_result, EFFECTS_TOTAL) end # check if there is a cycle and duplicated inference of `mi` @@ -1057,7 +1187,9 @@ function maybe_get_const_prop_profitable(interp::AbstractInterpreter, return nothing end mi = mi::MethodInstance - if !force && !const_prop_methodinstance_heuristic(interp, mi, arginfo, sv) + call_result = result.call_result + inf_result = call_result isa LocalInferenceResult ? call_result.result : nothing + if !force && !const_prop_methodinstance_heuristic(interp, inf_result, mi, arginfo, sv) add_remark!(interp, sv, "[constprop] Disabled by method instance heuristic") return nothing end @@ -1144,7 +1276,7 @@ function find_constrained_arg(cnd::Conditional, fargs::Vector{Any}, sv::Inferenc return nothing end -# checks if all argtypes has additional information other than what `Type` can provide +# checks if all argtypes have additional information other than what `Type` can provide function is_all_overridden(interp::AbstractInterpreter, (; fargs, argtypes)::ArgInfo, sv::AbsIntState) 𝕃ᵢ = typeinf_lattice(interp) for i in 1:length(argtypes) @@ -1161,7 +1293,7 @@ end function force_const_prop(interp::AbstractInterpreter, @nospecialize(f), method::Method) return is_aggressive_constprop(method) || InferenceParams(interp).aggressive_constant_propagation || - typename(typeof(f)).constprop_heuristic === Core.FORCE_CONST_PROP + !iszero(typename(typeof(f)).constprop_heuristic & Core.FORCE_CONST_PROP) end function const_prop_function_heuristic(interp::AbstractInterpreter, @nospecialize(f), @@ -1170,7 +1302,7 @@ function const_prop_function_heuristic(interp::AbstractInterpreter, @nospecializ heuristic = typename(typeof(f)).constprop_heuristic if length(argtypes) > 1 𝕃ᵢ = typeinf_lattice(interp) - if heuristic === Core.ARRAY_INDEX_HEURISTIC + if !iszero(heuristic & Core.ARRAY_INDEX_HEURISTIC) arrty = argtypes[2] # don't propagate constant index into indexing of non-constant array if arrty isa Type && arrty <: AbstractArray && !issingletontype(arrty) @@ -1183,14 +1315,15 @@ function const_prop_function_heuristic(interp::AbstractInterpreter, @nospecializ elseif ⊑(𝕃ᵢ, arrty, Array) || ⊑(𝕃ᵢ, arrty, GenericMemory) return false end - elseif heuristic === Core.ITERATE_HEURISTIC + end + if !iszero(heuristic & Core.ITERATE_HEURISTIC) itrty = argtypes[2] if ⊑(𝕃ᵢ, itrty, Array) || ⊑(𝕃ᵢ, itrty, GenericMemory) return false end end end - if !all_overridden && heuristic === Core.SAMETYPE_HEURISTIC + if !all_overridden && !iszero(heuristic & Core.SAMETYPE_HEURISTIC) # it is almost useless to inline the op when all the same type, # but highly worthwhile to inline promote of a constant length(argtypes) > 2 || return false @@ -1211,7 +1344,7 @@ end # where we would spend a lot of time, but are probably unlikely to get an improved # result anyway. function const_prop_methodinstance_heuristic(interp::AbstractInterpreter, - mi::MethodInstance, arginfo::ArgInfo, sv::AbsIntState) + inf_result::Union{InferenceResult,Nothing}, mi::MethodInstance, ::ArgInfo, sv::AbsIntState) method = mi.def::Method if method.is_for_opaque_closure # Not inlining an opaque closure can be very expensive, so be generous @@ -1241,11 +1374,10 @@ function const_prop_methodinstance_heuristic(interp::AbstractInterpreter, # was able to cut it down to something simple (inlineable in particular). # If so, there will be a good chance we might be able to const prop # all the way through and learn something new. - code = get(code_cache(interp), mi, nothing) - if isa(code, CodeInstance) - inferred = @atomic :monotonic code.inferred + if inf_result isa InferenceResult + inferred = inf_result.src # TODO propagate a specific `CallInfo` that conveys information about this call - if src_inlining_policy(interp, inferred, NoCallInfo(), IR_FLAG_NULL) + if src_inlining_policy(interp, mi, inferred, NoCallInfo(), IR_FLAG_NULL) return true end end @@ -1255,77 +1387,118 @@ end function semi_concrete_eval_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState) - world = frame_world(sv) - mi_cache = WorldView(code_cache(interp), world) - codeinst = get(mi_cache, mi, nothing) - if codeinst !== nothing - irsv = IRInterpretationState(interp, codeinst, mi, arginfo.argtypes, world) - if irsv !== nothing - assign_parentchild!(irsv, sv) - rt, (nothrow, noub) = ir_abstract_constant_propagation(interp, irsv) - @assert !(rt isa Conditional || rt isa MustAlias) "invalid lattice element returned from irinterp" - if !(isa(rt, Type) && hasintersect(rt, Bool)) - ir = irsv.ir - # TODO (#48913) enable double inlining pass when there are any calls - # that are newly resolved by irinterp - # state = InliningState(interp) - # ir = ssa_inlining_pass!(irsv.ir, state, propagate_inbounds(irsv)) - effects = result.effects - if nothrow - effects = Effects(effects; nothrow=true) - end - if noub - effects = Effects(effects; noub=ALWAYS_TRUE) - end - exct = refine_exception_type(result.exct, effects) - semi_concrete_result = SemiConcreteResult(codeinst, ir, effects, spec_info(irsv)) - const_edge = nothing # TODO use the edges from irsv? - return ConstCallResult(rt, exct, semi_concrete_result, effects, const_edge) - end - end - end - return nothing -end - -function const_prop_result(inf_result::InferenceResult) - @assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge" - const_prop_result = ConstPropResult(inf_result) - return ConstCallResult(inf_result.result, inf_result.exc_result, const_prop_result, - inf_result.ipo_effects, inf_result.ci_as_edge) + call_result = result.call_result + call_result isa LocalInferenceResult || return nothing + inf_result = call_result.result + edge = result.edge + edge isa CodeInstance || return nothing + proof = inference_proof(call_result) + inferred = inf_result.src + src_inlining_policy(interp, mi, inferred, NoCallInfo(), IR_FLAG_NULL) || return nothing # hack to work-around test failures caused by #58183 until both it and #48913 are fixed + irsv = IRInterpretationState(interp, edge, mi, arginfo.argtypes, inferred, + proof_worlds(proof)) + irsv === nothing && return nothing + assign_parentchild!(irsv, sv) + rt, (nothrow, noub) = ir_abstract_constant_propagation(interp, irsv) + @assert !(rt isa Conditional || rt isa MustAlias) "invalid lattice element returned from irinterp" + if !(isa(rt, Type) && hasintersect(rt, Bool)) + ir = irsv.ir + # TODO (#48913) enable double inlining pass when there are any calls + # that are newly resolved by irinterp + # state = InliningState(interp) + # ir = ssa_inlining_pass!(irsv.ir, state, propagate_inbounds(irsv)) + effects = result.effects + if nothrow + effects = Effects(effects; nothrow=true) + end + if noub + effects = Effects(effects; noub=ALWAYS_TRUE) + end + exct = refine_exception_type(result.exct, effects) + proof_edges = Any[] + add_inference_proof!(proof_edges, proof, edge) + for info in ir.stmts.info + add_edges!(proof_edges, info) + end + append!(proof_edges, irsv.edges) + semi_concrete_proof = LocalInferenceProof(irsv.valid_worlds, + Core.svec(proof_edges...)) + semi_concrete_result = SemiConcreteResult(edge, ir, effects, spec_info(irsv); + proof = semi_concrete_proof) + return ConstCallResult(rt, exct, semi_concrete_result, effects) + end + nothing +end + +function const_prop_result(local_result::LocalInferenceResult) + inf_result = local_result.result + return ConstCallResult(inf_result.result, inf_result.exc_result, local_result, + inf_result.ipo_effects) end # return cached result of constant analysis -return_localcache_result(::AbstractInterpreter, inf_result::InferenceResult, ::AbsIntState) = - const_prop_result(inf_result) +return_localcache_result(::AbstractInterpreter, local_result::LocalInferenceResult, ::AbsIntState) = + const_prop_result(local_result) + +function const_prop_inference_proof(frame::InferenceState, result::MethodCallResult, + concrete_eval_result::Union{Nothing,ConstCallResult}) + proof_edges = Any[] + valid_worlds = frame.valid_worlds + # This proof is cached independently of the executable target chosen at a + # particular call site, so it must remain self-contained. Pairing/elision is + # only valid later, when the cached result is attached to a concrete CallInfo. + add_result_proof!(proof_edges, result.call_result) + if result.call_result !== nothing + valid_worlds = intersect(valid_worlds, + proof_worlds(inference_proof(result.call_result))) + end + if concrete_eval_result !== nothing + add_result_proof!(proof_edges, concrete_eval_result.const_result) + valid_worlds = intersect(valid_worlds, + proof_worlds(inference_proof(concrete_eval_result.const_result))) + end + append!(proof_edges, frame.edges) + return LocalInferenceProof(valid_worlds, Core.svec(proof_edges...)) +end function compute_forwarded_argtypes(interp::AbstractInterpreter, arginfo::ArgInfo, sv::AbsIntState) 𝕃ᵢ = typeinf_lattice(interp) - return has_conditional(𝕃ᵢ, sv) ? ConditionalSimpleArgtypes(arginfo, sv) : SimpleArgtypes(arginfo.argtypes) + if has_mustalias(𝕃ᵢ, sv) || has_conditional(𝕃ᵢ, sv) + return ForwardableArgtypes(arginfo, sv) + end + return SimpleArgtypes(arginfo.argtypes) end function const_prop_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState, concrete_eval_result::Union{Nothing,ConstCallResult}=nothing) - inf_cache = get_inference_cache(interp) 𝕃ᵢ = typeinf_lattice(interp) forwarded_argtypes = compute_forwarded_argtypes(interp, arginfo, sv) # use `cache_argtypes` that has been constructed for fresh regular inference if available - volatile_inf_result = result.volatile_inf_result - if volatile_inf_result !== nothing - cache_argtypes = volatile_inf_result.inf_result.argtypes + call_result = result.call_result + if call_result isa LocalInferenceResult + cache_argtypes = call_result.result.argtypes else cache_argtypes = matching_cache_argtypes(𝕃ᵢ, mi) end argtypes = matching_cache_argtypes(𝕃ᵢ, mi, forwarded_argtypes, cache_argtypes) - inf_result = cache_lookup(𝕃ᵢ, mi, argtypes, inf_cache) - if inf_result !== nothing + argtypes = get_nospecializeinfer_argtypes(argtypes, cache_argtypes, mi.def::Method) + inf_result = constprop_cache_lookup(𝕃ᵢ, mi, argtypes, get_inference_cache(interp), + get_inference_world(interp)) + if inf_result === missing + # a previous const-prop attempt hit a cycle and produced a limited result; + # don't re-attempt the same work that would lead to the same limited outcome + add_remark!(interp, sv, "[constprop] Found cached but limited constant inference result") + return nothing + elseif inf_result isa LocalInferenceResult # found the cache for this constant prop' - if inf_result.result === nothing - add_remark!(interp, sv, "[constprop] Found cached constant inference in a cycle") - return nothing - end - @assert inf_result.linfo === mi "MethodInstance for cached inference result does not match" + @assert inf_result.result.linfo === mi "MethodInstance for cached inference result does not match" return return_localcache_result(interp, inf_result, sv) + elseif inf_result isa InferenceResult + # Raw entries are internal sentinels for an unresolved constant-inference cycle. + @assert inf_result.result === nothing + add_remark!(interp, sv, "[constprop] Found cached constant inference in a cycle") + return nothing end overridden_by_const = falses(length(argtypes)) for i = 1:length(argtypes) @@ -1350,39 +1523,62 @@ function const_prop_call(interp::AbstractInterpreter, sv.time_paused += frame.time_paused add_remark!(interp, sv, "[constprop] Fresh constant inference hit a cycle") @assert frame.frameid != 0 && frame.cycleid == frame.frameid - callstack = frame.callstack::Vector{AbsIntState} + callstack = frame.callstack @assert callstack[end] === frame && length(callstack) == frame.frameid pop!(callstack) + # add to the cache to record that this will always fail + inf_result.cache_world = get_inference_world(interp) + push!(get_inference_cache(interp), inf_result) + return nothing + end + if inf_result.tombstone + # This const-prop attempt resolved but hit a cycle and produced a limited result. + # The tombstoned entry is already cached by `promotecache!` via the normal local + # cache mechanism, so `constprop_cache_lookup` will find it on subsequent lookups. + add_remark!(interp, sv, "[constprop] Constant inference produced a limited result") return nothing end - existing_edge = result.edge - inf_result.ci_as_edge = codeinst_as_edge(interp, frame, existing_edge) @assert frame.frameid != 0 && frame.cycleid == frame.frameid @assert frame.parentid == sv.frameid @assert inf_result.result !== nothing - # ConditionalSimpleArgtypes is allowed, because the only case in which it modifies - # the argtypes is when one of the argtypes is a `Conditional`, which case - # concrete_eval_result will not be available. - if concrete_eval_result !== nothing && isa(forwarded_argtypes, Union{SimpleArgtypes, ConditionalSimpleArgtypes}) + # ForwardableArgtypes is allowed, because the only case in which it modifies + # the argtypes is when one of the argtypes is a `Conditional` or `MustAlias`, in + # which case concrete_eval_result will not be available (all args must be `Const`). + if concrete_eval_result !== nothing && isa(forwarded_argtypes, Union{SimpleArgtypes,ForwardableArgtypes}) # override return type and effects with concrete evaluation result if available inf_result.result = concrete_eval_result.rt inf_result.ipo_effects = concrete_eval_result.effects end - typ = inf_result.result - return const_prop_result(inf_result) + # The caller may retain regular-inference return/exception facts while taking + # only an effect (or another component) from constant propagation. Make the + # const result's proof certify every input to that component-wise merge, not + # merely the const-prop frame itself. + proof = const_prop_inference_proof(frame, result, concrete_eval_result) + local_result = LocalInferenceResult(inf_result, proof, get_inference_world(interp)) + push!(get_inference_cache(interp), local_result) + return const_prop_result(local_result) end -# TODO implement MustAlias forwarding - -struct ConditionalSimpleArgtypes +struct ForwardableArgtypes arginfo::ArgInfo sv::InferenceState end +function find_mustalias_target_arg(ma::MustAlias, fargs::Vector{Any}, sv::InferenceState) + slot = ma.slot + for i in 1:length(fargs) + arg = ssa_def_slot(fargs[i], sv) + if isa(arg, SlotNumber) && slot_id(arg) == slot + return i + end + end + return nothing +end + function matching_cache_argtypes(𝕃::AbstractLattice, mi::MethodInstance, - conditional_argtypes::ConditionalSimpleArgtypes, + forwardable_argtypes::ForwardableArgtypes, cache_argtypes::Vector{Any}) - (; arginfo, sv) = conditional_argtypes + (; arginfo, sv) = forwardable_argtypes (; fargs, argtypes) = arginfo given_argtypes = Vector{Any}(undef, length(argtypes)) def = mi.def::Method @@ -1404,11 +1600,22 @@ function matching_cache_argtypes(𝕃::AbstractLattice, mi::MethodInstance, # TODO bail out here immediately rather than just propagating Bottom ? given_argtypes[i] = Bottom else - given_argtypes[i] = Conditional(slotid, thentype, elsetype) + given_argtypes[i] = Conditional(slotid, #= ssadef =# 0, thentype, elsetype) end continue end end + # forward `MustAlias` if it conveys aliasing on any other argument + if isa(argtype, MustAlias) && fargs !== nothing + slotid = find_mustalias_target_arg(argtype, fargs, sv) + if slotid !== nothing + sigt = widenconst(slotid > nargs ? argtypes[slotid] : cache_argtypes[slotid]) + if ⊑(𝕃, argtype.vartyp, sigt) + given_argtypes[i] = MustAlias(slotid, 0, argtype.vartyp, argtype.fldidx, argtype.fldtyp) + continue + end + end + end given_argtypes[i] = widenslotwrapper(argtype) end return pick_const_args!(𝕃, given_argtypes, cache_argtypes) @@ -1423,6 +1630,9 @@ function ssa_def_slot(@nospecialize(arg), sv::InferenceState) init = arg.id arg = code[init] end + if isa(arg, Argument) + arg = SlotNumber(arg.n) + end if arg isa SlotNumber # found this kind of pattern: # %init = SlotNumber(x) @@ -1466,7 +1676,7 @@ function ssa_def_slot(@nospecialize(arg), sv::InferenceState) end # No slots in irinterp -ssa_def_slot(@nospecialize(arg), sv::IRInterpretationState) = nothing +ssa_def_slot(@nospecialize(arg), ::IRInterpretationState) = nothing struct AbstractIterationResult cti::Vector{Any} @@ -1482,9 +1692,9 @@ AbstractIterationResult(cti::Vector{Any}, info::MaybeAbstractIterationInfo) = # Union of Tuples of the same length is converted to Tuple of Unions. # returns an array of types function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft), @nospecialize(typ), - sv::AbsIntState) + vtypes::Union{VarTable,Nothing}, sv::AbsIntState) if isa(typ, PartialStruct) - widet = typ.typ + widet = unwrap_unionall(typ.typ) if isa(widet, DataType) if widet.name === Tuple.name return Future(AbstractIterationResult(typ.fields, nothing)) @@ -1568,150 +1778,229 @@ function precise_container_type(interp::AbstractInterpreter, @nospecialize(itft) end return Future(AbstractIterationResult(Any[Vararg{eltype(tti0)}], nothing)) else - return abstract_iteration(interp, itft, typ, sv) - end + return abstract_iteration(interp, itft, typ, vtypes, sv) + end +end + +# State and captured context for the `iterate(arg)` / `iterate(arg, state)` continuations +# (`InferIterate` / `InferIterate2Arg` below). User-derived data (`iteratef`, `itertype`) is +# held here behind `@nospecialize` so that these continuations are not specialized on user +# types (cf. `CallInferenceState`, `AbstractApplyState`, which exist for the same reason). +mutable struct AbstractIterationState + stateordonet + stateordonet_widened + valtype + statetype + iteratef + itertype + may_have_terminated::Bool + nextstate::UInt8 + const vtypes::Union{VarTable,Nothing} + const iterateresult::Future{AbstractIterationResult} + const call1future::Future{CallMeta} + const calls::Vector{CallMeta} + const ret::Vector{Any} + call2future::Future{CallMeta} + function AbstractIterationState( + @nospecialize(iteratef), @nospecialize(itertype), vtypes::Union{VarTable,Nothing}, + iterateresult::Future{AbstractIterationResult}, call1future::Future{CallMeta} + ) + return new( + #=stateordonet=#Bottom, #=stateordonet_widened=#Bottom, #=valtype=#Bottom, #=statetype=#Bottom, + iteratef, itertype, #=may_have_terminated=#false, #=nextstate=#0x00, + vtypes, iterateresult, call1future, #=calls=#CallMeta[], #=ret=#Any[]) + end +end + +# continuation for `iterate(arg)` +struct InferIterate + state::AbstractIterationState +end +# continuation for `iterate(arg, state)` +struct InferIterate2Arg + state::AbstractIterationState end # simulate iteration protocol on container type up to fixpoint -function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @nospecialize(itertype), sv::AbsIntState) +function abstract_iteration(interp::AbstractInterpreter, @nospecialize(itft), @nospecialize(itertype), + vtypes::Union{VarTable,Nothing}, sv::AbsIntState) if isa(itft, Const) iteratef = itft.val else return Future(AbstractIterationResult(Any[Vararg{Any}], nothing, Effects())) end @assert !isvarargtype(itertype) - iterateresult = Future{AbstractIterationResult}() - call1future = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[itft, itertype]), StmtInfo(true, false), sv)::Future - function inferiterate(interp, sv) - call1 = call1future[] - stateordonet = call1.rt - # Return Bottom if this is not an iterator. - # WARNING: Changes to the iteration protocol must be reflected here, - # this is not just an optimization. - # TODO: this doesn't realize that Array, GenericMemory, SimpleVector, Tuple, and NamedTuple do not use the iterate protocol - if stateordonet === Bottom - iterateresult[] = AbstractIterationResult(Any[Bottom], AbstractIterationInfo(CallMeta[CallMeta(Bottom, Any, call1.effects, call1.info)], true)) + call1future = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[itft, itertype]), StmtInfo(true, false), vtypes, sv)::Future{CallMeta} + state = AbstractIterationState(iteratef, itertype, vtypes, iterateresult, call1future) + inferiterate = InferIterate(state) + # continue making progress as soon as possible, on iterate(arg) + if !(isready(call1future) && inferiterate(interp, sv)) + push!(sv.tasks, inferiterate) + end + return iterateresult +end + +function (inferiterate::InferIterate)(interp, sv) + state = inferiterate.state + call1 = state.call1future[] + # Return Bottom if this is not an iterator. + # WARNING: Changes to the iteration protocol must be reflected here, + # this is not just an optimization. + # TODO: this doesn't realize that Array, GenericMemory, SimpleVector, Tuple, and NamedTuple do not use the iterate protocol + if call1.rt === Bottom + state.iterateresult[] = AbstractIterationResult(Any[Bottom], AbstractIterationInfo(CallMeta[CallMeta(Bottom, Any, call1.effects, call1.info)], true)) + return true + end + push!(state.calls, call1) + state.stateordonet = call1.rt + state.stateordonet_widened = widenconst(call1.rt) + # continue making progress as much as possible, on iterate(arg, state) + inferiterate_2arg = InferIterate2Arg(state) + inferiterate_2arg(interp, sv) || push!(sv.tasks, inferiterate_2arg) + return true +end + +function (inferiterate_2arg::InferIterate2Arg)(interp, sv) + state = inferiterate_2arg.state + 𝕃ᵢ = typeinf_lattice(interp) + iteratef = state.iteratef + itertype = state.itertype + vtypes = state.vtypes + calls = state.calls + ret = state.ret + iterateresult = state.iterateresult + if state.nextstate === 0x1 + state.nextstate = 0xff + @goto state1 + elseif state.nextstate === 0x2 + state.nextstate = 0xff + @goto state2 + else + @assert state.nextstate === 0x0 + state.nextstate = 0xff + end + + # Try to unroll the iteration up to max_tuple_splat, which covers any finite + # length iterators, or interesting prefix + while true + if state.stateordonet_widened === Nothing + iterateresult[] = AbstractIterationResult(ret, AbstractIterationInfo(calls, true)) return true end - stateordonet_widened = widenconst(stateordonet) - calls = CallMeta[call1] - valtype = statetype = Bottom - ret = Any[] - 𝕃ᵢ = typeinf_lattice(interp) - may_have_terminated = false - local call2future::Future{CallMeta} - - nextstate::UInt8 = 0x0 - function inferiterate_2arg(interp, sv) - if nextstate === 0x1 - nextstate = 0xff - @goto state1 - elseif nextstate === 0x2 - nextstate = 0xff - @goto state2 - else - @assert nextstate === 0x0 - nextstate = 0xff - end - - # Try to unroll the iteration up to max_tuple_splat, which covers any finite - # length iterators, or interesting prefix - while true - if stateordonet_widened === Nothing - iterateresult[] = AbstractIterationResult(ret, AbstractIterationInfo(calls, true)) - return true - end - if Nothing <: stateordonet_widened || length(ret) >= InferenceParams(interp).max_tuple_splat - break - end - if !isa(stateordonet_widened, DataType) || !(stateordonet_widened <: Tuple) || isvatuple(stateordonet_widened) || length(stateordonet_widened.parameters) != 2 - break - end - nstatetype = getfield_tfunc(𝕃ᵢ, stateordonet, Const(2)) - # If there's no new information in this statetype, don't bother continuing, - # the iterator won't be finite. - if ⊑(𝕃ᵢ, nstatetype, statetype) - iterateresult[] = AbstractIterationResult(Any[Bottom], AbstractIterationInfo(calls, false), EFFECTS_THROWS) + if Nothing <: state.stateordonet_widened || length(ret) >= InferenceParams(interp).max_tuple_splat + break + end + if (!isa(state.stateordonet_widened, DataType) || + !(state.stateordonet_widened <: Tuple) || + isvatuple(state.stateordonet_widened) || + length(state.stateordonet_widened.parameters) != 2) + break + end + nstatetype = getfield_tfunc(𝕃ᵢ, state.stateordonet, Const(2)) + # If there's no new information in this statetype, don't bother continuing, + # the iterator won't be finite. + if ⊑(𝕃ᵢ, nstatetype, state.statetype) + iterateresult[] = AbstractIterationResult(Any[Bottom], AbstractIterationInfo(calls, false), EFFECTS_THROWS) + return true + end + state.valtype = getfield_tfunc(𝕃ᵢ, state.stateordonet, Const(1)) + push!(ret, state.valtype) + state.statetype = nstatetype + state.call2future = abstract_call_known( + interp, iteratef, ArgInfo(nothing, Any[Const(iteratef), itertype, state.statetype]), + StmtInfo(true, false), vtypes, sv)::Future{CallMeta} + if !isready(state.call2future) + state.nextstate = 0x1 + return false + @label state1 + end + let call = state.call2future[] + push!(calls, call) + state.stateordonet = call.rt + state.stateordonet_widened = widenconst(state.stateordonet) + end + end + # From here on, we start asking for results on the widened types, rather than + # the precise (potentially const) state type + # statetype and valtype are reinitialized in the first iteration below from the + # (widened) stateordonet, which has not yet been fully analyzed in the loop above + state.valtype = state.statetype = Bottom + state.may_have_terminated = Nothing <: state.stateordonet_widened + while state.valtype !== Any + nounion = typeintersect(state.stateordonet_widened, Tuple{Any,Any}) + if nounion !== Union{} && !isa(nounion, DataType) + # nounion is of a type we cannot handle + state.valtype = Any + break + end + if nounion === Union{} || (nounion.parameters[1] <: state.valtype && nounion.parameters[2] <: state.statetype) + # reached a fixpoint or iterator failed/gave invalid answer + if !hasintersect(state.stateordonet_widened, Nothing) + # ... but cannot terminate + if state.may_have_terminated + # ... and iterator may have terminated prior to this loop, but not during it + state.valtype = Bottom + else + # ... or cannot have terminated prior to this loop + iterateresult[] = AbstractIterationResult(Any[Bottom], AbstractIterationInfo(calls, false), Effects()) return true end - valtype = getfield_tfunc(𝕃ᵢ, stateordonet, Const(1)) - push!(ret, valtype) - statetype = nstatetype - call2future = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[Const(iteratef), itertype, statetype]), StmtInfo(true, false), sv)::Future - if !isready(call2future) - nextstate = 0x1 - return false - @label state1 - end - let call = call2future[] - push!(calls, call) - stateordonet = call.rt - stateordonet_widened = widenconst(stateordonet) - end - end - # From here on, we start asking for results on the widened types, rather than - # the precise (potentially const) state type - # statetype and valtype are reinitialized in the first iteration below from the - # (widened) stateordonet, which has not yet been fully analyzed in the loop above - valtype = statetype = Bottom - may_have_terminated = Nothing <: stateordonet_widened - while valtype !== Any - nounion = typeintersect(stateordonet_widened, Tuple{Any,Any}) - if nounion !== Union{} && !isa(nounion, DataType) - # nounion is of a type we cannot handle - valtype = Any - break - end - if nounion === Union{} || (nounion.parameters[1] <: valtype && nounion.parameters[2] <: statetype) - # reached a fixpoint or iterator failed/gave invalid answer - if !hasintersect(stateordonet_widened, Nothing) - # ... but cannot terminate - if may_have_terminated - # ... and iterator may have terminated prior to this loop, but not during it - valtype = Bottom - else - # ... or cannot have terminated prior to this loop - iterateresult[] = AbstractIterationResult(Any[Bottom], AbstractIterationInfo(calls, false), Effects()) - return true - end - end - break - end - valtype = tmerge(valtype, nounion.parameters[1]) - statetype = tmerge(statetype, nounion.parameters[2]) - call2future = abstract_call_known(interp, iteratef, ArgInfo(nothing, Any[Const(iteratef), itertype, statetype]), StmtInfo(true, false), sv)::Future - if !isready(call2future) - nextstate = 0x2 - return false - @label state2 - end - let call = call2future[] - push!(calls, call) - stateordonet = call.rt - stateordonet_widened = widenconst(stateordonet) - end - end - if valtype !== Union{} - push!(ret, Vararg{valtype}) end - iterateresult[] = AbstractIterationResult(ret, AbstractIterationInfo(calls, false)) - return true - end # function inferiterate_2arg - # continue making progress as much as possible, on iterate(arg, state) - inferiterate_2arg(interp, sv) || push!(sv.tasks, inferiterate_2arg) - return true - end # inferiterate - # continue making progress as soon as possible, on iterate(arg) - if !(isready(call1future) && inferiterate(interp, sv)) - push!(sv.tasks, inferiterate) + break + end + state.valtype = tmerge(state.valtype, nounion.parameters[1]) + state.statetype = tmerge(state.statetype, nounion.parameters[2]) + state.call2future = abstract_call_known( + interp, iteratef, ArgInfo(nothing, Any[Const(iteratef), itertype, state.statetype]), + StmtInfo(true, false), vtypes, sv)::Future{CallMeta} + if !isready(state.call2future) + state.nextstate = 0x2 + return false + @label state2 + end + let call = state.call2future[] + push!(calls, call) + state.stateordonet = call.rt + state.stateordonet_widened = widenconst(state.stateordonet) + end + end + if state.valtype !== Union{} + push!(ret, Vararg{state.valtype}) + end + iterateresult[] = AbstractIterationResult(ret, AbstractIterationInfo(calls, false)) + return true +end + +mutable struct AbstractApplyState + res + exctype + i::Int + j::Int + nextstate::UInt8 + all_effects::Effects + ctypes::Vector{Vector{Any}} + ctypes´::Vector{Vector{Any}} + infos::Vector{Vector{MaybeAbstractIterationInfo}} + infos´::Vector{Vector{MaybeAbstractIterationInfo}} + argtypesi::Vector{Any} + ctfuture::Future{AbstractIterationResult} + callfuture::Future{CallMeta} + function AbstractApplyState( + res, exctype, i::Int, j::Int, nextstate::UInt8, all_effects::Effects, + ctypes::Vector{Vector{Any}}, ctypes´::Vector{Vector{Any}}, + infos::Vector{Vector{MaybeAbstractIterationInfo}}, + infos´::Vector{Vector{MaybeAbstractIterationInfo}}, + ) + @nospecialize res exctype + return new(res, exctype, i, j, nextstate, all_effects, ctypes, ctypes´, infos, infos´) end - return iterateresult end # do apply(af, fargs...), where af is a function value function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si::StmtInfo, - sv::AbsIntState, max_methods::Int=get_max_methods(interp, sv)) + vtypes::Union{VarTable,Nothing}, sv::AbsIntState, + max_methods::Int=get_max_methods(interp, sv)) itft = Core.Box(argtype_by_index(argtypes, 2)) aft = argtype_by_index(argtypes, 3) (itft.contents === Bottom || aft === Bottom) && return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) @@ -1725,66 +2014,55 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si:: return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) end end - res = Union{} splitunions = 1 < unionsplitcost(typeinf_lattice(interp), aargtypes) <= InferenceParams(interp).max_apply_union_enum - ctypes::Vector{Vector{Any}} = [Any[aft]] - infos::Vector{Vector{MaybeAbstractIterationInfo}} = Vector{MaybeAbstractIterationInfo}[MaybeAbstractIterationInfo[]] - all_effects::Effects = EFFECTS_TOTAL retinfos = ApplyCallInfo[] retinfo = UnionSplitApplyCallInfo(retinfos) - exctype = Union{} - ctypes´::Vector{Vector{Any}} = Vector{Any}[] - infos´::Vector{Vector{MaybeAbstractIterationInfo}} = Vector{MaybeAbstractIterationInfo}[] - local ti, argtypesi - local ctfuture::Future{AbstractIterationResult} - local callfuture::Future{CallMeta} - applyresult = Future{CallMeta}() - # split the rest into a resumable state machine - i::Int = 1 - j::Int = 1 - nextstate::UInt8 = 0x0 + state = AbstractApplyState(Union{}, Union{}, 1, 1, 0x00, EFFECTS_TOTAL, + Vector{Any}[Any[aft]], Vector{Any}[], + Vector{MaybeAbstractIterationInfo}[MaybeAbstractIterationInfo[]], Vector{MaybeAbstractIterationInfo}[]) + function infercalls(interp, sv) # n.b. Remember that variables will lose their values across restarts, # so be sure to manually hoist any values that must be preserved and do # not rely on program order. # This is a little more complex than the closure continuations often used elsewhere, but avoids needing to manage all of that indentation - if nextstate === 0x1 - nextstate = 0xff + if state.nextstate === 0x1 + state.nextstate = 0xff @goto state1 - elseif nextstate === 0x2 - nextstate = 0xff + elseif state.nextstate === 0x2 + state.nextstate = 0xff @goto state2 - elseif nextstate === 0x3 - nextstate = 0xff + elseif state.nextstate === 0x3 + state.nextstate = 0xff @goto state3 else - @assert nextstate === 0x0 - nextstate = 0xff - end - while i <= length(aargtypes) - argtypesi = (splitunions ? uniontypes(aargtypes[i]) : Any[aargtypes[i]]) - i += 1 - j = 1 - while j <= length(argtypesi) - ti = argtypesi[j] - j += 1 + @assert state.nextstate === 0x0 + state.nextstate = 0xff + end + while state.i <= length(aargtypes) + state.argtypesi = (splitunions ? uniontypes(aargtypes[state.i]) : Any[aargtypes[state.i]]) + state.i += 1 + state.j = 1 + while state.j <= length(state.argtypesi) + ti = state.argtypesi[state.j] + state.j += 1 if !isvarargtype(ti) - ctfuture = precise_container_type(interp, itft.contents, ti, sv)::Future - if !isready(ctfuture) - nextstate = 0x1 + state.ctfuture = precise_container_type(interp, itft.contents, ti, vtypes, sv)::Future{AbstractIterationResult} + if !isready(state.ctfuture) + state.nextstate = 0x1 return false @label state1 end - (;cti, info, ai_effects) = ctfuture[] + (;cti, info, ai_effects) = state.ctfuture[] else - ctfuture = precise_container_type(interp, itft.contents, unwrapva(ti), sv)::Future - if !isready(ctfuture) - nextstate = 0x2 + state.ctfuture = precise_container_type(interp, itft.contents, unwrapva(ti), vtypes, sv)::Future{AbstractIterationResult} + if !isready(state.ctfuture) + state.nextstate = 0x2 return false @label state2 end - (;cti, info, ai_effects) = ctfuture[] + (;cti, info, ai_effects) = state.ctfuture[] # We can't represent a repeating sequence of the same types, # so tmerge everything together to get one type that represents # everything. @@ -1797,41 +2075,41 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si:: end cti = Any[Vararg{argt}] end - all_effects = merge_effects(all_effects, ai_effects) + state.all_effects = merge_effects(state.all_effects, ai_effects) if info !== nothing for call in info.each - all_effects = merge_effects(all_effects, call.effects) + state.all_effects = merge_effects(state.all_effects, call.effects) end end if any(@nospecialize(t) -> t === Bottom, cti) continue end - for k = 1:length(ctypes) - ct = ctypes[k] + for k = 1:length(state.ctypes) + ct = state.ctypes[k] if isvarargtype(ct[end]) # This is vararg, we're not gonna be able to do any inlining, # drop the info info = nothing tail = tuple_tail_elem(typeinf_lattice(interp), unwrapva(ct[end]), cti) - push!(ctypes´, push!(ct[1:(end - 1)], tail)) + push!(state.ctypes´, push!(ct[1:(end - 1)], tail)) else - push!(ctypes´, append!(ct[:], cti)) + push!(state.ctypes´, append!(ct[:], cti)) end - push!(infos´, push!(copy(infos[k]), info)) + push!(state.infos´, push!(copy(state.infos[k]), info)) end end # swap for the new array and empty the temporary one - ctypes´, ctypes = ctypes, ctypes´ - infos´, infos = infos, infos´ - empty!(ctypes´) - empty!(infos´) + state.ctypes´, state.ctypes = state.ctypes, state.ctypes´ + state.infos´, state.infos = state.infos, state.infos´ + empty!(state.ctypes´) + empty!(state.infos´) end - all_effects.nothrow || (exctype = Any) + state.all_effects.nothrow || (state.exctype = Any) - i = 1 - while i <= length(ctypes) - ct = ctypes[i] - if bail_out_apply(interp, InferenceLoopState(res, all_effects), sv) + state.i = 1 + while state.i <= length(state.ctypes) + ct = state.ctypes[state.i] + if bail_out_apply(interp, InferenceLoopState(state.res, state.all_effects), sv) add_remark!(interp, sv, "_apply_iterate inference reached maximally imprecise information: bailing on analysis of more methods.") # there is unanalyzed candidate, widen type and effects to the top let retinfo = NoCallInfo() # NOTE this is necessary to prevent the inlining processing @@ -1849,23 +2127,23 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si:: break end end - callfuture = abstract_call(interp, ArgInfo(nothing, ct), si, sv, max_methods)::Future - if !isready(callfuture) - nextstate = 0x3 + state.callfuture = abstract_call(interp, ArgInfo(nothing, ct), si, vtypes, sv, max_methods)::Future{CallMeta} + if !isready(state.callfuture) + state.nextstate = 0x3 return false @label state3 end - let (; info, rt, exct, effects) = callfuture[] - push!(retinfos, ApplyCallInfo(info, infos[i])) - res = tmerge(typeinf_lattice(interp), res, rt) - exctype = tmerge(typeinf_lattice(interp), exctype, exct) - all_effects = merge_effects(all_effects, effects) + let (; info, rt, exct, effects) = state.callfuture[] + push!(retinfos, ApplyCallInfo(info, state.infos[state.i])) + state.res = tmerge(typeinf_lattice(interp), state.res, rt) + state.exctype = tmerge(typeinf_lattice(interp), state.exctype, exct) + state.all_effects = merge_effects(state.all_effects, effects) end - i += 1 + state.i += 1 end # TODO: Add a special info type to capture all the iteration info. # For now, only propagate info if we don't also union-split the iteration - applyresult[] = CallMeta(res, exctype, all_effects, retinfo) + applyresult[] = CallMeta(state.res, state.exctype, state.all_effects, retinfo) return true end # function infercalls # start making progress on the first call @@ -1951,7 +2229,7 @@ end end function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs, argtypes)::ArgInfo, - sv::AbsIntState) + vtypes::Union{VarTable,Nothing}, sv::AbsIntState) @nospecialize f la = length(argtypes) 𝕃ᵢ = typeinf_lattice(interp) @@ -1992,7 +2270,9 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs fldidx = maybe_const_fldidx(vartyp, a3.val) if fldidx !== nothing # wrap this aliasable field into `MustAlias` for possible constraint propagations - return MustAlias(var, vartyp, fldidx, rt) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(var)] + return MustAlias(var, vtyp.ssadef, vartyp, fldidx, rt) end end end @@ -2007,7 +2287,9 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs if isa(a, SlotNumber) cndt = isa_condition(a2, a3, InferenceParams(interp).max_union_splitting, rt) if cndt !== nothing - return Conditional(a, cndt.thentype, cndt.elsetype) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(a)] + return Conditional(a, vtyp.ssadef, cndt.thentype, cndt.elsetype) end end if isa(a2, MustAlias) @@ -2025,7 +2307,9 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs # !(x isa T) implies !(Type{a2} <: T) # TODO: complete splitting, based on which portions of the Union a3 for which isa_tfunc returns Const(true) or Const(false) instead of Bool elsetype = typesubtract(a3, Type{widenconst(a2)}, InferenceParams(interp).max_union_splitting) - return Conditional(b, a3, elsetype) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(b)] + return Conditional(b, vtyp.ssadef, a3, elsetype) end end elseif f === (===) @@ -2036,16 +2320,20 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs # if doing a comparison to a singleton, consider returning a `Conditional` instead if isa(aty, Const) if isa(b, SlotNumber) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(b)] cndt = egal_condition(aty, bty, InferenceParams(interp).max_union_splitting, rt) - return Conditional(b, cndt.thentype, cndt.elsetype) + return Conditional(b, vtyp.ssadef, cndt.thentype, cndt.elsetype) elseif isa(bty, MustAlias) && !isa(rt, Const) # skip refinement when the field is known precisely (just optimization) cndt = egal_condition(aty, bty.fldtyp, InferenceParams(interp).max_union_splitting) return form_mustalias_conditional(bty, cndt.thentype, cndt.elsetype) end elseif isa(bty, Const) if isa(a, SlotNumber) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(a)] cndt = egal_condition(bty, aty, InferenceParams(interp).max_union_splitting, rt) - return Conditional(a, cndt.thentype, cndt.elsetype) + return Conditional(a, vtyp.ssadef, cndt.thentype, cndt.elsetype) elseif isa(aty, MustAlias) && !isa(rt, Const) # skip refinement when the field is known precisely (just optimization) cndt = egal_condition(bty, aty.fldtyp, InferenceParams(interp).max_union_splitting) return form_mustalias_conditional(aty, cndt.thentype, cndt.elsetype) @@ -2076,18 +2364,24 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs if isa(b, SlotNumber) thentype = rt === Const(false) ? Bottom : widenslotwrapper(bty) elsetype = rt === Const(true) ? Bottom : widenslotwrapper(bty) - return Conditional(b, thentype, elsetype) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(b)] + return Conditional(b, vtyp.ssadef, thentype, elsetype) elseif isa(a, SlotNumber) thentype = rt === Const(false) ? Bottom : widenslotwrapper(aty) elsetype = rt === Const(true) ? Bottom : widenslotwrapper(aty) - return Conditional(a, thentype, elsetype) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(a)] + return Conditional(a, vtyp.ssadef, thentype, elsetype) end elseif f === Core.Intrinsics.not_int aty = argtypes[2] if isa(aty, Conditional) thentype = rt === Const(false) ? Bottom : aty.elsetype elsetype = rt === Const(true) ? Bottom : aty.thentype - return Conditional(aty.slot, thentype, elsetype) + @assert vtypes !== nothing + vtyp = vtypes[aty.slot] + return Conditional(aty.slot, vtyp.ssadef, thentype, elsetype) end elseif f === isdefined a = ssa_def_slot(fargs[2], sv) @@ -2110,7 +2404,9 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs elsetype = elsetype ⊔ ty end end - return Conditional(a, thentype, elsetype) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(a)] + return Conditional(a, vtyp.ssadef, thentype, elsetype) else thentype = form_partially_defined_struct(𝕃ᵢ, argtype2, argtypes[3]) if thentype !== nothing @@ -2120,7 +2416,9 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs elseif rt === Const(true) elsetype = Bottom end - return Conditional(a, thentype, elsetype) + @assert vtypes !== nothing + vtyp = vtypes[slot_id(a)] + return Conditional(a, vtyp.ssadef, thentype, elsetype) end end end @@ -2184,8 +2482,11 @@ function abstract_call_unionall(interp::AbstractInterpreter, argtypes::Vector{An canconst = true if isa(a3, Const) body = a3.val + elseif isconstType(a3) + # the body value is pinned exactly (`===`) + body = type_parameter(a3) elseif isType(a3) - body = a3.parameters[1] + body = type_parameter(a3) canconst = false else return CallMeta(Any, Any, Effects(EFFECTS_TOTAL; nothrow), call.info) @@ -2215,40 +2516,40 @@ function get_ci_abi(ci::CodeInstance) (def::MethodInstance).specTypes end -function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState) +function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, vtypes::Union{VarTable,Nothing}, sv::AbsIntState) argtypes = arginfo.argtypes ft′ = argtype_by_index(argtypes, 2) ft = widenconst(ft′) ft === Bottom && return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) types = argtype_by_index(argtypes, 3) + our_world = get_inference_world(interp) if types isa Const && types.val isa Union{Method, CodeInstance} method_or_ci = types.val if isa(method_or_ci, CodeInstance) - our_world = sv.world.this argtype = argtypes_to_type(pushfirst!(argtype_tail(argtypes, 4), ft)) specsig = get_ci_abi(method_or_ci) defdef = get_ci_mi(method_or_ci).def - exct = method_or_ci.exctype + exct_ci = method_or_ci.exctype if !hasintersect(argtype, specsig) return Future(CallMeta(Bottom, TypeError, EFFECTS_THROWS, NoCallInfo())) elseif !(argtype <: specsig) || ((!isa(method_or_ci.def, ABIOverride) && isa(defdef, Method)) && !(argtype <: defdef.sig)) - exct = Union{exct, TypeError} + exct_ci = Union{exct_ci, TypeError} end callee_valid_range = WorldRange(method_or_ci.min_world, method_or_ci.max_world) if !(our_world in callee_valid_range) if our_world < first(callee_valid_range) - update_valid_age!(sv, WorldRange(first(sv.world.valid_worlds), first(callee_valid_range)-1)) + update_valid_age!(sv, our_world, WorldRange(first(sv.valid_worlds), first(callee_valid_range)-1)) else - update_valid_age!(sv, WorldRange(last(callee_valid_range)+1, last(sv.world.valid_worlds))) + update_valid_age!(sv, our_world, WorldRange(last(callee_valid_range)+1, last(sv.valid_worlds))) end return Future(CallMeta(Bottom, ErrorException, EFFECTS_THROWS, NoCallInfo())) end # TODO: When we add curing, we may want to assume this is nothrow - if (method_or_ci.owner === Nothing && method_ir_ci.def.def isa Method) - exct = Union{exct, ErrorException} + if (method_or_ci.owner === Nothing && method_or_ci.def.def isa Method) + exct_ci = Union{exct_ci, ErrorException} end - update_valid_age!(sv, callee_valid_range) - return Future(CallMeta(method_or_ci.rettype, exct, Effects(decode_effects(method_or_ci.ipo_purity_bits), nothrow=(exct===Bottom)), + update_valid_age!(sv, our_world, callee_valid_range) + return Future(CallMeta(method_or_ci.rettype, exct_ci, Effects(decode_effects(method_or_ci.ipo_purity_bits), nothrow=(exct_ci===Bottom)), InvokeCICallInfo(method_or_ci))) else method = method_or_ci::Method @@ -2262,7 +2563,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt end else hasintersect(widenconst(types), Union{Method, CodeInstance}) && return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) - (types, isexact, isconcrete, istype) = instanceof_tfunc(argtype_by_index(argtypes, 3), false) + types, isexact, _, _ = instanceof_tfunc(argtype_by_index(argtypes, 3), false) isexact || return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) unwrapped = unwrap_unionall(types) types === Bottom && return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) @@ -2274,13 +2575,12 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt nargtype === Bottom && return Future(CallMeta(Bottom, TypeError, EFFECTS_THROWS, NoCallInfo())) nargtype isa DataType || return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) # other cases are not implemented below isdispatchelem(ft) || return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below - ft = ft::DataType lookupsig = rewrap_unionall(Tuple{ft, unwrapped.parameters...}, types)::Type nargtype = Tuple{ft, nargtype.parameters...} argtype = Tuple{ft, argtype.parameters...} matched, valid_worlds = findsup(lookupsig, method_table(interp)) matched === nothing && return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) - update_valid_age!(sv, valid_worlds) + update_valid_age!(sv, our_world, valid_worlds) method = matched.method end tienv = ccall(:jl_type_intersection_with_env, Any, (Any, Any), nargtype, method.sig)::SimpleVector @@ -2292,7 +2592,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt lookupsig_box = Core.Box(lookupsig) invokecall = InvokeCall(types) return Future{CallMeta}(mresult, interp, sv) do result, interp, sv - (; rt, exct, effects, edge, volatile_inf_result) = result + (; rt, exct, effects, edge, call_result) = result local ft′ = ft′_box.contents sig = match.spec_types argtypes′ = invoke_rewrite(arginfo.argtypes) @@ -2309,22 +2609,23 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt f = singleton_type(ft′) const_call_result = abstract_call_method_with_const_args(interp, result, f, arginfo′, si, match, sv, invokecall) - const_result = volatile_inf_result if const_call_result !== nothing - const_edge = nothing + const_result = nothing if const_call_result.rt ⊑ rt - (; rt, effects, const_result, const_edge) = const_call_result + (; rt, effects, const_result) = const_call_result end if const_call_result.exct ⋤ exct - (; exct, const_result, const_edge) = const_call_result + (; exct, const_result) = const_call_result end - if const_edge !== nothing - edge = const_edge - update_valid_age!(sv, world_range(const_edge)) + if const_result !== nothing + update_valid_age!(sv, get_inference_world(interp), + proof_worlds(inference_proof(const_result))) + call_result = const_result end end - rt = from_interprocedural!(interp, rt, sv, arginfo′, sig) - info = InvokeCallInfo(edge, match, const_result, lookupsig_box.contents) + rt = from_interprocedural!(interp, rt, sv, arginfo′, sig, vtypes) + info = InvokeCallInfo(edge, match, call_result, lookupsig_box.contents, + result.needs_mi_edge) if !match.fully_covers effects = Effects(effects; nothrow=false) exct = exct ⊔ TypeError @@ -2340,11 +2641,11 @@ function invoke_rewrite(xs::Vector{Any}) return newxs end -function abstract_finalizer(interp::AbstractInterpreter, argtypes::Vector{Any}, sv::AbsIntState) +function abstract_finalizer(interp::AbstractInterpreter, argtypes::Vector{Any}, vtypes, sv::AbsIntState) if length(argtypes) == 3 finalizer_argvec = Any[argtypes[2], argtypes[3]] - call = abstract_call(interp, ArgInfo(nothing, finalizer_argvec), StmtInfo(false, false), sv, #=max_methods=#1)::Future - return Future{CallMeta}(call, interp, sv) do call, interp, sv + call = abstract_call(interp, ArgInfo(nothing, finalizer_argvec), StmtInfo(false, false), vtypes, sv, #=max_methods=#1)::Future + return Future{CallMeta}(call, interp, sv) do call, _, _ return CallMeta(Nothing, Any, Effects(), FinalizerInfo(call.info, call.effects)) end end @@ -2369,7 +2670,7 @@ function abstract_throw(interp::AbstractInterpreter, argtypes::Vector{Any}, ::Ab return Future(CallMeta(Union{}, exct, EFFECTS_THROWS, NoCallInfo())) end -function abstract_throw_methoderror(interp::AbstractInterpreter, argtypes::Vector{Any}, ::AbsIntState) +function abstract_throw_methoderror(::AbstractInterpreter, argtypes::Vector{Any}, ::AbsIntState) exct = if length(argtypes) == 1 ArgumentError elseif !isvarargtype(argtypes[2]) @@ -2429,6 +2730,9 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s end end +# The binding lookup code uses the current world to bound its scan to only those worlds that are currently valid +binding_world_hints(world::UInt, sv::AbsIntState) = WorldWithRange(world, sv.valid_worlds) + @nospecs function abstract_eval_get_binding_type(interp::AbstractInterpreter, sv::AbsIntState, M, s) @nospecialize M s ⊑ = partialorder(typeinf_lattice(interp)) @@ -2438,21 +2742,19 @@ end return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) end gr = GlobalRef(M, s) - (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition - local rt - kind = binding_kind(partition) - if is_some_guard(kind) || kind == PARTITION_KIND_DECLARED - # We do not currently assume an invalidation for guard -> defined transitions - # rt = Const(nothing) - rt = Type - elseif is_some_const_binding(kind) - rt = Const(Any) - else - rt = Const(partition_restriction(partition)) - end - rt + world = get_inference_world(interp) + valid_worlds, (_, partition) = binding_access_range(gr, binding_world_hints(world, sv), false) + update_valid_age!(sv, world, valid_worlds) + kind = binding_kind(partition) + if is_some_guard(kind) || kind == PARTITION_KIND_DECLARED + # We do not currently assume an invalidation for guard -> defined transitions + # rt = Const(nothing) + rt = Type + elseif is_some_const_binding(kind) + rt = Const(Any) + else + rt = Const(partition_restriction(partition)) end - update_valid_age!(sv, valid_worlds) return CallMeta(rt, Union{}, EFFECTS_TOTAL, GlobalAccessInfo(convert(Core.Binding, gr))) elseif !hasintersect(widenconst(M), Module) || !hasintersect(widenconst(s), Symbol) return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) @@ -2486,7 +2788,7 @@ function abstract_eval_setglobal!(interp::AbstractInterpreter, sv::AbsIntState, (rt, exct) = global_assignment_rt_exct(interp, sv, saw_latestworld, gr, v) return CallMeta(rt, exct, Effects(setglobal!_effects, nothrow=exct===Bottom), GlobalAccessInfo(convert(Core.Binding, gr))) end - return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) + return CallMeta(Union{}, Union{TypeError, ErrorException}, EFFECTS_THROWS, NoCallInfo()) end ⊑ = partialorder(typeinf_lattice(interp)) if !(hasintersect(widenconst(M), Module) && hasintersect(widenconst(s), Symbol)) @@ -2521,28 +2823,62 @@ function abstract_eval_setglobal!(interp::AbstractInterpreter, sv::AbsIntState, end end -function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, - @nospecialize(M), @nospecialize(s), @nospecialize(v)) - scm = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) - scm.rt === Bottom && return scm - gcm = abstract_eval_getglobal(interp, sv, saw_latestworld, M, s) - return CallMeta(gcm.rt, Union{scm.exct,gcm.exct}, merge_effects(scm.effects, gcm.effects), scm.info) -end - -function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, - @nospecialize(M), @nospecialize(s), @nospecialize(v), @nospecialize(order)) - scm = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v, order) - scm.rt === Bottom && return scm - gcm = abstract_eval_getglobal(interp, sv, saw_latestworld, M, s, order) - return CallMeta(gcm.rt, Union{scm.exct,gcm.exct}, merge_effects(scm.effects, gcm.effects), scm.info) +# Shared model for the read-modify-write global builtins (`swapglobal!`, `replaceglobal!`). +# The runtime resolves a single binding for the whole operation: `jl_get_binding_wr` (a +# `write=true` walk, which never follows imports -- a store through an import throws) picks +# the slot that is both read and written, so the read is modeled from that same own +# partition rather than from the leaf a `getglobal` would resolve. The read is observable +# without a successful store (`replaceglobal!` returns the old value when the comparison +# fails), so its exception type and effects are always merged in; conversely, when the store +# can never succeed the old value is never returned, so the result type is `Bottom`. +# Returns the operation's `CallMeta` -- whose `rt` is the type of the value read -- paired +# with the binding's declared type, or `nothing` if the partition does not declare one. +function abstract_eval_rmwglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, + @nospecialize(M), @nospecialize(s), @nospecialize(v)) + if isa(M, Const) && isa(s, Const) + M, s = M.val, s.val + if M isa Module && s isa Symbol + gr = GlobalRef(M, s) + info = GlobalAccessInfo(convert(Core.Binding, gr)) + if saw_latestworld + return Pair{CallMeta,Any}(CallMeta(Any, Any, + merge_effects(generic_getglobal_effects, setglobal!_effects), info), nothing) + end + world = get_inference_world(interp) + valid_worlds, (b, partition) = binding_access_range(gr, binding_world_hints(world, sv), true) + update_valid_age!(sv, world, valid_worlds) + rte = abstract_eval_partition_load(interp, b, partition) + (srt, sexct) = global_assignment_binding_rt_exct(interp, partition, v) + exct = Union{rte.exct, sexct} + effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=exct===Bottom)) + T = binding_kind(partition) == PARTITION_KIND_GLOBAL ? partition_restriction(partition) : nothing + return Pair{CallMeta,Any}(CallMeta(srt === Bottom ? Bottom : rte.rt, exct, effects, info), T) + end + return Pair{CallMeta,Any}(CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()), nothing) + end + ⊑ = partialorder(typeinf_lattice(interp)) + if !(hasintersect(widenconst(M), Module) && hasintersect(widenconst(s), Symbol)) + return Pair{CallMeta,Any}(CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()), nothing) + elseif M ⊑ Module && s ⊑ Symbol + exct = Union{UndefVarError, ErrorException} + else + exct = Union{UndefVarError, TypeError, ErrorException} + end + return Pair{CallMeta,Any}(CallMeta(Any, exct, + merge_effects(generic_getglobal_effects, setglobal!_effects), NoCallInfo()), nothing) end function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) if !isvarargtype(argtypes[end]) - if length(argtypes) == 4 - return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif length(argtypes) == 5 - return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) + if length(argtypes) in (4, 5) + cm = abstract_eval_rmwglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + sg = cm.first + if length(argtypes) == 5 + # a swap both loads and stores, so the order is validated once, as the runtime + # does with `jl_get_atomic_order_checked(order, #=loading=#1, #=storing=#1)` + sg = merge_exct(sg, global_order_exct(argtypes[5], #=loading=#true, #=storing=#true)) + end + return sg else return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) end @@ -2579,30 +2915,10 @@ end function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) if !isvarargtype(argtypes[end]) if length(argtypes) in (5, 6, 7) - (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] - T = nothing - if isa(M, Const) && isa(s, Const) - M, s = M.val, s.val - M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) - s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) - gr = GlobalRef(M, s) - v′ = RefValue{Any}(v) - (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition - partition_T = nothing - partition_rte = abstract_eval_partition_load(interp, binding, partition) - if binding_kind(partition) == PARTITION_KIND_GLOBAL - partition_T = partition_restriction(partition) - end - partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v′[])[2]} - partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) - Pair{RTEffects, Any}(partition_rte, partition_T) - end - update_valid_age!(sv, valid_worlds) - effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=rte.exct===Bottom)) - sg = CallMeta(Any, rte.exct, effects, GlobalAccessInfo(convert(Core.Binding, gr))) - else - sg = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) - end + # only the `desired` value (`argtypes[5]`) is type-checked against the binding; + # `expected` is merely compared + cm = abstract_eval_rmwglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[5]) + sg = cm.first if length(argtypes) >= 6 goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#true) sg = merge_exct(sg, goe) @@ -2611,6 +2927,8 @@ function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntSta goe = global_order_exct(argtypes[7], #=loading=#true, #=storing=#false) sg = merge_exct(sg, goe) end + sg.rt === Bottom && return sg + T = cm.second rt = T === nothing ? ccall(:jl_apply_cmpswap_type, Any, (Any,), S) where S : ccall(:jl_apply_cmpswap_type, Any, (Any,), T) @@ -2636,8 +2954,8 @@ end # call where the function is known exactly function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), - arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState, - max_methods::Int = get_max_methods(interp, f, sv)) + arginfo::ArgInfo, si::StmtInfo, vtypes::Union{VarTable,Nothing}, + sv::AbsIntState, max_methods::Int = get_max_methods(interp, f, sv)) (; fargs, argtypes) = arginfo argtypes::Vector{Any} = arginfo.argtypes # declare type because the closure below captures `argtypes` fargs = arginfo.fargs @@ -2645,14 +2963,15 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), 𝕃ᵢ = typeinf_lattice(interp) if isa(f, Builtin) if f === _apply_iterate - return abstract_apply(interp, argtypes, si, sv, max_methods) + return abstract_apply(interp, argtypes, si, vtypes, sv, max_methods) elseif f === invoke - return abstract_invoke(interp, arginfo, si, sv) + return abstract_invoke(interp, arginfo, si, vtypes, sv) elseif f === modifyfield! || f === Core.modifyglobal! || + f === Core.modifyglobal_partition || f === Core.memoryrefmodify! || f === atomic_pointermodify - return abstract_modifyop!(interp, f, argtypes, si, sv) + return abstract_modifyop!(interp, f, argtypes, si, vtypes, sv) elseif f === Core.finalizer - return abstract_finalizer(interp, argtypes, sv) + return abstract_finalizer(interp, argtypes, vtypes, sv) elseif f === applicable return abstract_applicable(interp, argtypes, sv, max_methods) elseif f === throw @@ -2679,8 +2998,10 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return Future(abstract_eval_isdefinedglobal(interp, sv, si.saw_latestworld, argtypes)) elseif f === Core.get_binding_type return Future(abstract_eval_get_binding_type(interp, sv, argtypes)) + elseif f === Core._task + return abstract_eval_task_builtin(interp, arginfo, si, vtypes, sv) end - rt = abstract_call_builtin(interp, f, arginfo, sv) + rt = abstract_call_builtin(interp, f, arginfo, vtypes, sv) ft = popfirst!(argtypes) effects = builtin_effects(𝕃ᵢ, f, argtypes, rt) if effects.nothrow @@ -2717,40 +3038,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), elseif isa(f, Core.OpaqueClosure) # calling an OpaqueClosure about which we have no information returns no information return Future(CallMeta(typeof(f).parameters[2], Any, Effects(), NoCallInfo())) - elseif f === TypeVar && !isvarargtype(argtypes[end]) - # Manually look through the definition of TypeVar to - # make sure to be able to get `PartialTypeVar`s out. - 2 ≤ la ≤ 4 || return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) - # make sure generic code is prepared for inlining if needed later - let T = Any[Type{TypeVar}, Any, Any, Any] - resize!(T, la) - atype = Tuple{T...} - T[1] = Const(TypeVar) - let call = abstract_call_gf_by_type(interp, f, ArgInfo(nothing, T), si, atype, sv, max_methods)::Future - return Future{CallMeta}(call, interp, sv) do call, interp, sv - n = argtypes[2] - ub_var = Const(Any) - lb_var = Const(Union{}) - if la == 4 - ub_var = argtypes[4] - lb_var = argtypes[3] - elseif la == 3 - ub_var = argtypes[3] - end - pT = typevar_tfunc(𝕃ᵢ, n, lb_var, ub_var) - typevar_argtypes = Any[n, lb_var, ub_var] - effects = builtin_effects(𝕃ᵢ, Core._typevar, typevar_argtypes, pT) - if effects.nothrow - exct = Union{} - else - exct = builtin_exct(𝕃ᵢ, Core._typevar, typevar_argtypes, pT) - end - return CallMeta(pT, exct, effects, call.info) - end - end - end elseif f === UnionAll - let call = abstract_call_gf_by_type(interp, f, ArgInfo(nothing, Any[Const(UnionAll), Any, Any]), si, Tuple{Type{UnionAll}, Any, Any}, sv, max_methods)::Future + let call = abstract_call_gf_by_type(interp, f, ArgInfo(nothing, Any[Const(UnionAll), Any, Any]), si, Tuple{Type{UnionAll}, Any, Any}, vtypes, sv, max_methods)::Future return Future{CallMeta}(call, interp, sv) do call, interp, sv return abstract_call_unionall(interp, argtypes, call) end @@ -2765,12 +3054,12 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return return_type_tfunc(interp, argtypes, si, sv) elseif la == 3 && f === Core.:(!==) # mark !== as exactly a negated call to === - let callfuture = abstract_call_gf_by_type(interp, f, ArgInfo(fargs, Any[Const(f), Any, Any]), si, Tuple{typeof(f), Any, Any}, sv, max_methods)::Future, - rtfuture = abstract_call_known(interp, (===), arginfo, si, sv, max_methods)::Future + let callfuture = abstract_call_gf_by_type(interp, f, ArgInfo(fargs, Any[Const(f), Any, Any]), si, Tuple{typeof(f), Any, Any}, vtypes, sv, max_methods)::Future, + rtfuture = abstract_call_known(interp, (===), arginfo, si, vtypes, sv, max_methods)::Future return Future{CallMeta}(isready(callfuture) && isready(rtfuture), interp, sv) do interp, sv local rty = rtfuture[].rt if isa(rty, Conditional) - return CallMeta(Conditional(rty.slot, rty.elsetype, rty.thentype), Bottom, EFFECTS_TOTAL, NoCallInfo()) # swap if-else + return CallMeta(Conditional(rty.slot, rty.ssadef, rty.elsetype, rty.thentype), Bottom, EFFECTS_TOTAL, NoCallInfo()) # swap if-else elseif isa(rty, Const) return CallMeta(Const(rty.val === false), Bottom, EFFECTS_TOTAL, MethodResultPure()) end @@ -2778,26 +3067,73 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), end end elseif la == 3 && f === Core.:(>:) - # mark issupertype as a exact alias for issubtype + # mark issupertype as an exact alias for issubtype # swap T1 and T2 arguments and call <: - if fargs !== nothing && length(fargs) == 3 - fargs = Any[<:, fargs[3], fargs[2]] - else - fargs = nothing + atype = argtypes_to_type(argtypes) + let call = abstract_call_gf_by_type(interp, f, ArgInfo(fargs, Any[Const(f), Any, Any]), si, Tuple{typeof(f), Any, Any}, vtypes, sv, max_methods)::Future + if fargs !== nothing && length(fargs) == 3 + fargs_reverse = Any[<:, fargs[3], fargs[2]] + else + fargs_reverse = nothing + end + argtypes_reverse = Any[typeof(<:), argtypes[3], argtypes[2]] + call_reverse = abstract_call_known(interp, <:, ArgInfo(fargs_reverse, argtypes_reverse), si, vtypes, sv, max_methods) + return Future{CallMeta}(isready(call) && isready(call_reverse), interp, sv) do interp, sv + return call_reverse[] + end + end + end + atype = argtypes_to_type(argtypes) + call = abstract_call_gf_by_type(interp, f, arginfo, si, atype, vtypes, sv, max_methods)::Future + # Improve some results with custom tfuncs, + # now that we've inferred the target function to generate source code, + # which might be needed for inlining / invoke / dispatch. + if f === TypeVar && !isvarargtype(argtypes[end]) + # Manually look through the definition of TypeVar to + # make sure to be able to get `PartialTypeVar`s out. + 2 ≤ la ≤ 4 || return Future{CallMeta}(call, sv, interp) do call, sv, interp + return CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo()) + end + # make sure generic code is prepared for inlining if needed later + let T = Any[Type{TypeVar}, Any, Any, Any] + resize!(T, la) + atype = Tuple{T...} + T[1] = Const(TypeVar) + return Future{CallMeta}(call, interp, sv) do call, interp, sv + n = argtypes[2] + ub_var = Const(Any) + lb_var = Const(Union{}) + if la == 4 + ub_var = argtypes[4] + lb_var = argtypes[3] + elseif la == 3 + ub_var = argtypes[3] + end + pT = typevar_tfunc(𝕃ᵢ, n, lb_var, ub_var) + typevar_argtypes = Any[n, lb_var, ub_var] + effects = builtin_effects(𝕃ᵢ, Core._typevar, typevar_argtypes, pT) + if effects.nothrow + exct = Union{} + else + exct = builtin_exct(𝕃ᵢ, Core._typevar, typevar_argtypes, pT) + end + return CallMeta(pT, exct, effects, call.info) + end end - argtypes = Any[typeof(<:), argtypes[3], argtypes[2]] - return abstract_call_known(interp, <:, ArgInfo(fargs, argtypes), si, sv, max_methods) elseif la == 2 && f === Core.typename - return Future(CallMeta(typename_static(argtypes[2]), Bottom, EFFECTS_TOTAL, MethodResultPure())) + return Future{CallMeta}(call, interp, sv) do call, interp, sv + return CallMeta(typename_static(argtypes[2]), Bottom, EFFECTS_TOTAL, MethodResultPure()) + end elseif f === Core._hasmethod - return Future(_hasmethod_tfunc(interp, argtypes, sv)) + return Future{CallMeta}(call, interp, sv) do call, interp, sv + return _hasmethod_tfunc(interp, argtypes, sv) + end end - atype = argtypes_to_type(argtypes) - return abstract_call_gf_by_type(interp, f, arginfo, si, atype, sv, max_methods)::Future + return call end -function abstract_call_opaque_closure(interp::AbstractInterpreter, - closure::PartialOpaque, arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState, check::Bool=true) +function abstract_call_opaque_closure(interp::AbstractInterpreter, closure::PartialOpaque, + arginfo::ArgInfo, si::StmtInfo, vtypes::Union{VarTable,Nothing}, sv::AbsIntState, check::Bool=true) sig = argtypes_to_type(arginfo.argtypes) tt = closure.typ ocargsig = rewrap_unionall((unwrap_unionall(tt)::DataType).parameters[1], tt) @@ -2818,24 +3154,24 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, mresult = abstract_call_method(interp, ocmethod, sig, Core.svec(), false, si, sv) ocsig_box = Core.Box(ocsig) return Future{CallMeta}(mresult, interp, sv) do result, interp, sv - (; rt, exct, effects, volatile_inf_result, edge, edgecycle) = result + (; rt, exct, effects, call_result, edge, edgecycle) = result 𝕃ₚ = ipo_lattice(interp) ⊑, ⋤, ⊔ = partialorder(𝕃ₚ), strictneqpartialorder(𝕃ₚ), join(𝕃ₚ) - const_result = volatile_inf_result if !edgecycle const_call_result = abstract_call_method_with_const_args(interp, result, #=f=#nothing, arginfo, si, match, sv) if const_call_result !== nothing - const_edge = nothing + const_result = nothing if const_call_result.rt ⊑ rt - (; rt, effects, const_result, const_edge) = const_call_result + (; rt, effects, const_result) = const_call_result end if const_call_result.exct ⋤ exct - (; exct, const_result, const_edge) = const_call_result + (; exct, const_result) = const_call_result end - if const_edge !== nothing - edge = const_edge - update_valid_age!(sv, world_range(const_edge)) + if const_result !== nothing + update_valid_age!(sv, get_inference_world(interp), + proof_worlds(inference_proof(const_result))) + call_result = const_result end end end @@ -2848,14 +3184,13 @@ function abstract_call_opaque_closure(interp::AbstractInterpreter, exct = exct ⊔ TypeError end end - rt = from_interprocedural!(interp, rt, sv, arginfo, match.spec_types) - info = OpaqueClosureCallInfo(edge, match, const_result) + rt = from_interprocedural!(interp, rt, sv, arginfo, match.spec_types, vtypes) + info = OpaqueClosureCallInfo(edge, match, call_result, result.needs_mi_edge) return CallMeta(rt, exct, effects, info) end end function most_general_argtypes(closure::PartialOpaque) - ret = Any[] cc = widenconst(closure) argt = (unwrap_unionall(cc)::DataType).parameters[1] if !isa(argt, DataType) || argt.name !== typename(Tuple) @@ -2865,13 +3200,13 @@ function most_general_argtypes(closure::PartialOpaque) end function abstract_call_unknown(interp::AbstractInterpreter, @nospecialize(ft), - arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState, - max_methods::Int) + arginfo::ArgInfo, si::StmtInfo, vtypes::Union{VarTable,Nothing}, + sv::AbsIntState, max_methods::Int) if isa(ft, PartialOpaque) newargtypes = copy(arginfo.argtypes) newargtypes[1] = ft.env return abstract_call_opaque_closure(interp, - ft, ArgInfo(arginfo.fargs, newargtypes), si, sv, #=check=#true) + ft, ArgInfo(arginfo.fargs, newargtypes), si, vtypes, sv, #=check=#true) end wft = widenconst(ft) if hasintersect(wft, Builtin) @@ -2887,20 +3222,20 @@ function abstract_call_unknown(interp::AbstractInterpreter, @nospecialize(ft), # non-constant function, but the number of arguments is known and the `f` is not a builtin or intrinsic atype = argtypes_to_type(arginfo.argtypes) atype === Bottom && return Future(CallMeta(Union{}, Union{}, EFFECTS_THROWS, NoCallInfo())) # accidentally unreachable - return abstract_call_gf_by_type(interp, nothing, arginfo, si, atype, sv, max_methods)::Future + return abstract_call_gf_by_type(interp, nothing, arginfo, si, atype, vtypes, sv, max_methods)::Future end # call where the function is any lattice element function abstract_call(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, - sv::AbsIntState, max_methods::Int=typemin(Int)) + vtypes::Union{VarTable,Nothing}, sv::AbsIntState, max_methods::Int=typemin(Int)) ft = widenslotwrapper(arginfo.argtypes[1]) f = singleton_type(ft) if f === nothing max_methods = max_methods == typemin(Int) ? get_max_methods(interp, sv) : max_methods - return abstract_call_unknown(interp, ft, arginfo, si, sv, max_methods) + return abstract_call_unknown(interp, ft, arginfo, si, vtypes, sv, max_methods) end max_methods = max_methods == typemin(Int) ? get_max_methods(interp, f, sv) : max_methods - return abstract_call_known(interp, f, arginfo, si, sv, max_methods) + return abstract_call_known(interp, f, arginfo, si, vtypes, sv, max_methods) end function sp_type_rewrap(@nospecialize(T), mi::MethodInstance, isreturn::Bool) @@ -2924,19 +3259,10 @@ function sp_type_rewrap(@nospecialize(T), mi::MethodInstance, isreturn::Bool) if !isempty(mi.sparam_vals) sparam_vals = Any[isvarargtype(v) ? TypeVar(:N, Union{}, Any) : v for v in mi.sparam_vals] + free_sps_before = find_free_typevars(mi.specTypes) T = ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), T, spsig, sparam_vals) isref && isreturn && T === Any && return Bottom # catch invalid return Ref{T} where T = Any - for v in sparam_vals - if isa(v, TypeVar) - T = UnionAll(v, T) - end - end - if has_free_typevars(T) - fv = ccall(:jl_find_free_typevars, Vector{Any}, (Any,), T) - for v in fv - T = UnionAll(v, T) - end - end + T = rewrap_free_typevars(T, free_sps_before) else T = rewrap_unionall(T, spsig) end @@ -2959,7 +3285,7 @@ function abstract_eval_cfunction(interp::AbstractInterpreter, e::Expr, sstate::S # this may be the wrong world for the call, # but some of the result is likely to be valid anyways # and that may help generate better codegen - abstract_call(interp, ArgInfo(nothing, at), StmtInfo(false, false), sv)::Future + abstract_call(interp, ArgInfo(nothing, at), StmtInfo(false, false), sstate.vtypes, sv)::Future rt = e.args[1] isconcretetype(rt) || (rt = Any) return RTEffects(rt, Any, EFFECTS_UNKNOWN) @@ -2987,6 +3313,8 @@ function abstract_eval_special_value(interp::AbstractInterpreter, @nospecialize( elseif isa(e, GlobalRef) # No need for an edge since an explicit GlobalRef will be picked up by the source scan return abstract_eval_globalref(interp, e, sstate.saw_latestworld, sv) + elseif isa(e, Core.BindingPartition) + return abstract_eval_partition_load(interp, partition_owner(e), e) end if isa(e, QuoteNode) e = e.value @@ -2997,16 +3325,11 @@ function abstract_eval_special_value(interp::AbstractInterpreter, @nospecialize( end function abstract_eval_value_expr(interp::AbstractInterpreter, e::Expr, sv::AbsIntState) - if e.head === :call && length(e.args) ≥ 1 - # TODO: We still have non-linearized cglobal - @assert e.args[1] === Core.tuple || e.args[1] === GlobalRef(Core, :tuple) - else - @assert e.head !== :(=) - # Some of our tests expect us to handle invalid IR here and error later - # - permit that for now. - # @assert false "Unexpected EXPR head in value position" - merge_effects!(interp, sv, EFFECTS_UNKNOWN) - end + @assert e.head !== :(=) + # Some of our tests expect us to handle invalid IR here and error later + # - permit that for now. + # @assert false "Unexpected EXPR head in value position" + merge_effects!(interp, sv, EFFECTS_UNKNOWN) return Any end @@ -3053,8 +3376,8 @@ function abstract_call(interp::AbstractInterpreter, arginfo::ArgInfo, sstate::St add_curr_ssaflag!(sv, IR_FLAG_UNUSED) end si = StmtInfo(!unused, sstate.saw_latestworld) - call = abstract_call(interp, arginfo, si, sv)::Future - Future{Any}(call, interp, sv) do call, interp, sv + call = abstract_call(interp, arginfo, si, sstate.vtypes, sv)::Future + Future{Any}(call, interp, sv) do call, _, sv # this only is needed for the side-effect, sequenced before any task tries to consume the return value, # which this will do even without returning this Future sv.stmt_info[sv.currpc] = call.info @@ -3072,25 +3395,31 @@ function abstract_eval_call(interp::AbstractInterpreter, e::Expr, sstate::Statem end arginfo = ArgInfo(ea, argtypes) call = abstract_call(interp, arginfo, sstate, sv)::Future - return Future{RTEffects}(call, interp, sv) do call, interp, sv + return Future{RTEffects}(call, interp, sv) do call, _, _ (; rt, exct, effects, refinements) = call return RTEffects(rt, exct, effects, refinements) end end +function is_field_pointerfree(dt::DataType, fidx::Int) + dt.layout::Ptr{Cvoid} == C_NULL && return false + DataTypeFieldDesc(dt)[fidx].isptr && return false + ft = fieldtype(dt, fidx) + return ft isa DataType && datatype_pointerfree(ft) +end + function abstract_eval_new(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) 𝕃ᵢ = typeinf_lattice(interp) - rt, isexact = instanceof_tfunc(abstract_eval_value(interp, e.args[1], sstate, sv), true) + rt, _... = instanceof_tfunc(abstract_eval_value(interp, e.args[1], sstate, sv), true) ut = unwrap_unionall(rt) exct = Union{ErrorException,TypeError} if isa(ut, DataType) && !isabstracttype(ut) ismutable = ismutabletype(ut) fcount = datatype_fieldcount(ut) nargs = length(e.args) - 1 - has_any_uninitialized = (fcount === nothing || (fcount > nargs && (let t = rt - any(i::Int -> !is_undefref_fieldtype(fieldtype(t, i)), (nargs+1):fcount) - end))) + has_any_uninitialized = fcount === nothing || (fcount > nargs && + any(i::Int->is_field_pointerfree(ut, i), (nargs+1):fcount)) if has_any_uninitialized # allocation with undefined field is inconsistent always consistent = ALWAYS_FALSE @@ -3101,12 +3430,16 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, sstate::Stateme else consistent = ALWAYS_TRUE # immutable allocation is consistent end - if isconcretedispatch(rt) - nothrow = true - @assert fcount !== nothing && fcount ≥ nargs "malformed :new expression" # syntactically enforced by the front-end + # `:new` can carry `PartialStruct` even when `rt` isn't `isconcretedispatch` — + # partially-instantiated parametric types (e.g. `Generator{Vector{Int}, F<:OC{Tuple{Int}, T} where T}`) still + # have well-defined field count, and field-level extended lattice elements carry + # information beyond the declared type. + if fcount !== nothing + nothrow = isconcretedispatch(rt) + @assert nargs ≤ fcount "malformed :new expression" # syntactically enforced by the front-end ats = Vector{Any}(undef, nargs) local anyrefine = false - local allconst = true + local allconst = isconcretedispatch(rt) for i = 1:nargs at = widenslotwrapper(abstract_eval_value(interp, e.args[i+1], sstate, sv)) ft = fieldtype(rt, i) @@ -3124,7 +3457,7 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, sstate::Stateme end ats[i] = at end - if fcount == nargs && consistent === ALWAYS_TRUE && allconst + if allconst && fcount == nargs && consistent === ALWAYS_TRUE argvals = Vector{Any}(undef, nargs) for j in 1:nargs argvals[j] = (ats[j]::Const).val @@ -3159,6 +3492,8 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, sstate::Stateme end end rt = PartialStruct(𝕃ᵢ, rt, undefs, ats) + else + rt = refine_partial_type(rt) end else rt = refine_partial_type(rt) @@ -3208,6 +3543,63 @@ function abstract_eval_splatnew(interp::AbstractInterpreter, e::Expr, sstate::St return RTEffects(rt, Any, effects) end +# Effects of the `Core._task` builtin itself; the deferred body does not run here. +# Creating a task returns a fresh mutable object, inherits the current task's scope +# and forks its RNG state (advancing the parent's internal RNG via `jl_rng_split`), +# so the call is not consistent, not effect-free, and accesses task state. It may +# throw, e.g. for an invalid stack size, but always terminates and has no UB. +const TASK_BUILTIN_EFFECTS = Effects(EFFECTS_TOTAL; + consistent=ALWAYS_FALSE, effect_free=ALWAYS_FALSE, nothrow=false, + notaskstate=false, inaccessiblememonly=ALWAYS_FALSE) + +function abstract_eval_task_builtin(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, + vtypes::Union{VarTable,Nothing}, sv::AbsIntState) + argtypes = arginfo.argtypes + la = length(argtypes) + isva = !isempty(argtypes) && isvarargtype(argtypes[end]) + if isva + la > 5 && return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) + size_arg = argtype_by_index(argtypes, 3) + elseif !(3 <= la <= 4) + return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) + else + size_arg = argtypes[3] + end + if !hasintersect(widenconst(size_arg), Int) + return Future(CallMeta(Bottom, Any, EFFECTS_THROWS, NoCallInfo())) + end + if isva && la < 5 + # Without a fixed invoke target, the vararg may supply any of the required + # arguments, so only retain the builtin's guaranteed return type. + return Future(CallMeta(Task, Any, TASK_BUILTIN_EFFECTS, NoCallInfo())) + end + func_arg = argtypes[2] + + # Handle the fixed Method/CodeInstance/Type argument (4th parameter) as invoke. + # A trailing vararg may be empty; non-empty tails throw an arity error before + # the deferred invoke can run. + if la >= 4 + invoke_args = Any[Const(Core.invoke), func_arg, argtypes[4]] + invoke_arginfo = ArgInfo(nothing, invoke_args) + invoke_future = abstract_invoke(interp, invoke_arginfo, si, vtypes, sv) + return Future{CallMeta}(task_callmeta, invoke_future, interp, sv) + end + + # Otherwise use abstract_call for function analysis + callinfo_future = abstract_call(interp, ArgInfo(nothing, Any[func_arg]), StmtInfo(true, si.saw_latestworld), vtypes, sv, #=max_methods=#1) + return Future{CallMeta}(task_callmeta, callinfo_future, interp, sv) +end + +# Convert the `CallMeta` of the task body call into the `CallMeta` of the `Core._task` +# call that creates it, tracking the body's result type with a `PartialTask` and keeping +# its call information for the inlining pass. +function task_callmeta(call::CallMeta, ::AbstractInterpreter, ::AbsIntState) + fetch_type = widenconst(call.rt) + rt_result = fetch_type === Any ? Task : PartialTask(fetch_type) + info_result = TaskCallInfo(call.info) + return CallMeta(rt_result, Any, TASK_BUILTIN_EFFECTS, info_result) +end + function abstract_eval_new_opaque_closure(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) 𝕃ᵢ = typeinf_lattice(interp) @@ -3233,8 +3625,8 @@ function abstract_eval_new_opaque_closure(interp::AbstractInterpreter, e::Expr, argtypes = most_general_argtypes(rt) pushfirst!(argtypes, rt.env) callinfo = abstract_call_opaque_closure(interp, rt, - ArgInfo(nothing, argtypes), StmtInfo(true, false), sv, #=check=#false)::Future - Future{Any}(callinfo, interp, sv) do callinfo, interp, sv + ArgInfo(nothing, argtypes), StmtInfo(true, false), sstate.vtypes, sv, #=check=#false)::Future + Future{Any}(callinfo, interp, sv) do callinfo, _, sv sv.stmt_info[sv.currpc] = OpaqueClosureCreateInfo(callinfo) nothing end @@ -3265,7 +3657,7 @@ function abstract_eval_isdefined_expr(::AbstractInterpreter, e::Expr, sstate::St elseif !vtyp.undef rt = Const(true) # definitely assigned previously else # form `Conditional` to refine `vtyp.undef` in the then branch - rt = Conditional(sym, widenslotwrapper(vtyp.typ), widenslotwrapper(vtyp.typ); isdefined=true) + rt = Conditional(sym, vtyp.ssadef, widenslotwrapper(vtyp.typ), widenslotwrapper(vtyp.typ); isdefined=true) end return RTEffects(rt, Union{}, EFFECTS_TOTAL) end @@ -3301,7 +3693,7 @@ function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, mod::Module, if allow_import !== true gr = GlobalRef(mod, sym) partition = lookup_binding_partition!(interp, gr, sv) - if allow_import !== true && is_some_binding_imported(binding_kind(partition)) + if allow_import !== true && !is_leaf_partition(partition) if allow_import === false rt = Const(false) else @@ -3311,7 +3703,10 @@ function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, mod::Module, end end - (valid_worlds, rte) = abstract_load_all_consistent_leaf_partitions(interp, gr, sv.world) + world = get_inference_world(interp) + valid_worlds, (leaf_b, leaf_p) = binding_access_range(gr, binding_world_hints(world, sv), false) + update_valid_age!(sv, world, valid_worlds) + rte = abstract_eval_partition_load(interp, leaf_b, leaf_p) if rte.exct == Union{} rt = Const(true) elseif rte.rt === Union{} && rte.exct === UndefVarError @@ -3436,11 +3831,10 @@ function abstract_eval_statement_expr(interp::AbstractInterpreter, e::Expr, ssta return abstract_eval_new_opaque_closure(interp, e, sstate, sv) elseif ehead === :foreigncall return abstract_eval_foreigncall(interp, e, sstate, sv) + elseif ehead === :foreignglobal + return abstract_eval_foreignglobal(interp, e, sstate, sv) elseif ehead === :cfunction return abstract_eval_cfunction(interp, e, sstate, sv) - elseif ehead === :method - rt = (length(e.args) == 1) ? Any : Method - return RTEffects(rt, Any, EFFECTS_UNKNOWN) elseif ehead === :copyast return abstract_eval_copyast(interp, e, sstate, sv) elseif ehead === :invoke || ehead === :invoke_modify @@ -3457,11 +3851,8 @@ function abstract_eval_statement_expr(interp::AbstractInterpreter, e::Expr, ssta return abstract_eval_static_parameter(interp, e, sv) elseif ehead === :gc_preserve_begin || ehead === :aliasscope return RTEffects(Any, Union{}, Effects(EFFECTS_TOTAL; consistent=ALWAYS_FALSE, effect_free=EFFECT_FREE_GLOBALLY)) - elseif ehead === :gc_preserve_end || ehead === :leave || ehead === :pop_exception || - ehead === :global || ehead === :popaliasscope + elseif ehead === :gc_preserve_end || ehead === :leave || ehead === :pop_exception || ehead === :popaliasscope return RTEffects(Nothing, Union{}, Effects(EFFECTS_TOTAL; effect_free=EFFECT_FREE_GLOBALLY)) - elseif ehead === :globaldecl - return RTEffects(Nothing, Any, EFFECTS_UNKNOWN) elseif ehead === :thunk return RTEffects(Any, Any, Effects()) end @@ -3485,61 +3876,33 @@ function refine_partial_type(@nospecialize t) return t end -abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::IRInterpretationState) = nothing - -function abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::AbsIntState) - if isexpr(e, :call) - n = length(e.args) - argtypes = Vector{Any}(undef, n) - callresult = Future{CallMeta}() - i::Int = 1 - nextstate::UInt8 = 0x0 - local ai, res - function evalargs(interp, sv) - if nextstate === 0x1 - @goto state1 - elseif nextstate === 0x2 - @goto state2 - end - while i <= n - ai = abstract_eval_nonlinearized_foreigncall_name(interp, e.args[i], sstate, sv) - if !isready(ai) - nextstate = 0x1 - return false - @label state1 - end - argtypes[i] = ai[].rt - i += 1 - end - res = abstract_call(interp, ArgInfo(e.args, argtypes), sstate, sv) - if !isready(res) - nextstate = 0x2 - return false - @label state2 - end - callresult[] = res[] - return true - end - evalargs(interp, sv) || push!(sv.tasks, evalargs) - return callresult - else - return Future(abstract_eval_basic_statement(interp, e, sstate, sv)) - end -end - function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) callee = e.args[1] - if isexpr(callee, :call) && length(callee.args) > 1 && callee.args[1] == GlobalRef(Core, :tuple) - # NOTE these expressions are not properly linearized - abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[2], sstate, sv) - if length(callee.args) > 2 - abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[3], sstate, sv) + if isexpr(callee, :tuple) + if length(callee.args) >= 1 + # Evaluate the arguments to constrain the world, effects, and other info for codegen, + # but note there is an implied `if !=(C_NULL)` branch here that might read data + # in a different world (the exact cache behavior is unspecified), so we do not use + # these results to refine reachability of the subsequent foreigncall. + abstract_eval_value(interp, callee.args[1], sstate, sv) + if length(callee.args) >= 2 + abstract_eval_value(interp, callee.args[2], sstate, sv) + #TODO: implement abstract_eval_nonlinearized_foreigncall_name correctly? + # lib_effects = abstract_call(interp, ArgInfo(e.args, Any[typeof(Libdl.dlopen), lib]), sstate, sv)::Future + end end else abstract_eval_value(interp, callee, sstate, sv) end mi = frame_instance(sv) t = sp_type_rewrap(e.args[2], mi, true) + let fptr = e.args[1] + if !isexpr(fptr, :tuple) + if !hasintersect(widenconst(abstract_eval_value(interp, fptr, sstate, sv)), Ptr) + return RTEffects(Bottom, Any, EFFECTS_THROWS) + end + end + end for i = 3:length(e.args) if abstract_eval_value(interp, e.args[i], sstate, sv) === Bottom return RTEffects(Bottom, Any, EFFECTS_THROWS) @@ -3549,13 +3912,30 @@ function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate: abstract_eval_value(interp, x, sstate, sv) end cconv = e.args[5] - if isa(cconv, QuoteNode) && (v = cconv.value; isa(v, Tuple{Symbol, UInt16, Bool})) + if isa(cconv, QuoteNode) && (v = cconv.value; + isa(v, Union{Tuple{Symbol, UInt16, Bool}, Tuple{Symbol, UInt16, Bool, Bool}, + Tuple{Symbol, UInt16, Bool, Bool, Bool}})) override = decode_effects_override(v[2]) effects = override_effects(effects, override) end return RTEffects(t, Any, effects) end +function abstract_eval_foreignglobal(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) + arg = e.args[1] + # Evaluate the arguments to constrain the world for codegen + if isexpr(arg, :tuple) + for elt in arg.args + abstract_eval_value(interp, elt, sstate, sv) + #TODO: implement abstract_eval_nonlinearized_foreigncall_name correctly? + # (see foreigncall implementation above) + end + else + abstract_eval_value(interp, arg, sstate, sv) + end + return RTEffects(Ptr{Cvoid}, Any, EFFECTS_UNKNOWN) +end + function abstract_eval_phi(interp::AbstractInterpreter, phi::PhiNode, sstate::StatementState, sv::AbsIntState) rt = Union{} for i in 1:length(phi.values) @@ -3604,48 +3984,136 @@ world_range(ci::CodeInfo) = WorldRange(ci.min_world, ci.max_world) world_range(ci::CodeInstance) = WorldRange(ci.min_world, ci.max_world) world_range(compact::IncrementalCompact) = world_range(compact.ir) -function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) - worlds = world_range(src) - partition = lookup_binding_partition(min_world(worlds), g) +# Like `walk_binding_partition` but drops the WorldRange tracking — IR-only callers don't use it. +# +# Walk imports to the leaf partition, also reporting whether `getglobal` would +# deprecation-warn for the access: the deprecation flag is ORed across the walk but +# suppressed once an explicit import is passed (the `import`/`using: x` site warns +# instead), mirroring the runtime `jl_walk_binding_inplace_depwarn`. +@inline function walk_to_leaf_partition_depwarn(binding::Core.Binding, partition::Core.BindingPartition, world::UInt) + passed_explicit = false + depwarn = false + while true + kind = binding_kind(partition) + if !passed_explicit + depwarn |= (partition.kind & PARTITION_FLAG_DEPWARN) != 0 + end + is_leaf_partition(partition) && break + is_some_explicit_imported(kind) && (passed_explicit = true) + binding = partition_restriction(partition)::Core.Binding + partition = lookup_binding_partition(world, binding) + end + return (binding, partition, depwarn) +end + +@inline function walk_to_leaf_partition(binding::Core.Binding, partition::Core.BindingPartition, world::UInt) + binding, partition, _ = walk_to_leaf_partition_depwarn(binding, partition, world) + return (binding, partition) +end + +# Whether `partition` is a leaf partition is the inverse of asking if it is imported. +# A non-leaf partition isn't resolved to a specific behavior, so queries that reach that case are necessarily conservative, +# thus the compiler usually avoids attempting those queries itself by using walk_to_leaf_partition. +@inline is_leaf_partition(partition::Core.BindingPartition) = + !is_some_binding_imported(binding_kind(partition)) - (valid_worlds, rte) = abstract_load_all_consistent_leaf_partitions(nothing, g, WorldWithRange(min_world(worlds), worlds)) - if min_world(valid_worlds) > min_world(worlds) || max_world(valid_worlds) < max_world(worlds) - return Any +@inline function partition_rt(partition::Core.BindingPartition) + is_leaf_partition(partition) || return Any + kind = binding_kind(partition) + (is_some_guard(kind) || kind == PARTITION_KIND_DECLARED) && return Any + if is_defined_const_binding(kind) + kind == PARTITION_KIND_BACKDATED_CONST && return Any + return Const(partition_restriction(partition)) + end + return partition_restriction(partition) +end + +@inline function partition_rt_widened(partition::Core.BindingPartition) + is_leaf_partition(partition) || return Any + kind = binding_kind(partition) + (is_some_guard(kind) || kind == PARTITION_KIND_DECLARED) && return Any + if is_defined_const_binding(kind) + kind == PARTITION_KIND_BACKDATED_CONST && return Any + return Core.Typeof(partition_restriction(partition)) end + return partition_restriction(partition) +end - return rte.rt +# IR-level GlobalRef queries. Not abstract eval (unsound there): inference already +# guarantees consistency across the world range, so we just look up at max_world. +@inline function globalref_leaf_partition(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) + world = max_world(world_range(src)) + binding = convert(Core.Binding, g) + partition = lookup_binding_partition(world, binding) + _, leaf_partition = walk_to_leaf_partition(binding, partition, world) + return leaf_partition end +globalref_rt(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) = + partition_rt(globalref_leaf_partition(g, src)) + +# `widenconst`-compatible variant — skips the `Const(...)` box on defined-const bindings. +globalref_rt_widened(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) = + partition_rt_widened(globalref_leaf_partition(g, src)) + +# `singleton_type`-compatible variant — skips the `Const(...)` box on defined-const +# bindings, and (like `singleton_type`) also unwraps `Type{T}` / singleton restrictions. +function partition_singleton(partition::Core.BindingPartition) + is_leaf_partition(partition) || return nothing + kind = binding_kind(partition) + (is_some_guard(kind) || kind == PARTITION_KIND_DECLARED) && return nothing + if is_defined_const_binding(kind) + kind == PARTITION_KIND_BACKDATED_CONST && return nothing + return partition_restriction(partition) + end + return singleton_type(partition_restriction(partition)) +end + +globalref_singleton(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) = + partition_singleton(globalref_leaf_partition(g, src)) + function lookup_binding_partition!(interp::AbstractInterpreter, g::Union{GlobalRef, Core.Binding}, sv::AbsIntState) - partition = lookup_binding_partition(get_inference_world(interp), g) - update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world)) + world = get_inference_world(interp) + partition = lookup_binding_partition(world, g) + update_valid_age!(sv, world, WorldRange(partition.min_world, partition.max_world)) partition end -function walk_binding_partition(imported_binding::Core.Binding, partition::Core.BindingPartition, world::UInt) +function walk_binding_partition(b::Core.Binding, partition::Core.BindingPartition, world::UInt, write::Bool) valid_worlds = WorldRange(partition.min_world, partition.max_world) - while is_some_binding_imported(binding_kind(partition)) - imported_binding = partition_restriction(partition)::Core.Binding - partition = lookup_binding_partition(world, imported_binding) - valid_worlds = intersect(valid_worlds, WorldRange(partition.min_world, partition.max_world)) + if !write + while !is_leaf_partition(partition) + b = partition_restriction(partition)::Core.Binding + partition = lookup_binding_partition(world, b) + valid_worlds = intersect(valid_worlds, WorldRange(partition.min_world, partition.max_world)) + end end - return Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(valid_worlds, imported_binding=>partition) + return Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}( + valid_worlds, b=>partition) end + function abstract_eval_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState) b = convert(Core.Binding, g) partition = lookup_binding_partition!(interp, b, sv) - valid_worlds, (_, partition) = walk_binding_partition(b, partition, get_inference_world(interp)) - update_valid_age!(sv, valid_worlds) + world = get_inference_world(interp) + valid_worlds, (_, partition) = walk_binding_partition(b, partition, world, false) + update_valid_age!(sv, world, valid_worlds) return partition end -function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing}, binding::Core.Binding, partition::Core.BindingPartition) +abstract_eval_partition_load(interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition) = + abstract_eval_partition_load(binding, partition, InferenceParams(interp).assume_bindings_static) + +function abstract_eval_partition_load(binding::Core.Binding, partition::Core.BindingPartition, assume_bindings_static::Bool) kind = binding_kind(partition) isdepwarn = (partition.kind & PARTITION_FLAG_DEPWARN) != 0 local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) + if !is_leaf_partition(partition) + return RTEffects(Any, UndefVarError, local_getglobal_effects) + end if is_some_guard(kind) - if interp !== nothing && InferenceParams(interp).assume_bindings_static + if assume_bindings_static return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) else # We do not currently assume an invalidation for guard -> defined transitions @@ -3676,8 +4144,8 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing} rt = partition_restriction(partition) effects = local_getglobal_effects end - if (interp !== nothing && InferenceParams(interp).assume_bindings_static && - kind in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) && + if (assume_bindings_static && + is_some_global(kind) && isdefined(binding, :value)) exct = Union{} effects = Effects(generic_getglobal_effects; nothrow=true) @@ -3689,80 +4157,99 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing} return RTEffects(rt, exct, effects) end -function scan_specified_partitions(query::F1, walk_binding_partition::F2, - interp::Union{AbstractInterpreter,Nothing}, g::GlobalRef, wwr::WorldWithRange) where {F1,F2} - local total_validity, rte, binding_partition - binding = convert(Core.Binding, g) +# This key is our definition for whether two binding partition behave identically under a global access. +# Shared between invalidations and inference, so they have a common vocabulary of what it means to be valid interchangeably. +# It ignores flag such as `export`/`public`, so those queries must not be folded by inference. + +# The first element is the maximum information inference/codegen reads from a partition. +# The second element is the maximum information inference/codegen can write via a partition. +# +# The identity for `DECLARED` matters because the load half of the key does not separate it from a +# deprecated guard-like partition (both are `Any`/`UndefVarError` with `effect_free` false), +# while a store to the former is `nothrow` and a store to the latter throws. +# +# Within `DECLARED` the key deliberately says less than it does for `GLOBAL`: it does not separate a +# deprecated partition from an undeprecated one. That is correct as things stand since nothing ever freezes a +# `DECLARED` partition, and would need to fixed if we ever want to start optimizing these. +function binding_access_key(b::Core.Binding, p::Core.BindingPartition) + kind = binding_kind(p) + slot = is_some_global(kind) ? b : nothing + return Pair{Any,Any}(abstract_eval_partition_load(b, p, false), slot) +end + +# Evaluate the query "which world range is a global access to `g` valid over, and which partition does it resolve to." +# n.b. Passing `write=false` never returns a wider valid world range than `write=true`, but it may change the behavior of the returned partition. +# n.b. The returned partition might not actually be in the lookup world, but it does have identical access behavior to the partition that is. +binding_access_range(g::GlobalRef, wwr::WorldWithRange, write::Bool) = + binding_access_range(convert(Core.Binding, g), wwr, write) +function binding_access_range(binding::Core.Binding, wwr::WorldWithRange, write::Bool) lookup_world = max_world(wwr.valid_worlds) - while true - # Partitions are ordered newest-to-oldest so start at the top - binding_partition = @isdefined(binding_partition) ? - lookup_binding_partition(lookup_world, binding, binding_partition) : - lookup_binding_partition(lookup_world, binding) - while lookup_world >= binding_partition.min_world && (!@isdefined(total_validity) || min_world(total_validity) > min_world(wwr.valid_worlds)) - partition_validity, (leaf_binding, leaf_partition) = walk_binding_partition(binding, binding_partition, lookup_world) - @assert lookup_world in partition_validity - this_rte = query(interp, leaf_binding, leaf_partition) - if @isdefined(rte) - if this_rte === rte - total_validity = union(total_validity, partition_validity) - lookup_world = min_world(total_validity) - 1 - continue - end - if min_world(total_validity) <= wwr.this - @goto out - end + wwr_min = min_world(wwr.valid_worlds) + binding_partition = lookup_binding_partition(lookup_world, binding) + partition_validity, leaf = walk_binding_partition(binding, binding_partition, lookup_world, write) + @assert lookup_world in partition_validity + total_validity = partition_validity + total_min = min_world(total_validity) + # The partition found in the lookup world already covers everything asked for, so no + # earlier partition can widen the answer. Returning here rather than falling into the + # scan below is what keeps the query cheap: `binding_access_key` allocates, and this is + # the common case (the binding was last repartitioned before `wwr_min`), so + # materializing a key that is never compared would dominate the cost of the query. + total_min <= wwr_min && return total_validity, leaf + (leaf_binding, leaf_partition) = leaf + key = binding_access_key(leaf_binding, leaf_partition) + lookup_world = total_min - 1 + + # Scan backwards to find the largest sub-range of valid_worlds that + # gives a consistent key and contains wwr.this. + while total_min > wwr_min + if lookup_world < binding_partition.min_world + binding_partition = lookup_binding_partition(lookup_world, binding, binding_partition) + end + while lookup_world >= binding_partition.min_world && total_min > wwr_min + this_partition_validity, this_leaf = walk_binding_partition(binding, binding_partition, lookup_world, write) + (this_leaf_binding, this_leaf_partition) = this_leaf + @assert lookup_world in this_partition_validity + this_key = binding_access_key(this_leaf_binding, this_leaf_partition) + if this_key === key + total_validity = union(total_validity, this_partition_validity) + else + # Key changed: if we already cover wwr.this, return current range + total_min <= wwr.this && @goto out + # Otherwise the old key wasn't for our world, start fresh + total_validity = this_partition_validity + leaf = this_leaf + key = this_key end - total_validity = partition_validity - lookup_world = min_world(total_validity) - 1 - rte = this_rte + total_min = min_world(total_validity) + lookup_world = total_min - 1 end - min_world(total_validity) > min_world(wwr.valid_worlds) || break end @label out - return Pair{WorldRange, typeof(rte)}(total_validity, rte) -end - -scan_leaf_partitions(query::F, ::Nothing, g::GlobalRef, wwr::WorldWithRange) where F = - scan_specified_partitions(query, walk_binding_partition, nothing, g, wwr) -scan_leaf_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F = - scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) - -function scan_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F - walk_binding_partition = function (b::Core.Binding, partition::Core.BindingPartition, world::UInt) - Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}( - WorldRange(partition.min_world, partition.max_world), b=>partition) - end - return scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) + return total_validity, leaf end -abstract_load_all_consistent_leaf_partitions(interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) = - scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr) -abstract_load_all_consistent_leaf_partitions(::Nothing, g::GlobalRef, wwr::WorldWithRange) = - scan_leaf_partitions(abstract_eval_partition_load, nothing, g, wwr) - -function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) +function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState{I}) where {I<:AbstractInterpreter} if saw_latestworld return RTEffects(Any, Any, generic_getglobal_effects) end - # For inference purposes, we don't particularly care which global binding we end up loading, we only - # care about its type. However, we would still like to terminate the world range for the particular - # binding we end up reaching such that codegen can emit a simpler pointer load. - (valid_worlds, ret) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, sv.world) - update_valid_age!(sv, valid_worlds) - return ret + # For inference purposes, we don't particularly care which global partition we end up loading, we + # only care about its type, but we still narrow `valid_worlds` to the binding's access range. + # The optimizer would have to narrow to that anyways in order to be valid to optimize this load to a single pointer. + world = get_inference_world(interp::I) + valid_worlds, (leaf_b, leaf_p) = binding_access_range(g, binding_world_hints(world, sv), false) + update_valid_age!(sv, world, valid_worlds) + return abstract_eval_partition_load(interp, leaf_b, leaf_p) end function global_assignment_rt_exct(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, g::GlobalRef, @nospecialize(newty)) if saw_latestworld - return Pair{Any,Any}(newty, ErrorException) - end - newty′ = RefValue{Any}(newty) - (valid_worlds, ret) = scan_partitions(interp, g, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition - global_assignment_binding_rt_exct(interp, partition, newty′[]) + return Pair{Any,Any}(newty, Union{TypeError, ErrorException}) end - update_valid_age!(sv, valid_worlds) - return ret + world = get_inference_world(interp) + valid_worlds, (_, partition) = binding_access_range(g, binding_world_hints(world, sv), true) + update_valid_age!(sv, world, valid_worlds) + return global_assignment_binding_rt_exct(interp, partition, newty) end function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partition::Core.BindingPartition, @nospecialize(newty)) @@ -3770,15 +4257,16 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio if is_some_guard(kind) return Pair{Any,Any}(newty, ErrorException) elseif is_some_const_binding(kind) || is_some_imported(kind) - return Pair{Any,Any}(Bottom, ErrorException) + # N.B.: Backdating should not improve inference in an earlier world + return Pair{Any,Any}(kind == PARTITION_KIND_BACKDATED_CONST ? newty : Bottom, ErrorException) end ty = kind == PARTITION_KIND_DECLARED ? Any : partition_restriction(partition) wnewty = widenconst(newty) if !hasintersect(wnewty, ty) - return Pair{Any,Any}(Bottom, ErrorException) + return Pair{Any,Any}(Bottom, TypeError) elseif !(wnewty <: ty) retty = tmeet(typeinf_lattice(interp), newty, ty) - return Pair{Any,Any}(retty, ErrorException) + return Pair{Any,Any}(retty, TypeError) end return Pair{Any,Any}(newty, Bottom) end @@ -3821,7 +4309,7 @@ end @goto injectresult end if isa(stmt, NewvarNode) - changes = StateUpdate(stmt.slot, VarState(Bottom, true)) + changes = StateUpdate(stmt.slot, VarState(Bottom, frame.currpc, #= undef =# true)) elseif isa(stmt, PhiNode) add_curr_ssaflag!(frame, IR_FLAGS_REMOVABLE) # Implement convergence for PhiNodes. In particular, PhiNodes need to tmerge over @@ -3841,12 +4329,7 @@ end (; rt, exct, effects, refinements) = abstract_eval_special_value(interp, stmt, sstate, frame) else hd = stmt.head - if hd === :method - fname = stmt.args[1] - if isa(fname, SlotNumber) - changes = StateUpdate(fname, VarState(Any, false)) - end - elseif (hd === :code_coverage_effect || + if (hd === :code_coverage_effect || # :boundscheck can be narrowed to Bool (hd !== :boundscheck && is_meta_expr(stmt))) rt = Nothing @@ -3890,7 +4373,7 @@ end end end if lhs !== nothing && rt !== Bottom - changes = StateUpdate(lhs::SlotNumber, VarState(rt, false)) + changes = StateUpdate(lhs::SlotNumber, VarState(rt, frame.currpc, #= undef =# false)) end end return AbstractEvalBasicStatementResult(rt, exct, effects, changes, refinements, currsaw_latestworld) @@ -3921,7 +4404,7 @@ end @nospecializeinfer function widenreturn(𝕃ᵢ::MustAliasesLattice, @nospecialize(rt), info::BestguessInfo) if isa(rt, MustAlias) - if 1 ≤ rt.slot ≤ info.nargs + if 1 ≤ rt.slot ≤ info.nargs && rt.ssadef == 0 rt = InterMustAlias(rt) else rt = widenmustalias(rt) @@ -3956,13 +4439,13 @@ end rt = widenconditional(rt) end end - if isa(rt, Conditional) + if isa(rt, Conditional) && rt.ssadef == 0 rt = InterConditional(rt.slot, rt.thentype, rt.elsetype) elseif is_lattice_bool(𝕃ᵢ, rt) rt = bool_rt_to_conditional(rt, info) end end - if isa(rt, Conditional) + if isa(rt, Conditional) && rt.ssadef == 0 rt = InterConditional(rt) end isa(rt, InterConditional) && return rt @@ -4030,6 +4513,8 @@ end fields[i] = a end anyrefine && return PartialStruct(𝕃ᵢ, rt.typ, _getundefs(rt), fields) + elseif isa(rt, PartialTask) + return rt # already widened, by construction end if isa(rt, PartialOpaque) return rt # XXX: this case was missed in #39512 @@ -4066,23 +4551,45 @@ function handle_control_backedge!(interp::AbstractInterpreter, frame::InferenceS return nothing end -function update_bbstate!(𝕃ᵢ::AbstractLattice, frame::InferenceState, bb::Int, vartable::VarTable, saw_latestworld::Bool) +# Intersect `dest` alias table with `src` in-place (meet operation at CFG join points). +# A slot is considered aliased in the merged state only if it has the same alias on all +# incoming paths. Returns true if `dest` changed. +function intersect_alias_tables!(dest::Vector{Int}, src::Vector{Int}) + changed = false + for i in 1:length(dest) + if dest[i] != 0 && dest[i] != src[i] + dest[i] = 0 + changed = true + end + end + return changed +end + +function update_bbstate!( + 𝕃ᵢ::AbstractLattice, vartable::VarTable, slot_aliases::Vector{Int}, bb::Int, + saw_latestworld::Bool, frame::InferenceState + ) frame.bb_saw_latestworld[bb] |= saw_latestworld - bbtable = frame.bb_vartables[bb] - if bbtable === nothing + bbstate = frame.bb_states[bb] + if bbstate === nothing # if a basic block hasn't been analyzed yet, # we can update its state a bit more aggressively - frame.bb_vartables[bb] = copy(vartable) + frame.bb_states[bb] = BBEntryState(copy(vartable), copy(slot_aliases)) return true else - return stupdate!(𝕃ᵢ, bbtable, vartable) + pc = first(frame.cfg.blocks[bb].stmts) + # Minus sign marks this as a "virtual" PC so that it is + # not confused with a real assignment at this PC. + changed = stupdate!(𝕃ᵢ, bbstate.vartable, vartable, -pc) + changed |= intersect_alias_tables!(bbstate.aliases, slot_aliases) + return changed end end function init_vartable!(vartable::VarTable, frame::InferenceState) nargtypes = length(frame.result.argtypes) for i = 1:length(vartable) - vartable[i] = VarState(Bottom, i > nargtypes) + vartable[i] = VarState(Bottom, #= ssadef =# typemin(Int), i > nargtypes) end return vartable end @@ -4133,6 +4640,7 @@ end function update_exc_bestguess!(interp::AbstractInterpreter, @nospecialize(exct), frame::InferenceState) 𝕃ₚ = ipo_lattice(interp) handler = gethandler(frame) + exct = widenslotwrapper(exct) if handler === nothing if !⊑(𝕃ₚ, exct, frame.exc_bestguess) frame.exc_bestguess = tmerge(𝕃ₚ, frame.exc_bestguess, exct) @@ -4153,14 +4661,17 @@ function update_exc_bestguess!(interp::AbstractInterpreter, @nospecialize(exct), end end -function propagate_to_error_handler!(currstate::VarTable, currsaw_latestworld::Bool, frame::InferenceState, 𝕃ᵢ::AbstractLattice) +function propagate_to_error_handler!( + 𝕃ᵢ::AbstractLattice, currstate::VarTable, slot_aliases::Vector{Int}, + currsaw_latestworld::Bool, frame::InferenceState, + ) # If this statement potentially threw, propagate the currstate to the # exception handler, BEFORE applying any state changes. curr_hand = gethandler(frame) if curr_hand !== nothing enter = frame.src.code[curr_hand.enter_idx]::EnterNode exceptbb = block_for_inst(frame.cfg, enter.catch_dest) - if update_bbstate!(𝕃ᵢ, frame, exceptbb, currstate, currsaw_latestworld) + if update_bbstate!(𝕃ᵢ, currstate, slot_aliases, exceptbb, currsaw_latestworld, frame) push!(frame.ip, exceptbb) end end @@ -4178,11 +4689,14 @@ end struct CurrentState result::Future{RTEffects} currstate::VarTable + slot_aliases::Vector{Int} currsaw_latestworld::Bool bbstart::Int bbend::Int - CurrentState(result::Future{RTEffects}, currstate::VarTable, currsaw_latestworld::Bool, bbstart::Int, bbend::Int) = - new(result, currstate, currsaw_latestworld, bbstart, bbend) + CurrentState( + result::Future{RTEffects}, currstate::VarTable, slot_aliases::Vector{Int}, + currsaw_latestworld::Bool, bbstart::Int, bbend::Int + ) = new(result, currstate, slot_aliases, currsaw_latestworld, bbstart, bbend) CurrentState() = new() end @@ -4193,7 +4707,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr bbs = frame.cfg.blocks nbbs = length(bbs) 𝕃ᵢ = typeinf_lattice(interp) - states = frame.bb_vartables + states = frame.bb_states saw_latestworld = frame.bb_saw_latestworld currbb = frame.currbb currpc = frame.currpc @@ -4204,6 +4718,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr bbend = nextresult.bbend currstate = nextresult.currstate currsaw_latestworld = nextresult.currsaw_latestworld + slot_aliases = nextresult.slot_aliases stmt = frame.src.code[currpc] result = abstract_eval_basic_statement(interp, stmt, StatementState(currstate, currsaw_latestworld), frame, nextresult.result) @goto injected_result @@ -4212,12 +4727,14 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr if currbb != 1 currbb = frame.currbb = _bits_findnext(W.bits, 1)::Int # next basic block end - currstate = copy(states[currbb]::VarTable) + currstate = copy((states[currbb]::BBEntryState).vartable) currsaw_latestworld = saw_latestworld[currbb] + slot_aliases = copy((states[1]::BBEntryState).aliases) while currbb <= nbbs delete!(W, currbb) bbstart = first(bbs[currbb].stmts) bbend = last(bbs[currbb].stmts) + init_slot_aliases!(slot_aliases, frame, currbb) currpc = bbstart - 1 while currpc < bbend @@ -4246,9 +4763,10 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr end orig_condt = condt if !(isa(condt, Const) || isa(condt, Conditional)) && isa(condslot, SlotNumber) + vtyp = currstate[slot_id(condslot)] # if this non-`Conditional` object is a slot, we form and propagate # the conditional constraint on it - condt = Conditional(condslot, Const(true), Const(false)) + condt = Conditional(condslot, vtyp.ssadef, Const(true), Const(false)) end condval = maybe_extract_const_bool(condt) nothrow = (condval !== nothing) || ⊑(𝕃ᵢ, orig_condt, Bool) @@ -4256,7 +4774,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr add_curr_ssaflag!(frame, IR_FLAG_NOTHROW) else update_exc_bestguess!(interp, TypeError, frame) - propagate_to_error_handler!(currstate, currsaw_latestworld, frame, 𝕃ᵢ) + propagate_to_error_handler!(𝕃ᵢ, currstate, slot_aliases, currsaw_latestworld, frame) merge_effects!(interp, frame, EFFECTS_THROWS) end @@ -4293,30 +4811,32 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr # We continue with the true branch, but process the false # branch here. - if isa(condt, Conditional) - else_change = conditional_change(𝕃ᵢ, currstate, condt, #=then_or_else=#false) + if isa(condt, Conditional) && conditional_valid(condt, currstate) + else_change = conditional_change(𝕃ᵢ, currstate, condt, :else) if else_change !== nothing elsestate = copy(currstate) - stoverwrite1!(elsestate, else_change) + strefine1!(elsestate, else_change) + propagate_aliased_condition!(𝕃ᵢ, elsestate, condt, :else, slot_aliases) elseif condslot isa SlotNumber elsestate = copy(currstate) else elsestate = currstate end if condslot isa SlotNumber # refine the type of this conditional object itself for this else branch - stoverwrite1!(elsestate, condition_object_change(currstate, condt, condslot, #=then_or_else=#false)) + strefine1!(elsestate, condition_object_change(currstate, condt, condslot, :else)) end - else_changed = update_bbstate!(𝕃ᵢ, frame, falsebb, elsestate, currsaw_latestworld) - then_change = conditional_change(𝕃ᵢ, currstate, condt, #=then_or_else=#true) + else_changed = update_bbstate!(𝕃ᵢ, elsestate, slot_aliases, falsebb, currsaw_latestworld, frame) + then_change = conditional_change(𝕃ᵢ, currstate, condt, :then) thenstate = currstate if then_change !== nothing - stoverwrite1!(thenstate, then_change) + strefine1!(thenstate, then_change) + propagate_aliased_condition!(𝕃ᵢ, thenstate, condt, :then, slot_aliases) end if condslot isa SlotNumber # refine the type of this conditional object itself for this then branch - stoverwrite1!(thenstate, condition_object_change(currstate, condt, condslot, #=then_or_else=#true)) + strefine1!(thenstate, condition_object_change(currstate, condt, condslot, :then)) end else - else_changed = update_bbstate!(𝕃ᵢ, frame, falsebb, currstate, currsaw_latestworld) + else_changed = update_bbstate!(𝕃ᵢ, currstate, slot_aliases, falsebb, currsaw_latestworld, frame) end if else_changed handle_control_backedge!(interp, frame, currpc, stmt.dest) @@ -4362,7 +4882,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr sstate = StatementState(currstate, currsaw_latestworld) result = abstract_eval_basic_statement(interp, stmt, sstate, frame) if result isa Future{RTEffects} - return CurrentState(result, currstate, currsaw_latestworld, bbstart, bbend) + return CurrentState(result, currstate, slot_aliases, currsaw_latestworld, bbstart, bbend) else @label injected_result (; rt, exct, effects, changes, refinements, currsaw_latestworld) = result @@ -4374,7 +4894,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr # TODO: assert that these conditions match. For now, we assume the `nothrow` flag # to be correct, but allow the exct to be an over-approximation. end - propagate_to_error_handler!(currstate, currsaw_latestworld, frame, 𝕃ᵢ) + propagate_to_error_handler!(𝕃ᵢ, currstate, slot_aliases, currsaw_latestworld, frame) end if rt === Bottom ssavaluetypes[currpc] = Bottom @@ -4386,14 +4906,15 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr end if changes !== nothing stoverwrite1!(currstate, changes) + update_alias_table!(slot_aliases, stmt, frame.src.code) end if refinements isa SlotRefinement - apply_refinement!(𝕃ᵢ, refinements.slot, refinements.typ, currstate, changes) + apply_refinement!(𝕃ᵢ, refinements.slot, refinements.typ, currstate, changes, slot_aliases) elseif refinements isa Vector{Any} for i = 1:length(refinements) newtyp = refinements[i] newtyp === nothing && continue - apply_refinement!(𝕃ᵢ, SlotNumber(i), newtyp, currstate, changes) + apply_refinement!(𝕃ᵢ, SlotNumber(i), newtyp, currstate, changes, slot_aliases) end end if rt === nothing @@ -4410,7 +4931,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr # Case 2: Directly branch to a different BB begin @label branch - if update_bbstate!(𝕃ᵢ, frame, nextbb, currstate, currsaw_latestworld) + if update_bbstate!(𝕃ᵢ, currstate, slot_aliases, nextbb, currsaw_latestworld, frame) push!(W, nextbb) end end @@ -4421,11 +4942,11 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr currbb == -1 && break # the working set is empty currbb > nbbs && break - nexttable = states[currbb] - if nexttable === nothing + nextstate = states[currbb] + if nextstate === nothing init_vartable!(currstate, frame) else - stoverwrite!(currstate, nexttable) + stoverwrite!(currstate, nextstate.vartable) end end end # while currbb <= nbbs @@ -4433,24 +4954,130 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr return CurrentState() end -function apply_refinement!(𝕃ᵢ::AbstractLattice, slot::SlotNumber, @nospecialize(newtyp), - currstate::VarTable, currchanges::Union{Nothing,StateUpdate}) +function apply_refinement!( + 𝕃ᵢ::AbstractLattice, slot::SlotNumber, @nospecialize(newtyp), + currstate::VarTable, currchanges::Union{Nothing,StateUpdate}, + slot_aliases::Vector{Int} + ) if currchanges !== nothing && currchanges.var == slot return # type propagation from statement (like assignment) should have the precedence end - vtype = currstate[slot_id(slot)] + slotid = slot_id(slot) + vtype = currstate[slotid] oldtyp = vtype.typ ⊏ = strictpartialorder(𝕃ᵢ) if newtyp ⊏ oldtyp - stmtupdate = StateUpdate(slot, VarState(newtyp, vtype.undef)) - stoverwrite1!(currstate, stmtupdate) + refinement = StateRefinement(slotid, newtyp, vtype.undef) + strefine1!(currstate, refinement) + for i in 1:length(currstate) + slot_aliases[i] == slotid || continue + alias_vtype = currstate[i] + if newtyp ⊏ alias_vtype.typ + strefine1!(currstate, StateRefinement(i, newtyp, alias_vtype.undef)) + end + end + end +end + +function init_slot_aliases!(slot_aliases::Vector{Int}, frame::InferenceState, bb::Int) + entry = frame.bb_states[bb] + if entry !== nothing + copyto!(slot_aliases, entry.aliases) + else + fill!(slot_aliases, 0) + end +end + +function clear_slot_aliases!(aliases::Vector{Int}, slot::Int) + for i in 1:length(aliases) + if aliases[i] == slot + aliases[i] = 0 + end + end + aliases[slot] = 0 + return aliases +end + +# Core transfer function: update an alias table for a single statement. +# Handles assignments (`y = x`) and `NewvarNode` declaration +function update_alias_table!(aliases::Vector{Int}, @nospecialize(stmt), code::Vector{Any}) + if isa(stmt, NewvarNode) + # When a slot is killed, also clear any slots that alias it, since + # those aliases are now stale (the target has a new undefined value). + clear_slot_aliases!(aliases, slot_id(stmt.slot)) + return + end + lhs = rhs = nothing + if isexpr(stmt, :(=)) && length(stmt.args) == 2 + lhs = stmt.args[1] + rhs = stmt.args[2] + end + isa(lhs, SlotNumber) || return + lhs_id = slot_id(lhs) + # When a slot is reassigned, clear any slots that were aliasing it. + # They still hold the OLD value of lhs, so the alias lhs_id→... is stale. + clear_slot_aliases!(aliases, lhs_id) + rhs === nothing && return + while isa(rhs, SSAValue) + rhs = code[rhs.id] + end + if isa(rhs, SlotNumber) + rhs_id = slot_id(rhs) + rhs_alias = aliases[rhs_id] + aliases[lhs_id] = rhs_alias == 0 ? rhs_id : rhs_alias + end +end + +# When a Conditional refines slot `x`, propagate the same refinement to all slots aliased to `x`. +function propagate_aliased_condition!( + 𝕃ᵢ::AbstractLattice, state::VarTable, condt::Conditional, then_or_else::Symbol, + slot_aliases::Vector{Int} + ) + condslot = condt.slot + for i in 1:length(state) + slot_aliases[i] == condslot || continue + alias_condt = Conditional(i, state[i].ssadef, condt.thentype, condt.elsetype) + alias_change = conditional_change(𝕃ᵢ, state, alias_condt, then_or_else) + if alias_change !== nothing + strefine1!(state, alias_change) + end end end -function conditional_change(𝕃ᵢ::AbstractLattice, currstate::VarTable, condt::Conditional, then_or_else::Bool) +""" + conditional_valid(condt::Conditional, currstate::VarTable) -> Bool + +Check whether a `Conditional` is still valid for refining the type of its slot. + +A `Conditional` becomes invalid when the slot it references has been reassigned +since the conditional was created. This is detected by comparing the `ssadef` +(reaching definition) stored in the `Conditional` with the current `ssadef` of +the slot in `currstate`. If they differ, the slot now holds a different object +and the conditional's type constraints no longer apply. + +For example: +```julia +x = foo() # ssadef=1 +cond = x isa Int # creates Conditional(slot=x, ssadef=1, ...) +x = bar() # ssadef=2, x is now a different object +if cond # conditional_valid returns false here + # Cannot assume x::Int because x was reassigned +end +``` +""" +function conditional_valid(condt::Conditional, currstate::VarTable) + @assert condt.ssadef != typemin(Int) + return currstate[condt.slot].ssadef == condt.ssadef +end + +function conditional_change(𝕃ᵢ::AbstractLattice, currstate::VarTable, condt::Conditional, then_or_else::Symbol) vtype = currstate[condt.slot] oldtyp = vtype.typ - newtyp = then_or_else ? condt.thentype : condt.elsetype + newtyp = if then_or_else === :then + condt.thentype + elseif then_or_else === :else + condt.elsetype + else @assert false end if iskindtype(newtyp) # this code path corresponds to the special handling for `isa(x, iskindtype)` check # implemented within `abstract_call_builtin` @@ -4458,6 +5085,10 @@ function conditional_change(𝕃ᵢ::AbstractLattice, currstate::VarTable, condt # approximate test for `typ ∩ oldtyp` being better than `oldtyp` # since we probably formed these types with `typesubstract`, # the comparison is likely simple + elseif condt.isdefined && then_or_else === :then && vtype.undef + # For `@isdefined slot`, the type may not be a refinement + # but the `.undef` information still can be + return StateRefinement(condt.slot, oldtyp, #= undef =# false) else return nothing end @@ -4467,29 +5098,33 @@ function conditional_change(𝕃ᵢ::AbstractLattice, currstate::VarTable, condt newtyp = tmerge(𝕃ᵢ, newtyp, LimitedAccuracy(Bottom, oldtyp.causes)) end # if this `Conditional` is from `@isdefined condt.slot`, refine its `undef` information - newundef = condt.isdefined ? !then_or_else : vtype.undef - return StateUpdate(SlotNumber(condt.slot), VarState(newtyp, newundef), #=conditional=#true) + newundef = condt.isdefined ? (then_or_else === :else) : vtype.undef + return StateRefinement(condt.slot, newtyp, newundef) end function condition_object_change(currstate::VarTable, condt::Conditional, - condslot::SlotNumber, then_or_else::Bool) + condslot::SlotNumber, then_or_else::Symbol) vtype = currstate[slot_id(condslot)] - newcondt = Conditional(condt.slot, - then_or_else ? condt.thentype : Union{}, - then_or_else ? Union{} : condt.elsetype) - return StateUpdate(condslot, VarState(newcondt, vtype.undef)) + if then_or_else === :then + thentype = condt.thentype + elsetype = Union{} + elseif then_or_else === :else + thentype = Union{} + elsetype = condt.elsetype + else @assert false end + newcondt = Conditional(condt.slot, condt.ssadef, thentype, elsetype) + return StateRefinement(slot_id(condslot), newcondt, vtype.undef) end # make as much progress on `frame` as possible (by handling cycles) warnlength::Int = 2500 -function typeinf(interp::AbstractInterpreter, frame::InferenceState) +function typeinf(interp::AbstractInterpreter, frame::InferenceState{I}) where {I<:AbstractInterpreter} time_before = _time_ns() - callstack = frame.callstack::Vector{AbsIntState} + callstack = frame.callstack nextstates = CurrentState[] takenext = frame.frameid minwarn = warnlength - takeprev = 0 - while takenext >= frame.frameid + @zone "CC: ABSTRACT_INTERPRET" while takenext >= frame.frameid callee = takenext == 0 ? frame : callstack[takenext]::InferenceState if !isempty(callstack) if length(callstack) - frame.frameid >= minwarn diff --git a/Compiler/src/abstractlattice.jl b/Compiler/src/abstractlattice.jl index 4d0accedfc765..e91bf0391cb75 100644 --- a/Compiler/src/abstractlattice.jl +++ b/Compiler/src/abstractlattice.jl @@ -24,13 +24,13 @@ is_valid_lattice_norec(::ConstsLattice, @nospecialize(elem)) = isa(elem, Const) """ struct PartialsLattice{𝕃<:AbstractLattice} <: AbstractLattice -A lattice extending a base lattice `𝕃` and adjoining `PartialStruct` and `PartialOpaque`. +A lattice extending a base lattice `𝕃` and adjoining `PartialStruct`, `PartialOpaque`, and `PartialTask`. """ struct PartialsLattice{𝕃<:AbstractLattice} <: AbstractLattice parent::𝕃 end widenlattice(𝕃::PartialsLattice) = 𝕃.parent -is_valid_lattice_norec(::PartialsLattice, @nospecialize(elem)) = isa(elem, PartialStruct) || isa(elem, PartialOpaque) +is_valid_lattice_norec(::PartialsLattice, @nospecialize(elem)) = isa(elem, PartialStruct) || isa(elem, PartialOpaque) || isa(elem, PartialTask) """ struct ConditionalsLattice{𝕃<:AbstractLattice} <: AbstractLattice @@ -80,8 +80,8 @@ const AnyConditionalsLattice{𝕃<:AbstractLattice} = Union{ConditionalsLattice{ const AnyMustAliasesLattice{𝕃<:AbstractLattice} = Union{MustAliasesLattice{𝕃}, InterMustAliasesLattice{𝕃}} const SimpleInferenceLattice = typeof(PartialsLattice(ConstsLattice())) -const BaseInferenceLattice = typeof(ConditionalsLattice(SimpleInferenceLattice.instance)) -const IPOResultLattice = typeof(InterConditionalsLattice(SimpleInferenceLattice.instance)) +const BaseInferenceLattice = typeof(MustAliasesLattice(ConditionalsLattice(SimpleInferenceLattice.instance))) +const IPOResultLattice = typeof(InterMustAliasesLattice(InterConditionalsLattice(SimpleInferenceLattice.instance))) """ struct InferenceLattice{𝕃<:AbstractLattice} <: AbstractLattice @@ -96,7 +96,7 @@ widenlattice(𝕃::InferenceLattice) = 𝕃.parent is_valid_lattice_norec(::InferenceLattice, @nospecialize(elem)) = isa(elem, LimitedAccuracy) """ - tmeet(𝕃::AbstractLattice, a, b::Type) + tmeet(𝕃::AbstractLattice, a, b::AnyType) Compute the lattice meet of lattice elements `a` and `b` over the lattice `𝕃`, dropping any results that will not be inhabited at runtime. @@ -107,7 +107,7 @@ Note that currently `b` is restricted to being a type """ function tmeet end -function tmeet(::JLTypeLattice, @nospecialize(a::Type), @nospecialize(b::Type)) +function tmeet(::JLTypeLattice, @nospecialize(a::AnyType), @nospecialize(b::AnyType)) ti = typeintersect(a, b) valid_as_lattice(ti, true) || return Bottom return ti @@ -150,7 +150,7 @@ If `𝕃` is `JLTypeLattice`, this is equivalent to subtyping. """ function ⊑ end -@nospecializeinfer ⊑(::JLTypeLattice, @nospecialize(a::Type), @nospecialize(b::Type)) = a <: b +@nospecializeinfer ⊑(::JLTypeLattice, @nospecialize(a::AnyType), @nospecialize(b::AnyType)) = a <: b """ ⊏(𝕃::AbstractLattice, a, b)::Bool @@ -183,7 +183,7 @@ end """ has_nontrivial_extended_info(𝕃::AbstractLattice, t)::Bool -Determines whether the given lattice element `t` of `𝕃` has non-trivial extended lattice +Determine whether the given lattice element `t` of `𝕃` has non-trivial extended lattice information that would not be available from the type itself. """ @nospecializeinfer has_nontrivial_extended_info(𝕃::AbstractLattice, @nospecialize t) = @@ -191,13 +191,16 @@ information that would not be available from the type itself. @nospecializeinfer function has_nontrivial_extended_info(𝕃::PartialsLattice, @nospecialize t) isa(t, PartialStruct) && return true isa(t, PartialOpaque) && return true + isa(t, PartialTask) && return true return has_nontrivial_extended_info(widenlattice(𝕃), t) end @nospecializeinfer function has_nontrivial_extended_info(𝕃::ConstsLattice, @nospecialize t) isa(t, PartialTypeVar) && return true if isa(t, Const) val = t.val - return !issingletontype(typeof(val)) && !(isa(val, Type) && hasuniquerep(val)) + # a type-valued `Const` may pin `=== val` beyond its widening (which for + # an open `val` is only the `==`-class `Type{val}`) + return !issingletontype(typeof(val)) end return has_nontrivial_extended_info(widenlattice(𝕃), t) end @@ -206,7 +209,7 @@ end """ is_const_prop_profitable_arg(𝕃::AbstractLattice, t)::Bool -Determines whether the given lattice element `t` of `𝕃` has new extended lattice information +Determine whether the given lattice element `t` of `𝕃` has new extended lattice information that should be forwarded along with constant propagation. """ @nospecializeinfer is_const_prop_profitable_arg(𝕃::AbstractLattice, @nospecialize t) = @@ -223,6 +226,7 @@ that should be forwarded along with constant propagation. # return false end isa(t, PartialOpaque) && return true + isa(t, PartialTask) && return true return is_const_prop_profitable_arg(widenlattice(𝕃), t) end @nospecializeinfer function is_const_prop_profitable_arg(𝕃::ConstsLattice, @nospecialize t) @@ -246,12 +250,17 @@ end @nospecializeinfer function is_forwardable_argtype(𝕃::PartialsLattice, @nospecialize x) isa(x, PartialStruct) && return true isa(x, PartialOpaque) && return true + isa(x, PartialTask) && return true return is_forwardable_argtype(widenlattice(𝕃), x) end @nospecializeinfer function is_forwardable_argtype(𝕃::ConstsLattice, @nospecialize x) isa(x, Const) && return true return is_forwardable_argtype(widenlattice(𝕃), x) end +@nospecializeinfer function is_forwardable_argtype(𝕃::MustAliasesLattice, @nospecialize x) + isa(x, MustAlias) && return true + return is_forwardable_argtype(widenlattice(𝕃), x) +end @nospecializeinfer is_forwardable_argtype(::JLTypeLattice, @nospecialize x) = false """ @@ -260,7 +269,7 @@ end Appropriately converts inferred type of a return value `rt` to such a type that we know we can store in the cache and is valid and good inter-procedurally, -E.g. if `rt isa Conditional` then `rt` should be converted to `InterConditional` +e.g. if `rt isa Conditional` then `rt` should be converted to `InterConditional` or the other cacheable lattice element. External lattice `𝕃ᵢ::ExternalLattice` may overload: diff --git a/base/invalidation.jl b/Compiler/src/bindinginvalidations.jl similarity index 73% rename from base/invalidation.jl rename to Compiler/src/bindinginvalidations.jl index 0a44449748c2f..fa29ea9025fce 100644 --- a/base/invalidation.jl +++ b/Compiler/src/bindinginvalidations.jl @@ -1,26 +1,18 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -struct GlobalRefIterator - mod::Module -end -IteratorSize(::Type{GlobalRefIterator}) = SizeUnknown() -globalrefs(mod::Module) = GlobalRefIterator(mod) - -function iterate(gri::GlobalRefIterator, i = 1) - m = gri.mod - table = ccall(:jl_module_get_bindings, Ref{SimpleVector}, (Any,), m) - i > length(table) && return nothing - b = table[i] - b === nothing && return iterate(gri, i+1) - return ((b::Core.Binding).globalref, i+1) -end +using ..Compiler: _uncompressed_ir, specializations, get_ci_mi, convert, generating_output, has_image_globalref, + PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED, + BINDING_FLAG_ANY_IMPLICIT_EDGES, binding_kind, partition_restriction, is_some_imported, + is_some_binding_imported, is_some_implicit, SizeUnknown, maybe_add_binding_backedge!, walk_binding_partition, binding_access_key, userefs, + MaybeCompressed +using .Core: SimpleVector, CodeInfo function foreachgr(visit, src::CodeInfo) stmts = src.code for i = 1:length(stmts) stmt = stmts[i] isa(stmt, GlobalRef) && visit(stmt) - for ur in Compiler.userefs(stmt) + for ur in userefs(stmt) arg = ur[] isa(arg, GlobalRef) && visit(arg) end @@ -35,7 +27,7 @@ function anygr(visit, src::CodeInfo) visit(stmt) && return true continue end - for ur in Compiler.userefs(stmt) + for ur in userefs(stmt) arg = ur[] isa(arg, GlobalRef) && visit(arg) && return true end @@ -65,11 +57,11 @@ end function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalidated_bpart::Core.BindingPartition, new_max_world::UInt) invalidate_all = false binding = convert(Core.Binding, gr) - if isdefined(method, :source) + if isdefined(method, :source) && isa(method.source, MaybeCompressed) src = _uncompressed_ir(method) invalidate_all = should_invalidate_code_for_globalref(gr, src) end - if invalidate_all && !Base.generating_output() + if invalidate_all && !generating_output() @atomic method.did_scan_source |= 0x4 end invalidated_any = false @@ -99,15 +91,13 @@ export_affecting_partition_flags(bpart::Core.BindingPartition) = function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) gr = b.globalref - (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world) - (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1) + (_, (ib, ibpart)) = walk_binding_partition(b, invalidated_bpart, new_max_world, false) + (_, (nb, nbpart)) = walk_binding_partition(b, new_bpart, new_max_world+1, false) - # `abstract_eval_partition_load` is the maximum amount of information that inference - # reads from a binding partition. If this information does not change - we do not need to - # invalidate any code that inference created, because we know that the result will not change. - need_to_invalidate_code = - Compiler.abstract_eval_partition_load(nothing, ib, ibpart) !== - Compiler.abstract_eval_partition_load(nothing, nb, nbpart) + # `binding_access_key` captures everything inference/codegen accesses about a binding partition. + # If this information does not change, we do not need to invalidate any code that inference created + # because we know that the result will not change. + need_to_invalidate_code = binding_access_key(ib, ibpart) !== binding_access_key(nb, nbpart) need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !== export_affecting_partition_flags(new_bpart) @@ -155,9 +145,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core latest_bpart = user_binding.partitions latest_bpart.max_world == typemax(UInt) || continue is_some_implicit(binding_kind(latest_bpart)) || continue - new_bpart = need_to_invalidate_export ? - ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) : - latest_bpart + new_bpart = ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) if need_to_invalidate_code || new_bpart !== latest_bpart push!(queued_bindings, (convert(Core.Binding, user_binding), latest_bpart, new_bpart)) end @@ -173,20 +161,14 @@ end invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) = invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) -function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance}) - meth = isa(edge, Method) ? edge : get_ci_mi(edge).def - ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth) - return nothing -end - -function binding_was_invalidated(b::Core.Binding) - # At least one partition is required for invalidation - !isdefined(b, :partitions) && return false - b.partitions.min_world > unsafe_load(cglobal(:jl_require_world, UInt)) -end - -function scan_new_method!(method::Method, image_backedges_only::Bool) +# Reconstruct, for a method whose source this process has not scanned yet, the invalidation +# that a binding change would have triggered had this code been present when it happened. +# Not every access records an edge (one inference left on the runtime path does not), so this +# source scan is what covers them, and it must ask the same question the edge check asks: +# `binding_changed_since_require_world`, not just "was this binding repartitioned". +function scan_new_method!(method::Method, world::UInt, image_backedges_only::Bool) isdefined(method, :source) || return + isa(method.source, MaybeCompressed) || return if image_backedges_only && !has_image_globalref(method) return end @@ -194,8 +176,8 @@ function scan_new_method!(method::Method, image_backedges_only::Bool) mod = method.module foreachgr(src) do gr::GlobalRef b = convert(Core.Binding, gr) - if binding_was_invalidated(b) - # TODO: We could turn this into an addition if condition. For now, use it as a reasonably cheap + if binding_changed_since_require_world(b, world) + # TODO: We could turn this into an additional if condition. For now, use it as a reasonably cheap # additional consistency check @assert !image_backedges_only @atomic method.did_scan_source |= 0x4 @@ -205,17 +187,14 @@ function scan_new_method!(method::Method, image_backedges_only::Bool) @atomic method.did_scan_source |= 0x1 end -function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool) - if image_backedges_only && Base.generating_output(true) +function scan_new_methods!(internal_methods::Vector{Any}, world::UInt, image_backedges_only::Bool) + if image_backedges_only && generating_output(true) # Replacing image bindings is forbidden during incremental precompilation - skip backedge insertion return end for method in internal_methods if isa(method, Method) - scan_new_method!(method, image_backedges_only) + scan_new_method!(method, world, image_backedges_only) end end - for tme::Core.TypeMapEntry in extext_methods - scan_new_method!(tme.func::Method, image_backedges_only) - end end diff --git a/Compiler/src/bootstrap.jl b/Compiler/src/bootstrap.jl index 74943fc765f17..ea14c42a26636 100644 --- a/Compiler/src/bootstrap.jl +++ b/Compiler/src/bootstrap.jl @@ -7,6 +7,8 @@ function activate_codegen!() ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel) + # Register the new unified compile and emit function + ccall(:jl_set_compile_and_emit_func, Cvoid, (Any,), compile_and_emit_native) Core.eval(Compiler, quote let typeinf_world_age = Base.tls_world_age() @eval Core.OptimizedGenerics.CompilerPlugins.typeinf(::Nothing, mi::MethodInstance, source_mode::UInt8) = @@ -24,8 +26,10 @@ function bootstrap!() ssa_inlining_pass!_tt = Tuple{typeof(ssa_inlining_pass!), IRCode, InliningState{NativeInterpreter}, Bool} optimize_tt = Tuple{typeof(optimize), NativeInterpreter, OptimizationState{NativeInterpreter}, InferenceResult} typeinf_ext_tt = Tuple{typeof(typeinf_ext), NativeInterpreter, MethodInstance, UInt8} - typeinf_tt = Tuple{typeof(typeinf), NativeInterpreter, InferenceState} - typeinf_edge_tt = Tuple{typeof(typeinf_edge), NativeInterpreter, Method, Any, SimpleVector, InferenceState, Bool, Bool} + typeinf_tt = Tuple{typeof(typeinf), NativeInterpreter, InferenceState{NativeInterpreter}} + typeinf_edge_tt = Tuple{ + typeof(typeinf_edge), NativeInterpreter, Method, Any, SimpleVector, + InferenceState{NativeInterpreter}, Bool, Bool, Bool} fs = Any[ # we first create caches for the optimizer, because they contain many loop constructions # and they're better to not run in interpreter even during bootstrapping @@ -46,7 +50,6 @@ function bootstrap!() end end starttime = time() - methods = Any[] world = get_world_counter() for f in fs if isa(f, DataType) && f.name === typename(Tuple) diff --git a/Compiler/src/cicache.jl b/Compiler/src/cicache.jl index 4d188dbbc3450..8191b95e8eebf 100644 --- a/Compiler/src/cicache.jl +++ b/Compiler/src/cicache.jl @@ -1,26 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -""" - struct InternalCodeCache - -Internally, each `MethodInstance` keep a unique global cache of code instances -that have been created for the given method instance, stratified by world age -ranges. This struct abstracts over access to this cache. -""" -struct InternalCodeCache - owner::Any # `jl_egal` is used for comparison -end - -function setindex!(cache::InternalCodeCache, ci::CodeInstance, mi::MethodInstance) - @assert ci.owner === cache.owner - m = mi.def - if isa(m, Method) - ccall(:jl_push_newly_inferred, Cvoid, (Any,), ci) - end - ccall(:jl_mi_cache_insert, Cvoid, (Any, Any), mi, ci) - return cache -end - struct WorldRange min_world::UInt max_world::UInt @@ -34,13 +13,13 @@ in(world::UInt, wr::WorldRange) = wr.min_world <= world <= wr.max_world min_world(wr::WorldRange) = first(wr) max_world(wr::WorldRange) = last(wr) -function intersect(a::WorldRange, b::WorldRange) +@inline function intersect(a::WorldRange, b::WorldRange) ret = WorldRange(max(a.min_world, b.min_world), min(a.max_world, b.max_world)) @assert ret.min_world <= ret.max_world return ret end -function union(a::WorldRange, b::WorldRange) +@inline function union(a::WorldRange, b::WorldRange) if b.min_world < a.min_world (b, a) = (a, b) end @@ -49,39 +28,43 @@ function union(a::WorldRange, b::WorldRange) end """ - struct WorldView + struct InternalCodeCache -Takes a given cache and provides access to the cache contents for the given -range of world ages, rather than defaulting to the current active world age. +Internally, each `MethodInstance` keeps a unique global cache of code instances +that have been created for the given method instance, stratified by world age +ranges. This struct abstracts over access to this cache. """ -struct WorldView{Cache} - cache::Cache +struct InternalCodeCache + owner::Any # `jl_egal` is used for comparison worlds::WorldRange - WorldView(cache::Cache, range::WorldRange) where Cache = new{Cache}(cache, range) + InternalCodeCache(@nospecialize(owner), wr::WorldRange) = new(owner, wr) + InternalCodeCache(@nospecialize(owner), args...) = new(owner, WorldRange(args...)) +end + +function setindex!(cache::InternalCodeCache, ci::CodeInstance, mi::MethodInstance) + @assert ci.owner === cache.owner + m = mi.def + if isa(m, Method) + ccall(:jl_push_newly_inferred, Cvoid, (Any,), ci) + end + ccall(:jl_mi_cache_insert, Cvoid, (Any, Any), mi, ci) + return cache end -WorldView(cache, args...) = WorldView(cache, WorldRange(args...)) -WorldView(wvc::WorldView, wr::WorldRange) = WorldView(wvc.cache, wr) -WorldView(wvc::WorldView, args...) = WorldView(wvc.cache, args...) -function haskey(wvc::WorldView{InternalCodeCache}, mi::MethodInstance) - return ccall(:jl_rettype_inferred, Any, (Any, Any, UInt, UInt), wvc.cache.owner, mi, first(wvc.worlds), last(wvc.worlds)) !== nothing +function haskey(wvc::InternalCodeCache, mi::MethodInstance) + return ccall(:jl_rettype_inferred, Any, (Any, Any, UInt, UInt), wvc.owner, mi, first(wvc.worlds), last(wvc.worlds)) !== nothing end -function get(wvc::WorldView{InternalCodeCache}, mi::MethodInstance, default) - r = ccall(:jl_rettype_inferred, Any, (Any, Any, UInt, UInt), wvc.cache.owner, mi, first(wvc.worlds), last(wvc.worlds)) +function get(wvc::InternalCodeCache, mi::MethodInstance, default) + r = ccall(:jl_rettype_inferred, Any, (Any, Any, UInt, UInt), wvc.owner, mi, first(wvc.worlds), last(wvc.worlds)) if r === nothing return default end return r::CodeInstance end -function getindex(wvc::WorldView{InternalCodeCache}, mi::MethodInstance) +function getindex(wvc::InternalCodeCache, mi::MethodInstance) r = get(wvc, mi, nothing) r === nothing && throw(KeyError(mi)) return r::CodeInstance end - -function setindex!(wvc::WorldView{InternalCodeCache}, ci::CodeInstance, mi::MethodInstance) - setindex!(wvc.cache, ci, mi) - return wvc -end diff --git a/Compiler/src/effects.jl b/Compiler/src/effects.jl index 9aea4cb204ec6..cb50cb01f1acf 100644 --- a/Compiler/src/effects.jl +++ b/Compiler/src/effects.jl @@ -11,28 +11,32 @@ The output represents the state of different effect properties in the following - `+e` (green): `ALWAYS_TRUE` - `-e` (red): `ALWAYS_FALSE` - `?e` (yellow): `EFFECT_FREE_IF_INACCESSIBLEMEMONLY` -3. `nothrow` (`n`): +3. `reset_safe` (`re`): + - `+re` (green): `ALWAYS_TRUE` + - `-re` (red): `ALWAYS_FALSE` + - `?re` (yellow): `RESET_SAFE_IF_INACCESSIBLEMEMONLY` +4. `nothrow` (`n`): - `+n` (green): `true` - `-n` (red): `false` -4. `terminates` (`t`): +5. `terminates` (`t`): - `+t` (green): `true` - `-t` (red): `false` -5. `notaskstate` (`s`): +6. `notaskstate` (`s`): - `+s` (green): `true` - `-s` (red): `false` -6. `inaccessiblememonly` (`m`): +7. `inaccessiblememonly` (`m`): - `+m` (green): `ALWAYS_TRUE` - `-m` (red): `ALWAYS_FALSE` - `?m` (yellow): `INACCESSIBLEMEM_OR_ARGMEMONLY` -7. `noub` (`u`): +8. `noub` (`u`): - `+u` (green): `true` - `-u` (red): `false` - `?u` (yellow): `NOUB_IF_NOINBOUNDS` -8. `:nonoverlayed` (`o`): +9. `:nonoverlayed` (`o`): - `+o` (green): `ALWAYS_TRUE` - `-o` (red): `ALWAYS_FALSE` - `?o` (yellow): `CONSISTENT_OVERLAY` -9. `:nortcall` (`r`): +10. `:nortcall` (`r`): - `+r` (green): `true` - `-r` (red): `false` """ @@ -47,7 +51,7 @@ of the method being analyzed. They are represented as `Bool` or `UInt8` bits wit following meanings: - `consistent::UInt8`: * `ALWAYS_TRUE`: this method is guaranteed to return or terminate consistently. - * `ALWAYS_FALSE`: this method may be not return or terminate consistently, and there is + * `ALWAYS_FALSE`: this method might not return or terminate consistently, and there is no need for further analysis with respect to this effect property as this conclusion will not be refined anyway. * `CONSISTENT_IF_NOTRETURNED`: the `:consistent`-cy of this method can later be refined to @@ -62,6 +66,15 @@ following meanings: will not be refined anyway. * `EFFECT_FREE_IF_INACCESSIBLEMEMONLY`: the `:effect-free`-ness of this method can later be refined to `ALWAYS_TRUE` in a case when `:inaccessiblememonly` is proven. +- `reset_safe::UInt8` + * The execution of this function may be interrupted and reset to an earlier cancellation + point at any point in the function. The interpretation is similar to `:effect_free`, + but has different guarantees. + N.B.: this bit describes the function's IPO contract only. Machinery the runtime + inserts implicitly to execute it (allocation, write barriers, runtime library calls) + is not covered by the contract and may not be safe to abandon - but this analysis + does not need to model that: it is deferred to codegen and the runtime (see + `llvm-cancellation-lowering.cpp`). - `nothrow::Bool`: this method is guaranteed to not throw an exception. If the execution of this method may raise `MethodError`s and similar exceptions, then the method is not considered as `:nothrow`. @@ -71,7 +84,7 @@ following meanings: - `terminates::Bool`: this method is guaranteed to terminate. - `notaskstate::Bool`: this method does not access any state bound to the current task and may thus be moved to a different task without changing observable - behavior. Note that this currently implies that `noyield` as well, since + behavior. Note that this currently implies `noyield` as well, since yielding modifies the state of the current task, though this may be split in the future. - `inaccessiblememonly::UInt8`: @@ -92,11 +105,11 @@ following meanings: assertions (such as `:consistent` or `:effect_free`) as well, but we do not model this, and they assume the absence of undefined behavior. - `nonoverlayed::UInt8`: - * `ALWAYS_TRUE`: this method is guaranteed to not invoke any methods that defined in an + * `ALWAYS_TRUE`: this method is guaranteed to not invoke any methods that are defined in an [overlayed method table](@ref OverlayMethodTable). * `CONSISTENT_OVERLAY`: this method may invoke overlayed methods, but all such overlayed methods are `:consistent` with their non-overlayed original counterparts - (see [`Base.@assume_effects`](@ref) for the exact definition of `:consistenct`-cy). + (see [`Base.@assume_effects`](@ref) for the exact definition of `:consistent`-cy). * `ALWAYS_FALSE`: this method may invoke overlayed methods. - `nortcall::Bool`: this method does not call `Core.Compiler.return_type`, and it is guaranteed that any other methods this method might call also do not call @@ -106,9 +119,9 @@ Note that the representations above are just internal implementation details and to change in the future. See [`Base.@assume_effects`](@ref) for more detailed explanation on the definitions of these properties. -Along the abstract interpretation, `Effects` at each statement are analyzed locally and they +During abstract interpretation, `Effects` at each statement are analyzed locally and they are merged into the single global `Effects` that represents the entire effects of the -analyzed method (see the implementation of `merge_effects!`). Each effect property is +analyzed method (see the implementation of `merge_effects`). Each effect property is initialized with `ALWAYS_TRUE`/`true` and then transitioned towards `ALWAYS_FALSE`/`false`. Note that within the current flow-insensitive analysis design, effects detected by local analysis on each statement usually taint the global conclusion conservatively. @@ -119,6 +132,7 @@ $(effects_key_string) struct Effects consistent::UInt8 effect_free::UInt8 + reset_safe::UInt8 nothrow::Bool terminates::Bool notaskstate::Bool @@ -129,6 +143,7 @@ struct Effects function Effects( consistent::UInt8, effect_free::UInt8, + reset_safe::UInt8, nothrow::Bool, terminates::Bool, notaskstate::Bool, @@ -139,6 +154,7 @@ struct Effects return new( consistent, effect_free, + reset_safe, nothrow, terminates, notaskstate, @@ -175,14 +191,18 @@ const NOUB_IF_NOINBOUNDS = 0x01 << 1 # :nonoverlayed bits const CONSISTENT_OVERLAY = 0x01 << 1 -const EFFECTS_TOTAL = Effects(ALWAYS_TRUE, ALWAYS_TRUE, true, true, true, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true) -const EFFECTS_THROWS = Effects(ALWAYS_TRUE, ALWAYS_TRUE, false, true, true, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true) -const EFFECTS_UNKNOWN = Effects(ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_TRUE, false) # unknown mostly, but it's not overlayed at least (e.g. it's not a call) +# :reset_safe bits +const RESET_SAFE_IF_INACCESSIBLEMEMONLY = 0x01 << 1 -function Effects(effects::Effects=Effects( - ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false); +const EFFECTS_TOTAL = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true, true, true, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true) +const EFFECTS_THROWS = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, false, true, true, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true) +const EFFECTS_UNKNOWN = Effects(ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_TRUE, false) # unknown mostly, but it's not overlayed at least (e.g. it's not a call) +const EFFECTS_MINIMAL = Effects(ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false, false, false, ALWAYS_FALSE, ALWAYS_FALSE, ALWAYS_FALSE, false) + +function Effects(effects::Effects=EFFECTS_MINIMAL; consistent::UInt8 = effects.consistent, effect_free::UInt8 = effects.effect_free, + reset_safe::UInt8 = effects.reset_safe, nothrow::Bool = effects.nothrow, terminates::Bool = effects.terminates, notaskstate::Bool = effects.notaskstate, @@ -193,6 +213,7 @@ function Effects(effects::Effects=Effects( return Effects( consistent, effect_free, + reset_safe, nothrow, terminates, notaskstate, @@ -218,13 +239,21 @@ function is_better_effects(new::Effects, old::Effects) end end if new.effect_free == ALWAYS_TRUE - any_improved |= old.consistent != ALWAYS_TRUE + any_improved |= old.effect_free != ALWAYS_TRUE elseif new.effect_free == EFFECT_FREE_IF_INACCESSIBLEMEMONLY old.effect_free == ALWAYS_TRUE && return false any_improved |= old.effect_free != EFFECT_FREE_IF_INACCESSIBLEMEMONLY elseif new.effect_free != old.effect_free return false end + if new.reset_safe == ALWAYS_TRUE + any_improved |= old.reset_safe != ALWAYS_TRUE + elseif new.reset_safe == RESET_SAFE_IF_INACCESSIBLEMEMONLY + old.reset_safe == ALWAYS_TRUE && return false + any_improved |= old.reset_safe != RESET_SAFE_IF_INACCESSIBLEMEMONLY + elseif new.reset_safe != old.reset_safe + return false + end if new.nothrow any_improved |= !old.nothrow elseif new.nothrow != old.nothrow @@ -276,6 +305,7 @@ function merge_effects(old::Effects, new::Effects) return Effects( merge_effectbits(old.consistent, new.consistent), merge_effectbits(old.effect_free, new.effect_free), + merge_effectbits(old.reset_safe, new.reset_safe), merge_effectbits(old.nothrow, new.nothrow), merge_effectbits(old.terminates, new.terminates), merge_effectbits(old.notaskstate, new.notaskstate), @@ -295,6 +325,7 @@ merge_effectbits(old::Bool, new::Bool) = old & new is_consistent(effects::Effects) = effects.consistent === ALWAYS_TRUE is_effect_free(effects::Effects) = effects.effect_free === ALWAYS_TRUE +is_reset_safe(effects::Effects) = effects.reset_safe === ALWAYS_TRUE is_nothrow(effects::Effects) = effects.nothrow is_terminates(effects::Effects) = effects.terminates is_notaskstate(effects::Effects) = effects.notaskstate @@ -331,6 +362,8 @@ is_consistent_if_inaccessiblememonly(effects::Effects) = !iszero(effects.consist is_effect_free_if_inaccessiblememonly(effects::Effects) = !iszero(effects.effect_free & EFFECT_FREE_IF_INACCESSIBLEMEMONLY) +is_reset_safe_if_inaccessiblememonly(effects::Effects) = !iszero(effects.reset_safe & RESET_SAFE_IF_INACCESSIBLEMEMONLY) + is_inaccessiblemem_or_argmemonly(effects::Effects) = effects.inaccessiblememonly === INACCESSIBLEMEM_OR_ARGMEMONLY is_consistent_overlay(effects::Effects) = effects.nonoverlayed === CONSISTENT_OVERLAY @@ -345,13 +378,15 @@ function encode_effects(e::Effects) ((e.inaccessiblememonly % UInt32) << 8) | ((e.noub % UInt32) << 10) | ((e.nonoverlayed % UInt32) << 12) | - ((e.nortcall % UInt32) << 14) + ((e.nortcall % UInt32) << 14) | + ((e.reset_safe % UInt32) << 15) end function decode_effects(e::UInt32) return Effects( UInt8((e >> 0) & 0x07), UInt8((e >> 3) & 0x03), + UInt8((e >> 15) & 0x03), Bool((e >> 5) & 0x01), Bool((e >> 6) & 0x01), Bool((e >> 7) & 0x01), diff --git a/Compiler/src/inferenceresult.jl b/Compiler/src/inferenceresult.jl index 77f897e4035a5..44c25e19620e4 100644 --- a/Compiler/src/inferenceresult.jl +++ b/Compiler/src/inferenceresult.jl @@ -1,22 +1,41 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -function matching_cache_argtypes(𝕃::AbstractLattice, mi::MethodInstance) +function matching_cache_argtypes(::AbstractLattice, mi::MethodInstance) (; def, specTypes) = mi return most_general_argtypes(isa(def, Method) ? def : nothing, specTypes) end +# For `@nospecializeinfer` methods, widen the `@nospecialize`'d argument positions back to +# `cache_argtypes` values to respect the `@nospecializeinfer` semantics. +# This also ensures that the constprop `argtypes` have the same length as `cache_argtypes`. +function get_nospecializeinfer_argtypes(argtypes::Vector{Any}, cache_argtypes::Vector{Any}, + method::Method) + is_nospecializeinfer(method) || return argtypes + nargs = Int(method.nargs) + new_argtypes = Vector{Any}(undef, length(cache_argtypes)) + for i = 1:length(cache_argtypes) + i_arg = min(i - 1, nargs - 1) # 0-indexed, 0 is the function slot + if i_arg > 0 && !iszero(method.nospecialize & (1 << (i_arg - 1))) + new_argtypes[i] = cache_argtypes[i] + else + new_argtypes[i] = argtypes[i] + end + end + return new_argtypes +end + struct SimpleArgtypes argtypes::Vector{Any} end # Like `SimpleArgtypes`, but allows the argtypes to be wider than the current call. # As a result, it is not legal to refine the cache result with information more -# precise than was it deducible from the `WidenedSimpleArgtypes`. +# precise than what was deducible from the `WidenedSimpleArgtypes`. struct WidenedArgtypes argtypes::Vector{Any} end -function matching_cache_argtypes(𝕃::AbstractLattice, mi::MethodInstance, +function matching_cache_argtypes(𝕃::AbstractLattice, ::MethodInstance, simple_argtypes::Union{SimpleArgtypes, WidenedArgtypes}, cache_argtypes::Vector{Any}) (; argtypes) = simple_argtypes @@ -89,7 +108,7 @@ function is_argtype_match(𝕃::AbstractLattice, end end -function va_process_argtypes(𝕃::AbstractLattice, given_argtypes::Vector{Any}, nargs::UInt, isva::Bool) +function va_process_argtypes(𝕃::AbstractLattice, given_argtypes::Vector{Any}, nargs::UInt, isva::Bool, mi::MethodInstance) nargs = Int(nargs) if isva || (!isempty(given_argtypes) && isvarargtype(given_argtypes[end])) isva_given_argtypes = Vector{Any}(undef, nargs) @@ -100,6 +119,11 @@ function va_process_argtypes(𝕃::AbstractLattice, given_argtypes::Vector{Any}, newarg = widenconditional(newarg) end end + if isva && has_mustalias(𝕃) && isa(newarg, MustAlias) + if newarg.slot > (nargs-isva) + newarg = widenmustalias(newarg) + end + end isva_given_argtypes[i] = newarg end if isva @@ -115,12 +139,23 @@ function va_process_argtypes(𝕃::AbstractLattice, given_argtypes::Vector{Any}, end end end + if has_mustalias(𝕃) + for i = last:length(given_argtypes) + newarg = given_argtypes[i] + if isa(newarg, MustAlias) && newarg.slot > (nargs-isva) + given_argtypes[i] = widenmustalias(newarg) + end + end + end end isva_given_argtypes[nargs] = tuple_tfunc(𝕃, given_argtypes[last:end]) end return isva_given_argtypes end - @assert length(given_argtypes) == nargs "invalid `given_argtypes` for `mi`" + if length(given_argtypes) != nargs + println(given_argtypes, " != ", nargs, " for ", mi) + throw(AssertionError("invalid `given_argtypes` for `mi`")) + end return given_argtypes end @@ -149,7 +184,7 @@ function most_general_argtypes(method::Union{Method,Nothing}, @nospecialize(spec # replace singleton types with their equivalent Const object atyp = Const(atyp.instance) elseif isconstType(atyp) - atyp = Const(atyp.parameters[1]) + atyp = Const(type_parameter(atyp)) else atyp = elim_free_typevars(rewrap_unionall(atyp, specTypes)) end @@ -178,23 +213,48 @@ function elim_free_typevars(@nospecialize t) end end -function cache_lookup(𝕃::AbstractLattice, mi::MethodInstance, given_argtypes::Vector{Any}, - cache::Vector{InferenceResult}) - method = mi.def::Method +function constprop_cache_lookup(𝕃::AbstractLattice, mi::MethodInstance, + given_argtypes::Vector{Any}, cache::InferenceCache, + world::UInt) nargtypes = length(given_argtypes) - for cached_result in cache - cached_result.tombstone && continue # ignore deleted entries (due to LimitedAccuracy) - cached_result.linfo === mi || continue + indices = get_indices(cache, mi) + found_tombstone = false + for idx in indices + cached = cache.results[idx] + cached_result = cached isa LocalInferenceResult ? cached.result : cached + cached_result.cache_world == world || continue + valid_worlds = cached isa LocalInferenceResult ? + proof_worlds(cached.proof) : cached_result.valid_worlds + world in valid_worlds || continue cache_argtypes = cached_result.argtypes - @assert length(cache_argtypes) == nargtypes "invalid `cache_argtypes` for `mi`" - cache_overridden_by_const = cached_result.overridden_by_const::BitVector + if length(cache_argtypes) != nargtypes + # A `MethodInstance` whose `specTypes` ends in an unbounded `Vararg` (i.e. its + # trailing varargs are not specialized to a fixed arity) can be + # const-propagated at multiple arities, producing cached results whose + # `argtypes` differ in length (e.g. one ending in a `Vararg` element, another + # expanded to concrete arguments). Such entries describe distinct argument + # refinements, and a query of a different arity cannot reuse them, so skip. + # Any other length mismatch would indicate a genuine bug. + @assert isvarargtype((unwrap_unionall(mi.specTypes)::DataType).parameters[end]) "invalid `cache_argtypes` for `mi`" + @goto next_cache + end + cache_overridden_by_const = cached_result.overridden_by_const + cache_overridden_by_const === nothing && continue + cache_overridden_by_const = cache_overridden_by_const::BitVector for i in 1:nargtypes if !is_argtype_match(𝕃, given_argtypes[i], cache_argtypes[i], cache_overridden_by_const[i]) @goto next_cache end end - return cached_result + # Don't return tombstoned entries as cache items: they represent rejected work + # (due to LimitedAccuracy). Instead, record that a tombstone was found so the + # caller can avoid re-attempting the same const-prop that would hit the same limit. + if cached_result.tombstone + found_tombstone = true + @goto next_cache + end + return cached @label next_cache end - return nothing + return found_tombstone ? missing : nothing end diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index 8a8e5354e3d59..4d318951e213d 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -94,8 +94,8 @@ end This struct is intended as a memory- and GC-pressure-efficient mechanism for incrementally computing def-use maps. The idea is that the def-use map -is constructed into two passes over the IR. In the first, we simply count the -the number of uses, computing the number of uses for each def as well as the +is constructed in two passes over the IR. In the first, we simply count the +number of uses, computing the number of uses for each def as well as the total number of uses. In the second pass, we actually fill in the def-use information. @@ -209,6 +209,18 @@ to enable flow-sensitive analysis. """ const VarTable = Vector{VarState} +""" + BBEntryState + +Bundles the per-basic-block variable-type table ([`VarTable`](@ref)) and slot-alias table +(`Vector{Int}`) into a single value. This ensures that both components are always present or +absent together. +""" +struct BBEntryState + vartable::VarTable + aliases::Vector{Int} +end + struct StatementState vtypes::Union{VarTable,Nothing} saw_latestworld::Bool @@ -217,7 +229,6 @@ end const CACHE_MODE_NULL = 0x00 # not cached, optimization optional const CACHE_MODE_GLOBAL = 0x01 << 0 # cached globally, optimization required const CACHE_MODE_LOCAL = 0x01 << 1 # cached locally, optimization required -const CACHE_MODE_VOLATILE = 0x01 << 2 # not cached, optimization required abstract type Handler end get_enter_idx(handler::Handler) = get_enter_idx_impl(handler)::Int @@ -259,10 +270,16 @@ end intersect(world::WorldWithRange, valid_worlds::WorldRange) = WorldWithRange(world.this, intersect(world.valid_worlds, valid_worlds)) -mutable struct InferenceState +# `InferenceState` and `IRInterpretationState` are defined together in a `typegroup` +# block so each can reference the other in its `callstack` field type. They are not +# meant to be subtyped from outside, so rather than introducing an abstract supertype +# the shared `AbsIntState{I}` is a `Union` type alias (defined further below). +typegroup + +mutable struct InferenceState{I<:AbstractInterpreter} #= information about this method instance =# linfo::MethodInstance - world::WorldWithRange + valid_worlds::WorldRange mod::Module sptypes::Vector{VarState} slottypes::Vector{Any} @@ -275,9 +292,23 @@ mutable struct InferenceState currpc::Int ip::BitSet#=TODO BoundedMinPrioritySet=# # current active instruction pointers handler_info::Union{Nothing,HandlerInfo{TryCatchFrame}} - ssavalue_uses::Vector{BitSet} # ssavalue sparsity and restart info + ssavalue_uses::SSAUses # ssavalue sparsity and restart info + # Per-basic-block entry state. `nothing` if the BB has not been analyzed yet. + # Populated lazily during the main inference loop by `update_bbstate!`, which merges + # the current exit state into each successor. Both the variable-type table and the + # slot-alias table are bundled together in a `BBEntryState` so that the type system + # guarantees they are always in sync. + # + # Slot alias tracking: + # `aliases[i] == j` means slot `i` currently holds the same value as slot `j`. + # `aliases[i] == 0` means slot `i` is not known to be aliased to any other slot. + # The table is always kept "flat": aliases always point directly to the root slot, not + # through a chain, so a single lookup suffices to find all aliases of a given slot. + # The working alias table for the BB currently being analyzed is kept as a local + # variable `slot_aliases` in `typeinf_local` (analogous to `currstate`). # TODO: Could keep this sparsely by doing structural liveness analysis ahead of time. - bb_vartables::Vector{Union{Nothing,VarTable}} # nothing if not analyzed yet + bb_states::Vector{Union{Nothing,BBEntryState}} + bb_saw_latestworld::Vector{Bool} ssavaluetypes::Vector{Any} ssaflags::Vector{UInt32} @@ -288,10 +319,10 @@ mutable struct InferenceState tasks::Vector{WorkThunk} pclimitations::IdSet{InferenceState} # causes of precision restrictions (LimitedAccuracy) on currpc ssavalue limitations::IdSet{InferenceState} # causes of precision restrictions (LimitedAccuracy) on return - cycle_backedges::Vector{Tuple{InferenceState, Int}} # call-graph backedges connecting from callee to caller + cycle_backedges::Vector{Tuple{InferenceState{I}, Int}} # call-graph backedges connecting from callee to caller # IPO tracking of in-process work, shared with all frames given AbstractInterpreter - callstack #::Vector{AbsIntState} + callstack::Vector{Union{InferenceState{I},IRInterpretationState{I}}} parentid::Int # index into callstack of the parent frame that originally added this frame (call cycle_parent to extract the current parent of the SCC) frameid::Int # index into callstack at which this object is found (or zero, if this is not a cached frame and has no parent) cycleid::Int # index into the callstack of the topmost frame in the cycle (all frames in the same cycle share the same cycleid) @@ -315,12 +346,16 @@ mutable struct InferenceState insert_coverage::Bool # The interpreter that created this inference state. Not looked at by - # NativeInterpreter. But other interpreters may use this to detect cycles + # NativeInterpreter. But other interpreters may use this to detect cycles. + # Stored as the abstract supertype to keep reads boxed (the type parameter + # `I` lets callers narrow with `frame.interp::I` only at the specific call + # sites that need concrete dispatch, without propagating concrete unboxed + # values through every method that takes `interp`). interp::AbstractInterpreter # src is assumed to be a newly-allocated CodeInfo, that can be modified in-place to contain intermediate results - function InferenceState(result::InferenceResult, src::CodeInfo, cache_mode::UInt8, - interp::AbstractInterpreter) + function InferenceState{I}(result::InferenceResult, src::CodeInfo, cache_mode::UInt8, + interp::I) where {I<:AbstractInterpreter} mi = result.linfo world = get_inference_world(interp) if world == typemax(UInt) @@ -340,34 +375,35 @@ mutable struct InferenceState ssavalue_uses = find_ssavalue_uses(code, nssavalues) nstmts = length(code) edges = [] - stmt_info = CallInfo[ NoCallInfo() for i = 1:nstmts ] + stmt_info = CallInfo[ NoCallInfo() for _ = 1:nstmts ] nslots = length(src.slotflags) slottypes = Vector{Any}(undef, nslots) - bb_saw_latestworld = Bool[false for i = 1:length(cfg.blocks)] - bb_vartables = Union{Nothing,VarTable}[ nothing for i = 1:length(cfg.blocks) ] - bb_vartable1 = bb_vartables[1] = VarTable(undef, nslots) + bb_saw_latestworld = Bool[false for _ = 1:length(cfg.blocks)] + bb_vartable1 = VarTable(undef, nslots) + bb_states = Union{Nothing,BBEntryState}[nothing for _ = 1:length(cfg.blocks)] + bb_states[1] = BBEntryState(bb_vartable1, zeros(Int, nslots)) argtypes = result.argtypes - argtypes = va_process_argtypes(typeinf_lattice(interp), argtypes, src.nargs, src.isva) + argtypes = va_process_argtypes(typeinf_lattice(interp), argtypes, src.nargs, src.isva, mi) nargtypes = length(argtypes) for i = 1:nslots argtyp = (i > nargtypes) ? Bottom : argtypes[i] if argtyp === Bool && has_conditional(typeinf_lattice(interp)) - argtyp = Conditional(i, Const(true), Const(false)) + argtyp = Conditional(i, #= ssadef =# 0, Const(true), Const(false)) end slottypes[i] = argtyp - bb_vartable1[i] = VarState(argtyp, i > nargtypes) + bb_vartable1[i] = VarState(argtyp, #= ssadef =# 0, i > nargtypes) end - src.ssavaluetypes = ssavaluetypes = Any[ NOT_FOUND for i = 1:nssavalues ] + src.ssavaluetypes = ssavaluetypes = Any[ NOT_FOUND for _ = 1:nssavalues ] ssaflags = copy(src.ssaflags) unreachable = BitSet() pclimitations = IdSet{InferenceState}() limitations = IdSet{InferenceState}() - cycle_backedges = Tuple{InferenceState,Int}[] - callstack = AbsIntState[] + cycle_backedges = Tuple{InferenceState{I},Int}[] + callstack = Union{InferenceState{I},IRInterpretationState{I}}[] tasks = WorkThunk[] valid_worlds = WorldRange(1, get_world_counter()) @@ -392,8 +428,8 @@ mutable struct InferenceState parentid = frameid = cycleid = 0 this = new( - mi, WorldWithRange(world, valid_worlds), mod, sptypes, slottypes, src, cfg, spec_info, - currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, bb_saw_latestworld, ssavaluetypes, ssaflags, edges, stmt_info, + mi, valid_worlds, mod, sptypes, slottypes, src, cfg, spec_info, + currbb, currpc, ip, handler_info, ssavalue_uses, bb_states, bb_saw_latestworld, ssavaluetypes, ssaflags, edges, stmt_info, tasks, pclimitations, limitations, cycle_backedges, callstack, parentid, frameid, cycleid, result, unreachable, bestguess, exc_bestguess, ipo_effects, _time_ns(), 0.0, 0, 0, @@ -401,9 +437,6 @@ mutable struct InferenceState interp) # some more setups - if !iszero(cache_mode & CACHE_MODE_LOCAL) - push!(get_inference_cache(interp), result) - end if !iszero(cache_mode & CACHE_MODE_GLOBAL) push!(callstack, this) this.cycleid = this.frameid = length(callstack) @@ -412,13 +445,69 @@ mutable struct InferenceState # Apply generated function restrictions if src.min_world != 1 || src.max_world != typemax(UInt) # From generated functions - update_valid_age!(this, WorldRange(src.min_world, src.max_world)) + update_valid_age!(this, world, WorldRange(src.min_world, src.max_world)) end return this end end +# TODO add `result::InferenceResult` and put the irinterp result into the inference cache? +mutable struct IRInterpretationState{I<:AbstractInterpreter} + const spec_info::SpecInfo + const ir::IRCode + const mi::MethodInstance + valid_worlds::WorldRange + curridx::Int + time_caches::Float64 + time_paused::UInt64 + const argtypes_refined::Vector{Bool} + const sptypes::Vector{VarState} + const tpdum::TwoPhaseDefUseMap + const ssa_refined::BitSet + const lazyreachability::LazyCFGReachability + const tasks::Vector{WorkThunk} + const edges::Vector{Any} + callstack::Vector{Union{InferenceState{I},IRInterpretationState{I}}} + frameid::Int + parentid::Int + interp::AbstractInterpreter # see comment on `InferenceState.interp` + + function IRInterpretationState{I}( + interp::I, spec_info::SpecInfo, ir::IRCode, + mi::MethodInstance, argtypes::Vector{Any}, min_world::UInt, max_world::UInt + ) where {I<:AbstractInterpreter} + curridx = 1 + given_argtypes = Vector{Any}(undef, length(argtypes)) + for i = 1:length(given_argtypes) + given_argtypes[i] = widenslotwrapper(argtypes[i]) + end + if isa(mi.def, Method) + argtypes_refined = Bool[!⊑(optimizer_lattice(interp), ir.argtypes[i], given_argtypes[i]) + for i = 1:length(given_argtypes)] + else + argtypes_refined = Bool[false for _ = 1:length(given_argtypes)] + end + empty!(ir.argtypes) + append!(ir.argtypes, given_argtypes) + tpdum = TwoPhaseDefUseMap(length(ir.stmts)) + ssa_refined = BitSet() + lazyreachability = LazyCFGReachability(ir) + valid_worlds = WorldRange(min_world, max_world == typemax(UInt) ? get_world_counter() : max_world) + if !(get_inference_world(interp) in valid_worlds) + error("invalid age range update") + end + tasks = WorkThunk[] + edges = Any[] + callstack = Union{InferenceState{I},IRInterpretationState{I}}[] + return new{I}(spec_info, ir, mi, valid_worlds, + curridx, 0.0, 0, argtypes_refined, ir.sptypes, tpdum, + ssa_refined, lazyreachability, tasks, edges, callstack, 0, 0, interp) + end +end + +end # typegroup + gethandler(frame::InferenceState, pc::Int=frame.currpc) = gethandler(frame.handler_info, pc) gethandler(::Nothing, ::Int) = nothing function gethandler(handler_info::HandlerInfo, pc::Int) @@ -447,12 +536,11 @@ const compute_trycatch = ComputeTryCatch{SimpleHandler}() compute_trycatch(ir.stmts.stmt, ir.cfg.blocks) """ - (::ComputeTryCatch{Handler})(code, [, bbs]) -> handler_info::Union{Nothing,HandlerInfo{Handler}} + (::ComputeTryCatch{Handler})(code[, bbs]) -> handler_info::Union{Nothing,HandlerInfo{Handler}} const compute_trycatch = ComputeTryCatch{SimpleHandler}() Given the code of a function, compute, at every statement, the current -try/catch handler, and the current exception stack top. This function returns -a tuple of: +try/catch handler, and the current exception stack top. This function returns a `HandlerInfo` with: 1. `handler_info.handler_at`: A statement length vector of tuples `(catch_handler, exception_stack)`, which are indices into `handlers` @@ -526,7 +614,7 @@ function (::ComputeTryCatch{Handler})(code::Vector{Any}, bbs::Union{Vector{Basic l = stmt.catch_dest (bbs !== nothing) && (l != 0) && (l = first(bbs[l].stmts)) # We assigned a handler number above. Here we just merge that - # with out current handler information. + # with our current handler information. if l != 0 handler_at[l] = (cur_stacks[1], handler_at[l][2]) end @@ -549,7 +637,7 @@ function (::ComputeTryCatch{Handler})(code::Vector{Any}, bbs::Union{Vector{Basic l += 1 end cur_hand = cur_stacks[1] - for i = 1:l + for _ = 1:l cur_hand = handler_at[get_enter_idx(handlers[cur_hand])][1] end cur_stacks = (cur_hand, cur_stacks[2]) @@ -579,18 +667,23 @@ should_insert_coverage(mod::Module, debuginfo::DebugInfo) = should_instrument(mo function should_instrument(mod::Module, debuginfo::DebugInfo, only_if_affects_optimizer::Bool=false) instrumentation_enabled(mod, only_if_affects_optimizer) && return true - JLOptions().code_coverage == 3 || JLOptions().malloc_log == 3 || return false - # path-specific coverage mode: if any line falls in a tracked file enable coverage for all + # an instrumented image covers every path (see instrumentation_enabled) + generating_output() && return false + JLOptions().malloc_log == 3 || return false + # Path-specific allocation tracking: instrument methods with a matching source file. return _should_instrument(debuginfo) end _should_instrument(loc::Symbol) = is_file_tracked(loc) _should_instrument(loc::Method) = _should_instrument(loc.file) _should_instrument(loc::MethodInstance) = _should_instrument(loc.def) -_should_instrument(loc::Module) = false +_should_instrument(::Module) = false +_should_instrument(::Nothing) = false function _should_instrument(info::DebugInfo) linetable = info.linetable - linetable === nothing || (_should_instrument(linetable) && return true) + # a byte-precise linetable is a compressed `String` that carries no file + # information of its own; only recurse into nested `DebugInfo` + linetable isa DebugInfo && _should_instrument(linetable) && return true _should_instrument(info.def) && return true return false end @@ -604,6 +697,8 @@ function InferenceState(result::InferenceResult, cache_mode::UInt8, interp::Abst maybe_validate_code(mi, src, "lowered") return InferenceState(result, src, cache_mode, interp) end +InferenceState(result::InferenceResult, src::CodeInfo, cache_mode::UInt8, interp::I) where {I<:AbstractInterpreter} = + InferenceState{I}(result, src, cache_mode, interp) InferenceState(result::InferenceResult, cache_mode::Symbol, interp::AbstractInterpreter) = InferenceState(result, convert_cache_mode(cache_mode), interp) InferenceState(result::InferenceResult, src::CodeInfo, cache_mode::Symbol, interp::AbstractInterpreter) = @@ -614,161 +709,453 @@ function convert_cache_mode(cache_mode::Symbol) return CACHE_MODE_GLOBAL elseif cache_mode === :local return CACHE_MODE_LOCAL - elseif cache_mode === :volatile - return CACHE_MODE_VOLATILE elseif cache_mode === :no return CACHE_MODE_NULL end error("unexpected `cache_mode` is given") end -""" - constrains_param(var::TypeVar, sig, covariant::Bool, type_constrains::Bool) - -Check if `var` will be constrained to have a definite value -in any concrete leaftype subtype of `sig`. - -It is used as a helper to determine whether type intersection is guaranteed to be able to -find a value for a particular type parameter. -A necessary condition for type intersection to not assign a parameter is that it only -appears in a `Union[All]` and during subtyping some other union component (that does not -constrain the type parameter) is selected. +# How the type filling a position is compared against the method parameter, +# which determines when an occurrence of a static parameter there pins it: +# * `MATCH_EGAL`: the position must be equal to the instantiated parameter +# (an invariant slot), so any successful match determines the variable. +# * `MATCH_INHABITED`: the position is filled by an inhabited type `X` +# (possibly abstract), which contributes only a lower bound: it pins the +# variable up to type equality, but only when the variable's declared +# lower bound is `Union{}`. +# * `MATCH_TYPEOF`: the position is filled by `typeof` of an argument — a +# concrete inhabited type. +const MATCH_EGAL = 0 +const MATCH_INHABITED = 1 +const MATCH_TYPEOF = 2 -The `type_constrains` flag determines whether Type{T} is considered to be constraining -`T`. This is not true in general, because of the existence of types with free type -parameters, however, some callers would like to ignore this corner case. """ -function constrains_param(var::TypeVar, @nospecialize(typ), covariant::Bool, type_constrains::Bool=false) - typ === var && return true - while typ isa UnionAll - covariant && constrains_param(var, typ.var.ub, covariant, type_constrains) && return true - # typ.var.lb doesn't constrain var - typ = typ.body + constrains_var(var::TypeVar, t, match::Int, nonempty_vararg::Bool=false) + +Check if `var` will be constrained to have a definite value when the position +`t` (a signature body under `var`'s binder) is filled according to `match` +(`MATCH_EGAL`, `MATCH_INHABITED`, or `MATCH_TYPEOF`); see the `MATCH_*` +constants for what each match kind means. + +Subtyping pins a variable through two channels (`eff_constrained` in +`subtype.c`): an invariant occurrence (`MATCH_EGAL`, matched by type equality) +and a covariant occurrence (`MATCH_TYPEOF`, filled with a `typeof`-produced +argument type). + +`MATCH_INHABITED` is a refinement based on filtering out non-inhabited types +(notably `Union{}`) when the field must be inhabited (because the constructor +inhabits it). + +`constrains_var` is a conservative static approximation of that dynamic rule. +In particular an invariant (`MATCH_EGAL`) `Union` arm is always treated as +absorbable (`Union{T,Int}` matched against `Int` does not pin `T`), even when +it is disjoint from the other arms so that every match either exposes it or +pins `var = Union{}` by absorbing it. Per issues #58427 and #59023: a parameter +that some match pins only as `Union{}` is still worth reporting since it may be +reachable with `invoke` and is consistent with the covariant occurrence rule. + +`nonempty_vararg` credits the signature's own trailing `Vararg` as if it +matched at least one argument (it is not propagated into nested positions, +where an empty tuple can always occur); it is an assumption the caller must +justify (see `Test.detect_unbound_args`). + +`inhabited_params` assumes that no invariant type parameter position of the +matching arguments is filled with `Union{}`, so that an occurrence of an inner +variable there always pins it to an inhabited type (see `pin_grade`), and the +descent through an inner variable's upper bound is also taken for +`MATCH_INHABITED` positions: a `Type{<:T}` slot then pins `T` even though the +call `f(Union{})` would not. +""" +function constrains_var(var::TypeVar, @nospecialize(t), match::Int, nonempty_vararg::Bool=false, + inhabited_params::Bool=false) + if t === var + # equality pins unconditionally; a lower-bound contribution pins only + # when nothing else can union into the least solution + return match === MATCH_EGAL || var.lb === Union{} end - if typ isa Union - # for unions, verify that both options would constrain var - ba = constrains_param(var, typ.a, covariant, type_constrains) - bb = constrains_param(var, typ.b, covariant, type_constrains) - (ba && bb) && return true - elseif typ isa DataType - # return true if any param constrains var - fc = length(typ.parameters) - if fc > 0 - if typ.name === Tuple.name - # vararg tuple needs special handling - for i in 1:(fc - 1) - p = typ.parameters[i] - constrains_param(var, p, covariant, type_constrains) && return true - end - lastp = typ.parameters[fc] - vararg = unwrap_unionall(lastp) - if vararg isa Core.TypeofVararg && isdefined(vararg, :N) - constrains_param(var, vararg.N, covariant, type_constrains) && return true - # T = vararg.parameters[1] doesn't constrain var - else - constrains_param(var, lastp, covariant, type_constrains) && return true + while t isa UnionAll + # only a concrete `typeof` argument (or any inhabited type, under + # `inhabited_params`) can carry the pin through an inner variable's + # range; the cheap occurs-check gates the `pin_grade` body traversal + # just for performance since constrains_var would be false + if (match === MATCH_TYPEOF || (match === MATCH_INHABITED && inhabited_params)) && + t.var.lb === Union{} && has_typevar(t.var.ub, var) + pin = pin_grade(t.var, t.body, nonempty_vararg, inhabited_params) + # an inhabited (possibly abstract) type filling this position can + # pin the inner variable at most as inhabited + match === MATCH_INHABITED && (pin = min(pin, PIN_INHABITED)) + if pin == PIN_TYPEOF && constrains_var(var, t.var.ub, MATCH_TYPEOF, false, inhabited_params) + return true + elseif pin == PIN_INHABITED && constrains_var(var, t.var.ub, MATCH_INHABITED, false, inhabited_params) + return true + end + end + if t.var === var + # `var` is rebound: occurrences below belong to the inner variable + return false + end + # other occurrences in the bounds of `t.var` do not reliably pin `var` + t = t.body + end + if t isa Union + # each arm is individually reachable, so every arm must pin `var`; an + # `MATCH_EGAL` arm the other arms can absorb pins nothing, but when + # absorption is impossible the assignments it admits pin `var`, possibly + # only as `var = Union{}` (issue #58427), which we deliberately still + # report — so in both cases require every arm to pin `var`, accepting + # the false positive for arms every match must expose (issue #59023) + return constrains_var(var, t.a, match, false, inhabited_params) && + constrains_var(var, t.b, match, false, inhabited_params) + end + if isTypeEq(t) # TypeEgal cannot constrain (or contain) a parameter + return constrains_var(var, type_parameter(t), MATCH_EGAL, false, inhabited_params) + end + t isa DataType || return false + if t.name === Tuple.name + for p in t.parameters + if p isa Core.TypeofVararg + # the argument count pins `N` exactly (including zero) + isdefined(p, :N) && constrains_var(var, p.N, match, false, inhabited_params) && return true + if match === MATCH_TYPEOF && nonempty_vararg && isdefined(p, :T) + constrains_var(var, p.T, MATCH_TYPEOF, false, inhabited_params) && return true end else - if typ.name === typename(Type) && typ.parameters[1] === var && var.ub === Any - # Types with free type parameters are <: Type cause the typevar - # to be unconstrained because Type{T} with free typevars is illegal - return type_constrains - end - for i in 1:fc - p = typ.parameters[i] - constrains_param(var, p, false, type_constrains) && return true + constrains_var(var, p, match, false, inhabited_params) && return true + end + end + else + for p in t.parameters + constrains_var(var, p, MATCH_EGAL, false, inhabited_params) && return true + end + end + return false +end + +# How strongly the position(s) of `v` in `t` pin its value, for every call: +# * `PIN_TYPEOF`: `v` takes a `typeof`-produced argument type — a concrete +# type, in particular not `Union{}` and without free type variables. This +# holds for a required covariant argument slot (directly, under nested +# `Tuple`s, or in all arms of a `Union`), or as the sole upper bound of +# another variable in such a slot. +# * `PIN_INHABITED`: `v` takes an *inhabited* type (not `Union{}`), though +# possibly an abstract one: `v` is an invariant parameter of a constructor +# in a required covariant slot, and that constructor declares a field of +# exactly that parameter's type which every inner constructor initializes, +# so a value of the parameter type must exist inside the argument. +# * `PIN_NONE`: neither is guaranteed; `v` may be `Union{}` (e.g. `Type{v}` +# positions matched by the argument `Union{}`, or invariant parameters +# without such a field), leaving its bounds unconstraining. +# With `inhabited_params`, any invariant parameter occurrence of `v` (at any +# depth, in any constructor) is assumed to be filled with an inhabited type, +# so it grades `PIN_INHABITED` without the field requirement. +const PIN_NONE = 0 +const PIN_INHABITED = 1 +const PIN_TYPEOF = 2 + +function pin_grade(v::TypeVar, @nospecialize(t), nonempty_vararg::Bool=false, inhabited_params::Bool=false) + # Implementation note: a covariant occurrence of `v` reports `PIN_TYPEOF` + # (concrete typeof value) computed in isolation. Strictly this over-reports + # when `v` also occurs elsewhere. We don't take the minimum (that would + # need a non-local scan of all of `v`'s occurrences), because the + # over-grade never changes a `constrains_var` result: `PIN_TYPEOF` vs + # `PIN_INHABITED` is consumed only by the triangular descent that pins an + # outer variable `W` appearing in `v`'s upper bound, and either (1) `v.ub` + # is a structured type mentioning `W` (`v<:Vector{W}`, `v<:Type{W}`) — the + # only place concreteness is load-bearing; or (2) `v.ub` is `W` or a + # `Union`, where concreteness is not needed (`v<:W` gives `W = v`, defined; + # `v<:Union{...W...}` is handled by the union-arm rule regardless of grade). + t === v && return PIN_TYPEOF + grade = PIN_NONE + while t isa UnionAll + if t.var.ub === v && t.var.lb === Union{} + # `v` is exactly the sole upper bound of `t.var`, so the least + # solution for `v` is `t.var`'s value, pinned at the same grade: a + # concrete `t.var` (`PIN_TYPEOF`) makes `v` concrete, an + # inhabited-but-abstract one (`PIN_INHABITED`) makes `v` inhabited + # (mirroring the two-grade handling in `constrains_var`) + g = pin_grade(t.var, t.body, nonempty_vararg, inhabited_params) + g == PIN_TYPEOF && return PIN_TYPEOF + grade = max(grade, g) + end + # `v` is rebound below: deeper occurrences belong to the inner variable, + # but the where-clauses peeled so far still count + t.var === v && return grade + t = t.body + end + if t isa Union + # a where-clause position (`grade`) pins on its own; the union itself + # pins only if every arm does + return max(grade, min(pin_grade(v, t.a, false, inhabited_params), + pin_grade(v, t.b, false, inhabited_params))) + end + if isTypeEq(t) + # a `Type{X}` position is filled by a type, `Union{}` included + inhabited_params || return grade + return max(grade, min(PIN_INHABITED, pin_grade(v, type_parameter(t), false, inhabited_params))) + end + t isa DataType || return grade + isabstracttype(t) && !inhabited_params && return grade + if t.name === Tuple.name + for p in t.parameters + if p isa Core.TypeofVararg + if nonempty_vararg && isdefined(p, :T) + grade = max(grade, pin_grade(v, p.T, false, inhabited_params)) end + else + grade = max(grade, pin_grade(v, p, false, inhabited_params)) + end + grade == PIN_TYPEOF && break + end + return grade + end + for i in 1:length(t.parameters) + p = t.parameters[i] + if p === v + if inhabited_params || some_field_requires_inhabited_param(t, i) + return max(grade, PIN_INHABITED) end + elseif inhabited_params + # a nested invariant position is filled by a type, so it pins at + # most as inhabited even where `v` occurs covariantly inside + g = min(PIN_INHABITED, pin_grade(v, p, false, inhabited_params)) + g == PIN_INHABITED && return max(grade, g) end end + return grade +end + +# Whether the `i`-th type parameter of `t`'s is constrained to be inhabited +# by being the declared type of one of its always-initialized fields, such +# that instances are guaranteed to contain a value of it. +function some_field_requires_inhabited_param(t::DataType, i::Int) + base = unwrap_unionall(t.name.wrapper)::DataType + tv = base.parameters[i] + tv isa TypeVar || return false + ftypes = datatype_fieldtypes(base) + for j in 1:datatype_min_ninitialized(base) + ftypes[j] === tv && return true + end return false end const EMPTY_SPTYPES = VarState[] +function type_sptype_to_egal(@nospecialize(ty)) + isType(ty) || return ty + p = type_parameter(ty) + p isa Type && !has_free_typevars(p) || return ty + return Core.TypeEgal{p} +end + +function type_value_to_egal(@nospecialize(v), @nospecialize(fallback)) + v isa Type || return fallback + has_free_typevars(v) && return fallback + return Core.TypeEgal{v} +end + +# Compute the abstract value `ty` for a sparam whose inferred env entry carries +# a TypeVar (either the sig's own `vᵢ` for unspecialized MIs, or a possibly +# narrowed `output_tvar` from subtyping). First try to sharpen via +# `arg::Type{vᵢ}` / `Vararg{_,vᵢ}` patterns in `sigtypes`; otherwise fall back +# to `Type{output_tvar}` rewrapped against free typevars of `specTypes`. +function sptype_for_tvar(vᵢ::TypeVar, output_tvar::TypeVar, sigtypes::Core.SimpleVector, + @nospecialize(specTypes), v_egal::Bool=false) + for j = 1:length(sigtypes) + sⱼ = sigtypes[j] + if isType(sⱼ) && type_parameter(sⱼ) === vᵢ + # `arg::Type{T}` pins the sparam to the arg's type + return fieldtype(specTypes, j) + elseif (va = va_from_vatuple(sⱼ)) !== nothing + # `::Tuple{.., Vararg{_,vᵢ}}` means `vᵢ` is the Int length + if isdefined(va, :N) && va.N === vᵢ + return Int + end + end + end + if Any === output_tvar.ub && Bottom === output_tvar.lb + # `Bottom <: T <: Any` additionally allows non-Type tvars + return Any + end + if output_tvar.lb === output_tvar.ub + lb = output_tvar.lb + # `TypeVar` bounds are declared `Union{TypeVar, Type}`, so a pinned + # plain value (which would be egal-keyed and `Const`-able) cannot occur. + @assert isa(lb, Union{Type, TypeVar}) + # A pinned TypeVar compares by identity (`==` is `===` for TypeVars), + # but denotes a stable object only when it is free in `specTypes` (part + # of the call's own types, which is what the runtime sparam read + # yields); one that would get rewrapped below stands for a different + # binding per instantiation and stays `==`-known. + if isa(lb, TypeVar) && is_free_typevar_in_spec(lb, specTypes) + return Const(lb) + end + # `X <: T <: X` pins the var to `X` up to type equality: an `S == X` rep + # argument also matches this MethodInstance and would bind the var to `S` + # (#61323) + ty = rewrap_free_typevars(TypeEq{lb}, find_free_typevars(specTypes)) + return v_egal ? type_value_to_egal(lb, ty) : ty + end + return rewrap_free_typevars(TypeEq{output_tvar}, find_free_typevars(specTypes)) +end + +function type_arg_parameter(@nospecialize(t)) + t = unwrap_unionall(t) + isType(t) || return nothing + return type_parameter(t) +end + +function has_invariant_sparam_occurrence(@nospecialize(t), v::TypeVar) + t = unwrap_unionall(t) + t isa DataType || return false + t.name === Tuple.name && return false + for p in t.parameters + has_typevar(p, v) && return true + end + return false +end + +# TODO: This analysis should move into subtyping, which already has this +# information when it computes the env: it knows for each binding whether the +# matched position determines the value by identity. Propagate that outwards +# (alongside the `svec(tvar, constrained)` markers) instead of re-deriving it +# syntactically here. +function sparam_definitely_egal_from_spec(v::TypeVar, sigtypes::Core.SimpleVector, + @nospecialize(specTypes)) + spec = unwrap_unionall(specTypes) + spec isa DataType || return false + for i = 1:min(length(sigtypes), length(spec.parameters)) + sigarg = sigtypes[i] + specarg = spec.parameters[i] + sigarg_unwrapped = unwrap_unionall(sigarg) + if sigarg_unwrapped === v + isdispatchelem(specarg) && return true + continue + end + if sigarg_unwrapped isa TypeVar && has_invariant_sparam_occurrence(sigarg_unwrapped.ub, v) + return true + end + if has_invariant_sparam_occurrence(sigarg, v) + return true + end + sig_type_arg = type_arg_parameter(sigarg) + sig_type_arg === nothing && continue + if sig_type_arg === v + specarg isa Core.TypeEgal && return true + continue + end + if sig_type_arg isa TypeVar && has_invariant_sparam_occurrence(sig_type_arg.ub, v) + return true + end + has_invariant_sparam_occurrence(sig_type_arg, v) && return true + end + return false +end + +function is_free_typevar_in_spec(v::TypeVar, @nospecialize(specTypes)) + for tv in find_free_typevars(specTypes) + tv === v && return true + end + return false +end + function sptypes_from_meth_instance(mi::MethodInstance) def = mi.def isa(def, Method) || return EMPTY_SPTYPES # toplevel sig = def.sig - if isempty(mi.sparam_vals) - isa(sig, UnionAll) || return EMPTY_SPTYPES - # mi is unspecialized - spvals = Any[] - sig′ = sig - while isa(sig′, UnionAll) - push!(spvals, sig′.var) - sig′ = sig′.body - end - else - spvals = mi.sparam_vals - end + # mi is unspecialized: no subtyping has run, so we don't have the + # `svec(tvar, constrained)` markers that specialized env entries carry. + isempty(mi.sparam_vals) && return sptypes_from_unspecialized(sig) + sigtypes = (unwrap_unionall(sig)::DataType).parameters + spvals = mi.sparam_vals nvals = length(spvals) sptypes = Vector{VarState}(undef, nvals) + temp = sig for i = 1:nvals v = spvals[i] - if v isa TypeVar - temp = sig - for j = 1:i-1 - temp = temp.body + undef = true + # An `svec(inner, constrained)` marker from subtyping/intersection + # means the sparam value is uncertain; `inner` is either the env + # TypeVar (carrying identity and bounds) or a DataType with free + # typevars (when the var got pinned to a value that still contains + # call-site tvars). `constrained` is true iff every concrete subtype + # of the call site will pin this var to a definite value. + v_tvar = nothing + v_constrained = false + if isa(v, SimpleVector) + v_inner = v[1] + v_constrained = v[2]::Bool + if isa(v_inner, TypeVar) + v_tvar = v_inner + else + # DataType-with-free-tvars case: route through the generic + # `has_free_typevars(v)` path by unwrapping. + v = v_inner end + end + if v isa TypeVar && is_free_typevar_in_spec(v, mi.specTypes) + # A plain TypeVar env entry that is free in `specTypes` is the exact + # runtime value of this sparam (bound from a typevar embedded in the + # call's argument types); TypeVars compare by identity, so `Const(v)` + # — not `Type{v}` — is its precise abstraction. + ty = Const(v) + undef = false + v_egal = false + elseif v_tvar !== nothing || has_free_typevars(v) vᵢ = (temp::UnionAll).var - sigtypes = (unwrap_unionall(temp)::DataType).parameters - for j = 1:length(sigtypes) - sⱼ = sigtypes[j] - if isType(sⱼ) && sⱼ.parameters[1] === vᵢ - # if this parameter came from `arg::Type{T}`, - # then `arg` is more precise than `Type{T} where lb<:T<:ub` - ty = fieldtype(mi.specTypes, j) - @goto ty_computed - elseif (va = va_from_vatuple(sⱼ)) !== nothing - # if this parameter came from `::Tuple{.., Vararg{T,vᵢ}}`, - # then `vᵢ` is known to be `Int` - if isdefined(va, :N) && va.N === vᵢ - ty = Int - @goto ty_computed - end - end - end - ub = unwraptv_ub(v) - if has_free_typevars(ub) - ub = Any - end - lb = unwraptv_lb(v) - if has_free_typevars(lb) - lb = Bottom - end - if Any === ub && lb === Bottom - ty = Any + if v_tvar !== nothing + v_egal = sparam_definitely_egal_from_spec(vᵢ, sigtypes, mi.specTypes) + ty = sptype_for_tvar(vᵢ, v_tvar, sigtypes, mi.specTypes, v_egal) else - tv = TypeVar(v.name, lb, ub) - ty = UnionAll(tv, Type{tv}) + ty = rewrap_free_typevars(TypeEq{v}, find_free_typevars(mi.specTypes)) + v_egal = false end - @label ty_computed - undef = !(let sig=sig - # if the specialized signature `linfo.specTypes` doesn't contain any free - # type variables, we can use it for a more accurate analysis of whether `v` - # is constrained or not, otherwise we should use `def.sig` which always - # doesn't contain any free type variables - if !has_free_typevars(mi.specTypes) - sig = mi.specTypes - end - @assert !has_free_typevars(sig) - constrains_param(v, sig, #=covariant=#true) - end) + undef = !v_constrained && !constrains_var(vᵢ, temp.body, MATCH_TYPEOF) elseif isvarargtype(v) - # if this parameter came from `func(..., ::Vararg{T,v})`, + # this parameter came from `func(..., ::Vararg{T,v})`, # so the type is known to be `Int` ty = Int undef = false + v_egal = false else ty = Const(v) undef = false + v_egal = false + end + if !undef && v_egal + ty = type_sptype_to_egal(ty) end - sptypes[i] = VarState(ty, undef) + sptypes[i] = VarState(ty, typemin(Int), undef) + temp = (temp::UnionAll).body end return sptypes end +# Separate path for unspecialized MIs (empty `sparam_vals`): no subtyping has +# run, so we can't use the svec-wrapped env entries produced by intersection. +function sptypes_from_unspecialized(@nospecialize sig) + isa(sig, UnionAll) || return EMPTY_SPTYPES + nvals = unionall_depth(sig) + sptypes = Vector{VarState}(undef, nvals) + params = (unwrap_unionall(sig)::DataType).parameters + temp = sig + for i = 1:nvals + vᵢ = (temp::UnionAll).var + ty = sptype_for_tvar(vᵢ, vᵢ, params, sig) + undef = !constrains_var(vᵢ, (temp::UnionAll).body, MATCH_TYPEOF) + sptypes[i] = VarState(ty, typemin(Int), undef) + temp = (temp::UnionAll).body + end + return sptypes +end + +function sp_at_idx(sig::UnionAll, idx::Int) + while idx != 1 + sig = sig.body::UnionAll + idx -= 1 + end + return sig.var +end + function va_from_vatuple(@nospecialize(t)) @_foldable_meta t = unwrap_unionall(t) @@ -814,60 +1201,19 @@ end # IRInterpretationState # ===================== - -# TODO add `result::InferenceResult` and put the irinterp result into the inference cache? -mutable struct IRInterpretationState - const spec_info::SpecInfo - const ir::IRCode - const mi::MethodInstance - world::WorldWithRange - curridx::Int - time_caches::Float64 - time_paused::UInt64 - const argtypes_refined::Vector{Bool} - const sptypes::Vector{VarState} - const tpdum::TwoPhaseDefUseMap - const ssa_refined::BitSet - const lazyreachability::LazyCFGReachability - const tasks::Vector{WorkThunk} - const edges::Vector{Any} - callstack #::Vector{AbsIntState} - frameid::Int - parentid::Int - - function IRInterpretationState(interp::AbstractInterpreter, - spec_info::SpecInfo, ir::IRCode, mi::MethodInstance, argtypes::Vector{Any}, - world::UInt, min_world::UInt, max_world::UInt) - curridx = 1 - given_argtypes = Vector{Any}(undef, length(argtypes)) - for i = 1:length(given_argtypes) - given_argtypes[i] = widenslotwrapper(argtypes[i]) - end - if isa(mi.def, Method) - argtypes_refined = Bool[!⊑(optimizer_lattice(interp), ir.argtypes[i], given_argtypes[i]) - for i = 1:length(given_argtypes)] - else - argtypes_refined = Bool[false for i = 1:length(given_argtypes)] - end - empty!(ir.argtypes) - append!(ir.argtypes, given_argtypes) - tpdum = TwoPhaseDefUseMap(length(ir.stmts)) - ssa_refined = BitSet() - lazyreachability = LazyCFGReachability(ir) - valid_worlds = WorldRange(min_world, max_world == typemax(UInt) ? get_world_counter() : max_world) - tasks = WorkThunk[] - edges = Any[] - callstack = AbsIntState[] - return new(spec_info, ir, mi, WorldWithRange(world, valid_worlds), - curridx, 0.0, 0, argtypes_refined, ir.sptypes, tpdum, - ssa_refined, lazyreachability, tasks, edges, callstack, 0, 0) - end -end - -function IRInterpretationState(interp::AbstractInterpreter, - codeinst::CodeInstance, mi::MethodInstance, argtypes::Vector{Any}, world::UInt) - @assert codeinst.def === mi "method instance is not synced with code instance" - src = @atomic :monotonic codeinst.inferred +# (the struct itself is defined in the `typegroup` block alongside `InferenceState`) + +IRInterpretationState(interp::I, spec_info::SpecInfo, ir::IRCode, + mi::MethodInstance, argtypes::Vector{Any}, + min_world::UInt, max_world::UInt) where {I<:AbstractInterpreter} = + IRInterpretationState{I}(interp, spec_info, ir, mi, argtypes, min_world, max_world) + +function IRInterpretationState( + interp::AbstractInterpreter, codeinst::CodeInstance, mi::MethodInstance, + argtypes::Vector{Any}, @nospecialize(src), + valid_worlds::WorldRange = WorldRange(codeinst.min_world, codeinst.max_world) + ) + @assert get_ci_mi(codeinst) === mi "method instance is not synced with code instance" if isa(src, String) src = _uncompressed_ir(codeinst, src) else @@ -875,19 +1221,19 @@ function IRInterpretationState(interp::AbstractInterpreter, end spec_info = SpecInfo(src) ir = inflate_ir(src, mi) - argtypes = va_process_argtypes(optimizer_lattice(interp), argtypes, src.nargs, src.isva) - return IRInterpretationState(interp, spec_info, ir, mi, argtypes, world, - codeinst.min_world, codeinst.max_world) + argtypes = va_process_argtypes(optimizer_lattice(interp), argtypes, src.nargs, src.isva, mi) + return IRInterpretationState(interp, spec_info, ir, mi, argtypes, + first(valid_worlds), last(valid_worlds)) end # AbsIntState # =========== -const AbsIntState = Union{InferenceState,IRInterpretationState} +const AbsIntState{I<:AbstractInterpreter} = Union{InferenceState{I}, IRInterpretationState{I}} function print_callstack(frame::AbsIntState) print("=================== Callstack: ==================\n") - frames = frame.callstack::Vector{AbsIntState} + frames = frame.callstack for idx = (frame.frameid == 0 ? 0 : 1):length(frames) sv = (idx == 0 ? frame : frames[idx]) idx == frame.frameid && print("*") @@ -899,7 +1245,7 @@ function print_callstack(frame::AbsIntState) end print("] ") print(frame_instance(sv)) - is_cached(sv) || print(" [uncached]") + is_cached(sv) || print(" [not globally cached]") sv.parentid == idx - 1 || print(" [parent=", sv.parentid, "]") isempty(callers_in_cycle(sv)) || print(" [cycle=", sv.cycleid, "]") println() @@ -918,11 +1264,11 @@ function frame_module(sv::AbsIntState) return def.module end -frame_parent(sv::AbsIntState) = sv.parentid == 0 ? nothing : (sv.callstack::Vector{AbsIntState})[sv.parentid] +frame_parent(sv::AbsIntState) = sv.parentid == 0 ? nothing : sv.callstack[sv.parentid] function cycle_parent(sv::InferenceState) sv.parentid == 0 && return nothing - callstack = sv.callstack::Vector{AbsIntState} + callstack = sv.callstack sv = callstack[sv.cycleid]::InferenceState sv.parentid == 0 && return nothing return callstack[sv.parentid] @@ -931,17 +1277,17 @@ cycle_parent(sv::IRInterpretationState) = frame_parent(sv) # add the orphan child to the parent and the parent to the child -function assign_parentchild!(child::InferenceState, parent::AbsIntState) +function assign_parentchild!(child::InferenceState{I}, parent::AbsIntState{I}) where {I<:AbstractInterpreter} @assert child.frameid in (0, 1) - child.callstack = callstack = parent.callstack::Vector{AbsIntState} + child.callstack = callstack = parent.callstack child.parentid = parent.frameid push!(callstack, child) child.cycleid = child.frameid = length(callstack) nothing end -function assign_parentchild!(child::IRInterpretationState, parent::AbsIntState) +function assign_parentchild!(child::IRInterpretationState{I}, parent::AbsIntState{I}) where {I<:AbstractInterpreter} @assert child.frameid in (0, 1) - child.callstack = callstack = parent.callstack::Vector{AbsIntState} + child.callstack = callstack = parent.callstack child.parentid = parent.frameid push!(callstack, child) child.frameid = length(callstack) @@ -963,9 +1309,6 @@ spec_info(sv::IRInterpretationState) = sv.spec_info propagate_inbounds(sv::AbsIntState) = spec_info(sv).propagate_inbounds method_for_inference_limit_heuristics(sv::AbsIntState) = spec_info(sv).method_for_inference_limit_heuristics -frame_world(sv::InferenceState) = sv.world.this -frame_world(sv::IRInterpretationState) = sv.world.this - function is_effect_overridden(sv::AbsIntState, effect::Symbol) if is_effect_overridden(frame_instance(sv), effect) return true @@ -983,11 +1326,17 @@ is_effect_overridden(override::EffectsOverride, effect::Symbol) = getfield(overr has_conditional(𝕃::AbstractLattice, ::InferenceState) = has_conditional(𝕃) has_conditional(::AbstractLattice, ::IRInterpretationState) = false +has_mustalias(𝕃::AbstractLattice, ::InferenceState) = has_mustalias(𝕃) +has_mustalias(::AbstractLattice, ::IRInterpretationState) = false # work towards converging the valid age range for sv -function update_valid_age!(sv::AbsIntState, valid_worlds::WorldRange) - sv.world = intersect(sv.world, valid_worlds) - return sv.world.valid_worlds +function update_valid_age!(sv::AbsIntState, world, valid_worlds::WorldRange) + valid_worlds = intersect(sv.valid_worlds, valid_worlds) + if !(world in valid_worlds) + error("invalid age range update") + end + sv.valid_worlds = valid_worlds + return valid_worlds end """ @@ -998,17 +1347,17 @@ Iterate through all callers of the given `AbsIntState` in the abstract interpret ascending the tree from the given `AbsIntState`). Note that cycles may be visited in any order. """ -struct AbsIntStackUnwind - callstack::Vector{AbsIntState} - AbsIntStackUnwind(sv::AbsIntState) = new(sv.callstack::Vector{AbsIntState}) +struct AbsIntStackUnwind{I<:AbstractInterpreter} + callstack::Vector{AbsIntState{I}} + AbsIntStackUnwind(sv::AbsIntState{I}) where {I<:AbstractInterpreter} = new{I}(sv.callstack) end function iterate(unw::AbsIntStackUnwind, frame::Int=length(unw.callstack)) frame == 0 && return nothing return (unw.callstack[frame], frame - 1) end -struct AbsIntCycle - frames::Vector{AbsIntState} +struct AbsIntCycle{I<:AbstractInterpreter} + frames::Vector{AbsIntState{I}} cycleid::Int cycletop::Int end @@ -1026,7 +1375,7 @@ interpretation stack (including the given `AbsIntState` itself) that are part of the same cycle, only if it is part of a cycle with multiple frames. """ function callers_in_cycle(sv::InferenceState) - callstack = sv.callstack::Vector{AbsIntState} + callstack = sv.callstack cycletop = cycleid = sv.cycleid while cycletop < length(callstack) frame = callstack[cycletop + 1] @@ -1036,7 +1385,7 @@ function callers_in_cycle(sv::InferenceState) end return AbsIntCycle(callstack, cycletop == cycleid ? 0 : cycleid, cycletop) end -callers_in_cycle(sv::IRInterpretationState) = AbsIntCycle(sv.callstack::Vector{AbsIntState}, 0, 0) +callers_in_cycle(sv::IRInterpretationState) = AbsIntCycle(sv.callstack, 0, 0) get_curr_ssaflag(sv::InferenceState) = sv.ssaflags[sv.currpc] get_curr_ssaflag(sv::IRInterpretationState) = sv.ir.stmts[sv.curridx][:flag] @@ -1067,7 +1416,7 @@ end merge_effects!(::AbstractInterpreter, ::IRInterpretationState, ::Effects) = return decode_statement_effects_override(sv::InferenceState) = decode_statement_effects_override(sv.src.ssaflags[sv.currpc]) -decode_statement_effects_override(sv::IRInterpretationState) = decode_statement_effects_override(UInt32(0)) +decode_statement_effects_override(::IRInterpretationState) = decode_statement_effects_override(UInt32(0)) struct InferenceLoopState rt @@ -1090,8 +1439,8 @@ bail_out_apply(::AbstractInterpreter, state::InferenceLoopState, ::InferenceStat bail_out_apply(::AbstractInterpreter, state::InferenceLoopState, ::IRInterpretationState) = state.rt === Any -add_remark!(::AbstractInterpreter, ::InferenceState, remark) = return -add_remark!(::AbstractInterpreter, ::IRInterpretationState, remark) = return +add_remark!(::AbstractInterpreter, ::InferenceState, _remark) = return +add_remark!(::AbstractInterpreter, ::IRInterpretationState, _remark) = return function get_max_methods(interp::AbstractInterpreter, @nospecialize(f), sv::AbsIntState) fmax = get_max_methods_for_func(f) @@ -1117,6 +1466,15 @@ function get_max_methods_for_func(@nospecialize(f)) end return nothing end + +# Whether `f` is marked to only allow inference of call sites with fully concrete +# argument types. `f === nothing` means the callee value is unknown. +function is_concrete_only(@nospecialize(f)) + f === nothing && return false + isa(f, DataType) && return f.name.concrete_only + return typeof(f).name.concrete_only +end + get_max_methods_for_module(sv::AbsIntState) = get_max_methods_for_module(frame_module(sv)) function get_max_methods_for_module(mod::Module) max_methods = ccall(:jl_get_module_max_methods, Cint, (Any,), mod) % Int @@ -1190,7 +1548,7 @@ end """ doworkloop(args...) -Run a tasks inside the abstract interpreter, returning false if there are none. +Run a task inside the abstract interpreter, returning false if there are none. Tasks will be run in DFS post-order tree order, such that all child tasks will be run in the order scheduled, prior to running any subsequent tasks. This allows tasks to generate more child tasks, which will be run before anything else. diff --git a/Compiler/src/methodtable.jl b/Compiler/src/methodtable.jl index 24a8b1ecf8242..2ca7c32137dab 100644 --- a/Compiler/src/methodtable.jl +++ b/Compiler/src/methodtable.jl @@ -1,8 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license struct MethodLookupResult - # Really Vector{Core.MethodMatch}, but it's easier to represent this as - # and work with Vector{Any} on the C side. + # Really Vector{Core.MethodMatch}, but it's easier to represent this as, + # and work with, Vector{Any} on the C side. matches::Vector{Any} valid_worlds::WorldRange ambig::Bool @@ -32,7 +32,7 @@ end struct OverlayMethodTable <: MethodTableView Overlays the internal method table such that specific queries can be redirected to an -external table, e.g., to override existing method. +external table, e.g., to override existing methods. """ struct OverlayMethodTable <: MethodTableView world::UInt @@ -42,7 +42,9 @@ end struct MethodMatchKey sig # ::Type limit::Int - MethodMatchKey(@nospecialize(sig), limit::Int) = new(sig, limit) + include_ambiguous::Bool + MethodMatchKey(@nospecialize(sig), limit::Int, include_ambiguous::Bool) = + new(sig, limit, include_ambiguous) end """ @@ -58,20 +60,23 @@ end CachedMethodTable(table::T) where T = CachedMethodTable{T}(IdDict{MethodMatchKey, Union{Nothing,MethodLookupResult}}(), table) """ - findall(sig::Type, view::MethodTableView; limit::Int=-1) -> + findall(sig::Type, view::MethodTableView; + limit::Int=-1, include_ambiguous::Bool=false) -> matches::MethodLookupResult or nothing -Find all methods in the given method table `view` that are applicable to the given signature `sig`. -If no applicable methods are found, an empty result is returned. -If the number of applicable methods exceeded the specified `limit`, `nothing` is returned. -Note that the default setting `limit=-1` does not limit the number of applicable methods. -`overlayed` indicates if any of the matching methods comes from an overlayed method table. +Find all methods in `view` that are applicable to `sig`. If no applicable methods +are found, an empty result is returned. If the number of applicable methods exceeds +`limit`, `nothing` is returned. The default `limit=-1` does not limit the number of +applicable methods. `include_ambiguous=true` retains fully ambiguous matches that are +normally filtered out of the result. """ -findall(@nospecialize(sig::Type), table::InternalMethodTable; limit::Int=-1) = - _findall(sig, nothing, table.world, limit) +findall(@nospecialize(sig::Type), table::InternalMethodTable; + limit::Int=-1, include_ambiguous::Bool=false) = + _findall(sig, nothing, table.world, limit, include_ambiguous) -function findall(@nospecialize(sig::Type), table::OverlayMethodTable; limit::Int=-1) - result = _findall(sig, table.mt, table.world, limit) +function findall(@nospecialize(sig::Type), table::OverlayMethodTable; + limit::Int=-1, include_ambiguous::Bool=false) + result = _findall(sig, table.mt, table.world, limit, include_ambiguous) result === nothing && return nothing nr = length(result) if nr ≥ 1 && result[nr].fully_covers @@ -79,36 +84,46 @@ function findall(@nospecialize(sig::Type), table::OverlayMethodTable; limit::Int return result end # fall back to the internal method table - fallback_result = _findall(sig, nothing, table.world, limit) + fallback_result = _findall(sig, nothing, table.world, limit, include_ambiguous) fallback_result === nothing && return nothing - # merge the fallback match results with the internal method table + # merge the fallback match results with the internal method table, + # filtering out base methods that are fully covered by overlay methods + overlay_matches = result.matches + filtered = filter(fallback_result.matches) do base_match::MethodMatch + dominated = any(overlay_matches) do overlay_match::MethodMatch + base_match.method.sig <: overlay_match.method.sig + end + return !dominated + end return MethodLookupResult( - vcat(result.matches, fallback_result.matches), + vcat(overlay_matches, filtered), WorldRange( max(result.valid_worlds.min_world, fallback_result.valid_worlds.min_world), min(result.valid_worlds.max_world, fallback_result.valid_worlds.max_world)), result.ambig | fallback_result.ambig) end -function _findall(@nospecialize(sig::Type), mt::Union{Nothing,MethodTable}, world::UInt, limit::Int) +function _findall(@nospecialize(sig::Type), mt::Union{Nothing,MethodTable}, world::UInt, + limit::Int, include_ambiguous::Bool) _min_val = RefValue{UInt}(typemin(UInt)) _max_val = RefValue{UInt}(typemax(UInt)) _ambig = RefValue{Int32}(0) - ms = _methods_by_ftype(sig, mt, limit, world, false, _min_val, _max_val, _ambig) + ms = _methods_by_ftype(sig, mt, limit, world, include_ambiguous, _min_val, _max_val, _ambig) isa(ms, Vector) || return nothing return MethodLookupResult(ms, WorldRange(_min_val[], _max_val[]), _ambig[] != 0) end -function findall(@nospecialize(sig::Type), table::CachedMethodTable; limit::Int=-1) +function findall(@nospecialize(sig::Type), table::CachedMethodTable; + limit::Int=-1, include_ambiguous::Bool=false) if isconcretetype(sig) # as for concrete types, we cache result at on the next level - return findall(sig, table.table; limit) + return findall(sig, table.table; limit, include_ambiguous) end - key = MethodMatchKey(sig, limit) + key = MethodMatchKey(sig, limit, include_ambiguous) if haskey(table.cache, key) return table.cache[key] else - return table.cache[key] = findall(sig, table.table; limit) + return table.cache[key] = findall(sig, table.table; limit, include_ambiguous) end end diff --git a/Compiler/src/opaque_closure.jl b/Compiler/src/opaque_closure.jl index d0a375c2a54b5..21f2640037893 100644 --- a/Compiler/src/opaque_closure.jl +++ b/Compiler/src/opaque_closure.jl @@ -48,6 +48,8 @@ function Core.OpaqueClosure(ir::IRCode, @nospecialize env...; end src.slotflags = fill(zero(UInt8), nargtypes) src.slottypes = copy(ir.argtypes) + src.min_world = ir.valid_worlds.min_world + src.max_world = ir.valid_worlds.max_world src.isva = isva src.nargs = UInt(nargtypes) src = ir_to_codeinf!(src, ir) diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 27fb1e2168639..de2b3e2380f38 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -39,6 +39,9 @@ const IR_FLAG_NOUB = one(UInt32) << 10 #const IR_FLAG_CONSISTENTOVERLAY = one(UInt32) << 12 # This statement is :nortcall const IR_FLAG_NORTCALL = one(UInt32) << 13 +# This statement is proven :reset_safe +const IR_FLAG_RESET_SAFE = one(UInt32) << 14 +# Reserved: one(UInt32) << 15 used for RSIIMO below # An optimization pass has updated this statement in a way that may # have exposed information that inference did not see. Re-running # inference on this statement may be profitable. @@ -50,15 +53,21 @@ const IR_FLAG_UNUSED = one(UInt32) << 17 const IR_FLAG_EFIIMO = one(UInt32) << 18 # This statement is :inaccessiblememonly == INACCESSIBLEMEM_OR_ARGMEMONLY const IR_FLAG_INACCESSIBLEMEM_OR_ARGMEM = one(UInt32) << 19 +# This statement is :reset_safe == RESET_SAFE_IF_INACCESSIBLEMEMONLY +const IR_FLAG_RSIIMO = one(UInt32) << 20 const NUM_IR_FLAGS = 3 # sync with julia.h const IR_FLAGS_EFFECTS = IR_FLAG_CONSISTENT | IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW | - IR_FLAG_TERMINATES | IR_FLAG_NOUB | IR_FLAG_NORTCALL + IR_FLAG_TERMINATES | IR_FLAG_NOUB | IR_FLAG_NORTCALL | IR_FLAG_RESET_SAFE const IR_FLAGS_REMOVABLE = IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW | IR_FLAG_TERMINATES +# N.B.: RSIIMO is deliberately not included: `has_flag` requires all bits, and +# nothing consumes an escape-analysis outcome for reset-safety yet. When such +# a consumer exists, it needs its own (RSIIMO | INACCESSIBLEMEM_OR_ARGMEM) +# qualification, not membership in this conjunction. const IR_FLAGS_NEEDS_EA = IR_FLAG_EFIIMO | IR_FLAG_INACCESSIBLEMEM_OR_ARGMEM has_flag(curr::UInt32, flag::UInt32) = (curr & flag) == flag @@ -66,7 +75,7 @@ has_flag(curr::UInt32, flag::UInt32) = (curr & flag) == flag function iscallstmt(@nospecialize stmt) stmt isa Expr || return false head = stmt.head - return head === :call || head === :invoke || head === :foreigncall + return head === :call || head === :invoke || head === :foreigncall || head === :foreignglobal end function flags_for_effects(effects::Effects) @@ -79,6 +88,20 @@ function flags_for_effects(effects::Effects) elseif is_effect_free_if_inaccessiblememonly(effects) flags |= IR_FLAG_EFIIMO end + # N.B.: Inference does not yet model `reset_safe` separately from + # `effect_free`, so until it does, only statements that are additionally + # proven effect-free may be treated as reset-safe (execution can safely + # be reset across them, since they have no externally visible effects). + # This flag describes the statement's IPO contract only: machinery the + # runtime inserts implicitly to execute it (allocation, write barriers, + # runtime library calls), whose frames must never be abandoned + # asynchronously, is handled separately by eagerly dropping the published + # reset context around it (see llvm-cancellation-lowering.cpp). + if is_reset_safe(effects) && is_effect_free(effects) + flags |= IR_FLAG_RESET_SAFE + elseif is_reset_safe_if_inaccessiblememonly(effects) && is_effect_free_if_inaccessiblememonly(effects) + flags |= IR_FLAG_RSIIMO + end if is_nothrow(effects) flags |= IR_FLAG_NOTHROW end @@ -99,14 +122,9 @@ end const TOP_TUPLE = GlobalRef(Core, :tuple) -# This corresponds to the type of `CodeInfo`'s `inlining_cost` field -const InlineCostType = UInt16 -const MAX_INLINE_COST = typemax(InlineCostType) -const MIN_INLINE_COST = InlineCostType(10) -const MaybeCompressed = Union{CodeInfo, String} - -is_inlineable(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_inlining_cost, InlineCostType, (Any,), src) != MAX_INLINE_COST +inlining_cost(@nospecialize src) = + src isa Union{MaybeCompressed,UInt8} ? ccall(:jl_ir_inlining_cost, InlineCostType, (Any,), src) : MAX_INLINE_COST +is_inlineable(@nospecialize src) = inlining_cost(src) != MAX_INLINE_COST set_inlineable!(src::CodeInfo, val::Bool) = src.inlining_cost = (val ? MIN_INLINE_COST : MAX_INLINE_COST) @@ -132,7 +150,18 @@ is_declared_noinline(@nospecialize src::MaybeCompressed) = ##################### # return whether this src should be inlined. If so, retrieve_ir_for_inlining must return an IRCode from it -function src_inlining_policy(interp::AbstractInterpreter, + +function src_inlining_policy(interp::AbstractInterpreter, mi::MethodInstance, + @nospecialize(src), @nospecialize(info::CallInfo), stmt_flag::UInt32) + # If we have a generator, but we can't invoke it (because argument type information is lacking), + # don't inline so we defer its invocation to runtime where we'll have precise type information. + if isa(mi.def, Method) && hasgenerator(mi) + may_invoke_generator(mi) || return false + end + return src_inlining_policy(interp, src, info, stmt_flag) +end + +function src_inlining_policy(::AbstractInterpreter, @nospecialize(src), @nospecialize(info::CallInfo), stmt_flag::UInt32) isa(src, OptimizationState) && (src = src.src) if isa(src, MaybeCompressed) @@ -147,47 +176,38 @@ end struct InliningState{Interp<:AbstractInterpreter} edges::Vector{Any} - world::UInt interp::Interp opt_cache::IdDict{MethodInstance,CodeInstance} end function InliningState(sv::InferenceState, interp::AbstractInterpreter, opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) - return InliningState(sv.edges, frame_world(sv), interp, opt_cache) + return InliningState(sv.edges, interp, opt_cache) end function InliningState(interp::AbstractInterpreter, opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) - return InliningState(Any[], get_inference_world(interp), interp, opt_cache) + return InliningState(Any[], interp, opt_cache) end struct OptimizerCache{CodeCache} - wvc::WorldView{CodeCache} - owner + cache::CodeCache opt_cache::IdDict{MethodInstance,CodeInstance} function OptimizerCache( - wvc::WorldView{CodeCache}, - @nospecialize(owner), + cache::CodeCache, opt_cache::IdDict{MethodInstance,CodeInstance}) where CodeCache - new{CodeCache}(wvc, owner, opt_cache) + return new{CodeCache}(cache, opt_cache) end end -function get((; wvc, owner, opt_cache)::OptimizerCache, mi::MethodInstance, default) +function get((; cache, opt_cache)::OptimizerCache, mi::MethodInstance, default) if haskey(opt_cache, mi) - codeinst = opt_cache[mi] - @assert codeinst.min_world ≤ wvc.worlds.min_world && - wvc.worlds.max_world ≤ codeinst.max_world && - codeinst.owner === owner - @assert isdefined(codeinst, :inferred) && codeinst.inferred === nothing - return codeinst + return opt_cache[mi] # this is incomplete right now, but will be finished (by finish_cycle) before caching anything end - return get(wvc, mi, default) + return get(cache, mi, default) end # get `code_cache(::AbstractInterpreter)` from `state::InliningState` function code_cache(state::InliningState) - cache = WorldView(code_cache(state.interp), state.world) - owner = cache_owner(state.interp) - return OptimizerCache(cache, owner, state.opt_cache) + cache = code_cache(state.interp) + return OptimizerCache(cache, state.opt_cache) end mutable struct OptimizationResult @@ -212,7 +232,7 @@ mutable struct OptimizationState{Interp<:AbstractInterpreter} inlining::InliningState{Interp} cfg::CFG unreachable::BitSet - bb_vartables::Vector{Union{Nothing,VarTable}} + bb_states::Vector{Union{Nothing,BBEntryState}} insert_coverage::Bool end function OptimizationState(sv::InferenceState, interp::AbstractInterpreter, @@ -220,14 +240,14 @@ function OptimizationState(sv::InferenceState, interp::AbstractInterpreter, inlining = InliningState(sv, interp, opt_cache) return OptimizationState(sv.linfo, sv.src, nothing, sv.stmt_info, sv.mod, sv.sptypes, sv.slottypes, inlining, sv.cfg, - sv.unreachable, sv.bb_vartables, sv.insert_coverage) + sv.unreachable, sv.bb_states, sv.insert_coverage) end function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractInterpreter, opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) # prepare src for running optimization passes if it isn't already nssavalues = src.ssavaluetypes if nssavalues isa Int - src.ssavaluetypes = Any[ Any for i = 1:nssavalues ] + src.ssavaluetypes = Any[ Any for _ = 1:nssavalues ] else nssavalues = length(src.ssavaluetypes::Vector{Any}) end @@ -235,9 +255,9 @@ function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractIn nslots = length(src.slotflags) slottypes = src.slottypes if slottypes === nothing - slottypes = Any[ Any for i = 1:nslots ] + slottypes = Any[ Any for _ = 1:nslots ] end - stmt_info = CallInfo[ NoCallInfo() for i = 1:nssavalues ] + stmt_info = CallInfo[ NoCallInfo() for _ = 1:nssavalues ] # cache some useful state computations def = mi.def mod = isa(def, Method) ? def.module : def @@ -246,14 +266,14 @@ function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractIn inlining = InliningState(interp, opt_cache) cfg = compute_basic_blocks(src.code) unreachable = BitSet() - bb_vartables = Union{VarTable,Nothing}[] - for block = 1:length(cfg.blocks) - push!(bb_vartables, VarState[ - VarState(slottypes[slot], src.slotflags[slot] & SLOT_USEDUNDEF != 0) + nbbstate = zeros(Int, nslots) + bb_states = Union{BBEntryState,Nothing}[ + BBEntryState(VarState[ + VarState(slottypes[slot], typemin(Int), src.slotflags[slot] & SLOT_USEDUNDEF != 0) for slot = 1:nslots - ]) - end - return OptimizationState(mi, src, nothing, stmt_info, mod, sptypes, slottypes, inlining, cfg, unreachable, bb_vartables, false) + ], nbbstate) + for _ = 1:length(cfg.blocks)] + return OptimizationState(mi, src, nothing, stmt_info, mod, sptypes, slottypes, inlining, cfg, unreachable, bb_states, false) end function OptimizationState(mi::MethodInstance, interp::AbstractInterpreter) world = get_inference_world(interp) @@ -263,6 +283,7 @@ function OptimizationState(mi::MethodInstance, interp::AbstractInterpreter) end function argextype end # imported by EscapeAnalysis +function argextype_widened end # imported by EscapeAnalysis function try_compute_field end # imported by EscapeAnalysis include("ssair/heap.jl") @@ -274,8 +295,8 @@ include("ssair/EscapeAnalysis.jl") include("ssair/passes.jl") include("ssair/irinterp.jl") -function ir_to_codeinf!(opt::OptimizationState, frame::InferenceState, edges::SimpleVector) - ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp, frame.result, opt.optresult)) +function ir_to_codeinf!(opt::OptimizationState{I}, frame::InferenceState{I}, edges::SimpleVector) where {I<:AbstractInterpreter} + ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp::I, frame.result, opt.optresult)) end function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector, inlining_cost::InlineCostType) @@ -341,8 +362,8 @@ function new_expr_effect_flags(𝕃ₒ::AbstractLattice, args::Vector{Any}, src: typ, isexact = instanceof_tfunc(atyp, true) if !isexact atyp = unwrap_unionall(widenconst(atyp)) - if isType(atyp) && isTypeDataType(atyp.parameters[1]) - typ = atyp.parameters[1] + if isType(atyp) && isTypeDataType(type_parameter(atyp)) + typ = type_parameter(atyp) else return (false, false, false) end @@ -383,6 +404,10 @@ function stmt_effect_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), @nospe # GlobalRef was moved to statement position, it is probably not `const`, # so we can't say much about it anyway. return (false, false, false) + elseif isa(stmt, Core.BindingPartition) + # A resolved global read: its effects come from the partition kind. + (; effects) = abstract_eval_partition_load(partition_owner(stmt), stmt, false) + return (is_consistent(effects), is_removable_if_unused(effects), is_nothrow(effects)) elseif isa(stmt, Expr) (; head, args) = stmt if head === :static_parameter @@ -395,7 +420,7 @@ function stmt_effect_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), @nospe f = argextype(args[1], src) f = singleton_type(f) f === nothing && return (false, false, false) - if f === Intrinsics.cglobal || f === Intrinsics.llvmcall + if f === Intrinsics.llvmcall # TODO: these are not yet linearized return (false, false, false) end @@ -422,6 +447,8 @@ function stmt_effect_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), @nospe terminates = is_terminates(effects) removable = effect_free & nothrow & terminates return (consistent, removable, nothrow) + elseif head === :foreignglobal + return (false, false, false) elseif head === :new_opaque_closure length(args) < 4 && return (false, false, false) typ = argextype(args[1], src) @@ -462,7 +489,7 @@ function recompute_effects_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), end if !iscallstmt(stmt) # There is a bit of a subtle point here, which is that some non-call - # statements (e.g. PiNode) can be UB:, however, we consider it + # statements (e.g. PiNode) can be UB, however, we consider it # illegal to introduce such statements that actually cause UB (for any # input). Ideally that'd be handled at insertion time (TODO), but for # the time being just do that here. @@ -518,7 +545,9 @@ function argextype( elseif isa(x, QuoteNode) return Const(x.value) elseif isa(x, GlobalRef) - return abstract_eval_globalref_type(x, src) + return globalref_rt(x, src) + elseif isa(x, Core.BindingPartition) + return partition_rt(x) elseif isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) return Any elseif isa(x, PiNode) @@ -527,6 +556,17 @@ function argextype( return Const(x) end end + +# `widenconst(argextype(x, src, ...))` without the throwaway `Const` for GlobalRef args. +@inline function argextype_widened(@nospecialize(x), + src::Union{IRCode,IncrementalCompact,CodeInfo}, sptypes::Vector{VarState}) + isa(x, GlobalRef) && return globalref_rt_widened(x, src) + isa(x, Core.BindingPartition) && return partition_rt_widened(x) + return widenconst(argextype(x, src, sptypes)) +end +@inline argextype_widened(@nospecialize(x), ir::IRCode) = argextype_widened(x, ir, ir.sptypes) +@inline argextype_widened(@nospecialize(x), compact::IncrementalCompact) = + argextype_widened(x, compact, compact.ir.sptypes) function abstract_eval_ssavalue(s::SSAValue, src::CodeInfo) ssavaluetypes = src.ssavaluetypes if ssavaluetypes isa Int @@ -543,7 +583,7 @@ abstract_eval_ssavalue(s::SSAValue, src::Union{IRCode,IncrementalCompact}) = typ Called at the end of optimization to store the resulting IR back into the OptimizationState. """ -function finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) +function finishopt!(::AbstractInterpreter, opt::OptimizationState, ir::IRCode) opt.optresult = OptimizationResult(ir, ccall(:jl_ir_flag_inlining, UInt8, (Any,), opt.src), false) return nothing end @@ -667,7 +707,7 @@ GetNativeEscapeCache(interp::AbstractInterpreter) = GetNativeEscapeCache(code_ca function ((; code_cache)::GetNativeEscapeCache)(codeinst::Union{CodeInstance,MethodInstance}) if codeinst isa MethodInstance codeinst = get(code_cache, codeinst, nothing) - codeinst isa CodeInstance || return false + codeinst === nothing && return false end argescapes = traverse_analysis_results(codeinst) do @nospecialize result return result isa EscapeAnalysis.ArgEscapeCache ? result : nothing @@ -675,7 +715,7 @@ function ((; code_cache)::GetNativeEscapeCache)(codeinst::Union{CodeInstance,Met if argescapes !== nothing return argescapes end - effects = decode_effects(codeinst.ipo_purity_bits) + effects = codeinst isa CodeInstance ? decode_effects(codeinst.ipo_purity_bits) : codeinst.ipo_effects if is_effect_free(effects) && is_inaccessiblememonly(effects) # We might not have run EA on simple frames without any escapes (e.g. when optimization # is skipped when result is constant-folded by abstract interpretation). If those @@ -726,6 +766,8 @@ function iscall_with_boundscheck(@nospecialize(stmt), sv::PostOptAnalysisState) nargs = 4 elseif f === memoryrefset! nargs = 5 + elseif f === memoryrefunset! + nargs = 4 else return false end @@ -809,7 +851,7 @@ function scan_non_dataflow_flags!(inst::Instruction, sv::PostOptAnalysisState) stmt = inst[:stmt] if !needs_ea_validation if !isterminator(stmt) && stmt !== nothing - # ignore control flow node – they are not removable on their own and thus not + # ignore control flow nodes – they are not removable on their own and thus do not # have `IR_FLAG_EFFECT_FREE` but still do not taint `:effect_free`-ness of # the whole method invocation sv.all_effect_free &= has_flag(flag, IR_FLAG_EFFECT_FREE) @@ -850,10 +892,14 @@ function scan_inconsistency!(inst::Instruction, sv::PostOptAnalysisState) # Special case: For `getfield` and memory operations, we allow inconsistency of the :boundscheck argument (; inconsistent, tpdum) = sv if iscall_with_boundscheck(stmt, sv) - for i = 1:(length(stmt.args)-1) + for i = 1:length(stmt.args) val = stmt.args[i] + # SSAValue should be the only permitted argument type which can be inconsistent found here. + # Others (e.g. GlobalRef) should have been moved to statement position. See stmt_effect_flags. if isa(val, SSAValue) - stmt_inconsistent |= val.id in inconsistent + if i < length(stmt.args) # not the boundscheck argument (which is last) + stmt_inconsistent |= val.id in inconsistent + end count!(tpdum, val) end end @@ -880,7 +926,7 @@ function ((; sv)::ScanStmt)(inst::Instruction, lstmt::Int, bb::Int) if isa(stmt, EnterNode) # try/catch not yet modeled give_up_refinements!(sv) - return nothing + return true # don't bail out early -- can cause tpdum counts to be off end scan_non_dataflow_flags!(inst, sv) @@ -926,10 +972,11 @@ function ((; sv)::ScanStmt)(inst::Instruction, lstmt::Int, bb::Int) end end - # bail out early if there are no possibilities to refine the effects - if !any_refinable(sv) - return nothing - end + # Do not bail out early, as this can cause tpdum counts to be off. + # # bail out early if there are no possibilities to refine the effects + # if !any_refinable(sv) + # return nothing + # end return true end @@ -937,25 +984,24 @@ end function check_inconsistentcy!(sv::PostOptAnalysisState, scanner::BBScanner) (; ir, inconsistent, tpdum) = sv + sv.all_retpaths_consistent || return scan!(ScanStmt(sv), scanner, false) + sv.all_retpaths_consistent || return complete!(tpdum); push!(scanner.bb_ip, 1) populate_def_use_map!(tpdum, scanner) stmt_ip = BitSetBoundedMinPrioritySet(length(ir.stmts)) for def in inconsistent - for use in tpdum[def] - if !(use in inconsistent) - push!(inconsistent, use) - append!(stmt_ip, tpdum[use]) - end - end - end + append!(stmt_ip, tpdum[def]) + end lazydomtree = LazyDomtree(ir) while !isempty(stmt_ip) idx = popfirst!(stmt_ip) + idx in inconsistent && continue # already processed inst = ir[SSAValue(idx)] stmt = inst[:stmt] if iscall_with_boundscheck(stmt, sv) + # recompute inconsistent flags for call while skipping boundscheck (last) argument any_non_boundscheck_inconsistent = false for i = 1:(length(stmt.args)-1) val = stmt.args[i] @@ -967,19 +1013,18 @@ function check_inconsistentcy!(sv::PostOptAnalysisState, scanner::BBScanner) any_non_boundscheck_inconsistent || continue elseif isa(stmt, ReturnNode) sv.all_retpaths_consistent = false + return elseif isa(stmt, GotoIfNot) bb = block_for_inst(ir, idx) cfg = ir.cfg blockliveness = BlockLiveness(cfg.blocks[bb].succs, nothing) for succ in iterated_dominance_frontier(cfg, blockliveness, get!(lazydomtree)) visit_bb_phis!(ir, succ) do phiidx::Int - push!(inconsistent, phiidx) - push!(stmt_ip, phiidx) + phiidx in inconsistent || push!(stmt_ip, phiidx) end end end - sv.all_retpaths_consistent || break - append!(inconsistent, tpdum[idx]) + push!(inconsistent, idx) append!(stmt_ip, tpdum[idx]) end end @@ -998,11 +1043,11 @@ function ipo_dataflow_analysis!(interp::AbstractInterpreter, opt::OptimizationSt completed_scan = scan!(ScanStmt(sv), scanner, true) if !completed_scan - if sv.all_retpaths_consistent - check_inconsistentcy!(sv, scanner) - else + # finish scanning for all_retpaths_consistent computation + check_inconsistentcy!(sv, scanner) + if !sv.all_retpaths_consistent # No longer any dataflow concerns, just scan the flags - scan!(scanner, false) do inst::Instruction, lstmt::Int, bb::Int + scan!(scanner, false) do inst::Instruction, ::Int, ::Int scan_non_dataflow_flags!(inst, sv) # bail out early if there are no possibilities to refine the effects if !any_refinable(sv) @@ -1017,7 +1062,7 @@ function ipo_dataflow_analysis!(interp::AbstractInterpreter, opt::OptimizationSt end # run the optimization work -function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult) +function optimize(interp::AbstractInterpreter, opt::OptimizationState{I}, caller::InferenceResult) where {I<:AbstractInterpreter} @zone "CC: OPTIMIZER" ir = run_passes_ipo_safe(opt.src, opt) ipo_dataflow_analysis!(interp, opt, ir, caller) finishopt!(interp, opt, ir) @@ -1053,7 +1098,7 @@ function run_passes_ipo_safe( __stage__ = 0 # used by @pass # NOTE: The pass name MUST be unique for `optimize_until::String` to work - @pass "CC: CONVERT" ir = convert_to_ircode(ci, sv) + @pass "CC: CONVERT" ir = convert_to_ircode!(ci, sv) @pass "CC: SLOT2REG" ir = slot2reg(ir, ci, sv) # TODO: Domsorting can produce an updated domtree - no need to recompute here @pass "CC: COMPACT_1" ir = compact!(ir) @@ -1061,6 +1106,7 @@ function run_passes_ipo_safe( # @zone "CC: VERIFY 2" verify_ir(ir) @pass "CC: COMPACT_2" ir = compact!(ir) @pass "CC: SROA" ir = sroa_pass!(ir, sv.inlining) + @pass "CC: GLOBALS" ir = reformulate_globals_pass!(ir, sv) @pass "CC: ADCE" (ir, made_changes) = adce_pass!(ir, sv.inlining) if made_changes @pass "CC: COMPACT_3" ir = compact!(ir, true) @@ -1075,6 +1121,363 @@ function run_passes_ipo_safe( return ir end +# Optimized conversion of a argument type to a singleton value, or nothing +function _global_call_singleton(@nospecialize(callee), ir::Union{IRCode,IncrementalCompact}) + isa(callee, QuoteNode) && return callee.value + isa(callee, GlobalRef) && return globalref_singleton(callee, ir) + isa(callee, Core.BindingPartition) && return partition_singleton(callee) + return singleton_type(argextype(callee, ir)) +end + +# The atomic-order argument at position `i`. +function order_arg(stmt::Expr, i::Int, ir::IRCode) + a = stmt.args[i] + o = _global_call_singleton(a, ir) + return isa(o, Symbol) ? QuoteNode(o) : a +end + +# The atomic-order argument at position `i`, which may be absent (reported as returning nothing). +function optional_order_arg(stmt::Expr, i::Int, ir::IRCode) + i > length(stmt.args) && return nothing + o = order_arg(stmt, i, ir) + return o === nothing ? QuoteNode(nothing) : o +end + +# Recognize a statement that reads a global binding value. +function recognize_global_read(@nospecialize(stmt), ir::IRCode) + if isa(stmt, GlobalRef) + return Pair{GlobalRef,Any}(stmt, QuoteNode(:unordered)) + end + isa(stmt, Expr) && stmt.head === :call || return nothing + na = length(stmt.args) + (na == 3 || na == 4) || return nothing + f = _global_call_singleton(stmt.args[1], ir) + if f === Core.getglobal + M = _global_call_singleton(stmt.args[2], ir) + s = _global_call_singleton(stmt.args[3], ir) + if isa(M, Module) && isa(s, Symbol) + order = na == 4 ? order_arg(stmt, 4, ir) : QuoteNode(:monotonic) + return Pair{GlobalRef,Any}(GlobalRef(M, s), order) + end + elseif f === Core.getfield && na == 3 + M = _global_call_singleton(stmt.args[2], ir) + s = _global_call_singleton(stmt.args[3], ir) + if isa(M, Module) && isa(s, Symbol) + return Pair{GlobalRef,Any}(GlobalRef(M, s), QuoteNode(:unordered)) + end + end + return nothing +end + +# The module and name a store names, taken from the two arguments following the callee at +# `base` (see `recognize_global_write`), as a `GlobalRef`, or nothing if either is unproven. +function _global_write_target(stmt::Expr, base::Int, ir::IRCode) + M = _global_call_singleton(stmt.args[base+1], ir) + s = _global_call_singleton(stmt.args[base+2], ir) + (isa(M, Module) && isa(s, Symbol)) ? GlobalRef(M, s) : nothing +end + +# A description of a recognized store to a global binding. +struct GlobalWriteInfo + g::GlobalRef + op::Symbol + order::Any + failorder::Any + value::Any + cmp::Any + # the reduce function's code instance, for a store recognized as an `:invoke_modify` + # node; `nothing` for the plain call form. + invoke::Any +end + +# Build the reformulated store from GlobalWriteInfo. +# An `:invoke_modify` store is rebuilt as one, keeping the reduce function's code instance +# ahead of the call. +function build_global_write_partition_call(part::Core.BindingPartition, w::GlobalWriteInfo) + callee = GlobalRef(Core, w.op) + ex = w.invoke === nothing ? Expr(:call, callee, QuoteNode(part)) : + Expr(:invoke_modify, w.invoke, callee, QuoteNode(part)) + (w.op === :replaceglobal_partition || w.op === :modifyglobal_partition) && push!(ex.args, w.cmp) + push!(ex.args, w.value) + if w.order !== nothing + push!(ex.args, w.order) + w.failorder === nothing || push!(ex.args, w.failorder) + end + return ex +end + +# Recognize a statement that writes a global binding through one of the store operator builtins. +# `modifyglobal!` upgrades like the rest, in both the plain call form and the `:invoke_modify` +# form the inliner gives it -- the code instance that node carries for the reduce function +# rides along on the upgraded one. +function recognize_global_write(@nospecialize(stmt), ir::IRCode) + if isexpr(stmt, :call) + base = 1 + invoke = nothing + elseif isexpr(stmt, :invoke_modify) + base = 2 + invoke = stmt.args[1] + else + return nothing + end + na = length(stmt.args) - base + 3 <= na <= 6 || return nothing # cheap arity check for the narrowest and widest list accepted below + f = _global_call_singleton(stmt.args[base], ir) + if f === Core.modifyglobal! && (na == 4 || na == 5) + g = _global_write_target(stmt, base, ir) + g === nothing && return nothing + order = optional_order_arg(stmt, base+5, ir) + return GlobalWriteInfo(g, :modifyglobal_partition, order, nothing, stmt.args[base+4], stmt.args[base+3], invoke) + elseif invoke !== nothing + return nothing # only `modifyglobal!` is invoked this way + elseif f === Core.setglobal! && (na == 3 || na == 4) + g = _global_write_target(stmt, base, ir) + g === nothing && return nothing + order = optional_order_arg(stmt, base+4, ir) + return GlobalWriteInfo(g, :setglobal_partition, order, nothing, stmt.args[base+3], nothing, nothing) + elseif f === Core.swapglobal! && (na == 3 || na == 4) + g = _global_write_target(stmt, base, ir) + g === nothing && return nothing + order = optional_order_arg(stmt, base+4, ir) + return GlobalWriteInfo(g, :swapglobal_partition, order, nothing, stmt.args[base+3], nothing, nothing) + elseif f === Core.replaceglobal! && (4 <= na <= 6) + g = _global_write_target(stmt, base, ir) + g === nothing && return nothing + order = optional_order_arg(stmt, base+5, ir) + failorder = optional_order_arg(stmt, base+6, ir) + return GlobalWriteInfo(g, :replaceglobal_partition, order, failorder, stmt.args[base+4], stmt.args[base+3], nothing) + elseif f === Core.setglobalonce! && (3 <= na <= 5) + g = _global_write_target(stmt, base, ir) + g === nothing && return nothing + order = optional_order_arg(stmt, base+4, ir) + failorder = optional_order_arg(stmt, base+5, ir) + return GlobalWriteInfo(g, :setglobalonce_partition, order, failorder, stmt.args[base+3], nothing, nothing) + end + return nothing +end + +# Recognize a definedness query on a global binding: +# `isdefinedglobal(M, s[, allow_import[, order]])` or `isdefined(M::Module, s)`. +# Only an `allow_import === true` query reformulates (it walks imports to the leaf, matching the resolution below). +# An `allow_import === false` query is left on the runtime path (if not already folded by inference), which is conservatively correct. +function recognize_global_isdefined(@nospecialize(stmt), ir::IRCode) + isa(stmt, Expr) && stmt.head === :call || return nothing + na = length(stmt.args) + f = _global_call_singleton(stmt.args[1], ir) + if f === Core.isdefinedglobal && (3 <= na <= 5) + M = _global_call_singleton(stmt.args[2], ir) + s = _global_call_singleton(stmt.args[3], ir) + (isa(M, Module) && isa(s, Symbol)) || return nothing + if na >= 4 + _global_call_singleton(stmt.args[4], ir) === true || return nothing + end + order = na == 5 ? order_arg(stmt, 5, ir) : QuoteNode(:unordered) + return Pair{GlobalRef,Any}(GlobalRef(M, s), order) + elseif f === Core.isdefined && na == 3 + M = _global_call_singleton(stmt.args[2], ir) + s = _global_call_singleton(stmt.args[3], ir) + (isa(M, Module) && isa(s, Symbol)) || return nothing + return Pair{GlobalRef,Any}(GlobalRef(M, s), QuoteNode(:unordered)) + end + return nothing +end + +# Narrow the validity of the code being optimized to `valid_worlds`, the optimizer's +# counterpart of `update_valid_age!`. `sv.src` carries the frame's `valid_worlds` at this +# point (set by `finishinfer!`), and `finish_nocycle`/`finish_cycle` intersect it back into +# the frame after optimization. +function narrow_valid_worlds!(sv::OptimizationState, world::UInt, valid_worlds::WorldRange) + src = sv.src + valid_worlds = intersect(world_range(src), valid_worlds) + if !(world in valid_worlds) + error("invalid age range update") + end + src.min_world = first(valid_worlds) + src.max_world = last(valid_worlds) + return valid_worlds +end + +# Resolve a read of `g` to a leaf `Core.BindingPartition` that fully captures the behavior. +# Return the leaf partition to use, whether `getglobal` would deprecation-warn for this access, +# and whether the walk crossed an import (determining the name to use for errors). +const ResolvedRead = Tuple{Core.BindingPartition,Bool,Bool} + +function reformulate_read(g::GlobalRef, sv::OptimizationState, world::UInt, edges::Vector{Any}, + cache::IdDict{Core.Binding,Union{ResolvedRead,Nothing}}) + binding = convert(Core.Binding, g) + haskey(cache, binding) && return cache[binding] + p = resolve_read(g, binding, world) + if p !== nothing + push!(edges, binding) + valid_worlds, _ = binding_access_range(g, WorldWithRange(world, world_range(sv.src)), false) + narrow_valid_worlds!(sv, world, valid_worlds) + end + cache[binding] = p + return p +end + +function resolve_read(g::GlobalRef, binding::Core.Binding, world::UInt) + # A world-1 constant (builtin/intrinsic/core type) is immutable: leave it as a bare + # `GlobalRef` for codegen to embed directly, with no `BindingPartition` and no edge. + world1_const(g) && return nothing + partition = lookup_binding_partition(world, binding) + leaf_binding, leaf, depwarn = walk_to_leaf_partition_depwarn(binding, partition, world) + kind = binding_kind(leaf) + # Freeze only what codegen embeds by value (a real constant) or by slot (a typed global, + # whose identity `binding_access_key` tracks). A backdated constant is neither: inference + # types it as an untyped global, which keeps it as a runtime `getglobal` and also keeps + # the backdate admonition. + # `PARTITION_KIND_DECLARED` (an untyped `global x`) is notably also omitted here, per the comment in `binding_access_key`. + if !(is_defined_const_binding(kind) && kind !== PARTITION_KIND_BACKDATED_CONST) && + kind !== PARTITION_KIND_GLOBAL + return nothing + end + # `leaf_binding !== binding` means the walk crossed an import, so the leaf no longer names + # the access the source asked for in UndefVarError. That only matters for a leaf that can + # actually be undefined (not a constant). + return (leaf, depwarn, kind === PARTITION_KIND_GLOBAL && leaf_binding !== binding) +end + +# The store counterpart of `reformulate_read`. +# `PARTITION_KIND_DECLARED` (an untyped `global x`) is notably omitted here per the comment in `binding_access_key`. +function reformulate_write(g::GlobalRef, sv::OptimizationState, world::UInt, edges::Vector{Any}, + cache::IdDict{Core.Binding,Union{Core.BindingPartition,Nothing}}) + binding = convert(Core.Binding, g) + haskey(cache, binding) && return cache[binding] + partition = lookup_binding_partition(world, binding) + p = binding_kind(partition) === PARTITION_KIND_GLOBAL ? partition : nothing + if p !== nothing + push!(edges, binding) + valid_worlds, _ = binding_access_range(g, WorldWithRange(world, world_range(sv.src)), true) + narrow_valid_worlds!(sv, world, valid_worlds) + end + cache[binding] = p + return p +end + +# Preserve depwarn effect explicitly. +function emit_depwarn_partition!(ir::IRCode, idx::Int, p::Core.BindingPartition) + insert_node!(ir, idx, NewInstruction( + Expr(:call, GlobalRef(Core, :depwarn_partition), QuoteNode(p)), Nothing)) + return nothing +end + +function _reformulate_globals!(ir::IRCode, opt::OptimizationState) + world = get_inference_world(opt.inlining.interp) + edges = opt.inlining.edges + read_cache = IdDict{Core.Binding,Union{ResolvedRead,Nothing}}() + write_cache = IdDict{Core.Binding,Union{Core.BindingPartition,Nothing}}() + for idx = 1:length(ir.stmts) + inst = ir[SSAValue(idx)] + stmt = inst[:stmt] + r = recognize_global_read(stmt, ir) + if r !== nothing + rr = reformulate_read(r.first, opt, world, edges, read_cache) + if rr !== nothing + (p, depwarn, imported) = rr + depwarn && emit_depwarn_partition!(ir, idx, p) + # Decide if simple `p` has the right semantics, or needs the full `getglobal_partition` call to preserve semantics. + stmt = (r.second !== QuoteNode(:unordered) || imported) ? + Expr(:call, GlobalRef(Core, :getglobal_partition), + QuoteNode(r.first), QuoteNode(p), r.second) : p + inst[:stmt] = stmt + r.second isa QuoteNode && continue # optimize the loop in the common case + end + # fall through: a read call may still contain nested `GlobalRef` operands worth rewriting (including for `order`). + end + d = recognize_global_isdefined(stmt, ir) + if d !== nothing + rr = reformulate_read(d.first, opt, world, edges, read_cache) + if rr !== nothing + stmt = Expr(:call, GlobalRef(Core, :isdefinedglobal_partition), QuoteNode(rr[1]), d.second) + inst[:stmt] = stmt + d.second isa QuoteNode && continue # optimize the loop in the common case + end + # fall through: as for a read, a non-constant `order` operand may still be a `GlobalRef` worth rewriting. + end + w = recognize_global_write(stmt, ir) + if w !== nothing + part = reformulate_write(w.g, opt, world, edges, write_cache) + if part !== nothing + (part.kind & PARTITION_FLAG_DEPWARN) != 0 && emit_depwarn_partition!(ir, idx, part) + # Decide if simple `p = val` has the right semantics, or needs the full call form to preserve all semantics. + stmt = (w.op === :setglobal_partition && w.order === nothing) ? + Expr(:(=), part, w.value) : build_global_write_partition_call(part, w) + inst[:stmt] = stmt + end + # An unresolved store keeps the runtime path it came in on: nothing froze, so + # there is no partition to name its target with, and every `Core.*_partition` + # builtin takes one. + # fall through: a write call may still contain other nested `GlobalRef` operands worth rewriting. + end + + # `ccall`/`cglobal` name their target with a nested `Expr(:tuple, name, library)`, + # with special semantics, since that GlobalRef is permitted to have side-effects and throw, + # but we still want to apply the same GlobalRef -> BindingPartition transform optimization. + if isexpr(stmt, :foreigncall) || isexpr(stmt, :foreignglobal) + target = stmt.args[1] + if isexpr(target, :tuple) + newargs = nothing + for i = 1:length(target.args) + use = target.args[i] + isa(use, GlobalRef) || continue + rr = reformulate_read(use, opt, world, edges, read_cache) + rr === nothing && continue + (p, depwarn, _) = rr + depwarn && continue # skip optimizing since we don't have a good place to put the depwarn node -- this should end up on a cold branch in codegen anyways + if newargs === nothing + newargs = copy(target.args) + end + newargs[i] = p + end + if newargs !== nothing + newtarget = Expr(:tuple) + newtarget.args = newargs + stmt.args[1] = newtarget + inst[:stmt] = stmt + end + end + end + + # Rewrite (effect-free) `GlobalRef` operands nested inside this statement. + urs = userefs(stmt) + changed = false + for ur in urs + use = ur[] + if isa(use, GlobalRef) + rr = reformulate_read(use, opt, world, edges, read_cache) + if rr !== nothing + (p, _, _) = rr + ur[] = p + changed = true + end + end + end + if changed + inst[:stmt] = urs[] + end + end + return nothing +end + +# Rather than repeat the `bb_saw_latestworld` analysis, +# skip reformulating any IR that carries a `:latestworld` marker at all. +# It isn't usually hot code, and usually bails early anyways. +function has_latestworld(ir::IRCode) + for idx = 1:length(ir.stmts) + isexpr(ir[SSAValue(idx)][:stmt], :latestworld) && return true + end + return false +end + +function reformulate_globals_pass!(ir::IRCode, opt::OptimizationState) + has_latestworld(ir) || _reformulate_globals!(ir, opt) + # Reflect any narrowing on the IR itself, which inherited the frame's range at conversion. + valid_worlds = world_range(opt.src) + valid_worlds == ir.valid_worlds || (ir = IRCode(ir, valid_worlds)) + return ir +end + function strip_trailing_junk!(code::Vector{Any}, ssavaluetypes::Vector{Any}, ssaflags::Vector, debuginfo::DebugInfoStream, cfg::CFG, info::Vector{CallInfo}) # Remove `nothing`s at the end, we don't handle them well # (we expect the last instruction to be a terminator) @@ -1123,10 +1526,10 @@ function changed_lineinfo(di::DebugInfo, codeloc::Int, prevloc::Int) edge === prev[2] || return true # change to this edge linetable = di.linetable # check for change to line number here - if linetable === nothing || line == 0 + if !(linetable isa DebugInfo) || line == 0 line == prevline || return true else - changed_lineinfo(linetable::DebugInfo, Int(line), Int(prevline)) && return true + changed_lineinfo(linetable, Int(line), Int(prevline)) && return true end # check for change to edge here edge == 0 && return false # no edge here @@ -1136,10 +1539,11 @@ function changed_lineinfo(di::DebugInfo, codeloc::Int, prevloc::Int) end end -function convert_to_ircode(ci::CodeInfo, sv::OptimizationState) +function convert_to_ircode!(ci::CodeInfo, sv::OptimizationState) # Update control-flow to reflect any unreachable branches. ssavaluetypes = ci.ssavaluetypes::Vector{Any} - ci.code = code = copy_exprargs(ci.code) + # ci is always a fresh private copy so we can reuse it here. + code = ci.code di = DebugInfoStream(sv.linfo, ci.debuginfo, length(code)) codelocs = di.codelocs ssaflags = ci.ssaflags @@ -1326,7 +1730,6 @@ end function slot2reg(ir::IRCode, ci::CodeInfo, sv::OptimizationState) # need `ci` for the slot metadata, IR for the code - svdef = sv.linfo.def @zone "CC: DOMTREE_1" domtree = construct_domtree(ir) defuse_insts = scan_slot_def_use(Int(ci.nargs), ci, ir.stmts.stmt) 𝕃ₒ = optimizer_lattice(sv.inlining.interp) @@ -1373,8 +1776,8 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp # and are likely to combine with the operations around them, # so reduce their cost by half. cost = T_IFUNC_COST[iidx] - if cost == 0 || nargs < 3 || - (f === Intrinsics.cglobal || f === Intrinsics.llvmcall) # these hold malformed IR, so argextype will crash on them + if cost == 0 || nargs < 3 || f === Intrinsics.llvmcall + # holds malformed IR, so argextype will crash on it return cost end aty2 = widenconditional(argextype(ex.args[2], src, sptypes)) @@ -1411,7 +1814,10 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp elseif f === Core.memoryrefset! && length(ex.args) >= 3 atyp = argextype(ex.args[2], src, sptypes) return isknowntype(atyp) ? 5 : params.inline_nonleaf_penalty - elseif f === typeassert && isconstType(widenconst(argextype(ex.args[3], src, sptypes))) + elseif f === Core.memoryrefunset! && length(ex.args) >= 3 + atyp = argextype(ex.args[2], src, sptypes) + return isknowntype(atyp) ? 5 : params.inline_nonleaf_penalty + elseif f === typeassert && isconstType(argextype_widened(ex.args[3], src, sptypes)) return 1 end fidx = find_tfunc(f) @@ -1429,10 +1835,15 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp return params.inline_nonleaf_penalty elseif head === :foreigncall foreigncall = ex.args[1] - if foreigncall isa QuoteNode && foreigncall.value === :jl_string_ptr - return 1 + if isexpr(foreigncall, :tuple, 1) + foreigncall = foreigncall.args[1] + if foreigncall isa QuoteNode && foreigncall.value === :jl_string_ptr + return 1 + end end return 20 + elseif head === :foreignglobal + return 1 elseif head === :invoke || head === :invoke_modify # Calls whose "return type" is Union{} do not actually return: # they are errors. Since these are not part of the typical @@ -1442,7 +1853,12 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp extyp = line == -1 ? Any : argextype(SSAValue(line), src, sptypes) return extyp === Union{} ? 0 : UNKNOWN_CALL_COST elseif head === :(=) - return statement_cost(ex.args[2], -1, src, sptypes, params) + # A resolved store to a global should cost the same or less than the `setglobal!` tfunc declared. + lhs = ex.args[1] + cost = (isa(lhs, GlobalRef) || isa(lhs, Core.BindingPartition)) ? 3 : 0 + rhs = ex.args[2] + isa(rhs, Expr) && (cost += statement_cost(rhs, -1, src, sptypes, params)) + return cost elseif head === :copyast return 100 end @@ -1566,17 +1982,16 @@ function renumber_ir_elements!(body::Vector{Any}, ssachangemap::Vector{Int}, lab end elseif isa(el, EnterNode) tgt = el.catch_dest - if tgt != 0 - was_deleted = labelchangemap[tgt] == typemin(Int) - if was_deleted - @assert !isdefined(el, :scope) - body[i] = nothing + if tgt != 0 && labelchangemap[tgt] == typemin(Int) + @assert !isdefined(el, :scope) + body[i] = nothing # the enclosing catch block was deleted + else + # renumber the catch destination (tgt == 0 stays frame-less) and the scope operand + newdest = tgt == 0 ? 0 : tgt + labelchangemap[tgt] + if isdefined(el, :scope) && isa(el.scope, SSAValue) + body[i] = EnterNode(newdest, SSAValue(el.scope.id + ssachangemap[el.scope.id])) else - if isdefined(el, :scope) && isa(el.scope, SSAValue) - body[i] = EnterNode(tgt + labelchangemap[tgt], SSAValue(el.scope.id + ssachangemap[el.scope.id])) - else - body[i] = EnterNode(el, tgt + labelchangemap[tgt]) - end + body[i] = EnterNode(el, newdest) end end elseif isa(el, Expr) diff --git a/Compiler/src/precompile.jl b/Compiler/src/precompile.jl new file mode 100644 index 0000000000000..9504bd85dc052 --- /dev/null +++ b/Compiler/src/precompile.jl @@ -0,0 +1,492 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +compile_hint(@nospecialize(argt::Type)) = ccall(:jl_compile_hint, Int32, (Any,), argt) != 0 + +# Utility functions for type manipulation +function count_union_components(t::Union) + count = 0 + while true + count += 1 + t = t.b + if !isa(t, Union) + count += 1 + break + end + end + return count +end + +function nth_union_component(t::Union, n::Int) + current = 1 + while current < n && isa(t, Union) + current += 1 + t = t.b + end + if current == n + return isa(t, Union) ? t.a : t + else + error("Index out of bounds for Union type") + end +end + +# Port of _compile_all_tvar_union +# f(...) where {T<:Union{...}} is a common pattern +function compile_all_tvar_union(methsig) + tvarslen = unionall_depth(methsig) + if tvarslen == 0 + return compile_hint(methsig) + end + + sigbody = methsig + env = Vector{Any}(undef, 2 * tvarslen) + idx = Vector{Int}(undef, tvarslen) + + # Initialize environment + for i in 1:tvarslen + if !isa(sigbody, UnionAll) + return false + end + idx[i] = 1 + var = sigbody.var + env[2*i - 1] = var + + # Get upper bound + tv = var + while isa(tv, TypeVar) + tv = tv.ub + end + + # Any as TypeVar is common and not useful here to try to analyze further + if isabstracttype(tv) && !isType(tv) + return false + end + + env[2*i] = tv + sigbody = sigbody.body + end + + all_success = true + incr = false + + while !incr + # Generate all combinations + for i in 1:tvarslen + incr = true + tv = env[2*i - 1] + while isa(tv, TypeVar) + tv = tv.ub + end + + if isa(tv, Union) + l = count_union_components(tv) + j = idx[i] + env[2*i] = nth_union_component(tv, j) + j += 1 + + if incr + if j > l + idx[i] = 1 + else + idx[i] = j + incr = false + end + end + end + end + + # Try to instantiate and compile + sig = try + ccall(:jl_instantiate_type_with, Any, (Any, Ptr{Any}, Cint), + sigbody, env, tvarslen) + catch + nothing + end + if isa(sig, DataType) && ccall(:jl_has_concrete_subtype, Cint, (Any,), sig) != 0 + success = compile_hint(sig) + all_success = all_success && success + else + all_success = false + end + end + + return all_success +end + +# Port of _compile_all_union +function compile_all_union(sig) + sigbody = unwrap_unionall(sig)::DataType + + if !isa(sigbody, Type) || !isa(sigbody, DataType) + return compile_all_tvar_union(sig) + end + + count_unions = 0 + union_size = 1 + params = sigbody.parameters + + for ty in params + if isa(ty, Union) + count_unions += 1 + union_size *= count_union_components(ty) + elseif isa(ty, DataType) && + ((!isconcretetype(ty) || iskindtype(ty)) && !isType(ty)) + return false # no amount of union splitting will help + end + end + + if union_size <= 1 || union_size > 8 + return compile_all_tvar_union(sig) + end + + idx = zeros(Int, count_unions) + all_success = true + + incr = false + while !incr + # Generate parameter combinations + new_params = Vector{Any}(undef, length(params)) + idx_ctr = 1 + incr = true + + for (i, ty) in enumerate(params) + if isa(ty, Union) + l = count_union_components(ty) + j = idx[idx_ctr] + new_params[i] = nth_union_component(ty, j + 1) # 1-based indexing + j += 1 + + if incr + if j >= l + idx[idx_ctr] = 0 + else + idx[idx_ctr] = j + incr = false + end + end + idx_ctr += 1 + else + new_params[i] = ty + end + end + + # Create new signature and try to compile + # Reconstruct tuple type + new_sigbody = Tuple{new_params...} + # Rewrap in UnionAll if needed + methsig = rewrap_unionall(new_sigbody, sig) + success = compile_all_tvar_union(methsig) + all_success = all_success && success + end + return all_success +end + +# Complete method collection implementation +function collect_all_method_defs(newmodules, mod_array) + allmeths = Any[] + + function method_visitor(method) + method = method::Method + if newmodules !== nothing + method.module in newmodules || return true + end + if isdefined(method, :external_mt) + return true # Continue iteration + end + push!(allmeths, method) + return true + end + + # Always visit the global method table first + visit(method_visitor, Core.methodtable) + + # If mod_array is provided, iterate through modules looking for MethodTable objects + #if mod_array !== nothing + # function visit_methodtable(mt::Core.MethodTable) + # if mt !== Core.methodtable # Skip global method table since we already visited it + # visit(method_visitor, mt) + # end + # end + + # function foreach_mtable_in_module(mod::Module) + # # Get all bindings in the module and look for MethodTable objects + # for name in names(mod, all=true, imported=true) + # if isdefined(mod, name) + # val = getglobal(mod, name) + # if isa(val, Module) && val !== mod && parentmodule(val) === mod + # # Recursively visit submodules + # foreach_mtable_in_module(val) + # elseif isa(val, Core.MethodTable) + # # Visit this method table + # visit_methodtable(val) + # end + # end + # end + # end + + # # Iterate through provided modules + # for mod in mod_array + # if isa(mod, Module) + # # Only visit toplevel modules (where parent == mod) + # if parentmodule(mod) === mod + # foreach_mtable_in_module(mod) + # end + # end + # end + #end + return allmeths +end + +function infer_all_method_defs!(all::Bool, allmeths, world::UInt, worklist) + # Process collected methods and create method instances + for m in allmeths + m = m::Method + + # Skip macro methods unless specifically requested + if !all && !iszero(ccall(:jl_method_is_macro, Cint, (Any,), m)) + continue + end + + if !isdefined(m, :source) + continue + end + + # Check if this method has a single compilable specialization + if isa(m.sig, DataType) && isa_compileable_sig(m.sig, Core.svec(), m) + # Method has a single compilable specialization, e.g. its definition + # signature is concrete. in this case we can just hint it. + ccall(:jl_compile_method_sig, Cvoid, (Any, Any, Any, Csize_t), + m, m.sig, Core.svec(), world) + else + # Try to create leaf signatures using union expansion from the signature declaration and compile those + compile_all_union(m.sig) + + if all + # Also compile fully generic fallback if requested + unspec = ccall(:jl_get_unspecialized, Any, (Any,), m) + if unspec !== nothing + push!(worklist, unspec) + end + + end + end + end +end + +function enqueue_specializations!(all::Bool, newmethods, worklist) + for method in newmethods + method = method::Method + + # skip all macros + if !all && !iszero(ccall(:jl_method_is_macro, Cint, (Any,), method)) + continue + end + + # Check for special methods that should always be compiled + if (method.name === :__init__ || isdefined(method, :ccallable)) && isdispatchtuple(method.sig) + # Get method instance for __init__ methods and @ccallable functions + mi = specialize_method(method, method.sig, Core.svec())::MethodInstance + push!(worklist, mi) + else + # Process existing specializations + specializations = method.specializations + if isa(specializations, Core.SimpleVector) + for i = 1:length(specializations) + mi = specializations[i] + if mi !== nothing + enqueue_specialization!(all, worklist, mi::MethodInstance) + end + end + elseif isa(specializations, MethodInstance) + enqueue_specialization!(all, worklist, specializations) + end + end + + # Handle ccallable methods + if isdefined(method, :ccallable) + push!(worklist, method.ccallable) + end + end +end + +function enqueue_specialization!(all::Bool, worklist, mi::MethodInstance) + if mi.precompile + push!(worklist, mi) + return true + end + codeinst = isdefined(mi, :cache) ? mi.cache : nothing + while codeinst !== nothing + do_compile = false + if codeinst.owner !== nothing + # This code instance is from a foreign interpreter, so we skip it + elseif use_const_api(codeinst) # Check if invoke is jl_fptr_const_return + do_compile = true + elseif codeinst.invoke != C_NULL + do_compile = true + elseif !do_compile && isdefined(codeinst, :inferred) + inferred = codeinst.inferred + # Check compilation options and inlining cost + if (all || inferred === nothing || + ((isa(inferred, String) || isa(inferred, CodeInfo) || isa(inferred, UInt8)) && + ccall(:jl_ir_inlining_cost, UInt16, (Any,), inferred) == typemax(UInt16))) + do_compile = true + end + end + if do_compile + push!(worklist, mi) + return true + end + # Move to the next code instance in the chain + codeinst = isdefined(codeinst, :next) ? codeinst.next : nothing + end + return true +end + +# Main unified compilation and emission function +function compile_and_emit_native(worlds::Vector{UInt}, + trim_mode::UInt8, + external_linkage::Bool, + newmodules, # Vector{Module} or Nothing + mod_array, # Vector{Module} or Nothing + all::Bool, + module_init_order::Vector{Any}, # Vector{Module} + ext_foreign_cis::Vector{Any}) # Vector{CodeInstance} + @nospecialize + latestworld = worlds[end] + + # Step 1: Precompile all __init__ methods that will be required + for mod in module_init_order + if Core.invoke_in_world(latestworld, isdefined, mod, :__init__) + f = Core.invoke_in_world(latestworld, getglobal, mod, :__init__) + # Get module compile setting + setting = ccall(:jl_get_module_compile, Cint, (Any,), mod) + # When trimming, the entrypoint list is the only thing that keeps `__init__` + # alive so keep them regardless of compile option + if setting != JL_OPTIONS_COMPILE_OFF && (trim_mode != TRIM_NO || setting != JL_OPTIONS_COMPILE_MIN) + tt = Tuple{Core.Typeof(f)} + compile_hint(tt) + trim_mode == 0x00 || add_entrypoint(tt) + end + end + end + + # Step 2: Collect all method definitions, filtered by worklist if provided + newmethods = collect_all_method_defs(newmodules, mod_array) + + # Step 3: Collect set of method instances that seem worth compiling + specialization_worklist = [] + if trim_mode == 0x00 + if newmodules === nothing + infer_all_method_defs!(all, newmethods, latestworld, specialization_worklist) + else + # Compute new_used using queue_used with global newly_inferred + new_used = ccall(:jl_compute_new_used_ci, Any, ()) + if new_used !== nothing + for i in 1:length(new_used::Vector{Any}) + ci = new_used[i] + if ci isa MethodInstance + push!(specialization_worklist, ci) + elseif ci isa CodeInstance + if ci.owner !== nothing + # enqueue_specialization will skip over CIs from foreign interpreters + # and currently will visit at most one (do_compile) CI per method instance + if ci.max_world === typemax(UInt) + push!(ext_foreign_cis, ci) + end + else + enqueue_specialization!(all, specialization_worklist, get_ci_mi(ci)) + end + end + end + end + end + enqueue_specializations!(all, newmethods, specialization_worklist) + + # Process the specialization worklist and prepare final tocompile worklist + tocompile = [] + for item in specialization_worklist + if isa(item, Core.MethodInstance) + processed_mi = process_method_instance_for_compilation(item, latestworld) + if processed_mi !== nothing + push!(tocompile, processed_mi) + end + else + # Handle SimpleVector (ccallable entries) + push!(tocompile, item::Core.SimpleVector) + end + end + else # trimming mode + # array of MethodInstances and ccallable aliases to include in the output + tocompile = [] + + # Process entrypoint method instances + for mi in _entrypoint_mis + # Add the method instance to compile list + push!(tocompile, mi) + + # Check if this method has a ccallable annotation + if isdefined(mi.def, :ccallable) && mi.def.ccallable !== nothing + push!(tocompile, mi.def.ccallable) + end + end + end + + # Step 4: Perform type inference on tocompile to create codeinfos + # Returns svec(codeinfos, cis): the interleaved CodeInstance/CodeInfo work + # list for codegen, plus the ordered CodeInstances to place in the method + # caches of the output image. + result = try + typeinf_ext_toplevel(tocompile, worlds, trim_mode, external_linkage) + catch exc + # Handle trimming failures + isa(exc, Core.TrimFailure) || rethrow() + # The verification check failed. The error message should already have + # been printed, so give up here and exit (w/o a stack trace). + invokelatest(invokelatest(getglobal, Base, :exit), 1) + end + + return result + +end + +# Helper function to process method instances for compilation +function process_method_instance_for_compilation(mi::MethodInstance, world::UInt) + method = mi.def::Method + if !(isdefined(method, :unspecialized) && mi === method.unspecialized) + if !isa_compileable_sig(mi.specTypes, mi.sparam_vals, method) + # Try to get a compileable specialization + mi = ccall(:jl_get_specialization1, Any, (Any, Csize_t), + mi.specTypes, world)::Union{Nothing,MethodInstance} + end + end + return mi +end + +const _entrypoint_mis = Vector{MethodInstance}() + +# Add a method signature as an entrypoint for compilation. +function add_entrypoint(types::Type) + world = get_world_counter() + # Get the method instance for this signature + mi = ccall(:jl_get_compile_hint_specialization, Any, + (Any, Csize_t, Cint), + types, world, 1) + if mi === nothing + return false + end + push!(_entrypoint_mis, mi::MethodInstance) + return true +end + +function add_ccallable_entrypoints!() + # Collect all methods with ccallable annotations + ccallable_methods = Any[] + visit(Core.methodtable) do method + method = method::Method + if isdefined(method, :ccallable) + # Add the ccallable tuple signature + ccallable_sig = method.ccallable[2] # Second element is the signature + add_entrypoint(ccallable_sig) + end + return true + end +end diff --git a/Compiler/src/reflection_interface.jl b/Compiler/src/reflection_interface.jl index 3fc182685e598..6cd21355918c2 100644 --- a/Compiler/src/reflection_interface.jl +++ b/Compiler/src/reflection_interface.jl @@ -42,7 +42,7 @@ end function statement_costs!(interp::AbstractInterpreter, cost::Vector{Int}, body::Vector{Any}, src::Union{CodeInfo, IRCode}, match::Core.MethodMatch) params = OptimizationParams(interp) - sptypes = VarState[VarState(sp, false) for sp in match.sparams] + sptypes = VarState[VarState(sp, #= ssadef =# typemin(Int), false) for sp in match.sparams] return statement_costs!(cost, body, src, sptypes, params) end diff --git a/base/staticdata.jl b/Compiler/src/reinfer.jl similarity index 71% rename from base/staticdata.jl rename to Compiler/src/reinfer.jl index a9ba58f3d82aa..079c78778ba61 100644 --- a/base/staticdata.jl +++ b/Compiler/src/reinfer.jl @@ -1,10 +1,15 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +using ..Compiler.Base +using ..Compiler: _findsup, store_backedges, JLOptions, get_world_counter, + _methods_by_ftype, get_methodtable, get_ci_mi, should_instrument, + morespecific, RefValue, get_require_world, Vector, IdDict, + binding_access_range, is_leaf_partition, WorldWithRange, WorldRange, min_world, max_world using .Core: CodeInstance, MethodInstance -using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable, get_ci_mi, morespecific +const CI_FLAGS_NATIVE_CACHE_VALID = 0b1000 const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1 -const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing) +const _jl_debug_method_invalidation = RefValue{Union{Nothing,Vector{Any}}}(nothing) debug_method_invalidation(onoff::Bool) = _jl_debug_method_invalidation[] = onoff ? Any[] : nothing @@ -50,8 +55,7 @@ end function VerifyMethodInitialState(codeinst::CodeInstance) mi = get_ci_mi(codeinst) def = mi.def::Method - callees = codeinst.edges - VerifyMethodInitialState(codeinst, mi, def, callees) + VerifyMethodInitialState(codeinst, mi, def, codeinst.edges) end function VerifyMethodWorkState(dummy_cause::CodeInstance) @@ -63,56 +67,64 @@ function VerifyMethodResultState() end +function binding_access_range_at(b::Core.Binding, world::UInt) + wr, _ = binding_access_range(b, WorldWithRange(world, WorldRange(get_require_world(), world)), false) + return wr +end + +# Whether an access to `b` can resolve differently now than it did in any process that serialized code against it. +function binding_changed_since_require_world(b::Core.Binding, world::UInt) + require_world = get_require_world() + # Fast path: this binding has not been repartitioned since the require world at all, and it + # resolves without crossing an import, so no walk is needed to know its range reaches back. + # A non-leaf partition has to take the slow path: the walk continues into the binding it + # imports, which may itself have been repartitioned after the require world even though `b` + # was not. + if isdefined(b, :partitions) + p = b.partitions + p.min_world <= require_world && is_leaf_partition(p) && return false + end + return min_world(binding_access_range_at(b, world)) > require_world +end + # Restore backedges to external targets -# `edges` = [caller1, ...], the list of worklist-owned code instances internally -# `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally -function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{Any}}, extext_methods::Vector{Any}, internal_methods::Vector{Any}) +# `internal_methods` = [caller1, ...], the list of worklist-owned code instances internally +function insert_backedges(internal_methods::Vector{Any}) # determine which CodeInstance objects are still valid in our image # to enable any applicable new codes backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) - Base.scan_new_methods!(extext_methods, internal_methods, backedges_only) - workspace = VerifyMethodWorkspace() - _insert_backedges(edges, workspace) - if ext_ci_list !== nothing - _insert_backedges(ext_ci_list, workspace, #=external=#true) - end + scan_new_methods!(internal_methods, get_world_counter(), backedges_only) + scan_new_code!(internal_methods, VerifyMethodWorkspace()) + nothing end -function _insert_backedges(edges::Vector{Any}, workspace::VerifyMethodWorkspace, external::Bool=false) - for i = 1:length(edges) - codeinst = edges[i]::CodeInstance +function scan_new_code!(internal_methods::Vector{Any}, workspace::VerifyMethodWorkspace) + for i = 1:length(internal_methods) + codeinst = internal_methods[i] + codeinst isa CodeInstance || continue + # codeinst.owner === nothing || continue validation_world = get_world_counter() verify_method_graph(codeinst, validation_world, workspace) # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. @ccall jl_promote_ci_to_current(codeinst::Any, validation_world::UInt)::Cvoid - minvalid = codeinst.min_world - maxvalid = codeinst.max_world - # Finally, if this CI is still valid in some world age and belongs to an external method(specialization), - # poke it that mi's cache - if maxvalid ≥ minvalid && external - caller = get_ci_mi(codeinst) - @assert isdefined(codeinst, :inferred) # See #53586, #53109 - inferred = @ccall jl_rettype_inferred( - codeinst.owner::Any, caller::Any, minvalid::UInt, maxvalid::UInt)::Any - if inferred !== nothing - # We already got a code instance for this world age range from - # somewhere else - we don't need this one. - else - @ccall jl_mi_cache_insert(caller::Any, codeinst::Any)::Cvoid - end - end end end function verify_method_graph(codeinst::CodeInstance, validation_world::UInt, workspace::VerifyMethodWorkspace) - @assert isempty(workspace.stack); @assert isempty(workspace.visiting); - @assert isempty(workspace.initial_states); @assert isempty(workspace.work_states); @assert isempty(workspace.result_states) + @assert isempty(workspace.stack) "workspace corrupted" + @assert isempty(workspace.visiting) "workspace corrupted" + @assert isempty(workspace.initial_states) "workspace corrupted" + @assert isempty(workspace.work_states) "workspace corrupted" + @assert isempty(workspace.result_states) "workspace corrupted" child_cycle, minworld, maxworld = verify_method(codeinst, validation_world, workspace) @assert child_cycle == 0 - @assert isempty(workspace.stack); @assert isempty(workspace.visiting); - @assert isempty(workspace.initial_states); @assert isempty(workspace.work_states); @assert isempty(workspace.result_states) + @assert isempty(workspace.stack) "workspace corrupted" + @assert isempty(workspace.visiting) "workspace corrupted" + @assert isempty(workspace.initial_states) "workspace corrupted" + @assert isempty(workspace.work_states) "workspace corrupted" + @assert isempty(workspace.result_states) "workspace corrupted" nothing end @@ -132,26 +144,33 @@ function gen_staged_sig(def::Method, mi::MethodInstance) end function needs_instrumentation(codeinst::CodeInstance, mi::MethodInstance, def::Method, validation_world::UInt) + # foreign CIs (owner !== nothing) aren't run as native code here, so instrumenting them is moot + codeinst.owner === nothing || return false if JLOptions().code_coverage != 0 || JLOptions().malloc_log != 0 # test if the code needs to run with instrumentation, in which case we cannot use existing generated code if isdefined(def, :debuginfo) ? # generated_only functions do not have debuginfo, so fall back to considering their codeinst debuginfo though this may be slower and less reliable - Compiler.should_instrument(def.module, def.debuginfo) : - isdefined(codeinst, :debuginfo) && Compiler.should_instrument(def.module, codeinst.debuginfo) + should_instrument(def.module, def.debuginfo) : + isdefined(codeinst, :debuginfo) && should_instrument(def.module, codeinst.debuginfo) + # Compatible image code already has the requested counters. + # Allocation tracking still needs fresh instrumentation. + if JLOptions().malloc_log == 0 && ccall(:jl_codeinst_coverage_compatible, Cint, (Any,), codeinst) != 0 + return false + end return true end gensig = gen_staged_sig(def, mi) if gensig !== nothing # if this is defined by a generator, try to consider forcing re-running the generators too, to add coverage for them - minworld = Ref{UInt}(1) - maxworld = Ref{UInt}(typemax(UInt)) - has_ambig = Ref{Int32}(0) + minworld = RefValue{UInt}(1) + maxworld = RefValue{UInt}(typemax(UInt)) + has_ambig = RefValue{Int32}(0) result = _methods_by_ftype(gensig, nothing, -1, validation_world, #=ambig=#false, minworld, maxworld, has_ambig) if result !== nothing for k = 1:length(result) match = result[k]::Core.MethodMatch genmethod = match.method # no, I refuse to refuse to recurse into your cursed generated function generators and will only test one level deep here - if isdefined(genmethod, :debuginfo) && Compiler.should_instrument(genmethod.module, genmethod.debuginfo) + if isdefined(genmethod, :debuginfo) && should_instrument(genmethod.module, genmethod.debuginfo) return true end end @@ -162,7 +181,7 @@ function needs_instrumentation(codeinst::CodeInstance, mi::MethodInstance, def:: end # Test all edges relevant to a method: -# - Visit the entire call graph, starting from edges[idx] to determine if that method is valid +# - Visit the entire call graph, starting from `codeinst` to determine if that method is valid # - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable # and slightly modified with an early termination option once the computation reaches its minimum function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace::VerifyMethodWorkspace) @@ -194,10 +213,8 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace continue end - minworld, maxworld = Base.get_require_world(), validation_world - if haskey(workspace.visiting, initial.codeinst) - workspace.result_states[current_depth] = VerifyMethodResultState(workspace.visiting[initial.codeinst], minworld, maxworld) + workspace.result_states[current_depth] = VerifyMethodResultState(workspace.visiting[initial.codeinst], UInt(1), validation_world) workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) continue end @@ -206,10 +223,13 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace depth = length(workspace.stack) workspace.visiting[initial.codeinst] = depth + # unable to backdate before require_world, since Bindings are not able to track that information + minworld, maxworld = get_require_world(), validation_world + # Check for invalidation of GlobalRef edges if (initial.def.did_scan_source & 0x1) == 0x0 backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) - Base.scan_new_method!(initial.def, backedges_only) + scan_new_method!(initial.def, validation_world, backedges_only) end if (initial.def.did_scan_source & 0x4) != 0x0 maxworld = 0 @@ -220,13 +240,13 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace end # Process all non-CodeInstance edges - if !isempty(initial.callees) && maxworld != Base.get_require_world() + if !isempty(initial.callees) && maxworld != get_require_world() matches = [] j = 1 while j <= length(initial.callees) local min_valid2::UInt, max_valid2::UInt edge = initial.callees[j] - @assert !(edge isa Method) + @assert !(edge isa Method) "unexpected Method edge indicates corrupt edges list creation" if edge isa CodeInstance # Convert CodeInstance to MethodInstance for validation (like original) @@ -246,16 +266,16 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace edge = sig elseif edge isa Core.Binding j += 1 - min_valid2 = minworld - max_valid2 = maxworld - if !Base.binding_was_invalidated(edge) - if isdefined(edge, :partitions) - min_valid2 = edge.partitions.min_world - max_valid2 = edge.partitions.max_world - end - else + # Check that what of and how this code accessed the leaf partition is still valid. + wr = binding_access_range_at(edge, validation_world) + if min_world(wr) > get_require_world() + # Nothing can be backdated before the require world, so an access + # whose range does not reach it cannot be shown valid at all. min_valid2 = 1 max_valid2 = 0 + else + min_valid2 = min_world(wr) + max_valid2 = max_world(wr) end else callee = initial.callees[j+1] @@ -330,12 +350,16 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace child = pop!(workspace.stack) if result.result_maxworld ≠ 0 @atomic :monotonic child.min_world = result.result_minworld + # Finally, if this CI is still valid in some world age and marked as valid in the native cache, poke it in that mi's cache now + if child.flags & CI_FLAGS_NATIVE_CACHE_VALID == CI_FLAGS_NATIVE_CACHE_VALID + @ccall jl_mi_cache_insert(get_ci_mi(child)::Any, child::Any)::Cvoid + end end @atomic :monotonic child.max_world = result.result_maxworld - if result.result_maxworld == validation_world && validation_world == get_world_counter() - Compiler.store_backedges(child, child.edges) + if result.result_maxworld == validation_world && validation_world == get_world_counter() && isdefined(child, :edges) + store_backedges(child, child.edges) end - @assert workspace.visiting[child] == length(workspace.stack) + 1 + @assert workspace.visiting[child] == length(workspace.stack) + 1 "internal error maintaining workspace" delete!(workspace.visiting, child) invalidations = _jl_debug_method_invalidation[] if invalidations !== nothing && result.result_maxworld < validation_world @@ -423,13 +447,8 @@ function method_morespecific_via_interferences(method1::Method, method2::Method) if method1 === method2 return false end - ms = method_in_interferences_recursive(method1, method2, IdSet{Method}()) - # slow check: @assert ms === morespecific(method1, method2) || typeintersect(method1.sig, method2.sig) === Union{} || typeintersect(method2.sig, method1.sig) === Union{} - return ms -end -# Returns true if method1 is in method2's interferences (meaning !morespecific(method2, method1)) -function method_in_interferences_recursive(method1::Method, method2::Method, visited::IdSet{Method}) + # Check direct interferences first if method_in_interferences(method2, method1) return false end @@ -437,24 +456,46 @@ function method_in_interferences_recursive(method1::Method, method2::Method, vis return true end - # Recursively check through interference graph - method2 in visited && return false + visited = Method[] push!(visited, method2) - interferences = method2.interferences - for k = 1:length(interferences) - isassigned(interferences, k) || break - method3 = interferences[k]::Method - if method_in_interferences(method2, method3) - continue # only follow edges to morespecific methods in search of the morespecific target (skip ambiguities) - end - if method_in_interferences_recursive(method1, method3, visited) - return true # found method1 in the interference graph + + workqueue = Method[method2] + while !isempty(workqueue) + current = pop!(workqueue) + interferences = current.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break + method3 = interferences[k]::Method + + # Check if we're already visiting this interference method (cycle prevention and memoization) + method3 in visited && continue + push!(visited, method3) + + if method_in_interferences(current, method3) + continue # only follow edges to morespecific methods in search of the morespecific target (skip ambiguities) + end + + # Check direct interferences for this interference method + if method_in_interferences(method3, method1) + continue # return false for this path + end + if method_in_interferences(method1, method3) + return true # found method1 in the interference graph + end + + push!(workqueue, method3) end end + # slow check: @assert ms === morespecific(method1, method2) || typeintersect(method1.sig, method2.sig) === Union{} || typeintersect(method2.sig, method1.sig) === Union{} return false end +# Max interference-set size for which n==1 uses the interference fast path instead of +# `ml_matches`: the scan probes every member with `typeintersect`, so above this size the +# pruned `ml_matches` lookup is cheaper (~8 is the empirical crossover). +const VERIFY_INTERF_CAP = 8 + function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt, fully_covers::Bool, matches::Vector{Any}) # verify that these edges intersect with the same methods as before mi = nothing @@ -471,36 +512,48 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n if _jl_debug_method_invalidation[] === nothing && world == get_world_counter() return UInt(1), UInt(0) end - elseif n == 1 - # first, fast-path a check if the expected method simply dominates its sig anyways - # so the result of ml_matches is already simply known - let t = expecteds[i], meth, minworld, maxworld - meth = get_method_from_edge(t) - if !(t isa Method) - if t isa CodeInstance - mi = get_ci_mi(t)::MethodInstance - else - mi = t::MethodInstance + else # n >= 1 + if n == 1 + # first, fast-path a check if the expected method simply dominates its sig anyways + # so the result of ml_matches is already simply known + let t = expecteds[i], meth, minworld, maxworld + meth = get_method_from_edge(t) + if !(t isa Method) + if t isa CodeInstance + mi = get_ci_mi(t)::MethodInstance + else + mi = t::MethodInstance + end + # Fast path is legal when fully_covers=true + if fully_covers && !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world "expected method not present in verification world" + maxworld = typemax(UInt) + return minworld, maxworld + end end # Fast path is legal when fully_covers=true - if fully_covers && !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) + if fully_covers && !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) minworld = meth.primary_world - @assert minworld ≤ world + @assert minworld ≤ world "expected method not present in verification world" maxworld = typemax(UInt) return minworld, maxworld end end - # Fast path is legal when fully_covers=true - if fully_covers && !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) - minworld = meth.primary_world - @assert minworld ≤ world - maxworld = typemax(UInt) - return minworld, maxworld - end end - elseif n > 1 - # Try the interference set fast path: check if all interference sets are covered by expecteds + # Try the interference set fast path (used by both n==1, when the O(1) checks above + # did not resolve it, and n>1): the result is unchanged as long as no interfering + # method intersects sig outside of what the expected method(s) cover. interference_fast_path_success = fully_covers + if interference_fast_path_success && n == 1 + # Skip to ml_matches for large interference sets (see VERIFY_INTERF_CAP). The set + # is packed, so isassigned(., cap+1) tests "size > cap" without any typeintersect. + let interf = get_method_from_edge(expecteds[i]).interferences, cap = VERIFY_INTERF_CAP + if length(interf) > cap && isassigned(interf, cap + 1) + interference_fast_path_success = false + end + end + end # If it didn't fail yet, then check that all interference methods are either expected, or not applicable. if interference_fast_path_success local interference_minworld::UInt = 1 @@ -551,7 +604,7 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n end if interference_fast_path_success # All interference sets are covered by expecteds, can return success - @assert interference_minworld ≤ world + @assert interference_minworld ≤ world "expected method not present in verification world" maxworld = typemax(UInt) return interference_minworld, maxworld end @@ -559,9 +612,9 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n end # next, compare the current result of ml_matches to the old result lim = _jl_debug_method_invalidation[] !== nothing ? Int(typemax(Int32)) : n - minworld = Ref{UInt}(1) - maxworld = Ref{UInt}(typemax(UInt)) - has_ambig = Ref{Int32}(0) + minworld = RefValue{UInt}(1) + maxworld = RefValue{UInt}(typemax(UInt)) + has_ambig = RefValue{Int32}(0) result = _methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig) if result === nothing empty!(matches) @@ -612,13 +665,13 @@ const METHOD_SIG_LATEST_WHICH = 0x1 const METHOD_SIG_LATEST_ONLY = 0x2 function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt, matches::Vector{Any}) - @assert invokesig isa Type + @assert invokesig isa Type "corrupt edges list" local minworld::UInt, maxworld::UInt empty!(matches) if invokesig === expected.sig && !iszero(expected.dispatch_status & METHOD_SIG_LATEST_WHICH) # the invoke match is `expected` for `expected->sig`, unless `expected` is replaced minworld = expected.primary_world - @assert minworld ≤ world + @assert minworld ≤ world "expected method not present in verification world" maxworld = typemax(UInt) else mt = get_methodtable(expected) @@ -626,7 +679,7 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn minworld = 1 maxworld = 0 else - matched, valid_worlds = Compiler._findsup(invokesig, mt, world) + matched, valid_worlds = _findsup(invokesig, mt, world) minworld, maxworld = valid_worlds.min_world, valid_worlds.max_world if matched === nothing maxworld = 0 @@ -641,3 +694,9 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn end return minworld, maxworld end + +# Wrapper to call insert_backedges in typeinf_world for external calls +function insert_backedges_typeinf(internal_methods::Vector{Any}) + args = Any[insert_backedges, internal_methods] + return ccall(:jl_call_in_typeinf_world, Any, (Ptr{Any}, Cint), args, length(args)) +end diff --git a/Compiler/src/sort.jl b/Compiler/src/sort.jl index 6c8571f6198e6..9d2edee188876 100644 --- a/Compiler/src/sort.jl +++ b/Compiler/src/sort.jl @@ -3,7 +3,7 @@ # reference on sorted binary search: # https://www.tbray.org/ongoing/When/200x/2003/03/22/Binary -# index of the first value of vector a that is greater than or equal to x; +# index of the first value of vector v that is greater than or equal to x; # returns lastindex(v)+1 if x is greater than all values in v. function searchsortedfirst(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer hi = hi + T(1) @@ -22,7 +22,7 @@ function searchsortedfirst(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::key return lo end -# index of the last value of vector a that is less than or equal to x; +# index of the last value of vector v that is less than or equal to x; # returns firstindex(v)-1 if x is less than all values of v. function searchsortedlast(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer u = T(1) diff --git a/Compiler/src/ssair/EscapeAnalysis.jl b/Compiler/src/ssair/EscapeAnalysis.jl index 4ce972937700c..c8ad50569d1e1 100644 --- a/Compiler/src/ssair/EscapeAnalysis.jl +++ b/Compiler/src/ssair/EscapeAnalysis.jl @@ -13,7 +13,7 @@ export using Base: Base # imports -import Base: ==, copy, getindex, setindex! +import Base: ==, !=, copy, getindex, hash, setindex! # usings using Core using Core: Builtin, IntrinsicFunction, SimpleVector, ifelse, sizeof @@ -23,12 +23,12 @@ using Base: # Base definitions @nospecialize, @specialize, BitSet, IdDict, IdSet, UnitRange, Vector, delete!, empty!, enumerate, first, get, get!, hasintersect, haskey, isassigned, isempty, length, max, min, missing, println, push!, pushfirst!, - !, !==, &, *, +, -, :, <, <<, >, |, ∈, ∉, ∩, ∪, ≠, ≤, ≥, ⊆ + !, !==, %, &, *, +, -, :, <, <<, >, |, ⊻, ∈, ∉, ∩, ∪, ≠, ≤, ≥, ⊆ using ..Compiler: # Compiler specific definitions AbstractLattice, Compiler, IRCode, IR_FLAG_NOTHROW, - argextype, fieldcount_noerror, has_flag, intrinsic_nothrow, is_meta_expr_head, - is_identity_free_argtype, isexpr, setfield!_nothrow, singleton_type, try_compute_field, - try_compute_fieldidx, widenconst + argextype, argextype_widened, fieldcount_noerror, has_flag, intrinsic_nothrow, + is_meta_expr_head, is_identity_free_argtype, isexpr, setfield!_nothrow, singleton_type, + try_compute_field, try_compute_fieldidx function include(x::String) if !isdefined(Base, :end_base_include) @@ -63,7 +63,7 @@ A lattice for escape information, which holds the following properties: * `x.AliasInfo::Unindexable` records all the possible values that can be aliased to fields/elements of `x` without precise index information - `x.Liveness::BitSet`: records SSA statement numbers where `x` should be live, e.g. to be used as a call argument, to be returned to a caller, or preserved for `:foreigncall`: - * `isempty(x.Liveness)`: `x` is never be used in this call frame (the bottom) + * `isempty(x.Liveness)`: `x` is never used in this call frame (the bottom) * `0 ∈ x.Liveness` also has the special meaning that it's a call argument of the currently analyzed call frame (and thus it's visible from the caller immediately). * `pc ∈ x.Liveness`: `x` may be used at the SSA statement at `pc` @@ -186,7 +186,7 @@ end # we need to make sure this `==` operator corresponds to lattice equality rather than object equality, # otherwise `propagate_changes` can't detect the convergence x::EscapeInfo == y::EscapeInfo = begin - # fast pass: better to avoid top comparison + # fast path: better to avoid top comparison x === y && return true x.Analyzed === y.Analyzed || return false x.ReturnEscape === y.ReturnEscape || return false @@ -220,13 +220,31 @@ x::EscapeInfo == y::EscapeInfo = begin return true end +const hashei_seed = 0x2fe2deaeb40ac445 % UInt +function hash(x::EscapeInfo, h::UInt) + h ⊻= hashei_seed + h = hash(x.Analyzed, h) + h = hash(x.ReturnEscape, h) + h = hash(x.ThrownEscape, h) + xa = x.AliasInfo + if isa(xa, Bool) + h = hash(xa, h) + elseif isa(xa, IndexableFields) + h = hash(xa.infos, h) + else + h = hash((xa::Unindexable).info, h) + end + h = hash(x.Liveness, h) + return h +end + """ x::EscapeInfo ⊑ₑ y::EscapeInfo -> Bool The non-strict partial order over [`EscapeInfo`](@ref). """ x::EscapeInfo ⊑ₑ y::EscapeInfo = begin - # fast pass: better to avoid top comparison + # fast path: better to avoid top comparison if y === ⊤ return true elseif x === ⊤ @@ -285,7 +303,7 @@ end x::EscapeInfo ⊏ₑ y::EscapeInfo -> Bool The strict partial order over [`EscapeInfo`](@ref). -This is defined as the irreflexive kernel of `⊏ₑ`. +This is defined as the irreflexive kernel of `⊑ₑ`. """ x::EscapeInfo ⊏ₑ y::EscapeInfo = x ⊑ₑ y && !(y ⊑ₑ x) @@ -303,7 +321,7 @@ x::EscapeInfo ⋤ₑ y::EscapeInfo = !(y ⊑ₑ x) Computes the join of `x` and `y` in the partial order defined by [`EscapeInfo`](@ref). """ x::EscapeInfo ⊔ₑ y::EscapeInfo = begin - # fast pass: better to avoid top join + # fast path: better to avoid top join if x === ⊤ || y === ⊤ return ⊤ elseif x === ⊥ @@ -415,7 +433,7 @@ end """ iridx(x, estate::EscapeState) -> xidx::Union{Int,Nothing} -Tries to convert analyzable IR element `x::Union{Argument,SSAValue}` to +Try to convert analyzable IR element `x::Union{Argument,SSAValue}` to its unique identifier number `xidx` that is valid in the analysis context of `estate`. Returns `nothing` if `x` isn't maintained by `estate` and thus unanalyzable (e.g. `x::GlobalRef`). @@ -584,6 +602,8 @@ function analyze_escapes(ir::IRCode, nargs::Int, 𝕃ₒ::AbstractLattice, get_e escape_new!(astate, pc, stmt.args) elseif head === :foreigncall escape_foreigncall!(astate, pc, stmt.args) + elseif head === :foreignglobal + escape_foreignglobal!(astate, pc, stmt.args) elseif head === :throw_undef_if_not # XXX when is this expression inserted ? add_escape_change!(astate, stmt.args[1], ThrownEscape(pc)) elseif is_meta_expr_head(head) @@ -620,7 +640,7 @@ function analyze_escapes(ir::IRCode, nargs::Int, 𝕃ₒ::AbstractLattice, get_e escape_edges!(astate, pc, stmt.values) elseif isa(stmt, UpsilonNode) escape_val_ifdefined!(astate, pc, stmt) - elseif isa(stmt, GlobalRef) # global load + elseif isa(stmt, GlobalRef) || isa(stmt, Core.BindingPartition) # global load add_escape_change!(astate, SSAValue(pc), ⊤) elseif isa(stmt, SSAValue) escape_val!(astate, pc, stmt) @@ -778,9 +798,9 @@ function add_liveness_change!(astate::AnalysisState, @nospecialize(x), livepc::I end function add_alias_change!(astate::AnalysisState, @nospecialize(x), @nospecialize(y)) - if isa(x, GlobalRef) + if isa(x, GlobalRef) || isa(x, Core.BindingPartition) return add_escape_change!(astate, y, ⊤) - elseif isa(y, GlobalRef) + elseif isa(y, GlobalRef) || isa(y, Core.BindingPartition) return add_escape_change!(astate, x, ⊤) end estate = astate.estate @@ -884,7 +904,7 @@ is_nothrow(ir::IRCode, pc::Int) = has_flag(ir[SSAValue(pc)], IR_FLAG_NOTHROW) Propagates escapes via exceptions that can happen in `tryregions`. Naively it seems enough to propagate escape information imposed on `:the_exception` object, -but actually there are several other ways to access to the exception object such as +but actually there are several other ways to access the exception object such as `Base.current_exceptions` and manual catch of `rethrow`n object. For example, escape analysis needs to account for potential escape of the allocated object via `rethrow_escape!()` call in the example below: @@ -930,7 +950,7 @@ function escape_exception!(astate::AnalysisState, tryregions::Vector{UnitRange{I for i in 1:length(escapes) x = escapes[i] xt = x.ThrownEscape - xt === TOP_THROWN_ESCAPE && @goto propagate_exception_escape # fast pass + xt === TOP_THROWN_ESCAPE && @goto propagate_exception_escape # fast path for pc in xt for region in tryregions pc ∈ region && @goto propagate_exception_escape # early break because of AllEscape @@ -963,8 +983,8 @@ function escape_invoke!(astate::AnalysisState, pc::Int, args::Vector{Any}) # to consider the possibility of aliasing between them and the return value. for argidx = first_idx:last_idx arg = args[argidx] - if arg isa GlobalRef - continue # :effect_free guarantees that nothings escapes to the global scope + if arg isa GlobalRef || arg isa Core.BindingPartition + continue # :effect_free guarantees that nothing escapes to the global scope end if !is_identity_free_argtype(argextype(arg, astate.ir)) add_alias_change!(astate, ret, arg) @@ -1022,7 +1042,24 @@ function from_interprocedural(argescape::ArgEscapeInfo, pc::Int) return EscapeInfo(#=Analyzed=#true, #=ReturnEscape=#false, ThrownEscape, AliasInfo, Liveness) end -# escape every argument `(args[6:length(args[3])])` and the name `args[1]` +# the only possible 'escape' here is really just that it can return a +# bitcast of its one (pointer) argument, but be conservative anyway +function escape_foreignglobal!(astate::AnalysisState, pc::Int, args::Vector{Any}) + nargs = length(args) + if nargs != 1 + # invalid foreignglobal, no escape + return + end + name = args[1] + nothrow = is_nothrow(astate.ir, pc) + name_info = nothrow ? ⊥ : ThrownEscape(pc) + if !isexpr(name, :tuple) + add_escape_change!(astate, name, name_info) + add_liveness_change!(astate, name, pc) + end +end + +# escape every argument `(args[6:5+length(args[3])])` and the name `args[1]` # TODO: we can apply a similar strategy like builtin calls to specialize some foreigncalls function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) nargs = length(args) @@ -1037,8 +1074,10 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) # NOTE array allocations might have been proven as nothrow (https://github.com/JuliaLang/julia/pull/43565) nothrow = is_nothrow(astate.ir, pc) name_info = nothrow ? ⊥ : ThrownEscape(pc) - add_escape_change!(astate, name, name_info) - add_liveness_change!(astate, name, pc) + if !isexpr(name, :tuple) + add_escape_change!(astate, name, name_info) + add_liveness_change!(astate, name, pc) + end for i = 1:nargs # we should escape this argument if it is directly called, # otherwise just impose ThrownEscape if not nothrow @@ -1143,7 +1182,7 @@ function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) if isa(AliasInfo, Bool) AliasInfo && @goto conservative_propagation # AliasInfo of this object hasn't been analyzed yet: set AliasInfo now - typ = widenconst(argextype(obj, astate.ir)) + typ = argextype_widened(obj, astate.ir) nflds = fieldcount_noerror(typ) if nflds === nothing AliasInfo = Unindexable() @@ -1172,7 +1211,7 @@ function escape_new!(astate::AnalysisState, pc::Int, args::Vector{Any}) elseif isa(AliasInfo, Unindexable) AliasInfo = copy(AliasInfo) @label escape_unindexable_def - # fields are known partially: propagate escape information imposed on recorded possibilities to all fields values + # fields are known partially: propagate escape information imposed on recorded possibilities to all field values info = AliasInfo.info objinfo′ = ignore_aliasinfo(objinfo) for i in 2:nargs @@ -1253,7 +1292,7 @@ function escape_builtin!(::typeof(getfield), astate::AnalysisState, pc::Int, arg length(args) ≥ 3 || return false ir, estate = astate.ir, astate.estate obj = args[2] - typ = widenconst(argextype(obj, ir)) + typ = argextype_widened(obj, ir) if hasintersect(typ, Module) # global load add_escape_change!(astate, SSAValue(pc), ⊤) end @@ -1305,15 +1344,18 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar if isa(obj, SSAValue) || isa(obj, Argument) objinfo = estate[obj] else - # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively - add_escape_change!(astate, val, ⊤) + # unanalyzable object (e.g. obj::GlobalRef): + # escape field value conservatively. Force the change so it applies even to an + # identity-free value (e.g. a `String` stored into a global): the value genuinely + # escapes to global scope, matching the aliasing path taken when `obj` is an SSA value. + add_escape_change!(astate, val, ⊤, #=force=#true) @goto add_thrown_escapes end AliasInfo = objinfo.AliasInfo if isa(AliasInfo, Bool) AliasInfo && @goto conservative_propagation # AliasInfo of this object hasn't been analyzed yet: set AliasInfo now - typ = widenconst(argextype(obj, ir)) + typ = argextype_widened(obj, ir) AliasInfo, fidx = analyze_fields(ir, typ, args[3]) if isa(AliasInfo, IndexableFields) @goto escape_indexable_def @@ -1321,7 +1363,7 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar @goto escape_unindexable_def end elseif isa(AliasInfo, IndexableFields) - typ = widenconst(argextype(obj, ir)) + typ = argextype_widened(obj, ir) AliasInfo, fidx = reanalyze_fields(AliasInfo, ir, typ, args[3]) isa(AliasInfo, Unindexable) && @goto escape_unindexable_def @label escape_indexable_def diff --git a/Compiler/src/ssair/disjoint_set.jl b/Compiler/src/ssair/disjoint_set.jl index e000d7e8a582f..041539564f3f2 100644 --- a/Compiler/src/ssair/disjoint_set.jl +++ b/Compiler/src/ssair/disjoint_set.jl @@ -14,7 +14,7 @@ using Base: OneTo, collect, zero, zeros, one, typemax # A forest of disjoint sets of integers # # Since each element is an integer, we can use arrays -# instead of dictionary (for efficiency) +# instead of a dictionary (for efficiency) # # Disjoint sets over other key types can be implemented # based on an IntDisjointSet through a map from the key @@ -46,7 +46,7 @@ length(s::IntDisjointSet) = length(s.parents) """ num_groups(s::IntDisjointSet) -Get a number of groups. +Get the number of groups. """ num_groups(s::IntDisjointSet) = s.ngroups eltype(::Type{IntDisjointSet{T}}) where {T<:Integer} = T @@ -73,7 +73,7 @@ end """ find_root!(s::IntDisjointSet{T}, x::T) -Find the root element of the subset that contains an member `x`. +Find the root element of the subset that contains a member `x`. Path compression happens here. """ find_root!(s::IntDisjointSet{T}, x::T) where {T<:Integer} = find_root_impl!(s.parents, x) diff --git a/Compiler/src/ssair/domtree.jl b/Compiler/src/ssair/domtree.jl index 38a07da5fb075..2691fe2b0f41f 100644 --- a/Compiler/src/ssair/domtree.jl +++ b/Compiler/src/ssair/domtree.jl @@ -2,7 +2,7 @@ # This file implements the Semi-NCA (SNCA) dominator tree construction # described in Georgiadis' PhD thesis [LG05], which itself is a simplification -# of the Simple Lenguare-Tarjan (SLT) algorithm [LG79]. This algorithm matches +# of the Simple Lengauer-Tarjan (SLT) algorithm [LG79]. This algorithm matches # the algorithm choice in LLVM and seems to be a sweet spot in implementation # simplicity and efficiency. # @@ -238,16 +238,35 @@ function (T::Type{<:GenericDomTree})() return T(DFSTree(0), SNCAData[], BBNumber[], DomTreeNode[]) end -function construct_domtree(blocks::Vector{BasicBlock}) - return update_domtree!(blocks, DomTree(), true, 0) +""" +Reusable scratch buffers for (re)computing a dominator tree. Kept separate from +the `GenericDomTree` so that the working storage can be reused across successive +dynamic updates (by threading the same cache through), and dropped entirely once +the tree is finalized. A fresh cache is used by default. + +The buffers are reset on each use, so a cache must not be shared between +concurrent computations. `worklist` is shared between `SNCA!` and +`compute_domtree_nodes!`, which run sequentially (both `PreNumber` and +`BBNumber` are `Int`). +""" +struct DomTreeCache + ancestors::Vector{PreNumber} + idoms_pre::Vector{PreNumber} + worklist::Vector{Tuple{Int, Int}} +end +DomTreeCache() = DomTreeCache(PreNumber[], PreNumber[], Tuple{Int, Int}[]) + +function construct_domtree(blocks::Vector{BasicBlock}; cache::DomTreeCache=DomTreeCache()) + return update_domtree!(blocks, DomTree(), true, 0; cache) end -function construct_postdomtree(blocks::Vector{BasicBlock}) - return update_domtree!(blocks, PostDomTree(), true, 0) +function construct_postdomtree(blocks::Vector{BasicBlock}; cache::DomTreeCache=DomTreeCache()) + return update_domtree!(blocks, PostDomTree(), true, 0; cache) end function update_domtree!(blocks::Vector{BasicBlock}, domtree::GenericDomTree{IsPostDom}, - recompute_dfs::Bool, max_pre::PreNumber) where {IsPostDom} + recompute_dfs::Bool, max_pre::PreNumber; + cache::DomTreeCache=DomTreeCache()) where {IsPostDom} if recompute_dfs DFS!(domtree.dfs_tree, blocks, IsPostDom) end @@ -256,34 +275,50 @@ function update_domtree!(blocks::Vector{BasicBlock}, domtree::GenericDomTree{IsP max_pre = length(domtree.dfs_tree) end - SNCA!(domtree, blocks, max_pre) - compute_domtree_nodes!(domtree) + SNCA!(domtree, blocks, max_pre, cache) + compute_domtree_nodes!(domtree; cache) return domtree end -function compute_domtree_nodes!(domtree::GenericDomTree{IsPostDom}) where {IsPostDom} - # Compute children - copy!(domtree.nodes, - DomTreeNode[DomTreeNode() for _ in 1:length(domtree.idoms_bb)]) +function compute_domtree_nodes!(domtree::GenericDomTree{IsPostDom}; + cache::DomTreeCache=DomTreeCache()) where {IsPostDom} + # Compute children. Reuse any existing `children` vectors (e.g. when + # recomputing the domtree during a dynamic update) to avoid reallocating + # them, and reset the levels to the default of 1. + nodes = domtree.nodes + new_len = length(domtree.idoms_bb) + old_len = length(nodes) + for i in 1:min(old_len, new_len) + children = nodes[i].children + empty!(children) + nodes[i] = DomTreeNode(1, children) + end + resize!(nodes, new_len) + for i in (old_len+1):new_len + nodes[i] = DomTreeNode() + end for (idx, idom) in Iterators.enumerate(domtree.idoms_bb) ((!IsPostDom && idx == 1) || idom == 0) && continue - push!(domtree.nodes[idom].children, idx) + push!(nodes[idom].children, idx) end # n.b. now issorted(domtree.nodes[*].children) since idx is sorted above # Recursively set level + worklist = cache.worklist if IsPostDom for (node, idom) in enumerate(domtree.idoms_bb) idom == 0 || continue - update_level!(domtree.nodes, node, 1) + update_level!(domtree.nodes, node, 1, worklist) end else - update_level!(domtree.nodes, 1, 1) + update_level!(domtree.nodes, 1, 1, worklist) end return domtree.nodes end -function update_level!(nodes::Vector{DomTreeNode}, node::BBNumber, level::Int) - worklist = Tuple{BBNumber, Int}[(node, level)] +function update_level!(nodes::Vector{DomTreeNode}, node::BBNumber, level::Int, + worklist::Vector{Tuple{BBNumber, Int}}) + empty!(worklist) + push!(worklist, (node, level)) while !isempty(worklist) (node, level) = pop!(worklist) nodes[node] = DomTreeNode(level, nodes[node].children) @@ -304,7 +339,7 @@ pseudocode in [LG05] is not entirely accurate. The best way to understand what's happening is to read [LT79], then the description of SLT in [LG05] (warning: inconsistent notation), then the description of Semi-NCA. """ -function SNCA!(domtree::GenericDomTree{IsPostDom}, blocks::Vector{BasicBlock}, max_pre::PreNumber) where {IsPostDom} +function SNCA!(domtree::GenericDomTree{IsPostDom}, blocks::Vector{BasicBlock}, max_pre::PreNumber, cache::DomTreeCache) where {IsPostDom} D = domtree.dfs_tree state = domtree.snca_state # There may be more blocks than are reachable in the DFS / dominator tree @@ -342,7 +377,8 @@ function SNCA!(domtree::GenericDomTree{IsPostDom}, blocks::Vector{BasicBlock}, m # Calculate semidominators, but only for blocks with preorder number up to # max_pre - ancestors = copy(D.to_parent_pre) + ancestors = cache.ancestors + copy!(ancestors, D.to_parent_pre) relevant_blocks = IsPostDom ? (1:max_pre) : (2:max_pre) for w::PreNumber in reverse(relevant_blocks) semi_w = ancestors[w] @@ -368,7 +404,8 @@ function SNCA!(domtree::GenericDomTree{IsPostDom}, blocks::Vector{BasicBlock}, m if length(ancestors) <= 32 snca_compress!(state, ancestors, v_pre, last_linked) else - snca_compress_worklist!(state, ancestors, v_pre, last_linked) + snca_compress_worklist!(state, ancestors, v_pre, last_linked, + cache.worklist) end end @@ -382,7 +419,8 @@ function SNCA!(domtree::GenericDomTree{IsPostDom}, blocks::Vector{BasicBlock}, m # Compute immediate dominators, which for a node must be the nearest common # ancestor in the (immediate) dominator tree between its semidominator and # its parent (see Lemma 2.6 in [LG05]). - idoms_pre = copy(D.to_parent_pre) + idoms_pre = cache.idoms_pre + copy!(idoms_pre, D.to_parent_pre) for v in (IsPostDom ? (1:n_nodes) : (2:n_nodes)) idom = idoms_pre[v] vsemi = state[v].semi @@ -425,11 +463,13 @@ end function snca_compress_worklist!( state::Vector{SNCAData}, ancestors::Vector{PreNumber}, - v::PreNumber, last_linked::PreNumber) + v::PreNumber, last_linked::PreNumber, + worklist::Vector{Tuple{PreNumber, PreNumber}}) # TODO: There is a smarter way to do this u = ancestors[v] - worklist = Tuple{PreNumber, PreNumber}[(u,v)] @assert u < v + empty!(worklist) + push!(worklist, (u, v)) while !isempty(worklist) u, v = last(worklist) if u >= last_linked @@ -448,7 +488,7 @@ end "Given updated blocks, update the given dominator tree with an inserted edge." function domtree_insert_edge!(domtree::DomTree, blocks::Vector{BasicBlock}, - from::BBNumber, to::BBNumber) + from::BBNumber, to::BBNumber; cache::DomTreeCache=DomTreeCache()) # `from` is unreachable, so `from` and `to` aren't in domtree if bb_unreachable(domtree, from) return domtree @@ -463,10 +503,10 @@ function domtree_insert_edge!(domtree::DomTree, blocks::Vector{BasicBlock}, if to_pre == 0 || (from_pre < to_pre && from_post < to_post) # The DFS tree is invalidated by the edge insertion, so run from # scratch - update_domtree!(blocks, domtree, true, 0) + update_domtree!(blocks, domtree, true, 0; cache) else # DFS tree is still valid, so update only affected nodes - update_domtree!(blocks, domtree, false, to_pre) + update_domtree!(blocks, domtree, false, to_pre; cache) end return domtree @@ -474,7 +514,7 @@ end "Given updated blocks, update the given dominator tree with a deleted edge." function domtree_delete_edge!(domtree::DomTree, blocks::Vector{BasicBlock}, - from::BBNumber, to::BBNumber) + from::BBNumber, to::BBNumber; cache::DomTreeCache=DomTreeCache()) # `from` is unreachable, so `from` and `to` aren't in domtree if bb_unreachable(domtree, from) return domtree @@ -484,7 +524,7 @@ function domtree_delete_edge!(domtree::DomTree, blocks::Vector{BasicBlock}, if is_parent(domtree.dfs_tree, from, to) # The `from` block is the parent of the `to` block in the DFS tree, so # deleting the edge invalidates the DFS tree, so start from scratch - update_domtree!(blocks, domtree, true, 0) + update_domtree!(blocks, domtree, true, 0; cache) elseif on_semidominator_path(domtree, from, to) # Recompute semidominators for blocks with preorder number up to that # of `to` block. Semidominators for blocks with preorder number greater @@ -493,7 +533,7 @@ function domtree_delete_edge!(domtree::DomTree, blocks::Vector{BasicBlock}, # `to` would be lower than those of these blocks, and `to` is not their # parent in the DFS tree). to_pre = domtree.dfs_tree.to_pre[to] - update_domtree!(blocks, domtree, false, to_pre) + update_domtree!(blocks, domtree, false, to_pre; cache) end # Otherwise, dominator tree is not affected @@ -597,7 +637,7 @@ end """ dominates(domtree::DomTree, bb1::Int, bb2::Int)::Bool -Checks if `bb1` dominates `bb2`. +Check if `bb1` dominates `bb2`. `bb1` and `bb2` are indexes into the `CFG` blocks. `bb1` dominates `bb2` if the only way to enter `bb2` is via `bb1`. (Other blocks may be in between, e.g `bb1->bbx->bb2`). @@ -608,7 +648,7 @@ dominates(domtree::DomTree, bb1::BBNumber, bb2::BBNumber) = """ postdominates(domtree::PostDomTree, bb1::Int, bb2::Int)::Bool -Checks if `bb1` post-dominates `bb2`. +Check if `bb1` post-dominates `bb2`. `bb1` and `bb2` are indexes into the `CFG` blocks. `bb1` post-dominates `bb2` if every pass from `bb2` to the exit is via `bb1`. (Other blocks may be in between, e.g `bb2->bbx->bb1->exit`). @@ -629,7 +669,7 @@ end bb_unreachable(domtree::DomTree, bb::BBNumber) = bb != 1 && domtree.dfs_tree.to_pre[bb] == 0 -"Iterable data structure that walks though all dominated blocks" +"Iterable data structure that walks through all dominated blocks" struct DominatedBlocks domtree::DomTree worklist::Vector{BBNumber} diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 251767d577157..267bae4734251 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -28,8 +28,7 @@ end struct ConstantCase val::Any - edge::CodeInstance - ConstantCase(@nospecialize(val), edge::CodeInstance) = new(val, edge) + ConstantCase(@nospecialize(val)) = new(val) end struct SomeCase @@ -325,7 +324,12 @@ function ir_prepare_inlining!(insert_node!::Inserter, inline_target::Union{IRCod debuginfo = inline_target isa IRCode ? inline_target.debuginfo : inline_target.ir.debuginfo topline = new_inlined_at = ir_inline_linetable!(debuginfo, di, inlined_at) if should_insert_coverage(def.module, di) - insert_node!(NewInstruction(Expr(:code_coverage_effect), Nothing, topline)) + # The inlinee's first marker also records its definition line + # (coverageVisitStmt in codegen). Hit mode needs no extra entry marker; + # count mode keeps it to record each call. + if JLOptions().code_coverage_mode != 0 || !has_coverage_effect(ir) + insert_node!(NewInstruction(Expr(:code_coverage_effect), Nothing, topline)) + end end spvals_ssa = nothing if !validate_sparams(mi.sparam_vals) @@ -506,7 +510,7 @@ For example, given the following method definition: g(x::T, y::T) where T<:Integer = ... -it is _invalid_ to optimize a cal site like `g(x::Any, y::Any)` into: +it is _invalid_ to optimize a call site like `g(x::Any, y::Any)` into: if isa(x, Integer) && isa(y, Integer) [inlined/resolved g(x::Integer, y::Integer)] @@ -720,7 +724,7 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}}, # replace singleton types with their equivalent Const object p = Const(p.instance) elseif isconstType(p) - p = Const(p.parameters[1]) + p = Const(type_parameter(p)) end push!(def_argtypes, p) end @@ -767,146 +771,138 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}}, return new_argtypes end +function has_typeegal_slot(@nospecialize(atype)) + for p in (atype::DataType).parameters + p isa Core.TypeEgal && return true + end + return false +end + function compileable_specialization(code::Union{MethodInstance,CodeInstance}, effects::Effects, et::InliningEdgeTracker, @nospecialize(info::CallInfo), state::InliningState) - mi = code isa CodeInstance ? code.def : code + mi = code isa CodeInstance ? get_ci_mi(code) : code mi_invoke = mi method, atype, sparams = mi.def::Method, mi.specTypes, mi.sparam_vals if OptimizationParams(state.interp).compilesig_invokes new_atype = get_compileable_sig(method, atype, sparams) new_atype === nothing && return nothing if atype !== new_atype - sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), new_atype, method.sig)::SimpleVector - sparams = sp_[2]::SimpleVector + (_, sparams) = typeintersect_env(new_atype, method.sig) mi_invoke = specialize_method(method, new_atype, sparams) mi_invoke === nothing && return nothing - code = mi_invoke end else # If this caller does not want us to optimize calls to use their # declared compilesig, then it is also likely they would handle sparams # incorrectly if there were any unknown typevars, so we conservatively return nothing - if any(@nospecialize(t)->isa(t, TypeVar), mi.sparam_vals) + if any(@nospecialize(t)->isa(t, SimpleVector), mi.sparam_vals) return nothing end end + if unionall_depth(method.sig) != length(sparams) || !validate_sparams(sparams) + return nothing + end # prefer using a CodeInstance gotten from the cache, since that is where the invoke target should get compiled to normally # TODO: can this code be gotten directly from inference sometimes? - code = get(code_cache(state), mi_invoke, nothing) - if !isa(code, CodeInstance) - #println("missing code for ", mi_invoke, " for ", mi) - code = mi_invoke + # A normalized compileable signature can have a less precise ABI for TypeEgal + # arguments, forcing boxed argument passing for non-recursive invokes, so a + # directly supplied inferred edge for the actual call signature wins there. + keep_direct_edge = code isa CodeInstance && mi !== mi_invoke && has_typeegal_slot(atype) + if !keep_direct_edge + cached = get(code_cache(state), mi_invoke, nothing) + if cached isa CodeInstance + code = cached + elseif !(code isa CodeInstance && code.def === mi_invoke) + #println("missing code for ", mi_invoke, " for ", mi) + code = mi_invoke + end end add_inlining_edge!(et, code) # to the code and edges return InvokeCase(code, effects, info) end -struct InferredResult +struct InferredCode src::Any # CodeInfo or IRCode effects::Effects - edge::CodeInstance - InferredResult(@nospecialize(src), effects::Effects, edge::CodeInstance) = new(src, effects, edge) -end -@inline function get_cached_result(state::InliningState, mi::MethodInstance) - code = get(code_cache(state), mi, nothing) - if code isa CodeInstance - if use_const_api(code) - # in this case function can be inlined to a constant - return ConstantCase(quoted(code.rettype_const), code) - end - return code - end - return nothing + InferredCode(@nospecialize(src), effects::Effects) = new(src, effects) end -@inline function get_local_result(inf_result::InferenceResult) - @assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge" +@inline function get_local_code(local_result::LocalInferenceResult) + inf_result = local_result.result effects = inf_result.ipo_effects if is_foldable_nothrow(effects) res = inf_result.result if isa(res, Const) && is_inlineable_constant(res.val) # use constant calling convention - return ConstantCase(quoted(res.val), inf_result.ci_as_edge) + return SomeCase(quoted(res.val)) end end - return InferredResult(inf_result.src, effects, inf_result.ci_as_edge) + return InferredCode(inf_result.src, effects) +end + +function add_inlining_dispatch_edge!(edges::Vector{Any}, mi::MethodInstance, + @nospecialize(info::CallInfo)) + if info isa InvokeCallInfo + add_invoke_edge!(edges, info.atype, mi) + elseif info isa VirtualMethodMatchInfo + add_inlining_dispatch_edge!(edges, mi, info.info) + elseif info isa MethodMatchInfo || info isa UnionSplitInfo + # A standalone `MethodInstance` edge claims `mi.specTypes` has a single + # fully-covering match, which is false when this call matched several methods. + # Encode the lookup instead; `mi_edge` keeps the invalidation target. + _add_edges_impl(edges, info, #=mi_edge=#true) + else + add_one_edge!(edges, mi) + end + return nothing end # the general resolver for usual and const-prop'ed calls -function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult,VolatileInferenceResult}, +function resolve_todo(mi::MethodInstance, call_result::Union{Nothing,LocalInferenceResult}, + call_edge::Union{Nothing,MethodInstance,CodeInstance}, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) et = InliningEdgeTracker(state) - - preserve_local_sources = true - if isa(result, InferenceResult) - inferred_result = get_local_result(result) - elseif isa(result, VolatileInferenceResult) - inferred_result = get_local_result(result.inf_result) - # volatile inference result can be inlined destructively - preserve_local_sources = !result.inf_result.is_src_volatile | OptimizationParams(state.interp).preserve_local_sources - else - inferred_result = get_cached_result(state, mi) + target = call_edge === nothing ? mi : call_edge + + if call_result === nothing + # there is no cached source available for this, but there might be code for the compilation sig + item = compileable_specialization(target, Effects(), et, info, state) + if item !== nothing && call_edge === nothing + # The inlining edge added above certifies the selected method body, but + # a target synthesized by the optimizer must separately certify how the + # runtime call selected that method. Add this after the identity edge so + # edge deduplication cannot upgrade the dispatch edge into an invoke edge. + add_inlining_dispatch_edge!(et.edges, mi, info) + end + return item end - if inferred_result isa ConstantCase - add_inlining_edge!(et, inferred_result.edge) - return inferred_result - elseif inferred_result isa InferredResult - (; src, effects, edge) = inferred_result - elseif inferred_result isa CodeInstance - src = @atomic :monotonic inferred_result.inferred - effects = decode_effects(inferred_result.ipo_purity_bits) - edge = inferred_result - else # there is no cached source available for this, but there might be code for the compilation sig - return compileable_specialization(mi, Effects(), et, info, state) + + # The local result's proof justifies its inferred facts and retained source. The + # ordinary call edge remains a separate executable target. + add_inlining_edge!(et, target) + add_inference_proof!(et.edges, inference_proof(call_result), target) + inferred_result = get_local_code(call_result) + if inferred_result isa SomeCase + return ConstantCase(inferred_result.val) end + (; src, effects) = inferred_result # the duplicated check might have been done already within `analyze_method!`, but still # we need it here too since we may come here directly using a constant-prop' result if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) - return compileable_specialization(edge, effects, et, info, state) + return compileable_specialization(target, effects, et, info, state) end - src_inlining_policy(state.interp, src, info, flag) || - return compileable_specialization(edge, effects, et, info, state) + src_inlining_policy(state.interp, mi, src, info, flag) || + return compileable_specialization(target, effects, et, info, state) - add_inlining_edge!(et, edge) - if inferred_result isa CodeInstance - ir, spec_info, debuginfo = retrieve_ir_for_inlining(inferred_result, src) - else - ir, spec_info, debuginfo = retrieve_ir_for_inlining(mi, src, preserve_local_sources) - end - return InliningTodo(mi, ir, spec_info, debuginfo, effects) -end - -# the special resolver for :invoke-d call -function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::UInt32, - state::InliningState) - if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) - return nothing - end - - et = InliningEdgeTracker(state) - - cached_result = get_cached_result(state, mi) - if cached_result isa ConstantCase - add_inlining_edge!(et, cached_result.edge) - return cached_result - elseif cached_result isa CodeInstance - src = @atomic :monotonic cached_result.inferred - effects = decode_effects(cached_result.ipo_purity_bits) - else # there is no cached source available, bail out - return nothing - end - - src_inlining_policy(state.interp, src, info, flag) || return nothing - ir, spec_info, debuginfo = retrieve_ir_for_inlining(cached_result, src) - add_inlining_edge!(et, cached_result) + ir, spec_info, debuginfo = retrieve_ir_for_inlining(mi, src, true) return InliningTodo(mi, ir, spec_info, debuginfo, effects) end function validate_sparams(sparams::SimpleVector) for i = 1:length(sparams) spᵢ = sparams[i] - (isa(spᵢ, TypeVar) || isvarargtype(spᵢ)) && return false + (isa(spᵢ, SimpleVector) || has_free_typevars(spᵢ) || isvarargtype(spᵢ)) && return false end return true end @@ -918,10 +914,42 @@ function may_have_fcalls(m::Method) return ccall(:jl_ir_flag_has_fcall, Bool, (Any,), src) end -function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, - @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; - allow_typevars::Bool, - volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) +function has_coverage_effect(ir::IRCode) + for idx in 1:length(ir.stmts) + isexpr(ir.stmts[idx][:stmt], :code_coverage_effect) && return true + end + return false +end + +function strip_coverage_effects!(src::CodeInfo) + for idx in eachindex(src.code) + isexpr(src.code[idx], :code_coverage_effect) && (src.code[idx] = nothing) + end + return src +end + +function strip_coverage_effects!(ir::IRCode) + for idx in 1:length(ir.stmts) + isexpr(ir.stmts[idx][:stmt], :code_coverage_effect) && + (ir.stmts[idx][:stmt] = nothing) + end + return ir +end + +function strip_untracked_coverage_effects!(mi::MethodInstance, src::Union{CodeInfo,IRCode}, + debuginfo::DebugInfo) + method = mi.def::Method + should_insert_coverage(method.module, debuginfo) && return src + return strip_coverage_effects!(src) +end + +function analyze_method!( + call_result::Union{Nothing,LocalInferenceResult}, + call_edge::Union{Nothing,MethodInstance,CodeInstance}, + match::MethodMatch, argtypes::Vector{Any}, + @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; + allow_typevars::Bool + ) method = match.method # Check that we have the correct number of arguments @@ -951,32 +979,40 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any}, # Get the specialization for this method signature # (later we will decide what to do with it) mi = specialize_method(match) - return resolve_todo(mi, volatile_inf_result, info, flag, state) + return resolve_todo(mi, call_result, call_edge, info, flag, state) end function retrieve_ir_for_inlining(cached_result::CodeInstance, src::String) src = _uncompressed_ir(cached_result, src) - return inflate_ir!(src, cached_result.def), SpecInfo(src), src.debuginfo + mi = get_ci_mi(cached_result) + strip_untracked_coverage_effects!(mi, src, src.debuginfo) + return inflate_ir!(src, mi), SpecInfo(src), src.debuginfo end function retrieve_ir_for_inlining(cached_result::CodeInstance, src::CodeInfo) - return inflate_ir!(copy(src), cached_result.def), SpecInfo(src), src.debuginfo + mi = get_ci_mi(cached_result) + src = copy(src) + strip_untracked_coverage_effects!(mi, src, src.debuginfo) + return inflate_ir!(src, mi), SpecInfo(src), src.debuginfo end function retrieve_ir_for_inlining(mi::MethodInstance, src::CodeInfo, preserve_local_sources::Bool) if preserve_local_sources src = copy(src) end + strip_untracked_coverage_effects!(mi, src, src.debuginfo) return inflate_ir!(src, mi), SpecInfo(src), src.debuginfo end function retrieve_ir_for_inlining(mi::MethodInstance, ir::IRCode, preserve_local_sources::Bool) if preserve_local_sources ir = copy(ir) end + ir.debuginfo.def = mi + debuginfo = DebugInfo(ir.debuginfo, length(ir.stmts)) + strip_untracked_coverage_effects!(mi, ir, debuginfo) # COMBAK this is not correct, we should make `InferenceResult` propagate `SpecInfo` spec_info = let m = mi.def::Method SpecInfo(Int(m.nargs), m.isva, false, nothing) end - ir.debuginfo.def = mi - return ir, spec_info, DebugInfo(ir.debuginfo, length(ir.stmts)) + return ir, spec_info, debuginfo end function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, preserve_local_sources::Bool) result = opt.optresult @@ -994,7 +1030,7 @@ function handle_single_case!(todo::Vector{Pair{Int,Any}}, ir[SSAValue(idx)][:stmt] = case.val elseif isa(case, InvokeCase) is_foldable_nothrow(case.effects) && inline_const_if_inlineable!(ir[SSAValue(idx)]) && return nothing - isinvoke && rewrite_invoke_exprargs!(stmt) + isinvoke && invoke_rewrite!(stmt) if stmt.head === :invoke stmt.args[1] = case.invoke else @@ -1005,13 +1041,13 @@ function handle_single_case!(todo::Vector{Pair{Int,Any}}, elseif case === nothing # Do, well, nothing else - isinvoke && rewrite_invoke_exprargs!(stmt) + isinvoke && invoke_rewrite!(stmt) push!(todo, idx=>(case::InliningTodo)) end return nothing end -rewrite_invoke_exprargs!(expr::Expr) = (expr.args = invoke_rewrite(expr.args); expr) +invoke_rewrite!(expr::Expr) = (expr.args = invoke_rewrite(expr.args); expr) function is_valid_type_for_apply_rewrite(@nospecialize(typ), params::OptimizationParams) if isa(typ, Const) && (v = typ.val; isa(v, SimpleVector)) @@ -1073,7 +1109,6 @@ function call_sig(ir::IRCode, stmt::Expr) has_free_typevars(ft) && return nothing f = singleton_type(ft) f === Core.Intrinsics.llvmcall && return nothing - f === Core.Intrinsics.cglobal && return nothing argtypes = Vector{Any}(undef, length(stmt.args)) argtypes[1] = ft for i = (offset+1):length(stmt.args) @@ -1164,8 +1199,20 @@ end function handle_invoke_call!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @nospecialize(info), flag::UInt32, sig::Signature, state::InliningState) + # InvokeCICallInfo indicates that `abstract_invoke` already analyzed the call + # and determined it is of the form `invoke(f, ::CodeInstance, args...)` + # where the argtypes and worldages are valid for the context, and the invoke + # pointer is set. Therefore, we can simply transform this into an + # `Expr(:invoke, ...)` + if info isa InvokeCICallInfo + stmt.head = :invoke + stmt.args = [info.edge, stmt.args[2], stmt.args[4:end]...] + # Transformed to :invoke, now handle it as such + handle_invoke_expr!(todo, ir, idx, stmt, info, flag, sig, state) + return nothing + end nspl = nsplit(info) - nspl == 0 && return nothing # e.g. InvokeCICallInfo + nspl == 0 && return nothing @assert nspl == 1 mresult = getsplit(info, 1) match = mresult.matches[1] @@ -1175,22 +1222,22 @@ function handle_invoke_call!(todo::Vector{Pair{Int,Any}}, end result = getresult(info, 1) if isa(result, ConcreteResult) - item = concrete_result_item(result, info, state) + item = concrete_result_item(result, info.edge, match, info, state) elseif isa(result, SemiConcreteResult) item = semiconcrete_result_item(result, info, flag, state) else argtypes = invoke_rewrite(sig.argtypes) - if isa(result, ConstPropResult) + if isa(result, LocalInferenceResult) mi = result.result.linfo validate_sparams(mi.sparam_vals) || return nothing if Union{} !== argtypes_to_type(argtypes) <: mi.def.sig - item = resolve_todo(mi, result.result, info, flag, state) + item = resolve_todo(mi, result, info.edge, info, flag, state) handle_single_case!(todo, ir, idx, stmt, item, true) return nothing end end - volatile_inf_result = result isa VolatileInferenceResult ? result : nothing - item = analyze_method!(match, argtypes, info, flag, state; allow_typevars=false, volatile_inf_result) + item = analyze_method!(result, info.edge, match, argtypes, info, flag, state; + allow_typevars=false) end handle_single_case!(todo, ir, idx, stmt, item, true) return nothing @@ -1219,6 +1266,36 @@ function narrow_opaque_closure!(ir::IRCode, stmt::Expr, @nospecialize(info::Call stmt.args[3] = newT end end + return nothing +end + +function handle_task_call!(ir::IRCode, idx::Int, stmt::Expr, info::TaskCallInfo, state::InliningState) + length(stmt.args) == 3 || return nothing + # Extract the CodeInstance from the inference result if available + info_edge = extract_indirect_invoke(info.info) + info_edge === nothing && return nothing + info, edge = info_edge + case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state) + case === nothing && return nothing + # The runtime (`jl_f_invoke`) only accepts Method/CodeInstance/Type targets, so + # decline if compileable_specialization only found an uncached MethodInstance. + case.invoke isa CodeInstance || return nothing + # Append the CodeInstance as a third argument to the _task call + # Core._task(func, size) becomes Core._task(func, size, ci) + push!(stmt.args, case.invoke) + ir[SSAValue(idx)][:stmt] = stmt + return nothing +end + +function extract_indirect_invoke(@nospecialize info::CallInfo) + info isa MethodResultPure && (info = info.info) + info isa MethodMatchInfo || return nothing + length(info.edges) == length(info.results) == 1 || return nothing + match = info.results[1]::MethodMatch + match.fully_covers || return nothing + edge = info.edges[1] + edge === nothing && return nothing + return info, edge end # As a matter of convenience, this pass also computes effect-freenes. @@ -1287,8 +1364,10 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag f !== Core.finalizer && f !== modifyfield! && f !== Core.modifyglobal! && + f !== Core.modifyglobal_partition && f !== Core.memoryrefmodify! && - f !== atomic_pointermodify) + f !== atomic_pointermodify && + f !== Core._task) # No inlining defined for most builtins (just invoke/apply/typeassert/finalizer), so attempt an early exit for them return nothing end @@ -1306,39 +1385,33 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag return stmt, sig end -function handle_any_const_result!(cases::Vector{InliningCase}, - @nospecialize(result), match::MethodMatch, argtypes::Vector{Any}, - @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; - allow_typevars::Bool) - if isa(result, ConcreteResult) - return handle_concrete_result!(cases, result, match, info, state) - elseif isa(result, SemiConcreteResult) - return handle_semi_concrete_result!(cases, result, match, info, flag, state) - elseif isa(result, ConstPropResult) - return handle_const_prop_result!(cases, result, match, info, flag, state; allow_typevars) +function handle_any_call_result!( + cases::Vector{InliningCase}, @nospecialize(call_result::Union{Nothing,InferredCallResult}), + call_edge::Union{Nothing,CodeInstance}, match::MethodMatch, argtypes::Vector{Any}, + @nospecialize(info::CallInfo), + flag::UInt32, state::InliningState; + allow_typevars::Bool + ) + if isa(call_result, ConcreteResult) + return handle_concrete_result!(cases, call_result, call_edge, match, info, state) + elseif isa(call_result, SemiConcreteResult) + return handle_semi_concrete_result!(cases, call_result, match, info, flag, state) else - @assert result === nothing || result isa VolatileInferenceResult - return handle_match!(cases, match, argtypes, info, flag, state; allow_typevars, volatile_inf_result = result) + return handle_call_result!(cases, call_result, call_edge, match, argtypes, info, flag, + state; allow_typevars) end end -function info_effects(@nospecialize(result), match::MethodMatch, state::InliningState) - if isa(result, ConcreteResult) - return result.effects - elseif isa(result, SemiConcreteResult) - return result.effects - elseif isa(result, ConstPropResult) - return result.result.ipo_effects - else - mi = specialize_method(match; preexisting=true) - if isa(mi, MethodInstance) - code = get(code_cache(state), mi, nothing) - if code isa CodeInstance - return decode_effects(code.ipo_purity_bits) - end - end +function info_effects(@nospecialize(call_result::Union{Nothing,InferredCallResult}), match::MethodMatch, state::InliningState) + if call_result === nothing return Effects() - end + elseif isa(call_result, LocalInferenceResult) + return call_result.result.ipo_effects + elseif isa(call_result, ConcreteResult) + return call_result.effects + elseif isa(call_result, SemiConcreteResult) + return call_result.effects + else error("Unknown InferredCallResult type") end end function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig::Signature, @@ -1361,8 +1434,9 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig local split_fully_covered = false for (j, match) in enumerate(meth) all_result_count += 1 - result = getresult(info, all_result_count) - joint_effects = merge_effects(joint_effects, info_effects(result, match, state)) + call_result = getresult(info, all_result_count) + call_edge = getedge(info, all_result_count) + joint_effects = merge_effects(joint_effects, info_effects(call_result, match, state)) split_fully_covered |= match.fully_covers if !validate_sparams(match.sparams) if match.fully_covers @@ -1378,8 +1452,9 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig elseif !(match.spec_types <: match.method.sig) # the requirement for correct union-split handled_all_cases = false else - handled_all_cases &= handle_any_const_result!(cases, - result, match, argtypes, info, flag, state; allow_typevars=false) + handled_all_cases &= handle_any_call_result!(cases, + call_result, call_edge, match, argtypes, info, flag, state; + allow_typevars=false) end end fully_covered &= split_fully_covered @@ -1393,9 +1468,11 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig # so try to handle it by bypassing validate_sparams (i, j, k) = revisit_idx match = getsplit(info, i)[j] - result = getresult(info, k) - handled_all_cases &= handle_any_const_result!(cases, - result, match, argtypes, info, flag, state; allow_typevars=true) + call_result = getresult(info, k) + call_edge = getedge(info, k) + handled_all_cases &= handle_any_call_result!(cases, + call_result, call_edge, match, argtypes, info, flag, state; + allow_typevars=true) end if !fully_covered # We will emit an inline MethodError in this case, but that info already came inference, so we must already have the uncovered edge for it @@ -1418,28 +1495,20 @@ function handle_call!(todo::Vector{Pair{Int,Any}}, handle_cases!(todo, ir, idx, stmt, atype, cases, handled_all_cases, fully_covered, joint_effects) end -function handle_match!(cases::Vector{InliningCase}, - match::MethodMatch, argtypes::Vector{Any}, @nospecialize(info::CallInfo), flag::UInt32, - state::InliningState; - allow_typevars::Bool, volatile_inf_result::Union{Nothing,VolatileInferenceResult}) +function handle_call_result!( + cases::Vector{InliningCase}, call_result::Union{Nothing,LocalInferenceResult}, + call_edge::Union{Nothing,CodeInstance}, match::MethodMatch, argtypes::Vector{Any}, + @nospecialize(info::CallInfo), flag::UInt32, + state::InliningState; + allow_typevars::Bool + ) # We may see duplicated dispatch signatures here when a signature gets widened # during abstract interpretation: for the purpose of inlining, we can just skip # processing this dispatch candidate (unless unmatched type parameters are present) !allow_typevars && any(case::InliningCase->case.sig === match.spec_types, cases) && return true - item = analyze_method!(match, argtypes, info, flag, state; allow_typevars, volatile_inf_result) - item === nothing && return false - push!(cases, InliningCase(match.spec_types, item)) - return true -end -function handle_const_prop_result!(cases::Vector{InliningCase}, result::ConstPropResult, - match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState; - allow_typevars::Bool) - mi = result.result.linfo - if !validate_sparams(mi.sparam_vals) - (allow_typevars && !may_have_fcalls(mi.def::Method)) || return false - end - item = resolve_todo(mi, result.result, info, flag, state) + item = analyze_method!(call_result, call_edge, match, argtypes, info, flag, state; + allow_typevars) item === nothing && return false push!(cases, InliningCase(match.spec_types, item)) return true @@ -1448,8 +1517,10 @@ end function semiconcrete_result_item(result::SemiConcreteResult, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) code = result.edge - mi = code.def + mi = get_ci_mi(code) et = InliningEdgeTracker(state) + add_inlining_edge!(et, code) + add_inference_proof!(et.edges, inference_proof(result), code) if (!OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) || # For `NativeInterpreter`, `SemiConcreteResult` may be produced for @@ -1458,10 +1529,9 @@ function semiconcrete_result_item(result::SemiConcreteResult, (is_declared_noinline(mi.def::Method) && !is_stmt_inline(flag))) return compileable_specialization(code, result.effects, et, info, state) end - src_inlining_policy(state.interp, result.ir, info, flag) || + src_inlining_policy(state.interp, mi, result.ir, info, flag) || return compileable_specialization(code, result.effects, et, info, state) - add_inlining_edge!(et, result.edge) preserve_local_sources = OptimizationParams(state.interp).preserve_local_sources ir, _, debuginfo = retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) return InliningTodo(mi, ir, result.spec_info, debuginfo, result.effects) @@ -1469,7 +1539,7 @@ end function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult, match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState) - mi = result.edge.def + mi = get_ci_mi(result.edge) validate_sparams(mi.sparam_vals) || return false item = semiconcrete_result_item(result, info, flag, state) item === nothing && return false @@ -1478,8 +1548,9 @@ function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiC end function handle_concrete_result!(cases::Vector{InliningCase}, result::ConcreteResult, - match::MethodMatch, @nospecialize(info::CallInfo), state::InliningState) - case = concrete_result_item(result, info, state) + call_edge::Union{Nothing,CodeInstance}, match::MethodMatch, + @nospecialize(info::CallInfo), state::InliningState) + case = concrete_result_item(result, call_edge, match, info, state) case === nothing && return false push!(cases, InliningCase(match.spec_types, case)) return true @@ -1488,13 +1559,20 @@ end may_inline_concrete_result(result::ConcreteResult) = isdefined(result, :result) && is_inlineable_constant(result.result) -function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallInfo), state::InliningState) +function concrete_result_item(result::ConcreteResult, + call_edge::Union{Nothing,CodeInstance}, match::MethodMatch, + @nospecialize(info::CallInfo), state::InliningState) + target = result.edge === nothing ? call_edge : result.edge + target === nothing && (target = specialize_method(match)) + target === nothing && return nothing + et = InliningEdgeTracker(state) + add_inlining_edge!(et, target) + add_inference_proof!(et.edges, inference_proof(result), target) if !may_inline_concrete_result(result) - et = InliningEdgeTracker(state) - return compileable_specialization(result.edge, result.effects, et, info, state) + return compileable_specialization(target, result.effects, et, info, state) end @assert result.effects === EFFECTS_TOTAL - return ConstantCase(quoted(result.result), result.edge) + return ConstantCase(quoted(result.result)) end function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @@ -1521,33 +1599,22 @@ function handle_opaque_closure_call!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, info::OpaqueClosureCallInfo, flag::UInt32, sig::Signature, state::InliningState) result = info.result - if isa(result, ConstPropResult) - mi = result.result.linfo - validate_sparams(mi.sparam_vals) || return nothing - item = resolve_todo(mi, result.result, info, flag, state) - elseif isa(result, ConcreteResult) - item = concrete_result_item(result, info, state) + if isa(result, ConcreteResult) + item = concrete_result_item(result, info.edge, info.match, info, state) elseif isa(result, SemiConcreteResult) - item = item = semiconcrete_result_item(result, info, flag, state) + item = semiconcrete_result_item(result, info, flag, state) else - @assert result === nothing || result isa VolatileInferenceResult - volatile_inf_result = result - item = analyze_method!(info.match, sig.argtypes, info, flag, state; allow_typevars=false, volatile_inf_result) + item = analyze_method!(result, info.edge, info.match, sig.argtypes, info, flag, state; + allow_typevars=false) end handle_single_case!(todo, ir, idx, stmt, item) return nothing end function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOpInfo, state::InliningState) - info = info.info - info isa MethodResultPure && (info = info.info) - info isa ConstCallInfo && (info = info.call) - info isa MethodMatchInfo || return nothing - length(info.edges) == length(info.results) == 1 || return nothing - match = info.results[1]::MethodMatch - match.fully_covers || return nothing - edge = info.edges[1] - edge === nothing && return nothing + info_edge = extract_indirect_invoke(info.info) + info_edge === nothing && return nothing + info, edge = info_edge case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state) case === nothing && return nothing stmt.head = :invoke_modify @@ -1561,6 +1628,10 @@ function handle_finalizer_call!(ir::IRCode, idx::Int, stmt::Expr, info::Finalize # Finalizers don't return values, so if their execution is not observable, # we can just not register them if is_removable_if_unused(info.effects) + # The inferred effects are what make deleting this registration legal. + # Preserve their dispatch and inference proof so redefining the finalizer + # invalidates code in which the registration was removed. + add_edges!(state.edges, info.info) ir[SSAValue(idx)] = nothing return nothing end @@ -1590,7 +1661,9 @@ function handle_finalizer_call!(ir::IRCode, idx::Int, stmt::Expr, info::Finalize push!(stmt.args, true) push!(stmt.args, code) end - elseif isa(item1, InvokeCase) + elseif isa(item1, InvokeCase) && item1.invoke isa CodeInstance + # like handle_task_call!, an uncached MethodInstance is unusable here, since + # `try_resolve_finalizer!` requires a CodeInstance in this argument position push!(stmt.args, false) push!(stmt.args, item1.invoke) elseif isa(item1, ConstantCase) @@ -1600,14 +1673,26 @@ function handle_finalizer_call!(ir::IRCode, idx::Int, stmt::Expr, info::Finalize return nothing end +# the special resolver for :invoke-d call function handle_invoke_expr!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @nospecialize(info::CallInfo), flag::UInt32, sig::Signature, state::InliningState) - mi = stmt.args[1] - if !(mi isa MethodInstance) - mi = (mi::CodeInstance).def + edge = stmt.args[1] + mi = isa(edge, MethodInstance) ? edge : get_ci_mi(edge::CodeInstance) + call_result = nothing + let info = info + info isa MethodResultPure && (info = info.info) + if isa(info, InvokeCallInfo) + call_result = info.result + elseif isa(info, MethodMatchInfo) + # We didn't preserve the converted info when inserting :invoke node so we cannot recover this with accuracy. + # Since that info isn't used, but this is enough for the "apply `ssa_inlining_pass` multiple times" test + if length(info.edges) == length(info.results) == 1 + call_result = getresult(info, 1) + end + end end - case = resolve_todo(mi, info, flag, state) - handle_single_case!(todo, ir, idx, stmt, case, false) + item = resolve_todo(mi, call_result, edge, info, flag, state) + handle_single_case!(todo, ir, idx, stmt, item) return nothing end @@ -1625,13 +1710,13 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) todo = Pair{Int, Any}[] for idx in 1:length(ir.stmts) - flag = ir.stmts[idx][:flag] + inst = ir.stmts[idx] + flag = inst[:flag] simpleres = process_simple!(todo, ir, idx, flag, state) simpleres === nothing && continue stmt, sig = simpleres - - info = ir.stmts[idx][:info] + info = inst[:info] # `NativeInterpreter` won't need this, but provide a support for `:invoke` exprs here # for external `AbstractInterpreter`s that may run the inlining pass multiple times @@ -1655,6 +1740,8 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) handle_opaque_closure_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, ModifyOpInfo) handle_modifyop!_call!(ir, idx, stmt, info, state) + elseif isa(info, TaskCallInfo) + handle_task_call!(ir, idx, stmt, info, state) elseif sig.f === Core.invoke handle_invoke_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, FinalizerInfo) @@ -1720,6 +1807,9 @@ function early_inline_special_case(ir::IRCode, stmt::Expr, flag::UInt32, elseif ⊑(optimizer_lattice(state.interp), cond, Bool) && stmt.args[3] === stmt.args[4] return SomeCase(stmt.args[3]) end + elseif (f === Core.task_result_type && length(argtypes) == 2 && + ⊑(optimizer_lattice(state.interp), argtypes[2], Task)) + return SomeCase(quoted(instanceof_tfunc(type)[1])) end return nothing end @@ -1755,12 +1845,12 @@ function late_inline_special_case!(ir::IRCode, idx::Int, stmt::Expr, flag::UInt3 length(stmt.args) == 2 ? Any : stmt.args[end]) return SomeCase(typevar_call) elseif f === UnionAll && length(argtypes) == 3 && ⊑(optimizer_lattice(state.interp), argtypes[2], TypeVar) - unionall_call = Expr(:foreigncall, QuoteNode(:jl_type_unionall), Any, svec(Any, Any), + unionall_call = Expr(:foreigncall, Expr(:tuple, QuoteNode(:jl_type_unionall)), Any, svec(Any, Any), 0, QuoteNode(:ccall), stmt.args[2], stmt.args[3]) return SomeCase(unionall_call) elseif is_return_type(f) if isconstType(type) - return SomeCase(quoted(type.parameters[1])) + return SomeCase(quoted(type_parameter(type))) elseif isa(type, Const) return SomeCase(quoted(type.val)) end @@ -1773,15 +1863,32 @@ struct SSASubstitute arg_replacements::Vector{Any} spvals_ssa::Union{Nothing,SSAValue} inlined_at::NTuple{3,Int32} # TODO: add a map also, so that ssaidx doesn't need to equal inlined_idx? + # lazily-computed `sptypes_from_meth_instance(mi)`, shared across all + # marker-sparam substitutions of this inlined item + sptypes_cache::RefValue{Union{Nothing,Vector{VarState}}} +end +SSASubstitute(mi::MethodInstance, arg_replacements::Vector{Any}, + spvals_ssa::Union{Nothing,SSAValue}, inlined_at::NTuple{3,Int32}) = + SSASubstitute(mi, arg_replacements, spvals_ssa, inlined_at, + RefValue{Union{Nothing,Vector{VarState}}}(nothing)) + +function cached_sptypes(ssa_substitute::SSASubstitute) + sptypes = ssa_substitute.sptypes_cache[] + sptypes === nothing || return sptypes + return ssa_substitute.sptypes_cache[] = sptypes_from_meth_instance(ssa_substitute.mi) end -function insert_spval!(insert_node!::Inserter, spvals_ssa::SSAValue, spidx::Int, do_isdefined::Bool) +function insert_spval!(insert_node!::Inserter, spvals_ssa::SSAValue, spidx::Int, + do_isdefined::Bool, @nospecialize(typ = Any)) ret = insert_node!( - removable_if_unused(NewInstruction(Expr(:call, Core._svec_ref, spvals_ssa, spidx), Any))) + removable_if_unused(NewInstruction(Expr(:call, Core._svec_ref, spvals_ssa, spidx), typ))) tcheck_not = nothing if do_isdefined + # The caller handles guaranteed-defined static parameters before this + # fallback. At runtime, SimpleVector is the undefined sentinel for + # sparams. tcheck = insert_node!( - removable_if_unused(NewInstruction(Expr(:call, Core.isa, ret, Core.TypeVar), Bool))) + removable_if_unused(NewInstruction(Expr(:call, Core.isa, ret, Core.SimpleVector), Bool))) tcheck_not = insert_node!( removable_if_unused(NewInstruction(Expr(:call, not_int, tcheck), Bool))) end @@ -1800,22 +1907,32 @@ function ssa_substitute_op!(insert_node!::Inserter, subst_inst::Instruction, @no if head === :static_parameter spidx = e.args[1]::Int val = sparam_vals[spidx] - if !isa(val, TypeVar) && val !== Vararg + val_uncertain = isa(val, SimpleVector) || has_free_typevars(val) + if !val_uncertain && val !== Vararg return quoted(val) else flag = subst_inst[:flag] - maybe_undef = !has_flag(flag, IR_FLAG_NOTHROW) && isa(val, TypeVar) - (ret, tcheck_not) = insert_spval!(insert_node!, ssa_substitute.spvals_ssa::SSAValue, spidx, maybe_undef) + if isa(val, SimpleVector) + spstate = cached_sptypes(ssa_substitute)[spidx] + maybe_undef = spstate.undef && !has_flag(flag, IR_FLAG_NOTHROW) + typ = spstate.undef ? Any : spstate.typ + else + maybe_undef = !has_flag(flag, IR_FLAG_NOTHROW) && val_uncertain + typ = Any + end + (ret, tcheck_not) = insert_spval!(insert_node!, ssa_substitute.spvals_ssa::SSAValue, spidx, maybe_undef, typ) if maybe_undef insert_node!( - NewInstruction(Expr(:throw_undef_if_not, val.name, tcheck_not), Nothing)) + NewInstruction(Expr(:throw_undef_if_not, sp_at_idx(ssa_substitute.mi.def.sig, spidx).name, tcheck_not), Nothing)) end return ret end elseif head === :isdefined && isa(e.args[1], Expr) && e.args[1].head === :static_parameter spidx = (e.args[1]::Expr).args[1]::Int val = sparam_vals[spidx] - if !isa(val, TypeVar) + if !isa(val, SimpleVector) + return true + elseif val[2]::Bool return true else (_, tcheck_not) = insert_spval!(insert_node!, ssa_substitute.spvals_ssa::SSAValue, spidx, true) diff --git a/Compiler/src/ssair/ir.jl b/Compiler/src/ssair/ir.jl index e6a8ffe6d539e..631008c512bb7 100644 --- a/Compiler/src/ssair/ir.jl +++ b/Compiler/src/ssair/ir.jl @@ -153,7 +153,7 @@ end function is_valid_phiblock_stmt(@nospecialize(stmt)) isa(stmt, PhiNode) && return true isa(stmt, Union{UpsilonNode, PhiCNode, SSAValue}) && return false - isa(stmt, Expr) && return is_value_pos_expr_head(stmt.head) + isa(stmt, Expr) && return false return true end @@ -192,7 +192,7 @@ mutable struct DebugInfoStream # DebugInfoStream(def::Union{MethodInstance,Nothing}, di::DebugInfo, nstmts::Int) = # if debuginfo_file1(di.def) === debuginfo_file1(di.def) # new(def, di.linetable, Core.svec(di.edges...), getdebugidx(di, 0), - # ccall(:jl_uncompress_codelocs, Any, (Any, Int), di.codelocs, nstmts)::Vector{Int32}) + # ccall(:jl_uncompress_codelocs, Any, (Any, Int), di, nstmts)::Vector{Int32}) # else function DebugInfoStream(def::Union{MethodInstance,Nothing}, di::DebugInfo, nstmts::Int) codelocs = zeros(Int32, nstmts * 3) @@ -208,8 +208,14 @@ Core.DebugInfo(di::DebugInfoStream, nstmts::Int) = DebugInfo(something(di.def), di.linetable, Core.svec(di.edges...), ccall(:jl_compress_codelocs, Any, (Int32, Any, Int), di.firstline, di.codelocs, nstmts)::String) -getdebugidx(debuginfo::DebugInfo, pc::Int) = - ccall(:jl_uncompress1_codeloc, NTuple{3,Int32}, (Any, Int), debuginfo.codelocs, pc) +function getdebugidx(debuginfo::DebugInfo, pc::Int) + if debuginfo.linetable isa String + # drop provenance for compatibility + ((pc <= 0 ? -1 : source_location(debuginfo, pc).line), 0, 0) + else + ccall(:jl_uncompress1_codeloc, NTuple{3,Int32}, (Any, Int), debuginfo, pc) + end +end function getdebugidx(debuginfo::DebugInfoStream, pc::Int) if 3 <= 3pc <= length(debuginfo.codelocs) @@ -221,6 +227,53 @@ function getdebugidx(debuginfo::DebugInfoStream, pc::Int) end end +has_prev_debuginfo(di, pc::Int) = prev_debuginfo(di, pc)[1] !== nothing +has_edge_debuginfo(di, pc::Int) = edge_debuginfo(di, pc)[1] !== nothing + +"(debuginfo, nextpc) from the previous step in the compiler" +function prev_debuginfo(di, pc::Int) + di.linetable isa Core.DebugInfo || return (nothing, 0) + pc > 0 || return (nothing, 0) + nextpc::Int = getdebugidx(di, pc)[1] + (di.linetable, nextpc) +end + +"(debuginfo, nextpc) of inlinee" +function edge_debuginfo(di, pc::Int) + _, eid::Int, epc::Int = getdebugidx(di, pc) + # XXX: eid > 0 should imply epc > 0 + (eid > 0 && epc > 0) || return (nothing, 0) + (di.edges[eid]::DebugInfo, epc) +end + +# All 1-based. 0 if unavailable. +struct SourceLocation + byte::Int + byte_end::Int + col::Int + col_end::Int + line::Int + line_end::Int +end + +function source_location(di::Union{DebugInfo,DebugInfoStream}, pc::Int) + while has_prev_debuginfo(di, pc) + di, pc = prev_debuginfo(di, pc) + end + if di isa DebugInfoStream + # DebugInfoStream with linetable=nothing does not contain source + # information on its own. + return SourceLocation(0,0,0,0,0,0) + end + @assert pc > 0 "no source for pc<=0" + (l1, c1) = ccall(:jl_cdi_firstxy, NTuple{2, Int32}, (Any, Int32), di, pc) + if c1 <= 0 + return SourceLocation(0,0,0,0,l1,0) + end + (b1, b2) = ccall(:jl_cdi_bytespan, NTuple{2, Int32}, (Any, Int32), di, pc) + (l2, c2) = ccall(:jl_cdi_byte_to_xy, NTuple{2, Int32}, (Any, Int32), di, b2) + return SourceLocation(b1,b2,c1,c2,l1,l2) +end # SSA values that need renaming struct OldSSAValue @@ -242,7 +295,7 @@ on where they appear: ii. a `NewSSAValue` with negative `id` refers to post-compaction `new_node` node. 2. In non-compacted nodes, - i. a `NewSSAValue` with positive `id` refers to the index of an already-compacted instructions. + i. a `NewSSAValue` with positive `id` refers to the index of an already-compacted instruction. ii. a `NewSSAValue` with negative `id` has the same meaning as in compacted nodes. """ struct NewSSAValue @@ -444,6 +497,10 @@ struct IRCode @assert di.codelocs === stmts.line return new(stmts, ir.argtypes, ir.sptypes, di, cfg, new_nodes, ir.meta, ir.valid_worlds) end + # The same IR restricted to a narrower `valid_worlds` (see `reformulate_globals_pass!`). + function IRCode(ir::IRCode, valid_worlds::WorldRange) + return new(ir.stmts, ir.argtypes, ir.sptypes, ir.debuginfo, ir.cfg, ir.new_nodes, ir.meta, valid_worlds) + end global function copy(ir::IRCode) di = ir.debuginfo stmts = copy(ir.stmts) @@ -527,42 +584,69 @@ end struct OOBToken end; const OOB_TOKEN = OOBToken() struct UndefToken end; const UNDEF_TOKEN = UndefToken() -@noinline function _useref_getindex(@nospecialize(stmt), op::Int) +# Split into op==1 and op>1 cases to avoid redundant comparisons. Most IR node +# types only have a single operand, so the op==1 path can return directly without +# checking `op == 1 || return OOB_TOKEN` for each type. The op>1 path only needs +# to handle Expr, PhiNode, and PhiCNode which can have multiple operands. +function _useref_getindex(@nospecialize(stmt), op::Int) + if op == 1 + return _useref_getindex_op1(stmt) + else + return _useref_getindex_opN(stmt, op) + end +end + +@noinline function _useref_getindex_op1(@nospecialize(stmt)) if isa(stmt, Expr) && stmt.head === :(=) rhs = stmt.args[2] - if isa(rhs, Expr) - if is_relevant_expr(rhs) - op > length(rhs.args) && return OOB_TOKEN - return rhs.args[op] - end + if isa(rhs, Expr) && is_relevant_expr(rhs) + length(rhs.args) < 1 && return OOB_TOKEN + return rhs.args[1] end - op == 1 || return OOB_TOKEN return rhs elseif isa(stmt, Expr) # @assert is_relevant_expr(stmt) - op > length(stmt.args) && return OOB_TOKEN - return stmt.args[op] + length(stmt.args) < 1 && return OOB_TOKEN + return stmt.args[1] elseif isa(stmt, GotoIfNot) - op == 1 || return OOB_TOKEN return stmt.cond elseif isa(stmt, ReturnNode) isdefined(stmt, :val) || return OOB_TOKEN - op == 1 || return OOB_TOKEN return stmt.val elseif isa(stmt, EnterNode) isdefined(stmt, :scope) || return OOB_TOKEN - op == 1 || return OOB_TOKEN return stmt.scope elseif isa(stmt, PiNode) isdefined(stmt, :val) || return OOB_TOKEN - op == 1 || return OOB_TOKEN return stmt.val elseif isa(stmt, Union{AnySSAValue, GlobalRef}) - op == 1 || return OOB_TOKEN return stmt elseif isa(stmt, UpsilonNode) isdefined(stmt, :val) || return OOB_TOKEN - op == 1 || return OOB_TOKEN return stmt.val + elseif isa(stmt, PhiNode) + length(stmt.values) < 1 && return OOB_TOKEN + isassigned(stmt.values, 1) || return UNDEF_TOKEN + return stmt.values[1] + elseif isa(stmt, PhiCNode) + length(stmt.values) < 1 && return OOB_TOKEN + isassigned(stmt.values, 1) || return UNDEF_TOKEN + return stmt.values[1] + else + return OOB_TOKEN + end +end + +@noinline function _useref_getindex_opN(@nospecialize(stmt), op::Int) + if isa(stmt, Expr) && stmt.head === :(=) + rhs = stmt.args[2] + if isa(rhs, Expr) && is_relevant_expr(rhs) + op > length(rhs.args) && return OOB_TOKEN + return rhs.args[op] + end + return OOB_TOKEN + elseif isa(stmt, Expr) # @assert is_relevant_expr(stmt) + op > length(stmt.args) && return OOB_TOKEN + return stmt.args[op] elseif isa(stmt, PhiNode) op > length(stmt.values) && return OOB_TOKEN isassigned(stmt.values, op) || return UNDEF_TOKEN @@ -581,10 +665,10 @@ function is_relevant_expr(e::Expr) return e.head in (:call, :invoke, :invoke_modify, :new, :splatnew, :(=), :(&), :gc_preserve_begin, :gc_preserve_end, - :foreigncall, :isdefined, :copyast, + :foreigncall, :foreignglobal, :isdefined, :copyast, :throw_undef_if_not, - :cfunction, :method, :pop_exception, - :leave, :const, :globaldecl, + :cfunction, :pop_exception, + :leave, :new_opaque_closure) end @@ -697,6 +781,9 @@ struct CFGTransformState bb_rename_pred::Vector{Int} bb_rename_succ::Vector{Int} domtree::Union{Nothing, DomTree} + # Scratch buffers reused for the in-place domtree updates performed while + # killing edges. Non-`nothing` exactly when `domtree` is. + domtree_cache::Union{Nothing, DomTreeCache} end # N.B.: Takes ownership of the CFG array @@ -704,9 +791,9 @@ function CFGTransformState!(blocks::Vector{BasicBlock}, allow_cfg_transforms::Bo if allow_cfg_transforms bb_rename = Vector{Int}(undef, length(blocks)) cur_bb = 1 - domtree = construct_domtree(blocks) + dfs = DFS(blocks) for i = 1:length(bb_rename) - if bb_unreachable(domtree, i) + if i != 1 && dfs.to_pre[i] == 0 # if i is unreachable bb_rename[i] = -1 else bb_rename[i] = cur_bb @@ -732,14 +819,18 @@ function CFGTransformState!(blocks::Vector{BasicBlock}, allow_cfg_transforms::Bo let blocks = blocks, bb_rename = bb_rename result_bbs = BasicBlock[blocks[i] for i = 1:length(blocks) if bb_rename[i] != -1] end + # Reuse the same cache for the initial construction and any later + # in-place updates while killing edges during compaction. + domtree_cache = DomTreeCache() # TODO: This could be done by just renaming the domtree - domtree = construct_domtree(result_bbs) + domtree = construct_domtree(result_bbs; cache=domtree_cache) else bb_rename = Vector{Int}() result_bbs = blocks domtree = nothing + domtree_cache = nothing end - return CFGTransformState(allow_cfg_transforms, allow_cfg_transforms, result_bbs, bb_rename, bb_rename, domtree) + return CFGTransformState(allow_cfg_transforms, allow_cfg_transforms, result_bbs, bb_rename, bb_rename, domtree, domtree_cache) end mutable struct IncrementalCompact @@ -795,7 +886,7 @@ mutable struct IncrementalCompact bb_rename = Vector{Int}() pending_nodes = NewNodeStream() pending_perm = Int[] - return new(code, parent.result, CFGTransformState(false, false, parent.cfg_transform.result_bbs, bb_rename, bb_rename, nothing), + return new(code, parent.result, CFGTransformState(false, false, parent.cfg_transform.result_bbs, bb_rename, bb_rename, nothing, nothing), ssa_rename, parent.used_ssas, parent.late_fixup, perm, 1, parent.new_new_nodes, parent.new_new_used_ssas, pending_nodes, pending_perm, @@ -1382,16 +1473,28 @@ function kill_edge!(ir::IRCode, from::Int, to::Int, callback=nothing) kill_edge!(ir.cfg.blocks, from, to, callback) end -# N.B.: from and to are non-renamed indices -function kill_edge!(compact::IncrementalCompact, active_bb::Int, from::Int, to::Int) +@inline function compacted_stmt_range(compact::IncrementalCompact, bb::BasicBlock, active_bb::Int, to::Int) + to == active_bb && return StmtRange(first(bb.stmts), compact.result_idx - 1) + return bb.stmts +end + +""" + kill_edge_terminator!(compact::IncrementalCompact, active_bb::Int, from::Int, to::Int) + +Kill a CFG edge while compacting a terminator in `active_bb`. Assumes all PhiNode +block statements in `to` have already been processed, so the active BB may only +scan the compacted prefix when `to == active_bb`. `from` and `to` are non-renamed indices. +""" +function kill_edge_terminator!(compact::IncrementalCompact, active_bb::Int, from::Int, to::Int) # Note: We recursively kill as many edges as are obviously dead. - (; bb_rename_pred, bb_rename_succ, result_bbs, domtree) = compact.cfg_transform + (; bb_rename_pred, bb_rename_succ, result_bbs, domtree, domtree_cache) = compact.cfg_transform preds = result_bbs[bb_rename_succ[to]].preds succs = result_bbs[bb_rename_pred[from]].succs deleteat!(preds, findfirst(x::Int->x==bb_rename_pred[from], preds)::Int) deleteat!(succs, findfirst(x::Int->x==bb_rename_succ[to], succs)::Int) if domtree !== nothing - domtree_delete_edge!(domtree, result_bbs, bb_rename_pred[from], bb_rename_succ[to]) + domtree_delete_edge!(domtree, result_bbs, bb_rename_pred[from], bb_rename_succ[to]; + cache=domtree_cache::DomTreeCache) end # Check if the block is now dead if length(preds) == 0 || (domtree !== nothing && bb_unreachable(domtree, bb_rename_succ[to])) @@ -1399,7 +1502,7 @@ function kill_edge!(compact::IncrementalCompact, active_bb::Int, from::Int, to:: for succ in copy(to_succs) new_succ = findfirst(x::Int->x==succ, bb_rename_pred) new_succ === nothing && continue - kill_edge!(compact, active_bb, to, new_succ) + kill_edge_terminator!(compact, active_bb, to, new_succ) end empty!(preds) empty!(to_succs) @@ -1421,12 +1524,16 @@ function kill_edge!(compact::IncrementalCompact, active_bb::Int, from::Int, to:: # Remove this edge from all phi nodes in `to` block # NOTE: It is possible for `to` to contain only `nothing` statements, # so we must be careful to stop at its last statement - if to < active_bb - stmts = result_bbs[bb_rename_succ[to]].stmts + if to <= active_bb + bb = result_bbs[bb_rename_succ[to]] + stmts = compacted_stmt_range(compact, bb, active_bb, to) idx = first(stmts) while idx <= last(stmts) stmt = compact.result[idx][:stmt] - stmt === nothing && continue + if stmt === nothing + idx += 1 + continue + end isa(stmt, PhiNode) || break i = findfirst(x::Int32->x==bb_rename_pred[from], stmt.edges) if i !== nothing @@ -1456,6 +1563,17 @@ struct Refined Refined(@nospecialize(val)) = new(val) end + +# Whether `gr` names a primordial constant: a constant binding of `Core` or `Core.Intrinsics` +# (a builtin, intrinsic, or core type) covering world age 1. The runtime keeps these immutable, +# so a read needs no edge and codegen can embed the value. +function world1_const(gr::GlobalRef) + (gr.mod === Core || gr.mod === Core.Intrinsics) || return false + b = convert(Core.Binding, gr) + isdefined(b, :partitions) || return false + return binding_kind(lookup_binding_partition(UInt(1), b)) === PARTITION_KIND_CONST +end + function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instruction, idx::Int, processed_idx::Int, active_bb::Int, do_rename_ssa::Bool) stmt = inst[:stmt] (; result, ssa_rename, late_fixup, used_ssas, new_new_used_ssas) = compact @@ -1474,6 +1592,19 @@ function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instr result[result_idx][:stmt] = GotoNode(label) result_idx += 1 elseif isa(stmt, GlobalRef) + # Fold GlobalRef inline only when it is a world1_constant, + # otherwise it should be a BindingPartition before this transformation. + total_flags = IR_FLAG_CONSISTENT | IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW + flag = result[result_idx][:flag] + if has_flag(flag, total_flags) && world1_const(stmt) + ssa_rename[idx] = stmt + else + ssa_rename[idx] = SSAValue(result_idx) + result[result_idx][:stmt] = stmt + result_idx += 1 + end + elseif isa(stmt, Core.BindingPartition) + # A constant binding can be moved into argument position. total_flags = IR_FLAG_CONSISTENT | IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW flag = result[result_idx][:flag] if has_flag(flag, total_flags) @@ -1502,14 +1633,14 @@ function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instr if cond ssa_rename[idx] = nothing result[result_idx][:stmt] = nothing - kill_edge!(compact, active_bb, active_bb, stmt.dest) + kill_edge_terminator!(compact, active_bb, active_bb, stmt.dest) # Don't increment result_idx => Drop this statement else label = bb_rename_succ[stmt.dest] @assert label > 0 ssa_rename[idx] = SSAValue(result_idx) result[result_idx][:stmt] = GotoNode(label) - kill_edge!(compact, active_bb, active_bb, active_bb+1) + kill_edge_terminator!(compact, active_bb, active_bb, active_bb+1) result_idx += 1 end else @@ -1582,7 +1713,7 @@ function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instr ssa_rename[idx] = pi_val return result_idx end - elseif !isa(pi_val, AnySSAValue) && !isa(pi_val, GlobalRef) + elseif !isa(pi_val, AnySSAValue) && !isa(pi_val, GlobalRef) && !isa(pi_val, Core.BindingPartition) pi_val′ = isa(pi_val, QuoteNode) ? pi_val.value : pi_val stmttyp = stmt.typ if isa(stmttyp, Const) ? pi_val′ === stmttyp.val : typeof(pi_val′) === stmttyp @@ -1684,7 +1815,10 @@ function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instr stmt = ssa_rename[stmt.id] end elseif isa(stmt, NewSSAValue) - stmt = SSAValue(stmt.id) + if stmt.id > 0 + # Negative ids reference new_new_nodes and must remain NewSSAValue. + stmt = SSAValue(stmt.id) + end else # Constant assign, replace uses of this ssa value with its result end diff --git a/Compiler/src/ssair/irinterp.jl b/Compiler/src/ssair/irinterp.jl index 3d72da72625be..25f9388fb3b43 100644 --- a/Compiler/src/ssair/irinterp.jl +++ b/Compiler/src/ssair/irinterp.jl @@ -1,12 +1,21 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -function collect_limitations!(@nospecialize(typ), ::IRInterpretationState) - @assert !isa(typ, LimitedAccuracy) "irinterp is unable to handle heavy recursion correctly" +function collect_limitations!(@nospecialize(typ), sv::IRInterpretationState) + if isa(typ, LimitedAccuracy) + parent = frame_parent(sv) + while parent isa IRInterpretationState + parent = frame_parent(parent) + end + if parent isa InferenceState + union!(parent.pclimitations, typ.causes) + end + return typ.typ + end return typ end function concrete_eval_invoke(interp::AbstractInterpreter, ci::CodeInstance, argtypes::Vector{Any}, parent::IRInterpretationState) - world = frame_world(parent) + world = get_inference_world(interp) effects = decode_effects(ci.ipo_purity_bits) if (is_foldable(effects) && is_all_const_arg(argtypes, #=start=#1) && (is_nonoverlayed(interp) || is_nonoverlayed(effects))) @@ -18,14 +27,24 @@ function concrete_eval_invoke(interp::AbstractInterpreter, ci::CodeInstance, arg end return Pair{Any,Tuple{Bool,Bool}}(Const(value), (true, true)) else - mi = ci.def + mi = get_ci_mi(ci) if is_constprop_edge_recursed(mi, parent) return Pair{Any,Tuple{Bool,Bool}}(nothing, (is_nothrow(effects), is_noub(effects))) end - newirsv = IRInterpretationState(interp, ci, mi, argtypes, world) + src = ci_get_source(interp, ci) + newirsv = IRInterpretationState(interp, ci, mi, argtypes, src) if newirsv !== nothing assign_parentchild!(newirsv, parent) - return ir_abstract_constant_propagation(interp, newirsv) + result = ir_abstract_constant_propagation(interp, newirsv) + update_valid_age!(parent, world, newirsv.valid_worlds) + proof_edges = Any[] + for info in newirsv.ir.stmts.info + add_edges!(proof_edges, info) + end + append!(proof_edges, newirsv.edges) + proof = LocalInferenceProof(newirsv.valid_worlds, Core.svec(proof_edges...)) + add_inference_proof!(parent.edges, proof) + return result end return Pair{Any,Tuple{Bool,Bool}}(nothing, (is_nothrow(effects), is_noub(effects))) end @@ -35,13 +54,15 @@ function abstract_eval_invoke_inst(interp::AbstractInterpreter, inst::Instructio stmt = inst[:stmt]::Expr ci = stmt.args[1] if ci isa MethodInstance - world = frame_world(irsv) - mi_cache = WorldView(code_cache(interp), world) + mi_cache = code_cache(interp) code = get(mi_cache, ci, nothing) code === nothing && return Pair{Any,Tuple{Bool,Bool}}(nothing, (false, false)) + code = code::CodeInstance else code = ci::CodeInstance end + update_valid_age!(irsv, get_inference_world(interp), proof_worlds(code)) + add_inference_proof!(irsv.edges, code) argtypes = collect_argtypes(interp, stmt.args[2:end], StatementState(nothing, false), irsv) argtypes === nothing && return Pair{Any,Tuple{Bool,Bool}}(Bottom, (false, false)) return concrete_eval_invoke(interp, code, argtypes, irsv) @@ -55,7 +76,7 @@ end function abstract_call(interp::AbstractInterpreter, arginfo::ArgInfo, sstate::StatementState, irsv::IRInterpretationState) si = StmtInfo(true, sstate.saw_latestworld) # TODO better job here? - call = abstract_call(interp, arginfo, si, irsv)::Future + call = abstract_call(interp, arginfo, si, sstate.vtypes, irsv)::Future Future{Any}(call, interp, irsv) do call, interp, irsv irsv.ir.stmts[irsv.curridx][:info] = call.info nothing @@ -148,7 +169,7 @@ function reprocess_instruction!(interp::AbstractInterpreter, inst::Instruction, rt = nothing if isa(stmt, Expr) head = stmt.head - if (head === :call || head === :foreigncall || head === :new || head === :splatnew || + if (head === :call || head === :foreigncall || head === :foreignglobal || head === :new || head === :splatnew || head === :static_parameter || head === :isdefined || head === :boundscheck) @assert isempty(irsv.tasks) # TODO: this whole function needs to be converted to a stackless design to be a valid AbsIntState, but this should work here for now result = abstract_eval_statement_expr(interp, stmt, StatementState(nothing, false), irsv) @@ -186,6 +207,9 @@ function reprocess_instruction!(interp::AbstractInterpreter, inst::Instruction, return false elseif head === :leave return false + elseif head === :(=) + # a store (which should be to a global at this point) is not refinable + return false else Core.println(stmt) error("reprocess_instruction!: unhandled expression found") @@ -294,7 +318,7 @@ end function populate_def_use_map!(tpdum::TwoPhaseDefUseMap, scanner::BBScanner) scan!(scanner, false) do inst::Instruction, lstmt::Int, bb::Int - for ur in userefs(inst) + for ur in userefs(inst[:stmt]) val = ur[] if isa(val, SSAValue) push!(tpdum[val.id], inst.idx) @@ -458,7 +482,7 @@ function ir_abstract_constant_propagation(interp::AbstractInterpreter, irsv::IRI end if irsv.frameid != 0 - callstack = irsv.callstack::Vector{AbsIntState} + callstack = irsv.callstack @assert callstack[end] === irsv && length(callstack) == irsv.frameid pop!(callstack) end diff --git a/Compiler/src/ssair/legacy.jl b/Compiler/src/ssair/legacy.jl index 64a39c72ad9eb..540b17c181c98 100644 --- a/Compiler/src/ssair/legacy.jl +++ b/Compiler/src/ssair/legacy.jl @@ -5,7 +5,7 @@ inflate_ir!(ci::CodeInfo, sptypes::Vector{VarState}, argtypes::Vector{Any}) -> ir::IRCode Inflates `ci::CodeInfo`-IR to `ir::IRCode`-format. -This should be used with caution as it is a in-place transformation where the fields of +This should be used with caution as it is an in-place transformation where the fields of the original `ci::CodeInfo` are modified. """ function inflate_ir!(ci::CodeInfo, mi::MethodInstance) @@ -13,7 +13,7 @@ function inflate_ir!(ci::CodeInfo, mi::MethodInstance) if ci.slottypes === nothing argtypes = va_process_argtypes(fallback_lattice, matching_cache_argtypes(fallback_lattice, mi), - ci.nargs, ci.isva) + ci.nargs, ci.isva, mi) else argtypes = ci.slottypes[1:ci.nargs] end diff --git a/Compiler/src/ssair/passes.jl b/Compiler/src/ssair/passes.jl index f16163554b75b..b580ee34039f2 100644 --- a/Compiler/src/ssair/passes.jl +++ b/Compiler/src/ssair/passes.jl @@ -2,8 +2,7 @@ function is_known_call(@nospecialize(x), @nospecialize(func), ir::Union{IRCode,IncrementalCompact}) isexpr(x, :call) || return false - ft = argextype(x.args[1], ir) - return singleton_type(ft) === func + return _global_call_singleton(x.args[1], ir) === func end function is_known_invoke_or_call(@nospecialize(x), @nospecialize(func), ir::Union{IRCode,IncrementalCompact}) @@ -11,8 +10,7 @@ function is_known_invoke_or_call(@nospecialize(x), @nospecialize(func), ir::Unio (isinvoke || isexpr(x, :call)) || return false narg = isinvoke ? 2 : 1 length(x.args) < narg && return false - ft = argextype(x.args[narg], ir) - return singleton_type(ft) === func + return _global_call_singleton(x.args[narg], ir) === func end struct SSAUse @@ -185,7 +183,7 @@ function find_def_for_use( end function collect_leaves(compact::IncrementalCompact, @nospecialize(val), @nospecialize(typeconstraint), 𝕃ₒ::AbstractLattice, - predecessors::Pre = ((@nospecialize(def), compact::IncrementalCompact) -> isa(def, PhiNode) ? def.values : nothing)) where {Pre} + predecessors::Pre = ((@nospecialize(def), _compact::IncrementalCompact) -> isa(def, PhiNode) ? def.values : nothing)) where {Pre} if isa(val, Union{OldSSAValue, SSAValue}) val, typeconstraint = simple_walk_constraint(compact, val, typeconstraint) end @@ -195,10 +193,10 @@ end abstract type WalkerCallback end struct TrivialWalker <: WalkerCallback end -(::TrivialWalker)(@nospecialize(def), @nospecialize(defssa::AnySSAValue)) = nothing +(::TrivialWalker)(@nospecialize(_def), @nospecialize(_defssa::AnySSAValue)) = nothing struct PiWalker <: WalkerCallback end -function (::PiWalker)(@nospecialize(def), @nospecialize(defssa::AnySSAValue)) +function (::PiWalker)(@nospecialize(def), @nospecialize(_defssa::AnySSAValue)) if isa(def, PiNode) return LiftedValue(def.val) end @@ -228,7 +226,7 @@ function simple_walk(compact::IncrementalCompact, @nospecialize(defssa::AnySSAVa is_old(compact, defssa) && (def = OldSSAValue(def.id)) end defssa = def - elseif isa(def, Union{PhiNode, PhiCNode, GlobalRef}) + elseif isa(def, Union{PhiNode, PhiCNode, GlobalRef, Core.BindingPartition}) return defssa else new_def = walker_callback(def, defssa) @@ -250,7 +248,7 @@ mutable struct TypeConstrainingWalker <: WalkerCallback typeconstraint::Any TypeConstrainingWalker(@nospecialize(typeconstraint::Any)) = new(typeconstraint) end -function (walker_callback::TypeConstrainingWalker)(@nospecialize(def), @nospecialize(defssa::AnySSAValue)) +function (walker_callback::TypeConstrainingWalker)(@nospecialize(def), @nospecialize(_defssa::AnySSAValue)) if isa(def, PiNode) walker_callback.typeconstraint = typeintersect(walker_callback.typeconstraint, widenconst(def.typ)) @@ -304,7 +302,7 @@ function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospe if is_old(compact, defssa) && isa(val, SSAValue) val = OldSSAValue(val.id) end - edge_typ = widenconst(argextype(val, compact)) + edge_typ = argextype_widened(val, compact) hasintersect(edge_typ, typeconstraint) || continue push!(possible_predecessors, n) end @@ -351,7 +349,7 @@ function record_immutable_preserve!(new_preserves::Vector{Any}, def::Expr, compa args = isexpr(def, :new) ? def.args : def.args[2:end] for i = 1:length(args) arg = args[i] - if !isbitstype(widenconst(argextype(arg, compact))) + if !isbitstype(argextype_widened(arg, compact)) push!(new_preserves, arg) end end @@ -361,20 +359,25 @@ function already_inserted(compact::IncrementalCompact, old::OldSSAValue) already_inserted_ssa(compact, compact.idx-1)(0, old) end -function already_inserted_ssa(compact::IncrementalCompact, processed_idx::Int) - return function did_already_insert(phi_arg::Int, old::OldSSAValue) - id = old.id - if id <= length(compact.ir.stmts) - return id <= processed_idx - end - id -= length(compact.ir.stmts) - if id <= length(compact.ir.new_nodes) - return did_already_insert(phi_arg, OldSSAValue(compact.ir.new_nodes.info[id].pos)) - end - id -= length(compact.ir.new_nodes) - @assert id <= length(compact.pending_nodes) - return !(id in compact.pending_perm) +function _already_inserted_ssa(compact::IncrementalCompact, processed_idx::Int, + phi_arg::Int, old::OldSSAValue) + id = old.id + if id <= length(compact.ir.stmts) + return id <= processed_idx end + id -= length(compact.ir.stmts) + if id <= length(compact.ir.new_nodes) + return _already_inserted_ssa(compact, processed_idx, phi_arg, + OldSSAValue(compact.ir.new_nodes.info[id].pos)) + end + id -= length(compact.ir.new_nodes) + @assert id <= length(compact.pending_nodes) + return !(id in compact.pending_perm) +end + +function already_inserted_ssa(compact::IncrementalCompact, processed_idx::Int) + return (phi_arg::Int, old::OldSSAValue) -> + _already_inserted_ssa(compact, processed_idx, phi_arg, old) end function is_pending(compact::IncrementalCompact, old::OldSSAValue) @@ -437,7 +440,7 @@ function lift_leaves(compact::IncrementalCompact, field::Int, lift_arg!(compact, leaf, cache_key, def, 1+field, lifted_leaves) continue # NOTE we can enable this, but most `:splatnew` expressions are transformed into - # `:new` expressions by the inlinear + # `:new` expressions by the inliner # elseif isexpr(def, :splatnew) && length(def.args) == 2 && isa(def.args[2], AnySSAValue) # tplssa = def.args[2]::AnySSAValue # tplexpr = compact[tplssa][:stmt] @@ -475,7 +478,7 @@ function lift_leaves(compact::IncrementalCompact, field::Int, end elseif isa(leaf, QuoteNode) leaf = leaf.value - elseif isa(leaf, GlobalRef) + elseif isa(leaf, GlobalRef) || isa(leaf, Core.BindingPartition) typ = argextype(leaf, compact) if isa(typ, Const) leaf = typ.val @@ -560,6 +563,13 @@ function lift_comparison!(::typeof(===), compact::IncrementalCompact, lhs, rhs = args[2], args[3] vl = argextype(lhs, compact) vr = argextype(rhs, compact) + result = egal_tfunc(𝕃ₒ, vl, vr) + if isa(result, Const) + compact[idx] = result.val + compact[SSAValue(idx)][:type] = result + add_flag!(compact[SSAValue(idx)], IR_FLAG_REFINED) + return + end if isa(vl, Const) isa(vr, Const) && return val = rhs @@ -601,7 +611,7 @@ end function lift_comparison_leaves!(@specialize(tfunc), compact::IncrementalCompact, @nospecialize(val), @nospecialize(cmp), idx::Int, 𝕃ₒ::AbstractLattice) - typeconstraint = widenconst(argextype(val, compact)) + typeconstraint = argextype_widened(val, compact) if isa(val, Union{OldSSAValue, SSAValue}) val, typeconstraint = simple_walk_constraint(compact, val, typeconstraint) end @@ -625,9 +635,8 @@ function lift_comparison_leaves!(@specialize(tfunc), end end - # perform lifting (lifted_val, nest) = perform_lifting!(compact, - visited_philikes, cmp, Bool, lifted_leaves::LiftedLeaves, val, nothing) + visited_philikes, cmp, nothing, Bool, lifted_leaves::LiftedLeaves, val, nothing) compact[idx] = (lifted_val::LiftedValue).val @@ -776,12 +785,14 @@ end function perform_lifting!(compact::IncrementalCompact, visited_philikes::Vector{AnySSAValue}, @nospecialize(cache_key), + lifting_cache::Union{IdDict{Pair{AnySSAValue,Pair{Any,Any}}, AnySSAValue}, Nothing}, @nospecialize(result_t), lifted_leaves::Union{LiftedLeaves, LiftedDefs}, @nospecialize(stmt_val), lazydomtree::Union{LazyDomtree,Nothing}, walker_callback::WalkerCallback = TrivialWalker()) reverse_mapping = IdDict{AnySSAValue, Int}() for id in 1:length(visited_philikes) reverse_mapping[visited_philikes[id]] = id end + @assert !isempty(lifted_leaves) "perform_lifting! requires at least one lifted leaf" # Check if all the lifted leaves are the same local the_leaf @@ -796,6 +807,7 @@ function perform_lifting!(compact::IncrementalCompact, end end + @assert @isdefined(the_leaf) if all_same && isa(the_leaf, LiftedValue) dominates_all = true the_leaf_val = the_leaf.val @@ -828,17 +840,37 @@ function perform_lifting!(compact::IncrementalCompact, if isa(the_leaf_val, OldSSAValue) the_leaf = LiftedValue(simple_walk(compact, the_leaf_val)) end - return Pair{Any, PhiNest}(the_leaf, PhiNest(visited_philikes, Vector{LiftedPhilike}(undef, 0), lifted_leaves, reverse_mapping, walker_callback)) + return Pair{Union{LiftedValue,Nothing}, PhiNest}(the_leaf, PhiNest(visited_philikes, Vector{LiftedPhilike}(undef, 0), lifted_leaves, reverse_mapping, walker_callback)) end end # Insert PhiNodes nphilikes = length(visited_philikes) lifted_philikes = Vector{LiftedPhilike}(undef, nphilikes) + # The cache key includes stmt_val to distinguish liftings through the same phi + # node with different type constraints (e.g. direct access vs through a PiNode). + # stmt_val has not been walked yet at this point, so it reflects the original + # getfield argument and thus the type constraint used by collect_leaves. + local cache_key_with_val = nothing + if lifting_cache !== nothing + cache_key_with_val = Pair{Any,Any}(cache_key, stmt_val) + end for i = 1:nphilikes old_ssa = visited_philikes[i] old_inst = compact[old_ssa] old_node = old_inst[:stmt]::Union{PhiNode,Expr} + if lifting_cache !== nothing && cache_key_with_val !== nothing + ckey = Pair{AnySSAValue,Pair{Any,Any}}(old_ssa, cache_key_with_val) + if ckey in keys(lifting_cache) + ssa = lifting_cache[ckey] + if isa(old_node, PhiNode) + lifted_philikes[i] = LiftedPhilike(ssa, old_node, false) + else + lifted_philikes[i] = LiftedPhilike(ssa, IfElseCall(old_node), false) + end + continue + end + end if isa(old_node, PhiNode) new_node = PhiNode() ssa = insert_node!(compact, old_ssa, removable_if_unused(NewInstruction(new_node, result_t))) @@ -856,22 +888,82 @@ function perform_lifting!(compact::IncrementalCompact, ssa = insert_node!(compact, old_ssa, new_inst, #= attach_after =# true) lifted_philikes[i] = LiftedPhilike(ssa, IfElseCall(new_node), true) end + if lifting_cache !== nothing && cache_key_with_val !== nothing + ckey = Pair{AnySSAValue,Pair{Any,Any}}(old_ssa, cache_key_with_val) + lifting_cache[ckey] = ssa + end end # Fixup the stmt itself if isa(stmt_val, Union{SSAValue, OldSSAValue}) stmt_val = simple_walk(compact, stmt_val, LiftedLeaveWalker(lifted_leaves, reverse_mapping, walker_callback)) end + stmt_val = resolve_lifted_stmt_val(stmt_val, lifted_leaves, reverse_mapping, lifted_philikes) - if stmt_val in keys(lifted_leaves) - stmt_val = lifted_leaves[stmt_val] - elseif isa(stmt_val, AnySSAValue) && stmt_val in keys(reverse_mapping) - stmt_val = LiftedValue(lifted_philikes[reverse_mapping[stmt_val]].ssa) - else - error() + return Pair{Union{LiftedValue,Nothing}, PhiNest}(stmt_val, PhiNest(visited_philikes, lifted_philikes, lifted_leaves, reverse_mapping, walker_callback)) +end + +function resolve_lifted_stmt_val(@nospecialize(stmt_val), lifted_leaves::LiftedLeaves, + reverse_mapping::IdDict{AnySSAValue, Int}, + lifted_philikes::Vector{LiftedPhilike}) + if haskey(lifted_leaves, stmt_val) + return lifted_leaves[stmt_val] + elseif isa(stmt_val, AnySSAValue) && haskey(reverse_mapping, stmt_val) + return LiftedValue(lifted_philikes[reverse_mapping[stmt_val]].ssa) end + error("failed to resolve lifted statement value") +end +function resolve_lifted_stmt_val(@nospecialize(stmt_val), lifted_defs::LiftedDefs, + reverse_mapping::IdDict{AnySSAValue, Int}, + lifted_philikes::Vector{LiftedPhilike}) + @assert !haskey(lifted_defs, stmt_val) "definition lifting should only resolve through a lifted phi-like value" + if isa(stmt_val, AnySSAValue) && haskey(reverse_mapping, stmt_val) + return LiftedValue(lifted_philikes[reverse_mapping[stmt_val]].ssa) + end + error("failed to resolve lifted definition value") +end - return Pair{Any, PhiNest}(stmt_val, PhiNest(visited_philikes, lifted_philikes, lifted_leaves, reverse_mapping, walker_callback)) +# Handle _apply_iterate calls: convert arguments to use `Core.svec`. +# The behavior of `Core.svec` (with boxing) better matches the ABI of codegen. +function lift_apply_args!(compact::IncrementalCompact, idx::Int, stmt::Expr) + compact[idx] = nothing + for i in 4:length(stmt.args) # Skip `_apply_iterate`, `iterate`, and the function + arg = stmt.args[i] + arg_type = argextype_widened(arg, compact) + if isa(arg_type, DataType) && arg_type.name === Tuple.name + svec_args = nothing + if isa(arg, SSAValue) + arg_stmt = compact[arg][:stmt] + if is_known_call(arg_stmt, Core.tuple, compact) + svec_args = copy(arg_stmt.args) + end + end + if svec_args === nothing + # Fallback path: generate getfield calls for tuple elements + tuple_length = length(arg_type.parameters) + if tuple_length > 0 && !isvarargtype(arg_type.parameters[tuple_length]) + svec_args = Vector{Any}(undef, tuple_length + 1) + for j in 1:tuple_length + getfield_call = Expr(:call, GlobalRef(Core, :getfield), arg, j) + getfield_type = arg_type.parameters[j] + inst = compact[SSAValue(idx)] + getfield_ssa = insert_node!(compact, SSAValue(idx), NewInstruction(getfield_call, getfield_type, NoCallInfo(), inst[:line], inst[:flag])) + svec_args[j + 1] = getfield_ssa + end + end + end + if svec_args !== nothing + svec_args[1] = GlobalRef(Core, :svec) + new_svec_call = Expr(:call) + new_svec_call.args = svec_args + inst = compact[SSAValue(idx)] + new_svec_ssa = insert_node!(compact, SSAValue(idx), NewInstruction(new_svec_call, SimpleVector, NoCallInfo(), inst[:line], inst[:flag])) + stmt.args[i] = new_svec_ssa + end + end + end + compact[idx] = stmt + nothing end function lift_svec_ref!(compact::IncrementalCompact, idx::Int, stmt::Expr) @@ -945,15 +1037,14 @@ function keyvalue_predecessors(@nospecialize(key), 𝕃ₒ::AbstractLattice) if length(def.args) in (5, 6) collection = def.args[end-2] set_key = def.args[end-1] - set_val_idx = length(def.args) elseif length(def.args) == 4 - collection = def.args[end-1] # Key is deleted + # collection = def.args[end-1] # TODO: Model this return nothing elseif length(def.args) == 3 - collection = def.args[end] # The whole collection is deleted + # collection = def.args[end] # TODO: Model this return nothing else @@ -972,7 +1063,7 @@ end struct KeyValueWalker <: WalkerCallback compact::IncrementalCompact end -function (walker_callback::KeyValueWalker)(@nospecialize(def), @nospecialize(defssa::AnySSAValue)) +function (walker_callback::KeyValueWalker)(@nospecialize(def), @nospecialize(_defssa::AnySSAValue)) if is_known_invoke_or_call(def, Core.OptimizedGenerics.KeyValue.set, walker_callback.compact) @assert length(def.args) in (5, 6) return LiftedValue(def.args[end-2]) @@ -996,17 +1087,29 @@ function lift_keyvalue_get!(compact::IncrementalCompact, idx::Int, stmt::Expr, result_t = tmerge(𝕃ₒ, result_t, argextype(v.val, compact)) end + # Extract the wrapper type (e.g. Some{V}) from the inferred return type + # Union{Nothing, Some{V}} by subtracting Nothing. Bail out if the result + # is not a valid single-field concrete wrapper type. + get_rtype = widenconst(compact[SSAValue(idx)][:type]) + wrapper_typ = typesubtract(get_rtype, Nothing, 0) + isconcretetype(wrapper_typ) || return + fieldcount(wrapper_typ) == 1 || return + ⊑(𝕃ₒ, result_t, fieldtype(wrapper_typ, 1)) || return + (lifted_val, nest) = perform_lifting!(compact, - visited_philikes, key, result_t, lifted_leaves, collection, nothing, + visited_philikes, key, nothing, result_t, lifted_leaves, collection, nothing, KeyValueWalker(compact)) - compact[idx] = lifted_val === nothing ? nothing : Expr(:call, GlobalRef(Core, :tuple), lifted_val.val) - finish_phi_nest!(compact, nest) if lifted_val !== nothing - if !⊑(𝕃ₒ, compact[SSAValue(idx)][:type], tuple_tfunc(𝕃ₒ, Any[result_t])) - add_flag!(compact[SSAValue(idx)], IR_FLAG_REFINED) - end + newexpr = Expr(:new, wrapper_typ, lifted_val.val) + compact[idx] = newexpr + compact[SSAValue(idx)][:type] = wrapper_typ + add_flag!(compact[SSAValue(idx)], IR_FLAG_REFINED) + refine_new_effects!(𝕃ₒ, compact, idx, newexpr) + else + compact[idx] = nothing end + finish_phi_nest!(compact, nest) return end @@ -1052,7 +1155,7 @@ end argdef = compact[rarg][:stmt] else isType(arg) || return nothing - arg = arg.parameters[1] + arg = type_parameter(arg) end is_known_call(argdef, Core.apply_type, compact) || return nothing @@ -1219,6 +1322,8 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) defuses = nothing # will be initialized once we encounter mutability in order to reduce dynamic allocations # initialization of domtree is delayed to avoid the expensive computation in many cases lazydomtree = LazyDomtree(ir) + lifting_cache = IdDict{Pair{AnySSAValue,Pair{Any,Any}}, AnySSAValue}() + def_lifting_cache = IdDict{Pair{AnySSAValue,Pair{Any,Any}}, AnySSAValue}() scope_mapping::Union{Vector{SSAValue}, Nothing} = nothing for ((old_idx, idx), stmt) in compact # If we encounter any EnterNode with set :scope, propagate the current scope for all basic blocks, so @@ -1244,7 +1349,7 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) if isa(stmt, EnterNode) if isdefined(stmt, :scope) if scope_mapping === nothing - scope_mapping = SSAValue[SSAValue(0) for i = 1:length(compact.cfg_transform.result_bbs)] + scope_mapping = SSAValue[SSAValue(0) for _ = 1:length(compact.cfg_transform.result_bbs)] end update_scope_mapping!(scope_mapping, bb+1, SSAValue(idx)) end @@ -1330,7 +1435,7 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) push!(preserved, preserved_arg.id) continue elseif isexpr(def, :new) - typ = unwrap_unionall(widenconst(argextype(SSAValue(defidx), compact))) + typ = unwrap_unionall(argextype_widened(SSAValue(defidx), compact)) if typ isa DataType && !ismutabletype(typ) record_immutable_preserve!(new_preserves, def, compact) push!(preserved, preserved_arg.id) @@ -1377,6 +1482,9 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) compact[SSAValue(idx)] = (compact[enter_ssa][:stmt]::EnterNode).scope elseif isexpr(stmt, :new) refine_new_effects!(𝕃ₒ, compact, idx, stmt) + elseif is_known_call(stmt, Core._apply_iterate, compact) + length(stmt.args) >= 4 || continue + lift_apply_args!(compact, idx, stmt) end continue end @@ -1387,15 +1495,14 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) # analyze `getfield` / `isdefined` / `setfield!` call val = stmt.args[2] end - struct_typ = widenconst(argextype(val, compact)) - struct_argtyp = argument_datatype(struct_typ) - if struct_argtyp === nothing + struct_typ = argextype_widened(val, compact) + struct_typ_name = argument_datatypename(struct_typ) + if struct_typ_name === nothing if isa(struct_typ, Union) && is_isdefined lift_comparison!(isdefined, compact, idx, stmt, 𝕃ₒ) end continue end - struct_typ_name = struct_argtyp.name struct_typ_name.atomicfields == C_NULL || continue # TODO: handle more if !((field_ordering === :unspecified) || @@ -1450,21 +1557,22 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) end (lifted_val, nest) = perform_lifting!(compact, - visited_philikes, field, result_t, lifted_leaves, val, lazydomtree) + visited_philikes, field, lifting_cache, result_t, lifted_leaves, val, lazydomtree) should_delete_node = false line = compact[SSAValue(idx)][:line] if lifted_val !== nothing && !⊑(𝕃ₒ, compact[SSAValue(idx)][:type], result_t) - compact[idx] = lifted_val === nothing ? nothing : lifted_val.val + compact[idx] = lifted_val.val + compact[SSAValue(idx)][:type] = result_t add_flag!(compact[SSAValue(idx)], IR_FLAG_REFINED) elseif lifted_val === nothing || isa(lifted_val.val, AnySSAValue) # Save some work in a later compaction, by inserting this into the renamer now, # but only do this if we didn't set the REFINED flag, to save work for irinterp # in revisiting only the renamings that came through *this* idx. - compact.ssa_rename[old_idx] = lifted_val === nothing ? nothing : lifted_val.val + compact.ssa_rename[old_idx] = lifted_val === nothing ? nothing : lifted_val.val::AnySSAValue should_delete_node = true else - compact[idx] = lifted_val === nothing ? nothing : lifted_val.val + compact[idx] = lifted_val.val end finish_phi_nest!(compact, nest) @@ -1479,7 +1587,7 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) lifted_leaves_def[k] = v === nothing ? false : true end (def_val, nest) = perform_lifting!(compact, - visited_philikes, field, Bool, lifted_leaves_def, val, lazydomtree) + visited_philikes, field, def_lifting_cache, Bool, lifted_leaves_def, val, lazydomtree) def_val = (def_val::LiftedValue).val finish_phi_nest!(compact, nest) end @@ -1516,7 +1624,7 @@ function sroa_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) end ir = complete(compact) # remove any use that has been optimized away by the DCE - for (intermediaries, defuse) in values(defuses) + for (_, defuse) in values(defuses) filter!(x -> ir[SSAValue(x.idx)][:stmt] !== nothing, defuse.uses) end sroa_mutables!(ir, defuses, used_ssas, lazydomtree, inlining) @@ -1532,7 +1640,7 @@ end function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, code::CodeInstance, @nospecialize(info::CallInfo), inlining::InliningState, attach_after::Bool) - mi = code.def + mi = get_ci_mi(code) et = InliningEdgeTracker(inlining) if code isa CodeInstance if use_const_api(code) @@ -1540,12 +1648,15 @@ function try_inline_finalizer!(ir::IRCode, argexprs::Vector{Any}, idx::Int, add_inlining_edge!(et, code) return true end - src = @atomic :monotonic code.inferred + # COMBAK: this has awkward and unreliable global cache effects, but + # this doesn't respect the bottom-up inliner order so we do not have + # CallInfo anymore. See `handle_finalizer_call!` too. + src = ci_get_source(inlining.interp, code) else return false end - src_inlining_policy(inlining.interp, src, info, IR_FLAG_NULL) || return false + src_inlining_policy(inlining.interp, mi, src, info, IR_FLAG_NULL) || return false src, spec_info, di = retrieve_ir_for_inlining(code, src) # For now: Require finalizer to only have one basic block @@ -1598,6 +1709,25 @@ function reachable_blocks(cfg::CFG, from_bb::Int, to_bb::Int) return visited end +function _update_finalizer_insert!(ir::IRCode, lazypostdomtree::LazyPostDomtree, + finalizer_idx::Int, insert_bb::Int, + insert_idx::Union{Int,Nothing}, x::Union{Int,SSAUse}) + defuse_idx = x isa SSAUse ? x.idx : x + defuse_idx == finalizer_idx && return insert_bb, insert_idx + defuse_bb = block_for_inst(ir, defuse_idx) + new_insert_bb = nearest_common_dominator(get!(lazypostdomtree), + insert_bb, defuse_bb) + if new_insert_bb == insert_bb && insert_idx !== nothing + insert_idx = max(insert_idx::Int, defuse_idx) + elseif new_insert_bb == defuse_bb + insert_idx = defuse_idx + else + insert_idx = nothing + end + insert_bb = new_insert_bb + return insert_bb, insert_idx +end + function try_resolve_finalizer!(ir::IRCode, alloc_idx::Int, finalizer_idx::Int, defuse::SSADefUse, inlining::InliningState, lazydomtree::LazyDomtree, lazypostdomtree::LazyPostDomtree, @nospecialize(info::CallInfo)) @@ -1619,24 +1749,12 @@ function try_resolve_finalizer!(ir::IRCode, alloc_idx::Int, finalizer_idx::Int, # Check #2: The insertion block for the finalizer is the post-dominator of all uses insert_bb::Int = finalizer_bb insert_idx::Union{Int,Nothing} = finalizer_idx - function note_defuse!(x::Union{Int,SSAUse}) - defuse_idx = x isa SSAUse ? x.idx : x - defuse_idx == finalizer_idx && return nothing - defuse_bb = block_for_inst(ir, defuse_idx) - new_insert_bb = nearest_common_dominator(get!(lazypostdomtree), - insert_bb, defuse_bb) - if new_insert_bb == insert_bb && insert_idx !== nothing - insert_idx = max(insert_idx::Int, defuse_idx) - elseif new_insert_bb == defuse_bb - insert_idx = defuse_idx - else - insert_idx = nothing - end - insert_bb = new_insert_bb - nothing + for x in defuse.uses + insert_bb, insert_idx = _update_finalizer_insert!(ir, lazypostdomtree, finalizer_idx, insert_bb, insert_idx, x) + end + for x in defuse.defs + insert_bb, insert_idx = _update_finalizer_insert!(ir, lazypostdomtree, finalizer_idx, insert_bb, insert_idx, x) end - foreach(note_defuse!, defuse.uses) - foreach(note_defuse!, defuse.defs) insert_bb != 0 || return nothing # verify post-dominator of all uses exists if !OptimizationParams(inlining.interp).assume_fatal_throw @@ -1673,7 +1791,7 @@ function try_resolve_finalizer!(ir::IRCode, alloc_idx::Int, finalizer_idx::Int, finalizer_stmt = ir[SSAValue(finalizer_idx)][:stmt] argexprs = Any[finalizer_stmt.args[2], finalizer_stmt.args[3]] - flag = info isa FinalizerInfo ? flags_for_effects(info.effects) : IR_FLAG_NULL + flag = isa(info, FinalizerInfo) ? flags_for_effects(info.effects) : IR_FLAG_NULL if length(finalizer_stmt.args) >= 4 inline = finalizer_stmt.args[4] if inline === nothing @@ -1683,7 +1801,7 @@ function try_resolve_finalizer!(ir::IRCode, alloc_idx::Int, finalizer_idx::Int, if inline::Bool && try_inline_finalizer!(ir, argexprs, loc, ci, info, inlining, attach_after) # the finalizer body has been inlined else - newinst = add_flag(NewInstruction(Expr(:invoke, ci, argexprs...), Nothing), flag) + newinst = add_flag(NewInstruction(Expr(:invoke, ci, argexprs...), Any), flag) insert_node!(ir, loc, newinst, attach_after) end end @@ -1717,10 +1835,9 @@ function sroa_mutables!(ir::IRCode, defuses::IdDict{Int,Tuple{SPCSet,SSADefUse}} # Find the type for this allocation defexpr = ir[SSAValue(defidx)][:stmt] isexpr(defexpr, :new) || continue - typ = unwrap_unionall(ir.stmts[defidx][:type]) # Could still end up here if we tried to setfield! on an immutable, which would # error at runtime, but is not illegal to have in the IR. - typ = widenconst(typ) + typ = unwrap_unionall(widenconst(ir.stmts[defidx][:type])) ismutabletype(typ) || continue typ = typ::DataType # Check if there are any uses we did not account for. If so, the variable @@ -1873,7 +1990,7 @@ function sroa_mutables!(ir::IRCode, defuses::IdDict{Int,Tuple{SPCSet,SSADefUse}} elseif use.kind === :preserve newval = compute_value_for_use(ir, domtree, allblocks, du, phinodes, fidx, use.idx) - if !isbitstype(widenconst(argextype(newval, ir))) + if !isbitstype(argextype_widened(newval, ir)) if preserve_uses === nothing preserve_uses = IdDict{Int, Vector{Any}}() end @@ -2144,7 +2261,7 @@ function adce_pass!(ir::IRCode, inlining::Union{Nothing,InliningState}=nothing) if !isassigned(stmt.values, i) # Should be impossible to have something used only by PiNodes that's undef push!(to_drop, i) - elseif !hasintersect(widenconst(argextype(stmt.values[i], compact)), + elseif !hasintersect(argextype_widened(stmt.values[i], compact), widenconst(t)) push!(to_drop, i) end @@ -2201,7 +2318,7 @@ function is_bb_empty(ir::IRCode, bb::BasicBlock) end # TODO: This is terrible, we should change the IR for GotoIfNot to gain an else case -function is_legal_bb_drop(ir::IRCode, bbidx::Int, bb::BasicBlock) +function is_legal_bb_drop(_ir::IRCode, bbidx::Int, _bb::BasicBlock) # For the time being, don't drop the first bb, because it has special predecessor semantics. bbidx == 1 && return false return true @@ -2529,14 +2646,13 @@ function cfg_simplify!(ir::IRCode) end cresult_bbs = BasicBlock[ - BasicBlock(StmtRange(bb_starts[i], - i+1 > length(bb_starts) ? length(compact.result) : bb_starts[i+1]-1), + BasicBlock(StmtRange(bb_starts[i], bb_starts[i+1]-1), compute_preds(bbs, result_bbs, bb_rename_pred, i), compute_succs(merged_succ, bbs, result_bbs, bb_rename_succ, i)) for i = 1:length(result_bbs)] # Fixup terminators for any blocks that would have caused double edges - for (bbidx, (new_bb, old_bb)) in enumerate(zip(cresult_bbs, result_bbs)) + for (bbidx, new_bb) in enumerate(cresult_bbs) @assert length(new_bb.succs) <= 2 length(new_bb.succs) <= 1 && continue if new_bb.succs[1] == new_bb.succs[2] @@ -2559,8 +2675,7 @@ function cfg_simplify!(ir::IRCode) # Run instruction compaction to produce the result, # but we're messing with the CFG # so we don't want compaction to do so independently - compact = IncrementalCompact(ir, CFGTransformState(true, false, cresult_bbs, bb_rename_pred, bb_rename_succ, nothing)) - result_idx = 1 + compact = IncrementalCompact(ir, CFGTransformState(true, false, cresult_bbs, bb_rename_pred, bb_rename_succ, nothing, nothing)) for (idx, orig_bb) in enumerate(result_bbs) ms = orig_bb bb_start = true @@ -2575,7 +2690,6 @@ function cfg_simplify!(ir::IRCode) compact.result[compact.result_idx][:stmt] = nothing elseif isa(stmt, PhiNode) phi = stmt - values = phi.values (; ssa_rename, late_fixup, used_ssas, new_new_used_ssas) = compact ssa_rename[i] = SSAValue(compact.result_idx) already_inserted = function (branch::Int, val::OldSSAValue) @@ -2640,7 +2754,7 @@ function cfg_simplify!(ir::IRCode) compact.result[compact.result_idx][:stmt] = nothing end end - # We always increase the result index to ensure a predicatable + # We always increase the result index to ensure a predictable # placement of the resulting nodes. compact.result_idx += 1 end diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index 6731054ee8f30..8ad436903a1f5 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -8,12 +8,12 @@ using Base, Core.IR import Base: show using Base: isexpr, prec_decl, show_unquoted, with_output_color -using .Compiler: ALWAYS_FALSE, ALWAYS_TRUE, argextype, BasicBlock, block_for_inst, - CachedMethodTable, CFG, compute_basic_blocks, DebugInfoStream, Effects, - EMPTY_SPTYPES, getdebugidx, IncrementalCompact, InferenceResult, InferenceState, - InvalidIRError, IRCode, LimitedAccuracy, NativeInterpreter, scan_ssa_use!, - singleton_type, sptypes_from_meth_instance, StmtRange, Timings, VarState, widenconst, - get_ci_mi, get_ci_abi +using .Compiler: ALWAYS_FALSE, ALWAYS_TRUE, BasicBlock, CFG, CachedMethodTable, + DebugInfoStream, EMPTY_SPTYPES, Effects, IRCode, IncrementalCompact, InferenceResult, + InferenceState, InvalidIRError, LimitedAccuracy, NativeInterpreter, StmtRange, + Timings, VarState, argextype, block_for_inst, compute_basic_blocks, edge_debuginfo, + get_ci_abi, get_ci_mi, has_prev_debuginfo, prev_debuginfo, scan_ssa_use!, + singleton_type, source_location, sptypes_from_meth_instance, widenconst @nospecialize @@ -48,6 +48,20 @@ end const inlined_apply_iterate_types = Union{Array,Memory,Tuple,NamedTuple,Core.SimpleVector} +# There is no @enum at this level of bootstrapping yet, so we use plain constants instead. +const SSAWarnTypeClass = UInt8 +const SSA_WARN_TYPE_STABLE = SSAWarnTypeClass(0) +const SSA_WARN_TYPE_MILD = SSAWarnTypeClass(1) +const SSA_WARN_TYPE_STRONG = SSAWarnTypeClass(2) + +function ssa_warn_type_class(io::IO, idx::Int) + levels = get(io, :ssa_warn_levels, nothing) + if levels isa AbstractVector{SSAWarnTypeClass} && isassigned(levels, idx) + return levels[idx] + end + return SSA_WARN_TYPE_STABLE +end + function builtin_call_has_dispatch( @nospecialize(f), args::Vector{Any}, @@ -76,16 +90,25 @@ function builtin_call_has_dispatch( end function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,CodeInfo,IncrementalCompact}, - sptypes::Vector{VarState}, used::BitSet, maxlength_idx::Int, color::Bool, show_type::Bool, label_dynamic_calls::Bool) + sptypes::Vector{VarState}, used::BitSet, maxlength_idx::Int, color::Bool, show_type::Bool, label_dynamic_calls::Bool, color_warntype::Bool) if idx in used idx_s = string(idx) pad = " "^(maxlength_idx - length(idx_s) + 1) - print(io, "%", idx_s, pad, "= ") + cls = ssa_warn_type_class(io, idx) + if color && color_warntype && cls === SSA_WARN_TYPE_STRONG + printstyled(io, "%", idx_s; color=:light_red, bold=true) + elseif color && color_warntype && cls === SSA_WARN_TYPE_MILD + printstyled(io, "%", idx_s; color=Base.warn_color(), bold=true) + else + print(io, "%", idx_s) + end + print(io, pad, "= ") else print(io, " "^(maxlength_idx + 4)) end # TODO: `indent` is supposed to be the full width of the leader for correct alignment indent = 16 + io = color_warntype ? io : Base.IOContext(io, :ssa_warn_levels => nothing) if !color && stmt isa PiNode # when the outer context is already colored (green, for pending nodes), don't use the usual coloring printer print(io, "π (") @@ -99,7 +122,6 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,Co ci = stmt.args[1] if ci isa Core.CodeInstance printstyled(io, " invoke "; color = :light_black) - mi = get_ci_mi(ci) abi = get_ci_abi(ci) else printstyled(io, "dynamic invoke "; color = :yellow) @@ -115,7 +137,7 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,Co skip_ftype = (length(sig) == 0) # doesn't exist... skip_ftype = skip_ftype || ( # ... or, f prints as a user-accessible value... - (f isa GlobalRef) && + (f isa GlobalRef || f isa Core.BindingPartition) && # ... and matches the value of the singleton type of the invoked MethodInstance (singleton_type(ft) === singleton_type(sig[1]) !== nothing) ) @@ -221,7 +243,7 @@ end """ Compute line number annotations for an IRCode or CodeInfo. -This functions compute three sets of annotations for each IR line. Take the following +This function computes three sets of annotations for each IR line. Take the following example (taken from `@code_typed sin(1.0)`): ``` @@ -282,7 +304,6 @@ function compute_ir_line_annotations(code::Union{IRCode,CodeInfo}) loc_annotations = String[] loc_methods = String[] loc_lineno = String[] - cur_group = 1 last_lineno = 0 last_stack = LineInfoNode[] # nb. only file, line, and method are populated in this last_printed_depth = 0 @@ -301,7 +322,7 @@ function compute_ir_line_annotations(code::Union{IRCode,CodeInfo}) x = min(length(last_stack), length(stack)) depth = length(stack) - 1 # Compute the last depth that was in common - first_mismatch = let last_stack=last_stack + first_mismatch = let last_stack=last_stack, stack=stack findfirst(i->last_stack[i] != stack[i], 1:x) end # If the first mismatch is the last stack frame, that might just @@ -360,7 +381,7 @@ end Base.show(io::IO, code::Union{IRCode, IncrementalCompact}) = show_ir(io, code) # A line_info_preprinter for disabling line info printing -lineinfo_disabled(io::IO, linestart::String, idx::Int) = "" +lineinfo_disabled(::IO, _linestart::String, _idx::Int) = "" # utility function to extract the file name from a DebugInfo object function debuginfo_file1(debuginfo::Union{DebugInfo,DebugInfoStream}) @@ -378,14 +399,18 @@ function debuginfo_file1(debuginfo::Union{DebugInfo,DebugInfoStream}) end # utility function to extract the first line number and file of a block of code -function debuginfo_firstline(debuginfo::Union{DebugInfo,DebugInfoStream}) - linetable = debuginfo.linetable - while linetable != nothing - debuginfo = linetable - linetable = debuginfo.linetable +function debuginfo_firstline(di::DebugInfoStream) + if di.linetable isa DebugInfo + di = di.linetable + debuginfo_firstline(di) + else + # likely doesn't contain any usable provenance + debuginfo_file1(di), di.firstline end - codeloc = getdebugidx(debuginfo, 0) - return debuginfo_file1(debuginfo), codeloc[1] +end +function debuginfo_firstline(di::DebugInfo) + firstline = ccall(:jl_cdi_firstline_all, Int32, (Any,), di) + debuginfo_file1(di), firstline end struct LineInfoNode @@ -394,30 +419,31 @@ struct LineInfoNode line::Int32 end -# utility function for converting a debuginfo object a particular pc to list of LineInfoNodes representing the inlining info at that pc for function `def` +# utility function for converting a debuginfo object at a particular pc to a list of LineInfoNodes representing the inlining info at that pc for function `def` # which is either `nothing` (macro-expand), a module (top-level), a Method (unspecialized code) or a MethodInstance (specialized code) # Returns `false` if the line info should not be updated with this info because this # statement has no effect on the line numbers. The `scopes` will still be populated however # with as much information as was available about the inlining at that statement. -function append_scopes!(scopes::Vector{LineInfoNode}, pc::Int, debuginfo, @nospecialize(def)) +function append_scopes!(scopes::Vector{LineInfoNode}, pc::Int, di, @nospecialize(def)) doupdate = true - while true - debuginfo.def isa Symbol || (def = debuginfo.def) - codeloc = getdebugidx(debuginfo, pc) - line::Int = codeloc[1] - inl_to::Int = codeloc[2] - doupdate &= line != 0 || inl_to != 0 # disabled debug info--no update - if debuginfo.linetable === nothing || pc <= 0 || line < 0 - line < 0 && (doupdate = false; line = 0) # broken debug info - push!(scopes, LineInfoNode(def, debuginfo_file1(debuginfo), Int32(line))) + while di !== nothing + di.def isa Symbol || (def = di.def) + if pc <= 0 + # TODO: assert false + return false + elseif !has_prev_debuginfo(di, pc) + line = source_location(di, pc).line # TODO: column ignored here + (line <= 0) && (doupdate = false; line = 0) # broken debug info + push!(scopes, LineInfoNode(def, debuginfo_file1(di), Int32(line))) else - doupdate = append_scopes!(scopes, line, debuginfo.linetable::DebugInfo, def) && doupdate + di2, pc2 = prev_debuginfo(di, pc) + doupdate &= append_scopes!(scopes, pc2, di2, def) end - inl_to == 0 && return doupdate def = :var"macro expansion" - debuginfo = debuginfo.edges[inl_to] - pc::Int = codeloc[3] + di, pc = edge_debuginfo(di, pc) + doupdate |= di !== nothing end + return doupdate end # utility wrapper around `append_scopes!` that returns an empty list instead of false @@ -459,7 +485,6 @@ function DILineInfoPrinter(debuginfo, def, showtypes::Bool=false) #context_depth[] = 0 nframes = length(DI) nctx::Int = 0 - pop_skips = 0 # compute the size of the matching prefix in the inlining information stack for i = 1:min(length(context), nframes) CtxLine = context[i] @@ -534,37 +559,33 @@ function DILineInfoPrinter(debuginfo, def, showtypes::Bool=false) started::Bool = false if !update_line_only && showtypes && !isa(frame.method, Symbol) && nctx != 1 print(io, linestart) - with_output_color(linecolor, io) do io - print(io, indent("│")) - print(io, "┌ invoke ", frame.method) - println(io) - end + printstyled(io, indent("│"), color=linecolor) + printstyled(io, "┌ invoke ", frame.method, color=linecolor) + println(io) started = true end print(io, linestart) - with_output_color(linecolor, io) do io - print(io, indent("│")) - push!(context, frame) - if update_line_only - update_line_only = false - else - context_depth[] += 1 - nctx != 1 && print(io, started ? "│" : "┌") - end - print(io, " @ ", frame.file) - if frame.line != typemax(frame.line) && frame.line != 0 - print(io, ":", frame.line) - end - print(io, " within `", method_name(frame), "`") - if collapse - method = method_name(frame) - while nctx < nframes - frame = DI[nframes - nctx] - method_name(frame) === method || break - nctx += 1 - push!(context, frame) - print(io, " @ ", frame.file, ":", frame.line) - end + printstyled(io, indent("│"), color=linecolor) + push!(context, frame) + if update_line_only + update_line_only = false + else + context_depth[] += 1 + nctx != 1 && printstyled(io, started ? "│" : "┌", color=linecolor) + end + printstyled(io, " @ ", frame.file, color=linecolor) + if frame.line != typemax(frame.line) && frame.line != 0 + printstyled(io, ":", frame.line, color=linecolor) + end + printstyled(io, " within `", method_name(frame), "`", color=linecolor) + if collapse + method = method_name(frame) + while nctx < nframes + frame = DI[nframes - nctx] + method_name(frame) === method || break + nctx += 1 + push!(context, frame) + printstyled(io, " @ ", frame.file, ":", frame.line, color=linecolor) end end println(io) @@ -600,6 +621,7 @@ end printed as part of the IR or not - `bb_color`: color used for printing the basic block brackets on the left - `label_dynamic_calls`: whether to label calls as dynamic / builtin / intrinsic +- `color_warntype`: whether to color the SSA value index based on the stability of its type """ struct IRShowConfig line_info_preprinter @@ -607,19 +629,22 @@ struct IRShowConfig should_print_stmt bb_color::Symbol label_dynamic_calls::Bool + color_warntype::Bool IRShowConfig( line_info_preprinter, line_info_postprinter=default_expr_type_printer; should_print_stmt=Returns(true), bb_color::Symbol=:light_black, - label_dynamic_calls=true + label_dynamic_calls=true, + color_warntype=false ) = new( line_info_preprinter, line_info_postprinter, should_print_stmt, bb_color, - label_dynamic_calls + label_dynamic_calls, + color_warntype ) end @@ -678,12 +703,66 @@ end function show_ir_stmt(io::IO, code::Union{IRCode, CodeInfo, IncrementalCompact}, idx::Int, config::IRShowConfig, sptypes::Vector{VarState}, used::BitSet, cfg::CFG, bb_idx::Int; pop_new_node! = Returns(nothing), only_after::Bool=false) return show_ir_stmt(io, code, idx, config.line_info_preprinter, config.line_info_postprinter, - sptypes, used, cfg, bb_idx; pop_new_node!, only_after, config.bb_color, config.label_dynamic_calls) + sptypes, used, cfg, bb_idx; pop_new_node!, only_after, config.bb_color, config.label_dynamic_calls, config.color_warntype) +end + +function _print_ir_indentation(io::IO, cfg::CFG, bb_idx::Int, max_bb_idx_size::Int, bb_color, + line_info_preprinter, idx::Int, i::Int; final::Bool=true) + # Compute BB guard rail + if bb_idx > length(cfg.blocks) + # If invariants are violated, print a special leader + linestart = " "^(max_bb_idx_size + 2) # not inside a basic block bracket + inlining_indent = line_info_preprinter(io, linestart, i == 1 ? idx : 0) + printstyled(io, "!!! ", "─"^max_bb_idx_size, color=bb_color) + else + bbrange = cfg.blocks[bb_idx].stmts + # Print line info update + linestart = idx == first(bbrange) ? " " : sprint(io -> printstyled(io, "│ ", color=bb_color), context=io) + linestart *= " "^max_bb_idx_size + # idx == 0 means only indentation is printed, so we don't print linfos + # multiple times if the are new nodes + inlining_indent = line_info_preprinter(io, linestart, i == 1 ? idx : 0) + + if i == 1 && idx == first(bbrange) + bb_idx_str = string(bb_idx) + bb_pad = max_bb_idx_size - length(bb_idx_str) + bb_type = length(cfg.blocks[bb_idx].preds) <= 1 ? "─" : "┄" + printstyled(io, bb_idx_str, " ", bb_type, "─"^bb_pad, color=bb_color) + elseif final && idx == last(bbrange) # print separator + printstyled(io, "└", "─"^(1 + max_bb_idx_size), color=bb_color) + else + printstyled(io, "│ ", " "^max_bb_idx_size, color=bb_color) + end + end + print(io, inlining_indent, " ") + return nothing +end + +function _print_ir_new_node(io::IO, node, code, sptypes::Vector{VarState}, used::BitSet, maxlength_idx::Int, + label_dynamic_calls::Bool, line_info_postprinter, cfg::CFG, bb_idx::Int, + max_bb_idx_size::Int, bb_color, line_info_preprinter, idx::Int, i::Int; final::Bool=true) + _print_ir_indentation(io, cfg, bb_idx, max_bb_idx_size, bb_color, line_info_preprinter, idx, i; final) + + node_idx, new_node_inst, new_node_type = node + @assert new_node_inst !== UNDEF # we filtered these out earlier + show_type = should_print_ssa_type(new_node_inst) + with_output_color(:green, io) do io′ + print_stmt(io′, node_idx, new_node_inst, code, sptypes, used, maxlength_idx, false, show_type, label_dynamic_calls, false) + end + + if new_node_type === UNDEF + # Try to be robust against errors + printstyled(io, "::#UNDEF", color=:red) + else + line_info_postprinter(io; type = new_node_type, used = node_idx in used, show_type, idx = node_idx) + end + println(io) + return nothing end function show_ir_stmt(io::IO, code::Union{IRCode, CodeInfo, IncrementalCompact}, idx::Int, line_info_preprinter, line_info_postprinter, sptypes::Vector{VarState}, used::BitSet, cfg::CFG, bb_idx::Int; pop_new_node! = Returns(nothing), only_after::Bool=false, - bb_color=:light_black, label_dynamic_calls::Bool=true) + bb_color=:light_black, label_dynamic_calls::Bool=true, color_warntype::Bool=false) stmt = _stmt(code, idx) type = _type(code, idx) max_bb_idx_size = length(string(length(cfg.blocks))) @@ -703,58 +782,11 @@ function show_ir_stmt(io::IO, code::Union{IRCode, CodeInfo, IncrementalCompact}, end i = 1 - function print_indentation(final::Bool=true) - # Compute BB guard rail - if bb_idx > length(cfg.blocks) - # If invariants are violated, print a special leader - linestart = " "^(max_bb_idx_size + 2) # not inside a basic block bracket - inlining_indent = line_info_preprinter(io, linestart, i == 1 ? idx : 0) - printstyled(io, "!!! ", "─"^max_bb_idx_size, color=bb_color) - else - bbrange = cfg.blocks[bb_idx].stmts - # Print line info update - linestart = idx == first(bbrange) ? " " : sprint(io -> printstyled(io, "│ ", color=bb_color), context=io) - linestart *= " "^max_bb_idx_size - # idx == 0 means only indentation is printed, so we don't print linfos - # multiple times if the are new nodes - inlining_indent = line_info_preprinter(io, linestart, i == 1 ? idx : 0) - - if i == 1 && idx == first(bbrange) - bb_idx_str = string(bb_idx) - bb_pad = max_bb_idx_size - length(bb_idx_str) - bb_type = length(cfg.blocks[bb_idx].preds) <= 1 ? "─" : "┄" - printstyled(io, bb_idx_str, " ", bb_type, "─"^bb_pad, color=bb_color) - elseif final && idx == last(bbrange) # print separator - printstyled(io, "└", "─"^(1 + max_bb_idx_size), color=bb_color) - else - printstyled(io, "│ ", " "^max_bb_idx_size, color=bb_color) - end - end - print(io, inlining_indent, " ") - end - # first, print new nodes that are to be inserted before the current statement - function print_new_node(node; final::Bool=true) - print_indentation(final) - - node_idx, new_node_inst, new_node_type = node - @assert new_node_inst !== UNDEF # we filtered these out earlier - show_type = should_print_ssa_type(new_node_inst) - let maxlength_idx=maxlength_idx, show_type=show_type - with_output_color(:green, io) do io′ - print_stmt(io′, node_idx, new_node_inst, code, sptypes, used, maxlength_idx, false, show_type, label_dynamic_calls) - end - end - - if new_node_type === UNDEF # try to be robust against errors - printstyled(io, "::#UNDEF", color=:red) - else - line_info_postprinter(io; type = new_node_type, used = node_idx in used, show_type, idx = node_idx) - end - println(io) - end while (next = pop_new_node!(idx)) !== nothing - only_after || print_new_node(next; final=false) + only_after || _print_ir_new_node(io, next, code, sptypes, used, maxlength_idx, label_dynamic_calls, + line_info_postprinter, cfg, bb_idx, max_bb_idx_size, bb_color, + line_info_preprinter, idx, i; final=false) i += 1 end @@ -766,12 +798,12 @@ function show_ir_stmt(io::IO, code::Union{IRCode, CodeInfo, IncrementalCompact}, # FIXME: `only_after` is hack so that we can call this function to print uncompacted # attach-after nodes when the current node has already been compated already if !only_after - print_indentation(next===nothing) + _print_ir_indentation(io, cfg, bb_idx, max_bb_idx_size, bb_color, line_info_preprinter, idx, i; final=next===nothing) if code isa CodeInfo stmt = statement_indices_to_labels(stmt, cfg) end show_type = type !== nothing && should_print_ssa_type(stmt) - print_stmt(io, idx, stmt, code, sptypes, used, maxlength_idx, true, show_type, label_dynamic_calls) + print_stmt(io, idx, stmt, code, sptypes, used, maxlength_idx, true, show_type, label_dynamic_calls, color_warntype) if type !== nothing # ignore types for pre-inference code if type === UNDEF # This is an error, but can happen if passes don't update their type information @@ -786,7 +818,9 @@ function show_ir_stmt(io::IO, code::Union{IRCode, CodeInfo, IncrementalCompact}, # finally, print new nodes that are to be inserted after the current statement while next !== nothing - print_new_node(next) + _print_ir_new_node(io, next, code, sptypes, used, maxlength_idx, label_dynamic_calls, + line_info_postprinter, cfg, bb_idx, max_bb_idx_size, bb_color, + line_info_preprinter, idx, i) i += 1 next = pop_new_node!(idx; attach_after=true) end @@ -859,7 +893,7 @@ function inline_linfo_printer(code::Union{IRCode,CodeInfo}) max_lineno_width = maximum(length, loc_lineno) max_method_width = maximum(length, loc_methods) - function (io::IO, indent::String, idx::Int) + function (io::IO, _indent::String, idx::Int) cols = (displaysize(io)::Tuple{Int,Int})[2] if idx == 0 @@ -960,6 +994,23 @@ function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(io, ir) finish_show_ir(io, cfg, config) end +function warntype_type_class(@nospecialize(type)) + if type isa Union && is_expected_union(type) + return SSA_WARN_TYPE_MILD + elseif type isa Type && (!Base.isdispatchelem(type) || type == Core.Box) + return SSA_WARN_TYPE_STRONG + else + return SSA_WARN_TYPE_STABLE + end +end + +function ssa_warntype_class(code::CodeInfo, idx::Int) + types = code.ssavaluetypes + types isa Vector || return SSA_WARN_TYPE_STABLE + isassigned(types, idx) || return SSA_WARN_TYPE_STABLE + return warntype_type_class(types[idx]) +end + function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(io, ci); pop_new_node! = Returns(nothing)) used = stmts_used(io, ci) @@ -968,7 +1019,12 @@ function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(io, c sptypes = if parent isa MethodInstance sptypes_from_meth_instance(parent) else EMPTY_SPTYPES end - let io = IOContext(io, :maxssaid=>length(ci.code)) + ssa_warn_levels = fill(SSA_WARN_TYPE_STABLE, length(ci.code)) + for idx in used + checkbounds(Bool, ssa_warn_levels, idx) || continue + ssa_warn_levels[idx] = ssa_warntype_class(ci, idx) + end + let io = IOContext(io, :maxssaid=>length(ci.code), :ssa_warn_levels=>ssa_warn_levels) show_ir_stmts(io, ci, 1:length(ci.code), config, sptypes, used, cfg, 1; pop_new_node!) end finish_show_ir(io, cfg, config) @@ -1062,7 +1118,7 @@ function show_ir(io::IO, compact::IncrementalCompact, config::IRShowConfig=defau finish_show_ir(io, uncompacted_cfg, config) end -function effectbits_letter(effects::Effects, name::Symbol, suffix::Char) +function effectbits_letter(effects::Effects, name::Symbol, suffix::Union{Char, String}) ft = fieldtype(Effects, name) if ft === UInt8 prefix = getfield(effects, name) === ALWAYS_TRUE ? '+' : @@ -1094,6 +1150,8 @@ function Base.show(io::IO, e::Effects) print(io, ',') printstyled(io, effectbits_letter(e, :effect_free, 'e'); color=effectbits_color(e, :effect_free)) print(io, ',') + printstyled(io, effectbits_letter(e, :reset_safe, "re"); color=effectbits_color(e, :reset_safe)) + print(io, ',') printstyled(io, effectbits_letter(e, :nothrow, 'n'); color=effectbits_color(e, :nothrow)) print(io, ',') printstyled(io, effectbits_letter(e, :terminates, 't'); color=effectbits_color(e, :terminates)) @@ -1154,7 +1212,7 @@ function Base.show(io::IO, mi_info::Timings.InferenceFrameInfo) show_tuple_as_call(io, def.name, mi.specTypes; argnames, qualified=true) end else - di = mi.cache.inferred.debuginfo + di = mi.cache.debuginfo file, line = debuginfo_firstline(di) file = string(file) line = isempty(file) || line < 0 ? "" : "$file:$line" @@ -1172,7 +1230,7 @@ const __debuginfo = Dict{Symbol, Any}( # :full => src -> statementidx_lineinfo_printer(src), # and add variable slot information :source => src -> statementidx_lineinfo_printer(src), # :oneliner => src -> statementidx_lineinfo_printer(PartialLineInfoPrinter, src), - :none => src -> lineinfo_disabled, + :none => _src -> lineinfo_disabled, ) const default_debuginfo = Ref{Symbol}(:none) debuginfo(sym) = sym === :default ? default_debuginfo[] : sym @@ -1182,7 +1240,7 @@ const __debuginfo = Dict{Symbol, Any}( :source => src -> statementidx_lineinfo_printer(src), :source_inline => src -> inline_linfo_printer(src), # :oneliner => src -> statementidx_lineinfo_printer(PartialLineInfoPrinter, src), - :none => src -> lineinfo_disabled, + :none => _src -> lineinfo_disabled, ) const debuginfo_modes = [:none, :source, :source_inline] @@ -1208,3 +1266,15 @@ function get_debuginfo_printer(mode::Symbol) end get_debuginfo_printer(src, mode::Symbol) = get_debuginfo_printer(mode)(src) + +# True if one can be pretty certain that the compiler handles this union well, +# i.e. must be small with concrete types. +function is_expected_union(u::Union) + Base.unionlen(u) < 4 || return false + for x in Base.uniontypes(u) + if !Base.isdispatchelem(x) || x == Core.Box + return false + end + end + return true +end diff --git a/Compiler/src/ssair/slot2ssa.jl b/Compiler/src/ssair/slot2ssa.jl index 16a964b4d72f1..86fca631b8dcf 100644 --- a/Compiler/src/ssair/slot2ssa.jl +++ b/Compiler/src/ssair/slot2ssa.jl @@ -3,15 +3,13 @@ mutable struct SlotInfo defs::Vector{Int} uses::Vector{Int} - any_newvar::Bool end -SlotInfo() = SlotInfo(Int[], Int[], false) +SlotInfo() = SlotInfo(Int[], Int[]) function scan_entry!(result::Vector{SlotInfo}, idx::Int, @nospecialize(stmt)) # NewvarNodes count as defs for the purpose # of liveness analysis (i.e. they kill use chains) if isa(stmt, NewvarNode) - result[slot_id(stmt.slot)].any_newvar = true push!(result[slot_id(stmt.slot)].defs, idx) return elseif isexpr(stmt, :(=)) @@ -172,7 +170,7 @@ function typ_for_val(@nospecialize(x), ci::CodeInfo, ir::IRCode, idx::Int, slott end return (ci.ssavaluetypes::Vector{Any})[idx] end - isa(x, GlobalRef) && return abstract_eval_globalref_type(x, ci) + isa(x, GlobalRef) && return globalref_rt(x, ci) isa(x, SSAValue) && return (ci.ssavaluetypes::Vector{Any})[x.id] isa(x, Argument) && return slottypes[x.n] isa(x, NewSSAValue) && return types(ir)[new_to_regular(x, length(ir.stmts))] @@ -218,7 +216,7 @@ so it needs a ϕ-node. Now, the key insight of that algorithm is that we have two defs, in blocks `A` and `B`, and `A` dominates `B`, then we do not need to recurse into `B`, because the set of potential backedges from a subtree rooted at `B` (to outside the subtree) is a strict -subset of those backedges from a subtree rooted at `A` (out outside the subtree rooted +subset of those backedges from a subtree rooted at `A` (outside the subtree rooted at `A`). Note however that this does not work the other way. Thus, the algorithm needs to make sure that we always visit `B` before `A`. @@ -578,7 +576,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, # No uses => no need for phi nodes isempty(slot.uses) && continue # TODO: Restore this optimization - if false # length(slot.defs) == 1 && slot.any_newvar + if false # length(slot.defs) == 1 if slot.defs[] == 0 typ = sv.slottypes[idx] ssaval = Argument(idx) @@ -608,13 +606,13 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, # The slot is live-in into this block. We need to # Create a PhiC node in the catch entry block and # an upsilon node in the corresponding enter block - varstate = sv.bb_vartables[li] - if varstate === nothing + bbstate = sv.bb_states[li] + if bbstate === nothing continue end node = PhiCNode(Any[]) insertpoint = first_insert_for_bb(code, cfg, li) - vt = varstate[idx] + vt = bbstate.vartable[idx] phic_ssa = NewSSAValue( insert_node!(ir, insertpoint, NewInstruction(node, vt.typ)).id - length(ir.stmts)) @@ -640,9 +638,9 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, for block in phiblocks push!(phi_slots[block], idx) node = PhiNode() - varstate = sv.bb_vartables[block] - @assert varstate !== nothing - vt = varstate[idx] + bbstate = sv.bb_states[block] + @assert bbstate !== nothing + vt = bbstate.vartable[idx] ssaval = NewSSAValue(insert_node!(ir, first_insert_for_bb(code, cfg, block), NewInstruction(node, vt.typ)).id - length(ir.stmts)) undef_node = undef_ssaval = nothing @@ -662,10 +660,8 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, initial_incoming_vals = Pair{Any, Any}[ if 0 in defuses[x].defs Pair{Any, Any}(Argument(x), true) - elseif !defuses[x].any_newvar - Pair{Any, Any}(UNDEF_TOKEN, false) else - Pair{Any, Any}(SSAValue(-2), false) + Pair{Any, Any}(UNDEF_TOKEN, false) end for x in 1:length(ci.slotflags) ] worklist = Tuple{Int, Int, Vector{Pair{Any, Any}}}[(1, 0, initial_incoming_vals)] @@ -673,7 +669,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, new_nodes = ir.new_nodes @zone "CC: SSA_RENAME" while !isempty(worklist) (item, pred, incoming_vals) = pop!(worklist) - if sv.bb_vartables[item] === nothing + if sv.bb_states[item] === nothing continue end # Rename existing phi nodes first, because their uses occur on the edge @@ -696,11 +692,6 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, for (idx, slot) in Iterators.enumerate(phi_slots[item]) (; ssaval, node, undef_ssaval, undef_node) = new_phi_nodes[item][idx] (incoming_val, incoming_def) = incoming_vals[slot] - if incoming_val === SSAValue(-1) - # Optimistically omit this path. - # Liveness analysis would probably have prevented us from inserting this phi node - continue - end push!(node.edges, pred) if incoming_val === UNDEF_TOKEN resize!(node.values, length(node.values)+1) @@ -726,13 +717,11 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, has_pinode = fill(false, length(sv.slottypes)) for slot in live_slots[item] (ival, idef) = incoming_vals[slot] - (ival === SSAValue(-1)) && continue - (ival === SSAValue(-2)) && continue (ival === UNDEF_TOKEN) && continue - varstate = sv.bb_vartables[item] - @assert varstate !== nothing - typ = varstate[slot].typ + bbstate = sv.bb_states[item] + @assert bbstate !== nothing + typ = bbstate.vartable[slot].typ if !⊑(𝕃ₒ, sv.slottypes[slot], typ) node = PiNode(ival, typ) ival = NewSSAValue(insert_node!(ir, @@ -782,14 +771,15 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, if isa(arg1, SlotNumber) id = slot_id(arg1) val = stmt.args[2] - typ = typ_for_val(val, ci, ir, idx, sv.slottypes) # Having UNDEF_TOKEN appear on the RHS is possible if we're on a dead branch. # Do something reasonable here, by marking the LHS as undef as well. if val !== UNDEF_TOKEN thisdef = true + typ = typ_for_val(val, ci, ir, idx, sv.slottypes) thisval = make_ssa!(ci, code, idx, typ) else code[idx] = nothing + typ = Union{} thisval = UNDEF_TOKEN thisdef = false end @@ -804,10 +794,6 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, new_phic_nodes[leave_block]) if cidx !== nothing node = thisdef ? UpsilonNode(thisval) : UpsilonNode() - if incoming_vals[id] === UNDEF_TOKEN - node = UpsilonNode() - typ = Union{} - end insert = new_phic_nodes[leave_block][cidx].insert push!(insert.node.values, NewSSAValue(insert_node!(ir, idx, NewInstruction(node, typ), true).id - length(ir.stmts))) diff --git a/Compiler/src/ssair/tarjan.jl b/Compiler/src/ssair/tarjan.jl index e73039868c367..3197733c22125 100644 --- a/Compiler/src/ssair/tarjan.jl +++ b/Compiler/src/ssair/tarjan.jl @@ -57,7 +57,7 @@ bb_unreachable(reach::CFGReachability, bb::Int) = reach.scc[bb] == 0 bb_in_irreducible_loop(reach::CFGReachability, bb::Int) = reach.irreducible[bb] -# Returns `true` if a node is 'rooted' as reachable, i.e. it is has an incoming +# Returns `true` if a node is 'rooted' as reachable, i.e. it has an incoming # edge from a resolved SCC other than its own (or it is BasicBlock #1). # # `tarjan!` takes the transitive closure of this relation in order to detect diff --git a/Compiler/src/ssair/verify.jl b/Compiler/src/ssair/verify.jl index 3aa1e00e3f2d3..e9389c8cd1aa5 100644 --- a/Compiler/src/ssair/verify.jl +++ b/Compiler/src/ssair/verify.jl @@ -25,7 +25,24 @@ if !isdefined(@__MODULE__, Symbol("@verify_error")) end end -is_toplevel_expr_head(head::Symbol) = head === :global || head === :method || head === :thunk +# Whether a bare `GlobalRef` is something the front-end might have emitted in argument position +# (an inline `(top ...)`/`(core ...)` operand), even though that is unsound: the value is read by +# a lookup at codegen/run time with no invalidation edge. +value_position_globalref_from_lowering(gr::GlobalRef) = (m = gr.mod; m === Core || m === Core.Intrinsics || m === Base || is_top_module(m)) +is_top_module(m::Module) = ccall(:jl_istopmod, UInt8, (Any,), m) != 0 + +# Whether `gr` currently resolves (through imports, at the latest world) to a defined constant. +# A single lookup, not a scan over `ir.valid_worlds`: the value-position exception this gates is +# unsound regardless of world, so no range check could rescue it. +function is_const_globalref_now(gr::GlobalRef) + b = convert(Core.Binding, gr) + isdefined(b, :partitions) || return false + world = get_world_counter() + _, leaf = walk_to_leaf_partition(b, lookup_binding_partition(world, b), world) + return is_defined_const_binding(binding_kind(leaf)) +end + +is_toplevel_expr_head(head::Symbol) = head === :thunk is_value_pos_expr_head(head::Symbol) = head === :static_parameter function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, use_idx::Int, printed_use_idx::Int, print::Bool, isforeigncall::Bool, arg_idx::Int, allow_frontend_forms::Bool, @nospecialize(raise_error)) @@ -66,26 +83,34 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, raise_error() end elseif isa(op, GlobalRef) - if op.mod !== Core && op.mod !== Base - (valid_worlds, alldef) = scan_leaf_partitions(nothing, op, WorldWithRange(min_world(ir.valid_worlds), ir.valid_worlds)) do _, _, bpart - is_defined_const_binding(binding_kind(bpart)) - end - if !alldef || max_world(valid_worlds) < max_world(ir.valid_worlds) || min_world(valid_worlds) > min_world(ir.valid_worlds) - @verify_error "Unbound or partitioned GlobalRef not allowed in value position" - raise_error() - end + # A bare `GlobalRef` names a binding, not a value: reading it takes a lookup, so it is + # not a constant and is not valid in value position. An ordinary global read is its own + # statement (lowering hoists it -- see `valid-ir-argument?` in julia-syntax.scm), and + # once resolved it is a `Core.BindingPartition` (`reformulate_globals_pass!`), which + # carries its value or slot with it and needs no lookup. + # + # Two exceptions are admitted. A world-1 constant (`world1_const`) is immutable, so the + # optimizer deliberately leaves it bare for codegen to embed. A top-module/intrinsics + # reference lowering emits inline as a `(top ...)`/`(core ...)` operand is unsound but + # relied on by the frontend; require only that it is a constant now, since the exception + # is unsound regardless of world and a scan over `ir.valid_worlds` could not make it sound. + if !(world1_const(op) || (value_position_globalref_from_lowering(op) && is_const_globalref_now(op))) + @verify_error "GlobalRef not allowed in value position (only a world-1 constant or a top-module constant emitted by lowering)" + raise_error() end + elseif isa(op, Core.BindingPartition) + # The resolved form of a global read (from `reformulate_globals_pass!`), carrying an + # invalidation edge. Valid in value position for any binding kind, unlike a bare + # `GlobalRef`. elseif isa(op, Expr) # Only Expr(:boundscheck) is allowed in value position - if isforeigncall && arg_idx == 1 && op.head === :call - # Allow a tuple in symbol position for foreigncall - this isn't actually - # a real call - it's interpreted in global scope by codegen. However, - # we do need to keep this a real use, because it could also be a pointer. - elseif !is_value_pos_expr_head(op.head) - if !allow_frontend_forms || op.head !== :opaque_closure_method - @verify_error "Expr not allowed in value position" - raise_error() - end + if isforeigncall && arg_idx == 1 && op.head === :tuple + # Allow a tuple literal in symbol position for foreigncall - this + # is syntax for a literal value or globalref - it is interpreted in + # global scope by codegen. + elseif !allow_frontend_forms || op.head !== :opaque_closure_method + @verify_error "Expr not allowed in value position" + raise_error() end elseif isa(op, Union{OldSSAValue, NewSSAValue}) @verify_error "At statement %$use_idx: Left over SSA marker ($op)" @@ -338,16 +363,14 @@ function verify_ir(ir::IRCode, print::Bool=true, end if is_phinode_block && !is_valid_phiblock_stmt(stmt) - if !isa(stmt, Expr) || !is_value_pos_expr_head(stmt.head) - # Go back and check that all non-PhiNodes are valid value-position - for validate_idx in firstidx:(lastphi-1) - validate_stmt = ir[SSAValue(validate_idx)][:stmt] - isa(validate_stmt, PhiNode) && continue - check_op(ir, domtree, validate_stmt, bb, idx, idx, print, false, 0, - allow_frontend_forms, raise_error) - end - is_phinode_block = false + # Go back and check that all non-PhiNodes are valid value-position + for validate_idx in firstidx:(lastphi-1) + validate_stmt = ir[SSAValue(validate_idx)][:stmt] + isa(validate_stmt, PhiNode) && continue + check_op(ir, domtree, validate_stmt, bb, idx, idx, print, false, 0, + allow_frontend_forms, raise_error) end + is_phinode_block = false end if isa(stmt, PhiCNode) for i = 1:length(stmt.values) @@ -374,16 +397,25 @@ function verify_ir(ir::IRCode, print::Bool=true, isforeigncall = false if isa(stmt, Expr) if stmt.head === :(=) - @verify_error "Assignment should have been removed during SSA conversion" - raise_error() + # Local (slot/SSA) assignments are removed during SSA conversion, + # but a store to a global -- a `GlobalRef` or a `BindingPartition` + # frozen by `reformulate_globals_pass!` -- is a legitimate `:(=)`. + lhs = stmt.args[1] + if !(isa(lhs, GlobalRef) || isa(lhs, Core.BindingPartition)) + @verify_error "Assignment should have been removed during SSA conversion" + raise_error() + end elseif stmt.head === :isdefined if length(stmt.args) > 2 @verify_error "malformed isdefined" raise_error() end - if stmt.args[1] isa GlobalRef - # undefined GlobalRef is OK in isdefined - continue + let v = stmt.args[1] + # a GlobalRef or static_parameter isdefined check does + # not evaluate its argument + if v isa GlobalRef || isexpr(v, :static_parameter) + continue + end end elseif stmt.head === :throw_undef_if_not if length(stmt.args) > 3 @@ -399,18 +431,8 @@ function verify_ir(ir::IRCode, print::Bool=true, # blocks, which isn't allowed for regular SSA values, so # we skip the validation below. continue - elseif stmt.head === :foreigncall + elseif stmt.head === :foreigncall || stmt.head === :foreignglobal isforeigncall = true - elseif stmt.head === :isdefined && length(stmt.args) == 1 && - isexpr(stmt.args[1], :static_parameter) - # a GlobalRef or static_parameter isdefined check does not evaluate its argument - continue - elseif stmt.head === :call - f = stmt.args[1] - if f isa GlobalRef && f.name === :cglobal - # TODO: these are not yet linearized - continue - end elseif stmt.head === :leave for i in 1:length(stmt.args) arg = stmt.args[i] diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 525fe0cc222b7..cdb221fd13832 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -24,6 +24,140 @@ end struct NoCallInfo <: CallInfo end add_edges_impl(::Vector{Any}, ::NoCallInfo) = nothing +# InferredCallResult is defined in types.jl with LocalInferenceResult + +struct ConcreteResult <: InferredCallResult + edge::Union{Nothing,CodeInstance} + effects::Effects + proof::Union{Nothing,InferenceProof} + result + function ConcreteResult(edge::Union{Nothing,CodeInstance}, effects::Effects; + proof::Union{Nothing,InferenceProof}=nothing) + @assert edge !== nothing || proof !== nothing + return new(edge, effects, proof) + end + function ConcreteResult(edge::Union{Nothing,CodeInstance}, effects::Effects, + @nospecialize(val); + proof::Union{Nothing,InferenceProof}=nothing) + @assert edge !== nothing || proof !== nothing + return new(edge, effects, proof, val) + end +end + +struct SemiConcreteResult <: InferredCallResult + edge::CodeInstance + ir::IRCode + effects::Effects + spec_info::SpecInfo + proof::Union{Nothing,InferenceProof} + SemiConcreteResult(edge::CodeInstance, ir::IRCode, effects::Effects, spec_info::SpecInfo; + proof::Union{Nothing,InferenceProof}=nothing) = + new(edge, ir, effects, spec_info, proof) +end + +inference_proof(result::ConcreteResult) = + result.proof === nothing ? result.edge::CodeInstance : result.proof +inference_proof(result::SemiConcreteResult) = something(result.proof, result.edge) + +function record_invoke_edge!(invokes::IdDict{Any,Vector{Any}}, + @nospecialize(signature), @nospecialize(target)) + signatures = get!(Vector{Any}, invokes, target) + for previous in signatures + previous == signature && return false + end + push!(signatures, signature) + return true +end + +function _materialize_inference_edges!(edges::Vector{Any}, source, + seen_proofs::IdSet{LocalInferenceProof}, + standalone::IdSet{Any}, + invokes::IdDict{Any,Vector{Any}}) + i = 1 + while i <= length(source) + edge = source[i] + if edge isa LocalInferenceProof + i += 1 + edge in seen_proofs && continue + push!(seen_proofs, edge) + _materialize_inference_edges!( + edges, edge.edges, seen_proofs, standalone, invokes) + elseif edge isa Int + # Encoded lookup groups are positional: `nmatches, atype, matches...`. + # Preserve each complete group, while recording its targets so an + # identical standalone proof edge later in the stream can be omitted. + n = abs(edge) + last = i + 1 + n + @assert last <= length(source) + for j = i:last + push!(edges, source[j]) + end + for j = i + 2:last + target = source[j] + if target isa Union{Method,MethodInstance,CodeInstance,Core.Binding} + push!(standalone, target) + end + end + i = last + 1 + elseif edge isa Union{Method,MethodInstance,CodeInstance,Core.Binding} + i += 1 + edge in standalone && continue + push!(standalone, edge) + push!(edges, edge) + else + # Everything else is an invoke signature paired with its target. Keep + # distinct signatures, and deduplicate only the identical pair. + @assert i < length(source) + target = source[i + 1] + i += 2 + record_invoke_edge!(invokes, edge, target) || continue + push!(edges, edge, target) + end + end + return nothing +end + +function materialize_inference_edges(source) + has_local_proof = false + for edge in source + if edge isa LocalInferenceProof + has_local_proof = true + break + end + end + if !has_local_proof + return source isa SimpleVector ? source : Core.svec(source...) + end + edges = Any[] + sizehint!(edges, length(source)) + _materialize_inference_edges!(edges, source, + IdSet{LocalInferenceProof}(), IdSet{Any}(), IdDict{Any,Vector{Any}}()) + return Core.svec(edges...) +end + +function add_inference_proof!(edges::Vector{Any}, proof::CodeInstance, + @nospecialize(paired_edge=nothing)) + # This CI certifies the inferred facts of its defining method; it does not + # necessarily represent ordinary dispatch on its MethodInstance signature + # (for example, it may have been reached through `invoke`). Encode the proof + # as an identity edge to that method, just as we do for an inlined CI. + proof === paired_edge || add_inlining_edge!(edges, proof) + return nothing +end + +function add_inference_proof!(edges::Vector{Any}, proof::LocalInferenceProof, + @nospecialize(paired_edge=nothing)) + any(edge -> edge === proof, edges) || push!(edges, proof) + return nothing +end + +function add_result_proof!(edges::Vector{Any}, result::Union{Nothing,InferredCallResult}, + @nospecialize(paired_edge=nothing)) + result === nothing && return nothing + add_inference_proof!(edges, inference_proof(result), paired_edge) + return nothing +end + """ info::MethodMatchInfo <: CallInfo @@ -38,13 +172,68 @@ struct MethodMatchInfo <: CallInfo atype fullmatch::Bool edges::Vector{Union{Nothing,CodeInstance}} + needs_mi_edges::BitVector + call_results::Vector{Union{Nothing,InferredCallResult}} function MethodMatchInfo( results::MethodLookupResult, mt::MethodTable, @nospecialize(atype), fullmatch::Bool) edges = fill!(Vector{Union{Nothing,CodeInstance}}(undef, length(results)), nothing) - return new(results, mt, atype, fullmatch, edges) + needs_mi_edges = falses(length(results)) + call_results = fill!(Vector{Union{Nothing,InferredCallResult}}(undef, length(results)), nothing) + return new(results, mt, atype, fullmatch, edges, needs_mi_edges, call_results) end end add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) = _add_edges_impl(edges, info) + +function method_match_edge(info::MethodMatchInfo, i::Int, mi_edge::Bool) + edge = info.edges[i] + edge !== nothing && return edge + match = info.results[i] + # A proof-carrying result may carry facts about this specialization without an + # executable CI of its own. The same is true when a scheduled call consumed + # provisional SCC facts before a CI or local result was available. A completed + # tombstone similarly carries its proof separately from the dispatch target. Keep + # the MethodInstance as an invalidation target in these cases; a bare Method in an + # encoded lookup is intentionally ignored by the backedge iterator and therefore + # cannot certify those facts. + return (mi_edge || info.needs_mi_edges[i] || info.call_results[i] !== nothing) ? + specialize_method(match) : match.method +end + +function has_encoded_lookup(edges::Vector{Any}, info::MethodMatchInfo, + encoded_nmatches::Int, mi_edge::Bool) + i = 1 + while i <= length(edges) + entry = edges[i] + if entry isa Int + n = abs(entry) + next_i = i + 2 + n + if next_i - 1 <= length(edges) && entry === encoded_nmatches && + edges[i + 1] == info.atype + matches = true + for j = 1:n + if edges[i + 1 + j] !== method_match_edge(info, j, mi_edge) + matches = false + break + end + end + matches && return true + end + i = next_i + else + i += 1 + end + end + return false +end + +function add_method_match_proofs!(edges::Vector{Any}, info::MethodMatchInfo, + mi_edge::Bool) + for i = 1:length(info.call_results) + add_result_proof!(edges, info.call_results[i], method_match_edge(info, i, mi_edge)) + end + return nothing +end + function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Bool=false) if !fully_covering(info) exists = false @@ -74,29 +263,24 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo end if mi.specTypes === m.spec_types add_one_edge!(edges, edge) + add_result_proof!(edges, info.call_results[1], edge) return nothing end end # add check for whether this lookup already existed in the edges list # encode nmatches as negative if fully_covers is false encoded_nmatches = fully_covering(info) ? nmatches : -nmatches - for i in 1:length(edges) - if edges[i] === encoded_nmatches && edges[i+1] == info.atype - # TODO: must also verify the CodeInstance match too - return nothing - end - end - push!(edges, encoded_nmatches, info.atype) - for i = 1:nmatches - edge = info.edges[i] - m = info.results[i] - if edge === nothing - edge = mi_edge ? specialize_method(m) : m.method - else - @assert edge.def.def === m.method + if !has_encoded_lookup(edges, info, encoded_nmatches, mi_edge) + push!(edges, encoded_nmatches, info.atype) + for i = 1:nmatches + edge = method_match_edge(info, i, mi_edge) + if edge isa CodeInstance + @assert edge.def.def === info.results[i].method + end + push!(edges, edge) end - push!(edges, edge) end + add_method_match_proofs!(edges, info, mi_edge) nothing end function add_one_edge!(edges::Vector{Any}, edge::MethodInstance) @@ -126,18 +310,23 @@ function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) # found edge we can upgrade edges[i] = edge return - elseif true # XXX compare `CodeInstance` identify? + elseif edgeᵢ_orig === edge + # Only the identical CodeInstance certifies the same inference proof. return end + # Different CodeInstances for the same MethodInstance may certify + # different inference facts. Keep both, irrespective of whether their + # current forward-edge streams happen to compare equal. end i += 1 end push!(edges, edge) nothing end -nsplit_impl(info::MethodMatchInfo) = 1 +nsplit_impl(::MethodMatchInfo) = 1 getsplit_impl(info::MethodMatchInfo, idx::Int) = (@assert idx == 1; info.results) -getresult_impl(::MethodMatchInfo, ::Int) = nothing +getresult_impl(info::MethodMatchInfo, idx::Int) = info.call_results[idx] +getedge_impl(info::MethodMatchInfo, idx::Int) = info.edges[idx] """ info::UnionSplitInfo <: CallInfo @@ -157,57 +346,32 @@ _add_edges_impl(edges::Vector{Any}, info::UnionSplitInfo, mi_edge::Bool=false) = for split in info.split; _add_edges_impl(edges, split, mi_edge); end nsplit_impl(info::UnionSplitInfo) = length(info.split) getsplit_impl(info::UnionSplitInfo, idx::Int) = getsplit(info.split[idx], 1) -getresult_impl(::UnionSplitInfo, ::Int) = nothing - -abstract type ConstResult end - -struct ConstPropResult <: ConstResult - result::InferenceResult -end - -struct ConcreteResult <: ConstResult - edge::CodeInstance - effects::Effects - result - ConcreteResult(edge::CodeInstance, effects::Effects) = new(edge, effects) - ConcreteResult(edge::CodeInstance, effects::Effects, @nospecialize val) = new(edge, effects, val) -end - -struct SemiConcreteResult <: ConstResult - edge::CodeInstance - ir::IRCode - effects::Effects - spec_info::SpecInfo -end - -# XXX Technically this does not represent a result of constant inference, but rather that of -# regular edge inference. It might be more appropriate to rename `ConstResult` and -# `ConstCallInfo` to better reflect the fact that they represent either of local or -# volatile inference result. -struct VolatileInferenceResult <: ConstResult - inf_result::InferenceResult +function getresult_impl(info::UnionSplitInfo, idx::Int) + for split in info.split + n = length(split.call_results) + if idx ≤ n + return split.call_results[idx] + else + idx -= n + end + end end - -""" - info::ConstCallInfo <: CallInfo - -The precision of this call was improved using constant information. -In addition to the original call information `info.call`, this info also keeps the results -of constant inference `info.results::Vector{Union{Nothing,ConstResult}}`. -""" -struct ConstCallInfo <: CallInfo - call::Union{MethodMatchInfo,UnionSplitInfo} - results::Vector{Union{Nothing,ConstResult}} +function getedge_impl(info::UnionSplitInfo, idx::Int) + for split in info.split + n = length(split.edges) + if idx ≤ n + return split.edges[idx] + else + idx -= n + end + end + return nothing end -add_edges_impl(edges::Vector{Any}, info::ConstCallInfo) = add_edges!(edges, info.call) -nsplit_impl(info::ConstCallInfo) = nsplit(info.call) -getsplit_impl(info::ConstCallInfo, idx::Int) = getsplit(info.call, idx) -getresult_impl(info::ConstCallInfo, idx::Int) = info.results[idx] """ info::MethodResultPure <: CallInfo -This struct represents a method result constant was proven to be effect-free. +This struct represents a method result that was proven to be a pure (effect-free) constant. """ struct MethodResultPure <: CallInfo info::CallInfo @@ -279,64 +443,89 @@ struct InvokeCICallInfo <: CallInfo end add_edges_impl(edges::Vector{Any}, info::InvokeCICallInfo) = add_inlining_edge!(edges, info.edge) -nsplit_impl(info::InvokeCICallInfo) = 0 +nsplit_impl(::InvokeCICallInfo) = 0 """ info::InvokeCallInfo Represents a resolved call to `Core.invoke`, carrying the `info.match::MethodMatch` of the method that has been processed. -Optionally keeps `info.result::InferenceResult` that keeps constant information. +Optionally keeps a proof-carrying `info.result` with constant information. """ struct InvokeCallInfo <: CallInfo edge::Union{Nothing,CodeInstance} match::MethodMatch - result::Union{Nothing,ConstResult} + result::Union{Nothing,InferredCallResult} atype # ::Type + needs_mi_edge::Bool # targetless body-derived facts require an invalidation target end +InvokeCallInfo(edge::Union{Nothing,CodeInstance}, match::MethodMatch, + result::Union{Nothing,InferredCallResult}, @nospecialize(atype)) = + InvokeCallInfo(edge, match, result, atype, false) add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo) = _add_edges_impl(edges, info) function _add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo, mi_edge::Bool=false) edge = info.edge if edge === nothing - edge = mi_edge ? specialize_method(info.match) : info.match.method + edge = (mi_edge || info.needs_mi_edge || info.result !== nothing) ? + specialize_method(info.match) : info.match.method end add_invoke_edge!(edges, info.atype, edge) + add_result_proof!(edges, info.result, edge) nothing end function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::Union{MethodInstance,Method}) - for i in 2:length(edges) + i = 1 + while i <= length(edges) edgeᵢ = edges[i] + if edgeᵢ isa Int + i += 2 + abs(edgeᵢ) + continue + end edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def) - edgeᵢ isa MethodInstance || edgeᵢ isa Method || continue + if !(edgeᵢ isa MethodInstance || edgeᵢ isa Method) + i += 1 + continue + end if edgeᵢ === edge - edge_minus_1 = edges[i-1] + i == 1 && (i += 1; continue) + edge_minus_1 = edges[i - 1] if edge_minus_1 isa Type && edge_minus_1 == atype return # found existing covered edge end end + i += 1 end push!(edges, atype) push!(edges, edge) nothing end function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::CodeInstance) - for i in 2:length(edges) + i = 1 + while i <= length(edges) edgeᵢ_orig = edgeᵢ = edges[i] + if edgeᵢ isa Int + i += 2 + abs(edgeᵢ) + continue + end edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def) if ((edgeᵢ isa MethodInstance && edgeᵢ === edge.def) || (edgeᵢ isa Method && edgeᵢ === edge.def.def)) - edge_minus_1 = edges[i-1] + i == 1 && (i += 1; continue) + edge_minus_1 = edges[i - 1] if edge_minus_1 isa Type && edge_minus_1 == atype if edgeᵢ_orig isa MethodInstance || edgeᵢ_orig isa Method # found edge we can upgrade edges[i] = edge return - elseif true # XXX compare `CodeInstance` identify? + elseif edgeᵢ_orig === edge return end + # A distinct CodeInstance for the same MethodInstance may carry a + # distinct proof, so retain it as a separate invoke edge. end end + i += 1 end push!(edges, atype) push!(edges, edge) @@ -348,6 +537,10 @@ function add_inlining_edge!(edges::Vector{Any}, edge::MethodInstance) i = 1 while i <= length(edges) edgeᵢ = edges[i] + if edgeᵢ isa Int + i += 2 + abs(edgeᵢ) + continue + end if edgeᵢ isa Method && edgeᵢ === edge.def # found edge we can upgrade edges[i] = edge @@ -369,6 +562,10 @@ function add_inlining_edge!(edges::Vector{Any}, edge::CodeInstance) i = 1 while i <= length(edges) edgeᵢ = edges[i] + if edgeᵢ isa Int + i += 2 + abs(edgeᵢ) + continue + end if edgeᵢ isa Method && edgeᵢ === edge.def.def # found edge we can upgrade edges[i] = edge @@ -379,11 +576,12 @@ function add_inlining_edge!(edges::Vector{Any}, edge::CodeInstance) edges[i] = edge return end - if edgeᵢ isa CodeInstance && edgeᵢ.def === edge.def - # found existing edge - # XXX compare `CodeInstance` identify? + if edgeᵢ === edge + # found the identical existing edge return end + # A distinct CodeInstance for the same MethodInstance may carry a + # distinct proof, so do not deduplicate it by `def` alone. i += 1 end # add_invoke_edge alone @@ -392,31 +590,40 @@ function add_inlining_edge!(edges::Vector{Any}, edge::CodeInstance) nothing end -nsplit_impl(info::InvokeCallInfo) = 1 +nsplit_impl(::InvokeCallInfo) = 1 getsplit_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; MethodLookupResult(Core.MethodMatch[info.match], WorldRange(typemin(UInt), typemax(UInt)), false)) getresult_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; info.result) - +getedge_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; info.edge) """ info::OpaqueClosureCallInfo Represents a resolved call of opaque closure, carrying the `info.match::MethodMatch` of the method that has been processed. -Optionally keeps `info.result::InferenceResult` that keeps constant information. +Optionally keeps a proof-carrying `info.result` with constant information. """ struct OpaqueClosureCallInfo <: CallInfo edge::Union{Nothing,CodeInstance} match::MethodMatch - result::Union{Nothing,ConstResult} + result::Union{Nothing,InferredCallResult} + needs_mi_edge::Bool # targetless body-derived facts require an invalidation target end +OpaqueClosureCallInfo(edge::Union{Nothing,CodeInstance}, match::MethodMatch, + result::Union{Nothing,InferredCallResult}) = + OpaqueClosureCallInfo(edge, match, result, false) function add_edges_impl(edges::Vector{Any}, info::OpaqueClosureCallInfo) edge = info.edge + if edge === nothing && (info.needs_mi_edge || info.result !== nothing) + edge = specialize_method(info.match) + end if edge !== nothing add_one_edge!(edges, edge) end + add_result_proof!(edges, info.result, edge) nothing end +getedge_impl(info::OpaqueClosureCallInfo, idx::Int) = (@assert idx == 1; info.edge) """ info::OpaqueClosureCreateInfo <: CallInfo @@ -472,6 +679,7 @@ add_edges_impl(::Vector{Any}, ::FinalizerInfo) = nothing Represents a resolved call of one of: - `modifyfield!(obj, name, op, x, [order])` - `modifyglobal!(mod, var, op, x, order)` + - `Core.modifyglobal_partition(partition, op, x, [order])` - `memoryrefmodify!(memref, op, x, order, boundscheck)` - `Intrinsics.atomic_pointermodify(ptr, op, x, order)` @@ -482,11 +690,26 @@ struct ModifyOpInfo <: CallInfo end add_edges_impl(edges::Vector{Any}, info::ModifyOpInfo) = add_edges!(edges, info.info) +""" + info::TaskCallInfo <: CallInfo + +Represents a resolved call of `Core._task(f, size)`. `info.info` wraps the call +information of the deferred `f()` call that runs when the created task is scheduled. +""" +struct TaskCallInfo <: CallInfo + info::CallInfo # the callinfo for the deferred `f()` call +end +add_edges_impl(edges::Vector{Any}, info::TaskCallInfo) = add_edges!(edges, info.info) + struct VirtualMethodMatchInfo <: CallInfo info::Union{MethodMatchInfo,UnionSplitInfo,InvokeCallInfo} end add_edges_impl(edges::Vector{Any}, info::VirtualMethodMatchInfo) = _add_edges_impl(edges, info.info, #=mi_edge=#true) +nsplit_impl(info::VirtualMethodMatchInfo) = nsplit(info.info) +getsplit_impl(info::VirtualMethodMatchInfo, idx::Int) = getsplit(info.info, idx) +getresult_impl(info::VirtualMethodMatchInfo, idx::Int) = getresult(info.info, idx) +getedge_impl(info::VirtualMethodMatchInfo, idx::Int) = getedge(info.info, idx) """ info::GlobalAccessInfo <: CallInfo diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index 71719d75144b3..ddb3b83fd3acd 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -67,6 +67,7 @@ end const DATATYPE_TYPES_FIELDINDEX = fieldindex(DataType, :types) const DATATYPE_NAME_FIELDINDEX = fieldindex(DataType, :name) +const DATATYPE_SUPER_FIELDINDEX = fieldindex(DataType, :super) ########## # tfuncs # @@ -74,7 +75,9 @@ const DATATYPE_NAME_FIELDINDEX = fieldindex(DataType, :name) # Note that in most places in the compiler here, we'll assume that T=Type{S} is well-formed, # and implies that `S <: Type`, not `1::Type{1}`, for example. -# This means that isType(T) implies we can call subtype on T.parameters[1], etc. +# This means that isType(T) implies we can call subtype on type_parameter(T), etc. +# Use isTypeEq(T) or isTypeEgal(T) where equality-only and egality-certain +# type-object kinds need to be distinguished. function add_tfunc(f::IntrinsicFunction, minarg::Int, maxarg::Int, @nospecialize(tfunc), cost::Int) idx = reinterpret(Int32, f) + 1 @@ -95,7 +98,7 @@ add_tfunc(Core.throw_methoderror, 1, INT_INF, @nospecs((𝕃::AbstractLattice, x # if isexact is false, the actual runtime type may (will) be a subtype of t # if isconcrete is true, the actual runtime type is definitely concrete (unreachable if not valid as a typeof) # if istype is true, the actual runtime value will definitely be a type (e.g. this is false for Union{Type{Int}, Int}) -function instanceof_tfunc(@nospecialize(t), astag::Bool=false, @nospecialize(troot) = t) +function instanceof_tfunc(@nospecialize(t), astag::Bool=false, @nospecialize(troot = t)) if isa(t, Const) if isa(t.val, Type) && valid_as_lattice(t.val, astag) return t.val, true, isconcretetype(t.val), true @@ -108,8 +111,13 @@ function instanceof_tfunc(@nospecialize(t), astag::Bool=false, @nospecialize(tro return Bottom, true, true, false # runtime unreachable elseif t === typeof(Bottom) || !hasintersect(t, Type) return Bottom, true, false, false # literal Bottom or non-Type - elseif isType(t) - tp = t.parameters[1] + elseif isTypeEgal(t) + # the value is exactly (`===`) the closed type `tp` + tp = type_parameter(t) + valid_as_lattice(tp, astag) || return Bottom, true, false, false # runtime unreachable / throws on non-Type + return tp, true, isconcretetype(tp), true + elseif isTypeEq(t) + tp = type_parameter(t) valid_as_lattice(tp, astag) || return Bottom, true, false, false # runtime unreachable / throws on non-Type if troot isa UnionAll # Free `TypeVar`s inside `Type` has violated the "diagonal" rule. @@ -132,6 +140,14 @@ function instanceof_tfunc(@nospecialize(t), astag::Bool=false, @nospecialize(tro isexact = true end end + # If this is a NamedTuple type with known names but an unknown tuple type + # parameter, use the length of the names to constrain the tuple type. + if t′′ isa DataType && t′′.name === _NAMEDTUPLE_NAME && t′′.parameters[1] isa Tuple && has_free_typevars(t′′) + names = t′′.parameters[1]::Tuple + n = length(names) + nt_bound = NamedTuple{names, T} where T<:NTuple{n, Any} + tr = typeintersect(tr, nt_bound) + end return tr, isexact, isconcrete, istype elseif isa(t, Union) ta, isexact_a, isconcrete_a, istype_a = instanceof_tfunc(unwraptv(t.a), astag, troot) @@ -233,7 +249,7 @@ end function not_tfunc(𝕃::AbstractLattice, @nospecialize(b)) if isa(b, Conditional) - return Conditional(b.slot, b.elsetype, b.thentype) + return Conditional(b.slot, b.ssadef, b.elsetype, b.thentype) elseif isa(b, Const) return Const(not_int(b.val)) end @@ -315,7 +331,7 @@ add_tfunc(Core.Intrinsics.llvmcall, 3, INT_INF, llvmcall_tfunc, 10) @nospecs cglobal_tfunc(𝕃::AbstractLattice, fptr) = Ptr{Cvoid} @nospecs function cglobal_tfunc(𝕃::AbstractLattice, fptr, t) isa(t, Const) && return isa(t.val, Type) ? Ptr{t.val} : Ptr - return isType(t) ? Ptr{t.parameters[1]} : Ptr + return isType(t) ? Ptr{type_parameter(t)} : Ptr end add_tfunc(Core.Intrinsics.cglobal, 1, 2, cglobal_tfunc, 5) @@ -354,14 +370,14 @@ end if isa(x, Conditional) y = widenconditional(y) if isa(y, Const) - y.val === false && return Conditional(x.slot, x.elsetype, x.thentype) + y.val === false && return Conditional(x.slot, x.ssadef, x.elsetype, x.thentype) y.val === true && return x return Const(false) end elseif isa(y, Conditional) x = widenconditional(x) if isa(x, Const) - x.val === false && return Conditional(y.slot, y.elsetype, y.thentype) + x.val === false && return Conditional(y.slot, y.ssadef, y.elsetype, y.thentype) x.val === true && return y return Const(false) end @@ -371,8 +387,8 @@ end @nospecs function egal_tfunc(𝕃::ConstsLattice, x, y) if isa(x, Const) && isa(y, Const) return Const(x.val === y.val) - elseif (isa(x, Const) && y === typeof(x.val) && issingletontype(x)) || - (isa(y, Const) && x === typeof(y.val) && issingletontype(y)) + elseif (isa(x, Const) && y === typeof(x.val) && issingletontype(y)) || + (isa(y, Const) && x === typeof(y.val) && issingletontype(x)) return Const(true) end return egal_tfunc(widenlattice(𝕃), x, y) @@ -408,7 +424,7 @@ end if arg1 isa MustAlias arg1 = widenmustalias(arg1) end - arg1t = arg1 isa Const ? typeof(arg1.val) : isconstType(arg1) ? typeof(arg1.parameters[1]) : widenconst(arg1) + arg1t = arg1 isa Const ? typeof(arg1.val) : isconstType(arg1) ? typeof(type_parameter(arg1)) : widenconst(arg1) a1 = unwrap_unionall(arg1t) if isa(a1, DataType) && !isabstracttype(a1) if a1 === Module @@ -486,7 +502,12 @@ function sizeof_nothrow(@nospecialize(x)) x = widenconst(x) return !hasintersect(x, Type) end + xw = widenconst(x) + isType(xw) && !isconstType(xw) && return false x = unwrap_unionall(t) + # instances are variable-sized, so the type itself has no definite size + x === Core.CancellationTokenSource && return false + x === Core.WaitEntryN && return false if isconcrete if isa(x, DataType) && x.layout != C_NULL # there's just a few concrete types with an opaque layout @@ -506,11 +527,12 @@ function sizeof_nothrow(@nospecialize(x)) return true end -function _const_sizeof(@nospecialize(x)) +# f shall be Core.sizeof or Core.bitsizeof +function _const_sizeof(@nospecialize(f), @nospecialize(x)) # Constant GenericMemory does not have constant size isa(x, GenericMemory) && return Int size = try - Core.sizeof(x) + f(x) catch ex # Might return # "argument is an abstract type; size is indeterminate" or @@ -520,18 +542,18 @@ function _const_sizeof(@nospecialize(x)) end return Const(size) end -@nospecs function sizeof_tfunc(𝕃::AbstractLattice, x) +@nospecs function size_tfunc(𝕃::AbstractLattice, x, f) x = widenmustalias(x) - isa(x, Const) && return _const_sizeof(x.val) - isa(x, Conditional) && return _const_sizeof(Bool) - isconstType(x) && return _const_sizeof(x.parameters[1]) + isa(x, Const) && return _const_sizeof(f, x.val) + isa(x, Conditional) && return _const_sizeof(f, Bool) + isconstType(x) && return _const_sizeof(f, type_parameter(x)) xu = unwrap_unionall(x) if isa(xu, Union) - return tmerge(sizeof_tfunc(𝕃, rewrap_unionall(xu.a, x)), - sizeof_tfunc(𝕃, rewrap_unionall(xu.b, x))) + return tmerge(size_tfunc(𝕃, rewrap_unionall(xu.a, x), f), + size_tfunc(𝕃, rewrap_unionall(xu.b, x), f)) end - # Core.sizeof operates on either a type or a value. First check which - # case we're in. + # Core.sizeof or Core.bitsizeof operate on either a type or a value. + # First check which case we're in. t, exact = instanceof_tfunc(x, false) if t !== Bottom # The value corresponding to `x` at runtime could be a type. @@ -539,24 +561,27 @@ end x = unwrap_unionall(t) if exact && isa(x, Union) isinline = uniontype_layout(x)[1] - return isinline ? Const(Int(Core.sizeof(x))) : Bottom + return isinline ? Const(Int(f(x))) : Bottom end isa(x, DataType) || return Int - (isconcretetype(x) || isprimitivetype(x)) && return _const_sizeof(x) + (isconcretetype(x) || isprimitivetype(x)) && return _const_sizeof(f, x) else x = widenconst(x) - x !== DataType && isconcretetype(x) && return _const_sizeof(x) - isprimitivetype(x) && return _const_sizeof(x) + x !== DataType && isconcretetype(x) && return _const_sizeof(f, x) + isprimitivetype(x) && return _const_sizeof(f, x) end return Int end +@nospecs sizeof_tfunc(𝕃::AbstractLattice, x) = size_tfunc(𝕃, x, Core.sizeof) +@nospecs bitsizeof_tfunc(𝕃::AbstractLattice, x) = size_tfunc(𝕃, x, Core.bitsizeof) add_tfunc(Core.sizeof, 1, 1, sizeof_tfunc, 1) +add_tfunc(Core.bitsizeof, 1, 1, bitsizeof_tfunc, 1) @nospecs function nfields_tfunc(𝕃::AbstractLattice, x) isa(x, Const) && return Const(nfields(x.val)) isa(x, Conditional) && return Const(0) xt = widenconst(x) x = unwrap_unionall(xt) - isconstType(x) && return Const(nfields(x.parameters[1])) + isconstType(x) && return Const(nfields(type_parameter(x))) if isa(x, DataType) && !isabstracttype(x) if x.name === Tuple.name isvatuple(x) && return Int @@ -580,7 +605,20 @@ end add_tfunc(nfields, 1, 1, nfields_tfunc, 1) add_tfunc(Core._expr, 1, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->Expr), 100) add_tfunc(svec, 0, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->SimpleVector), 20) -@nospecs function _svec_ref_tfunc(𝕃::AbstractLattice, s, i) + +@nospecs function _svec_len_tfunc(::AbstractLattice, s) + if isa(s, Const) && isa(s.val, SimpleVector) + return Const(length(s.val)) + end + return Int +end +add_tfunc(Core._svec_len, 1, 1, _svec_len_tfunc, 1) +@nospecs function _svec_len_nothrow(𝕃::AbstractLattice, s) + ⊑ = partialorder(𝕃) + return s ⊑ SimpleVector +end + +@nospecs function _svec_ref_tfunc(::AbstractLattice, s, i) if isa(s, Const) && isa(i, Const) s, i = s.val, i.val if isa(s, SimpleVector) && isa(i, Int) @@ -590,7 +628,7 @@ add_tfunc(svec, 0, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->SimpleVec return Any end add_tfunc(Core._svec_ref, 2, 2, _svec_ref_tfunc, 1) -@nospecs function typevar_tfunc(𝕃::AbstractLattice, n, lb_arg, ub_arg) +@nospecs function typevar_tfunc(::AbstractLattice, n, lb_arg, ub_arg) lb = Union{} ub = Any ub_certain = lb_certain = true @@ -601,8 +639,10 @@ add_tfunc(Core._svec_ref, 2, 2, _svec_ref_tfunc, 1) lb = lb_arg.val else lb_arg = widenslotwrapper(lb_arg) - if isType(lb_arg) - lb = lb_arg.parameters[1] + if isTypeEgal(lb_arg) + lb = type_parameter(lb_arg) + elseif isTypeEq(lb_arg) + lb = type_parameter(lb_arg) lb_certain = false else return TypeVar @@ -612,8 +652,10 @@ add_tfunc(Core._svec_ref, 2, 2, _svec_ref_tfunc, 1) ub = ub_arg.val else ub_arg = widenslotwrapper(ub_arg) - if isType(ub_arg) - ub = ub_arg.parameters[1] + if isTypeEgal(ub_arg) + ub = type_parameter(ub_arg) + elseif isTypeEq(ub_arg) + ub = type_parameter(ub_arg) ub_certain = false else return TypeVar @@ -636,7 +678,7 @@ end ⊑ = partialorder(𝕃) b = widenconst(b) (b ⊑ TypeVar) && return true - if isType(b) + if isType(b) || b === typeof(Union{}) return true end return false @@ -695,7 +737,7 @@ function pointer_eltype(@nospecialize(ptr)) end @nospecs function pointerarith_tfunc(𝕃::AbstractLattice, ptr, offset) - return ptr + return widenconst(ptr) end @nospecs function pointerref_tfunc(𝕃::AbstractLattice, a, i, align) return pointer_eltype(a) @@ -703,7 +745,7 @@ end @nospecs function pointerset_tfunc(𝕃::AbstractLattice, a, v, i, align) return a end -@nospecs function atomic_fence_tfunc(𝕃::AbstractLattice, order) +@nospecs function atomic_fence_tfunc(𝕃::AbstractLattice, order, syncscope) return Nothing end @nospecs function atomic_pointerref_tfunc(𝕃::AbstractLattice, a, order) @@ -744,7 +786,7 @@ add_tfunc(add_ptr, 2, 2, pointerarith_tfunc, 1) add_tfunc(sub_ptr, 2, 2, pointerarith_tfunc, 1) add_tfunc(pointerref, 3, 3, pointerref_tfunc, 4) add_tfunc(pointerset, 4, 4, pointerset_tfunc, 5) -add_tfunc(atomic_fence, 1, 1, atomic_fence_tfunc, 4) +add_tfunc(atomic_fence, 2, 2, atomic_fence_tfunc, 4) add_tfunc(atomic_pointerref, 2, 2, atomic_pointerref_tfunc, 4) add_tfunc(atomic_pointerset, 3, 3, atomic_pointerset_tfunc, 5) add_tfunc(atomic_pointerswap, 3, 3, atomic_pointerswap_tfunc, 5) @@ -763,6 +805,8 @@ add_tfunc(donotdelete, 0, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->No return widenconditional(val) elseif setting === :type return Any + elseif setting === :blackbox + return widenconst(val) else return Bottom end @@ -771,7 +815,7 @@ add_tfunc(compilerbarrier, 2, 2, compilerbarrier_tfunc, 5) add_tfunc(Core.finalizer, 2, 4, @nospecs((𝕃::AbstractLattice, args...)->Nothing), 5) @nospecs function compilerbarrier_nothrow(setting, val) - return isa(setting, Const) && contains_is((:type, :const, :conditional), setting.val) + return isa(setting, Const) && contains_is((:type, :const, :conditional, :blackbox), setting.val) end # more accurate typeof_tfunc for vararg tuples abstract only in length @@ -797,11 +841,10 @@ end @nospecs function typeof_tfunc(𝕃::AbstractLattice, t) isa(t, Const) && return Const(typeof(t.val)) t = widenconst(t) - if isType(t) - tp = t.parameters[1] - if hasuniquerep(tp) - return Const(typeof(tp)) - end + if isconstType(t) + return Const(typeof(type_parameter(t))) + elseif isTypeEq(t) + # the value is only `== tp`, so its `typeof` is not pinned down (#61323) elseif isa(t, DataType) if isconcretetype(t) return Const(t) @@ -843,6 +886,23 @@ end end add_tfunc(typeof, 1, 1, typeof_tfunc, 1) +@nospecs function has_free_typevars_tfunc(𝕃::AbstractLattice, t) + isa(t, Const) && return Const(has_free_typevars(t.val)) + t = widenconst(t) + if isType(t) + p = type_parameter(t) + # a free typevar in the lattice element stands for closed runtime values + has_free_typevars(p) && return Bool + return Const(has_free_typevars(p)) + elseif t === TypeVar + return Const(true) + elseif !hasintersect(t, Type) && !hasintersect(t, TypeVar) && !hasintersect(t, TypeofVararg) + return Const(false) + end + return Bool +end +add_tfunc(has_free_typevars, 1, 1, has_free_typevars_tfunc, 1) + @nospecs function typeassert_tfunc(𝕃::AbstractLattice, v, t) t = instanceof_tfunc(t, true)[1] t === Any && return v @@ -854,7 +914,7 @@ add_tfunc(typeassert, 2, 2, typeassert_tfunc, 4) ⊑ = partialorder(𝕃) # ty, exact = instanceof_tfunc(t, true) # return exact && v ⊑ ty - if (isType(t) && !has_free_typevars(t) && v ⊑ t.parameters[1]) || + if (isType(t) && !has_free_typevars(t) && v ⊑ type_parameter(t)) || (isa(t, Const) && isa(t.val, Type) && v ⊑ t.val) return true end @@ -925,8 +985,8 @@ add_tfunc(<:, 2, 2, subtype_tfunc, 10) end function try_compute_fieldidx(@nospecialize(typ), @nospecialize(field)) - typ = argument_datatype(typ) - typ === nothing && return nothing + typ = unwraptv(typ) + typ isa Union || typ isa UnionAll || typ isa DataType || return nothing if isa(field, Symbol) field = fieldindex(typ, field, false) field == 0 && return nothing @@ -1008,7 +1068,7 @@ end nflds = fieldcount_noerror(sty) ismod = false else - sv = (s00::DataType).parameters[1] + sv = type_parameter(s00) sty = typeof(sv) nflds = nfields(sv) ismod = sv isa Module @@ -1040,7 +1100,7 @@ end if isa(s, Union) return getfield_nothrow(𝕃, rewrap_unionall(s.a, s00), name, boundscheck) && getfield_nothrow(𝕃, rewrap_unionall(s.b, s00), name, boundscheck) - elseif isType(s) && isTypeDataType(s.parameters[1]) + elseif isa(s, Core.TypeEgal) && isTypeDataType(type_parameter(s)) s = s0 = DataType end if isa(s, DataType) @@ -1097,7 +1157,9 @@ end function _getfield_tfunc_const(@nospecialize(sv), name::Const) nv = _getfield_fieldindex(typeof(sv), name) nv === nothing && return Bottom - if isa(sv, DataType) && nv == DATATYPE_TYPES_FIELDINDEX && isdefined(sv, nv) + # `types` and `super` are write-once: non-const (filled lazily), but once + # a defined value is observed it can never change, so folding it is sound + if isa(sv, DataType) && (nv == DATATYPE_TYPES_FIELDINDEX || nv == DATATYPE_SUPER_FIELDINDEX) && isdefined(sv, nv) return Const(getfield(sv, nv)) end if !isa(sv, Module) && isconst(typeof(sv), nv) @@ -1138,11 +1200,20 @@ end if nv < 1 return Bottom elseif nv ≤ length(s00.fields) + setfield && isconst(sty, nv) && return Bottom return unwrapva(s00.fields[nv]) end end end s00 = s + elseif isa(s00, PartialTask) + # N.B.: Do not use `PartialTask.fetch_type` to refine any field load here (e.g. + # `:result`). Task fields are mutable, so `fetch_type` is not an invariant of the + # current field contents; it is only sound as the *checked* side of the `typeassert` + # in `fetch` (via `task_result_type_tfunc`). Refining the load itself would let the + # optimizer prove that typeassert and delete it, turning a field mutation into + # silent type confusion instead of a runtime `TypeError`. + s00 = Task end return _getfield_tfunc(widenlattice(𝕃), s00, name, setfield) end @@ -1177,14 +1248,25 @@ end end if isType(s) if isconstType(s) - sv = (s00::DataType).parameters[1] + sv = type_parameter(s) if isa(name, Const) r = _getfield_tfunc_const(sv, name) r !== nothing && return r end s = typeof(sv) else - sv = s.parameters[1] + sv = type_parameter(s) + if isa(sv, DataType) && isa(name, Const) && + _getfield_fieldindex(DataType, name) == DATATYPE_SUPER_FIELDINDEX && + # read without forcing: on a deferred instantiation whose slot + # is still unset the modeled `getfield` throws, so no precision + # is lost by falling through (see issue #61347) + isdefined(sv, :super) && + (svsuper = getfield(sv, :super); !has_free_typevars(svsuper)) + # only `DataType` reps reach `.super` without throwing, and the + # `.super`s of `==`-equal `DataType`s are `==`-equal (if not egal) + return Type{svsuper} + end if isTypeDataType(sv) && isa(name, Const) nv = _getfield_fieldindex(DataType, name)::Int if nv == DATATYPE_NAME_FIELDINDEX @@ -1273,18 +1355,6 @@ end return rewrap_unionall(R, s00) end -# checks if a field of this type is guaranteed to be defined to a value -# and that access to an uninitialized field will cause an `UndefRefError` or return zero -# - is_undefref_fieldtype(String) === true -# - is_undefref_fieldtype(Integer) === true -# - is_undefref_fieldtype(Any) === true -# - is_undefref_fieldtype(Int) === false -# - is_undefref_fieldtype(Union{Int32,Int64}) === false -# - is_undefref_fieldtype(T) === false -function is_undefref_fieldtype(@nospecialize ftyp) - return !has_free_typevars(ftyp) && !allocatedinline(ftyp) -end - @nospecs function setfield!_tfunc(𝕃::AbstractLattice, o, f, v, order) if !isvarargtype(order) hasintersect(widenconst(order), Symbol) || return Bottom @@ -1344,15 +1414,25 @@ end return getfield_tfunc(𝕃, o, f) end @nospecs function modifyfield!_tfunc(𝕃::AbstractLattice, o, f, op, v, order=Symbol) + # the stored value is `op(o.f, v)`, so check only that `o.f` is writable at all + setfield!_tfunc(𝕃, o, f, Any) === Bottom && return Bottom o′ = widenconst(o) - T = _fieldtype_tfunc(𝕃, o′, f, isconcretetype(o′)) + exact = isconcretetype(o′) + egal = isa(o, Const) || exact + T = _fieldtype_tfunc(𝕃, o′, f, exact, egal) T === Bottom && return Bottom PT = Const(Pair) return instanceof_tfunc(apply_type_tfunc(𝕃, Any[PT, T, T]), true)[1] end @nospecs function replacefield!_tfunc(𝕃::AbstractLattice, o, f, x, v, success_order=Symbol, failure_order=Symbol) + # `replacefield!` type-checks the replacement `v` before the comparison, so a + # non-writable field or a `v` that cannot be stored always throws (even when the + # comparison would have failed) + setfield!_tfunc(𝕃, o, f, v) === Bottom && return Bottom o′ = widenconst(o) - T = _fieldtype_tfunc(𝕃, o′, f, isconcretetype(o′)) + exact = isconcretetype(o′) + egal = isa(o, Const) || exact + T = _fieldtype_tfunc(𝕃, o′, f, exact, egal) T === Bottom && return Bottom PT = Const(ccall(:jl_apply_cmpswap_type, Any, (Any,), T) where T) return instanceof_tfunc(apply_type_tfunc(𝕃, Any[PT, T]), true)[1] @@ -1363,7 +1443,7 @@ end return Bool end -@nospecs function abstract_modifyop!(interp::AbstractInterpreter, ff, argtypes::Vector{Any}, si::StmtInfo, sv::AbsIntState) +@nospecs function abstract_modifyop!(interp::AbstractInterpreter, ff, argtypes::Vector{Any}, si::StmtInfo, vtypes::Union{VarTable,Nothing}, sv::AbsIntState) if ff === modifyfield! minargs = 5 maxargs = 6 @@ -1374,6 +1454,11 @@ end maxargs = 6 op_argi = 4 v_argi = 5 + elseif ff === Core.modifyglobal_partition + minargs = 4 + maxargs = 5 + op_argi = 3 + v_argi = 4 elseif ff === Core.memoryrefmodify! minargs = 6 maxargs = 6 @@ -1407,6 +1492,21 @@ end GT = abstract_eval_get_binding_type(interp, sv, o, f).rt RT = isa(GT, Const) ? Pair{GT.val, GT.val} : Pair TF = isa(GT, Const) ? GT.val : Any + elseif ff === Core.modifyglobal_partition + GT = nothing + p = unwrapva(argtypes[2]) + if isa(p, Const) && isa(p.val, Core.BindingPartition) + p = p.val::Core.BindingPartition + kind = binding_kind(p) + if kind === PARTITION_KIND_GLOBAL + restriction = partition_restriction(p) + isa(restriction, Type) && (GT = restriction) + elseif kind === PARTITION_KIND_DECLARED + GT = Any + end + end + RT = GT === nothing ? Pair : Pair{GT, GT} + TF = GT === nothing ? Any : GT elseif ff === Core.memoryrefmodify! o = unwrapva(argtypes[2]) RT = memoryrefmodify!_tfunc(𝕃ᵢ, o, Any, Any, Symbol, Bool) @@ -1424,7 +1524,7 @@ end # as well as compute the info for the method matches op = unwrapva(argtypes[op_argi]) v = unwrapva(argtypes[v_argi]) - callinfo = abstract_call(interp, ArgInfo(nothing, Any[op, TF, v]), StmtInfo(true, si.saw_latestworld), sv, #=max_methods=#1) + callinfo = abstract_call(interp, ArgInfo(nothing, Any[op, TF, v]), StmtInfo(true, si.saw_latestworld), vtypes, sv, #=max_methods=#1) TF = Core.Box(TF) RT = Core.Box(RT) return Future{CallMeta}(callinfo, interp, sv) do callinfo, interp, sv @@ -1452,6 +1552,14 @@ add_tfunc(modifyfield!, 4, 5, modifyfield!_tfunc, 3) add_tfunc(replacefield!, 4, 6, replacefield!_tfunc, 3) add_tfunc(setfieldonce!, 3, 5, setfieldonce!_tfunc, 3) +function fieldtype_egal_lattice(@nospecialize(s0)) + s = widenconst(s0) + if isa(s, Union) + return fieldtype_egal_lattice(s.a) && fieldtype_egal_lattice(s.b) + end + return isa(s, Core.TypeEgal) +end + @nospecs function fieldtype_nothrow(𝕃::AbstractLattice, s0, name) s0 === Bottom && return true # unreachable ⊑ = partialorder(𝕃) @@ -1474,16 +1582,18 @@ add_tfunc(setfieldonce!, 3, 5, setfieldonce!_tfunc, 3) s, exact = instanceof_tfunc(s0, false) s === Bottom && return false # always - return _fieldtype_nothrow(s, exact, name) + egal = isa(s0, Const) || fieldtype_egal_lattice(s0) + return _fieldtype_nothrow(s, exact, egal, name) end -function _fieldtype_nothrow(@nospecialize(s), exact::Bool, name::Const) +function _fieldtype_nothrow(@nospecialize(s), exact::Bool, egal::Bool, name::Const) u = unwrap_unionall(s) if isa(u, Union) - a = _fieldtype_nothrow(u.a, exact, name) - b = _fieldtype_nothrow(u.b, exact, name) + a = _fieldtype_nothrow(u.a, exact, egal, name) + b = _fieldtype_nothrow(u.b, exact, egal, name) return exact ? (a || b) : (a && b) end + egal || return false u isa DataType || return false isabstracttype(u) && return false if u.name === _NAMEDTUPLE_NAME && !isconcretetype(u) @@ -1539,15 +1649,24 @@ end s, exact = instanceof_tfunc(s0, false) s === Bottom && return Bottom - return _fieldtype_tfunc(𝕃, s, name, exact) -end - -@nospecs function _fieldtype_tfunc(𝕃::AbstractLattice, s, name, exact::Bool) + # `exact` from a `Type{X}` element still admits `==`-equal but non-egal reps + # of `X`, whose `fieldtype` results are only `==`-equal to the stored type; + # the `Const` folds below additionally require an egality-certain argument + # value (#61323) + egal = isa(s0, Const) || isa(widenconst(s0), Core.TypeEgal) + return _fieldtype_tfunc(𝕃, s, name, exact, egal) +end + +# `egal` must be true only if `s` is the type of a runtime value (a canonical +# tag, whose stored field types are interned) or the egality-certain value of a +# `fieldtype` argument; a merely `==`-certain `fieldtype` argument yields field +# types that are `==` but not necessarily `===` the stored ones (#61323). +@nospecs function _fieldtype_tfunc(𝕃::AbstractLattice, s, name, exact::Bool, egal::Bool) exact = exact && !has_free_typevars(s) u = unwrap_unionall(s) if isa(u, Union) - ta0 = _fieldtype_tfunc(𝕃, rewrap_unionall(u.a, s), name, exact) - tb0 = _fieldtype_tfunc(𝕃, rewrap_unionall(u.b, s), name, exact) + ta0 = _fieldtype_tfunc(𝕃, rewrap_unionall(u.a, s), name, exact, egal) + tb0 = _fieldtype_tfunc(𝕃, rewrap_unionall(u.b, s), name, exact, egal) ta0 ⊑ tb0 && return tb0 tb0 ⊑ ta0 && return ta0 ta, exacta, _, istypea = instanceof_tfunc(ta0, false) @@ -1560,13 +1679,14 @@ end end return Any end + isType(u) && return Bottom # type objects have no fields u isa DataType || return Any if isabstracttype(u) # Abstract types have no fields exact && return Bottom # Type{...} without free typevars has no subtypes, so it is actually # exact, even if `exact` is false. - isType(u) && !has_free_typevars(u.parameters[1]) && return Bottom + isType(u) && !has_free_typevars(type_parameter(u)) && return Bottom return Any end if u.name === _NAMEDTUPLE_NAME && !isconcretetype(u) @@ -1585,15 +1705,30 @@ end end t = Bottom for i in 1:length(ftypes) - ft1 = unwrapva(ftypes[i]) + fti = ftypes[i] + ft1 = unwrapva(fti) + if !(isa(ft1, Type) || isa(ft1, TypeVar)) + if !isvarargtype(fti) && u.name === Tuple.name + # A genuine tuple field may be a value parameter (e.g. + # `Tuple{1:2}`); `fieldtype` returns that value rather than + # throwing. Type identity compares non-type parameters by + # egality, so the stored value is `===` the parameter even + # when the argument type is only `==`-certain (#61323) -- the + # `==`-vs-`===` ambiguity is specific to type-valued fields. + t = tmerge(t, Const(ft1)) + t === Any && break + continue + end + # Malformed Vararg types like `NTuple{<:Any, 3}` have non-Type + # components (e.g., `3`); `fieldtype` would throw at runtime. + continue + end exactft1 = exact || (!has_free_typevars(ft1) && u.name !== Tuple.name) ft1 = rewrap_unionall(ft1, s) if exactft1 - if hasuniquerep(ft1) - ft1 = Const(ft1) # ft unique via type cache - else - ft1 = Type{ft1} - end + # `fieldtype` returns exactly (`===`) the stored type, but only + # an egality-certain argument pins which stored rep is returned + ft1 = egal ? Const(ft1) : Type{ft1} elseif ft1 isa Type || ft1 isa TypeVar if ft1 === Any && u.name === Tuple.name # Tuple{:x} is possible in this case @@ -1624,18 +1759,22 @@ end return Bottom else ft = ftypes[fld] + if !(isa(ft, Type) || isa(ft, TypeVar)) && u.name === Tuple.name + # a value parameter in a genuine tuple field (see the loop above): + # non-type parameters are compared by egality, so this is `Const` + # even for an `==`-only argument + return Const(ft) + end end - if !isa(ft, Type) && !isa(ft, TypeVar) - return Const(ft) + if !(isa(ft, Type) || isa(ft, TypeVar)) + return Bottom # see non-`Const` case above end exactft = exact || (!has_free_typevars(ft) && u.name !== Tuple.name) ft = rewrap_unionall(ft, s) if exactft - if hasuniquerep(ft) - return Const(ft) # ft unique via type cache - end - return Type{ft} + # only an egality-certain argument pins the stored rep (see above) + return egal ? Const(ft) : Type{ft} end if u.name === Tuple.name && ft === Any # Tuple{:x} is possible @@ -1647,9 +1786,78 @@ add_tfunc(fieldtype, 2, 3, fieldtype_tfunc, 0) # Like `valid_tparam`, but in the type domain. valid_tparam_type(T::DataType) = valid_typeof_tparam(T) +valid_tparam_type(T::TypeEq) = true +valid_tparam_type(T::Core.TypeEgal) = true valid_tparam_type(U::Union) = valid_tparam_type(U.a) && valid_tparam_type(U.b) valid_tparam_type(U::UnionAll) = valid_tparam_type(unwrap_unionall(U)) +function typeeq_apply_type_nothrow(𝕃::AbstractLattice, argtypes::Vector{Any}) + length(argtypes) == 2 || return false + ai = widenslotwrapper(widenconditional(argtypes[2])) + # a bare `TypeEgal{T}` element pins the argument value like `Const(T)` does + ai = maybe_singleton_const(ai) + if isa(ai, Const) + v = ai.val + return isa(v, Type) || isa(v, TypeVar) || valid_tparam(v) + end + isTypeEq(ai) && return true + isa(ai, PartialTypeVar) && return true + ai = widenconst(ai) + return (⊑(𝕃, ai, AnyType) || ⊑(𝕃, ai, TypeVar) || + (isa(ai, Type) && valid_tparam_type(ai))) +end + +function typeeq_apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}) + length(argtypes) == 2 || return Bottom + ai = widenslotwrapper(argtypes[2]) + # a bare `TypeEgal{T}` element pins the argument value like `Const(T)` does + ai = maybe_singleton_const(ai) + if isa(ai, Const) + v = ai.val + (isa(v, Type) || isa(v, TypeVar) || valid_tparam(v)) || return Bottom + return Const(apply_type(TypeEq, v)) + end + if isTypeEq(ai) + # the argument is only `== X`, so the result is only `== Type{X}` (#61323) + return Type{ai} + end + if isa(ai, PartialTypeVar) + (ai.lb_certain && ai.ub_certain) || return TypeEq + return Type{TypeEq{ai.tv}} + end + ai = widenconst(ai) + if ⊑(𝕃, ai, AnyType) || ⊑(𝕃, ai, TypeVar) || (isa(ai, Type) && valid_tparam_type(ai)) + return TypeEq + end + if isa(ai, Type) && isconcretetype(ai) + return Bottom + end + return TypeEq +end + +# like `TypeEq`, but only closed type values are valid parameters +function typeegal_apply_type_nothrow(𝕃::AbstractLattice, argtypes::Vector{Any}) + length(argtypes) == 2 || return false + ai = widenslotwrapper(widenconditional(argtypes[2])) + if isa(ai, Const) + v = ai.val + return isa(v, Type) && !has_free_typevars(v) + end + # for a non-constant parameter we cannot rule out non-types or free typevars + return false +end + +function typeegal_apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}) + length(argtypes) == 2 || return Bottom + ai = widenslotwrapper(argtypes[2]) + if isa(ai, Const) + v = ai.val + (isa(v, Type) && !has_free_typevars(v)) || return Bottom + return Const(Core.apply_type(Core.TypeEgal, v)) + end + return Core.TypeEgal +end + function apply_type_nothrow(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospecialize(rt)) rt === Type && return false length(argtypes) >= 1 || return false @@ -1657,13 +1865,15 @@ function apply_type_nothrow(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospe if isa(headtypetype, Const) headtype = headtypetype.val elseif isconstType(headtypetype) - headtype = headtypetype.parameters[1] + headtype = type_parameter(headtypetype) else return false end # We know the apply_type is well formed. Otherwise our rt would have been # Bottom (or Type). (headtype === Union) && return true + headtype === TypeEq && return typeeq_apply_type_nothrow(𝕃, argtypes) + headtype === Core.TypeEgal && return typeegal_apply_type_nothrow(𝕃, argtypes) isa(rt, Const) && return true u = headtype # TODO: implement optimization for isvarargtype(u) and istuple occurrences (which are valid but are not UnionAll) @@ -1677,7 +1887,7 @@ function apply_type_nothrow(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospe return false end elseif (isa(ai, Const) && isa(ai.val, Type)) || isconstType(ai) - ai = isa(ai, Const) ? ai.val : (ai::DataType).parameters[1] + ai = isa(ai, Const) ? ai.val : type_parameter(ai) if has_free_typevars(u.var.lb) || has_free_typevars(u.var.ub) return false end @@ -1695,7 +1905,14 @@ function apply_type_nothrow(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospe end else istype || return false - if isa(u.var.ub, TypeVar) || !(T <: u.var.ub) + if isa(u.var.ub, TypeVar) + return false + end + Tub = T + while isa(Tub, TypeVar) + Tub = Tub.ub + end + if !(Tub <: u.var.ub) return false end if exact ? !(u.var.lb <: T) : !(u.var.lb === Bottom) @@ -1721,7 +1938,11 @@ function apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}; if isa(headtypetype, Const) headtype = headtypetype.val elseif isconstType(headtypetype) - headtype = headtypetype.parameters[1] + headtype = type_parameter(headtypetype) + elseif isTypeEq(headtypetype) + # an `==`-only head cannot be pinned down; a `Union` head can also collapse + # `Union{T}` to a bare `TypeVar` + return type_parameter(headtypetype) == Union ? Union{Type, TypeVar} : Type else return Any end @@ -1743,7 +1964,7 @@ function apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}; end end else - if !isType(ai) + if !(isTypeEq(ai) || (isTypeEgal(ai) && type_parameter(ai) isa Type)) if !isa(ai, Type) || hasintersect(ai, Type) || hasintersect(ai, TypeVar) hasnonType = true else @@ -1760,9 +1981,14 @@ function apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}; allconst = true for i = 2:largs ai = argtypes[i] - if isType(ai) - aty = ai.parameters[1] - allconst &= hasuniquerep(aty) + if isTypeEgal(ai) + aty = type_parameter(ai) + elseif isTypeEq(ai) + aty = type_parameter(ai) + # `Union` instantiation does not canonicalize its arguments the + # way datatype instantiation does (`Union{S}` is `S` itself), so + # an `==`-only argument leaves the result only `==`-certain + allconst = false else aty = (ai::Const).val end @@ -1770,6 +1996,12 @@ function apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}; end return allconst ? Const(ty) : Type{ty} end + if headtype === TypeEq + return typeeq_apply_type_tfunc(𝕃, argtypes) + end + if headtype === Core.TypeEgal + return typeegal_apply_type_tfunc(𝕃, argtypes) + end if 1 < unionsplitcost(𝕃, argtypes) ≤ max_union_splitting rt = Bottom for split_argtypes = switchtupleunion(𝕃, argtypes) @@ -1780,6 +2012,44 @@ function apply_type_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}; end return _apply_type_tfunc(𝕃, headtype, argtypes) end + +function apply_type_arg_value(@nospecialize(t)) + if isa(t, Const) + val = t.val + elseif isType(t) + val = type_parameter(t) + else + val = singleton_type(t) + val === nothing && return nothing + end + return (isa(val, Type) || isvarargtype(val)) ? val : nothing +end + +function partial_typeofvararg_value(@nospecialize(t)) + isa(t, PartialStruct) || return nothing + t.typ === TypeofVararg || return nothing + length(t.fields) >= 1 || return nothing + undefs = _getundefs(t) + undefs[1] === false || return nothing + T = apply_type_arg_value(t.fields[1]) + isa(T, Type) || return nothing + try + if length(t.fields) >= 2 && undefs[2] === false + N = t.fields[2] + isa(N, Const) && isa(N.val, Int) || return nothing + return Core.apply_type(Vararg, T, N.val) + end + # `Vararg{T}` claims the `N` field is absent; an unknown defined-ness + # (e.g. from merging `Vararg{T}` and `Vararg{T,n}` partials) may stand + # for a value that does carry `N` + (length(undefs) >= 2 && undefs[2] === true) || return nothing + return Core.apply_type(Vararg, T) + catch ex + ex isa InterruptException && rethrow() + return nothing + end +end + @nospecs function _apply_type_tfunc(𝕃::AbstractLattice, headtype, argtypes::Vector{Any}) largs = length(argtypes) istuple = headtype === Tuple @@ -1789,6 +2059,7 @@ end uw = unwrap_unionall(headtype) uncertain = false canconst = true + anyeq = false # some argument is only known up to type equality (`==`) tparams = Any[] outervars = TypeVar[] @@ -1809,10 +2080,16 @@ end ua = headtype for i = 2:largs ai = widenslotwrapper(argtypes[i]) - if isType(ai) - aip1 = ai.parameters[1] + if isTypeEgal(ai) + push!(tparams, type_parameter(ai)) + elseif isTypeEq(ai) + aip1 = type_parameter(ai) canconst &= !has_free_typevars(aip1) + anyeq = true push!(tparams, aip1) + elseif istuple && (pva = partial_typeofvararg_value(ai)) !== nothing + anyeq = true + push!(tparams, pva) elseif isa(ai, Const) && (isa(ai.val, Type) || isa(ai.val, TypeVar) || valid_tparam(ai.val) || (istuple && isvarargtype(ai.val))) push!(tparams, ai.val) @@ -1820,12 +2097,19 @@ end canconst = false push!(tparams, ai.tv) else + if widenconst(ai) <: TypeVar && widenconst(ai) !== Union{} + # A TypeVar value of unknown identity used as a type parameter + # yields a type with a free typevar, which `jl_isa` excludes + # from every closed `Type{...}` form this function could + # construct; only the top kind forms are sound here. + return isvarargtype(headtype) ? TypeofVararg : Type + end uncertain = true unw = unwrap_unionall(ai) isT = isType(unw) # compute our desired upper bound value if isT - ub = rewrap_unionall(unw.parameters[1], ai) + ub = rewrap_unionall(type_parameter(unw), ai) else ub = Any end @@ -1834,7 +2118,7 @@ end # outer type, use the wrapper type, instead of letting it nest more # complexity here. This is not monotonic, but seems to work out pretty well. if isT - ub = unwrap_unionall(unw.parameters[1]) + ub = unwrap_unionall(type_parameter(unw)) if ub isa DataType ub = ub.name.wrapper unw = Type{unwrap_unionall(ub)} @@ -1865,13 +2149,13 @@ end # make sure vars introduced here are unique if contains_is(outervars, tai.var) ai = rename_unionall(ai) - unw = unwrap_unionall(ai)::DataType + unw = unwrap_unionall(ai) # ub = rewrap_unionall(unw, ai) break end tai = tai.body end - push!(tparams, unw.parameters[1]) + push!(tparams, type_parameter(unw)) while isa(ai, UnionAll) push!(outervars, ai.var) ai = ai.body @@ -1931,11 +2215,28 @@ end end end end - !uncertain && canconst && return Const(appl) + # An `==`-only (`Type{X}`) argument still yields a `Const`: datatype instantiation + # normalizes and `==`-deduplicates its (invariant) parameters, so all `S == X` reps + # apply to the same (egal) object. Wrapper (`Type{...}`), `UnionAll` and covariant + # `Tuple` results do not canonicalize and stay egal-distinct across reps (#61323). + !uncertain && canconst && + (!anyeq || (isa(appl, DataType) && appl.name !== Tuple.name)) && return Const(appl) if isvarargtype(appl) + !uncertain && canconst && !anyeq && return Const(appl) + if !uncertain && canconst && 2 <= largs <= 3 + fields = Any[widenslotwrapper(argtypes[2]), largs == 3 ? widenslotwrapper(argtypes[3]) : Any] + undefs = Union{Nothing,Bool}[false, largs == 3 ? false : true] + return PartialStruct(𝕃, TypeofVararg, undefs, fields) + end return TypeofVararg end if istuple + if !uncertain && canconst + # every parameter is exactly known and only `==`-certainty (`anyeq`) + # blocked the `Const` fold above: the invariant parameters still pin + # the tuple up to type equality (`S == X` implies `Tuple{S} == Tuple{X}`) + return Type{appl} + end return Type{<:appl} end ans = Type{appl} @@ -1960,15 +2261,8 @@ function tuple_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}) # UnionAll context is missing around this. pop!(argtypes) end - all_are_const = true - for i in 1:length(argtypes) - if !isa(argtypes[i], Const) - all_are_const = false - break - end - end - if all_are_const - return Const(ntuple(i::Int->argtypes[i].val, length(argtypes))) + if is_all_const_arg(argtypes, 1) # repeated from builtin_tfunction for the benefit of callers that use this tfunc directly + return Const(tuple(collect_const_args(argtypes, 1)...)) end params = Vector{Any}(undef, length(argtypes)) anyinfo = false @@ -1990,10 +2284,15 @@ function tuple_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}) # here we should turn such `Type{...}`-parameters to valid parameters, e.g. # (::Type{Int},) -> Tuple{DataType} (or PartialStruct for more accuracy) # (::Union{Type{Int32},Type{Int64}}) -> Tuple{Type} - if isType(x) + if isTypeEgal(x) anyinfo = true - xparam = x.parameters[1] - if hasuniquerep(xparam) || xparam === Bottom + params[i] = typeof(type_parameter(x)) + elseif isTypeEq(x) + anyinfo = true + xparam = type_parameter(x) + # an `==`-only type value has no pinned `typeof` (#61323), except + # `Type{Union{}}` whose only value is `Union{}` + if xparam === Bottom params[i] = typeof(xparam) else params[i] = Type @@ -2032,6 +2331,10 @@ end hasintersect(widenconst(item), memoryrefget_tfunc(𝕃, mem, order, boundscheck)) || return Bottom return item end +@nospecs function memoryrefunset!_tfunc(𝕃::AbstractLattice, mem, order, boundscheck) + memoryref_builtin_common_errorcheck(mem, order, boundscheck) || return Bottom + return Nothing +end @nospecs function memoryrefswap!_tfunc(𝕃::AbstractLattice, mem, v, order, boundscheck) memoryrefset!_tfunc(𝕃, mem, v, order, boundscheck) === Bottom && return Bottom return memoryrefget_tfunc(𝕃, mem, order, boundscheck) @@ -2059,6 +2362,7 @@ end add_tfunc(Core.memoryrefget, 3, 3, memoryrefget_tfunc, 20) add_tfunc(Core.memoryrefset!, 4, 4, memoryrefset!_tfunc, 20) +add_tfunc(Core.memoryrefunset!, 3, 3, memoryrefunset!_tfunc, 20) add_tfunc(Core.memoryrefswap!, 4, 4, memoryrefswap!_tfunc, 20) add_tfunc(Core.memoryrefmodify!, 5, 5, memoryrefmodify!_tfunc, 20) add_tfunc(Core.memoryrefreplace!, 6, 6, memoryrefreplace!_tfunc, 20) @@ -2231,6 +2535,8 @@ function memoryrefop_builtin_common_nothrow(𝕃::AbstractLattice, argtypes::Vec # If we could potentially throw undef ref errors, bail out now. array_type_undefable(memtype) && return false end + # memoryrefunset! has no value-typed arg to check; element type is always + # compatible with `nothing` since we are only zeroing pointer slots. # If we have @inbounds (last argument is false), we're allowed to assume # we don't throw bounds errors. if isa(boundscheck, Const) @@ -2279,6 +2585,8 @@ function _builtin_nothrow(𝕃::AbstractLattice, @nospecialize(f::Builtin), argt return memtype ⊑ GenericMemoryRef elseif f === memoryrefset! return memoryrefop_builtin_common_nothrow(𝕃, argtypes, f) + elseif f === memoryrefunset! + return memoryrefop_builtin_common_nothrow(𝕃, argtypes, f) elseif f === memoryrefget return memoryrefop_builtin_common_nothrow(𝕃, argtypes, f) elseif f === memoryref_isassigned @@ -2313,7 +2621,7 @@ function _builtin_nothrow(𝕃::AbstractLattice, @nospecialize(f::Builtin), argt return subtype_nothrow(𝕃, argtypes[1], argtypes[2]) elseif f === isdefined return isdefined_nothrow(𝕃, argtypes) - elseif f === Core.sizeof + elseif f === Core.sizeof || f === Core.bitsizeof na == 1 || return false return sizeof_nothrow(argtypes[1]) elseif f === Core.ifelse @@ -2334,19 +2642,26 @@ function _builtin_nothrow(𝕃::AbstractLattice, @nospecialize(f::Builtin), argt elseif f === Core.compilerbarrier na == 2 || return false return compilerbarrier_nothrow(argtypes[1], nothing) + elseif f === Core._svec_len + na == 1 || return false + return _svec_len_nothrow(𝕃, argtypes[1]) elseif f === Core._svec_ref na == 2 || return false return _svec_ref_tfunc(𝕃, argtypes[1], argtypes[2]) isa Const + elseif f === Core.task_result_type + na == 1 || return false + return argtypes[1] ⊑ Task end return false end -# known to be always effect-free (in particular nothrow) +# known to be always effect-free (in particular also nothrow) const _PURE_BUILTINS = Any[ tuple, svec, ===, typeof, + has_free_typevars, nfields, ] @@ -2355,11 +2670,13 @@ const _CONSISTENT_BUILTINS = Any[ svec, # SimpleVector is immutable, thus svecs of egal arguments are egal ===, typeof, + has_free_typevars, nfields, fieldtype, apply_type, isa, UnionAll, + Core.bitsizeof, Core.sizeof, Core.ifelse, (<:), @@ -2370,6 +2687,8 @@ const _CONSISTENT_BUILTINS = Any[ donotdelete, memoryrefnew, memoryrefoffset, + Core._svec_len, + Core._svec_ref, ] # known to be effect-free (but not necessarily nothrow) @@ -2385,6 +2704,7 @@ const _EFFECT_FREE_BUILTINS = [ memoryrefget, memoryref_isassigned, isdefined, + Core.bitsizeof, Core.sizeof, Core.ifelse, Core._typevar, @@ -2393,25 +2713,32 @@ const _EFFECT_FREE_BUILTINS = [ throw, Core.throw_methoderror, getglobal, + Core.getglobal_partition, + Core.isdefinedglobal_partition, compilerbarrier, + Core._svec_len, Core._svec_ref, + Core.task_result_type, ] const _INACCESSIBLEMEM_BUILTINS = Any[ (<:), (===), apply_type, + Core.bitsizeof, Core.ifelse, Core.sizeof, svec, fieldtype, isa, nfields, + Core.task_result_type, throw, Core.throw_methoderror, tuple, typeassert, typeof, + has_free_typevars, compilerbarrier, Core._typevar, donotdelete, @@ -2424,10 +2751,12 @@ const _ARGMEM_BUILTINS = Any[ memoryrefget, memoryref_isassigned, memoryrefset!, + memoryrefunset!, modifyfield!, replacefield!, setfield!, swapfield!, + Core._svec_len, Core._svec_ref, ] @@ -2516,6 +2845,20 @@ function isdefined_effects(𝕃::AbstractLattice, argtypes::Vector{Any}) return Effects(EFFECTS_TOTAL; consistent, nothrow, inaccessiblememonly) end +function is_relocatable_ptr_field(ty, fld) + # Special case: these fields are not mutated, but they are the only native pointer fields + # that have relocations in staticdata.c, so they can change between processes. + if hasintersect(widenconst(ty), Core.TypeName) && + (Const(:constfields) ⊑ fld || Const(fieldindex(Core.TypeName, :constfields)) ⊑ fld || + Const(:atomicfields) ⊑ fld || Const(fieldindex(Core.TypeName, :atomicfields)) ⊑ fld) + return true + elseif hasintersect(widenconst(ty), DataType) && + (Const(:layout) ⊑ fld || Const(fieldindex(DataType, :layout)) ⊑ fld) + return true + end + return false +end + function getfield_effects(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospecialize(rt)) length(argtypes) < 2 && return EFFECTS_THROWS obj = argtypes[1] @@ -2527,8 +2870,12 @@ function getfield_effects(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospeci noub=ALWAYS_FALSE) end # :consistent if the argtype is immutable - consistent = (is_immutable_argtype(obj) || is_mutation_free_argtype(obj)) ? - ALWAYS_TRUE : CONSISTENT_IF_INACCESSIBLEMEMONLY + if is_relocatable_ptr_field(argtypes[1], argtypes[2]) + consistent = ALWAYS_FALSE + else + consistent = (is_immutable_argtype(obj) || is_mutation_free_argtype(obj)) ? + ALWAYS_TRUE : CONSISTENT_IF_INACCESSIBLEMEMONLY + end noub = ALWAYS_TRUE bcheck = getfield_boundscheck(argtypes) nothrow = getfield_nothrow(𝕃, argtypes, bcheck) @@ -2545,7 +2892,7 @@ function getfield_effects(𝕃::AbstractLattice, argtypes::Vector{Any}, @nospeci noub = ALWAYS_FALSE end end - if hasintersect(widenconst(obj), Module) + if hasintersect(widenconst(obj), Module) || is_relocatable_ptr_field(argtypes[1], argtypes[2]) # Modeled more precisely in abstract_eval_getglobal inaccessiblememonly = ALWAYS_FALSE elseif is_mutation_free_argtype(obj) @@ -2565,21 +2912,24 @@ const _EFFECTS_KNOWN_BUILTINS = Any[ # _apply_iterate, # Core._call_in_world_total, # Core._compute_sparams, - # Core._defaultctors, # Core._equiv_typedef, Core._expr, # Core._primitivetype, # Core._setsuper!, # Core._structtype, + Core._svec_len, Core._svec_ref, + Core._task, # Core._typebody!, Core._typevar, apply_type, compilerbarrier, Core.current_scope, + Core.depwarn_partition, donotdelete, Core.finalizer, Core.get_binding_type, + Core.getglobal_partition, Core.ifelse, # Core.invoke_in_world, # invokelatest, @@ -2593,6 +2943,8 @@ const _EFFECTS_KNOWN_BUILTINS = Any[ memoryrefset!, # Core.memoryrefsetonce!, # Core.memoryrefswap!, + memoryrefunset!, + Core.bitsizeof, Core.sizeof, svec, Core.throw_methoderror, @@ -2600,21 +2952,29 @@ const _EFFECTS_KNOWN_BUILTINS = Any[ fieldtype, getfield, getglobal, + has_free_typevars, # invoke, isa, isdefined, # isdefinedglobal, + Core.isdefinedglobal_partition, modifyfield!, # modifyglobal!, + # Core.modifyglobal_partition, nfields, replacefield!, # replaceglobal!, + Core.replaceglobal_partition, setfield!, # setfieldonce!, # setglobal!, + Core.setglobal_partition, # setglobalonce!, + Core.setglobalonce_partition, swapfield!, # swapglobal!, + Core.swapglobal_partition, + Core.task_result_type, throw, tuple, typeassert, @@ -2650,6 +3010,32 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argty 2 ≤ length(argtypes) ≤ 3 || return EFFECTS_THROWS # Modeled more precisely in abstract_eval_getglobal return generic_getglobal_effects + elseif f === Core.getglobal_partition + length(argtypes) == 3 || return EFFECTS_THROWS + partition = argtypes[2] + if isa(partition, Const) && isa(partition.val, Core.BindingPartition) + p = partition.val::Core.BindingPartition + # Same effects as the read this partition encodes, but the explicit memory order + # may be invalid or non-atomic, so it is not `nothrow`. + return Effects(abstract_eval_partition_load(partition_owner(p), p, false).effects; nothrow=false) + end + return generic_getglobal_effects + elseif f === Core.setglobal_partition + return setglobal!_effects + elseif f === Core.swapglobal_partition || f === Core.replaceglobal_partition || + f === Core.setglobalonce_partition + # These also read the binding, so they are no more `:consistent` than a read is + # (matching `abstract_eval_swapglobal!` and friends). + return merge_effects(generic_getglobal_effects, setglobal!_effects) + elseif f === Core.isdefinedglobal_partition + length(argtypes) == 2 || return EFFECTS_THROWS + return generic_isdefinedglobal_effects + elseif f === Core.depwarn_partition + length(argtypes) == 1 || return EFFECTS_THROWS + # A deprecation warning is an observable side effect (printing to stderr, and + # throwing under `--depwarn=error`), so this call must never be removed as unused. + return Effects(EFFECTS_TOTAL; consistent=ALWAYS_FALSE, effect_free=ALWAYS_FALSE, + nothrow=false, inaccessiblememonly=ALWAYS_FALSE) elseif f === Core.get_binding_type length(argtypes) == 2 || return EFFECTS_THROWS # Modeled more precisely in abstract_eval_get_binding_type @@ -2672,17 +3058,19 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argty consistent = ALWAYS_FALSE, notaskstate = false, nothrow) + elseif f === Core._task + return TASK_BUILTIN_EFFECTS else if contains_is(_CONSISTENT_BUILTINS, f) consistent = ALWAYS_TRUE - elseif f === memoryrefget || f === memoryrefset! || f === memoryref_isassigned || f === Core._svec_ref + elseif f === memoryrefget || f === memoryrefset! || f === memoryrefunset! || f === memoryref_isassigned || f === Core._svec_len || f === Core._svec_ref consistent = CONSISTENT_IF_INACCESSIBLEMEMONLY elseif f === Core._typevar || f === Core.memorynew consistent = CONSISTENT_IF_NOTRETURNED else consistent = ALWAYS_FALSE end - if f === setfield! || f === memoryrefset! + if f === setfield! || f === memoryrefset! || f === memoryrefunset! effect_free = EFFECT_FREE_IF_INACCESSIBLEMEMONLY elseif contains_is(_EFFECT_FREE_BUILTINS, f) || contains_is(_PURE_BUILTINS, f) effect_free = ALWAYS_TRUE @@ -2697,7 +3085,7 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argty else inaccessiblememonly = ALWAYS_FALSE end - if f === memoryrefnew || f === memoryrefget || f === memoryrefset! || f === memoryref_isassigned + if f === memoryrefnew || f === memoryrefget || f === memoryrefset! || f === memoryrefunset! || f === memoryref_isassigned noub = memoryop_noub(f, argtypes) ? ALWAYS_TRUE : ALWAYS_FALSE else noub = ALWAYS_TRUE @@ -2718,7 +3106,7 @@ function memoryop_noub(@nospecialize(f), argtypes::Vector{Any}) return true end expected_nargs = 3 - elseif f === memoryrefget || f === memoryref_isassigned + elseif f === memoryrefget || f === memoryref_isassigned || f === memoryrefunset! expected_nargs = 3 else @assert f === memoryrefset! "unexpected memoryop is given" @@ -2736,9 +3124,8 @@ function memoryop_noub(@nospecialize(f), argtypes::Vector{Any}) return false end -function current_scope_tfunc(interp::AbstractInterpreter, sv::InferenceState) +function current_scope_tfunc(::AbstractInterpreter, sv::InferenceState) pc = sv.currpc - handler_info = sv.handler_info while true pchandler = gethandler(sv, pc) if pchandler === nothing @@ -2760,7 +3147,7 @@ function current_scope_tfunc(interp::AbstractInterpreter, sv::InferenceState) pc = pchandler.enter_idx end end -current_scope_tfunc(interp::AbstractInterpreter, sv) = Any +current_scope_tfunc(::AbstractInterpreter, ::IRInterpretationState) = Any hasvarargtype(argtypes::Vector{Any}) = !isempty(argtypes) && isvarargtype(argtypes[end]) @@ -2784,11 +3171,12 @@ end function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtypes::Vector{Any}, sv::Union{AbsIntState, Nothing}) 𝕃ᵢ = typeinf_lattice(interp) - if isa(f, IntrinsicFunction) - if is_pure_intrinsic_infer(f) && all(@nospecialize(a) -> isa(a, Const), argtypes) - argvals = anymap(@nospecialize(a) -> (a::Const).val, argtypes) + # Early constant evaluation for foldable builtins with all const args + if isa(f, IntrinsicFunction) ? is_pure_intrinsic_infer(f) : (contains_is(_PURE_BUILTINS, f) || (contains_is(_CONSISTENT_BUILTINS, f) && contains_is(_EFFECT_FREE_BUILTINS, f))) + if is_all_const_arg(argtypes, 1) + argvals = collect_const_args(argtypes, 1) try - # unroll a few cases which have specialized codegen + # unroll a few common cases for better codegen if length(argvals) == 1 return Const(f(argvals[1])) elseif length(argvals) == 2 @@ -2802,6 +3190,8 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp return Bottom end end + end + if isa(f, IntrinsicFunction) iidx = Int(reinterpret(Int32, f)) + 1 if iidx < 0 || iidx > length(T_IFUNC) # unknown intrinsic @@ -2828,6 +3218,7 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp end tf = T_FFUNC_VAL[fidx] end + if hasvarargtype(argtypes) if length(argtypes) - 1 > tf[2] # definitely too many arguments @@ -2946,12 +3337,12 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V end if f === Intrinsics.bitcast - ty, isexact, isconcrete = instanceof_tfunc(argtypes[1], true) + ty, _, isconcrete, _ = instanceof_tfunc(argtypes[1], true) xty = widenconst(argtypes[2]) if !isconcrete return Union{ErrorException, TypeError} end - if !(isprimitivetype(ty) && isprimitivetype(xty) && Core.sizeof(ty) === Core.sizeof(xty)) + if !(isprimitivetype(ty) && isprimitivetype(xty) && Core.bitsizeof(ty) === Core.bitsizeof(xty)) return ErrorException end return Union{} @@ -2962,7 +3353,7 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V Intrinsics.sitofp, Intrinsics.fptrunc, Intrinsics.fpext) # If !isconcrete, `ty` may be Union{} at runtime even if we have # isprimitivetype(ty). - ty, isexact, isconcrete = instanceof_tfunc(argtypes[1], true) + ty, _, isconcrete, _ = instanceof_tfunc(argtypes[1], true) if !isconcrete return Union{ErrorException, TypeError} end @@ -2971,16 +3362,22 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V return ErrorException end - # fpext, fptrunc, fptoui, fptosi, uitofp, and sitofp have further - # restrictions on the allowed types. + # fpext, sext_int, zext_int, fptrunc, trunc_int, fptoui, fptosi, uitofp, and sitofp + # have further restrictions on the allowed types. if f === Intrinsics.fpext && !(ty <: CORE_FLOAT_TYPES && xty <: CORE_FLOAT_TYPES && Core.sizeof(ty) > Core.sizeof(xty)) return ErrorException end + if (f === Intrinsics.sext_int || f === Intrinsics.zext_int) && !(Core.bitsizeof(ty) > Core.bitsizeof(xty)) + return ErrorException + end if f === Intrinsics.fptrunc && !(ty <: CORE_FLOAT_TYPES && xty <: CORE_FLOAT_TYPES && Core.sizeof(ty) < Core.sizeof(xty)) return ErrorException end + if f === Intrinsics.trunc_int && !(Core.bitsizeof(ty) < Core.bitsizeof(xty)) + return ErrorException + end if (f === Intrinsics.fptoui || f === Intrinsics.fptosi) && !(xty <: CORE_FLOAT_TYPES) return ErrorException end @@ -2992,18 +3389,26 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V end if f === Intrinsics.have_fma - ty, isexact, isconcrete = instanceof_tfunc(argtypes[1], true) + ty, _, isconcrete, _ = instanceof_tfunc(argtypes[1], true) if !(isconcrete && isprimitivetype(ty)) return TypeError end return Union{} end + if f === Intrinsics.add_ptr || f === Intrinsics.sub_ptr + if !(argtypes[1] ⊑ Ptr && argtypes[2] ⊑ UInt) + return TypeError + end + return Union{} + end + # The remaining intrinsics are math/bits/comparison intrinsics. # All the non-floating point intrinsics work on primitive values of the same type. isshift = f === shl_int || f === lshr_int || f === ashr_int argtype1 = widenconst(argtypes[1]) isprimitivetype(argtype1) || return ErrorException + f === bswap_int && Core.bitsizeof(argtype1) % 16 != 0 && return ErrorException if contains_is(_FLOAT_INTRINSICS, f) argtype1 <: CORE_FLOAT_TYPES || return ErrorException end @@ -3080,7 +3485,7 @@ function return_type_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, s return Future(UNKNOWN) end - af_argtype = isa(tt, Const) ? tt.val : (tt::DataType).parameters[1] + af_argtype = isa(tt, Const) ? tt.val : type_parameter(tt) if !isa(af_argtype, DataType) || !(af_argtype <: Tuple) return Future(UNKNOWN) end @@ -3092,16 +3497,39 @@ function return_type_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, s argtypes_vec = Any[af_argtype.parameters...] isempty(argtypes_vec) && push!(argtypes_vec, Union{}) aft = argtypes_vec[1] + # e.g. `return_type(Tuple{Vararg{Any}})`: there is no function type to model + isvarargtype(aft) && return Future(UNKNOWN) end - if !(isa(aft, Const) || (isType(aft) && !has_free_typevars(aft)) || - (isconcretetype(aft) && !(aft <: Builtin) && !iskindtype(aft))) - return Future(UNKNOWN) - end - # effects are not an issue if we know this statement will get removed, but if it does not get removed, # then this could be recursively re-entering inference (via concrete-eval), which will not terminate RT_CALL_EFFECTS = Effects(EFFECTS_TOTAL; nortcall=false) + if isa(aft, PartialOpaque) + argtypes_vec[1] = aft = widenconst(aft) + end + aftw = widenconst(aft) + if hasintersect(aftw, Core.OpaqueClosure) + # Match `return_type(::OpaqueClosure, ::DataType)`: observe the return type + # declared by the OC type without inspecting the opaque closure source. + uaft = unwrap_unionall(aftw) + if isa(uaft, DataType) && aftw <: Core.OpaqueClosure + ocargt = rewrap_unionall(uaft.parameters[1], aftw) + if !hasintersect(af_argtype, ocargt) + return Future(CallMeta(Const(Union{}), Union{}, RT_CALL_EFFECTS, NoCallInfo())) + end + rt = rewrap_unionall(uaft.parameters[2], aftw) + if aftw isa DataType + return Future(CallMeta(Const(rt), Union{}, RT_CALL_EFFECTS, NoCallInfo())) + else + return Future(CallMeta(Type{<:rt}, Union{}, RT_CALL_EFFECTS, NoCallInfo())) + end + end + end + if !(isa(aft, Const) || (isType(aft) && !has_free_typevars(aft)) || + (isconcretetype(aft) && !(aft <: Builtin) && !iskindtype(aft))) + return Future(UNKNOWN) + end + if contains_is(argtypes_vec, Union{}) return Future(CallMeta(Const(Union{}), Union{}, RT_CALL_EFFECTS, NoCallInfo())) end @@ -3113,9 +3541,13 @@ function return_type_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, s old_restrict = sv.restrict_abstract_call_sites sv.restrict_abstract_call_sites = false end - call = abstract_call(interp, ArgInfo(nothing, argtypes_vec), si, sv, #=max_methods=#-1) + # TODO: Could pass vtypes here to enable Conditional/MustAlias refinements + # in return_type inference. Currently passing `nothing` which means any + # slot-dependent refinements will be widened. This is conservative but + # may miss some precision opportunities. + call = abstract_call(interp, ArgInfo(nothing, argtypes_vec), si, nothing, sv, #=max_methods=#-1) tt = Core.Box(tt) - return Future{CallMeta}(call, interp, sv) do call, interp, sv + return Future{CallMeta}(call, interp, sv) do call, _, sv if isa(sv, InferenceState) sv.restrict_abstract_call_sites = old_restrict end @@ -3138,7 +3570,7 @@ function return_type_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, s # input arguments were known for certain # XXX: this doesn't imply we know anything about rt return CallMeta(Const(rt), Union{}, RT_CALL_EFFECTS, info) - elseif isType(rt) + elseif isTypeEq(rt) return CallMeta(Type{rt}, Union{}, RT_CALL_EFFECTS, info) else return CallMeta(Type{<:rt}, Union{}, RT_CALL_EFFECTS, info) @@ -3162,7 +3594,7 @@ function abstract_applicable(interp::AbstractInterpreter, argtypes::Vector{Any}, rt = Bool # too many matches to analyze else (; valid_worlds, applicable) = matches - update_valid_age!(sv, valid_worlds) + update_valid_age!(sv, get_inference_world(interp), valid_worlds) napplicable = length(applicable) if napplicable == 0 rt = Const(false) # never any matches @@ -3193,7 +3625,7 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv else return CallMeta(Any, Any, Effects(), NoCallInfo()) end - (types, isexact, isconcrete, istype) = instanceof_tfunc(argtype_by_index(argtypes, typeidx), false) + (types, isexact, _, _) = instanceof_tfunc(argtype_by_index(argtypes, typeidx), false) isexact || return CallMeta(Bool, Any, Effects(), NoCallInfo()) unwrapped = unwrap_unionall(types) if types === Bottom || !(unwrapped isa DataType) || unwrapped.name !== Tuple.name @@ -3204,7 +3636,7 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv types = rewrap_unionall(Tuple{ft, unwrapped.parameters...}, types)::Type end match, valid_worlds = findsup(types, method_table(interp)) - update_valid_age!(sv, valid_worlds) + update_valid_age!(sv, get_inference_world(interp), valid_worlds) if match === nothing rt = Const(false) vresults = MethodLookupResult(Any[], valid_worlds, true) @@ -3223,7 +3655,7 @@ function typename_static(@nospecialize(t)) t isa Const && return _typename(t.val) t isa Conditional && return Bool.name t = unwrap_unionall(widenconst(t)) - return isType(t) ? _typename(t.parameters[1]) : Core.TypeName + return isType(t) ? _typename(type_parameter(t)) : Core.TypeName end function global_order_exct(@nospecialize(o), loading::Bool, storing::Bool) @@ -3260,8 +3692,35 @@ add_tfunc(swapglobal!, 3, 4, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) add_tfunc(modifyglobal!, 4, 5, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) add_tfunc(replaceglobal!, 4, 6, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) add_tfunc(setglobalonce!, 3, 5, @nospecs((𝕃::AbstractLattice, args...)->Bool), 3) +@nospecs function getglobal_partition_tfunc(𝕃::AbstractLattice, gr, partition, order) + isa(partition, Const) || return Any + p = partition.val + isa(p, Core.BindingPartition) || return Any + return partition_rt(p) +end +add_tfunc(Core.getglobal_partition, 3, 3, getglobal_partition_tfunc, 0) +# as for `modifyglobal!`, the result is modeled by `abstract_modifyop!` instead +add_tfunc(Core.modifyglobal_partition, 3, 4, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) +add_tfunc(Core.setglobal_partition, 2, 3, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) +add_tfunc(Core.swapglobal_partition, 2, 3, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) +add_tfunc(Core.replaceglobal_partition, 3, 5, @nospecs((𝕃::AbstractLattice, args...)->Any), 3) +add_tfunc(Core.setglobalonce_partition, 2, 4, @nospecs((𝕃::AbstractLattice, args...)->Bool), 3) +add_tfunc(Core.isdefinedglobal_partition, 2, 2, @nospecs((𝕃::AbstractLattice, args...)->Bool), 1) +add_tfunc(Core.depwarn_partition, 1, 1, @nospecs((𝕃::AbstractLattice, args...)->Nothing), 1) add_tfunc(Core.get_binding_type, 2, 2, @nospecs((𝕃::AbstractLattice, args...)->Type), 0) +@nospecs function task_result_type_tfunc(𝕃::AbstractLattice, T) + hasintersect(widenconst(T), Task) || return Union{} + if T isa PartialTask + # fetch_type is widened at construction, but re-widen defensively since + # PartialTask objects also arrive from cached (serialized) rettype_const + # and from external AbstractInterpreters + return Const(widenconst(T.fetch_type)) + end + return Type +end +add_tfunc(Core.task_result_type, 1, 1, task_result_type_tfunc, 0) + # foreigncall # =========== @@ -3270,7 +3729,7 @@ add_tfunc(Core.get_binding_type, 2, 2, @nospecs((𝕃::AbstractLattice, args...) const FOREIGNCALL_ARG_START = 6 -function foreigncall_effects(@nospecialize(abstract_eval), e::Expr) +function foreigncall_effects(@nospecialize(abstract_eval), ::Expr) # `:foreigncall` can potentially perform all sorts of operations, including calling # overlay methods, but the `:foreigncall` itself is not dispatched, and there is no # concern that the method calls that potentially occur within the `:foreigncall` will @@ -3293,7 +3752,7 @@ function new_genericmemory_nothrow(@nospecialize(abstract_eval), args::Vector{An 0 < dimval < typemax(Int) || return false tot, ovflw = Intrinsics.checked_smul_int(dimval, elsz) ovflw && return false - isboxed = 1; isunion = 2 + isunion = 2 tot, ovflw = Intrinsics.checked_sadd_int(tot, arrayelem == isunion ? 1 + dimval : 1) ovflw && return false return true diff --git a/Compiler/src/timing.jl b/Compiler/src/timing.jl index 8b8240b025a66..2d21c4b98284a 100644 --- a/Compiler/src/timing.jl +++ b/Compiler/src/timing.jl @@ -6,7 +6,14 @@ if ccall(:jl_timing_enabled, Cint, ()) != 0 file = QuoteNode(file) # XXX: This buffer must be large enough to store any jl_timing_block_t (runtime-checked) - buffer = (0, 0, 0, 0, 0, 0, 0) + # Inlined Sys.isapple but bootstrapping + kernel = ccall(:jl_get_UNAME, Symbol, ()) + buffer = if kernel === :Apple || kernel === :Darwin + # Oslog needs a little extra space to store any buffered data + (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + else + (0, 0, 0, 0, 0, 0, 0) + end buffer_size = Core.sizeof(buffer) return quote if $event[] === C_NULL @@ -32,7 +39,7 @@ if ccall(:jl_timing_enabled, Cint, ()) != 0 return getzonedexpr(name, ex, :unknown_julia_function, __source__.file, __source__.line, 0) end else - macro zone(name, ex::Expr) + macro zone(_, ex::Expr) return esc(ex) end end diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 9849d22a9ce67..b2f2643146bca 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -17,7 +17,6 @@ using ..Compiler: -, +, :, Vector, length, first, empty!, push!, pop!, @inline, # What we record for any given frame we infer during type inference. struct InferenceFrameInfo mi::Core.MethodInstance - world::UInt64 sptypes::Vector{Compiler.VarState} slottypes::Vector{Any} nargs::Int @@ -26,7 +25,6 @@ end function _typeinf_identifier(frame::Compiler.InferenceState) mi_info = InferenceFrameInfo( frame.linfo, - frame_world(sv), copy(frame.sptypes), copy(frame.slottypes), length(frame.result.argtypes), @@ -75,7 +73,7 @@ start the ROOT() timer again. `ROOT()` measures all time spent _outside_ inferen function reset_timings() end push!(_timings, Timing( # The MethodInstance for ROOT(), and default empty values for other fields. - InferenceFrameInfo(ROOTmi, 0x0, Compiler.VarState[], Any[Core.Const(ROOT)], 1), + InferenceFrameInfo(ROOTmi, Compiler.VarState[], Any[Core.Const(ROOT)], 1), _time_ns())) function close_current_timer() end function enter_new_timer(frame) end @@ -91,45 +89,69 @@ If set to `true`, record per-method-instance timings within type inference in th __set_measure_typeinf(onoff::Bool) = __measure_typeinf__[] = onoff const __measure_typeinf__ = RefValue{Bool}(false) -function result_edges(interp::AbstractInterpreter, caller::InferenceState) +function internal_result_edges(caller::InferenceState) result = caller.result opt = result.src if isa(opt, OptimizationState) - return Core.svec(opt.inlining.edges...) + return opt.inlining.edges else - return Core.svec(caller.edges...) + return caller.edges end end +result_edges(::AbstractInterpreter, caller::InferenceState) = + materialize_inference_edges(internal_result_edges(caller)) + function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt, time_before::UInt64) result = caller.result - #@assert last(result.valid_worlds) <= get_world_counter() || isempty(caller.edges) - if caller.cache_mode === CACHE_MODE_LOCAL - @assert !isdefined(result, :ci) - result.src = transform_result_for_local_cache(interp, result) - elseif isdefined(result, :ci) + valid_worlds = caller.valid_worlds + min_world, max_world = first(valid_worlds), last(valid_worlds) + result.valid_worlds = valid_worlds + caller.src.min_world = min_world + caller.src.max_world = max_world + if isdefined(result, :ci) edges = result_edges(interp, caller) + #@assert max_world <= get_world_counter() || isempty(edges) ci = result.ci mi = result.linfo - # if we aren't cached, we don't need this edge - # but our caller might, so let's just make it anyways - if last(result.valid_worlds) >= validation_world - # if we can record all of the backedges in the global reverse-cache, - # we can now widen our applicability in the global cache too - store_backedges(ci, edges) + result_type = result.result + result_type isa LimitedAccuracy && (result_type = result_type.typ) + @assert !(result_type === nothing) + const_flag = is_result_constabi_eligible(result) + if isa(result_type, Const) + rettype_const = result_type.val + const_flags = const_flag ? 0x3 : 0x2 + elseif isa(result_type, PartialTask) + rettype_const = result_type + const_flags = 0x2 + elseif isa(result_type, PartialOpaque) + rettype_const = result_type + const_flags = 0x2 + elseif isconstType(result_type) + rettype_const = type_parameter(result_type) + const_flags = 0x2 + elseif isa(result_type, PartialStruct) + rettype_const = (_getundefs(result_type), result_type.fields) + const_flags = 0x2 + elseif isa(result_type, InterConditional) + rettype_const = result_type + const_flags = 0x2 + elseif isa(result_type, InterMustAlias) + rettype_const = result_type + const_flags = 0x2 + else + rettype_const = nothing + const_flags = 0x0 end inferred_result = nothing - uncompressed = result.src - const_flag = is_result_constabi_eligible(result) debuginfo = nothing - discard_src = caller.cache_mode === CACHE_MODE_NULL || const_flag + const_flag = is_result_constabi_eligible(result) + discard_src = caller.cache_mode === CACHE_MODE_NULL || (const_flag && may_discard_trees(interp)) if !discard_src inferred_result = transform_result_for_cache(interp, result, edges) if inferred_result !== nothing - uncompressed = inferred_result - debuginfo = get_debuginfo(inferred_result) - # Inlining may fast-path the global cache via `VolatileInferenceResult`, so store it back here result.src = inferred_result + debuginfo = get_debuginfo(inferred_result) else if isa(result.src, OptimizationState) debuginfo = get_debuginfo(ir_to_codeinf!(result.src)) @@ -145,60 +167,114 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation resize!(inferred_result.slotnames, nslots) end inferred_result = maybe_compress_codeinfo(interp, mi, inferred_result) - result.is_src_volatile = false elseif ci.owner === nothing - # The global cache can only handle objects that codegen understands + # The global cache can only handle objects that codegen understands (nothing or CodeInfo) inferred_result = nothing end + else + result.src = nothing end if debuginfo === nothing debuginfo = DebugInfo(mi) end - min_world, max_world = first(result.valid_worlds), last(result.valid_worlds) + # if we aren't cached, we don't need this edge + # but our caller might, so let's just make it anyways + if max_world >= validation_world + # if we can record all of the backedges in the global reverse-cache, + # we can now widen our applicability in the global cache too + store_backedges(ci, edges) + end ipo_effects = encode_effects(result.ipo_effects) time_now = _time_ns() time_self_ns = caller.time_self_ns + (time_now - time_before) time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 - ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), - ci, inferred_result, const_flag, min_world, max_world, ipo_effects, - result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) - if is_cached(caller) # CACHE_MODE_GLOBAL - cache_result!(interp, result, ci) + ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), + ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, inferred_result, + const_flags, min_world, max_world, + ipo_effects, result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) + elseif caller.cache_mode === CACHE_MODE_LOCAL + result.src = transform_result_for_local_cache(interp, result) + end + nothing +end + +function promotecache!(interp::AbstractInterpreter, caller::InferenceState) + result = caller.result + if isdefined(result, :ci) + ci = result.ci + mi = result.linfo + # Only an early winner kept `ci` out of `opt_cache`, so only that case may + # still be demoted here. If a winner appeared during optimization, `ci` may + # already occur in optimized IR and must be published even if equivalent. + if result.replacement_ci !== nothing + equivalent = find_equivalent_cached_ci( + interp, ci, result.valid_worlds) + if equivalent === nothing + # The early winner was sufficient to keep this provisional CI out + # of optimizer state, but it cannot represent the completed result's + # return ABI. Publish the now-complete CI normally. + caller.cache_mode |= CACHE_MODE_GLOBAL + caller.cache_mode &= ~CACHE_MODE_LOCAL + result.replacement_ci = nothing + else + result.replacement_ci = equivalent + end + end + if !iszero(caller.cache_mode & CACHE_MODE_GLOBAL) + code_cache(interp)[mi] = ci end + # A globally committed exact CI becomes visible before its reservation is + # released and any waiters wake. A publication loser is still filled as a + # session-local ephemeral CI before its reservation is released: it may carry + # source required by this interpreter even when the global winner does not. engine_reject(interp, ci) codegen = codegen_cache(interp) - if !discard_src && codegen !== nothing && (isa(uncompressed, CodeInfo) || isa(uncompressed, OptimizationState)) + if codegen !== nothing + uncompressed = result.src if isa(uncompressed, OptimizationState) - uncompressed = ir_to_codeinf!(uncompressed, edges) + uncompressed = ir_to_codeinf!(uncompressed, ci.edges) + result.src = uncompressed end - # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work - codegen[ci] = uncompressed - if bootstrapping_compiler && inferred_result == nothing - # This is necessary to get decent bootstrapping performance - # when compiling the compiler to inject everything eagerly - # where codegen can start finding and using it right away - if mi.def isa Method && isa_compileable_sig(mi) && is_cached(caller) - ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), ci, uncompressed) + if isa(uncompressed, CodeInfo) + # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work + codegen[ci] = uncompressed + if bootstrapping_compiler + # This is necessary to get decent bootstrapping performance + # when compiling the compiler to inject everything eagerly + # where codegen can start finding and using it right away + if mi.def isa Method && isa_compileable_sig(mi) && is_cached(caller) + ccall(:jl_add_codeinsts_to_jit, Cvoid, (Any, Any), Any[ci], Any[uncompressed]) + end end end end end - return nothing -end - -function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) - mi = result.linfo - code_cache(interp)[mi] = ci + if !iszero(caller.cache_mode & CACHE_MODE_LOCAL) + if result.tombstone + # Preserve rejected constant-propagation work as an internal marker. It is + # never exposed as a completed call result. + result.cache_world = get_inference_world(interp) + push!(get_inference_cache(interp), result) + elseif result.result !== nothing && result.overridden_by_const === nothing + # Local work is reusable only through its explicit dependency proof. Its + # provisional CI, if any, is deliberately not allowed to escape as a target. + proof = LocalInferenceProof(result.valid_worlds, + Core.svec(internal_result_edges(caller)...)) + local_result = LocalInferenceResult(result, proof, get_inference_world(interp)) + push!(get_inference_cache(interp), local_result) + end + # Successful constant propagation is published by `const_prop_call` only after + # concrete-evaluation overrides have been finalized. + end + nothing end function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstance, src::CodeInfo) user_edges = src.edges edges = user_edges isa SimpleVector ? user_edges : user_edges === nothing ? Core.svec() : Core.svec(user_edges...) - const_flag = false di = src.debuginfo rettype = Any exctype = Any - rettype_const = nothing const_flags = 0x0 ipo_effects = zero(UInt32) min_world = src.min_world @@ -211,10 +287,8 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan # we can now widen our applicability in the global cache too store_backedges(ci, edges) end - ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), - ci, rettype, exctype, nothing, const_flags, min_world, max_world, ipo_effects, nothing, di, edges) - ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), - ci, nothing, const_flag, min_world, max_world, ipo_effects, nothing, 0.0, 0.0, 0.0, di, edges) + ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), + ci, rettype, exctype, nothing, nothing, const_flags, min_world, max_world, ipo_effects, nothing, 0.0, 0.0, 0.0, di, edges) code_cache(interp)[mi] = ci codegen = codegen_cache(interp) if codegen !== nothing @@ -224,31 +298,75 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan return nothing end -function finish_nocycle(::AbstractInterpreter, frame::InferenceState, time_before::UInt64) +function finish_nocycle(interp::AbstractInterpreter, frame::InferenceState{I}, time_before::UInt64) where {I<:AbstractInterpreter} opt_cache = IdDict{MethodInstance,CodeInstance}() - finishinfer!(frame, frame.interp, frame.cycleid, opt_cache) + finishinfer!(frame, interp::I, frame.cycleid, opt_cache) opt = frame.result.src - if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` - optimize(frame.interp, opt, frame.result) + if opt isa OptimizationState # implies `may_optimize(interp) === true` + optimize(interp::I, opt::OptimizationState{I}, frame.result) + # check the valid_worlds hasn't been narrowed by added :invoke edges or resolving a global access + valid_worlds = intersect(frame.valid_worlds, world_range(opt.src)) + valid_worlds = intersect(valid_worlds, compute_recursive_worlds(opt.inlining.edges)) + update_valid_age!(frame, get_inference_world(interp::I), valid_worlds) end empty!(opt_cache) validation_world = get_world_counter() - finish!(frame.interp, frame, validation_world, time_before) - if isdefined(frame.result, :ci) + finish!(interp::I, frame, validation_world, time_before) + promotecache!(interp::I, frame) + if (!iszero(frame.cache_mode & CACHE_MODE_GLOBAL) && + isdefined(frame.result, :ci)) # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. ccall(:jl_promote_ci_to_current, Cvoid, (Any, UInt), frame.result.ci, validation_world) end if frame.cycleid != 0 - frames = frame.callstack::Vector{AbsIntState} + frames = frame.callstack @assert frames[end] === frame pop!(frames) end return nothing end -function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cycleid::Int, time_before::UInt64) +function propagate_unpublished_cycle_proof!( + frames::Vector{AbsIntState{I}}, cycleid::Int, world::UInt, + cycle_valid_worlds::WorldRange + ) where {I<:AbstractInterpreter} + unpublished = InferenceState{I}[] + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState + if iszero(caller.cache_mode & CACHE_MODE_GLOBAL) + push!(unpublished, caller) + end + end + isempty(unpublished) && return nothing + + proof_edges = Any[] + for frameid = cycleid:length(frames) + append!(proof_edges, internal_result_edges(frames[frameid]::InferenceState)) + end + cycle_proof = LocalInferenceProof(cycle_valid_worlds, Core.svec(proof_edges...)) + # An unpublished member needs the shared fixed-point proof for later local + # reuse. Any caller that directly consumed that member while it was in progress + # needs the same proof. Other published SCC members remain connected through + # their ordinary CI edges, so copying the union onto every member would only + # multiply serialized edge streams by the SCC size. + consumers = IdSet{InferenceState{I}}() + for callee in unpublished + push!(consumers, callee) + for (caller, _) in callee.cycle_backedges + push!(consumers, caller) + end + end + for caller in consumers + add_inference_proof!(internal_result_edges(caller), cycle_proof) + update_valid_age!(caller, world, proof_worlds(cycle_proof)) + end + return nothing +end + +function finish_cycle(interp::AbstractInterpreter, frames::Vector{AbsIntState{I}}, cycleid::Int, time_before::UInt64) where {I<:AbstractInterpreter} + world = get_inference_world(interp::I) cycle_valid_worlds = WorldRange() cycle_valid_effects = EFFECTS_TOTAL for frameid = cycleid:length(frames) @@ -258,14 +376,14 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei # all frames in the cycle should have the same bits of `valid_worlds` and `effects` # that are simply the intersection of each partial computation, without having # dependencies on each other (unlike rt and exct) - cycle_valid_worlds = intersect(cycle_valid_worlds, caller.world.valid_worlds) + cycle_valid_worlds = intersect(cycle_valid_worlds, caller.valid_worlds) cycle_valid_effects = merge_effects(cycle_valid_effects, caller.ipo_effects) end opt_cache = IdDict{MethodInstance,CodeInstance}() for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState - adjust_cycle_frame!(caller, cycle_valid_worlds, cycle_valid_effects) - finishinfer!(caller, caller.interp, cycleid, opt_cache) + adjust_cycle_frame!(caller, world, cycle_valid_worlds, cycle_valid_effects) + finishinfer!(caller, caller.interp::I, cycleid, opt_cache) time_now = _time_ns() caller.time_self_ns += (time_now - time_before) time_before = time_now @@ -276,7 +394,9 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei caller = frames[frameid]::InferenceState opt = caller.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` - optimize(caller.interp, opt, caller.result) + optimize(caller.interp::I, opt::OptimizationState{I}, caller.result) + cycle_valid_worlds = intersect(cycle_valid_worlds, world_range(opt.src)) + cycle_valid_worlds = intersect(cycle_valid_worlds, compute_recursive_worlds(opt.inlining.edges)) time_now = _time_ns() caller.time_self_ns += (time_now - time_before) time_before = time_now @@ -286,6 +406,13 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei caller.time_paused = UInt64(0) caller.time_caches = 0.0 end + # Calls to an already in-progress SCC member consume its current facts + # immediately and record only a cycle backedge. Usually the member's published + # CI later certifies those facts. If any member will not publish its exact CI + # (for example, it was demoted in favor of a reentrant cached winner or its + # source was tombstoned), preserve the SCC's external dependencies explicitly. + # One shared proof for the whole SCC avoids constructing a cyclic proof graph. + propagate_unpublished_cycle_proof!(frames, cycleid, world, cycle_valid_worlds) empty!(opt_cache) cycletop = frames[cycleid]::InferenceState time_start = cycletop.time_start @@ -296,16 +423,23 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei caller.time_start = time_start caller.time_caches = time_caches caller.time_paused = time_paused - finish!(caller.interp, caller, validation_world, time_before) - if isdefined(caller.result, :ci) - push!(cis, caller.result.ci) - end + update_valid_age!(caller, world, cycle_valid_worlds) + finish!(caller.interp::I, caller, validation_world, time_before) end if cycletop.parentid != 0 parent = frames[cycletop.parentid] parent.time_caches += time_caches parent.time_paused += time_paused end + # After everything is finished, promote the work into visible caches + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState + promotecache!(caller.interp::I, caller) + if (!iszero(caller.cache_mode & CACHE_MODE_GLOBAL) && + isdefined(caller.result, :ci)) + push!(cis, caller.result.ci) + end + end # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. @@ -314,8 +448,8 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei return nothing end -function adjust_cycle_frame!(sv::InferenceState, cycle_valid_worlds::WorldRange, cycle_valid_effects::Effects) - update_valid_age!(sv, cycle_valid_worlds) +function adjust_cycle_frame!(sv::InferenceState, world::UInt, cycle_valid_worlds::WorldRange, cycle_valid_effects::Effects) + update_valid_age!(sv, world, cycle_valid_worlds) sv.ipo_effects = cycle_valid_effects # traverse the callees of this cycle that are tracked within `sv.cycle_backedges` # and adjust their statements so that they are consistent with the new `cycle_valid_effects` @@ -401,7 +535,8 @@ function inline_cost_model(interp::AbstractInterpreter, result::InferenceResult, end function transform_result_for_local_cache(interp::AbstractInterpreter, result::InferenceResult) - if is_result_constabi_eligible(result) + ## XXX: this must perform the exact same operations as transform_result_for_cache to avoid introducing soundness bugs + if may_discard_trees(interp) && is_result_constabi_eligible(result) return nothing end src = result.src @@ -419,7 +554,7 @@ function transform_result_for_cache(interp::AbstractInterpreter, result::Inferen if isa(src, OptimizationState) opt = src inlining_cost = compute_inlining_cost(interp, result, opt.optresult) - discard_optimized_result(interp, opt, inlining_cost) && return nothing + discard_optimized_result(interp, inlining_cost) && return nothing src = ir_to_codeinf!(opt) end if isa(src, CodeInfo) @@ -433,24 +568,33 @@ function transform_result_for_cache(interp::AbstractInterpreter, result::Inferen return src end -function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) +function discard_optimized_result(interp::AbstractInterpreter, inlining_cost::InlineCostType) may_discard_trees(interp) || return false - return inlining_cost == MAX_INLINE_COST + inlining_cost == MAX_INLINE_COST || return false + precompile_keep_ir(interp) && return false + return true end function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInfo) def = mi.def isa(def, Method) || return ci # don't compress toplevel code can_discard_trees = may_discard_trees(interp) - cache_the_tree = !can_discard_trees || is_inlineable(ci) - cache_the_tree || return nothing + inlineable = is_inlineable(ci) + if can_discard_trees && !inlineable + # Precompile-keep-ir mode: retain non-inlineable IR as raw CodeInfo so + # irgen's typeinf_ext can reuse it instead of re-inferring. + # jl_finalize_precompile_inferred nulls it before save. + precompile_keep_ir(interp) && return ci + return nothing + end + # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? may_compress(interp) && return ccall(:jl_compress_ir, String, (Any, Any), def, ci) return ci end function cycle_fix_limited(@nospecialize(typ), sv::InferenceState, cycleid::Int) if typ isa LimitedAccuracy - frames = sv.callstack::Vector{AbsIntState} + frames = sv.callstack causes = typ.causes for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState @@ -594,14 +738,17 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: end end result = me.result - result.valid_worlds = me.world.valid_worlds result.result = bestguess ipo_effects = result.ipo_effects = me.ipo_effects = adjust_effects(me) result.exc_result = me.exc_bestguess = refine_exception_type(me.exc_bestguess, ipo_effects) - me.src.rettype = widenconst(ignorelimited(bestguess)) - me.src.ssaflags = me.ssaflags - me.src.min_world = first(me.world.valid_worlds) - me.src.max_world = last(me.world.valid_worlds) + src = me.src + src.rettype = widenconst(ignorelimited(bestguess)) + src.ssaflags = me.ssaflags + valid_worlds = me.valid_worlds + result.valid_worlds = valid_worlds + min_world, max_world = first(valid_worlds), last(valid_worlds) + src.min_world = min_world + src.max_world = max_world istoplevel = !(me.linfo.def isa Method) istoplevel || compute_edges!(me) # don't add backedges to toplevel method instance @@ -609,13 +756,15 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: # A parent may be cached still, but not this intermediate work: # we can throw everything else away now. Caching anything can confuse later # heuristics to consider it worth trying to pursue compiling this further and - # finding infinite work as a result. Avoiding caching helps to ensure there is only - # a finite amount of work that can be discovered later (although potentially still a - # large multiplier on it). + # finding infinite work as a result. Avoiding global caching helps to ensure there + # is only a finite amount of work that can be discovered later (although potentially + # still a large multiplier on it). We still allow local caching so that tombstoned + # entries can be found by `constprop_cache_lookup` to prevent re-attempting the same + # const-prop work that would hit the same limit. result.src = nothing result.tombstone = true - me.cache_mode = CACHE_MODE_NULL - set_inlineable!(me.src, false) + me.cache_mode &= ~CACHE_MODE_GLOBAL + set_inlineable!(src, false) else # annotate fulltree with type information, # either because we are the outermost code, or we might use this later @@ -629,52 +778,29 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: if doopt result.src = OptimizationState(me, interp, opt_cache) else - result.src = me.src # for reflection etc. + result.src = src # for reflection etc. end end - maybe_validate_code(me.linfo, me.src, "inferred") + maybe_validate_code(me.linfo, src, "inferred") - # finish populating inference results into the CodeInstance if possible, and maybe cache that globally for use elsewhere + # check global cache again for :invoke use, and put in the opt_cache if it wasn't there at this time if isdefined(result, :ci) - result_type = result.result - result_type isa LimitedAccuracy && (result_type = result_type.typ) - @assert !(result_type === nothing) - if isa(result_type, Const) - rettype_const = result_type.val - const_flags = is_result_constabi_eligible(result) ? 0x3 : 0x2 - elseif isa(result_type, PartialOpaque) - rettype_const = result_type - const_flags = 0x2 - elseif isconstType(result_type) - rettype_const = result_type.parameters[1] - const_flags = 0x2 - elseif isa(result_type, PartialStruct) - rettype_const = (_getundefs(result_type), result_type.fields) - const_flags = 0x2 - elseif isa(result_type, InterConditional) - rettype_const = result_type - const_flags = 0x2 - elseif isa(result_type, InterMustAlias) - rettype_const = result_type - const_flags = 0x2 - else - rettype_const = nothing - const_flags = 0x0 - end - - di = nothing - edges = empty_edges # `edges` will be updated within `finish!` ci = result.ci - min_world, max_world = first(result.valid_worlds), last(result.valid_worlds) - ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), - ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags, - min_world, max_world, - encode_effects(result.ipo_effects), result.analysis_results, di, edges) - if is_cached(me) # CACHE_MODE_GLOBAL - already_cached = is_already_cached(me.interp, result, ci) - if already_cached - me.cache_mode = CACHE_MODE_VOLATILE + ipo_effects = encode_effects(result.ipo_effects) + # populate a few fields that won't change again (and are inspected by optimization) + @atomic :monotonic ci.ipo_purity_bits = ipo_effects + ci.analysis_results = result.analysis_results + if !iszero(me.cache_mode & CACHE_MODE_GLOBAL) + ci = result.ci + replacement_ci = find_cached_ci(interp, result) + if replacement_ci !== nothing + # Make the publication decision before this CI can enter `opt_cache`. + # The provisional object remains private to this inference session and + # is filled only after optimization, so no published caller can observe + # it as an executable edge. + result.replacement_ci = replacement_ci + me.cache_mode = CACHE_MODE_LOCAL else opt_cache[result.linfo] = ci end @@ -683,17 +809,13 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: nothing end -function is_already_cached(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) - # check if the existing linfo metadata is also sufficient to describe the current inference result - # to decide if it is worth caching this right now - mi = result.linfo - cache = WorldView(code_cache(interp), result.valid_worlds) - if haskey(cache, mi) - # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later - @assert isdefined(cache[mi], :inferred) - return true - end - return false +function find_cached_ci(interp::AbstractInterpreter, result::InferenceResult) + # Any inferred global CI covering this result is sufficient to defer publication + # until the new result's ABI is known. `promotecache!` then keeps an equivalent + # result local, or publishes the completed CI normally when no equivalent winner + # exists. + return find_cached_ci(interp, result.linfo, result.valid_worlds, + SOURCE_MODE_NOT_REQUIRED) end # Iterate a series of back-edges that need registering, based on the provided forward edge list. @@ -741,30 +863,42 @@ function Base.iterate(it::ForwardToBackedgeIterator, i::Int = 1) end # record the backedges + +function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance}) + meth = isa(edge, Method) ? edge : get_ci_mi(edge).def + ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth) + return nothing +end + function store_backedges(caller::CodeInstance, edges::SimpleVector) isa(get_ci_mi(caller).def, Method) || return # don't add backedges to toplevel method instance backedges = ForwardToBackedgeIterator(edges) - for (i, (invokesig, item)) in enumerate(backedges) - # check for any duplicate edges we've already registered - duplicate_found = false - for (i′, (invokesig′, item′)) in enumerate(backedges) - i == i′ && break - if item′ === item && invokesig′ == invokesig - duplicate_found = true - break + # `Compiler` is loaded before `Set` during bootstrap, so keep the signatures + # for each identity-keyed dependency in a small vector. + seen = IdDict{Any,Vector{Any}}() + for (invokesig, item) in backedges + if haskey(seen, item) + signatures = seen[item] + duplicate_found = false + for signature in signatures + if signature == invokesig + duplicate_found = true + break + end end + duplicate_found && continue + push!(signatures, invokesig) + else + seen[item] = Any[invokesig] end - - if !duplicate_found - if item isa Core.Binding - maybe_add_binding_backedge!(item, caller) - elseif item isa MethodTable - ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any), invokesig, caller) - else - item::MethodInstance - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) - end + if item isa Core.Binding + maybe_add_binding_backedge!(item, caller) + elseif item isa MethodTable + ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any), invokesig, caller) + else + item::MethodInstance + ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) end end nothing @@ -775,13 +909,27 @@ function compute_edges!(sv::InferenceState) for i in 1:length(sv.stmt_info) add_edges!(edges, sv.stmt_info[i]) end - user_edges = sv.src.edges + user_edges = sv.src.edges::Union{Nothing, SimpleVector, Vector{Any}} if user_edges !== nothing && user_edges !== empty_edges append!(edges, user_edges) end nothing end +function compute_recursive_worlds(edges::Vector{Any}) + range = WorldRange(typemin(UInt), typemax(UInt)) + for edge in edges + if edge isa CodeInstance + wr = WorldRange(edge.min_world, edge.max_world) + iszero(last(wr.max_world)) && continue # part of the current cycle, not yet valid + range = intersect(range, wr) + elseif edge isa LocalInferenceProof + range = intersect(range, edge.valid_worlds) + end + end + return range +end + function record_slot_assign!(sv::InferenceState) # look at all assignments to slots # and union the set of types stored there @@ -833,11 +981,11 @@ function find_dominating_assignment(id::Int, idx::Int, sv::InferenceState) end # annotate types of all symbols in AST, preparing for optimization -function type_annotate!(interp::AbstractInterpreter, sv::InferenceState) - # widen `Conditional`s from `slottypes` +function type_annotate!(::AbstractInterpreter, sv::InferenceState) + # widen slot wrappers from `slottypes` slottypes = sv.slottypes for i = 1:length(slottypes) - slottypes[i] = widenconditional(slottypes[i]) + slottypes[i] = widenslotwrapper(slottypes[i]) end # compute the required type for each slot @@ -870,13 +1018,14 @@ function type_annotate!(interp::AbstractInterpreter, sv::InferenceState) end end - # widen slot wrappers (`Conditional` and `MustAlias`) in `bb_vartables` - for varstate in sv.bb_vartables - if varstate !== nothing + # widen slot wrappers (`Conditional` and `MustAlias`) in `bb_states` + for bbstate in sv.bb_states + if bbstate !== nothing + vartable = bbstate.vartable for slot in 1:nslots - vt = varstate[slot] + vt = vartable[slot] widened_type = widenslotwrapper(ignorelimited(vt.typ)) - varstate[slot] = VarState(widened_type, vt.undef) + vartable[slot] = VarState(widened_type, vt.ssadef, vt.undef) end end end @@ -885,17 +1034,10 @@ function type_annotate!(interp::AbstractInterpreter, sv::InferenceState) end function merge_call_chain!(::AbstractInterpreter, parent::InferenceState, child::InferenceState) - # add backedge of parent <- child - # then add all backedges of parent <- parent.parent - frames = parent.callstack::Vector{AbsIntState} + # update all cycleid to be in the same group + frames = parent.callstack @assert child.callstack === frames ancestorid = child.cycleid - while true - add_cycle_backedge!(parent, child) - parent.cycleid === ancestorid && break - child = parent - parent = cycle_parent(child)::InferenceState - end # ensure that walking the callstack has the same cycleid (DAG) for frameid = reverse(ancestorid:length(frames)) frame = frames[frameid]::InferenceState @@ -906,14 +1048,13 @@ function merge_call_chain!(::AbstractInterpreter, parent::InferenceState, child: end function add_cycle_backedge!(caller::InferenceState, frame::InferenceState) - update_valid_age!(caller, frame.world.valid_worlds) backedge = (caller, caller.currpc) contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge) return frame end -function is_same_frame(interp::AbstractInterpreter, mi::MethodInstance, frame::InferenceState) - return mi === frame_instance(frame) && cache_owner(interp) === cache_owner(frame.interp) +function is_same_frame(interp::I, mi::MethodInstance, frame::InferenceState) where {I<:AbstractInterpreter} + return mi === frame_instance(frame) && cache_owner(interp) === cache_owner(frame.interp::I) end function poison_callstack!(infstate::InferenceState, topmost::InferenceState) @@ -921,57 +1062,128 @@ function poison_callstack!(infstate::InferenceState, topmost::InferenceState) nothing end -# Walk through `mi`'s upstream call chain, starting at `parent`. If a parent -# frame matching `mi` is encountered, then there is a cycle in the call graph -# (i.e. `mi` is a descendant callee of itself). Upon encountering this cycle, -# we "resolve" it by merging the call chain, which entails updating each intermediary -# frame's `cycleid` field and adding the appropriate backedges. Finally, -# we return `mi`'s pre-existing frame. If no cycles are found, `nothing` is -# returned instead. -function resolve_call_cycle!(interp::AbstractInterpreter, mi::MethodInstance, parent::AbsIntState) +# Find `mi` in the contiguous inference portion of `parent`'s upstream call chain. +function find_call_cycle(interp::AbstractInterpreter, mi::MethodInstance, parent::AbsIntState) # TODO (#48913) implement a proper recursion handling for irinterp: # This works most of the time currently just because the irinterp code doesn't get used much with # `@assume_effects`, so it never sees a cycle normally, but that may not be a sustainable solution. - parent isa InferenceState || return false - frames = parent.callstack::Vector{AbsIntState} + parent isa InferenceState || return nothing + frames = parent.callstack uncached = false for frameid = reverse(1:length(frames)) frame = frames[frameid] isa(frame, InferenceState) || break uncached |= !is_cached(frame) # ensure we never add a (globally) uncached frame to a cycle if is_same_frame(interp, mi, frame) - if uncached - # our attempt to speculate into a constant call lead to an undesired self-cycle - # that cannot be converged: if necessary, poison our call-stack (up to the discovered duplicate frame) - # with the limited flag and abort (set return type to Any) now - poison_callstack!(parent, frame) - return true - end - merge_call_chain!(interp, parent, frame) - return frame + return frame, uncached end end - return false + return nothing +end + +# Resolve a cycle by merging its call chain and return `mi`'s pre-existing frame. +# Return `true` for an unresolvable cycle and `false` when no cycle was found. +function resolve_call_cycle!(interp::AbstractInterpreter, mi::MethodInstance, parent::AbsIntState) + cycle = find_call_cycle(interp, mi, parent) + cycle === nothing && return false + frame, uncached = cycle + if uncached + # our attempt to speculate into a constant call lead to an undesired self-cycle + # that cannot be converged: if necessary, poison our call-stack (up to the discovered duplicate frame) + # with the limited flag and abort (set return type to Any) now + poison_callstack!(parent::InferenceState, frame) + return true + end + merge_call_chain!(interp, parent::InferenceState, frame) + return frame end ipo_effects(code::CodeInstance) = decode_effects(code.ipo_purity_bits) # return cached result of regular inference -function return_cached_result(interp::AbstractInterpreter, method::Method, codeinst::CodeInstance, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool) +function return_cached_result(interp::AbstractInterpreter, method::Method, codeinst::CodeInstance, @nospecialize(src), caller::AbsIntState, edgecycle::Bool, edgelimited::Bool, edgerecursed::Bool) rt = cached_return_type(codeinst) exct = codeinst.exctype effects = ipo_effects(codeinst) - update_valid_age!(caller, WorldRange(min_world(codeinst), max_world(codeinst))) + valid_worlds = WorldRange(min_world(codeinst), max_world(codeinst)) + if src !== nothing + # Create an InferenceResult to preserve cached source lookup + inf_result = InferenceResult(codeinst.def, typeinf_lattice(interp)) + inf_result.result = rt + inf_result.exc_result = exct + inf_result.src = src::CodeInfo + inf_result.ipo_effects = effects + inf_result.ci = codeinst + inf_result.valid_worlds = valid_worlds + local_result = LocalInferenceResult(inf_result, codeinst, get_inference_world(interp)) + push!(get_inference_cache(interp), local_result) + else + local_result = nothing + end + update_valid_age!(caller, get_inference_world(interp), valid_worlds) caller.time_caches += reinterpret(Float16, codeinst.time_infer_total) caller.time_caches += reinterpret(Float16, codeinst.time_infer_cache_saved) - return Future(MethodCallResult(interp, caller, method, rt, exct, effects, codeinst, edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, rt, exct, effects, codeinst, + edgecycle, edgelimited, edgerecursed, local_result)) +end + +function return_cached_result(interp::AbstractInterpreter, method::Method, + local_result::LocalInferenceResult, + codeinst::Union{Nothing,CodeInstance}, + caller::AbsIntState, edgecycle::Bool, edgelimited::Bool, + edgerecursed::Bool) + inf_result = local_result.result + rt = inf_result.result + exct = inf_result.exc_result + effects = inf_result.ipo_effects + world = get_inference_world(interp) + update_valid_age!(caller, world, proof_worlds(local_result.proof)) + if codeinst !== nothing + update_valid_age!(caller, world, WorldRange(codeinst.min_world, codeinst.max_world)) + caller.time_caches += reinterpret(Float16, codeinst.time_infer_total) + caller.time_caches += reinterpret(Float16, codeinst.time_infer_cache_saved) + end + return Future(MethodCallResult(interp, caller, method, rt, exct, effects, + codeinst, edgecycle, edgelimited, edgerecursed, local_result)) +end + +function lookup_cached_edge(interp::AbstractInterpreter, method::Method, + mi::MethodInstance, caller::AbsIntState, force_inline::Bool, + edgecycle::Bool, edgelimited::Bool, edgerecursed::Bool) + local_result = lookup_local_inference_result(interp, mi) + codeinst = get(code_cache(interp), mi, nothing) + if !(codeinst isa CodeInstance) + local_result === nothing && return nothing, nothing + return return_cached_result(interp, method, local_result, nothing, caller, + edgecycle, edgelimited, edgerecursed), nothing + end + @assert codeinst.def === mi "MethodInstance for cached edge does not match" + + if local_result !== nothing + return return_cached_result(interp, method, local_result, codeinst, caller, + edgecycle, edgelimited, edgerecursed), nothing + end + + inferred = @atomic :monotonic codeinst.inferred + need_inlineable_code = (may_optimize(interp) && + (force_inline || is_inlineable(inferred) || use_const_api(codeinst))) + if need_inlineable_code + src = ci_get_source(interp, codeinst, inferred) + src === nothing && return nothing, codeinst + return return_cached_result(interp, method, codeinst, src, caller, + edgecycle, edgelimited, edgerecursed), nothing + end + return return_cached_result(interp, method, codeinst, nothing, caller, + edgecycle, edgelimited, edgerecursed), nothing end + function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method, @nospecialize(rt), @nospecialize(exct), effects::Effects, edge::Union{Nothing,CodeInstance}, edgecycle::Bool, edgelimited::Bool, - volatile_inf_result::Union{Nothing,VolatileInferenceResult}=nothing) - if edge === nothing + edgerecursed::Bool, call_result::Union{Nothing,InferredCallResult} = nothing; + force_edgecycle::Bool = true, needs_mi_edge::Bool = false) + if force_edgecycle && edge === nothing && call_result === nothing edgecycle = edgelimited = true end @@ -985,7 +1197,7 @@ function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method effects = Effects(effects; terminates=true) elseif edgecycle # Some sort of recursion was detected. - if edge !== nothing && !edgelimited && !is_edge_recursed(edge, sv) + if (edge !== nothing || call_result isa LocalInferenceResult) && !edgelimited && !edgerecursed # no `MethodInstance` cycles -- don't taint :terminate else # we cannot guarantee that the call will terminate @@ -993,64 +1205,121 @@ function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method end end - return MethodCallResult(rt, exct, effects, edge, edgecycle, edgelimited, volatile_inf_result) + return MethodCallResult(rt, exct, effects, edge, edgecycle, edgelimited, call_result; + needs_mi_edge) end -# allocate a dummy `edge::CodeInstance` to be added by `add_edges!`, reusing an existing_edge if possible -# TODO: fill this in fully correctly (currently IPO info such as effects and return types are lost) -function codeinst_as_edge(interp::AbstractInterpreter, sv::InferenceState, @nospecialize existing_edge) - mi = sv.linfo - min_world, max_world = first(sv.world.valid_worlds), last(sv.world.valid_worlds) - if max_world >= get_world_counter() - max_world = typemax(UInt) - end - edges = Core.svec(sv.edges...) - if existing_edge isa CodeInstance - # return an existing_edge, if the existing edge has more restrictions already (more edges and narrower worlds) - if existing_edge.min_world >= min_world && - existing_edge.max_world <= max_world && - existing_edge.edges == edges - return existing_edge +function completed_inference_result(interp::AbstractInterpreter, frame::InferenceState) + result = frame.result + @assert result.result !== nothing + @assert !result.tombstone + if (!iszero(frame.cache_mode & CACHE_MODE_GLOBAL) && isdefined(result, :ci) && + get(code_cache(interp, result.valid_worlds), result.linfo, nothing) === result.ci) + proof = result.ci + else + cache = get_inference_cache(interp) + indices = get_indices(cache, result.linfo) + for i in length(indices):-1:1 + cached = cache.results[indices[i]] + if cached isa LocalInferenceResult && cached.result === result + return cached + end end - end - ci = CodeInstance(mi, cache_owner(interp), Any, Any, nothing, nothing, zero(Int32), - min_world, max_world, zero(UInt32), nothing, nothing, edges) - if max_world == typemax(UInt) - # if we can record all of the backedges in the global reverse-cache, - # we can now widen our applicability in the global cache too - # TODO: this should probably come after we decide this edge is even useful - store_backedges(ci, edges) - end - return ci + proof = LocalInferenceProof(result.valid_worlds, + Core.svec(internal_result_edges(frame)...)) + end + return LocalInferenceResult(result, proof, get_inference_world(interp)) +end + +function _schedule_edge_infer_task!(caller::AbsIntState, frame::InferenceState, result::InferenceResult, + method::Method, edge_ci::Union{Nothing,CodeInstance}, + edgecycle::Bool, edgelimited::Bool, edgerecursed::Bool) + mresult = Future{MethodCallResult}() + push!(caller.tasks, function get_infer_result(interp, caller) + update_valid_age!(caller, get_inference_world(interp), frame.valid_worlds) + isinferred = is_inferred(frame) + effects = nothing + call_result = nothing + edge = edge_ci + if isinferred + if !result.tombstone + call_result = completed_inference_result(interp, frame) + update_valid_age!(caller, get_inference_world(interp), + proof_worlds(call_result.proof)) + else + # A limited source can still produce clean return/effect facts. Retain + # their dependencies without exposing the tombstoned source as reusable. + proof = LocalInferenceProof(result.valid_worlds, + Core.svec(internal_result_edges(frame)...)) + add_inference_proof!(caller.edges, proof) + update_valid_age!(caller, get_inference_world(interp), proof_worlds(proof)) + end + effects = result.ipo_effects # effects are adjusted already within `finish` for ipo_effects + if !iszero(frame.cache_mode & CACHE_MODE_GLOBAL) && isdefined(result, :ci) + edge = result.ci + else + cached = get(code_cache(interp), result.linfo, nothing) + cached isa CodeInstance && (edge = cached) + end + else + # Do not expose this frame's provisional CI. The ordinary Method/MI lookup + # edge is sufficient until the cycle resolves, and an existing published + # target may still be reused when one was available at scheduling time. + effects = adjust_effects(effects_for_cycle(frame.ipo_effects), method) + add_cycle_backedge!(caller, frame) + end + # Both provisional SCC results and completed tombstones can contribute body + # facts without a CI target or reusable local result. Their transitive proof + # edges do not replace the dispatch dependency on this MethodInstance. + needs_mi_edge = edge === nothing && call_result === nothing + if edge !== nothing + update_valid_age!(caller, get_inference_world(interp), proof_worlds(edge)) + end + bestguess = frame.bestguess + exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) + # Propagate newly inferred source to the inliner, allowing efficient inlining + # without deserialization. + # A missing target here is deliberate; preserve the cycle/limiting decision made + # by `abstract_call_method` instead of inferring a new cycle from target absence. + mresult[] = MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, + edge, edgecycle, edgelimited, edgerecursed, call_result; + force_edgecycle=false, needs_mi_edge) + return true + end) + return mresult end # compute (and cache) an inferred AST and return the current best estimate of the result type -function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool) +function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, caller::AbsIntState, edgecycle::Bool, edgelimited::Bool, edgerecursed::Bool) mi = specialize_method(method, atype, sparams) cache_mode = CACHE_MODE_GLOBAL # cache edge targets globally by default force_inline = is_stmt_inline(get_curr_ssaflag(caller)) edge_ci = nothing - # check cache with SOURCE_MODE_NOT_REQUIRED source_mode - let codeinst = get(code_cache(interp), mi, nothing) - if codeinst isa CodeInstance # return existing rettype if the code is already inferred - inferred = @atomic :monotonic codeinst.inferred - if inferred === nothing && force_inline - # we already inferred this edge before and decided to discard the inferred code, - # nevertheless we re-infer it here again in order to propagate the re-inferred - # source to the inliner as a volatile result - cache_mode = CACHE_MODE_VOLATILE - edge_ci = codeinst - else - @assert codeinst.def === mi "MethodInstance for cached edge does not match" - return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) - end + cached, missing_source_edge = lookup_cached_edge(interp, method, mi, caller, + force_inline, edgecycle, edgelimited, edgerecursed) + if cached !== nothing + return cached + elseif missing_source_edge !== nothing + # Reuse a sourceless result only when `mi` is already active in this + # interpreter's inference cycle; otherwise local inference may recover source. + if edgerecursed && find_call_cycle(interp, mi, caller) !== nothing + return return_cached_result(interp, method, missing_source_edge, nothing, caller, + edgecycle, edgelimited, edgerecursed) end + # A globally published executable target exists, but its source was discarded. + # Re-infer only the source/facts and certify that local work with a local proof. + cache_mode = CACHE_MODE_LOCAL + edge_ci = missing_source_edge end if !InferenceParams(interp).force_enable_inference && ccall(:jl_get_module_infer, Cint, (Any,), method.module) == 0 add_remark!(interp, caller, "[typeinf_edge] Inference is disabled for the target module") - return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited, edgerecursed)) end - if !is_cached(caller) && frame_parent(caller) === nothing + if !edgerecursed && !edgelimited + # the callstack walk proved there is no cycle to resolve, as long as + # `atype` was not coarsened to an on-stack specialization after that walk + frame = false + elseif !is_cached(caller) && frame_parent(caller) === nothing # this caller exists to return to the user # (if we asked resolve_call_cycle!, it might instead detect that there is a cycle that it can't merge) frame = false @@ -1063,19 +1332,16 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize reserve_start = _time_ns() # subtract engine_reserve (thread-synchronization) time from callers to avoid double-counting ci_from_engine = engine_reserve(interp, mi) caller.time_paused += (_time_ns() - reserve_start) - edge_ci = ci_from_engine - codeinst = get(code_cache(interp), mi, nothing) - if codeinst isa CodeInstance # return existing rettype if the code is already inferred + cached, missing_source_edge = lookup_cached_edge(interp, method, mi, caller, + force_inline, edgecycle, edgelimited, edgerecursed) + if cached !== nothing + engine_reject(interp, ci_from_engine) + return cached + elseif missing_source_edge !== nothing engine_reject(interp, ci_from_engine) ci_from_engine = nothing - inferred = @atomic :monotonic codeinst.inferred - if inferred === nothing && force_inline - cache_mode = CACHE_MODE_VOLATILE - edge_ci = codeinst - else - @assert codeinst.def === mi "MethodInstance for cached edge does not match" - return return_cached_result(interp, method, codeinst, caller, edgecycle, edgelimited) - end + cache_mode = CACHE_MODE_LOCAL + edge_ci = missing_source_edge end else ci_from_engine = nothing @@ -1083,6 +1349,9 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize result = InferenceResult(mi, typeinf_lattice(interp)) if ci_from_engine !== nothing result.ci = ci_from_engine + elseif !iszero(cache_mode & CACHE_MODE_GLOBAL) + result.ci = ccall(:jl_new_codeinst_uninit, Any, (Any, Any), + mi, cache_owner(interp))::CodeInstance end frame = InferenceState(result, cache_mode, interp) # always use the cache for edge targets if frame === nothing @@ -1091,44 +1360,33 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize if ci_from_engine !== nothing engine_reject(interp, ci_from_engine) end - return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited, edgerecursed)) end assign_parentchild!(frame, caller) # the actual inference task for this edge is going to be scheduled within `typeinf_local` via the callstack queue # while splitting off the rest of the work for this caller into a separate workq thunk - let mresult = Future{MethodCallResult}() - push!(caller.tasks, function get_infer_result(interp, caller) - update_valid_age!(caller, frame.world.valid_worlds) - local isinferred = is_inferred(frame) - local edge = isinferred ? edge_ci : nothing - local effects = isinferred ? frame.result.ipo_effects : # effects are adjusted already within `finish` for ipo_effects - adjust_effects(effects_for_cycle(frame.ipo_effects), method) - local bestguess = frame.bestguess - local exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) - # propagate newly inferred source to the inliner, allowing efficient inlining w/o deserialization: - # note that this result is cached globally exclusively, so we can use this local result destructively - local volatile_inf_result = if isinferred && edge_ci isa CodeInstance - result.ci_as_edge = edge_ci # set the edge for the inliner usage - VolatileInferenceResult(result) - end - mresult[] = MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, - edge, edgecycle, edgelimited, volatile_inf_result) - return true - end) - return mresult - end + return _schedule_edge_infer_task!(caller, frame, result, method, edge_ci, edgecycle, edgelimited, edgerecursed) elseif frame === true # unresolvable cycle add_remark!(interp, caller, "[typeinf_edge] Unresolvable cycle") - return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) + return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited, edgerecursed)) end # return the current knowledge about this cycle frame = frame::InferenceState - update_valid_age!(caller, frame.world.valid_worlds) + update_valid_age!(caller, get_inference_world(interp), frame.valid_worlds) effects = adjust_effects(effects_for_cycle(frame.ipo_effects), method) bestguess = frame.bestguess exc_bestguess = refine_exception_type(frame.exc_bestguess, effects) - return Future(MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, nothing, edgecycle, edgelimited)) + add_cycle_backedge!(caller, frame) + result = frame.result + edge = get(code_cache(interp), result.linfo, nothing) + edge isa CodeInstance || (edge = nothing) + if edge !== nothing + update_valid_age!(caller, get_inference_world(interp), proof_worlds(edge)) + end + return Future(MethodCallResult(interp, caller, method, bestguess, exc_bestguess, effects, + edge, edgecycle, edgelimited, edgerecursed; + force_edgecycle=false, needs_mi_edge=edge === nothing)) end # The `:terminates` effect bit must be conservatively tainted unless recursion cycle has @@ -1147,6 +1405,8 @@ function cached_return_type(code::CodeInstance) return PartialStruct(fallback_lattice, rettype, undefs, fields) elseif isa(rettype_const, PartialOpaque) && rettype <: Core.OpaqueClosure return rettype_const + elseif isa(rettype_const, PartialTask) && rettype <: Task + return rettype_const elseif isa(rettype_const, InterConditional) && rettype !== InterConditional return rettype_const elseif isa(rettype_const, InterMustAlias) && rettype !== InterMustAlias @@ -1159,14 +1419,14 @@ end #### entry points for inferring a MethodInstance given a type signature #### """ - codeinfo_for_const(interp::AbstractInterpreter, mi::MethodInstance, worlds::WorldRange, @nospecialize(val)) + codeinfo_for_const(interp::AbstractInterpreter, mi::MethodInstance, worlds::WorldRange, edges::SimpleVector, @nospecialize(val)) Return a fake CodeInfo that just contains `return \$val`. This function is used in various reflection APIs when asking for the code of a function that inference has found to just return a constant. For such functions, no code is actually stored - the constant is used directly. However, because this is an ABI implementation detail, it is nice to maintain consistency and just synthesize a CodeInfo when the reflection APIs ask for them - this function does that. """ -function codeinfo_for_const(interp::AbstractInterpreter, mi::MethodInstance, @nospecialize(val)) +function codeinfo_for_const(::AbstractInterpreter, mi::MethodInstance, worlds::WorldRange, edges::SimpleVector, @nospecialize(val)) method = mi.def::Method tree = ccall(:jl_new_code_info_uninit, Ref{CodeInfo}, ()) tree.code = Any[ ReturnNode(quoted(val)) ] @@ -1177,7 +1437,11 @@ function codeinfo_for_const(interp::AbstractInterpreter, mi::MethodInstance, @no tree.debuginfo = DebugInfo(mi) tree.ssaflags = [IR_FLAG_NULL] tree.rettype = Core.Typeof(val) - tree.edges = Core.svec() + tree.min_world = first(worlds) + tree.max_world = last(worlds) + tree.edges = edges + tree.nargs = UInt(nargs) + tree.isva = method.isva set_inlineable!(tree, true) tree.parent = mi return tree @@ -1243,11 +1507,12 @@ function typeinf_frame(interp::AbstractInterpreter, mi::MethodInstance, run_opti if run_optimizer if result_is_constabi(interp, frame.result) rt = frame.result.result::Const - src = codeinfo_for_const(interp, frame.linfo, rt.val) + edges = materialize_inference_edges(frame.edges) + src = codeinfo_for_const(interp, frame.linfo, frame.valid_worlds, edges, rt.val) else opt = OptimizationState(frame, interp) optimize(interp, opt, frame.result) - src = ir_to_codeinf!(opt, frame, Core.svec(opt.inlining.edges...)) + src = ir_to_codeinf!(opt, frame, materialize_inference_edges(opt.inlining.edges)) end result.src = frame.src = src end @@ -1296,33 +1561,64 @@ end """ ci_has_source(interp::AbstractInterpreter, code::CodeInstance) -Determine whether this CodeInstance is something that could be compiled from -source that interp has. +Determine whether this CodeInstance is something that will return something +compileable by ci_get_source. """ function ci_has_source(interp::AbstractInterpreter, code::CodeInstance) codegen = codegen_cache(interp) codegen === nothing && return false use_const_api(code) && return true - haskey(codegen, code) && return true + inf = get(codegen, code, nothing) + inf === nothing || return true inf = @atomic :monotonic code.inferred if isa(inf, String) inf = _uncompressed_ir(code, inf) end - if code.owner === nothing - if isa(inf, CodeInfo) - codegen[code] = inf - return true - end - elseif inf !== nothing + if isa(inf, CodeInfo) + codegen[code] = inf return true end return false end +# Get source if available for inlining, otherwise return nothing +# populates codegen cache for code, if successful +function ci_get_source(interp::AbstractInterpreter, code::CodeInstance, @nospecialize src) + codegen = codegen_cache(interp) + if codegen !== nothing + inf = get(codegen, code, nothing) + inf === nothing || return inf + end + if use_const_api(code) + return codeinfo_for_const(interp, get_ci_mi(code), WorldRange(code.min_world, code.max_world), code.edges, code.rettype_const) + end + if isa(src, String) + src = _uncompressed_ir(code, src) + end + if isa(src, CodeInfo) + if codegen !== nothing + codegen[code] = src + end + return src + elseif isa(src, IRCode) + error("IRCode is unexpected") + end + return nothing +end + +function ci_get_source(interp::AbstractInterpreter, code::CodeInstance) + return ci_get_source(interp, code, isdefined(code, :inferred) ? code.inferred : nothing) +end + function ci_has_invoke(code::CodeInstance) return (@atomic :monotonic code.invoke) !== C_NULL end +const CI_FLAGS_FROM_IMAGE = 0b0100 +function ci_from_image(code::CodeInstance) + return (@atomic :monotonic code.flags) & CI_FLAGS_FROM_IMAGE != 0 +end + function ci_meets_requirement(interp::AbstractInterpreter, code::CodeInstance, source_mode::UInt8) source_mode == SOURCE_MODE_NOT_REQUIRED && return true source_mode == SOURCE_MODE_ABI && return ci_has_abi(interp, code) @@ -1330,29 +1626,160 @@ function ci_meets_requirement(interp::AbstractInterpreter, code::CodeInstance, s return false end +function ci_worlds_cover(code::CodeInstance, valid_worlds::WorldRange) + min_world = @atomic :acquire code.min_world + max_world = @atomic :acquire code.max_world + return min_world <= first(valid_worlds) && last(valid_worlds) <= max_world +end + +function ci_cache_head(mi::MethodInstance) + isdefined(mi, :cache, :acquire) || return nothing + return @atomic :acquire mi.cache +end + +function ci_cache_next(code::CodeInstance) + isdefined(code, :next, :acquire) || return nothing + return @atomic :acquire code.next +end + +function find_cached_ci(interp::AbstractInterpreter, mi::MethodInstance, + valid_worlds::WorldRange, source_mode::UInt8) + cache = code_cache(interp, valid_worlds) + return find_cached_ci(interp, cache, mi, valid_worlds, source_mode) +end + +function find_cached_ci(interp::AbstractInterpreter, cache, mi::MethodInstance, + valid_worlds::WorldRange, source_mode::UInt8) + code = get(cache, mi, nothing) + if (code isa CodeInstance && + ci_worlds_cover(code, valid_worlds) && + isdefined(code, :inferred, :acquire) && + ci_meets_requirement(interp, code, source_mode)) + return code + end + return nothing +end + +function find_cached_ci(interp::AbstractInterpreter, cache::OverlayCodeCache, + mi::MethodInstance, valid_worlds::WorldRange, source_mode::UInt8) + return find_cached_ci(interp, cache.globalcache, mi, valid_worlds, source_mode) +end + +function find_cached_ci(interp::AbstractInterpreter, cache::InternalCodeCache, + mi::MethodInstance, valid_worlds::WorldRange, source_mode::UInt8) + # `jl_rettype_inferred` returns the first inferred CI spanning the requested + # worlds, without considering whether it carries source or an ABI. Once a + # source-less CI is at the head of the native cache, repeatedly looking only at + # that entry would publish a new source-capable CI for every ABI/source request. + # Walk the native cache chain to find the first entry that satisfies the full + # request. Cache wrappers dispatch back to this implementation through their + # underlying global cache. A concurrent `jl_mi_cache_insert` may briefly make + # the chain circular while moving an existing CI; like the C-side walkers + # (e.g. `jl_get_ci_equiv`), keep following `next` until the writer's fixup + # store lands. + code = ci_cache_head(mi) + while code !== nothing + if (ci_worlds_cover(code, valid_worlds) && + code.owner === cache.owner && + isdefined(code, :inferred, :acquire) && + ci_meets_requirement(interp, code, source_mode)) + return code + end + code = ci_cache_next(code) + end + return nothing +end + +function ci_is_equivalent_winner(candidate::CodeInstance, ci::CodeInstance, + valid_worlds::WorldRange) + return (candidate !== ci && + ci_worlds_cover(candidate, valid_worlds) && + candidate.def === ci.def && + candidate.owner === ci.owner && + isdefined(candidate, :inferred, :acquire) && + isdefined(candidate, :rettype) && + candidate.rettype === ci.rettype) +end + +function find_equivalent_cached_ci(interp::AbstractInterpreter, cache, + ci::CodeInstance, valid_worlds::WorldRange) + candidate = get(cache, get_ci_mi(ci), nothing) + return (candidate isa CodeInstance && + ci_is_equivalent_winner(candidate, ci, valid_worlds)) ? candidate : nothing +end + +function find_equivalent_cached_ci(interp::AbstractInterpreter, + cache::OverlayCodeCache, ci::CodeInstance, + valid_worlds::WorldRange) + return find_equivalent_cached_ci( + interp, cache.globalcache, ci, valid_worlds) +end + +function find_equivalent_cached_ci(::AbstractInterpreter, + ::InternalCodeCache, ci::CodeInstance, + valid_worlds::WorldRange) + candidate = ci_cache_head(get_ci_mi(ci)) + while candidate !== nothing + ci_is_equivalent_winner(candidate, ci, valid_worlds) && return candidate + candidate = ci_cache_next(candidate) + end + return nothing +end + +function find_equivalent_cached_ci(interp::AbstractInterpreter, ci::CodeInstance, + valid_worlds::WorldRange) + cache = code_cache(interp, valid_worlds) + return find_equivalent_cached_ci(interp, cache, ci, valid_worlds) +end + +function find_local_cached_ci(interp::AbstractInterpreter, mi::MethodInstance, + valid_worlds::WorldRange, source_mode::UInt8) + cache = get_inference_cache(interp) + indices = get_indices(cache, mi) + world = get_inference_world(interp) + for i in length(indices):-1:1 + cached = cache.results[indices[i]] + cached isa LocalInferenceResult || continue + result = cached.result + result.overridden_by_const === nothing || continue + result.cache_world == world || continue + world in proof_worlds(cached.proof) || continue + isdefined(result, :ci) || continue + code = result.ci + if (ci_worlds_cover(code, valid_worlds) && + isdefined(code, :inferred, :acquire) && + ci_meets_requirement(interp, code, source_mode)) + return code + end + end + return nothing +end + +function find_typeinf_cached_ci(interp::AbstractInterpreter, mi::MethodInstance, + valid_worlds::WorldRange, source_mode::UInt8) + code = find_cached_ci(interp, mi, valid_worlds, source_mode) + code === nothing || return code + return find_local_cached_ci(interp, mi, valid_worlds, source_mode) +end + # compute (and cache) an inferred AST and return type function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mode::UInt8) start_time = ccall(:jl_typeinf_timing_begin, UInt64, ()) - let code = get(code_cache(interp), mi, nothing) - if code isa CodeInstance - # see if this code already exists in the cache - if ci_meets_requirement(interp, code, source_mode) - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return code - end + valid_worlds = WorldRange(get_inference_world(interp)) + let code = find_typeinf_cached_ci(interp, mi, valid_worlds, source_mode) + if code !== nothing + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64, Cint), start_time, 0) + return code end end def = mi.def ci = engine_reserve(interp, mi) # check cache again if it is still new after reserving in the engine - let code = get(code_cache(interp), mi, nothing) - if code isa CodeInstance - # see if this code already exists in the cache - if ci_meets_requirement(interp, code, source_mode) - engine_reject(interp, ci) - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return code - end + let code = find_typeinf_cached_ci(interp, mi, valid_worlds, source_mode) + if code !== nothing + engine_reject(interp, ci) + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64, Cint), start_time, 0) + return code end end if !InferenceParams(interp).force_enable_inference @@ -1362,8 +1789,10 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod finish!(interp, mi, ci, src) else engine_reject(interp, ci) + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64, Cint), start_time, 0) + return nothing end - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64, Cint), start_time, 0) return ci end end @@ -1372,18 +1801,27 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod frame = InferenceState(result, #=cache_mode=#:global, interp) if frame === nothing engine_reject(interp, ci) - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64, Cint), start_time, 0) return nothing end typeinf(interp, frame) - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64, Cint), start_time, 0) - ci = result.ci # reload from result in case it changed + publication_winner = result.replacement_ci + ci = if (publication_winner !== nothing && + ci_meets_requirement(interp, publication_winner, source_mode)) + publication_winner + else + result.ci + end codegen = codegen_cache(interp) @assert frame.cache_mode != CACHE_MODE_NULL - @assert is_result_constabi_eligible(result) || codegen === nothing || haskey(codegen, ci) - @assert is_result_constabi_eligible(result) == use_const_api(ci) @assert isdefined(ci, :inferred) "interpreter did not fulfill our expectations" + @assert codegen === nothing || ci_meets_requirement(interp, ci, source_mode) + if ci === result.ci + @assert is_result_constabi_eligible(result) || codegen === nothing || haskey(codegen, ci) + @assert is_result_constabi_eligible(result) == use_const_api(ci) + end return ci end @@ -1420,8 +1858,7 @@ function compileable_specialization_for_call(interp::AbstractInterpreter, @nospe compileable_atype = get_compileable_sig(match.method, match.spec_types, match.sparams) compileable_atype === nothing && return nothing if match.spec_types !== compileable_atype - sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), compileable_atype, match.method.sig)::SimpleVector - sparams = sp_[2]::SimpleVector + (_, sparams) = typeintersect_env(compileable_atype, match.method.sig) mi = specialize_method(match.method, compileable_atype, sparams) else mi = specialize_method(match.method, compileable_atype, match.sparams) @@ -1454,25 +1891,64 @@ markinspected!(queue::CompilationQueue, item) = push!(queue.inspected, item) isinspected(queue::CompilationQueue, item) = item in queue.inspected Base.isempty(queue::CompilationQueue) = isempty(queue.tocompile) +function has_valid_abi_sparams(mi::MethodInstance) + isa(mi.specTypes, UnionAll) && return false + def = mi.def + isa(def, Method) || return true + unionall_depth(def.sig) == length(mi.sparam_vals) || return false + for i = 1:length(mi.sparam_vals) + sp = mi.sparam_vals[i] + if isa(sp, SimpleVector) || isvarargtype(sp) + return false + end + end + return true +end + # collect a list of all code that is needed along with CodeInstance to codegen it fully function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vector{VarState}; - invokelatest_queue::Union{CompilationQueue,Nothing} = nothing) + invokelatest_queue::Union{CompilationQueue,Nothing} = nothing, + enqueue_unprepared_invokes::Bool = false, + external_linkage::Bool = false) src = ci.code for i = 1:length(src) stmt = src[i] isexpr(stmt, :(=)) && (stmt = stmt.args[2]) if isexpr(stmt, :invoke) || isexpr(stmt, :invoke_modify) edge = stmt.args[1] - edge isa CodeInstance && isdefined(edge, :inferred) && push!(workqueue, edge) + # If this CodeInstance is already compiled in the image, and we can + # link to it, we should do that instead of compiling it again. With + # invoke_modify, we need to compile it regardless. + if edge isa CodeInstance && has_valid_abi_sparams(get_ci_mi(edge)) && + (isexpr(stmt, :invoke_modify) || + !(external_linkage && ci_from_image(edge) && ci_has_invoke(edge))) && + (enqueue_unprepared_invokes || + ci_has_invoke(edge) || ci_has_source(workqueue.interp, edge) || + !iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), get_ci_mi(edge), edge))) + # The globally-cached check keeps batches closed under invoke + # edges even when the edge's source lives only in another + # interpreter's codegen cache (activation clears `inferred`, + # so `ci_has_source` cannot see it): the drain loops re-infer + # such edges, preventing them from silently leaking to + # permanent `tojlinvoke` fallbacks at link time. Uncached + # speculative edges must NOT be enqueued unconditionally: + # resolved invoke edges of recursion that inference widened + # (e.g. self-recursion with a growing tuple argument) form an + # unbounded chain of fresh signatures, and re-inferring each + # one would enqueue the next forever. + push!(workqueue, edge) + elseif enqueue_unprepared_invokes && edge isa MethodInstance && has_valid_abi_sparams(edge) + push!(workqueue, edge) + end end invokelatest_queue === nothing && continue if isexpr(stmt, :call) farg = stmt.args[1] !applicable(argextype, farg, ci, sptypes) && continue # TODO: Why is this failing during bootstrap - ftyp = widenconst(argextype(farg, ci, sptypes)) + ftyp = argextype_widened(farg, ci, sptypes) - if ftyp === typeof(Core.finalizer) && length(stmt.args) == 3 + if ftyp === typeof(Core.finalizer) && 3 <= length(stmt.args) <= 5 finalizer = argextype(stmt.args[2], ci, sptypes) obj = argextype(stmt.args[3], ci, sptypes) atype = argtypes_to_type(Any[finalizer, obj]) @@ -1481,7 +1957,7 @@ function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vec continue end elseif isexpr(stmt, :cfunction) && length(stmt.args) == 5 - (pointer_type, f, rt, at, call_type) = stmt.args + (_, f, _, at, _) = stmt.args linfo = ci.parent linfo isa MethodInstance || continue @@ -1493,6 +1969,12 @@ function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vec push!(argtypes, sp_type_rewrap(at[i], linfo, #= isreturn =# false)) end atype = argtypes_to_type(argtypes) + elseif isexpr(stmt, :new) + # When creating a struct of Function type, check to see if we should + # proactively compile the lambda + t, _, _, _ = instanceof_tfunc(argextype(stmt.args[1], ci, sptypes)) + t <: Function || continue + atype = Tuple{t, Vararg} else # TODO: handle other StmtInfo like OpaqueClosure? continue @@ -1507,6 +1989,35 @@ function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vec end end +""" + jit_cache_root!(cache, ci::CodeInstance) + +Establish a GC root for `ci` that is adequate for handing it to the JIT. + +The JIT retains raw, non-GC-visible pointers to every CodeInstance it emits code +for (in its symbol table and in the debuginfo address map used for backtraces +and profiling), for the lifetime of the process. Every CodeInstance passed to +`jl_add_codeinsts_to_jit` must therefore remain GC-reachable permanently. +[`add_codeinsts_to_jit!`](@ref) calls this function for each CodeInstance it is +about to emit that is not already rooted through the native `mi.cache` chain +(which guarantees the required lifetime on its own); the executable cache that +holds the CodeInstance is responsible for guaranteeing an equivalent lifetime. + +The generic fallback conservatively promotes the CodeInstance to a global root, +which matches the lifetime of the code emitted for it (JIT code is never +freed). A custom cache whose entries are process-rooted by other means may +override this with a no-op. +""" +function jit_cache_root!(cache, ci::CodeInstance) + ccall(:jl_as_global_root, Any, (Any, Cint), ci, 1) + return nothing +end +# Entries in the native `mi.cache` chain are already rooted for the lifetime of +# the process through their MethodInstance. +jit_cache_root!(::InternalCodeCache, ::CodeInstance) = nothing +jit_cache_root!(cache::OverlayCodeCache, ci::CodeInstance) = + jit_cache_root!(cache.globalcache, ci) + function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UInt8) source_mode == SOURCE_MODE_ABI || return ci ci isa CodeInstance && !ci_has_invoke(ci) || return ci @@ -1514,51 +2025,65 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn codegen === nothing && return ci workqueue = CompilationQueue(; interp) push!(workqueue, ci) + codeinsts, srcs = Any[], Any[] while !isempty(workqueue) # ci_has_real_invoke(ci) && return ci # optimization: cease looping if ci happens to get compiled (not just jl_fptr_wait_for_compiled, but fully jl_is_compiled_codeinst) callee = pop!(workqueue) ci_has_invoke(callee) && continue isinspected(workqueue, callee) && continue - src = get(codegen, callee, nothing) + if !has_valid_abi_sparams(get_ci_mi(callee)) + markinspected!(workqueue, callee) + continue + end + let cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance + if cached !== callee + markinspected!(workqueue, callee) + continue + end + end + src = ci_get_source(interp, callee) if !isa(src, CodeInfo) - src = @atomic :monotonic callee.inferred - if isa(src, String) - src = _uncompressed_ir(callee, src) + newcallee = typeinf_ext(workqueue.interp, callee.def, source_mode) # always SOURCE_MODE_ABI + if newcallee isa CodeInstance + callee === ci && (ci = newcallee) # ci stopped meeting the requirements after typeinf_ext last checked, try again with newcallee + push!(workqueue, newcallee) end - if !isa(src, CodeInfo) - newcallee = typeinf_ext(workqueue.interp, callee.def, source_mode) # always SOURCE_MODE_ABI - if newcallee isa CodeInstance - callee === ci && (ci = newcallee) # ci stopped meeting the requirements after typeinf_ext last checked, try again with newcallee - push!(workqueue, newcallee) - end - if newcallee !== callee - markinspected!(workqueue, callee) - end - continue + if newcallee !== callee + markinspected!(workqueue, callee) end + continue end markinspected!(workqueue, callee) mi = get_ci_mi(callee) sptypes = sptypes_from_meth_instance(mi) collectinvokes!(workqueue, src, sptypes) if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance - if cached === callee - # make sure callee is gc-rooted and cached, as required by jl_add_codeinst_to_jit + valid_worlds = WorldRange(get_inference_world(workqueue.interp)) + cached = find_equivalent_cached_ci( + workqueue.interp, callee, valid_worlds) + if cached === nothing + # make sure callee is cached, as required by jl_add_codeinsts_to_jit code_cache(workqueue.interp)[mi] = callee else # use an existing CI from the cache, if there is available one that is compatible callee === ci && (ci = cached) callee = cached end + # `callee` is about to be emitted while absent from the native + # `mi.cache` chain; the executable cache it lives in must root it + # for the lifetime of the process (see `jit_cache_root!`). + jit_cache_root!(code_cache(workqueue.interp), callee) end - ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), callee, src) + push!(codeinsts, callee) + push!(srcs, src) end + ccall(:jl_add_codeinsts_to_jit, Cvoid, (Any, Any), codeinsts, srcs) return ci end function typeinf_ext_toplevel(interp::AbstractInterpreter, mi::MethodInstance, source_mode::UInt8) - ci = typeinf_ext(interp, mi, source_mode) + mi2 = ccall(:jl_normalize_to_compilable_mi, Any, (Any,), mi)::MethodInstance + ci = typeinf_ext(interp, mi2, source_mode) ci = add_codeinsts_to_jit!(interp, ci, source_mode) return ci end @@ -1572,6 +2097,8 @@ end function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; invokelatest_queue::Union{CompilationQueue,Nothing} = nothing, + enqueue_unprepared_invokes::Bool = false, + external_linkage::Bool, ) interp = workqueue.interp world = get_inference_world(interp) @@ -1593,11 +2120,9 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; invokelatest_queue === nothing && continue (rt::Type, sig::Type) = item # make a best-effort attempt to enqueue the relevant code for the ccallable - ptr = ccall(:jl_get_specialization1, - #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), - sig, world, #= mt_cache =# 0) - if ptr !== C_NULL - mi = unsafe_pointer_to_objref(ptr)::MethodInstance + mi = ccall(:jl_get_specialization1, Any, (Any, Csize_t), sig, world) + if mi !== nothing + mi = mi::MethodInstance ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) ci isa CodeInstance && push!(invokelatest_queue, ci) end @@ -1608,9 +2133,13 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; callee = item isinspected(workqueue, callee) && continue mi = get_ci_mi(callee) + if !has_valid_abi_sparams(mi) + markinspected!(workqueue, callee) + continue + end # now make sure everything has source code, if desired if use_const_api(callee) - src = codeinfo_for_const(interp, mi, callee.rettype_const) + src = codeinfo_for_const(interp, mi, WorldRange(callee.min_world, callee.max_world), callee.edges, callee.rettype_const) else src = get(interp.codegen, callee, nothing) if src === nothing @@ -1628,11 +2157,13 @@ function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; markinspected!(workqueue, callee) if src isa CodeInfo sptypes = sptypes_from_meth_instance(mi) - collectinvokes!(workqueue, src, sptypes; invokelatest_queue) + collectinvokes!(workqueue, src, sptypes; invokelatest_queue, + enqueue_unprepared_invokes, external_linkage) # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, world)::CodeInstance - if cached === callee + cached = find_equivalent_cached_ci( + interp, callee, WorldRange(world)) + if cached === nothing code_cache(interp)[mi] = callee else # Use an existing CI from the cache, if there is available one that is compatible @@ -1653,8 +2184,11 @@ const TRIM_NO = 0x0 const TRIM_SAFE = 0x1 const TRIM_UNSAFE = 0x2 const TRIM_UNSAFE_WARN = 0x3 -function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::UInt8) - inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) +function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::UInt8, external_linkage::Bool) + # During `--trim`, infer against an isolated cache namespace. The owner is re-stamped + # back to `nothing` at serialization time (see `src/staticdata.c`). + cache_owner = trim_mode == TRIM_NO ? nothing : :trim + inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO, cache_owner) # Create an "invokelatest" queue to enable eager compilation of speculative # invokelatest calls such as from `Core.finalizer` and `ccallable` @@ -1670,31 +2204,104 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m ) append!(workqueue, methods) - compile!(codeinfos, workqueue; invokelatest_queue) + compile!(codeinfos, workqueue; invokelatest_queue, external_linkage, + enqueue_unprepared_invokes = trim_mode != TRIM_NO) end if invokelatest_queue !== nothing # This queue is intentionally aliased, to handle e.g. a `finalizer` calling `Core.finalizer` # (it will enqueue into itself and immediately drain) - compile!(codeinfos, invokelatest_queue; invokelatest_queue) + compile!(codeinfos, invokelatest_queue; invokelatest_queue, external_linkage, + enqueue_unprepared_invokes = trim_mode != TRIM_NO) end if trim_mode != TRIM_NO && trim_mode != TRIM_UNSAFE verify_typeinf_trim(codeinfos, trim_mode == TRIM_UNSAFE_WARN) end - return codeinfos + + # Build the ordered list of CodeInstances to store in the image's method + # caches. This is kept as its own array (rather than being recovered from + # `codeinfos` by the caller) so the set and ordering of cached entries can be + # chosen independently of what native code gets emitted. The linked list of + # each MethodInstance's cache is rebuilt from this order during serialization + # (see jl_rewrite_mi_caches in staticdata.c). + # + # First the compiled entries, in compilation order: every entry paired with a + # CodeInfo in `codeinfos` is inferred and about to be compiled, so it lands in + # the single invoke+inferred group that jl_mi_cache_insert (gf.c) keeps at the + # front, and the compilation order already lists higher-`max_world` entries + # first (worlds are processed newest-first). + cis = Any[] + seen = IdSet{CodeInstance}() + for i = 1:length(codeinfos) + item = codeinfos[i] + if item isa CodeInstance && !(item in seen) + push!(seen, item) + push!(cis, item) + end + end + + # Then walk each CodeInstance's forward `edges` recursively and append every + # reachable CodeInstance. These callees are inferred but were not compiled + # (no `invoke` assigned), so appending them after the compiled entries places + # them in gf.c's inferred (post-invoke) group within each MethodInstance's + # cache. Preserving them keeps the inference results callers depend on from + # being dropped by the cache-clearing pass in staticdata.c. `cis` doubles as + # the worklist, so edges discovered from appended entries are visited too. + # + # Skip under `--trim` where inferred-but-not-compiled entries are not useful + # at runtime without a Compiler / JIT. + if trim_mode == TRIM_NO + i = 1 + while i <= length(cis) + ci = cis[i]::CodeInstance + if isdefined(ci, :edges) + edges = ci.edges + for j = 1:length(edges) + isassigned(edges, j) || continue + edge = edges[j] + if edge isa CodeInstance && !(edge in seen) + push!(seen, edge) + push!(cis, edge) + end + end + end + i += 1 + end + end + + # Keep cache CodeInstances that represent an existing cache entry: either they are + # already linked into their MethodInstance's cache, or jl_get_ci_equiv finds + # another equivalent already cached for them. + # This hack avoids putting badly inferred edges in the cache, while still trying to populate the cache sufficiently. + filter!(cis) do ci + ci = ci::CodeInstance + mi = get_ci_mi(ci) + return !iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, ci)) || + ccall(:jl_get_ci_equiv, Any, (Any, UInt), ci, 0x0)::CodeInstance !== ci + end + + return Core.svec(codeinfos, cis) end const _verify_trim_world_age = RefValue{UInt}(typemax(UInt)) -verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = Core._call_in_world(_verify_trim_world_age[], verify_typeinf_trim, stdout, codeinfos, onlywarn) +verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = Core._call_in_world(_verify_trim_world_age[], verify_typeinf_trim, Base.stderr, codeinfos, onlywarn) + +function _return_type_opaque_closure(@nospecialize(oc::Core.OpaqueClosure), t::DataType) + ocargt, ocrt = typeof(oc).parameters + hasintersect(t, ocargt) || return Union{} + return ocrt +end function return_type(@nospecialize(f), t::DataType) # this method has a special tfunc + isa(f, Core.OpaqueClosure) && return _return_type_opaque_closure(f, t) world = tls_world_age() args = Any[_return_type, NativeInterpreter(world), Tuple{Core.Typeof(f), t.parameters...}] return ccall(:jl_call_in_typeinf_world, Any, (Ptr{Any}, Cint), args, length(args)) end function return_type(@nospecialize(f), t::DataType, world::UInt) + isa(f, Core.OpaqueClosure) && return _return_type_opaque_closure(f, t) return return_type(Tuple{Core.Typeof(f), t.parameters...}, world) end diff --git a/Compiler/src/typelattice.jl b/Compiler/src/typelattice.jl index f4c3b051d3e3f..aeabd5834034d 100644 --- a/Compiler/src/typelattice.jl +++ b/Compiler/src/typelattice.jl @@ -4,9 +4,9 @@ # structs/constants # ##################### -# N.B.: Const/PartialStruct/InterConditional are defined in Core, to allow them to be used -# inside the global code cache. -import Core: Const, InterConditional, PartialStruct +# N.B.: Const/PartialStruct/InterConditional/InterMustAlias/PartialTask are defined in Core, +# to allow them to be used inside the global code cache. +import Core: Const, InterConditional, PartialStruct, InterMustAlias, PartialTask function may_form_limited_typ(@nospecialize(aty), @nospecialize(bty), @nospecialize(xty)) if aty isa LimitedAccuracy @@ -32,7 +32,7 @@ the type of `SlotNumber(cnd.slot)` will be limited by `cnd.thentype` and in the false branch, it will be limited by `cnd.elsetype`. Example: ```julia -let cond = isa(x::Union{Int, Float}, Int)::Conditional(x, Int, Float) +let cond = isa(x::Union{Int, Float}, Int)::Conditional(x, _, Int, Float) if cond # May assume x is `Int` now else @@ -43,41 +43,44 @@ end """ struct Conditional slot::Int + ssadef::Int thentype elsetype # `isdefined` indicates this `Conditional` is from `@isdefined slot`, implying that # the `undef` information of `slot` can be improved in the then branch. # Since this is only beneficial for local inference, it is not translated into `InterConditional`. isdefined::Bool - function Conditional(slot::Int, @nospecialize(thentype), @nospecialize(elsetype); + function Conditional(slot::Int, ssadef::Int, @nospecialize(thentype), @nospecialize(elsetype); isdefined::Bool=false) assert_nested_slotwrapper(thentype) assert_nested_slotwrapper(elsetype) limited = may_form_limited_typ(thentype, elsetype, Bool) limited !== nothing && return limited - return new(slot, thentype, elsetype, isdefined) + return new(slot, ssadef, thentype, elsetype, isdefined) end end -Conditional(var::SlotNumber, @nospecialize(thentype), @nospecialize(elsetype); isdefined::Bool=false) = - Conditional(slot_id(var), thentype, elsetype; isdefined) +Conditional(var::SlotNumber, ssadef::Int, @nospecialize(thentype), @nospecialize(elsetype); isdefined::Bool=false) = + Conditional(slot_id(var), ssadef, thentype, elsetype; isdefined) const AnyConditional = Union{Conditional,InterConditional} -Conditional(cnd::InterConditional) = Conditional(cnd.slot, cnd.thentype, cnd.elsetype) -InterConditional(cnd::Conditional) = InterConditional(cnd.slot, cnd.thentype, cnd.elsetype) +function InterConditional(cnd::Conditional) + @assert cnd.ssadef == 0 + InterConditional(cnd.slot, cnd.thentype, cnd.elsetype) +end """ alias::MustAlias -This lattice element wraps a reference to object field while recoding the identity of the +This lattice element wraps a reference to object field while recording the identity of the parent object. It allows certain constraints that can be imposed on the object field type by built-in functions like `isa` and `===` to be propagated to another reference to the same object field. One important note is that this lattice element assumes the invariant that the field of wrapped slot object never changes until the slot object is re-assigned. This means, the wrapped object field should be constant as inference currently doesn't track any memory -effects on per-object basis. Particularly `maybe_const_fldidx` takes the lift to check if +effects on per-object basis. Particularly `maybe_const_fldidx` has the task of checking if a given lattice element is eligible to be wrapped by `MustAlias`. Example: -```juila +```julia let alias = getfield(x::Some{Union{Nothing,String}}, :value)::MustAlias(x, Some{Union{Nothing,String}}, 1, Union{Nothing,String}) if alias === nothing # May assume `getfield(x, :value)` is `nothing` now @@ -90,48 +93,30 @@ N.B. currently this lattice element is only used in abstractinterpret, not in op """ struct MustAlias slot::Int + ssadef::Int vartyp::Any fldidx::Int fldtyp::Any - function MustAlias(slot::Int, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) + function MustAlias(slot::Int, ssadef::Int, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) assert_nested_slotwrapper(vartyp) assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" limited = may_form_limited_typ(vartyp, fldtyp, fldtyp) limited !== nothing && return limited - return new(slot, vartyp, fldidx, fldtyp) + return new(slot, ssadef, vartyp, fldidx, fldtyp) end end -MustAlias(var::SlotNumber, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) = - MustAlias(slot_id(var), vartyp, fldidx, fldtyp) - -""" - alias::InterMustAlias - -This lattice element used in a very similar way as `InterConditional`, but corresponds to `MustAlias`. -""" -struct InterMustAlias - slot::Int - vartyp::Any - fldidx::Int - fldtyp::Any - function InterMustAlias(slot::Int, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) - assert_nested_slotwrapper(vartyp) - assert_nested_slotwrapper(fldtyp) - # @assert !isalreadyconst(vartyp) "vartyp is already const" - # @assert !isalreadyconst(fldtyp) "fldtyp is already const" - limited = may_form_limited_typ(vartyp, fldtyp, fldtyp) - limited !== nothing && return limited - return new(slot, vartyp, fldidx, fldtyp) - end -end -InterMustAlias(var::SlotNumber, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) = - InterMustAlias(slot_id(var), vartyp, fldidx, fldtyp) +MustAlias(var::SlotNumber, ssadef::Int, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) = + MustAlias(slot_id(var), ssadef, vartyp, fldidx, fldtyp) const AnyMustAlias = Union{MustAlias,InterMustAlias} -MustAlias(alias::InterMustAlias) = MustAlias(alias.slot, alias.vartyp, alias.fldidx, alias.fldtyp) -InterMustAlias(alias::MustAlias) = InterMustAlias(alias.slot, alias.vartyp, alias.fldidx, alias.fldtyp) +function InterMustAlias(alias::MustAlias) + @assert alias.ssadef == 0 + limited = may_form_limited_typ(alias.vartyp, alias.fldtyp, alias.fldtyp) + limited !== nothing && return limited + InterMustAlias(alias.slot, alias.vartyp, alias.fldidx, alias.fldtyp) +end struct PartialTypeVar tv::TypeVar @@ -145,8 +130,20 @@ end struct StateUpdate var::SlotNumber vtype::VarState - conditional::Bool - StateUpdate(var::SlotNumber, vtype::VarState, conditional::Bool=false) = new(var, vtype, conditional) +end + +""" +Similar to `StateUpdate`, except with the additional guarantee that object identity +is preserved by the update (i.e. `x (before) === x (after)`). +""" +struct StateRefinement + slot::Int + # XXX: This should be an intersection of the old type with the new + # (i.e. newtyp ⊑ oldtyp) + newtyp + undef::Bool + + StateRefinement(slot::Int, @nospecialize(newtyp), undef::Bool) = new(slot, newtyp, undef) end """ @@ -234,13 +231,6 @@ end return typ end -@nospecializeinfer function widenwrappedslotwrapper(@nospecialize typ) - if isa(typ, LimitedAccuracy) - return LimitedAccuracy(widenslotwrapper(typ.typ), typ.causes) - end - return widenslotwrapper(typ) -end - # Conditional # =========== @@ -284,6 +274,7 @@ end return false end +is_same_conditionals(a::Conditional, b::Conditional) = a.slot == b.slot && a.ssadef == b.ssadef is_same_conditionals(a::C, b::C) where C<:AnyConditional = a.slot == b.slot @nospecializeinfer is_lattice_bool(lattice::AbstractLattice, @nospecialize(typ)) = typ !== Bottom && ⊑(lattice, typ, Bool) @@ -331,41 +322,37 @@ end return fldidx end -@nospecializeinfer function form_mustalias_conditional(alias::MustAlias, @nospecialize(thentype), @nospecialize(elsetype)) - (; slot, vartyp, fldidx) = alias +@nospecializeinfer function form_mustalias_refinement(alias::MustAlias, @nospecialize(newtyp)) + newtyp === Union{} && return nothing + (; vartyp, fldidx) = alias if isa(vartyp, PartialStruct) - fields = vartyp.fields - thenfields = thentype === Bottom ? nothing : copy(fields) - elsefields = elsetype === Bottom ? nothing : copy(fields) + fields = copy(vartyp.fields) undefs = copy(_getundefs(vartyp)) if 1 ≤ fldidx ≤ length(fields) - thenfields === nothing || (thenfields[fldidx] = thentype) - elsefields === nothing || (elsefields[fldidx] = elsetype) + fields[fldidx] = newtyp undefs[fldidx] = false end - return Conditional(slot, - thenfields === nothing ? Bottom : PartialStruct(fallback_lattice, vartyp.typ, undefs, thenfields), - elsefields === nothing ? Bottom : PartialStruct(fallback_lattice, vartyp.typ, undefs, elsefields)) + return PartialStruct(fallback_lattice, vartyp.typ, undefs, fields) else vartyp_widened = widenconst(vartyp) - thenfields = thentype === Bottom ? nothing : Any[] - elsefields = elsetype === Bottom ? nothing : Any[] + fields = Any[] for i in 1:fieldcount(vartyp_widened) - if i == fldidx - thenfields === nothing || push!(thenfields, thentype) - elsefields === nothing || push!(elsefields, elsetype) - else - t = fieldtype(vartyp_widened, i) - thenfields === nothing || push!(thenfields, t) - elsefields === nothing || push!(elsefields, t) - end + push!(fields, i == fldidx ? newtyp : fieldtype(vartyp_widened, i)) end - return Conditional(slot, - thenfields === nothing ? Bottom : PartialStruct(fallback_lattice, vartyp_widened, thenfields), - elsefields === nothing ? Bottom : PartialStruct(fallback_lattice, vartyp_widened, elsefields)) + undefs = partialstruct_init_undefs(vartyp_widened, fields) + undefs === nothing && return nothing + return PartialStruct(fallback_lattice, vartyp_widened, undefs, fields) end end +@nospecializeinfer function form_mustalias_conditional(alias::MustAlias, @nospecialize(thentype), @nospecialize(elsetype)) + thentype_r = thentype === Bottom ? Bottom : form_mustalias_refinement(alias, thentype) + elsetype_r = elsetype === Bottom ? Bottom : form_mustalias_refinement(alias, elsetype) + thentype_r === nothing && (thentype_r = Bottom) + elsetype_r === nothing && (elsetype_r = Bottom) + return Conditional(alias.slot, alias.ssadef, thentype_r, elsetype_r) +end + function issubalias(a::AnyMustAlias, b::AnyMustAlias) return a.slot == b.slot && a.fldidx == b.fldidx && a.vartyp ⊑ b.vartyp && a.fldtyp ⊑ b.fldtyp @@ -469,7 +456,8 @@ end return isa(b, Type) && a.typ <: b elseif isa(b, PartialStruct) if isa(a, Const) - widea = widenconst(a)::DataType + widea = widenconst(a) + isa(widea, DataType) || return false wideb = widenconst(b) wideb′ = unwrap_unionall(wideb)::DataType widea.name === wideb′.name || return false @@ -514,6 +502,14 @@ end elseif isa(b, PartialOpaque) return false end + if isa(a, PartialTask) + if isa(b, PartialTask) + return ⊑(lattice, a.fetch_type, b.fetch_type) + end + return ⊑(widenlattice(lattice), Task, b) + elseif isa(b, PartialTask) + return false + end return ⊑(widenlattice(lattice), a, b) end @@ -581,6 +577,11 @@ end return is_lattice_equal(lattice, a.env, b.env) end isa(b, PartialOpaque) && return false + if isa(a, PartialTask) + isa(b, PartialTask) || return false + return is_lattice_equal(lattice, a.fetch_type, b.fetch_type) + end + isa(b, PartialTask) && return false return is_lattice_equal(widenlattice(lattice), a, b) end @@ -609,7 +610,7 @@ end # lattice operations # ================== -@nospecializeinfer function tmeet(lattice::PartialsLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(lattice::PartialsLattice, @nospecialize(v), @nospecialize(t::AnyType)) if isa(v, PartialStruct) has_free_typevars(t) && return v widev = widenconst(v) @@ -643,11 +644,14 @@ end ti = typeintersect(widev, t) valid_as_lattice(ti, true) || return Bottom return PartialOpaque(ti, v.env, v.parent, v.source) + elseif isa(v, PartialTask) + has_free_typevars(t) && return v + return Task <: t ? v : Bottom end return tmeet(widenlattice(lattice), v, t) end -@nospecializeinfer function tmeet(lattice::ConstsLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(lattice::ConstsLattice, @nospecialize(v), @nospecialize(t::AnyType)) if isa(v, Const) if !has_free_typevars(t) && !isa(v.val, t) return Bottom @@ -657,7 +661,7 @@ end tmeet(widenlattice(lattice), widenconst(v), t) end -@nospecializeinfer function tmeet(lattice::ConditionalsLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(lattice::ConditionalsLattice, @nospecialize(v), @nospecialize(t::AnyType)) if isa(v, Conditional) if !(Bool <: t) return Bottom @@ -667,26 +671,26 @@ end tmeet(widenlattice(lattice), v, t) end -@nospecializeinfer function tmeet(𝕃::MustAliasesLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(𝕃::MustAliasesLattice, @nospecialize(v), @nospecialize(t::AnyType)) if isa(v, MustAlias) v = widenmustalias(v) end return tmeet(widenlattice(𝕃), v, t) end -@nospecializeinfer function tmeet(lattice::InferenceLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(lattice::InferenceLattice, @nospecialize(v), @nospecialize(t::AnyType)) # TODO: This can probably happen and should be handled @assert !isa(v, LimitedAccuracy) tmeet(widenlattice(lattice), v, t) end -@nospecializeinfer function tmeet(lattice::InterConditionalsLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(lattice::InterConditionalsLattice, @nospecialize(v), @nospecialize(t::AnyType)) # TODO: This can probably happen and should be handled @assert !isa(v, AnyConditional) tmeet(widenlattice(lattice), v, t) end -@nospecializeinfer function tmeet(𝕃::InterMustAliasesLattice, @nospecialize(v), @nospecialize(t::Type)) +@nospecializeinfer function tmeet(𝕃::InterMustAliasesLattice, @nospecialize(v), @nospecialize(t::AnyType)) if isa(v, InterMustAlias) v = widenmustalias(v) end @@ -700,11 +704,16 @@ Widens extended lattice element `x` to native `Type` representation. """ widenconst(::AnyConditional) = Bool widenconst(a::AnyMustAlias) = widenconst(widenmustalias(a)) -widenconst(c::Const) = (v = c.val; isa(v, Type) ? Type{v} : typeof(v)) +# a closed type value widens to the egality kind, mirroring how `jl_inst_arg_tuple_type` +# keys runtime dispatch (`Const(v) ⊑ TypeEgal{v} ⊑ Type{v}`); an open one only to its +# `==`-class `Type{v}` +widenconst(c::Const) = (v = c.val; isa(v, Type) ? + (has_free_typevars(v) ? Type{v} : Core.TypeEgal{v}) : typeof(v)) widenconst(::PartialTypeVar) = TypeVar widenconst(t::Core.PartialStruct) = t.typ widenconst(t::PartialOpaque) = t.typ -@nospecializeinfer widenconst(@nospecialize t::Type) = t +widenconst(t::PartialTask) = Task +@nospecializeinfer widenconst(@nospecialize t::AnyType) = t widenconst(::TypeVar) = error("unhandled TypeVar") widenconst(::TypeofVararg) = error("unhandled Vararg") widenconst(::LimitedAccuracy) = error("unhandled LimitedAccuracy") @@ -713,34 +722,28 @@ widenconst(::LimitedAccuracy) = error("unhandled LimitedAccuracy") # state management # #################### -function smerge(lattice::AbstractLattice, sa::Union{NotFound,VarState}, sb::Union{NotFound,VarState}) +function smerge(lattice::AbstractLattice, sa::Union{NotFound,VarState}, sb::Union{NotFound,VarState}, join_pc::Int) sa === sb && return sa sa === NOT_FOUND && return sb sb === NOT_FOUND && return sa - return VarState(tmerge(lattice, sa.typ, sb.typ), sa.undef | sb.undef) + return VarState(tmerge(lattice, sa.typ, sb.typ), sa.ssadef == sb.ssadef ? sa.ssadef : join_pc, sa.undef | sb.undef) end -@nospecializeinfer @inline schanged(lattice::AbstractLattice, @nospecialize(n), @nospecialize(o)) = - (n !== o) && (o === NOT_FOUND || (n !== NOT_FOUND && !(n.undef <= o.undef && ⊑(lattice, n.typ, o.typ)))) - -# remove any lattice elements that wrap the reassigned slot object from the vartable -function invalidate_slotwrapper(vt::VarState, changeid::Int, ignore_conditional::Bool) - newtyp = ignorelimited(vt.typ) - if (!ignore_conditional && isa(newtyp, Conditional) && newtyp.slot == changeid) || - (isa(newtyp, MustAlias) && newtyp.slot == changeid) - newtyp = @noinline widenwrappedslotwrapper(vt.typ) - return VarState(newtyp, vt.undef) - end - return nothing -end +@nospecializeinfer @inline schanged(lattice::AbstractLattice, @nospecialize(n), @nospecialize(o), join_pc::Int) = + (n !== o) && (o === NOT_FOUND || (n !== NOT_FOUND && !(n.undef <= o.undef && (n.ssadef === o.ssadef || o.ssadef === join_pc) && ⊑(lattice, n.typ, o.typ)))) -function stupdate!(lattice::AbstractLattice, state::VarTable, changes::VarTable) +function stupdate!(lattice::AbstractLattice, state::VarTable, changes::VarTable, join_pc::Int) changed = false for i = 1:length(state) newtype = changes[i] oldtype = state[i] - if schanged(lattice, newtype, oldtype) - state[i] = smerge(lattice, oldtype, newtype) + # In addition to computing the type, the merge here computes the "reaching definition" + # for a slot. The provided `join_pc` is a "virtual" PC, which corresponds to the ϕ-block + # that would exist at the beginning of the BasicBlock. + # + # This effectively applies the "path-convergence criterion" for SSA construction. + if schanged(lattice, newtype, oldtype, join_pc) + state[i] = smerge(lattice, oldtype, newtype, join_pc) changed = true end end @@ -755,16 +758,17 @@ function stoverwrite!(state::VarTable, newstate::VarTable) end function stoverwrite1!(state::VarTable, change::StateUpdate) - changeid = slot_id(change.var) - for i = 1:length(state) - invalidated = invalidate_slotwrapper(state[i], changeid, change.conditional) - if invalidated !== nothing - state[i] = invalidated - end - end - # and update the type of it - newtype = change.vtype - state[changeid] = newtype + # Note: We no longer need to invalidate Conditional/MustAlias in other slots + # that reference this slot. The ssadef tracking handles this: when a slot is + # reassigned, its ssadef changes, and any Conditional/MustAlias referencing + # the old ssadef will be detected as stale by conditional_valid(). + state[slot_id(change.var)] = change.vtype + return state +end + +function strefine1!(state::VarTable, refinement::StateRefinement) + (; newtyp, undef, slot) = refinement + state[slot] = VarState(newtyp, state[slot].ssadef, undef) return state end @@ -773,13 +777,13 @@ end # type. # Legacy constructor -function Core.PartialStruct(𝕃::AbstractLattice, @nospecialize(typ), fields::Vector{Any}) +function Core.PartialStruct(𝕃::AbstractLattice, @nospecialize(typ::Type), fields::Vector{Any}) undefs = partialstruct_init_undefs(typ, fields) undefs === nothing && error("This object never exists at runtime") return PartialStruct(𝕃, typ, undefs, fields) end -function Core.PartialStruct(::AbstractLattice, @nospecialize(typ), undefs::Vector{Union{Nothing,Bool}}, fields::Vector{Any}) +function Core.PartialStruct(::AbstractLattice, @nospecialize(typ::Type), undefs::Vector{Union{Nothing,Bool}}, fields::Vector{Any}) for i = 1:length(fields) assert_nested_slotwrapper(fields[i]) end diff --git a/Compiler/src/typelimits.jl b/Compiler/src/typelimits.jl index af5e7964be0a6..30a4c414d81a5 100644 --- a/Compiler/src/typelimits.jl +++ b/Compiler/src/typelimits.jl @@ -27,7 +27,7 @@ function limit_type_size(@nospecialize(t), @nospecialize(compare), @nospecialize t <: r || (r = Any) # final escape hatch end #@assert r === _limit_type_size(r, t, source) # this monotonicity constraint is slightly stronger than actually required, - # since we only actually need to demonstrate that repeated application would reaches a fixed point, + # since we only actually need to demonstrate that repeated application would reach a fixed point, #not that it is already at the fixed point return r end @@ -53,6 +53,8 @@ function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int) # see if it is derived from the body # also handle the var here, since this construct bounds the mindepth to the smallest possible value return is_derived_type(t, c.var.ub, mindepth) || is_derived_type(t, c.body, mindepth) + elseif isType(c) + return is_derived_type(t, type_parameter(c), mindepth) elseif isa(c, DataType) if mindepth > 0 mindepth -= 1 @@ -84,7 +86,7 @@ end # The goal of this function is to return a type of greater "size" and less "complexity" than # both `t` or `c` over the lattice defined by `sources`, `depth`, and `allowed_tuplelen`. function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVector, depth::Int, allowed_tuplelen::Int) - @assert isa(t, Type) && isa(c, Type) "unhandled TypeVar / Vararg" + @assert isa(t, AnyType) && isa(c, AnyType) "unhandled TypeVar / Vararg" if t === c return t # quick egal test elseif t === Union{} @@ -103,45 +105,46 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec # then unwrap `t` # NOTE that `TypeVar` / `Vararg` are handled separately to catch the logic errors if isa(c, UnionAll) - return __limit_type_size(t, c.body, sources, depth, allowed_tuplelen)::Type + return __limit_type_size(t, c.body, sources, depth, allowed_tuplelen)::AnyType end if isa(t, UnionAll) tbody = __limit_type_size(t.body, c, sources, depth, allowed_tuplelen) tbody === t.body && return t - return UnionAll(t.var, tbody)::Type + return UnionAll(t.var, tbody)::AnyType elseif isa(t, Union) if isa(c, Union) a = __limit_type_size(t.a, c.a, sources, depth, allowed_tuplelen) b = __limit_type_size(t.b, c.b, sources, depth, allowed_tuplelen) return Union{a, b} end + elseif isType(t) + # Type is fairly important, so do not widen it as fast as other types if avoidable + # (this branch also covers `TypeEgal`, whose `Type{...}` widenings are supertypes) + tt = type_parameter(t) + ttu = unwrap_unionall(tt) # TODO: use a helper that preserves nested Type structure after #50692 is fixed + # must forbid nesting through this if we detect that potentially occurring + # we already know !is_derived_type_from_any so refuse to recurse here + if isType(ttu) + return Type{<:Type} + elseif !isa(ttu, DataType) + return Type + end + # try to peek into c to get a comparison object, but if we can't perhaps t is already simple enough on its own + if isType(c) + ct = type_parameter(c) + else + ct = Union{} + end + Qt = __limit_type_size(tt, ct, sources, depth + 1, 0) + Qt === tt && return t + Qt === Any && return Type + # Can't form Type{<:Qt} just yet, without first make sure we limited the depth + # enough, since this moves Qt outside of Type for is_derived_type_from_any + Qt = __limit_type_size(tt, ct, sources, depth + 2, 0) + Qt === Any && return Type + return Type{<:Qt} elseif isa(t, DataType) - if isType(t) - # Type is fairly important, so do not widen it as fast as other types if avoidable - tt = t.parameters[1] - ttu = unwrap_unionall(tt) # TODO: use argument_datatype(tt) after #50692 fixed - # must forbid nesting through this if we detect that potentially occurring - # we already know !is_derived_type_from_any so refuse to recurse here - if !isa(ttu, DataType) - return Type - elseif isType(ttu) - return Type{<:Type} - end - # try to peek into c to get a comparison object, but if we can't perhaps t is already simple enough on its own - if isType(c) - ct = c.parameters[1] - else - ct = Union{} - end - Qt = __limit_type_size(tt, ct, sources, depth + 1, 0) - Qt === tt && return t - Qt === Any && return Type - # Can't form Type{<:Qt} just yet, without first make sure we limited the depth - # enough, since this moves Qt outside of Type for is_derived_type_from_any - Qt = __limit_type_size(tt, ct, sources, depth + 2, 0) - Qt === Any && return Type - return Type{<:Qt} - elseif isa(c, DataType) + if isa(c, DataType) tP = t.parameters cP = c.parameters if t.name === c.name && !isempty(cP) @@ -190,6 +193,8 @@ end # helper function of `_limit_type_size`, which has the right to take and return `TypeVar` / `Vararg` function __limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVector, depth::Int, allowed_tuplelen::Int) + isa(t, SimpleVector) && (t = t[1]) + isa(c, SimpleVector) && (c = c[1]) cN = 0 if isvarargtype(c) # Tuple{Vararg{T}} --> Tuple{T} is OK isdefined(c, :N) && (cN = c.N) @@ -263,27 +268,28 @@ function type_more_complex(@nospecialize(t), @nospecialize(c), sources::SimpleVe elseif isa(t, Int) && isa(c, Int) return t !== 1 && !(0 <= t < c) # alternatively, could use !(abs(t) <= abs(c) || abs(t) < n) for some n end + if isType(t) + # Type is fairly important, so do not widen it as fast as other types if avoidable + tt = type_parameter(t) + # ttu = unwrap_unionall(tt) # TODO: use a helper that preserves nested Type structure after #50692 is fixed + if isType(c) + ct = type_parameter(c) + else + ct = Union{} + tupledepth == 0 && return true # cannot allow nesting + end + # allow creating variation within a nested Type, but not very deep + if tupledepth > 1 + tupledepth = 1 + else + tupledepth = 0 + end + return type_more_complex(tt, ct, sources, depth + 1, tupledepth, 0) + end # base case for data types if isa(t, DataType) tP = t.parameters - if isType(t) - # Type is fairly important, so do not widen it as fast as other types if avoidable - tt = tP[1] - ttu = unwrap_unionall(tt) # TODO: use argument_datatype(tt) after #50692 fixed - if isType(c) - ct = c.parameters[1] - else - ct = Union{} - tupledepth == 0 && return true # cannot allow nesting - end - # allow creating variation within a nested Type, but not very deep - if tupledepth > 1 - tupledepth = 1 - else - tupledepth = 0 - end - return type_more_complex(tt, ct, sources, depth + 1, tupledepth, 0) - elseif isa(c, DataType) && t.name === c.name + if isa(c, DataType) && t.name === c.name cP = c.parameters length(cP) < length(tP) && return true isempty(tP) && return false @@ -338,6 +344,7 @@ end typea === typeb && return true if typea isa PartialStruct aty = widenconst(typea) + issimplertype(𝕃, aty, widenconst(typeb)) || return false if typeb isa Const || typeb isa PartialStruct @assert n_initialized(typea) ≤ n_initialized(typeb) "typeb ⊑ typea is assumed" elseif typeb isa PartialStruct @@ -396,6 +403,9 @@ end return false end return false + elseif typea isa PartialTask + typeb isa PartialTask || return false + return issimplertype(𝕃, typea.fetch_type, typeb.fetch_type) end return true end @@ -500,16 +510,16 @@ end # type-lattice for Conditional wrapper (NOTE never be merged with InterConditional) if isa(typea, Conditional) && isa(typeb, Const) if typeb.val === true - typeb = Conditional(typea.slot, Any, Union{}) + typeb = Conditional(typea.slot, typea.ssadef, Any, Union{}) elseif typeb.val === false - typeb = Conditional(typea.slot, Union{}, Any) + typeb = Conditional(typea.slot, typea.ssadef, Union{}, Any) end end if isa(typeb, Conditional) && isa(typea, Const) if typea.val === true - typea = Conditional(typeb.slot, Any, Union{}) + typea = Conditional(typeb.slot, typeb.ssadef, Any, Union{}) elseif typea.val === false - typea = Conditional(typeb.slot, Union{}, Any) + typea = Conditional(typeb.slot, typeb.ssadef, Union{}, Any) end end if isa(typea, Conditional) && isa(typeb, Conditional) @@ -517,7 +527,7 @@ end thentype = tmerge(widenlattice(lattice), typea.thentype, typeb.thentype) elsetype = tmerge(widenlattice(lattice), typea.elsetype, typeb.elsetype) if thentype !== elsetype - return Conditional(typea.slot, thentype, elsetype) + return Conditional(typea.slot, typea.ssadef, thentype, elsetype) end end val = maybe_extract_const_bool(typea) @@ -718,6 +728,21 @@ end typeb = widenlattice(wl, typeb) end + # type-lattice for PartialTask wrapper + apt = isa(typea, PartialTask) + bpt = isa(typeb, PartialTask) + if apt && bpt + # Both are PartialTask - merge their fetch types + merged_fetch_type = tmerge(lattice, typea.fetch_type, typeb.fetch_type) + # Any carries no additional type information - return Task + merged_fetch_type === Any && return Task + return PartialTask(merged_fetch_type) + elseif apt + typea = Task + elseif bpt + typeb = Task + end + return tmerge(wl, typea, typeb) end @@ -733,7 +758,7 @@ end return tmerge(wl, typea, typeb) end -@nospecializeinfer function tmerge(lattice::JLTypeLattice, @nospecialize(typea::Type), @nospecialize(typeb::Type)) +@nospecializeinfer function tmerge(lattice::JLTypeLattice, @nospecialize(typea::AnyType), @nospecialize(typeb::AnyType)) # it's always ok to form a Union of two concrete types act = isconcretetype(typea) bct = isconcretetype(typeb) @@ -759,43 +784,51 @@ end heighta = 0 while a !== Any heighta += 1 - a = a.super + a = datatype_super(a) end b = unwrap_unionall(bname.wrapper) heightb = 0 while b !== Any b.name === aname && return aname heightb += 1 - b = b.super + b = datatype_super(b) end a = unwrap_unionall(aname.wrapper) while heighta > heightb - a = a.super + a = datatype_super(a) heighta -= 1 end return a.name === bname ? bname : nothing end -@nospecializeinfer @noinline function tmerge_types_slow(@nospecialize(typea::Type), @nospecialize(typeb::Type)) +@nospecializeinfer @noinline function tmerge_types_slow(@nospecialize(typea::AnyType), @nospecialize(typeb::AnyType)) # collect the list of types from past tmerge calls returning Union # and then reduce over that list types = Any[] _uniontypes(typea, types) _uniontypes(typeb, types) typenames = Vector{Core.TypeName}(undef, length(types)) + all_datatypes = true for i in 1:length(types) # check that we will be able to analyze (and simplify) everything - # bail if everything isn't a well-formed DataType + # bail if everything isn't a well-formed nominal kind ti = types[i] uw = unwrap_unionall(ti) - uw isa DataType || return Any - ti <: uw.name.wrapper || return Any - typenames[i] = uw.name + if uw isa DataType + ti <: uw.name.wrapper || return Any + typenames[i] = uw.name + elseif isType(uw) + typenames[i] = TypeEq.name + all_datatypes = false + else + return Any + end end u = Union{types...} if issimpleenoughtype(u) return u end + all_datatypes || return Any # see if any of the union elements have the same TypeName # in which case, simplify this tmerge by replacing it with # the widest possible version of itself (the wrapper) @@ -832,11 +865,11 @@ end uw = unwrap_unionall(wr)::DataType ui = unwrap_unionall(ti)::DataType while ui.name !== ijname - ui = ui.super + ui = datatype_super(ui) end uj = unwrap_unionall(tj)::DataType while uj.name !== ijname - uj = uj.super + uj = datatype_super(uj) end p = Vector{Any}(undef, length(uw.parameters)) usep = true diff --git a/Compiler/src/types.jl b/Compiler/src/types.jl index 7bbd59b742a93..b733688691b88 100644 --- a/Compiler/src/types.jl +++ b/Compiler/src/types.jl @@ -1,5 +1,4 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# const WorkThunk = Any # #@eval struct WorkThunk @@ -8,6 +7,12 @@ const WorkThunk = Any # end # (p::WorkThunk)() = p.thunk() +# This corresponds to the type of `CodeInfo`'s `inlining_cost` field +const InlineCostType = UInt16 +const MAX_INLINE_COST = typemax(InlineCostType) +const MIN_INLINE_COST = InlineCostType(10) +const MaybeCompressed = Union{CodeInfo, String} + """ AbstractInterpreter @@ -65,14 +70,23 @@ SpecInfo(src::CodeInfo) = SpecInfo( A special wrapper that represents a local variable of a method being analyzed. This does not participate in the native type system nor the inference lattice, and it thus should be always unwrapped to `v.typ` when performing any type or lattice operations on it. -`v.undef` represents undefined-ness of this static parameter. If `true`, it means that the + +`v.undef` represents undefined-ness of this local variable. If `true`, it means that the variable _may_ be undefined at runtime, otherwise it is guaranteed to be defined. If `v.typ === Bottom` it means that the variable is strictly undefined. + +`v.ssadef` represents the "reaching definition" for the variable. +If zero, then the value comes from an argument. +If negative, this refers to a "virtual ϕ-block" preceding the given index, +that would have been inserted as the value of this slot in a truly SSA-form IR. +If a slot has the same `ssadef` at two different points of execution, +the slot contents are guaranteed to share identity (`x₀ === x₁`). """ struct VarState typ + ssadef::Int undef::Bool - VarState(@nospecialize(typ), undef::Bool) = new(typ, undef) + VarState(@nospecialize(typ), ssadef::Int, undef::Bool) = new(typ, ssadef, undef) end struct AnalysisResults @@ -85,6 +99,9 @@ struct AnalysisResults end const NULL_ANALYSIS_RESULTS = AnalysisResults(nothing) +# Abstract type for completed call inference results that can be stored in CallInfo +abstract type InferredCallResult end + """ result::InferenceResult @@ -109,21 +126,92 @@ mutable struct InferenceResult ipo_effects::Effects # if inference is finished effects::Effects # if optimization is finished analysis_results::AnalysisResults # AnalysisResults with e.g. result::ArgEscapeCache if optimized, otherwise NULL_ANALYSIS_RESULTS - is_src_volatile::Bool # `src` has been cached globally as the compressed format already, allowing `src` to be used destructively + cache_world::UInt # exact inference world for session-local reuse, or zero tombstone::Bool + replacement_ci::Union{Nothing,CodeInstance} # global publication winner selected before optimizer exposure #=== uninitialized fields ===# - ci::CodeInstance # CodeInstance if this result may be added to the cache - ci_as_edge::CodeInstance # CodeInstance as the edge representing locally cached result + ci::CodeInstance # CodeInstance that will contain the result in full function InferenceResult(mi::MethodInstance, argtypes::Vector{Any}, overridden_by_const::Union{Nothing,BitVector}) result = exc_result = src = nothing valid_worlds = WorldRange() ipo_effects = effects = Effects() analysis_results = NULL_ANALYSIS_RESULTS return new(mi, argtypes, overridden_by_const, result, exc_result, src, - valid_worlds, ipo_effects, effects, analysis_results, #=is_src_volatile=#false, false) + valid_worlds, ipo_effects, effects, analysis_results, zero(UInt), + false, nothing) + end +end + +""" + LocalInferenceProof + +A dependency certificate for inference work that is not represented by a published +[`CodeInstance`](@ref). `edges` is an internal forward-edge stream whose entries may +include other `LocalInferenceProof`s. Keeping those references as a shared graph avoids +repeatedly copying the transitive dependency closure; the graph is flattened only when a +result is published as a `CodeInstance`. The type is mutable only to give each proof a +stable identity; both fields are immutable. +""" +mutable struct LocalInferenceProof + const valid_worlds::WorldRange + const edges::SimpleVector + function LocalInferenceProof(valid_worlds::WorldRange, edges::SimpleVector) + # Local proofs are not registered with the global invalidation machinery. + # Inference states cap their range at the current world counter, and exact-world + # local-cache reuse relies on that cap to keep an unregistered proof from + # claiming validity in a future world. + @assert first(valid_worlds) <= last(valid_worlds) <= get_world_counter() + for edge in edges + edge_worlds = if edge isa LocalInferenceProof + edge.valid_worlds + elseif edge isa CodeInstance && edge.min_world <= edge.max_world + # Skip a provisional CI (`min_world > max_world`); an SCC fills it + # before any proof containing it can escape. + WorldRange(edge.min_world, edge.max_world) + else + continue + end + @assert (first(edge_worlds) <= first(valid_worlds) && + last(valid_worlds) <= last(edge_worlds)) + end + return new(valid_worlds, edges) end end + +const InferenceProof = Union{CodeInstance,LocalInferenceProof} + +proof_worlds(proof::CodeInstance) = WorldRange(proof.min_world, proof.max_world) +proof_worlds(proof::LocalInferenceProof) = proof.valid_worlds + +""" + LocalInferenceResult + +The completed, reusable view of an [`InferenceResult`](@ref). `result` owns the inferred +source and facts, while `proof` certifies them. The executable edge for a particular call +is deliberately stored separately in its `CallInfo`. +""" +struct LocalInferenceResult <: InferredCallResult + result::InferenceResult + proof::InferenceProof + function LocalInferenceResult(result::InferenceResult, proof::InferenceProof, world::UInt) + @assert result.result !== nothing "cannot complete an unfinished InferenceResult" + @assert !result.tombstone "cannot complete a tombstoned InferenceResult" + @assert world in proof_worlds(proof) "inference proof does not cover its cache world" + @assert iszero(result.cache_world) || result.cache_world == world + result.cache_world = world + if proof isa CodeInstance + @assert proof.def === result.linfo "CodeInstance proof does not match InferenceResult" + else + @assert first(proof.valid_worlds) <= last(proof.valid_worlds) + end + return new(result, proof) + end +end + +inference_proof(result::LocalInferenceResult) = result.proof + +const InferenceCacheEntry = Union{InferenceResult,LocalInferenceResult} function InferenceResult(mi::MethodInstance, 𝕃::AbstractLattice=fallback_lattice) argtypes = matching_cache_argtypes(𝕃, mi) return InferenceResult(mi, argtypes, #=overridden_by_const=#nothing) @@ -144,6 +232,52 @@ function traverse_analysis_results(callback, (;analysis_results)::Union{Inferenc return nothing end +""" + InferenceCache + +A cache for local inference work that maintains an index for fast lookups by +`MethodInstance`. Successful entries are stored as [`LocalInferenceResult`](@ref), while +raw `InferenceResult` entries are retained only as internal unresolved-cycle or tombstone +markers for constant propagation. Entries are reused only at their exact `cache_world`; +their static world range is not itself registered for invalidation while they remain local. +""" +struct InferenceCache + results::Vector{InferenceCacheEntry} + # Index from MethodInstance to indices in `results` where linfo === mi + index::IdDict{MethodInstance, Vector{Int}} +end + +InferenceCache() = InferenceCache(Vector{InferenceCacheEntry}(), IdDict{MethodInstance, Vector{Int}}()) + +function Base.push!(cache::InferenceCache, entry::InferenceCacheEntry) + if entry isa InferenceResult + @assert (entry.result === nothing || entry.tombstone) "completed InferenceResult must be wrapped before caching" + end + push!(cache.results, entry) + result = entry isa LocalInferenceResult ? entry.result : entry + mi = result.linfo + idx = length(cache.results) + if haskey(cache.index, mi) + push!(cache.index[mi], idx) + else + cache.index[mi] = Int[idx] + end + return cache +end + +Base.length(cache::InferenceCache) = length(cache.results) +Base.isempty(cache::InferenceCache) = isempty(cache.results) + +Base.iterate(cache::InferenceCache) = iterate(cache.results) +Base.iterate(cache::InferenceCache, state) = iterate(cache.results, state) +Base.eltype(::Type{InferenceCache}) = InferenceCacheEntry +Base.getindex(cache::InferenceCache, i::Int) = cache.results[i] + +# Get indices for a specific MethodInstance (returns empty vector if not found) +function get_indices(cache::InferenceCache, mi::MethodInstance) + return get(cache.index, mi, Int[]) +end + """ inf_params::InferenceParams @@ -207,6 +341,7 @@ struct InferenceParams assume_bindings_static::Bool ignore_recursion_hardlimit::Bool force_enable_inference::Bool + cache_owner::Any function InferenceParams( max_methods::Int, @@ -219,6 +354,7 @@ struct InferenceParams assume_bindings_static::Bool, ignore_recursion_hardlimit::Bool, force_enable_inference::Bool, + @nospecialize(cache_owner), ) return new( max_methods, @@ -231,6 +367,7 @@ struct InferenceParams assume_bindings_static, ignore_recursion_hardlimit, force_enable_inference, + cache_owner, ) end end @@ -245,7 +382,8 @@ function InferenceParams( #=aggressive_constant_propagation::Bool=# false, #=assume_bindings_static::Bool=# false, #=ignore_recursion_hardlimit::Bool=# false, - #=force_enable_inference::Bool=# false + #=force_enable_inference::Bool=# false, + #=cache_owner=# nothing ); max_methods::Int = params.max_methods, max_union_splitting::Int = params.max_union_splitting, @@ -257,6 +395,7 @@ function InferenceParams( assume_bindings_static::Bool = params.assume_bindings_static, ignore_recursion_hardlimit::Bool = params.ignore_recursion_hardlimit, force_enable_inference::Bool = params.force_enable_inference, + cache_owner = params.cache_owner, ) return InferenceParams( max_methods, @@ -269,6 +408,7 @@ function InferenceParams( assume_bindings_static, ignore_recursion_hardlimit, force_enable_inference, + cache_owner, ) end @@ -385,7 +525,7 @@ struct NativeInterpreter <: AbstractInterpreter method_table::CachedMethodTable{InternalMethodTable} # Cache of inference results for this particular interpreter - inf_cache::Vector{InferenceResult} + inf_cache::InferenceCache codegen::IdDict{CodeInstance,CodeInfo} # Parameters for inference and optimization @@ -406,7 +546,7 @@ function NativeInterpreter(world::UInt = get_world_counter(); # incorrect, fail out loudly. @assert world <= curr_max_world method_table = CachedMethodTable(InternalMethodTable(world)) - inf_cache = Vector{InferenceResult}() # Initially empty cache + inf_cache = InferenceCache() # Initially empty cache codegen = IdDict{CodeInstance,CodeInfo}() return NativeInterpreter(world, method_table, inf_cache, codegen, inf_params, opt_params) end @@ -416,7 +556,7 @@ InferenceParams(interp::NativeInterpreter) = interp.inf_params OptimizationParams(interp::NativeInterpreter) = interp.opt_params get_inference_world(interp::NativeInterpreter) = interp.world get_inference_cache(interp::NativeInterpreter) = interp.inf_cache -cache_owner(interp::NativeInterpreter) = nothing +cache_owner(interp::NativeInterpreter) = interp.inf_params.cache_owner engine_reserve(interp::AbstractInterpreter, mi::MethodInstance) = engine_reserve(mi, cache_owner(interp)) engine_reserve(mi::MethodInstance, @nospecialize owner) = ccall(:jl_engine_reserve, Any, (Any, Any), mi, owner)::CodeInstance @@ -439,6 +579,11 @@ function add_remark! end may_optimize(::AbstractInterpreter) = true may_compress(::AbstractInterpreter) = true may_discard_trees(::AbstractInterpreter) = true +may_discard_trees(::NativeInterpreter) = + ccall(:jl_get_type_infer_preserve_ir, Int8, ()) == 0 +precompile_keep_ir(::AbstractInterpreter) = false +precompile_keep_ir(::NativeInterpreter) = + ccall(:jl_get_precompile_keep_ir, Int8, ()) != 0 """ method_table(interp::AbstractInterpreter)::MethodTableView @@ -460,7 +605,7 @@ and may be safely discarded between calls to this function. By default, a value of `nothing` is returned indicating that `CodeInstance`s should not be added to the JIT. Attempting to execute them via `invoke` will result in an error. """ -codegen_cache(interp::AbstractInterpreter) = nothing +codegen_cache(::AbstractInterpreter) = nothing codegen_cache(interp::NativeInterpreter) = interp.codegen """ @@ -477,6 +622,19 @@ but `AbstractInterpreter` doesn't provide a specific interface for configuring i """ function bail_out_toplevel_call end, function bail_out_call end, function bail_out_apply end +""" + widen_call_result(interp::AbstractInterpreter, si::StmtInfo, state::CallInferenceState, + sv::AbsIntState) -> Bool + +Decide whether to widen `state.rettype` of the currently-inferred call to `Any` before +returning the result to the enclosing frame. By default this returns +`call_result_unused(si) && !(state.rettype === Bottom)`: when the call has no SSA consumer, +precise return type information is locally useless, so widening lets downstream `=== Any` +short-circuits (e.g. the cycle backedge revisit filter in `update_cycle_worklists!`) elide +redundant work; `Bottom` is preserved so that always-throw behavior remains observable. +""" +function widen_call_result end + """ infer_compilation_signature(::AbstractInterpreter)::Bool @@ -492,10 +650,52 @@ typeinf_lattice(::AbstractInterpreter) = InferenceLattice(BaseInferenceLattice.i ipo_lattice(::AbstractInterpreter) = InferenceLattice(IPOResultLattice.instance) optimizer_lattice(::AbstractInterpreter) = SimpleInferenceLattice.instance +""" + OverlayCodeCache(globalcache, localcache) + +Compatibility wrapper for custom interpreters that keep an executable cache and an +inference cache together. Executable lookup delegates exclusively to `globalcache`; +completed local inference results are accessed through `get_inference_cache` instead. +""" +struct OverlayCodeCache{Cache} + globalcache::Cache + localcache::InferenceCache +end + +function setindex!(cache::OverlayCodeCache, ci::CodeInstance, mi::MethodInstance) + setindex!(cache.globalcache, ci, mi) + return cache +end +haskey(cache::OverlayCodeCache, mi::MethodInstance) = haskey(cache.globalcache, mi) +get(cache::OverlayCodeCache, mi::MethodInstance, default) = + get(cache.globalcache, mi, default) +getindex(cache::OverlayCodeCache, mi::MethodInstance) = getindex(cache.globalcache, mi) + +code_cache(interp::AbstractInterpreter, #=extended_range=#::WorldRange) = code_cache(interp) + function code_cache(interp::AbstractInterpreter) - cache = InternalCodeCache(cache_owner(interp)) - worlds = WorldRange(get_inference_world(interp)) - return WorldView(cache, worlds) + return InternalCodeCache(cache_owner(interp), get_inference_world(interp)) +end + +function code_cache(interp::NativeInterpreter, extended_range::WorldRange) + @assert get_inference_world(interp) in extended_range + return InternalCodeCache(cache_owner(interp), extended_range) +end + +function lookup_local_inference_result(interp::AbstractInterpreter, mi::MethodInstance) + cache = get_inference_cache(interp) + indices = get_indices(cache, mi) + world = get_inference_world(interp) + for i in length(indices):-1:1 + cached = cache.results[indices[i]] + cached isa LocalInferenceResult || continue + result = cached.result + result.overridden_by_const === nothing || continue + result.cache_world == world || continue + world in proof_worlds(cached.proof) || continue + return cached + end + return nothing end get_escape_cache(interp::AbstractInterpreter) = GetNativeEscapeCache(interp) @@ -513,7 +713,8 @@ function add_edges!(edges::Vector{Any}, info::CallInfo) end nsplit(info::CallInfo) = nsplit_impl(info)::Union{Nothing,Int} getsplit(info::CallInfo, idx::Int) = getsplit_impl(info, idx)::MethodLookupResult -getresult(info::CallInfo, idx::Int) = getresult_impl(info, idx)#=::Union{Nothing,ConstResult}=# +getresult(info::CallInfo, idx::Int) = getresult_impl(info, idx)#=::Union{Nothing,InferredCallResult}=# +getedge(info::CallInfo, idx::Int) = getedge_impl(info, idx)::Union{Nothing,CodeInstance} add_edges_impl(::Vector{Any}, ::CallInfo) = error(""" All `CallInfo` is required to implement `add_edges_impl(::Vector{Any}, ::CallInfo)`""") @@ -522,5 +723,6 @@ getsplit_impl(::CallInfo, ::Int) = error(""" A `info::CallInfo` that implements `nsplit_impl(info::CallInfo)::Int` must implement `getsplit_impl(info::CallInfo, idx::Int)::MethodLookupResult` in order to correctly opt in to inlining""") getresult_impl(::CallInfo, ::Int) = nothing +getedge_impl(::CallInfo, ::Int) = nothing @specialize diff --git a/Compiler/src/typeutils.jl b/Compiler/src/typeutils.jl index cf1f92967166e..7c345e15db984 100644 --- a/Compiler/src/typeutils.jl +++ b/Compiler/src/typeutils.jl @@ -4,24 +4,12 @@ # lattice utilities # ##################### -# true if Type{T} is inlineable as constant T -# requires that T is a singleton, s.t. T == S implies T === S -isconstType(@nospecialize t) = isType(t) && hasuniquerep(t.parameters[1]) - -# test whether type T has a unique representation, s.t. T == S implies T === S -function hasuniquerep(@nospecialize t) - # typeof(Bottom) is special since even though it is a leaftype, - # at runtime, it might be Type{Union{}} instead, so don't attempt inference of it - t === typeof(Union{}) && return false - t === Union{} && return true - isa(t, TypeVar) && return false # TypeVars are identified by address, not equality - iskindtype(typeof(t)) || return true # non-types are always compared by egal in the type system - isconcretetype(t) && return true # these are also interned and pointer comparable - if isa(t, DataType) && t.name !== Tuple.name && !isvarargtype(t) # invariant DataTypes - return all(hasuniquerep, t.parameters) - end - return false -end +# true if a value of this type is known to be exactly (`===`) the type `T`, so it +# is inlineable as the constant `T`. A general `Type{T}` is not: it also matches +# `S == T` with `S !== T` reps, e.g. `Union{U,V} where {U<:T,V<:T}` (#61323). +# Use `Core.TypeEgal{T}` when that exactness is required; `Type{Union{}}` is the +# exception, since the bottom object is unique. +isconstType(@nospecialize t) = isTypeEgal(t) || (isTypeEq(t) && type_parameter(t) === Union{}) """ isTypeDataType(@nospecialize t)::Bool @@ -32,8 +20,8 @@ we have `isa(S, DataType)`. In particular, if a statement is typed as `Type{t}` will be a `DataType` at runtime (and not e.g. a `Union` or `UnionAll` typeequal to it). """ function isTypeDataType(@nospecialize t) - isa(t, DataType) || return false isType(t) && return false + isa(t, DataType) || return false # Could be Union{} at runtime t === Core.TypeofBottom && return false # Return true if `t` is not covariant @@ -46,9 +34,9 @@ has_extended_info(@nospecialize x) = (!isa(x, Type) && !isvarargtype(x)) || isTy # some of these queries, this check can be used to somewhat protect against making incorrect # decisions based on incorrect subtyping. Note that this check, itself, is broken for # certain combinations of `a` and `b` where one/both isa/are `Union`/`UnionAll` type(s)s. -isnotbrokensubtype(@nospecialize(a), @nospecialize(b)) = (!iskindtype(b) || !isType(a) || hasuniquerep(a.parameters[1]) || b <: a) +isnotbrokensubtype(@nospecialize(a), @nospecialize(b)) = (!iskindtype(b) || !isType(a) || b <: a) -function argtypes_to_type(argtypes::Array{Any,1}) +function argtypes_to_type(argtypes::Vector{Any}) argtypes = anymap(@nospecialize(a) -> isvarargtype(a) ? a : widenconst(a), argtypes) filter!(@nospecialize(x) -> !isvarargtype(x) || valid_as_lattice(unwrapva(x), true), argtypes) all(@nospecialize(x) -> isvarargtype(x) || valid_as_lattice(x, true), argtypes) || return Bottom @@ -76,11 +64,13 @@ function valid_as_lattice(@nospecialize(x), astag::Bool=false) # operations that might remove the Union itself) return true end + if isType(x) + p = type_parameter(x) + p isa Type || p isa TypeVar || return false + return true + end if x isa DataType - if isType(x) - p = x.parameters[1] - p isa Type || p isa TypeVar || return false - elseif astag && isstructtype(x) + if astag && isstructtype(x) datatype_fieldtypes(x) # force computation of has_concrete_subtype to be updated now return has_concrete_subtype(x) end @@ -171,7 +161,7 @@ function typesubtract(@nospecialize(a), @nospecialize(b), max_union_splitting::I end _typename(@nospecialize a) = Union{} -_typename(a::TypeVar) = Core.TypeName +_typename(::TypeVar) = Core.TypeName function _typename(a::Union) ta = _typename(a.a) tb = _typename(a.b) @@ -182,6 +172,7 @@ function _typename(a::Union) end _typename(union::UnionAll) = _typename(union.body) _typename(a::DataType) = Const(a.name) +_typename(a::TypeEq) = Core.TypeName function tuple_tail_elem(𝕃::AbstractLattice, @nospecialize(init), ct::Vector{Any}) t = init @@ -192,16 +183,122 @@ function tuple_tail_elem(𝕃::AbstractLattice, @nospecialize(init), ct::Vector{ return Vararg{widenconst(t)} end +# Given `fargs` from `ArgInfo` and optionally `argtypes`, compute alias groups +# for argument positions. Returns `nothing` when no aliasing is possible (both +# callers treat this as the identity grouping), otherwise a `Vector{Int}` where +# `groups[i]` is the index of the leader for position `i`. `groups[i] == i` means +# leader (or non-aliased); `groups[i] < i` means follower. +# +# Aliasing is detected from two sources: +# 1. IR identity: same `SlotNumber`/`SSAValue` in `fargs` +# 2. `MustAlias` in `argtypes`: same `(slot, ssadef, fldidx)` means same value +function compute_alias_groups( + na::Int, + fargs::Union{Nothing,Vector{Any}}, + argtypes::Union{Nothing,Vector{Any}} + ) + # Fast path: skip the allocation when no aliasing is possible (the common + # case on the inference hot path). + any_alias_candidate(na, fargs, argtypes) || return nothing + groups = Vector{Int}(undef, na) + for i = 1:na + groups[i] = i + end + fargs !== nothing && merge_fargs_alias_groups!(groups, fargs) + argtypes !== nothing && merge_mustalias_groups!(groups, argtypes) + return groups +end + +# Cheap, non-allocating necessary condition for a non-identity grouping: a shared +# `SlotNumber`/`SSAValue` in `fargs`, or at least two `MustAlias` in `argtypes`. +# Over-approximating only costs a needless allocation, never correctness. +function any_alias_candidate( + na::Int, + fargs::Union{Nothing,Vector{Any}}, + argtypes::Union{Nothing,Vector{Any}} + ) + if fargs !== nothing + for i = 1:na + arg_i = fargs[i] + if arg_i isa SlotNumber || arg_i isa SSAValue + for j in 1:i-1 + fargs[j] === arg_i && return true + end + end + end + end + if argtypes !== nothing + nmustalias = 0 + for i = 1:na + if argtypes[i] isa MustAlias + nmustalias += 1 + nmustalias == 2 && return true + end + end + end + return false +end + +# Detect aliasing from IR identity: same `SlotNumber`/`SSAValue` in `fargs`. +function merge_fargs_alias_groups!(groups::Vector{Int}, fargs::Vector{Any}) + for i = 1:length(groups) + arg_i = fargs[i] + if arg_i isa SlotNumber || arg_i isa SSAValue + for j in 1:i-1 + if fargs[j] === arg_i + groups[i] = groups[j] + break + end + end + end + end + return groups +end + +# Detect additional aliasing from `MustAlias` lattice elements in `argtypes`: +# two positions with the same `(slot, ssadef, fldidx)` must refer to the same value. +function merge_mustalias_groups!(groups::Vector{Int}, argtypes::Vector{Any}) + for i = 1:length(groups) + groups[i] == i || continue # already a follower + ti = argtypes[i] + ti isa MustAlias || continue + for j in 1:i-1 + groups[j] == j || continue # only match against leaders + tj = argtypes[j] + tj isa MustAlias || continue + if ti.slot == tj.slot && ti.ssadef == tj.ssadef && ti.fldidx == tj.fldidx + # Merge: make j the leader for i, and re-point any + # existing followers of i to j as well + for k in i:length(groups) + if groups[k] == i + groups[k] = j + end + end + break + end + end + end + return groups +end + # Gives a cost function over the effort to switch a tuple-union representation # as a cartesian product, relative to the size of the original representation. # Thus, we count the longest element as being roughly invariant to being inside # or outside of the Tuple/Union nesting, though somewhat more expensive to be -# outside than inside because the representation is larger (because and it +# outside than inside because the representation is larger (because it # informs the callee whether any splitting is possible). -function unionsplitcost(𝕃::AbstractLattice, argtypes::Union{SimpleVector,Vector{Any}}) +function unionsplitcost(𝕃::AbstractLattice, argtypes::Union{SimpleVector,Vector{Any}}; + fargs::Union{Nothing,Vector{Any}}=nothing) + na = length(argtypes) + groups = compute_alias_groups(na, fargs, argtypes isa Vector{Any} ? argtypes : nothing) nu = 1 max = 2 - for ti in argtypes + for i in 1:na + # skip followers: their type is constrained by their leader + if groups !== nothing && groups[i] != i + continue + end + ti = argtypes[i] if has_extended_unionsplit(𝕃) && !isvarargtype(ti) ti = widenconst(ti) end @@ -225,9 +322,15 @@ function switchtupleunion(@nospecialize(ty)) return _switchtupleunion(JLTypeLattice(), Any[tparams...], length(tparams), [], ty) end -switchtupleunion(𝕃::AbstractLattice, argtypes::Vector{Any}) = _switchtupleunion(𝕃, argtypes, length(argtypes), [], nothing) +function switchtupleunion(𝕃::AbstractLattice, argtypes::Vector{Any}; + fargs::Union{Nothing,Vector{Any}}=nothing) + na = length(argtypes) + groups = compute_alias_groups(na, fargs, argtypes) + return _switchtupleunion(𝕃, argtypes, na, [], nothing, groups) +end -function _switchtupleunion(𝕃::AbstractLattice, t::Vector{Any}, i::Int, tunion::Vector{Any}, @nospecialize(origt)) +function _switchtupleunion(𝕃::AbstractLattice, t::Vector{Any}, i::Int, tunion::Vector{Any}, + @nospecialize(origt), groups::Union{Nothing,Vector{Int}}=nothing) if i == 0 if origt === nothing push!(tunion, copy(t)) @@ -235,23 +338,53 @@ function _switchtupleunion(𝕃::AbstractLattice, t::Vector{Any}, i::Int, tunion tpl = rewrap_unionall(Tuple{t...}, origt) push!(tunion, tpl) end + return tunion + end + + if groups !== nothing && groups[i] != i + # If this position is a follower (aliased to an earlier position), + # its type is already set by the leader — just recurse without iterating. + _switchtupleunion(𝕃, t, i - 1, tunion, origt, groups) else origti = ti = t[i] - # TODO remove this to implement callsite refinement of MustAlias + followers = Int[] + if groups !== nothing + for j in 1:length(t) + if groups[j] == i # Collect follower indices for this leader + push!(followers, j) + end + end + end + # TODO Generalize this to allow callsite union-splitting of MustAlias if isa(ti, Union) + origtypes = Any[t[j] for j in followers] for ty in uniontypes(ti) t[i] = ty - _switchtupleunion(𝕃, t, i - 1, tunion, origt) + for j in followers + t[j] = ty + end + _switchtupleunion(𝕃, t, i - 1, tunion, origt, groups) end t[i] = origti - elseif has_extended_unionsplit(𝕃) && !isa(ti, Const) && !isvarargtype(ti) && isa(widenconst(ti), Union) - for ty in uniontypes(ti) + for (k, j) in enumerate(followers) + t[j] = origtypes[k] + end + elseif (has_extended_unionsplit(𝕃) && !isa(ti, Const) && !isvarargtype(ti) && + (wty = widenconst(ti); isa(wty, Union))) + origtypes = Any[t[j] for j in followers] + for ty in uniontypes(wty) t[i] = ty - _switchtupleunion(𝕃, t, i - 1, tunion, origt) + for j in followers + t[j] = ty + end + _switchtupleunion(𝕃, t, i - 1, tunion, origt, groups) end t[i] = origti + for (k, j) in enumerate(followers) + t[j] = origtypes[k] + end else - _switchtupleunion(𝕃, t, i - 1, tunion, origt) + _switchtupleunion(𝕃, t, i - 1, tunion, origt, groups) end end return tunion @@ -330,6 +463,7 @@ function _is_immutable_type(@nospecialize ty) if isa(ty, Union) return _is_immutable_type(ty.a) && _is_immutable_type(ty.b) end + isType(ty) && return false return !isabstracttype(ty) && !ismutabletype(ty) end diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index fe8966c32fc17..e9d69ce41b141 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -1,5 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# supertype access that forces a deferred supertype (an instantiation of a +# self-referential definition fills `super` lazily, see issue #61347); a +# plain field read on such an instantiation would see an undefined field +datatype_super(x::DataType) = ccall(:jl_datatype_super, Any, (Any,), x)::DataType + ########### # generic # ########### @@ -21,7 +26,7 @@ function contains_is(itr, @nospecialize(x)) return false end -anymap(f::Function, a::Array{Any,1}) = Any[ f(a[i]) for i in 1:length(a) ] +anymap(f::Function, a::Vector{Any}) = Any[ f(a[i]) for i in 1:length(a) ] ############ # inlining # @@ -167,9 +172,8 @@ isa_compileable_sig(@nospecialize(atype), sparams::SimpleVector, method::Method) !iszero(ccall(:jl_isa_compileable_sig, Int32, (Any, Any, Any), atype, sparams, method)) isa_compileable_sig(m::MethodInstance) = (def = m.def; !isa(def, Method) || isa_compileable_sig(m.specTypes, m.sparam_vals, def)) -isa_compileable_sig(m::ABIOverride) = false +isa_compileable_sig(::ABIOverride) = false -has_typevar(@nospecialize(t), v::TypeVar) = ccall(:jl_has_typevar, Cint, (Any, Any), t, v) != 0 """ is_declared_inline(method::Method)::Bool @@ -215,7 +219,7 @@ is_no_constprop(method::Union{Method,CodeInfo}) = method.constprop == 0x02 if isa(ft, Const) return ft.val elseif isconstType(ft) - return ft.parameters[1] + return type_parameter(ft) elseif issingletontype(ft) return ft.instance end @@ -223,12 +227,10 @@ is_no_constprop(method::Union{Method,CodeInfo}) = method.constprop == 0x02 end @nospecializeinfer function maybe_singleton_const(@nospecialize(t)) - if isa(t, DataType) - if issingletontype(t) - return Const(t.instance) - elseif isconstType(t) - return Const(t.parameters[1]) - end + if isa(t, DataType) && issingletontype(t) + return Const(t.instance) + elseif isconstType(t) + return Const(type_parameter(t)) end return t end @@ -268,8 +270,60 @@ function foreach_anyssa(@specialize(f), @nospecialize(stmt)) end end +# Uses of each SSA value, in compressed-sparse-row form: the uses of ssa `i` are +# `data[offsets[i]:offsets[i+1]-1]`. Entries are use occurrences, not distinct +# statement indices, so a statement using the same value twice lists its line +# twice; harmless for the consumers (`isempty` and idempotent block re-enqueue). +struct SSAUses + offsets::Vector{Int} + data::Vector{Int} +end + +# Non-escaping view over one value's uses; supports `isempty` and iteration. +struct SSAUseList + data::Vector{Int} + start::Int + stop::Int +end + +@inline function getindex(uses::SSAUses, i::Int) + offsets = uses.offsets + return SSAUseList(uses.data, offsets[i], offsets[i+1] - 1) +end + +@inline isempty(l::SSAUseList) = l.stop < l.start +@inline length(l::SSAUseList) = l.stop - l.start + 1 +@inline function iterate(l::SSAUseList, i::Int = l.start) + i > l.stop && return nothing + return (l.data[i], i + 1) +end + function find_ssavalue_uses(body::Vector{Any}, nvals::Int) - uses = BitSet[ BitSet() for i = 1:nvals ] + # count uses per value + counts = zeros(Int, nvals) + foreach_ssavalue_use(body) do id::Int, _line::Int + counts[id] += 1 + end + # prefix-sum the counts into row pointers + offsets = Vector{Int}(undef, nvals + 1) + tot = 1 + for i = 1:nvals + offsets[i] = tot + tot += counts[i] + end + offsets[nvals + 1] = tot + # scatter line numbers into `data`, reusing `counts` as write cursors + data = Vector{Int}(undef, tot - 1) + fill!(counts, 0) + foreach_ssavalue_use(body) do id::Int, line::Int + data[offsets[id] + counts[id]] = line + counts[id] += 1 + end + return SSAUses(offsets, data) +end + +# Call `f(ssa_id, line)` for each SSA value used as an operand in `body`. +function foreach_ssavalue_use(@specialize(f), body::Vector{Any}) for line in 1:length(body) e = body[line] if isa(e, ReturnNode) @@ -279,17 +333,17 @@ function find_ssavalue_uses(body::Vector{Any}, nvals::Int) e = e.cond end if isa(e, SSAValue) - push!(uses[e.id], line) + f(e.id, line) elseif isa(e, Expr) - find_ssavalue_uses!(uses, e, line) + foreach_ssavalue_use(f, e, line) elseif isa(e, PhiNode) - find_ssavalue_uses!(uses, e, line) + foreach_ssavalue_use(f, e, line) end end - return uses + return nothing end -function find_ssavalue_uses!(uses::Vector{BitSet}, e::Expr, line::Int) +function foreach_ssavalue_use(@specialize(f), e::Expr, line::Int) head = e.head is_meta_expr_head(head) && return skiparg = (head === :(=)) @@ -297,22 +351,24 @@ function find_ssavalue_uses!(uses::Vector{BitSet}, e::Expr, line::Int) if skiparg skiparg = false elseif isa(a, SSAValue) - push!(uses[a.id], line) + f(a.id, line) elseif isa(a, Expr) - find_ssavalue_uses!(uses, a, line) + foreach_ssavalue_use(f, a, line) end end + return nothing end -function find_ssavalue_uses!(uses::Vector{BitSet}, e::PhiNode, line::Int) +function foreach_ssavalue_use(@specialize(f), e::PhiNode, line::Int) values = e.values for i = 1:length(values) isassigned(values, i) || continue val = values[i] if isa(val, SSAValue) - push!(uses[val.id], line) + f(val.id, line) end end + return nothing end # using a function to ensure we can infer this @@ -328,9 +384,14 @@ end inlining_enabled() = (JLOptions().can_inline == 1) function instrumentation_enabled(m::Module, only_if_affects_optimizer::Bool) - generating_output() && return false # don't alter caches + if generating_output() + # an image is instrumented for every scope or not at all + # (jl_image_coverage_config), and the scope is applied when its counters + # are registered; the generating process itself is not instrumented + return only_if_affects_optimizer && ccall(:jl_image_coverage_config, UInt8, ()) != 0 + end cov = JLOptions().code_coverage - if cov == 1 # user + if cov == 1 || cov == 3 # user instrumentation; @path filters reports m = moduleroot(m) m === Core && return false isdefined(Main, :Base) && m === Main.Base && return false diff --git a/Compiler/src/validation.jl b/Compiler/src/validation.jl index 067d9460b52ec..67b2072679644 100644 --- a/Compiler/src/validation.jl +++ b/Compiler/src/validation.jl @@ -8,8 +8,6 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :static_parameter => 1:1, :(&) => 1:1, :(=) => 2:2, - :method => 1:4, - :const => 1:2, :new => 1:typemax(Int), :splatnew => 2:2, :the_exception => 0:0, @@ -21,9 +19,8 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :boundscheck => 0:1, :copyast => 1:1, :meta => 0:typemax(Int), - :global => 1:1, - :globaldecl => 1:2, :foreigncall => 5:typemax(Int), # name, RT, AT, nreq, (cconv, effects, gc_safe), args..., roots... + :foreignglobal => 1:1, # name :cfunction => 5:5, :isdefined => 1:2, :code_coverage_effect => 0:0, @@ -52,8 +49,6 @@ const SLOTFLAGS_MISMATCH = "length(slotnames) < length(slotflags)" const SSAVALUETYPES_MISMATCH = "not all SSAValues in AST have a type in ssavaluetypes" const SSAVALUETYPES_MISMATCH_UNINFERRED = "uninferred CodeInfo ssavaluetypes field does not equal the number of present SSAValues" const SSAFLAGS_MISMATCH = "not all SSAValues have a corresponding `ssaflags`" -const NON_TOP_LEVEL_METHOD = "encountered `Expr` head `:method` in non-top-level code (i.e. `nargs` > 0)" -const NON_TOP_LEVEL_GLOBAL = "encountered `Expr` head `:global` in non-top-level code (i.e. `nargs` > 0)" const SIGNATURE_NARGS_MISMATCH = "method signature does not match number of method arguments" const SLOTNAMES_NARGS_MISMATCH = "CodeInfo for method contains fewer slotnames than the number of method arguments" const INVALID_SIGNATURE_OPAQUE_CLOSURE = "invalid signature of method for opaque closure - `sig` field must always be set to `Tuple`" @@ -87,16 +82,11 @@ end function _validate_val!(@nospecialize(x), errors, ssavals::BitSet) if isa(x, Expr) if x.head === :call || x.head === :invoke || x.head === :invoke_modify - f = x.args[1] - if f isa GlobalRef && (f.name === :cglobal) && x.head === :call - # TODO: these are not yet linearized - else - for arg in x.args - if !is_valid_argument(arg) - push!(errors, InvalidCodeError(INVALID_CALL_ARG, arg)) - else - _validate_val!(arg, errors, ssavals) - end + for arg in x.args + if !is_valid_argument(arg) + push!(errors, InvalidCodeError(INVALID_CALL_ARG, arg)) + else + _validate_val!(arg, errors, ssavals) end end end @@ -123,10 +113,6 @@ function validate_code!(errors::Vector{InvalidCodeError}, c::CodeInfo, is_top_le for x in c.code if isa(x, Expr) head = x.head - if !is_top_level - head === :method && push!(errors, InvalidCodeError(NON_TOP_LEVEL_METHOD)) - head === :global && push!(errors, InvalidCodeError(NON_TOP_LEVEL_GLOBAL)) - end narg_bounds = get(VALID_EXPR_HEADS, head, -1:-1) nargs = length(x.args) if narg_bounds == -1:-1 @@ -148,9 +134,9 @@ function validate_code!(errors::Vector{InvalidCodeError}, c::CodeInfo, is_top_le validate_val!(rhs) elseif head === :call || head === :invoke || x.head === :invoke_modify || head === :gc_preserve_end || head === :meta || - head === :inbounds || head === :foreigncall || head === :cfunction || - head === :const || head === :leave || head === :pop_exception || - head === :method || head === :global || head === :static_parameter || + head === :inbounds || head === :foreigncall || head === :foreignglobal || head === :cfunction || + head === :leave || head === :pop_exception || + head === :static_parameter || head === :new || head === :splatnew || head === :thunk || head === :loopinfo || head === :throw_undef_if_not || head === :code_coverage_effect || head === :inline || head === :noinline validate_val!(x) @@ -183,6 +169,7 @@ function validate_code!(errors::Vector{InvalidCodeError}, c::CodeInfo, is_top_le elseif isa(x, SlotNumber) elseif isa(x, Argument) elseif isa(x, GlobalRef) + elseif isa(x, Core.BindingPartition) elseif isa(x, LineNumberNode) elseif isa(x, PiNode) elseif isa(x, PhiCNode) @@ -243,11 +230,11 @@ end validate_code(args...) = validate_code!(Vector{InvalidCodeError}(), args...) -is_valid_lvalue(@nospecialize(x)) = isa(x, SlotNumber) || isa(x, GlobalRef) +is_valid_lvalue(@nospecialize(x)) = isa(x, SlotNumber) || isa(x, GlobalRef) || isa(x, Core.BindingPartition) function is_valid_argument(@nospecialize(x)) if isa(x, SlotNumber) || isa(x, Argument) || isa(x, SSAValue) || - isa(x, GlobalRef) || isa(x, QuoteNode) || (isa(x, Expr) && is_value_pos_expr_head(x.head)) || + isa(x, GlobalRef) || isa(x, Core.BindingPartition) || isa(x, QuoteNode) || isa(x, Number) || isa(x, AbstractString) || isa(x, AbstractChar) || isa(x, Tuple) || isa(x, Type) || isa(x, Core.Box) || isa(x, Module) || x === nothing return true @@ -260,7 +247,7 @@ end function is_valid_rvalue(@nospecialize(x)) is_valid_argument(x) && return true if isa(x, Expr) && x.head in (:new, :splatnew, :the_exception, :isdefined, :call, - :invoke, :invoke_modify, :foreigncall, :cfunction, :gc_preserve_begin, :copyast, + :invoke, :invoke_modify, :foreigncall, :foreignglobal, :cfunction, :gc_preserve_begin, :copyast, :new_opaque_closure) return true end diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index eb775bfa290ce..0d2ed3eb92c0a 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -4,22 +4,22 @@ import ..Compiler: verify_typeinf_trim, NativeInterpreter, argtypes_to_type, com using ..Compiler: # operators - !, !=, !==, +, :, <, <=, ==, =>, >, >=, ∈, ∉, + !, !=, !==, %, -, +, :, <, <=, ==, =>, >, >=, ^, ∈, ∉, # types Array, Builtin, Callable, Cint, CodeInfo, CodeInstance, Csize_t, Exception, GenericMemory, GlobalRef, IdDict, IdSet, IntrinsicFunction, Method, MethodInstance, NamedTuple, Pair, PhiCNode, PhiNode, PiNode, QuoteNode, SSAValue, SimpleVector, String, Tuple, VarState, Vector, # functions - argextype, empty!, error, get, get_ci_mi, get_world_counter, getindex, getproperty, - hasintersect, haskey, in, isdispatchelem, isempty, isexpr, iterate, length, map!, max, + argextype, argextype_widened, empty!, error, get, get_ci_mi, get_world_counter, getglobal, getindex, getproperty, + hasintersect, haskey, in, isdefinedglobal, isdispatchelem, isempty, isexpr, iterate, length, map!, max, pop!, popfirst!, push!, pushfirst!, reinterpret, reverse!, reverse, setindex!, setproperty!, similar, singleton_type, sptypes_from_meth_instance, sp_type_rewrap, - unsafe_pointer_to_objref, widenconst, isconcretetype, + unsafe_pointer_to_objref, isconcretetype, # misc @nospecialize, @assert, C_NULL -using ..IRShow: LineInfoNode, print, show, println, append_scopes!, IOContext, IO, normalize_method_name -using ..Base: Base, sourceinfo_slotnames +using ..IRShow: LineInfoNode, print, show, println, append_scopes!, IOContext, IO, normalize_method_name, is_expected_union +using ..Base: Base, sourceinfo_slotnames, printstyled using ..Base.StackTraces: StackFrame ## declarations ## @@ -47,78 +47,192 @@ const runtime_functions = Symbol[ :jl_apply, ] -## code for pretty printing ## +# Check if a type is "stable" - uses same classification as code_warntype +function is_type_stable(@nospecialize(typ)) + typ isa Core.Const && return true + typ === Union{} && return false + typ == Core.Box && return false + typ isa Union && return is_expected_union(typ) + return isdispatchelem(typ) +end + +# Color printing for types - same colors as code_warntype: +# - red bold: unstable/abstract types +# - yellow: expected unions (small unions of concrete types) +# - light_black: stable concrete types +function print_type_colored(io::IO, @nospecialize(typ)) + color = get(io, :color, false)::Bool + if !color + print(io, typ) + elseif typ === Union{} + printstyled(io, typ; color=:red) + elseif typ == Core.Box + printstyled(io, typ; color=:red, bold=true) + elseif typ isa Union && is_expected_union(typ) + printstyled(io, typ; color=:yellow) + elseif isdispatchelem(typ) + printstyled(io, typ; color=:light_black) + else + printstyled(io, typ; color=:red, bold=true) + end +end + -# wrap a statement in a typeassert for printing clarity, unless that info seems already obvious -function mapssavaluetypes(codeinfo::CodeInfo, sptypes::Vector{VarState}, stmt) - @nospecialize stmt - newstmt = mapssavalues(codeinfo, sptypes, stmt) - typ = widenconst(argextype(stmt, codeinfo, sptypes)) - if newstmt isa Expr - if newstmt.head ∈ (:quote, :inert) - return newstmt +# Print a value with optional stable coloring (light_black if stable) +function print_value(io::IO, @nospecialize(x), stable::Bool) + stable ? printstyled(io, x; color=:light_black) : print(io, x) +end + +# Unwrap SSAValue and PiNode to get the underlying statement +function unwrap_stmt(codeinfo::CodeInfo, @nospecialize(stmt)) + while true + if stmt isa SSAValue + stmt = codeinfo.code[stmt.id] + isexpr(stmt, :(=)) && (stmt = stmt.args[2]) + elseif stmt isa PiNode + stmt = stmt.val + else + return stmt end - elseif newstmt isa GlobalRef && isdispatchelem(typ) - return newstmt - elseif newstmt isa Union{Int, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64, String, QuoteNode} - return newstmt - elseif newstmt isa Callable - return newstmt end - return Expr(:(::), newstmt, typ) end -# map the ssavalues in a (value-producing) statement to the expression they came from, summarizing some things to avoid excess printing -function mapssavalues(codeinfo::CodeInfo, sptypes::Vector{VarState}, stmt) - @nospecialize stmt - if stmt isa SSAValue - return mapssavalues(codeinfo, sptypes, codeinfo.code[stmt.id]) - elseif stmt isa PiNode - return mapssavalues(codeinfo, sptypes, stmt.val) - elseif stmt isa Expr - stmt.head ∈ (:quote, :inert) && return stmt - newstmt = Expr(stmt.head) - if stmt.head === :foreigncall - return Expr(:call, :ccall, mapssavalues(codeinfo, sptypes, stmt.args[1])) - elseif stmt.head ∉ (:new, :method, :toplevel, :thunk) - newstmt.args = map!(similar(stmt.args), stmt.args) do arg - @nospecialize arg - return mapssavaluetypes(codeinfo, sptypes, arg) +# Convert Core.Argument to slot name symbol if available +function argument_name(codeinfo::CodeInfo, arg::Core.Argument) + slotnames = codeinfo.slotnames + if slotnames !== nothing && arg.n <= length(slotnames) + return slotnames[arg.n] + end + return arg +end + +const MAX_NESTING_DEPTH = 1 + +function is_call_expr(codeinfo::CodeInfo, @nospecialize(stmt)) + stmt = unwrap_stmt(codeinfo, stmt) + return stmt isa Expr && stmt.head ∈ (:call, :invoke, :foreigncall, :foreignglobal, :new) +end + +function has_unstable_arg(codeinfo::CodeInfo, sptypes::Vector{VarState}, args, startidx::Int) + for i in (startidx + 1):length(args) + is_type_stable(argextype_widened(args[i], codeinfo, sptypes)) || return true + end + return false +end + +function print_value(io::IO, codeinfo::CodeInfo, @nospecialize(stmt), stable::Bool) + stmt = unwrap_stmt(codeinfo, stmt) + if stmt isa GlobalRef + # TODO: This is not correct. + print_value(io, stmt.name, stable) + elseif stmt isa Core.Argument + print_value(io, argument_name(codeinfo, stmt), stable) + elseif stmt isa QuoteNode && stmt.value isa Symbol + print_value(io, Base.repr(stmt.value), stable) # Show :symbol with colon + else + print_value(io, Base.repr(stmt), stable) + end +end + +# Print value with type annotation: value::type +function print_typed(io::IO, codeinfo::CodeInfo, sptypes::Vector{VarState}, @nospecialize(stmt), stable::Bool; + depth::Int=0, indent::Int=0) + typ = argextype_widened(stmt, codeinfo, sptypes) + arg_stable = is_type_stable(typ) + # Fold deeply nested stable calls + if arg_stable && depth >= MAX_NESTING_DEPTH && is_call_expr(codeinfo, stmt) + printstyled(io, "(…)"; color=:light_black) + else + print_stmt_colored(io, codeinfo, sptypes, stmt; depth=depth, stable=arg_stable, indent=indent) + end + printstyled(io, "::"; color=:light_black) + print_type_colored(io, typ) +end + +function should_elide_type(@nospecialize(stmt), @nospecialize(typ)) + stmt isa GlobalRef && isdispatchelem(typ) && return true + stmt isa Callable && return true + return false +end + +function print_stmt_colored(io::IO, codeinfo::CodeInfo, sptypes::Vector{VarState}, @nospecialize(stmt); + depth::Int=0, stable::Bool=false, indent::Int=0) + stmt = unwrap_stmt(codeinfo, stmt) + + if !is_call_expr(codeinfo, stmt) + return print_value(io, codeinfo, stmt, stable) + end + + if stmt.head === :foreigncall + print_value(io, "ccall", stable) + print(io, "(") + length(stmt.args) >= 1 && print_value(io, codeinfo, stmt.args[1], stable) + print(io, ")") + else + startidx = stmt.head === :invoke ? 2 : 1 + farg = startidx <= length(stmt.args) ? stmt.args[startidx] : nothing + nargs = length(stmt.args) - startidx + + # Print function call + if farg !== nothing + fstmt = unwrap_stmt(codeinfo, farg) + ftyp = argextype_widened(farg, codeinfo, sptypes) + needs_type = !should_elide_type(fstmt, ftyp) + needs_type && print(io, "(") + print_value(io, codeinfo, farg, stable) + if needs_type + printstyled(io, "::"; color=:light_black) + print_type_colored(io, ftyp) + print(io, ")") end - if newstmt.head === :invoke - # why is the fancy printing for this not in show_unquoted? - popfirst!(newstmt.args) - newstmt.head = :call + end + print(io, "(") + use_multiline = !stable && nargs > 0 && has_unstable_arg(codeinfo, sptypes, stmt.args, startidx) + for i in (startidx + 1):length(stmt.args) + i > startidx + 1 && print(io, ",") + if use_multiline + println(io) + print(io, " " ^ (indent + 1)) + elseif i > startidx + 1 + print(io, " ") end + print_typed(io, codeinfo, sptypes, stmt.args[i], stable; depth=depth+1, indent=indent+1) end - return newstmt - elseif stmt isa PhiNode - return PhiNode() - elseif stmt isa PhiCNode - return PhiNode() + use_multiline && nargs > 0 && (println(io); print(io, " " ^ indent)) + print(io, ")") end - return stmt end -function verify_print_stmt(io::IOContext{IO}, codeinfo::CodeInfo, sptypes::Vector{VarState}, stmtidx::Int) - if codeinfo.slotnames !== nothing - io = IOContext(io, :SOURCE_SLOTNAMES => sourceinfo_slotnames(codeinfo)) - end - print(io, mapssavaluetypes(codeinfo, sptypes, SSAValue(stmtidx))) +function verify_print_stmt(io::IO, codeinfo::CodeInfo, sptypes::Vector{VarState}, stmtidx::Int) + codeinfo.slotnames !== nothing && (io = IOContext(io, :SOURCE_SLOTNAMES => sourceinfo_slotnames(codeinfo))) + stmt = unwrap_stmt(codeinfo, codeinfo.code[stmtidx]) + typ = argextype_widened(SSAValue(stmtidx), codeinfo, sptypes) + print_stmt_colored(io, codeinfo, sptypes, stmt) + print(io, "::") + print_type_colored(io, typ) end -function verify_print_error(io::IOContext{IO}, desc::CallMissing, parents::ParentMap) +function verify_print_error(io::IO, desc::CallMissing, parents::ParentMap, warn::Bool) (; codeinst, codeinfo, sptypes, stmtidx, desc) = desc frames = verify_create_stackframes(codeinst, stmtidx, parents) - print(io, desc, " from statement ") + color = warn ? Base.warn_color() : Base.error_color() + printstyled(io, desc; color=color, bold=true) + print(io, " from statement ") verify_print_stmt(io, codeinfo, sptypes, stmtidx) + print(io, "\n") Base.show_backtrace(io, frames) print(io, "\n\n") nothing end -function verify_print_error(io::IOContext{IO}, desc::CCallableMissing, parents::ParentMap) - print(io, desc.desc, " for ", desc.sig, " => ", desc.rt, "\n\n") +function verify_print_error(io::IO, desc::CCallableMissing, ::ParentMap, warn::Bool) + color = warn ? Base.warn_color() : Base.error_color() + printstyled(io, desc.desc; color=color, bold=true) + print(io, " for ") + print_type_colored(io, desc.sig) + print(io, " => ") + print_type_colored(io, desc.rt) + print(io, "\n\n") nothing end @@ -126,6 +240,7 @@ function verify_create_stackframes(codeinst::CodeInstance, stmtidx::Int, parents scopes = LineInfoNode[] frames = StackFrame[] parent = (codeinst, stmtidx) + visited = IdSet{Tuple{CodeInstance,Int}}() while parent !== nothing codeinst, stmtidx = parent di = codeinst.debuginfo @@ -139,7 +254,13 @@ function verify_create_stackframes(codeinst::CodeInstance, stmtidx::Int, parents push!(frames, sf) end empty!(scopes) - parent = get(parents, codeinst, nothing) + + new_parent = get(parents, codeinst, nothing) + if haskey(visited, new_parent) + break + end + push!(visited, new_parent) + parent = new_parent end return frames end @@ -160,6 +281,7 @@ function may_dispatch(@nospecialize ftyp) Core.finalizer isa ftyp || Core.modifyfield! isa ftyp || Core.modifyglobal! isa ftyp || + Core.modifyglobal_partition isa ftyp || Core.memoryrefmodify! isa ftyp else return true @@ -186,12 +308,14 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, ci = get(caches, edge_mi, nothing) ci isa CodeInstance && continue # assume that only this_world matters for trim end + elseif edge isa MethodInstance + ci = get(caches, edge, nothing) + ci isa CodeInstance && continue end # TODO: check for calls to Base.atexit? elseif isexpr(stmt, :call) - error = "unresolved call" farg = stmt.args[1] - ftyp = widenconst(argextype(farg, codeinfo, sptypes)) + ftyp = argextype_widened(farg, codeinfo, sptypes) if ftyp <: IntrinsicFunction #TODO: detect if f !== Core.Intrinsics.atomic_pointermodify (see statement_cost), otherwise error continue @@ -206,12 +330,12 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, # args[1] is _apply_iterate object # args[2] is invoke object farg = stmt.args[3] - ftyp = widenconst(argextype(farg, codeinfo, sptypes)) + ftyp = argextype_widened(farg, codeinfo, sptypes) if may_dispatch(ftyp) error = "unresolved call to function" else for i in 4:length(stmt.args) - atyp = widenconst(argextype(stmt.args[i], codeinfo, sptypes)) + atyp = argextype_widened(stmt.args[i], codeinfo, sptypes) if !(atyp <: Union{SimpleVector, GenericMemory, Array, Tuple, NamedTuple}) error = "unresolved argument to call" break @@ -220,7 +344,7 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, end end elseif Core.finalizer isa ftyp - if length(stmt.args) == 3 + if 3 <= length(stmt.args) <= 5 finalizer = argextype(stmt.args[2], codeinfo, sptypes) obj = argextype(stmt.args[3], codeinfo, sptypes) atype = argtypes_to_type(Any[finalizer, obj]) @@ -230,9 +354,8 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, ci = get(caches, mi, nothing) ci isa CodeInstance && continue end - - error = "unresolved finalizer registered" end + error = "unresolved finalizer registered" elseif Core._apply isa ftyp error = "trim verification not yet implemented for builtin `Core._apply`" elseif Core._call_in_world_total isa ftyp @@ -247,9 +370,13 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, error = "trim verification not yet implemented for builtin `Core.modifyfield!`" elseif Core.modifyglobal! isa ftyp error = "trim verification not yet implemented for builtin `Core.modifyglobal!`" + elseif Core.modifyglobal_partition isa ftyp + error = "trim verification not yet implemented for builtin `Core.modifyglobal_partition`" elseif Core.memoryrefmodify! isa ftyp error = "trim verification not yet implemented for builtin `Core.memoryrefmodify!`" else @assert false "unexpected builtin" end + else + error = "unresolved call" end extyp = argextype(SSAValue(i), codeinfo, sptypes) if extyp === Union{} @@ -257,7 +384,7 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, end elseif isexpr(stmt, :cfunction) length(stmt.args) != 5 && continue # required by IR legality - (pointer_type, f, rt, at, call_type) = stmt.args + f, at = stmt.args[2], stmt.args[4] at isa SimpleVector || continue # required by IR legality ft = argextype(f, codeinfo, sptypes) @@ -279,9 +406,17 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, error = "unresolved cfunction" elseif isexpr(stmt, :foreigncall) foreigncall = stmt.args[1] - if foreigncall isa QuoteNode - if foreigncall.value in runtime_functions - error = "disallowed ccall into a runtime function" + if isexpr(foreigncall, :tuple, 1) + foreigncall = foreigncall.args[1] + if foreigncall isa String + foreigncall = QuoteNode(Symbol(foreigncall)) + end + if foreigncall isa QuoteNode + if foreigncall.value in runtime_functions + error = "disallowed ccall into a runtime function" + end + else + error = "disallowed ccall with non-constant name and no library" end end elseif isexpr(stmt, :new_opaque_closure) @@ -308,7 +443,10 @@ function get_verify_typeinf_trim(codeinfos::Vector{Any}) item = codeinfos[i] if item isa CodeInstance push!(inspected, item) - if item.owner === nothing && item.min_world <= this_world <= item.max_world + # Trim inference caches its results under the `:trim` symbol as the owner (see + # `typeinf_ext_toplevel`), so the `CodeInstance`s handed to us here carry that + # owner rather than `nothing`. + if item.owner === :trim && item.min_world <= this_world <= item.max_world mi = get_ci_mi(item) if mi === item.def caches[mi] = item @@ -324,12 +462,10 @@ function get_verify_typeinf_trim(codeinfos::Vector{Any}) elseif item isa SimpleVector rt = item[1]::Type sig = item[2]::Type - ptr = ccall(:jl_get_specialization1, - #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), - sig, this_world, #= mt_cache =# 0) + mi = ccall(:jl_get_specialization1, Any, (Any, Csize_t), sig, this_world) asrt = Any - valid = if ptr !== C_NULL - mi = unsafe_pointer_to_objref(ptr)::MethodInstance + valid = if mi !== nothing + mi = mi::MethodInstance ci = get(caches, mi, nothing) if ci isa CodeInstance # TODO: should we find a way to indicate to the user that this gets called via ccallable? @@ -369,9 +505,8 @@ function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) warn, desc = desc severity = warn ? 2 : 1 no = (counts[severity] += 1) - print(io, warn ? "Verifier warning #" : "Verifier error #", no, ": ") - # TODO: should we coalesce any of these stacktraces to minimize spew? - verify_print_error(io, desc, parents) + printstyled(io, "Error #", no, ": "; color=Base.error_color(), bold=true) # TODO: should we coalesce any of these stacktraces to minimize spew? + verify_print_error(io, desc, parents, warn) end ## TODO: compute and display the minimum and/or full call graph instead of merely the first parent stacktrace? @@ -384,11 +519,19 @@ function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) let severity = 0 if counts[1] > 0 || counts[2] > 0 - print("Trim verify finished with ") - print(counts[1], counts[1] == 1 ? " error" : " errors") - print(", ") - print(counts[2], counts[2] == 1 ? " warning" : " warnings") - print(".\n") + print(io, "Trim verify finished with ") + if counts[1] > 0 + printstyled(io, counts[1], counts[1] == 1 ? " error" : " errors"; color=Base.error_color(), bold=true) + else + print(io, counts[1], counts[1] == 1 ? " error" : " errors") + end + print(io, ", ") + if counts[2] > 0 + printstyled(io, counts[2], counts[2] == 1 ? " warning" : " warnings"; color=Base.warn_color(), bold=true) + else + print(io, counts[2], counts[2] == 1 ? " warning" : " warnings") + end + print(io, ".\n") severity = 2 end if counts[1] > 0 diff --git a/Compiler/test/AbstractInterpreter.jl b/Compiler/test/AbstractInterpreter.jl index 12da527225a7e..6cab73836fb5f 100644 --- a/Compiler/test/AbstractInterpreter.jl +++ b/Compiler/test/AbstractInterpreter.jl @@ -34,6 +34,41 @@ end @MethodTable OVERLAY_MT Compiler.method_table(interp::MTOverlayInterp) = Compiler.OverlayMethodTable(Compiler.get_inference_world(interp), OVERLAY_MT) +# `include_ambiguous` preserves ambiguous matches across method-table views and cache modes. +ambiguous_lookup(x::Integer, y) = 1 +ambiguous_lookup(x, y::Integer) = 2 +ambiguous_overlay(x, y) = 0 +@overlay OVERLAY_MT ambiguous_overlay(x::Integer, y) = 1 +@overlay OVERLAY_MT ambiguous_overlay(x, y::Integer) = 2 + +@testset "ambiguous method lookup" begin + world = Base.get_world_counter() + sig = Tuple{typeof(ambiguous_lookup),Integer,Integer} + internal = Compiler.InternalMethodTable(world) + filtered = Compiler.findall(sig, internal) + inclusive = Compiler.findall(sig, internal; include_ambiguous=true) + @test filtered !== nothing + @test Compiler.length(filtered) == 0 + @test inclusive !== nothing + @test inclusive.ambig + @test Compiler.length(inclusive) == 2 + @test Set(match.method for match in inclusive) == Set(methods(ambiguous_lookup)) + + cached = Compiler.CachedMethodTable(internal) + @test Compiler.length(Compiler.findall(sig, cached)) == 0 + @test Compiler.length(Compiler.findall(sig, cached; include_ambiguous=true)) == 2 + @test length(cached.cache) == 2 + + overlay_sig = Tuple{typeof(ambiguous_overlay),Integer,Integer} + overlay = Compiler.OverlayMethodTable(world, OVERLAY_MT) + overlay_matches = Compiler.findall(overlay_sig, overlay; include_ambiguous=true) + @test overlay_matches !== nothing + @test overlay_matches.ambig + @test Compiler.length(overlay_matches) == 2 + base_method = only(methods(ambiguous_overlay)) + @test all(match -> match.method !== base_method, overlay_matches) +end + function Compiler.add_remark!(interp::MTOverlayInterp, ::Compiler.InferenceState, remark) if interp.meta !== nothing # Core.println(remark) @@ -104,6 +139,15 @@ overlay_match(::Any) = nothing overlay_match(x) end |> only === Union{Nothing,Missing} +# overlay method should shadow the base method with the same signature, +# filtering it out from method match results +overlay_shadow_zero() = Any[] +overlay_shadow_zero(xs::Vector{Int}...) = Int[xs[i][j] for i=eachindex(xs) for j=eachindex(xs[i])] +@overlay OVERLAY_MT overlay_shadow_zero() = error() +@test Base.infer_return_type((Vector{Vector{Int}},); interp=MTOverlayInterp()) do x + overlay_shadow_zero(x...) +end == Vector{Int} + # partial concrete evaluation @test Base.return_types(; interp=MTOverlayInterp()) do isbitstype(Int) ? nothing : missing @@ -157,7 +201,7 @@ gpu_factorial3(x::Int) = myfactorial(x, raise_on_gpu3) @test Base.infer_effects(gpu_factorial2, (Int,); interp=MTOverlayInterp()) |> Compiler.is_consistent_overlay let effects = Base.infer_effects(gpu_factorial3, (Int,); interp=MTOverlayInterp()) # check if `@consistent_overlay` together works with `@assume_effects` - # N.B. the overlaid `raise_on_gpu3` is not :foldable otherwise since `error_on_gpu` is (intetionally) undefined. + # N.B. the overlaid `raise_on_gpu3` is not :foldable otherwise since `error_on_gpu` is (intentionally) undefined. @test Compiler.is_consistent_overlay(effects) @test Compiler.is_foldable(effects) end @@ -408,10 +452,10 @@ Compiler.nsplit_impl(info::NoinlineCallInfo) = Compiler.nsplit(info.info) Compiler.getsplit_impl(info::NoinlineCallInfo, idx::Int) = Compiler.getsplit(info.info, idx) Compiler.getresult_impl(info::NoinlineCallInfo, idx::Int) = Compiler.getresult(info.info, idx) -function Compiler.abstract_call(interp::NoinlineInterpreter, - arginfo::Compiler.ArgInfo, si::Compiler.StmtInfo, sv::Compiler.InferenceState, max_methods::Int) +function Compiler.abstract_call(interp::NoinlineInterpreter, arginfo::Compiler.ArgInfo, si::Compiler.StmtInfo, + vtypes::Union{Compiler.VarTable,Nothing}, sv::Compiler.InferenceState, max_methods::Int) ret = @invoke Compiler.abstract_call(interp::Compiler.AbstractInterpreter, - arginfo::Compiler.ArgInfo, si::Compiler.StmtInfo, sv::Compiler.InferenceState, max_methods::Int) + arginfo::Compiler.ArgInfo, si::Compiler.StmtInfo, vtypes::Union{Compiler.VarTable,Nothing}, sv::Compiler.InferenceState, max_methods::Int) return Compiler.Future{Compiler.CallMeta}(ret, interp, sv) do ret, interp, sv if sv.mod in noinline_modules(interp) (;rt, exct, effects, info) = ret @@ -494,29 +538,32 @@ struct CustomData inferred CustomData(@nospecialize inferred) = new(inferred) end -function Compiler.transform_result_for_cache(interp::CustomDataInterp, result::Compiler.InferenceResult, edges::Core.SimpleVector) +function Compiler.transform_result_for_cache( + interp::CustomDataInterp, result::Compiler.InferenceResult, edges::Core.SimpleVector) inferred_result = @invoke Compiler.transform_result_for_cache( interp::Compiler.AbstractInterpreter, result::Compiler.InferenceResult, edges::Core.SimpleVector) return CustomData(inferred_result) end -function Compiler.src_inlining_policy(interp::CustomDataInterp, @nospecialize(src), - @nospecialize(info::Compiler.CallInfo), stmt_flag::UInt32) +function Compiler.src_inlining_policy( + interp::CustomDataInterp, @nospecialize(src), @nospecialize(info::Compiler.CallInfo), + stmt_flag::UInt32) if src isa CustomData src = src.inferred end - return @invoke Compiler.src_inlining_policy(interp::Compiler.AbstractInterpreter, src::Any, - info::Compiler.CallInfo, stmt_flag::UInt32) + return @invoke Compiler.src_inlining_policy( + interp::Compiler.AbstractInterpreter, src::Any, info::Compiler.CallInfo, + stmt_flag::UInt32) end Compiler.retrieve_ir_for_inlining(cached_result::CodeInstance, src::CustomData) = Compiler.retrieve_ir_for_inlining(cached_result, src.inferred) Compiler.retrieve_ir_for_inlining(mi::MethodInstance, src::CustomData, preserve_local_sources::Bool) = Compiler.retrieve_ir_for_inlining(mi, src.inferred, preserve_local_sources) let src = code_typed((Int,); interp=CustomDataInterp()) do x - return sin(x) + cos(x) + return (@noinline sin(x)) + (@noinline cos(x)) end |> only |> first @test count(isinvoke(:sin), src.code) == 1 @test count(isinvoke(:cos), src.code) == 1 - @test count(isinvoke(:+), src.code) == 0 + @test_broken count(isinvoke(:+), src.code) == 0 end # ephemeral cache mode @@ -525,9 +572,9 @@ func_ext_cache1(a) = func_ext_cache2(a) * cos(a) func_ext_cache2(a) = sin(a) let interp = DebugInterp() @test Base.infer_return_type(func_ext_cache1, (Float64,); interp) === Float64 - @test isdefined(interp, :code_cache) + @test isdefined(interp, :global_cache) found = false - for (mi, codeinst) in interp.code_cache.dict + for (mi, codeinst) in interp.global_cache.dict if mi.def.name === :func_ext_cache2 found = true break @@ -538,7 +585,7 @@ end @newinterp InvokeInterp struct InvokeOwner end -codegen = IdDict{CodeInstance, CodeInfo}() +global codegen::IdDict{CodeInstance, CodeInfo} = IdDict{CodeInstance, CodeInfo}() Compiler.cache_owner(::InvokeInterp) = InvokeOwner() Compiler.codegen_cache(::InvokeInterp) = codegen let interp = InvokeInterp() @@ -562,3 +609,143 @@ let interp = InvokeInterp() @test isa(result, String) @test contains(result, "[1] error(::Char, ::Char, ::Char, ::Char)") end + +# Global publication and per-interpreter source/ABI capability are selected separately, +# including when a winner appears while inference is running. +@newinterp SourceModeWinnerInterp +global source_mode_codegen::IdDict{CodeInstance,CodeInfo} = IdDict{CodeInstance,CodeInfo}() +Compiler.codegen_cache(::SourceModeWinnerInterp) = source_mode_codegen + +source_mode_inadequate_winner(x::Int) = x + 1 +let interp = SourceModeWinnerInterp() + mi = Base.method_instance(source_mode_inadequate_winner, (Int,)) + inadequate = Core.CodeInstance(mi, Compiler.cache_owner(interp), Int, Any, + nothing, nothing, zero(Int32), UInt(1), typemax(UInt), zero(UInt32), + nothing, nothing, Core.svec()) + Compiler.code_cache(interp)[mi] = inadequate + @test !Compiler.ci_has_source(interp, inadequate) + + ci = Compiler.typeinf_ext(interp, mi, Compiler.SOURCE_MODE_ABI) + @test ci !== inadequate + @test Compiler.ci_has_source(interp, ci) + @test iszero(@ccall jl_mi_cache_has_ci(mi::Any, ci::Any)::Cint) + # The capability-blind global winner remains unique. The source-capable result + # is session-local and can be reused by this interpreter without allowing it to + # escape into the global executable cache. + @test get(Compiler.code_cache(interp), mi, nothing) === inadequate + @test Compiler.typeinf_ext(interp, mi, Compiler.SOURCE_MODE_ABI) === ci + overlay = Compiler.OverlayCodeCache( + Compiler.code_cache(interp), Compiler.InferenceCache()) + valid_worlds = Compiler.WorldRange(interp.world) + @test Compiler.find_cached_ci(interp, overlay, mi, + valid_worlds, Compiler.SOURCE_MODE_ABI) === nothing + @test Compiler.find_local_cached_ci(interp, mi, + valid_worlds, Compiler.SOURCE_MODE_ABI) === ci + + # JIT compilation uses the local source to compile the ABI-equivalent global + # winner; the local CI itself remains outside the executable cache. + @test Compiler.typeinf_ext_toplevel( + interp, mi, Compiler.SOURCE_MODE_ABI) === inadequate + @test Compiler.ci_has_invoke(inadequate) + @test iszero(@ccall jl_mi_cache_has_ci(mi::Any, ci::Any)::Cint) + # `inadequate` lives in the native `mi.cache` chain (an `InternalCodeCache` + # with a custom owner), which already roots it for the process lifetime, so + # the JIT handoff must not have leaked a redundant global root for it. + @test (@ccall jl_as_global_root(inadequate::Any, 0::Cint)::Ptr{Cvoid}) == C_NULL +end + +# A source-inadequate winner with a different return ABI cannot suppress normal +# publication. The completed CI is globally inserted and promoted before JIT use. +source_mode_nonequivalent_winner(x::Int) = x + 1 +let interp = SourceModeWinnerInterp() + mi = Base.method_instance(source_mode_nonequivalent_winner, (Int,)) + inadequate = Core.CodeInstance(mi, Compiler.cache_owner(interp), Any, Any, + nothing, nothing, zero(Int32), UInt(1), typemax(UInt), zero(UInt32), + nothing, nothing, Core.svec()) + Compiler.code_cache(interp)[mi] = inadequate + + ci = Compiler.typeinf_ext_toplevel(interp, mi, Compiler.SOURCE_MODE_ABI) + @test ci !== inadequate + @test ci.rettype === Int + @test !iszero(@ccall jl_mi_cache_has_ci(mi::Any, ci::Any)::Cint) + @test ci.max_world == typemax(UInt) + + @eval source_mode_world_bump_62338() = nothing + newer = SourceModeWinnerInterp(; world=Base.get_world_counter()) + @test Compiler.typeinf_ext_toplevel( + newer, mi, Compiler.SOURCE_MODE_ABI) === ci +end + +# Equivalent-winner selection also goes through the cache abstraction rather than +# assuming that every executable cache is the native MethodInstance chain. +@newinterp SourceModeEphemeralInterp true +global source_mode_ephemeral_codegen::IdDict{CodeInstance,CodeInfo} = + IdDict{CodeInstance,CodeInfo}() +Compiler.codegen_cache(::SourceModeEphemeralInterp) = source_mode_ephemeral_codegen +source_mode_ephemeral_winner(x::Int) = x + 1 +let interp = SourceModeEphemeralInterp() + mi = Base.method_instance(source_mode_ephemeral_winner, (Int,)) + winner = Core.CodeInstance(mi, Compiler.cache_owner(interp), Int, Any, + nothing, nothing, zero(Int32), UInt(1), typemax(UInt), zero(UInt32), + nothing, nothing, Core.svec()) + Compiler.code_cache(interp)[mi] = winner + + @test Compiler.typeinf_ext_toplevel( + interp, mi, Compiler.SOURCE_MODE_ABI) === winner + @test Compiler.code_cache(interp)[mi] === winner + @test Compiler.ci_has_invoke(winner) + # The JIT retains raw pointers to the emitted `winner` for the lifetime of + # the process, and this ephemeral cache dies with `interp`, so the JIT + # handoff must have promoted `winner` to a global root (`jit_cache_root!`). + @test (@ccall jl_as_global_root(winner::Any, 0::Cint)::Ptr{Cvoid}) != C_NULL + empty!(source_mode_ephemeral_codegen) +end + +const source_mode_interp_ref = Ref{Any}() +const source_mode_winner_ref = Ref{Any}() +@generated function source_mode_publish_winner() + interp = source_mode_interp_ref[]::SourceModeWinnerInterp + winner = source_mode_winner_ref[]::Core.CodeInstance + Compiler.code_cache(interp)[winner.def] = winner + return :(nothing) +end +source_mode_qualifying_winner(x::Int) = (source_mode_publish_winner(); x + 1) +let interp = SourceModeWinnerInterp() + mi = Base.method_instance(source_mode_qualifying_winner, (Int,)) + winner = Core.CodeInstance(mi, Compiler.cache_owner(interp), Int, Any, + nothing, nothing, zero(Int32), UInt(1), typemax(UInt), zero(UInt32), + nothing, nothing, Core.svec()) + source_mode_codegen[winner] = Compiler.retrieve_code_info(mi, interp.world) + source_mode_interp_ref[] = interp + source_mode_winner_ref[] = winner + + ci = Compiler.typeinf_ext(interp, mi, Compiler.SOURCE_MODE_ABI) + @test ci === winner + local_results = [ + entry for entry in Compiler.get_inference_cache(interp).results + if entry isa Compiler.LocalInferenceResult && entry.result.linfo === mi + ] + @test length(local_results) == 1 + local_result = only(local_results) + @test local_result.result.replacement_ci === winner + @test iszero(@ccall jl_mi_cache_has_ci(mi::Any, local_result.result.ci::Any)::Cint) + source_mode_interp_ref[] = nothing + source_mode_winner_ref[] = nothing + empty!(source_mode_codegen) +end + +# The executable cache for a custom interpreter remains CodeInstance-only even when +# inference also retains completed local source/proof entries. +using REPL.REPLCompletions: completions +@newinterp OverlayCacheInterp true +@test let + interp = OverlayCacheInterp() + # `completions` has a call graph deep enough to exercise repeated global and local + # cache lookups for the same MethodInstances. + f = completions + args = ("", 0) + mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, + Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} + Compiler.typeinf_ext_toplevel(interp, mi, Compiler.SOURCE_MODE_NOT_REQUIRED) + true +end diff --git a/Compiler/test/CompilerLoadingTest/Manifest.toml b/Compiler/test/CompilerLoadingTest/Manifest.toml index 7fb3452a61017..a3f26e8cbc6bd 100644 --- a/Compiler/test/CompilerLoadingTest/Manifest.toml +++ b/Compiler/test/CompilerLoadingTest/Manifest.toml @@ -1,16 +1,22 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-DEV" -manifest_format = "2.0" +julia_version = "1.14.0-DEV" +manifest_format = "2.1" project_hash = "10c2816629fed766649b89eb6670e7001df6ea18" [[deps.Compiler]] path = "../.." uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" -version = "0.0.1" +version = "0.1.1" + + [deps.Compiler.syntax] + julia_version = "1.14.0" [[deps.CompilerLoadingTest]] deps = ["Compiler"] path = "." uuid = "95defb8a-f82d-44d7-b2c9-37d658f648c1" version = "0.0.0" + + [deps.CompilerLoadingTest.syntax] + julia_version = "1.14.0-DEV" diff --git a/Compiler/test/CompilerLoadingTest/Project.toml b/Compiler/test/CompilerLoadingTest/Project.toml index 5dca932dc7997..40d1af4434292 100644 --- a/Compiler/test/CompilerLoadingTest/Project.toml +++ b/Compiler/test/CompilerLoadingTest/Project.toml @@ -3,3 +3,6 @@ uuid = "95defb8a-f82d-44d7-b2c9-37d658f648c1" [deps] Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" + +[sources] +Compiler = {path = "../.."} diff --git a/Compiler/test/EAUtils.jl b/Compiler/test/EAUtils.jl index 990a7de3b8141..ec2e589a2cec0 100644 --- a/Compiler/test/EAUtils.jl +++ b/Compiler/test/EAUtils.jl @@ -11,8 +11,8 @@ using .Compiler: EscapeAnalysis as EA # imports import .Compiler: - AbstractInterpreter, NativeInterpreter, WorldView, WorldRange, InferenceParams, - OptimizationParams, get_world_counter, get_inference_cache, ipo_dataflow_analysis! + AbstractInterpreter, InferenceParams, OptimizationParams, + get_world_counter, get_inference_cache, ipo_dataflow_analysis! # usings using Core.IR using .Compiler: InferenceResult, InferenceState, OptimizationState, IRCode @@ -31,7 +31,7 @@ mutable struct EscapeAnalyzer <: AbstractInterpreter const world::UInt const inf_params::InferenceParams const opt_params::OptimizationParams - const inf_cache::Vector{InferenceResult} + const inf_cache::Compiler.InferenceCache const token::EscapeAnalyzerCacheToken const entry_mi::Union{Nothing,MethodInstance} result::EscapeResultForEntry @@ -39,7 +39,7 @@ mutable struct EscapeAnalyzer <: AbstractInterpreter entry_mi::Union{Nothing,MethodInstance}=nothing) inf_params = InferenceParams() opt_params = OptimizationParams() - inf_cache = InferenceResult[] + inf_cache = Compiler.InferenceCache() return new(world, inf_params, opt_params, inf_cache, cache_token, entry_mi) end end @@ -296,7 +296,7 @@ while caching the analysis results. controls the world age to use when looking up methods, use current world age if not specified. - `cache_token::EscapeAnalyzerCacheToken = GLOBAL_EA_CACHE_TOKEN`: specifies the cache token to use, by default a global token is used so that the analysis - can use the caches from previous invocations. If you with to use a fresh cache and perform + can use the caches from previous invocations. If you wish to use a fresh cache and perform a new analysis, specify a new `EscapeAnalyzerCacheToken` instance. - `interp::EscapeAnalyzer = EscapeAnalyzer(world, cache_token)`: specifies the escape analyzer to use. @@ -340,7 +340,7 @@ Note that this version does not cache the analysis results. controls the world age to use when looking up methods, use current world age if not specified. - `cache_token::EscapeAnalyzerCacheToken = GLOBAL_EA_CACHE_TOKEN`: specifies the cache token to use, by default a global token is used so that the analysis - can use the caches from previous invocations. If you with to use a fresh cache and perform + can use the caches from previous invocations. If you wish to use a fresh cache and perform a new analysis, specify a new `EscapeAnalyzerCacheToken` instance. - `interp::AbstractInterpreter = EscapeAnalyzer(world, cache_token)`: specifies the abstract interpreter to use, by default a new `EscapeAnalyzer` with an empty cache is created. diff --git a/Compiler/test/EscapeAnalysis.jl b/Compiler/test/EscapeAnalysis.jl index 07855d3362881..642e72e1f792a 100644 --- a/Compiler/test/EscapeAnalysis.jl +++ b/Compiler/test/EscapeAnalysis.jl @@ -35,13 +35,14 @@ let utils_ex = quote Core.eval(@__MODULE__, utils_ex) end -using .EscapeAnalysis: EscapeInfo, IndexableFields +using .EscapeAnalysis: EscapeInfo, IndexableFields, Unindexable, + NoEscape, ArgEscape, ReturnEscape, ThrownEscape, AllEscape isϕ(@nospecialize x) = isa(x, Core.PhiNode) """ is_load_forwardable(x::EscapeInfo) -> Bool -Queries if `x` is elibigle for store-to-load forwarding optimization. +Queries if `x` is eligible for store-to-load forwarding optimization. """ function is_load_forwardable(x::EscapeInfo) AliasInfo = x.AliasInfo @@ -682,6 +683,13 @@ end end end @test has_all_escape(result.state[Argument(2)]) + result = @eval M begin + $code_escapes((String,)) do s + $(M.___xxx___.Rx)[] = s + nothing + end + end + @test has_all_escape(result.state[Argument(2)]) end # field escape @@ -810,7 +818,7 @@ end r = only(findall(isreturn, result.ir.stmts.stmt)) findall(1:length(result.ir.stmts)) do i if isnew(result.ir.stmts[i][:stmt]) - t = result.ir.stmts[i][:type] + t = Compiler.widenconst(result.ir.stmts[i][:type]) return t === SafeRef{String} || # o1 t === SafeRef{SafeRef} # o2 end @@ -1069,7 +1077,7 @@ end @test is_load_forwardable(result.state[SSAValue(i)]) end for i in findall(isnew, result.ir.stmts.stmt) - if result.ir[SSAValue(i)][:type] <: SafeRef + if Compiler.widenconst(result.ir[SSAValue(i)][:type]) <: SafeRef @test is_load_forwardable(result.state[SSAValue(i)]) end end @@ -1091,7 +1099,7 @@ end @test is_load_forwardable(result.state[SSAValue(i)]) end for i in findall(isnew, result.ir.stmts.stmt) - if result.ir[SSAValue(i)][:type] <: SafeRef + if Compiler.widenconst(result.ir[SSAValue(i)][:type]) <: SafeRef @test is_load_forwardable(result.state[SSAValue(i)]) end end @@ -1388,7 +1396,7 @@ end @test has_return_escape(result.state[Argument(3)], r) # baz @test has_return_escape(result.state[Argument(4)], r) # qux for new in findall(isnew, result.ir.stmts.stmt) - if !(result.ir[SSAValue(new)][:type] <: Base.RefValue) + if !(Compiler.widenconst(result.ir[SSAValue(new)][:type]) <: Base.RefValue) @test is_load_forwardable(result.state[SSAValue(new)]) end end @@ -1448,7 +1456,7 @@ let result = @code_escapes compute(MPoint, 1+.5im, 2+.5im, 2+.25im, 4+.75im) for i in findall(1:length(result.ir.stmts)) do idx inst = result.ir[SSAValue(idx)] stmt = inst[:stmt] - return (isnew(stmt) || isϕ(stmt)) && inst[:type] <: MPoint + return (isnew(stmt) || isϕ(stmt)) && Compiler.widenconst(inst[:type]) <: MPoint end @test is_load_forwardable(result.state[SSAValue(i)]) end @@ -1464,7 +1472,7 @@ end # idxs = findall(1:length(result.ir.stmts)) do idx # inst = result.ir[SSAValue(idx)] # stmt = inst[:stmt] -# return isnew(stmt) && inst[:type] <: MPoint +# return isnew(stmt) && Compiler.widenconst(inst[:type]) <: MPoint # end # @assert length(idxs) == 2 # @test count(i->is_load_forwardable(result.state[SSAValue(i)]), idxs) == 1 @@ -1481,7 +1489,7 @@ let result = @code_escapes compute!(MPoint(1+.5im, 2+.5im), MPoint(2+.25im, 4+.7 for i in findall(1:length(result.ir.stmts)) do idx inst = result.ir[SSAValue(idx)] stmt = inst[:stmt] - return isnew(stmt) && inst[:type] <: MPoint + return isnew(stmt) && Compiler.widenconst(inst[:type]) <: MPoint end @test is_load_forwardable(result.state[SSAValue(i)]) end @@ -1709,4 +1717,15 @@ end @test (@code_escapes scope_folding()) isa EAUtils.EscapeResult @test (@code_escapes scope_folding_opt()) isa EAUtils.EscapeResult +@testset "EscapeInfo hash consistent with ==" begin + @test hash(NoEscape()) == hash(NoEscape()) + @test hash(EscapeInfo(NoEscape(), IndexableFields(2))) == + hash(EscapeInfo(NoEscape(), IndexableFields(2))) + @test hash(EscapeInfo(NoEscape(), Unindexable())) == + hash(EscapeInfo(NoEscape(), Unindexable())) + @test allunique(hash.([NoEscape(), ArgEscape(), ReturnEscape(1), ThrownEscape(1), + AllEscape(), EscapeInfo(NoEscape(), IndexableFields(2)), + EscapeInfo(NoEscape(), Unindexable())])) +end + end # module test_EA diff --git a/Compiler/test/abioverride.jl b/Compiler/test/abioverride.jl index feb992b27ee43..01be7cafbbef0 100644 --- a/Compiler/test/abioverride.jl +++ b/Compiler/test/abioverride.jl @@ -4,7 +4,7 @@ using Base.Meta include("irutils.jl") # In this test, we will manually construct a CodeInstance that specializes the `myplus` -# method on a constant for the second argument and test various, interfaces surrounding +# method on a constant for the second argument and test various interfaces surrounding # CodeInstances with ABI overrides. myplus(x::Int, y::Int) = x + y @@ -54,7 +54,7 @@ let world = Base.tls_world_age() ccall(:jl_mi_cache_insert, Cvoid, (Any, Any), mi, new_ci) # Poke the source code into the JIT for it - ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), new_ci, new_source) + ccall(:jl_add_codeinsts_to_jit, Cvoid, (Any, Any), Any[new_ci], Any[new_source]) end @test contains(repr(new_ci), "ABI Overridden") diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index eb5eae29cdc54..ceaee5ebb2718 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -22,7 +22,7 @@ end # The tests below assume a certain format and safepoint_on_entry=true breaks that. function get_llvm(@nospecialize(f), @nospecialize(t), raw=true, dump_module=false, optimize=true) params = Base.CodegenParams(safepoint_on_entry=false, gcstack_arg = false, debug_info_level=Cint(2)) - d = InteractiveUtils._dump_function(InteractiveUtils.ArgInfo(f, t), false, false, raw, dump_module, :att, optimize, :none, false, params) + d = InteractiveUtils._dump_function(InteractiveUtils.ArgInfo(f, t), false, false, raw, dump_module, :att, optimize, :none, false, "", params) sprint(print, d) end @@ -134,14 +134,14 @@ if !is_debug_build && opt_level > 0 # Array test_loads_no_call(strip_debug_calls(get_llvm(sizeof, Tuple{Vector{Int}})), [Iptr]) # As long as the eltype is known we don't need to load the elsize, but do need to check isvector - @test_skip test_loads_no_call(strip_debug_calls(get_llvm(sizeof, Tuple{Array{Any}})), ["atomic $Iptr", "ptr", "ptr", Iptr, Iptr, "ptr", Iptr]) + @test_skip test_loads_no_call(strip_debug_calls(get_llvm(sizeof, Tuple{Array{Any}})), ["atomic volatile $Iptr", "ptr", "ptr", Iptr, Iptr, "ptr", Iptr]) # Memory test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory{Int}})), [Iptr]) # As long as the eltype is known we don't need to load the elsize test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory{Any}})), [Iptr]) # Check that we load the elsize and isunion from the typeof layout - test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory})), [Iptr, "atomic $Iptr", "ptr", "i32", "i16"]) - test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory})), [Iptr, "atomic $Iptr", "ptr", "i32", "i16"]) + test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory})), [Iptr, "atomic volatile $Iptr", "ptr", "i32", "i16"]) + test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory})), [Iptr, "atomic volatile $Iptr", "ptr", "i32", "i16"]) # Primitive Type size should be folded to a constant test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Ptr})), String[]) @@ -410,7 +410,10 @@ function g_dict_hash_alloc() end # Warm up f_dict_hash_alloc(); g_dict_hash_alloc(); -@test abs((@allocated f_dict_hash_alloc()) / (@allocated g_dict_hash_alloc()) - 1) < 0.3 +# Take the minimum of several runs so that a one-time allocation inside the +# measured call (e.g. lazy compilation of a call target) does not skew the ratio +min_dict_hash_alloc(f) = minimum(@allocated(f()) for _ in 1:3) +@test abs(min_dict_hash_alloc(f_dict_hash_alloc) / min_dict_hash_alloc(g_dict_hash_alloc) - 1) < 0.3 # returning an argument shouldn't alloc a new box @noinline f33829(x) = (global called33829 = true; x) @@ -802,7 +805,8 @@ function f34459(args...) Base.pointerset(args[1], 1, 1, 1) return end -@test !occursin("jl_f_tuple", get_llvm(f34459, Tuple{Ptr{Int}, Type{Int}}, true, false, false)) +# only the egality type `TypeEgal{Int}` is a ghost that codegen can elide (#61323) +@test !occursin("jl_f_tuple", get_llvm(f34459, Tuple{Ptr{Int}, Core.TypeEgal{Int}}, true, false, false)) # issue #48394: incorrectly-inferred getproperty shouldn't introduce invalid cgval_t # when dealing with unions of ghost values @@ -883,7 +887,7 @@ let io = IOBuffer() code_llvm(io,foo54166, (Vector{Union{Missing,Int}}, Int, Int), dump_module=true, raw=true) str = String(take!(io)) @test !occursin("jtbaa_unionselbyte", str) - @test occursin("jtbaa_arrayselbyte", str) + @test occursin("jtbaa_memoryselbyte", str) end ex54166 = Union{Missing, Int64}[missing -2; missing -2]; @@ -1073,5 +1077,153 @@ end let io = IOBuffer() code_llvm(io, (x, y) -> (@atomic x[1] = y; nothing), (AtomicMemory{Pair{Any,Any}}, Pair{Any,Any},), raw=true, optimize=false) str = String(take!(io)) - @test occursin("julia.write_barrier", str) + @test occursin("julia.field_write_barrier", str) +end + +# Aggregate barriers identify the stored payload's references, including locked elements. +struct FieldBarrierElement + tag::Int + a::Any + b::Any + c::Any +end +unset_field_barrier(r, ::Val{order}) where {order} = Core.memoryrefunset!(r, order, false) +set_field_barrier(r, x, ::Val{order}) where {order} = (Core.memoryrefset!(r, x, order, false); nothing) +swap_field_barrier(r, x, ::Val{order}) where {order} = Core.memoryrefswap!(r, x, order, false) +set_object_field_barrier(r, x) = (r[] = x; nothing) + +@testset "aggregate field barriers" begin + T = FieldBarrierElement + function check_slots(ir, slot_as; clear=false) + lines = split(ir, '\n') + barriers = filter(line -> occursin("call void", line) && occursin("@julia.field_write_barrier", line), lines) + @test length(barriers) == 1 + isempty(barriers) && return + slots = [m.captures[1] for m in eachmatch(r"ptr addrspace\((?:11|13)\) (%[^ ,]+), ptr addrspace\(10\)", only(barriers))] + @test length(slots) == 3 + @test occursin("@julia.field_write_barrier.p$slot_as", only(barriers)) + if clear + @test length(collect(eachmatch(r"ptr addrspace\(10\) null", only(barriers)))) == 3 + end + geps = Dict(m.captures[1] => (m.captures[2], parse(Int, m.captures[3])) + for m in eachmatch(r"(%[^ ,]+) = getelementptr(?: inbounds)? i8, ptr addrspace\((?:11|13)\) (%[^ ,]+), i(?:32|64) ([0-9]+)", ir)) + payloads = String[] + offsets = Int[] + for slot in slots + @test haskey(geps, slot) + haskey(geps, slot) || continue + payload, offset = geps[slot] + push!(payloads, payload) + push!(offsets, offset) + end + @test offsets == [fieldoffset(T, i) for i in 2:4] + @test length(unique(payloads)) == 1 + if clear && !isempty(payloads) + @test any(line -> occursin("store ", line) && occursin("zeroinitializer, ptr addrspace(13) $(first(payloads)),", line), lines) + end + end + for (M, order) in ((Memory{T}, :not_atomic), (AtomicMemory{T}, :sequentially_consistent)) + R = typeof(GenericMemoryRef(M(undef, 0))) + check_slots(get_llvm(unset_field_barrier, Tuple{R,Val{order}}, true, false, false), 13; clear=true) + for f in (set_field_barrier, swap_field_barrier) + check_slots(get_llvm(f, Tuple{R,T,Val{order}}, true, false, false), 13) + end + end + check_slots(get_llvm(set_object_field_barrier, Tuple{Base.RefValue{T},T}, true, false, false), 11) +end + +# Cancellation-token clears and rebinds must barrier the slot before storing it. +cancellation_binding_barrier(src) = Core.cancellation_point!(src) +@testset "cancellation binding barriers" begin + ir = get_llvm(cancellation_binding_barrier, Tuple{Union{Nothing,Core.CancellationTokenSource}}, true, false, false) + lines = split(ir, '\n') + barriers = findall(line -> occursin("call void", line) && occursin("@julia.field_write_barrier.p11", line), lines) + @test length(barriers) == 2 + casts = Dict(m.captures[1] => m.captures[2] + for m in eachmatch(r"(%[^ ,]+) = addrspacecast ptr (%[^ ,]+) to ptr addrspace\(11\)", ir)) + for i in barriers + operands = match(r"@julia.field_write_barrier.p11\(ptr addrspace\(10\) [^,]+, ptr addrspace\(11\) ([^,]+), ptr addrspace\(10\) (.*)\)", lines[i]) + @test operands !== nothing + operands === nothing && continue + slot, child = operands.captures + destination = haskey(casts, slot) ? "ptr $(casts[slot])" : "ptr addrspace(11) $slot" + @test occursin("store atomic ptr addrspace(10) $child, $destination", lines[i + 1]) + end +end + +# Test phi node codegen for union types with inline roots +function union_phi_inline_roots(x::Bool) + if x + return ("Q8", 1) + else + return ("Q10", Ref(5)) + end +end +@test union_phi_inline_roots(true) === ("Q8", 1) + +mutable struct AnyBoxEA val::Any end +function preserve_any_ea(x) + b = AnyBoxEA(x) + GC.@preserve b begin + return b.val + end +end +function loop_preserve_any_ea(n) + s = "v" + for _ in 1:n + s = preserve_any_ea(s)::String + end + s +end +loop_preserve_any_ea(10) +@test (@allocated loop_preserve_any_ea(10)) == 0 + +# blackbox compiles to zero-cost inline asm, not a runtime call +@testset "blackbox codegen" begin + # Scalar blackbox: should produce inline asm, no call + blackbox_int(x::Int) = Base.blackbox(x) + ir_int = get_llvm(blackbox_int, Tuple{Int}) + @test !occursin("call ", strip_debug_calls(ir_int)) || occursin("asm", ir_int) + @test !occursin("jl_", ir_int) + + # Pointer/boxed blackbox: should produce julia.blackbox intrinsic (lowered to asm after GC) + blackbox_str(x::String) = Base.blackbox(x) + ir_str = get_llvm(blackbox_str, Tuple{String}, true, false, false) + @test occursin("julia.blackbox", ir_str) + + # blackbox preserves value identity at runtime + @test Base.blackbox(42) == 42 + @test Base.blackbox([1,2,3]) == [1,2,3] + + # blackbox does not allocate for isbits types + @test (@allocated Base.blackbox(42)) == 0 + + # No gc frame needed for scalar blackbox + @test !occursin("%gcframe", get_llvm(blackbox_int, Tuple{Int})) + + # Struct blackbox: uses memory clobber for unboxed aggregates + struct BlackboxTestStruct + a::Float64 + b::Float64 + end + blackbox_struct(x::BlackboxTestStruct) = Base.blackbox(x) + @test blackbox_struct(BlackboxTestStruct(1.0, 2.0)) == BlackboxTestStruct(1.0, 2.0) + ir_struct = get_llvm(blackbox_struct, Tuple{BlackboxTestStruct}) + @test occursin("~{memory}", ir_struct) + @test !occursin("jl_", strip_debug_calls(ir_struct)) + + # blackbox barriers Julia-level constprop: return type must be Int, not Const(42) + @test Base.return_types() do; Base.blackbox(42); end |> only === Int +end + +# sret parameters must have an alignment attribute (required by LLVM LangRef). +@testset "sret alignment attribute" begin + struct SretAlignTest + a::Float32 + b::Float32 + c::Float32 + end + @noinline f_srettest(x::Float32) = SretAlignTest(x, x+1, x+2) + ir = get_llvm(f_srettest, Tuple{Float32}, true, true, true) + @test occursin(r"sret\([^)]+\) align \d+", ir) end diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index 18cfc0e8d5388..bc06ef845e65e 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -46,6 +46,17 @@ end return nothing end +# `supertype(::UnionAll)` can throw (its recursion hits `supertype(::Union)` for +# inputs like `Union{S,T} where {S,T}`), so it must be `:foldable` rather than +# `:total` and a dead call to it must not be eliminated (issue #61988) +@test Compiler.is_foldable(Base.infer_effects(supertype, (UnionAll,))) +@test !Compiler.is_nothrow(Base.infer_effects(supertype, (UnionAll,))) +@test !fully_eliminated((UnionAll,)) do x + supertype(x) + return nothing +end +@test_throws MethodError (x -> (supertype(x); nothing))(Union{S,T} where {S,T}) + # Test that a missing methtable identification gets tainted # appropriately struct FCallback; f::Union{Nothing, Function}; end @@ -266,6 +277,9 @@ end |> Compiler.is_consistent @test Base.infer_effects() do Maybe{String}()[] end |> Compiler.is_consistent +@test Base.infer_effects() do + Maybe{Some{Base.RefValue{Int}}}() +end |> Compiler.is_consistent let f() = Maybe{String}()[] @test Base.return_types() do f() # this call should be concrete evaluated @@ -369,7 +383,7 @@ let effects = Base.infer_effects(f_glob_assign_int, (); optimize=false) @test !Compiler.is_effect_free(effects) @test Compiler.is_nothrow(effects) end -# effects modeling for for setglobal! +# effects modeling for setglobal! global SETGLOBAL!_NOTHROW::Int = 0 let effects = Base.infer_effects(; optimize=false) do setglobal!(@__MODULE__, :SETGLOBAL!_NOTHROW, 42) @@ -927,7 +941,8 @@ unknown_sparam_nothrow1(x::Ref{T}) where T = (T; nothing) unknown_sparam_nothrow2(x::Ref{Ref{T}}) where T = (T; nothing) @test Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Type{Int},))) @test Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Type{<:Integer},))) -@test !Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Type,))) +@test Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Type{Ref{T}} where {T},))) +@test Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Type,))) @test !Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Nothing,))) @test !Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Union{Type{Int},Nothing},))) @test !Compiler.is_nothrow(Base.infer_effects(unknown_sparam_throw, (Any,))) @@ -1008,6 +1023,19 @@ end @isdefined($(gensym("some_undef_symbol"))) end |> !Compiler.is_consistent +# `@isdefined`-guarded read of a slot whose value is a `MustAlias` must still refine +# the slot's `undef` info +function isdefined_alias_loop(t::Tuple) + local prev + s = "" + for x in t + @isdefined(prev) && (s = prev) + prev = x + end + return s +end +@test Compiler.is_nothrow(Base.infer_effects(isdefined_alias_loop, (Tuple{String,String},))) + # Effects of Base.hasfield (#50198) hf50198(s) = hasfield(typeof((;x=1, y=2)), s) f50198() = (hf50198(Ref(:x)[]); nothing) @@ -1046,9 +1074,13 @@ end |> Compiler.is_nothrow # Effects for :compilerbarrier f1_compilerbarrier(b) = Base.compilerbarrier(:type, b) f2_compilerbarrier(b) = Base.compilerbarrier(:conditional, b) +f3_compilerbarrier(b) = Base.compilerbarrier(:blackbox, b) @test !Compiler.is_consistent(Base.infer_effects(f1_compilerbarrier, (Bool,))) @test Compiler.is_consistent(Base.infer_effects(f2_compilerbarrier, (Bool,))) +# :blackbox is not consistent (prevents CSE/constant-folding) but is nothrow +@test !Compiler.is_consistent(Base.infer_effects(f3_compilerbarrier, (Bool,))) +@test Compiler.is_nothrow(Base.infer_effects(f3_compilerbarrier, (Bool,))) # Optimizer-refined effects function f1_optrefine(b) @@ -1440,6 +1472,11 @@ let effects = Base.infer_effects(Core.Intrinsics.pointerset, Tuple{Vararg{Any}}) @test Compiler.is_consistent(effects) @test !Compiler.is_effect_free(effects) end +@test Compiler.intrinsic_nothrow(Core.Intrinsics.add_ptr, Any[Ptr{Int}, UInt]) +@test Compiler.intrinsic_nothrow(Core.Intrinsics.sub_ptr, Any[Ptr{Int}, UInt]) +@test !Compiler.intrinsic_nothrow(Core.Intrinsics.add_ptr, Any[UInt, UInt]) +@test !Compiler.intrinsic_nothrow(Core.Intrinsics.sub_ptr, Any[UInt, UInt]) +@test Compiler.is_nothrow(Base.infer_effects(+, Tuple{Ptr{UInt8}, UInt})) # effects modeling for atomic intrinsics # these functions especially need to be marked !effect_free since they imply synchronization for atomicfunc = Any[ @@ -1460,7 +1497,7 @@ let effects = Base.infer_effects(Core.Intrinsics.atomic_pointermodify, Tuple{Var end # JuliaLang/julia#57780 -let effects = Base.infer_effects(Base._unsetindex!, (MemoryRef{String},)) +let effects = Base.infer_effects(Base.unsetindex!, (MemoryRef{String},)) @test !Compiler.is_effect_free(effects) end @@ -1470,12 +1507,167 @@ end @test Base.infer_effects(invokelatest, Tuple{Vararg{Any}}) == Compiler.Effects() @test Base.infer_effects(invoke, Tuple{Vararg{Any}}) == Compiler.Effects() +bitsizeof_int() = Core.bitsizeof(Int) +let effects = Base.infer_effects(bitsizeof_int) + @test Compiler.is_foldable_nothrow(effects) + @test Compiler.is_inaccessiblememonly(effects) +end + # Core._svec_ref effects modeling (required for external abstract interpreter that doesn't run optimization) let effects = Base.infer_effects((Core.SimpleVector,Int); optimize=false) do svec, i Core._svec_ref(svec, i) end - @test !Compiler.is_consistent(effects) + @test Compiler.is_consistent(effects) + @test Compiler.is_effect_free(effects) + @test !Compiler.is_nothrow(effects) + @test Compiler.is_terminates(effects) +end + +@test Compiler.is_nothrow(Base.infer_effects(length, (Core.SimpleVector,))) + + +# https://github.com/JuliaLang/julia/issues/60009 +function null_offset(offset) + Ptr{UInt8}(C_NULL) + offset +end +@test null_offset(Int(100)) == Ptr{UInt8}(UInt(100)) + +# https://github.com/JuliaLang/julia/issues/61435 +function catch_error_61435(f, x) + try + f(x) + catch + return :caught + end +end +let f = (x) -> Core.Intrinsics.sext_int(Int16, x) + @test Compiler.is_nothrow(Base.infer_effects(f, (Int8,))) + @test !Compiler.is_nothrow(Base.infer_effects(f, (Int16,))) + @test !Compiler.is_nothrow(Base.infer_effects(f, (Int32,))) + @test catch_error_61435(f, Int16(0)) === :caught +end +let f = (x) -> Core.Intrinsics.zext_int(UInt16, x) + @test Compiler.is_nothrow(Base.infer_effects(f, (UInt8,))) + @test !Compiler.is_nothrow(Base.infer_effects(f, (UInt16,))) + @test !Compiler.is_nothrow(Base.infer_effects(f, (UInt32,))) + @test catch_error_61435(f, UInt16(0)) === :caught +end +let f = (x) -> Core.Intrinsics.trunc_int(Int16, x) + @test !Compiler.is_nothrow(Base.infer_effects(f, (Int8,))) + @test !Compiler.is_nothrow(Base.infer_effects(f, (Int16,))) + @test Compiler.is_nothrow(Base.infer_effects(f, (Int32,))) + @test catch_error_61435(f, Int16(0)) === :caught +end + +# Intrinsic width checks use logical primitive widths rather than storage sizes. +primitive type EffectsUInt17 17 end +primitive type EffectsUInt23 23 end +let f = (x) -> Core.Intrinsics.bitcast(EffectsUInt17, x) + @test !Compiler.is_nothrow(Base.infer_effects(f, (EffectsUInt23,))) + @test Base.infer_exception_type(f, (EffectsUInt23,)) === ErrorException + @test !fully_eliminated((EffectsUInt23,)) do x + f(x) + return nothing + end +end +let f = (x) -> Core.Intrinsics.zext_int(EffectsUInt23, x) + @test Compiler.is_nothrow(Base.infer_effects(f, (EffectsUInt17,))) +end +let f = (x) -> Core.Intrinsics.sext_int(EffectsUInt23, x) + @test Compiler.is_nothrow(Base.infer_effects(f, (EffectsUInt17,))) +end +let f = (x) -> Core.Intrinsics.trunc_int(EffectsUInt17, x) + @test Compiler.is_nothrow(Base.infer_effects(f, (EffectsUInt23,))) +end + +# issue #57324 +module Issue57324 +struct T <: AbstractVector{Float64} + m::Memory{UInt64} +end +function f(w) + r = Base.OneTo(w.m[1]) + setindex!(w, 0.0, r[1]) +end +Base.setindex!(w::T, v, i::Int) = _setindex!(w, i) +function _setindex!(w, i) + w.m[w.m[1]] = 0 > i ? nothing : 0 + w +end +Base.size(::T) = (0,) +end +let effects = Base.infer_effects(Issue57324.f, (Issue57324.T,)) + @test Compiler.is_terminates(effects) + @test Compiler.is_notaskstate(effects) + @test Compiler.is_nortcall(effects) +end + +# issue #61590 +@test !Compiler.is_consistent(Base.infer_effects(getproperty, (Core.TypeName, Symbol))) +@test !Compiler.is_consistent(Base.infer_effects(getfield, (Core.TypeName, Symbol))) + +# task_result_type effects modeling (should have !consistent effect) +let effects = Base.infer_effects(Core.task_result_type, (Task,)) + @test !Compiler.is_consistent(effects) # !consistent bit should be set @test Compiler.is_effect_free(effects) + @test Compiler.is_nothrow(effects) + @test Compiler.is_terminates(effects) +end +let effects = Base.infer_effects(Core.task_result_type, (Union{Task,Int},)) + @test Compiler.is_effect_free(effects) + @test !Compiler.is_nothrow(effects) +end +for argtypes in ((), (Int,), (Task, Task)) + @test !Compiler.is_nothrow(Base.infer_effects(Core.task_result_type, argtypes)) +end + +# Core._task effects modeling: creating a task terminates and has no UB, but +# accesses task state (scope inheritance, parent RNG split) and may throw +let effects = Base.infer_effects(Core._task, (Function, Int)) + @test !Compiler.is_consistent(effects) + @test !Compiler.is_effect_free(effects) @test !Compiler.is_nothrow(effects) @test Compiler.is_terminates(effects) + @test !Compiler.is_notaskstate(effects) + @test Compiler.is_noub(effects) +end + +# Every `*_partition` builtin the reformulation pass emits must have its effects modeled by +# `builtin_effects`, so that re-deriving the flags of a reformulated statement is no more +# pessimistic than the `getglobal`/`setglobal!` it replaced. +module PartitionEffects + const c = 42 + global g::Int = 1 +end +let 𝕃 = Compiler.SimpleInferenceLattice.instance, + Const = Compiler.Const, + part(name) = Base.lookup_binding_partition(Base.get_world_counter(), + convert(Core.Binding, GlobalRef(PartitionEffects, name))) + for f in (Core.getglobal_partition, Core.setglobal_partition, Core.swapglobal_partition, + Core.replaceglobal_partition, Core.setglobalonce_partition, + Core.isdefinedglobal_partition, Core.depwarn_partition) + @test f in Compiler._EFFECTS_KNOWN_BUILTINS + end + + effects = Compiler.builtin_effects(𝕃, Core.getglobal_partition, + Any[Const(GlobalRef(PartitionEffects, :c)), Const(part(:c)), Const(:monotonic)], Int) + @test Compiler.is_effect_free(effects) + @test Compiler.is_consistent(effects) + @test !Compiler.is_nothrow(effects) # the memory order argument may be invalid + effects = Compiler.builtin_effects(𝕃, Core.getglobal_partition, + Any[Const(GlobalRef(PartitionEffects, :g)), Const(part(:g)), Const(:monotonic)], Int) + @test Compiler.is_effect_free(effects) + @test !Compiler.is_consistent(effects) + # Only a plain store is `:consistent`; the read-modify-write forms return the old value. + for f in (Core.setglobal_partition, Core.swapglobal_partition, Core.replaceglobal_partition, + Core.setglobalonce_partition) + effects = Compiler.builtin_effects(𝕃, f, Any[Const(part(:g)), Int], Int) + @test !Compiler.is_effect_free(effects) + @test Compiler.is_consistent(effects) === (f === Core.setglobal_partition) + end + Base.deprecate(PartitionEffects, :c) + effects = Compiler.builtin_effects(𝕃, Core.getglobal_partition, + Any[Const(GlobalRef(PartitionEffects, :c)), Const(part(:c)), Const(:monotonic)], Int) + @test !Compiler.is_effect_free(effects) + Base.deprecate(PartitionEffects, :c, 0) end diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index 58cd3db49c371..444433363bc4c 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6,6 +6,7 @@ using Test include("setup_Compiler.jl") include("irutils.jl") +include("newinterp.jl") # tests for Compiler correctness and precision using .Compiler: Conditional, ⊑ @@ -26,6 +27,572 @@ end @test Compiler.limit_type_size(Ref{Complex{T} where T}, Ref, Ref, 100, 0) == Ref @test Compiler.limit_type_size(Ref{Complex{T} where T}, Ref{Complex{T} where T}, Ref, 100, 0) == Ref{Complex{T} where T} +# Local-cache entries contain reusable source plus a proof, while executable targets +# remain exclusively in the global CodeInstance cache. +local_wrapper62338(x::Int) = x + 1 +let + precompile(local_wrapper62338, (Int,)) + interp = Compiler.NativeInterpreter(Base.get_world_counter()) + mi = Base.method_instance(local_wrapper62338, (Int,)) + ci = get(Compiler.code_cache(interp), mi, nothing) + @test ci isa Core.CodeInstance + + pending = Compiler.InferenceResult(mi, Compiler.typeinf_lattice(interp)) + pending.ci = ci + push!(Compiler.get_inference_cache(interp), pending) + @test get(Compiler.code_cache(interp), mi, nothing) === ci + + artifact = Compiler.InferenceResult(mi, Compiler.typeinf_lattice(interp)) + artifact.result = Int + artifact.exc_result = Any + artifact.src = Compiler.retrieve_code_info(mi, Base.get_world_counter()) + artifact.ci = ci + artifact.valid_worlds = Compiler.WorldRange(ci.min_world, Base.get_world_counter()) + proof = Compiler.LocalInferenceProof(artifact.valid_worlds, Core.svec()) + local_result = Compiler.LocalInferenceResult(artifact, proof, Base.get_world_counter()) + push!(Compiler.get_inference_cache(interp), local_result) + got = Compiler.lookup_local_inference_result(interp, mi) + @test got === local_result + @test got.proof === proof + @test got.result.ci === ci + @test get(Compiler.code_cache(interp), mi, nothing) === ci + @test_throws AssertionError push!(Compiler.get_inference_cache(interp), artifact) +end + +# A nested inference session can publish a global CI while an outer SCC member is still +# in progress. No provisional CI that is later kept local may escape into a published +# caller, and every exact CI used by a published SCC edge must itself be published. +wrapper62338_bump() = 0 +wrapper62338_a(x::Int) = x <= 0 ? wrapper62338_bump() : wrapper62338_req(x - 1) +wrapper62338_req(x::Int) = x <= 0 ? 1 : wrapper62338_v(x - 1) + 1 +wrapper62338_v(x::Int) = x <= 0 ? 2 : wrapper62338_w(x - 1) + 2 +wrapper62338_trigger(x::Int) = wrapper62338_v(x) +@generated function wrapper62338_gencache(x) + precompile(wrapper62338_trigger, (Int,)) + return :(x) +end +wrapper62338_w(x::Int) = x <= 0 ? 3 : (r = wrapper62338_a(x - 1); wrapper62338_gencache(x); r + 3) +wrapper62338_root(x::Int) = wrapper62338_a(x) + wrapper62338_v(x) +wrapper62338_bump() = 1 +let interp = Compiler.NativeInterpreter(Base.get_world_counter()) + @test code_typed(wrapper62338_root, (Int,); interp) isa Vector + cycle_methods = (which(wrapper62338_a, (Int,)), which(wrapper62338_req, (Int,)), + which(wrapper62338_v, (Int,)), which(wrapper62338_w, (Int,))) + cache_entries = Compiler.get_inference_cache(interp).results + local_cycle_entries = [ + entry for entry in cache_entries + if entry isa Compiler.LocalInferenceResult && entry.result.linfo.def in cycle_methods + ] + local_cycle_results = map(entry -> entry.result, local_cycle_entries) + @test !isempty(local_cycle_results) + @test any(result -> result.replacement_ci !== nothing, local_cycle_results) + @test all(result -> result.replacement_ci === nothing || + isdefined(result.replacement_ci, :inferred), local_cycle_results) + + published = Core.CodeInstance[] + for f in (wrapper62338_a, wrapper62338_req, wrapper62338_v, wrapper62338_w, + wrapper62338_root) + mi = Base.method_instance(f, (Int,)) + ci = get(Compiler.code_cache(interp), mi, nothing) + if ci isa Core.CodeInstance + push!(published, ci) + end + live_cis = Core.CodeInstance[] + if isdefined(mi, :cache) + cached_ci = mi.cache + while cached_ci isa Core.CodeInstance + if cached_ci.owner === Compiler.cache_owner(interp) && + cached_ci.min_world <= interp.world <= cached_ci.max_world + push!(live_cis, cached_ci) + end + isdefined(cached_ci, :next) || break + cached_ci = cached_ci.next + end + end + @test length(live_cis) <= 1 + end + @test !isempty(published) + bump_method = which(wrapper62338_bump, ()) + @test all(local_cycle_entries) do entry + edges = Compiler.materialize_inference_edges(entry.proof.edges) + any(edges) do edge + edge === bump_method || + (edge isa Core.MethodInstance && edge.def === bump_method) || + (edge isa Core.CodeInstance && edge.def.def === bump_method) + end + end + for caller in published, edge in caller.edges + if edge isa Core.CodeInstance + @test !iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), edge.def, edge)) + end + end + for result in local_cycle_results + isdefined(result, :ci) || continue + if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), result.linfo, result.ci)) + @test all(caller -> all(edge -> edge !== result.ci, caller.edges), published) + end + end +end + +# A limited source may still return clean facts. Its scheduled consumer must retain the +# dependency proof even though the tombstoned source itself is not reusable. +module LimitedSrcTombstoneProof62338 + const FLAG = true + callee() = FLAG ? 1 : 2 + bystander() = 3 + entry() = 1 +end + +# A deferred mutual-SCC edge may consume provisional facts even when the original +# recursion heuristic did not mark the call as an edge cycle. Preserve that as an +# independent invalidation requirement. +let + interp = Compiler.NativeInterpreter(Base.get_world_counter()) + outer_mi = Base.method_instance(LimitedSrcTombstoneProof62338.entry, ()) + outer_result = Compiler.InferenceResult(outer_mi, Compiler.typeinf_lattice(interp)) + outer = Compiler.InferenceState(outer_result, Compiler.CACHE_MODE_LOCAL, interp) + + child_mi = Base.method_instance(LimitedSrcTombstoneProof62338.callee, ()) + child_result = Compiler.InferenceResult(child_mi, Compiler.typeinf_lattice(interp)) + child = Compiler.InferenceState(child_result, Compiler.CACHE_MODE_LOCAL, interp) + Compiler.assign_parentchild!(child, outer) + + mresult = Compiler._schedule_edge_infer_task!( + outer, child, child.result, child_mi.def, nothing, false, false, false) + @test Compiler.doworkloop(interp, outer) + @test isready(mresult) + scheduled = mresult[] + @test !scheduled.edgecycle + @test scheduled.needs_mi_edge +end + +let + world = Base.get_world_counter() + inf_params = Compiler.InferenceParams(; cache_owner=LimitedSrcTombstoneProof62338) + interp = Compiler.NativeInterpreter(world; inf_params) + binding = convert(Core.Binding, GlobalRef(LimitedSrcTombstoneProof62338, :FLAG)) + + outer_mi = Base.method_instance(LimitedSrcTombstoneProof62338.entry, ()) + outer_result = Compiler.InferenceResult(outer_mi, Compiler.typeinf_lattice(interp)) + outer_result.ci = Compiler.engine_reserve(interp, outer_mi) + outer = Compiler.InferenceState(outer_result, Compiler.CACHE_MODE_GLOBAL, interp) + + child_mi = Base.method_instance(LimitedSrcTombstoneProof62338.callee, ()) + child_result = Compiler.InferenceResult(child_mi, Compiler.typeinf_lattice(interp)) + child = Compiler.InferenceState(child_result, Compiler.CACHE_MODE_LOCAL, interp) + Compiler.assign_parentchild!(child, outer) + + bystander_mi = Base.method_instance(LimitedSrcTombstoneProof62338.bystander, ()) + bystander_result = Compiler.InferenceResult( + bystander_mi, Compiler.typeinf_lattice(interp)) + bystander = Compiler.InferenceState( + bystander_result, Compiler.CACHE_MODE_GLOBAL, interp) + Compiler.assign_parentchild!(bystander, child) + + # Model a clean return with a limited intermediate statement, then finalize it + # through the ordinary tombstone path. + empty!(child.ip) + fill!(child.ssavaluetypes, Any) + causes = Compiler.IdSet{Compiler.InferenceState}() + push!(causes, outer) + child.ssavaluetypes[1] = Compiler.LimitedAccuracy(Int, causes) + child.bestguess = Core.Const(1) + child.exc_bestguess = Union{} + child.ipo_effects = Compiler.EFFECTS_TOTAL + push!(child.edges, binding) + Compiler.finishinfer!(child, interp, child.cycleid, + IdDict{Core.MethodInstance,Core.CodeInstance}()) + + @test child.result.tombstone + @test child.ssavaluetypes[1] isa Compiler.LimitedAccuracy + @test child.result.result === Core.Const(1) + @test child.result.src === nothing + + # Model the direct, already-in-progress SCC edge. Unlike the deferred task + # below, this path consumed the child's clean facts before its tombstone was + # known and therefore needs proof propagation when the SCC is finalized. + Compiler.add_cycle_backedge!(outer, child) + cycle_worlds = child.valid_worlds + Compiler.propagate_unpublished_cycle_proof!( + child.callstack, 2, world, cycle_worlds) + cycle_proofs = filter(edge -> edge isa Compiler.LocalInferenceProof, outer.edges) + @test length(cycle_proofs) == 1 + @test any(edge -> edge === binding, + Compiler.materialize_inference_edges(only(cycle_proofs).edges)) + @test !any(edge -> edge isa Compiler.LocalInferenceProof, bystander.edges) + + resize!(outer.callstack, 1) + mresult = Compiler._schedule_edge_infer_task!( + outer, child, child.result, child_mi.def, nothing, false, false, false) + + @test Compiler.typeinf(interp, outer) + @test isready(mresult) + @test mresult[].needs_mi_edge + ci = outer.result.ci + @test get(Compiler.code_cache(interp), outer_mi, nothing) === ci + @test ci.rettype_const === 1 + @test any(edge -> edge === binding, ci.edges) +end + +# Local constant inference keeps its dependency proof separate from the executable target. +module LocalProofConstpropCache + const SINK = Ref{Any}() + const VALUE = "v1" + + @noinline Base.@constprop :aggressive function readglobal(M::Module, s::Symbol) + SINK[] = s + return getglobal(M, s) + end + + probe_twice() = ( + readglobal(LocalProofConstpropCache, :VALUE)::String, + readglobal(LocalProofConstpropCache, :VALUE)::String, + ) + probe_once() = readglobal(LocalProofConstpropCache, :VALUE)::String +end + +let interp = Compiler.NativeInterpreter(Base.get_world_counter()) + mi = Base.method_instance(LocalProofConstpropCache.probe_twice, ()) + frame = Compiler.typeinf_frame(interp, mi, false) + infos = [info for info in frame.stmt_info if info isa Compiler.MethodMatchInfo] + @test length(infos) == 2 + + results = [only(info.call_results) for info in infos] + targets = [only(info.edges) for info in infos] + callee_mi = Base.method_instance(LocalProofConstpropCache.readglobal, (Module, Symbol)) + cache_entries = Compiler.get_inference_cache(interp).results + cached = [ + entry for entry in cache_entries + if entry isa Compiler.LocalInferenceResult && + entry.result.linfo === callee_mi && + entry.result.overridden_by_const !== nothing + ] + @test length(cached) == 1 + @test cached[1] === results[1] === results[2] + @test targets[1] === targets[2] + @test targets[1] === get(Compiler.code_cache(interp), callee_mi, nothing) + @test all(i -> Compiler.getedge(infos[i], 1) === targets[i], eachindex(infos)) + + proofs = map(Compiler.inference_proof, results) + binding = convert(Core.Binding, GlobalRef(LocalProofConstpropCache, :VALUE)) + @test all(proof -> proof isa Compiler.LocalInferenceProof, proofs) + @test all(proofs) do proof + edges = Compiler.materialize_inference_edges(proof.edges) + any(edge -> edge === binding, edges) + end + callee_edges = [ + edge for edge in frame.edges + if edge isa Core.CodeInstance && edge.def === callee_mi + ] + @test only(callee_edges) === targets[1] +end + +let interp = Compiler.NativeInterpreter(Base.get_world_counter()) + callee_mi = Base.method_instance(LocalProofConstpropCache.readglobal, (Module, Symbol)) + caller_mi = Base.method_instance(LocalProofConstpropCache.probe_once, ()) + frame = Compiler.typeinf_frame(interp, caller_mi, true) + src = frame.src + invokes = [stmt for stmt in src.code if stmt isa Expr && stmt.head === :invoke] + @test length(invokes) == 1 + target = only(invokes).args[1] + @test target === get(Compiler.code_cache(interp), callee_mi, nothing) + + binding = convert(Core.Binding, GlobalRef(LocalProofConstpropCache, :VALUE)) + @test any(edge -> edge === binding, src.edges) + callee_edges = [ + edge for edge in src.edges + if edge isa Core.CodeInstance && edge.def === callee_mi + ] + @test only(callee_edges) === target +end + +stmtinfo_edge_target(x::Int) = x +function stmtinfo_codeinstance(mi::Core.MethodInstance, owner, + edges::Core.SimpleVector=Core.svec()) + return Core.CodeInstance(mi, owner, Any, Any, nothing, nothing, zero(Int32), + typemin(UInt), typemax(UInt), zero(UInt32), nothing, nothing, edges) +end + +@testset "inference proof edge materialization" begin + world = Base.get_world_counter() + atype = Tuple{typeof(stmtinfo_edge_target),Int} + match = only(Base._methods_by_ftype(atype, -1, world)) + mi = Compiler.specialize_method(match) + interp = Compiler.NativeInterpreter(world) + owner = Compiler.cache_owner(interp) + ci1 = stmtinfo_codeinstance(mi, owner) + ci2 = stmtinfo_codeinstance(mi, owner) + @test_throws AssertionError Compiler.LocalInferenceProof( + Compiler.WorldRange(world, world + 1), Core.svec()) + + @testset "constprop cache worlds" begin + lattice = Compiler.typeinf_lattice(interp) + argtypes = Compiler.matching_cache_argtypes(lattice, mi) + argtypes[2] = Compiler.Const(1) + overridden = falses(length(argtypes)) + overridden[2] = true + + result = Compiler.InferenceResult(mi, copy(argtypes), overridden) + result.result = Int + result.valid_worlds = Compiler.WorldRange(UInt(1), UInt(3)) + proof = Compiler.LocalInferenceProof(result.valid_worlds, Core.svec()) + local_result = Compiler.LocalInferenceResult(result, proof, UInt(2)) + cache = Compiler.InferenceCache() + push!(cache, local_result) + @test Compiler.constprop_cache_lookup( + lattice, mi, argtypes, cache, UInt(2)) === local_result + @test Compiler.constprop_cache_lookup( + lattice, mi, argtypes, cache, UInt(3)) === nothing + + tombstone = Compiler.InferenceResult(mi, copy(argtypes), overridden) + tombstone.result = Int + tombstone.tombstone = true + tombstone.cache_world = UInt(2) + tombstone.valid_worlds = Compiler.WorldRange(UInt(1), UInt(3)) + cache = Compiler.InferenceCache() + push!(cache, tombstone) + @test ismissing(Compiler.constprop_cache_lookup( + lattice, mi, argtypes, cache, UInt(2))) + @test Compiler.constprop_cache_lookup( + lattice, mi, argtypes, cache, UInt(3)) === nothing + end + + @testset "proof streams and paired edges" begin + plain_edges = Core.svec(ci1, ci2) + @test Compiler.materialize_inference_edges(plain_edges) === plain_edges + + edges = Any[] + Compiler.add_inference_proof!(edges, ci1, ci1) + @test isempty(edges) + Compiler.add_inference_proof!(edges, ci2, ci1) + @test edges == Any[match.method.sig, ci2] + + encoded = Core.svec(-1, atype, ci1, atype, mi) + proof = Compiler.LocalInferenceProof(Compiler.WorldRange(UInt(1), UInt(2)), encoded) + Compiler.add_inference_proof!(edges, proof, ci1) + @test edges[end] === proof + flat = Compiler.materialize_inference_edges(edges) + @test length(flat) == 2 + length(encoded) + @test all(i -> flat[i + 2] === encoded[i], eachindex(encoded)) + + leaf = Compiler.LocalInferenceProof(Compiler.WorldRange(world), Core.svec(ci2)) + root = Compiler.LocalInferenceProof(Compiler.WorldRange(world), Core.svec(leaf, leaf)) + internal_edges = Any[] + Compiler.add_inference_proof!(internal_edges, root) + Compiler.add_inference_proof!(internal_edges, root) + @test internal_edges == Any[root] + @test Compiler.materialize_inference_edges(internal_edges) == Core.svec(ci2) + + duplicate_leaf = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(ci2)) + duplicate_root = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(leaf, duplicate_leaf)) + @test Compiler.materialize_inference_edges(duplicate_root.edges) == + Core.svec(ci2) + + invoke_leaf1 = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(atype, ci1)) + invoke_leaf2 = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(atype, ci1)) + invoke_root = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(invoke_leaf1, invoke_leaf2)) + @test Compiler.materialize_inference_edges(invoke_root.edges) == + Core.svec(atype, ci1) + + encoded_leaf = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(1, atype, ci1)) + standalone_leaf = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(ci1)) + encoded_root = Compiler.LocalInferenceProof( + Compiler.WorldRange(world), Core.svec(encoded_leaf, standalone_leaf)) + @test Compiler.materialize_inference_edges(encoded_root.edges) == + Core.svec(1, atype, ci1) + + regular_inf = Compiler.InferenceResult(mi, Compiler.typeinf_lattice(interp)) + regular_inf.result = Int + regular_inf.valid_worlds = Compiler.WorldRange(world - 2, world) + regular_proof = Compiler.LocalInferenceProof( + regular_inf.valid_worlds, Core.svec(ci2)) + regular_result = Compiler.LocalInferenceResult(regular_inf, regular_proof, world) + frame_result = Compiler.InferenceResult(mi, Compiler.typeinf_lattice(interp)) + frame = Compiler.InferenceState(frame_result, Compiler.CACHE_MODE_LOCAL, interp) + push!(frame.edges, ci1) + method_result = Compiler.MethodCallResult( + Int, Any, Compiler.Effects(), ci1, false, false, regular_result) + concrete_proof = Compiler.LocalInferenceProof( + Compiler.WorldRange(world - 1, world), Core.svec(match.method)) + concrete_result = Compiler.ConcreteResult( + ci1, Compiler.Effects(); proof=concrete_proof) + concrete_call = Compiler.ConstCallResult( + Int, Any, concrete_result, Compiler.Effects()) + composite = Compiler.const_prop_inference_proof( + frame, method_result, concrete_call) + @test composite.valid_worlds == Compiler.WorldRange(world - 1, world) + @test Compiler.materialize_inference_edges(composite.edges) == + Core.svec(ci2, match.method, ci1) + + concrete = Compiler.ConcreteResult(ci1, Compiler.Effects(); proof) + @test Compiler.inference_proof(concrete) === proof + @test !isdefined(concrete, :result) + concrete_with_value = Compiler.ConcreteResult(ci1, Compiler.Effects(), 1; proof) + @test concrete_with_value.result === 1 + end + + @testset "lookup edges precede their proofs" begin + results = Compiler.MethodLookupResult(Any[match], + Compiler.WorldRange(typemin(UInt), typemax(UInt)), false) + singleton = Compiler.MethodMatchInfo(results, Core.methodtable, atype, true) + singleton.edges[1] = ci1 + singleton.call_results[1] = Compiler.ConcreteResult(ci2, Compiler.Effects()) + edges = Any[] + Compiler.add_edges!(edges, singleton) + @test edges == Any[ci1, match.method.sig, ci2] + + singleton.call_results[1] = Compiler.ConcreteResult(ci1, Compiler.Effects()) + empty!(edges) + Compiler.add_edges!(edges, singleton) + @test edges == Any[ci1] + + flat_stream = Core.svec(1, atype, ci2) + flat = Compiler.LocalInferenceProof(Compiler.WorldRange(world), flat_stream) + singleton.call_results[1] = Compiler.ConcreteResult(ci1, Compiler.Effects(); proof=flat) + empty!(edges) + Compiler.add_edges!(edges, singleton) + @test edges == Any[ci1, flat] + materialized = Compiler.materialize_inference_edges(edges) + @test materialized[1] === ci1 + @test all(i -> materialized[i + 1] === flat_stream[i], eachindex(flat_stream)) + + invoke = Compiler.InvokeCallInfo(ci1, match, + Compiler.ConcreteResult(ci2, Compiler.Effects()), atype) + empty!(edges) + Compiler.add_edges!(edges, invoke) + @test edges == Any[atype, ci1, match.method.sig, ci2] + + local_inf = Compiler.InferenceResult(mi, Compiler.typeinf_lattice(interp)) + local_inf.result = Int + local_inf.valid_worlds = Compiler.WorldRange(world) + local_proof = Compiler.LocalInferenceProof(local_inf.valid_worlds, Core.svec()) + local_result = Compiler.LocalInferenceResult(local_inf, local_proof, world) + + multi_results = Compiler.MethodLookupResult( + Any[match, match], Compiler.WorldRange(), false) + multi = Compiler.MethodMatchInfo(multi_results, Core.methodtable, atype, true) + multi.call_results[1] = local_result + empty!(edges) + Compiler.add_edges!(edges, multi) + @test edges[3] === mi + has_mi_backedge = false + flat_edges = Compiler.materialize_inference_edges(edges) + for (_, edge) in Compiler.ForwardToBackedgeIterator(flat_edges) + has_mi_backedge |= edge === mi + end + @test has_mi_backedge + + multi.call_results[1] = nothing + multi.needs_mi_edges[1] = true + empty!(edges) + Compiler.add_edges!(edges, multi) + @test edges[3] === mi + flat_edges = Compiler.materialize_inference_edges(edges) + @test any(Compiler.ForwardToBackedgeIterator(flat_edges)) do (_, edge) + edge === mi + end + multi.needs_mi_edges[1] = false + + # Each CallInfo type implements the targetless-facts upgrade separately; + # cover both triggers (attached result, needs-mi-edge bit) per type, and + # every result representation (local, targetless concrete) at least once. + targetless_concrete = Compiler.ConcreteResult( + nothing, Compiler.Effects(); proof=local_proof) + + invoke_concrete = Compiler.InvokeCallInfo( + nothing, match, targetless_concrete, atype) + empty!(edges) + Compiler.add_edges!(edges, invoke_concrete) + @test edges[2] === mi + flat_edges = Compiler.materialize_inference_edges(edges) + @test any(Compiler.ForwardToBackedgeIterator(flat_edges)) do (_, edge) + edge === mi + end + + invoke_provisional = Compiler.InvokeCallInfo( + nothing, match, nothing, atype, true) + empty!(edges) + Compiler.add_edges!(edges, invoke_provisional) + @test edges[2] === mi + + opaque_local = Compiler.OpaqueClosureCallInfo(nothing, match, local_result) + empty!(edges) + Compiler.add_edges!(edges, opaque_local) + @test first(edges) === mi + + opaque_provisional = Compiler.OpaqueClosureCallInfo( + nothing, match, nothing, true) + empty!(edges) + Compiler.add_edges!(edges, opaque_provisional) + @test first(edges) === mi + end + + @testset "lookup identity and edge access" begin + results = Compiler.MethodLookupResult(Any[match], Compiler.WorldRange(), false) + info1 = Compiler.MethodMatchInfo(results, Core.methodtable, atype, false) + info1.edges[1] = ci1 + info2 = Compiler.MethodMatchInfo(results, Core.methodtable, atype, false) + info2.edges[1] = ci2 + + edges = Any[] + Compiler.add_edges!(edges, info1) + Compiler.add_edges!(edges, info1) + @test count(edge -> edge isa Int, edges) == 1 + Compiler.add_edges!(edges, info2) + @test count(edge -> edge isa Int, edges) == 2 + starts = findall(edge -> edge isa Int, edges) + @test edges[starts[1] + 2] === ci1 + @test edges[starts[2] + 2] === ci2 + + @test Compiler.getedge(info1, 1) === ci1 + split = Compiler.UnionSplitInfo([info1, info2]) + @test Compiler.getedge(split, 1) === ci1 + @test Compiler.getedge(split, 2) === ci2 + @test Compiler.getedge(Compiler.InvokeCallInfo(ci1, match, nothing, atype), 1) === ci1 + @test Compiler.getedge(Compiler.OpaqueClosureCallInfo(ci2, match, nothing), 1) === ci2 + @test Compiler.getedge(Compiler.VirtualMethodMatchInfo(split), 2) === ci2 + end + + @testset "encoded groups are immutable units" begin + encoded = Any[1, atype, mi] + Compiler.add_one_edge!(encoded, ci1) + Compiler.add_one_edge!(encoded, ci2) + Compiler.add_one_edge!(encoded, ci1) + @test encoded == Any[1, atype, mi, ci1, ci2] + + invoke_edges = Any[1, atype, mi] + Compiler.add_invoke_edge!(invoke_edges, atype, ci1) + Compiler.add_invoke_edge!(invoke_edges, atype, ci2) + Compiler.add_invoke_edge!(invoke_edges, atype, ci1) + @test invoke_edges == Any[1, atype, mi, atype, ci1, atype, ci2] + + inline_edges = Any[1, atype, mi] + Compiler.add_inlining_edge!(inline_edges, ci1) + Compiler.add_inlining_edge!(inline_edges, ci2) + Compiler.add_inlining_edge!(inline_edges, ci1) + @test inline_edges[3] === mi + @test inline_edges[5] === ci1 + @test inline_edges[7] === ci2 + + upgraded = Any[mi] + Compiler.add_one_edge!(upgraded, ci1) + @test only(upgraded) === ci1 + invoke_upgraded = Any[atype, mi] + Compiler.add_invoke_edge!(invoke_upgraded, atype, ci1) + @test invoke_upgraded[2] === ci1 + inline_upgraded = Any[match.method] + Compiler.add_inlining_edge!(inline_upgraded, ci1) + @test only(inline_upgraded) === ci1 + end +end + let comparison = Tuple{X, X} where X<:Tuple sig = Tuple{X, X} where X<:comparison ref = Tuple{X, X} where X @@ -51,7 +618,7 @@ let t = Tuple{Ref{T},T,T} where T, c = Tuple{Ref, T, T} where T # #36407 end # obtain Vararg with 2 undefined fields -let va = ccall(:jl_type_intersection_with_env, Any, (Any, Any), Tuple{Tuple}, Tuple{Tuple{Vararg{Any, N}}} where N)[2][1] +let va = Base.typeintersect_env(Tuple{Tuple}, Tuple{Tuple{Vararg{Any, N}}} where N)[2][1] @test Compiler.__limit_type_size(Tuple, va, Core.svec(va, Union{}), 2, 2) === Tuple end @@ -105,12 +672,12 @@ end # issue #51694 @test Compiler.type_more_complex( - Base.Generator{Base.Iterators.Flatten{Array{Bool, 1}}, typeof(identity)}, - Base.Generator{Array{Bool, 1}, typeof(identity)}, + Base.Generator{Base.Iterators.Flatten{Vector{Bool}}, typeof(identity)}, + Base.Generator{Vector{Bool}, typeof(identity)}, Core.svec(), 0, 0, 0) @test Compiler.type_more_complex( - Base.Generator{Base.Iterators.Flatten{Base.Generator{Array{Bool, 1}, typeof(identity)}}, typeof(identity)}, - Base.Generator{Array{Bool, 1}, typeof(identity)}, + Base.Generator{Base.Iterators.Flatten{Base.Generator{Vector{Bool}, typeof(identity)}}, typeof(identity)}, + Base.Generator{Vector{Bool}, typeof(identity)}, Core.svec(), 0, 0, 0) let # 40336 @@ -136,8 +703,6 @@ end @test Compiler.limit_type_size(Type{Union{Int,Type{Int}}}, Union{Type{Int},Type{Type{Int}}}, Union{}, 0, 0) == Type @test Compiler.limit_type_size(Type{Union{Int,Type{Int}}}, Type{Union{Type{Int},Type{Type{Int}}}}, Union{}, 0, 0) == Type{Union{Int, Type{Int}}} @test Compiler.limit_type_size(Type{Union{Int,Type{Int}}}, Type{Type{Int}}, Union{}, 0, 0) == Type - - @test Compiler.limit_type_size(Type{Any}, Union{}, Union{}, 0, 0) == Compiler.limit_type_size(Type{Any}, Any, Union{}, 0, 0) == Compiler.limit_type_size(Type{Any}, Type, Union{}, 0, 0) == @@ -306,8 +871,8 @@ let fT(x::T) where {T} = T @test fT(Any) === DataType @test fT(Int) === DataType - @test fT(Type{Any}) === DataType - @test fT(Type{Int}) === DataType + @test fT(Type{Any}) === Core.TypeEq + @test fT(Type{Int}) === Core.TypeEq ff(x::Type{T}) where {T} = T @test ff(Type{Any}) === Type{Any} @@ -419,7 +984,7 @@ end # issue #12826 f12826(v::Vector{I}) where {I<:Integer} = v[1] -@test Base.return_types(f12826,Tuple{Array{I,1} where I<:Integer})[1] == Integer +@test Base.return_types(f12826,Tuple{Vector{I} where I<:Integer})[1] == Integer # non-terminating inference, issue #14009 @@ -506,6 +1071,11 @@ let f(T) = Type{T} @test Base.return_types(f, Tuple{Type{Int}}) == Any[Type{Type{Int}}] end +# Keep tuple iteration precise when joining ordinary values with kind values. +@test Core.Compiler.tmerge(String, Type) == Union{String, Type} +@test Base.return_types(iterate, Tuple{Tuple{String, Type}, Int}) == + Any[Union{Nothing, Tuple{Union{String, Type}, Int}}] + # issue #9222 function SimpleTest9222(pdedata, mu_actual::Vector{T1}, nu_actual::Vector{T1}, v0::Vector{T1}, epsilon::T1, beta::Vector{T1}, @@ -546,7 +1116,69 @@ const NInt1{N} = Tuple{Int, Vararg{Int, N}} fNInt(x::NInt) = (x...,) gNInt() = fNInt(x) @test Base.return_types(gNInt, ()) == Any[NInt] -@test Base.return_types(eltype, (NInt,)) == Any[Union{Type{Int}, Type{Union{}}}] # issue 21763 +# issue 21763: a closed equality-keyed query folds `eltype`'s `@isdefined(E)` sparam +# guard — every `==`-equal rep of the argument binds the var (BOUND_EQ), even +# though its value is only `==`-certain. The value-typed entry must stay `Type`: its +# `Type{<:NInt}` argtype includes `Tuple{}`, which matches without binding the var. +@test Base.return_types(Base._eltype_ntuple, (Type{Tuple{Int}},)) == Any[Type{Int}] +@test Base.return_types(eltype, (Type{NInt{1}},)) == Any[Type{Int}] +@test Base.return_types(eltype, (NInt,)) == Any[Type] +f21763_def(t::Type{<:Tuple{Vararg{E}}}) where E = @isdefined(E) ? E : :undef +@test Base.return_types(f21763_def, (Type{Tuple{Int}},)) == Any[Type{Int}] +@test Base.return_types(f21763_def, (Type{<:NInt},)) == Any[Union{Symbol, Type{Int}}] +# the un-pinning is a property of the `Type{<:X}` range, not of `Vararg`: a +# fixed-length tuple range still admits the `Union{}` member, which binds +# nothing, so `E` is reached only through the range var's bound and stays undef +f21763_def_fixed(t::Type{<:Tuple{E}}) where E = @isdefined(E) ? E : :undef +@test Base.return_types(f21763_def_fixed, (Type{Tuple{Int}},)) == Any[Type{Int}] +@test Base.return_types(f21763_def_fixed, (Type{<:Tuple{Int}},)) == Any[Union{Symbol, Type{Int}}] + +# `fieldtype` of an `==`-only `Type{X}` element must not fold to an egal constant: +# an `==`-equal rep of `X` yields a fieldtype that is `==` but not `===` the stored +# one (#61323) +let rep = Tuple{Tuple{S}} where S<:Int + @test rep == Tuple{Tuple{Int}} && rep !== Tuple{Tuple{Int}} + @test fieldtype(rep, 1) == Tuple{Int} && fieldtype(rep, 1) !== Tuple{Int} + @test Base.return_types((Type{Tuple{Tuple{Int}}},)) do t + fieldtype(t, 1) + end == Any[Type{Tuple{Int}}] + fldrep(t) = fieldtype(t, 1) === Tuple{Int} + fldreparr(tarr, i) = fldrep(tarr[i]) + @test fldreparr(Type{Tuple{Tuple{Int}}}[rep, Tuple{Tuple{Int}}], 1) === false + @test fldreparr(Type{Tuple{Tuple{Int}}}[rep, Tuple{Tuple{Int}}], 2) === true +end + +# a bare `TypeEgal{T}` lattice element pins a `Type{...}` construction argument +# like `Const(T)` does, while an `==`-only `Type{T}` element must not +let apply_type_tfunc = Compiler.apply_type_tfunc + 𝕃 = Compiler.fallback_lattice + rt = apply_type_tfunc(𝕃, Const(Type), Core.TypeEgal{Int}) + @test rt isa Compiler.Const && rt.val === Type{Int} + @test apply_type_tfunc(𝕃, Const(Type), Type{Int}) == Type{Type{Int}} +end + +# Generated-function static parameters from TypeEgal signatures need exact +# datatype-parameter bindings. +struct P62001{T,N} end +@generated generated_type_sparam62001(::Type{P62001{T,N}}) where {T,N} = :(Val{$T}()) +const P62001Int4 = P62001{Int,4} +@test only(Base.return_types(generated_type_sparam62001, Tuple{Core.TypeEgal{P62001Int4}})) === Val{Int} + +# Invariant datatype parameters preserve the exact runtime spelling of the +# `Type{Union{}}`/`Core.TypeofBottom` alias family when forming static params. +struct TypeofBottomParam62001{S} end +typeofbottom_param62001(::Type{TypeofBottomParam62001{S}}) where {S} = + S === Type{Union{}} ? 1 : "" +@test typeofbottom_param62001(TypeofBottomParam62001{Type{Union{}}}) == 1 +@test only(Base.return_types(typeofbottom_param62001, + Tuple{Type{TypeofBottomParam62001{Type{Union{}}}}})) === Int +@test only(Base.return_types(typeofbottom_param62001, + Tuple{Core.TypeEgal{TypeofBottomParam62001{Type{Union{}}}}})) === Int + +# Structural TypeofVararg results should remain usable when constructing Tuple types. +vararg_tail62001(X::Tuple{S,Vararg{S}}) where S = + X[2:end]::Tuple{Vararg{eltype(X::Tuple{Any,Vararg{Any}})}} +@test only(Base.return_types(vararg_tail62001, Tuple{Tuple{Int,Vararg{Int}}})) == Tuple{Vararg{Int}} # issue #17572 function f17572(::Type{Val{A}}) where A @@ -576,9 +1208,9 @@ f16530b(c) = fieldtype(Foo16530b, c) let T = Vector{Tuple{Vararg{Float64,dim}}} where dim @test f16530a() == T @test f16530a(:c) == T - @test Base.return_types(f16530a, ()) == Any[Type{T}] - @test Base.return_types(f16530b, ()) == Any[Type{T}] - @test Base.return_types(f16530b, (Symbol,)) == Any[Type{T}] + @test Base.return_types(f16530a, ()) == Any[Core.TypeEgal{T}] + @test Base.return_types(f16530b, ()) == Any[Core.TypeEgal{T}] + @test Base.return_types(f16530b, (Symbol,)) == Any[Core.TypeEgal{T}] end @test f16530a(:d) == Vector @@ -594,8 +1226,8 @@ let T1 = Tuple{Int, Float64}, @test f18037(1) === T1 @test f18037(2) === T2 - @test Base.return_types(f18037, ()) == Any[Type{T1}] - @test Base.return_types(f18037, (Int,)) == Any[Union{Type{T1},Type{T2}}] + @test Base.return_types(f18037, ()) == Any[Core.TypeEgal{T1}] + @test Base.return_types(f18037, (Int,)) == Any[Union{Core.TypeEgal{T1},Core.TypeEgal{T2}}] end # issue #18015 @@ -731,7 +1363,7 @@ for (codetype, all_ssa) in Any[ local i for i = 1:length(code.ssavaluetypes) typ = code.ssavaluetypes[i] - @test isa(typ, Type) || isa(typ, Const) || isa(typ, Conditional) || typ + @test isa(typ, Type) || isa(typ, Const) || isa(typ, Conditional) context=typ end test_inferred_static(codetype, all_ssa) end @@ -781,7 +1413,7 @@ mutable struct HasAbstractlyTypedField x::Union{Int,String} end f_infer_abstract_fieldtype() = fieldtype(HasAbstractlyTypedField, :x) -@test Base.return_types(f_infer_abstract_fieldtype, ()) == Any[Type{Union{Int,String}}] +@test Base.return_types(f_infer_abstract_fieldtype, ()) == Any[Core.TypeEgal{Union{Int,String}}] let fieldtype_tfunc(@nospecialize args...) = Compiler.fieldtype_tfunc(Compiler.fallback_lattice, args...), fieldtype_nothrow(@nospecialize(s0), @nospecialize(name)) = Compiler.fieldtype_nothrow( @@ -790,30 +1422,79 @@ let fieldtype_tfunc(@nospecialize args...) = @test fieldtype_tfunc(Union{Type{Int32}, Int32}, Const(:x)) == Union{} @test fieldtype_tfunc(Union{Type{Base.RefValue{T}}, Type{Int32}} where {T<:Array}, Const(:x)) == Type{<:Array} @test fieldtype_tfunc(Union{Type{Base.RefValue{T}}, Type{Int32}} where {T<:Real}, Const(:x)) == Type{<:Real} + # the `Type{...}` elements are only `==`-certain, so no `Const` fold (#61323) @test fieldtype_tfunc(Union{Type{Base.RefValue{<:Array}}, Type{Int32}}, Const(:x)) == Type{Array} - @test fieldtype_tfunc(Union{Type{Base.RefValue{<:Real}}, Type{Int32}}, Const(:x)) == Const(Real) + @test fieldtype_tfunc(Union{Type{Base.RefValue{<:Real}}, Type{Int32}}, Const(:x)) == Type{Real} @test fieldtype_tfunc(Const(Union{Base.RefValue{<:Real}, Type{Int32}}), Const(:x)) == Const(Real) @test fieldtype_tfunc(Type{Union{Base.RefValue{T}, Type{Int32}}} where {T<:Real}, Const(:x)) == Type{<:Real} @test fieldtype_tfunc(Type{<:Tuple}, Const(1)) == Any @test fieldtype_tfunc(Type{<:Tuple}, Any) == Any - @test fieldtype_nothrow(Type{Base.RefValue{<:Real}}, Const(:x)) + @test !fieldtype_nothrow(Type{Base.RefValue{<:Real}}, Const(:x)) + @test fieldtype_nothrow(Core.TypeEgal{Base.RefValue{<:Real}}, Const(:x)) @test !fieldtype_nothrow(Type{Union{}}, Const(:x)) @test !fieldtype_nothrow(Union{Type{Base.RefValue{T}}, Int32} where {T<:Real}, Const(:x)) @test !fieldtype_nothrow(Union{Type{Base.RefValue{<:Real}}, Int32}, Const(:x)) @test fieldtype_nothrow(Const(Union{Base.RefValue{<:Real}, Int32}), Const(:x)) @test !fieldtype_nothrow(Type{Union{Base.RefValue{T}, Int32}} where {T<:Real}, Const(:x)) # improvable? - @test fieldtype_nothrow(Union{Type{Base.RefValue{T}}, Type{Base.RefValue{Any}}} where {T<:Real}, Const(:x)) - @test fieldtype_nothrow(Union{Type{Base.RefValue{<:Real}}, Type{Base.RefValue{Any}}}, Const(:x)) + @test !fieldtype_nothrow(Union{Type{Base.RefValue{T}}, Type{Base.RefValue{Any}}} where {T<:Real}, Const(:x)) + @test !fieldtype_nothrow(Union{Type{Base.RefValue{<:Real}}, Type{Base.RefValue{Any}}}, Const(:x)) + @test fieldtype_nothrow(Union{Core.TypeEgal{Base.RefValue{Int}}, Core.TypeEgal{Base.RefValue{Any}}}, Const(:x)) @test fieldtype_nothrow(Const(Union{Base.RefValue{<:Real}, Base.RefValue{Any}}), Const(:x)) - @test fieldtype_nothrow(Type{Union{Base.RefValue{T}, Base.RefValue{Any}}} where {T<:Real}, Const(:x)) + @test !fieldtype_nothrow(Type{Union{Base.RefValue{T}, Base.RefValue{Any}}} where {T<:Real}, Const(:x)) @test !fieldtype_nothrow(Type{Tuple{}}, Const(1)) - @test fieldtype_nothrow(Type{Tuple{Int}}, Const(1)) - @test fieldtype_nothrow(Type{Tuple{Vararg{Int}}}, Const(1)) - @test fieldtype_nothrow(Type{Tuple{Vararg{Int}}}, Const(2)) - @test fieldtype_nothrow(Type{Tuple{Vararg{Int}}}, Const(42)) + @test !fieldtype_nothrow(Type{Tuple{Int}}, Const(1)) + @test fieldtype_nothrow(Core.TypeEgal{Tuple{Int}}, Const(1)) + @test !fieldtype_nothrow(Type{Tuple{Vararg{Int}}}, Const(1)) + @test fieldtype_nothrow(Core.TypeEgal{Tuple{Vararg{Int}}}, Const(1)) + @test fieldtype_nothrow(Core.TypeEgal{Tuple{Vararg{Int}}}, Const(2)) + @test fieldtype_nothrow(Core.TypeEgal{Tuple{Vararg{Int}}}, Const(42)) @test !fieldtype_nothrow(Type{<:Tuple{Vararg{Int}}}, Const(1)) @test TypeVar <: fieldtype_tfunc(Any, Any) -end + # JuliaLang/julia#30807: malformed types like `NTuple{<:Any, 3}` should not crash `fieldtype_tfunc` + @test fieldtype_tfunc(Const(NTuple{<:Any, 3}), Const(1)) == Union{} + @test fieldtype_tfunc(Const(NTuple{<:Any, 3}), Int) == Union{} + # a value-parameter tuple field (e.g. `Tuple{1:2}`) makes `fieldtype` return + # that value rather than throw, so the tfunc must not fold to `Union{}` (#62001) + @test fieldtype_tfunc(Const(Tuple{1:2, 3:4}), Const(1)) == Const(1:2) + @test fieldtype_tfunc(Const(Tuple{1:2, 3:4}), Const(2)) == Const(3:4) + @test fieldtype_tfunc(Const(Tuple{1:2, 3:4}), Const(3)) == Union{} # out of bounds + @test fieldtype_tfunc(Const(Tuple{1:2, 3:4}), Int) == UnitRange{Int} + @test fieldtype_tfunc(Const(Tuple{Int, 5, Char}), Const(2)) == Const(5) + # non-type parameters are `===`-pinned by type identity, so a value field is + # `Const` even when the argument type is only `==`-certain (#61323) + @test fieldtype_tfunc(Type{Tuple{1:2, 3:4}}, Const(1)) == Const(1:2) + @test fieldtype_tfunc(Type{Tuple{1:2, 3:4}}, Int) == UnitRange{Int} +end + +# #62001: const-propagating a value-parameter tuple type through `fieldtypes` +# (which reads each field via `fieldtype`) must yield valid IR rather than fold +# the non-throwing value reads to `Union{}` +let (ci, rt) = only(code_typed(() -> fieldtypes(Tuple{1:2, 3:4}), (); optimize=true)) + @test rt == Tuple{UnitRange{Int}, UnitRange{Int}} +end + +const FieldRep62001 = Tuple{Tuple{S}} where S<:Int +mutable struct FieldRepA62001 + x::FieldRep62001 +end +mutable struct FieldRepB62001 + x::Tuple{Tuple{Int}} +end +@test fieldtype(FieldRepA62001, :x) == fieldtype(FieldRepB62001, :x) +@test fieldtype(FieldRepA62001, :x) !== fieldtype(FieldRepB62001, :x) +let rt = Compiler.modifyfield!_tfunc(Compiler.fallback_lattice, + Union{FieldRepA62001, FieldRepB62001}, Compiler.Const(:x), Any, Any, Symbol) + @test rt == Pair{Tuple{Tuple{Int}}, Tuple{Tuple{Int}}} +end + +# JuliaLang/julia#30807: malformed types like `NTuple{<:Any, 3}` should not crash `fieldtype_tfunc` +struct Issue30807 + xs::NTuple{<:Any, 3} + Issue30807(xs...) = new(xs) +end +@test Base.infer_return_type((Int,Int,Int)) do x, y, z + Issue30807(x, y, z) +end === Union{} # issue #11480 @noinline f11480(x,y) = x @@ -990,11 +1671,11 @@ err20033(x::Float64...) = prod(x) @test Base._return_type(tuple, Tuple{Int,Int8,Int}) === Tuple{Int,Int8,Int} # issue #21088 -@test Base._return_type(typeof, Tuple{Int}) == Type{Int} +@test Base._return_type(typeof, Tuple{Int}) == Core.TypeEgal{Int} # Inference of constant svecs @eval fsvecinf() = $(QuoteNode(Core.svec(Tuple{Int,Int}, Int)))[1] -@test only(Base.return_types(fsvecinf, Tuple{})) == Type{Tuple{Int,Int}} +@test only(Base.return_types(fsvecinf, Tuple{})) == Core.TypeEgal{Tuple{Int,Int}} # nfields tfunc on `DataType` let f = ()->Val{nfields(DataType[Int][1])} @@ -1023,7 +1704,7 @@ end # issue #21410 f21410(::V, ::Pair{V,E}) where {V, E} = E @test only(Base.return_types(f21410, Tuple{Ref, Pair{Ref{T},Ref{T}} where T<:Number})) == - Type{E} where E <: (Ref{T} where T<:Number) + Type{Ref{T}} where T<:Number # issue #21369 function inf_error_21369(arg) @@ -1298,7 +1979,7 @@ function test_const_return(@nospecialize(f), @nospecialize(t), @nospecialize(val continue end end - @test false || "Side effect expressions found $ex" + @test false context="Side effect expressions found $ex" return end end @@ -1315,6 +1996,8 @@ function find_call(code::Core.CodeInfo, @nospecialize(func), narg) end elseif isa(farg, Core.SSAValue) farg = Compiler.widenconst(code.ssavaluetypes[farg.id]) + elseif isa(farg, Core.BindingPartition) + farg = typeof(Base.partition_restriction(farg)) else farg = typeof(farg) end @@ -1334,6 +2017,7 @@ test_const_return(()->sizeof(1 < 2), Tuple{}, 1) test_const_return(()->fieldtype(Dict{Int64,Nothing}, :age), Tuple{}, UInt) test_const_return(@eval(()->Core.sizeof($(Array{Int,0}(undef)))), Tuple{}, 2 * sizeof(Int)) test_const_return(@eval(()->Core.sizeof($(Matrix{Float32}(undef, 2, 2)))), Tuple{}, 4 * sizeof(Int)) +primitive type BitsizeofUInt17 17 end # TODO: do we want to implement these? # test_const_return(@eval(()->sizeof($(Array{Int,0}(undef)))), Tuple{}, sizeof(Int)) # test_const_return(@eval(()->sizeof($(Matrix{Float32}(undef, 2, 2)))), Tuple{}, 4 * 2 * 2) @@ -1345,6 +2029,15 @@ function sizeof_typeref(typeref) end @test @inferred(sizeof_typeref(Ref{DataType}(Int))) == sizeof(Int) @test find_call(only(code_typed(sizeof_typeref, (Ref{DataType},)))[1], Core.sizeof, 2) +# Make sure Core.bitsizeof with a ::DataType as inferred input type is inferred but not constant. +function bitsizeof_typeref(typeref) + return Core.bitsizeof(typeref[]) +end +@test bitsizeof_typeref(Ref{DataType}(BitsizeofUInt17)) == 17 +let (src, rt) = only(code_typed(bitsizeof_typeref, (Ref{DataType},))) + @test rt === Int + @test find_call(src, Core.bitsizeof, 2) +end # Constant `Vector` can be resized and shouldn't be optimized to a constant. const constvec = [1, 2, 3] @eval function sizeof_constvec() @@ -1381,7 +2074,7 @@ let isa_tfunc(@nospecialize xs...) = @test isa_tfunc(typeof(Union{}), Union{}) === Union{} # any result is ok @test isa_tfunc(typeof(Union{}), Type{typeof(Union{})}) === Const(true) @test isa_tfunc(typeof(Union{}), Const(typeof(Union{}))) === Const(true) - let c = Conditional(0, Const(Union{}), Const(Union{})) + let c = Conditional(#= slot =# 0, #= ssadef =# 0, Const(Union{}), Const(Union{})) @test isa_tfunc(c, Const(Bool)) === Const(true) @test isa_tfunc(c, Type{Bool}) === Const(true) @test isa_tfunc(c, Const(Real)) === Const(true) @@ -1433,7 +2126,7 @@ let subtype_tfunc(@nospecialize xs...) = @test subtype_tfunc(Type{Union{}}, Any) === Const(true) # Union{} <: Any @test subtype_tfunc(Type{Union{}}, Union{Type{Int64}, Type{Float64}}) === Const(true) @test subtype_tfunc(Type{Union{}}, Union{Type{T}, Type{Float64}} where T) === Const(true) - let c = Conditional(0, Const(Union{}), Const(Union{})) + let c = Conditional(#= slot =# 0, #= ssadef =# 0, Const(Union{}), Const(Union{})) @test subtype_tfunc(c, Const(Bool)) === Const(true) # any result is ok end @test subtype_tfunc(Type{Val{1}}, Type{Val{T}} where T) === Bool @@ -1465,8 +2158,8 @@ let egal_tfunc @test egal_tfunc(Array, Array) == Bool @test egal_tfunc(Array, AbstractArray{Int}) == Bool @test egal_tfunc(Array{Real}, AbstractArray{Int}) === Const(false) - @test egal_tfunc(Array{Real, 2}, AbstractArray{Real, 2}) === Bool - @test egal_tfunc(Array{Real, 2}, AbstractArray{Int, 2}) === Const(false) + @test egal_tfunc(Matrix{Real}, AbstractMatrix{Real}) === Bool + @test egal_tfunc(Matrix{Real}, AbstractMatrix{Int}) === Const(false) @test egal_tfunc(DataType, Int) === Const(false) @test egal_tfunc(DataType, Const(Int)) === Bool @test egal_tfunc(DataType, Const(Array)) === Const(false) @@ -1477,7 +2170,7 @@ let egal_tfunc @test egal_tfunc(Type{Union{Float32, Float64}}, Type{Union{Float32, Float64}}) === Bool @test egal_tfunc(typeof(Union{}), typeof(Union{})) === Bool # could be improved @test egal_tfunc(Const(typeof(Union{})), Const(typeof(Union{}))) === Const(true) - let c = Conditional(0, Const(Union{}), Const(Union{})) + let c = Conditional(#= slot =# 0, #= ssadef =# 0, Const(Union{}), Const(Union{})) @test egal_tfunc(c, Const(Bool)) === Const(false) @test egal_tfunc(c, Type{Bool}) === Const(false) @test egal_tfunc(c, Const(Real)) === Const(false) @@ -1488,17 +2181,17 @@ let egal_tfunc @test egal_tfunc(c, Bool) === Bool @test egal_tfunc(c, Any) === Bool end - let c = Conditional(0, Union{}, Const(Union{})) # === Const(false) - @test egal_tfunc(c, Const(false)) === Conditional(c.slot, c.elsetype, Union{}) - @test egal_tfunc(c, Const(true)) === Conditional(c.slot, Union{}, c.elsetype) + let c = Conditional(#= slot =# 0, #= ssadef =# 0, Union{}, Const(Union{})) # === Const(false) + @test egal_tfunc(c, Const(false)) === Conditional(c.slot, c.ssadef, c.elsetype, Union{}) + @test egal_tfunc(c, Const(true)) === Conditional(c.slot, c.ssadef, Union{}, c.elsetype) @test egal_tfunc(c, Const(nothing)) === Const(false) @test egal_tfunc(c, Int) === Const(false) @test egal_tfunc(c, Bool) === Bool @test egal_tfunc(c, Any) === Bool end - let c = Conditional(0, Const(Union{}), Union{}) # === Const(true) - @test egal_tfunc(c, Const(false)) === Conditional(c.slot, Union{}, c.thentype) - @test egal_tfunc(c, Const(true)) === Conditional(c.slot, c.thentype, Union{}) + let c = Conditional(#= slot =# 0, #= ssadef =# 0, Const(Union{}), Union{}) # === Const(true) + @test egal_tfunc(c, Const(false)) === Conditional(c.slot, c.ssadef, Union{}, c.thentype) + @test egal_tfunc(c, Const(true)) === Conditional(c.slot, c.ssadef, c.thentype, Union{}) @test egal_tfunc(c, Const(nothing)) === Const(false) @test egal_tfunc(c, Int) === Const(false) @test egal_tfunc(c, Bool) === Bool @@ -1527,16 +2220,24 @@ let nfields_tfunc(@nospecialize xs...) = Compiler.nfields_tfunc(Compiler.fallback_lattice, xs...) sizeof_tfunc(@nospecialize xs...) = Compiler.sizeof_tfunc(Compiler.fallback_lattice, xs...) + bitsizeof_tfunc(@nospecialize xs...) = + Compiler.bitsizeof_tfunc(Compiler.fallback_lattice, xs...) sizeof_nothrow(@nospecialize xs...) = Compiler.sizeof_nothrow(xs...) @test sizeof_tfunc(Const(Ptr)) === sizeof_tfunc(Union{Ptr, Int, Type{Ptr{Int8}}, Type{Int}}) === Const(Sys.WORD_SIZE ÷ 8) @test sizeof_tfunc(Type{Ptr}) === Const(sizeof(Ptr)) - @test sizeof_nothrow(Union{Ptr, Int, Type{Ptr{Int8}}, Type{Int}}) + @test bitsizeof_tfunc(Type{BitsizeofUInt17}) === Const(17) + @test bitsizeof_tfunc(DataType) === Int + @test !sizeof_nothrow(Union{Ptr, Int, Type{Ptr{Int8}}, Type{Int}}) + @test sizeof_nothrow(Union{Ptr, Int, Core.TypeEgal{Ptr{Int8}}, Core.TypeEgal{Int}}) @test sizeof_nothrow(Const(Ptr)) - @test sizeof_nothrow(Type{Ptr}) - @test sizeof_nothrow(Type{Union{Ptr{Int}, Int}}) + @test !sizeof_nothrow(Type{Ptr}) + @test sizeof_nothrow(Core.TypeEgal{Ptr}) + @test !sizeof_nothrow(Type{Union{Ptr{Int}, Int}}) + @test sizeof_nothrow(Core.TypeEgal{Union{Ptr{Int}, Int}}) @test !sizeof_nothrow(Const(Tuple)) - @test sizeof_nothrow(Type{Vector{Int}}) + @test !sizeof_nothrow(Type{Vector{Int}}) + @test sizeof_nothrow(Core.TypeEgal{Vector{Int}}) @test !sizeof_nothrow(Type{Union{Int, String}}) @test sizeof_nothrow(String) @test !sizeof_nothrow(Type{String}) @@ -1550,14 +2251,22 @@ let nfields_tfunc(@nospecialize xs...) = @test nfields_tfunc(Number) === Int @test nfields_tfunc(Int) === Const(0) @test nfields_tfunc(Complex) === Const(2) - @test nfields_tfunc(Type{Type{Int}}) === Const(nfields(DataType)) + # only the egality kind `TypeEgal{X}` pins the value to exactly `X` (#61323) + @test nfields_tfunc(Type{Type{Int}}) === Int + @test nfields_tfunc(Core.TypeEgal{Type{Int}}) === Const(nfields(Type{Int})) @test nfields_tfunc(UnionAll) === Const(2) @test nfields_tfunc(DataType) === Const(nfields(DataType)) - @test nfields_tfunc(Type{Int}) === Const(nfields(DataType)) - @test nfields_tfunc(Type{Integer}) === Const(nfields(DataType)) + @test nfields_tfunc(Type{Int}) === Int + @test nfields_tfunc(Core.TypeEgal{Int}) === Const(nfields(DataType)) + @test nfields_tfunc(Type{Integer}) === Int + @test nfields_tfunc(Core.TypeEgal{Integer}) === Const(nfields(DataType)) @test nfields_tfunc(Type{Complex}) === Int @test nfields_tfunc(typeof(Union{})) === Const(0) + # `Type{Union{}}` stays egality-pinned (unique bottom object, see `isconstType`); + # `TypeEgal{Union{}}` normalizes to `typeof(Union{})` at construction @test nfields_tfunc(Type{Union{}}) === Const(0) + @test Core.TypeEgal{Union{}} === typeof(Union{}) + @test nfields_tfunc(Core.TypeEgal{Union{}}) === Const(0) @test nfields_tfunc(Tuple{Int, Vararg{Int}}) === Int @test nfields_tfunc(Tuple{Int, Integer}) === Const(2) @test nfields_tfunc(Union{Tuple{Int, Float64}, Tuple{Int, Int}}) === Const(2) @@ -1580,7 +2289,7 @@ let typeof_tfunc(@nospecialize xs...) = end f_typeof_tfunc(x) = typeof(x) -@test Base.return_types(f_typeof_tfunc, (Union{<:T, Int} where T<:Complex,)) == Any[Union{Type{Int}, Type{Complex{T}} where T<:Real}] +@test Base.return_types(f_typeof_tfunc, (Union{<:T, Int} where T<:Complex,)) == Any[Union{Core.TypeEgal{Int}, Type{Complex{T}} where T<:Real}] # memoryref_tfunc, memoryrefget_tfunc, memoryrefset!_tfunc, memoryref_isassigned, memoryrefoffset_tfunc let memoryref_tfunc(@nospecialize xs...) = Compiler.memoryref_tfunc(Compiler.fallback_lattice, xs...) @@ -1655,7 +2364,9 @@ end let tuple_tfunc(@nospecialize xs...) = Compiler.tuple_tfunc(Compiler.fallback_lattice, Any[xs...]) - @test Compiler.widenconst(tuple_tfunc(Type{Int})) === Tuple{DataType} + # only the egality kind `TypeEgal{X}` pins the element's `typeof` (#61323) + @test Compiler.widenconst(tuple_tfunc(Type{Int})) === Tuple{Type} + @test Compiler.widenconst(tuple_tfunc(Core.TypeEgal{Int})) === Tuple{DataType} # https://github.com/JuliaLang/julia/issues/44705 @test tuple_tfunc(Union{Type{Int32},Type{Int64}}) === Tuple{Type} @test tuple_tfunc(DataType) === Tuple{DataType} @@ -1673,7 +2384,10 @@ g23024(TT::Tuple{DataType}) = f23024(TT[1], v23024) @test !Compiler.isconstType(Type{typeof(Union{})}) # could be Core.TypeofBottom or Type{Union{}} at runtime @test !isa(Compiler.getfield_tfunc(Compiler.fallback_lattice, Type{Core.TypeofBottom}, Compiler.Const(:name)), Compiler.Const) -@test Base.return_types(supertype, (Type{typeof(Union{})},)) == Any[Any] +# values of `Type{typeof(Union{})}` include `UnionAll` spellings (#33136), so +# `supertype(::UnionAll)` is applicable too; the egal-pinned type stays precise +@test Base.return_types(supertype, (Type{typeof(Union{})},)) == Any[Any, Type{Core.AnyType}] +@test Base.return_types(supertype, (Core.TypeEgal{typeof(Union{})},)) == Any[Core.TypeEgal{Core.AnyType}] # issue #23685 struct Node23685{T} @@ -1771,7 +2485,7 @@ let getfield_tfunc(@nospecialize xs...) = @test getfield_tfunc(ARef{Int},Const(:x),Bool,Bool) === Union{} end -using Core: Const +using Core: Const, PartialStruct mutable struct XY{X,Y} x::X y::Y @@ -1801,6 +2515,7 @@ let setfield!_tfunc(@nospecialize xs...) = @test setfield!_tfunc(ABCDconst, Const(:c), Any) === Any @test setfield!_tfunc(ABCDconst, Const(3), Any) === Any @test setfield!_tfunc(ABCDconst, Symbol, Any) === Any + @test setfield!_tfunc(PartialStruct(Compiler.fallback_lattice, ABCDconst, Any[Const(42), Int, Any, Union{Int,Nothing}]), Const(:a), Int) === Union{} @test setfield!_tfunc(ABCDconst, Int, Any) === Any @test setfield!_tfunc(Union{Base.RefValue{Any},Some{Any}}, Const(:x), Int) === Int @test setfield!_tfunc(Union{Base.RefValue,Some{Any}}, Const(:x), Int) === Int @@ -1884,6 +2599,37 @@ let setfield!_nothrow(@nospecialize xs...) = @test !setfield!_nothrow(Any, Any, Int) end +mutable struct AtomicFields + @atomic a::Int + b::Int + const c::Int +end +let modifyfield!_tfunc(@nospecialize xs...) = + Compiler.modifyfield!_tfunc(Compiler.fallback_lattice, xs...) + replacefield!_tfunc(@nospecialize xs...) = + Compiler.replacefield!_tfunc(Compiler.fallback_lattice, xs...) + cmpswap_Int = ccall(:jl_apply_cmpswap_type, Any, (Any,), Int) + # writable fields (`@atomic` and plain) of a mutable struct + @test modifyfield!_tfunc(AtomicFields, Const(:a), Any, Any) === Pair{Int,Int} + @test replacefield!_tfunc(AtomicFields, Const(:a), Int, Int) === cmpswap_Int + @test modifyfield!_tfunc(AtomicFields, Const(:b), Any, Any) === Pair{Int,Int} + @test replacefield!_tfunc(AtomicFields, Const(:b), Int, Int) === cmpswap_Int + # `replacefield!` type-checks the replacement value unconditionally, so a value that + # can never be stored always throws + @test replacefield!_tfunc(AtomicFields, Const(:a), Int, String) === Union{} + @test replacefield!_tfunc(AtomicFields, Const(:b), Int, String) === Union{} + # `const` fields can never be written, so the operation always throws + @test modifyfield!_tfunc(AtomicFields, Const(:c), Any, Any) === Union{} + @test modifyfield!_tfunc(AtomicFields, Const(3), Any, Any) === Union{} + @test replacefield!_tfunc(AtomicFields, Const(:c), Int, Int) === Union{} + @test replacefield!_tfunc(AtomicFields, Const(3), Int, Int) === Union{} + # immutable types can never be written, so the operation always throws + @test modifyfield!_tfunc(Some{Int}, Const(:value), Any, Any) === Union{} + @test replacefield!_tfunc(Some{Int}, Const(:value), Int, Int) === Union{} + @test modifyfield!_tfunc(Some, Const(:value), Any, Any) === Union{} + @test replacefield!_tfunc(Some, Const(:value), Any, Any) === Union{} +end + struct Foo_22708 x::Ptr{Foo_22708} end @@ -2024,7 +2770,10 @@ g26339(T) = T === Int ? 1 : "" @test Base.return_types(f26339, (Int,)) == Any[String] @test Base.return_types(g26339, (Int,)) == Any[String] @test Base.return_types(f26339, (Type{Int},)) == Any[String] -@test Base.return_types(g26339, (Type{Int},)) == Any[Int] +# only the egality kind pins the argument to `=== Int`; a `Type{Int}` query also +# admits `S == Int` reps with `S !== Int` (#61323), so there `T === Int` cannot fold +@test Base.return_types(g26339, (Core.TypeEgal{Int},)) == Any[Int] +@test Base.return_types(g26339, (Type{Int},)) == Any[Union{Int, String}] @test Base.return_types(f26339, (Type{Union{}},)) == Any[Int] @test Base.return_types(g26339, (Type{Union{}},)) == Any[String] @test Base.return_types(f26339, (typeof(Union{}),)) == Any[Int] @@ -2032,6 +2781,80 @@ g26339(T) = T === Int ? 1 : "" @test Base.return_types(f26339, (Type,)) == Any[Union{Int, String}] @test Base.return_types(g26339, (Type,)) == Any[Union{Int, String}] +@noinline call_type_int61323(f, x) = f(Base.inferencebarrier(x)::Type{Int}) + +# JuliaLang/julia#61323: a `Type{X}`-typed (`==`-only) value must not be treated as +# `=== X`, neither directly nor through a static parameter bound from it, while +# egality-pinned (`Const`/dispatch) queries keep folding. `S == Int` but `S !== Int`: +let S = (Union{T, U} where {T<:Int, U<:Int}) + @test S == Int && S !== Int + garg61323(x) = x === Int ? 1 : "" + fsparam61323(::Type{T}) where {T} = T === Int ? 1 : "" + for f in Any[garg61323, fsparam61323] + @test f(Int) === 1 + @test f(S) === "" + @test call_type_int61323(f, S) === "" + @test Base.return_types(f, (Type{Int},)) == Any[Union{Int, String}] + @test Base.return_types(f, (Core.TypeEgal{Int},)) == Any[Int] + end + @test Base.return_types(() -> garg61323(Int)) == Any[Int] + @test Base.return_types(() -> fsparam61323(Int)) == Any[Int] + # instantiation `==`-deduplicates parameters, so this folds for every rep, + # consistently with the runtime + fapply61323(::Type{T}) where {T} = Ref{T} === Ref{Int} ? 1 : "" + @test fapply61323(Int) === 1 + @test fapply61323(S) === 1 + @test call_type_int61323(fapply61323, S) === 1 + @test Base.return_types(fapply61323, (Type{Int},)) == Any[Int] +end + +# Defined type-valued static parameters should retain egality so DataType +# parameter reads can fold through StaticArrays-like Size computation. +tuple_svec62001(::Type{T}) where {T<:Tuple} = T.parameters +Base.@pure tuple_tuple62001(::Type{T}) where {T<:Tuple} = (tuple_svec62001(T)...,) +struct Size62001{S} end +Base.@pure Size62001(s::Tuple) = Size62001{s}() +Size62001(::Type{T}) where {T<:Tuple} = Size62001{tuple_tuple62001(T)}() +abstract type StaticArray62001{S<:Tuple,T,N} <: AbstractArray{T,N} end +struct SArray62001{S<:Tuple,T,N,L} <: StaticArray62001{S,T,N} + data::NTuple{L,T} +end +const SMatrix62001{N,M,T,L} = SArray62001{Tuple{N,M},T,2,L} +Size62001(::Type{SA}) where {SA <: StaticArray62001} = error() +Size62001(::Type{SA}) where {SA <: StaticArray62001{S}} where {S<:Tuple} = + @isdefined(S) ? Size62001(S) : error() +Size62001(a::T) where {T<:AbstractArray} = Size62001(T) +Base.getindex(a::SArray62001, i::Int) = getfield(a, :data)[i] +@generated function _getindex_scalar62001(::Size62001{S}, a::StaticArray62001, + inds::Int...) where S + if length(inds) == 0 + return :(a[1]) + end + stride = 1 + ind_expr = :() + for i in 1:length(inds) + if i == 1 + ind_expr = :(inds[1]) + else + ind_expr = :($ind_expr + $stride * (inds[$i] - 1)) + end + stride *= S[i] + end + return :(a[$ind_expr]) +end +Base.getindex(a::StaticArray62001, inds::Int...) = _getindex_scalar62001(Size62001(a), a, inds...) +struct StaticKernel62001{N,T} + P::SMatrix62001{N,N,T} +end +static_matrix_getindex62001(k::StaticKernel62001{N,T}, i::Int, j::Int) where {N,T} = k.P[i,j] +let A = SMatrix62001{2,2,Float64,4}((1.0, 2.0, 3.0, 4.0)) + @test only(Base.return_types(Size62001, Tuple{Type{SMatrix62001{2,2,Float64}}})) == Size62001{(2,2)} + @test only(Base.return_types(Size62001, Tuple{SMatrix62001{2,2,Float64}})) == Size62001{(2,2)} + @test only(Base.return_types(getindex, Tuple{SMatrix62001{2,2,Float64},Int,Int})) === Float64 + @test only(Base.return_types(static_matrix_getindex62001, + Tuple{typeof(StaticKernel62001{2,Float64}(A)),Int,Int})) === Float64 +end + # Test that Conditional doesn't get widened to Bool too quickly f25261() = (1, 1) f25261(s) = i == 1 ? (1, 2) : nothing @@ -2229,10 +3052,30 @@ end end @testset "`from_interprocedural!`: translate inter-procedural information" begin - # TODO come up with a test case to check the functionality of `collect_limitations!` - # one heavy test case would be to use https://github.com/aviatesk/JET.jl and - # check `julia /path/to/JET/jet /path/to/JET/src/JET.jl` doesn't result in errors - # because of nested `LimitedAccuracy`es + # LimitedAccuracy from nested IR interpretation should be recorded on the + # nearest enclosing inference frame. + limited_parent62001(x) = x + function make_irsv62001(interp, mi, argtypes, world) + src = Compiler.retrieve_code_info(mi, world) + spec_info = Compiler.SpecInfo(src) + ir = Compiler.inflate_ir(src, mi) + return Compiler.IRInterpretationState(interp, spec_info, ir, mi, + Any[argtypes...], UInt(1), world) + end + let interp = Compiler.NativeInterpreter(), world = Base.get_world_counter() + match = only(Base._methods_by_ftype(Tuple{typeof(limited_parent62001),Int}, -1, world)) + mi = Compiler.specialize_method(match) + parent = Compiler.InferenceState(Compiler.InferenceResult(mi, Compiler.typeinf_lattice(interp)), + :global, interp) + child = make_irsv62001(interp, mi, (Core.Const(limited_parent62001), Int), world) + Compiler.assign_parentchild!(child, parent) + grandchild = make_irsv62001(interp, mi, (Core.Const(limited_parent62001), Int), world) + Compiler.assign_parentchild!(grandchild, child) + causes = IdSet{Compiler.InferenceState}() + push!(causes, parent) + @test Compiler.collect_limitations!(Compiler.LimitedAccuracy(String, causes), grandchild) === String + @test parent in parent.pclimitations + end # `InterConditional` handling: `abstract_invoke` ispositive(a) = isa(a, Int) && a > 0 @@ -2245,7 +3088,7 @@ end # the `fargs = nothing` edge case @test Base.return_types((Any,)) do a Base._return_type(invoke, Tuple{typeof(ispositive), Type{Tuple{Any}}, Any}) - end |> only == Type{Bool} + end |> only == Core.TypeEgal{Bool} # `InterConditional` handling: `abstract_call_opaque_closure` @test Base.return_types((Any,)) do a @@ -2279,46 +3122,36 @@ mutable struct AliasableConstField{S,T} f2::T end -using .Compiler: - InferenceLattice, MustAliasesLattice, InterMustAliasesLattice, - BaseInferenceLattice, SimpleInferenceLattice, IPOResultLattice, typeinf_lattice, ipo_lattice, optimizer_lattice - -include("newinterp.jl") -@newinterp MustAliasInterpreter -Compiler.typeinf_lattice(::MustAliasInterpreter) = InferenceLattice(MustAliasesLattice(BaseInferenceLattice.instance)) -Compiler.ipo_lattice(::MustAliasInterpreter) = InferenceLattice(InterMustAliasesLattice(IPOResultLattice.instance)) -Compiler.optimizer_lattice(::MustAliasInterpreter) = SimpleInferenceLattice.instance - # lattice # ------- using .Compiler: MustAlias, Const, PartialStruct, ⊑, tmerge -let 𝕃ᵢ = InferenceLattice(MustAliasesLattice(BaseInferenceLattice.instance)) +let 𝕃ᵢ = Compiler.InferenceLattice(Compiler.MustAliasesLattice(Compiler.BaseInferenceLattice.instance)) ⊑(@nospecialize(a), @nospecialize(b)) = Compiler.:⊑(𝕃ᵢ, a, b) tmerge(@nospecialize(a), @nospecialize(b)) = Compiler.tmerge(𝕃ᵢ, a, b) isa_tfunc(@nospecialize xs...) = Compiler.isa_tfunc(𝕃ᵢ, xs...) ifelse_tfunc(@nospecialize xs...) = Compiler.ifelse_tfunc(𝕃ᵢ, xs...) - @test (MustAlias(2, AliasableField{Any}, 1, Int) ⊑ Int) - @test !(Int ⊑ MustAlias(2, AliasableField{Any}, 1, Int)) - @test (Int ⊑ MustAlias(2, AliasableField{Any}, 1, Any)) - @test (Const(42) ⊑ MustAlias(2, AliasableField{Any}, 1, Int)) - @test !(MustAlias(2, AliasableField{Any}, 1, Any) ⊑ Int) - @test tmerge(MustAlias(2, AliasableField{Any}, 1, Any), Const(nothing)) === Any - @test tmerge(MustAlias(2, AliasableField{Any}, 1, Int), Const(nothing)) === Union{Int,Nothing} - @test tmerge(Const(nothing), MustAlias(2, AliasableField{Any}, 1, Any)) === Any - @test tmerge(Const(nothing), MustAlias(2, AliasableField{Any}, 1, Int)) === Union{Int,Nothing} + @test (MustAlias(2, 0, AliasableField{Any}, 1, Int) ⊑ Int) + @test !(Int ⊑ MustAlias(2, 0, AliasableField{Any}, 1, Int)) + @test (Int ⊑ MustAlias(2, 0, AliasableField{Any}, 1, Any)) + @test (Const(42) ⊑ MustAlias(2, 0, AliasableField{Any}, 1, Int)) + @test !(MustAlias(2, 0, AliasableField{Any}, 1, Any) ⊑ Int) + @test tmerge(MustAlias(2, 0, AliasableField{Any}, 1, Any), Const(nothing)) === Any + @test tmerge(MustAlias(2, 0, AliasableField{Any}, 1, Int), Const(nothing)) === Union{Int,Nothing} + @test tmerge(Const(nothing), MustAlias(2, 0, AliasableField{Any}, 1, Any)) === Any + @test tmerge(Const(nothing), MustAlias(2, 0, AliasableField{Any}, 1, Int)) === Union{Int,Nothing} tmerge(Const(AbstractVector{<:Any}), Const(AbstractVector{T} where {T})) # issue #56913 - @test isa_tfunc(MustAlias(2, AliasableField{Any}, 1, Bool), Const(Bool)) === Const(true) - @test isa_tfunc(MustAlias(2, AliasableField{Any}, 1, Bool), Type{Bool}) === Const(true) - @test isa_tfunc(MustAlias(2, AliasableField{Any}, 1, Int), Type{Bool}) === Const(false) - @test ifelse_tfunc(MustAlias(2, AliasableField{Any}, 1, Bool), Int, Int) === Int - @test ifelse_tfunc(MustAlias(2, AliasableField{Any}, 1, Int), Int, Int) === Union{} + @test isa_tfunc(MustAlias(2, 0, AliasableField{Any}, 1, Bool), Const(Bool)) === Const(true) + @test isa_tfunc(MustAlias(2, 0, AliasableField{Any}, 1, Bool), Type{Bool}) === Const(true) + @test isa_tfunc(MustAlias(2, 0, AliasableField{Any}, 1, Int), Type{Bool}) === Const(false) + @test ifelse_tfunc(MustAlias(2, 0, AliasableField{Any}, 1, Bool), Int, Int) === Int + @test ifelse_tfunc(MustAlias(2, 0, AliasableField{Any}, 1, Int), Int, Int) === Union{} end maybeget_mustalias_tmerge(x::AliasableField) = x.f maybeget_mustalias_tmerge(x) = x -@test Base.return_types((Union{Nothing,AliasableField{Any}},); interp=MustAliasInterpreter()) do x +@test Base.return_types((Union{Nothing,AliasableField{Any}},)) do x isa(maybeget_mustalias_tmerge(x)#=::Any, not MustAlias=#, Int) && throw() x end |> only === Union{Nothing,AliasableField{Any}} @@ -2327,38 +3160,38 @@ end |> only === Union{Nothing,AliasableField{Any}} # -------------- # simple intra-procedural case -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField,)) do a if isa(getfield(a, :f), Int) return getfield(a, :f) end return 0 end |> only === Int -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField,)) do a if isa(getfield(a, 1), Int) return getfield(a, 1) end return 0 end |> only === Int -@test Base.return_types((AliasableField{Union{Some{Int},Nothing}},); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField{Union{Some{Int},Nothing}},)) do a if isa(getfield(a, 1), Some) return getfield(a, 1) end throw() end |> only === Some{Int} -@test Base.return_types((Tuple{Any},); interp=MustAliasInterpreter()) do t +@test Base.return_types((Tuple{Any},)) do t if isa(getfield(t, 1), Int) return getfield(t, 1) end return 0 end |> only === Int -@test Base.return_types((Any,); interp=MustAliasInterpreter()) do a +@test Base.return_types((Any,)) do a x = AliasableFields(a, 0) # x::PartialStruct(AliasableFields, Any[Any, Const(0)]) if isa(getfield(x, :f1), Int) # x::PartialStruct(AliasableFields, Any[Int, Const(0)]) return getfield(x, :f1) end return 0 end |> only === Int -@test Base.return_types((Any,Any); interp=MustAliasInterpreter()) do a, b +@test Base.return_types((Any,Any)) do a, b x = AliasableFields(a, b) # x::AliasableFields if isa(getfield(x, :f1), Int) # x::PartialStruct(AliasableFields, Any[Int, Any]) if isa(getfield(x, :f2), Int) # x::PartialStruct(AliasableFields, Any[Int, Int]) @@ -2367,7 +3200,7 @@ end |> only === Int end return 0, 0 end |> only === Tuple{Int,Int} -@test Base.return_types((Any,); interp=MustAliasInterpreter()) do a +@test Base.return_types((Any,)) do a x = AliasableConstField(a, 0) if isa(getfield(x, :f1), Int) return getfield(x, :f1) @@ -2376,14 +3209,14 @@ end |> only === Tuple{Int,Int} end |> only === Int # shouldn't use refinement information when not worthwhile -@test Base.return_types((AliasableField{Int},); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField{Int},)) do a if isa(getfield(a, :f), Any) return getfield(a, :f) # shouldn't be ::Any end return 0 end |> only === Int # shouldn't assume anything about mutable field -@test Base.return_types((Any,Any); interp=MustAliasInterpreter()) do a, b +@test Base.return_types((Any,Any)) do a, b x = AliasableConstField{Any,Any}(a, b) if isa(getfield(x, :f2), Int) setfield!(x, :f2, z::Any) @@ -2392,7 +3225,7 @@ end |> only === Int return 0 end |> only === Any # when abstract type, we shouldn't assume anything -@test Base.return_types((Any,); interp=MustAliasInterpreter()) do a +@test Base.return_types((Any,)) do a if isa(getfield(a, :mayexist), Int) return getfield(a, :mayexist) end @@ -2400,26 +3233,26 @@ end |> only === Any end |> only === Any # works inter-procedurally -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField,)) do a if isa(a.f, Int) return a.f end return 0 end |> only === Int -@test Base.return_types((Tuple{Any},); interp=MustAliasInterpreter()) do t +@test Base.return_types((Tuple{Any},)) do t if isa(t[1], Int) return t[1] end return 0 end |> only === Int -@test Base.return_types((Any,); interp=MustAliasInterpreter()) do a +@test Base.return_types((Any,)) do a x = AliasableFields(a, 0) # x::PartialStruct(AliasableFields, Any[Any, Const(0)]) if isa(x.f1, Int) # x::PartialStruct(AliasableFields, Any[Int, Const(0)]) return x.f1 end return 0 end |> only === Int -@test Base.return_types((Any,Any); interp=MustAliasInterpreter()) do a, b +@test Base.return_types((Any,Any)) do a, b x = AliasableFields(a, b) # x::AliasableFields if isa(x.f1, Int) # x::PartialStruct(AliasableFields, Any[Int, Any]) if isa(x.f2, Int) # x::PartialStruct(AliasableFields, Any[Int, Int]) @@ -2428,7 +3261,7 @@ end |> only === Int end return 0, 0 end |> only === Tuple{Int,Int} -@test Base.return_types((Any,); interp=MustAliasInterpreter()) do a +@test Base.return_types((Any,)) do a x = AliasableConstField(a, 0) if isa(x.f1, Int) return x.f1 @@ -2436,7 +3269,7 @@ end |> only === Tuple{Int,Int} return 0 end |> only === Int getf(a) = a.f -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField,)) do a if isa(getf(a), Int) return getf(a) end @@ -2447,13 +3280,13 @@ end |> only === Int merge_same_aliases(b, a) = b ? _merge_same_aliases1(a) : _merge_same_aliases2(a) # MustAlias(a, Const(:f1), Union{Int,Nothing}) _merge_same_aliases1(a) = (@assert isa(a.f, Int); a.f) # ::MustAlias(a, Const(:f1), Int) _merge_same_aliases2(a) = (@assert isa(a.f, Nothing); a.f) # ::MustAlias(a, Const(:f1), Nothing) -@test Base.return_types((Bool,AliasableField,); interp=MustAliasInterpreter()) do b, a +@test Base.return_types((Bool,AliasableField,)) do b, a return merge_same_aliases(b, a) # ::Union{Int,Nothing} end |> only === Union{Nothing,Int} # call-site refinement isaint(a) = isa(a, Int) -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do a +@test Base.return_types((AliasableField,)) do a if isaint(a.f) return a.f end @@ -2462,17 +3295,17 @@ end |> only === Int # handle multiple call-site refinement targets isasome(_) = true isasome(::Nothing) = false -@test_broken Base.return_types((AliasableField{Union{Int,Nothing}},); interp=MustAliasInterpreter()) do a +@test Base.infer_return_type((AliasableField{Union{Int,Nothing}},)) do a if isasome(a.f) return a.f end return 0 -end |> only === Int +end === Int # appropriate lattice order -@test Base.return_types((AliasableField{Any},); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField{Any},)) do x v = x.f # ::MustAlias(2, AliasableField{Any}, 1, Any) - if isa(v, Int) # ::Conditional(3, Int, Any) + if isa(v, Int) # ::Conditional(3, _, Int, Any) v = v # ::Int (∵ Int ⊑ MustAlias(2, AliasableField{Any}, 1, Any)) else v = 42 @@ -2482,28 +3315,28 @@ end |> only === Int # complicated callsite refinement cases from_interconditional_check11(y::Int, ::AliasableField) = y > 0 -@test Base.return_types((AliasableField{Any},); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField{Any},)) do x if from_interconditional_check11(x.f, x) return x.f end return 0 end |> only === Int from_interconditional_check12(::AliasableField, y::Int) = y > 0 -@test Base.return_types((AliasableField{Any},); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField{Any},)) do x if from_interconditional_check12(x, x.f) return x.f end return 0 end |> only === Int from_interconditional_check21(y, ::Union{Int,String}) = isa(y, Int) -@test Base.return_types((AliasableField{Any},); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField{Any},)) do x if from_interconditional_check21(x.f, x.f) return x.f end return 0 end |> only === Int from_interconditional_check22(::Union{Int,String}, y) = isa(y, Int) -@test Base.return_types((AliasableField{Any},); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField{Any},)) do x if from_interconditional_check22(x.f, x.f) return x.f end @@ -2516,7 +3349,7 @@ struct JET509 list::Union{Tuple{},Vector{Int}} end jet509_hasitems(list) = length(list) >= 1 -@test Base.return_types((JET509,); interp=MustAliasInterpreter()) do ilist::JET509 +@test Base.return_types((JET509,)) do ilist::JET509 list = ilist.list if jet509_hasitems(list) return list @@ -2525,7 +3358,7 @@ jet509_hasitems(list) = length(list) >= 1 end |> only == Vector{Int} # don't form nested slot wrappers -@test Base.infer_return_type((NullableAliasableFields{NullableAliasableFields},); interp=MustAliasInterpreter()) do x +@test Base.infer_return_type((NullableAliasableFields{NullableAliasableFields},)) do x y = getfield(x, :f1) if isdefined(y, :f2) && isa(getfield(y, :f2), Int) return getfield(y, :f2) @@ -2537,52 +3370,52 @@ end == Int # -------------- # simple symmetric tests -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField,)) do x if x.f === 0 return x.f end return 0 end |> only === Int -@test Base.return_types((AliasableField,); interp=MustAliasInterpreter()) do x +@test Base.return_types((AliasableField,)) do x if 0 === x.f return x.f end return 0 end |> only === Int # NOTE we prioritize constraints on aliased field over those on slots themselves -@test Base.return_types((AliasableField,Int,); interp=MustAliasInterpreter()) do x, a +@test Base.return_types((AliasableField,Int,)) do x, a if x.f === a return x.f end return 0 end |> only === Int -@test Base.return_types((AliasableField,Int,); interp=MustAliasInterpreter()) do x, a +@test Base.return_types((AliasableField,Int,)) do x, a if a === x.f return x.f end return 0 end |> only === Int -@test_broken Base.return_types((AliasableField{Union{Nothing,Int}},); interp=MustAliasInterpreter()) do x +@test Base.infer_return_type((AliasableField{Union{Nothing,Int}},)) do x if !isnothing(x.f) return x.f end return 0 -end |> only === Int -@test_broken Base.return_types((AliasableField{Union{Some{Int},Nothing}},); interp=MustAliasInterpreter()) do x +end === Int +@test Base.infer_return_type((AliasableField{Union{Some{Int},Nothing}},)) do x if !isnothing(x.f) return x.f end throw() -end |> only === Some{Int} +end === Some{Int} # handle the edge case @eval intermustalias_edgecase(_) = $(Compiler.InterMustAlias(2, Some{Any}, 1, Int)) -Base.return_types(intermustalias_edgecase, (Any,); interp=MustAliasInterpreter()) # create cache -@test Base.return_types((Any,); interp=MustAliasInterpreter()) do x +Base.return_types(intermustalias_edgecase, (Any,)) # create cache +@test Base.return_types((Any,)) do x intermustalias_edgecase(x) end |> only === Compiler.InterMustAlias -@test Base.infer_return_type((AliasableField,Integer,); interp=MustAliasInterpreter()) do a, x +@test Base.infer_return_type((AliasableField,Integer,)) do a, x s = (;x) if getfield(a, :f) isa Symbol return getfield(s, getfield(a, :f)) @@ -2591,17 +3424,73 @@ end |> only === Compiler.InterMustAlias end == Integer # `isdefined` accuracy for `MustAlias` -@test Base.infer_return_type((Any,); interp=MustAliasInterpreter()) do x +@test Base.infer_return_type((Any,)) do x xx = Ref{Any}(x) xxx = Some{Any}(xx) Val(isdefined(xxx.value, :x)) end == Val{true} +# Test union splitting for MustAlias +struct GetSomethingA; x::Union{Nothing,Int}; end +struct GetSomethingB; x::Int; end +getsomethingx(a::GetSomethingA) = something(a.x, 0) +getsomethingx(b::GetSomethingB) = b.x +@test Base.infer_return_type((Union{GetSomethingA,GetSomethingB},)) do x + getsomethingx(x) +end == Int + +# https://github.com/JuliaLang/julia/issues/59975 +struct Issue59975; a; end +function issue59975(x::Issue59975) + if x.a isa Int + return x.a + end + return 0 +end +@test Base.infer_return_type(issue59975, (Issue59975,)) == Int + @testset "issue #56913: `BoundsError` in type inference" begin R = UnitRange{Int} - @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, Vararg{R}}) - @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, R, Vararg{R}}) -end + @test Core.TypeEgal{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, Vararg{R}}) + @test Core.TypeEgal{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, R, Vararg{R}}) +end + +# issue #61953: `constprop_cache_lookup` asserted that all cached const-prop results for a +# `MethodInstance` share their `argtypes` length. That is false for an `mi` whose `specTypes` +# ends in an unbounded `Vararg` (its trailing varargs are not specialized to a fixed arity): +# such an `mi` can be const-propagated at multiple arities, yielding cached results whose +# `argtypes` differ in length. This is a reduction of the original report (inferring +# `Polyhedra.points` over a `CDDLib.CDDGeneratorMatrix`): a varargs `constructpolyhedron` whose +# trailing iterators are a large `Union` keyed on a union-constrained coefficient type var, so +# type intersection widens differing-arity calls to the same unbounded-`Vararg` `mi`. +module Issue61953 + const CoefT = Union{Float64, Rational{BigInt}} + abstract type Rep{T} end + abstract type VRep{T} <: Rep{T} end + struct Line{T, AT<:AbstractVector{T}} end + struct Ray{T, AT<:AbstractVector{T}} end + abstract type AbstractRepIterator{T, ElemT} end + struct AllRepIterator{T, ElemT, LinElemT, LRT<:AbstractRepIterator{T, LinElemT}, RT<:AbstractRepIterator{T, ElemT}} end + const ElemIt{ElemT} = Union{AllRepIterator{<:Any, ElemT}, AbstractRepIterator{<:Any, ElemT}, AbstractVector{ElemT}} + const It{T} = Union{ElemIt{<:AbstractVector{T}}, ElemIt{<:Line{T}}, ElemIt{<:Ray{T}}} + const SINK = Ref{Any}(nothing) + mkrep(::Type{R}, d, it...) where {R} = R(length(it)) + function constructpolyhedron(RepT::Type{<:Rep{T}}, d, p::Tuple{Vararg{Rep}}, it::It{T}...) where {T} + SINK[] = d # observable effect, so const-prop is preferred over (semi-)concrete eval + return mkrep(RepT, d, it...)::RepT + end + mutable struct ConcreteV{T<:CoefT} <: VRep{T}; x::Int; end + # a statically-unknown-length iterator collection (splat yields a trailing `Vararg`)... + itervar(p::VRep{T}) where {T} = Base.inferencebarrier(())::Tuple{Vararg{It{T}}} + # ...and a fixed-length one (splat yields concrete trailing arguments) + iterfix(p::VRep{T}) where {T} = ntuple(_ -> Base.inferencebarrier(nothing)::It{T}, Val(3)) + asrep(p::VRep{T}) where {T} = Base.inferencebarrier(p)::VRep{T} + abstractrep(::Type{T}) where {T} = Base.inferencebarrier(ConcreteV{T})::Type{<:VRep{T}} + cvar(p::VRep{T}) where {T} = constructpolyhedron(abstractrep(T), 2, (asrep(p),), itervar(p)...) + cfix(p::VRep{T}) where {T} = constructpolyhedron(abstractrep(T), 2, (asrep(p),), iterfix(p)...) + driver(p::VRep{<:CoefT}) = (cvar(p), cfix(p)) +end +@test Base.infer_return_type(Issue61953.driver, Tuple{Issue61953.ConcreteV{<:Issue61953.CoefT}}) <: Tuple function f25579(g) h = g[] @@ -2629,7 +3518,7 @@ function h25579(g) return t ? typeof(h) : typeof(h) end @test Base.return_types(h25579, (Base.RefValue{Union{Nothing, Int}},)) == - Any[Type{Float64}] + Any[Core.TypeEgal{Float64}] f26172(v) = Val{length(Base.tail(ntuple(identity, v)))}() # Val(M-1) g26172(::Val{0}) = () @@ -2766,7 +3655,7 @@ end |> only == Union{Int,Tuple{Any,Any}} let A = Core.Const(true) B = Core.InterConditional(2, Tuple, Union{}) C = Core.InterConditional(2, Any, Union{}) - L = ipo_lattice(Compiler.NativeInterpreter()) + L = Compiler.ipo_lattice(Compiler.NativeInterpreter()) @test !⊑(L, A, B) @test ⊑(L, B, A) @test tmerge(L, A, B) == C @@ -2812,12 +3701,13 @@ end |> only === Int # correct `apply_type` inference of `NamedTuple{(), <:Any}` @test (() -> NamedTuple{(), <:Any})() isa UnionAll -# Don't pessimize apply_type to anything worse than Type (or TypeVar) and yield Bottom for invalid Unions -@test only(Base.return_types(Core.apply_type, Tuple{Type{Union}})) == Type{Union{}} +# Don't pessimize apply_type to anything worse than Type (or TypeVar). An `==`-only +# `Type{Union}` head isn't pinned down (#61323), so these widen to the Type/TypeVar floor. +@test only(Base.return_types(Core.apply_type, Tuple{Type{Union}})) == Union{Type,TypeVar} @test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Any})) == Union{Type,TypeVar} -@test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Any,Any})) == Type -@test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Int})) == Union{} -@test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Any,Int})) == Union{} +@test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Any,Any})) == Union{Type,TypeVar} +@test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Int})) == Union{Type,TypeVar} +@test only(Base.return_types(Core.apply_type, Tuple{Type{Union},Any,Int})) == Union{Type,TypeVar} @test only(Base.return_types(Core.apply_type, Tuple{Any})) == Any @test only(Base.return_types(Core.apply_type, Tuple{Any,Any})) == Any @@ -2839,8 +3729,8 @@ let apply_type_tfunc = Compiler.apply_type_tfunc @test apply_type_tfunc(𝕃, Const(Val), Type{Union{Int,Pair{Pair{Pair{Pair{A,B},C},D},E}}} where {A,B,C,D,E}) == Type{Val{_A}} where _A end @test only(Base.return_types(keys, (Dict{String},))) == Base.KeySet{String, T} where T<:(Dict{String}) -@test only(Base.return_types((r)->similar(Array{typeof(r[])}, 1), (Base.RefValue{Array{Int}},))) == Vector{<:Array{Int}} -@test only(Base.return_types((r)->similar(Array{typeof(r[])}, 1), (Base.RefValue{Array{<:Real}},))) == Vector{<:Array{<:Real}} +@test only(Base.return_types((r)->similar(Array{typeof(r[])}, 1), (Base.RefValue{Array{Int}},))) == Vector{Array{Int, N}} where N +@test only(Base.return_types((r)->similar(Array{typeof(r[])}, 1), (Base.RefValue{Array{<:Real}},))) == Vector{Array{T, N}} where {T<:Real, N} # test complexity limit on apply_type on a function capturing functions returning functions @test only(Base.return_types(Base.afoldl, (typeof((m, n) -> () -> Returns(nothing)(m, n)), Function, Function, Vararg{Function}))) === Function @@ -3358,7 +4248,7 @@ f30394(foo::T1, ::Type{T2}) where {T2, T1 <: T2} = foo f30394(foo, T2) = f30394(foo.foo_inner, T2) -@test Base.return_types(f30394, (Foo30394_2, Type{Base30394})) == Any[Base30394] +@test Base.return_types(f30394, (Foo30394_2, Core.TypeEgal{Base30394})) == Any[Base30394] # PR #30385 @@ -3509,11 +4399,11 @@ end struct MixedKeyDict{T<:Tuple} #<: AbstractDict{Any,Any} dicts::T end -Base.merge(f::Function, d::MixedKeyDict, others::MixedKeyDict...) = _merge(f, (), d.dicts, (d->d.dicts).(others)...) -Base.merge(f, d::MixedKeyDict, others::MixedKeyDict...) = _merge(f, (), d.dicts, (d->d.dicts).(others)...) +Base.mergewith(f::Function, d::MixedKeyDict, others::MixedKeyDict...) = _merge(f, (), d.dicts, (d->d.dicts).(others)...) +Base.mergewith(f, d::MixedKeyDict, others::MixedKeyDict...) = _merge(f, (), d.dicts, (d->d.dicts).(others)...) function _merge(f, res, d, others...) ofsametype, remaining = _alloftype(Base.heads(d), ((),), others...) - return _merge(f, (res..., merge(f, ofsametype...)), Base.tail(d), remaining...) + return _merge(f, (res..., mergewith(f, ofsametype...)), Base.tail(d), remaining...) end _merge(f, res, ::Tuple{}, others...) = _merge(f, res, others...) _merge(f, res, d) = MixedKeyDict((res..., d...)) @@ -3537,9 +4427,9 @@ _alloftype(ofdesiredtype, accumulated) = ofdesiredtype, Base.front(accumulated) let d = MixedKeyDict((Dict(1 => 3), Dict(4. => 2))) e = MixedKeyDict((Dict(1 => 7), Dict(5. => 9))) - @test merge(+, d, e).dicts == (Dict(1 => 10), Dict(4.0 => 2, 5.0 => 9)) + @test mergewith(+, d, e).dicts == (Dict(1 => 10), Dict(4.0 => 2, 5.0 => 9)) f = MixedKeyDict((Dict(2 => 7), Dict(5. => 11))) - @test merge(+, d, e, f).dicts == (Dict(1 => 10, 2 => 7), Dict(4.0 => 2, 5.0 => 20)) + @test mergewith(+, d, e, f).dicts == (Dict(1 => 10, 2 => 7), Dict(4.0 => 2, 5.0 => 20)) end # Issue #31974 @@ -3812,7 +4702,7 @@ f36531(args...) = tuple((args...)...) partial_return_1(x) = (x, 1) partial_return_2(x) = Val{partial_return_1(x)[2]} -@test Base.return_types(partial_return_2, (Int,)) == Any[Type{Val{1}}] +@test Base.return_types(partial_return_2, (Int,)) == Any[Core.TypeEgal{Val{1}}] # Soundness and precision of abstract_iteration f41839() = (1:100...,) @@ -3855,7 +4745,7 @@ function f_typ_assert(x::Int) y = y::Any Val{y[2]} end -@test Base.return_types(f_typ_assert, (Int,)) == Any[Type{Val{1}}] +@test Base.return_types(f_typ_assert, (Int,)) == Any[Core.TypeEgal{Val{1}}] function f_typ_assert2(x::Any) y = (x::Union{Int, Float64}, 1) @@ -3892,7 +4782,8 @@ end for badf in [getfield_const_typename_bad1, getfield_const_typename_bad2] local badf local code = code_typed(badf, Tuple{})[1].first.code - @test Meta.isexpr(code[1], :call) + # the invalid `getfield` call is not constant-folded away + @test any(x -> Meta.isexpr(x, :call), code) @test code[end] === Core.ReturnNode() @test_throws TypeError badf() end @@ -3932,7 +4823,7 @@ apply_fargs(f, args...) = f(args...) @test only(Base.return_types(apply_fargs, Tuple{typeof(Core.apply_type), Vararg})) == Any @test only(Base.return_types(apply_fargs, Tuple{typeof(Core.apply_type), Any, Vararg})) == Any @test only(Base.return_types(apply_fargs, Tuple{typeof(Core.apply_type), Any, Any, Vararg})) == Any -f_apply_cglobal(args...) = cglobal(args...) +f_apply_cglobal(args...) = Core.Intrinsics.cglobal(args...) @test only(Base.return_types(f_apply_cglobal, Tuple{Vararg{Type{Int}}})) == Ptr @test only(Base.return_types(f_apply_cglobal, Tuple{Any, Vararg{Type{Int}}})) == Ptr @test only(Base.return_types(f_apply_cglobal, Tuple{Any, Type{Int}, Vararg{Type{Int}}})) == Ptr{Int} @@ -3952,7 +4843,8 @@ f37532(T, x) = (Core.bitcast(Ptr{T}, x); x) f37943(x::Any, i::Int) = getfield((x::Pair{false, Int}), i) g37943(i::Int) = fieldtype(Pair{false, T} where T, i) @test only(Base.return_types(f37943, Tuple{Any, Int})) === Union{} -@test only(Base.return_types(g37943, Tuple{Int})) === Union{Type{Union{}}, Type{Any}} +# the runtime-constructed `where`-type argument is only `==`-certain (#61323) +@test only(Base.return_types(g37943, Tuple{Int})) == Union{Type{Union{}}, Type{Any}} # Don't let PartialStruct prevent const prop f_partial_struct_constprop(a, b) = (a[1]+b[1], nothing) @@ -4079,6 +4971,156 @@ end end end +@testset "alias-aware union splitting" begin + let 𝕃 = Compiler.fallback_lattice + argtypes = Any[typeof(+), Union{Int,Float64}, Union{Int,Float64}] + # without aliasing: 2 * 2 = 4 + @test Compiler.unionsplitcost(𝕃, argtypes) == 4 + @test length(Compiler.switchtupleunion(𝕃, argtypes)) == 4 + # with aliasing: only one independent union, cost should be 2 + slot = SlotNumber(2) + fargs = Any[SSAValue(1), slot, slot] + @test Compiler.unionsplitcost(𝕃, argtypes; fargs) == 2 + tunion = Compiler.switchtupleunion(𝕃, argtypes; fargs) + @test length(tunion) == 2 + @test Any[typeof(+), Int, Int] in tunion + @test Any[typeof(+), Float64, Float64] in tunion + end + + # unionsplitcost with 3 aliased args: cost 2 instead of 8 + let 𝕃 = Compiler.fallback_lattice + slot = SlotNumber(2) + fargs = Any[SSAValue(1), slot, slot, slot] + argtypes = Any[typeof(+), Union{Int,Float64}, Union{Int,Float64}, Union{Int,Float64}] + # without aliasing: 2*2*2 = 8 + @test Compiler.unionsplitcost(𝕃, argtypes) == 8 + @test length(Compiler.switchtupleunion(𝕃, argtypes)) == 8 + # with aliasing: only one independent union, cost should be 2 + @test Compiler.unionsplitcost(𝕃, argtypes; fargs) == 2 + tunion = Compiler.switchtupleunion(𝕃, argtypes; fargs) + @test length(tunion) == 2 + @test Any[typeof(+), Int, Int, Int] in tunion + @test Any[typeof(+), Float64, Float64, Float64] in tunion + end + + # MustAlias-based aliasing: different SSAValues but same (slot, ssadef, fldidx) + # e.g. f(a.x, a.x) where two getfield calls produce different SSAValues + let 𝕃 = Compiler.MustAliasesLattice(Compiler.fallback_lattice) + ma1 = Compiler.MustAlias(2, 0, AliasableField{Union{Int,Float64}}, 1, Union{Int,Float64}) + ma2 = Compiler.MustAlias(2, 0, AliasableField{Union{Int,Float64}}, 1, Union{Int,Float64}) + fargs = Any[SSAValue(1), SSAValue(2), SSAValue(3)] + argtypes = Any[typeof(+), ma1, ma2] + @test Compiler.unionsplitcost(𝕃, argtypes; fargs) == 2 + tunion = Compiler.switchtupleunion(𝕃, argtypes; fargs) + @test length(tunion) == 2 + @test Any[typeof(+), Int, Int] in tunion + @test Any[typeof(+), Float64, Float64] in tunion + end + + # MustAlias with different slot should NOT be aliased + let 𝕃 = Compiler.MustAliasesLattice(Compiler.fallback_lattice) + ma1 = Compiler.MustAlias(2, 0, AliasableField{Union{Int,Float64}}, 1, Union{Int,Float64}) + ma2 = Compiler.MustAlias(3, 0, AliasableField{Union{Int,Float64}}, 1, Union{Int,Float64}) + fargs = Any[SSAValue(1), SSAValue(2), SSAValue(3)] + argtypes = Any[typeof(+), ma1, ma2] + @test Compiler.unionsplitcost(𝕃, argtypes; fargs) == 4 + @test length(Compiler.switchtupleunion(𝕃, argtypes; fargs)) == 4 + end +end + +# Integration test: alias-aware splitting eliminates impossible cross-type methods +# When `a::Union{A,B}`, `f(a, a, a)` can only ever call the diagonal methods. +# The cross-type method `f(::A, ::B, ::A)` is impossible when all args are aliased. +# Without alias-aware splitting, inference includes `f(::A, ::B, ::A)` and returns +# `Union{Int, String}`. With alias-aware splitting, only diagonal combinations are +# considered, giving the precise return type `Int`. +struct AliasUnionSplitA end +struct AliasUnionSplitB end +alias_union_split_f(::AliasUnionSplitA, ::AliasUnionSplitA) = 1 +alias_union_split_f(::AliasUnionSplitB, ::AliasUnionSplitB) = 2 +alias_union_split_f(::AliasUnionSplitA, ::AliasUnionSplitB) = "bad1" +alias_union_split_f(::AliasUnionSplitB, ::AliasUnionSplitA) = "bad2" +@test Base.infer_return_type((Union{AliasUnionSplitA,AliasUnionSplitB},)) do a + alias_union_split_f(a, a) +end == Int +# MustAlias integration: `getfield` produces MustAlias which enables alias detection +# across different SSAValues that access the same field of the same slot +@test Base.infer_return_type((AliasableField{Union{AliasUnionSplitA,AliasUnionSplitB}},)) do x + alias_union_split_f(getfield(x, :f), getfield(x, :f)) +end == Int +# `getproperty` (`x.f`) produces MustAlias via InterMustAlias pipeline +@test Base.infer_return_type((AliasableField{Union{AliasUnionSplitA,AliasUnionSplitB}},)) do x + alias_union_split_f(x.f, x.f) +end == Int + +# Slot alias conditional refinement: when `y = x` and a branch refines `x`, +# the refinement should propagate to `y` as well. +@testset "slot alias conditional refinement" begin + # basic case: `y = x; if x isa Int; sin(y)` should infer y::Int + @test Base.infer_return_type((Any,)) do x + y = x + if x isa Int + return sin(y) + end + end == Union{Nothing, Float64} + # else branch should also refine aliases + @test Base.infer_return_type((Union{Int,String},)) do x + y = x + if x isa Int + return y + else + return y + end + end == Union{Int, String} + # chained alias: z = y = x, refining x should also refine z + @test Base.infer_return_type((Union{Int,Float64},)) do x + z = y = x + if x isa Int + return z + end + return nothing + end == Union{Nothing, Int} + # alias should be broken by reassignment + @test Base.infer_return_type((Any,)) do x + y = x + if x isa Int + y = "hello" + return y + end + return nothing + end == Union{Nothing, String} + # typeassert-based refinement should also propagate to aliases + @test Base.infer_return_type((Any,)) do x + y = x + typeassert(x, Int) + return y + end == Int + # cross-BB: alias established before join point should survive to condition check + @test Base.infer_return_type((Union{Int,String}, Bool)) do x, cond + if cond + y = x + else + y = x + end + if x isa Int + return sin(y) + end + return nothing + end == Union{Nothing, Float64} + # cross-BB: alias broken on one path → no refinement at join point + @test Base.infer_return_type((Union{Int,String}, Bool)) do x, cond + if cond + y = x + else + y = "broken" + end + if x isa Int + return y + end + return nothing + end == Union{Nothing, String, Int} +end + @testset "constant prop' for union split signature" begin # indexing into tuples really relies on constant prop', and we will get looser result # (`Union{Int,String,Char}`) if constant prop' doesn't happen for splitunion signatures @@ -4260,6 +5302,16 @@ end == Int callsig_backprop_invalidation_outer(a) end ≠ Int +# MustAlias signature constraint propagation: +# when a call like `f(x.value)` constrains `x.value` via the method signature, +# the refinement should propagate back to the slot `x` as a PartialStruct +check_int_positive(x::Int) = x > 0 || error("x must be positive") +# basic case: field type should be narrowed after the call +@test Base.infer_return_type((Some{Any},)) do x + check_int_positive(x.value) + return sin(x.value) +end == Float64 + # https://github.com/JuliaLang/julia/issues/37866 function issue37866(v::Vector{Union{Nothing,Float64}}) for x in v @@ -4332,7 +5384,7 @@ let # Test the presence of PhiNodes in lowered IR by taking the above function, ci.slottypes = Any[ Any for i = 1:length(ci.slotflags) ] ci.ssavaluetypes = Any[Any for i = 1:ci.ssavaluetypes] sv = Compiler.OptimizationState(mi, Compiler.NativeInterpreter()) - ir = Compiler.convert_to_ircode(ci, sv) + ir = Compiler.convert_to_ircode!(ci, sv) ir = Compiler.slot2reg(ir, ci, sv) ir = Compiler.compact!(ir) Compiler.replace_code_newstyle!(ci, ir) @@ -4351,10 +5403,10 @@ invoke_constprop(a::Any, typ::Bool) = typ ? Any : :any invoke_constprop(a::Number, typ::Bool) = typ ? Number : :number @test Base.return_types((Any,)) do a @invoke invoke_constprop(a::Any, true::Bool) -end |> only === Type{Any} +end |> only == Core.TypeEgal{Any} @test Base.return_types((Any,)) do a @invoke invoke_constprop(a::Number, true::Bool) -end |> only === Type{Number} +end |> only == Core.TypeEgal{Number} @test Base.return_types((Any,)) do a @invoke invoke_constprop(a::Any, false::Bool) end |> only === Symbol @@ -4389,9 +5441,10 @@ function call_func_itr(func, itr) end global inline_checker = c -> c # untyped global, a call of this func will prevent inlining -# if `f` is inlined, `GlobalRef(m, :inline_checker)` should appear within the body of `invokef` +# if `f` is inlined, a read of `inline_checker` should appear within the body of `invokef` function is_inline_checker(@nospecialize stmt) - isa(stmt, GlobalRef) && stmt.name === :inline_checker + (isa(stmt, GlobalRef) && stmt.name === :inline_checker) || + (isa(stmt, Core.BindingPartition) && Base.partition_owner(stmt).globalref.name === :inline_checker) end function func_nospecialized(@nospecialize a) @@ -4425,38 +5478,48 @@ Base.@nospecializeinfer func_nospecializeinfer_constprop(@nospecialize a) = func itr_dispatchonly = Any[sin, muladd, "foo", nothing, missing] # untyped container can cause excessive runtime dispatch itr_withinfernce = tuple(sin, muladd, "foo", nothing, missing) # typed container can cause excessive inference +function count_inferred(m::Method) + count = 0 + for mi in Base.specializations(m) + isdefined(mi, :cache) || continue + # inferred methods come first in the cache by construction, so no iteratation needed + count += isdefined(mi.cache, :inferred) + end + return count +end + @testset "compilation annotations" begin @testset "@nospecialize" begin # `@nospecialize` should suppress runtime dispatches of `nospecialize` @test call_func_itr(func_nospecialized, itr_dispatchonly) == 2 - @test length(Base.specializations(only(methods((func_nospecialized))))) == 1 + @test length(Base.specializations(only(methods(func_nospecialized)))) == 1 # `@nospecialize` should allow inference to happen @test call_func_itr(func_nospecialized, itr_withinfernce) == 2 - @test length(Base.specializations(only(methods((func_nospecialized))))) == 6 + @test length(Base.specializations(only(methods(func_nospecialized)))) == 6 @test count(is_inline_checker, @get_code call_func_itr(func_nospecialized, itr_dispatchonly)) == 0 # `@nospecialize` should allow inlinining @test call_func_itr(func_nospecialized_inline, itr_dispatchonly) == 2 - @test length(Base.specializations(only(methods((func_nospecialized_inline))))) == 1 + @test length(Base.specializations(only(methods(func_nospecialized_inline)))) == 1 @test call_func_itr(func_nospecialized_inline, itr_withinfernce) == 2 - @test length(Base.specializations(only(methods((func_nospecialized_inline))))) == 6 + @test length(Base.specializations(only(methods(func_nospecialized_inline)))) == 6 @test count(is_inline_checker, @get_code call_func_itr(func_nospecialized_inline, itr_dispatchonly)) == 5 end @testset "@nospecializeinfer" begin # `@nospecialize` should suppress runtime dispatches of `nospecialize` @test call_func_itr(func_nospecializeinfer, itr_dispatchonly) == 2 - @test length(Base.specializations(only(methods((func_nospecializeinfer))))) == 1 + @test length(Base.specializations(only(methods(func_nospecializeinfer)))) == 1 # `@nospecializeinfer` suppresses inference also @test call_func_itr(func_nospecializeinfer, itr_withinfernce) == 2 - @test length(Base.specializations(only(methods((func_nospecializeinfer))))) == 1 + @test count_inferred(only(methods(func_nospecializeinfer))) == 1 @test !any(is_inline_checker, @get_code call_func_itr(func_nospecializeinfer, itr_dispatchonly)) # `@nospecializeinfer` should allow inlinining @test call_func_itr(func_nospecializeinfer_inline, itr_dispatchonly) == 2 @test length(Base.specializations(only(methods((func_nospecializeinfer_inline))))) == 1 @test call_func_itr(func_nospecializeinfer_inline, itr_withinfernce) == 2 - @test length(Base.specializations(only(methods((func_nospecializeinfer_inline))))) == 1 + @test count_inferred(only(methods(func_nospecializeinfer_inline))) == 1 @test any(is_inline_checker, @get_code call_func_itr(func_nospecializeinfer_inline, itr_dispatchonly)) # `@nospecializeinfer` should allow constprop @@ -4469,7 +5532,7 @@ itr_withinfernce = tuple(sin, muladd, "foo", nothing, missing) # typed container end @test call_func_itr(func_nospecializeinfer_constprop, itr_withinfernce) == 0 for m = methods(func_nospecializeinfer_constprop) - @test length(Base.specializations(m)) == 1 + @test count_inferred(m) == 1 end end end @@ -5037,6 +6100,15 @@ g_max_methods(x) = f_max_methods(x) @test only(Base.return_types(g_max_methods, Tuple{Int})) === Int @test only(Base.return_types(g_max_methods, Tuple{Any})) === Any +# Test that `Core.TypeName.concrete_only` makes inference give up at call sites with +# non-concrete argument types while keeping concrete call sites precise +function f_concrete_only end +typeof(f_concrete_only).name.concrete_only = true +f_concrete_only(x) = 1 +g_concrete_only(x) = f_concrete_only(x) +@test only(Base.return_types(g_concrete_only, Tuple{Int})) === Int +@test only(Base.return_types(g_concrete_only, Tuple{Integer})) === Any + # Test that a module-wise `@max_methods` works as expected module Test43370 using Test @@ -5068,6 +6140,22 @@ let 𝕃ᵢ = Compiler.fallback_lattice @test t.fields == Any[Const(42), Int] end +# issue #60715 +let 𝕃 = Compiler.fallback_lattice + local fn, fn1, pn, pn1 + F(n) = iszero(n) ? Union{} : Tuple{Int, Union{Int, F(n-1)}} + P(n) = (f = F(n); Compiler.PartialStruct(𝕃, f, [Const(0), fieldtype(f, 2)])) + + n = 0 + while Compiler.issimpleenoughtype(F(n+1)) + n += 1 + fn, fn1, pn, pn1 = F(n), F(n+1), P(n), P(n+1) + @test Compiler.:⊑(𝕃, pn, pn1) + end + @test !Compiler.issimplertype(𝕃, pn1, pn) + @test !isa(Compiler.tmerge(𝕃, pn, pn1), Compiler.PartialStruct) +end + foo_empty_vararg(i...) = i[2] bar_empty_vararg(i) = foo_empty_vararg(10, 20, 30, i...) @test bar_empty_vararg(Union{}[]) === 20 @@ -5104,7 +6192,7 @@ end)[2] == Union{} # compilerbarrier builtin import Core: compilerbarrier # runtime semantics -for setting = (:type, :const, :conditional) +for setting = (:type, :const, :conditional, :blackbox) @test compilerbarrier(setting, 42) == 42 @test compilerbarrier(setting, :sym) == :sym end @@ -5141,6 +6229,13 @@ for setting = (#=:type, :const,=# :conditional,) compilerbarrier($(QuoteNode(setting)), 42) end end +# :blackbox preserves type information (unlike :type) but strips Const +@test Base.return_types((Int,)) do a + compilerbarrier(:blackbox, a) +end |> only === Int +@test Base.return_types() do + compilerbarrier(:blackbox, 42) +end |> only === Int # must not be Const(42) # https://github.com/JuliaLang/julia/issues/46426 @noinline typebarrier() = Base.inferencebarrier(0.0) @@ -5219,11 +6314,11 @@ end @testset "#45956: non-linearized cglobal needs special treatment for stmt effects" begin function foo() cglobal((a, )) - ccall(0, Cvoid, (Nothing,), b) + ccall(C_NULL, Cvoid, (Nothing,), b) end @test only(code_typed() do cglobal((a, )) - ccall(0, Cvoid, (Nothing,), b) + ccall(C_NULL, Cvoid, (Nothing,), b) end)[2] === Nothing end @@ -5235,8 +6330,8 @@ end |> only === Union{} @test Base.return_types((Tuple{typeof(typeof),Float64},)) do args f = args[1] # ::MustAlias v = args[2] # ::MustAlias - f(v) # ::Type{Float64} -end |> only === Type{Float64} + f(v) # exactly `Float64` +end |> only == Core.TypeEgal{Float64} # Issue #46839: `abstract_invoke` should handle incorrect call type @test only(Base.return_types(()->invoke(BitSet, Any, x), ())) === Union{} @@ -5312,16 +6407,91 @@ function issue49027(::Type{<:Issue49027{Ty}}) where Ty end return nothing end -@test only(Base.return_types(issue49027, (Type{Issue49027{TypeVar(:Ty)}},))) >: Nothing -@test isnothing(issue49027(Issue49027{TypeVar(:Ty)})) +@test_skip only(Base.return_types(issue49027, (Type{Issue49027{TypeVar(:Ty)}},))) >: Nothing +@test_skip isnothing(issue49027(Issue49027{TypeVar(:Ty)})) function issue49027_integer(::Type{<:Issue49027{Ty}}) where Ty<:Integer if @isdefined Ty # should be false when `Ty` is given as a free type var. return Ty::DataType end nothing end -@test only(Base.return_types(issue49027_integer, (Type{Issue49027{TypeVar(:Ty,Int)}},))) >: Nothing -@test isnothing(issue49027_integer(Issue49027{TypeVar(:Ty,Int)})) +@test_skip only(Base.return_types(issue49027_integer, (Type{Issue49027{TypeVar(:Ty,Int)}},))) >: Nothing +@test_skip isnothing(issue49027_integer(Issue49027{TypeVar(:Ty,Int)})) + +struct TypeVarParam62001 end +myeltype62001(::Type{Vector{T}}) where T = @isdefined(T) ? T : TypeVarParam62001 +paramtype62001(::Type{V}) where V<:Vector = + isa(V, UnionAll) ? myeltype62001(Base.unwrap_unionall(V)) : myeltype62001(V) +# A static parameter may be exactly a TypeVar object from the input. +typevar_length62001(::Type{NTuple{N, VecElement{T}}}) where {N, T} = N + 32 +let T = Base.unwrap_unionall(Vector).parameters[1] + @test myeltype62001(Base.unwrap_unionall(Vector)) === T + @test paramtype62001(Vector{Int8}) === Int8 + @test paramtype62001(Vector) === T + @test only(Base.return_types(myeltype62001, (Type{Base.unwrap_unionall(Vector)},))) === TypeVar +end +let N = TypeVar(:N), T = TypeVar(:T) + @test_throws MethodError typevar_length62001(NTuple{N, VecElement{T}}) +end +# A TypeVar-valued sparam used in type application: the result is a type with +# a free typevar, which no closed `Type{...}`/existential form contains, so +# inference must keep the typevar's identity (or give up to the top kind +# forms) rather than invent a fresh existential. +applysparam62001(::Type{Vector{T}}) where T = Vector{T} +let v = Base.unwrap_unionall(Vector) + @test applysparam62001(v) === v + @test only(Base.return_types(applysparam62001, (Type{v},))) == Type{v} +end +# Identityless TypeVar values as type parameters widen to the top kind forms. +applytypevar62001(tv::TypeVar) = Vector{tv} +applytypevar62001b(tv::TypeVar) = isa(Vector{tv}, Type{Vector{_A}} where _A) +applytypevar62001c(tv::TypeVar) = Vararg{tv} +let x = TypeVar(:x) + @test applytypevar62001(x).parameters[1] === x + @test applytypevar62001b(x) === false + @test applytypevar62001c(x) isa Core.TypeofVararg + @test only(Base.return_types(applytypevar62001, (TypeVar,))) == Type + @test only(Base.return_types(applytypevar62001c, (TypeVar,))) == Core.TypeofVararg +end +# A pinned-TypeVar env marker (`svec(TypeVar(name, w, w), constrained)`) keeps +# `Const` identity only when `w` is free in `specTypes`: there the object is +# part of the call's own types. A pinned var that is not free in `specTypes` +# gets rewrapped and stands for a different binding per instantiation, so it is +# only `==`-known (the TypeEq path). +pinnedtv62001(::Type{Vector{T}}) where T = T +let m = only(methods(pinnedtv62001)) + tv = TypeVar(:tv) + atype = Tuple{typeof(pinnedtv62001), Type{Vector{tv}}} + mi = Compiler.specialize_method(m, atype, Core.svec(Core.svec(TypeVar(:T, tv, tv), true))) + st = only(Compiler.sptypes_from_meth_instance(mi)) + @test st.typ isa Core.Const && st.typ.val === tv + + S = TypeVar(:S) + atype = UnionAll(S, Tuple{typeof(pinnedtv62001), Type{Vector{S}}}) + mi = Compiler.specialize_method(m, atype, Core.svec(Core.svec(TypeVar(:T, S, S), true))) + st = only(Compiler.sptypes_from_meth_instance(mi)) + @test !(st.typ isa Core.Const) + @test st.typ == (Core.TypeEq{X} where X) +end + +# Expanding a generated function whose MethodInstance carries a pinned sparam +# uncertainty marker must hand the generator the marker's `==`-representative, +# not the raw `svec(tvar, constrained)` env entry. +@generated pinnedstaged62001(::Type{Val{S}}) where {S} = QuoteNode(S) +let m = only(methods(pinnedstaged62001)) + X = Tuple{Tuple{Int}} + atype = Tuple{typeof(pinnedstaged62001), Core.TypeEgal{Val{X}}} + mi = Compiler.specialize_method(m, atype, Core.svec(Core.svec(TypeVar(:S, X, X), true))) + src = ccall(:jl_code_for_staged, Any, (Any, UInt, Ptr{Cvoid}), mi, Base.get_world_counter(), C_NULL) + @test src isa Core.CodeInfo + retvals = Any[v isa QuoteNode ? v.value : v for v in + (stmt.val for stmt in src.code if stmt isa Core.ReturnNode)] + @test any(v -> v === X, retvals) +end + +# Closed type-valued arguments should make `Core.Typeof` infer the `TypeEgal` branch. +@test @inferred(Core.has_free_typevars(Pair)) === false +@test @inferred(Core.Typeof(Pair)) === Core.TypeEgal{Pair} function fapplicable end gapplicable() = Val(applicable(fapplicable)) @@ -5392,7 +6562,7 @@ Base.@constprop :aggressive function issue48679(x, b) end @test Base.return_types((Float64,)) do x issue48679(x, false) -end |> only == Type{Float64} +end |> only == Core.TypeEgal{Float64} Base.@constprop :aggressive @noinline _issue48679_const(b, y::Union{Nothing,T}) where {T} = b ? nothing : T::Type Base.@constprop :aggressive function issue48679_const(x, b) @@ -5403,7 +6573,7 @@ Base.@constprop :aggressive function issue48679_const(x, b) end @test Base.return_types((Float64,)) do x issue48679_const(x, false) -end |> only == Type{Float64} +end |> only == Core.TypeEgal{Float64} # `invoke` call in irinterp @noinline _irinterp_invoke(x::Any) = :any @@ -5413,7 +6583,7 @@ Base.@constprop :aggressive Base.@assume_effects :foldable function irinterp_inv end @test Base.return_types((Int,)) do x irinterp_invoke(x, true) -end |> only == Type{Int} +end |> only == Core.TypeEgal{Int} # recursion detection for semi-concrete interpretation # avoid direct infinite loop via `concrete_eval_invoke` @@ -5448,12 +6618,12 @@ end |> only === Tuple{Int,Symbol} else return T end -end) == Type{Nothing} +end) == Core.TypeEgal{Nothing} # Test that Base._return_type inference works for the 1-arg version @test Base.return_types() do Base._return_type(Tuple{typeof(+), Int, Int}) -end |> only == Type{Int} +end |> only == Core.TypeEgal{Int} # Test that NamedTuple abstract iteration works for PartialStruct/Const function nt_splat_const() @@ -5528,7 +6698,7 @@ Base.@propagate_inbounds f_issue50544(::Type{Issue50544{T}}, i) where T = T.para g_issue50544(T...) = Issue50544{Tuple{T...}} h_issue50544(x::T) where T = g_issue50544(f_issue50544(T, 1), f_issue50544(T, 2, 1)) let x = Issue50544((1, Issue50544((2.0, 'x')))) - @test only(Base.return_types(h_issue50544, (typeof(x),))) == Type{Issue50544{Tuple{Int,Float64}}} + @test only(Base.return_types(h_issue50544, (typeof(x),))) == Core.TypeEgal{Issue50544{Tuple{Int,Float64}}} end # refine const-prop'ed `PartialStruct` with declared method signature type @@ -6012,7 +7182,7 @@ end # issue #53585 let t = ntuple(i -> i % 8 == 1 ? Int64 : Float64, 4000) - @test only(Base.return_types(Base.promote_typeof, t)) == Type{Float64} + @test only(Base.return_types(Base.promote_typeof, t)) == Core.TypeEgal{Float64} @test only(Base.return_types(vcat, t)) == Vector{Float64} end @@ -6063,13 +7233,15 @@ function test_func_cached_conditional(y) end; let interp = CachedConditionalInterp(); @test Base.infer_return_type(test_func_cached_conditional, (Any,); interp) == Tuple{Float64, Float64} - @test count(interp.inf_cache) do result + @test count(interp.inf_cache) do entry + result = entry isa Compiler.LocalInferenceResult ? entry.result : entry result.linfo.def.name === :func_cached_conditional end == 1 end # fieldcount on `Tuple` should constant fold, even though `.fields` not const -@test fully_eliminated(Base.fieldcount, Tuple{Type{Tuple{Nothing, Int, Int}}}) +# (only for the egality kind; an `==`-only `Type{T}` no longer folds, #61323) +@test fully_eliminated(Base.fieldcount, Tuple{Core.TypeEgal{Tuple{Nothing, Int, Int}}}) # Vararg-constprop regression from MutableArithmetics (#54341) global SIDE_EFFECT54341::Int @@ -6338,13 +7510,13 @@ function issue56387(nt::NamedTuple, field::Symbol=:a) end types[index] end -@test Base.infer_return_type(issue56387, (typeof((;a=1)),)) == Type{Int} +@test Base.infer_return_type(issue56387, (typeof((;a=1)),)) == Core.TypeEgal{Int} # `apply_type_tfunc` with `Union` in its arguments let apply_type_tfunc = Compiler.apply_type_tfunc 𝕃 = Compiler.fallback_lattice Const = Core.Const - @test apply_type_tfunc(𝕃, Any[Const(Vector), Union{Type{Int},Type{Nothing}}]) == Union{Type{Vector{Int}},Type{Vector{Nothing}}} + @test apply_type_tfunc(𝕃, Any[Const(Vector), Union{Type{Int},Type{Nothing}}]) == Union{Core.TypeEgal{Vector{Int}},Core.TypeEgal{Vector{Nothing}}} end @test Base.infer_return_type((Bool,Int,)) do b, y @@ -6379,6 +7551,44 @@ end === Int @test Base.infer_return_type((String,)) do x swapglobal!(@__MODULE__, :swapglobal!_xxx, x) end === Union{} +# a swap does both a load and store, so its order is validated once for both +@test Base.infer_exception_type((Int,)) do x + swapglobal!(@__MODULE__, :swapglobal!_xxx, x, :unordered) +end >: ConcurrencyViolationError +@test !(Base.infer_exception_type((Int,)) do x + swapglobal!(@__MODULE__, :swapglobal!_xxx, x, :acquire_release) +end >: ConcurrencyViolationError) + +# `replaceglobal!` reads the binding it writes, so it can throw `UndefVarError` +@test Base.infer_exception_type((Module,)) do m + replaceglobal!(m, :swapglobal!_xxx, 1, 2) +end >: UndefVarError +# the `desired` value is type-checked before the comparison, so a bad store never returns +@test Base.infer_return_type((String,)) do x + replaceglobal!(@__MODULE__, :swapglobal!_xxx, 1, x) +end === Union{} +@test Base.infer_return_type((Int,)) do x + replaceglobal!(@__MODULE__, :swapglobal!_xxx, 1, x) +end === ccall(:jl_apply_cmpswap_type, Any, (Any,), Int) +# a store through an import throws before anything is read, so no old value is returned +module RMWGlobalImportSource + global rmwglobal_imported::Int = 1 + global rmwglobal_used::Int = 2 +end +import .RMWGlobalImportSource: rmwglobal_imported +using .RMWGlobalImportSource: rmwglobal_used +@test Base.infer_return_type((Int,)) do x + replaceglobal!(@__MODULE__, :rmwglobal_imported, 1, x) +end === Union{} +@test Base.infer_exception_type((Int,)) do x + replaceglobal!(@__MODULE__, :rmwglobal_imported, 1, x) +end >: ErrorException +@test Base.infer_return_type((Int,)) do x + swapglobal!(@__MODULE__, :rmwglobal_used, x) +end === Union{} +@test Base.infer_exception_type((Int,)) do x + swapglobal!(@__MODULE__, :rmwglobal_used, x) +end >: ErrorException @newinterp AssumeBindingsStaticInterp Compiler.InferenceParams(::AssumeBindingsStaticInterp) = Compiler.InferenceParams(; assume_bindings_static=true) @@ -6458,11 +7668,12 @@ end global invalid_setglobal!_exct_modeling::Int @test Base.infer_exception_type((Float64,)) do x setglobal!(@__MODULE__, :invalid_setglobal!_exct_modeling, x) -end == ErrorException +end == TypeError # Issue #58257 - Hang in inference during BindingPartition resolution module A58257 module B58257 + const age = Base.get_world_counter() using ..A58257 # World age here is N end @@ -6474,9 +7685,293 @@ end ## The sequence of events is critical here. A58257.get! # Creates binding partition in A, N+1:∞ A58257.B58257.get! # Creates binding partition in A.B, N+1:∞ -Base.invoke_in_world(UInt(38678), getglobal, A58257, :get!) # Expands binding partition in A through isa(x, Core.BindingPartition), readcode) + @test !any(x -> isa(x, GlobalRef), readcode) + # the store reformulates to `BindingPartition = value`, even though `global g::Int` + # creates a one-world `PARTITION_KIND_DECLARED` partition just before the `= 0` + # assignment: the write must resolve the typed-global partition. + @test any(x -> Meta.isexpr(x, :(=)) && isa(x.args[1], Core.BindingPartition), writecode) +end +# The `load_consistent` bound must still span flag-only (`public`/`export`) partition +# changes, so adding an `export` does not needlessly invalidate the store. +module ReformGlobalsExport + global g::Int = 0 + getg() = g + setg!(v) = setglobal!(ReformGlobalsExport, :g, v) + export g +end +let setg! = ReformGlobalsExport.setg! + b = convert(Core.Binding, GlobalRef(ReformGlobalsExport, :g)) + exported_min = Base.lookup_binding_partition(Base.get_world_counter(), b).min_world + writeci = code_typed(setg!, (Int,))[1][1] + # store still reformulates + @test any(x -> Meta.isexpr(x, :(=)) && isa(x.args[1], Core.BindingPartition), writeci.code) + # and its validity spans back across the `export` flag-only partition boundary + @test writeci.min_world < exported_min +end +# A user-inserted `BindingPartition` should get handled the same as `GlobalRef` in IR. +module ReformGlobalsSplice + global gdecl::Int # declared but unassigned: reading it throws UndefVarError +end +let b = convert(Core.Binding, GlobalRef(ReformGlobalsSplice, :gdecl)), + part = Base.lookup_binding_partition(Base.get_world_counter(), b), + readpart = @eval function () + v = $(part) + return v + end + effects = Base.infer_effects(readpart, ()) + @test !Compiler.is_consistent(effects) + @test !Compiler.is_nothrow(effects) + @test Base.infer_return_type(readpart, ()) == Int + @test Base.infer_exception_type(readpart, ()) == UndefVarError + @test_throws UndefVarError readpart() +end +# The partition queries may be handed a non-leaf (import) partition, whose restriction is +# another binding rather than a value or a declared type: the `Core.*_partition` builtins are +# ordinary functions, and a `Core.BindingPartition` can be spliced into hand-built code. Such +# an access is resolved by an import walk at the calling world, and inference has no edge to +# cover that walk, so it answers conservatively -- exactly as for an unresolved global read. +module ReformGlobalsImport + module Inner + export gi + global gi::Int = 5 + const ci = identity + end + using .Inner + import .Inner: ci +end +let b = convert(Core.Binding, GlobalRef(ReformGlobalsImport, :gi)), + part = Base.lookup_binding_partition(Base.get_world_counter(), b), + readpart = @eval function () + v = $(part) + return v + end + @test !Compiler.is_leaf_partition(part) + @test Compiler.partition_singleton(part) === nothing + @test Compiler.partition_rt(part) === Any + @test Compiler.partition_rt_widened(part) === Any + @test Base.infer_return_type(readpart, ()) === Any + @test Base.infer_exception_type(readpart, ()) === UndefVarError + @test !Compiler.is_nothrow(Base.infer_effects(readpart, ())) + @test readpart() === 5 +end +# Inference does not walk an imported partition in value position -- it has no edge to +# cover the world-dependent walk -- so the callee stays dynamic even though the import +# currently resolves to a constant. It must still run. +let b = convert(Core.Binding, GlobalRef(ReformGlobalsImport, :ci)), + part = Base.lookup_binding_partition(Base.get_world_counter(), b), + callpart = @eval function (x) + f = $(part) + return f(x) + end + @test !Compiler.is_leaf_partition(part) + @test Compiler.partition_singleton(part) === nothing + @test callpart(42) === 42 +end + +# A read through an import still freezes its leaf, but as the call form carrying the source +# `GlobalRef`: a bare partition would name the module the binding was imported from in the +# `UndefVarError`, not the one the source asked for. +module ReformGlobalsImportName + module Inner + export gu + global gu::Int # declared, never assigned + end + using .Inner + getgu() = gu +end +let code = code_typed(ReformGlobalsImportName.getgu, ())[1][1].code + @test !any(x -> isa(x, Core.BindingPartition), code) + @test any(code) do x + Meta.isexpr(x, :call) && x.args[1] === GlobalRef(Core, :getglobal_partition) && + x.args[2] === QuoteNode(GlobalRef(ReformGlobalsImportName, :gu)) && + isa(x.args[3], QuoteNode) && isa(x.args[3].value, Core.BindingPartition) + end + # the frozen partition is still the leaf, so no import walk happens at run time + part = only(x.args[3].value for x in code + if Meta.isexpr(x, :call) && x.args[1] === GlobalRef(Core, :getglobal_partition)) + @test Compiler.is_leaf_partition(part) + @test Base.partition_owner(part) === convert(Core.Binding, GlobalRef(ReformGlobalsImportName.Inner, :gu)) + # and the error names the module the source asked for, as the unoptimized read does + err = try; ReformGlobalsImportName.getgu(); catch e; e; end + @test err isa UndefVarError && err.var === :gu && err.scope === ReformGlobalsImportName + err2 = try; getglobal(ReformGlobalsImportName, :gu); catch e; e; end + @test err2 isa UndefVarError && err2.var === :gu && err2.scope === ReformGlobalsImportName +end +# The same read in operand position: an operand slot cannot hold a call, so the call form is +# inserted as its own statement and its value used, keeping the partition frozen. +module ReformGlobalsImportOperand + module Inner + export go + global go::Int # declared, never assigned + end + using .Inner + useo() = identity(go) +end +let code = code_typed(ReformGlobalsImportOperand.useo, ())[1][1].code + @test any(code) do x + Meta.isexpr(x, :call) && x.args[1] === GlobalRef(Core, :getglobal_partition) && + x.args[2] === QuoteNode(GlobalRef(ReformGlobalsImportOperand, :go)) && + isa(x.args[3], QuoteNode) && isa(x.args[3].value, Core.BindingPartition) + end + err = try; ReformGlobalsImportOperand.useo(); catch e; e; end + @test err isa UndefVarError && err.var === :go && err.scope === ReformGlobalsImportOperand +end +# An imported *constant* is always defined, so it has no error to misname and keeps the +# bare-partition form. +module ReformGlobalsImportConst + module Inner + export cu + const cu = 42 + end + using .Inner + getcu() = cu +end +@test ReformGlobalsImportConst.getcu() === 42 + +# An atomic order inference cannot prove constant does not keep an access on the runtime +# path: it rides along on the reformulated node as an ordinary operand, which the +# `Core.*_partition` builtin validates exactly as the access it replaces would. Only the +# inlined access is given up (codegen falls back to a generic builtin call), not the frozen +# partition or its invalidation edge. +module ReformGlobalsOrder + global g::Int = 0 + getg(o) = getglobal(ReformGlobalsOrder, :g, o) + setg!(v, o) = setglobal!(ReformGlobalsOrder, :g, v, o) + swapg!(v, o) = swapglobal!(ReformGlobalsOrder, :g, v, o) + replaceg!(c, v, o, fo) = replaceglobal!(ReformGlobalsOrder, :g, c, v, o, fo) + onceg!(v, o) = setglobalonce!(ReformGlobalsOrder, :g, v, o) + definedg(o) = isdefinedglobal(ReformGlobalsOrder, :g, true, o) + # a literal `nothing` operand: an invalid order, not an absent one + @eval setgnothing!(v) = setglobal!(ReformGlobalsOrder, :g, v, $(nothing)) +end +# the sole reformulated `Core.` call in the optimized code of `f(tt...)` +function only_partition_call(@nospecialize(f), @nospecialize(tt::Tuple), name::Symbol) + code = code_typed(f, tt)[1][1].code + return only(x for x in code + if Meta.isexpr(x, :call) && x.args[1] === GlobalRef(Core, name)) +end +is_frozen_partition(@nospecialize x) = isa(x, QuoteNode) && isa(x.value, Core.BindingPartition) +let M = ReformGlobalsOrder + # every access still freezes its partition, with the unproven order as an operand + read = only_partition_call(M.getg, (Symbol,), :getglobal_partition) + @test is_frozen_partition(read.args[3]) && read.args[4] === Core.Argument(2) + store = only_partition_call(M.setg!, (Int,Symbol), :setglobal_partition) + @test is_frozen_partition(store.args[2]) && store.args[4] === Core.Argument(3) + swap = only_partition_call(M.swapg!, (Int,Symbol), :swapglobal_partition) + @test is_frozen_partition(swap.args[2]) && swap.args[4] === Core.Argument(3) + repl = only_partition_call(M.replaceg!, (Int,Int,Symbol,Symbol), :replaceglobal_partition) + @test is_frozen_partition(repl.args[2]) && repl.args[5] === Core.Argument(4) && + repl.args[6] === Core.Argument(5) + once = only_partition_call(M.onceg!, (Int,Symbol), :setglobalonce_partition) + @test is_frozen_partition(once.args[2]) && once.args[4] === Core.Argument(3) + defined = only_partition_call(M.definedg, (Symbol,), :isdefinedglobal_partition) + @test is_frozen_partition(defined.args[2]) && defined.args[3] === Core.Argument(2) + # a `nothing` order is carried as an order, not mistaken for an absent argument + setnothing = only_partition_call(M.setgnothing!, (Int,), :setglobal_partition) + @test is_frozen_partition(setnothing.args[2]) && setnothing.args[4] === QuoteNode(nothing) + # and every forwarded order is validated as the access it replaced would validate it + @test M.getg(:acquire) === 0 + @test_throws ConcurrencyViolationError M.getg(:not_atomic) + @test_throws TypeError M.getg(1) + @test M.setg!(3, :release) === 3 + @test M.getg(:monotonic) === 3 + @test_throws ConcurrencyViolationError M.setg!(4, :acquire) + @test_throws TypeError M.setg!(4, 1) + @test_throws TypeError M.setgnothing!(4) + @test M.swapg!(4, :acquire_release) === 3 + @test M.replaceg!(4, 5, :acquire_release, :monotonic) === (old = 4, success = true) + @test !M.onceg!(6, :release) + @test M.definedg(:acquire) + @test_throws ConcurrencyViolationError M.definedg(:not_atomic) + @test M.getg(:acquire) === 5 +end + +# `modifyglobal!` reformulates like the other stores, in both the plain call form and the +# `:invoke_modify` form the inliner gives it: the module and name become the frozen partition, +# and an `:invoke_modify` keeps the reduce function's code instance so codegen still calls the +# specialized op. A store that does not resolve names no partition, so it is left alone. +module ReformGlobalsModify + module Inner + export mi + global mi::Int = 0 + end + using .Inner # a store to `mi` is an error, so it never resolves + global g::Int = 0 + addg!(v) = modifyglobal!(ReformGlobalsModify, :g, +, v) + addmi!(v) = modifyglobal!(ReformGlobalsModify, :mi, +, v) + addgop!(op, v) = modifyglobal!(ReformGlobalsModify, :g, op, v) +end +# the callee is the `GlobalRef` the reformulation writes, or the builtin itself for a call +# whose callee inlining already resolved +is_modify_partition_callee(@nospecialize callee) = + callee === GlobalRef(Core, :modifyglobal_partition) || callee === Core.modifyglobal_partition +is_modify_partition_call(@nospecialize x) = + iscall(is_modify_partition_callee, x) || + (Meta.isexpr(x, :invoke_modify) && is_modify_partition_callee(x.args[2])) +# the sole reformulated modify statement in the optimized code of `f(tt...)` +only_modify_call(@nospecialize(f), @nospecialize(tt::Tuple)) = + only(x for x in code_typed(f, tt)[1][1].code if is_modify_partition_call(x)) +let M = ReformGlobalsModify + # a constant reduce function is statically dispatched, so the node keeps its code + # instance, and the resolved store names the frozen partition + inv = only_modify_call(M.addg!, (Int,)) + @test Meta.isexpr(inv, :invoke_modify) && isa(inv.args[1], Core.CodeInstance) + @test is_frozen_partition(inv.args[3]) && inv.args[5] === Core.Argument(2) + # a store that does not resolve is left on the runtime path, code instance and all + unrescode = code_typed(M.addmi!, (Int,))[1][1].code + unres = only(x for x in unrescode if Meta.isexpr(x, :invoke_modify)) + @test !any(is_modify_partition_call, unrescode) + @test isa(unres.args[1], Core.CodeInstance) + # a reduce function that is not statically dispatched has no code instance, so the store + # reformulates as a plain call + dyn = only_modify_call(M.addgop!, (Any,Int)) + @test Meta.isexpr(dyn, :call) && is_frozen_partition(dyn.args[2]) + # and each form runs, enforcing the declared type as `modifyglobal!` does + @test Base.infer_return_type(M.addg!, (Int,)) === Pair{Int,Int} + @test M.addg!(2) === (0 => 2) + # the resolved store records exactly one edge to the binding it froze + b = convert(Core.Binding, GlobalRef(M, :g)) + @test count(==(b), collect(Base.method_instance(M.addg!, (Int,)).cache.edges)) == 1 + @test M.addgop!(+, 5) === (2 => 7) + @test M.addgop!(-, 7) === (7 => 0) + @test_throws TypeError M.addgop!((_, _) -> 1.5, 1) + @test_throws "cannot assign a value to imported variable" M.addmi!(1) + @test M.Inner.mi === 0 + # the builtin is modeled by `abstract_modifyop!`, like `modifyglobal!`: it takes the + # declared type from the frozen partition, and its reduce function is inlined the same + # way (so a hand-written call becomes an `:invoke_modify` too) + callpart = @eval v -> Core.modifyglobal_partition($(QuoteNode(inv.args[3].value)), +, v) + @test Base.infer_return_type(callpart, (Int,)) === Pair{Int,Int} + @test Meta.isexpr(only_modify_call(callpart, (Int,)), :invoke_modify) + @test callpart(3) === (0 => 3) + @test M.g === 3 +end + +# There should be exactly one edge to `g`, even though it is referenced twice. +module ReformGlobalsEdges + global g::Int = 0 + getgg() = g + g +end +let f = ReformGlobalsEdges.getgg + @test f() === 0 + ci = Base.method_instance(f, ()).cache + b = convert(Core.Binding, GlobalRef(ReformGlobalsEdges, :g)) + @test count(==(b), collect(ci.edges)) == 1 +end + function tt57873(a::Vector{String}, pref) ret = String[] for j in a @@ -6484,7 +7979,7 @@ function tt57873(a::Vector{String}, pref) end return ret end -let code = Compiler.typeinf_ext_toplevel(Any[Core.svec(Any,Tuple{typeof(tt57873),Vector{String},Tuple{String}})], [Base.get_world_counter()], Base.Compiler.TRIM_NO) +let code = Compiler.typeinf_ext_toplevel(Any[Core.svec(Any,Tuple{typeof(tt57873),Vector{String},Tuple{String}})], [Base.get_world_counter()], Base.Compiler.TRIM_NO, false)[1] @test !isempty(code) ## If we were to run trim here, we should fail with: # Verifier error #1: unresolved invoke from statement tt57873(::Vector{String}, ::Tuple{String, String})::Vector{String} @@ -6515,4 +8010,216 @@ end <: Bool Core.get_binding_type(m, n, xs...) end <: Type +# issue #59269 +function haskey_inference_test() + kwargs = Core.compilerbarrier(:const, Base.pairs((; item = false))) + return haskey(kwargs, :item) ? nothing : Any[] +end +@inferred haskey_inference_test() + +# JuliaLang/julia#55548: invalidate stale slot wrapper types in `ssavaluetypes` +_issue55548_proj1(a, b) = a +function issue55548(a) + a = Base.inferencebarrier(a)::Union{Int64,Float64} + if _issue55548_proj1(isa(a, Int64), (a = Base.inferencebarrier(1.0)::Union{Int64,Float64}; true)) + return a + end + return 2 +end +@test Float64 <: Base.infer_return_type(issue55548, (Int,)) +@test issue55548(Int64(0)) === 1.0 + +# issue #60883: conditional propagation through wrapper functions +mutable struct A60883 + a::Int +end +inner60883(a, b) = iszero(a.a) && !b +outer60883(a, b) = inner60883(a, b) +function issue60883() + a = A60883(0) + b = iszero(a.a) + if outer60883(a, b) else end + return b # should not be narrowed to Const(false) +end +@test issue60883() === true + +throwconditional(c, x) = c ? throw(x isa Int) : throw(x isa Float64) +@test Base.infer_exception_type((Bool, Any)) do c, x + throwconditional(c, x) +end == Bool + +# A frame owned by another inference cache must not trigger sourceless cached-edge reuse. +recursed_edge_cache_owner61177(x::Int) = x + 1 +let + world = Base.get_world_counter() + owner_a, owner_b = Ref(nothing), Ref(nothing) + interp_a = Compiler.NativeInterpreter(world; + inf_params=Compiler.InferenceParams(; cache_owner=owner_a)) + interp_b = Compiler.NativeInterpreter(world; + inf_params=Compiler.InferenceParams(; cache_owner=owner_b)) + target_mi = Base.method_instance(recursed_edge_cache_owner61177, (Int,)) + target_ci = Core.CodeInstance(target_mi, owner_b, Int, Any, + nothing, nothing, zero(Int32), UInt(1), typemax(UInt), zero(UInt32), + nothing, nothing, Core.svec()) + Compiler.code_cache(interp_b)[target_mi] = target_ci + + caller = Compiler.InferenceState( + Compiler.InferenceResult(target_mi, Compiler.typeinf_lattice(interp_a)), + :global, interp_a) + Compiler.add_curr_ssaflag!(caller, Compiler.IR_FLAG_INLINE) + reinferred = Compiler.abstract_call_method(interp_b, target_mi.def, target_mi.specTypes, + Core.svec(), false, Compiler.StmtInfo(true, false), caller) + @test !isready(reinferred) + @test caller.callstack[end].cache_mode == Compiler.CACHE_MODE_LOCAL +end + +# issue #61177: effects of a recursive `@inline` function must not degrade on re-inference +f61177(@nospecialize x) = x isa Int ? @inline(f61177(x + 1)) + x : 0 +let eff = Base.infer_effects(f61177) + @test Compiler.is_consistent(eff) + @test Compiler.is_effect_free(eff) + @test Compiler.is_nothrow(eff) + @test !Compiler.is_terminates(eff) + @test eff == Base.infer_effects(f61177) +end + +# issue #60715 +let + f() = 1; f(_, x...) = (0, f(x...)) + @test f(1, 2, 3) == (0, (0, (0, 1))) +end + +# aviatesk/JETLS.jl/issues/618 +Base.@nospecializeinfer function jetls618(a, @nospecialize(rest...)) + if a > 0 + z = a + length(rest) + else + z = 0 + end + println(z) + return z +end +@test Base.infer_return_type() do + jetls618(1,2,3), jetls618(1,2,3,4) +end == Tuple{Int,Int} + +# issue #60252 +f60252(f, nt::NamedTuple) = NamedTuple{keys(nt)}(f(v) for v in values(nt)) +@inferred f60252(identity, (a=1, b=2)) +f60252_2(t::Tuple) = NamedTuple{(:a, :b), typeof(t)}(t) +@test Base.infer_return_type(f60252_2, (Tuple{Vararg{Int64}},)) == @NamedTuple{a::Int64, b::Int64} + +# perform post const-prop' concrete evaluation when effects are further improved by const-prop' +@noinline function concrete_eval_eligible_if_false(x::Float64, n::Int, y::Bool) + if y # this prevents initial concrete-evaluation + println("x = ", x) + end + s = 0.0 + Base.@assume_effects :terminates_locally for i = 1:n + s += sin(x) + end + return s +end +@test Base.infer_return_type() do + Val(concrete_eval_eligible_if_false(42., 5, false) == 5sin(42.)) +end == Val{true} + +# Const-prop' `PartialStruct` of well-formed types that are `!isconcretedispatch`: +# Test with an example using `OpaqueClosure`, which would be represented as `PartialOpaque`, +# which will be a field of `PartialStruct` representing `Some`. Here, since this `oc` has +# untyped argument types, the return type cannot be derived by eager inference in the +# current OC framework, so inference will fail unless `PartialOpaque` is propagated all the +# way to `call_someoc`. +call_someoc(some, x) = some.value(x) +@test Base.infer_return_type() do + oc = Base.Experimental.@opaque x -> 2x + call_someoc(Some(oc), 1) +end == Int +# A somewhat artificial example, but a test case that exercises the above code path without +# using `OpaqueClosure` +struct UntypedBoxWithParam{T} + x::Some{Any} + UntypedBoxWithParam{T}(x) where T = new{T}(Some{Any}(x)) +end +readbox(box::UntypedBoxWithParam) = box.x.value +@test Base.infer_return_type((Type,Int)) do T, x + readbox(UntypedBoxWithParam{T}(x)) +end == Int + +# A constructor call where one argument is `Any`-typed forces the corresponding +# sparam (M) to be unresolved. Inference must still tighten the *other* sparam +# (B) from its declared `<:Tuple` to `<:Tuple{Vector}`. +module NestedTVarSPtype +struct ParamStruct{N,M,A<:Tuple,B<:Tuple,C<:Tuple} + output_size::NTuple{N,Int} + temparray_size::NTuple{M,Int} + output_indices::A + temparray_indices::B + data_indices::C +end +mk(tempinds::Vector, tempsize) = + ParamStruct((1,), (tempsize,), (Colon(),), (tempinds,), (1:1,)) +end # module NestedTVarSPtype +let rt = Base.infer_return_type(NestedTVarSPtype.mk, (Vector, Any)) + @test rt <: (NestedTVarSPtype.ParamStruct{1, 1, Tuple{Colon}, B, Tuple{UnitRange{Int}}} where B<:Tuple{Vector}) +end + +# `Compiler.return_type` on an `OpaqueClosure` should model the declared +# return type stored in the OC type without inspecting the OC source. +@test Base.infer_return_type() do + oc = Base.Experimental.@opaque Tuple{Int}->Real x -> 2x + Compiler.return_type(oc, Tuple{Int}) +end == Core.TypeEgal{Real} +# When the OC is still a `PartialOpaque`, but its declared return type parameter is +# not exact, do not use the source to recover the runtime-selected OC type. +@test Base.infer_return_type() do + oc = Base.Experimental.@opaque x::Int -> 2x + Compiler.return_type(oc, Tuple{Int}) +end == Type +@test Base.infer_return_type((Core.OpaqueClosure{Tuple{Int},Real},)) do oc + Compiler.return_type(oc, Tuple{Int}) +end == Core.TypeEgal{Real} +@test Base.infer_return_type((Core.OpaqueClosure{Tuple{Int},<:Real},)) do oc + Compiler.return_type(oc, Tuple{Int}) +end == Type{<:Real} +@test Base.infer_return_type((Core.OpaqueClosure{Tuple{Int},Real},)) do oc + Compiler.return_type(oc, Tuple{String}) +end == Type{Union{}} + +# `return_type_tfunc` should bail out (rather than crash inference) when the queried +# signature has no function type to model +@test Base.infer_return_type() do + Compiler.return_type(Tuple{Vararg{Any}}) +end == Type +@test Base.infer_return_type() do + Compiler.return_type(Tuple) +end == Type + +@test Base.infer_return_type(Core.task_result_type, (Task,)) === Type +task_returner() = Task(() -> "hello") +@test Base.infer_return_type((typeof(task_returner),)) do f + Core.task_result_type(f()) +end === Core.TypeEgal{String} +@test Base.infer_return_type((typeof(task_returner),)) do f + fetch(f()) +end === String +@test Base.infer_return_type((Int,)) do i + fetch(Threads.@spawn sin(i)) +end === Float64 + +# Unknown splats must be handled conservatively, while a fixed invoke target remains precise. +splatted_task_inference(xs::Tuple) = Core._task(xs...) +@test Base.infer_return_type(splatted_task_inference, (Tuple,)) === Task +splatted_task_invalid_size(rest::Tuple) = Core._task(identity, "invalid", rest...) +@test Base.infer_return_type(splatted_task_invalid_size, (Tuple,)) === Union{} +splatted_task_target() = 42 +splatted_task_target(xs...) = xs +function splatted_task_invoke(@nospecialize(rest::Tuple)) + targets = (Tuple{Vararg}, rest...) + t = Core._task(splatted_task_target, 0, targets...) + t.donenotify = Base.ThreadSynchronizer() + return fetch(schedule(t)) +end +@test Base.infer_return_type(splatted_task_invoke, (Tuple,)) === Tuple{} + end # module inference diff --git a/Compiler/test/inline.jl b/Compiler/test/inline.jl index 0a88907965f5a..a6cbe15036686 100644 --- a/Compiler/test/inline.jl +++ b/Compiler/test/inline.jl @@ -10,6 +10,41 @@ include("setup_Compiler.jl") include("irutils.jl") include("newinterp.jl") +# Inlined IR drops coverage effects when the process does not instrument the +# inlinee. Every coverage scope instruments this user module. +coverage_strip_callee() = nothing +let mi = Base.method_instance(coverage_strip_callee, ()), + src = make_codeinfo(Any[Expr(:code_coverage_effect), ReturnNode(nothing)]; + ssavaluetypes=Any[Nothing, Nothing], + slottypes=Any[typeof(coverage_strip_callee)], slotnames=[:self]) + ir, _, _ = Compiler.retrieve_ir_for_inlining(mi, src, true) + instrumented = Base.JLOptions().code_coverage != 0 + @test any(stmt -> isexpr(stmt, :code_coverage_effect), ir.stmts.stmt) == instrumented + @test isexpr(src.code[1], :code_coverage_effect) +end + +# Hit mode omits the inlining entry marker when the inlinee carries its own +# markers; count mode always inserts it. The rule only applies when the process +# instruments the callee. +coverage_entry_callee() = nothing +if Base.JLOptions().code_coverage != 0 + let mi = Base.method_instance(coverage_entry_callee, ()) + for has_marker in (false, true) + code = has_marker ? Any[Expr(:code_coverage_effect), ReturnNode(nothing)] : + Any[ReturnNode(nothing)] + src = make_codeinfo(code; ssavaluetypes=Any[Nothing for _ in code], + slottypes=Any[typeof(coverage_entry_callee)], slotnames=[:self]) + callee_ir, spec_info, di = Compiler.retrieve_ir_for_inlining(mi, src, true) + @test any(stmt -> isexpr(stmt, :code_coverage_effect), callee_ir.stmts.stmt) == has_marker + ir = make_ircode(Any[ReturnNode(nothing)]) + Compiler.ir_prepare_inlining!(Compiler.InsertBefore(ir, SSAValue(1)), ir, callee_ir, + spec_info, di, mi, (Int32(1), Int32(0), Int32(0)), Any[]) + inserted = any(stmt -> isexpr(stmt, :code_coverage_effect), ir.new_nodes.stmts.stmt) + @test inserted == (!has_marker || Base.JLOptions().code_coverage_mode != 0) + end + end +end + """ Helper to walk the AST and call a function on every node. """ @@ -152,7 +187,7 @@ end (src, _) = only(code_typed(sum27403, Tuple{Vector{Int}})) @test !any(src.code) do x - x isa Expr && x.head === :invoke && !(x.args[2] in (Core.GlobalRef(Base, :throw_boundserror), Base.throw_boundserror)) + isexpr(x, :invoke) && !(isinvoke(:throw_boundserror, x) || isinvoke(:_throw_boundserror_indices, x)) end end @@ -265,7 +300,7 @@ function foo_apply_apply_type_svec() end @test fully_eliminated(foo_apply_apply_type_svec, Tuple{}; retval=NTuple{3, Float32}) -# The that inlining doesn't drop ambiguity errors (#30118) +# Test that inlining doesn't drop ambiguity errors (#30118) c30118(::Tuple{Ref{<:Type}, Vararg}) = nothing c30118(::Tuple{Ref, Ref}) = nothing b30118(x...) = c30118(x) @@ -316,7 +351,7 @@ end const _a_global_array = [1] f_inline_global_getindex() = _a_global_array[1] let ci = code_typed(f_inline_global_getindex, Tuple{})[1].first - @test any(x->(isexpr(x, :call) && x.args[1] in (GlobalRef(Base, :memoryrefget), Base.memoryrefget)), ci.code) + @test any(iscall((ci, Base.memoryrefget)), ci.code) end # Issue #29114 & #36087 - Inlining of non-tuple splats @@ -631,10 +666,11 @@ g41299(f::Tf, args::Vararg{Any,N}) where {Tf,N} = f(args...) # https://github.com/JuliaLang/julia/issues/42078 # idempotency of callsite inlining -function getcache(mi::Core.MethodInstance) +function getcacheci(mi::Core.MethodInstance) cache = Compiler.code_cache(Compiler.NativeInterpreter()) codeinst = Compiler.get(cache, mi, nothing) - return isnothing(codeinst) ? nothing : codeinst + codeinst === nothing && return nothing + return codeinst end @noinline f42078(a) = sum(sincos(a)) let @@ -652,7 +688,7 @@ let end let # make sure to discard the inferred source mi = only(methods(f42078)).specializations::Core.MethodInstance - codeinst = getcache(mi)::Core.CodeInstance + codeinst = getcacheci(mi)::Core.CodeInstance @atomic codeinst.inferred = nothing end @@ -763,7 +799,7 @@ end # Issue #42264 - crash on certain union splits let f(x) = (x...,) # Test splatting with a Union of non-{Tuple, SimpleVector} types that require creating new `iterate` calls - # in inlining. For this particular case, we're relying on `iterate(::CaretesianIndex)` throwing an error, such + # in inlining. For this particular case, we're relying on `iterate(::CartesianIndex)` throwing an error, such # that the original apply call is not union-split, but the inserted `iterate` call is. @test code_typed(f, Tuple{Union{Int64, CartesianIndex{1}, CartesianIndex{3}}})[1][2] == Tuple{Int64} end @@ -819,7 +855,7 @@ end # test single, non-dispatchtuple callsite inlining @constprop :none @inline test_single_nondispatchtuple(@nospecialize(t)) = - isa(t, DataType) && t.name === Type.body.name + isa(t, DataType) && t.name === Core.TypeEq.name let src = code_typed1((Any,)) do x test_single_nondispatchtuple(x) @@ -830,7 +866,7 @@ let end @constprop :aggressive @inline test_single_nondispatchtuple(c, @nospecialize(t)) = - c && isa(t, DataType) && t.name === Type.body.name + c && isa(t, DataType) && t.name === Core.TypeEq.name let src = code_typed1((Any,)) do x test_single_nondispatchtuple(true, x) @@ -1731,7 +1767,7 @@ let src = code_typed1(with_unmatched_typeparam) break end end - @test isnothing(found) || (source=src, statement=found) + @test isnothing(found) context=(; source=src, statement=found) end function twice_sitofp(x::Int, y::Int) @@ -1846,6 +1882,55 @@ let src = code_typed1((AtomicMemoryRef{Int},)) do a @test count(isinvokemodify(:+), src.code) == 1 end +# Core._task handling +# =================== +# Test that _task inlines properly with const prop +f_task_invoke() = 42 +let src = code_typed1(()) do + return Task(f_task_invoke) + end + m = which(f_task_invoke, ()) + @test count(e -> begin + if iscall((src, Core._task), e) && e isa Expr && e.head === :call && length(e.args) == 4 + ci = e.args[4] + if ci isa CodeInstance && ci.def.def === m + return true + end + end + return false + end, src.code) == 1 +end + +# Test that no invoke target is injected when the single method match does not fully +# cover the argument type: the task must fall back to generic dispatch so that +# non-callable bodies still raise a MethodError when the task runs. +abstract type TaskCallable end +struct TaskCallableImpl <: TaskCallable end +(::TaskCallableImpl)() = 1 +let src = code_typed1((TaskCallable,)) do f + return Task(f) + end + @test count(e -> iscall((src, Core._task), e) && length((e::Expr).args) == 3, src.code) == 1 +end + +# Test that task_result_type gets inlined to its constant value +let src = code_typed1((Task,)) do t; Core.task_result_type(t); end + # Should be inlined to the `Any` constant, with no call to task_result_type + @test count(iscall((src, Core.task_result_type)), src.code) == 0 + @test src.code[end] == ReturnNode(Any) +end +let src = code_typed1((Union{Task,Int},)) do t; Core.task_result_type(t); end + # The Int path throws, so the call cannot be folded away. + @test count(iscall((src, Core.task_result_type)), src.code) == 1 +end +for src in ( + code_typed1((Int,)) do i; Core.task_result_type(i); end, + code_typed1(()) do; Core.task_result_type(); end, + code_typed1((Task,Task)) do t, u; Core.task_result_type(t, u); end) + # Invalid argument types and arities must retain the throwing call. + @test count(iscall((src, Core.task_result_type)), src.code) == 1 +end + # apply `ssa_inlining_pass` multiple times func_mul_int(a::Int, b::Int) = Core.Intrinsics.mul_int(a, b) multi_inlining1(a::Int, b::Int) = @noinline func_mul_int(a, b) @@ -1856,7 +1941,7 @@ let i::Int, continue_::Bool i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) @test i !== nothing # now delete the callsite flag, and see the second inlining pass can inline the call - @eval Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE + ir.stmts[i][:flag] &= ~Compiler.IR_FLAG_NOINLINE inlining = Compiler.InliningState(interp) ir = Compiler.ssa_inlining_pass!(ir, inlining, false) @test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) === nothing @@ -1879,22 +1964,12 @@ let i::Int, continue_::Bool ir, = only(Base.code_ircode(multi_inlining2, (Int,Int); optimize_until="CC: INLINING", interp)) i = findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) @test i !== nothing - # now delete the callsite flag, and see the second inlining pass can inline the call - @eval Compiler $ir.stmts[$i][:flag] &= ~IR_FLAG_NOINLINE + # now delete the callsite flag, and see the second inlining pass does not inline the call, since inference recorded it should not + ir.stmts[i][:flag] &= ~Compiler.IR_FLAG_NOINLINE inlining = Compiler.InliningState(interp) ir = Compiler.ssa_inlining_pass!(ir, inlining, false) - @test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) === nothing - @test (i = findfirst(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt)) !== nothing - lins = Compiler.IRShow.buildLineInfoNode(ir.debuginfo, nothing, i) - @test_broken (continue_ = length(lins) == 3) # see TODO in `ir_inline_linetable!` - if continue_ - def1 = lins[1].method - @test def1 isa Core.MethodInstance && def1.def.name === :multi_inlining2 - def2 = lins[2].method - @test def2 isa Core.MethodInstance && def2.def.name === :call_func_mul_int - def3 = lins[3].method - @test def3 isa Core.MethodInstance && def3.def.name === :call_func_mul_int - end + @test findfirst(isinvoke(:func_mul_int), ir.stmts.stmt) !== nothing + @test findfirst(iscall((ir, Core.Intrinsics.mul_int)), ir.stmts.stmt) === nothing end # Test special purpose inliner for Core.ifelse @@ -2008,7 +2083,41 @@ let src = code_typed1(make_issue47349(Val{4}()), (Any,)) end @test Base.return_types((Int,)) do x make_issue47349(Val(4))((x,nothing,Int)) - end |> only === Type{Int} + end |> only == Core.TypeEgal{Int} +end + +# JIT preparation should keep resolved invoke edges so inlined Type-argument +# calls do not allocate. +struct FastReadBuffer62001 + data::Vector{UInt8} + position::Base.RefValue{Int} +end +FastReadBuffer62001() = FastReadBuffer62001(UInt8[0x01, 0x02], Ref(0)) +@inline function read_byte62001(buf::FastReadBuffer62001, ::Type{UInt8}) + nextpos = buf.position[] + 1 + nextpos > length(buf.data) && throw(EOFError()) + buf.position[] = nextpos + @inbounds return buf.data[nextpos] +end +let buf = FastReadBuffer62001() + let src = code_typed1(Base.allocated, + Tuple{typeof(read_byte62001), FastReadBuffer62001, Core.TypeEgal{UInt8}}) + @test count(src.code) do @nospecialize x + Meta.isexpr(x, :invoke) && + (x.args[1]::Core.CodeInstance).def.specTypes == + Tuple{typeof(read_byte62001), FastReadBuffer62001, Core.TypeEgal{UInt8}} + end == 1 + end + for _ in 1:5 + buf.position[] = 0 + read_byte62001(buf, UInt8) + end + for _ in 1:5 + buf.position[] = 0 + Base.allocated(read_byte62001, buf, UInt8) + end + buf.position[] = 0 + @test Base.allocated(read_byte62001, buf, UInt8) == 0 end # Test that irinterp can make use of constant results even if they're big @@ -2115,7 +2224,10 @@ for run_finalizer_escape_test in (run_finalizer_escape_test1, run_finalizer_esca global finalizer_escape::Int = 0 let src = code_typed1(run_finalizer_escape_test, Tuple{Bool, Bool}) - @test any(iscall((src, Core.setglobal!)), src.code) + # `reformulate_globals_pass!` rewrites the resolved default `setglobal!` store to + # an `Expr(:(=), ::Core.BindingPartition, value)` form. + @test any(x -> iscall((src, Core.setglobal!))(x) || + (Meta.isexpr(x, :(=)) && isa(x.args[1], Core.BindingPartition)), src.code) end let @@ -2314,4 +2426,95 @@ let src = code_typed1(g_noinline_invoke, (Union{Symbol,Nothing},)) @test !any(@nospecialize(x)->isa(x,GlobalRef), src.code) end +path = Ref{Symbol}(:unknown) +function f59018_generator(x) + if @generated + # a runtime-dispatched type-valued argument reaches the generator as the + # egality kind `Core.TypeEgal{T}`; by-type expansion uses `Type{T}` + if x isa Core.TypeEq || x isa Core.TypeEgal + path[] = :generator + return Core.sizeof(Base.type_parameter(x)) + end + else + path[] = :fallback + return Core.sizeof(x isa Union{Core.TypeEq, Core.TypeEgal} ? Base.type_parameter(x) : x.parameters[1]) + end +end +f59018() = f59018_generator(Base.inferencebarrier(Int64)) +let src = code_typed1(f59018, ()) + # We should hit a dynamic dispatch, because not enough information + # is available to expand the generator during compilation. + @test iscall((src, f59018_generator), src.code[end - 1]) + @test path[] === :unknown + @test f59018() === 8 + @test path[] === :generator +end + +# https://github.com/JuliaLang/julia/issues/58915 +f58915(nt) = @inline Base.setindex(nt, 2, :next) +# This function should fully-inline, i.e. it should have only built-in / intrinsic calls +# and no invokes or dynamic calls of user code +let src = code_typed1(f58915, Tuple{@NamedTuple{next::UInt32,prev::UInt32}}) + # Any calls should be built-in calls + @test count(iscall(f->!isa(singleton_type(argextype(f, src)), Core.Builtin)), src.code) == 0 + # There should be no invoke at all + @test count(isinvoke(Returns(true)), src.code) == 0 +end + +# https://github.com/JuliaLang/julia/issues/58915#issuecomment-3061421895 +let src = code_typed1(Base.setindex, (@NamedTuple{next::UInt32,prev::UInt32}, Int, Symbol)) + @test count(isinvoke(:merge_fallback), src.code) == 0 + @test count(iscall((src, Base.merge_fallback)), src.code) == 0 +end + +# @nospecialize annotation on uunamed arguments +# https://github.com/JuliaLang/julia/issues/44428 +@noinline _issue44428_1(@nospecialize _::Any) = println(Base.inferencebarrier(0)) +@noinline _issue44428_2(@nospecialize ::Any) = println(Base.inferencebarrier(0)) +@noinline _issue44428_3(@nospecialize _) = println(Base.inferencebarrier(0)) +function issue44428(x) + _issue44428_1(x) + _issue44428_2(x) + _issue44428_3(x) +end +let src = code_typed1(issue44428, (Any,)) + @test count(isinvoke(:_issue44428_1), src.code) == 1 + @test count(isinvoke(:_issue44428_2), src.code) == 1 + @test count(isinvoke(:_issue44428_3), src.code) == 1 + @test count(x->Meta.isexpr(x,:call), src.code) == 0 +end + +# issue #61552 +let mi = Compiler.specialize_method(only(methods(ndims, (Matrix{Float64},))), + Tuple{typeof(ndims), Matrix{Float64}}, Core.svec()) + codeinst = getcacheci(mi)::Core.CodeInstance + @test Compiler.use_const_api(codeinst) + @test codeinst.inferred === nothing + interp = Compiler.NativeInterpreter() + @test Compiler.ci_get_source(interp, codeinst) isa Core.CodeInfo +end + + +# `statement_cost` prices the reformulated global-access builtin calls via their registered +# tfunc cost: `getglobal_partition` reads are free (matching plain `getglobal`), and the +# `setglobal_partition` store matches the store builtins' tfunc cost, as does the `:(=)` form. +let m = Module() + Core.eval(m, :(global gcost::Int = 0)) + gr = GlobalRef(m, :gcost) + b = convert(Core.Binding, gr) + part = Base.lookup_binding_partition(Base.get_world_counter(), b) + src = code_typed(() -> nothing, ())[1][1] + params = Compiler.OptimizationParams() + sptypes = Compiler.VarState[] + getcall = Expr(:call, GlobalRef(Core, :getglobal_partition), QuoteNode(gr), QuoteNode(part), + QuoteNode(:acquire)) + setcall = Expr(:call, GlobalRef(Core, :setglobal_partition), QuoteNode(part), 0, + QuoteNode(:release)) + @test Compiler.statement_cost(getcall, -1, src, sptypes, params) == + Compiler.T_FFUNC_COST[Compiler.find_tfunc(Core.getglobal_partition)] == 0 + store_cost = Compiler.statement_cost(setcall, -1, src, sptypes, params) + assign_cost = Compiler.statement_cost(Expr(:(=), part, 0), -1, src, sptypes, params) + @test store_cost == Compiler.T_FFUNC_COST[Compiler.find_tfunc(Core.setglobal_partition)] + @test assign_cost == Compiler.T_FFUNC_COST[Compiler.find_tfunc(Core.setglobal!)] +end end # module inline_tests diff --git a/Compiler/test/interpreter_exec.jl b/Compiler/test/interpreter_exec.jl index b1d450f8f4286..7983a19c32ec3 100644 --- a/Compiler/test/interpreter_exec.jl +++ b/Compiler/test/interpreter_exec.jl @@ -112,3 +112,58 @@ let m = Meta.@lower 1 + 1 @test :b === @eval $m @test isempty(current_exceptions()) end + +# The interpreter must enforce the same memory-order rules as codegen. +iscall_gr(x, name) = Meta.isexpr(x, :call) && x.args[1] == GlobalRef(Core, name) +global exec_gp_g::Int = 1 +exec_gp_badread() = getglobal(@__MODULE__, :exec_gp_g, :not_atomic) +exec_gp_badwrite() = setglobal!(@__MODULE__, :exec_gp_g, 5, :not_atomic) +exec_gp_goodswap() = swapglobal!(@__MODULE__, :exec_gp_g, 7, :sequentially_consistent) +let ci = code_typed(exec_gp_badread, ())[1][1] + @assert any(x -> iscall_gr(x, :getglobal_partition), ci.code) + @test_throws ConcurrencyViolationError exec_gp_badread() +end +let ci = code_typed(exec_gp_badwrite, ())[1][1] + @assert any(x -> iscall_gr(x, :setglobal_partition), ci.code) + @test_throws ConcurrencyViolationError exec_gp_badwrite() + @test exec_gp_g === 1 # the failed store must not have written +end +let ci = code_typed(exec_gp_goodswap, ())[1][1] + @assert any(x -> iscall_gr(x, :swapglobal_partition), ci.code) + @test exec_gp_goodswap() === 1 # swap returns the old value + @test exec_gp_g === 7 +end +global exec_gp_def::Int = 3 +exec_gp_isdef() = isdefinedglobal(@__MODULE__, :exec_gp_def, true, :acquire) +let ci = code_typed(exec_gp_isdef, ())[1][1] + @assert any(x -> iscall_gr(x, :isdefinedglobal_partition), ci.code) + @test exec_gp_isdef() === true +end + +# A reformulated `modifyglobal!` reaches the interpreter as an `:invoke_modify` node carrying +# the frozen partition. The interpreter drops the node's code instance and calls the builtin, +# so the store must still run; a hand-built thunk is always interpreted, so it exercises that. +global exec_gp_m::Int = 2 +exec_gp_add(a::Int, b::Int) = a + b +@assert exec_gp_add(1, 2) === 3 # so the method instance below has a cached code instance +let m = Meta.@lower(1 + 1), add_ci = Base.method_instance(exec_gp_add, (Int, Int)).cache, + part = Base.lookup_binding_partition(Base.get_world_counter(), + convert(Core.Binding, GlobalRef(@__MODULE__, :exec_gp_m))) + @assert Meta.isexpr(m, :thunk) + @assert add_ci isa Core.CodeInstance + src = m.args[1]::CodeInfo + src.code = Any[ + Expr(:invoke_modify, add_ci, GlobalRef(Core, :modifyglobal_partition), + QuoteNode(part), GlobalRef(@__MODULE__, :exec_gp_add), 1), + ReturnNode(SSAValue(1)), + ] + nstmts = length(src.code) + src.ssavaluetypes = nstmts + src.ssaflags = fill(zero(UInt32), nstmts) + src.debuginfo = Core.DebugInfo(:none) + @test (@eval $m) === (2 => 3) + @test exec_gp_m === 3 +end +# the builtin names its target with a partition, and only that +@test_throws TypeError Core.modifyglobal_partition(GlobalRef(@__MODULE__, :exec_gp_m), +, 1) +@test exec_gp_m === 3 diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index b51411db1da05..8d63232fd2083 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -14,12 +14,12 @@ struct InvalidationTester <: Compiler.AbstractInterpreter world::UInt inf_params::Compiler.InferenceParams opt_params::Compiler.OptimizationParams - inf_cache::Vector{Compiler.InferenceResult} + inf_cache::Compiler.InferenceCache function InvalidationTester(; world::UInt = Base.get_world_counter(), inf_params::Compiler.InferenceParams = Compiler.InferenceParams(), opt_params::Compiler.OptimizationParams = Compiler.OptimizationParams(), - inf_cache::Vector{Compiler.InferenceResult} = Compiler.InferenceResult[]) + inf_cache::Compiler.InferenceCache = Compiler.InferenceCache()) return new(world, inf_params, opt_params, inf_cache) end end @@ -30,6 +30,61 @@ Compiler.get_inference_world(interp::InvalidationTester) = interp.world Compiler.get_inference_cache(interp::InvalidationTester) = interp.inf_cache Compiler.cache_owner(::InvalidationTester) = InvalidationTesterToken() +# Local constprop proofs expose same-module binding dependencies on the published caller. +module LocalProofBindingInvalidation61752 + _getproperty(M::Module, s::Symbol) = getglobal(M, s) + + const VALUE = "v1" + probe() = _getproperty(LocalProofBindingInvalidation61752, :VALUE)::String +end + +let interp = InvalidationTester() + @test Base.infer_return_type(LocalProofBindingInvalidation61752.probe, (); interp) === String + + mi = Base.method_instance(LocalProofBindingInvalidation61752.probe, ()) + ci = mi.cache + @test ci.owner === InvalidationTesterToken() + @test ci.max_world === typemax(UInt) + + binding = convert(Core.Binding, + GlobalRef(LocalProofBindingInvalidation61752, :VALUE)) + @test any(edge -> edge === binding, ci.edges) + callee_mi = Base.method_instance( + LocalProofBindingInvalidation61752._getproperty, (Module, Symbol)) + @test any(edge -> edge isa Core.CodeInstance && edge.def === callee_mi, ci.edges) + + world_before = Base.get_world_counter() + @eval LocalProofBindingInvalidation61752 const VALUE = "v2" + @test ci.max_world != typemax(UInt) + @test ci.max_world <= world_before +end + +# Eliding a finalizer registration based on inferred effects must keep the proof +# for those effects on the caller's CodeInstance. +module FinalizerEffectInvalidation62338 + mutable struct Target end + callback(::Target) = nothing + register(x::Target) = finalizer(callback, x) +end + +let + inf_params = Compiler.InferenceParams( + ; cache_owner=FinalizerEffectInvalidation62338) + interp = Compiler.NativeInterpreter(Base.get_world_counter(); inf_params) + mi = Base.method_instance(FinalizerEffectInvalidation62338.register, + (FinalizerEffectInvalidation62338.Target,)) + ci = Compiler.typeinf_ext(interp, mi, Compiler.SOURCE_MODE_GET_SOURCE) + src = Compiler.ci_get_source(interp, ci) + @test !any(iscall((src, Core.finalizer)), src.code) + @test ci.max_world == typemax(UInt) + + world_before = Base.get_world_counter() + @eval FinalizerEffectInvalidation62338 callback(::Target) = + (global callback_ran = true; nothing) + @test world_before < Base.get_world_counter() + @test ci.max_world < Base.get_world_counter() +end + # basic functionality test # ------------------------ @@ -116,17 +171,16 @@ begin @test any(iscall((src, pr48932_callee)), src.code) end - let mi = only(Base.specializations(Base.only(Base.methods(pr48932_callee)))) - # Base.method_instance(pr48932_callee, (Any,)) + let mi = only(Base.method_instances(pr48932_callee, Tuple, Base.get_world_counter())) ci = mi.cache @test isdefined(ci, :next) - @test ci.owner === InvalidationTesterToken() + @test ci.owner === nothing @test ci.max_world == typemax(UInt) # In cache due to Base.return_types(pr48932_callee, (Any,)) ci = ci.next @test !isdefined(ci, :next) - @test ci.owner === nothing + @test ci.owner === InvalidationTesterToken() @test ci.max_world == typemax(UInt) end let mi = Base.method_instance(pr48932_caller, (Int,)) @@ -150,11 +204,11 @@ begin # Base.method_instance(pr48932_callee, (Any,)) ci = mi.cache @test isdefined(ci, :next) - @test ci.owner === nothing + @test ci.owner === InvalidationTesterToken() @test_broken ci.max_world == typemax(UInt) ci = ci.next @test !isdefined(ci, :next) - @test ci.owner === InvalidationTesterToken() + @test ci.owner === nothing @test_broken ci.max_world == typemax(UInt) end @@ -224,11 +278,11 @@ begin take!(GLOBAL_BUFFER) let mi = only(Base.specializations(Base.only(Base.methods(pr48932_callee_inferable)))) ci = mi.cache @test isdefined(ci, :next) - @test ci.owner === InvalidationTesterToken() + @test ci.owner === nothing @test ci.max_world == typemax(UInt) ci = ci.next @test !isdefined(ci, :next) - @test ci.owner === nothing + @test ci.owner === InvalidationTesterToken() @test ci.max_world == typemax(UInt) end let mi = Base.method_instance(pr48932_caller_unuse, (Int,)) @@ -249,11 +303,11 @@ begin take!(GLOBAL_BUFFER) let mi = Base.method_instance(pr48932_caller_unuse, (Int,)) ci = mi.cache @test isdefined(ci, :next) - @test ci.owner === nothing + @test ci.owner === InvalidationTesterToken() @test_broken ci.max_world == typemax(UInt) ci = ci.next @test !isdefined(ci, :next) - @test ci.owner === InvalidationTesterToken() + @test ci.owner === nothing @test_broken ci.max_world == typemax(UInt) end @test isnothing(pr48932_caller_unuse(42)) @@ -281,17 +335,17 @@ begin take!(GLOBAL_BUFFER) @test any(isinvoke(:pr48932_callee_inlined), src.code) end - let mi = Base.method_instance(pr48932_callee_inlined, (Int,)) + let mi = only(Base.method_instances(pr48932_callee_inlined, (Any,), Base.get_world_counter())) ci = mi.cache @test isdefined(ci, :next) - @test ci.owner === InvalidationTesterToken() + @test ci.owner === nothing @test ci.max_world == typemax(UInt) ci = ci.next @test !isdefined(ci, :next) - @test ci.owner === nothing + @test ci.owner === InvalidationTesterToken() @test ci.max_world == typemax(UInt) end - let mi = Base.method_instance(pr48932_caller_inlined, (Int,)) + let mi = only(Base.method_instances(pr48932_caller_inlined, (Int,), Base.get_world_counter())) ci = mi.cache @test !isdefined(ci, :next) @test ci.owner === InvalidationTesterToken() @@ -302,18 +356,18 @@ begin take!(GLOBAL_BUFFER) @test "42" == String(take!(GLOBAL_BUFFER)) # test that we added the backedge from `pr48932_callee_inlined` to `pr48932_caller_inlined`: - # this redefinition below should invalidate the cache of `pr48932_callee_inlined` but not that of `pr48932_caller_inlined` + # this redefinition below should invalidate the cache of both `pr48932_callee_inlined` and `pr48932_caller_inlined` @noinline pr48932_callee_inlined(@nospecialize x) = (print(GLOBAL_BUFFER, x); nothing) @test isempty(Base.specializations(Base.only(Base.methods(pr48932_callee_inlined, Tuple{Any})))) let mi = Base.method_instance(pr48932_caller_inlined, (Int,)) ci = mi.cache @test isdefined(ci, :next) - @test ci.owner === nothing + @test ci.owner === InvalidationTesterToken() @test ci.max_world != typemax(UInt) ci = ci.next @test !isdefined(ci, :next) - @test ci.owner === InvalidationTesterToken() + @test ci.owner === nothing @test ci.max_world != typemax(UInt) end @@ -325,3 +379,135 @@ end # This test checks for invalidation of recursive backedges. However, unfortunately, the original failure # manifestation was an unreliable segfault or an assertion failure, so we don't have a more compact test. @test success(`$(Base.julia_cmd()) -e 'Base.typejoin(x, ::Type) = 0; exit()'`) + +# Test drop_all_caches functionality +@testset "drop_all_caches" begin + # Run in subprocess to avoid disrupting the main test process + script = """ + # Define test functions + drop_cache_test_f(x) = x + 1 + drop_cache_test_g(x) = drop_cache_test_f(x) * 2 + + # Compile the functions and capture stderr + drop_cache_test_g(5) == 12 || error("failure") + + println(stderr, "==DROPPING ALL CACHES==") + + # Drop all caches + Base.drop_all_caches() + + # Functions should still work (but will be recompiled on next call) + drop_cache_test_g(5) == 12 || error("failure") + + println(stderr, "SUCCESS: drop_all_caches test passed") + exit(0) + """ + + io = Pipe() + # Run the test in a subprocess because Base.drop_all_caches() is extreme. + # Drain stderr concurrently: the trace-compile output can exceed the pipe + # buffer, and the child blocks in its atexit uv loop until it is read. + result = run(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --trace-compile=stderr -e "$script"`, stderr=io), wait=false) + close(io.in) + reader = @async read(io, String) + @test success(result) + err = fetch(reader)::String + # println(err) + err_before, err_after = split(err, "==DROPPING ALL CACHES==") + @test occursin("SUCCESS: drop_all_caches test passed", err_after) + @test occursin("precompile(Tuple{typeof(Main.drop_cache_test_g), $Int})", err_before) + @test occursin("precompile(Tuple{typeof(Main.drop_cache_test_g), $Int}) # recompile", err_after) +end + +# Test that backedge compaction clears mi.backedges when all backedges are removed +begin + pr61102_callee(x) = 2x + pr61102_caller(x) = pr61102_callee(x) + pr61102_caller(0) + callee_mi = Base.method_instance(pr61102_callee, (Int,)) + @test isdefined(callee_mi, :backedges) + pr61102_callee(x::Int) = 3x + @test !isdefined(callee_mi, :backedges) +end + +# `Core.TypeName.concrete_only`: inference records no backedge at call sites with +# non-concrete argument types, so adding a more-specific method later does not +# invalidate the caller's compiled code +abstract type COStyle end +struct CODefStyle <: COStyle end +struct COFill end +function co_callee end +typeof(co_callee).name.concrete_only = true +co_callee(::COStyle, op, x) = 1 +struct COBox + s::COStyle + x::Any +end +co_caller(b::COBox) = co_callee(b.s, zero, b.x) + +# the non-concrete call site gives up to `Any` +@test Base.return_types((COBox,); interp=InvalidationTester()) do b + co_caller(b) +end |> only === Any + +let mi = Base.method_instance(co_caller, (COBox,)) + ci = mi.cache + @test ci.owner === InvalidationTesterToken() + @test ci.max_world == typemax(UInt) +end + +# add a more-specific method; the caller must remain valid +co_callee(::CODefStyle, op, x::COFill) = 2 +let mi = Base.method_instance(co_caller, (COBox,)) + ci = mi.cache + @test ci.owner === InvalidationTesterToken() + @test ci.max_world == typemax(UInt) +end + +# Unit tests for `Compiler.ReinferUtils` +# -------------------------------------------------------- + +module BindingRevalidationTest + module Flagged + const x = 1 + end + module Provider + export only_via_using + const only_via_using = 2 + end + module Consumer + end +end + +# An `export`/`public` flip repartitions a binding without changing what any lookup of it +# finds, so the access range must still span it. +let gr = GlobalRef(BindingRevalidationTest.Flagged, :x) + b = convert(Core.Binding, gr) + w1 = Base.get_world_counter() + r1, _ = Compiler.binding_access_range(gr, Compiler.WorldWithRange(w1, Compiler.WorldRange(UInt(1), w1)), false) + p1 = b.partitions.min_world + Base.set_binding_visibility!(BindingRevalidationTest.Flagged, :x, :export) + w2 = Base.get_world_counter() + # A new partition really was created ... + @test b.partitions.min_world > p1 + # ... but the access range is unchanged, because the flip does not change the access key. + r2, _ = Compiler.binding_access_range(gr, Compiler.WorldWithRange(w2, Compiler.WorldRange(UInt(1), w2)), false) + @test Compiler.min_world(r2) == Compiler.min_world(r1) +end + +# A binding that has never been resolved is not short-circuited to "unchanged": the +# revalidation predicate resolves it, and judges the resolution it gets. +let RU = Compiler.ReinferUtils + gr = GlobalRef(BindingRevalidationTest.Consumer, :only_via_using) + b = convert(Core.Binding, gr) + @test !isdefined(b, :partitions) + @eval BindingRevalidationTest.Consumer using ..Provider + @test RU.binding_changed_since_require_world(b, Base.get_world_counter()) +end + +# A binding untouched since the require world is unchanged, via the cheap fast path. +let RU = Compiler.ReinferUtils + b = convert(Core.Binding, GlobalRef(Base, :sin)) + @test b.partitions.min_world <= Base.get_require_world() + @test !RU.binding_changed_since_require_world(b, Base.get_world_counter()) +end diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index 758efaab9ab6b..bfbac73609f62 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -987,19 +987,9 @@ function pi_on_argument(x) end return -2 end -let code = code_typed(pi_on_argument, Tuple{Any})[1].first.code, - nisa = 0, found_pi = false - for stmt in code - if Meta.isexpr(stmt, :call) - callee = stmt.args[1] - if (callee === isa || callee === :isa || (isa(callee, GlobalRef) && - callee.name === :isa)) - nisa += 1 - end - elseif stmt === Core.PiNode(Core.Argument(2), Core.Argument) - found_pi = true - end - end +let src = code_typed(pi_on_argument, Tuple{Any})[1].first + nisa = count(iscall((src, isa)), src.code) + found_pi = any(==(Core.PiNode(Core.Argument(2), Core.Argument)), src.code) @test nisa == 1 @test found_pi end @@ -1071,7 +1061,7 @@ let # Test for https://github.com/JuliaLang/julia/issues/43402 end refs = map(Core.SSAValue, findall(@nospecialize(x)->Meta.isexpr(x, :new), src.code)) - some_ccall = findfirst(@nospecialize(x) -> Meta.isexpr(x, :foreigncall) && x.args[1] == :(:some_ccall), src.code) + some_ccall = findfirst(@nospecialize(x) -> Meta.isexpr(x, :foreigncall) && x.args[1] == Expr(:tuple, :(:some_ccall)), src.code) @assert some_ccall !== nothing stmt = src.code[some_ccall] nccallargs = length(stmt.args[3]::Core.SimpleVector) @@ -1181,7 +1171,7 @@ end @test Compiler.is_effect_free(Base.infer_effects(getfield, (Complex{Int}, Symbol))) -# We consider a potential deprecatio warning an effect, so for completely unknown getglobal, +# We consider a potential deprecation warning an effect, so for completely unknown getglobal, # we taint the effect_free bit. @test !Compiler.is_effect_free(Base.infer_effects(getglobal, (Module, Symbol))) @@ -1337,6 +1327,19 @@ end @test wrap1_wrap1_wrapper(true, 1, 1.0) === 1.0 @test wrap1_wrap1_wrapper(false, 1, 1.0) === 1 +# Regression test for #61740: `sroa_mutables!` previously asserted +# `widenconst(:type)::DataType`, which broke after #61719 extended +# `PartialStruct` to wrap parametric (UnionAll) types from `:new`. +mutable struct MutBox61740{T} + const x::Some{Any} + y::Int + MutBox61740{T}(x, y) where T = new{T}(Some{Any}(x), y) +end +read_mutbox61740(box::MutBox61740) = box.x.value +@test Base.infer_return_type((Type, Int)) do T, x + read_mutbox61740(MutBox61740{T}(x, 0)) +end === Int + # Test unswitching-union optimization within SRO Apass function sroaunswitchuniontuple(c, x1, x2) t = c ? (x1,) : (x2,) @@ -1386,7 +1389,33 @@ end @test foo(true, 1) == 2 # ifelse folding -@test Compiler.is_removable_if_unused(Base.infer_effects(exp, (Float64,))) +# Math functions that should be removable if unused (nothrow + effect-free). +# Test all IEEEFloat types for single-argument functions. +@testset "math functions removable if unused: $f($T)" for (f, T) in Iterators.product( + (exp, exp2, exp10, expm1, sinh, cosh, tanh, cbrt, frexp, modf, significand, rad2deg, deg2rad), + (Float16, Float32, Float64), +) + @test Compiler.is_removable_if_unused(Base.infer_effects(f, (T,))) +end +# ldexp takes (T, Int); test all float types +@testset "ldexp($T, Int) removable if unused" for T in (Float16, Float32, Float64) + @test Compiler.is_removable_if_unused(Base.infer_effects(ldexp, (T, Int))) +end +# asinh is nothrow for Float32/Float64: non-finite inputs handled early; all log/log1p +# calls receive positive arguments. Float16 promotes via a separate method. +@testset "asinh($T) removable if unused" for T in (Float32, Float64) + @test Compiler.is_removable_if_unused(Base.infer_effects(asinh, (T,))) +end +# hypot(Float32/Float16): _hypot uses sqrt(muladd(x,x,y*y)); argument is always ≥ 0. +# hypot(Float64) uses a more complex algorithm and is intentionally excluded here. +@testset "hypot($T, $T) removable if unused" for T in (Float16, Float32) + @test Compiler.is_removable_if_unused(Base.infer_effects(hypot, (T, T))) +end +# unsafe_trunc(::Type{<:Integer}, ::Float64) is nothrow: the bit-shift result fits +# within `Int` so `% Int` rather than `Int(...)` keeps the conversion non-throwing. +@testset "unsafe_trunc($T, Float64) removable if unused" for T in (UInt128, Int128) + @test Compiler.is_removable_if_unused(Base.infer_effects(unsafe_trunc, (Type{T}, Float64))) +end @test !Compiler.is_inlineable(code_typed1(exp, (Float64,))) @test fully_eliminated(; retval=Core.Argument(2)) do x::Float64 return Core.ifelse(true, x, exp(x)) @@ -1499,13 +1528,12 @@ let code = Any[ # Simulate the important results from inference interp = Compiler.NativeInterpreter() sv = Compiler.OptimizationState(mi, src, interp) - slot_id = 4 - for block_id = 3:5 - # (_4 !== nothing) conditional narrows the type, triggering PiNodes - sv.bb_vartables[block_id][slot_id] = VarState(Bool, #= maybe_undef =# false) - end + # (_4 !== nothing) conditional narrows the type, triggering PiNodes + sv.bb_states[#=block_id=#3].vartable[#=slot_id=#4] = VarState(Bool, #=def=#5, #=maybe_undef=#false) + sv.bb_states[#=block_id=#4].vartable[#=slot_id=#4] = VarState(Bool, #=def=#7, #=maybe_undef=#false) + sv.bb_states[#=block_id=#5].vartable[#=slot_id=#4] = VarState(Bool, #=def=#7, #=maybe_undef=#false) - ir = Compiler.convert_to_ircode(src, sv) + ir = Compiler.convert_to_ircode!(src, sv) ir = Compiler.slot2reg(ir, src, sv) ir = Compiler.compact!(ir) @@ -1854,6 +1882,20 @@ let (ir,rt) = only(Base.code_ircode((Int,)) do y @test rt == Union{Nothing,Float64} end +# issue #62082: a frame-less (`catch_dest == 0`) EnterNode's scope operand must be +# renumbered too, else `Core.current_scope()` reads a stale value in the scoped region +let sval = ScopedValue(1) + @noinline observe_scope() = Core.current_scope() + function scope_renumber(c::Bool) + if c + error("x") + end + @with sval => 2 observe_scope() + end + @test scope_renumber(false) isa Base.ScopedValues.Scope + @test_throws ErrorException scope_renumber(true) +end + # Test that adce_pass! sets Refined on PhiNode values let code = Any[ # Basic Block 1 @@ -2121,3 +2163,86 @@ let src = code_typed1(foosvalconstprop, ()) end @test count(is_constfield_load, src.code) == 0 end + +# JuliaLang/julia#58330: propagate SROA type refinements through scoped value reads and comparisons +const sval58330 = ScopedValue(1) +struct SROAEgalNonConst + x::Any +end + +@testset "SROA type refinement propagation" begin + let (ir, _) = only(Base.code_ircode(()) do + @with sval58330 => 2 sval58330[] + end) + ret = only(filter(isreturn, ir.stmts.stmt)) + @test singleton_type(Compiler.argextype(ret.val, ir)) === 2 + end + + let (ir, _) = only(Base.code_ircode(()) do + with(sval58330 => 2) do + sval58330[] + end + end) + ret = only(filter(isreturn, ir.stmts.stmt)) + @test singleton_type(Compiler.argextype(ret.val, ir)) === 2 + end + + let (ir, _) = only(Base.code_ircode( + (Bool, Int, Float64, String); optimize_until="CC: SROA") do b, x, y, z + local val + if b + val = SROAEgalNonConst(x) + else + val = SROAEgalNonConst(y) + end + val.x === z + end) + ret = only(filter(isreturn, ir.stmts.stmt)) + @test singleton_type(Compiler.argextype(ret.val, ir)) === false + end +end +# JuliaLang/julia #59548 +# Rewrite `Core._apply_iterate` to use `Core.svec` instead of `tuple` to better match +# the codegen ABI +let src = code_typed1((Vector{Any},)) do xs + println(stdout, xs...) + end + @test count(iscall((src, Core.svec)), src.code) == 1 +end +let src = code_typed1((Vector{Any},)) do xs + println(stdout, 1, xs...) # convert tuples represented by `PartialStruct` + end + @test count(iscall((src, Core.svec)), src.code) == 1 +end + +# Negative NewSSAValue ids must be preserved during compaction +function f_57827(op, init, x) + v = op(init, x) + i = 0 + while i < 1 + v = op(v, x) + i += 1 + end + return v +end +let rf = (acc, x) -> ifelse(x > acc[1], (x,), (acc[1],)) + @test f_57827(rf, (0.0,), 1) === (1,) + ir = first(only(Base.code_ircode(f_57827, (typeof(rf), Tuple{Float64}, Int64); optimize_until="CC: SROA"))) + @test ir isa Compiler.IRCode +end + +# Test that SROA lifting cache deduplicates phi nodes when multiple +# getfield calls access the same field of the same phi node. +struct LiftCachePoint + x::Float64 + y::Float64 +end +let src = code_typed1((Bool,)) do cond + p = cond ? LiftCachePoint(1.0, 2.0) : LiftCachePoint(3.0, 4.0) + return abs(p.x) + p.x * 2.0 + end + @test count(isnew, src.code) == 0 + @test !any(iscall((src, getfield)), src.code) + # the lifting cache should deduplicate: only 1 phi for `p.x`, not 2 + @test count(x -> isa(x, Core.PhiNode), src.code) == 1 +end diff --git a/Compiler/test/newinterp.jl b/Compiler/test/newinterp.jl index 1e0c50192fb5b..95eb7ca579305 100644 --- a/Compiler/test/newinterp.jl +++ b/Compiler/test/newinterp.jl @@ -14,11 +14,10 @@ When the `ephemeral_cache=true` option is specified, `NewInterpreter` will hold `CodeInstance` in an ephemeral non-integrated cache, rather than in the integrated `Compiler.InternalCodeCache`. Keep in mind that ephemeral cache lacks support for invalidation and doesn't persist across -sessions. However it is an usual Julia object of the type `code_cache::IdDict{MethodInstance,CodeInstance}`, +sessions. However it is a usual Julia object of the type `global_cache::IdDict{MethodInstance,CodeInstance}`, making it easier for debugging and inspecting the compiler behavior. """ macro newinterp(InterpName, ephemeral_cache::Bool=false) - cache_token = QuoteNode(gensym(string(InterpName, "CacheToken"))) InterpCacheName = esc(Symbol(string(InterpName, "Cache"))) InterpName = esc(InterpName) C = Core @@ -34,18 +33,18 @@ macro newinterp(InterpName, ephemeral_cache::Bool=false) world::UInt inf_params::$Compiler.InferenceParams opt_params::$Compiler.OptimizationParams - inf_cache::Vector{$Compiler.InferenceResult} - $(ephemeral_cache && :(code_cache::$InterpCacheName)) + inf_cache::$Compiler.InferenceCache + $(ephemeral_cache && :(global_cache::$InterpCacheName)) function $InterpName(meta = nothing; world::UInt = Base.get_world_counter(), inf_params::$Compiler.InferenceParams = $Compiler.InferenceParams(), opt_params::$Compiler.OptimizationParams = $Compiler.OptimizationParams(), - inf_cache::Vector{$Compiler.InferenceResult} = $Compiler.InferenceResult[], + inf_cache::$Compiler.InferenceCache = $Compiler.InferenceCache(), $(ephemeral_cache ? - Expr(:kw, :(code_cache::$InterpCacheName), :($InterpCacheName())) : + Expr(:kw, :(global_cache::$InterpCacheName), :($InterpCacheName())) : Expr(:kw, :_, :nothing))) return $(ephemeral_cache ? - :(new(meta, world, inf_params, opt_params, inf_cache, code_cache)) : + :(new(meta, world, inf_params, opt_params, inf_cache, global_cache)) : :(new(meta, world, inf_params, opt_params, inf_cache))) end end @@ -53,13 +52,13 @@ macro newinterp(InterpName, ephemeral_cache::Bool=false) $Compiler.OptimizationParams(interp::$InterpName) = interp.opt_params $Compiler.get_inference_world(interp::$InterpName) = interp.world $Compiler.get_inference_cache(interp::$InterpName) = interp.inf_cache - $Compiler.cache_owner(::$InterpName) = $cache_token + $Compiler.cache_owner(::$InterpName) = $InterpName $(ephemeral_cache && quote - $Compiler.code_cache(interp::$InterpName) = $Compiler.WorldView(interp.code_cache, $Compiler.WorldRange(interp.world)) - $Compiler.get(wvc::$Compiler.WorldView{$InterpCacheName}, mi::$C.MethodInstance, default) = get(wvc.cache.dict, mi, default) - $Compiler.getindex(wvc::$Compiler.WorldView{$InterpCacheName}, mi::$C.MethodInstance) = getindex(wvc.cache.dict, mi) - $Compiler.haskey(wvc::$Compiler.WorldView{$InterpCacheName}, mi::$C.MethodInstance) = haskey(wvc.cache.dict, mi) - $Compiler.setindex!(wvc::$Compiler.WorldView{$InterpCacheName}, ci::$C.CodeInstance, mi::$C.MethodInstance) = setindex!(wvc.cache.dict, ci, mi) + $Compiler.code_cache(interp::$InterpName) = interp.global_cache + $Compiler.get(cache::$InterpCacheName, mi::$C.MethodInstance, default) = get(cache.dict, mi, default) + $Compiler.getindex(cache::$InterpCacheName, mi::$C.MethodInstance) = getindex(cache.dict, mi) + $Compiler.haskey(cache::$InterpCacheName, mi::$C.MethodInstance) = haskey(cache.dict, mi) + $Compiler.setindex!(cache::$InterpCacheName, ci::$C.CodeInstance, mi::$C.MethodInstance) = setindex!(cache.dict, ci, mi) end) end end diff --git a/Compiler/test/precompile.jl b/Compiler/test/precompile.jl new file mode 100644 index 0000000000000..9a2ccd709e3a7 --- /dev/null +++ b/Compiler/test/precompile.jl @@ -0,0 +1,34 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Tests for Compiler/src/precompile.jl: the logic that selects and compiles code for +# system and package images (`--output-o`, `--output-ji`), formerly precompile_utils.c. + +include("setup_Compiler.jl") + +using Test + +@testset "compile_all_tvar_union" begin + # When generating a system image, every method definition is compiled for the + # signatures obtainable by expanding `where T<:Union{...}` typevars. Typevars with + # an abstract upper bound are not worth instantiating: doing so would compile a + # fully generic specialization for every such method, which is wasteful and can + # even fail codegen (e.g. `LLVMPtr{T, Any}` has no valid address space). + # A typevar used as a type parameter makes the instantiated signature a dispatch + # tuple (`Type{TVParam{Any}}` is concrete), which is how e.g. `LLVMPtr{T, Any}` + # methods ended up being compiled. + struct TVParam{T} end + f_tvar_any(::Type{TVParam{T}}) where {T} = T + @test !Compiler.compile_all_tvar_union(Tuple{typeof(f_tvar_any), Type{TVParam{T}}} where {T}) + @test isempty(Base.specializations(only(methods(f_tvar_any)))) + + f_tvar_abstract(::Type{TVParam{T}}) where {T<:Real} = T + @test !Compiler.compile_all_tvar_union(Tuple{typeof(f_tvar_abstract), Type{TVParam{T}}} where {T<:Real}) + @test isempty(Base.specializations(only(methods(f_tvar_abstract)))) + + # ... but typevars bounded by a union of concrete types are expanded and compiled + f_tvar_union(x::T) where {T<:Union{Int,Float64}} = x + @test Compiler.compile_all_tvar_union(Tuple{typeof(f_tvar_union), T} where {T<:Union{Int,Float64}}) + specs = collect(Base.specializations(only(methods(f_tvar_union)))) + @test Set(mi.specTypes for mi in specs) == + Set([Tuple{typeof(f_tvar_union), Int}, Tuple{typeof(f_tvar_union), Float64}]) +end diff --git a/Compiler/test/special_loading.jl b/Compiler/test/special_loading.jl index ba8cbc635eae8..ca29618a44d17 100644 --- a/Compiler/test/special_loading.jl +++ b/Compiler/test/special_loading.jl @@ -2,7 +2,8 @@ # Only run when testing Base compiler if Base.identify_package("Compiler") === nothing - mktempdir() do dir + include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "test", "tempdepot.jl")) + mkdepottempdir() do dir withenv("JULIA_DEPOT_PATH" => dir * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do cd(joinpath(@__DIR__, "CompilerLoadingTest")) do @test success(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --project=. compiler_loading_test.jl`; stdout, stderr)) diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index 7aca2b8977a4e..17667344225e9 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -5,7 +5,7 @@ include("irutils.jl") using Test -using .Compiler: CFG, BasicBlock, NewSSAValue +using .Compiler: BasicBlock, CFG, NewSSAValue make_bb(preds, succs) = BasicBlock(Compiler.StmtRange(0, 0), preds, succs) @@ -109,6 +109,21 @@ let cfg = CFG(BasicBlock[ @test length(compact.cfg_transform.result_bbs) == 4 && 0 in compact.cfg_transform.result_bbs[3].preds end +# Test that removing a self-edge during compaction only scans compacted phi statements. +let code = Any[ + # Block 1 + Compiler.GotoNode(2), + # Block 2 + Core.PhiNode(Int32[1, 3], Any[1, 2]), + Compiler.GotoIfNot(true, 2), + # Block 3 + Compiler.ReturnNode(0), + ] + ir = make_ircode(code) + ir = Compiler.compact!(ir, true) + @test Compiler.verify_ir(ir) === nothing +end + # Issue #32579 - Optimizer bug involving type constraints function f32579(x::Int, b::Bool) if b @@ -155,11 +170,10 @@ let code = Any[ @test Compiler.verify_ir(ir) === nothing end -# Test that the verifier doesn't choke on cglobals (which aren't linearized) +# Test that the verifier accepts a syntactic-tuple first argument to :foreignglobal (cglobal) let code = Any[ - Expr(:call, GlobalRef(Main, :cglobal), - Expr(:call, Core.tuple, :(:c)), Nothing), - Compiler.ReturnNode() + Expr(:foreignglobal, Expr(:tuple, QuoteNode(:c))), + Compiler.ReturnNode() ] ir = make_ircode(code) @test Compiler.verify_ir(ir) === nothing @@ -175,6 +189,16 @@ let code = Any[ @test_throws ["IR verification failed.", "Code location: "] Compiler.verify_ir(ir, false) end +# Test that static_parameter in value position is non-canonical +let code = Any[ + Expr(:call, identity, Expr(:static_parameter, 1)) + ReturnNode(SSAValue(1)) + ] + ir = make_ircode(code; verify=false) + ir = Compiler.compact!(ir, true) + @test_throws ["IR verification failed.", "Code location: "] Compiler.verify_ir(ir, false) +end + # Issue #29107 let code = Any[ # Block 1 @@ -299,17 +323,6 @@ let code = Any[ oc = Core.OpaqueClosure(ir) @test oc(false, 1, 1) == 2 @test_throws "potential throw" oc(true, 1, 1) - - let buf = IOBuffer() - oc = Core.OpaqueClosure(ir; slotnames=Symbol[:ocfunc, :x, :y, :z]) - try - oc(true, 1, 1) - catch - Base.show_backtrace(buf, catch_backtrace()) - end - s = String(take!(buf)) - @test occursin("(x::Bool, y::$Int, z::$Int)", s) - end end # Test dynamic update of domtree with edge insertions and deletions in the @@ -357,7 +370,7 @@ let cfg = CFG(BasicBlock[ Compiler.cfg_delete_edge!(cfg, 6, 5) Compiler.domtree_delete_edge!(domtree, cfg.blocks, 6, 5) @test domtree.idoms_bb == Compiler.naive_idoms(cfg.blocks) == [0, 1, 1, 3, 2, 4] - # Add edge back (testing second case for insertion) + # Add edge back (testing last case for insertion) Compiler.cfg_insert_edge!(cfg, 6, 5) Compiler.domtree_insert_edge!(domtree, cfg.blocks, 6, 5) @test domtree.idoms_bb == Compiler.naive_idoms(cfg.blocks) == [0, 1, 1, 3, 1, 4] @@ -787,6 +800,54 @@ end end @test f_unreachable_phinode_edge2(1, 2) == 2 +# Preserve an undefined incoming value when slot2ssa creates a PhiNode (#55388). +function undef_phinode55388(nextstate, loopcond, valuecond, returncond) + nextstate && @goto state3 + while loopcond + ct = valuecond ? [] : nothing + if returncond + return + @label state3 + end + Base.donotdelete(ct) + end + nothing +end +let src = code_typed1(undef_phinode55388, (Bool, Bool, Bool, Bool)) + @test any(src.code) do stmt + isa(stmt, PhiNode) || return false + return any(i -> !isassigned(stmt.values, i), eachindex(stmt.edges)) + end +end +@test_throws UndefVarError undef_phinode55388(true, true, true, false) +@test undef_phinode55388(false, false, false, false) === nothing + +# ... and the same for a slot whose definitions live inside an exception region, +# which reaches the `PhiCNode`/`UpsilonNode` path instead (#55388). +@noinline maythrow55388(x) = x === nothing ? throw(ArgumentError("nothing")) : x +function undef_phicnode55388(nextstate, loopcond, valuecond) + nextstate && @goto state3 + while loopcond + local ct + try + ct = maythrow55388(valuecond ? [] : nothing) + catch + ct = nothing + end + @label state3 + Base.donotdelete(ct) + end + nothing +end +let src = code_typed1(undef_phicnode55388, (Bool, Bool, Bool)) + @test any(src.code) do stmt + isa(stmt, PhiNode) || return false + return any(i -> !isassigned(stmt.values, i), eachindex(stmt.edges)) + end +end +@test_throws UndefVarError undef_phicnode55388(true, true, true) +@test undef_phicnode55388(false, false, false) === nothing + global global_error_switch::Bool = true function gen_must_throw_phinode_edge(world::UInt, source, _) ci = make_codeinfo(Any[ @@ -816,11 +877,56 @@ end global global_error_switch = false @test f_must_throw_phinode_edge() == 1 +# Test that IRShow debuginfo printing works with IRCode owned by the active Compiler module. +function irshow_debuginfo_smoke(x) + y = x + 1 + return y +end +let ir = first(only(Base.code_ircode(irshow_debuginfo_smoke, (Int,)))) + output = sprint(Compiler.IRShow.show_ir, ir, + Compiler.IRShow.default_config(ir; debuginfo=:source_inline)) + @test occursin("return", output) +end + +function roundtrip_di(codelocs, firstline, nstmts) + str = ccall(:jl_compress_codelocs, + Any, (Int32, Any, Int), firstline, codelocs, nstmts)::String; + di = Core.DebugInfo(:foo, nothing, Core.svec(), str) + return ccall(:jl_uncompress_codelocs, Any, (Any, Int), di, nstmts) +end + # Test roundtrip of debuginfo compression -let cl = Int32[32, 1, 1, 1000, 240, 230] - str = ccall(:jl_compress_codelocs, Any, (Int32, Any, Int), 378, cl, 2)::String; - cl2 = ccall(:jl_uncompress_codelocs, Any, (Any, Int), str, 2) - @test cl == cl2 +let cl = Int32[32, 1, 1, 1000, 240, 230, 0, 0, 0] + @test roundtrip_di(cl, -1, 3) == cl + @test roundtrip_di(cl, 0, 3) == cl + @test roundtrip_di(cl, 1, 3) == cl + @test roundtrip_di(cl, 32, 3) == cl + @test roundtrip_di(cl, 33, 3) == cl +end +let cl = Int32[0,0,0,255,0,0,256,0,0,257,0,0] + @test roundtrip_di(cl, -1, 4) == cl + @test roundtrip_di(cl, 0, 4) == cl + @test roundtrip_di(cl, 1, 4) == cl + @test roundtrip_di(cl, 32, 4) == cl + @test roundtrip_di(cl, 33, 4) == cl +end + +# Test line comparisons with byte-precise debuginfo. +let sbt = String(UInt8[ + 1, 0, 0, 0, # byte offset + 1, 0, 0, 0, # line offset + 2, 0, 0, 0, # number of locations + 1, 1, # byte and span encoding lengths + 0, 1, 2, 1, # byte spans + 0, 2, # line starts + ]) + codelocs = Int32[1, 0, 0, 2, 0, 0] + str = ccall(:jl_compress_codelocs, + Any, (Int32, Any, Int), Int32(-1), codelocs, 2)::String + di = Core.DebugInfo(:foo, sbt, Core.svec(), str) + @test !Compiler.changed_lineinfo(di, 1, 1) + @test Compiler.changed_lineinfo(di, 2, 1) + @test !Compiler._should_instrument(di) end @test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = "nonexisting pass name") @@ -845,3 +951,37 @@ end let ir = Base.code_ircode(_worker_task57153, (), optimize_until="CC: COMPACT_2")[1].first @test findfirst(x->x==0, ir.cfg.blocks[1].preds) !== nothing end + +# Tests that CFG edge cleanup during compaction doesn't corrupt iteration codegen. +Trips_60660 = let + Ts = (Float64, Float32) + [(Ta, Tb, Tc) for Ta in Ts for Tb in Ts for Tc in Ts] +end +@test Trips_60660 == [ + (Float64, Float64, Float64), + (Float64, Float64, Float32), + (Float64, Float32, Float64), + (Float64, Float32, Float32), + (Float32, Float64, Float64), + (Float32, Float64, Float32), + (Float32, Float32, Float64), + (Float32, Float32, Float32), +] + +# Tests that phi-edge cleanup during compaction terminates when the phi block +# begins with a `nothing` statement rather than a PhiNode (#62818). +function f62818(c) + r = Ref(false) + while true + if c !== nothing + r[] = true + end + if r[] + continue + else + break + end + end +end +@test only(Base.return_types(f62818, Tuple{Nothing})) === Nothing +@test f62818(nothing) === nothing diff --git a/Compiler/test/tarjan.jl b/Compiler/test/tarjan.jl index 8fe940463b558..6a34bd85971cf 100644 --- a/Compiler/test/tarjan.jl +++ b/Compiler/test/tarjan.jl @@ -86,7 +86,7 @@ function test_reachability(V, E; deletions = 2E ÷ 3, all_checks=false) if all_checks # checks for internal data structures - O(E^2) - # Nodes should be mutually reachable iff they are in the same SCompiler. + # Nodes should be mutually reachable iff they are in the same SCC. scc = reachability.scc reachable_nodes = BitSet(v for v = 1:V if !bb_unreachable(reachability, v)) for i ∈ reachable_nodes diff --git a/Compiler/test/testgroups b/Compiler/test/testgroups index 4656448016cd3..30cf0a5384510 100644 --- a/Compiler/test/testgroups +++ b/Compiler/test/testgroups @@ -9,6 +9,7 @@ inference inline interpreter_exec invalidation +precompile irpasses newinterp ssair diff --git a/Compiler/test/validation.jl b/Compiler/test/validation.jl index f01ff85e4321c..4796a2b678170 100644 --- a/Compiler/test/validation.jl +++ b/Compiler/test/validation.jl @@ -4,6 +4,15 @@ using Test, Core.IR include("setup_Compiler.jl") +@testset "stdio validation" begin + for s in (:stdout, :stderr, :print, :println, :write) + @test getglobal(Compiler, s) === getglobal(Core, s) + @test isconst(Compiler, s) + end + @test Compiler.stdin === devnull + @test isconst(Compiler, :stdin) +end + function f22938(a, b, x...) nothing nothing @@ -116,10 +125,11 @@ end @test errors[1].kind === Compiler.SSAFLAGS_MISMATCH end -@testset "NON_TOP_LEVEL_METHOD" begin +@testset "INVALID_EXPR_HEAD for :method" begin + # :method is no longer a valid Expr head (replaced by Core.define_method builtin) c = copy(c0) c.code[1] = Expr(:method, :dummy) errors = Compiler.validate_code(c) @test length(errors) == 1 - @test errors[1].kind === Compiler.NON_TOP_LEVEL_METHOD + @test errors[1].kind === Compiler.INVALID_EXPR_HEAD end diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl index a84afd6933266..ce42410b5ef54 100644 --- a/Compiler/test/verifytrim.jl +++ b/Compiler/test/verifytrim.jl @@ -18,9 +18,32 @@ let infos = Any[] @test isempty(parents) end +struct ScopedTrimEnv + value::Int +end + +const scoped_trim_env = Base.ScopedValues.ScopedValue(ScopedTrimEnv(1)) + +function scoped_trim_read() + return Base.ScopedValues.with(scoped_trim_env => ScopedTrimEnv(2)) do + scoped_trim_env[].value + end +end + +let infos = typeinf_ext_toplevel( + Any[Base.method_instance(scoped_trim_read, ())], + [Base.get_world_counter()], + TRIM_UNSAFE, + false, +)[1] + errors, _ = get_verify_typeinf_trim(infos) + @test scoped_trim_read() == 2 + @test isempty(errors) +end + finalizer(@nospecialize(f), @nospecialize(o)) = Core.finalizer(f, o) -let infos = typeinf_ext_toplevel(Any[Core.svec(Nothing, Tuple{typeof(finalizer), typeof(identity), Any})], [Base.get_world_counter()], TRIM_UNSAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Nothing, Tuple{typeof(finalizer), typeof(identity), Any})], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) @test !isempty(errors) # unresolvable finalizer @@ -29,23 +52,25 @@ let infos = typeinf_ext_toplevel(Any[Core.svec(Nothing, Tuple{typeof(finalizer), @test !warn @test desc isa CallMissing @test occursin("finalizer", desc.desc) - repr = sprint(verify_print_error, desc, parents) - @test occursin( - r"""^unresolved finalizer registered from statement \(Core.finalizer\)\(f::Any, o::Any\)::Nothing - Stacktrace: - \[1\] finalizer\(f::Any, o::Any\)""", repr) + repr = sprint(verify_print_error, desc, parents, warn) + # New format uses multiline for unstable types + @test occursin(r"^unresolved finalizer registered from statement (Core\.)?finalizer\("s, repr) + @test occursin(r"f::Any"s, repr) + @test occursin(r"o::Any"s, repr) + @test occursin(r"::Nothing\n\nStacktrace:"s, repr) + @test occursin(r"\[1\] finalizer\(f::Any, o::Any\)"s, repr) end # test that basic `cfunction` generation is allowed, when the dispatch target can be resolved make_cfunction() = @cfunction(+, Float64, (Int64,Int64)) -let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfunction)})], [Base.get_world_counter()], TRIM_UNSAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfunction)})], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) @test isempty(errors) end # use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel make_cfunction_bad(@nospecialize(f::Any)) = @cfunction($f, Float64, (Int64,Int64))::Base.CFunction -let infos = typeinf_ext_toplevel(Any[Core.svec(Base.CFunction, Tuple{typeof(make_cfunction_bad), Any})], [Base.get_world_counter()], TRIM_UNSAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.CFunction, Tuple{typeof(make_cfunction_bad), Any})], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) @test !isempty(errors) # missing cfunction @@ -53,25 +78,24 @@ let infos = typeinf_ext_toplevel(Any[Core.svec(Base.CFunction, Tuple{typeof(make @test !is_warning @test desc isa CallMissing @test occursin("cfunction", desc.desc) - repr = sprint(verify_print_error, desc, parents) - @test occursin(r"""^unresolved cfunction from statement \$\(Expr\(:cfunction, Base.CFunction, :\(f::Any\), Float64, :\(svec\(Int64, Int64\)::Core.SimpleVector\), :\(:ccall\)\)\)::Base.CFunction - Stacktrace: - \[1\] make_cfunction_bad\(f::Any\)""", repr) + repr = sprint(verify_print_error, desc, parents, is_warning) + @test occursin(r"^unresolved cfunction from statement"s, repr) + @test occursin(r"::Base.CFunction\n\nStacktrace:"s, repr) + @test occursin(r"\[1\] make_cfunction_bad\(f::Any\)"s, repr) resize!(infos, 1) @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type errors, parents = get_verify_typeinf_trim(infos) - desc = only(errors) - @test !desc.first - desc = desc.second + (warn, desc) = only(errors) + @test !warn @test desc isa CCallableMissing @test desc.rt == Base.CFunction @test desc.sig == Tuple{typeof(make_cfunction_bad), Any} @test occursin("unresolved ccallable", desc.desc) - repr = sprint(verify_print_error, desc, parents) + repr = sprint(verify_print_error, desc, parents, warn) @test repr == "unresolved ccallable for Tuple{$(typeof(make_cfunction_bad)), Any} => Base.CFunction\n\n" end -let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] @test length(infos) > 4 errors, parents = get_verify_typeinf_trim(infos) @test isempty(errors) @@ -79,45 +103,49 @@ let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Bas resize!(infos, 1) @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type errors, parents = get_verify_typeinf_trim(infos) - desc = only(errors) - @test !desc.first - desc = desc.second + (warn, desc) = only(errors) + @test !warn @test desc isa CCallableMissing @test desc.rt == Base.SecretBuffer @test desc.sig == Tuple{Type{Base.SecretBuffer}} @test occursin("unresolved ccallable", desc.desc) - repr = sprint(verify_print_error, desc, parents) + repr = sprint(verify_print_error, desc, parents, warn) @test repr == "unresolved ccallable for Tuple{Type{Base.SecretBuffer}} => Base.SecretBuffer\n\n" end -let infos = typeinf_ext_toplevel(Any[Core.svec(Float64, Tuple{typeof(+), Int32, Int64})], [Base.get_world_counter()], TRIM_UNSAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Float64, Tuple{typeof(+), Int32, Int64})], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) - desc = only(errors) - @test !desc.first - desc = desc.second + (warn, desc) = only(errors) + @test !warn @test desc isa CCallableMissing @test desc.rt == Int64 @test desc.sig == Tuple{typeof(+), Int32, Int64} @test occursin("ccallable declared return type", desc.desc) - repr = sprint(verify_print_error, desc, parents) + repr = sprint(verify_print_error, desc, parents, warn) @test repr == "ccallable declared return type does not match inference for Tuple{typeof(+), Int32, Int64} => Int64\n\n" end -let infos = typeinf_ext_toplevel(Any[Core.svec(Int64, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_UNSAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Int64, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) - desc = only(errors) - @test desc.first - desc = desc.second + (warn, desc) = only(errors) + @test warn # this is a warning since Union{Int64, UInt64} <: Int64 is false but not an error @test desc isa CCallableMissing @test occursin("ccallable declared return type", desc.desc) - repr = sprint(verify_print_error, desc, parents) + repr = sprint(verify_print_error, desc, parents, warn) @test repr == "ccallable declared return type does not match inference for Tuple{typeof(ifelse), Bool, Int64, UInt64} => Union{Int64, UInt64}\n\n" end -let infos = typeinf_ext_toplevel(Any[Core.svec(Union{Int64,UInt64}, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_SAFE) +let infos = typeinf_ext_toplevel(Any[Core.svec(Union{Int64,UInt64}, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_SAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) @test isempty(errors) - infos = typeinf_ext_toplevel(Any[Core.svec(Real, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_SAFE) + infos = typeinf_ext_toplevel(Any[Core.svec(Real, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_SAFE, false)[1] errors, parents = get_verify_typeinf_trim(infos) @test isempty(errors) end + + +mi = Base.method_instance(sum, (Vector{Union{Int64,Float64, Float32,UInt32}},)) +let infos = typeinf_ext_toplevel(Any[mi], [Base.get_world_counter()], TRIM_UNSAFE, false)[1] + errors, parents = get_verify_typeinf_trim(infos) + @test !isempty(errors) +end diff --git a/HISTORY.md b/HISTORY.md index dd666eb3312ed..6002238cc0a28 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,165 @@ +Julia v1.13 Release Notes +========================= + +New language features +--------------------- + +* New `@__FUNCTION__` macro to refer to the innermost enclosing function ([#58940]). +* The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, + is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL + ([JuliaLang/JuliaSyntax.jl#525], [#57143]). +* Support for Unicode 17 ([#59534]). + +Language changes +---------------- + +* The `hash` algorithm and its values have changed for certain types, most notably `AbstractString`. Any `hash` specializations for equal types to those that changed, such as some third-party string packages, may need to be deleted ([#57509], [#59691]). +* The `hash(::AbstractString)` function is now a zero-copy / zero-cost function, based upon providing a correct implementation of the `codeunit` and `iterate` functions. Third-party string packages should migrate to the new algorithm by deleting their existing overrides of the `hash` function ([#59691]). + +Command-line option changes +--------------------------- + +* The option `--sysimage-native-code=no` has been deprecated. +* The `JULIA_CPU_TARGET` environment variable now supports a `sysimage` keyword to match (or extend) the CPU target used to build the current system image ([#58970]). +* The `--code-coverage=all` option now automatically throws away sysimage caches so that code coverage can be accurately measured on methods within the sysimage. It is thrown away after startup (and after startup.jl), before any user code is executed ([#59234]). +* New `--trace-eval` command-line option to show expressions being evaluated during top-level evaluation. Supports `--trace-eval=loc` or just `--trace-eval` (show location only), `--trace-eval=full` (show full expressions), and `--trace-eval=no` (disable tracing). Also adds `Base.TRACE_EVAL` global control that takes priority over the command-line option and can be set to `:no`, `:loc`, `:full`, or `nothing` (to use command-line setting) ([#57137]). +* Julia now automatically enables verbose debugging options (`--trace-eval` and `JULIA_TEST_VERBOSE`) when CI debugging has been triggered. i.e. via the "debug logging" UI toggle is enabled on github actions re-runs. Other platforms are supported too ([#59551]). + +Multi-threading changes +----------------------- + +* A new `AbstractSpinLock` is defined with `SpinLock <: AbstractSpinLock` ([#55944]). +* A new `PaddedSpinLock <: AbstractSpinLock` is defined. It has extra padding to avoid false sharing ([#55944]). +* On Apple Silicon, `Sys.CPU_THREADS` and `Sys.EFFECTIVE_CPU_THREADS` now count all CPU cores rather + than only the highest-performance tier. This affects the the following defaults: `--threads=auto` + (`JULIA_NUM_THREADS=auto`), the `--gcthreads` default, `--procs=auto`, `Distributed.addprocs()`, + and `JULIA_NUM_PRECOMPILE_TASKS`. Set `JULIA_CPU_THREADS` to override the detected count. The + performance-core heuristic has been improved and now lives in LinearAlgebra, where it continues + to size the default BLAS thread pool ([#62891], [JuliaLang/LinearAlgebra.jl#1686](https://github.com/JuliaLang/LinearAlgebra.jl/pull/1686)). + +New library functions +--------------------- + +* `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock` ([#56845]). +* `Iterators.nth` function to access the `n`-th element of a generic iterable ([#56580]). +* `ispositive(::Real)` and `isnegative(::Real)` are provided for performance and convenience ([#53677]). +* The `fieldindex` function (to get the index of a struct's field) is now exported ([#58119]). +* `Base.donotdelete` is now public. It prevents dead code elimination of its arguments ([#55774]). +* `Sys.sysimage_target()` returns the CPU target string used to build the current system image ([#58970]). +* `Iterators.findeach` is a lazy version of `findall` ([#54124]). + +New library features +-------------------- + +* `fieldoffset` now also accepts the field name as a symbol as `fieldtype` already did ([#58100]). +* `sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect; they previously threw ([#56978]). +* `Base.AbstractOneTo` is added as a supertype of one-based axes, with `Base.OneTo` as its subtype ([#56902]). +* `takestring!(::IOBuffer)` removes the content from the buffer, returning the content as a `String`. +* `chopprefix` and `chopsuffix` can now also accept an `AbstractChar` as the prefix/suffix to remove. +* The `macroexpand` (with default true) and the new `macroexpand!` (with default false) + functions now support a `legacyscope` boolean keyword argument to control whether to run + the legacy scope resolution pass over the result. The legacy scope resolution code has + known design bugs and will be disabled by default in a future version. Users should + migrate now by calling `legacyscope=false` or using `macroexpand!`. This may often require + fixes to the code calling `macroexpand` with `Meta.unescape` and `Meta.reescape` or by + updating tests to expect `hygienic-scope` or `escape` markers might appear in the result. +* `Base.ScopedValues.LazyScopedValue{T}` is introduced for scoped values that compute their default using a + `OncePerProcess{T}` callback, allowing for lazy initialization of the default value. `AbstractScopedValue` is + now the abstract base type for both `ScopedValue` and `LazyScopedValue` ([#59372]). +* New `Base.active_manifest()` function to return the path of the active manifest, like `Base.active_project()`. + Also can return the manifest that would be used for a given project file ([#57937]). + +Standard library changes +------------------------ + +* `mod(x::AbstractFloat, -Inf)` now returns `x` (as long as `x` is finite). This aligns with the C standard and is considered a bug fix ([#47102]). +* Indexless `getindex` and `setindex!` (i.e. `A[]`) on `ReinterpretArray` now correctly throw a `BoundsError` when there is more than one element ([#58814]). +* `randperm!` and `randcycle!` now support non-`Array` `AbstractArray` inputs, assuming they are mutable and their indices are one-based ([#58596]). +* `shuffle` now accepts `NTuple` arguments ([#56906]). + +#### REPL + +* The Julia REPL now supports bracketed paste on Windows, which should significantly speed up pasting large code blocks into the REPL ([#59825]). +* The REPL now provides syntax highlighting for input as you type. See the REPL docs for more info about customization. +* The REPL now supports automatic insertion of closing brackets, parentheses, and quotes. See the REPL docs for more info about customization. +* History searching has been rewritten to use a new interactive modal dialogue, using a fzf-like style. +* The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]). +* Display of repeated frames and cycles in stack traces has been improved by bracketing them in the trace and treating them consistently ([#55841]). +* The superscript character U+107A5 𐞥 (MODIFIER LETTER SMALL Q), which was already supported in the language, can now be accessed at the REPL with `\^q` ([#59544]). + +#### Test + +* `Test` now supports the `JULIA_TEST_VERBOSE` environment variable. When set to `true`, + it enables verbose testset entry/exit messages with timing information and sets the default `verbose=true` + for `DefaultTestSet` to show detailed hierarchical test summaries ([#59295]). +* Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). +* Transparent test sets (`@testset let`) now show context when tests error ([#58727]). +* `@test_throws` now supports a three-argument form `@test_throws ExceptionType pattern expr` to test both exception type and message pattern in one call ([#59117]). +* The testset stack was changed to use `ScopedValue` rather than task local storage ([#53462]). + +#### InteractiveUtils + +* Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}`. Type-annotated variables as in `f(val::Int; kw::Float64)` are not evaluated if the type annotation provides the necessary information, making this syntax compatible with signatures found in stacktraces ([#57909], [#58222]). +* Code introspection macros such as `@code_lowered` and `@code_typed` now have a much better support for broadcasting expressions, including broadcasting assignments of the form `x .+= f(y)` ([#58349]). + +#### Dates + +* `isoweekdate`, `isoyear`, `weeksinyear` are now implemented and exported for week based calendars, following [ISO week date](https://en.wikipedia.org/wiki/ISO_week_date) ([#48507]). + +External dependencies +--------------------- + +* 7-Zip updated from p7zip v17.06 to upstream 7-Zip v25.01. On Windows, the full 7z.exe/7z.dll bundle is replaced with standalone 7za.exe, which supports fewer formats but unifies cross-platform behavior ([#60025]). + +Deprecated or removed +--------------------- + +* The method `merge(combine::Callable, d::AbstractDict...)` is now deprecated to favor `mergewith` instead ([#59775]). + + +[#47102]: https://github.com/JuliaLang/julia/issues/47102 +[#48507]: https://github.com/JuliaLang/julia/issues/48507 +[#53462]: https://github.com/JuliaLang/julia/issues/53462 +[#53677]: https://github.com/JuliaLang/julia/issues/53677 +[#54124]: https://github.com/JuliaLang/julia/issues/54124 +[#55774]: https://github.com/JuliaLang/julia/issues/55774 +[#55841]: https://github.com/JuliaLang/julia/issues/55841 +[#55944]: https://github.com/JuliaLang/julia/issues/55944 +[#56580]: https://github.com/JuliaLang/julia/issues/56580 +[#56845]: https://github.com/JuliaLang/julia/issues/56845 +[#56902]: https://github.com/JuliaLang/julia/issues/56902 +[#56906]: https://github.com/JuliaLang/julia/issues/56906 +[#56978]: https://github.com/JuliaLang/julia/issues/56978 +[#57137]: https://github.com/JuliaLang/julia/issues/57137 +[#57143]: https://github.com/JuliaLang/julia/issues/57143 +[#57509]: https://github.com/JuliaLang/julia/issues/57509 +[#57825]: https://github.com/JuliaLang/julia/issues/57825 +[#57839]: https://github.com/JuliaLang/julia/issues/57839 +[#57909]: https://github.com/JuliaLang/julia/issues/57909 +[#57937]: https://github.com/JuliaLang/julia/issues/57937 +[#58100]: https://github.com/JuliaLang/julia/issues/58100 +[#58119]: https://github.com/JuliaLang/julia/issues/58119 +[#58181]: https://github.com/JuliaLang/julia/issues/58181 +[#58222]: https://github.com/JuliaLang/julia/issues/58222 +[#58349]: https://github.com/JuliaLang/julia/issues/58349 +[#58596]: https://github.com/JuliaLang/julia/issues/58596 +[#58727]: https://github.com/JuliaLang/julia/issues/58727 +[#58814]: https://github.com/JuliaLang/julia/issues/58814 +[#58940]: https://github.com/JuliaLang/julia/issues/58940 +[#58970]: https://github.com/JuliaLang/julia/issues/58970 +[#59117]: https://github.com/JuliaLang/julia/issues/59117 +[#59234]: https://github.com/JuliaLang/julia/issues/59234 +[#59295]: https://github.com/JuliaLang/julia/issues/59295 +[#59372]: https://github.com/JuliaLang/julia/issues/59372 +[#59534]: https://github.com/JuliaLang/julia/issues/59534 +[#59544]: https://github.com/JuliaLang/julia/issues/59544 +[#59551]: https://github.com/JuliaLang/julia/issues/59551 +[#59691]: https://github.com/JuliaLang/julia/issues/59691 +[#59775]: https://github.com/JuliaLang/julia/issues/59775 +[#59825]: https://github.com/JuliaLang/julia/issues/59825 +[#60025]: https://github.com/JuliaLang/julia/issues/60025 + + Julia v1.12 Release Notes ========================= @@ -8,7 +170,7 @@ New language features entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). Not all code is expected to work with this option, and since it is experimental you may encounter problems. * Redefinition of constants is now well defined and follows world age semantics ([#57253]). Additional redefinitions - (e.g. of types) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). + (e.g. of types) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1/manual/worldage/). * A new keyword argument `usings::Bool` has been added to `names`, returning all names visible via `using` ([#54609]). * The `@atomic` macro family now supports reference assignment syntax, e.g. `@atomic :monotonic v[3] += 4`, @@ -29,17 +191,11 @@ New language features * `Threads.@spawn` now takes a `:samepool` argument to specify the same threadpool as the caller. `Threads.@spawn :samepool foo()` which is shorthand for `Threads.@spawn Threads.threadpool() foo()` ([#57109]). * The `@ccall` macro can now take a `gc_safe` argument, that if set to true allows the runtime to run garbage collection concurrently to the `ccall` ([#49933]). +* A single method covering multiple functions is now allowed in more cases. See issue #54620. ([#58131]). Language changes ---------------- -* Julia now defaults to 1 "interactive" thread, in addition to the 1 default "worker" thread. i.e. `-t1,1`. - This means in default configuration the main task and repl (when in interactive mode), which both run on - thread 1, now run within the `interactive` threadpool. The libuv IO loop also runs on thread 1, - helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Asking for specifically 1 thread - (`-t1`/`JULIA_NUM_THREADS=1`) or passing `0` will disable the interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0` - , or `-tauto,0` etc. Asking for more than 1 thread will enable the interactive thread so - `-t2` will set the equivalent of `-t2,1` ([#57087]). * When a method is replaced with an exactly equivalent one, the old method is not deleted. Instead, the new method takes priority and becomes more specific than the old method. Thus if the new method is deleted later, the old method will resume operating. This can be useful in mocking frameworks (as in SparseArrays, @@ -87,6 +243,14 @@ Command-line option changes Multi-threading changes ----------------------- +* Julia now defaults to 1 "interactive" thread, in addition to the 1 default "worker" thread. i.e. `-t1,1`. + This means in default configuration the main task and repl (when in interactive mode), which both run on + thread 1, now run within the `interactive` threadpool. The libuv IO loop also runs on thread 1, + helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Asking for specifically 1 thread + (`-t1`/`JULIA_NUM_THREADS=1`) or `0` interactive threads will disable the interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0` + , or `-tauto,0` etc. Asking for more than 1 thread will enable the interactive thread so + `-t2` will set the equivalent of `-t2,1`. As a reminder, buffers + [should not be managed based on `threadid()`](https://docs.julialang.org/en/v1/manual/multi-threading/#Using-@threads-without-data-races) ([#57087]). * New types are defined to handle the pattern of code that must run once per process, called a `OncePerProcess{T}` type, which allows defining a function that should be run exactly once the first time it is called, and then always return the same result value of type `T` diff --git a/JuliaLowering/.gitignore b/JuliaLowering/.gitignore new file mode 100644 index 0000000000000..541c1af8ce456 --- /dev/null +++ b/JuliaLowering/.gitignore @@ -0,0 +1,2 @@ +/Manifest.toml +/Manifest-v*.toml diff --git a/JuliaLowering/LICENSE b/JuliaLowering/LICENSE new file mode 100644 index 0000000000000..62a00d053a428 --- /dev/null +++ b/JuliaLowering/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 JuliaHub and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/JuliaLowering/Project.toml b/JuliaLowering/Project.toml new file mode 100644 index 0000000000000..4add188a89eef --- /dev/null +++ b/JuliaLowering/Project.toml @@ -0,0 +1,22 @@ +name = "JuliaLowering" +uuid = "f3c80556-a63f-4383-b822-37d64f81a311" +authors = ["Claire Foster and contributors"] +version = "1.0.0-DEV" + +[deps] +JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" + +[sources] +JuliaSyntax = {path = "../JuliaSyntax"} + +[compat] +julia = "1" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[targets] +test = ["Test", "FileWatching", "Markdown", "REPL"] diff --git a/JuliaLowering/README.md b/JuliaLowering/README.md new file mode 100644 index 0000000000000..560d2c1b63a74 --- /dev/null +++ b/JuliaLowering/README.md @@ -0,0 +1,245 @@ +# JuliaLowering + +JuliaLowering.jl is an implementation of Julia's macro expansion and lowering +passes. It's intended to replace the original femtolisp implementation of macro +expansion and lowering, but it's not active by default yet. + +## Goals + +* Bring automatic hygiene to Julia macros +* Preserve code provenance through lowering (more than just line number nodes). + * Lowered IR should be traceable to the corresponding lowering input (AST). + Combining this with the AST-to-source text provenance from JuliaSyntax lets + us produce byte-precise error messages in lowering and downstream. + * Give each expression its own provenance rather than tracking provenance per + statement. +* Retain compatibility (albeit not bug-for-bug compatibility) with existing code + and macros + +## Trying it out + +```julia +# ./julia --project=./JuliaLowering +julia> using Pkg; Pkg.instantiate(); using JuliaLowering + +julia> JuliaLowering.include_string(Main, """ + macro macro_with_no_escape(x) + x + end + + global @macro_with_no_escape(g) = 1 + @show g + """) +g = 1 +1 + +julia> JuliaLowering.include_string(Main, """ + function f(x,(y,z)=(1,2),(a,b)) + end + """) +ERROR: LoweringError: +function f(x,(y,z)=(1,2),(a,b)) +# └───┘ ── all function parameters after an optional parameter must also be optional +end + +julia> JuliaLowering.include_string(Main, """ + function f(x,(y,z)=(1,2),(a,b)=(3,4)) + function g(@nospecialize(x:Int)) + end + end + """) +ERROR: LoweringError: +function f(x,(y,z)=(1,2),(a,b)=(3,4)) + function g(@nospecialize(x:Int)) +# └───┘ ── expected identifier or `identifier::type` + end +end +``` + +You can also activate JuliaLowering as default in the REPL: + +``` +julia> @activate JuliaLowering +``` + +Further input will be lowered with JuliaLowering, but note that all the good +things about doing so are TODO, since parser output takes a round-trip through +`Expr` (the current focus is finding and fixing lowering bugs where that doesn't +matter). + +## Syntax type + +JuliaLowering uses a new type for syntax (`SyntaxTree`), which is similar to the +existing `Expr`, but: +1. Every node is of the same type. With `expr::Expr`, you may have + `expr.args[1]::Symbol` with `string(expr.args[1]) === "foo"`. In + JuliaLowering, the equivalent would be `kind(syntax[1]) === K"Identifier"` + with `syntax_name(syntax[1]) === "foo"`. +2. For code provenance reasons, it contains a reference to the syntax it was + lowered from. This forms a linked list back to either a LineNumberNode (if + this was converted from an Expr) or a JuliaSyntax structure containing source + text information. This information does not affect lowering. +3. For macro hygiene, syntax versioning, and code provenance reasons, it + contains a reference to a "syntax context", which is internal to lowering and + unique per macro expansion and top-level thunk. + +This is like a stripped-down version of Racket's [syntax +objects](https://docs.racket-lang.org/reference/syntax-model.html), which are +also a layer atop simple symbols and expressions. Compared to Racket, though: +- We don't put lexical scope information in the AST +- Racket uses a ["sets of scopes"](https://www-old.cs.utah.edu/plt/scope-sets/) + model for macro hygiene. We only use a single "scope" per piece of syntax for + macro hygiene (JuliaLowering calls these scopes "layers" to distinguish them + from lexical scopes). + +This syntax type is the argument and return type for hygienic "new" macros. +Quoting with `quote` and `:` will continue to produce `Expr` and `Symbol` like +before, but a new kind of quote/unquote producing `SyntaxTree` may be added. +For now, `JuliaLowering.@legacy_quote_to_syntax :(x + $y)` can be used instead +with some limitations. + +This syntax type can represent exactly the same syntax as `Expr` can, but with +additional hygiene that may change semantics. `Expr` syntax converted to +`SyntaxTree` must be able to round-trip to the exact same `Expr` syntax as +macros see it. (Within reason, LineNumberNodes may differ.) `SyntaxTree` should +similarly be able to round-trip through `Expr` with degraded provenance, +assuming the tree contains zero or one hygiene layers. (Erasing hygiene by +converting a tree with >1 hygiene layer should be an error.) + +## New macros with automatic hygiene + +Julia's old (pre-JuliaLowering) macros are already partially hygienic according +to the usual definition of macro hygiene: an expansion shouldn't accidentally +introduce or reference names from the calling context [1]. Unfortunately, the +old macro expander does this by using the macro-definition context for the +entire expansion, including parts of the expansion that were passed by the +macro-caller context. This means almost every useful macro needs to wrap +arguments (or whatever pieces of them show up in the expansion) in `escape` +nodes. Worse yet, `escape` nodes change the AST structure, and may be passed to +inner macrocalls, so any macro that takes input needs to unwrap and rewrap every +piece of syntax it observes to correctly handle hygiene, which doesn't happen in +practice ([Julia issue #37691](https://github.com/JuliaLang/julia/issues/37691)). + +As mentioned above, JuliaLowering gives every piece of syntax a "syntax context" +containing a hygiene layer before expansion. A new layer is generated per macro +expansion, but is only applied to "new" syntax in the expansion. For example, +the argument to `@addone` retains its local scope or module in the new expander, +but behaves badly in the old expander: + +```julia +# A macro like this is likely making a mistake by not escaping `x` +module M +macro addone(x) + :(let y = 1 + $x + y + end) +end +end + +# Old expander produces `(+ M.x_in mangled_y)` == UndefVarError +@macroexpand let x_in = 0 + M.@addone x_in +end + +# Old expander produces `(+ mangled_y mangled_y)` == 0 +@macroexpand let y = 0 + M.@addone y +end +``` + +Later, in scope resolution, the `(name, layer)` pair is used instead of just the +name (the old expander attempts to mangle the name). The end result is that the +vast majority of escaping done in old `Expr` macros is no longer necessary in +the equivalent new-style macros. + +Unhygienic macros (any macro where calling `@define_something` defines some +local visible to the calling context, with a name not passed to the call) are +useful too. Like in the old system, breaking hygiene in the new system is +opt-in. + +[1] "partially hygienic" because the reverse is not true. + +## AST input specification, error messages + +A new AST validator is available in [validation.jl](./src/validation.jl). This +runs right before desugaring, since a malformed AST before that stage could be +fixed by some macro at any time, so it can't be validated. + +AST validation is partially to make the rest of lowering easier (allowing the +transformation passes to assume the code's structure), but it also serves as +documentation of the Julia AST. Some lowering errors, but not many, are thrown +from later in lowering. + +`LoweringError` with `internal=false` is thrown for all bad user input, +including malformed AST. Lowering-internal assertions use `@jl_assert`, which +can be compiled out (this also throws `LoweringError`, but with +`internal=true`). + +## Compatibility + +Assuming it doesn't depend on a bug or internal piece of flisp lowering, all +lowering input should continue to work under JuliaLowering. + +JuliaLowering doesn't guarantee (and usually doesn't produce) the same IR as +flisp given the same input. + +Macro compatibility is less simple to implement, but the same "old code +continues to work" guarantee applies. +- The syntax version of the `macro ... end` definition determines which of the + two signatures it has. Lowering with flisp, all syntax has version + `JL_OLD_SYNTAX_VERSION` and the `Expr` signature. +- A macro author can choose to implement their macro with the new signature, the + old signature, or both (but is responsible for both being equivalent if so). + Adding a new macro to a project running with the old syntax version would look + something like this: + ```julia + macro m(x) + # x::Expr + esc(x.args[1]) + end + @syntax_version some_version_number macro m(x) + # x::SyntaxTree + x[1] + end + ``` +- If no macro with the new signature exists, JuliaLowering converts all macro + arguments to `Expr`/`Symbol`/etc. syntax, expands the old macro, and converts + the expansion back to `SyntaxTree` with degraded provenance and all syntax + with version `JL_OLD_SYNTAX_VERSION`. + +JuliaLowering should guarantee that replacing an `Expr` macro with an equivalent +`SyntaxTree` one (with or without deleting the old macro) doesn't break existing +callers of the macro running with JuliaLowering. Of course, deleting an +existing `Expr` macro may break your code on older Julia versions using flisp, +and not all `SyntaxTree` macros can be written as `Expr` macros. + +Macro compatibility and the way of creating `SyntaxTree` are still under +development. This document will be updated if one of the promises or interfaces +above changes. + +### Why do we need to rewrite macros to get provenance and automatic hygiene? + +Ideally, `Expr` would continue to work, `esc` could be made a no-op in places +JuliaLowering auto-escapes, provenance flows freely, and macro authors live +happily ever after---what gives? + +1. Adding fields to `Expr` to carry the same information as `SyntaxTree` is not + out of the question, but doing that to `Symbol` is. The paper [Towards the + Essence of Hygiene](https://michaeldadams.org/papers/hygiene/hygiene-2015-popl-authors-copy.pdf) + describes how simple names are not enough in a hygienic system. + +2. Even if the point above wasn't a problem, auto-escaping unescaped caller-provided + syntax breaks the behaviour of current unescaped caller-provided syntax. For + example, the following is expected to resolve `foo` in `M`: and not in + `@__MODULE__`: + + ```julia + module M + macro m(x); x; end + end + + M.@m foo + ``` + + The `@addone` example above was framed as user error, but also illustrates + the breakage. diff --git a/JuliaLowering/src/JuliaLowering.jl b/JuliaLowering/src/JuliaLowering.jl new file mode 100644 index 0000000000000..650d15c94d5c2 --- /dev/null +++ b/JuliaLowering/src/JuliaLowering.jl @@ -0,0 +1,60 @@ +# Use a baremodule because we're implementing `include` and `eval` +baremodule JuliaLowering + +using Base +# We define a separate _include() for use in this module to avoid mixing method +# tables with the public `JuliaLowering.include()` API +const _include = Base.IncludeInto(JuliaLowering) + +if parentmodule(JuliaLowering) === Base + using Base.JuliaSyntax +else + using JuliaSyntax +end + +using .JuliaSyntax: @KSet_str, @stm, Kind, SourceAttrType, SourceRef, + SyntaxList, SyntaxTree, byte_range, children, filename, first_byte, + flattened_provenance, head, highlight, + is_leaf, is_literal, kind, last_byte, mapchildren, mapsyntax, newleaf, + newnode, node_string, numchildren, provenance, setmeta, setmeta!, getmeta, + CompileHints, source_location, sourcefile, sourceref, mapindex, mktree, + ScopeLayer, SyntaxContext, is_base_layer, base_layer, escape_layer, + syntax_module, is_flisp_compat, adopt_scope, remove_context, fill_context!, + fill_context, JL_NEW_SYNTAX_VERSION, JL_OLD_SYNTAX_VERSION + +const DEBUG = true + +# Falls back to `Union{}` so that `loc isa MacroSource` is always false on Julia < 1.14 +# where `Core.MacroSource` is not defined. +const MacroSource = isdefinedglobal(Core, :MacroSource) ? Core.MacroSource : Union{} + +const TypeEqOf = isdefinedglobal(Core, :TypeEqOf) ? "TypeEqOf" : "Typeof" + +_include("kinds.jl") +_register_kinds() + +_include("ast.jl") +_include("bindings.jl") +_include("utils.jl") +_include("validation.jl") + +_include("macro_expansion.jl") +_include("desugaring.jl") +_include("scope_analysis.jl") +_include("binding_analysis.jl") +_include("closure_conversion.jl") +_include("linear_ir.jl") +_include("runtime.jl") +_include("syntax_macros.jl") + +_include("eval.jl") +_include("compat.jl") +_include("hooks.jl") + +function __init__() + _register_kinds() +end + +_include("precompile.jl") + +end diff --git a/JuliaLowering/src/ast.jl b/JuliaLowering/src/ast.jl new file mode 100644 index 0000000000000..891927f0eef57 --- /dev/null +++ b/JuliaLowering/src/ast.jl @@ -0,0 +1,499 @@ +#------------------------------------------------------------------------------- +# @jl_assert: Produce an internal error that surfaces one or more trees. +# Example: `@jl_assert 1 === 1 (tree1, "message1"), tree2, (tree3, "message3")` +@static if DEBUG + macro jl_assert(cond, args...) + usage = "usage: @jl_assert(condition, tree|(tree, message)...)" + @assert(!isempty(args), usage) + sts = Expr(:call, SyntaxList) + msgs = Expr(:call, Base.vect) + for a in args + if Meta.isexpr(a, :tuple, 2) + push!(sts.args, a.args[1]) + push!(msgs.args, a.args[2]) + else + push!(sts.args, a) + push!(msgs.args, string(a)) + end + end + # just add assertion string to first msg + msgs.args[2] = Expr( + :string, "`jl_assert(", QuoteNode(cond), ", _)`: ", msgs.args[2]) + :($(esc(cond)) ? nothing : begin + throw(LoweringError($(esc(sts)), $(esc(msgs)), true)) + end) + end +else + # allow @jl_assert false in value position to not change rettype + macro jl_assert(cond, args...) + cond === false ? :(throw("@jl_assert false")) : nothing + end +end + +abstract type AbstractLoweringContext end + +""" +Bindings for the current lambda being processed. + +Lowering passes prior to scope resolution return `nothing` and bindings are +collected later. +""" +current_lambda_bindings(::AbstractLoweringContext) = nothing + +""" +Unique symbolic identity for a variable, constant, label, or other entity +""" +const IdTag = Int + +""" +Lexical scope ID +""" +const ScopeId = Int + +const DEFAULT_NODE = SyntaxTree( + K"None", nothing, nothing, LineNumberNode(0), nothing) + +""" + @mknode(old; attr=val...) + +Create a node `new` that is an immutable update of `old`, but setting `old` as +its provenance, and setting jl_source to macrocall's location. `attrs` may +override `old`'s fields (so if `old` is not provided, some attrs are required.) + +This is the main operation used by syntax transformations in lowering. +""" +macro mknode(attrs, old) + Base.remove_linenums!(old) + Base.remove_linenums!(attrs) + old_gs = gensym() + if !(isnothing(attrs) || attrs isa Expr && Meta.isexpr(attrs, :parameters)) + throw(ArgumentError("usage: @mknode(old; attr=val...)")) + end + out_args = Vector(undef, fieldcount(SyntaxTree)) + for (i, n) in enumerate(fieldnames(SyntaxTree)) + out_args[i] = (DEBUG && n === :jl_source) ? __source__ : + n === :source ? old_gs : + Expr(:(.), old_gs, QuoteNode(n)) + end + seen_attrs = Set{Symbol}() + attrs isa Expr && for a in attrs.args + (aname, aval) = if Meta.isexpr(a, :(kw), 2) && a.args[1] isa Symbol + (a.args[1]::Symbol, a.args[2]) + elseif a isa Symbol + (a, a) + else + throw(ArgumentError("usage: @mknode(old; attr=val...)")) + end + aname in seen_attrs && throw(ArgumentError("duplicate attr provided $__source__")) + push!(seen_attrs, aname) + out_args[Base.fieldindex(SyntaxTree, aname)] = aval + end + old === DEFAULT_NODE && !((:kind, :source, :context) ⊆ seen_attrs) && + throw(ArgumentError("brand-new node from @mknode requires more attrs $__source__")) + + out = Expr(:let, + Expr(:block, Expr(:(=), old_gs, old)), + Expr(:block, Expr(:call, SyntaxTree, out_args...))) + DEBUG && (out.args[end] = Expr(:call, _debug_check_attrs, out.args[end])) + esc(out) +end +macro mknode(x) + (old, attrs) = Meta.isexpr(x, :parameters) ? (DEFAULT_NODE, x) : (x, nothing) + esc(Expr(:macrocall, var"@mknode", __source__, attrs, old)) +end + +function _debug_check_attrs(x) + assert_syntaxtree(x, false) + x +end + +function JuliaSyntax.newleaf(prov, k, @nospecialize(value)) + context = prov isa SyntaxTree ? prov.context : nothing + @jl_assert k === K"Value" || value !== nothing ( + prov, "only Value may contain nothing") + if k == K"Identifier" || k == K"BindingId" || k == K"Value" || + k == K"core" || k == K"top" || k == K"Symbol" || k == K"globalref" || + k == K"Placeholder" || k == K"label" || k == K"symboliclabel" || + k == K"symbolicgoto" + @mknode(;kind=k, source=prov, context, value) + elseif k in KSet"TOMBSTONE SourceLocation latestworld latestworld_if_toplevel + softscope nothing" + @mknode(;kind=k, source=prov, context) + else + val = k == K"Integer" ? convert(Int, value) : + k == K"Float" ? convert(Float64, value) : + k == K"String" ? convert(String, value) : + k == K"Char" ? convert(Char, value) : + k == K"Bool" ? value : + k == K"LambdaBindings" ? value : + k == K"Slots" ? value : + k == K"SSAValue" ? value : + k == K"slot" ? value : + k == K"static_parameter" ? value : + k == K"VERSION" ? value : + error("Unexpected leaf kind `$k`") + @mknode(;kind=k, source=prov, value=val, context) + end +end + +function syntax_name(st) + @jl_assert kind(st) in KSet""" + Identifier Placeholder Symbol core top globalref symboliclabel symbolicgoto + unknown_head + """ st + st.value::String +end + +# Convenience functions to create leaf nodes referring to identifiers within +# the Core and Top modules. +nothing_(ctx, ex) = newleaf(ex, K"nothing") + +# Assign `ex` to an SSA variable. +# Return (variable, assignment_node) +function assign_tmp(ctx::AbstractLoweringContext, ex, name="tmp") + var = ssavar(ctx, ex, name) + assign_var = @mknode(;source=ex, context=ex.context, kind=K"=", + children=SyntaxList(var, ex)) + var, assign_var +end + +function emit_assign_tmp(stmts::SyntaxList, ctx, ex, name="tmp") + if is_ssa(ctx, ex) + return ex + end + var = ssavar(ctx, ex, name) + push!(stmts, newnode(ex, K"=", SyntaxList(var, ex))) + var +end + +#------------------------------------------------------------------------------- +# @ast macro + +# Fallbacks to give comprehensible error messages for use with the @ast macro +function _push_nodeid!(::Vector{SyntaxTree}, ex) + error("Attempt to use `$(repr(ex))` of type `$(typeof(ex))` as an AST node. Try annotating with `::K\"your_intended_kind\"?`") +end +function _push_nodeid!(::Vector{SyntaxTree}, ex::AbstractVector{<:SyntaxTree}) + error("Attempt to use vector as an AST node. Did you mean to splat this? (content: `$(repr(ex))`)") +end +function _push_nodeid!(ids::Vector{SyntaxTree}, st::SyntaxTree) + push!(ids, st) +end +function _push_nodeid!(::Vector{SyntaxTree}, ::Nothing) + nothing +end +function _append_nodeids!(ids::Vector{SyntaxTree}, vals) + for v in vals + _push_nodeid!(ids, v) + end +end +function _append_nodeids!(ids::Vector{SyntaxTree}, vals::SyntaxList) + append!(ids, vals) +end + +function _match_kind(srcref, ex, jl_line) + kws = Expr(:parameters) + seen = Set{Symbol}() + if Meta.isexpr(ex, :call) + kind = ex.args[1] + args = ex.args[2:end] + if Meta.isexpr(args[1], :parameters) + for a in args[1].args + a isa Symbol && push!(seen, a) + Meta.isexpr(a, :kw, 2) && a.args[1] isa Symbol && push!(seen, a.args[1]) + end + append!(kws.args, args[1].args) + popfirst!(args) + end + if length(args) == 1 && !Meta.isexpr(args[1], :kw) + srcref = args[1] + elseif length(args) > 1 + error("Unexpected srcref argument in `$ex`") + end + else + kind = ex + end + :source in seen || push!(kws.args, Expr(:kw, :source, srcref)) + :kind in seen || push!(kws.args, Expr(:kw, :kind, kind)) + :context in seen || push!(kws.args, Expr( + :kw, :context, Expr(:., srcref, QuoteNode(:context)))) + DEBUG && push!(kws.args, Expr(:kw, :jl_source, jl_line)) + return kws +end + +function _expand_ast_tree(ctx, srcref, tree, jl_line::QuoteNode) + if Meta.isexpr(tree, :(::)) + # Leaf node + if length(tree.args) == 2 + val = tree.args[1] + kindspec = tree.args[2] + else + val = nothing + kindspec = tree.args[1] + end + let kws = _match_kind(srcref, kindspec, jl_line) + !isnothing(val) && push!(kws.args, Expr(:kw, :value, val)) + Expr(:macrocall, var"@mknode", jl_line.value, kws) + end + elseif Meta.isexpr(tree, :call) && tree.args[1] === :(=>) + # Leaf node with copied attributes + kind = tree.args[3] + srcref2 = tree.args[2] + kws = Expr(:parameters, Expr(:kw, :kind, kind), Expr(:kw, :children, nothing)) + DEBUG && push!(kws.args, Expr(:kw, :jl_source, jl_line)) + Expr(:macrocall, var"@mknode", jl_line.value, kws, srcref2) + elseif Meta.isexpr(tree, (:vcat, :hcat, :vect)) + # Interior node + flatargs = [] + for a in tree.args + if Meta.isexpr(a, :row) + append!(flatargs, a.args) + else + push!(flatargs, a) + end + end + children_ex = :(let child_ids = Vector{$SyntaxTree}() + end) + child_stmts = children_ex.args[2].args + for a in flatargs[2:end] + child = _expand_ast_tree(ctx, srcref, a, jl_line) + if Meta.isexpr(child, :(...)) + push!(child_stmts, :($_append_nodeids!(child_ids, $(child.args[1])))) + else + push!(child_stmts, :($_push_nodeid!(child_ids, $child))) + end + end + push!(child_stmts, :(child_ids)) + let kws = _match_kind(srcref, flatargs[1], jl_line) + push!(kws.args, Expr(:kw, :children, children_ex)) + Expr(:macrocall, var"@mknode", jl_line.value, kws) + end + elseif Meta.isexpr(tree, :(:=)) + ctx === nothing && throw(ArgumentError( + "@ast requires ctx arg for `:=` assignments $jl_line")) + lhs = tree.args[1] + rhs = _expand_ast_tree(ctx, srcref, tree.args[2], jl_line) + ssadef = gensym("ssadef") + quote + ($lhs, $ssadef) = assign_tmp($ctx, $rhs, $(string(lhs))) + $ssadef + end + elseif Meta.isexpr(tree, :macrocall) + tree + elseif tree isa Expr + Expr(tree.head, map(a->_expand_ast_tree(ctx, srcref, a, jl_line), tree.args)...) + else + tree + end +end + +""" + @ast ctx srcref tree + +Syntactic s-expression shorthand for constructing a `SyntaxTree` AST. + +* `ctx` - Lowering context +* `srcref` - Reference to the source code from which this AST was derived. + +The `tree` contains syntax of the following forms: +* `[kind child₁ child₂]` - construct an interior node with children +* `value :: kind` - construct a leaf node +* `ex => kind` - convert a leaf node to the given `kind`, copying attributes + from it and also using `ex` as the source reference. +* `var := ex` - Set `var=ssavar(...)` and return an assignment node `\$var=ex`. + `var` may be used outside `@ast` +* `cond ? ex1 : ex2` - Conditional; `ex1` and `ex2` will be recursively expanded. + `if ... end` and `if ... else ... end` also work with this. + +Any `kind` can be replaced with an expression of the form +* `kind(srcref)` - override the source reference for this node and its children +* `kind(;attr=val)` - set an additional attribute +* `kind(srcref; attr₁=val₁, attr₂=val₂)` - the general form + + +# Examples + +``` +@ast ctx srcref [ + K"toplevel" + [K"using" + [K"importpath" + "Base" ::K"Identifier"(src) + ] + ] + [K"function" + [K"call" + "eval" ::K"Identifier" + "x" ::K"Identifier" + ] + [K"call" + "eval" ::K"core" + mn =>K"Identifier" + "x" ::K"Identifier" + ] + ] +] +``` +""" +macro ast(ctx, srcref, tree) + @gensym ctx_gs srcref_gs + assigns = if ctx isa Symbol && all(==('_'), string(ctx)) + :(let $srcref_gs = $srcref::$SyntaxTree + $(_expand_ast_tree(nothing, srcref_gs, tree, QuoteNode(__source__))) + end) + else + :(let $ctx_gs = $ctx, $srcref_gs = $srcref::$SyntaxTree + $(_expand_ast_tree(ctx_gs, srcref_gs, tree, QuoteNode(__source__))) + end) + end |> esc +end + +name_hint(name) = JuliaSyntax.CompileHints(:name_hint, name) + +#------------------------------------------------------------------------------- +# Predicates and accessors working on expression trees + +function is_quoted(ex) + kind(ex) in KSet"Symbol quote top core globalref inert + syntaxinert meta inbounds inline noinline loopinfo" +end + +function extension_type(ex) + @jl_assert kind(ex) == K"assert" ex + @jl_assert numchildren(ex) >= 1 ex + @jl_assert kind(ex[1]) == K"Symbol" ex + syntax_name(ex[1]) +end + +function is_eventually_call(ex::SyntaxTree) + k = kind(ex) + return k == K"call" || ((k == K"where" || k == K"::") && is_eventually_call(ex[1])) +end + +function find_parameters_ind(exs) + i = length(exs) + while i >= 1 + k = kind(exs[i]) + if k == K"parameters" + return i + elseif k != K"do" + break + end + i -= 1 + end + return 0 +end + +function has_parameters(ex::SyntaxTree) + find_parameters_ind(children(ex)) != 0 +end + +function has_parameters(args::AbstractVector) + find_parameters_ind(args) != 0 +end + +function any_assignment(exs) + any(kind(e) == K"=" for e in exs) +end + +function is_valid_modref(ex) + return kind(ex) == K"." && kind(ex[2]) == K"Symbol" && + (kind(ex[1]) == K"Identifier" || is_valid_modref(ex[1])) +end + +function is_core_Any(ex) + kind(ex) === K"core" && syntax_name(ex) === "Any" +end + +function is_simple_atom(ctx, ex) + k = kind(ex) + # TODO thismodule + is_literal(k) || k == K"Symbol" || k == K"Value" || is_ssa(ctx, ex) || + k == K"nothing" +end + +function is_identifier_like(ex) + k = kind(ex) + k == K"Identifier" || k == K"BindingId" || k == K"Placeholder" +end + +function decl_var(ex) + kind(ex) == K"::" ? ex[1] : ex +end + +# Given the signature of a `function`, return the symbol that will ultimately +# be assigned to in local/global scope, if any. +function assigned_function_name(ex) + while kind(ex) == K"where" + # f() where T + ex = ex[1] + end + if kind(ex) == K"::" && numchildren(ex) == 2 + # f()::T + ex = ex[1] + end + if kind(ex) != K"call" + throw(LoweringError(ex, "Expected call syntax in function signature")) + end + ex = ex[1] + if kind(ex) == K"curly" + # f{T}() + ex = ex[1] + end + if kind(ex) == K"::" || kind(ex) == K"." + # (obj::CallableType)(args) + # A.b.c(args) + nothing + elseif is_identifier_like(ex) + ex + else + throw(LoweringError(ex, "Unexpected name in function signature")) + end +end + +# Remove empty parameters block, eg, in the arg list of `f(x, y;)` +function remove_empty_parameters(args) + i = length(args) + while i > 0 && kind(args[i]) == K"parameters" && numchildren(args[i]) == 0 + i -= 1 + end + args[1:i] +end + +function to_symbol(ctx, ex) + @ast ctx ex ex=>K"Symbol" +end + +#------------------------------------------------------------------------------- +# Context wrapper which helps to construct a list of statements to be executed +# prior to some expression. Useful when we need to use subexpressions multiple +# times. +struct StatementListCtx{Ctx} <: AbstractLoweringContext + ctx::Ctx + stmts::Vector{SyntaxTree} +end + +function Base.getproperty(ctx::StatementListCtx, field::Symbol) + if field === :ctx + getfield(ctx, :ctx) + elseif field === :stmts + getfield(ctx, :stmts) + else + getproperty(getfield(ctx, :ctx), field) + end +end + +function emit(ctx::StatementListCtx, ex) + push!(ctx.stmts, ex) +end + +function emit_assign_tmp(ctx::StatementListCtx, ex, name="tmp") + emit_assign_tmp(ctx.stmts, ctx.ctx, ex, name) +end + +with_stmts(ctx, stmts) = StatementListCtx(ctx, stmts) +with_stmts(ctx::StatementListCtx, stmts) = StatementListCtx(ctx.ctx, stmts) + +function with_stmts(ctx) + StatementListCtx(ctx, SyntaxList()) +end diff --git a/JuliaLowering/src/binding_analysis.jl b/JuliaLowering/src/binding_analysis.jl new file mode 100644 index 0000000000000..0427a168b61cd --- /dev/null +++ b/JuliaLowering/src/binding_analysis.jl @@ -0,0 +1,381 @@ +#------------------------------------------------------------------------------- +# Syntactic-block-local dominance analysis to optimize Box allocations. +# Nearly identical (by design) to flisp's `lambda-optimize-vars!` in julia-syntax.scm. +# +# This pass attempts to prove +# (for the special case of assigned-once variables): +# 1. A variable is always defined at every use / capture +# 2. A variable will not be modified anywhere after any capture +# +# Within inner syntactic blocks, an outer assignment effectively "guards" the +# variable so that uses / captures do not taint the "always-defined" status. +# +# This "guard" behavior is disabled when a variable was introduced outside of +# an enclosing loop, since that would allow condition (2) to be violated. In +# contrast, straight-line captures (i.e. an assignment followed by a capture +# in the same block) is allowed even in loops, since it's unconditional. +# +# In the implementation, any variables in `unused` / `live` are considered +# "always-defined- when-used-or-captured-and-only-modified-once-dynamically". +# These variables may temporarily lose their status when considering uses / +# captures in inner blocks, but this is restored later if dominated by an +# outer assignment. + +""" + analyze_def_and_use!(ctx, ex) + +Perform tree-based def-use analysis to find captured variables that are assigned +before any closure captures them and not modified afterward. For such variables, +as an abuse of binding flags we can mark them as `unboxed=true` to avoid +unnecessary `Core.Box` allocations during closure conversion. + +This is called on the outermost lambda, and recursively processes nested lambdas. +""" +function analyze_def_and_use!(ctx, ex) + @stm ex begin + [K"lambda" _ _ _ body _...] -> begin + _analyze_nested_lambdas!(ctx, body) + _analyze_lambda_vars!(ctx, ex) + end + [K"toplevel_lambda" _ _ _ body _...] -> begin + _analyze_nested_lambdas!(ctx, body) + _analyze_lambda_vars!(ctx, ex) + end + [K"generated_lambda" _ _ _ body _...] -> begin + _analyze_nested_lambdas!(ctx, body) + _analyze_lambda_vars!(ctx, ex) + end + end +end + +function _analyze_nested_lambdas!(ctx, ex) + k = kind(ex) + if k in KSet"lambda toplevel_lambda generated_lambda" + analyze_def_and_use!(ctx, ex) + elseif !is_leaf(ex) && !is_quoted(ex) + for child in children(ex) + _analyze_nested_lambdas!(ctx, child) + end + end +end + +""" + DefUseState + +State for def-use analysis (flisp-compatible tables for tracking variable def and use). + +Fields: +- `unused`: candidate variables not yet used (read) in current block +- `live`: variables that have been assigned in current block +- `seen`: all variables we've seen assigned +- `decl`: variables scoped in current scope (via `local` or an argument) +- `decl_outside_loop`: variables scoped in scope outside loop (via `local` or an argument) +- `args`: argument variables (never undefined, special handling in mark_used!) +""" +mutable struct DefUseState + const lambda_id::ScopeId + const unused::Set{IdTag} + const live::Set{IdTag} + const seen::Set{IdTag} + decl::Set{IdTag} + decl_outside_loop::Set{IdTag} + const args::Set{IdTag} + + function DefUseState(lambda_id, ctx, candidates) + unused = copy(candidates) + live = Set{IdTag}() + seen = Set{IdTag}() + decl = Set{IdTag}() + decl_outside_loop = Set{IdTag}() + args = Set{IdTag}() + # Initialize decl and args with arguments since they're implicitly declared outside any loop + for id in candidates + binfo = get_binding(ctx, id) + if binfo.kind == :argument + push!(decl, id) + push!(args, id) + end + end + return new(lambda_id, unused, live, seen, decl, decl_outside_loop, args) + end +end + +# At CFG merge points, we lose certainty about which path was taken, +# so variables assigned in one branch may not have been assigned. +# Move live variables back to unused to require re-assignment. +# NOTE: This is NOT needed at branch points (return/break/goto) because +# code after them is unreachable - only at merge points (if/while/label). +function du_kill!(state::DefUseState) + union!(state.unused, state.live) + empty!(state.live) +end + +# Restore live to a previous state, moving new additions back to unused +function du_restore!(state::DefUseState, prev) + for id in state.decl_outside_loop + if (id in prev) && !(id in state.unused) + # This variable was 'used' inside this branch, but it's declared + # outside of a loop so it may see the dominating assignment execute + # multiple times. Invalidate it here for soundness. + delete!(prev, id) + end + end + for id in state.live + if !(id in prev) + push!(state.unused, id) + end + end + empty!(state.live) + union!(state.live, prev) +end + +# At the beginning of a loop, move all active decls into the "decl_outside_loop" set. +function du_enter_loop!(state::DefUseState) + prev_decl_outside_loop = state.decl_outside_loop + state.decl_outside_loop = state.decl + state.decl = copy(state.decl) + return prev_decl_outside_loop +end + +# At the end of a loop, restore the previous set of "declared" variables. +function du_leave_loop!(state::DefUseState, prev_decl_outside_loop) + state.decl = state.decl_outside_loop + state.decl_outside_loop = prev_decl_outside_loop +end + +# When a variable is used (read), remove from unused. +# Note: arguments are only "used" for purposes of this analysis when +# they are captured, since they are never undefined. +function du_mark_used!(state::DefUseState, var_id) + if var_id in state.unused && !(var_id in state.args) + delete!(state.unused, var_id) + end +end + +# When a variable is captured by a nested lambda before being assigned +function du_mark_captured!(state::DefUseState, var_id) + if var_id in state.unused + delete!(state.unused, var_id) + end +end + +# When a variable is assigned, move from unused to live +function du_assign!(state::DefUseState, var_id) + if var_id in state.unused + push!(state.live, var_id) + push!(state.seen, var_id) + delete!(state.unused, var_id) + end +end + +# Track local declarations for loop handling +function du_declare!(state::DefUseState, var_id) + if var_id in state.unused + push!(state.decl, var_id) + end +end + +# Returns whether e contained a symboliclabel +function du_visit!(ctx, state::DefUseState, e) + k = kind(e) + + if k == K"BindingId" + du_mark_used!(state, syntax_id(e)) + return false + + elseif k == K"symboliclabel" + # Must check BEFORE is_leaf since symboliclabel is a leaf node + du_kill!(state) + return true + + elseif k == K"label" + du_kill!(state) + return false + + elseif k in KSet"break symbolicgoto" + # this kill!() is not required for soundness since these are branch points + # not merge points, but it's here for parity with flisp + du_kill!(state) + return false + + elseif k == K"=" + # Visit RHS first, then record assignment + has_label = du_visit!(ctx, state, e[2]) + lhs = e[1] + if kind(lhs) == K"BindingId" + du_assign!(state, syntax_id(lhs)) + end + return has_label + + elseif k == K"lambda" + # Check captures from nested lambda + for (id, is_capt) in lambda_bindings(e[1]).locals_capt + if is_capt + du_mark_captured!(state, id) + end + end + # Don't recurse into nested lambdas - they have their own analysis + return false + + elseif k == K"local" + # Track local declarations for loop handling + # Note: For typed locals like `local x::T`, the K"local" node only + # contains the BindingId after desugaring. The type info is in + # a separate K"decl" node. So we only need to handle K"BindingId" here. + for child in children(e) + if kind(child) == K"BindingId" + du_declare!(state, syntax_id(child)) + end + end + return false + + elseif k == K"decl" + # Don't recurse into decl nodes - the BindingId is just a declaration, + # not a use. We only need to visit the type expression. + if numchildren(e) >= 2 + return du_visit!(ctx, state, e[2]) + end + return false + + elseif k == K"function_decl" + # [function_decl] defines and instantiates the closure type + @assert kind(e[1]) == K"BindingId" + func_id = syntax_id(e[1]) + func_id in state.seen && return false + ck = ClosureKey(func_id, state.lambda_id) + if haskey(ctx.closure_bindings, ck) + for lam in ctx.closure_bindings[ck].lambdas + for (id, capt) in lam.locals_capt + capt && du_mark_captured!(state, id) + end + end + end + return false + + elseif k == K"method_defs" + # XXX: the assignment is executed after the body, but flisp also makes + # the mistake of modelling the assignment as dominating the body, so we + # introduce boxes if it's corrected. + if kind(e[1]) === K"BindingId" + du_assign!(state, syntax_id(e[1])) + end + has_label = false + for child in children(e) + has_label |= du_visit!(ctx, state, child) + end + return has_label + + elseif k == K"no_method_defs" + du_assign!(state, syntax_id(e[1])) + return false + + elseif k == K"return" + has_label = numchildren(e) >= 1 ? du_visit!(ctx, state, e[1]) : false + du_kill!(state) # not necessary, but included for flisp parity + return has_label + + elseif k in KSet"if elseif trycatchelse tryfinally" + prev = copy(state.live) + has_label = false + for child in children(e) + has_label |= du_visit!(ctx, state, child) + du_kill!(state) + end + if has_label + # If there's a label inside, we could have skipped a prior + # variable initialization + return true + else + du_restore!(state, prev) + return false + end + + elseif k in KSet"_while _do_while" + prev = copy(state.live) + old_decl = du_enter_loop!(state) + has_label = false + for child in children(e) + has_label |= du_visit!(ctx, state, child) + end + du_leave_loop!(state, old_decl) + if has_label + du_kill!(state) + return true + else + du_restore!(state, prev) + return false + end + + elseif k == K"symbolicblock" + # Skip the first child (break target label) - it's not a @goto target + # No save/restore needed: the body always executes (break just exits early) + has_label = false + for child in children(e)[2:end] + has_label |= du_visit!(ctx, state, child) + end + return has_label + + elseif is_leaf(e) || is_quoted(e) || + k in KSet"local always_defined meta inbounds boundscheck noinline + loopinfo decl with_static_parameters toplevel_butfirst global + globalref constdecl atomic isdefined toplevel module error + gc_preserve_begin gc_preserve_end export public inline" + + # Forms that don't interact with locals or affect control flow (likely more than is necessary). + # flisp: `lambda-opt-ignored-exprs` + return false + + else + has_label = false + for child in children(e) + has_label |= du_visit!(ctx, state, child) + end + return has_label + end +end + +function _analyze_lambda_vars!(ctx::VariableAnalysisContext, ex) + # Collect candidate variables: captured and single-assigned + candidates = Set{IdTag}() + for (id, from_outer_lambda) in lambda_bindings(ex[1]).locals_capt + b = get_binding(ctx, id) + !b.is_captured && continue + from_outer_lambda && continue + if b.is_assigned_once && b.kind in (:local, :argument) + push!(candidates, id) + end + end + isempty(candidates) && return + + state = DefUseState(lambda_bindings(ex[1]).scope_id, ctx, candidates) + @stm ex begin + [K"lambda" _ _ _ body] -> du_visit!(ctx, state, body) + [K"lambda" _ _ _ body rett] -> (du_visit!(ctx, state, body); + du_visit!(ctx, state, rett)) + [K"toplevel_lambda" _ _ _ body] -> du_visit!(ctx, state, body) + [K"generated_lambda" _ _ _ body] -> du_visit!(ctx, state, body) + end + + for id in union(state.live, state.unused) + if id in state.seen + b = get_binding(ctx, id) + b.unboxed = true + b.is_always_defined = true + end + end + + # A single (scope-dominating) assignment implies unboxed even if we gave up above + for id in candidates + b = get_binding(ctx, id) + # XXX: This uses is-always-defined to imply that the assignment is defined + # everywhere in its scope, which then implies that the one definition + # executes only once dynamically. + # (i.e. it forbids single-assignment to `x` in an inner loop) + # + # If this flag becomes broader and only considers definedness-at-use + # then this check (taken from `julia-syntax.scm`) becomes unsound. + if b.kind === :local && b.is_always_defined && b.is_assigned_once + b.unboxed = true + end + end +end diff --git a/JuliaLowering/src/bindings.jl b/JuliaLowering/src/bindings.jl new file mode 100644 index 0000000000000..79e5f889e3994 --- /dev/null +++ b/JuliaLowering/src/bindings.jl @@ -0,0 +1,207 @@ +""" +Metadata about a binding +""" +mutable struct BindingInfo + const id::IdTag # Unique integer identifying this binding + const name::String + const kind::Symbol # :local :global :argument :static_parameter + const node_id::SyntaxTree + const mod::Union{Nothing,Module} # Set when `kind === :global` + type::Union{Nothing,SyntaxTree} # Type, for bindings declared like x::T = 10 + lambda_id::Int # from scope resolution; 0 if unresolved + is_const::Bool # Constant, cannot be reassigned + is_ssa::Bool # Single assignment, defined before use + is_internal::Bool # True for internal bindings generated by the compiler + is_ambiguous_local::Bool # Local, but would be global in soft scope (ie, the REPL) + unboxed::Bool # determined by binding_analysis.jl; flisp generally mutates `capt` instead + + # flisp: vinfo + is_nospecialize::Bool # @nospecialize on this argument (only valid for kind == :argument) + is_read::Bool + is_called::Bool + is_assigned::Bool + is_assigned_once::Bool + is_captured::Bool + is_always_defined::Bool + is_used_undef::Bool +end + +""" +Metadata about "entities" (variables, constants, etc) in the program. Each +entity is associated to a unique integer id, the BindingId. A binding will be +inferred for each *name* in the user's source program by symbolic analysis of +the source. + +However, bindings can also be introduced programmatically during lowering or +macro expansion: the primary key for bindings is the `BindingId` integer, not +a name. +""" +struct Bindings + info::Vector{BindingInfo} +end + +function BindingInfo(bindings::Bindings, + name::AbstractString, kind::Symbol, node_id::SyntaxTree; + mod::Union{Nothing,Module} = nothing, + type::Union{Nothing,SyntaxTree} = nothing, + lambda_id::Int = 0, + is_const::Bool = false, + is_ssa::Bool = false, + is_internal::Bool = false, + is_ambiguous_local::Bool = false, + unboxed::Bool = false, + is_nospecialize::Bool = false, + is_read::Bool = false, + is_called::Bool = false, + is_assigned::Bool = false, + is_assigned_once::Bool = false, + is_captured::Bool = false, + is_always_defined::Bool = is_ssa || kind === :argument, + is_used_undef::Bool = false) + bid = next_binding_id(bindings) + b = BindingInfo( + bid, name, kind, node_id, mod, type, lambda_id, is_const, is_ssa, + is_internal, is_ambiguous_local, unboxed, is_nospecialize, + is_read, is_called, is_assigned, is_assigned_once, is_captured, + is_always_defined, is_used_undef) + add_binding(bindings, b) + b +end + +function Base.show(io::IO, binfo::BindingInfo) + print(io, "BindingInfo(", binfo.id, + ", ", repr(binfo.name), + ", ", repr(binfo.kind), + ", ", binfo.node_id) + !isnothing(binfo.mod) && print(io, ", mod=", binfo.mod) + !isnothing(binfo.type) && print(io, ", type=", binfo.type) + print(io, ", lambda_id=", binfo.lambda_id) + binfo.is_const && print(io, ", is_const=true") + binfo.is_ssa && print(io, ", is_ssa=true") + binfo.is_internal && print(io, ", is_internal=true") + binfo.is_ambiguous_local && print(io, ", is_ambiguous_local=true") + binfo.unboxed && print(io, ", unboxed=true") + binfo.is_nospecialize && print(io, ", is_nospecialize=true") + binfo.is_read && print(io, ", is_read=true") + binfo.is_called && print(io, ", is_called=true") + binfo.is_assigned && print(io, ", is_assigned=true") + binfo.is_assigned_once && print(io, ", is_assigned_once=true") + binfo.is_captured && print(io, ", is_captured=true") + binfo.is_always_defined && print(io, ", is_always_defined=true") + binfo.is_used_undef && print(io, ", is_used_undef=true") + print(io, ")") +end + +Bindings() = Bindings(Vector{BindingInfo}()) + +next_binding_id(bindings::Bindings) = length(bindings.info) + 1 + +function add_binding(bindings::Bindings, binding) + if next_binding_id(bindings) != binding.id + error("Use next_binding_id() to create a valid binding id") + end + push!(bindings.info, binding) +end + +function syntax_id(ex::SyntaxTree) + @jl_assert kind(ex) in KSet"BindingId SSAValue slot static_parameter label" ex + ex.value::IdTag +end + +function get_binding(bindings::Bindings, x)::BindingInfo + id = if x isa SyntaxTree + @jl_assert kind(x) === K"BindingId" x + syntax_id(x) + else + x + end + bindings.info[id] +end + +function get_binding(ctx::AbstractLoweringContext, x)::BindingInfo + get_binding(ctx.bindings::Bindings, x) +end + +function _new_binding(bindings::Bindings, srcref::SyntaxTree, + name::AbstractString, kind::Symbol; kws...) + # A binding is only useful when it shows up in the tree, so create its tree + # node eagerly and share it among uses (see `binding_ex`) + bid = next_binding_id(bindings) + ex = @ast _ srcref bid::K"BindingId" + b = BindingInfo(bindings, name, kind, ex; kws...) + return b +end + +# Create a new SSA binding +function ssavar(ctx::AbstractLoweringContext, srcref, name="tmp") + binding_ex(ctx, _new_binding(ctx.bindings, srcref, name, :local; + is_ssa=true, is_internal=true)) +end + +# Create a new local mutable binding or lambda argument +function new_local_binding(ctx::AbstractLoweringContext, srcref, name; + kind=:local, kws...) + @jl_assert kind === :local || kind === :argument srcref + nameref = newleaf(srcref, K"Identifier", name) + b = _new_binding(ctx.bindings, nameref, name, kind; is_internal=true, kws...) + lbindings = current_lambda_bindings(ctx) + if !isnothing(lbindings) + init_lambda_binding(lbindings, b, false) + end + binding_ex(ctx, b) +end + +function new_global_binding(ctx::AbstractLoweringContext, srcref, name, mod; kws...) + nameref = newleaf(srcref, K"Identifier", name) + binding_ex(ctx, _new_binding( + ctx.bindings, nameref, name, :global; is_internal=true, mod=mod, kws...)) +end + +function binding_ex(ctx::AbstractLoweringContext, b::BindingInfo) + b.node_id +end +binding_ex(ctx, id::IdTag) = binding_ex(ctx, get_binding(ctx, id)) +binding_type_ex(ctx::AbstractLoweringContext, b::BindingInfo) = + b.type + +""" +Key to use when transforming names into bindings +""" +struct NameKey + name::String + layer::ScopeLayer +end + +function NameKey(ex::SyntaxTree) + @jl_assert kind(ex) in KSet"Identifier symboliclabel symbolicgoto" ex + NameKey(syntax_name(ex), (ex.context::SyntaxContext).layer) +end + +# One lambda's variables +struct LambdaBindings + # Binding ID of #self# + self::IdTag + # For finding the parent lambda in variable analysis + scope_id::ScopeId + # A map from every referenced local binding ID to whether the local is + # captured (true) or native to this lambda (false). References in inner + # lambdas count: `inner.locals_capt[id]` implies `haskey(locals_capt, id)` + # TODO: If we use scope ID as a lambda ID and give BindingInfo a field + # noting which lambda it belongs to, we could just make this a BitSet of + # vars present, where we tell if a binding is captured by comparing + # this.scope_id with the BindingInfo's scope_id. + locals_capt::Dict{IdTag,Bool} +end + +LambdaBindings(self::IdTag = 0, scope_id::ScopeId = 0) = + LambdaBindings(self, scope_id, Dict{IdTag,Bool}()) + +function init_lambda_binding(bindings::LambdaBindings, b::BindingInfo, capt::Bool) + bindings.locals_capt[b.id] = capt + b.lambda_id = bindings.scope_id +end + +function lambda_bindings(st::SyntaxTree) + @jl_assert kind(st) === K"LambdaBindings" st + st.value::LambdaBindings +end diff --git a/JuliaLowering/src/closure_conversion.jl b/JuliaLowering/src/closure_conversion.jl new file mode 100644 index 0000000000000..e1c6f6917af09 --- /dev/null +++ b/JuliaLowering/src/closure_conversion.jl @@ -0,0 +1,737 @@ +struct ClosureInfo + closure_key::ClosureKey + # Global name of the type of the closure + type_name::SyntaxTree + # Names of fields for use with getfield, in order + field_names::Vector{SyntaxTree} + # Map from the original BindingId of closed-over vars to the index of the + # associated field in the closure type. + field_inds::Dict{IdTag,Int} + capt_sp::Vector{SyntaxTree} +end + +mutable struct ClosureConversionCtx <: AbstractLoweringContext + const bindings::Bindings + const mod::Module + const closure_bindings::Dict{ClosureKey,ClosureBindings} + const capture_rewriting::Union{Nothing,ClosureInfo, + Vector{SyntaxTree}} + const top_bindings::LambdaBindings + const lambda_bindings::LambdaBindings + const sp_typevars::Dict{IdTag, IdTag} + # True if we're in a section of code which preserves top-level sequencing + # such that closure types can be emitted inline with other code. + const toplevel::Bool + # toplevel, or contained by method_defs and no lambda within it + const lifted::Bool + # True if this expression should not have toplevel effects, namely, it + # should not declare the globals it references. This allows generated + # functions to refer to globals that have already been declared, without + # triggering the "function body AST not pure" error. + const toplevel_pure::Bool + const toplevel_stmts::Vector{SyntaxTree} + const closure_infos::Dict{ClosureKey,ClosureInfo} + # Populated with function_decl, then unpopulated with the first + # corresponding method_defs + const closure_structs::Dict{ClosureKey,SyntaxTree} +end + +function current_lambda_bindings(ctx::ClosureConversionCtx) + ctx.lifted ? ctx.top_bindings : ctx.lambda_bindings +end + +# Access captured variable from inside a closure +function captured_var_access(ctx, ex) + cap_rewrite = ctx.capture_rewriting + if cap_rewrite isa ClosureInfo + field_sym = cap_rewrite.field_names[cap_rewrite.field_inds[syntax_id(ex)]] + @ast ctx ex [K"call" + "getfield"::K"core" + binding_ex(ctx, current_lambda_bindings(ctx).self) + field_sym + ] + else + interpolations = cap_rewrite + @jl_assert !isnothing(cap_rewrite) ex + if isempty(interpolations) || !is_same_identifier_like(interpolations[end], ex) + push!(interpolations, ex) + end + @ast ctx ex [K"captured_local" length(interpolations)::K"Integer"] + end +end + +function get_box_contents(ctx::ClosureConversionCtx, var, box_ex) + b = get_binding(ctx, var) + box = ssavar(ctx, box_ex) + undef_var = new_local_binding(ctx, var, b.name; is_used_undef=true) + box_access = + @ast ctx var [K"call" "getfield"::K"core" box "contents"::K"Symbol"] + if !isnothing(b.type) + box_access = @ast ctx var [K"call" + "typeassert"::K"core" + box_access + _convert_closures(ctx, renumber_assigned_ssavalues( + ctx, binding_type_ex(ctx, b))) + ] + end + @ast ctx var [K"block" + [K"=" box box_ex] + # Lower in an UndefVar check to a similarly named variable + # (ref #20016) so that closure lowering Box introduction + # doesn't impact the error message and the compiler is expected + # to fold away the extraneous null check + # + # TODO: Ideally the runtime would rely on provenance info for + # this error and we can remove the isdefined check. + [K"if" [K"call" "isdefined"::K"core" box "contents"::K"Symbol"] + (::K"TOMBSTONE") + [K"block" [K"newvar" undef_var] undef_var]] + box_access + ] +end + +# Convert `ex` to `type` by calling `convert(type, ex)` when necessary. +# +# Used for converting the right hand side of an assignment to a typed local or +# global and for converting the return value of a function call to the declared +# return type. +function convert_for_type_decl(ctx, srcref, ex, type, do_typeassert) + # Use a slot to permit union-splitting this in inference + tmp = new_local_binding(ctx, srcref, "tmp", is_always_defined=true) + + @ast ctx srcref [K"block" + type_tmp := renumber_assigned_ssavalues(ctx, type) + [K"=" tmp ex] + [K"if" + [K"call" "isa"::K"core" tmp type_tmp] + (::K"nothing") + [K"=" + tmp + if do_typeassert + [K"call" + "typeassert"::K"core" + [K"call" "convert"::K"top" type_tmp tmp] + type_tmp + ] + else + [K"call" "convert"::K"top" type_tmp tmp] + end + ] + ] + tmp + ] +end + +# TODO: Avoid producing redundant calls to declare_global +function make_globaldecl(ctx, src_ex, mod, name, strong=false, type=nothing) + decl = @ast ctx src_ex [K"block" + [K"call" + "declare_global"::K"core" + mod::K"Value" name::K"Symbol" strong::K"Bool" + type + ] + (::K"latestworld") + (::K"nothing") + ] + ctx.toplevel_pure && return newleaf(decl, K"TOMBSTONE") + if !ctx.toplevel + push!(ctx.toplevel_stmts, decl) + newleaf(decl, K"TOMBSTONE") + else + return decl + end +end + +function convert_global_assignment(ctx, ex, var, rhs0) + binfo = get_binding(ctx, var) + @jl_assert binfo.kind == :global ex var + stmts = SyntaxList() + decl = make_globaldecl(ctx, ex, binfo.mod, binfo.name, true) + if kind(decl) !== K"TOMBSTONE" + push!(stmts, decl) + end + rhs1 = if is_simple_atom(ctx, rhs0) + rhs0 + else + tmp = ssavar(ctx, rhs0) + push!(stmts, @ast ctx rhs0 [K"=" tmp rhs0]) + tmp + end + rhs = if binfo.is_const && isnothing(binfo.type) + # const global assignments without a type declaration don't need us to + # deal with the binding type at all. + rhs1 + else + type_var = ssavar(ctx, ex, "binding_type") + push!(stmts, @ast ctx ex [K"=" + type_var + [K"call" + "get_binding_type"::K"core" + binfo.mod::K"Value" + binfo.name::K"Symbol" + ] + ]) + do_typeassert = false # Global assignment type checking is done by the runtime + convert_for_type_decl(ctx, ex, rhs1, type_var, do_typeassert) + end + push!(stmts, @ast ctx ex [K"=" var rhs]) + @ast ctx ex [K"block" + stmts... + rhs1 + ] +end + +# Convert assignment to a closed variable to a `setfield!` call and generate +# `convert` calls for variables with declared types. +# +# When doing this, the original value needs to be preserved, to ensure the +# expression `a=b` always returns exactly `b`. +function convert_assignment(ctx, ex) + var = ex[1] + rhs0 = _convert_closures(ctx, ex[2]) + if kind(var) == K"Placeholder" + return @ast ctx ex [K"=" var rhs0] + end + @jl_assert kind(var) == K"BindingId" ex + binfo = get_binding(ctx, var) + if binfo.kind == :global + convert_global_assignment(ctx, ex, var, rhs0) + else + @jl_assert binfo.kind in (:local, :argument, :typevar) ex + boxed = is_boxed(binfo) + if isnothing(binfo.type) && !boxed + @ast ctx ex [K"=" var rhs0] + else + # Typed local + tmp_rhs0 = ssavar(ctx, rhs0) + rhs = isnothing(binfo.type) ? tmp_rhs0 : + convert_for_type_decl( + ctx, ex, tmp_rhs0, + _convert_closures(ctx, binding_type_ex(ctx, binfo)), + true) + assignment = if boxed + @ast ctx ex [K"call" + "setfield!"::K"core" + is_self_captured(ctx, var) ? captured_var_access(ctx, var) : var + "contents"::K"Symbol" + rhs + ] + else + @ast ctx ex [K"=" var rhs] + end + @ast ctx ex [K"block" + [K"=" tmp_rhs0 rhs0] + assignment + tmp_rhs0 + ] + end + end +end + +# Compute fields for a closure type, one field for each captured variable. +function closure_type_fields(ctx, srcref, closure_binds, is_opaque) + capt_locals = Set{IdTag}() + capt_sp = Set{IdTag}() + add_capt(id) = push!( + get_binding(ctx, id).kind !== :static_parameter || is_opaque ? + capt_locals : capt_sp, id) + for lambda_bindings in closure_binds.lambdas + for (id, is_capt) in lambda_bindings.locals_capt + is_capt && add_capt(id) + end + end + foreach(add_capt, closure_binds.capt_sp) + + field_syms = SyntaxList() + if is_opaque + field_orig_bindings = sort!(collect(capt_locals)) + # For opaque closures we don't try to generate sensible names for the + # fields as there's no closure type to generate. + for i in eachindex(field_orig_bindings) + push!(field_syms, @ast ctx srcref i::K"Integer") + end + else + field_names = Dict{String,IdTag}() + for id in sort!(collect(capt_locals)) + binfo = get_binding(ctx, id) + # We name each field of the closure after the variable which was closed + # over, for clarity. Adding a suffix can be necessary when collisions + # occur due to macro expansion and generated bindings + name0 = binfo.name + name = name0 + i = 1 + while haskey(field_names, name) + name = "$name0#$i" + i += 1 + end + field_names[name] = id + end + field_orig_bindings = Vector{IdTag}() + for (name,id) in sort!(collect(field_names)) + push!(field_syms, @ast ctx srcref name::K"Symbol") + push!(field_orig_bindings, id) + end + end + field_inds = Dict{IdTag,Int}() + field_is_box = Vector{Bool}() + for (i,id) in enumerate(field_orig_bindings) + push!(field_is_box, is_boxed(ctx, id)) + field_inds[id] = i + end + capt_sp2 = SyntaxList() + for sp in sort!(collect(capt_sp)) + push!(capt_sp2, binding_ex(ctx, sp)) + end + + return field_syms, field_orig_bindings, field_inds, field_is_box, capt_sp2 +end + +# No box needed for: +# - non-captured vars +# - static params (can't be reassigned) +# - any local our optimizations have determined to be unboxed +function is_boxed(binfo::BindingInfo) + binfo.kind === :static_parameter && return false + binfo.kind === :typevar && return false + binfo.unboxed && return false + binfo.kind === :argument && !binfo.is_assigned && return false + return binfo.is_captured +end + +function is_boxed(ctx, x) + is_boxed(get_binding(ctx, x)) +end + +# Is a field in the closure argument `self`. Exception: non-OC sparams are type +# params to the `self` type, and are rewritten later in linearization. +function is_self_captured(ctx, x) + b = get_binding(ctx, x) + out = get(current_lambda_bindings(ctx).locals_capt, b.id, false) + if out && (b.kind === :static_parameter || b.kind === :typevar) + ctx.capture_rewriting isa ClosureInfo && + haskey(ctx.capture_rewriting.field_inds, b.id) + else + out + end +end + +# Should comply with whatever `jl_demangle_typename` expects +function closure_type_name(ctx, ck) + stack = ctx.closure_bindings[ck].name_stack + counter() = module_unique_name(ctx.mod, first(stack)) + self = last(stack) + # This should probably be a stack of identifiers instead of strings so + # is_internal works. Hygiene likely doesn't matter here. + base = self == "#anon#" || self == "#->#" ? "#" * counter() : + startswith(self, '#') ? self : "#" * self + name_str = string(base, "#", counter()) +end + +function convert_local_function_decl(ctx, ex) + ck = closure_key(ctx, ex[1]) + haskey(ctx.closure_infos, ck) && return @ast ctx ex (::K"TOMBSTONE") + + closure_binds = ctx.closure_bindings[ck] + field_syms, field_orig_bindings, field_inds, field_is_box, capt_sp = + closure_type_fields(ctx, ex, closure_binds, false) + name_str = closure_type_name(ctx, ck) + global_clstruct = new_global_binding(ctx, ex, name_str, ctx.mod) + sp_syms = mapsyntax(sp->newleaf(sp, K"Symbol", + get_binding(ctx, syntax_id(sp)).name), + capt_sp) + define_clstruct = type_ex = @ast ctx ex [K"call" + eval_closure_type::K"Value" + ctx.mod::K"Value" + name_str::K"Symbol" + [K"call" "svec"::K"core" sp_syms...] + [K"call" "svec"::K"core" field_syms...] + [K"call" "svec"::K"core" [f::K"Bool" for f in field_is_box]...] + ] + if !ctx.toplevel + push!(ctx.toplevel_stmts, define_clstruct) + push!(ctx.toplevel_stmts, @ast ctx ex (::K"latestworld_if_toplevel")) + define_clstruct = nothing + end + ctx.closure_infos[ck] = + ClosureInfo(ck, global_clstruct, field_syms, field_inds, capt_sp) + type_params = mapsyntax(capt_sp) do sp + is_self_captured(ctx, sp) ? captured_var_access(ctx, sp) : sp + end + init_closure_args = SyntaxList() + for (id, boxed) in zip(field_orig_bindings, field_is_box) + field_val = binding_ex(ctx, id) + if is_self_captured(ctx, field_val) + # Access from outer closure if necessary but do not + # unbox to feed into the inner nested closure. + field_val = captured_var_access(ctx, field_val) + end + push!(init_closure_args, field_val) + if !boxed + push!(type_params, @ast ctx ex [K"call" + "_typeof_captured_variable"::K"core" + field_val]) + end + end + ctx.closure_structs[ck] = clstruct = ssavar(ctx, ex[1]) + @ast ctx ex [K"block" + define_clstruct + (::K"latestworld_if_toplevel") + closure_type := if isempty(type_params) + global_clstruct + else + [K"call" "apply_type"::K"core" global_clstruct type_params...] + end + [K"=" clstruct [K"new" closure_type init_closure_args...]] + (::K"TOMBSTONE") + ] +end + +# We want to change the order of children as little as necessary to get all +# top-level-only forms out to top level (extra movement is hard to reason about, +# as there is currently a somewhat brittle ordering of forms enforced by +# desugaring). For top-level `st`, this means setting up a new `toplevel_stmts` +# catcher for all children of `st` to add to. Otherwise, expressions use their +# parent's catcher. An exception to "as little as necessary" is made for loops +# for performance reasons. +function map_cl_convert(ctx::ClosureConversionCtx, ex) + if !ctx.toplevel + mapchildren(e->_convert_closures(ctx, e), ex) + elseif kind(ex) === K"_while" || kind(ex) === K"_do_while" + mapchildren(e->_convert_closures( + ClosureConversionCtx( + ctx.bindings, ctx.mod, + ctx.closure_bindings, ctx.capture_rewriting, ctx.top_bindings, + ctx.lambda_bindings, ctx.sp_typevars, false, ctx.lifted, + ctx.toplevel_pure, ctx.toplevel_stmts, ctx.closure_infos, + ctx.closure_structs), + e), ex) + else + toplevel_stmts = SyntaxList() + ctx2 = ClosureConversionCtx( + ctx.bindings, ctx.mod, + ctx.closure_bindings, ctx.capture_rewriting, ctx.top_bindings, + ctx.lambda_bindings, ctx.sp_typevars, true, ctx.lifted, + ctx.toplevel_pure, toplevel_stmts, ctx.closure_infos, + ctx.closure_structs) + res = mapchildren(e->_convert_closures(ctx2, e), ex) + if isempty(toplevel_stmts) + res + else + @ast ctx ex [K"block" toplevel_stmts... res] + end + end +end + +function _convert_closures(ctx::ClosureConversionCtx, ex) + k = kind(ex) + if k == K"BindingId" + b = get_binding(ctx, ex) + if ctx.lifted && haskey(ctx.sp_typevars, b.id) + binding_ex(ctx, ctx.sp_typevars[b.id]) + else + access = is_self_captured(ctx, ex) ? captured_var_access(ctx, ex) : ex + is_boxed(ctx, ex) ? get_box_contents(ctx, ex, access) : access + end + elseif is_leaf(ex) || k == K"inert" || k == K"syntaxinert" + ex + elseif k == K"=" + convert_assignment(ctx, ex) + elseif k == K"isdefined" + # Convert isdefined expr to function for closure converted variables + var = ex[1] + if kind(var) === K"static_parameter" + ex + elseif (binfo = get_binding(ctx, var); is_boxed(binfo)) + access = is_self_captured(ctx, var) ? captured_var_access(ctx, var) : var + @ast ctx ex [K"call" + "isdefined"::K"core" + access + "contents"::K"Symbol" + ] + elseif binfo.is_always_defined || is_self_captured(ctx, var) + # Captured but unboxed vars are always defined + @ast ctx ex true::K"Bool" + elseif binfo.kind == :global + # Normal isdefined won't work for globals (#56985) + @ast ctx ex [K"call" + "isdefinedglobal"::K"core" + ctx.mod::K"Value" + binfo.name::K"Symbol"] + else + ex + end + elseif k == K"decl" + @jl_assert kind(ex[1]) == K"BindingId" ex + binfo = get_binding(ctx, ex[1]) + if binfo.kind == :global + # flisp has this, but our K"assert" handling is in a previous pass + # [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] + make_globaldecl(ctx, ex, binfo.mod, binfo.name, true, _convert_closures(ctx, ex[2])) + else + newleaf(ex, K"TOMBSTONE") + end + elseif k == K"global" + # Leftover `global` forms become weak globals. + mod, name = if kind(ex[1]) == K"BindingId" + binfo = get_binding(ctx, ex[1]) + @jl_assert binfo.kind == :global ex + binfo.mod, binfo.name + else + # See note about using eval on Expr(:global/:const, GlobalRef(...)) + @jl_assert ex[1].value isa GlobalRef ex[1] + ex[1].value.mod, String(ex[1].value.name) + end + @ast ctx ex [K"unused_only" make_globaldecl(ctx, ex, mod, name, false)] + elseif k == K"local" + var = ex[1] + binfo = get_binding(ctx, var) + if is_boxed(binfo) + @ast ctx ex [K"=" var [K"call" "Box"::K"core"]] + elseif !binfo.is_always_defined + @ast ctx ex [K"newvar" var] + else + newleaf(ex, K"TOMBSTONE") + end + elseif k == K"lambda" || k == K"toplevel_lambda" || k == K"generated_lambda" + @jl_assert false (ex, "lambda should be at top level or in `method`") + elseif k == K"function_decl" + func_name = ex[1] + @jl_assert kind(func_name) == K"BindingId" ex + if haskey(ctx.closure_bindings, closure_key(ctx, func_name)) + convert_local_function_decl(ctx, ex) + else + @ast ctx ex [K"block" [K"method" func_name] (::K"TOMBSTONE")] + end + elseif k == K"method" + @jl_assert ctx.lifted ex + # The method sp svec needs every sp the body and sig capture + cr = ctx.capture_rewriting + sp_ids = IdTag[syntax_id(c) for c in children(ex[3][3])] + if cr isa ClosureInfo + append!(sp_ids, syntax_id(sp) for sp in cr.capt_sp) + end + sort!(sp_ids) + sps = SyntaxList() + for id in sp_ids + push!(sps, binding_ex(ctx, id)) + end + tvs = mapsyntax(c->binding_ex(ctx, ctx.sp_typevars[syntax_id(c)]), sps) + + # rm method table argument if it's a closure id, since it's unnecessary + # and requires the `(= id (new ...))` call to be lifted above the + # method. flisp might be messing up overlays when it does this, since + # it removes all locals, not just closure ids. + mtable = kind(ex[1]) === K"BindingId" && + haskey(ctx.closure_bindings, closure_key(ctx, ex[1])) ? + @ast(ctx, ex[1], (::K"nothing")) : _convert_closures(ctx, ex[1]) + @ast ctx ex [K"method" + mtable + [K"call" "svec"::K"core" + _convert_closures(ctx, ex[2]) + [K"call" "svec"::K"core" tvs...] + (::K"SourceLocation")] + closure_convert_lambda(ctx, ex[3], sps) + ] + elseif k == K"function_type" + func_name = ex[1] + if kind(func_name) == K"BindingId" && get_binding(ctx, func_name).kind === :local + ck = closure_key(ctx, ex[1]) + @jl_assert(haskey(ctx.closure_infos, ck), + (ex, "function_type of local without known closure type")) + ci = ctx.closure_infos[ck] + if isempty(ci.capt_sp) || ci !== ctx.capture_rewriting + ci.type_name + else + # flisp: fix-function-arg-type + tvs = mapsyntax( + sp->binding_ex(ctx, ctx.sp_typevars[syntax_id(sp)]), + ci.capt_sp) + @ast ctx ex [K"call" "apply_type"::K"core" ci.type_name tvs...] + end + else + @ast ctx ex [K"call" TypeEqOf::K"core" _convert_closures(ctx, func_name)] + end + elseif k == K"method_defs" + name = ex[1] + is_closure = kind(name) == K"BindingId" && get_binding(ctx, name).kind === :local + cap_rewrite = is_closure ? ctx.closure_infos[closure_key(ctx, name)] : nothing + ctx2 = ClosureConversionCtx( + ctx.bindings, ctx.mod, + ctx.closure_bindings, cap_rewrite, + ctx.top_bindings, ctx.lambda_bindings, ctx.sp_typevars, + ctx.toplevel, true, ctx.toplevel_pure, ctx.toplevel_stmts, + ctx.closure_infos, ctx.closure_structs) + tvs = map_cl_convert(ctx2, ex[2]) + assign_fname = !is_closure ? nothing : let ck = closure_key(ctx, name) + cl_ssa = get(ctx.closure_structs, ck, nothing) + if cl_ssa === nothing + nothing + else + delete!(ctx.closure_structs, ck) + convert_assignment(ctx, @ast ctx ex [K"=" name cl_ssa]) + end + end + if is_closure && !ctx.toplevel + push!(ctx2.toplevel_stmts, tvs) + push!(ctx2.toplevel_stmts, map_cl_convert(ctx2, ex[3])) + @ast ctx ex [K"block" assign_fname (::K"TOMBSTONE")] + else + @ast ctx ex [K"block" tvs map_cl_convert(ctx2, ex[3]) assign_fname] + end + elseif k == K"no_method_defs" + name = ex[1] + if kind(name) == K"BindingId" && get_binding(ctx, name).kind === :local + ck = closure_key(ctx, name) + cl_ssa = get(ctx.closure_structs, ck, nothing) + if cl_ssa === nothing + @ast ctx ex (::K"TOMBSTONE") + else + delete!(ctx.closure_structs, ck) + convert_assignment(ctx, @ast ctx ex [K"=" name cl_ssa]) + end + else + @ast ctx ex (::K"TOMBSTONE") + end + elseif k == K"_opaque_closure" + ck = closure_key(ctx, ex[1]) + closure_binds = ctx.closure_bindings[ck] + field_syms, field_orig_bindings, field_inds, _field_is_box, capt_sp = + closure_type_fields(ctx, ex, closure_binds, true) + + capture_rewrites = ClosureInfo( + ck, ex #=unused=#, field_syms, field_inds, capt_sp) + ctx2 = ClosureConversionCtx( + ctx.bindings, ctx.mod, + ctx.closure_bindings, capture_rewrites, ctx.top_bindings, + ctx.lambda_bindings, ctx.sp_typevars, false, false, + ctx.toplevel_pure, ctx.toplevel_stmts, ctx.closure_infos, + ctx.closure_structs) + argt = _convert_closures(ctx, ex[2]) + rt_lb = _convert_closures(ctx, ex[3]) + rt_ub = _convert_closures(ctx, ex[4]) + + init_closure_args = SyntaxList() + for id in field_orig_bindings + init_arg = binding_ex(ctx, id) + if is_self_captured(ctx, init_arg) + init_arg = captured_var_access(ctx, init_arg) + end + push!(init_closure_args, init_arg) + end + @ast ctx ex [K"new_opaque_closure" + argt # arg type tuple + rt_lb # return_lower_bound + rt_ub # return_upper_bound + ex[5] # allow_partial + [K"opaque_closure_method" + (::K"nothing") + ex[6] # nargs + ex[7] # is_va + ex[8] # functionloc + closure_convert_lambda(ctx2, ex[9], SyntaxList()) + ] + init_closure_args... + ] + else + map_cl_convert(ctx, ex) + end +end + +function closure_convert_lambda(ctx, ex, sps) + k = kind(ex) + @jl_assert k in KSet"lambda toplevel_lambda generated_lambda" ex + lbs = lambda_bindings(ex[1]) + interpolations = nothing + if isnothing(ctx.capture_rewriting) + # Global method which may capture locals + interpolations = SyntaxList() + cap_rewrite = interpolations + else + cap_rewrite = ctx.capture_rewriting + end + ctx2 = ClosureConversionCtx( + ctx.bindings, ctx.mod, + ctx.closure_bindings, cap_rewrite, ctx.top_bindings, + lbs, ctx.sp_typevars, + k === K"toplevel_lambda", k === K"toplevel_lambda", + ctx.toplevel_pure && k == K"generated_lambda", + ctx.toplevel_stmts, ctx.closure_infos, ctx.closure_structs) + lambda_children = SyntaxList() + push!(lambda_children, ex[1]) + push!(lambda_children, ex[2]) + push!(lambda_children, @ast ctx ex[3] [K"block" sps...]) + + # Add box initializations for arguments which are captured by an inner lambda + body_stmts = SyntaxList() + for arg in children(ex[2]) + kind(arg) != K"Placeholder" || continue + if is_boxed(ctx, arg) + push!(body_stmts, @ast ctx arg [K"=" + arg + [K"call" "Box"::K"core" arg] + ]) + end + end + # Convert body. + input_body_stmts = kind(ex[4]) != K"block" ? ex[4:4] : ex[4][1:end] + for e in input_body_stmts + push!(body_stmts, _convert_closures(ctx2, e)) + end + push!(lambda_children, @ast ctx2 ex[4] [K"block" body_stmts...]) + + if numchildren(ex) > 4 + # Convert return type + @jl_assert numchildren(ex) == 5 ex + push!(lambda_children, _convert_closures(ctx2, ex[5])) + end + + lam = @mknode(ex; children=lambda_children) + if !isnothing(interpolations) && !isempty(interpolations) + @ast ctx ex [K"call" + replace_captured_locals::K"Value" + lam + [K"call" + "svec"::K"core" + interpolations... + ] + ] + else + lam + end +end + + +""" +For each local function decl with closure key `ck`, we: +1. Declare the closure type, populating `closure_infos[ck]` +2. Define all methods +3. Instantiate the closure with `new`, storing it in `closure_structs[ck]`, and + assigning this to the function name + +Also in this pass: +* Deal with typed variables (K"decl") and their assignments +* Deal with const and non-const global assignments +* Lower variables captured by closures into boxes, etc, as necessary + +Invariants: +* This pass must not introduce new K"Identifier" - only K"BindingId". +* Any new binding IDs must be added to the enclosing lambda locals +""" +@fzone "JL: closures" function convert_closures( + ctx::VariableAnalysisContext, ex::SyntaxTree +) + # TODO: ctx.mod is used instead of syntax_module(ex) beyond this point, + # which is dubious + lbs = lambda_bindings(ex[1]) + ctx_out = ClosureConversionCtx(ctx.bindings, ctx.layer.mod, + ctx.closure_bindings, nothing, + lbs, lbs, ctx.sp_typevars, + false, true, true, SyntaxList(), + Dict{ClosureKey,ClosureInfo}(), + Dict{ClosureKey,SyntaxTree}()) + ex_out = closure_convert_lambda(ctx_out, ex, children(ex[3])) + if !isempty(ctx_out.toplevel_stmts) + throw(LoweringError(first(ctx_out.toplevel_stmts), "Top level code was found outside any top level context. `@generated` functions may not contain closures, including `do` syntax and generators/comprehension")) + end + ctx_out, flatten_blocks(ex_out) +end diff --git a/JuliaLowering/src/compat.jl b/JuliaLowering/src/compat.jl new file mode 100644 index 0000000000000..38dcc5f2a2658 --- /dev/null +++ b/JuliaLowering/src/compat.jl @@ -0,0 +1,825 @@ +const JS = JuliaSyntax + +function find_kind(s::String) + out = get(JS._kind_str_to_int, s, nothing) + return isnothing(out) ? nothing : JS.Kind(out) +end + +# flisp: dot-operators +# +# We work from the operator's name here (rather than its `Kind`) because by this +# point operators are represented uniformly as identifier-like names: this code +# also runs on trees converted from `Expr`, where an operator such as `.^` is +# simply the `Symbol` `:.^` with no token or `Kind` to inspect. `Base.isoperator` +# is the same operator-name test already used for `op=` in `est_to_dst` below; +# note we can't look up a `Kind` by name (eg via `find_kind`), since most +# operators no longer have their own kind - they share `K"Operator"`. +function is_dotted_operator(s::AbstractString) + return length(s) >= 2 && + s[1] === '.' && s[2] !== '.' && + Base.isoperator(s[2:end]) +end + +function is_eventually_call(e) + return e isa Expr && (e.head === :call || + e.head in (:escape, :where, :(::)) && is_eventually_call(e.args[1])) +end + +function est_syntax_name(st, default) + kind(st) in KSet"Identifier unknown_head" ? st.value::String : default +end + +function _scavenge_lnn(e::Expr) + e.head in (:macrocall, :quote, :inert) || for a in e.args + a isa LineNumberNode && return a + if a isa Expr + a_out = _scavenge_lnn(a) + a_out isa LineNumberNode && return a_out + end + end + return nothing +end +scavenge_lnn(@nospecialize(e)) = + something(e isa Expr ? _scavenge_lnn(e) : nothing, + LineNumberNode(0, :none)) + +_unescape_lnn(@nospecialize(e)) = + e isa LineNumberNode ? e : + (e isa Expr && + (e.head === :escape || e.head === Symbol("hygienic-scope")) && + length(e.args) > 0) ? _unescape_lnn(e.args[1]) : nothing + +function _get_inner_lnn(e::Expr, default::LineNumberNode) + e.head in (:function, :macro, :module, :(=)) || return default + length(e.args) >= 2 || return default + b = e.args[end] + b isa Expr || return default + b.head === :block || return default + length(b.args) >= 1 || return default + b_lnn = _unescape_lnn(b.args[1]) + return b_lnn isa LineNumberNode ? b_lnn : default +end + +# List of Expr-AST forms that are always converted to some SyntaxTree form and +# never inserted as an opaque `K"Value"`. Note no LineNumberNode, which appears +# unwrapped in a macrocall (possibly generated functions too, TODO check) +isa_lowering_ast_node(@nospecialize(e)) = + e isa Symbol || e isa QuoteNode || e isa Expr || e isa GlobalRef + +function is_expr_value(st::SyntaxTree) + k = kind(st) + return JuliaSyntax.is_literal(k) || k === K"Value" +end + +function expr_to_est(@nospecialize(e), src::SourceAttrType=scavenge_lnn(e)) + _expr_to_est(e, src, false)[1] +end + +# Adding more cases to this function is almost certainly wrong, since this +# operates on arbitrary heads and arguments throughout macro expansion, not +# well-formed syntax after expansion is done. Most of the complexity here is +# LineNumberNode absorption logic: linenodes are always considered provenance if +# unquoted, then removed in certain forms. If `src` is not an linenode, it is +# assumed to be a better provenance source, so linenodes in `e` are not used for +# provenance (but still removed). +function _expr_to_est(@nospecialize(e), src::SourceAttrType, quoted::Bool) + st = if e isa Symbol + newleaf(src, K"Identifier", String(e)) + elseif e isa QuoteNode + cid, _ = _expr_to_est(e.value, src, true) + newnode(src, K"inert", SyntaxList(cid)) + elseif e isa Expr + head_s = string(e.head) + st_k = find_kind(head_s) + src = old_src = src isa LineNumberNode ? _get_inner_lnn(e, src) : src + cs = SyntaxTree[] + rm_linenodes = e.head in (:block, :toplevel) + quoted |= e.head in (:quote, :inert) + for arg in e.args + if rm_linenodes && (lnn = quoted ? arg : _unescape_lnn(arg); + lnn isa LineNumberNode) + src isa LineNumberNode && (src = lnn) + else + cid, src = _expr_to_est(arg, src, quoted) + push!(cs, cid) + end + end + if isnothing(st_k) + @mknode(;kind=K"unknown_head", value=head_s, source=old_src, + children=cs, context=nothing) + else + @mknode(;kind=st_k, source=old_src, children=cs, context=nothing) + end + elseif e isa GlobalRef + # Represent globalref as K"Identifier" with :mod attribute + @mknode(;kind=K"Identifier", source=src, value=string(e.name), + mod=e.mod, context=nothing) + else + # We may want additional special cases for other types where + # `Base.isa_ast_node(e)`, but `K"Value"` should be fine for most, since + # most are produced in or after lowering + if e isa LineNumberNode && src isa LineNumberNode + # linenode outside of block or toplevel + src = e + end + newleaf(src, K"Value", e) + end + @jl_assert isa_lowering_ast_node(e) || is_expr_value(st) st + + return st, src +end + +# @__doc__ is brittle +_is_meta_doc_block(st) = @stm st begin + [K"block" [K"meta" [K"Identifier"]] _] -> syntax_name(st[1][1]) == "doc" + _ -> false +end + +# `suppress_linenodes` is true if `st`'s parent knows `st` is an exception to +# normal linenode rules. It only applies to `st`, and not transitively to its +# children. +function est_to_expr(st::SyntaxTree, suppress_linenodes=false) + k = kind(st) + if kind(st) === K"Identifier" + # @jl_assert scope layer is base + n = Symbol(syntax_name(st)) + mod = st.mod + !isnothing(mod) ? GlobalRef(mod, n) : n + elseif is_leaf(st) && is_expr_value(st) + v = st.value + # Let `st.value isa Symbol` (or other AST node). Since we enforce that + # this is never produced by the reverse Expr->SyntaxTree transformation, + # there is no lonely Expr for which `st` is the only SyntaxTree + # representation. This means we can pick some other expr this + # represents, namely Expr(`(inert ,st.value)) rather than + # Expr(st.value). + isa_lowering_ast_node(v) ? QuoteNode(v) : v + elseif k === K"inert" + QuoteNode(est_to_expr(st[1])) + else + # TODO: should handle post-lowering forms as well + @jl_assert !is_leaf(st) (st, "est_to_expr should only be used pre-desugaring") + # In a partially-expanded or quoted AST, there may be heads with no + # corresponding kind + head = Symbol((k === K"unknown_head" ? syntax_name(st) : untokenize(k))::String) + out = Expr(head) + + # (Move the following assumptions to the docs if they turn out accurate) + # The only mandatory LineNumberNode is the second macrocall argument. + # Other than that, optional linenodes may show up anywhere within: + # - `block`, unless the block is the first child of `for` or `let` + # - `toplevel` + # Macro authors are responsible for handling any linenodes that follow + # the rules above (but the presence of optional linenodes can't be + # counted upon). + need_lnns = head in (:block, :toplevel) && !suppress_linenodes && + !_is_meta_doc_block(st) + for (i, c) in enumerate(children(st)) + need_lnns && push!(out.args, source_location(LineNumberNode, c)) + let suppress_c = i == 1 && (k == K"for" || k == K"let") + push!(out.args, est_to_expr(c, suppress_c)) + end + end + # Add extra linenodes to some blocks for better provenance + if head === :block && length(out.args) == 0 && !suppress_linenodes + push!(out.args, source_location(LineNumberNode, st)) + elseif head in (:module, :function, :macro) && length(out.args) > 0 + let b = out.args[end] + b isa Expr && b.head === :block && pushfirst!( + b.args, source_location(LineNumberNode, st)) + end + elseif head in (:for, :while) && length(out.args) > 0 + let b = out.args[end] + b isa Expr && b.head === :block && push!( + b.args, source_location( + LineNumberNode, sourcefile(st), last_byte(st))) + end + end + out + end +end + +#------------------------------------------------------------------------------- +# EST->DST + +Base.@kwdef struct SyntaxCompatContext <: AbstractLoweringContext + toplevel::Bool=true +end +function with(ctx::SyntaxCompatContext; toplevel=ctx.toplevel) + SyntaxCompatContext(toplevel) +end + +# .op => (. op) +function dst_separate_dotop(ctx, st::SyntaxTree) + k = kind(st) + if k === K"Identifier" + dotop_s = syntax_name(st) + !is_dotted_operator(dotop_s) && return est_to_dst(ctx, st) + op_s = dotop_s[nextind(dotop_s,1):end] + op_leaf = newleaf(st, K"Identifier", op_s) + return @ast _ st [K"." op_leaf] + elseif k === K"Value" && st.value isa GlobalRef && + is_dotted_operator(string(st.value.name)) + @jl_assert false (st, "TODO: handle dotted globalref") + else + return est_to_dst(ctx, st) + end +end + +function dst_eq_to_in(ctx, st::SyntaxTree) + return @stm st begin + [K"filter" cond is...] -> + @ast _ st [K"filter" est_to_dst(ctx, cond) + [K"iteration" map_dst_eq_to_in(ctx, is)...]] + [K"=" l r] -> + @ast _ st [K"in" est_to_dst(ctx, l) est_to_dst(ctx, r)] + end +end +function map_dst_eq_to_in(ctx, sl) + mapsyntax(st->dst_eq_to_in(ctx, st), sl) +end + +function dst_iterspec(ctx, src::SyntaxTree, sl::AbstractVector{SyntaxTree}) + return if length(sl) === 1 && kind(sl[1]) === K"filter" + cond = sl[1][1] + iters = sl[1][2:end] + @ast _ sl[1] [K"filter" + [K"iteration" map_dst_eq_to_in(ctx, iters)...] + est_to_dst(ctx, cond) + ] + else + @ast _ src [K"iteration" map_dst_eq_to_in(ctx, sl)...] + end +end + +function dst_sink_parameters(ctx, sl::AbstractVector{SyntaxTree}) + out = map_est_to_dst(ctx, sl) + if !isempty(out) && kind(out[1]) === K"parameters" + push!(out, popfirst!(out)) + end + return out +end + +function dst_importpath(ctx, st::SyntaxTree) + return @stm st begin + [K"as" p name] -> + @ast _ st [K"as" dst_importpath(ctx, st[1]) est_to_dst(ctx, name)] + [K"." xs...] -> + @ast _ st [K"importpath" map_est_to_dst(ctx, xs)...] + end +end + +_dst_eq_to_kw(st::SyntaxTree) = @stm st begin + [K"=" l r] -> @ast _ st [K"kw" l r] + x -> x +end + +# flisp: tuple-to-arglist. Fix parsing mistakes where anon function arglist is +# parsed as a block instead of a tuple, or uses `=` instead of `kw`. Note +# return type annotations are not possible on an anonymous function. +# (a::T)->1 +# (::T)->1 +# (a...)->1 +# (a=1)->1 +# (a=1;)->1 +# (a=1;b=1)->1 +function _dst_fix_arglist(st::SyntaxTree) + @stm st begin + [K"::" [K"call" _...] _] -> st + [K"call" _...] -> st + [K"tuple" xs...] -> let fixed = mapsyntax(_dst_eq_to_kw, xs) + fixed == xs ? st : @ast _ st [K"tuple" fixed...] + end + [K"where" x tvs...] -> let fixed = _dst_fix_arglist(x) + fixed == x ? st : @ast _ st [K"where" fixed tvs...] + end + [K"block" x1 x2] -> + @ast _ st [K"tuple" _dst_eq_to_kw(x1) + [K"parameters" _dst_eq_to_kw(x2)]] + [K"block" x] -> @ast _ st [K"tuple" _dst_eq_to_kw(x)] + [K"block"] -> @ast _ st [K"tuple"] + [K"block" _...] -> @jl_assert false st + x -> @ast _ st [K"tuple" _dst_eq_to_kw(x)] + end +end + +_is_false(st::SyntaxTree) = kind(st) === K"Value" && st.value === false + +function _expand_literal_pow(st::SyntaxTree) + k = kind(st) + (k in KSet"call dotcall" && + numchildren(st) === 3 && + kind(st[1]) === K"Identifier" && syntax_name(st[1]) === "^" && + kind(st[3]) in KSet"Integer Value" && st[3].value isa Int) || return st + @ast _ st [k + "literal_pow"::K"top" + st[1] st[2] + [K"call" [K"call" "apply_type"::K"core" "Val"::K"top" st[3]]] + ] +end + +function est_to_dst_ident(ctx, st::SyntaxTree) + s = syntax_name(st) + if is_writeonly_est_name(s) + @mknode(st; kind=K"Placeholder") + elseif is_flisp_compat(st) && s === "#self#" && !ctx.toplevel + @mknode(st; kind=K"thisfunction", value=nothing, children=SyntaxList()) + else + st + end +end + +has_if_generated(st::SyntaxTree) = @stm st begin + (_, when=is_leaf(st)||is_quoted(st)) -> false + [K"function" _...] -> false + ([K"=" call _], when=is_eventually_call(call)) -> false + [K"->" _...] -> false + [K"if" [K"generated"] _ _] -> true + _ -> any(has_if_generated, children(st)) +end + +# The (if (generated) gen nongen) form is troublesome because everything +# surrounding it is implicitly quoted (with `gen` interpolated into it), so +# converting the function's AST before proper quoting is incorrect. +split_generated(st::SyntaxTree, gen_part) = @stm st begin + (_, when=is_leaf(st)||is_quoted(st)) -> st + [K"if" [K"generated"] gen nongen] -> if gen_part + @ast(_, st, [K"syntaxunquote" gen]) + else + nongen + end + _ -> mapchildren(x->split_generated(x, gen_part), st) +end + +# Set [no]specialize on a function parameter's identifier. `meta` is a symbol +# if we should set this arg's meta unconditionally, or a map identifier-string +# to symbol if we should only do it for some identifiers (function body >0 arg +# nospecialize), or nothing if we should just recurse to find meta forms. +# Exceptions with unconditional meta: set meta on the tuple for a destructuring +# arg, and the whole expression for (::T). +function apply_arg_meta(st, meta::Union{Nothing, Symbol, Dict{String, Symbol}}) + k = kind(st) + if k == K"Identifier" + if meta isa Symbol + setmeta(st, meta, true) + elseif isnothing(meta) + st + else + sym = get(meta, syntax_name(st), nothing) + !isnothing(sym) ? setmeta(st, sym, true) : st + end + elseif k == K"Placeholder" || k == K"tuple" || k == K"::" && numchildren(st) == 1 + meta isa Symbol ? setmeta(st, meta, true) : st + elseif k == K"..." || k == K"::" || k == K"=" || k == K"kw" + c1 = st[1] + out1 = apply_arg_meta(c1, meta) + c1 == out1 ? st : @ast _ st [k out1 st[2:end]...] + elseif k == K"meta" + # not specified what to do here if we get conflicting + # specialize/nospecialize + meta2 = Symbol(syntax_name(st[1])) + @jl_assert meta2 in (:specialize, :nospecialize) st + apply_arg_meta(st[2], meta2) + elseif k == K"parameters" + mapchildren(x->apply_arg_meta(x, meta), st) + else + @jl_assert false st + end +end + +function apply_arglist_meta(st, meta::Union{Nothing, Symbol, Dict{String, Symbol}}) + @stm st begin + [K"where" x tvs...] -> let fixed = apply_arglist_meta(x, meta) + fixed == x ? st : @ast _ st [K"where" fixed tvs...] + end + [K"::" x t] -> let fixed = apply_arglist_meta(x, meta) + fixed == x ? st : @ast _ st [K"::" fixed t] + end + [K"call" f args...] -> mapchildren(x-> + x == f ? strip_arg_meta(f) : apply_arg_meta(x, meta), st) + [K"tuple" _...] -> mapchildren(x->apply_arg_meta(x, meta), st) + end +end + +# flisp bug; underscore sparams are sometimes readable (see #60626). Should +# return `st` unchanged 99% of the time. +function force_readable_sparams(st) + kind(st) === K"where" && is_flisp_compat(st) || return st + sig, wheres = let (sig0, wheres0) = flatten_wheres(st) + sig0, mapsyntax(typevar_bounds, wheres0) + end + any(w->is_flisp_compat(w) && is_writeonly_est_name(syntax_name(w[1])), + wheres) || return st + + seen = Set{String}() + lt = @ast _ st "<:"::K"Identifier" + for i in eachindex(wheres) + n = wheres[i][1] + n_str = syntax_name(n) + lb = _mangle_writeonly(wheres[i][2], seen) + ub = _mangle_writeonly(wheres[i][3], seen) + is_flisp_compat(n) && is_writeonly_est_name(n_str) && push!(seen, n_str) + wheres[i] = @ast _ st [K"comparison" lb lt _mangle_writeonly(n, seen) lt ub] + end + mangle = args->mapsyntax(a->_mangle_writeonly_argt(a, seen), args) + sig2 = @stm sig begin + [K"::" [K"call" as...] t] -> @ast _ sig [K"::" [K"call" mangle(as)...] t] + [K"call" as...] -> @ast _ sig [K"call" mangle(as)...] + [K"tuple" as...] -> @ast _ sig [K"tuple" mangle(as)...] + end + @ast _ st [K"where" sig2 wheres...] +end +_mangle_writeonly_argt(st, seen) = @stm st begin + [K"parameters" _...] -> mapchildren(c->_mangle_writeonly_argt(c, seen), st) + [K"kw" x v] -> @ast _ st [K"kw" _mangle_writeonly_argt(x, seen) v] + [K"=" x v] -> @ast _ st [K"=" _mangle_writeonly_argt(x, seen) v] + [K"..." x] -> @ast _ st [K"..." _mangle_writeonly_argt(x, seen)] + [K"::" x t] -> @ast _ st [K"::" x _mangle_writeonly(t, seen)] + [K"::" t] -> @ast _ st [K"::" _mangle_writeonly(t, seen)] + [K"overlay" mt x] -> @ast _ st [K"overlay" mt _mangle_writeonly(x, seen)] + _ -> st +end +function _mangle_writeonly(st, seen) + k = kind(st) + if k === K"Identifier" && isnothing(st.mod) && is_flisp_compat(st) + n = syntax_name(st) + !(n in seen) ? st : @ast _ st (string(n, "FIXME#60626")::K"Identifier") + elseif is_leaf(st) || is_quoted(st) || k === K"->" || k === K"function" + st + else + mapchildren(c->_mangle_writeonly(c, seen), st) + end +end + +function _note_32026_hack!(st, expansion_sc::SyntaxContext) + k = kind(st) + if st.context.layer === expansion_sc.layer && + (k === K"function" || k === K"=" && is_eventually_call(st[1])) + setmeta!(st, :resolved_global_function_name, true) + end + st +end +_apply_32026_hack(st, sc::SyntaxContext) = @stm st begin + ([K"Identifier"], when=st.mod===nothing && st.context.layer===sc.layer) -> + @mknode(st; mod=sc.layer.mod) + [K"call" x args...] -> kind(x) === K"::" ? st : + @ast _ st [K"call" _apply_32026_hack(x, sc) args...] + [K"where" x args...] -> @ast _ st [K"where" _apply_32026_hack(x, sc) args...] + [K"::" x t] -> @ast _ st [K"::" _apply_32026_hack(x, sc) t] + [K"curly" x args...] -> @ast _ st [K"curly" _apply_32026_hack(x, sc) args...] + x -> x +end +function apply_32026_hack(st, orig) + is_flisp_compat(st) || return st + getmeta(orig, :resolved_global_function_name, false) || return st + @jl_assert is_flisp_compat(orig) orig + _apply_32026_hack(st, orig.context::SyntaxContext) +end + +function collect_body_meta(st) + argmeta_all = nothing + argmeta = nothing + mmetas = nothing + for c in children(st) + kind(c) === K"meta" || continue + spec = numchildren(c) >= 1 && kind(c[1]) === K"Identifier" ? + syntax_name(c[1]) : "" + if spec in ("specialize", "nospecialize") + meta = Symbol(spec) + if numchildren(c) == 1 + isnothing(argmeta_all) && (argmeta_all = meta) + else + isnothing(argmeta) && (argmeta = Dict{String, Symbol}()) + for id in c[2:end] + kind(id) === K"Identifier" && (argmeta[syntax_name(id)] = meta) + end + end + else + for m in children(c) + isnothing(mmetas) && (mmetas = SyntaxList()) + km = kind(m) + if km === K"purity" + push!(mmetas, m) + elseif kind(m) === K"Identifier" && syntax_name(m) in ( + "inline", "noinline", "propagate_inbounds", + "nospecializeinfer", "aggressive_constprop", "no_constprop") + push!(mmetas, @mknode(m; kind=K"Symbol")) + end + end + end + end + (isnothing(argmeta_all) ? argmeta : argmeta_all), mmetas +end + +# Absorb `meta` nodes from arguments and the function body into syntax `.meta` +# for easier desugaring. +function _dst_function_body(ctx, st, r, method_metas) + r2 = if has_if_generated(r) + gen, nongen = split_generated(r, true), split_generated(r, false) + @ast _ st [K"_generated_body" [K"syntaxquote" gen] est_to_dst(ctx, nongen)] + else + est_to_dst(ctx, r) + end + isnothing(method_metas) ? r2 : setmeta(r2, :method_metas, method_metas) +end + +function dst_raw_lambda(ctx, st, sps) + argl = map(x->expr_to_est(x::Symbol, st[1]), st[1].value::Vector) + @ast _ st [K"lambda" [K"block" argl...] [K"block" sps...] + est_to_dst(with(ctx; toplevel=false), st[2])] +end + +""" +Convert the Expr-like tree (EST) coming from macro expansion to the tree +desugaring expects (DST), where some forms have SyntaxNode structure and others +have Expr structure. + +We may drop cases from this conversion, for example, if... + +- syntax evolution changes a form in Expr and EST to use DST structure, so the + input we receive here is "already done" + +- desugaring changes to accept the EST form instead of the DST one, so we can + leave our input unchanged + +We can assume `st` has passed `valid_st1`. Errors arising from invalid AST +(including finding `macrocall/escape/quote` forms) should be handled there. +""" +function est_to_dst(ctx::SyntaxCompatContext, st::SyntaxTree) + rec = var"#self#" + return @stm st begin + [K"Identifier"] -> est_to_dst_ident(ctx, st) + [K"Value"] -> st.value === nothing ? newleaf(st, K"nothing") : st + (_, when=is_leaf(st)) -> st + ([K"unknown_head" l r], + when=(s=syntax_name(st); Base.isoperator(s))) -> let + (op_s, out_k) = s[1] === '.' ? + (s[nextind(s,1):prevind(s,end)], K".op=") : + (s[1:prevind(s,end)], K"op=") + + op_leaf = newleaf(st, K"Identifier", op_s) + @ast _ st [out_k rec(ctx, l) op_leaf rec(ctx, r)] + end + [K"comparison" cs0...] -> let cs = copy(cs0) + for (i, c) in enumerate(cs) + cs[i] = iseven(i) ? dst_separate_dotop(ctx, cs[i]) : rec(ctx, cs[i]) + end + @mknode(st; children=cs) + end + [K"'" x] -> + @ast _ st [K"call" "'"::K"Identifier"(st) rec(ctx, x)] + [K"." f [K"tuple" args...]] -> _expand_literal_pow( + @ast _ st [K"dotcall" rec(ctx, f) dst_sink_parameters(ctx, args)...]) + ([K"inert" [K"Identifier"]], when=isnothing(st[1].mod)) -> + @ast _ st st[1]=>K"Symbol" + [K"syntaxinert" _] -> st + [K"inert" _] -> st + [K"module" _...] -> st + [K"toplevel" _...] -> st + [K"for" [K"=" _ _] body] -> + @ast _ st [K"for" [K"iteration"(st[1]) dst_eq_to_in(ctx, st[1])] rec(ctx, body)] + [K"for" [K"block" iters...] body] -> + @ast _ st [K"for" + [K"iteration"(st[1]) map_dst_eq_to_in(ctx, iters)...] + rec(ctx, body) + ] + (_, when=(k = kind(st); k in KSet"tuple vect braces")) -> + @ast _ st [k dst_sink_parameters(ctx, children(st))...] + (_, when=(k = kind(st); k in KSet"curly ref")) -> + @ast _ st [k dst_separate_dotop(ctx, st[1]) + dst_sink_parameters(ctx, children(st)[2:end])...] + # tuple arg should not be converted or desugared + [K"foreigncall" [K"tuple" _...] args...] -> + @ast _ st [K"foreigncall" [K"foreignsymbol" st[1]] args...] + [K"foreignglobal" [K"tuple" _...]] -> + @ast _ st [K"foreignglobal" [K"foreignsymbol" st[1]]] + ([K"call" [K"Identifier"] sym args...], + when=(syntax_name(st[1]) === "ccall" || + syntax_name(st[1]) === "cglobal")) -> if kind(sym) === K"tuple" + @ast _ st [K"call" st[1] [K"foreignsymbol" st[2]] map_est_to_dst(ctx, args)...] + else + @ast _ st [K"call" st[1] rec(ctx, sym) map_est_to_dst(ctx, args)...] + end + [K"call" f args...] -> let + out_k, out_f = @stm dst_separate_dotop(ctx, f) begin + [K"." op] -> (K"dotcall", op) + f_sep -> (K"call", f_sep) + end + out = @ast _ st [out_k + out_f dst_sink_parameters(ctx, children(st)[2:end])... + ] + _expand_literal_pow(out) + end + [K"try" tryb cvar catchb rest...] -> let + has_catch = !(_is_false(cvar) && _is_false(catchb)) + cvar_out = _is_false(cvar) ? + newleaf(cvar, K"Placeholder") : rec(ctx, cvar) + has_finally = length(rest) >= 1 && !_is_false(rest[1]) + has_else = length(rest) === 2 + @ast _ st [K"try" rec(ctx, tryb) + has_catch ? [K"catch"(catchb) cvar_out rec(ctx, catchb)] : nothing + has_else ? [K"else"(rest[2]) rec(ctx, rest[2])] : nothing + has_finally ? [K"finally"(rest[1]) rec(ctx, rest[1])] : nothing + ] + end + [K"flatten" _] -> let + out_iters = SyntaxList() + next = st + while kind(next) === K"flatten" + push!(out_iters, dst_iterspec(ctx, next, next[1][2:end])) + next = next[1][1] + end + @jl_assert kind(next) === K"generator" st next + push!(out_iters, dst_iterspec(ctx, next, next[2:end])) + @ast _ st [K"generator" rec(ctx, next[1]) out_iters...] + end + [K"comprehension" xs...] -> let + arg = rec(ctx, length(xs) == 1 ? xs[1] : + @ast _ st [K"generator" children(st)...]) + if kind(arg) === K"generator" + @ast _ st [K"comprehension" arg] + else + @ast _ st [K"call" "collect"::K"top" arg] + end + end + [K"typed_comprehension" t0 g] -> let + t = rec(t0) + arg = rec(ctx, g) + if kind(arg) === K"generator" + @ast _ st [K"typed_comprehension" t arg] + else + @ast _ st [K"call" "collect"::K"top" t arg] + end + end + # hack: `[_ for _ in rhs]`, `[f(_) for _ in rhs]` works + ([K"generator" body [K"=" u2 rhs]], + when=is_flisp_compat(st) && + is_writeonly_est_name(est_syntax_name(u2, "")) && begin + u2name=est_syntax_name(u2, "") + func = @stm body begin + ([K"call" func [K"Identifier"]], + when=est_syntax_name(body[2], "")===u2name && + !is_dotted_operator(est_syntax_name(body[1], ""))) -> func + ([K"Identifier"], + when=est_syntax_name(body, "")===u2name) -> + @ast _ st ("identity"::K"top") + _ -> nothing + end + func !== nothing + end) -> @ast _ st [K"call" "Generator"::K"top" rec(ctx, func) rec(ctx, rhs)] + [K"generator" body iters...] -> + @ast _ st [K"generator" rec(ctx, body) dst_iterspec(ctx, st, iters)] + ([K"=" l r], when=(is_eventually_call(l))) -> let + f_ctx = with(ctx; toplevel=false) + # no fix_arglist needed, since this func can't be anonymous + arg_meta, method_metas = collect_body_meta(r) + l = force_readable_sparams(apply_arglist_meta(l, arg_meta)) + l = apply_32026_hack(l, st) + @ast _ st [K"function" + rec(ctx, l) + _dst_function_body(f_ctx, st, r, method_metas)] + end + [K"function" [K"Identifier"]] -> + @ast _ st [K"function" apply_32026_hack(st[1], st)] + [K"function" l r] -> let + f_ctx = with(ctx; toplevel=false) + arg_meta, method_metas = collect_body_meta(r) + l = force_readable_sparams( + apply_arglist_meta(_dst_fix_arglist(l), arg_meta)) + l = apply_32026_hack(l, st) + @ast _ st [K"function" + rec(ctx, l) + _dst_function_body(f_ctx, st, r, method_metas)] + end + [K"->" l r] -> let + f_ctx = with(ctx; toplevel=false) + arg_meta, method_metas = collect_body_meta(r) + l = force_readable_sparams( + apply_arglist_meta(_dst_fix_arglist(l), arg_meta)) + @ast _ st [K"->" + rec(ctx, l) + _dst_function_body(f_ctx, st, r, method_metas)] + end + [K"macro" l r] -> let + arg_meta, method_metas = collect_body_meta(r) + r2 = rec(with(ctx; toplevel=false), r) + isnothing(method_metas) || (r2 = setmeta(r2, :method_metas, method_metas)) + @ast _ st [K"macro" rec(ctx, apply_arglist_meta(l, arg_meta)) r2] + end + [K"do" [K"call" f args...] lam] -> let + @ast _ st [K"call" rec(ctx, f) rec(ctx, lam) dst_sink_parameters(ctx, args)...] + end + ([K"let" binds body], when=(kind(binds) !== K"block")) -> + @ast _ st [K"let" [K"block"(binds) rec(ctx, binds)] rec(ctx, body)] + (_, when=(kind(st) in KSet"using import")) -> let + # dot_importpath = (. _...) + # as_or_dotip = dot_importpath | (as dot_importpath name) + # replaces dot_importpath with (importpath _...) in + # (using as_or_dotip...) + # (using (: as_or_dotip as_or_dotip...)) + paths, maybe_colon = @stm st[1] begin + [K":" paths...] -> (paths, st[1]) + _ -> (children(st), nothing) + end + out_cs = mapsyntax(st->dst_importpath(ctx, st), paths) + if !isnothing(maybe_colon) + out_c1 = @ast _ maybe_colon [K":" out_cs...] + out_cs = SyntaxList(out_c1) + end + @mknode(st; children=out_cs) + end + # flisp macro expansion treated const as local, so names got mangled + # throughout the thunk. JL uses locals for this, so strip const. + ([K"const" [K"=" l r]], when=ctx.toplevel && is_flisp_compat(l) && + !is_base_layer(l.context::SyntaxContext)) -> + @ast _ st [K"=" rec(l) rec(r)] + + #----------------------------------------------------------------------- + # Heads not emitted from parsing + ([K"meta" s vs...], + when=(meta=est_syntax_name(s, ""); + !ctx.toplevel && meta in ("nospecialize", "specialize"))) -> + # Should be handled in the function case + newleaf(st, K"nothing") + ([K"meta" s gen], when=est_syntax_name(s, "") === "generated") -> + @ast _ st [K"meta" @mknode(s; kind=K"Symbol") rec(ctx, gen)] + [K"meta" syms...] -> + @ast _ st [K"meta" mapsyntax( + s->(kind(s) === K"Identifier" ? @mknode(s; kind=K"Symbol") : s), + syms)... + ] + [K"boundscheck" x] -> @mknode(st; children=SyntaxList()) + [K"inbounds" [K"Identifier"]] -> newnode(st, K"inbounds_pop", SyntaxList()) + [K"core" x] -> newleaf(st, K"core", syntax_name(x)) + [K"top" x] -> newleaf(st, K"top", syntax_name(x)) + [K"static_parameter" x] -> newleaf(st, K"static_parameter", x.value::IdTag) + [K"with-static-parameters" lam sps...] -> + dst_raw_lambda(ctx, lam, sps) + [K"lambda" _ _] -> dst_raw_lambda(ctx, st, SyntaxTree[]) + [K"copyast" [K"inert" ex]] -> @ast _ st [K"call" + interpolate_expr::K"Value" + [K"inert"(st[1]) ex] + ] + [K"symbolicgoto" lab] -> + @mknode(st; value=syntax_name(lab), children=nothing) + [K"symboliclabel" lab] -> + @mknode(st; value=syntax_name(lab), children=nothing) + [K"symbolicblock" id body] -> let s = syntax_name(id) + if is_writeonly_est_name(s) + @ast _ st [K"symbolicblock" id=>K"Placeholder" rec(ctx, body)] + else + @ast _ st [K"symbolicblock" id=>K"symboliclabel" rec(ctx, body)] + end + end + [K"unknown_head" cs...] -> let head = syntax_name(st) + if head === "latestworld-if-toplevel" + newleaf(st, K"latestworld_if_toplevel") + elseif head === "scope-block" + @ast _ st [K"scope_block" [K"neutral_scope"] + map_est_to_dst(ctx, cs)...] + else + @jl_assert(false, (st, string( + "unknown expr head (corresponding to no kind) between", + " macro-expansion and desugaring: "))) + end + end + ([K"latestworld"], when=!is_leaf(st)) -> newleaf(st, K"latestworld") + [K"cfunction" typ fptr rt at sym] -> let + # A symbol in fptr[1] does not observe hygiene or local scopes, but + # treating this as a binding is better for e.g. JETLS. + out_fptr = if kind(fptr) == K"inert" && numchildren(fptr) == 1 && + kind(fptr[1]) == K"Identifier" + sc = fptr[1].context::SyntaxContext + ident = @mknode(fptr[1]; mod=base_layer(sc).mod) + @ast _ fptr [K"static_eval"(fptr) ident] + else + rec(ctx, fptr) + end + @ast _ st [K"cfunction" + rec(ctx, typ) out_fptr + [K"static_eval"(rt; meta=name_hint("cfunction return type")) rec(ctx, rt)] + [K"static_eval"(at; meta=name_hint("cfunction argument type")) rec(ctx, at)] + rec(ctx, sym) + ] + end + + # avoid creating excess nodes + _ -> let out_cs = map_est_to_dst(ctx, children(st)) + out_cs == children(st) ? st : @mknode(st; children=out_cs) + end + end +end +function map_est_to_dst(ctx, sl) + mapsyntax(st->est_to_dst(ctx, st), sl) +end + +est_to_dst(st) = est_to_dst(SyntaxCompatContext(), st) + +#------------------------------------------------------------------------------- +# misc + +function purity_expr_to_flags(st::SyntaxTree) + @jl_assert kind(st) === K"purity" st + args = Bool[x.value for x in children(st)] + Base.encode_effects_override(Base.EffectsOverride(args...)) +end diff --git a/JuliaLowering/src/desugaring.jl b/JuliaLowering/src/desugaring.jl new file mode 100644 index 0000000000000..2adf04a183384 --- /dev/null +++ b/JuliaLowering/src/desugaring.jl @@ -0,0 +1,4589 @@ +# Lowering Pass 2 - syntax desugaring + +mutable struct DesugaringContext <: AbstractLoweringContext + const layer::ScopeLayer + const bindings::Bindings + const ssa_mapping::Dict{Int, IdTag} + const world::UInt +end + +# Translate a K"ssavalue" node from pre-lowered code into a normal SSA binding. +# Uses ctx.ssa_mapping to ensure the same external SSA id maps to the same binding. +function _resolve_ssavalue(ctx::DesugaringContext, ex) + binding_id = get!(ctx.ssa_mapping, ex[1].value) do + syntax_id(ssavar(ctx, ex)) + end + binding_ex(ctx, binding_id) +end + +# Return true when `x` and `y` are "the same identifier", but also works with +# bindings (and hence ssa vars). See also `is_identifier_like()` +function is_same_identifier_like(ex::SyntaxTree, y::SyntaxTree) + return (kind(ex) == K"Identifier" && kind(y) == K"Identifier" && NameKey(ex) == NameKey(y)) || + (kind(ex) == K"BindingId" && kind(y) == K"BindingId" && syntax_id(ex) == syntax_id(y)) +end + +function is_same_identifier_like(ex::SyntaxTree, name::AbstractString) + return kind(ex) == K"Identifier" && syntax_name(ex) == name +end + +# Hack. Scopes aren't resolved, so only use this where a false positive is +# still a correct answer. +function contains_identifier(ex::SyntaxTree, idents::AbstractVector{<:SyntaxTree}) + contains_unquoted(ex) do e + any(is_same_identifier_like(e, id) for id in idents) + end +end + +function contains_identifier(ex::SyntaxTree, idents...) + contains_unquoted(ex) do e + any(is_same_identifier_like(e, id) for id in idents) + end +end + +# Return true if `f(e)` is true for any unquoted child of `ex`, recursively. +function contains_unquoted(f::Function, ex::SyntaxTree) + if f(ex) + return true + elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert syntaxinert meta") + return any(contains_unquoted(f, e) for e in children(ex)) + else + return false + end +end + +# Identify some expressions that are safe to repeat +# +# TODO: Can we use this in more places? +function is_effect_free(ex) + k = kind(ex) + # TODO: metas + is_literal(k) || is_identifier_like(ex) || k == K"Symbol" || + k == K"inert" || k == K"syntaxinert" || k == K"top" || + k == K"core" || k == K"Value" || k == K"nothing" + # flisp also includes `a.b` with simple `a`, but this seems like a bug + # because this calls the user-defined getproperty? +end + +function check_no_parameters(ex::SyntaxTree, msg) + i = find_parameters_ind(children(ex)) + if i > 0 + throw(LoweringError(ex[i], msg)) + end +end + +function check_no_assignment(exs, msg="misplaced assignment statement in `[ ... ]`") + i = findfirst(kind(e) == K"=" || kind(e) == K"kw" for e in exs) + if !isnothing(i) + throw(LoweringError(exs[i], msg)) + end +end + +function new_internal_context(st::SyntaxTree) + sc_orig = st.context::SyntaxContext + SyntaxContext( + ScopeLayer(syntax_module(st), nothing), + # macro provenance: could use nothing, but this is easier for consumers + sc_orig.unexpanded, + # version: internal bindings are only used in syntax we create, so the + # version should be the latest one + JL_NEW_SYNTAX_VERSION, + true) +end + +# Generating a new_local_binding or ssaval should only be done if we can +# guarantee there's some scope it's declared in, and that it's not declared or +# used outside of that scope (binding capture is OK). This is the alternative. +function newsym(ctx, src::SyntaxTree, name::String; unused=false) + kind = unused ? K"Placeholder" : K"Identifier" + out = @mknode(; kind, source=src, value=name, children=nothing, + meta=src.meta, context=new_internal_context(src)) +end + +# In an flisp-compatible expansion, some explicit global declarations (and any +# initialization in the same expression) are unhygienic; they are declared in +# the macrocall module (unless wrapped in a top-level form). This is buggy +# (references in the same scope don't resolve to it, op-equal assignments don't +# work, etc.), but compatible. flisp: `unescape`, `unescape-global-lhs`. TODO: +# It would be cleaner to do this in compat.jl. +function relayer_global_if_unhygienic(ctx, st::SyntaxTree) + sc = st.context::SyntaxContext + relayered = SyntaxList() + # TODO: is_base_layer(sc) or sc.layer == ctx.layer? + (!is_flisp_compat(sc) || is_base_layer(sc)) && return st, relayered + sc2 = escape_layer(sc, true) + return _relayer_global_if_unhygienic(relayered, st, sc2), relayered +end +function _relayer_global_if_unhygienic(done::SyntaxList, st::SyntaxTree, sc::SyntaxContext) + k = kind(st) + if k === K"Identifier" && is_flisp_compat(st) && st.context::SyntaxContext !== sc + push!(done, st) + @mknode(st; context=sc) + elseif k === K"::" || k === K"kw" + n_done = length(done) + lhs = _relayer_global_if_unhygienic(done, st[1], sc) + n_done == length(done) ? st : (@ast _ st [k lhs st[2]]) + elseif k === K"tuple" || k === K"parameters" + mapchildren(e->_relayer_global_if_unhygienic(done, e, sc), st) + else + st + end +end + +#------------------------------------------------------------------------------- +# Destructuring + +# Convert things like `(x,y,z) = (a,b,c)` to assignments, eliminating the +# tuple. Includes support for slurping/splatting. This function assumes that +# `_tuple_sides_match` returns true, so the following have already been +# checked: +# * There's max one `...` on the left hand side +# * There's max one `...` on the right hand side, in the last place, or +# matched with an lhs... in the last place. (required so that +# pairwise-matching terms from the right is valid) +# * Neither side has any key=val terms or parameter blocks +# +# Tuple elimination must act /as if/ the right hand side tuple was first +# constructed followed by destructuring. In particular, any side effects due to +# evaluating the individual terms in the right hand side tuple must happen in +# order. +function tuple_to_assignments(ctx, ex, is_const) + lhs = ex[1] + rhs = ex[2] + wrap(asgn) = is_const ? (@ast ctx ex [K"const" asgn]) : asgn + + # Tuple elimination aims to turn assignments between tuples into lists of assignments. + # + # However, there's a complex interplay of side effects due to the + # individual assignments and these can be surprisingly complicated to + # model. For example `(x[i], y) = (f(), g)` can contain the following + # surprises: + # * `tmp = f()` calls `f` which might throw, or modify the bindings for + # `x` or `y`. + # * `x[i] = tmp` is lowered to `setindex!` which might throw or modify the + # bindings for `x` or `y`. + # * `g` might throw an `UndefVarError` + # + # Thus for correctness we introduce temporaries for all right hand sides + # with observable side effects and ensure they're evaluated in order. + n_lhs = numchildren(lhs) + n_rhs = numchildren(rhs) + stmts = SyntaxList() + rhs_tmps = SyntaxList() + for i in 1:n_rhs + rh = rhs[i] + r = if kind(rh) == K"..." + rh[1] + else + rh + end + k = kind(r) + if is_literal(k) || k == K"Symbol" || k == K"inert" || + k == K"syntaxinert" || k == K"top" || k == K"core" || + k == K"Value" + # Effect-free and nothrow right hand sides do not need a temporary + # (we require nothrow because the order of rhs terms is observable + # due to sequencing, thus identifiers are not allowed) + else + # Example rhs which need a temporary + # * `f()` - arbitrary side effects to any binding + # * `z` - might throw UndefVarError + tmp = emit_assign_tmp(stmts, ctx, r) + rh = kind(rh) == K"..." ? @ast(ctx, rh, [K"..." tmp]) : tmp + end + push!(rhs_tmps, rh) + end + + il = 0 + ir = 0 + while il < n_lhs + il += 1 + ir += 1 + lh = lhs[il] + if kind(lh) == K"..." + # Exactly one lhs `...` occurs in the middle somewhere, with a + # general rhs which has at least as many non-`...` terms or one + # `...` term at the end. + # Examples: + # (x, ys..., z) = (a, b, c, d) + # (x, ys..., z) = (a, bs...) + # (xs..., y) = (a, bs...) + # (xs...) = (a, b, c) + # in this case we can pairwise-match arguments from the end + # backward and emit a general tuple assignment for the middle. + jl = n_lhs + jr = n_rhs + while jl > il && jr > ir + if kind(lhs[jl]) == K"..." || kind(rhs_tmps[jr]) == K"..." + break + end + jl -= 1 + jr -= 1 + end + middle = emit_assign_tmp(stmts, ctx, + @ast(ctx, rhs, [K"tuple" rhs_tmps[ir:jr]...]), + "rhs_tmp" + ) + if il == jl + # (x, ys...) = (a,b,c) + # (x, ys...) = (a,bs...) + # (ys...) = () + push!(stmts, wrap(@ast ctx ex [K"=" lh[1] middle])) + else + # (x, ys..., z) = (a, b, c, d) + # (x, ys..., z) = (a, bs...) + # (xs..., y) = (a, bs...) + push!(stmts, wrap(@ast ctx ex [K"=" [K"tuple" lhs[il:jl]...] middle])) + end + # Continue with the remainder of the list of non-splat terms + il = jl + ir = jr + else + rh = rhs_tmps[ir] + if kind(rh) == K"..." + push!(stmts, wrap(@ast ctx ex [K"=" [K"tuple" lhs[il:end]...] rh[1]])) + break + else + push!(stmts, wrap(@ast ctx ex [K"=" lh rh])) + end + end + end + + @ast ctx ex [K"block" + stmts... + [K"removable" [K"tuple" rhs_tmps...]] + ] +end + +# Create an assignment `$lhs = $rhs` where `lhs` must be "simple". If `rhs` is +# a block, sink the assignment into the last statement of the block to keep +# more expressions at top level. `rhs` should already be expanded. +# +# flisp: sink-assignment +function sink_assignment(ctx, srcref, lhs, rhs) + @jl_assert is_identifier_like(lhs) lhs + if kind(rhs) == K"block" && numchildren(rhs) > 0 + @ast ctx srcref [K"block" + rhs[1:end-1]... + [K"=" lhs rhs[end]] + ] + else + @ast ctx srcref [K"=" lhs rhs] + end +end + +function _tuple_sides_match(lhs, rhs) + N = max(length(lhs), length(rhs)) + for i = 1:N+1 + if i > length(lhs) + # (x, y) = (a, b) # match + # (x,) = (a, b) # no match + return i > length(rhs) + elseif kind(lhs[i]) == K"..." + # (x, ys..., z) = (a, b) # match + # (x, ys...) = (a,) # match + return true + elseif i > length(rhs) + # (x, y) = (a,) # no match + # (x, y, zs...) = (a,) # no match + return false + elseif kind(rhs[i]) == K"..." + # (x, y) = (as...,) # match + # (x, y, z) = (a, bs...) # match + # (x, y) = (as..., b) # no match + return i == length(rhs) + end + end +end + +# Lower `(lhss...) = rhs` in contexts where `rhs` must be a tuple at runtime +# by assuming that `getfield(rhs, i)` works and is efficient. +function lower_tuple_assignment(ctx, assignment_srcref, lhss, rhs) + stmts = SyntaxList() + tmp = emit_assign_tmp(stmts, ctx, rhs, "rhs_tmp") + for (i, lh) in enumerate(lhss) + push!(stmts, @ast ctx assignment_srcref [K"=" + lh + [K"call" "getfield"::K"core" tmp i::K"Integer"] + ]) + end + newnode(assignment_srcref, K"block", stmts) +end + +# Implement destructuring with `lhs` a tuple expression (possibly with +# slurping) and `rhs` a general expression. +# +# Destructuring in this context is done via the iteration interface, though +# calls `Base.indexed_iterate()` to allow for a fast path in cases where the +# right hand side is directly indexable. +function _destructure(ctx, assignment_srcref, stmts, lhs, rhs, is_const) + n_lhs = numchildren(lhs) + iterstate = n_lhs > 0 ? new_local_binding(ctx, rhs, "iterstate") : nothing + + end_stmts = SyntaxList() + wrap(asgn) = is_const ? (@ast ctx assignment_srcref [K"const" asgn]) : asgn + + i = 0 + for lh in children(lhs) + i += 1 + if kind(lh) == K"..." + lh1 = if is_identifier_like(lh[1]) && !is_const + lh[1] + else + lhs_tmp = ssavar(ctx, lh[1], "lhs_tmp") + push!(end_stmts, expand_forms_2(ctx, wrap(@ast ctx lh[1] [K"=" lh[1] lhs_tmp]))) + lhs_tmp + end + if i == n_lhs + # Slurping as last lhs, eg, for `zs` in + # (x, y, zs...) = rhs + if kind(lh1) != K"Placeholder" + push!(stmts, expand_forms_2(ctx, + @ast ctx assignment_srcref [K"=" + lh1 + [K"call" + "rest"::K"top" + rhs + if i > 1 + iterstate + end + ] + ] + )) + end + else + # Slurping before last lhs. Eg, for `xs` in + # (xs..., y, z) = rhs + # For this we call + # (xs, tail) = Base.split_rest(...) + # then continue iteration with `tail` as new rhs. + tail = ssavar(ctx, lh, "tail") + push!(stmts, + expand_forms_2(ctx, + lower_tuple_assignment(ctx, + assignment_srcref, + (lh1, tail), + @ast ctx assignment_srcref [K"call" + "split_rest"::K"top" + rhs + (n_lhs - i)::K"Integer" + if i > 1 + iterstate + end + ] + ) + ) + ) + rhs = tail + n_lhs = n_lhs - i + i = 0 + end + else + # Normal case, eg, for `y` in + # (x, y, z) = rhs + lh1 = if is_identifier_like(lh) && !is_const + lh + # elseif is_eventually_call(lh) (TODO??) + else + lhs_tmp = ssavar(ctx, lh, "lhs_tmp") + push!(end_stmts, expand_forms_2(ctx, wrap(@ast ctx lh [K"=" lh lhs_tmp]))) + lhs_tmp + end + push!(stmts, + expand_forms_2(ctx, + lower_tuple_assignment(ctx, + assignment_srcref, + i == n_lhs ? (lh1,) : (lh1, iterstate), + @ast ctx assignment_srcref [K"call" + "indexed_iterate"::K"top" + rhs + i::K"Integer" + if i > 1 + iterstate + end + ] + ) + ) + ) + end + end + # Actual assignments must happen after the whole iterator is destructured + # (https://github.com/JuliaLang/julia/issues/40574) + append!(stmts, end_stmts) + stmts +end + +# Expands cases of property destructuring +function expand_property_destruct(ctx, ex) + @jl_assert numchildren(ex) == 2 ex + lhs = ex[1] + @jl_assert kind(lhs) == K"tuple" ex + if numchildren(lhs) != 1 + throw(LoweringError(lhs, "Property destructuring must use a single `;` before the property names, eg `(; a, b) = rhs`")) + end + params = lhs[1] + @jl_assert kind(params) == K"parameters" ex + rhs = ex[2] + stmts = SyntaxList() + rhs1 = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, rhs)) + for prop in children(params) + propname = kind(prop) == K"Identifier" ? prop : + kind(prop) == K"::" && kind(prop[1]) == K"Identifier" ? prop[1] : + throw(LoweringError(prop, "invalid assignment location")) + push!(stmts, expand_forms_2(ctx, @ast ctx rhs1 [K"=" + prop + [K"call" + "getproperty"::K"top" + rhs1 + propname=>K"Symbol" + ] + ])) + end + push!(stmts, @ast ctx rhs1 [K"removable" rhs1]) + newnode(ex, K"block", stmts) +end + +# Expands all cases of general tuple destructuring, eg +# (x,y) = (a,b) +function expand_tuple_destruct(ctx, ex, is_const) + lhs = ex[1] + @jl_assert kind(lhs) == K"tuple" ex + rhs = ex[2] + + num_slurp = 0 + for lh in children(lhs) + num_slurp += (kind(lh) == K"...") + if num_slurp > 1 + throw(LoweringError(lh, "multiple `...` in destructuring assignment are ambiguous")) + end + end + + if kind(rhs) == K"tuple" + num_splat = sum(kind(rh) == K"..." for rh in children(rhs); init=0) + if num_splat == 0 && (numchildren(lhs) - num_slurp) > numchildren(rhs) + throw(LoweringError(ex, "More variables on left hand side than right hand in tuple assignment")) + end + + if !any_assignment(children(rhs)) && !has_parameters(rhs) && + _tuple_sides_match(children(lhs), children(rhs)) + return expand_forms_2(ctx, tuple_to_assignments(ctx, ex, is_const)) + end + end + + stmts = SyntaxList() + rhs1 = if is_ssa(ctx, rhs) || + (is_identifier_like(rhs) && + !any(is_same_identifier_like(kind(l) == K"..." ? l[1] : l, rhs) + for l in children(lhs))) + rhs + else + emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, rhs)) + end + _destructure(ctx, ex, stmts, lhs, rhs1, is_const) + push!(stmts, @ast ctx rhs1 [K"removable" rhs1]) + newnode(ex, K"block", stmts) +end + +#------------------------------------------------------------------------------- +# Expand comparison chains + +function expand_scalar_compare_chain(ctx, srcref, terms, i) + comparisons = nothing + while i + 2 <= length(terms) + lhs = terms[i] + op = terms[i+1] + rhs = terms[i+2] + if kind(op) == K"." && numchildren(op) == 1 + break + end + comp = @ast ctx op [K"call" + op + lhs + rhs + ] + if isnothing(comparisons) + comparisons = comp + else + comparisons = @ast ctx srcref [K"&&" + comparisons + comp + ] + end + i += 2 + end + (comparisons, i) +end + +# Expanding comparison chains: (comparison a op b op c ...) +# +# We use && to combine pairs of adjacent scalar comparisons and .& to combine +# vector-vector and vector-scalar comparisons. Combining scalar comparisons are +# treated as having higher precedence than vector comparisons, thus: +# +# a < b < c ==> (a < b) && (b < c) +# a .< b .< c ==> (a .< b) .& (b .< c) +# a < b < c .< d .< e ==> (a < b && b < c) .& (c .< d) .& (d .< e) +# a .< b .< c < d < e ==> (a .< b) .& (b .< c) .& (c < d && d < e) +function expand_compare_chain(ctx, ex) + @jl_assert kind(ex) == K"comparison" ex + terms = children(ex) + @jl_assert numchildren(ex) >= 3 ex + @jl_assert isodd(numchildren(ex)) ex + i = 1 + comparisons = nothing + # Combine any number of dotted comparisons + while i + 2 <= length(terms) + if !(kind(terms[i+1]) == K"." && numchildren(terms[i+1]) == 1) + (comp, i) = expand_scalar_compare_chain(ctx, ex, terms, i) + else + lhs = terms[i] + op = terms[i+1] + rhs = terms[i+2] + i += 2 + comp = @ast ctx op [K"dotcall" + op[1] + lhs + rhs + ] + end + if isnothing(comparisons) + comparisons = comp + else + comparisons = @ast ctx ex [K"dotcall" + "&"::K"top" + # ^^ NB: Flisp bug. Flisp lowering essentially does + # adopt_scope("&"::K"Identifier", ctx.mod) + # here which seems wrong if the comparison chain arose from + # a macro in a different module. One fix would be to use + # adopt_scope("&"::K"Identifier", ex) + # to get the module of the comparison expression for the + # `&` operator. But a simpler option is probably to always + # use `Base.&` so we do that. + comparisons + comp + ] + end + end + comparisons +end + +#------------------------------------------------------------------------------- +# Expansion of array indexing +function _arg_to_temp(ctx, stmts, ex) + k = kind(ex) + if is_effect_free(ex) + ex + elseif k == K"..." + @ast ctx ex [k _arg_to_temp(ctx, stmts, ex[1])] + elseif k == K"kw" + @ast ctx ex [K"kw" ex[1] _arg_to_temp(ctx, stmts, ex[2])] + elseif k == K"parameters" + mapchildren(ex) do e + _arg_to_temp(ctx, stmts, e) + end + else + emit_assign_tmp(stmts, ctx, ex) + end +end + +# Make the *arguments* of an expression safe for multiple evaluation, for +# example +# +# a[f(x)] => (temp=f(x); a[temp]) +# +# Any assignments are added to `stmts` and a result expression returned which +# may be used in further desugaring. +function remove_argument_side_effects(ctx, stmts, ex) + if is_literal(ex) || is_identifier_like(ex) + ex + else + k = kind(ex) + if k == K"let" + emit_assign_tmp(stmts, ctx, ex) + else + args = SyntaxList() + for e in children(ex) + push!(args, _arg_to_temp(ctx, stmts, e)) + end + # TODO: Copy attributes? + @ast ctx ex [k args...] + end + end +end + +# Replace any `begin` or `end` symbols with an expression indexing the array +# `arr` in the `n`th index. `splats` are a list of the splatted arguments that +# precede index `n` `is_last` is true when this is this +# last index +function replace_beginend(ctx, ex, arr, n, splats, is_last) + k = kind(ex) + if k == K"Identifier" && syntax_name(ex) in ("begin", "end") + indexfunc = @ast ctx ex (syntax_name(ex) == "begin" ? "firstindex" : "lastindex")::K"top" + if length(splats) == 0 + if is_last && n == 1 + @ast ctx ex [K"call" indexfunc arr] + else + @ast ctx ex [K"call" indexfunc arr n::K"Integer"] + end + else + splat_lengths = SyntaxList() + for splat in splats + push!(splat_lengths, @ast ctx ex [K"call" "length"::K"top" splat]) + end + @ast ctx ex [K"call" + indexfunc + arr + [K"call" + "+"::K"top" + (n - length(splats))::K"Integer" + splat_lengths... + ] + ] + end + elseif is_leaf(ex) || is_quoted(ex) + ex + elseif k == K"ref" + # inside ref, only replace within the first argument + @ast ctx ex [k + replace_beginend(ctx, ex[1], arr, n, splats, is_last) + ex[2:end]... + ] + elseif k == K"kw" + # note from flisp + # TODO: this probably should not be allowed since keyword args aren't + # positional, but in this context we have just used their positions anyway + @ast ctx ex [K"kw" ex[1] replace_beginend(ctx, ex[2], arr, n, splats, is_last)] + else + mapchildren(e->replace_beginend(ctx, e, arr, n, splats, is_last), ex) + end +end + +# Go through indices and replace the `begin` or `end` symbol +# `arr` - array being indexed +# `idxs` - list of indices +# returns the expanded indices. Any statements that need to execute first are +# added to ctx.stmts. +function process_indices(sctx::StatementListCtx, arr, idxs) + has_splats = any(kind(i) == K"..." for i in idxs) + idxs_out = SyntaxList() + splats = SyntaxList() + for (n, idx0) in enumerate(idxs) + is_splat = kind(idx0) == K"..." + val = replace_beginend(sctx, is_splat ? idx0[1] : idx0, + arr, n, splats, n == length(idxs)) + idx = kind(val) === K"kw" || !has_splats || is_simple_atom(sctx, val) ? + val : emit_assign_tmp(sctx, val) + if is_splat + push!(splats, idx) + end + push!(idxs_out, is_splat ? @ast(sctx, idx0, [K"..." idx]) : idx) + end + return idxs_out +end + +# Expand things like `f()[i,end]`, add to `sctx.stmts` (temporaries for +# computing indices) and return +# * `arr` - The array (may be a temporary ssa value) +# * `idxs` - List of indices +function expand_ref_components(sctx::StatementListCtx, ex) + check_no_parameters(ex, "unexpected semicolon in array expression") + @jl_assert kind(ex) == K"ref" ex + @jl_assert numchildren(ex) >= 1 ex + arr = ex[1] + idxs = ex[2:end] + if any(contains_identifier(e, "begin", "end") for e in idxs) + arr = emit_assign_tmp(sctx, arr) + end + new_idxs = process_indices(sctx, arr, idxs) + return (arr, new_idxs) +end + +function expand_setindex(ctx, ex) + @jl_assert kind(ex) == K"=" && numchildren(ex) == 2 ex + lhs = ex[1] + sctx = with_stmts(ctx) + (arr, idxs) = expand_ref_components(sctx, lhs) + rhs = emit_assign_tmp(sctx, ex[2]) + @ast ctx ex [K"block" + sctx.stmts... + expand_forms_2(ctx, [K"call" + "setindex!"::K"top" + arr + rhs + idxs... + ]) + [K"removable" rhs] + ] +end + +#------------------------------------------------------------------------------- +# Expansion of broadcast notation `f.(x .+ y)` + +function expand_dotcall(ctx, ex) + k = kind(ex) + if k == K"dotcall" + @jl_assert numchildren(ex) >= 1 ex + farg = setmeta(ex[1], :is_called, true) + args = SyntaxList() + append!(args, ex[2:end]) + kws = remove_kw_args!(ctx, args) + @ast ctx ex [K"call" + (isnothing(kws) ? "broadcasted" : "broadcasted_kwsyntax")::K"top" + farg # todo: What about (z=f).(x,y) ? + (expand_dotcall(ctx, arg) for arg in args)... + if !isnothing(kws) + [K"parameters" + kws... + ] + end + ] + elseif k == K"comparison" + expand_dotcall(ctx, expand_compare_chain(ctx, ex)) + elseif k == K".&&" || k == K".||" + @ast ctx ex [K"call" + "broadcasted"::K"top" + (k == K".&&" ? "andand" : "oror")::K"top" + (expand_dotcall(ctx, arg) for arg in children(ex))... + ] + else + ex + end +end + +function expand_fuse_broadcast(ctx, ex) + if kind(ex) == K".=" || kind(ex) == K".op=" + @jl_assert numchildren(ex) == 2 ex + lhs = ex[1] + kl = kind(lhs) + rhs = expand_dotcall(ctx, ex[2]) + @ast ctx ex [K"block" + dest := if kl == K"ref" + sctx = with_stmts(ctx) + (arr, idxs) = expand_ref_components(sctx, lhs) + [K"block" + sctx.stmts... + [K"call" + "dotview"::K"top" + arr + idxs... + ] + ] + elseif kl == K"." && numchildren(lhs) == 2 + [K"call" + "dotgetproperty"::K"top" + children(lhs)... + ] + else + lhs + end + bc := if !(kind(rhs) == K"call" && kind(rhs[1]) == K"top" && syntax_name(rhs[1]) == "broadcasted") + # Ensure the rhs of .= is always wrapped in a call to `broadcasted()` + [K"call"(rhs) + "broadcasted"::K"top" + "identity"::K"top" + rhs + ] + else + rhs + end + [K"call" "materialize!"::K"top" dest bc] + dest + ] + else + @ast ctx ex [K"call" + "materialize"::K"top" + expand_dotcall(ctx, ex) + ] + end +end + +#------------------------------------------------------------------------------- +# Expansion of generators and comprehensions + +# Return any subexpression which is a 'return` statement, not including any +# inside quoted sections or method bodies. +function find_return(ex::SyntaxTree) + if kind(ex) == K"return" + return ex + elseif !is_leaf(ex) && !(kind(ex) in KSet"quote inert syntaxinert meta function ->") + for e in children(ex) + r = find_return(e) + if !isnothing(r) + return r + end + end + else + return nothing + end +end + +function check_no_return(ex) + r = find_return(ex) + if !isnothing(r) + throw(LoweringError(r, "`return` not allowed inside comprehension or generator")) + end +end + +function lhs_local_defs(ctx, lhs) + defs = SyntaxList() + foreach_lhs_name(lhs) do var + push!(defs, @ast ctx var [K"local" var]) + end + return defs +end + +# Return the anonymous function taking an iterated value, for use with the +# first argument to `Base.Generator` +function func_for_generator(ctx, body, iter_value_destructuring) + if is_same_identifier_like(iter_value_destructuring, body) + # Use Base.identity for generators which are filters such as + # `(x for x in xs if f(x))`. This avoids creating a new type. + @ast ctx body "identity"::K"top" + elseif !is_identifier_like(iter_value_destructuring) + # compat: arg::T should convert, not assert, and duplicated arg is OK + arg = newsym(ctx, iter_value_destructuring, "#generator#") + @ast ctx body [K"->" + [K"tuple" arg] + [K"block" + lhs_local_defs(ctx, iter_value_destructuring)... + [K"=" iter_value_destructuring arg] + body]] + else + @ast ctx body [K"->" [K"tuple" iter_value_destructuring] [K"block" body]] + end +end + +function expand_generator(ctx, ex) + @jl_assert numchildren(ex) >= 2 ex + body = ex[1] + check_no_return(body) + if numchildren(ex) > 2 + outervar_assignments = SyntaxList() + for iterspecs in ex[2:end-1] + for iterspec in children(iterspecs) + foreach_lhs_name(iterspec[1]) do var + @jl_assert kind(var) == K"Identifier" ex # Todo: K"BindingId"? + push!(outervar_assignments, @ast ctx var [K"=" var var]) + end + end + end + body = @ast ctx ex [K"let" + [K"block" + outervar_assignments... + ] + [K"block" + body + ] + ] + end + for iterspecs_ind in numchildren(ex):-1:2 + iterspecs = ex[iterspecs_ind] + filter_test = nothing + if kind(iterspecs) == K"filter" + filter_test = iterspecs[2] + iterspecs = iterspecs[1] + end + if kind(iterspecs) != K"iteration" + throw(LoweringError(ex, """Expected `K"iteration"` iteration specification in generator""")) + end + iter_ranges = SyntaxList() + iter_lhss = SyntaxList() + for iterspec in children(iterspecs) + @jl_assert kind(iterspec) == K"in" iterspec + @jl_assert numchildren(iterspec) == 2 iterspec + push!(iter_lhss, iterspec[1]) + push!(iter_ranges, iterspec[2]) + end + iter_value_destructuring = if numchildren(iterspecs) == 1 + iterspecs[1][1] + else + iter_lhss = SyntaxList() + for iterspec in children(iterspecs) + push!(iter_lhss, iterspec[1]) + end + @ast ctx iterspecs [K"tuple" iter_lhss...] + end + iter = if length(iter_ranges) > 1 + @ast ctx iterspecs [K"call" + "product"::K"top" + iter_ranges... + ] + else + iter_ranges[1] + end + if !isnothing(filter_test) + iter = @ast ctx ex [K"call" + "Filter"::K"top" + func_for_generator(ctx, filter_test, iter_value_destructuring) + iter + ] + end + body = @ast ctx ex [K"call" + "Generator"::K"top" + func_for_generator(ctx, body, iter_value_destructuring) + iter + ] + if iterspecs_ind < numchildren(ex) + body = @ast ctx ex [K"call" + "Flatten"::K"top" + body + ] + end + end + body +end + +function expand_comprehension_to_loops(ctx, ex) + @jl_assert kind(ex) == K"typed_comprehension" ex + element_type = ex[1] + gen = ex[2] + @jl_assert kind(gen) == K"generator" ex + body = gen[1] + check_no_return(body) + # TODO: check_no_break_continue + iterspecs = gen[2] + @jl_assert kind(iterspecs) == K"iteration" ex + new_iterspecs = SyntaxList() + iters = SyntaxList() + iter_defs = SyntaxList() + for iterspec in children(iterspecs) + iter = emit_assign_tmp(iter_defs, ctx, iterspec[2], "iter") + push!(iters, iter) + push!(new_iterspecs, @ast ctx iterspec [K"in" iterspec[1] iter]) + end + # Lower to nested for loops + idx = new_local_binding(ctx, iterspecs, "idx") + @ast ctx ex [K"block" + iter_defs... + full_iter := if length(iters) == 1 + iters[1] + else + [K"call" + "product"::K"top" + iters... + ] + end + iter_size := [K"call" "IteratorSize"::K"top" full_iter] + size_unknown := [K"call" "isa"::K"core" iter_size "SizeUnknown"::K"top"] + result := [K"call" "_array_for"::K"top" element_type full_iter iter_size] + [K"=" idx [K"call" "first"::K"top" [K"call" "LinearIndices"::K"top" result]]] + [K"for" [K"iteration" Iterators.reverse(new_iterspecs)...] + [K"block" + val := body + # TODO: inbounds setindex + [K"if" size_unknown + [K"call" "push!"::K"top" result val] + [K"call" "setindex!"::K"top" result val idx] + ] + #[K"call" "println"::K"top" [K"call" "typeof"::K"core" idx]] + [K"=" idx [K"call" "add_int"::K"top" idx 1::K"Integer"]] + ] + ] + result + ] +end + +# Mimics native lowerer's tuple-wrap function (julia-syntax.scm:2723-2736) +# Unwraps only ONE layer of `...` and wraps sequences of non-splat args in tuples. +# Example: `[a, b, xs..., c]` -> `[tuple(a, b), xs, tuple(c)]` +function _wrap_unsplatted_args(ctx, call_ex, args) + result = SyntaxList() + non_splat_run = SyntaxList() + for arg in args + if kind(arg) == K"..." + # Flush any accumulated non-splat args + if !isempty(non_splat_run) + push!(result, @ast ctx call_ex [K"call" "tuple"::K"core" non_splat_run...]) + non_splat_run = SyntaxList() + end + # Unwrap only ONE layer of `...` (corresponds to (cadr x) in native lowerer) + push!(result, arg[1]) + else + # Accumulate non-splat args + push!(non_splat_run, arg) + end + end + # Flush any remaining non-splat args + if !isempty(non_splat_run) + push!(result, @ast ctx call_ex [K"call" "tuple"::K"core" non_splat_run...]) + end + result +end + +function expand_splat(ctx, ex, topfunc, args) + # Matches native lowerer's algorithm + # https://github.com/JuliaLang/julia/blob/f362f47338de099cdeeb1b2d81b3ec1948443274/src/julia-syntax.scm#L2761-2762: + # 1. Unwrap one layer of `...` from each argument (via _wrap_unsplatted_args) + # 2. Create `_apply_iterate(iterate, f, wrapped_args...)` WITHOUT expanding args yet + # 3. Recursively expand the entire call - if any wrapped_arg still contains `...`, + # the recursive expansion will handle it, naturally building nested structure + # + # Example: tuple((xs...)...) recursion: + # Pass 1: unwrap outer `...` -> _apply_iterate(iterate, tuple, (xs...)) + # Pass 2: expand sees (xs...) in call context, unwraps again + # -> _apply_iterate(iterate, _apply_iterate, tuple(iterate, tuple), xs) + + wrapped_args = _wrap_unsplatted_args(ctx, ex, args) + + # Construct the unevaluated _apply_iterate call + result = @ast ctx ex [K"call" + "_apply_iterate"::K"core" + "iterate"::K"top" + topfunc + wrapped_args... + ] + + # Recursively expand the entire call (matching native's expand-forms) + return expand_forms_2(ctx, result) +end + +function expand_array(ctx, ex, topfunc) + args = children(ex) + check_no_assignment(args) + topfunc = @ast ctx ex topfunc::K"top" + if any(kind(arg) == K"..." for arg in args) + expand_splat(ctx, ex, topfunc, args) + else + @ast ctx ex [K"call" + topfunc + expand_forms_2(ctx, args)... + ] + end +end + +#------------------------------------------------------------------------------- +# Expansion of array concatenation notation `[a b ; c d]` etc + +function expand_vcat(ctx, ex) + check_no_parameters(ex, "unexpected semicolon in array expression") + check_no_assignment(children(ex)) + had_row = false + had_row_splat = false + is_typed = kind(ex) == K"typed_vcat" + eltype = is_typed ? ex[1] : nothing + elements = is_typed ? ex[2:end] : ex[1:end] + for e in elements + k = kind(e) + if k == K"row" + had_row = true + had_row_splat = had_row_splat || any(kind(e1) == K"..." for e1 in children(e)) + end + end + if had_row_splat + # In case there is splatting inside `hvcat`, collect each row as a + # separate tuple and pass those to `hvcat_rows` instead (ref #38844) + rows = SyntaxList() + for e in elements + if kind(e) == K"row" + push!(rows, @ast ctx e [K"tuple" children(e)...]) + else + push!(rows, @ast ctx e [K"tuple" e]) + end + end + fname = is_typed ? "typed_hvcat_rows" : "hvcat_rows" + @ast ctx ex [K"call" + fname::K"top" + eltype + rows... + ] + else + row_sizes = SyntaxList() + flat_elems = SyntaxList() + for e in elements + if kind(e) == K"row" + rowsize = numchildren(e) + append!(flat_elems, children(e)) + else + rowsize = 1 + push!(flat_elems, e) + end + push!(row_sizes, @ast ctx e rowsize::K"Integer") + end + if had_row + fname = is_typed ? "typed_hvcat" : "hvcat" + @ast ctx ex [K"call" + fname::K"top" + eltype + [K"tuple" row_sizes...] + flat_elems... + ] + else + fname = is_typed ? "typed_vcat" : "vcat" + @ast ctx ex [K"call" + fname::K"top" + eltype + flat_elems... + ] + end + end +end + +function ncat_contains_row(ex) + k = kind(ex) + if k == K"row" + return true + elseif k == K"nrow" + return any(ncat_contains_row(e) for e in children(ex)) + else + return false + end +end + +# flip first and second dimension for row major layouts +function nrow_flipdim(row_major, d) + return !row_major ? d : + d == 1 ? 2 : + d == 2 ? 1 : d +end + +function flatten_ncat_rows!(flat_elems, nrow_spans, row_major, parent_layout_dim, ex) + # Note that most of the checks for valid nesting here are also checked in + # the parser - they can only fail when nrcat is constructed + # programmatically (eg, by a macro). + k = kind(ex) + if k == K"row" + layout_dim = 1 + elems = children(ex) + parent_layout_dim != 1 || throw(LoweringError(ex,"Badly nested rows in `ncat`")) + elseif k == K"nrow" + dim = ex[1].value::Int + elems = children(ex)[2:end] + dim > 0 || throw(LoweringError(ex,"Unsupported dimension $dim in ncat")) + !row_major || dim != 2 || throw(LoweringError(ex,"2D `nrow` cannot be mixed with `row` in `ncat`")) + layout_dim = nrow_flipdim(row_major, dim) + elseif kind(ex) == K"..." + throw(LoweringError(ex, "Splatting ... in an `ncat` with multiple dimensions is not supported")) + else + push!(flat_elems, ex) + for ld in parent_layout_dim-1:-1:1 + push!(nrow_spans, (ld, 1)) + end + return + end + row_start = length(flat_elems) + parent_layout_dim > layout_dim || throw(LoweringError(ex, "Badly nested rows in `ncat`")) + for e in elems + if layout_dim == 1 + kind(e) ∉ KSet"nrow row" || throw(LoweringError(e,"Badly nested rows in `ncat`")) + end + flatten_ncat_rows!(flat_elems, nrow_spans, row_major, layout_dim, e) + end + n_elems_in_row = length(flat_elems) - row_start + for ld in parent_layout_dim-1:-1:layout_dim + push!(nrow_spans, (ld, n_elems_in_row)) + end +end + +# ncat comes in various layouts which we need to lower to special cases +# - one dimensional along some dimension +# - balanced column first or row first +# - ragged column first or row first +function expand_ncat(ctx, ex) + is_typed = kind(ex) == K"typed_ncat" + eltype = is_typed ? ex[1] : nothing + outer_dim = is_typed ? ex[2].value::Int : ex[1].value::Int + elements = is_typed ? ex[3:end] : ex[2:end] + hvncat_name = is_typed ? "typed_hvncat" : "hvncat" + @jl_assert outer_dim > 0 (ex,"Unsupported dimension in ncat") + if !any(kind(e) in KSet"row nrow" for e in elements) + # One-dimensional ncat along some dimension + # [a ;;; b ;;; c] + return @ast ctx ex [K"call" + hvncat_name::K"top" + eltype + outer_dim::K"Integer" + elements... + ] + end + # N-dimensional case. May be + # * column first or row first: + # [a;b ;;; c;d] + # [a b ;;; c d] + # * balanced or ragged: + # [a ; b ;;; c ; d] + # [a ; b ;;; c] + row_major = any(ncat_contains_row, elements) + @jl_assert !row_major || outer_dim != 2 (ex,"2D `nrow` cannot be mixed with `row` in `ncat`") + flat_elems = SyntaxList() + # `ncat` syntax nests lower dimensional `nrow` inside higher dimensional + # ones (with the exception of K"row" when `row_major` is true). Each nrow + # spans a number of elements and we first extract that. + nrow_spans = Vector{Tuple{Int,Int}}() + for e in elements + flatten_ncat_rows!(flat_elems, nrow_spans, row_major, + nrow_flipdim(row_major, outer_dim), e) + end + push!(nrow_spans, (outer_dim, length(flat_elems))) + # Construct the shape specification by postprocessing the flat list of + # spans. + sort!(nrow_spans, by=first) # depends on a stable sort + is_balanced = true + i = 1 + dim_lengths = zeros(Int, outer_dim) + prev_dimspan = 1 + while i <= length(nrow_spans) + layout_dim, dimspan = nrow_spans[i] + while i <= length(nrow_spans) && nrow_spans[i][1] == layout_dim + if dimspan != nrow_spans[i][2] + is_balanced = false + break + end + i += 1 + end + is_balanced || break + @jl_assert dimspan % prev_dimspan == 0 ex + dim_lengths[layout_dim] = Int(dimspan ÷ prev_dimspan) + prev_dimspan = dimspan + end + shape_spec = SyntaxList() + if is_balanced + if row_major + dim_lengths[1], dim_lengths[2] = dim_lengths[2], dim_lengths[1] + end + # For balanced concatenations, the shape is specified by the length + # along each dimension. + for dl in dim_lengths + push!(shape_spec, @ast ctx ex dl::K"Integer") + end + else + # For unbalanced/ragged concatenations, the shape is specified by the + # number of elements in each N-dimensional slice of the array, from layout + # dimension 1 to N. See the documentation for `hvncat` for details. + i = 1 + while i <= length(nrow_spans) + groups_for_dim = Int[] + layout_dim = nrow_spans[i][1] + while i <= length(nrow_spans) && nrow_spans[i][1] == layout_dim + push!(groups_for_dim, nrow_spans[i][2]) + i += 1 + end + push!(shape_spec, + @ast ctx ex [K"tuple" + [i::K"Integer" for i in groups_for_dim]... + ] + ) + end + end + @ast ctx ex [K"call" + hvncat_name::K"top" + eltype + [K"tuple" shape_spec...] + row_major::K"Bool" + flat_elems... + ] +end + +#------------------------------------------------------------------------------- +# Expand assignments + +# Expand UnionAll definitions, eg `X{T} = Y{T,T}` +function expand_unionall_def(ctx, srcref, lhs, rhs, is_const=true) + if numchildren(lhs) <= 1 + throw(LoweringError(lhs, "empty type parameter list in type alias")) + end + name = lhs[1] + expand_forms_2( + ctx, + @ast ctx srcref [K"block" + rr := [K"where" rhs lhs[2:end]...] + [is_const ? K"constdecl" : K"assign_or_constdecl_if_global" name rr] + [K"removable" rr] + ] + ) +end + +# Expand general assignment syntax, including +# * UnionAll definitions +# * Chained assignments +# * Setting of structure fields +# * Assignments to array elements +# * Destructuring +# * Typed variable declarations +function expand_assignment(ctx, ex, is_const=false) + @jl_assert numchildren(ex) == 2 ex + lhs = ex[1] + rhs = ex[2] + kl = kind(lhs) + if kl == K"curly" + expand_unionall_def(ctx, ex, lhs, rhs, is_const) + elseif kind(rhs) == K"=" + # Expand chains of assignments + # a = b = rhs ==> rr=rhs; b=rr; a=rr + stmts = SyntaxList() + rhs_end = rhs; while kind(rhs_end) === K"=" + rhs_end = rhs_end[2] + end + if !is_identifier_like(rhs_end) + rr = ssavar(ctx, rhs_end, "rhs") + assign_rr = @ast ctx rhs_end [K"=" rr rhs_end] + else + rr = rhs_end + assign_rr = nothing + end + ex_i = ex; while kind(ex_i) === K"=" + push!(stmts, @ast ctx ex_i [K"=" ex_i[1] rr]) + ex_i = ex_i[2] + end + # In const a = b = c, only a is const + is_const && (stmts[1] = @mknode(stmts[1]; kind=K"constdecl")) + + out = @ast ctx ex [K"block" assign_rr reverse!(stmts)... [K"removable" rr]] + expand_forms_2(ctx, out) + elseif kl == K"ssavalue" + sink_assignment(ctx, ex, _resolve_ssavalue(ctx, lhs), expand_forms_2(ctx, rhs)) + elseif is_identifier_like(lhs) + if is_const + rr = ssavar(ctx, ex) + @ast ctx ex [K"block" + sink_assignment(ctx, ex, rr, expand_forms_2(ctx, rhs)) + [K"constdecl" lhs rr] + [K"removable" rr] + ] + else + sink_assignment(ctx, ex, lhs, expand_forms_2(ctx, rhs)) + end + elseif kl == K"." + # a.b = rhs ==> setproperty!(a, :b, rhs) + @jl_assert !is_const (ex, "cannot declare `.` form const") + @jl_assert numchildren(lhs) == 2 lhs + a = lhs[1] + b = lhs[2] + stmts = SyntaxList() + # TODO: Do we need these first two temporaries? + if !is_identifier_like(a) + a = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, a), "a_tmp") + end + if kind(b) != K"Symbol" + b = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, b), "b_tmp") + end + if !is_identifier_like(rhs) && !is_literal(rhs) + rhs = emit_assign_tmp(stmts, ctx, expand_forms_2(ctx, rhs), "rhs_tmp") + end + @ast ctx ex [K"block" + stmts... + [K"call" "setproperty!"::K"top" a b rhs] + [K"removable" rhs] + ] + elseif kl == K"tuple" + if has_parameters(lhs) + expand_property_destruct(ctx, ex) + else + expand_tuple_destruct(ctx, ex, is_const) + end + elseif kl == K"ref" + # a[i1, i2] = rhs + @jl_assert !is_const (ex, "cannot declare ref form const") + expand_forms_2(ctx, expand_setindex(ctx, ex)) + elseif kl == K"::" && numchildren(lhs) == 2 + x = lhs[1] + T = lhs[2] + res = if is_const + expand_forms_2(ctx, @ast ctx ex [K"const" + [K"=" + lhs[1] + kind(lhs[1]) === K"Placeholder" ? rhs : + convert_for_type_decl(ctx, ex, rhs, T, true) + ]]) + elseif is_identifier_like(x) + # Identifier in lhs[1] is a variable type declaration, eg + # x::T = rhs + @ast ctx ex [K"block" + if kind(x) !== K"Placeholder" + [K"decl" x T] + end + [K"=" x rhs] + ] + else + # Otherwise just a type assertion, eg + # a[i]::T = rhs ==> (a[i]::T; a[i] = rhs) + # a[f(x)]::T = rhs ==> (tmp = f(x); a[tmp]::T; a[tmp] = rhs) + stmts = SyntaxList() + l1 = remove_argument_side_effects(ctx, stmts, lhs[1]) + # TODO: What about (f(z),y)::T = rhs? That's broken syntax and + # needs to be detected somewhere but won't be detected here. Maybe + # it shows that remove_argument_side_effects() is not the ideal + # solution here? + # TODO: handle underscore? + @ast ctx ex [K"block" + stmts... + [K"::" l1 lhs[2]] + [K"=" l1 rhs] + ] + end + expand_forms_2(ctx, res) + elseif kl == K"dotcall" + throw(LoweringError(lhs, "invalid dot call syntax on left hand side of assignment")) + elseif kl == K"typed_hcat" + throw(LoweringError(lhs, "invalid spacing in left side of indexed assignment")) + elseif kl == K"typed_vcat" || kl == K"typed_ncat" + throw(LoweringError(lhs, "unexpected `;` in left side of indexed assignment")) + elseif kl == K"vect" || kl == K"hcat" || kl == K"vcat" || kl == K"ncat" + throw(LoweringError(lhs, "use `(a, b) = ...` to assign multiple values")) + else + throw(LoweringError(lhs, "invalid assignment location")) + end +end + +function expand_update_operator(ctx, ex) + k = kind(ex) + dotted = k == K".op=" + + @jl_assert numchildren(ex) == 3 ex + lhs = ex[1] + op = ex[2] + rhs = ex[3] + + stmts = SyntaxList() + + declT = nothing + if kind(lhs) == K"::" + # eg `a[i]::T += 1` + declT = lhs[2] + decl_lhs = lhs + lhs = lhs[1] + end + + if kind(lhs) == K"ref" + # eg `a[end] = rhs` + sctx = with_stmts(ctx, stmts) + (arr, idxs) = expand_ref_components(sctx, lhs) + lhs = @ast ctx lhs [K"ref" arr idxs...] + end + + lhs = remove_argument_side_effects(ctx, stmts, lhs) + + if dotted + if !(kind(lhs) == K"ref" || (kind(lhs) == K"." && numchildren(lhs) == 2)) + # `f() .+= rhs` + lhs = emit_assign_tmp(stmts, ctx, lhs) + end + else + if kind(lhs) == K"tuple" && contains_unquoted( + e->kind(e) == K"BindingId" && get_binding(ctx, e).is_ssa, + lhs) + # If remove_argument_side_effects needed to replace an expression + # with an ssavalue, then it can't be updated by assignment + # (JuliaLang/julia#30062) + throw(LoweringError(lhs, "invalid multiple assignment location")) + end + end + + @ast ctx ex [K"block" + stmts... + [(dotted ? K".=" : K"=") + lhs + [(dotted ? K"dotcall" : K"call") + op + if isnothing(declT) + lhs + else + [K"::"(decl_lhs) lhs declT] + end + rhs + ] + ] + ] +end + +#------------------------------------------------------------------------------- +# Expand logical conditional statements + +# Flatten nested && or || nodes and expand their children +function expand_cond_children(ctx, ex, cond_kind=kind(ex), flat_children=SyntaxList()) + for e in children(ex) + if kind(e) == cond_kind + expand_cond_children(ctx, e, cond_kind, flat_children) + else + push!(flat_children, expand_forms_2(ctx, e)) + end + end + flat_children +end + +# Expand condition in, eg, `if` or `while` +function expand_condition(ctx, ex) + isblock = kind(ex) == K"block" && numchildren(ex) >= 1 + test = isblock ? ex[end] : ex + k = kind(test) + if k == K"&&" || k == K"||" + # `||` and `&&` get special lowering so that they compile directly to + # jumps rather than first computing a bool and then jumping. + cs = expand_cond_children(ctx, test) + test = isempty(cs) ? (@ast ctx ex (k === K"&&")::K"Bool") : + length(cs) == 1 ? (@ast ctx ex cs[1]) : + newnode(test, k, cs) + else + test = expand_forms_2(ctx, test) + end + if isblock + # Special handling so that the rules for `&&` and `||` can be applied + # to the last statement of a block + @ast ctx ex [K"block" mapsyntax(e->expand_forms_2(ctx,e), ex[1:end-1])... test] + else + test + end +end + +#------------------------------------------------------------------------------- +# Expand let blocks + +function expand_let(ctx, ex) + scope_type = numchildren(ex) == 3 && kind(ex[3]) === K"neutral_scope" ? + ex[3] : @ast ctx ex [K"hard_scope"] + @jl_assert numchildren(ex) == 2 || kind(scope_type) === K"neutral_scope" ex + bindings = ex[1] + @jl_assert kind(bindings) == K"block" bindings + blk = ex[2] + if numchildren(bindings) == 0 + return @ast ctx ex [K"scope_block" scope_type blk] + end + for binding in Iterators.reverse(children(bindings)) + kb = kind(binding) + if kb == K"::" || is_identifier_like(binding) + blk = @ast ctx ex [K"scope_block" scope_type + [K"local" binding] + blk + ] + elseif kb == K"=" + lhs = binding[1] + rhs = binding[2] + if is_identifier_like(lhs) + if kind(lhs) === K"Placeholder" + blk = @ast ctx binding [K"block" rhs blk] + else + blk = @ast ctx binding [K"block" + tmp := rhs + [K"scope_block"(ex) scope_type + [K"local"(lhs) lhs] + [K"always_defined" lhs] + [K"="(binding) lhs tmp] + blk + ] + ] + end + elseif kind(lhs) == K"::" + var = lhs[1] + if !is_identifier_like(var) + throw(LoweringError(var, "Invalid assignment location in let syntax")) + elseif kind(var) === K"Placeholder" + # do a typeassert/convert here? (this falls through flisp as + # a typed global...) + blk = @ast ctx binding [K"block" rhs blk] + else + blk = @ast ctx binding [K"block" + tmp := rhs + # type := lhs[2] + [K"scope_block"(ex) scope_type + # n.b. the declared type is referenced directly (not + # hoisted into a temporary) so that the declaration + # works for variables captured into other lambdas, where + # it is re-evaluated at each assignment like flisp does + [K"local"(lhs) [K"::" var lhs[2]]] + [K"always_defined" var] + [K"="(binding) var tmp] + blk + ] + ] + end + elseif kind(lhs) == K"tuple" + lhs_locals = SyntaxList() + foreach_lhs_name(lhs) do var + push!(lhs_locals, @ast ctx var [K"local" var]) + push!(lhs_locals, @ast ctx var [K"always_defined" var]) + end + blk = @ast ctx binding [K"block" + tmp := rhs + [K"scope_block"(ex) scope_type + lhs_locals... + [K"="(binding) lhs tmp] + blk + ] + ] + else + throw(LoweringError(lhs, "Invalid assignment location in let syntax")) + end + elseif kind(binding) == K"function" + sig = binding[1] + func_name = assigned_function_name(sig) + if isnothing(func_name) + # Some valid function syntaxes define methods on existing types and + # don't really make sense with let: + # let A.f() = 1 ... end + # let (obj::Callable)() = 1 ... end + throw(LoweringError(sig, "Function signature does not define a local function name")) + end + blk = @ast ctx binding [K"block" + [K"scope_block"(ex) scope_type + [K"local"(func_name) func_name] + # note no always_defined, as it's stronger than flisp's local-def + binding + blk + ] + ] + else + @jl_assert false (binding, "invalid binding in let") + end + end + return blk +end + +#------------------------------------------------------------------------------- +# Expand named tuples + +function _named_tuple_expr(ctx, srcref, names, values) + if isempty(names) + @ast ctx srcref [K"call" "NamedTuple"::K"core"] + else + @ast ctx srcref [K"call" + [K"curly" "NamedTuple"::K"core" [K"tuple" names...]] + # NOTE: don't use `tuple` head, so an assignment expression as a value + # doesn't turn this into another named tuple. + [K"call" "tuple"::K"core" values...] + ] + end +end + +function _merge_named_tuple(ctx, srcref, old, new) + if isnothing(old) + new + else + @ast ctx srcref [K"call" "merge"::K"top" old new] + end +end + +function expand_named_tuple(ctx, ex, kws; field_name="named tuple field", + element_name="named tuple element") + name_strs = Set{String}() + names = SyntaxList() + values = SyntaxList() + current_nt = nothing + for kw in kws + k = kind(kw) + appended_nt = nothing + name = value = nothing + if k == K"Identifier" + # x ==> x = x + name = to_symbol(ctx, kw) + value = kw + elseif k == K"kw" || k == K"=" + # syntax TODO: This should parse to K"kw" + # x = a + if kind(kw[1]) != K"Identifier" && kind(kw[1]) != K"Placeholder" + throw(LoweringError(kw[1], "invalid $field_name name")) + end + if kind(kw[2]) == K"..." + throw(LoweringError(kw[2], "`...` cannot be used in a value for a $field_name")) + end + name = to_symbol(ctx, kw[1]) + value = kw[2] + elseif k == K"." + # a.x ==> x=a.x + if kind(kw[2]) != K"Symbol" + throw(LoweringError(kw, "invalid $element_name")) + end + name = to_symbol(ctx, kw[2]) + value = kw + elseif k == K"call" && numchildren(kw) == 3 && + is_same_identifier_like(kw[1], "=>") + # a=>b ==> $a=b + appended_nt = _named_tuple_expr(ctx, kw, (kw[2],), (kw[3],)) + nothing, nothing + elseif k == K"..." + # args... ==> splat pairs + appended_nt = kw[1] + if isnothing(current_nt) && isempty(names) + # Must call merge to create NT from an initial splat + current_nt = _named_tuple_expr(ctx, ex, (), ()) + end + nothing, nothing + else + throw(LoweringError(kw, "Invalid $element_name")) + end + if !isnothing(name) && !isnothing(value) + if kind(name) == K"Symbol" + name_str = syntax_name(name) + if name_str in name_strs + throw(LoweringError(name, "Repeated $field_name name")) + end + push!(name_strs, name_str) + end + push!(names, name) + push!(values, value) + end + if !isnothing(appended_nt) + if !isempty(names) + current_nt = _merge_named_tuple(ctx, ex, current_nt, + _named_tuple_expr(ctx, ex, names, values)) + empty!(names) + empty!(values) + end + current_nt = _merge_named_tuple(ctx, ex, current_nt, appended_nt) + end + end + if !isempty(names) || isnothing(current_nt) + current_nt = _merge_named_tuple(ctx, ex, current_nt, + _named_tuple_expr(ctx, ex, names, values)) + end + @jl_assert !isnothing(current_nt) ex + current_nt +end + +#------------------------------------------------------------------------------- +# Call expansion + +function expand_kw_call(ctx, srcref, farg, args, kws) + @ast ctx srcref [K"block" + func := farg + kw_container := expand_named_tuple(ctx, srcref, kws; + field_name="keyword argument", + element_name="keyword argument") + if all(kind(kw) == K"..." for kw in kws) + # In this case need to check kws nonempty at runtime + [K"if" + [K"call" "isempty"::K"top" kw_container] + [K"call" func args...] + [K"call" "kwcall"::K"core" kw_container func args...] + ] + else + [K"call" "kwcall"::K"core" kw_container func args...] + end + ] +end + +# Special rule: Any becomes core.Any regardless of the module +# scope, and don't need GC roots. +function expand_ccall_argtype(ctx, ex) + if is_same_identifier_like(ex, "Any") + @ast ctx ex "Any"::K"core" + else + expand_forms_2(ctx, ex) + end +end + +# Expand the (sym, lib) argument to ccall / cglobal +function expand_csymbol(ctx, ex) + @stm ex begin + [K"static_eval" _] -> ex # already done + _ -> expand_forms_2(ctx, ex) + end +end + +function expand_ccall(ctx, ex) + @jl_assert kind(ex) == K"call" ex + if numchildren(ex) < 4 + throw(LoweringError(ex, "too few arguments to ccall")) + end + cfunc_name = ex[2] + # Detect calling convention if present. + known_conventions = ("cdecl", "stdcall", "fastcall", "thiscall", "llvmcall") + cconv = if kind(ex[3]) === K"cconv" + ex[3] + elseif any(is_same_identifier_like(ex[3], id) for id in known_conventions) + ex[3] + else + nothing + end + + if isnothing(cconv) + rt_idx = 3 + else + rt_idx = 4 + if numchildren(ex) < 5 + throw(LoweringError(ex, "too few arguments to ccall with calling convention specified")) + end + end + return_type = ex[rt_idx] + arg_type_tuple = ex[rt_idx+1] + args = ex[rt_idx+2:end] + if kind(arg_type_tuple) != K"tuple" + msg = "ccall argument types must be a tuple; try `(T,)`" + if kind(return_type) == K"tuple" + throw(LoweringError(return_type, msg*" and check if you specified a correct return type")) + else + throw(LoweringError(arg_type_tuple, msg)) + end + end + arg_types = children(arg_type_tuple) + vararg_type = nothing + if length(arg_types) >= 1 + va = arg_types[end] + if kind(va) == K"..." + @jl_assert numchildren(va) == 1 va + # Ok: vararg function + vararg_type = expand_ccall_argtype(ctx, va[1]) + arg_types = arg_types[1:end-1] + if length(arg_types) === 0 + throw(LoweringError(va, "C ABI prohibits vararg without one required argument")) + end + end + end + # todo: use multi-range errors here + if length(args) < length(arg_types) + throw(LoweringError(ex, "Too few arguments in ccall compared to argument types")) + elseif length(args) > length(arg_types) && isnothing(vararg_type) + throw(LoweringError(ex, "More arguments than types in ccall")) + end + sctx = with_stmts(ctx) + expanded_types = SyntaxList() + for argt in arg_types + if kind(argt) == K"..." + throw(LoweringError(argt, "only the trailing ccall argument type should have `...`")) + end + push!(expanded_types, expand_ccall_argtype(ctx, argt)) + end + for _ in length(arg_types)+1:length(args) + push!(expanded_types, vararg_type) + end + + # An improvement might be wrap the use of types in cconvert in a special + # K"global_scope" expression which modifies the scope resolution. This + # would at least make the rules self consistent if not pretty. + # + # One small improvement we make here is to emit temporaries for all the + # types used during expansion so at least we don't have their side effects + # more than once. + types_for_conv = SyntaxList() + for argt in expanded_types + push!(types_for_conv, emit_assign_tmp(sctx, argt)) + end + gc_roots = SyntaxList() + unsafe_args = SyntaxList() + for (i,arg) in enumerate(args) + if i > length(expanded_types) + raw_argt = expanded_types[end] + push!(expanded_types, raw_argt) + argt = types_for_conv[end] + else + raw_argt = expanded_types[i] + argt = types_for_conv[i] + end + exarg = expand_forms_2(ctx, arg) + if is_core_Any(raw_argt) + push!(unsafe_args, exarg) + else + cconverted_arg = emit_assign_tmp(sctx, + @ast ctx argt [K"call" + "cconvert"::K"top" + argt + exarg + ] + ) + push!(gc_roots, cconverted_arg) + push!(unsafe_args, + @ast ctx argt [K"call" + "unsafe_convert"::K"top" + argt + cconverted_arg + ] + ) + end + end + @ast ctx ex [K"block" + sctx.stmts... + [K"foreigncall" + expand_csymbol(ctx, cfunc_name) + [K"static_eval"(;meta=name_hint("ccall return type")) + expand_forms_2(ctx, return_type) + ] + [K"static_eval"(;meta=name_hint("ccall argument type")) + [K"call" + "svec"::K"core" + expanded_types... + ] + ] + (cconv !== nothing && kind(cconv) === K"cconv" ? cconv[2].value : + isnothing(vararg_type) ? 0 : + length(arg_types))::K"Integer" + if isnothing(cconv) + "ccall"::K"Symbol" + elseif kind(cconv) === K"cconv" + @ast ctx cconv [K"inert" cconv[1]] + else + cconv=>K"Symbol" + end + unsafe_args... + gc_roots... # GC roots + ] + ] +end + +function expand_cglobal(ctx, ex) + if numchildren(ex) == 2 + # cglobal(name) -> foreignglobal(name) + return @ast ctx ex [K"foreignglobal" + expand_csymbol(ctx, ex[2]) + ] + elseif numchildren(ex) == 3 + # cglobal(name, T) -> bitcast(Ptr{T}, foreignglobal(name)) + return @ast ctx ex [K"call" + "bitcast"::K"top" + [K"call" "apply_type"::K"core" "Ptr"::K"top" expand_forms_2(ctx, ex[3])] + [K"foreignglobal" expand_csymbol(ctx, ex[2])] + ] + else + throw(LoweringError(ex, "wrong number of arguments to cglobal")) + end +end + +function remove_kw_args!(ctx, args::SyntaxList) + kws = nothing + j = 0 + num_parameter_blocks = 0 + for i in 1:length(args) + arg = args[i] + k = kind(arg) + if k == K"kw" + if isnothing(kws) + kws = SyntaxList() + end + push!(kws, arg) + elseif k == K"parameters" + num_parameter_blocks += 1 + if num_parameter_blocks > 1 + throw(LoweringError(arg, "Cannot have more than one group of keyword arguments separated with `;`")) + end + if numchildren(arg) == 0 + continue # ignore empty parameters (issue #18845) + end + if isnothing(kws) + kws = SyntaxList() + end + append!(kws, children(arg)) + else + j += 1 + if j < i + args[j] = args[i] + end + end + end + resize!(args, j) + return kws +end + +function expand_call(ctx, ex) + farg = ex[1] + if kind(farg) === K"Identifier" && syntax_name(farg) === "ccall" + return expand_ccall(ctx, ex) + elseif kind(farg) === K"Identifier" && syntax_name(farg) === "cglobal" + return expand_cglobal(ctx, ex) + end + args = copy(ex[2:end]) + kws = remove_kw_args!(ctx, args) + if !isnothing(kws) + return expand_forms_2(ctx, expand_kw_call(ctx, ex, farg, args, kws)) + end + if any(kind(arg) == K"..." for arg in args) + # Splatting, eg, `f(a, xs..., b)` + expand_splat(ctx, ex, expand_forms_2(ctx, farg), args) + elseif kind(farg) == K"Identifier" && syntax_name(farg) === "include" + # world age special case + r = ssavar(ctx, ex) + @ast ctx ex [K"block" + [K"=" r [K"call" + expand_forms_2(ctx, farg) + expand_forms_2(ctx, args)... + ]] + (::K"latestworld_if_toplevel") + r + ] + else + @ast ctx ex [K"call" + expand_forms_2(ctx, farg) + expand_forms_2(ctx, args)... + ] + end +end + +#------------------------------------------------------------------------------- + +function expand_dot(ctx, ex) + @stm ex begin + # eg, `f = .+` + # Upstream TODO: Remove the (. +) representation and replace with use + # of DOTOP_FLAG? This way, `K"."` will be exclusively used for + # getproperty. + [K"." op] -> @ast ctx ex [K"call" "BroadcastFunction"::K"top" op] + [K"." l [K"syntaxinert" r]] -> + @ast ctx ex [K"call" "getproperty"::K"top" l [K"inert" r]] + [K"." l r] -> begin + @jl_assert is_leaf(r) || kind(r) in KSet"inert syntaxinert" ex + @ast ctx ex [K"call" "getproperty"::K"top" l r] + end + end +end + +#------------------------------------------------------------------------------- +# Expand for loops + +function expand_for(ctx, ex) + iterspecs = ex[1] + + @jl_assert kind(iterspecs) == K"iteration" ex + + # Loop variables not declared `outer` are reassigned for each iteration of + # the innermost loop in case the user assigns them to something else. + # (Maybe we should filter these to remove vars not assigned in the loop? + # But that would ideally happen after the variable analysis pass, not + # during desugaring.) + copied_vars = SyntaxList() + for iterspec in iterspecs[1:end-1] + @jl_assert kind(iterspec) == K"in" iterspec + lhs = iterspec[1] + if kind(lhs) != K"outer" + foreach_lhs_name(lhs) do var + push!(copied_vars, @ast ctx var [K"=" var var]) + end + end + end + + loop = ex[2] + for i in numchildren(iterspecs):-1:1 + iterspec = iterspecs[i] + lhs = iterspec[1] + + outer = kind(lhs) == K"outer" + lhs_local_defs = SyntaxList() + lhs_outer_defs = SyntaxList() + if outer + lhs = lhs[1] + end + foreach_lhs_name(lhs) do var + if outer + push!(lhs_outer_defs, @ast ctx var var) + else + push!(lhs_local_defs, @ast ctx var [K"local" var]) + end + end + + iter_ex = iterspec[2] + next = new_local_binding(ctx, iterspec, "next") + state = ssavar(ctx, iterspec, "state") + collection = ssavar(ctx, iter_ex, "collection") + + # Assign iteration vars and next state + body = @ast ctx iterspec [K"block" + lhs_local_defs... + lower_tuple_assignment(ctx, iterspec, (lhs, state), next) + loop + ] + + body = if i == numchildren(iterspecs) + # Innermost loop gets the continue label and copied vars + @ast ctx ex [K"symbolicblock" + "loop-cont"::K"symboliclabel" + [K"let" + [K"block" + copied_vars... + ] + body + [K"neutral_scope"] + ] + ] + else + # Outer loops get a scope block to contain the iteration vars + @ast ctx ex [K"scope_block" [K"neutral_scope"] body] + end + + loop = @ast ctx ex [K"block" + if outer + [K"assert" + "require_existing_locals"::K"Symbol" + lhs_outer_defs... + ] + end + [K"="(iter_ex) collection iter_ex] + # First call to iterate is unrolled + # next = top.iterate(collection) + [K"="(iterspec) next [K"call" "iterate"::K"top" collection]] + [K"if"(iterspec) # if next !== nothing + [K"call"(iterspec) + "not_int"::K"top" + [K"call" "==="::K"core" next (::K"nothing")] + ] + [K"_do_while"(ex) + [K"block" + body + # Advance iterator + [K"="(iterspec) next [K"call" "iterate"::K"top" collection state]] + ] + [K"call"(iterspec) + "not_int"::K"top" + [K"call" "==="::K"core" next (::K"nothing")] + ] + ] + ] + ] + end + + @ast ctx ex [K"symbolicblock" "loop-exit"::K"symboliclabel" + loop + ] +end + +#------------------------------------------------------------------------------- +# Expand try/catch/finally + +function match_try(ex) + @jl_assert numchildren(ex) > 1 (ex, "Invalid `try` form") + try_ = ex[1] + catch_ = nothing + finally_ = nothing + else_ = nothing + for e in ex[2:end] + k = kind(e) + if k == K"catch" && isnothing(catch_) + @jl_assert numchildren(e) == 2 (e, "Invalid `catch` form") + catch_ = e + elseif k == K"else" && isnothing(else_) + @jl_assert numchildren(e) == 1 e + else_ = e[1] + elseif k == K"finally" && isnothing(finally_) + @jl_assert numchildren(e) == 1 e + finally_ = e[1] + else + throw(LoweringError(ex, "Invalid clause in `try` form")) + end + end + (try_, catch_, else_, finally_) +end + +function _symboliclabel_defs(st, labels=Set{NameKey}()) + if kind(st) === K"symboliclabel" + push!(labels, NameKey(st)) + elseif !(is_leaf(st) || is_quoted(st)) + for c in children(st) + _symboliclabel_defs(c, labels) + end + end + labels +end +function _symboliclabel_refs(st, labels=Vector{SyntaxTree}()) + if kind(st) === K"symbolicgoto" + push!(labels, st) + elseif !(is_leaf(st) || is_quoted(st)) + for c in children(st) + _symboliclabel_refs(c, labels) + end + end + labels +end +function error_if_unmatched_symbolicgoto(ctx, st, hint) + refs = _symboliclabel_refs(st) + isempty(refs) && return nothing + defs = _symboliclabel_defs(st) + unmatched = nothing + for r in refs + NameKey(r) in defs || (unmatched = r) + end + isnothing(unmatched) || throw(LoweringError( + unmatched, "`goto` out of $hint block is not permitted with `finally`")) +end + +function expand_try(ctx, ex) + (try_, catch_, else_, finally_) = match_try(ex) + if !isnothing(finally_) + error_if_unmatched_symbolicgoto(ctx, try_, "a `try`") + !isnothing(catch_) && error_if_unmatched_symbolicgoto(ctx, catch_, "a `catch`") + !isnothing(else_) && error_if_unmatched_symbolicgoto(ctx, else_, "an `else`") + end + try_body = @ast ctx try_ [K"scope_block" [K"neutral_scope"] try_] + if isnothing(catch_) + try_block = try_body + else + exc_var = catch_[1] + catch_block = catch_[2] + if !is_identifier_like(exc_var) + throw(LoweringError(exc_var, "Expected an identifier as exception variable")) + end + try_block = @ast ctx ex [K"trycatchelse" + try_body + [K"scope_block"(catch_) [K"neutral_scope"] + if kind(exc_var) != K"Placeholder" + [K"block" + [K"="(exc_var) exc_var [K"call" current_exception::K"Value"]] + catch_block + ] + else + catch_block + end + ] + if !isnothing(else_) + [K"scope_block"(else_) [K"neutral_scope"] else_] + end + ] + end + + if isnothing(finally_) + try_block + else + @ast ctx ex [K"tryfinally" + try_block + [K"scope_block"(finally_) [K"neutral_scope"] finally_] + ] + end +end + +#------------------------------------------------------------------------------- +# Expand local/global/const declarations + +# Create local/global declarations, and possibly type declarations for each name +# on an assignment LHS. Works recursively with complex left hand side +# assignments containing tuple destructuring. Eg, given +# (x::T, (y::U, z)) +# strip out stmts = (local x) (decl x T) (local x) (decl y U) (local z) +function make_lhs_decls(ctx, stmts, declkind, declmeta, ex, type_decls=true) + @nospecialize declmeta + declname = @stm ex begin + [K"Identifier"] -> ex + [K"Placeholder"] -> nothing + ([K"::" [K"Identifier"] t], when=type_decls) -> let x = ex[1] + t2 = expand_forms_2(ctx, t) + push!(stmts, newnode(ex, K"decl", SyntaxList(x, t2))) + make_lhs_decls(ctx, stmts, declkind, declmeta, x, type_decls) + end + ([K"::" [K"Placeholder"] t], when=type_decls) -> let + # TODO: Currently, this ignores the LHS in `_::T = val`. + # We should probably do one of the following: + # - Throw a LoweringError if that's not too breaking + # - `convert(T, rhs)::T` and discard the result which is what + # `x::T = rhs` would do if x is never used again. + end + ([K"::" x t], when=!type_decls) -> + make_lhs_decls(ctx, stmts, declkind, declmeta, x, type_decls) + (_, when=kind(ex) in KSet"call curly where") -> + make_lhs_decls(ctx, stmts, declkind, declmeta, ex[1], type_decls) + [K"tuple" xs...] -> for x in xs + make_lhs_decls(ctx, stmts, declkind, declmeta, x, type_decls) + end + [K"parameters" xs...] -> for x in xs + make_lhs_decls(ctx, stmts, declkind, declmeta, x, type_decls) + end + [K"..." x] -> nothing # from recursion above + [K"ref" _ _...] -> nothing # decl is ignored; syntax TODO + [K"." _ _] -> nothing # decl is ignored; syntax TODO + end + + if !isnothing(declname) + stmt = @ast ctx ex [declkind(;meta=declmeta) declname] + push!(stmts, stmt) + end + return nothing +end + +# Separate decls and assignments (which require re-expansion) +# local x, (y=2), z ==> local x; local z; y = 2 +function expand_decls(ctx, ex) + declkind = kind(ex) + @jl_assert declkind in KSet"local global" ex + stmts = SyntaxList() + val_nothing = !(numchildren(ex) == 1 && is_leaf(children(ex)[1])) + for c in children(ex) + simple = kind(c) in KSet"Identifier :: Placeholder" + val_nothing &= simple + if declkind === K"global" + if kind(c) === K"=" + (lhs, relayered) = relayer_global_if_unhygienic(ctx, c[1]); + !isempty(relayered) && (c = @ast ctx c [K"=" lhs c[2]]) + elseif simple + (c, relayered) = relayer_global_if_unhygienic(ctx, c); + end + @isdefined(relayered) && for x in relayered + push!(stmts, @ast ctx x [K"relayered_global" x]) + end + end + lhs = @stm c begin + (_, when=simple) -> c + [K"=" x _] -> x + [K".=" x _] -> x + [K"op=" x _ _] -> x + [K".op=" x _ _] -> x + [K"function" x _...] -> x + end + # type decls are handled elsewhere unless simple + make_lhs_decls(ctx, stmts, declkind, ex.meta, lhs, simple) + simple || push!(stmts, expand_forms_2(ctx, c)) + end + # flisp quirk: if not a plain `global x` or `local x`, value is readable + val_nothing && push!(stmts, @ast ctx ex (::K"nothing")) + newnode(ex, K"block", stmts) +end + +# Iterate over the variable names assigned to from a "fancy assignment left hand +# side" such as nested tuple destructuring, curlies, and calls. +function foreach_lhs_name(f::Function, ex) + k = kind(ex) + if k == K"Placeholder" + # Ignored + elseif is_identifier_like(ex) + f(ex) + elseif (k === K"::" && numchildren(ex) === 2) || k in KSet"call curly where" + foreach_lhs_name(f, ex[1]) + elseif k in KSet"tuple parameters" + for c in children(ex) + foreach_lhs_name(f, c) + end + end + return nothing +end + +function expand_const_decl(ctx, ex) + if numchildren(ex) == 2 + # pre-desugared const + return @ast ctx ex [K"constdecl" ex[1] ex[2]] + end + @stm ex[1] begin + # const is ignored on function + [K"function" _...] -> expand_forms_2(ctx, ex[1]) + [K"global" [K"function" _...]] -> expand_forms_2(ctx, ex[1]) + + [K"global" x] -> let decls = SyntaxList() + @jl_assert kind(x) === K"=" ex + (lhs, relayered) = relayer_global_if_unhygienic(ctx, x[1]) + make_lhs_decls( + ctx, decls, K"global", ex[1].meta, lhs, false) + for x in relayered + push!(decls, @ast ctx x [K"relayered_global" x]) + end + x2 = @ast ctx x [K"=" lhs x[2]] + @ast ctx ex [K"block" decls... expand_assignment(ctx, x2, true)] + end + [K"=" _ _] -> expand_assignment(ctx, ex[1], true) + # Expr(:const, v) where v is a Symbol or a GlobalRef is an unfortunate + # remnant from the days when const-ness was a flag that could be set on + # any global. It creates a binding with kind PARTITION_KIND_UNDEF_CONST. + # TODO: deprecate and delete this "feature" + [K"Identifier"] -> @ast ctx ex [K"constdecl" ex[1]] + end +end + +#------------------------------------------------------------------------------- +# Expansion of function definitions + +# (where (where x a b) c d) -> (x, [c d a b]) +function flatten_wheres(ex) + tvs = SyntaxList() + while kind(ex) === K"where" + append!(tvs, ex[2:end]) + ex = ex[1] + end + return ex, tvs +end + +# Select static parameters which are used in function arguments `arg_types`, or +# transitively used. +# +# The transitive usage check probably doesn't guarantee that the types are +# inferable during dispatch as they may only be part of the bounds of another +# type. Thus we might get false positives here but we shouldn't get false +# negatives. +function select_used_typevars(uses::SyntaxList, typevars::SyntaxList) + used = BitVector(undef, length(typevars)) + for (i, tv) in enumerate(typevars) + @jl_assert kind(tv) === K"_typevar" tv + for u in uses + contains_identifier(u, tv[1]) && (used[i] = true) + end + end + # now find transitive uses + todo = findall(used) + while !isempty(todo) + tv_i = pop!(todo) + tv = typevars[tv_i] + # for each typevar `prev` before tv, if our bounds reference `prev` (and + # `prev` is not already used), add it to used and todo + for prev_i in 1:tv_i-1 + used[prev_i] && continue + prevname = typevars[prev_i][1] + if contains_identifier(tv[2], prevname) || + contains_identifier(tv[3], prevname) + used[prev_i] = true + push!(todo, prev_i) + end + end + end + return used +end + +used_typevars(uses::SyntaxList, tvs::SyntaxList) = + tvs[select_used_typevars(uses, tvs)] + +unused_typevars(uses::SyntaxList, tvs::SyntaxList) = + tvs[map(!, select_used_typevars(uses, tvs))] + +function make_assigns(ctx, ls::SyntaxList, rs::SyntaxList) + out = SyntaxList() + for (l, r) in zip(ls, rs) + push!(out, @ast ctx r [K"=" l r]) + end + out +end + +function scope_nest(ctx, assigns, body) + for a in Iterators.reverse(assigns) + body = @ast ctx a [K"let" [K"block" a] body] + end + body +end + +function pos_req_args(argl::SyntaxList) + last = lastindex(argl) + for i in eachindex(argl) + if kind(argl[i]) in KSet"kw ... parameters" + last = i-1 + break + end + end + argl[1:last] +end + +function pos_opt_args(argl::SyntaxList) + opt_start = length(pos_req_args(argl))+1 + opt_end = -1 + for i in opt_start:lastindex(argl) + if kind(argl[i]) === K"kw" + opt_end = i + end + end + @jl_assert let pos = kind(argl[end]) === K"parameters" ? argl[1:end-1] : argl + # no optargs, or optargs until the end (maybe excluding vararg, kws) + opt_end in (-1,lastindex(pos),lastindex(pos)-1) + end pos[end] + argl[opt_start:opt_end] +end + +# (_typevar name lb ub) -> (local (= name (call core TypeVar...))) +function assign_sparams(ctx, tvs) + out = SyntaxList() + for tv in tvs + @jl_assert kind(tv) === K"_typevar" tv + push!(out, @ast ctx tv [K"local" tv[1]]) + push!(out, @ast ctx tv [K"=" tv[1] bounds_to_typevar(ctx, tv)]) + end + out +end + +function method_def_sparams(ctx, src, tvs) + out = SyntaxList() + for tv in tvs + @jl_assert kind(tv) === K"_typevar" tv + push!(out, @ast ctx tv [K"typevar" tv[1] bounds_to_typevar(ctx, tv)]) + end + @ast ctx src [K"block" out...] +end + +# Hack: Normally just (block ex body), but needs special handling due to +# pre-quoted parts of generated function body, where we need to prepend +# desugarable AST to macro AST. Fortunately there are only two places (meta +# nkw, and destructuring arg assignments) we do this, so handle them manually. +function prepend_function_body(ctx, body, ex) + out = @stm body begin + [K"_generated_body" [K"syntaxquote" gen] nongen] -> begin + ex_est = @stm ex begin + [K"meta" [K"Symbol"] n] -> + @ast ctx ex [K"meta" "nkw"::K"Identifier" n] + # destructured arg assignments + [K"block" stmts... [K"nothing"]] -> + @ast ctx ex [K"block" stmts...] + _ -> @jl_assert false (ex, "unexpected prepend_function_body") + end + @ast ctx body [K"_generated_body" + [K"syntaxquote" [K"block" ex_est gen]] [K"block" ex nongen]] + end + _ -> @ast ctx body [K"block" ex body] + end + mm = getmeta(body, :method_metas, nothing) + isnothing(mm) || setmeta!(out, :method_metas, mm) + out +end + +# Prepend method metadata and retain it through recursive wrapper generation. +function prepend_method_metas(ctx, src, body, method_metas) + isnothing(method_metas) && return body + out = @stm body begin + [K"block" stmts...] -> + @ast ctx src [K"block" [K"meta" method_metas...] stmts...] + _ -> @ast ctx src [K"block" [K"meta" method_metas...] body] + end + setmeta(out, :method_metas, method_metas) +end + +# Produce all `method` exprs for the given `argl` +# - one wrapper per optional positional arg +# - one containing the body +# - possibly one generated method +function method_def_expr(ctx, src, mtable, sparams, argl, body, + rett=@ast(ctx, src, "Any"::K"core")) + @jl_assert length(argl) > 0 src + @jl_assert kind(argl[end]) !== K"parameters" src argl[end] + if length(pos_opt_args(argl)) > 0 + return optional_positional_defs( + ctx, src, mtable, sparams, argl, body, rett) + elseif kind(body) === K"_generated_body" + return generated_method_defs( + ctx, src, mtable, sparams, argl, body, rett) + end + # Needs to be done per method, not per function (may create ssavalues) + arg_types = mapsyntax(a->expand_forms_2(ctx, a[2]), argl) + @ast ctx src [K"method" mtable + [K"call" "svec"::K"core" arg_types...] + [K"lambda"(body) + [K"block" mapindex(argl, 1)...] + [K"block" mapindex(sparams, 1)...] + expand_forms_2(ctx, body) + is_core_Any(rett) ? nothing : expand_forms_2(ctx, rett)]] +end + +function _untyped_arg(a) + @jl_assert kind(a) === K"::" || kind(a) === K"_typevar" a + aname = setmeta(a[1], :nospecialize, true) + @ast _ a [K"::" aname "Any"::K"core"] +end + +function _expr_arg_syms(args) + out = SyntaxList() + for (i, a) in enumerate(args) + @jl_assert kind(a) === K"::" || kind(a) === K"_typevar" a + name = if kind(a[1]) === K"Placeholder" + UNUSED + elseif (a[1].context::SyntaxContext).internal && i > 1 + # we lose context, so deduplicate names (ignoring #self# to be + # safe). HACK: destructured args must match the desugared rhs + n = syntax_name(a[1]) + contains(n, "destructured") ? n : n*"#"*string(i) + else + syntax_name(a[1]) + end + push!(out, @mknode(a[1]; kind=K"Symbol", value=name)) + end + out +end + +# The Julia runtime associates the code generator with the non-generated method +# by adding (meta generated ...) to the non-generated body +# May need hygiene/provenance adjustments +function generated_method_defs(ctx, src, mtable, sparams, argl, body, rett) + @jl_assert kind(body) === K"_generated_body" && numchildren(body) == 2 body + gen_name = let mangled = reserve_module_binding_i( + ctx.layer.mod, + string("#", kind(mtable) === K"nothing" ? "_" : mtable, "@generator")) + new_global_binding(ctx, src, mangled, ctx.layer.mod) + end + + sc = src.context::SyntaxContext + gen_mdef = let arg1_name = newsym(ctx, argl[1], "#self#"), + gen_argl = SyntaxList( + @ast(ctx, src, [K"::" arg1_name [K"function_type" gen_name]]), + @ast(ctx, src, [K"::" + "__context__"::K"Identifier"(;context=sc) + SyntaxContext::K"Value" + ]), + mapsyntax(_untyped_arg, sparams)..., + mapsyntax(_untyped_arg, argl)...) + @jl_assert kind(body[1]) === K"syntaxquote" body + gen_body = est_to_dst(expand_syntaxquote(ctx, body[1][1])) + + method_def_expr(ctx, src, gen_name, SyntaxList(), gen_argl, gen_body, + @ast(ctx, src, "Any"::K"core")) + end + + nongen_mdef = let + nongen_body = @ast ctx body[2] [K"block" [K"meta" "generated"::K"Symbol" + [K"new" + GeneratedFunctionStub::K"Value" # Use stub type from JuliaLowering + SyntaxContext(ctx.layer.mod, sc.version)::K"Value" + gen_name + # Truncate provenance to just the source file range, as this + # will live permanently in the IR and we probably don't want + # the full provenance tree and intermediate expressions + # (TODO: More truncation. We certainly don't want to store the + # source file either.) + sourceref(src)::K"Value" + [K"call" "svec"::K"core" _expr_arg_syms(argl)...] + [K"call" "svec"::K"core" _expr_arg_syms(sparams)...]]] + body[2]] + method_def_expr(ctx, src, mtable, sparams, argl, nongen_body, rett) + end + + @ast ctx src [K"block" + [K"global" gen_name] + [K"function_decl" gen_name] + [K"method_defs" gen_name [K"block"] gen_mdef] + nongen_mdef] +end + +# Semantically, we want each wrapper method's body to call the method with one +# additional default (on top of its args, `passed`) until we reach the body +# method with all args filled. As an optimization, a wrapper can fill all +# remaining default values as long as we can rule out any of the additional +# default values depending on the values of non-`passed` arguments. +# +# flisp checks dependencies by searching each additional default for every +# subexpression of `arg::type` for every non-`passed` `arg` before it. We only +# check that static params in `::type` are not referenced in later defaults, and +# use `(let (= arg default) body)` to handle references to `arg`. (flisp likely +# does this search to accomplish what we do with scope_nest) +function optional_positional_defs(ctx, src, mtable, sparams, argl, body, rett) + opt = pos_opt_args(argl) + opt_decls = mapindex(opt, 1) + opt_names = mapindex(opt_decls, 1) + opt_defaults = mapindex(opt, 2) + + # the final optarg (index into `opt`) that might reference `sp` in its type + sp_known_by = zeros(Int, length(sparams)) + for sp_i in eachindex(sparams) + for (i, arg) in Iterators.reverse(enumerate(opt_decls)) + if contains_identifier(arg[2], sparams[sp_i][1]) + sp_known_by[sp_i] = i + break + end + end + end + # `deps[i] = j` is the largest `j (xs, va) + [K"parameters" xs...] -> (xs, nothing) + end + kw_decls = SyntaxList() + kw_syms = SyntaxList() + kw_defaults = SyntaxList() + for raw_a in kargl + a = expand_function_arg(ctx, raw_a, false) + @stm a begin + [K"kw" [K"::" n t] v] -> begin + push!(kw_decls, a[1]) + push!(kw_defaults, v) + end + [K"::" n t] -> begin + push!(kw_decls, a) + push!(kw_defaults, @ast ctx a [K"call" "throw"::K"core" + [K"call" "UndefKeywordError"::K"core" a[1]=>K"Symbol"]]) + end + end + end + kw_names = mapindex(kw_decls, 1) + kw_syms = mapsyntax(x->@mknode(x; kind=K"Symbol"), kw_names) + restkw_list = isnothing(restkw) ? SyntaxList() : + SyntaxList(@ast ctx restkw [K"::" + restkw [K"call" "pairs"::K"top" "NamedTuple"::K"core"]]) + + return (kw_decls, kw_names, kw_syms, kw_defaults, restkw_list) +end + +# Assumes `expand_function_arg` has run. Note that user-supplied +# "Vararg"::K"Identifier" is assumed to resolve to Core.Vararg +is_vararg_type_expr(st) = @stm st begin + [K"curly" x _...] -> is_vararg_type_expr(x) + [K"where" x _...] -> is_vararg_type_expr(x) + _ -> kind(st) in KSet"core Identifier" && syntax_name(st) == "Vararg" +end + +function keywords_method_def_expr(ctx, src, mtable, sparams, argl, body, rett, overlay) + kws = argl[end] + pargl = argl[1:end-1] + @jl_assert kind(kws) === K"parameters" src + pos_decls = mapsyntax(a->kind(a)===K"kw" ? a[1] : a, pargl) + # Mark the wrapper, not the body method, as the "self" arg to @__FUNCTION__. + # TODO: We could probably unify this with is_kwcall_self with a generic + # "closure not on first arg" flag if we're willing to pass the closure to + # the body method through this arg instead of the first. + pos_decls[1] = let p = pos_decls[1] + @ast ctx p [K"::" setmeta(p[1], :thisfunction_original, true) p[2]] + end + + # Positional names and splatted vararg so we can `(call f forward_pargl...)` + forward_pargl = let l = mapindex(pos_decls, 1) + pos_va = @stm argl[end-1] begin + [K"kw" [K"::" _... t] _...] -> is_vararg_type_expr(t) + [K"::" _... t] -> is_vararg_type_expr(t) + _ -> false + end + pos_va && (l[end] = @ast ctx l[end] [K"..." l[end]]) + l + end + (kw_decls, kw_names, kw_syms, kw_defaults, restkw) = expand_kw_args(ctx, kws) + ordered_defaults = any(val->contains_identifier(val, kw_names), kw_defaults) + pos_sparams = used_typevars(pargl, sparams) + prop_metas = getmeta(body, :method_metas, nothing) + + m1_name = let n = kind(mtable) === K"nothing" ? "_" : syntax_name(mtable), + mangled = string("#", n, "#kw_body#", module_unique_name(ctx.layer.mod)) + # probably not desirable, but fixes eval-into-closed-module + m1_sc = escape_layer(mtable.context::SyntaxContext, true) + @mknode(newsym(ctx, mtable, mangled); + context=SyntaxContext( + m1_sc.layer, m1_sc.unexpanded, m1_sc.version, true)) + end + # (1) Body method. This contains the actual function body, and requires + # every possible default to be filled. `rett` is only passed here since it + # can reference any argument. + mdefs1 = let arg1 = @ast ctx m1_name [K"::" m1_name [K"function_type" m1_name]] + nkw = @ast ctx kws [K"meta" "nkw"::K"Symbol" numchildren(kws)::K"Value"] + method_def_expr( + ctx, src, m1_name, sparams, + SyntaxList(arg1, kw_decls..., restkw..., pos_decls...), + prepend_function_body(ctx, body, nkw), rett) + end + # (2) nokw methods (one per optarg). Lowering wouldn't know to call + # Core.kwcall given no kws in a call, so this method initializes kw defaults + # and calls the body method. + mdefs2 = let rkw = isempty(restkw) ? nothing : + @ast ctx restkw[1] [K"call" + "pairs"::K"top" [K"call" "NamedTuple"::K"core"]] + body2 = if !ordered_defaults + @ast ctx src [K"call" m1_name kw_defaults... rkw forward_pargl...] + else + scope_nest(ctx, make_assigns(ctx, kw_names, kw_defaults), + @ast ctx src [K"call" m1_name kw_names... rkw forward_pargl...]) + end + nokw_body = prepend_method_metas( + ctx, src, @ast(ctx, src, [K"block" [K"return" body2]]), prop_metas) + method_def_expr( + ctx, src, mtable, pos_sparams, pargl, nokw_body) + end + # (3) Core.kwcall(arg2::NamedTuple, pargl...) methods (one per optarg). + # - for each kwarg: + # - kw_temp = if kwname in arg2, extract and typecheck it, else use default + # - collect excess kws (caller-provided fields in arg2 minus `kw_names`) + # - call body method using all kw_temps + # sig: (kwcall_self::typeof(Core.kwcall) kw_namedtuple pargl...) + mdefs3 = let + arg2_name = newsym(ctx, kws, "kws") + # If kwargs don't depend on each other, and their defaults don't contain + # assignments, then we can use ssavalues instead of slots + use_ssa_kw_temps = !ordered_defaults && + !any(val->contains_unquoted(e->kind(e) == K"=", val), kw_defaults) + kw_temps = use_ssa_kw_temps ? + mapsyntax(x->ssavar(ctx, x, syntax_name(x)), kw_names) : kw_names + tempslot = newsym(ctx, kws, "#kwtmp#") + keyword_only_spnames = mapindex(unused_typevars(pargl, sparams), 1) + + kw_assigns = SyntaxList() + for (tmp, sym, decl, default) in zip(kw_temps, kw_syms, kw_decls, kw_defaults) + get_kw = @ast ctx decl [K"call" "getfield"::K"core" arg2_name sym] + if !is_core_Any(decl[2]) && + !contains_identifier(decl[2], keyword_only_spnames) + # static parameters don't have values yet, so don't assert the + # declared kw type here if it contains any static params. bad + # types will trigger a MethodError when calling body instead. + get_kw = @ast ctx decl [K"block" + getkw_tmp := get_kw + [K"if" [K"call" "isa"::K"core" getkw_tmp decl[2]] + (::K"nothing") + [K"call" "throw"::K"core" + [K"new" "TypeError"::K"core" + "keyword argument"::K"Symbol" + sym decl[2] getkw_tmp]]] + getkw_tmp] + end + push!(kw_assigns, @ast ctx decl [K"=" tmp [K"block" + [K"if" [K"call" "isdefined"::K"core" arg2_name sym] + [K"=" tempslot get_kw] + [K"=" tempslot default]] + tempslot]]) + end + + # bundle and forward excess if there's a restkw, else throw kwerr + handle_excess = if !isempty(restkw) + excess_kw = ssavar(ctx, arg2_name, "excess_kw") + @ast ctx src [K"=" + excess_kw + [K"call" "pairs"::K"top" + isempty(kw_names) ? arg2_name : + [K"call" "structdiff"::K"top" arg2_name + [K"curly" "NamedTuple"::K"core" [K"tuple" kw_syms...]]]]] + else + @ast ctx src [K"if" + [K"call" "isempty"::K"top" + [K"call" "diff_names"::K"top" + [K"call" "keys"::K"top" arg2_name] + [K"tuple" kw_syms...]]] + (::K"nothing") + [K"call" "kwerr"::K"top" arg2_name forward_pargl...]] + end + final_call = @ast ctx kws [K"call" + m1_name + kw_temps... + isempty(restkw) ? nothing : excess_kw + forward_pargl...] + kwcall_body = if use_ssa_kw_temps + for n in kw_names + # If not using slots for the keyword argument values, still + # declare them for reflection purposes + push!(kw_assigns, @ast ctx n [K"local" setmeta(n, :is_internal, true)]) + end + @ast(ctx, src, [K"block" kw_assigns... handle_excess final_call]) + else + scope_nest(ctx, kw_assigns, + @ast ctx src [K"block" handle_excess final_call]) + end + kwcall_body = prepend_method_metas(ctx, src, kwcall_body, prop_metas) + # Core.kwcall method has its own first argument. Ensure closure + # conversion knows not to put the closure there. + let arg1_name = setmeta!( + newsym(ctx, kws, "#kwcall_self#"; unused=length(pos_opt_args(pargl)) == 0), + :is_kwcall_self, true) + arg1 = @ast ctx src [K"::" arg1_name + [K"call" "typeof"::K"core" "kwcall"::K"core"] + ] + arg2 = @ast ctx arg2_name [K"::" arg2_name "NamedTuple"::K"core"] + method_def_expr( + ctx, src, mtable, pos_sparams, + SyntaxList(arg1, arg2, pargl...), kwcall_body) + end + end + @ast ctx src [K"block" + [K"function_decl" m1_name] + # hack: define closure type for next decl + overlay || kind(mtable) === K"nothing" ? nothing : [K"no_method_defs" m1_name] + overlay || kind(mtable) === K"nothing" ? nothing : [K"function_decl" mtable] + [K"method_defs" m1_name method_def_sparams(ctx, src, sparams) mdefs1] + [K"method_defs" mtable method_def_sparams(ctx, src, pos_sparams) mdefs2] + [K"method_defs" mtable method_def_sparams(ctx, src, pos_sparams) mdefs3] + mtable + ] +end + +# string mangling is necessary until generated functions know about scope layers +# (hack, see _expr_arg_syms). +_lower_destructuring_arg(stmts, ctx, i, ex) = @stm ex begin + [K"tuple" _...] -> let arg2 = newsym(ctx, ex, "destructured#" * string(i)) + push!(stmts, @ast(ctx, ex, [K"local"(;meta=CompileHints(:is_destructured_arg, true)) + [K"=" ex arg2]])) + arg2 + end + [K"::" x t] -> @ast ctx ex [K"::" _lower_destructuring_arg(stmts, ctx, i, x) t] + [K"kw" x t] -> @ast ctx ex [K"kw" _lower_destructuring_arg(stmts, ctx, i, x) t] + [K"..." x] -> @ast ctx ex [K"..." _lower_destructuring_arg(stmts, ctx, i, x)] + _ -> ex +end + +function lower_destructuring_args!(ctx, args) + stmts = SyntaxList() + for (i, a) in enumerate(args) + args[i] = _lower_destructuring_arg(stmts, ctx, i, a) + end + # return `nothing` from the assignments (issue #26518) + !isempty(stmts) && push!(stmts, @ast ctx stmts[1] (::K"nothing")) + return stmts +end + +# `arg` is the first arg to a function's `call`. return (1) whether this is an +# :overlay expression, (2) the method table expression, and (3) the typed arg +# expression `(:: #self# t)` +function expand_function_arg1(ctx, arg) + if kind(arg) === K"overlay" + _, _, x = expand_function_arg1(ctx, arg[2]) + return true, expand_forms_2(ctx, arg[1]), x + end + aname = @stm arg begin + [K"::" [K"Identifier"] t] -> arg[1] + _ -> newsym(ctx, arg, "#self#") + end + atype = @stm arg begin + [K"::" t] -> t + [K"::" _ t] -> t + _ -> @ast ctx arg [K"function_type" arg] + end + # first arg to Expr(:method) + mt = @stm arg begin + [K"Identifier"] -> arg + [K"Value"] -> arg # TODO delete with globalref support + [K"Placeholder"] -> arg + _ -> @ast ctx arg (::K"nothing") + end + return false, mt, @ast ctx arg [K"::" aname atype] +end + +fix_argname(ctx, arg, used) = @stm arg begin + [K"Identifier"] -> arg + # Lowering should be able to use placeholder args as rvalues internally, + # e.g. for kw method dispatch. + ([K"Placeholder"], when=used) -> newsym(ctx, arg, "#arg#") + ([K"Placeholder"], when=!used) -> arg +end + +# flisp: fill-missing-argname, llist-types, llist-vars, dots->vararg +# +# Make an arg into `(:: x t)` or `(kw (:: x t) default)`. If `used`, the caller +# specifies that even placeholder/underscore arguments might be read from +# internally. Desugar type, but desugar default values later, since +# `default...` is unfortunately allowed, so do that in body desugaring. +expand_function_arg(ctx, arg, used) = @stm arg begin + [K"::" x t] -> + @ast ctx arg [K"::" fix_argname(ctx, x, used) t] + [K"::" t] -> let aname = newsym(ctx, arg, "#arg#"; unused=true) + @ast ctx arg [K"::" fix_argname(ctx, aname, used) t] + end + [K"kw" x v] -> + @ast ctx arg [K"kw" expand_function_arg(ctx, x, used) v] + # note: not correct for kwargs + [K"..." x] -> let inner = expand_function_arg(ctx, x, used) + @jl_assert kind(inner) === K"::" inner arg + @ast ctx x [K"::" inner[1] [K"curly" "Vararg"::K"core" inner[2]]] + end + _ -> @ast ctx arg [K"::" fix_argname(ctx, arg, used) "Any"::K"core"] +end + +# Normalize and expand all positional arguments to (:: identifier t), then call +# a helper to create the method(s). +function expand_function_def(ctx, src, raw_args, wheres, body, rett) + @jl_assert length(raw_args) >= 1 (body, "expected a self arg") + let arg_stmts = lower_destructuring_args!(ctx, raw_args) + if !isempty(arg_stmts) + blk = @ast ctx src [K"block" arg_stmts...] + body = prepend_function_body(ctx, body, blk) + end + end + (overlay, mtable, a1) = expand_function_arg1(ctx, raw_args[1]) + argl = SyntaxList(a1) + has_kws = kind(raw_args[end]) === K"parameters" && numchildren(raw_args[end]) > 0 + let force_used = length(pos_opt_args(raw_args)) > 0 || has_kws + for a in raw_args[2:end] + if kind(a) === K"parameters" + numchildren(a) >= 1 && push!(argl, a) + else + push!(argl, expand_function_arg(ctx, a, force_used)) + end + end + end + sparams = mapsyntax(typevar_bounds, wheres) + if has_kws + keywords_method_def_expr( + ctx, src, mtable, sparams, argl, body, rett, overlay) + elseif overlay + mtmp = ssavar(ctx, mtable) + @ast ctx src [K"block" + [K"method_defs" (::K"nothing") + method_def_sparams(ctx, src, sparams) + [K"block" [K"=" mtmp method_def_expr( + ctx, src, mtable, sparams, argl, body, rett)]]] + mtmp] + else + @ast ctx src [K"block" + (kind(mtable) === K"nothing") ? nothing : [K"function_decl" mtable] + [K"method_defs" mtable + method_def_sparams(ctx, src, sparams) + [K"block" method_def_expr(ctx, src, mtable, sparams, argl, body, rett)]] + [K"removable" mtable]] + end +end + +expand_opaque_closure(ctx, ex) = @stm ex begin + [K"opaque_closure" argt rt_lb rt_ub allow_partial lam] -> begin + @jl_assert kind(lam[1]) === K"tuple" ex + check_no_parameters(ex, lam[1]) + raw_args = append!(SyntaxList(), children(lam[1])) + arg_stmts = lower_destructuring_args!(ctx, raw_args) + + arg_names = SyntaxList(newsym(ctx, lam[1], "#self#")) + inner_arg_types = SyntaxList() + for a in raw_args + if kind(argt) !== K"nothing" && kind(a) === K"::" + throw(LoweringError(a, "opaque closure argument type may not be specified both in the method signature and separately")) + end + a2 = expand_function_arg(ctx, a, false) + if kind(a) === K"kw" || kind(a) === K"parameters" + throw(LoweringError( + a, "opaque closure cannot have optional or keyword arguments")) + end + @jl_assert kind(a2) === K"::" a2 + push!(inner_arg_types, a2[2]) + push!(arg_names, a2[1]) + end + + out_argt = kind(argt) !== K"nothing" ? argt : + @ast ctx lam[1] [K"curly" "Tuple"::K"core" inner_arg_types...] + out_rt_lb = kind(rt_lb) !== K"nothing" ? rt_lb : + @ast ctx lam[1] [K"curly" "Union"::K"core"] + out_rt_ub = kind(rt_ub) !== K"nothing" ? rt_ub : + @ast ctx lam[1] "Any"::K"core" + nargs = (length(arg_names)-1) # ignoring #self# + is_va = !isempty(raw_args) && kind(raw_args[end]) === K"..." + body = @ast ctx lam[2] [K"block" arg_stmts... lam[2]] + + @ast ctx ex [K"_opaque_closure" + ssavar(ctx, ex, "opaque_closure_id") # only a placeholder. Must be :local + expand_forms_2(ctx, out_argt) + expand_forms_2(ctx, out_rt_lb) + expand_forms_2(ctx, out_rt_ub) + allow_partial + nargs::K"Integer" + is_va::K"Bool" + ::K"SourceLocation"(lam) + [K"lambda"(lam) + [K"block" arg_names...] + [K"block"] + expand_forms_2(ctx, body)]] + end +end + +#------------------------------------------------------------------------------- +# Expand macro definitions + +# Name is hygienic-global in compat mode, hygienic otherwise +function _make_macro_name(ctx, ex) + k = kind(ex) + if k == K"Identifier" || k == K"Symbol" + if k === K"Identifier" && is_flisp_compat(ex) + @mknode(ex; kind=k, value="@$(syntax_name(ex))", + mod=syntax_module(ex)) + else + @mknode(ex; kind=k, value="@$(syntax_name(ex))") + end + elseif k == K"Placeholder" + @mknode(ex; kind=K"Identifier", value="@$(syntax_name(ex))") + elseif is_valid_modref(ex) + @jl_assert numchildren(ex) == 2 ex + @ast ctx ex [K"." ex[1] _make_macro_name(ctx, ex[2])] + else + @jl_assert false ex + end +end + +function expand_macro_def(ctx, ex) + if numchildren(ex) == 1 + # macro with zero methods + # `macro m end` + return @ast ctx ex [K"function" _make_macro_name(ctx, ex[1])] + end + (sig, name, args) = @stm ex begin + [K"macro" [K"call" n a...] _] -> (ex[1], n, remove_empty_parameters(a)) + _ -> @jl_assert false ex + end + + sc_ref = (kind(name) == K"." ? name[1] : name) + if is_flisp_compat(ex) + @ast ctx ex [K"function" + [K"call"(sig) + _make_macro_name(ctx, name) + [K"::" + adopt_scope(sc_ref, @ast(ctx, sig, "__source__"::K"Identifier")) + "LineNumberNode"::K"core" + ] + [K"::" + adopt_scope(sc_ref, @ast(ctx, sig, "__module__"::K"Identifier")) + "Module"::K"core" + ] + mapsyntax(e->apply_arg_meta(e, :nospecialize), args)... + ] + ex[2] + ] + else + @ast ctx ex [K"function" + [K"call"(sig) + _make_macro_name(ctx, name) + [K"::" + adopt_scope(sc_ref, @ast(ctx, sig, "__context__"::K"Identifier")) + MacroContext::K"Value" + ] + # We don't mark these @nospecialize because all arguments to + # new macros will be of type SyntaxTree + args... + ] + ex[2] + ] + end +end + +#------------------------------------------------------------------------------- +# Expand type definitions + +# argument to where expression -> (_typevar name expanded_lb expanded_ub) +# used, e.g. in all `sparams`, where flisp generally uses a list (name, lb, ub) +function typevar_bounds(ex) + any = @ast _ ex "Any"::K"core" + (name, lb, ub) = bounds = @stm ex begin + [K"Identifier"] -> (ex, any, any) + [K"Placeholder"] -> (ex, any, any) + ([K"comparison" lb op x _ ub], when=syntax_name(op)==="<:") -> (x, lb, ub) + ([K"comparison" ub op x _ lb], when=syntax_name(op)===">:") -> (x, lb, ub) + [K"<:" x ub] -> (x, any, ub) + [K">:" x lb] -> (x, lb, any) + end + @ast _ ex [K"_typevar" name lb ub] +end + +function bounds_to_typevar(ctx, ex) + @jl_assert kind(ex) === K"_typevar" ex + _bounds_to_typevar(ctx, ex, ex[1], ex[2], ex[3]) +end + +# Generate call to `TypeVar(name[, lb, ub])`. Note the resulting expression may +# contain SSA assignments, so can't be copied. +function _bounds_to_typevar(ctx, srcref, name, lb, ub) + @ast ctx srcref [K"call" + "TypeVar"::K"core" + name=>K"Symbol" + if !is_core_Any(lb) + expand_forms_2(ctx, lb) + end + if !is_core_Any(lb) || !is_core_Any(ub) + expand_forms_2(ctx, ub) + end + ] +end + +# Analyze type signatures such as `A{C} <: B where C` +# +# Return (name, typevar_names, typevar_stmts, supertype) where +# - `name` is the name of the type +# - `supertype` is the super type of the type +function analyze_type_sig(ctx, ex) + k = kind(ex) + if k == K"Identifier" + name = ex + type_params = () + supertype = @ast ctx ex "Any"::K"core" + elseif k == K"curly" && numchildren(ex) >= 1 && kind(ex[1]) == K"Identifier" + # name{type_params} + name = ex[1] + type_params = ex[2:end] + supertype = @ast ctx ex "Any"::K"core" + elseif k == K"<:" && numchildren(ex) == 2 + if kind(ex[1]) == K"Identifier" + name = ex[1] + type_params = () + supertype = ex[2] + elseif kind(ex[1]) == K"curly" && numchildren(ex[1]) >= 1 && kind(ex[1][1]) == K"Identifier" + name = ex[1][1] + type_params = ex[1][2:end] + supertype = ex[2] + end + end + @isdefined(name) || throw(LoweringError(ex, "invalid type signature")) + @isdefined(type_params) || throw(LoweringError(ex, "invalid type signature")) + @isdefined(supertype) || throw(LoweringError(ex, "invalid type signature")) + + return (name, type_params, supertype) +end + +# Expand type_params into (typevar_names, typevar_stmts) where +# - `typevar_names` are the names of the type's type parameters +# - `typevar_stmts` are a list of statements to define a `TypeVar` for each parameter +# name in `typevar_names`, to be emitted prior to uses of `typevar_names`. +function expand_typevars(ctx, type_params) + typevar_names = SyntaxList() + typevar_stmts = SyntaxList() + for param in type_params + bounds = typevar_bounds(param) + n = bounds[1] + push!(typevar_names, n) + push!(typevar_stmts, @ast ctx param [K"block" + [K"local" n] + [K"=" n bounds_to_typevar(ctx, bounds)] + ]) + end + return (typevar_names, typevar_stmts) +end + +function expand_abstract_or_primitive_type(ctx, ex) + is_abstract = kind(ex) == K"abstract" + if is_abstract + @jl_assert numchildren(ex) == 1 ex + else + @jl_assert kind(ex) == K"primitive" ex + @jl_assert numchildren(ex) == 2 ex + end + nbits = is_abstract ? nothing : ex[2] + name, type_params, supertype = analyze_type_sig(ctx, ex[1]) + name, _ = relayer_global_if_unhygienic(ctx, name) + typevar_names, typevar_stmts = expand_typevars(ctx, type_params) + newtype_var = ssavar(ctx, ex, "new_type") + @ast ctx ex [K"block" + [K"scope_block" [K"hard_scope"] + [K"block" + [K"local" name] + [K"always_defined" name] + typevar_stmts... + [K"=" + newtype_var + [K"call" + (is_abstract ? "_abstracttype" : "_primitivetype")::K"core" + syntax_module(name)::K"Value" + name=>K"Symbol" + [K"call" "svec"::K"core" typevar_names...] + if !is_abstract + nbits + end + ] + ] + [K"=" name newtype_var] + [K"call" "_setsuper!"::K"core" newtype_var supertype] + [K"call" "_typebody!"::K"core" name] + ] + ] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex] ] + [K"global" name] + [K"if" + [K"&&" + [K"call" + "isdefinedglobal"::K"core" + syntax_module(name)::K"Value" + name=>K"Symbol" + false::K"Bool"] + [K"call" "_equiv_typedef"::K"core" name newtype_var] + ] + nothing_(ctx, ex) + [K"constdecl" name newtype_var] + ] + nothing_(ctx, ex) + ] +end + +function _match_struct_field(x0) + type=nothing + docs=nothing + atomic=false + _const=false + x = x0 + while true + k = kind(x) + if k == K"Identifier" || k == K"Placeholder" + return (name=x, type=type, atomic=atomic, _const=_const, docs=docs) + elseif k == K"::" && numchildren(x) == 2 + isnothing(type) || throw(LoweringError(x0, "multiple types in struct field")) + type = x[2] + x = x[1] + elseif k == K"atomic" + atomic = true + x = x[1] + elseif k == K"const" + _const = true + x = x[1] + elseif k == K"doc" + docs = x[1] + x = x[2] + else + return nothing + end + end +end + +function _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, inner_defs, exs) + for e in exs + if kind(e) == K"block" + _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, + inner_defs, children(e)) + else + m = _match_struct_field(e) + if !isnothing(m) + # Struct field + for prev in field_names + if syntax_name(prev) == syntax_name(m.name) + throw(LoweringError(m.name, "duplicate field name")) + end + end + push!(field_names, m.name) + n = length(field_names) + push!(field_types, isnothing(m.type) ? @ast(ctx, e, "Any"::K"core") : m.type) + if m.atomic + push!(field_attrs, @ast ctx e n::K"Integer") + push!(field_attrs, @ast ctx e "atomic"::K"Symbol") + end + if m._const + push!(field_attrs, @ast ctx e n::K"Integer") + push!(field_attrs, @ast ctx e "const"::K"Symbol") + end + if !isnothing(m.docs) + push!(field_docs, @ast ctx e n::K"Integer") + push!(field_docs, @ast ctx e m.docs) + end + elseif kind(e) == K"string" || is_effect_free(e) + # effect-free code and docstrings should not add to `defs`, since + # that would prevent inner ctors from being generated + else + # Inner constructors and inner functions + # TODO: Disallow arbitrary expressions inside `struct`? + push!(inner_defs, e) + end + end + end +end + +# generate call to `convert()` for `(call new ...)` expressions +function _new_call_convert_arg(ctx, full_struct_type, field_type, field_index, val) + if is_core_Any(field_type) + return val + end + # kt = kind(field_type) + # TODO: Allow kt == K"Identifier" && kt in static_params to avoid fieldtype call? + @ast ctx field_type [K"block" + tmp_type := [K"call" + "fieldtype"::K"core" + full_struct_type + field_index::K"Integer" + ] + convert_for_type_decl(ctx, field_type, val, tmp_type, false) + ] +end + +function _is_new_call(ex) + kind(ex) == K"call" && + ((kind(ex[1]) == K"Identifier" && syntax_name(ex[1]) == "new") || + (kind(ex[1]) == K"curly" && kind(ex[1][1]) == K"Identifier" && syntax_name(ex[1][1]) == "new")) +end + +# Rewrite constructor signature, returning extra information needed for +# rewriting `new` calls in the body. Returns `(sig2, ctor_self)`, where: +# +# If `sig` is a constructor of `tname` like `tname{X,Y}(...)`, +# - sig2 is :((var"#ctor-self#"::Type{tname{X,Y}})(...)) +# - ctor_self is the symbol we generated above +# +# Otherwise, sig2 is sig, and ctor_self is nothing. +function rewrite_ctor_sig(ctx, sig, tname, global_tname, struct_typevars, wheres) + sig2 = sig + ctor_self = nothing + @stm sig begin + [K"::" x rett] -> let + call2, ctor_self = rewrite_ctor_sig( + ctx, x, tname, global_tname, struct_typevars, SyntaxList()) + sig2 = @ast(ctx, sig, [K"::" call2 rett]) + end + # recognize `(_::(Type{X{T}} where T))(...)` as an inner-style + # constructor for X (rewrite it to `X{T}(...) where T`) + ([K"call" [K"::" _ [K"where" _...]] args...], when=begin + t, inner_wheres = flatten_wheres(ex[1][2]) + isempty(wheres) && kind(t) === K"curly" && t[1].value === "Type" + end) -> let + append!(wheres, inner_wheres) + ex2 = @ast ctx ex [K"call" t[2] args...] + return rewrite_ctor_sig( + ctx, ex2, tname, global_tname, struct_typevars, wheres) + end + [K"call" [K"curly" name curlyargs...] args...] -> let + # if curlyargs is the wrong length, fall back to the ones in `new` + # TODO: this isn't quite the same as flisp, which passes curlyargs + # to new-call and checks there. We print the wrong message with + # `struct X{T}; X{T,U}() = new(); end`. + if (kind(name) !== K"::" && is_same_identifier_like(name, tname) && + length(curlyargs) == length(struct_typevars)) + @jl_assert is_leaf(name) (sig, "didn't find ctor name in sig") + ctor_self = newsym(ctx, sig, "#ctor-self#") + sig2 = @ast ctx sig [K"call" + [K"::" ctor_self + [K"curly" "Type"::K"core" + [K"curly" global_tname curlyargs...]]] + args...] + end + end + [K"call" name args...] -> let + if kind(name) !== K"::" && is_same_identifier_like(name, tname) + @jl_assert is_leaf(name) (sig, "didn't find ctor name in sig") + ctor_self = newsym(ctx, sig, "#ctor-self#") + sig2 = @ast ctx sig [K"call" + [K"::" ctor_self [K"curly" "Type"::K"core" global_tname]] + args...] + end + end + # anonymous function + [K"tuple" _...] -> (sig, nothing) + end + sig_out = isempty(wheres) ? sig2 : @ast ctx sig [K"where" sig2 wheres...] + return sig_out, ctor_self +end + +# Rewrite calls to `new` in bodies of inner constructors and inner functions +# into `new` or `splatnew` expressions. For example: +# +# struct X{T,S} +# X() = new() +# X() = new{A,B}() +# X{T,S}() where {T,S} = new() +# X{A,B}() = new() +# X{A}() = new() +# (t::Type{X})() = new{A,B}() +# f() = new() +# f() = new{A,B}() +# f() = new{Ts...}() +# end +# +# Map to the following +# +# X() = ERROR +# (#ctor-self#::Type{X})() = (new X{A,B}) +# (Type{X{T,S}}() where {T,S} = (new #ctor-self#) +# X{A,B}() = (new #ctor-self#) +# X{A}() = ERROR +# (t::Type{X})() = (new X{A,B}) +# f() = ERROR +# f() = (new X{A,B}) +# f() = (new X{Ts...}) +# +# TODO: Arguably the following "could also work", but any symbolic match of +# this case would be heuristic and rely on assuming Type == Core.Type. So +# runtime checks would really be required and flisp lowering doesn't catch +# this case either. +# +# (t::Type{X{A,B}})() = new() +function rewrite_ctor(ctx, ex, tname, global_tname, struct_typevars, field_types) + is_leaf(ex) && return ex + @stm ex begin + [K"inert" _] -> ex + [K"function" call body] -> let (sig, wheres) = flatten_wheres(call) + call2, ctor_self = + rewrite_ctor_sig(ctx, sig, tname, global_tname, struct_typevars, wheres) + body2 = _rewrite_ctor_new_calls( + ctx, body, global_tname, + mapsyntax(typevar_bounds, wheres), + struct_typevars, ctor_self, field_types) + @ast ctx ex [K"function" call2 body2] + end + x -> mapchildren(e->rewrite_ctor( + ctx, e, tname, global_tname, struct_typevars, field_types), ex) + end +end + +# possible TODO: flisp does rewrites +# new(args...) => new_call( +# global_tname, (), ctor_sparams, struct_typevars, map(rewrite, args), field_types, ctor_self) +# new{new_curlyargs...}(args...) => new_call( +# global_tname, new_curlyargs, ctor_sparams, struct_typevars, map(rewrite, args), field_types, ctor_self) +# +# This function should do as much as `new-call`, but does not use curlyargs +# or ctor_sparams, so may be missing something. +function _rewrite_ctor_new_calls(ctx, ex0, global_struct_name, ctor_sparams, + struct_typevars, ctor_self, field_types) + if is_leaf(ex0) + return ex0 + elseif !_is_new_call(ex0) + return mapchildren( + e->_rewrite_ctor_new_calls(ctx, e, global_struct_name, ctor_sparams, + struct_typevars, ctor_self, field_types), + ex0 + ) + end + # Rewrite a call to new() + e0args = children(ex0) + kw_arg_i = findfirst(e->(k = kind(e); k == K"kw"), e0args) + ex = if !isnothing(kw_arg_i) + throw(LoweringError(e0args[kw_arg_i], "`new` does not accept keyword arguments")) + elseif kind(e0args[end]) === K"parameters" # flisp oversight + if is_flisp_compat(ex0) + @mknode(ex0; children=e0args[1:end-1]) + else + throw(LoweringError(e0args[end], "`new` does not accept keyword arguments")) + end + else + ex0 + end + full_struct_type = if kind(ex[1]) == K"curly" + # new{A,B}(...) + new_type_params = ex[1][2:end] + n_type_splat = sum(kind(t) == K"..." for t in new_type_params; init=0) + n_type_nonsplat = length(new_type_params) - n_type_splat + if n_type_splat == 0 && n_type_nonsplat < length(struct_typevars) + throw(LoweringError(ex[1], "too few type parameters specified in `new{...}`")) + elseif n_type_nonsplat > length(struct_typevars) + throw(LoweringError(ex[1], "too many type parameters specified in `new{...}`")) + end + isempty(new_type_params) ? global_struct_name : + @ast ctx ex[1] [K"curly" global_struct_name new_type_params...] + elseif !isnothing(ctor_self) + # new(...) in constructors + ctor_self + else + # new(...) inside non-constructor inner functions + if isempty(struct_typevars) + global_struct_name + else + throw(LoweringError(ex[1], "too few type parameters specified in `new`")) + end + end + new_args = ex[2:end] + n_splat = sum(kind(t) == K"..." for t in new_args; init=0) + n_nonsplat = length(new_args) - n_splat + n_fields = length(field_types) + function throw_n_fields_error(desc) + @ast ctx ex [K"call" + "throw"::K"core" + [K"call" + "ArgumentError"::K"top" + "too $desc arguments in `new` (expected $n_fields)"::K"String" + ] + ] + end + if n_nonsplat > n_fields + return throw_n_fields_error("many") + else + # "Too few" args are allowed in partially initialized structs + end + if n_splat == 0 + @ast ctx ex [K"block" + struct_type := full_struct_type + [K"new" + struct_type + [_new_call_convert_arg(ctx, struct_type, type, i, name) + for (i, (name,type)) in enumerate(zip(ex[2:end], field_types))]... + ] + ] + else + fields_all_Any = all(is_core_Any, field_types) + if fields_all_Any + @ast ctx ex [K"block" + struct_type := full_struct_type + [K"splatnew" + struct_type + # Note: `jl_new_structt` ensures length of this tuple is + # exactly the number of fields. + [K"call" "tuple"::K"core" ex[2:end]...] + ] + ] + else + # `new` with splatted args which are symbolically not `Core.Any` + # (might be `Any` at runtime but we can't know that here.) + @ast ctx ex [K"block" + args := [K"call" "tuple"::K"core" ex[2:end]...] + n_args := [K"call" "nfields"::K"core" args] + [K"if" + [K"call" "ult_int"::K"top" n_args n_fields::K"Integer"] + throw_n_fields_error("few") + ] + [K"if" + [K"call" "ult_int"::K"top" n_fields::K"Integer" n_args] + throw_n_fields_error("many") + ] + struct_type := full_struct_type + [K"new" + struct_type + [_new_call_convert_arg(ctx, struct_type, type, i, + [K"call" "getfield"::K"core" args i::K"Integer"]) + for (i, type) in enumerate(field_types)]... + ] + ] + end + end +end + +function _constructor_min_initialized(ex::SyntaxTree) + if _is_new_call(ex) + if any(kind(e) == K"..." for e in ex[2:end]) + # Lowering ensures new with splats always inits all fields + # or in the case of splatnew this is enforced by the runtime. + typemax(Int) + else + numchildren(ex) - 1 + end + elseif !is_leaf(ex) + minimum((_constructor_min_initialized(e) for e in children(ex)), init=typemax(Int)) + else + typemax(Int) + end +end + +# Let S be a struct we're defining in module M. Below is a hack to allow its +# field types to refer to S as M.S. See #56497. +function _insert_fieldtype_struct_shim(ctx, sname, ex) + if kind(ex) == K"." && + numchildren(ex) == 2 && + kind(ex[2]) == K"Symbol" && + syntax_name(ex[2]) == syntax_name(sname) + @ast ctx ex [K"call" "struct_name_shim"::K"core" ex[1] ex[2] syntax_module(ex)::K"Value" sname] + elseif numchildren(ex) > 0 + mapchildren(e->_insert_fieldtype_struct_shim(ctx, sname, e), ex) + else + ex + end +end + +function insert_struct_shim(ctx, fieldtypes, name) + map(ex->_insert_fieldtype_struct_shim(ctx, name, ex), fieldtypes) +end + +# Used to handle TypeVar/TypeApp references during type resolution before real +# DataTypes exist. flisp: "Skips method bodies since constructors should use +# plain apply_type for correct effects inference." +function _replace_type_constructors(ctx, ex) + if is_leaf(ex) + return ex + end + k = kind(ex) + if k == K"call" && numchildren(ex) >= 1 && kind(ex[1]) == K"core" && syntax_name(ex[1]) == "apply_type" + new_head = @ast ctx ex[1] "apply_type_or_typeapp"::K"core" + new_children = SyntaxList() + push!(new_children, new_head) + for i in 2:numchildren(ex) + push!(new_children, _replace_type_constructors(ctx, ex[i])) + end + return @ast ctx ex [K"call" new_children...] + elseif k === K"method" || is_quoted(ex) + ex + else + return mapchildren(e->_replace_type_constructors(ctx, e), ex) + end +end + +struct TypeGroupEntry + sdef # struct definition syntax node + docs # nothing or K"doc" node + typevar_names # typevar names for this struct + typevar_stmts # typevar creation statements + field_names # field name syntax nodes + field_types # field type expressions + field_attrs # field attribute expressions + supertype # supertype expression + is_mutable::Bool + min_initialized::Int + inner_defs # inner constructor definitions + field_docs # field documentation +end + +function expand_typegroup_def(ctx, ex) + @jl_assert numchildren(ex) == 1 ex + body = flatten_blocks(ex[1]) + if kind(body) != K"block" + throw(LoweringError(body, "expected block for `typegroup` body")) + end + + # Collect and analyze struct definitions from block children. + # A child can be a bare K"struct" or a K"doc" wrapping a K"struct". + entries = TypeGroupEntry[] + struct_names = SyntaxList() # local name bindings (splatted into AST) + global_names = SyntaxList() # global name bindings (splatted into AST) + info_vars = SyntaxList() # SSA vars for struct info svecs (splatted into AST) + struct_mod_prev = nothing + + for child in children(body) + if kind(child) == K"struct" + sdef = child + docs = nothing + elseif kind(child) == K"doc" + @jl_assert numchildren(child) == 2 child + sdef = child[2] + if kind(sdef) != K"struct" + throw(LoweringError(sdef, "`typegroup` only supports `struct` definitions")) + end + docs = child + else + throw(LoweringError(child, "`typegroup` only supports `struct` definitions")) + end + + @jl_assert numchildren(sdef) == 3 sdef + is_mutable = sdef[1].value::Bool + type_sig = sdef[2] + type_body = sdef[3] + if kind(type_body) != K"block" + throw(LoweringError(type_body, "expected block for `struct` fields")) + end + struct_name, type_params, supertype = analyze_type_sig(ctx, type_sig) + typevar_names, typevar_stmts = expand_typevars(ctx, type_params) + field_names = SyntaxList() + field_types = SyntaxList() + field_attrs = SyntaxList() + field_docs = SyntaxList() + inner_defs = SyntaxList() + _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, + inner_defs, children(type_body)) + + min_initialized = minimum((_constructor_min_initialized(e) for e in inner_defs), + init=length(field_names)) + + push!(entries, TypeGroupEntry(sdef, docs, typevar_names, typevar_stmts, + field_names, field_types, field_attrs, + supertype, is_mutable, min_initialized, + inner_defs, field_docs)) + push!(struct_names, struct_name) + global_struct_name, _ = relayer_global_if_unhygienic(ctx, struct_name) + struct_mod = syntax_module(global_struct_name) + isnothing(struct_mod_prev) || struct_mod == struct_mod_prev || throw( + LoweringError(ex, "typegroup of types from multiple modules")) + struct_mod_prev = struct_mod + struct_globalref = @mknode(global_struct_name; mod=struct_mod) + push!(global_names, struct_globalref) + push!(info_vars, ssavar(ctx, sdef, "struct_info")) + end + n = length(entries) + if n == 0 + return nothing_(ctx, ex) + end + typegroup_mod = syntax_module(global_names[1]) + + # Build the lowered code + # + # Structure: + # 1. Assert toplevel-only + # 2. scope_block(hard) { + # a. Declare all names as locals + # b. Create TypeVar placeholders for each name + # c. For each struct: create TypeVar params, collect info into svec + # d. Call resolve_typegroup + # e. Bind to global constants + # f. latestworld + # g. Constructor definitions + # } + + stmts = SyntaxList() + + # 2a. Declare all names as locals + for name in struct_names + push!(stmts, @ast ctx name [K"local" name]) + end + + # 2b. Create TypeVar placeholders for each name + for name in struct_names + push!(stmts, @ast ctx name [K"=" name [K"call" "TypeVar"::K"core" name=>K"Symbol"]]) + end + + # 2c. For each struct: create scope_block with TypeVar params and collect info into svec + for i in 1:n + e = entries[i] + typevar_names = e.typevar_names + typevar_stmts = e.typevar_stmts + info_var = info_vars[i] + + inner_stmts = SyntaxList() + for tv_name in typevar_names + push!(inner_stmts, @ast ctx e.sdef [K"local" tv_name]) + end + append!(inner_stmts, typevar_stmts) + push!(inner_stmts, @ast ctx e.sdef [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" e.sdef]]) + push!(inner_stmts, @ast ctx e.sdef [K"=" + info_var + [K"call" "svec"::K"core" + [K"call" "svec"::K"core" typevar_names...] + [K"call" "svec"::K"core" [fname=>K"Symbol" for fname in e.field_names]...] + [K"call" "svec"::K"core" e.field_attrs...] + e.is_mutable::K"Bool" + e.min_initialized::K"Integer" + e.supertype + [K"call" "svec"::K"core" e.field_types...] + ] + ]) + + push!(stmts, @ast ctx e.sdef [K"scope_block" [K"hard_scope"] + [K"block" inner_stmts...] + ]) + end + + # 2d. Look up old types for redefinition equivalence check + old_type_vars = SyntaxList() + for i in 1:n + old_var = ssavar(ctx, ex, "old_type") + push!(stmts, @ast ctx ex [K"=" + old_var + [K"if" + [K"call" "isdefinedglobal"::K"core" + typegroup_mod::K"Value" + struct_names[i]=>K"Symbol" + false::K"Bool"] + global_names[i] + nothing_(ctx, ex) + ] + ]) + push!(old_type_vars, old_var) + end + # 2e. Call resolve_typegroup + resolve_tmp = ssavar(ctx, ex) + push!(stmts, @ast ctx ex [K"=" resolve_tmp + [K"call" "resolve_typegroup"::K"core" + typegroup_mod::K"Value" + [K"call" "svec"::K"core" struct_names...] + [K"call" "svec"::K"core" info_vars...] + [K"call" "svec"::K"core" old_type_vars...] + ] + ]) + + # 2f. Bind to global constants + for i in 1:n + prov = entries[i].sdef + push!(stmts, @ast ctx prov [K"=" struct_names[i] + [K"call" "getfield"::K"core" resolve_tmp i::K"Value"]]) + push!(stmts, @ast ctx prov [K"constdecl" global_names[i] struct_names[i]]) + end + + # 2f. latestworld + push!(stmts, @ast ctx ex (::K"latestworld")) + push!(stmts, nothing_(ctx, ex)) + + # 2g. Constructor definitions — placed outside the scope_block so that + # type names in constructor bodies resolve to globals, not captured locals. + fdef_stmts = SyntaxList() + for i in 1:n + e = entries[i] + if isempty(e.inner_defs) + push!(fdef_stmts, @ast ctx e.sdef [K"call" + "_defaultctors"::K"top" + global_names[i] + ::K"SourceLocation"(e.sdef) + ]) + else + inner_defs = e.inner_defs + for (def_i, def) in enumerate(inner_defs) + inner_defs[def_i] = + rewrite_ctor(ctx, def, struct_names[i], global_names[i], + e.typevar_names, e.field_types) + end + push!(fdef_stmts, @ast ctx e.sdef [K"scope_block" [K"hard_scope"] + [K"block" inner_defs...] + ]) + end + end + + push!(fdef_stmts, @ast ctx ex (::K"latestworld")) + + # 2h. Documentation — after constructors and latestworld so types are fully defined + for i in 1:n + e = entries[i] + if !isnothing(e.docs) || !isempty(e.field_docs) + push!(fdef_stmts, @ast ctx e.sdef [K"call"(isnothing(e.docs) ? e.sdef : e.docs) + bind_docs!::K"Value" + struct_names[i] + isnothing(e.docs) ? nothing_(ctx, e.sdef) : e.docs[1] + ::K"SourceLocation"(e.sdef) + [K"kw" + "field_docs"::K"Identifier" + [K"call" "svec"::K"core" e.field_docs...] + ] + ]) + end + end + + push!(fdef_stmts, nothing_(ctx, ex)) + + result = @ast ctx ex [K"block" + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] + mapsyntax(x->@ast(ctx, x, [K"global" x]), struct_names)... + [K"scope_block" [K"hard_scope"] [K"block" stmts...]] + fdef_stmts... + ] + + # Expand, then replace apply_type with apply_type_or_typeapp + expanded = expand_forms_2(ctx, result) + return _replace_type_constructors(ctx, expanded) +end + +function expand_struct_def(ctx, ex, docs) + @jl_assert numchildren(ex) == 3 ex + is_mutable = ex[1].value::Bool + type_sig = ex[2] + type_body = flatten_blocks(ex[3]) + if kind(type_body) != K"block" + throw(LoweringError(type_body, "expected block for `struct` fields")) + end + struct_name, type_params, supertype = analyze_type_sig(ctx, type_sig) + typevar_names, typevar_stmts = expand_typevars(ctx, type_params) + field_names = SyntaxList() + field_types = SyntaxList() + field_attrs = SyntaxList() + field_docs = SyntaxList() + inner_defs = SyntaxList() + _collect_struct_fields(ctx, field_names, field_types, field_attrs, field_docs, + inner_defs, children(type_body)) + min_initialized = minimum((_constructor_min_initialized(e) for e in inner_defs), + init=length(field_names)) + global_struct_name, _ = relayer_global_if_unhygienic(ctx, struct_name) + struct_mod = syntax_module(global_struct_name) + struct_globalref = @mknode(global_struct_name; mod=struct_mod) + + # Use the typegroup mechanism for ordinary structs to ensure safety + # when accessing incomplete types during definition (issue #60919). + # The struct name is a TypeVar placeholder during field type evaluation, + # preventing segfaults from accessing incomplete types. + info_var = ssavar(ctx, ex, "struct_info") + + stmts = SyntaxList() + + # Declare struct name as local and create TypeVar placeholder + push!(stmts, @ast ctx struct_name [K"local" struct_name]) + push!(stmts, @ast ctx struct_name [K"=" struct_name [K"call" "TypeVar"::K"core" struct_name=>K"Symbol"]]) + + # Inner scope_block for type parameters + info collection + inner_stmts = SyntaxList() + for tv_name in typevar_names + push!(inner_stmts, @ast ctx ex [K"local" tv_name]) + end + append!(inner_stmts, typevar_stmts) + push!(inner_stmts, @ast ctx ex [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]]) + push!(inner_stmts, @ast ctx ex [K"=" + info_var + [K"call" "svec"::K"core" + [K"call"(type_sig) "svec"::K"core" typevar_names...] + [K"call"(type_body) "svec"::K"core" [n=>K"Symbol" for n in field_names]...] + [K"call"(type_body) "svec"::K"core" field_attrs...] + is_mutable::K"Bool" + min_initialized::K"Integer" + supertype + [K"call" "svec"::K"core" insert_struct_shim(ctx, field_types, struct_name)...] + ] + ]) + push!(stmts, @ast ctx ex [K"scope_block" [K"hard_scope"] + [K"block" inner_stmts...] + ]) + + # Look up old type for redefinition equivalence check + old_type_var = ssavar(ctx, ex, "old_type") + push!(stmts, @ast ctx ex [K"=" + old_type_var + [K"if" + [K"call" "isdefinedglobal"::K"core" + struct_mod::K"Value" + struct_name=>K"Symbol" + false::K"Bool"] + struct_globalref + nothing_(ctx, ex) + ] + ]) + + # Call resolve_typegroup and extract the single result with getfield + push!(stmts, @ast ctx ex [K"=" + struct_name + [K"call" "getfield"::K"core" + [K"call" "resolve_typegroup"::K"core" + struct_mod::K"Value" + [K"call" "svec"::K"core" struct_name] + [K"call" "svec"::K"core" info_var] + [K"call" "svec"::K"core" old_type_var] + ] + 1::K"Integer" + ] + ]) + + # Bind to global constant + push!(stmts, @ast ctx ex [K"constdecl" struct_globalref struct_name]) + + # latestworld + nothing + push!(stmts, @ast ctx ex (::K"latestworld")) + push!(stmts, nothing_(ctx, ex)) + + # Constructor definitions — placed outside the scope_block so that + # type names in constructor bodies resolve to globals, not captured locals. + fdef_stmts = SyntaxList() + if isempty(inner_defs) + push!(fdef_stmts, @ast ctx ex [K"call" + "_defaultctors"::K"top" + struct_globalref + ::K"SourceLocation"(ex) + ]) + else + # For all functions within `struct`, rewrite `new` calls and + # constructor-like signatures + for (def_i, def) in enumerate(inner_defs) + inner_defs[def_i] = + rewrite_ctor(ctx, def, struct_name, struct_globalref, + typevar_names, field_types) + end + push!(fdef_stmts, @ast ctx ex [K"scope_block" [K"hard_scope"] + [K"block" inner_defs...] + ]) + end + push!(fdef_stmts, @ast ctx ex (::K"latestworld")) + + # Documentation — after constructors and latestworld so types are fully defined + if !isnothing(docs) || !isempty(field_docs) + push!(fdef_stmts, @ast ctx ex [K"call"(isnothing(docs) ? ex : docs) + bind_docs!::K"Value" + struct_name + isnothing(docs) ? nothing_(ctx, ex) : docs[1] + ::K"SourceLocation"(ex) + [K"kw" + "field_docs"::K"Identifier" + [K"call" "svec"::K"core" field_docs...] + ] + ]) + end + push!(fdef_stmts, nothing_(ctx, ex)) + + result = @ast ctx ex [K"block" + [K"global" struct_name] + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] + [K"scope_block" [K"hard_scope"] stmts...] + fdef_stmts... + ] + + # Expand, then replace apply_type with apply_type_or_typeapp + expanded = expand_forms_2(ctx, result) + return _replace_type_constructors(ctx, expanded) +end + +#------------------------------------------------------------------------------- +# Expand `where` syntax + +function expand_where(ctx, srcref, lhs, rhs) + bounds = typevar_bounds(rhs) + v = bounds[1] + @ast ctx srcref [K"let" + [K"block" [K"=" v bounds_to_typevar(ctx, bounds)]] + [K"call" "UnionAll"::K"core" v lhs] + ] +end + +function expand_wheres(ctx, ex) + body = ex[1] + @stm ex begin + [K"where" _ [K"_typevars" [K"block" names...] [K"block" stmts...]]] -> + for n in Iterators.reverse(names) + body = @ast ctx ex [K"call" "UnionAll"::K"core" n body] + end + [K"where" _ tvs...] -> + for v in Iterators.reverse(tvs) + body = expand_where(ctx, ex, body, v) + end + end + body +end + +# Match implicit where parameters for `Foo{<:Bar}` ==> `Foo{T} where T<:Bar` +function expand_curly(ctx, ex) + @jl_assert kind(ex) == K"curly" ex + check_no_parameters(ex, "unexpected semicolon in type parameter list") + check_no_assignment(children(ex), "misplaced assignment in type parameter list") + + typevar_stmts = SyntaxList() + type_args = SyntaxList() + implicit_typevars = SyntaxList() + + i = 1 + for e in children(ex) + k = kind(e) + if (k == K"<:" || k == K">:") && numchildren(e) == 1 + # `X{<:A}` and `X{>:A}` + name = @ast ctx e "#T$i"::K"Placeholder" + i += 1 + any = @ast ctx ex "Any"::K"core" + typevar = k == K"<:" ? + _bounds_to_typevar(ctx, e, name, any, e[1]) : + _bounds_to_typevar(ctx, e, name, e[1], any) + arg = emit_assign_tmp(typevar_stmts, ctx, typevar) + push!(implicit_typevars, arg) + else + arg = e + end + push!(type_args, arg) + end + + type = @ast ctx ex [K"call" "apply_type"::K"core" type_args...] + if !isempty(implicit_typevars) + type = @ast ctx ex [K"block" + typevar_stmts... + [K"where" type [K"_typevars" [K"block" implicit_typevars...] [K"block" typevar_stmts...]]] + ] + end + + return type +end + +#------------------------------------------------------------------------------- +# Expand import / using / export + +function expand_importpath(ctx, path) + @jl_assert kind(path) == K"importpath" path + @ast ctx path [K"." mapsyntax(_unplaceholder, children(path))...] +end + +function _unplaceholder(st) + k = kind(st) + k === K"Placeholder" || k === K"Symbol" ? @mknode(st; kind=K"Identifier") : + k === K"Identifier" ? st : @jl_assert false st +end + +# importer does not obey hygiene. Doesn't bother with relayering any imported +# items, as the runtime functions don't see hygiene anyway +function expand_import_or_using(ctx, ex) + if kind(ex[1]) == K":" + # import M: x.y as z, w + # (import (: (importpath M) (as (importpath x y) z) (importpath w))) + # => + # (call module_import + # false + # (call core.svec "M") + # (call core.svec 2 "x" "y" "z" 1 "w" "w")) + @jl_assert numchildren(ex[1]) >= 1 ex + from = ex[1][1] + from_path = @ast ctx from [K"inert" expand_importpath(ctx, from)] + paths = ex[1][2:end] + else + # import A.B + # (using (importpath A B)) + # (call eval_import true nothing (call core.svec 1 "w")) + @jl_assert numchildren(ex) >= 1 ex + from_path = nothing + paths = children(ex) + end + # Here we represent the paths as quoted `Expr` data structures + path_specs = SyntaxList() + for spec in paths + if kind(spec) == K"as" + @jl_assert numchildren(spec) == 2 spec + s2 = _unplaceholder(spec[2]) + path = @ast ctx spec [K"as" expand_importpath(ctx, spec[1]) s2] + else + path = expand_importpath(ctx, spec) + end + push!(path_specs, @ast ctx spec [K"inert" path]) + end + is_using = kind(ex) == K"using" + stmts = SyntaxList() + if isnothing(from_path) + for spec in path_specs + if is_using + push!(stmts, + @ast ctx spec [K"call" + eval_using ::K"Value" + ctx.layer.mod::K"Value" + spec + ] + ) + else + push!(stmts, + @ast ctx spec [K"call" + eval_import ::K"Value" + (!is_using) ::K"Bool" + ctx.layer.mod::K"Value" + (::K"nothing") + spec + ] + ) + end + # latestworld required between imports so that previous symbols + # become visible + push!(stmts, @ast ctx spec (::K"latestworld")) + end + else + push!(stmts, @ast ctx ex [K"call" + eval_import ::K"Value" + (!is_using) ::K"Bool" + ctx.layer.mod::K"Value" + from_path + path_specs... + ]) + push!(stmts, @ast ctx ex (::K"latestworld")) + end + @ast ctx ex [K"block" + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] + stmts... + [K"removable" (::K"nothing")] + ] +end + +# flisp: export is relayered, and no-esc public is a syntax error (we relayer) +function expand_public(ctx, ex) + identifiers = String[] + numchildren(ex) == 0 && return @ast ctx ex (::K"nothing") + mod = syntax_module(relayer_global_if_unhygienic(ctx, ex[1])[1]) + for e in children(ex) + @jl_assert kind(e) == K"Identifier" (ex, "Expected identifier") + syntax_module(relayer_global_if_unhygienic(ctx, e)[1]) !== mod && + throw(LoweringError( + ex, "unexpected public/export with names from multiple modules")) + push!(identifiers, syntax_name(e)) + end + @ast ctx ex [K"call" + eval_public::K"Value" + mod::K"Value" + (kind(ex) == K"export")::K"Bool" + identifiers::K"Value" + ] +end + +#------------------------------------------------------------------------------- +# Expand docstring-annotated expressions + +function isquotedmacrocall(ex) + kind(ex) == K"call" || return false + numchildren(ex) == 3 || return false + let (f, ex) = (ex[1], ex[3]) + kind(f) == K"Value" || return false + kind(ex) == K"inert" || return false + f.value === interpolate_expr || return false + kind(ex[1]) == K"macrocall" || return false + return true + end +end + +function expand_doc(ctx, ex, docex) + if kind(ex) in (K"Identifier", K".") + expand_forms_2(ctx, @ast ctx docex [K"call" + bind_static_docs!::K"Value" + (kind(ex) === K"." ? ex[1] : syntax_module(ex)::K"Value") + syntax_name((kind(ex) === K"." ? ex[2] : ex))::K"Symbol" + docex[1] + ::K"SourceLocation"(ex) + Union{}::K"Value" + ]) + elseif isquotedmacrocall(ex) + # TODO: implement proper `doc!` support here + expand_forms_2(ctx, ex, docex) + elseif is_eventually_call(ex) + TODO("docsystem rewrite") + else + expand_forms_2(ctx, ex, docex) + end +end + +#------------------------------------------------------------------------------- +# Desugaring's "big switch": expansion of some simple forms; dispatch to other +# expansion functions for the rest. + +""" +Lowering pass 2 - desugaring + +This pass simplifies expressions by expanding complicated syntax sugar into a +small set of core syntactic forms. For example, field access syntax `a.b` is +expanded to a function call `getproperty(a, :b)`. +""" +function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing) + @nospecialize docs + k = kind(ex) + if k == K"atomic" + throw(LoweringError(ex, "unimplemented or unsupported atomic declaration")) + elseif k == K"call" + expand_call(ctx, ex) + elseif k == K"dotcall" || k == K".&&" || k == K".||" || k == K".=" + expand_forms_2(ctx, expand_fuse_broadcast(ctx, ex)) + elseif k == K"." + expand_forms_2(ctx, expand_dot(ctx, ex)) + elseif k == K"?" + @jl_assert numchildren(ex) == 3 ex + expand_forms_2(ctx, @ast ctx ex [K"if" children(ex)...]) + elseif k == K"&&" || k == K"||" + cs = expand_cond_children(ctx, ex) + isempty(cs) && return @ast ctx ex (k === K"&&")::K"Bool" + length(cs) == 1 && return @ast ctx ex cs[1] + # Attributing correct provenance for `cs[1:end-1]` is tricky in cases + # like `a && (b && c)` because the expression constructed here arises + # from the source fragment `a && (b` which doesn't follow the tree + # structure. For now we attribute to the parent node. + cond = length(cs) == 2 ? + cs[1] : + newnode(ex, k, cs[1:end-1]) + # This transformation assumes the type assertion `cond::Bool` will be + # added by a later compiler pass (currently done in codegen) + if k == K"&&" + @ast ctx ex [K"if" cond cs[end] false::K"Bool"] + else + @ast ctx ex [K"if" cond true::K"Bool" cs[end]] + end + elseif k == K"::" + @jl_assert numchildren(ex) == 2 (ex, "`::` must be written `value::type` outside function argument lists") + @ast ctx ex [K"call" + "typeassert"::K"core" + expand_forms_2(ctx, ex[1]) + expand_forms_2(ctx, ex[2]) + ] + elseif k == K"<:" || k == K">:" || k == K"-->" + expand_forms_2(ctx, @ast ctx ex [K"call" + adopt_scope(ex, string(k)::K"Identifier") + children(ex)... + ]) + elseif k == K"op=" || k == K".op=" + expand_forms_2(ctx, expand_update_operator(ctx, ex)) + elseif k == K"=" + expand_assignment(ctx, ex) + elseif k == K"break" + @stm ex begin + [K"break"] -> + @ast ctx ex [K"break" "loop-exit"::K"symboliclabel"] + [K"break" [K"Placeholder"]] -> + @ast ctx ex [K"break" "loop-exit"::K"symboliclabel"] + [K"break" [K"Identifier"]] -> begin + @ast ctx ex [K"break" ex[1]=>K"symboliclabel"] + end + [K"break" [K"Placeholder"] val] -> + @ast ctx ex [K"break" "loop-exit"::K"symboliclabel" + expand_forms_2(ctx, val)] + [K"break" [K"Identifier"] val] -> begin + @ast ctx ex [K"break" ex[1]=>K"symboliclabel" + expand_forms_2(ctx, val)] + end + end + elseif k == K"continue" + @stm ex begin + [K"continue"] -> + @ast ctx ex [K"break" "loop-cont"::K"symboliclabel"] + [K"continue" [K"Placeholder"]] -> + @ast ctx ex [K"break" "loop-cont"::K"symboliclabel"] + [K"continue" [K"Identifier"]] -> + @ast ctx ex [K"break" string(syntax_name(ex[1]), "#cont")::K"symboliclabel"] + end + elseif k == K"comparison" + expand_forms_2(ctx, expand_compare_chain(ctx, ex)) + elseif k == K"doc" + @jl_assert numchildren(ex) == 2 ex + expand_doc(ctx, ex[2], ex) + elseif k == K"for" + expand_forms_2(ctx, expand_for(ctx, ex)) + elseif k == K"comprehension" + @jl_assert numchildren(ex) == 1 ex + @jl_assert kind(ex[1]) == K"generator" ex + @ast ctx ex [K"call" + "collect"::K"top" + expand_forms_2(ctx, ex[1]) + ] + elseif k == K"typed_comprehension" + @jl_assert numchildren(ex) == 2 ex + @jl_assert kind(ex[2]) == K"generator" ex + if numchildren(ex[2]) == 2 && kind(ex[2][2]) == K"iteration" + # Hack to lower simple typed comprehensions to loops very early, + # greatly reducing the number of functions and load on the compiler + expand_forms_2(ctx, expand_comprehension_to_loops(ctx, ex)) + else + @ast ctx ex [K"call" + "collect"::K"top" + expand_forms_2(ctx, ex[1]) + expand_forms_2(ctx, ex[2]) + ] + end + elseif k == K"generator" + expand_forms_2(ctx, expand_generator(ctx, ex)) + elseif k == K"function" + if numchildren(ex) == 1 + return @ast ctx ex [K"block" + [K"global_if_global" ex[1]] + [K"function_decl" ex[1]] + [K"no_method_defs" ex[1]] + ex[1]] + end + sig, wheres = flatten_wheres(ex[1]) + name, args, rett = @stm sig begin + [K"::" [K"call" f as...] t] -> (f, as, t) + [K"call" f as...] -> (f, as, @ast(ctx, sig, "Any"::K"core")) + [K"tuple" as...] -> (nothing, as, @ast(ctx, sig, "Any"::K"core")) + end + if isnothing(name) + name = newsym(ctx, sig, "#anon#") + @ast ctx ex [K"block" [K"local" name] expand_function_def( + ctx, ex, SyntaxList(name, args...), wheres, ex[2], rett)] + else + expand_function_def( + ctx, ex, SyntaxList(name, args...), wheres, ex[2], rett) + end + elseif k == K"->" + sig, wheres = flatten_wheres(ex[1]) + @jl_assert kind(sig) === K"tuple" ex + name = newsym(ctx, sig, "#->#") + rett = @ast(ctx, sig, "Any"::K"core") + @ast ctx ex [K"block" [K"local" name] expand_function_def( + ctx, ex, SyntaxList(name, children(sig)...), wheres, ex[2], rett)] + elseif k == K"macro" + @ast ctx ex [K"block" + [K"assert" + "global_toplevel_only"::K"Symbol" + [K"syntaxinert" ex] + ] + expand_forms_2(ctx, expand_macro_def(ctx, ex)) + ] + elseif k == K"if" || k == K"elseif" + @jl_assert numchildren(ex) >= 2 ex + @ast ctx ex [k + expand_condition(ctx, ex[1]) + expand_forms_2(ctx, ex[2:end])... + ] + elseif k == K"let" + expand_forms_2(ctx, expand_let(ctx, ex)) + elseif k == K"const" + expand_const_decl(ctx, ex) + elseif k == K"local" || k == K"global" + expand_decls(ctx, ex) + elseif k == K"where" + expand_forms_2(ctx, expand_wheres(ctx, ex)) + elseif k == K"string" + if numchildren(ex) == 1 && kind(ex[1]) == K"String" + ex[1] + else + expand_forms_2(ctx, @ast ctx ex [K"call" + "string"::K"top" + children(ex)... + ]) + end + elseif k == K"try" + expand_forms_2(ctx, expand_try(ctx, ex)) + elseif k == K"tuple" + if has_parameters(ex) + if numchildren(ex) > 1 + throw(LoweringError(ex[end], "unexpected semicolon in tuple - use `,` to separate tuple elements")) + end + expand_forms_2(ctx, expand_named_tuple(ctx, ex, children(ex[1]))) + elseif any_assignment(children(ex)) + expand_forms_2(ctx, expand_named_tuple(ctx, ex, children(ex))) + else + expand_forms_2(ctx, @ast ctx ex [K"call" + "tuple"::K"core" + children(ex)... + ]) + end + elseif k == K"$" + throw(LoweringError(ex, "`\$` expression outside string or quote block")) + elseif k == K"module" + throw(LoweringError(ex, "`module` is only allowed at top level")) + elseif k == K"import" || k == K"using" + expand_import_or_using(ctx, ex) + elseif k == K"export" || k == K"public" + expand_public(ctx, ex) + elseif k == K"abstract" || k == K"primitive" + expand_forms_2(ctx, expand_abstract_or_primitive_type(ctx, ex)) + elseif k == K"struct" + expand_struct_def(ctx, ex, docs) + elseif k == K"typegroup" + expand_typegroup_def(ctx, ex) + elseif k == K"ref" + sctx = with_stmts(ctx) + (arr, idxs) = expand_ref_components(sctx, ex) + expand_forms_2(ctx, + @ast ctx ex [K"block" + sctx.stmts... + [K"call" + "getindex"::K"top" + arr + idxs... + ] + ] + ) + elseif k == K"curly" + expand_forms_2(ctx, expand_curly(ctx, ex)) + elseif k == K"toplevel" + # Temporary: It would make more sense to return this unchanged once + # toplevel iteration over SyntaxTree exists, but for now, a call to + # `eval` lets JuliaLowering retain provenance and hygiene here. + ex2 = @ast ctx ex [K"block" + [K"assert" "toplevel_only"::K"Symbol" [K"syntaxinert" ex]] + [K"call" + eval::K"Value" + # a macro expanding to toplevel does not change the eval module, + # but does change the name resolution module + ctx.layer.mod::K"Value" + [K"syntaxinert" ex] + ] + ] + expand_forms_2(ctx, ex2) + elseif k == K"vect" + check_no_parameters(ex, "unexpected semicolon in array expression") + expand_array(ctx, ex, "vect") + elseif k == K"hcat" + expand_array(ctx, ex, "hcat") + elseif k == K"typed_hcat" + expand_array(ctx, ex, "typed_hcat") + elseif k == K"opaque_closure" + expand_opaque_closure(ctx, ex) + elseif k == K"vcat" || k == K"typed_vcat" + expand_forms_2(ctx, expand_vcat(ctx, ex)) + elseif k == K"ncat" || k == K"typed_ncat" + expand_forms_2(ctx, expand_ncat(ctx, ex)) + elseif k == K"while" + @jl_assert numchildren(ex) == 2 ex + @ast ctx ex [K"symbolicblock" "loop-exit"::K"symboliclabel" + [K"_while" + expand_condition(ctx, ex[1]) + [K"symbolicblock" "loop-cont"::K"symboliclabel" + [K"scope_block" [K"neutral_scope"] + expand_forms_2(ctx, ex[2]) + ] + ] + ] + ] + elseif k == K"inert" || k == K"syntaxinert" || k == K"foreignsymbol" + ex + elseif k == K"foreignglobal" + @ast ctx ex [K"foreignglobal" expand_csymbol(ctx, ex[1])] + elseif k == K"foreigncall" + # Assume user macros may produce this, but static_eval means desugaring + # has already occurred. + args = SyntaxList() + for i in 2:numchildren(ex) + c = ex[i] + if kind(c) === K"static_eval" + push!(args, c) + elseif i <= 3 + push!(args, @ast ctx ex [K"static_eval" expand_forms_2(ctx, c)]) + else + push!(args, expand_forms_2(ctx, c)) + end + end + @ast ctx ex [K"foreigncall" expand_csymbol(ctx, ex[1]) args...] + elseif k == K"gc_preserve" + @ast ctx ex [K"block" + s := [K"gc_preserve_begin" children(ex)[2:end]...] + r := expand_forms_2(ctx, children(ex)[1]) + [K"gc_preserve_end" s] + r + ] + elseif k == K"&" + throw(LoweringError(ex, "invalid syntax")) + elseif k == K"$" + throw(LoweringError(ex, "`\$` expression outside string or quote")) + elseif k == K"..." + throw(LoweringError(ex, "`...` expression outside call")) + elseif k == K"ssavalue" + _resolve_ssavalue(ctx, ex) + elseif is_leaf(ex) + ex + elseif k == K"return" + if numchildren(ex) == 0 + @ast ctx ex [K"return" (::K"nothing")] + elseif numchildren(ex) == 1 + mapchildren(e->expand_forms_2(ctx,e), ex) + else + throw(LoweringError(ex, "More than one argument to return")) + end + else + mapchildren(e->expand_forms_2(ctx,e), ex) + end +end + +function expand_forms_2(ctx::DesugaringContext, exs::Union{Tuple,AbstractVector}) + res = SyntaxList() + for e in exs + push!(res, expand_forms_2(ctx, e)) + end + res +end + +@fzone "JL: desugar" function expand_forms_2(ex::SyntaxTree, world::UInt) + sl = base_layer(ex.context::SyntaxContext) + ctx_out = DesugaringContext(sl, Bindings(), Dict{Int, IdTag}(), world) + vr = valid_st1(ex) + # surface only one error until we have pretty-printing for multiple + if !vr.ok + throw(LoweringError(vr.errors[1].sts, vr.errors[1].msgs, false)) + end + ex_out = expand_forms_2(ctx_out, est_to_dst(ex)) + if DEBUG + vr = valid_st2(ex_out) + !vr.ok && throw(LoweringError(vr.errors[1].sts, vr.errors[1].msgs, true)) + end + ctx_out, ex_out +end diff --git a/JuliaLowering/src/eval.jl b/JuliaLowering/src/eval.jl new file mode 100644 index 0000000000000..7c976d0b146b0 --- /dev/null +++ b/JuliaLowering/src/eval.jl @@ -0,0 +1,877 @@ +# Non-incremental lowering API for non-toplevel non-module expressions. +# May be removed? + +function lower(mod::Module, ex_in::SyntaxTree; expr_compat_mode::Bool=false, + soft_scope::Union{Nothing,Bool}=nothing) + ver = expr_compat_mode ? JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION + ex0 = rebase_layers(ex_in, mod, ver) + world = Base.get_world_counter() + ex1 = expand_forms_1(ex0, world, true) + ctx2, ex2 = expand_forms_2(ex1, world) + ctx3, ex3 = resolve_scopes(ctx2, ex2; soft_scope) + ctx4, ex4 = convert_closures(ctx3, ex3) + _ctx5, ex5 = linearize_ir(ctx4, ex4) + ex5 +end + +function macroexpand(mod::Module, ex_in::SyntaxTree; + expr_compat_mode::Bool=false, + ver::VersionNumber=expr_compat_mode ? + JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION, + recursive::Bool=true) + ex0 = rebase_layers(ex_in, mod, ver) + expand_forms_1(ex0, Base.get_world_counter(), recursive) +end + +"May be used in macros or from any module" +function macroexpand(st::SyntaxTree) + DEBUG && assert_expandable(st) + ctx = MacroExpansionContext(st, Base.get_world_counter(), true) + expand_forms_1(ctx, st) +end + +# If a top-level thunk has existing context, we can assume all syntax has the +# same base layer: either it was produced by a macro expansion and went through +# `apply_expansion_layer`, or it was produced by parsing (which we assume either +# adds zero or uniform context to the tree). + +# We ignore old the base layer's module, which should usually be the same as the +# current lowering module. (counterexample: macroexpand in mod A producing +# escaped :toplevel st, then eval st in mod B, but flisp does the same thing by +# spamming globalrefs to mod A throughout st). +function rebase_layers(st, mod::Module, ver::VersionNumber) + out = if st.context === nothing + # assert zero context + sc = SyntaxContext(mod, ver) + fill_context!(st, sc) + else + base = base_layer(st.context::SyntaxContext) + newbase = ScopeLayer(mod, nothing) + _rebase_layers( + st, Dict{ScopeLayer, ScopeLayer}(base=>newbase), + Dict{SyntaxContext, SyntaxContext}()) + end + DEBUG && assert_expandable(out) + out +end + +function _rebase_layers(st, slmap, scmap) + sc = st.context::SyntaxContext + sc2 = get(scmap, sc, nothing) + if isnothing(sc2) + sl2 = _get_sl!(slmap, sc.layer) + sc2 = scmap[sc] = SyntaxContext(sl2, sc.unexpanded, sc.version, sc.internal) + end + if is_leaf(st) || numchildren(st) == 0 + @mknode(st; context=sc2) + else + cs = mapsyntax(c->_rebase_layers(c, slmap, scmap), children(st)) + @mknode(st; context=sc2, children=cs) + end +end + +function _get_sl!(slmap, sl::ScopeLayer) + out = get(slmap, sl, nothing) + out isa ScopeLayer && return out + slmap[sl] = ScopeLayer( + sl.mod, isnothing(sl.escaped) ? nothing : _get_sl!(slmap, sl.escaped)) +end + +# Incremental lowering API which can manage toplevel and module expressions. +# +# This iteration API is oddly bespoke and arguably somewhat non-Julian for two +# reasons: +# +# * Lowering knows when new modules are required, and may request them with +# `:begin_module`. However `eval()` generates those modules so they need to +# be passed back into lowering. So we can't just use `Base.iterate()`. (Put a +# different way, we have a situation which is suited to coroutines but we +# don't want to use full Julia `Task`s for this.) +# * We might want to implement this `eval()` in Julia's C runtime code or early +# in bootstrap. Hence using SimpleVector and Symbol as the return values of +# `lower_step()` +# +# We might consider changing at least the second of these choices, depending on +# how we end up putting this into Base. + +struct LoweringIterator + ver::VersionNumber # later stored in module? + todo::Vector{Tuple{SyntaxTree, Bool, Int}} +end + +function lower_init(ex::SyntaxTree, ver) + LoweringIterator(ver, [(ex, false, 0)]) +end + +function lower_step(iter::LoweringIterator, mod::Module, world::UInt; + soft_scope::Union{Nothing,Bool}=nothing) + if isempty(iter.todo) + return Core.svec(:done) + end + + top_ex, is_module_body, child_idx = pop!(iter.todo) + if child_idx > 0 + if child_idx <= numchildren(top_ex) + push!(iter.todo, (top_ex, is_module_body, child_idx + 1)) + ex = top_ex[child_idx] + elseif is_module_body + return Core.svec(:end_module) + else + return lower_step(iter, mod, world; soft_scope) + end + else + ex = top_ex + end + + k = kind(ex) + if !(k in KSet"toplevel module") + ex = rebase_layers(ex, mod, iter.ver) + ex = expand_forms_1(ex, world, true) + k = kind(ex) + end + if k == K"toplevel" + push!(iter.todo, (ex, false, 1)) + return lower_step(iter, mod, world; soft_scope) + elseif k == K"module" + (version, notbare, mname, body) = @stm ex begin + [K"module" version nb_st mname body] -> + (version.value, nb_st.value, mname, body) + [K"module" nb_st mname body] -> + (nothing, nb_st.value, mname, body) + end + if kind(mname) != K"Identifier" + throw(LoweringError(mname, "Expected module name")) + end + newmod_name = Symbol(syntax_name(mname)) + loc = source_location(LineNumberNode, ex) + push!(iter.todo, (body, true, 1)) + return Core.svec(:begin_module, version, newmod_name, notbare, loc) + else + ctx2, ex2 = expand_forms_2(ex, world) + ctx3, ex3 = resolve_scopes(ctx2, ex2; soft_scope) + ctx4, ex4 = convert_closures(ctx3, ex3) + _ctx5, ex5 = linearize_ir(ctx4, ex4) + thunk = to_lowered_expr(ex5) + return Core.svec(:thunk, thunk) + end +end + + +#------------------------------------------------------------------------------- + +function codeinfo_has_image_globalref(@nospecialize(e)) + if e isa GlobalRef + return 0x00 !== @ccall jl_object_in_image(e.mod::Any)::UInt8 + elseif e isa Core.CodeInfo + return any(codeinfo_has_image_globalref, e.code) + else + return false + end +end + +function codeinfo_has_fcall(@nospecialize(e)) + if e isa Expr + if e.head === :(=) + return codeinfo_has_fcall(e.args[2]) + end + return e.head === :foreigncall || e.head === :foreignglobal || + e.head === :cfunction + end + return false +end + +const _CodeInfo_need_ver = v"1.12.0-DEV.512" +@static if VERSION < _CodeInfo_need_ver + function _CodeInfo(args...) + error("Constructing a CodeInfo using JuliaLowering currently requires Julia version $_CodeInfo_need_ver or greater") + end +else + # debuginfo changed completely as of https://github.com/JuliaLang/julia/pull/52415 + # nargs / isva was added as of https://github.com/JuliaLang/julia/pull/54341 + # field rettype added in https://github.com/JuliaLang/julia/pull/54655 + # field has_image_globalref added in https://github.com/JuliaLang/julia/pull/57433 + # CodeInfo constructor. TODO: Should be in Core + let + fns = fieldnames(Core.CodeInfo) + fts = fieldtypes(Core.CodeInfo) + conversions = [:(convert($t, $n)) for (t,n) in zip(fts, fns)] + + expected_fns = (:code, :debuginfo, :ssavaluetypes, :ssaflags, :slotnames, :slotflags, :slottypes, :rettype, :parent, :edges, :min_world, :max_world, :method_for_inference_limit_heuristics, :nargs, :propagate_inbounds, :has_fcall, :has_image_globalref, :nospecializeinfer, :isva, :inlining, :constprop, :purity, :inlining_cost) + expected_fts = (Vector{Any}, Core.DebugInfo, Any, Vector{UInt32}, Vector{Symbol}, Vector{UInt8}, Any, Any, Any, Any, UInt, UInt, Any, UInt, Bool, Bool, Bool, Bool, Bool, UInt8, UInt8, UInt16, UInt16) + code = if fns != expected_fns || fts != expected_fts + :(function _CodeInfo(args...) + error(string( + "JuliaLowering didn't recognize Core.CodeInfo's fields; ", + "it may need updating to match Core.CodeInfo.\n", + "expected field names: $($expected_fns)\n", + "expected field types: $($expected_fts)\n")) + end) + else + :(function _CodeInfo($(fns...)) + $(Expr(:new, :(Core.CodeInfo), conversions...)) + end) + end + + Core.eval(@__MODULE__, code) + end +end + +""" +Uncompressed form of DebugInfo's linetable::String. When compressing, some +conveniences are erased: +- `file` is not present +- `line_offset` is identical +- `spans` pairs (s1, s2) are stored `(s1-byte_offset, s2-s1+1)` +- `line_starts` are stored `x-byte_offset` +""" +struct SourceByteTable + file::Symbol + line_offset::Int32 + spans::Vector{Tuple{Int32,Int32}} + line_starts::Vector{Int32} + function SourceByteTable(file, line_offset, spans, line_starts) + @assert issorted(spans) + @assert allunique(spans) + @assert issorted(line_starts) + @assert allunique(line_starts) + @assert length(line_starts) > 0 + for s in spans + @assert 0 < s[2] "linenode provenance; expected SourceFile" + @assert 0 < s[1] <= s[2]+1 + end + if !isempty(spans) + @assert !isempty(line_starts) + min_byte = spans[begin][begin] + max_byte = maximum(maximum, spans) + @assert line_starts[begin] <= min_byte + for ls in line_starts[begin+1:end] + @assert min_byte < ls + @assert ls <= max_byte + end + else + # Not used for now + @assert false + end + + new(file, line_offset, spans, line_starts) + end +end +function SourceByteTable(sf::SourceFile, spans::Vector{Tuple{Int32, Int32}}) + # Trim all newlines outside SBT's range + line_starts = map(ls->Int32(ls+sf.byte_offset), sf.line_starts) + b0, _ = JuliaSyntax.source_line_range(sf, spans[1][1]) + first_line = sf.first_line + while length(line_starts) >= 2 && line_starts[2] <= b0 + popfirst!(line_starts) + first_line += 1 + end + max_byte = maximum(maximum, spans) + while !isempty(line_starts) && max_byte < line_starts[end] + pop!(line_starts) + end + SourceByteTable(Symbol(sf.filename), first_line, spans, line_starts) +end + +function _take32(io::IOBuffer, n::Integer) + n in (0, 1, 2, 4) || throw(ArgumentError("Unsupported byte count")) + v = Int32(0) + n >= 1 && (v |= Int32(read(io, UInt8))) + n >= 2 && (v |= Int32(read(io, UInt8))<<8) + n >= 4 && (v |= Int32(read(io, UInt8))<<16) + n >= 4 && (v |= Int32(read(io, UInt8))<<24) + return v +end + +function _push32(io::IOBuffer, v::Int32, n) + n in (0, 1, 2, 4) || throw(ArgumentError("Unsupported byte count")) + n >= 1 && write(io, v % UInt8) + n >= 2 && write(io, (v>>>8) % UInt8) + n >= 4 && write(io, (v>>>16) % UInt8) + n >= 4 && write(io, (v>>>24) % UInt8) + nothing +end + +_encoded_len(max::Int32) = Int32(max == 0 ? 0 : + max < typemax(UInt8) ? 1 : + max < typemax(UInt16) ? 2 : 4) + +function compress_sbt(sbt::SourceByteTable) + min_byte = sbt.line_starts[1] + max_byte = Int32(0) + max_span = Int32(0) + for (b1,b2) in sbt.spans + max_span = max(max_span, (b2+Int32(1))-b1) + max_byte = max(max_byte, b2) + end + + max_byte_rel = Int32(min_byte >= max_byte ? 1 : (max_byte - min_byte)) + nlocs::Int32 = length(sbt.spans) + encl_span = _encoded_len(max_span) + encl_byte = _encoded_len(max_byte_rel) + final_len = 14 + # header + (encl_byte + encl_span) * nlocs + + (encl_byte * length(sbt.line_starts)) + + io = IOBuffer(;sizehint=final_len) + _push32(io, min_byte, 4) + _push32(io, sbt.line_offset, 4) + _push32(io, nlocs, 4) + _push32(io, encl_byte, 1) + _push32(io, encl_span, 1) + for (b1, b2) in sbt.spans + _push32(io, b1 - min_byte, encl_byte) + _push32(io, b2 - b1 + Int32(1), encl_span) + end + for n in sbt.line_starts + _push32(io, n - min_byte, encl_byte) + end + + out = take!(io) + let l = length(out) + @assert l == final_len "wrong final length $l" + end + return String(out) +end + +function uncompress_sbt(di::Core.DebugInfo) + di.linetable isa String || throw(ArgumentError("linetable: expected string")) + io = IOBuffer(di.linetable) + byte_offset = _take32(io, 4) + line_offset = _take32(io, 4) + nlocs = _take32(io, 4) + byte_encl = _take32(io, 1) + span_encl = _take32(io, 1) + + let newlines_offset = (byte_encl + span_encl) * nlocs + @assert bytesavailable(io) >= newlines_offset "compressed string too short" + @assert byte_encl == 0 || + (bytesavailable(io) - newlines_offset) % byte_encl == 0 "bad newlines" + end + + out_spans = Tuple{Int32,Int32}[] + for i in 1:nlocs + s1 = _take32(io, byte_encl) + s2 = _take32(io, span_encl) + push!(out_spans, (s1+byte_offset, s1+byte_offset+s2-1)) + end + + out_newlines = Int32[] + while bytesavailable(io) > 0 + push!(out_newlines, _take32(io, byte_encl) + byte_offset) + end + return SourceByteTable(di.def, line_offset, out_spans, out_newlines) +end + +const LINENODE_SPAN_END = Int32(-5) + +# Byte-precise `DebugInfo` requires `Core.DebugInfo` to accept a `String` linetable, +# which is only available on recent Julia. On older versions (e.g. v1.12) we degrade to +# line-based `DebugInfo` so that lowering still produces a valid `CodeInfo`, at the cost of +# byte-precise source attribution. +const _has_byte_precise_debuginfo = + hasmethod(Core.DebugInfo, Tuple{Symbol, String, Core.SimpleVector, String}) + +function _di_pos(st::SyntaxTree) + src = JuliaSyntax.unexpanded_sourceref(st) + pos = if src isa SourceRef + (Int32(first_byte(src)), Int32(last_byte(src))) + elseif src isa LineNumberNode + (Int32(src.line), LINENODE_SPAN_END) + else + @jl_assert false st + end +end + +# TODO sourcefile(::LNN) should return Symbol, not LNN +function _di_sourcefile(st) + x = JuliaSyntax.unexpanded_sourceref(st) + x isa LineNumberNode ? x.file : x.file[]::SourceFile +end + +# A single pass over all IR to collect unique byte/line positions and CodeInfos +function collect_locs!(node_sources, codeinfos, top_sf, st) + if kind(st) === K"code_info" + push!(codeinfos, st) + # TODO: macro_source is ignored for now + get!(node_sources, st, _di_pos(st)) + for c in children(st[2]) + node_sources[c] = + if _di_sourcefile(c) !== top_sf + top_sf isa SourceFile && + @warn "inconsistent provenance for child" c st + node_sources[st] + else + _di_pos(c) + end + collect_locs!(node_sources, codeinfos, top_sf, c) + end + elseif !is_leaf(st) + # Non-toplevel codeinfo can contain nested codeinfo (opaque closures) + for c in children(st) + collect_locs!(node_sources, codeinfos, top_sf, c) + end + end + nothing +end + +function add_ci_debuginfo!(st::SyntaxTree, file::Symbol, + top_sbt::Union{String, Nothing}, + node_sources::Dict{SyntaxTree, Tuple{Int32, Int32}}, + spans::Vector{Tuple{Int32, Int32}}) + @jl_assert kind(st) === K"code_info" st + locs = let a = sizehint!(Vector{Int32}(), 3*numchildren(st[2])) + for c in children(st[2]) + if top_sbt isa String # precise provenance + push!(a, Int32(searchsortedfirst(spans, node_sources[c]))) + else + i = searchsortedfirst(spans, node_sources[c]) + @jl_assert spans[i][2] == LINENODE_SPAN_END (c, "lno with span end?") + push!(a, spans[i][1]) + end + push!(a, Int32(0)) + push!(a, Int32(0)) + end + a + end + + setmeta!(st, :debuginfo, Core.DebugInfo( + file, top_sbt, Core.svec(), + @ccall(jl_compress_codelocs((-1)::Int32, locs::Any, + numchildren(st[2])::Csize_t)::String))) +end + +# Populate `.debuginfo` on all K"code_info" in `st` +function add_debuginfo!(st::SyntaxTree) + @jl_assert kind(st) === K"code_info" st + node_sources = Dict{SyntaxTree, Tuple{Int32, Int32}}() + codeinfos = SyntaxList() + top_sf = _di_sourcefile(st) + collect_locs!(node_sources, codeinfos, top_sf, st) + byte_precise = _has_byte_precise_debuginfo && top_sf isa SourceFile + if !byte_precise && top_sf isa SourceFile + # Without byte-precise support, degrade each byte span to its line number + # so the line-based path below emits valid `DebugInfo` (same shape as the + # `LineNumberNode` case). + for id in collect(keys(node_sources)) + line = Int32(JuliaSyntax.source_line(top_sf, node_sources[id][1])) + node_sources[id] = (line, LINENODE_SPAN_END) + end + end + spans = sort!(unique(values(node_sources))) + if byte_precise + top_sbt = compress_sbt(SourceByteTable(top_sf, spans)) + file = Symbol(top_sf.filename) + else + top_sbt = nothing + file = top_sf isa SourceFile ? Symbol(top_sf.filename) : Symbol(top_sf) + end + for ci in codeinfos + add_ci_debuginfo!(ci, file, top_sbt, node_sources, spans) + end +end + +# flisp: jl_new_code_info_from_ir (method.c) +function compute_ssaflags(st::SyntaxTree) + @jl_assert kind(st) == K"block" st + stmts = children(st) + out = zeros(UInt32, length(stmts)) + inline_flags = Vector{Bool}() + inbounds_depth = 0 + purity_flags = Vector{UInt32}() + + # Note this should probably go in validation or be a user-facing + # loweringerror, but method.c only checks this in asserts builds, so we may + # need to allow these to be unbalanced + function checked_pop!(stk) + @jl_assert(!isempty(stk), (st, "ssaflags pop without push")) + pop!(stk) + end + for (i, stmt) in enumerate(stmts) + is_flag_stmt = true + @stm stmt begin + [K"inbounds" [K"Value"]] -> stmt[1].value::Bool ? + (inbounds_depth += 1) : # push + (inbounds_depth = 0) # clear + [K"inbounds_pop"] -> (inbounds_depth = max(0, inbounds_depth-1)) + [K"boundscheck" _...] -> nothing + [K"inline" [K"Value"]] -> stmt[1].value::Bool ? + push!(inline_flags, true) : checked_pop!(inline_flags) + [K"noinline" [K"Value"]] -> stmt[1].value::Bool ? + push!(inline_flags, false) : checked_pop!(inline_flags) + [K"purity"] -> checked_pop!(purity_flags) + [K"purity" _ _...] -> push!( + purity_flags, + UInt32(purity_expr_to_flags(stmt)) << Core.Compiler.NUM_IR_FLAGS) + _ -> is_flag_stmt = false + end + flag = UInt32(0) + if !isempty(inline_flags) + flag |= (inline_flags[end] ? + Core.Compiler.IR_FLAG_INLINE : Core.Compiler.IR_FLAG_NOINLINE) + end + if inbounds_depth != 0 + flag |= Core.Compiler.IR_FLAG_INBOUNDS + end + if !isempty(purity_flags) + for pf in purity_flags + flag |= pf + end + end + out[i] = is_flag_stmt ? UInt32(0) : flag + end + @jl_assert length(out) == length(stmts) st + @jl_assert length(inline_flags) == 0 st + @jl_assert length(purity_flags) == 0 st + out +end + +# Convert SyntaxTree to the CodeInfo+Expr data structures understood by the +# Julia runtime +function to_code_info(ex::SyntaxTree) + slots = ex[1].value::Vector{Slot} + meta = ex.meta + nargs = sum((s.kind==:argument for s in slots), init=0) + slotnames = Vector{Symbol}(undef, length(slots)) + slot_rename_inds = Dict{String,Int}() + slotflags = Vector{UInt8}(undef, length(slots)) + for (i, slot) in enumerate(slots) + name = slot.name + # TODO: Do we actually want unique names here? The C code in + # `jl_new_code_info_from_ir` has logic to simplify gensym'd names and + # use the empty string for compiler-generated bindings. + if name !== UNUSED + ni = get(slot_rename_inds, name, 0) + slot_rename_inds[name] = ni + 1 + if ni > 0 + name = "$name@$ni" + end + end + sname = Symbol(name) + slotnames[i] = sname + slotflags[i] = # Inference | Codegen + slot.is_read << 3 | # SLOT_USED | jl_vinfo_sa + slot.is_single_assign << 4 | # SLOT_ASSIGNEDONCE | - + slot.is_maybe_undef << 5 | # SLOT_USEDUNDEF | jl_vinfo_usedundef + slot.is_called << 6 # SLOT_CALLED | - + end + + stmts = map(_to_lowered_expr, children(ex[2])) + has_image_globalref = any(codeinfo_has_image_globalref, stmts) + ssaflags = compute_ssaflags(ex[2]) + propagate_inbounds = + get(meta, :propagate_inbounds, false) + has_fcall = any(codeinfo_has_fcall, stmts) + nospecializeinfer = + get(meta, :nospecializeinfer, false) + inlining = + get(meta, :inline, false) ? 0x01 : + get(meta, :noinline, false) ? 0x02 : 0x00 + constprop = + get(meta, :aggressive_constprop, false) ? 0x01 : + get(meta, :no_constprop, false) ? 0x02 : 0x00 + purity = + let eo = get(meta, :purity, nothing) + isnothing(eo) ? 0x0000 : eo::UInt16 + end + + # The following CodeInfo fields always get their default values for + # uninferred code. + ssavaluetypes = length(stmts) # Why does the runtime code do this? + slottypes = nothing + parent = nothing + method_for_inference_limit_heuristics = nothing + edges = nothing + min_world = Csize_t(1) + max_world = typemax(Csize_t) + isva = false + inlining_cost = 0xffff + rettype = Any + + @jl_assert(length(stmts) == numchildren(ex[2]), ex) + + _CodeInfo( + stmts, + getmeta(ex, :debuginfo, nothing), + ssavaluetypes, + ssaflags, + slotnames, + slotflags, + slottypes, + rettype, + parent, + edges, + min_world, + max_world, + method_for_inference_limit_heuristics, + nargs, + propagate_inbounds, + has_fcall, + has_image_globalref, + nospecializeinfer, + isva, + inlining, + constprop, + purity, + inlining_cost + ) +end + +""" +This pass convert's JuliaLowering's internal representation of untyped IR into +a form the Julia runtime understands. This is a necessary decoupling which +separates the development of JuliaLowering.jl from the evolution of the Julia +runtime itself. +""" +@fzone "JL: to_lowered_expr" function to_lowered_expr(ex::SyntaxTree) + @jl_assert kind(ex) in KSet"thunk code_info" ex + add_debuginfo!(kind(ex) === K"thunk" ? ex[1] : ex) + _to_lowered_expr(ex) +end + +function _to_lowered_expr(ex::SyntaxTree) + k = kind(ex) + if is_literal(k) + ex.value + elseif k == K"nothing" + nothing + elseif k == K"core" + GlobalRef(Core, Symbol(syntax_name(ex))) + elseif k == K"top" + GlobalRef(Base, Symbol(syntax_name(ex))) + elseif k == K"globalref" + GlobalRef(ex.mod::Module, Symbol(syntax_name(ex))) + elseif k == K"Identifier" + # TODO: assert false (only reachable from simdloop?) + Symbol(syntax_name(ex)) + elseif k == K"SourceLocation" + QuoteNode(source_location(LineNumberNode, ex)) + elseif k == K"Symbol" + QuoteNode(Symbol(syntax_name(ex))) + elseif k == K"slot" + Core.SlotNumber(syntax_id(ex)) + elseif k == K"static_parameter" + Expr(:static_parameter, syntax_id(ex)) + elseif k == K"SSAValue" + Core.SSAValue(syntax_id(ex)) + elseif k == K"return" + v = _to_lowered_expr(ex[1]) + @jl_assert Base.Compiler.is_valid_return(v) ex + Core.ReturnNode(v) + elseif k == K"inert" + est_to_expr(ex) + elseif k == K"syntaxinert" + ex[1] + elseif k == K"code_info" + to_code_info(ex) + elseif k == K"Value" + @jl_assert !isa_lowering_ast_node(ex.value) ( + ex, string("smuggling AST through Value is asking for trouble; ", + "find a SyntaxTree representation")) + ex.value isa LineNumberNode ? QuoteNode(ex.value) : ex.value + elseif k == K"goto" + Core.GotoNode(syntax_id(ex[1])) + elseif k == K"gotoifnot" + Core.GotoIfNot(_to_lowered_expr(ex[1]), syntax_id(ex[2])) + elseif k == K"enter" + catch_idx = syntax_id(ex[1]) + numchildren(ex) == 1 ? + Core.EnterNode(catch_idx) : + Core.EnterNode(catch_idx, _to_lowered_expr(ex[2])) + elseif k == K"newvar" + Core.NewvarNode(_to_lowered_expr(ex[1])) + elseif k == K"opaque_closure_method" + args = map(_to_lowered_expr, children(ex)) + # opaque_closure_method has special non-evaluated semantics for the + # `functionloc` line number node so we need to undo a level of quoting + arg4 = args[4] + @jl_assert arg4 isa QuoteNode ex + args[4] = arg4.value + Expr(:opaque_closure_method, args...) + elseif k == K"meta" + args = Any[_to_lowered_expr(e) for e in children(ex)] + # Unpack K"Symbol" QuoteNode as `Expr(:meta)` requires an identifier here. + arg1 = args[1] + @jl_assert (arg1 isa QuoteNode) ex + args[1] = arg1.value + Expr(:meta, args...) + elseif k == K"foreignsymbol" + @jl_assert kind(ex[1]) == K"tuple" ex + _foreignsymbol_expr(ex[1]) + elseif k == K"static_eval" + @jl_assert numchildren(ex) == 1 ex + _to_lowered_expr(ex[1]) + elseif k == K"cfunction" + # For a scope-resolved callable (`K"static_eval"`), drop the module tag + # and emit a bare Symbol so `method.c` resolves it in the method's + # module at eval time, matching Base `@cfunction`'s runtime semantics. + ret = Expr(:cfunction) + for (i, e) in enumerate(children(ex)) + if i == 2 && kind(e) == K"static_eval" && kind(e[1]) == K"globalref" + push!(ret.args, QuoteNode(Symbol(syntax_name(e[1])))) + else + push!(ret.args, _to_lowered_expr(e)) + end + end + return ret + elseif k in KSet"inline noinline inbounds inbounds_pop purity" + # only used in compute_ssaflags (see method.c) + nothing + else + # Allowed forms according to https://docs.julialang.org/en/v1/devdocs/ast/ + # + # call invoke static_parameter `=` method struct_type abstract_type + # primitive_type global const new splatnew isdefined + # enter leave pop_exception inbounds boundscheck loopinfo copyast meta + # lambda + head = k == K"call" ? :call : + k == K"new" ? :new : + k == K"splatnew" ? :splatnew : + k == K"=" ? :(=) : + k == K"leave" ? :leave : + k == K"isdefined" ? :isdefined : + k == K"loopinfo" ? :loopinfo : + k == K"thunk" ? :thunk : + k == K"boundscheck" ? :boundscheck : + k == K"latestworld" ? :latestworld : + k == K"pop_exception" ? :pop_exception : + k == K"captured_local" ? :captured_local : + k == K"gc_preserve_begin" ? :gc_preserve_begin : + k == K"gc_preserve_end" ? :gc_preserve_end : + k == K"foreigncall" ? :foreigncall : + k == K"foreignglobal" ? :foreignglobal : + k == K"cfunction" ? :cfunction : + k == K"aliasscope" ? :aliasscope : + k == K"popaliasscope" ? :popaliasscope : + k == K"new_opaque_closure" ? :new_opaque_closure : + nothing + if isnothing(head) + throw(LoweringError(ex, "Unhandled form for kind $k")) + end + ret = Expr(head) + for e in children(ex) + push!(ret.args, _to_lowered_expr(e)) + end + return ret + end +end + +# ultra-permissive conversion allowing unlowered structure, but lowered leaves +function _foreignsymbol_expr(ex) + if is_leaf(ex) || kind(ex) == K"inert" + _to_lowered_expr(ex) + else + k = kind(ex) + Expr(Symbol((k === K"unknown_head" ? syntax_name(ex) : untokenize(k))::String), + map(_foreignsymbol_expr, children(ex))...) + end +end + +#------------------------------------------------------------------------------- +# Our version of eval - should be upstreamed though? +@fzone "JL: eval" function eval(mod::Module, @nospecialize(ex); + soft_scope::Union{Nothing,Bool}=nothing, + expr_compat_mode::Bool=false) + # Run the `eval` driver in the lowering world. Any internal operations + # are required to `invokelatest` before executing any code that dispatches + # on user code / types. + ver = expr_compat_mode ? JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION + return invoke_in_lowering_world(_lower_and_eval, mod, ex, ver, soft_scope) +end + +# `ex` may be a `SyntaxTree` or an `Expr` (or `Expr` tree leaves of any type). +function _lower_and_eval(mod::Module, @nospecialize(ex), ver::VersionNumber, + soft_scope::Union{Nothing,Bool}) + st = ex isa SyntaxTree ? ex : expr_to_est(ex) + iter = lower_init(st, ver) + return _eval(mod, iter; soft_scope) +end + +function _eval(mod::Module, iter::LoweringIterator; soft_scope::Union{Nothing,Bool}=nothing) + modules = Module[mod] + result = nothing + while true + thunk = lower_step(iter, modules[end], Base.get_world_counter(); soft_scope)::Core.SimpleVector + type = thunk[1]::Symbol + if type == :done + break + elseif type == :begin_module + filename = something(thunk[5].file, :none) + mod = @ccall jl_begin_new_module( + modules[end]::Any, thunk[3]::Symbol, thunk[2]::Any, thunk[4]::Cint, + filename::Cstring, thunk[5].line::Cint)::Module + push!(modules, mod) + elseif type == :end_module + @ccall jl_end_new_module(modules[end]::Module)::Cvoid + result = pop!(modules) + else + @assert type == :thunk + result = Base.invokelatest(Core.eval, modules[end], thunk[2]) + end + end + @assert length(modules) === 1 + return result +end + +""" + include(mod::Module, path::AbstractString) + +Evaluate the contents of the input source file in the global scope of module +`mod`. Every module (except those defined with baremodule) has its own +definition of `include()` omitting the `mod` argument, which evaluates the file +in that module. Returns the result of the last evaluated expression of the +input file. During including, a task-local include path is set to the directory +containing the file. Nested calls to include will search relative to that path. +This function is typically used to load source interactively, or to combine +files in packages that are broken into multiple source files. +""" +function include(mod::Module, path::AbstractString) + path, prev = Base._include_dependency(mod, path) + code = read(path, String) + tls = task_local_storage() + tls[:SOURCE_PATH] = path + try + return include_string(mod, code, path) + finally + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + end +end + +""" + include_string(mod::Module, code::AbstractString, filename::AbstractString="string") + +Like `include`, except reads code from the given string rather than from a file. +""" +function include_string(mapexpr::Function, mod::Module, code::AbstractString, + filename::AbstractString; expr_compat_mode=false, + version::Union{VersionNumber, Nothing}=nothing) + # TODO: fix this hack. The normal way of getting the parser for this module + # only gives us Expr. We probably want the parser to always create + # SyntaxTree, then convert it to Expr if the version is too low. + version = if isnothing(version) && invokelatest( + isdefined, mod, Symbol("#_internal_julia_parse")) + vp = invokelatest(getglobal, mod, Symbol("#_internal_julia_parse")) + vp isa Base.VersionedParse ? vp.ver : VERSION + else + version isa VersionNumber ? version : VERSION + end + st = parseall(SyntaxTree, code; filename, version, ignore_warnings=true) + @jl_assert kind(st) === K"toplevel" st + if mapexpr !== identity + # TODO: Is there any way to support provenance here? + local last = nothing + for c in children(st) + last = eval(mod, expr_to_est(mapexpr(est_to_expr(c))); expr_compat_mode) + end + last + else + eval(mod, st; expr_compat_mode) + end +end +include_string(mod, code, filename="string"; kws...) = + include_string(identity, mod, code, filename; kws...) + +include(path::AbstractString) = include(JuliaLowering, path) diff --git a/JuliaLowering/src/hooks.jl b/JuliaLowering/src/hooks.jl new file mode 100644 index 0000000000000..d639e8832ef47 --- /dev/null +++ b/JuliaLowering/src/hooks.jl @@ -0,0 +1,75 @@ +# TODO: Allow `soft_scope::Union{Nothing,Bool}` to be passed through `jl_lower` C API + +""" +Becomes `Core._lower()` upon activating JuliaLowering. + +Returns an svec with the lowered code (usually expr) as its first element, and +(until integration is less experimental) whatever we want after it +""" +function core_lowering_hook(@nospecialize(code), mod::Module, file::String="none", + line::Int=0, world::UInt=typemax(Csize_t), _warn::Bool=false) + if !(code isa SyntaxTree || code isa Expr) + # e.g. LineNumberNode, integer... + return Core.svec(code) + end + + if _has_v1_13_hooks && Core._lower === core_lowering_hook && + unsafe_load(cglobal(:jl_lowering_world, Csize_t)) == 0 + # Refuse to run as `Core._lower` without a pinned world + error("`Core._lower` was set without pinning the lowering world; use `JuliaLowering.activate!()`") + end + local st0, st1 = nothing, nothing + try + st0 = code isa Expr ? expr_to_est(code, LineNumberNode(line, file)) : code + if kind(st0) in KSet"toplevel module" + return Core.svec(code) + elseif kind(st0) === K"doc" && numchildren(st0) >= 2 && kind(st0[2]) === K"module" + # TODO: this ignores module docstrings for now + return Core.svec(est_to_expr(st0[2])) + end + st0 = rebase_layers(st0, mod, JL_OLD_SYNTAX_VERSION) + st1 = expand_forms_1(st0, world, true) + ctx2, st2 = expand_forms_2(st1, world) + ctx3, st3 = resolve_scopes(ctx2, st2) + ctx4, st4 = convert_closures(ctx3, st3) + ctx5, st5 = linearize_ir(ctx4, st4) + ex = to_lowered_expr(st5) + return Core.svec(ex, st5, ctx5) + catch exc + @info("JuliaLowering threw given input:", code=code, file=file, + line=line, mod=mod, st0=st0, st1=st1) + if exc isa LoweringError && !exc.internal + return Core.svec(Expr(:error, sprint( + (io,err)->showerror(io,err; show_detail=false), exc))) + else + rethrow(exc) + end + + # TODO: Re-enable flisp fallback once we're done collecting errors + # @error("JuliaLowering failed — falling back to flisp!", + # exception=(exc,catch_backtrace()), + # code=code, file=file, line=line, mod=mod) + # return Base.fl_lower(code, mod, file, line, world, warn) + end +end + +# TODO: Write a parser hook here. The input to `core_lowering_hook` should +# eventually be a (convertible to) SyntaxTree, but we need to make updates to +# the parsing API to include a parameter for AST type. + +const _has_v1_13_hooks = isdefined(Core, :_lower) + +function activate!(enable=true) + if !_has_v1_13_hooks + error("Cannot use JuliaLowering without `Core._lower` binding or in $VERSION < 1.13") + end + + if enable + Core._setlowerer!(core_lowering_hook) + ccall(:jl_set_lowering_world, Cvoid, (Csize_t,), Base.get_world_counter()) + else + Core._setlowerer!(Base.fl_lower) + # Unlike JL, `jl_lower` dispatches the flisp wrapper at the latest world + ccall(:jl_set_lowering_world, Cvoid, (Csize_t,), 0) + end +end diff --git a/JuliaLowering/src/kinds.jl b/JuliaLowering/src/kinds.jl new file mode 100644 index 0000000000000..00b724f2cfbe6 --- /dev/null +++ b/JuliaLowering/src/kinds.jl @@ -0,0 +1,227 @@ +# The following kinds are used in intermediate forms by lowering but are not +# part of the surface syntax +function _register_kinds() + JuliaSyntax.register_kinds!(JuliaLowering, 1, [ + # "Syntax extensions" - expression kinds emitted by macros or macro + # expansion, and known to lowering. These are part of the AST API but + # without having surface syntax. + "BEGIN_EXTENSION_KINDS" + # atomic fields or accesses (see `@atomic`) + "atomic" + # Flag for @generated parts of a function + "generated" + # Temporary rooting of identifiers (GC.@preserve) + "gc_preserve" + "gc_preserve_begin" + "gc_preserve_end" + # A (quoted) `Symbol` + "Symbol" + "inbounds" + "inbounds_pop" # expr: (inbounds pop) with identifier "pop" + "boundscheck" + "inline" + "noinline" + "loopinfo" + "purity" + "aliasscope" + "popaliasscope" + # Call into foreign code + "foreigncall" + # Look up a symbol in foreign code + "foreignglobal" + # ccall convention + "cconv" + # Special form for constructing a function callable from C + "cfunction" + # Special form emitted by `Base.Experimental.@opaque` + "opaque_closure" + # Test whether a variable is defined + "isdefined" + # [K"throw_undef_if_not" var cond] + # This form is used internally in Core.Compiler but might be + # emitted by packages such as Diffractor. In principle it needs to + # be passed through lowering in a similar way to `isdefined` + "throw_undef_if_not" + # named labels for `@label` and `@goto` + "symboliclabel" + # Goto named label + "symbolicgoto" + # Labeled block for `@label name expr` (block break) + "symbolicblock" + # Internal initializer for struct types, for inner constructors/functions + "new" + "splatnew" + # Used for converting `esc()`'d expressions arising from old macro + # invocations during macro expansion (gone after macro expansion) + "escape" + # Used for converting the old-style macro hygienic-scope form (gone + # after macro expansion). + "hygienic-scope" + # Only produced by flisp macro expansion (which Core.@doc (cursed) + # invokes manually) + "copyast" + # An expression which will eventually be evaluated "statically" in + # the context of a CodeInfo and thus allows access only to globals + # and static parameters. Used for ccall, cfunction, cglobal + # TODO: Use this for GeneratedFunctionStub also? + "static_eval" + "islocal" + "isglobal" + "locals" + "thisfunction" + "overlay" + "syntaxquote" + "syntaxunquote" + "END_EXTENSION_KINDS" + + # The following kinds are internal to lowering + "BEGIN_LOWERING_KINDS" + # Semantic assertions used by lowering. The content of an assertion + # is not considered to be quoted, so use K"Symbol" etc inside where necessary. + "assert" + # Unique identifying integer for bindings (of variables, constants, etc) + "BindingId" + # Various heads harvested from flisp lowering. + # (TODO: May or may not need all these - assess later) + # Like block, but introduces a lexical scope; used during scope resolution. + "scope_block" + # Equivalent to Expr(:softscope). If found in the top-level thunk, + # all enclosed neutral scopes become soft scopes. :hardscope exists + # too, but is always produced inside scope-blocks, so we represent + # it with an attribute on our scope-blocks. + "softscope" + # [K"always_defined" x] is an assertion that variable `x` is assigned before use + # ('local-def in flisp implementation is K"local" plus K"always_defined" + "always_defined" + # `(relayered_global old::Identifier)` is used to tell scope + # resolution that any declaration conflicting with `(global old)` + # should fail, even though `old` was never actually declared + "relayered_global" + "_while" + "_do_while" + # (_typevar name lb ub). flisp usually uses 3-long lists for these, + # usually called `sparams` + "_typevar" + "_typevars" # used for supplying already-allocated `TypeVar`s to `where` + # (_generated_body (quote gen) nongen) to allow arglist-related desugaring + # to occur before the methods are created + "_generated_body" + "with_static_parameters" + # converted from nothing::K"Value" into desugaring. flisp: (null) + "nothing" + "top" + "core" + "lambda" + # "A source location literal" - a node which exists only to record + # a sourceref + "SourceLocation" + # [K"function_decl" name] + # Declare a zero-method generic function with global `name` or + # creates a closure object and assigns it to the local `name`. + "function_decl" + # [K"function_type" name] + # Evaluates to the type of the function or closure with given `name` + "function_type" + # [K"method_defs" name [K"block" typevars...] [K"block" body...]] + # The code in `body` defines methods for generic function `name`. + # If non-toplevel, all contained methods share a closure type. + # `typevars` are assigned-once top-level locals only referenced + # inside `K"method"`, but outside of `K"lambda"`, since any + # reference inside a lambda should resolve to the lambda's sparam + # shadowing it. + "method_defs" + # from `function f end`, tells closure conversion to give f its value + # (usually done with method_defs) + "no_method_defs" + # [K"typevar" name rhs] appears only in method_defs and gets special + # scope resolution: a sequence of K"sparam"s are similar to nested + # let-blocks, but without introducing a local scope. + "typevar" + "_opaque_closure" + # The enclosed statements must be executed at top level + "toplevel_butfirst" + # like v = val, except that if `v` turns out global (either + # implicitly or by explicit `global`), it gains an implicit `const` + "assign_or_constdecl_if_global" + "global_if_global" + "moved_local" + "label" + "trycatchelse" + "tryfinally" + # The contained block of code causes no side effects and can be + # removed by a later lowering pass if its value isn't used. + # (That is, it's removable in the same sense as + # `@assume_effects :removable`.) + "removable" + # Variable type declaration; `x::T = rhs` will be temporarily + # desugared to include `(decl x T)` + "decl" + # [K"captured_local" index] + # A local variable captured into a global method. Contains the + # `index` of the associated `Box` in the rewrite list. + "captured_local" + # Causes the linearization pass to conditionally emit a world age increment + "latestworld_if_toplevel" + # This has two forms: + # [K"constdecl" var val] => declare and assign constant + # [K"constdecl" var] => declare undefined constant + # var is GlobalRef Value or Identifier + "constdecl" + # Returned from statements that should error if the result is used. + "unused_only" + # Pre-lowered SSA value reference from Expr(:ssavalue, N). + # Translated to a BindingId during desugaring. + "ssavalue" + # Wraps the first argument of a foreigncall / foreignglobal when it + # should not be lowered (and should mostly be treated as :inert), but + # requires scope resolution and special conversion to Expr. + "foreignsymbol" + "thunk" + "with-static-parameters" + # Options specific to K"lambda" + "generated_lambda" + "toplevel_lambda" + + # Options specific to K"scope_block" + "hard_scope" + "neutral_scope" + + "LambdaBindings" + "Slots" + "END_LOWERING_KINDS" + + # The following kinds are emitted by lowering and used in Julia's untyped IR + "BEGIN_IR_KINDS" + # Identifier for a value which is only assigned once + "SSAValue" + # Local variable in a `CodeInfo` code object (including lambda arguments) + "slot" + # Static parameter to a `CodeInfo` code object ("type parameters" to methods) + "static_parameter" + # References/declares a global variable within a module + "globalref" + # Unconditional goto + "goto" + # Conditional goto + "gotoifnot" + # Exception handling + "enter" + "leave" + "pop_exception" + # Lowering targets for method definitions arising from `function` etc + "method" + # (re-)initialize a slot to undef + # See Core.NewvarNode + "newvar" + # Result of lowering a `K"lambda"` after bindings have been + # converted to slot/globalref/SSAValue. + "code_info" + # Internal initializer for opaque closures + "new_opaque_closure" + # Wrapper for the lambda of around opaque closure methods + "opaque_closure_method" + # World age increment (TODO: use top level assertion and only one latestworld kind) + "latestworld" + "END_IR_KINDS" + ]) +end diff --git a/JuliaLowering/src/linear_ir.jl b/JuliaLowering/src/linear_ir.jl new file mode 100644 index 0000000000000..72c42b1e36218 --- /dev/null +++ b/JuliaLowering/src/linear_ir.jl @@ -0,0 +1,1349 @@ +#------------------------------------------------------------------------------- +# Lowering pass 5: Flatten to linear IR + +# Must outline anything that can throw, e.g. globalrefs, static params +function is_valid_ir_argument(ctx, ex) + k = kind(ex) + if is_simple_atom(ctx, ex) || k in KSet"inert syntaxinert top core quote static_eval foreignsymbol" + true + elseif k == K"BindingId" + binfo = get_binding(ctx, ex) + bk = binfo.kind + bk === :slot + else + false + end +end + +function is_ssa(ctx, ex) + kind(ex) == K"BindingId" && get_binding(ctx, ex).is_ssa +end + +# Target to jump to, including info on try handler nesting and catch block +# nesting +struct JumpTarget + label::SyntaxTree + handler_token_stack::Vector{SyntaxTree} + catch_token_stack::Vector{SyntaxTree} + result_var::Union{SyntaxTree, Nothing} # for symbolicblock valued breaks +end + +function JumpTarget(label::SyntaxTree, ctx, result_var=nothing) + JumpTarget(label, copy(ctx.handler_token_stack), copy(ctx.catch_token_stack), result_var) +end + +struct JumpOrigin + goto::SyntaxTree + index::Int + handler_token_stack::Vector{SyntaxTree} + catch_token_stack::Vector{SyntaxTree} +end + +function JumpOrigin(goto::SyntaxTree, index, ctx) + JumpOrigin(goto, index, copy(ctx.handler_token_stack), copy(ctx.catch_token_stack)) +end + +struct FinallyHandler + tagvar::SyntaxTree + target::JumpTarget + exit_actions::Vector{Tuple{Symbol,Union{Nothing,SyntaxTree}}} +end + +function FinallyHandler(tagvar::SyntaxTree, target::JumpTarget) + FinallyHandler(tagvar, target, + Vector{Tuple{Symbol, Union{Nothing,SyntaxTree}}}()) +end + +""" +Context for creating linear IR. + +One of these is created per lambda expression to flatten the body down to +a sequence of statements (linear IR), which eventually becomes one CodeInfo. +""" +mutable struct LinearIRContext <: AbstractLoweringContext + const code::Vector{SyntaxTree} + const bindings::Bindings + const next_label_id::Base.RefValue{Int} + const is_toplevel_thunk::Bool + const lambda_bindings::LambdaBindings + const argmap::Dict{IdTag, IdTag} + const rettype_ssa::Base.RefValue{Union{Nothing,SyntaxTree}} + const break_targets::Dict{String, JumpTarget} + const break_label_stack::Vector{String} # tracks nesting order of symbolicblock labels + const handler_token_stack::Vector{SyntaxTree} + const catch_token_stack::Vector{SyntaxTree} + const finally_handlers::Vector{FinallyHandler} + const symbolic_jump_targets::Dict{String,JumpTarget} + const symbolic_jump_origins::Vector{JumpOrigin} + const symbolic_block_labels::Set{String} # labels that are symbolic blocks (not allowed as @goto targets) + const meta::Dict{Symbol, Any} + const mod::Module +end + +function rettype(ctx::LinearIRContext) + let r = ctx.rettype_ssa[] + isnothing(r) ? nothing : r + end +end + +function LinearIRContext(ctx, is_toplevel_thunk, lambda_bindings) + LinearIRContext(SyntaxList(), ctx.bindings, Ref(0), + is_toplevel_thunk, lambda_bindings, Dict{IdTag,IdTag}(), + Ref{Union{Nothing,SyntaxTree}}(nothing), + Dict{String,JumpTarget}(), String[], + SyntaxList(), SyntaxList(), + Vector{FinallyHandler}(), Dict{String,JumpTarget}(), + Vector{JumpOrigin}(), Set{String}(), Dict{Symbol, Any}(), ctx.mod) +end + +function current_lambda_bindings(ctx::LinearIRContext) + ctx.lambda_bindings +end + +function is_valid_body_ir_argument(ctx, ex) + if is_valid_ir_argument(ctx, ex) + true + elseif kind(ex) == K"BindingId" + get_binding(ctx, ex).is_always_defined + else + false + end +end + +function is_simple_arg(ctx, ex) + k = kind(ex) + return is_simple_atom(ctx, ex) || k == K"BindingId" || k == K"quote" || + k == K"inert" || k == K"syntaxinert" || k == K"top" || + k == K"core" || k == K"globalref" || k == K"static_eval" || + k == K"foreignsymbol" +end + +# flisp note: arguments are always counted as single-assign, so effects on +# arguments within compile_args are thrown out (intentional?) +function is_single_assign_var(ctx::LinearIRContext, ex) + kind(ex) == K"BindingId" || return false + binfo = get_binding(ctx, ex) + return binfo.kind == :argument || binfo.is_assigned_once +end + +function is_const_read_arg(ctx, ex) + k = kind(ex) + # Even if we have side effects, we know that singly-assigned + # locals cannot be affected by them so we can inline them anyway. + # TODO from flisp: "We could also allow const globals here" + return k == K"inert" || k == K"syntaxinert" || k == K"top" || + k == K"core" || k == K"static_eval" || k == K"foreignsymbol" || + is_simple_atom(ctx, ex) || is_single_assign_var(ctx, ex) +end + +function is_valid_ir_rvalue(ctx, lhs, rhs) + return is_ssa(ctx, lhs) || + is_valid_ir_argument(ctx, rhs) || + (kind(lhs) == K"BindingId" && + # FIXME: add: invoke ? + kind(rhs) in KSet"new splatnew cfunction isdefined call foreigncall foreignglobal gc_preserve_begin new_opaque_closure") +end + +function check_no_local_bindings(ctx, ex, msg) + contains_nonglobal_binding = contains_unquoted(ex) do e + kind(e) == K"BindingId" && get_binding(ctx, e).kind !== :global + end + if contains_nonglobal_binding + throw(LoweringError(ex, msg)) + end +end + +# evaluate the arguments of a call, creating temporary locations as needed +function compile_args(ctx, args) + # First check if all the arguments are simple (and therefore side-effect free). + # Otherwise, we need to use ssa values for all arguments to ensure proper + # left-to-right evaluation semantics. + all_simple = all(a->is_simple_arg(ctx, a), args) + args_out = SyntaxList() + for arg in args + arg_val = compile(ctx, arg, true, false) + if isnothing(arg_val) + # arguments that don't return a value, e.g. `f(return)` + push!(args_out, nothing_(ctx, arg)) + elseif ((all_simple || is_const_read_arg(ctx, arg_val)) && + is_valid_body_ir_argument(ctx, arg_val)) + push!(args_out, arg_val) + else + push!(args_out, emit_assign_tmp(ctx, arg_val)) + end + end + return args_out +end + +function emit(ctx::LinearIRContext, ex) + push!(ctx.code, ex) + return ex +end + +# Emit computation of ex, assigning the result to an ssavar and returning that +function emit_assign_tmp(ctx::LinearIRContext, ex, name="tmp") + tmp = ssavar(ctx, ex, name) + emit(ctx, @ast ctx ex [K"=" tmp ex]) + return tmp +end + +function compile_pop_exception(ctx, srcref, src_tokens, dest_tokens) + # It's valid to leave the context of src_tokens for the context of + # dest_tokens when src_tokens is the same or nested within dest_tokens. + # It's enough to check the token on the top of the dest stack. + n = length(dest_tokens) + jump_ok = n == 0 || (n <= length(src_tokens) && syntax_id(dest_tokens[n]) == syntax_id(src_tokens[n])) + jump_ok || throw(LoweringError(srcref, "Attempt to jump into catch block")) + if n < length(src_tokens) + @ast ctx srcref [K"pop_exception" src_tokens[n+1]] + else + nothing + end +end + +function compile_leave_handler(ctx, srcref, src_tokens, dest_tokens) + n = length(dest_tokens) + jump_ok = n == 0 || (n <= length(src_tokens) && syntax_id(dest_tokens[n]) == syntax_id(src_tokens[n])) + jump_ok || throw(LoweringError(srcref, "Attempt to jump into try block")) + if n < length(src_tokens) + @ast ctx srcref [K"leave" src_tokens[n+1:end]...] + else + nothing + end +end + +function emit_pop_exception(ctx::LinearIRContext, srcref, dest_tokens) + pexc = compile_pop_exception(ctx, srcref, ctx.catch_token_stack, dest_tokens) + if !isnothing(pexc) + emit(ctx, pexc) + end +end + +function emit_leave_handler(ctx::LinearIRContext, srcref, dest_tokens) + ex = compile_leave_handler(ctx, srcref, ctx.handler_token_stack, dest_tokens) + if !isnothing(ex) + emit(ctx, ex) + end +end + +# Enter the current finally block, either through the landing pad (on_exit == +# :rethrow) or via a jump (on_exit ∈ (:return, :break)). +# +# An integer tag is created to identify the current code path and select the +# on_exit action to be taken at finally handler exit. +function enter_finally_block(ctx, srcref, on_exit, value) + @jl_assert on_exit ∈ (:rethrow, :break, :return) srcref + handler = last(ctx.finally_handlers) + push!(handler.exit_actions, (on_exit, value)) + tag = length(handler.exit_actions) + emit(ctx, @ast ctx srcref [K"=" handler.tagvar tag::K"Integer"]) + if on_exit != :rethrow + emit_pop_exception(ctx, srcref, handler.target.catch_token_stack) + emit_leave_handler(ctx, srcref, handler.target.handler_token_stack[1:end-1]) + emit(ctx, @ast ctx srcref [K"goto" handler.target.label]) + end + tag +end + +# Helper function for emit_return +function _actually_return(ctx, ex) + # TODO: Handle the implicit return coverage hack for #53354 ? + if (rett = rettype(ctx); !isnothing(rett)) + ex = compile(ctx, convert_for_type_decl(ctx, rett, ex, rett, true), true, false) + end + simple_ret_val = isempty(ctx.catch_token_stack) ? + # returning lambda directly is needed for @generated + (is_valid_ir_argument(ctx, ex) || kind(ex) == K"lambda") : + is_simple_atom(ctx, ex) + if !simple_ret_val + ex = emit_assign_tmp(ctx, ex, "return_tmp") + end + emit_pop_exception(ctx, ex, SyntaxList()) + emit(ctx, @ast ctx ex [K"return" ex]) + return nothing +end + +function emit_return(ctx, srcref, ex) + # todo: Mark implicit returns + if isnothing(ex) + return + elseif isempty(ctx.handler_token_stack) + _actually_return(ctx, ex) + return + end + # TODO: What's this !is_ssa(ctx, ex) here about? + x = if is_simple_atom(ctx, ex) && !(is_ssa(ctx, ex) && !isempty(ctx.finally_handlers)) + ex + elseif !isempty(ctx.finally_handlers) + # todo: Why does flisp lowering create a mutable variable here even + # though we don't mutate it? + # tmp = ssavar(ctx, srcref, "returnval_via_finally") # <- can we use this? + tmp = new_local_binding(ctx, srcref, "returnval_via_finally") + emit(ctx, @ast ctx srcref [K"=" tmp ex]) + tmp + else + emit_assign_tmp(ctx, ex, "returnval_via_finally") + end + if !isempty(ctx.finally_handlers) + enter_finally_block(ctx, srcref, :return, x) + else + emit(ctx, @ast ctx srcref [K"leave" ctx.handler_token_stack...]) + _actually_return(ctx, x) + end + return nothing +end + +function emit_return(ctx, ex) + emit_return(ctx, ex, ex) +end + +function emit_break(ctx, ex) + name = syntax_name(ex[1]) + target = get(ctx.break_targets, name, nothing) + if isnothing(target) + if name == "loop-exit" + throw(LoweringError(ex, "`break` must be used inside a `while`, `for` loop, or `@label` block")) + elseif name == "loop-cont" + throw(LoweringError(ex, "`continue` must be used inside a `while` or `for` loop")) + elseif endswith(name, "#cont") + label = name[1:end-5] + throw(LoweringError(ex, "`continue $label` is not inside a `@label $label` loop")) + else + throw(LoweringError(ex, "`break $name` is not inside a `@label $name` block")) + end + end + # If targeting loop-exit, check for intervening named @label blocks + if name == "loop-exit" + for i in lastindex(ctx.break_label_stack):-1:1 + lbl = ctx.break_label_stack[i] + lbl == "loop-exit" && break + if lbl != "loop-cont" && !contains(lbl, '#') + throw(LoweringError(ex, + "plain `break` inside `@label $lbl` block is disallowed; use `break $lbl` to exit the block")) + end + end + end + # If targeting loop-cont, check for intervening loop-exit (@label block) + if name == "loop-cont" + for i in lastindex(ctx.break_label_stack):-1:1 + lbl = ctx.break_label_stack[i] + lbl == "loop-cont" && break + if lbl == "loop-exit" + throw(LoweringError(ex, "`continue` inside an anonymous `@label` block is not allowed")) + end + end + end + # Handle valued break (break name val) + if numchildren(ex) >= 2 + if isnothing(target.result_var) + throw(LoweringError(ex, "break with value not allowed for label `$name`")) + end + val = compile(ctx, ex[2], true, false) + emit_assignment(ctx, ex, target.result_var, val) + end + if (!isempty(ctx.finally_handlers) && length(target.handler_token_stack) < + length(last(ctx.finally_handlers).target.handler_token_stack)) + enter_finally_block(ctx, ex, :break, ex) + return + else + emit_pop_exception(ctx, ex, target.catch_token_stack) + emit_leave_handler(ctx, ex, target.handler_token_stack) + emit(ctx, @ast ctx ex [K"goto" target.label]) + end +end + +# `op` may be either K"=" (where global assignments are converted to setglobal!) +# or K"constdecl". flisp: emit-assignment-or-setglobal +function emit_simple_assignment(ctx, srcref, lhs, rhs, op=K"=") + binfo = get_binding(ctx, lhs) + if binfo.kind == :global + emit(ctx, @ast ctx srcref [ + K"call" + op == K"constdecl" ? "declare_const"::K"core" : "setglobal!"::K"core" + binfo.mod::K"Value" + binfo.name::K"Symbol" + rhs + ]) + else + emit(ctx, @ast ctx srcref [op lhs rhs]) + end +end + +function emit_assignment(ctx, srcref, lhs, rhs, op=K"=") + if !isnothing(rhs) + if is_valid_ir_rvalue(ctx, lhs, rhs) + emit_simple_assignment(ctx, srcref, lhs, rhs, op) + else + r = emit_assign_tmp(ctx, rhs) + emit_simple_assignment(ctx, srcref, lhs, r, op) + end + else + # in unreachable code (such as after return); still emit the assignment + # so that the structure of those uses is preserved + emit_simple_assignment(ctx, srcref, lhs, nothing_(ctx, srcref), op) + nothing + end +end + +function make_label(ctx, srcref) + id = ctx.next_label_id[] + ctx.next_label_id[] += 1 + newleaf(srcref, K"label", id) +end + +# flisp: make&mark-label +function emit_label(ctx, srcref) + if !isempty(ctx.code) + # Use current label if available + e = ctx.code[end] + if kind(e) == K"label" + return e + end + end + l = make_label(ctx, srcref) + emit(ctx, l) + l +end + +function emit_latestworld(ctx, srcref) + (isempty(ctx.code) || kind(last(ctx.code)) != K"latestworld") && + emit(ctx, kind(srcref) === K"latestworld" ? srcref : + newleaf(srcref, K"latestworld")) +end + +function compile_condition_term(ctx, ex) + cond = compile(ctx, ex, true, false) + isnothing(cond) && return nothing + if !is_valid_body_ir_argument(ctx, cond) + cond = emit_assign_tmp(ctx, cond) + end + return cond +end + +# flisp: emit-cond +function compile_conditional(ctx, ex, false_label) + if kind(ex) == K"block" && numchildren(ex) >= 1 + for i in 1:numchildren(ex)-1 + compile(ctx, ex[i], false, false) + end + test = ex[end] + else + test = ex + end + k = kind(test) + if k == K"||" + true_label = make_label(ctx, test) + for (i,e) in enumerate(children(test)) + c = compile_condition_term(ctx, e) + isnothing(c) && break + if i < numchildren(test) + next_term_label = make_label(ctx, test) + # Jump over short circuit + emit(ctx, @ast ctx e [K"gotoifnot" c next_term_label]) + # Short circuit to true + emit(ctx, @ast ctx e [K"goto" true_label]) + emit(ctx, next_term_label) + else + emit(ctx, @ast ctx e [K"gotoifnot" c false_label]) + end + end + emit(ctx, true_label) + elseif k == K"&&" + for e in children(test) + c = compile_condition_term(ctx, e) + isnothing(c) && break + emit(ctx, @ast ctx e [K"gotoifnot" c false_label]) + end + else + c = compile_condition_term(ctx, test) + isnothing(c) || emit(ctx, @ast ctx test [K"gotoifnot" c false_label]) + end +end + +# Lowering of exception handling must ensure that +# +# * Each `enter` is matched with a `leave` on every possible non-exceptional +# program path (including implicit returns generated in tail position). +# * Each catch block which is entered and handles the exception - by exiting +# via a non-exceptional program path - leaves the block with `pop_exception`. +# * Each `finally` block runs, regardless of any early `return` or jumps +# via `break`/`continue`/`goto` etc. +# +# These invariants are upheld by tracking the nesting using +# `handler_token_stack` and `catch_token_stack` and using these when emitting +# any control flow (return / goto) which leaves the associated block. +# +# The following special forms are emitted into the IR: +# +# (= tok (enter catch_label dynscope)) +# push exception handler with catch block at `catch_label` and dynamic +# scope `dynscope`, yielding a token which is used by `leave` and +# `pop_exception`. `dynscope` is only used in the special `tryfinally` form +# without associated source level syntax (see the `@with` macro) +# +# (leave tok) +# pop exception handler back to the state of the `tok` from the associated +# `enter`. Multiple tokens can be supplied to pop multiple handlers using +# `(leave tok1 tok2 ...)`. +# +# (pop_exception tok) - pop exception stack back to state of associated enter +# +# When an `enter` is encountered, the runtime pushes a new handler onto the +# `Task`'s exception handler stack which will jump to `catch_label` when an +# exception occurs. +# +# There are two ways that the exception-related task state can be restored +# +# 1. By encountering a `leave` which will restore the handler state with `tok`. +# 2. By throwing an exception. In this case the runtime will pop one handler +# automatically and jump to the catch label with the new exception pushed +# onto the exception stack. On this path the exception stack state must be +# restored back to the associated `enter` by encountering `pop_exception`. +# +# Note that the handler and exception stack represent two distinct types of +# exception-related state restoration which need to happen. Note also that the +# "handler state restoration" actually includes several pieces of runtime state +# including GC flags - see `jl_eh_restore_state` in the runtime for that. +# #### Lowering finally code paths +# +# When lowering `finally` blocks we want to emit the user's finally code once +# but multiple code paths may traverse the finally block. For example, consider +# the code +# +# ```julia +# function foo(x) +# while true +# try +# if x == 1 +# return f(x) +# elseif x == 2 +# g(x) +# continue +# else +# break +# end +# finally +# h() +# end +# end +# end +# ``` +# +# In this situation there's four distinct code paths through the finally block: +# 1. `return f(x)` needs to call `val = f(x)`, leave the `try` block, run `h()` then +# return `val`. +# 2. `continue` needs to call `h()` then jump to the start of the while loop +# 3. `break` needs to call `h()` then jump to the exit of the while loop +# 4. If an exception occurs in `f(x)` or `g(x)`, we need to call `h()` before +# falling back into the while loop. +# +# To deal with these we create a `finally_tag` variable to dynamically track +# which action to take after the finally block exits. Before jumping to the +# block we set this variable to a unique integer tag identifying the incoming +# code path. At the exit of the user's code (`h()` in this case) we perform the +# jump appropriate to the `break`, `continue` or `return` as necessary based on +# the tag. +function compile_try(ctx::LinearIRContext, ex, needs_value, in_tail_pos) + (try_block, catch_block, else_block, finally_block, catch_label, scope) = @stm ex begin + [K"trycatchelse" t c] -> (t, c, nothing, nothing, make_label(ctx, c), nothing) + [K"trycatchelse" t c e] -> (t, c, e, nothing, make_label(ctx, c), nothing) + [K"tryfinally" t f] -> (t, nothing, nothing, f, make_label(ctx, f), nothing) + [K"tryfinally" t f scope] -> (t, nothing, nothing, f, make_label(ctx, f), scope) + end + + has_finally_block = !isnothing(finally_block) + end_label = !in_tail_pos || has_finally_block ? make_label(ctx, ex) : nothing + try_result = needs_value && !in_tail_pos ? new_local_binding(ctx, ex, "try_result") : nothing + + enter_scope_arg = SyntaxList() + if scope !== nothing + args = SyntaxList() + push!(args, scope) + enter_scope_arg = compile_args(ctx, args) + end + # Exception handler block prefix + handler_token = ssavar(ctx, ex, "handler_token") + emit(ctx, @ast ctx ex [K"=" + handler_token + [K"enter" catch_label enter_scope_arg...] + ]) + push!(ctx.handler_token_stack, handler_token) + if has_finally_block + # TODO: Trivial finally block optimization from JuliaLang/julia#52593 (or + # support a special form for @with)? + finally_handler = FinallyHandler(new_local_binding(ctx, finally_block, "finally_tag"), + JumpTarget(end_label, ctx)) + push!(ctx.finally_handlers, finally_handler) + emit(ctx, @ast ctx finally_block [K"=" finally_handler.tagvar (-1)::K"Integer"]) + end + + # Try block code. + try_val = compile(ctx, try_block, needs_value, false) + # Exception handler block postfix + if isnothing(else_block) + if in_tail_pos + if !isnothing(try_val) + emit_return(ctx, try_val) + end + else + if needs_value && !isnothing(try_val) + emit_assignment(ctx, ex, try_result, try_val) + end + emit(ctx, @ast ctx ex [K"leave" handler_token]) + end + pop!(ctx.handler_token_stack) + else + if !isnothing(try_val) && (in_tail_pos || needs_value) + emit(ctx, try_val) # TODO: Only for any side effects ? + end + emit(ctx, @ast ctx ex [K"leave" handler_token]) + pop!(ctx.handler_token_stack) + # Else block code + else_val = compile(ctx, else_block, needs_value, in_tail_pos) + if !in_tail_pos + if needs_value && !isnothing(else_val) + emit_assignment(ctx, ex, try_result, else_val) + end + end + end + if !in_tail_pos + emit(ctx, @ast ctx ex [K"goto" end_label]) + end + + # Catch pad + # Emit either catch or finally block. A combined try/catch/finally block + # was split into separate trycatchelse and tryfinally blocks earlier. + emit(ctx, catch_label) # <- Exceptional control flow enters here + if has_finally_block + @assert @isdefined(finally_handler) "compiler hint" + # Attribute the postfix and prefix to the finally block as a whole. + srcref = finally_block + enter_finally_block(ctx, srcref, :rethrow, nothing) + emit(ctx, end_label) # <- Non-exceptional control flow enters here + pop!(ctx.finally_handlers) + compile(ctx, finally_block, false, false) + # Finally block postfix: Emit a branch for every code path which enters + # the block to dynamically decide which return/break/rethrow exit action to take + for (tag, (on_exit, value)) in Iterators.reverse(enumerate(finally_handler.exit_actions)) + next_action_label = !in_tail_pos || tag != 1 || on_exit != :return ? + make_label(ctx, srcref) : nothing + if !isnothing(next_action_label) + tmp = ssavar(ctx, srcref, "do_finally_action") + emit(ctx, @ast ctx srcref [K"=" tmp + [K"call" + "==="::K"core" + finally_handler.tagvar + tag::K"Integer" + ] + ]) + emit(ctx, @ast ctx srcref [K"gotoifnot" tmp next_action_label]) + end + if on_exit === :return + emit_return(ctx, value) + elseif on_exit === :break + emit_break(ctx, value) + elseif on_exit === :rethrow + emit(ctx, @ast ctx srcref [K"call" "rethrow"::K"top"]) + else + @jl_assert false finally_block + end + if !isnothing(next_action_label) + emit(ctx, next_action_label) + end + end + else + @assert !isnothing(catch_block) + push!(ctx.catch_token_stack, handler_token) + catch_val = compile(ctx, catch_block, needs_value, in_tail_pos) + if !isnothing(try_result) && !isnothing(catch_val) + emit_assignment(ctx, ex, try_result, catch_val) + end + if !in_tail_pos + emit(ctx, @ast ctx ex [K"pop_exception" handler_token]) + emit(ctx, end_label) + else + # (pop_exception done in emit_return) + end + pop!(ctx.catch_token_stack) + end + try_result +end + +# This pass behaves like an interpreter on the given code. +# To perform stateful operations, it calls `emit` to record that something +# needs to be done. In value position, it returns an expression computing +# the needed value. +function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos) + k = kind(ex) + if k == K"BindingId" || is_literal(k) || k == K"nothing" || + k == K"inert" || k == K"syntaxinert" || k == K"top" || + k == K"core" || k == K"Value" || k == K"Symbol" || + k == K"SourceLocation" || k == K"static_eval" || + k == K"foreignsymbol" || k == K"static_parameter" + ex1 = ex + if kind(ex1) == K"BindingId" + binfo = get_binding(ctx, ex1) + if haskey(ctx.argmap, binfo.id) + ex1 = newleaf(ex1, K"BindingId", ctx.argmap[binfo.id]) + end + end + if in_tail_pos + emit_return(ctx, ex1) + elseif needs_value + ex1 + else + if k == K"BindingId" && !is_ssa(ctx, ex1) + emit(ctx, ex1) # keep identifiers for undefined-var checking + end + nothing + end + elseif k == K"Placeholder" + if needs_value + throw(LoweringError(ex, "all-underscore identifiers are write-only and their values cannot be used in expressions")) + end + nothing + elseif k == K"TOMBSTONE" + @jl_assert !needs_value (ex,"TOMBSTONE encountered in value position") + nothing + elseif k == K"call" || k == K"new" || k == K"splatnew" || k == K"foreigncall" || + k == K"foreignglobal" || k == K"new_opaque_closure" || k == K"cfunction" + callex = newnode(ex, k, compile_args(ctx, children(ex))) + if in_tail_pos + emit_return(ctx, ex, callex) + elseif needs_value + callex + else + emit(ctx, callex) + nothing + end + elseif k == K"=" || k == K"constdecl" + lhs = ex[1] + res = if kind(lhs) == K"Placeholder" + compile(ctx, ex[2], needs_value, in_tail_pos) + elseif k == K"constdecl" && numchildren(ex) == 1 + # No RHS - make undefined constant + mod, name = if kind(ex[1]) == K"BindingId" + binfo = get_binding(ctx, ex[1]) + binfo.mod, binfo.name + else + @jl_assert kind(ex[1]) == K"Value" && typeof(ex[1].value) === GlobalRef ex + gr = ex[1].value + gr.mod, String(gr.name) + end + emit(ctx, @ast ctx ex [K"call" "declare_const"::K"core" + mod::K"Value" name::K"Symbol"]) + else + rhs = compile(ctx, ex[2], true, false) + if kind(lhs) == K"BindingId" + binfo = get_binding(ctx, lhs) + if haskey(ctx.argmap, binfo.id) + lhs = newleaf(lhs, K"BindingId", ctx.argmap[binfo.id]) + end + end + if needs_value && !isnothing(rhs) + r = emit_assign_tmp(ctx, rhs) + emit_simple_assignment(ctx, ex, lhs, r, k) + if in_tail_pos + emit_return(ctx, ex, r) + else + r + end + else + emit_assignment(ctx, ex, lhs, rhs, k) + end + end + k == K"constdecl" && emit_latestworld(ctx, ex) + res + elseif k == K"block" || k == K"scope_block" + nc = numchildren(ex) + if nc == 0 + if in_tail_pos + emit_return(ctx, nothing_(ctx, ex)) + elseif needs_value + nothing_(ctx, ex) + else + nothing + end + else + res = nothing + for i in 1:nc + islast = i == nc + res = compile(ctx, ex[i], islast && needs_value, islast && in_tail_pos) + end + res + end + elseif k == K"symbolicblock" + name = syntax_name(ex[1]) + # Skip duplicate check for default-scope labels (loop-exit, loop-cont) which allow nesting + if name != "loop-exit" && name != "loop-cont" + if haskey(ctx.symbolic_jump_targets, name) || name in ctx.symbolic_block_labels + throw(LoweringError(ex, "Label `$name` defined multiple times")) + end + push!(ctx.symbolic_block_labels, name) + end + end_label = make_label(ctx, ex) + need_value = needs_value || in_tail_pos + result_var = need_value ? new_local_binding(ctx, ex, "$(name)_result") : nothing + outer_target = get(ctx.break_targets, name, nothing) + ctx.break_targets[name] = JumpTarget(end_label, ctx, result_var) + push!(ctx.break_label_stack, name) + body_val = compile(ctx, ex[2], need_value, false) + pop!(ctx.break_label_stack) + if !isnothing(result_var) && !isnothing(body_val) + emit_assignment(ctx, ex, result_var, body_val) + end + if isnothing(outer_target) + delete!(ctx.break_targets, name) + else + ctx.break_targets[name] = outer_target + end + emit(ctx, end_label) + # Use isdefined to handle the case where initialization was + # skipped (e.g., by @goto jumping into a loop body). + if !isnothing(result_var) + defined_label = make_label(ctx, ex) + done_label = make_label(ctx, ex) + isdef = emit_assign_tmp(ctx, @ast ctx ex [K"isdefined" result_var]) + emit(ctx, @ast ctx ex [K"gotoifnot" isdef defined_label]) + emit(ctx, @ast ctx ex [K"goto" done_label]) + emit(ctx, defined_label) + emit_assignment(ctx, ex, result_var, nothing_(ctx, ex)) + emit(ctx, done_label) + end + if in_tail_pos + emit_return(ctx, ex, result_var) + nothing + elseif needs_value + result_var + end + elseif k == K"break" + emit_break(ctx, ex) + nothing + elseif k == K"symboliclabel" + label = emit_label(ctx, ex) + name = syntax_name(ex) + if haskey(ctx.symbolic_jump_targets, name) || name in ctx.symbolic_block_labels + throw(LoweringError(ex, "Label `$name` defined multiple times")) + end + push!(ctx.symbolic_jump_targets, name=>JumpTarget(label, ctx)) + if in_tail_pos + emit_return(ctx, ex, nothing_(ctx, ex)) + elseif needs_value + throw(LoweringError(ex, "misplaced label in value position")) + end + elseif k == K"symbolicgoto" + push!(ctx.symbolic_jump_origins, JumpOrigin(ex, length(ctx.code)+1, ctx)) + emit(ctx, newleaf(ex, K"TOMBSTONE")) # ? pop_exception + emit(ctx, newleaf(ex, K"TOMBSTONE")) # ? leave + emit(ctx, newleaf(ex, K"TOMBSTONE")) # ? goto + nothing + elseif k == K"return" + compile(ctx, ex[1], true, true) + nothing + elseif k == K"removable" + if needs_value + compile(ctx, ex[1], needs_value, in_tail_pos) + else + nothing + end + elseif k == K"if" || k == K"elseif" + @jl_assert numchildren(ex) <= 3 ex + has_else = numchildren(ex) > 2 + else_label = make_label(ctx, ex) + compile_conditional(ctx, ex[1], else_label) + if in_tail_pos + compile(ctx, ex[2], needs_value, in_tail_pos) + emit(ctx, else_label) + if has_else + compile(ctx, ex[3], needs_value, in_tail_pos) + else + emit_return(ctx, ex, nothing_(ctx, ex)) + end + nothing + else + val = needs_value && new_local_binding(ctx, ex, "if_val") + v1 = compile(ctx, ex[2], needs_value, in_tail_pos) + if needs_value + emit_assignment(ctx, ex, val, v1) + end + if has_else || needs_value + end_label = make_label(ctx, ex) + emit(ctx, @ast ctx ex [K"goto" end_label]) + else + end_label = nothing + end + emit(ctx, else_label) + v2 = if has_else + compile(ctx, ex[3], needs_value, in_tail_pos) + elseif needs_value + nothing_(ctx, ex) + end + if needs_value + emit_assignment(ctx, ex, val, v2) + end + if !isnothing(end_label) + emit(ctx, end_label) + end + val + end + elseif k == K"trycatchelse" || k == K"tryfinally" + compile_try(ctx, ex, needs_value, in_tail_pos) + elseif k == K"method" + @jl_assert ctx.is_toplevel_thunk (ex, "method not at top level") + mval = if numchildren(ex) == 1 + # Generic function declaration: define_method(module, name) + func_name = ex[1] + mod, name = if kind(func_name) == K"BindingId" + binfo = get_binding(ctx, func_name) + binfo.mod, binfo.name + elseif kind(func_name) == K"globalref" + func_name.mod, syntax_name(func_name) + else + ctx.mod, syntax_name(func_name) + end + emit_assign_tmp(ctx, @ast ctx ex [K"call" "define_method"::K"core" + mod::K"Value" name::K"Symbol"]) + else + @jl_assert numchildren(ex) == 3 ex + fname = ex[1] + sig = compile(ctx, ex[2], true, false) + if !is_valid_ir_argument(ctx, sig) + sig = emit_assign_tmp(ctx, sig) + end + lam = ex[3] + if kind(lam) == K"lambda" + lam = compile_lambda(ctx, lam) + else + lam = emit_assign_tmp(ctx, compile(ctx, lam, true, false)) + end + emit_assign_tmp(ctx, @ast ctx ex [K"call" "define_method"::K"core" + ctx.mod::K"Value" fname sig lam]) + end + emit_latestworld(ctx, ex) + out = if in_tail_pos + emit_return(ctx, mval) + elseif needs_value + mval + else + nothing + end + out + elseif k == K"opaque_closure_method" + @ast ctx ex [K"opaque_closure_method" + ex[1] + ex[2] + ex[3] + ex[4] + compile_lambda(ctx, ex[5]) + ] + elseif k in KSet"lambda generated_lambda toplevel_lambda" + lam = compile_lambda(ctx, ex) + if in_tail_pos + emit_return(ctx, lam) + elseif needs_value + lam + else + emit(ctx, lam) + end + elseif k == K"gc_preserve_begin" + newnode(ex, k, compile_args(ctx, children(ex))) + elseif k == K"gc_preserve_end" || k == K"loopinfo" + if needs_value + throw(LoweringError(ex, "misplaced kind $k in value position")) + end + emit(ctx, ex) + nothing + elseif k == K"meta" + if numchildren(ex) >= 1 + # Certain blessed forms are allowed to share a meta expression; + # others (nkw, optlevel) treat ex[1] as head and ex[2:end] as args + if kind(ex[1]) === K"purity" || + kind(ex[1]) === K"Symbol" && syntax_name(ex[1]) in ( + "inline", "noinline", "propagate_inbounds", + "nospecializeinfer", "aggressive_constprop", "no_constprop") + for c in children(ex) + if kind(c) === K"purity" + old = get(ctx.meta, :purity, UInt16(0)) + ctx.meta[:purity] = (old | purity_expr_to_flags(c))::UInt16 + elseif kind(c) === K"Symbol" + ctx.meta[Symbol(syntax_name(c))] = true + else + @jl_assert false c + end + end + else + emit(ctx, ex) + end + end + if needs_value + val = @ast ctx ex (::K"nothing") + if in_tail_pos + emit_return(ctx, val) + else + val + end + end + elseif k == K"inbounds" || k == K"inbounds_pop" || + k == K"inline" || k == K"noinline" || k == K"purity" || + k == K"aliasscope" || k == K"popaliasscope" + emit(ctx, ex) # if absorbed in flags, converted to nothing later + if needs_value + val = @ast ctx ex (::K"nothing") + if in_tail_pos + emit_return(ctx, val) + else + val + end + end + elseif k == K"_while" + end_label = make_label(ctx, ex) + top_label = emit_label(ctx, ex) + compile_conditional(ctx, ex[1], end_label) + compile(ctx, ex[2], false, false) + emit(ctx, @ast ctx ex [K"goto" top_label]) + emit(ctx, end_label) + if needs_value + compile(ctx, nothing_(ctx, ex), needs_value, in_tail_pos) + end + elseif k == K"_do_while" + end_label = make_label(ctx, ex) + top_label = emit_label(ctx, ex) + compile(ctx, ex[1], false, false) + compile_conditional(ctx, ex[2], end_label) + emit(ctx, @ast ctx ex [K"goto" top_label]) + emit(ctx, end_label) + if needs_value + compile(ctx, nothing_(ctx, ex), needs_value, in_tail_pos) + end + elseif k == K"isdefined" || k == K"captured_local" || + k == K"throw_undef_if_not" || k == K"boundscheck" + if in_tail_pos + emit_return(ctx, ex) + elseif needs_value + ex + end + elseif k == K"newvar" + @jl_assert !needs_value ex + is_duplicate = !isempty(ctx.code) && + (e = last(ctx.code); kind(e) == K"newvar" && syntax_id(e[1]) == syntax_id(ex[1])) + if !is_duplicate + # TODO: also exclude deleted vars + emit(ctx, ex) + end + elseif k == K"latestworld" + if needs_value + throw(LoweringError(ex, "misplaced latestworld")) + end + emit_latestworld(ctx, ex) + elseif k == K"latestworld_if_toplevel" + ctx.is_toplevel_thunk && emit_latestworld(ctx, ex) + elseif k == K"unused_only" + if needs_value && !in_tail_pos + throw(LoweringError( + ex, "global declaration doesn't read the variable and can't return a value")) + end + if needs_value && in_tail_pos && !ctx.is_toplevel_thunk + compile(ctx, ex[1], false, false) + compile(ctx, @ast(ctx, ex, (::K"nothing")), needs_value, in_tail_pos) + else + compile(ctx, ex[1], needs_value, in_tail_pos) + end + else + throw(LoweringError(ex, "Invalid syntax; $(repr(k))")) + end +end + +function _remove_vars_with_isdefined_check!(vars, ex) + if is_leaf(ex) || is_quoted(ex) || kind(ex) == K"static_eval" + return + elseif kind(ex) == K"isdefined" + delete!(vars, syntax_id(ex[1])) + else + for e in children(ex) + _remove_vars_with_isdefined_check!(vars, e) + end + end +end + +# Find newvar nodes that are unnecessary because +# 1. The variable is not captured and +# 2. The variable is assigned before any branches. +# +# This is used to remove newvar nodes that are not needed for re-initializing +# variables to undefined (see Julia issue #11065). It doesn't look for variable +# *uses*, because any variables used-before-def that also pass this test are +# *always* used undefined, and therefore don't need to be reinitialized. The +# one exception to that is `@isdefined`, which can observe an undefined +# variable without throwing an error. +function unnecessary_newvar_ids(ctx, stmts) + vars = Set{IdTag}() + ids_assigned_before_branch = Set{IdTag}() + for ex in stmts + _remove_vars_with_isdefined_check!(vars, ex) + k = kind(ex) + if k == K"newvar" + id = syntax_id(ex[1]) + if !get_binding(ctx, id).is_captured + push!(vars, id) + end + elseif k == K"goto" || k == K"gotoifnot" || (k == K"=" && kind(ex[2]) == K"enter") + empty!(vars) + elseif k == K"=" + id = syntax_id(ex[1]) + if id in vars + delete!(vars, id) + push!(ids_assigned_before_branch, id) + end + end + end + ids_assigned_before_branch +end + +# flisp: compile-body +function compile_body(ctx::LinearIRContext, ex) + compile(ctx, ex, true, true) + + # Fix up any symbolic gotos. (We can't do this earlier because the goto + # might precede the label definition in unstructured control flow.) + for origin in ctx.symbolic_jump_origins + name = syntax_name(origin.goto) + target = get(ctx.symbolic_jump_targets, name, nothing) + if isnothing(target) + # Check if it's a symbolic block label + if name in ctx.symbolic_block_labels + throw(LoweringError(origin.goto, "cannot use @goto to jump to @label block `$name`")) + end + throw(LoweringError(origin.goto, "label `$name` referenced but not defined")) + end + i = origin.index + pop_ex = compile_pop_exception(ctx, origin.goto, origin.catch_token_stack, + target.catch_token_stack) + if !isnothing(pop_ex) + @jl_assert kind(ctx.code[i]) == K"TOMBSTONE" ctx.code[i] + ctx.code[i] = pop_ex + i += 1 + end + leave_ex = compile_leave_handler(ctx, origin.goto, origin.handler_token_stack, + target.handler_token_stack) + if !isnothing(leave_ex) + @jl_assert kind(ctx.code[i]) == K"TOMBSTONE" ctx.code[i] + ctx.code[i] = leave_ex + i += 1 + end + @jl_assert kind(ctx.code[i]) == K"TOMBSTONE" ctx.code[i] + ctx.code[i] = @ast ctx origin.goto [K"goto" target.label] + end + + # Filter out unnecessary newvar nodes + ids_assigned_before_branch = unnecessary_newvar_ids(ctx, ctx.code) + filter!(ctx.code) do ex + !(kind(ex) == K"newvar" && syntax_id(ex[1]) in ids_assigned_before_branch) + end +end + +#------------------------------------------------------------------------------- + +# Recursively renumber an expression within linear IR +# flisp: renumber-stuff +function _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, ex) + k = kind(ex) + if k == K"BindingId" + id = syntax_id(ex) + if haskey(ssa_rewrites, id) + newleaf(ex, K"SSAValue", ssa_rewrites[id]) + else + new_id = get(slot_rewrites, id, nothing) + binfo = get_binding(ctx, id) + if !isnothing(new_id) + sk = binfo.kind == :local || binfo.kind == :argument ? K"slot" : + binfo.kind == :static_parameter ? K"static_parameter" : + throw(LoweringError(ex, "Found unexpected binding of kind $(binfo.kind)")) + newleaf(ex, sk, new_id) + else + if binfo.kind !== :global + throw(LoweringError(ex, "Found unexpected binding of kind $(binfo.kind)")) + end + @mknode(ex; kind=K"globalref", value=binfo.name, mod=binfo.mod) + end + end + elseif k == K"meta" || k == K"static_eval" + # Somewhat-hack for Expr(:meta, :generated, gen) which has + # weird top-level semantics for `gen`, but we still need to translate + # the binding it contains to a globalref. (TODO: use + # static_eval for this meta, somehow) + mapchildren(ex) do e + _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, e) + end + elseif is_literal(k) || is_quoted(k) + ex + elseif k == K"label" + @ast ctx ex label_table[syntax_id(ex)]::K"label" + elseif k == K"code_info" + ex + else + mapchildren(ex) do e + _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, e) + end + end +end + +# flisp: renumber-lambda, compact-ir +function renumber_body(ctx, input_code, slot_rewrites) + # Step 1: Remove any assignments to SSA variables, record the indices of labels + ssa_rewrites = Dict{IdTag,IdTag}() + label_table = Dict{Int,Int}() + code = SyntaxList() + for ex in input_code + k = kind(ex) + ex_out = nothing + if k == K"=" && (b = get_binding(ctx, ex[1]); b.is_ssa || b.kind == :typevar) + lhs_id = syntax_id(ex[1]) + @jl_assert(!haskey(ssa_rewrites, lhs_id), + (ex, "multiple assignments to ssavalue")) + @jl_assert ctx.is_toplevel_thunk || b.kind !== :typevar binding_ex(ctx, b) + if is_ssa(ctx, ex[2]) + # For SSA₁ = SSA₂, record that all uses of SSA₁ should be replaced by SSA₂ + ssa_rewrites[lhs_id] = ssa_rewrites[syntax_id(ex[2])] + else + # Otherwise, record which `code` index this SSA value refers to + ssa_rewrites[lhs_id] = length(code) + 1 + ex_out = ex[2] + end + elseif k == K"label" + label_table[syntax_id(ex)] = length(code) + 1 + elseif k == K"TOMBSTONE" + # remove statement + else + ex_out = ex + end + if !isnothing(ex_out) + push!(code, ex_out) + end + end + + # Step 2: + # * Translate any SSA uses and labels into indices in the code table + # * Translate locals into slot indices + for i in 1:length(code) + code[i] = _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, code[i]) + end + code +end + +struct Slot + name::String + kind::Symbol + is_nospecialize::Bool + is_read::Bool + is_single_assign::Bool + is_maybe_undef::Bool + is_called::Bool +end + +function compile_lambda(outer_ctx, ex) + k = kind(ex) + lbs = lambda_bindings(ex[1]) + lambda_args = ex[2] + static_parameters = ex[3] + ctx = LinearIRContext( + outer_ctx, k === K"toplevel_lambda", lbs) + if numchildren(ex) == 5 + tmp = ssavar(ctx, ex[5], "rett") + ctx.rettype_ssa[] = tmp + compile(ctx, @ast(ctx, ex[5], [K"=" tmp ex[5]]), false, false) + end + for arg in children(lambda_args) + kind(arg) == K"Placeholder" && continue + @jl_assert kind(arg) == K"BindingId" ex + binfo = get_binding(ctx, arg) + if binfo.is_assigned + @jl_assert !haskey(ctx.argmap, binfo.id) ex arg + ctx.argmap[binfo.id] = syntax_id(new_local_binding(ctx, binding_ex(ctx, binfo), binfo.name)) + end + end + compile_body(ctx, ex[4]) + for (id, remapped) in pairs(ctx.argmap) + binding = binding_ex(ctx, id) + local_slot = binding_ex(ctx, remapped) + pushfirst!(ctx.code, @ast ctx binding [K"=" local_slot binding]) + end + slots = Vector{Slot}() + slot_rewrites = Dict{IdTag,Int}() + for arg in children(lambda_args) + if kind(arg) == K"Placeholder" + # Unused functions arguments like: `_` or `::T` + push!(slots, Slot(UNUSED, :argument, + getmeta(arg, :nospecialize, false)::Bool, + false, false, false, false)) + else + @jl_assert kind(arg) == K"BindingId" ex arg + id = syntax_id(arg) + binfo = get_binding(ctx, id) + @jl_assert binfo.kind == :local || binfo.kind == :argument ex arg + push!(slots, Slot(binfo.name, :argument, binfo.is_nospecialize, + binfo.is_read, binfo.is_assigned_once, + binfo.is_used_undef, binfo.is_called)) + slot_rewrites[id] = length(slots) + end + end + # Sorting the lambda locals is required to remove dependence on Dict iteration order. + for (id, is_capt) in sort(collect(pairs(lbs.locals_capt)), by=first) + if !is_capt + binfo = get_binding(ctx.bindings, id) + if binfo.kind == :local + push!(slots, Slot(binfo.name, :local, false, + binfo.is_read, binfo.is_assigned_once, + binfo.is_used_undef, binfo.is_called)) + slot_rewrites[id] = length(slots) + end + end + end + for (i,arg) in enumerate(children(static_parameters)) + @jl_assert kind(arg) == K"BindingId" arg + id = syntax_id(arg) + info = get_binding(ctx.bindings, id) + @jl_assert info.kind == :static_parameter arg + slot_rewrites[id] = i + end + let ns_slots = SyntaxList() + for (i, s) in enumerate(slots) + if s.is_nospecialize + s.kind === :argument || throw(LoweringError( + ex, "nospecialize on non-argument")) + push!(ns_slots, newleaf(lambda_args[i], K"slot", i)) + end + end + if !isempty(ns_slots) + nargs = numchildren(lambda_args) + @jl_assert(length(ns_slots) < nargs, ex) + # all args but self + length(ns_slots) == nargs - 1 && empty!(ns_slots) + pushfirst!(ctx.code, + @ast ctx lambda_args [K"meta" "nospecialize"::K"Symbol" ns_slots...]) + end + end + code = renumber_body(ctx, ctx.code, slot_rewrites) + meta = CompileHints() + for (k, v) in ctx.meta + meta = CompileHints(meta, k, v) + end + out = @ast ctx ex [K"code_info"(;meta=meta) + slots::K"Slots" + [K"block"(ex[4]) code...] + ] + k === K"toplevel_lambda" ? @ast(ctx, ex, [K"thunk" out]) : out +end + +""" +This pass converts nested ASTs in the body of a lambda into a list of +statements (ie, Julia's linear/untyped IR). + +Most of the complexity of this pass is in lowering structured control flow (if, +loops, etc) to gotos and exception handling to enter/leave. We also convert +`K"BindingId"` into `K"slot"`, `K"globalref"` or `K"SSAValue"` as appropriate. +""" +@fzone "JL: linearize" function linearize_ir(ctx::ClosureConversionCtx, ex) + ctx_out = LinearIRContext(ctx, false, LambdaBindings()) + ex_out = compile_lambda(ctx_out, ex) + ctx_out, ex_out +end diff --git a/JuliaLowering/src/macro_expansion.jl b/JuliaLowering/src/macro_expansion.jl new file mode 100644 index 0000000000000..2e6b11765400b --- /dev/null +++ b/JuliaLowering/src/macro_expansion.jl @@ -0,0 +1,410 @@ +# One per pass +struct MacroExpansionContext <: AbstractLoweringContext + syntax_context::SyntaxContext + known_layers::Dict{ScopeLayer, Bool} + world::UInt + recursive::Bool +end + +function MacroExpansionContext(st, world, recursive) + sc = st.context::SyntaxContext + MacroExpansionContext( + sc, Dict{ScopeLayer, Bool}(base_layer(sc)=>true), + world, recursive) +end + +function collect_unquoted!(ctx, unquoted, ex, depth) + if kind(ex) == K"$" && depth == 0 + # children(ex) is usually length 1, but for double interpolation it may + # be longer and the children may contain K"..." expressions. Wrapping + # in a tuple groups the arguments together correctly in those cases. + push!(unquoted, @ast ctx ex [K"tuple" children(ex)...]) + else + inner_depth = kind(ex) == K"quote" ? depth + 1 : + kind(ex) == K"$" ? depth - 1 : + depth + for e in children(ex) + collect_unquoted!(ctx, unquoted, e, inner_depth) + end + end + return unquoted +end + +# TODO: Implementing interpolations with a macro could give us better provenance +function expand_quote(ctx, st) + unquoted = SyntaxList() + collect_unquoted!(ctx, unquoted, st, 0) + # not just optimizations; expected e.g. in `(. mod (quote field))` + if is_expr_value(st) + @jl_assert isempty(unquoted) st + st + elseif kind(st) === K"$" + numchildren(st) != 1 && throw(LoweringError( + st, raw"More than one value in bare `$` expression")) + kind(st[1]) === K"..." && throw(LoweringError( + st, raw"unexpected `...` in bare `$` expression")) + @ast ctx st st[1] + elseif kind(st) === K"Identifier" && st.mod === nothing + @jl_assert isempty(unquoted) st + @ast ctx st [K"inert" st] + else + @ast ctx st [K"call" interpolate_expr::K"Value" [K"inert" st] unquoted...] + end +end + +function collect_syntaxunquote!(ctx, unquoted, st, depth) + if kind(st) === K"syntaxunquote" && depth == 0 + numchildren(st) !== 1 && throw(LoweringError(st, "malformed syntaxunquote")) + push!(unquoted, @ast ctx st[1] [K"tuple" st[1]]) + else + inner_depth = kind(st) == K"syntaxquote" ? depth + 1 : + kind(st) == K"syntaxunquote" ? depth - 1 : depth + for c in children(st) + collect_syntaxunquote!(ctx, unquoted, c, inner_depth) + end + end + unquoted +end + +# compared to quote: (1) no need for a copy when no unquotes, since output +# should be immutable. (2) we do not optimize (syntaxquote (syntaxunquote x)) +# -> x, since x may not be a SyntaxTree +function expand_syntaxquote(ctx, st) + if kind(st) === K"syntaxunquote" + numchildren(st) != 1 && throw(LoweringError( + st, raw"More than one value in bare `syntaxunquote` expression")) + kind(st[1]) === K"..." && throw(LoweringError( + st, raw"unexpected `...` in bare `syntaxunquote` expression")) + end + unquoted = collect_syntaxunquote!(ctx, SyntaxList(), st, 0) + length(unquoted) == 0 ? @ast(ctx, st, [K"syntaxinert" st]) : + @ast ctx st [K"call" interpolate_syntax::K"Value" + [K"syntaxinert" st] unquoted...] +end + +# Passed to the user as an implicit macro argument +struct MacroContext <: AbstractLoweringContext + macrocall::SyntaxTree +end + +struct MacroExpansionError <: Exception + context::Union{Nothing,MacroContext} + ex::SyntaxTree + msg::String + "The source position relative to the node - may be `:begin` or `:end` or `:all`" + position::Symbol + "Error that occurred inside the macro function call (`nothing` if no inner exception)" + err + MacroExpansionError( + context::Union{Nothing,MacroContext}, ex::SyntaxTree, msg::AbstractString, position::Symbol, + @nospecialize err = nothing + ) = new(context, ex, msg, position, err) +end + +function MacroExpansionError(ex::SyntaxTree, msg::AbstractString; position=:all) + MacroExpansionError(nothing, ex, msg, position) +end + +function Base.showerror(io::IO, exc::MacroExpansionError) + print(io, "MacroExpansionError") + ctx = exc.context + if !isnothing(ctx) + # Use `Expr` formatting to pretty print the macro name for now - + # there's quite a lot of special cases. We could alternatively consider + # calling sourcetext() though that won't work well if it's a + # synthetically-generated macro name path. + macname_str = string(Expr( + :macrocall, est_to_expr(ctx.macrocall[1]), nothing)) + print(io, " while expanding ", macname_str, + " in module ", syntax_module(ctx.macrocall)) + end + print(io, ":\n") + # TODO: Display niceties: + # * Show the full provenance tree somehow, in addition to the primary + # source location we're showing here? + # * What if the expression doesn't arise from a source file? + # * How to deal with highlighting trivia? Could provide a token kind or + # child position within the raw tree? How to abstract this?? + src = sourceref(exc.ex) + if src isa LineNumberNode + highlight(io, src, note=exc.msg) + else + fb = first_byte(src) + lb = last_byte(src) + pos = exc.position + byterange = pos == :all ? (fb:lb) : + pos == :begin ? (fb:fb-1) : + pos == :end ? (lb+1:lb) : + error("Unknown position $pos") + highlight(io, src.file[], byterange, note=exc.msg) + end + if !isnothing(exc.err) + print(io, "\nCaused by:\n") + showerror(io, exc.err) + end +end + +function _eval_dot(world::UInt, ex::SyntaxTree) + if kind(ex) === K"." && numchildren(ex) == 2 + lhs = _eval_dot(world, ex[1]) + lhs isa Module || return nothing + rhs = kind(ex[2]) === K"inert" ? ex[2][1] : ex[2] + kind(rhs) in KSet"Identifier Symbol" || return nothing + _invoke_in_world(world, getproperty, lhs, Symbol(syntax_name(rhs))) + elseif kind(ex) === K"Value" + ex.value + elseif kind(ex) === K"Identifier" + _invoke_in_world(world, getproperty, + syntax_module(ex), Symbol(syntax_name(ex))) + else + nothing + end +end + +# If macroexpand(ex[1]) is an identifier or dot-expression, we can simply grab +# it from the correct module in ctx.world. Otherwise, we need to eval arbitrary +# code (which, TODO: does not use the correct world age, and it isn't clear the +# language is meant to support this). +function eval_macro_name(ctx, mctx::MacroContext, st0::SyntaxTree) + st = expand_forms_1(ctx, st0) + try + ed = _eval_dot(ctx.world, st) + if !isnothing(ed) + ed + else + # `ex` might contain a nontrivial mix of scopes so we can't just + # `eval()` it, as it's already been partially lowered by this point. + # Instead, we repeat the latter parts of `lower()` here. + mod = syntax_module(st)::Module + ctx2, st2 = expand_forms_2(st, ctx.world) + ctx3, st3 = resolve_scopes(ctx2, st2) + ctx4, st4 = convert_closures(ctx3, st3) + _ctx5, st5 = linearize_ir(ctx4, st4) + expr_form = to_lowered_expr(st5) + ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form) + end + catch err + throw(MacroExpansionError(mctx, st, "Macro not found", :all, err)) + end +end + +function _macrocall_expr_location(st::SyntaxTree) + @jl_assert kind(st) === K"macrocall" st + if kind(st[2]) === K"Value" + loc = st[2].value + if loc isa MacroSource + loc + elseif loc isa LineNumberNode + # Some macros, e.g. @cmd, don't play nicely with file == nothing + isnothing(loc.file) ? LineNumberNode(loc.line, :none) : loc + else + LineNumberNode(0, :none) + end + elseif kind(st[2]) === K"VERSION" + loc = source_location(LineNumberNode, st) + @static isdefinedglobal(Core, :MacroSource) ? Core.MacroSource(loc, st[2].value) : loc + else + LineNumberNode(0, :none) + end +end + +function expand_macro(ctx::MacroExpansionContext, st::SyntaxTree) + @jl_assert kind(st) === K"macrocall" st + numchildren(st) >= 2 || throw(LoweringError( + st, "`macrocall` requires a macro name and source location")) + sc_in = st.context::SyntaxContext + macname = st[1] + mctx = MacroContext(st) + macfunc = eval_macro_name(ctx, mctx, macname) + raw_args = st[3:end] + + # `ctx.world === typemax(UInt)` is our sentinel for "latest world" + macro_world = ctx.world === typemax(UInt) ? Base.get_world_counter() : ctx.world + has_new_macro = hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=macro_world) + + if has_new_macro + macro_args = [mctx, raw_args...] + macro_mi = lookup_method_instance(macfunc, macro_args, macro_world) + expanded = try + _invoke_in_world(ctx.world, macfunc, macro_args...) + catch exc + newexc = exc isa MacroExpansionError ? + MacroExpansionError(mctx, exc.ex, exc.msg, exc.position, exc.err) : + MacroExpansionError(mctx, st, "Error expanding macro", :all, exc) + rethrow(newexc) + end + st_out = if expanded isa SyntaxTree + expanded + else + expanded isa Expr && throw(LoweringError( + st, "implicit expr->syntaxtree: may later be allowed, but is probably a mistake today")) + expr_to_est(expanded, st) + end + else + macro_loc = _macrocall_expr_location(st) + macro_lnn = macro_loc isa MacroSource ? macro_loc.lno : macro_loc + macro_args = Any[macro_loc, base_layer(ctx.syntax_context).mod] + for arg in raw_args + @jl_assert kind(arg) !== K"VERSION" arg # handled in EST conversion + push!(macro_args, est_to_expr(arg)) + end + macro_mi = lookup_method_instance(macfunc, macro_args, macro_world) + st_out = try + _invoke_in_world(ctx.world, macfunc, macro_args...) + catch exc + if exc isa MethodError && exc.f === macfunc && !isempty( + methods_in_world(macfunc, Tuple{typeof(mctx), Vararg{Any}}, ctx.world, st)) + # If the macro has at least some methods implemented in the + # new style, assume the user meant to call one of those + # rather than any old-style macro methods which might exist + exc = MethodError(macfunc, (mctx, raw_args...,), ctx.world) + end + rethrow(LoadError( + string(macro_lnn.file), macro_lnn.line, + MacroExpansionError(mctx, st, "Error expanding macro", :all, exc))) + end + st_out = expr_to_est(st_out, macro_lnn) + end + # Module scope for the returned AST is the module where this particular + # method was defined (may be different from `parentmodule(macfunc)`) + mod_for_ast = macro_mi !== nothing ? macro_mi.def.module : parentmodule(macfunc) + sc2 = SyntaxContext( + ScopeLayer(mod_for_ast, sc_in.layer), st, + (has_new_macro ? JL_NEW_SYNTAX_VERSION : JL_OLD_SYNTAX_VERSION), false) + st_out2 = apply_expansion_layer(ctx, st_out, sc2, true, 0, 0) + st_out3 = !ctx.recursive ? st_out2 : expand_forms_1(ctx, st_out2) + has_new_macro || _note_32026_hack!(st_out3, sc2) + st_out3 +end + +function known_layer(ctx, sl::Union{Nothing, ScopeLayer}) + isnothing(sl) && return false + get!(ctx.known_layers, sl) do + known_layer(ctx, sl.escaped) + end +end + +""" +When a macro expands, we add a fresh layer to all new syntax in the expansion. +Any syntax that doesn't share a base layer with the top-level thunk is +considered "new". This is similar to racket's flip-scope operation, but +simpler and less powerful (we lose any layer we overwrite, but we gain the +invariant that all layers have the same root after every expansion, so +`escape` is well-defined). + +Implementation notes: + +- `escape` can never be resolved to a layer inside a macrocall, since we must + expand to know whether (old) more escapes will surround it or (new) the macro + moves the escape to another layer. + +- `escape` nodes coming in usually have no layer (old expansion). New + expansions can't create escapes, but can pass an argument containing `escape` + through, and arguments must have full context. Thus, if we see `escape` with + any context, we know the layer is uniform, and is the layer we want to escape + from, so we remove inner context. (if there's an an old macro requiring + caller-side `esc(arg)`, the new macro must also bump `esc(arg)`'s layer.) + +- We could try some shortcuts in module/toplevel, but note that macrocall/quote + need full context (a macro may extract an arbitrary child), and even when + `done`, module/toplevel may contain syntax with arbitrary layers that we must + clean up now (later, we lose the base layer used to detect new syntax) +""" +function apply_expansion_layer(ctx, st::SyntaxTree, sc_in::SyntaxContext, done, + qdepth, sqdepth) + @jl_assert known_layer(ctx, base_layer(sc_in)) st + sc0 = st.context + sc = (isnothing(sc0) || !known_layer(ctx, sc0.layer)) ? sc_in : sc0 + k = kind(st) + absorb_esc = done && qdepth == 0 && sqdepth == 0 + out = if is_leaf(st) + st.context !== sc ? @mknode(st; context=sc) : st + elseif k === K"escape" && absorb_esc + if numchildren(st) !== 1 + throw(LoweringError(st, "`escape` requires one argument")) + elseif is_base_layer(sc) + throw(LoweringError(st, "`escape` node in outer context")) + elseif !is_flisp_compat(sc) + throw(LoweringError(st, "new macros should not use `escape`")) + end + st1 = isnothing(sc0) ? st[1] : remove_context(st[1]) + apply_expansion_layer( + ctx, st1, escape_layer(sc, false), true, qdepth, sqdepth) + elseif k === K"hygienic-scope" && absorb_esc + if !(2 <= numchildren(st) <= 3) + throw(LoweringError(st, "`hygienic-scope` requires 2-3 children")) + elseif kind(st[2]) !== K"Value" || !(st[2].value isa Module) + throw(LoweringError(st, "`hygienic-scope` arg 2: expected Module")) + elseif !is_flisp_compat(sc) + throw(LoweringError(st, "new macros should not use `hygienic-scope`")) + end + new_sl = ScopeLayer(st[2].value::Module, sc.layer) + st1 = isnothing(sc0) ? st[1] : remove_context(st[1]) + sc2 = SyntaxContext(new_sl, sc.unexpanded, sc.version, sc.internal) + apply_expansion_layer(ctx, st1, sc2, true, qdepth, sqdepth) + else + done2 = done && !(k in KSet"macrocall inert syntaxinert") + qdepth2 = qdepth + (k === K"quote" ? 1 : k === K"$" ? -1 : 0) + sqdepth2 = sqdepth + (k === K"syntaxquote" ? 1 : k === K"syntaxunquote" ? -1 : 0) + @mknode(st; context=sc, children=mapsyntax(c->apply_expansion_layer( + ctx, c, sc_in, done2, qdepth2, sqdepth2), children(st))) + end + out +end + +""" +Expands macros and quote/interpolation forms. +""" +function expand_forms_1(ctx::MacroExpansionContext, st::SyntaxTree) + k = kind(st) + if is_leaf(st) + st + elseif k === K"macrocall" + expand_macro(ctx, st) + elseif (k === K"do" && numchildren(st) == 2 && kind(st[1]) === K"macrocall" && + kind(st[2]) === K"->") + mac_ex = @ast ctx st [ + K"macrocall" + st[1][1] # mac name + st[1][2] # loc + st[2] # do-lambda + children(st[1])[3:end]... + ] + expand_macro(ctx, mac_ex) + elseif k in KSet"inert syntaxinert toplevel module" + st + elseif k === K"quote" + if numchildren(st) !== 1 + throw(LoweringError(st, "`quote` requires one argument")) + end + expand_forms_1(ctx, expand_quote(ctx, st[1])) + elseif k === K"syntaxquote" + if numchildren(st) !== 1 + throw(LoweringError(st, "`syntaxquote` requires one argument")) + end + expand_forms_1(ctx, expand_syntaxquote(ctx, st[1])) + elseif k === K"escape" || k === K"hygienic-scope" + expand_forms_1( + ctx, apply_expansion_layer( + ctx, st, st.context::SyntaxContext, true, 0, 0)) + else + mapchildren(c->expand_forms_1(ctx, c), st) + end +end + +function assert_expandable(st, l=base_layer(st.context::SyntaxContext)) + @jl_assert st.context isa SyntaxContext (st, "expected syntax context") + @jl_assert base_layer(st.context::SyntaxContext) == l (st, "expected consistent layer") + !is_leaf(st) && for c in children(st) + assert_expandable(c, l) + end +end + +@fzone "JL: macroexpand" function expand_forms_1( + st::SyntaxTree, world::UInt, recursive::Bool) + + DEBUG && assert_expandable(st) + ctx = MacroExpansionContext(st, world, recursive) + st_out = expand_forms_1(ctx, st) + return st_out +end diff --git a/JuliaLowering/src/precompile.jl b/JuliaLowering/src/precompile.jl new file mode 100644 index 0000000000000..2a3fb65bd9614 --- /dev/null +++ b/JuliaLowering/src/precompile.jl @@ -0,0 +1,55 @@ +@static if Base.get_bool_env("JULIA_LOWERING_PRECOMPILE", true) + # Exercise lowering directly so this also works on runtimes where evaluating + # JuliaLowering output through `include_string` is not yet compatible. + thunks = String[ + """ + function foo(xxx, yyy) + @nospecialize xxx + return Pair{Any,Any}(typeof(xxx), typeof(yyy)) + end + """ + + """ + struct Foo + x::Int + Foo(x::Int) = new(x) + # Foo() = new() + end + """ + ] + for thunk in thunks + stream = JuliaSyntax.ParseStream(thunk) + JuliaSyntax.parse!(stream; rule=:all) + st0 = JuliaSyntax.build_tree(SyntaxTree, stream; filename=@__FILE__) + lwrst = lower(@__MODULE__, st0[1]) + lwr = to_lowered_expr(lwrst) + @assert Meta.isexpr(lwr, :thunk) && only(lwr.args) isa Core.CodeInfo + end + + @static if VERSION >= v"1.14.0-DEV.2635" + workload = raw""" + _precompile_kwf(x; y=1, z=2) = x + y + z + + function _precompile_destr(t) + (a, b) = t + a + b + end + + macro _precompile_plus1(ex) + :($(esc(ex)) + 1) + end + _precompile_usemac(x) = @_precompile_plus1(x) + + @generated function _precompile_genf(x) + :(x + 1) + end + + # Fire everything so inference and the generator run during the build. + _precompile_kwf(1; y = 2) + _precompile_destr((1, 2)) + _precompile_usemac(3) + _precompile_genf(1.0) + """ + include_string(@__MODULE__, workload, @__FILE__; expr_compat_mode=true) + end +end diff --git a/JuliaLowering/src/runtime.jl b/JuliaLowering/src/runtime.jl new file mode 100644 index 0000000000000..6ad742dc779bb --- /dev/null +++ b/JuliaLowering/src/runtime.jl @@ -0,0 +1,445 @@ +# Runtime support for +# 1. Functions called by the code emitted from lowering +# 2. Introspecting Julia's state during lowering +# +# These should probably all move to `Core` at some point. + +#------------------------------------------------------------------------------- +# Functions/types used by code emitted from lowering, but not called by it directly + +@inline function _invoke_in_world(w::UInt, f::F, @nospecialize(args...)) where {F} + if ccall(:jl_is_in_pure_context, Int8, ()) != 0 + # Similar to `Base.invoke_in_world` but also works inside generated-function + # expansion (see `jl_code_for_staged`) + return Core._call_in_world_total(w, f, args...) + end + return Base.invoke_in_world(w, f, args...) +end + +# Re-dispatch `f(args...)` at the pinned lowering world (see `jl_lowering_world`) +@inline function invoke_in_lowering_world(f::F, @nospecialize(args...)) where {F} + @static if VERSION >= v"1.14.0-DEV.2635" + w = unsafe_load(cglobal(:jl_lowering_world, Csize_t)) + if w == 0 + # Fallback when the Base lowering hook is not set up + w = Base.tls_world_age() + # FIXME: as a side effect, enabling the Base lowering hook now affects + # JuliaLowering execution not passing through the hook + end + return _invoke_in_world(w, f, args...) + else + f(args...) + end +end + +# Return the current exception. In JuliaLowering we use this rather than the +# special form `K"the_exception"` to reduce the number of special forms. +Base.@assume_effects :removable function current_exception() + @ccall jl_current_exception(current_task()::Any)::Any +end + +function __interpolate_expr(@nospecialize(ex), depth, @nospecialize(vals::Tuple), val_i) + if ex isa QuoteNode + out = __interpolate_expr(Expr(:inert, ex.value), depth, vals, val_i) + QuoteNode(only(out.args)) + elseif !(ex isa Expr) + ex + else + inner_depth = ex.head == :quote ? depth + 1 : + ex.head == :$ ? depth - 1 : depth + cs_out = Any[] + for e in ex.args + if e isa Expr && e.head == :$ && inner_depth == 0 + tup = vals[val_i[] += 1]::Tuple + for v in tup + push!(cs_out, v) + end + else + push!(cs_out, __interpolate_expr(e, inner_depth, vals, val_i)) + end + end + Expr(ex.head, cs_out...) + end +end +function _interpolate_expr(@nospecialize(ex), @nospecialize(values::Tuple)) + @jl_assert !Meta.isexpr(ex, :$) (expr_to_est(ex), "expand_quote should handle this") + __interpolate_expr(ex, 0, values, Ref(0)) +end +function interpolate_expr(@nospecialize(ex), @nospecialize(values...)) + return invoke_in_lowering_world(_interpolate_expr, ex, values) +end + +function __interpolate_syntax(st::SyntaxTree, depth, @nospecialize(vals), val_i) + is_leaf(st) && return st + k = kind(st) + inner_depth = k == K"syntaxquote" ? depth + 1 : + k == K"syntaxunquote" ? depth - 1 : depth + cs_out = SyntaxList() + for c in children(st) + if kind(c) == K"syntaxunquote" && inner_depth == 0 + tup = vals[val_i[] += 1]::Tuple + @jl_assert numchildren(c) == 1 st + @jl_assert kind(c[1]) === K"..." || length(tup) == 1 st + for v in tup + v2 = !(v isa SyntaxTree) ? expr_to_est(v, c) : v + push!(cs_out, v2) + end + else + push!(cs_out, __interpolate_syntax(c, inner_depth, vals, val_i)) + end + end + @mknode(st; children=cs_out) +end +function _interpolate_syntax(st::SyntaxTree, @nospecialize(vals::Tuple)) + # TODO: copy probably not required if immutable + st = mktree(st) + val_i = Ref(0) + out = __interpolate_syntax((@ast _ st [K"None" st]), 0, vals, val_i) + @jl_assert val_i[] == length(vals) st + @jl_assert numchildren(out) == 1 st + out[1] +end +function interpolate_syntax(st::SyntaxTree, @nospecialize(vals...)) + return invoke_in_lowering_world(_interpolate_syntax, st, vals) +end + +#-------------------------------------------------- +# Functions called by closure conversion +function eval_closure_type(mod::Module, closure_type_name::Symbol, + capt_sp, field_names, field_is_box) + type_params = Core.TypeVar[] + field_types = [] + for name in capt_sp + push!(type_params, Core.TypeVar(name)) + end + for (name, isbox) in zip(field_names, field_is_box) + if !isbox + T = Core.TypeVar(Symbol(name, "_type")) + push!(type_params, T) + push!(field_types, T) + else + push!(field_types, Core.Box) + end + end + type = Core._structtype(mod, closure_type_name, + Core.svec(type_params...), + Core.svec(field_names...), + Core.svec(), + false, + length(field_names)) + Core._setsuper!(type, Core.Function) + Core.declare_const(mod, closure_type_name, type) + Core._typebody!(type, Core.svec(field_types...)) + type +end + +# Interpolate captured local variables into the CodeInfo for a global method +function replace_captured_locals(ci_in::Core.CodeInfo, locals::Core.SimpleVector) + ci = copy(ci_in) + for (i, ex) in enumerate(ci.code) + ci.code[i] = _replace_captured_locals(ex, locals) + end + ci +end +function _replace_captured_locals(@nospecialize(e), locals) + if e isa Expr + if e.head === :captured_local + v = locals[e.args[1]::Int] + isa_lowering_ast_node(v) ? QuoteNode(v) : v + else + # could possibly limit to foreigncall + Expr(e.head, map(a->_replace_captured_locals(a, locals), e.args)...) + end + elseif e isa QuoteNode + QuoteNode(_replace_captured_locals(e.value, locals)) + else + e + end +end + +#-------------------------------------------------- +# Functions which create modules or mutate their bindings + +const _Base_has_eval_import = isdefined(Base, :_eval_import) + +function eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + if _Base_has_eval_import + Base._eval_import(imported, to, from, paths...) + else + head = imported ? :import : :using + ex = isnothing(from) ? + Expr(head, paths...) : + Expr(head, Expr(Symbol(":"), from, paths...)) + Core.eval(to, ex) + end +end + +function eval_using(to::Module, path::Expr) + if _Base_has_eval_import + Base._eval_using(to, path) + else + Core.eval(to, Expr(:using, path)) + end +end + +function eval_public(mod::Module, is_exported::Bool, identifiers) + # symbol jl_module_public is no longer exported as of #57765 + Core.eval(mod, Expr((is_exported ? :export : :public), map(Symbol, identifiers)...)) +end + +#-------------------------------------------------- +# Docsystem integration +function _bind_func_docs!(f, docstr, method_metadata::Core.SimpleVector) + mod = parentmodule(f) + bind = Base.Docs.Binding(mod, nameof(f)) + full_sig = method_metadata[1] + arg_sig = Tuple{full_sig[2:end]...} + lineno = method_metadata[3] + metadata = Dict{Symbol, Any}( + :linenumber => lineno.line, + :module => mod, + ) + if !isnothing(lineno.file) + push!(metadata, :path => string(lineno.file)) + end + Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), arg_sig) +end + +function bind_docs!(f::Function, docstr, method_metadata::Core.SimpleVector) + _bind_func_docs!(f, docstr, method_metadata) +end + +# Document constructors +function bind_docs!(::Type{Type{T}}, docstr, method_metadata::Core.SimpleVector) where T + _bind_func_docs!(T, docstr, method_metadata) +end + +function bind_docs!(type::Type, docstr, method_metadata::Core.SimpleVector) + _bind_func_docs!(type, docstr, method_metadata) +end + +function bind_docs!(type::Type, docstr, lineno::LineNumberNode; field_docs=Core.svec()) + mod = parentmodule(type) + bind = Base.Docs.Binding(mod, nameof(type)) + metadata = Dict{Symbol, Any}( + :linenumber => lineno, + :module => mod, + ) + if !isnothing(lineno.file) + push!(metadata, :path => string(lineno.file)) + end + if !isempty(field_docs) + fd = Dict{Symbol, Any}() + fns = fieldnames(type) + for i = 1:2:length(field_docs) + fd[fns[field_docs[i]]] = field_docs[i+1] + end + metadata[:fields] = fd + end + Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), Union{}) +end + +""" +Called in the unfortunate cases (K"call", K".", K"Identifier") where docstrings +change the semantics of the expressions they annotate, no longer requiring the +expression to execute. +""" +function bind_static_docs!(mod::Module, name::Symbol, docstr, lnn::LineNumberNode, sigtypes::Type) + metadata = Dict{Symbol, Any}( + :linenumber => lnn.line, + :module => mod, + :path => something(lnn.file, "none"), + ) + bind = Base.Docs.Binding(mod, name) + Docs.doc!(mod, bind, Base.Docs.docstr(docstr, metadata), sigtypes) +end + +#-------------------------------------------------- +# Runtime support infrastructure for `@generated` + +# An alternative to Core.GeneratedFunctionStub which works on SyntaxTree rather +# than Expr. +struct GeneratedFunctionStub + syntax_context::SyntaxContext + gen::Function + srcref::Union{LineNumberNode,SourceRef} + argnames::Core.SimpleVector + spnames::Core.SimpleVector +end + +function _gen_args_from_syms(ctx, src, args, sc) + out = SyntaxList() + for a in args + id = newleaf(src, K"Identifier", string(a)) + id = est_to_dst_ident(SyntaxCompatContext(), id) # support placeholders + id = @mknode(id; context=sc) + push!(out, id) + end + out +end + +# Call the `@generated` code generator function and wrap the results of the +# expression into a CodeInfo. +# +# `args` passed into stub by the Julia runtime are (parent_func, static_params..., arg_types...) +function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize args...) + # Some of the lowering pipeline from lower() and the pass-specific setup is + # re-implemented here because generated functions are very much (but not + # entirely) like macro expansion. + # + # TODO: Reduce duplication where possible. + + __module__ = source.module + sc = g.syntax_context + + # Run code generator - this acts like a macro expander + ex0 = g.gen(sc, args...) + + # Note that we expand in `tls_world_age()` (see Core.GeneratedFunctionStub) + world = Base.tls_world_age() + + # Lower the generated code in the lowering world + return invoke_in_lowering_world(_lower_generated_code, g, source, sc, + __module__, world, ex0) +end + +function _lower_generated_code(g::GeneratedFunctionStub, source::Method, + sc::SyntaxContext, __module__::Module, + world::UInt, @nospecialize(ex0)) + if ex0 isa Expr + ex0 = expr_to_est(ex0, source_location(LineNumberNode, g.srcref)) + end + # TODO: rebase mistake above? + if !(ex0 isa SyntaxTree) + ex0 isa Expr && throw(LoweringError( + ex0, "implicit expr->syntaxtree: may later be allowed, but is probably a mistake today")) + ex0 = expr_to_est(ex0, g.srcref) + end + + @jl_assert base_layer(sc).mod == __module__ ex0 + ex0 = JuliaSyntax.fill_context(ex0, sc) + ctx1 = MacroExpansionContext(ex0, world, true) + ex1 = expand_forms_1(ctx1, ex0) + # Desugaring + ctx2, ex2 = expand_forms_2(ex1, world) + + # Wrap expansion in a non-toplevel lambda and run scope resolution + ex2 = @ast ctx2 ex0 [K"generated_lambda" + [K"block" _gen_args_from_syms(ctx2, ex1, g.argnames, sc)...] + [K"block" _gen_args_from_syms(ctx2, ex1, g.spnames, sc)...] + ex2 + ] + ctx3, ex3 = resolve_scopes(ctx2, ex2) + + # Rest of lowering + ctx4, ex4 = convert_closures(ctx3, ex3) + _ctx5, ex5 = linearize_ir(ctx4, ex4) + ci = to_lowered_expr(ex5) + @assert ci isa Core.CodeInfo + + # See GeneratedFunctionStub code in base/expr.jl + ci.isva = source.isva + code = ci.code + bindings = IdSet{Core.Binding}() + for i = 1:length(code) + stmt = code[i] + if isa(stmt, GlobalRef) + push!(bindings, convert(Core.Binding, stmt)) + end + end + if !isempty(bindings) + ci.edges = Core.svec(bindings...) + end + + return ci +end + + +#------------------------------------------------------------------------------- +# The following functions are called directly by lowering to inspect Julia's state. + +# Get the binding for `name` if one is already resolved in module `mod`. Note +# that we cannot use `isdefined(::Module, ::Symbol)` here, because that causes +# binding resolution which is a massive side effect we must avoid in lowering. +function _get_module_binding(mod::Module, name::Symbol; create::Bool=false) + b = @ccall jl_get_module_binding(mod::Module, name::Symbol, create::Cint)::Ptr{Core.Binding} + b == C_NULL ? nothing : unsafe_pointer_to_objref(b) +end + +_module_binding_i_taken(mod::Module, basename::AbstractString, i::Int) = + _get_module_binding(mod, Symbol(basename, "##", i); create=false) !== nothing + +# Reserve a global binding named "$basename##$i" in module `mod` for some +# free `i`. We could scan 0:n here, but we instead try exponentially +# increasing `i` and backwards binary search to achieve `log(n)` performance +# +# TODO: Remove the use of this where possible. Currently this is used within +# lowering to create unique global names for keyword function bodies and +# closure types as a more local alternative to current-julia-module-counter. +# However, we should ideally defer it to eval-time to make lowering itself +# completely non-mutating. +function reserve_module_binding_i(mod::Module, basename::AbstractString) + hi = 0 + while _module_binding_i_taken(mod, basename, hi) + hi = 2hi + 1 + end + lo = hi == 0 ? 0 : (hi - 1) ÷ 2 + 1 + while lo < hi + mid = (lo + hi) ÷ 2 + if _module_binding_i_taken(mod, basename, mid) + lo = mid + 1 + else + hi = mid + end + end + name = "$basename##$hi" + # TODO: Fix the race condition here: We should really hold the Module's + # binding lock during this test-and-set type operation. But the binding + # lock is only accessible from C. See also the C code in + # `fl_module_unique_name`. + _get_module_binding(mod, Symbol(name); create=true) + return name +end + +# Even less likely to be deterministic than the above, but necessary to avoid +# quadratic behaviour where flisp doesn't already have it. See +# `fl_module_unique_name` +function module_unique_name(mod::Module) + @static if VERSION < v"1.14.0-DEV.3063" + # (JETLS) jl_module_next_counter is not exported before 1.14.0-DEV.3063 + reserve_module_binding_i(mod, "")[3:end] + else + string(@ccall(jl_module_next_counter(mod::Module)::UInt32)) + end +end +function module_unique_name(mod::Module, funcname::AbstractString) + occursin('#', funcname) ? module_unique_name(mod) : + reserve_module_binding_i(mod, funcname) +end + +# Return true if a `name` is defined in and *by* the module `mod`. +# Has no side effects, unlike isdefined() +# +# (This should do what fl_defined_julia_global does for flisp lowering) +function is_defined_and_owned_global(mod, name, world::UInt=Base.get_world_counter()) + return _invoke_in_world(world, Base.binding_kind, mod, name) === Base.PARTITION_KIND_GLOBAL +end + +function lookup_method_instance(func, args, world::Integer) + allargs = Vector{Any}(undef, length(args) + 1) + allargs[1] = func + allargs[2:end] = args + mi = @ccall jl_method_lookup(allargs::Ptr{Any}, length(allargs)::Csize_t, + world::Csize_t)::Ptr{Cvoid} + return mi == C_NULL ? nothing : unsafe_pointer_to_objref(mi) +end + +# Like `Base.methods()` but with world age support +function methods_in_world(func, arg_sig, world, err_ex) + out = Base._methods(func, arg_sig, -1, world) + @jl_assert(out isa Vector{Any}, + (err_ex, string( + "Base._methods returned non-vector;", + " bad world age provided? (", world, ")"))) + out::Vector{Any} +end diff --git a/JuliaLowering/src/scope_analysis.jl b/JuliaLowering/src/scope_analysis.jl new file mode 100644 index 0000000000000..6139f017111f2 --- /dev/null +++ b/JuliaLowering/src/scope_analysis.jl @@ -0,0 +1,954 @@ +# Lowering pass 3: scope and variable analysis +struct ScopeInfo + # index into ctx.scopes + id::ScopeId + # 0 if top-level thunk + parent_id::ScopeId + # Own ID if lambda, else some parent ID + lambda_id::ScopeId + # Tree introducing this scope + node_id::SyntaxTree + # True in the top-level scope, and any neutral scope nested within it not + # protected by a hard scope. Becomes soft if `ctx.enable_soft_scopes`. + is_permeable::Bool + # True for K"method_defs" and its non-lambda children where all new locals + # should participate in standard scope resolution, but then be associated + # with the top-level thunk by the end of this pass. + is_lifted::Bool + binding_assignments::Dict{IdTag, SyntaxTree} + assignments::Dict{NameKey, SyntaxTree} + # Map from variable names to binding IDs for resolution. Includes all + # locals, args, sparams, and explicit globals belonging to this scope. + # Variables captured from an outer scope are not included. The top-level + # scope also contains all globals for resolution to fall back to. + vars::Dict{NameKey,IdTag} + # See `LambdaBindings`. Nothing if not a lambda scope. This is the final + # collecting place for locals going in to closure conversion. + locals_capt::Union{Nothing, Dict{IdTag,Bool}} +end + +function ScopeInfo(ctx, parent_id, ex::SyntaxTree) + id = length(ctx.scopes) + 1 + k = kind(ex) + if parent_id == 0 + @jl_assert k in KSet"lambda toplevel_lambda generated_lambda" ex + lambda_id = id + is_permeable = k == K"toplevel_lambda" + is_lifted = false + lambda_bindings = Dict{IdTag, Bool}() + else + @jl_assert k in KSet"lambda method_defs scope_block" ex + parent = ctx.scopes[parent_id] + lambda_id = k === K"lambda" ? id : parent.lambda_id + is_permeable = (k === K"scope_block" && + kind(ex[1]) === K"neutral_scope" && + parent_id != 0 && parent.is_permeable) + is_lifted = k === K"method_defs" || + (k !== K"lambda" && parent.is_lifted) + lambda_bindings = k === K"lambda" ? Dict{IdTag, Bool}() : nothing + end + s = ScopeInfo( + id, parent_id, lambda_id, ex, is_permeable, is_lifted, + Dict{IdTag, SyntaxTree}(), Dict{NameKey, SyntaxTree}(), + Dict{NameKey,IdTag}(), lambda_bindings) + push!(ctx.scopes, s) + return s +end + +mutable struct ScopeResolutionContext <: AbstractLoweringContext + const layer::ScopeLayer + const bindings::Bindings + # Purely for display and deterministic ordering of scope layers + const layer_ids::Dict{ScopeLayer, Int} + # Every lexical scope, indexed by ScopeId + const scopes::Vector{ScopeInfo} + # Current stack of scopes to look for names in, innermost scope last + const scope_stack::Vector{ScopeId} + # Usually, globals in the top scope are ignored. This is a subset that may + # be assigned to without the `global` keyword in soft scopes due to being + # assigned to at top level, or passing the defined-and-owned-global check. + const soft_assignable_globals::Set{NameKey} + # Every static parameter corresponds to some typevar (top-level local) + # required to create this method + const sp_typevars::Dict{IdTag, IdTag} + # Typevars referenced in each typevar's bounds. Closures capturing a static + # parameter must also capture the sparams of its typevar's dependencies + const tv_deps::Dict{IdTag, Vector{IdTag}} + const enable_soft_scopes::Bool + const world::UInt +end + +function contains_softscope_marker(ex) + kind(ex) == K"softscope" && return true + needs_resolution(ex) && for c in children(ex) + contains_softscope_marker(c) && return true + end + return false +end + +top_scope(ctx) = ctx.scopes[1] +is_top_scope(scope::ScopeInfo) = scope.parent_id === 0 +enclosing_lambda(ctx, scope::ScopeInfo) = ctx.scopes[scope.lambda_id] +parent(ctx, scope::ScopeInfo) = is_top_scope(scope) ? nothing : + ctx.scopes[scope.parent_id] + +_var_str(v) = v === :local ? "local variable" : + v === :global ? "global variable" : + v === :argument ? "argument" : + v === :destructured_arg ? "destructured argument" : + v === :typevar ? "typevar" : + v === :static_parameter ? "static parameter" : "unknown" + +# Declare `ex` in `scope`, unless a binding already exists with the same name in +# scope, or anywhere. Throw an error if a name conflict occurs. The rules +# for conflict: declaring a local (or global) twice with the same name is a +# no-op, but doing so with an argument or static parameter is an error. A +# variable usually can't be two things in one scope, but flisp has quirks. +function explicit_declare_in_scope!(ctx, scope::ScopeInfo, ex, new_k::Symbol) + if kind(ex) === K"BindingId" + bid = syntax_id(ex) + b = get_binding(ctx, bid) + @jl_assert b.kind === new_k ex + @jl_assert b.lambda_id == 0 (ex, "cannot declare a BindingId in multiple scopes") + add_lambda_local!(ctx, scope, b) + return bid + elseif kind(ex) === K"Placeholder" + return nothing + end + bid = get(scope.vars, NameKey(ex), nothing) + old_k = isnothing(bid) ? nothing : get_binding(ctx, bid).kind + if isnothing(old_k) + if new_k === :argument + declare_in_scope!(ctx, scope, ex, :argument; + is_nospecialize=getmeta(ex, :nospecialize, false)) + else + real_k = new_k === :destructured_arg ? :local : new_k + declare_in_scope!(ctx, scope, ex, real_k) + end + elseif old_k === new_k + (new_k === :global || new_k === :local) && return bid + throw(LoweringError(ex, "function $(_var_str(new_k)) name not unique")) + # See note in test/scopes.jl: "globals may overlap args or sparams" + # elseif new_k === :global && old_k in (:argument, :static_parameter) + # declare_in_scope!(ctx, scope, ex, :global) + else + throw(LoweringError(ex, """ + $(_var_str(new_k)) name `$(NameKey(ex).name)` conflicts with an \ + existing $(_var_str(old_k)) from the same scope""")) + end +end + +# globals are added to both `scope` and the top scope (mainly so we can get the +# same binding for many unrelated global references). +function declare_in_scope!(ctx, scope::ScopeInfo, ex, bk::Symbol; + is_nospecialize::Bool=false, + is_ambiguous_local::Bool=false) + nk = NameKey(ex) + if bk === :global + mod = syntax_module(ex) + declaration_scope = top_scope(ctx) + else + declaration_scope = scope + mod = ex.mod isa Module ? + throw(LoweringError(ex, "cannot use GlobalRef as local identifier")) : nothing + end + is_internal = (ex.context::SyntaxContext).internal || + getmeta(ex, :is_internal, false)::Bool + b = _new_binding(ctx.bindings, ex, nk.name, bk; + mod, is_internal, is_nospecialize, is_ambiguous_local) + declaration_scope.vars[nk] = b.id + scope.vars[nk] = b.id + add_lambda_local!(ctx, scope, b) + return b.id +end + +function add_lambda_local!(ctx, scope::ScopeInfo, b) + if b.kind === :global || b.is_ssa + return + end + lam = scope.is_lifted ? top_scope(ctx) : enclosing_lambda(ctx, scope) + b.kind == :typevar && @jl_assert scope.is_lifted binding_ex(ctx, b) + @jl_assert !haskey(lam.locals_capt, b.id) ( + binding_ex(ctx, b), "adding lambda local twice") + lam.locals_capt[b.id] = false + b.lambda_id = lam.id + nothing +end + +function ensure_captured!(ctx, scope::ScopeInfo, b) + if b.kind === :global || b.kind === :typevar || b.is_ssa + return + end + lam = enclosing_lambda(ctx, scope) + if !haskey(lam.locals_capt, b.id) + # assert is opaque closure, or b not static_parameter + b.is_captured = true + lam.locals_capt[b.id] = true + s2 = parent(ctx, lam) + @jl_assert !isnothing(s2) ( + binding_ex(ctx, b), + "tried to capture local before declaration in any parent") + ensure_captured!(ctx, s2, b) + end + nothing +end + +function needs_resolution(ex) + kind(ex) === K"Identifier" || + !is_leaf(ex) && !is_quoted(ex) && !(kind(ex) in KSet"toplevel module") +end + +function resolve_name(ctx, ex; exclude_toplevel_globals=false) + # TODO: probably want to cache these lookups + nk = NameKey(ex) + for sid in Iterators.reverse(ctx.scope_stack) + bid = get(ctx.scopes[sid].vars, nk, nothing) + isnothing(bid) && continue + b = get_binding(ctx, bid) + if b.kind === :typevar + # only visible to lifted scopes in the same lambda (we should only + # hit this when we filter sparams with `used_typevars`) + s0 = ctx.scopes[ctx.scope_stack[end]] + s0.is_lifted && ctx.scopes[sid].lambda_id == s0.lambda_id || continue + end + if !exclude_toplevel_globals || sid !== top_scope(ctx).id || b.kind !== :global + return b + end + end + return nothing +end + +# Collect typevar bindings referenced in `ex` (a resolved typevar bound) +function _typevar_refs!(out, ctx, ex) + k = kind(ex) + if k == K"BindingId" + b = get_binding(ctx, ex) + b.kind === :typevar && !(b.id in out) && push!(out, b.id) + elseif !is_leaf(ex) && needs_resolution(ex) + foreach(e->_typevar_refs!(out, ctx, e), children(ex)) + end +end + +function _record_layer!(ctx, ex) + ex.context isa SyntaxContext || return + sl = ex.context.layer + get!(ctx.layer_ids, sl, length(ctx.layer_ids)+1) +end + +function _find_scope_decls!(ctx, scope, ex) + k = kind(ex) + _record_layer!(ctx, ex) + if k === K"local" && kind(ex[1]) === K"Identifier" + var_k = getmeta(ex, :is_destructured_arg, false) ? + :destructured_arg : :local + explicit_declare_in_scope!(ctx, scope, ex[1], var_k) + elseif k === K"global" && kind(ex[1]) === K"Identifier" + explicit_declare_in_scope!(ctx, scope, ex[1], :global) + elseif k === K"function_decl" + k1 = kind(ex[1]) + _record_layer!(ctx, ex[1]) + if k1 === K"BindingId" + b = get_binding(ctx, ex[1]) + @jl_assert b.is_ssa || b.kind === :global ( + ex, "allow local BindingId as function name?") + get!(scope.binding_assignments, b.id, ex[1]) + elseif k1 === K"Identifier" + ex[1].mod isa Module && + explicit_declare_in_scope!(ctx, scope, ex[1], :global) + get!(scope.assignments, NameKey(ex[1]), ex[1]) + get!(ctx.layer_ids, (ex[1].context::SyntaxContext).layer, + length(ctx.layer_ids)+1) + else + @jl_assert false (ex, "unknown kind in assignment") + end + elseif k in KSet"= constdecl assign_or_constdecl_if_global" + k1 = kind(ex[1]) + _record_layer!(ctx, ex[1]) + sc = ex[1].context::SyntaxContext + if k1 === K"BindingId" + b = get_binding(ctx, ex[1]) + get!(scope.binding_assignments, b.id, ex[1]) + elseif k1 === K"Identifier" + ex[1].mod === nothing && + get!(scope.assignments, NameKey(ex[1]), ex[1]) + elseif k1 === K"Placeholder" + # nothing to declare + else + @jl_assert false (ex, "unknown kind in assignment") + end + if !(k == K"constdecl" && numchildren(ex) == 1) + _find_scope_decls!(ctx, scope, ex[2]) + end + elseif needs_resolution(ex) && !(k in KSet"scope_block lambda method_defs") + for e in children(ex) + _find_scope_decls!(ctx, scope, e) + end + end + nothing +end + +# Produce a complete ScopeInfo and add it to the stack of active scopes. This +# means finding all variables declared and used in the scope `ex` and generating +# the (identifier,layer)=>binding_id mapping `scope.vars` +function enter_scope!(ctx, ex) + @jl_assert kind(ex) in KSet"lambda scope_block method_defs toplevel_lambda generated_lambda" ex + # Note that generated functions produce lambdas with this false + is_toplevel_thunk = kind(ex) === K"toplevel_lambda" + parent_id = (is_toplevel_thunk || isempty(ctx.scope_stack)) ? + 0 : ctx.scopes[ctx.scope_stack[end]].id + scope = ScopeInfo(ctx, parent_id, ex) + + #--------------------------------------------------------------------------- + # Find explicit decls that may influence assignment assignment resolution + if kind(ex) in KSet"lambda toplevel_lambda generated_lambda" + for c in children(ex[1]) + @jl_assert kind(c) in KSet"Identifier BindingId Placeholder" c + explicit_declare_in_scope!(ctx, scope, c, :argument) + end + for c in children(ex[2]) + kind(c) === K"Placeholder" && continue + @jl_assert kind(c) === K"Identifier" c + sp_id = explicit_declare_in_scope!(ctx, scope, c, :static_parameter) + p = parent(ctx, scope) + if !isnothing(p) # usually true, false for generated functions + ctx.sp_typevars[sp_id] = p.vars[NameKey(c)] + end + end + for c in children(ex)[3:end] + _find_scope_decls!(ctx, scope, c) + end + else + for c in children(ex) + _find_scope_decls!(ctx, scope, c) + end + end + push!(ctx.scope_stack, scope.id) # influences resolution below + + #--------------------------------------------------------------------------- + # Find assignment targets, possibly introducing implicit locals and globals + for (bid, _node_id) in sort!(collect(scope.binding_assignments); by=first) + # Mutable nameless bindings may be introduced in desugaring. These + # should be capturable, and may be local to the nearest lambda or + # global. Desugaring should ensure these are never used undef. + b = get_binding(ctx, bid) + b.lambda_id != 0 || add_lambda_local!(ctx, scope, b) + end + for (vk, node_id) in sort!(collect(scope.assignments); + by=x->let nk=x[1]; (nk.name, ctx.layer_ids[nk.layer]); end) + local ex = node_id + b = resolve_name(ctx, ex) + if b === nothing + sc = ex.context::SyntaxContext + # Top-level assignments are locals in hygienic expansions. We may + # need to adjust this, as flisp makes them name-mangled globals. + hygienic_toplevel = !is_base_layer(sc) && sc.layer !== ctx.layer + if is_toplevel_thunk && !hygienic_toplevel + # top-level assignments in no scope and no expansion + push!(ctx.soft_assignable_globals, vk) + declare_in_scope!(ctx, top_scope(ctx), ex, :global) + elseif scope.is_permeable && !hygienic_toplevel && + is_defined_and_owned_global( + syntax_module(sc), Symbol(vk.name), ctx.world) + # special soft scope rules: existing global variables are assigned to + if ctx.enable_soft_scopes + push!(ctx.soft_assignable_globals, vk) + declare_in_scope!(ctx, top_scope(ctx), ex, :global) + else + declare_in_scope!(ctx, scope, ex, :local; is_ambiguous_local=true) + end + else + declare_in_scope!(ctx, scope, ex, :local) + end + elseif b.kind === :global + if is_toplevel_thunk + # assign-existing and make visible to soft scope + push!(ctx.soft_assignable_globals, vk) + elseif !isnothing(resolve_name(ctx, ex; exclude_toplevel_globals=true)) || + (ctx.enable_soft_scopes && scope.is_permeable && + vk in ctx.soft_assignable_globals) + # assign-existing-global if this is an explicit global that + # isn't at top level, or if the soft scope exception applies + else + declare_in_scope!(ctx, scope, ex, :local; is_ambiguous_local = scope.is_permeable) + end + elseif b.kind === :static_parameter + throw(LoweringError(ex, "cannot overwrite a static parameter")) + elseif b.kind === :local || b.kind === :argument + # unambiguous assignment to existing variable + end + end + + return scope +end + +function add_local_decls!(ctx, stmts, srcref, scope) + # Add local decls to start of block so that closure conversion can + # initialize if necessary. + for id in sort!(collect(values(scope.vars))) + binfo = get_binding(ctx, id) + if binfo.kind == :local + push!(stmts, @ast ctx srcref [K"local" binding_ex(ctx, id)]) + end + end +end + +function _resolve_scopes(ctx::ScopeResolutionContext, ex::SyntaxTree, + @nospecialize(scope::Union{Nothing, ScopeInfo})) + k = kind(ex) + @jl_assert scope isa ScopeInfo || k === K"lambda" || + k === K"toplevel_lambda" || k === K"generated_lambda" ex + if k == K"Identifier" + if (mod = ex.mod; !isnothing(mod)) + return new_global_binding(ctx, ex, syntax_name(ex), mod) + end + b = resolve_name(ctx, ex) + # Unresolved names are assumed global + if isnothing(b) + gid = declare_in_scope!(ctx, top_scope(ctx), ex, :global) + b = get_binding(ctx, gid) + end + # Body-level @nospecialize sets :nospecialize metadata on identifiers. + # Propagate this to the binding so the slot gets the nospecialize flag. + if getmeta(ex, :nospecialize, false) && b.kind === :argument + b.is_nospecialize = true + end + if getmeta(ex, :is_called, false) + b.is_called = true + end + newleaf(ex, K"BindingId", b.id) + elseif k === K"BindingId" + ex + elseif k == K"softscope" + newleaf(ex, K"TOMBSTONE") + elseif !needs_resolution(ex) + ex + elseif k == K"local" + # Local declarations have a value of `nothing` according to flisp + # lowering. + # TODO: Should local decls be disallowed in value position? + @ast ctx ex (::K"nothing") + elseif k == K"decl" + ex_out = mapchildren(e->_resolve_scopes(ctx, e, scope), ex) + name = ex_out[1] + if kind(name) != K"Placeholder" + binfo = get_binding(ctx, name) + if binfo.kind == :global && !is_top_scope(enclosing_lambda(ctx, scope)) + throw(LoweringError(ex, "type declarations for global variables must be at top level, not inside a function")) + end + end + id = ex_out[1] + if kind(id) != K"Placeholder" + binfo = get_binding(ctx, id) + if !isnothing(binfo.type) && binfo.kind !== :global + throw(LoweringError(ex, "multiple type declarations found for `$(binfo.name)`")) + end + binfo.type = ex_out[2] + end + ex_out + elseif k == K"always_defined" + resolve_name(ctx, ex[1]).is_always_defined = true + newleaf(ex, K"TOMBSTONE") + elseif k in KSet"lambda toplevel_lambda generated_lambda" + # opaque closures are the exception + # scope isa ScopeInfo && @jl_assert scope.is_lifted ex + newscope = enter_scope!(ctx, ex) + arg_bindings = _resolve_scopes(ctx, ex[1], newscope) + sparam_bindings = SyntaxList() + for sp in children(ex[2]) + kind(sp) === K"Placeholder" && continue + push!(sparam_bindings, _resolve_scopes(ctx, sp, newscope)) + end + self_id = if numchildren(arg_bindings) === 0 + 0 + elseif getmeta(ex[1][1], :is_kwcall_self, false) + syntax_id(arg_bindings[3]) + else + syntax_id(arg_bindings[1]) + end + body_stmts = SyntaxList() + add_local_decls!(ctx, body_stmts, ex, newscope) + body = _resolve_scopes(ctx, ex[3], newscope) + if kind(body) == K"block" + append!(body_stmts, children(body)) + else + push!(body_stmts, body) + end + ret_var = numchildren(ex) == 4 ? + _resolve_scopes(ctx, ex[4], newscope) : nothing + pop!(ctx.scope_stack) + @ast ctx ex [k + LambdaBindings(self_id, newscope.id, newscope.locals_capt)::K"LambdaBindings" + arg_bindings + [K"block" sparam_bindings...] + [K"block" body_stmts...] + ret_var + ] + elseif k == K"scope_block" + newscope = enter_scope!(ctx, ex) + stmts = SyntaxList() + add_local_decls!(ctx, stmts, ex, newscope) + for e in children(ex)[2:end] + push!(stmts, _resolve_scopes(ctx, e, newscope)) + end + pop!(ctx.scope_stack) + @ast ctx ex [K"block" stmts...] + elseif k == K"method_defs" + newscope = enter_scope!(ctx, ex) + mname = _resolve_scopes(ctx, ex[1], scope) + tvs = SyntaxList() + for tv in children(ex[2]) # hack. flisp: replace-vars + rhs = _resolve_scopes(ctx, tv[2], newscope) + if kind(tv[1]) === K"Placeholder" + @ast ctx tv [K"=" tv[1] rhs] + else + bid = declare_in_scope!(ctx, newscope, tv[1], :typevar) + get_binding(ctx, bid).is_always_defined = true + deps = Vector{IdTag}() + _typevar_refs!(deps, ctx, rhs) + isempty(deps) || (ctx.tv_deps[bid] = deps) + push!(tvs, @ast ctx tv [K"=" binding_ex(ctx, bid) rhs]) + end + end + stmts = SyntaxList() + add_local_decls!(ctx, stmts, ex, newscope) + push!(stmts, _resolve_scopes(ctx, ex[3], newscope)) + pop!(ctx.scope_stack) + @ast ctx ex [K"method_defs" mname [K"block" tvs...] [K"block" stmts...]] + elseif k == K"islocal" + e1 = ex[1] + islocal = kind(e1) == K"Identifier" && + let b = resolve_name(ctx, e1) + !isnothing(b) && b.kind !== :global + end + @ast ctx ex islocal::K"Bool" + elseif k == K"isglobal" + e1 = ex[1] + isglobal = kind(e1) == K"Identifier" && + let b = resolve_name(ctx, e1) + isnothing(b) || b.kind === :global + end + @ast ctx ex isglobal::K"Bool" + elseif k == K"locals" + stmts = SyntaxList() + locals_dict = ssavar(ctx, ex, "locals_dict") + push!(stmts, @ast ctx ex [K"=" + locals_dict + [K"call" + [K"call" + "apply_type"::K"core" + "Dict"::K"top" + "Symbol"::K"core" + "Any"::K"core" + ] + ] + ]) + for sid in ctx.scope_stack + for id in sort!(collect(values(ctx.scopes[sid].vars))) + binfo = get_binding(ctx, id) + if binfo.kind == :global || binfo.is_internal || + binfo.kind == :typevar + continue + end + binding = binding_ex(ctx, id) + push!(stmts, @ast ctx ex [K"if" + [K"isdefined" binding] + [K"call" + "setindex!"::K"top" + locals_dict + binding + binfo.name::K"Symbol" + ] + ]) + end + end + push!(stmts, locals_dict) + newnode(ex, K"block", stmts) + elseif k == K"thisfunction" + lam = enclosing_lambda(ctx, scope::ScopeInfo).node_id + self_arg = lam[1][1] + for a in children(lam[1]) + getmeta(a, :thisfunction_original, false) && (self_arg = a) + end + return _resolve_scopes(ctx, self_arg, scope) + elseif k == K"assert" + etype = extension_type(ex) + if etype == "require_existing_locals" + for v in ex[2:end] + b = resolve_name(ctx, v) + if isnothing(b) || !(b.kind in (:local, :argument)) + throw(LoweringError(v, "`outer` annotations must match with a local variable in an outer scope but no such variable was found")) + end + end + elseif etype == "global_toplevel_only" + if !is_top_scope(scope) + e = ex[2][1] + throw(LoweringError(e, "$(kind(e)) is only allowed in global scope")) + end + elseif etype == "toplevel_only" + if !is_top_scope(enclosing_lambda(ctx, scope)) + e = ex[2][1] + throw(LoweringError(e, "this syntax is only allowed in top level code")) + end + else + @jl_assert false (ex, "unknown syntax assertion") + end + newleaf(ex, K"TOMBSTONE") + elseif k === K"relayered_global" + bid = get(scope.vars, NameKey(ex[1]), nothing) + !isnothing(bid) && let b = get_binding(ctx, bid) + b.kind !== :global && throw(LoweringError(ex, string( + "unhygienic global name `$(NameKey(ex[1]).name)` conflicts ", + "with an existing $(_var_str(b.kind))"))) + end + newleaf(ex, K"TOMBSTONE") + elseif k == K"function_decl" + resolved = mapchildren(e->_resolve_scopes(ctx, e, scope), ex) + name = resolved[1] + if kind(name) == K"BindingId" + bk = get_binding(ctx, name).kind + if bk == :argument + throw(LoweringError(name, "Cannot add method to a function argument")) + elseif bk == :global && !is_top_scope(enclosing_lambda(ctx, scope)) + throw(LoweringError(name, """ + Global method definition needs to be placed at the top \ + level, or use `eval()`""")) + end + end + resolved + elseif k == K"constdecl" + if !is_top_scope(enclosing_lambda(ctx, scope)) + throw(LoweringError(ex, "unsupported `const` inside function")) + end + resolved = mapchildren(e->_resolve_scopes(ctx, e, scope), ex) + if kind(resolved[1]) !== K"Placeholder" + @jl_assert kind(resolved[1]) === K"BindingId" resolved + if get_binding(ctx, syntax_id(resolved[1])).kind === :local + throw(LoweringError(ex, "unsupported `const` declaration on local variable")) + end + end + resolved + elseif k == K"assign_or_constdecl_if_global" + @jl_assert numchildren(ex) === 2 ex + id = _resolve_scopes(ctx, ex[1], scope) + assignment_kind = + kind(id) === K"Placeholder" || + (get_binding(ctx, id).kind !== :global) ? K"=" : K"constdecl" + @ast ctx ex _resolve_scopes(ctx, [assignment_kind ex[1] ex[2]], scope) + elseif k === K"global_if_global" + out = _resolve_scopes(ctx, ex[1], scope) + get_binding(ctx, out).kind !== :global ? (@ast ctx ex (::K"TOMBSTONE")) : + @ast ctx ex [K"global" out] + else + mapchildren(e->_resolve_scopes(ctx, e, scope), ex) + end +end + +function _resolve_scopes(ctx::ScopeResolutionContext, exs::AbstractVector, scope) + out = SyntaxList() + for e in exs + push!(out, _resolve_scopes(ctx, e, scope)) + end + out +end + +#------------------------------------------------------------------------------- +# Sub-pass to compute additional information about variable usage as required +# by closure conversion, etc +struct ClosureBindings + name_stack::Vector{String} # Names of functions the closure is nested within + lambdas::Vector{LambdaBindings} # Bindings for each method of the closure + capt_sp::Set{IdTag} +end + +# `binding` is that in `function_decl`, `method_defs[1]`, `method[1]`, +# `function_type[1]` when local +struct ClosureKey + binding::IdTag + lam::ScopeId +end + +ClosureBindings(name_stack) = + ClosureBindings(name_stack, Vector{LambdaBindings}(), Set{IdTag}()) + +mutable struct VariableAnalysisContext <: AbstractLoweringContext + const layer::ScopeLayer + const bindings::Bindings + const scopes::Vector{ScopeInfo} + const lambda_bindings::LambdaBindings + const lifted::Bool + # Stack of method definitions for closure naming + const method_def_stack::Vector{SyntaxTree} + const closure_key_stack::Vector{ClosureKey} + # Collection of information about each closure, principally which methods + # are part of the closure (and hence captures). + const closure_bindings::Dict{ClosureKey,ClosureBindings} + const sp_typevars::Dict{IdTag, IdTag} + const tv_deps::Dict{IdTag, Vector{IdTag}} + # Prevents infinite loops when analyzing a binding's type + const types_in_analysis::Set{IdTag} +end + +function init_closure_bindings!(ctx, fname) + bid = syntax_id(fname) + ck = closure_key(ctx, fname) + @jl_assert get_binding(ctx, bid).kind === :local fname + get!(ctx.closure_bindings, ck) do + name_stack = Vector{String}() + for parentname in ctx.method_def_stack + if kind(parentname) == K"BindingId" + push!(name_stack, get_binding(ctx, parentname).name) + end + end + push!(name_stack, get_binding(ctx, bid).name) + ClosureBindings(name_stack) + end +end + +# sparams, globals, and top-level locals interpolated into global methods are OK +# (the last may or may not work intentionally) +function static_eval_disallowed_binding(ctx, ex) + k = kind(ex) + if k == K"BindingId" + b = get_binding(ctx, syntax_id(ex)) + if b.kind != :global && b.kind != :static_parameter + lam = ctx.scopes[ctx.lambda_bindings.scope_id] + if is_top_scope(lam) || + !(b.lambda_id == top_scope(ctx).id && + enclosing_lambda(ctx, parent(ctx, lam)).id == top_scope(ctx).id) + return ex + end + end + elseif !is_leaf(ex) && !is_quoted(ex) + for e in children(ex) + r = static_eval_disallowed_binding(ctx, e) + if !isnothing(r) + return r + end + end + end + return nothing +end + +function add_assign!(b::BindingInfo) + b.is_assigned_once = !b.is_assigned + b.is_assigned = true +end + +# When a closure captures `T` and `T`'s typevar bound references `S`, it must +# capture `S` too +function expand_captured_sp_deps!(ctx, cb::ClosureBindings, scope) + sps = copy(cb.capt_sp) + for lb in cb.lambdas, (id, is_capt) in lb.locals_capt + is_capt && get_binding(ctx, id).kind === :static_parameter && push!(sps, id) + end + todo = collect(sps) + while !isempty(todo) + sp = pop!(todo) + owner = ctx.scopes[get_binding(ctx, sp).lambda_id] + for dep_tv in get(ctx.tv_deps, ctx.sp_typevars[sp], ()) + # The sparam for dep_tv in the same lambda that owns `sp` + dep_sp = nothing + for id in keys(owner.locals_capt) + b = get_binding(ctx, id) + if b.kind === :static_parameter && + get(ctx.sp_typevars, b.id, IdTag(0)) == dep_tv + dep_sp = id + break + end + end + isnothing(dep_sp) && throw(LoweringError( + binding_ex(ctx, dep_tv), "unimplemented capture in sparam bounds")) + dep_sp in sps && continue + push!(sps, dep_sp) + push!(cb.capt_sp, dep_sp) + ensure_captured!(ctx, scope, get_binding(ctx, dep_sp)) + push!(todo, dep_sp) + end + end +end + +function closure_key(ctx, ex) + @jl_assert kind(ex) === K"BindingId" ex + ClosureKey(syntax_id(ex), ctx.lambda_bindings.scope_id) +end +function current_closure_bindings(ctx) + isempty(ctx.closure_key_stack) && return nothing + get(ctx.closure_bindings, ctx.closure_key_stack[end], nothing) +end + +# Update ctx.bindings metadata based on binding usage +function analyze_variables!(ctx, ex) + k = kind(ex) + if k == K"BindingId" + b = get_binding(ctx, ex) + b.is_read = true + # The type of typed locals is invisible in the previous pass, + # but is filled in here. + scope = ctx.scopes[ctx.lambda_bindings.scope_id] + ensure_captured!(ctx, scope, b) + # b.kind === :static_parameter && ensure_captured!(ctx, scope, b) + @jl_assert (b.kind === :global || b.kind === :typevar || b.is_ssa || + haskey(ctx.lambda_bindings.locals_capt, b.id)) ex binding_ex(ctx, b.id) + if b.kind === :static_parameter && ctx.lifted + cb = current_closure_bindings(ctx) + isnothing(cb) || push!(cb.capt_sp, b.id) + end + if (b.kind === :local || b.kind === :argument) && !isnothing(b.type) && + !(b.id in ctx.types_in_analysis) + push!(ctx.types_in_analysis, b.id) + analyze_variables!(ctx, binding_type_ex(ctx, b)) + delete!(ctx.types_in_analysis, b.id) + end + elseif k == K"Identifier" + @jl_assert false ex + elseif !needs_resolution(ex) + return + elseif k == K"static_eval" || k == K"foreignsymbol" + badvar = static_eval_disallowed_binding(ctx, ex[1]) + if !isnothing(badvar) + default = k == K"foreignsymbol" ? + "function name and library expression" : "syntax" + name_hint = getmeta(ex, :name_hint, default)::String + throw(LoweringError(badvar, "$(name_hint) cannot reference local variable")) + end + analyze_variables!(ctx, ex[1]) + return + elseif k == K"local" || k == K"global" + # Presence of BindingId within local/global is ignored. + return + elseif k == K"=" + lhs = ex[1] + if kind(lhs) != K"Placeholder" + b = get_binding(ctx, lhs) + add_assign!(b) + scope = ctx.scopes[ctx.lambda_bindings.scope_id] + ensure_captured!(ctx, scope, b) + if !isnothing(b.type) + # Assignments introduce a variable's type later during closure + # conversion, but we must model that explicitly here. + analyze_variables!(ctx, binding_type_ex(ctx, b)) + end + end + analyze_variables!(ctx, ex[2]) + elseif k == K"function_decl" + name = ex[1] + b = get_binding(ctx, name) + if b.kind === :local + init_closure_bindings!(ctx, name) + end + add_assign!(b) + elseif k == K"function_type" + if kind(ex[1]) != K"BindingId" || get_binding(ctx, ex[1]).kind !== :local + analyze_variables!(ctx, ex[1]) + end + elseif k == K"constdecl" + if kind(ex[1]) !== K"Placeholder" + b = get_binding(ctx, ex[1]) + b.is_const = true + add_assign!(b) + end + analyze_variables!(ctx, ex[2]) + elseif k == K"call" + name = ex[1] + if kind(name) == K"BindingId" + get_binding(ctx, name).is_called = true + end + foreach(e->analyze_variables!(ctx, e), children(ex)) + elseif k == K"method_defs" + push!(ctx.method_def_stack, ex[1]) + is_closure = kind(ex[1]) == K"BindingId" && + get_binding(ctx, ex[1]).kind === :local + ctx2 = VariableAnalysisContext( + ctx.layer, ctx.bindings, ctx.scopes, + ctx.lambda_bindings, true, ctx.method_def_stack, + ctx.closure_key_stack, + ctx.closure_bindings, ctx.sp_typevars, ctx.tv_deps, + ctx.types_in_analysis) + if is_closure + push!(ctx.closure_key_stack, closure_key(ctx2, ex[1])) + cb = init_closure_bindings!(ctx2, ex[1]) + scope = ctx.scopes[ctx2.lambda_bindings.scope_id] + end + analyze_variables!(ctx2, ex[2]) + analyze_variables!(ctx2, ex[3]) + if is_closure + # All captures are known now; close them over typevar-bound deps + expand_captured_sp_deps!(ctx, cb, scope) + pop!(ctx.closure_key_stack) + end + pop!(ctx.method_def_stack) + elseif k == K"_opaque_closure" + name = ex[1] + init_closure_bindings!(ctx, name) + push!(ctx.method_def_stack, name) + push!(ctx.closure_key_stack, closure_key(ctx, ex[1])) + analyze_variables!(ctx, ex[2]) + analyze_variables!(ctx, ex[3]) + analyze_variables!(ctx, ex[4]) + analyze_variables!(ctx, ex[9]) + pop!(ctx.method_def_stack) + pop!(ctx.closure_key_stack) + elseif k in KSet"lambda toplevel_lambda generated_lambda" + lbs = lambda_bindings(ex[1]) + if !isempty(ctx.closure_key_stack) + # Record all lambdas for the same closure type in one place + ck = last(ctx.closure_key_stack) + if get_binding(ctx, ck.binding).kind === :local + push!(ctx.closure_bindings[ck].lambdas, lbs) + end + end + let ctx2 = VariableAnalysisContext( + ctx.layer, ctx.bindings, ctx.scopes, + lbs, false, ctx.method_def_stack, + ctx.closure_key_stack, ctx.closure_bindings, + ctx.sp_typevars, ctx.tv_deps, ctx.types_in_analysis) + foreach(e->analyze_variables!(ctx2, e), ex[4:end]) + end + else + for e in children(ex) + analyze_variables!(ctx, e) + end + end + nothing +end + +function resolve_scopes(ctx::ScopeResolutionContext, ex) + if !(kind(ex) in KSet"lambda toplevel_lambda generated_lambda") + # Wrap in a top level thunk if we're not already expanding a lambda. + # (Maybe this should be done elsewhere?) + ex = @ast ctx ex [K"toplevel_lambda" [K"block"] [K"block"] ex] + end + _resolve_scopes(ctx, ex, nothing) +end + +""" +This pass analyzes scopes and the names (locals/globals etc) used within them. + +Names of kind `K"Identifier"` are transformed into binding identifiers of +kind `K"BindingId"`. The associated `Bindings` table in the context records +metadata about each binding. + +This pass also records the set of binding IDs used locally within the +enclosing lambda form and information about variables captured by closures. +""" +@fzone "JL: resolve_scopes" function resolve_scopes(ctx::DesugaringContext, ex; + soft_scope::Union{Nothing,Bool}=nothing, + world::UInt=ctx.world) + enable_soft_scopes = soft_scope !== nothing ? soft_scope : contains_softscope_marker(ex) + ctx2 = ScopeResolutionContext(ctx.layer, ctx.bindings, + Dict{ScopeLayer, Int}(), + Vector{ScopeInfo}(), Vector{ScopeId}(), + Set{NameKey}(), Dict{IdTag, IdTag}(), + Dict{IdTag, Vector{IdTag}}(), + enable_soft_scopes, + world) + ex2 = resolve_scopes(ctx2, ex) + ctx3 = VariableAnalysisContext(ctx2.layer, ctx2.bindings, + ctx2.scopes, lambda_bindings(ex2[1]), true, + SyntaxList(), Vector{ClosureKey}(), + Dict{ClosureKey,ClosureBindings}(), + ctx2.sp_typevars, ctx2.tv_deps, Set{IdTag}()) + analyze_variables!(ctx3, ex2) + analyze_def_and_use!(ctx3, ex2) + ctx3, ex2 +end diff --git a/JuliaLowering/src/syntax_macros.jl b/JuliaLowering/src/syntax_macros.jl new file mode 100644 index 0000000000000..fd5397ed2601d --- /dev/null +++ b/JuliaLowering/src/syntax_macros.jl @@ -0,0 +1,441 @@ +# The following are versions of macros from Base which act as "standard syntax +# extensions": +# +# * They emit syntactic forms with special `Kind`s and semantics known to +# lowering +# * There is no other Julia surface syntax for these `Kind`s. + +# In order to implement these here without getting into bootstrapping problems, +# we just write them as plain old macro-named functions and add the required +# __context__ argument ourselves. +# +# TODO: @inline, @noinline, @inbounds, @simd, @ccall, @assume_effects +# +# TODO: Eventually move these to proper `macro` definitions and use +# `JuliaLowering.include()` or something. Then we'll be in the fun little world +# of bootstrapping but it shouldn't be too painful :) + +# Note that `@ast __context__ __context__.macrocall [K"foo" ...]` is unhygienic, +# since `@ast` is meant for internal lowering use (it requires an explicit +# provenance argument, and then copies any syntax context from the provenance to +# any created syntax). A real user-facing macro to replace it should use the +# provenance of the literal K"foo" expression in the file instead, and should +# not copy context (this is not hard to implement, but the provenance requires +# it and callers to be JL-lowered, which this file currently isn't.) + +function Base.var"@nospecialize"(__context__::MacroContext, exs::SyntaxTree...) + if length(exs) == 0 + @ast __context__ __context__.macrocall [K"meta" + "nospecialize"::K"Identifier"] + elseif length(exs) == 1 && kind(exs[1]) === K"=" + eq = exs[1] + @ast __context__ __context__.macrocall [K"meta" + "nospecialize"::K"Identifier" [K"kw"(eq) children(eq)...]] + else + @ast __context__ __context__.macrocall [K"meta" + "nospecialize"::K"Identifier" exs...] + end +end + +# TODO: support all forms that the original supports +# function Base.var"@atomic"(__context__::MacroContext, ex) +# @jl_assert kind(ex) == K"Identifier" || kind(ex) == K"::" (ex, "Expected identifier or declaration") +# @ast __context__ __context__.macrocall [K"atomic" ex] +# end + +# TODO: @label + +function Base.var"@goto"(__context__::MacroContext, ex) + @jl_assert kind(ex) == K"Identifier" ex + @ast __context__ ex [K"symbolicgoto" ex] +end + +function Base.var"@locals"(__context__::MacroContext) + @ast __context__ __context__.macrocall [K"locals"] +end + +@static if isdefined(Base, Symbol("@__FUNCTION__")) +function Base.var"@__FUNCTION__"(__context__::MacroContext) + @ast __context__ __context__.macrocall [K"thisfunction"] +end +end + +function Base.var"@isdefined"(__context__::MacroContext, ex) + @ast __context__ __context__.macrocall [K"isdefined" ex] +end + +function Base.var"@generated"(__context__::MacroContext) + @ast __context__ __context__.macrocall [K"generated"] +end +function Base.var"@generated"(__context__::MacroContext, ex) + if !(kind(ex) === K"function" || + kind(ex) === K"=" && is_eventually_call(ex[1])) + throw(LoweringError(ex, "Expected a function argument to `@generated`")) + end + @ast __context__ __context__.macrocall [K"function" + ex[1] + [K"block" + [K"if" [K"generated"] + ex[2] + [K"block" + [K"meta" "generated_only"::K"Identifier"] + [K"return" nothing::K"Value"] + ] + ] + ] + ] +end + +function Base.var"@cfunction"(__context__::MacroContext, callable, return_type, arg_types) + if kind(arg_types) != K"tuple" + throw(MacroExpansionError(arg_types, "@cfunction argument types must be a literal tuple")) + end + arg_types_svec = @ast __context__ arg_types [K"call" + [K"core" "svec"::K"Identifier"] + children(arg_types)... + ] + if kind(callable) == K"$" + fptr = callable[1] + typ = Base.CFunction + else + # Kinda weird semantics here - without `$`, the callable is a top level + # expression evaluated within the module where the `@cfunction` is + # expanded into. + fptr = @ast __context__ callable [K"inert" + callable + ] + typ = Ptr{Cvoid} + end + @ast __context__ __context__.macrocall [K"cfunction" + typ::K"Value" + fptr + return_type + arg_types_svec + [K"inert" "ccall"::K"Identifier"] + ] +end + +function ccall_macro_parse(ctx, exs) + gc_safe=false + opts = exs[1:end-1] + ex = exs[end] + for opt in opts + @stm opt begin + [K"=" [K"Identifier"] val] -> if syntax_name(opt[1]) != "gc_safe" + throw(MacroExpansionError(opt[1], "unknown option name for ccall")) + elseif !(kind(val) in KSet"Bool Value") + throw(MacroExpansionError(val, "gc_safe must be true or false")) + else + gc_safe = val.value + end + _ -> throw(MacroExpansionError(opt, "bad option to ccall")) + end + end + if length(opts) >= 2 + throw(MacroExpansionError(opts[2], "too many options provided to @ccall")) + end + + (func, argts, rettype) = @stm ex begin + [K"::" [K"call" f as...] r] -> let f_expanded = @stm f begin + [K"." lib sym] -> @ast ctx f [K"tuple" sym lib] + [K"inert" [K"Identifier"]] -> @ast ctx f [K"tuple" f] + [K"Identifier"] -> @ast ctx f [K"tuple" [K"inert" f]] + [K"$" x] -> let kx = kind(x) + if kx in KSet"tuple String string" || + (kx === K"Value" && x.value isa Tuple) || + kx == K"inert" && !(kind(x[1]) == K"Value" && x[1].value isa Ptr) + throw(MacroExpansionError( + f, "interpolated value should be a variable or expression, not a literal name or tuple")) + end + x + end + _ -> throw(MacroExpansionError( + f, "@ccall function name must be a symbol, a `.` node (e.g. `libc.printf`) or an interpolated function pointer (with `\$`)")) + end + (f_expanded, as, r) + end + [K"call" _...] -> throw(MacroExpansionError( + ex, "expected a return type annotation `::SomeType`", position=:end)) + _ -> throw(MacroExpansionError( + ex, "expected call expression with return type")) + end + + # detect varargs + varargs = nothing + argstart = 1 + if length(argts) > 0 && kind(argts[1]) == K"parameters" + varargs = children(argts[1]) + argstart = 2 + end + + # collect args and types + args = SyntaxList() + types = SyntaxList() + function pusharg!(at) + @stm at begin + [K"::" a t] -> (push!(args, a); push!(types, t)) + _ -> throw(MacroExpansionError( + at, "argument needs a type annotation")) + end + end + + for e in argts[argstart:end] + pusharg!(e) + end + + if !isnothing(varargs) + num_required_args = length(args) + if num_required_args == 0 + throw(MacroExpansionError( + argts[1], + "C ABI prohibits varargs without one required argument")) + end + for e in varargs + pusharg!(e) + end + else + num_required_args = 0 # Non-vararg call + end + + return func, rettype, types, args, gc_safe, num_required_args +end + +function ccall_macro_lower(ctx, ex, convention, func, rettype, types, args, gc_safe, num_required_args) + if convention isa Tuple + cconv_tuple = (convention..., gc_safe) + else + cconv_tuple = (convention, UInt16(0), gc_safe) + end + return @ast ctx ex [K"call" + "ccall"::K"Identifier" + func + [K"cconv" cconv_tuple::K"Value" num_required_args::K"Value"] + rettype + [K"tuple" types...] + args... + ] +end + +function Base.var"@ccall"(ctx::MacroContext) + throw(ArgumentError("@ccall needs a function signature with a return type")) +end + +function Base.var"@ccall"(ctx::MacroContext, exs...) + ccall_macro_lower(ctx, exs[end], :ccall, ccall_macro_parse(ctx, exs)...) +end + +function Base.GC.var"@preserve"(__context__::MacroContext, exs...) + idents = exs[1:end-1] + for e in idents + if kind(e) != K"Identifier" + throw(MacroExpansionError(e, "Preserved variable must be a symbol")) + end + end + @ast __context__ __context__.macrocall [K"gc_preserve" exs[end] exs[1:end-1]...] +end + +function Base.Experimental.var"@opaque"(__context__::MacroContext, ex) + @jl_assert kind(ex) == K"->" ex + @ast __context__ __context__.macrocall [K"opaque_closure" + nothing::K"Value" + nothing::K"Value" + nothing::K"Value" + true::K"Bool" + ex + ] +end + +# @eval should mostly ignore hygiene against our system's best wishes. Still +# attempt to preserve provenance. +function _at_eval_code(mc::MacroContext, mod_st::SyntaxTree, ex) + sc = mc.macrocall.context::SyntaxContext + val = remove_context(@ast mc mc.macrocall ("eval_result"::K"Identifier")) + q = _legacy_quote_to_syntax((@ast mc mc.macrocall [K"quote" ex]), 0, true) + new_sc = SyntaxContext(base_layer(sc).mod, sc.version) + @ast mc mc.macrocall [K"block" + [K"local" + [K"=" + val + [K"call" JuliaLowering.eval::K"Value" + mod_st + [K"call" JuliaSyntax.fill_context::K"Value" q new_sc::K"Value"] + ] + ] + ] + [K"unknown_head"(;value="latestworld-if-toplevel")] + val + ] +end +function Base.var"@eval"(__context__::MacroContext, ex) + sc = __context__.macrocall.context::SyntaxContext + mod = @ast __context__ __context__.macrocall base_layer(sc).mod::K"Value" + _at_eval_code(__context__, mod, ex) +end + +function Base.var"@eval"(__context__::MacroContext, mod, ex) + _at_eval_code(__context__, mod, ex) +end + +#-------------------------------------------------------------------------------- +# The following `@islocal` and `@inert` are macros for special syntax known to +# lowering which don't exist in Base but arguably should. +# +# For now we have our own versions +function var"@islocal"(__context__::MacroContext, ex) + @jl_assert kind(ex) == K"Identifier" ex + @ast __context__ __context__.macrocall [K"islocal" ex] +end + +""" +A non-interpolating quoted expression. + +For example, + +```julia +@inert quote + \$x +end +``` + +does not take `x` from the surrounding scope - instead it leaves the +interpolation `\$x` intact as part of the expression tree. + +TODO: What is the correct way for `@inert` to work? ie which of the following +should work? + +```julia +@inert quote + body +end + +@inert begin + body +end + +@inert x + +@inert \$x +``` + +The especially tricky cases involve nested interpolation ... +```julia +quote + @inert \$x +end + +@inert quote + quote + \$x + end +end + +@inert quote + quote + \$\$x + end +end +``` + +etc. Needs careful thought - we should probably just copy what lisp does with +quote+quasiquote 😅 +""" +function var"@inert"(__context__::MacroContext, ex) + @jl_assert kind(ex) == K"quote" ex + @ast __context__ __context__.macrocall [K"inert" ex] +end + +# `quote`/`inert` for syntaxtree +function var"@syntaxinert"(__context__::MacroContext, st) + @ast __context__ __context__.macrocall [K"syntaxinert" st] +end +function var"@syntaxquote"(__context__::MacroContext, st) + @ast __context__ __context__.macrocall [K"syntaxquote" st] +end +# not particularly good or useful, as @syntaxquote must expand first +function var"@syntaxunquote"(__context__::MacroContext, st) + @ast __context__ __context__.macrocall [K"syntaxunquote" st] +end + +# If the syntax version allows, convert quote/$ to syntaxquote/syntaxunquote. +# This is just a convenient way to create SyntaxTree with full provenance +# without dedicated surface syntax, mainly for testing metaprogramming in JL. +# It is insufficient in many ways, e.g. not all forms can be expressed (need +# surface syntax) +function var"@legacy_quote_to_syntax"(__context__::MacroContext, st) + @jl_assert kind(st) === K"quote" || kind(st) === K"inert" st + if is_flisp_compat(__context__.macrocall) + st + elseif kind(st) === K"inert" + @mknode(st; kind=K"syntaxinert") # parser simplifies quote to inert + else + _legacy_quote_to_syntax(st, 0, false) + end +end +function _legacy_quote_to_syntax(st::SyntaxTree, depth, force::Bool) + k = kind(st) + if k === K"quote" && depth == 0 && (force || !is_flisp_compat(st)) + @jl_assert numchildren(st) == 1 st + @mknode(st; kind=K"syntaxquote", children= + mapsyntax(c->_legacy_quote_to_syntax(c, depth+1, force), children(st))) + elseif k === K"$" && depth == 1 && (force || !is_flisp_compat(st)) + @jl_assert numchildren(st) == 1 (st, "bad multi-syntaxunquote") + @mknode(st; kind=K"syntaxunquote") + else + depth2 = k === K"quote" ? depth + 1 : k === K"$" ? depth - 1 : depth + cs = SyntaxList() + for c in children(st) + # Convert multi-unquote to single unquote + if depth2 == 1 && kind(c) === K"$" && numchildren(c) > 1 + for c2 in children(c) + push!(cs, @ast _ c [K"$" c2]) + end + else + push!(cs, c) + end + end + cs_out = mapsyntax(c->_legacy_quote_to_syntax(c, depth2, force), cs) + cs_out == children(st) ? st : @mknode(st; children=cs_out) + end +end +macro legacy_quote_to_syntax(x) + esc(x) +end + +""" +Retrieve the syntax version of the macrocall +""" +function var"@syntax_version"(__context__::MacroContext) + (__context__.macrocall.context::SyntaxContext).version +end +macro syntax_version() + JL_OLD_SYNTAX_VERSION +end + +""" +Set the syntax version of some syntax. This can be used to define macros +producing older syntax than the current version. +""" +function var"@syntax_version"(__context__::MacroContext, ver_st, st) + kind(st) === K"macro" || throw(LoweringError( + st, "`@syntax_version version macro` only supports macro definitions")) + ver = JuliaLowering.eval(syntax_module(ver_st), ver_st) + ver isa VersionNumber || throw(LoweringError( + ver_st, "version argument should be literal `v\"...\" call`")) + _ensure_syntax_version(st, ver) +end +macro syntax_version(_, x) + throw(ArgumentError("@syntax_version can't set version when lowering with flisp")) +end + +function _ensure_syntax_version(st, ver::VersionNumber) + st_sc = st.context::SyntaxContext + sc = st_sc.version == ver ? st_sc : + SyntaxContext(st_sc.layer, st_sc.unexpanded, ver, st_sc.internal) + + if is_leaf(st) || numchildren(st) == 0 + st_sc == sc ? st : @mknode(st; context=sc) + else + out = mapchildren(c->_ensure_syntax_version(c, ver), st) + (st_sc === sc && out === st) ? out : @mknode(st; context=sc) + end +end diff --git a/JuliaLowering/src/utils.jl b/JuliaLowering/src/utils.jl new file mode 100644 index 0000000000000..47b94eb79b631 --- /dev/null +++ b/JuliaLowering/src/utils.jl @@ -0,0 +1,396 @@ +attrsummary(name, _value) = string(name) +attrsummary(name, value::Number) = "$name=$value" +attrsummary(name, value::LineNumberNode) = "$name=L$(value.line)" +attrsummary(name, value::Module) = "$name=$value" + +function _value_string(ex) + k = kind(ex) + str = k == K"Identifier" ? syntax_name(ex) : + k == K"Placeholder" ? syntax_name(ex) : + k == K"SSAValue" ? "%" : + k == K"BindingId" ? "#" : + k == K"label" ? "label" : + k == K"nothing" ? "core.nothing" : + k == K"core" ? "core.$(syntax_name(ex))" : + k == K"top" ? "top.$(syntax_name(ex))" : + k == K"Symbol" ? ":$(syntax_name(ex))" : + k == K"globalref" ? "$(ex.mod).$(syntax_name(ex))" : + k == K"slot" ? "slot" : + k == K"Slots" ? "Slots" : + k == K"LambdaBindings" ? "LambdaBindings" : + k == K"latestworld" ? "latestworld" : + k == K"static_parameter" ? "static_parameter" : + k == K"symboliclabel" ? "label:$(syntax_name(ex))" : + k == K"symbolicgoto" ? "goto:$(syntax_name(ex))" : + k == K"SourceLocation" ? + "SourceLocation:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : + k == K"Value" ? + (ex.value isa SourceRef ? + "SourceRef:$(JuliaSyntax.filename(ex)):$(join(source_location(ex), ':'))" : + ex.value isa SyntaxContext ? "SyntaxContext(#=omitted=#)" : repr(ex.value)) : + ex.value !== nothing ? repr(ex.value) : "::K\"$(untokenize(k))\"" + + if kind(ex) in KSet"BindingId slot SSAValue static_parameter label" + idstr = subscript_str(syntax_id(ex)) + str = "$(str)$idstr" + end + if k == K"slot" || k == K"BindingId" + for p in provenance(ex) + if kind(p) == K"Identifier" + str = "$(str)/$(syntax_name(p))" + break + end + end + end + return str +end + +# Within JL, K"Placeholder" is used for never-read identifiers, but this magic +# symbol is used in the IR (its write-only properties are enforced in codegen). +const UNUSED = "#unused#" + +function _show_syntax_tree(io, ex, indent, show_kinds, @nospecialize(parent_sc)) + nodestr = kind(ex) === K"unknown_head" ? ("unknown_head:"*syntax_name(ex)) : + !is_leaf(ex) ? "[$(untokenize(head(ex)))]" : _value_string(ex) + + treestr = rpad(string(indent, nodestr), 40) + if show_kinds && is_leaf(ex) + treestr = treestr*" :: "*string(kind(ex)) + end + + std_attrs = Set([:value,:kind,:syntax_flags,:source,:context]) + attrstr = join([attrsummary(n, getproperty(ex, n)) + for n in fieldnames(typeof(ex)) if n ∉ std_attrs && + getproperty(ex, n) !== nothing], ",") + print(io, rpad(treestr, 60)) + print(io, " | ") + sc = ex.context + if sc isa SyntaxContext && sc !== parent_sc + print(io, sc) + print(io, ",") + end + print(io, attrstr) + println(io) + + if !is_leaf(ex) + new_indent = indent*" " + for n in children(ex) + _show_syntax_tree(io, n, new_indent, show_kinds, sc) + end + end +end + +function Base.show(io::IO, ::MIME"text/plain", ex::SyntaxTree, show_kinds=true) + assert_syntaxtree(ex) + _show_syntax_tree(io, ex, "", show_kinds, nothing) +end +function _show_syntax_tree_sexpr(io, ex) + if is_leaf(ex) + if JuliaSyntax.is_error(ex) + print(io, "(", untokenize(head(ex)), ")") + else + print(io, _value_string(ex)) + end + else + print(io, "(", untokenize(head(ex))) + for n in children(ex) + print(io, ' ') + _show_syntax_tree_sexpr(io, n) + end + print(io, ')') + end +end + +function Base.show(io::IO, ::MIME"text/x.sexpression", node::SyntaxTree) + assert_syntaxtree(node) + _show_syntax_tree_sexpr(io, node) +end + +function Base.show(io::IO, node::SyntaxTree) + assert_syntaxtree(node) + _show_syntax_tree_sexpr(io, node) +end + +#------------------------------------------------------------------------------- +# Error handling + +TODO(msg::AbstractString) = throw(ErrorException("Lowering TODO: $msg")) +TODO(ex::SyntaxTree, msg="") = throw(LoweringError(ex, "Lowering TODO: $msg")) + +""" +An error with detailed printing containing one or more SyntaxTrees and one +message per tree. If `!internal`, caused by bad user code in `syntax` (flisp: +`Expr(:error, msg)`). +""" +struct LoweringError <: Exception + sts::SyntaxList + msgs::Vector{String} + internal::Bool +end + +@noinline LoweringError(ex::SyntaxTree, msg::String) = + LoweringError(SyntaxList(ex), String[msg], false) + +function Base.showerror(io::IO, exc::LoweringError; show_detail=true) + println(io, exc.internal ? "internal lowering bug:" : "LoweringError:") + for i in eachindex(exc.sts) + st = exc.sts[i] + msg = exc.msgs[i] + src = sourceref(st) + highlight(io, src; note=msg) + if exc.internal || src isa LineNumberNode + print(io, "\nExpression:\n ") + show(io, MIME"text/x.sexpression"(), st) + # TODO: no parents available here; need to place them in LoweringError + parents = SyntaxList() + isempty(parents) || print(io, "\nContaining expressions:") + for p in parents + print(io, "\n ") + show(io, MIME"text/x.sexpression"(), p) + end + end + i !== lastindex(exc.sts) && print(io, "\n\n") + end + + if (show_detail || exc.internal) && !isempty(exc.sts) + print(io, "\n\nDetailed provenance:\n ") + _show_provtree(io, exc.sts[1], " ") + end +end + +function _show_provtree(io::IO, ex::SyntaxTree, indent) + print(io, ex) + if ex.jl_source !== nothing + printstyled(io, " @$(ex.jl_source)", color=:light_black) + end + prov = provenance(ex) + + print(io, "\n") + + src = ex.source + msrc = JuliaSyntax.macro_prov(ex) + printstyled(io, string( + indent, msrc === nothing ? "└─ " : "├─ "); color=:light_black) + if src isa SyntaxTree + _show_provtree(io, src, string(indent, msrc === nothing ? " " : "│ ")) + else + @jl_assert ex.source isa Union{LineNumberNode, SourceRef} ex + src = sourceref(ex) + fn = filename(src) + line, _ = source_location(src) + printstyled(io, "@ $fn:$line\n", color=:light_black) + end + if msrc isa SyntaxTree + printstyled(io, string(indent, "└─ "); color=:light_black) + _show_provtree(io, msrc, indent*" ") + end +end + +function showprov(io::IO, exs::AbstractVector; + note=nothing, include_location::Bool=true, highlight_kwargs...) + for (i,ex) in enumerate(Iterators.reverse(exs)) + sr = sourceref(ex) + if i > 1 + print(io, "\n\n") + end + k = kind(ex) + ex_note = !isnothing(note) ? note : + i > 1 && k == K"macrocall" ? "in macro expansion" : + i > 1 && k == K"$" ? "interpolated here" : + "in source" + highlight(io, sr; note=ex_note, highlight_kwargs...) + + if include_location + line, _ = source_location(sr) + locstr = "$(filename(sr)):$line" + JuliaSyntax._printstyled(io, "\n# @ $locstr", fgcolor=:light_black) + end + end +end + +function showprov(io::IO, ex::SyntaxTree; showprov_kwargs...) + showprov(io, flattened_provenance(ex); showprov_kwargs...) +end + +function subscript_str(i) + replace(string(i), + "0"=>"₀", "1"=>"₁", "2"=>"₂", "3"=>"₃", "4"=>"₄", + "5"=>"₅", "6"=>"₆", "7"=>"₇", "8"=>"₈", "9"=>"₉") +end + +function _deref_ssa(stmts, ex) + while kind(ex) == K"SSAValue" + ex = stmts[syntax_id(ex)] + end + ex +end + +function _is_define_method_call(e) + kind(e) == K"call" && numchildren(e) >= 1 && + kind(e[1]) == K"core" && syntax_name(e[1]) == "define_method" +end + +function _find_method_lambda(ex0, name) + ex = kind(ex0) === K"thunk" ? ex0[1] : ex0 + @jl_assert kind(ex) == K"code_info" ex + # Heuristic search through outer thunk for the method in question. + stmts = children(ex[2]) + for e in stmts + if _is_define_method_call(e) && numchildren(e) == 5 + # define_method(module, fname, sig, lam) + sig = _deref_ssa(stmts, e[4]) + @jl_assert kind(sig) == K"call" ex + arg_types = _deref_ssa(stmts, sig[2]) + @jl_assert kind(arg_types) == K"call" ex + self_type = _deref_ssa(stmts, arg_types[2]) + if kind(self_type) == K"globalref" && occursin(name, syntax_name(self_type)) + return e[5] + end + end + end +end + +function print_ir(io::IO, ex, method_filter=nothing) + @jl_assert kind(ex) == K"code_info" || kind(ex) == K"thunk" ex + if !isnothing(method_filter) + filtered = _find_method_lambda(ex, method_filter) + if isnothing(filtered) + @warn "Method not found with method filter $method_filter" + else + ex = filtered + end + end + _print_ir(io, ex, "") +end + +# TODO: JuliaLowering-the-module should always print the same way, ignoring parent modules +function _print_ir(io::IO, ex0, indent) + added_indent = " " + (ex, is_toplevel_thunk) = kind(ex0) === K"thunk" ? (ex0[1],true) : (ex0,false) + @jl_assert ((kind(ex) == K"lambda" || kind(ex) == K"code_info") + && kind(ex[2]) == K"block") ex + if !is_toplevel_thunk && kind(ex) == K"code_info" + slots = ex[1].value + print(io, indent, "slots: [") + for (i,slot) in enumerate(slots) + print(io, "slot$(subscript_str(i))/$(slot.name)") + flags = String[] + slot.is_nospecialize && push!(flags, "nospecialize") + !slot.is_read && push!(flags, "!read") + slot.is_single_assign && push!(flags, "single_assign") + slot.is_maybe_undef && push!(flags, "maybe_undef") + slot.is_called && push!(flags, "called") + if !isempty(flags) + print(io, "($(join(flags, ",")))") + end + if i < length(slots) + print(io, " ") + end + end + println(io, "]") + end + stmts = children(ex[2]) + for (i, e) in enumerate(stmts) + lno = rpad(i, 3) + if _is_define_method_call(e) && numchildren(e) == 5 + # define_method(module, fname, sig, lam) + print(io, indent, lno, " (call core.define_method ", + string(e[2]), " ", string(e[3]), " ", string(e[4])) + if kind(e[5]) == K"lambda" || kind(e[5]) == K"code_info" + println(io) + print(io, indent, " --- code_info") + println(io) + _print_ir(io, e[5], indent*added_indent) + else + println(io, " ", string(e[5]), ")") + end + elseif kind(e) == K"opaque_closure_method" + @jl_assert numchildren(e) == 5 e + print(io, indent, lno, " --- opaque_closure_method ") + for i=1:4 + print(io, " ", e[i]) + end + println(io) + _print_ir(io, e[5], indent*added_indent) + elseif kind(e) == K"code_info" + println(io, indent, lno, " --- ", "code_info") + _print_ir(io, e, indent*added_indent) + else + code = string(e) + println(io, indent, lno, " ", code) + end + end +end + +# Wrap a function body in Base.Compiler.@zone for profiling +if isdefined(Base.Compiler, Symbol("@zone")) && DEBUG + macro fzone(str, f) + @assert(f isa Expr && f.head === :function && length(f.args) === 2 && str isa String, + "usage: @fzone name_string ") + esc(Expr(:function, f.args[1], + # Use source of our caller, not of this macro. + Expr(:macrocall, :(Base.Compiler.var"@zone"), __source__, str, f.args[2]))) + end +else + macro fzone(str, f) + esc(f) + end +end + +function _flatten_blocks(st::SyntaxTree) + if kind(st) === K"block" + out = SyntaxList() + for c in children(st) + append!(out, _flatten_blocks(c)) + end + # special case: an empty final block has value nothing + if (length(children(st)) > 0 && kind(st[end]) === K"block" && + numchildren(st[end]) == 0) + push!(out, @ast _ st[end] (::K"nothing")) + end + return out + elseif is_quoted(st) + SyntaxList(st) + else + SyntaxList(mapchildren(flatten_blocks, st)) + end +end + +# Splat the contents of any block in `st` whose parent is also a block +function flatten_blocks(st::SyntaxTree) + if kind(st) === K"block" + @mknode(st; children=_flatten_blocks(st)) + elseif is_quoted(st) + st + else + mapchildren(flatten_blocks, st) + end +end + +# Hack. Used for assignment to variables with `decl`, since the type may change +# between assignments. flisp: renumber-assigned-ssavalues +function renumber_assigned_ssavalues(ctx, st) + ssamap = Dict{IdTag, IdTag}() + _find_assigned_ssavars!(ctx, ssamap, st) + isempty(ssamap) && return st + _replace_binding_ids(ctx, ssamap, st) +end +function _find_assigned_ssavars!(ctx, ssamap, st) + (is_leaf(st) || is_quoted(st)) && return + if kind(st) == K"=" && kind(st[1]) == K"BindingId" + b = get_binding(ctx, st[1]) + b.is_ssa || return + ssamap[b.id] = syntax_id(ssavar(ctx, st[1], b.name)) + end + foreach(e->_find_assigned_ssavars!(ctx, ssamap, e), children(st)) +end +function _replace_binding_ids(ctx, ssamap, st) + if kind(st) == K"BindingId" + id = get(ssamap, syntax_id(st), nothing) + isnothing(id) ? st : newleaf(st, K"BindingId", id) + elseif is_leaf(st) || is_quoted(st) + st + else + mapchildren(e->_replace_binding_ids(ctx, ssamap, e), st) + end +end diff --git a/JuliaLowering/src/validation.jl b/JuliaLowering/src/validation.jl new file mode 100644 index 0000000000000..714d249634f64 --- /dev/null +++ b/JuliaLowering/src/validation.jl @@ -0,0 +1,1402 @@ +struct ValidationDiagnostic + sts::SyntaxList + msgs::Vector{String} + loc::LineNumberNode # for noting where failures come from in this file +end +ValidationDiagnostic(st::SyntaxTree, msg, loc) = + ValidationDiagnostic(SyntaxList(st), String[msg], loc) + +""" +The type returned by all `vst` functions. There are three answers this can +represent: + - valid `(true, nothing)` + - known-invalid `(false, [errors...])` + - don't know/not my job `(false, nothing)`. +""" +struct ValidationResult + ok::Bool + errors::Union{Nothing, Vector{ValidationDiagnostic}} +end + +pass() = ValidationResult(true, nothing) +unknown() = ValidationResult(false, nothing) +@noinline fail(st::SyntaxTree, msg="invalid syntax", loc=nothing) = + ValidationResult(false, [ValidationDiagnostic( + st, msg, something(loc, LineNumberNode(0)))]) +macro fail(st, msg) + esc(:($fail($st, $msg, $(QuoteNode(__source__))))) +end + +is_known(vr::ValidationResult) = vr.ok || vr.errors !== nothing + +function Base.var"&"(vr1::ValidationResult, vr2::ValidationResult) + errors_out = isnothing(vr1.errors) ? vr2.errors : + isnothing(vr2.errors) ? vr1.errors : + union(vr1.errors, vr2.errors) + ValidationResult(vr1.ok & vr2.ok, errors_out) +end + +""" +Note that this, unlike `&`, is only a true `|` in the case that at least one +argument is `ok` or at most one argument produces errors. If both sides are +`!ok` with errors, use the first argument's errors. +""" +function Base.var"|"(vr1::ValidationResult, vr2::ValidationResult) + vr1.ok && return vr1 + vr2.ok && return vr2 + ValidationResult(false, !isnothing(vr1.errors) ? vr1.errors : vr2.errors) +end + +abstract type ValidationContext end + +function Base.all(f::T, vcx::ValidationContext, itr; kws...) where {T<:Function} + ok = pass() + for i in itr + ok &= f(vcx, i; kws...) + end + return ok +end + +#------------------------------------------------------------------------------- +# Post-macro-expansion (st1) + +""" +This context contains recursive flags that would otherwise require keyword +arguments to all validation functions, usually to remember the kinds of +structures we're in. + +By default, assume we are validating a usual lowering input (top-level) that has +been macroexpanded. +""" +Base.@kwdef struct Validation1Context <: ValidationContext + toplevel::Bool=true # not in any lambda body + in_gscope::Bool=true # not in any scope; implies toplevel + in_loop::Bool=false # any break/continue allowed + in_symblock::Bool=false # labeled break allowed + inner_cond::Bool=false # methods not allowed in control flow in an outer + # function. true in if (args 2-3), &&, || (arg 2+) + return_ok::Bool=true # yes usually (even outside of functions), no in + # comprehensions/generators + # syntax TODO: no return in finally? type decls? + # assign_ok::Bool=true # no in vect, curly, [typed_]h/v/ncat + + # fixme: flisp happens to allow reading of underscore sparam names if they + # are used in function signature types or other sparam bounds. See #60626. + # + # Mod._ is also readable + readable_underscore::Bool=false + + # vst0 shares this context type since macro expansion doesn't recurse + # into some forms, and most parts of the AST are the same. + unexpanded::Bool=false +end + +function with(vcx::Validation1Context; + toplevel =vcx.toplevel, + in_gscope =vcx.in_gscope, + in_loop =vcx.in_loop, + in_symblock =vcx.in_symblock, + inner_cond =vcx.inner_cond, + return_ok =vcx.return_ok, + readable_underscore=vcx.readable_underscore, + unexpanded =vcx.unexpanded) + Validation1Context( + toplevel, in_gscope, in_loop, in_symblock, inner_cond, return_ok, + readable_underscore, unexpanded) +end + +""" +Executable grammar of the input language to lowering (post-macro-expansion). + +This should serve three purposes: +(1) A readable reference for the julia AST structure (for e.g. macro authors). +(2) A set of assumptions we can use in lowering (a guard against many forms of + invalid input). If `valid_st1(st)` returns true, lowering is expected to + produce correct output given `st` (possibly by throwing a LoweringError). +(3) The place we throw helpful user-facing errors given malformed ASTs. + +Only AST structure is checked. Roughly, this means node kinds and child counts +given this node's parents (to a finite depth) and the current +Validation1Context. + +We don't check some other things: +- This pass assumes that required attributes exist, that leaf-only (or not) + kinds are leaves (or not). See `assert_syntaxtree`. +- Scope issues are caught later in lowering, e.g. declaring something local and + global. +- Checking that certain forms don't appear in value position is also handled + later in lowering. +""" +function valid_st1(st::SyntaxTree) + DEBUG && assert_syntaxtree(st) + vr = vst1(Validation1Context(), st) + @jl_assert is_known(vr) st + return vr +end + +vst1(vcx::Validation1Context, st::SyntaxTree)::ValidationResult = @stm st begin + [K"Identifier"] -> vst1_ident(vcx, st) + (_, when=is_expr_value(st)) -> pass() + [K"block" xs...] -> all(vst1, vcx, xs) + [K"let" [K"block" decls...] body] -> + all(vst1_symdecl_or_assign, vcx, decls) & + vst1(with(vcx; in_gscope=false), body) + [K"let" decl body] -> + vst1_symdecl_or_assign(vcx, decl) & + vst1(with(vcx; in_gscope=false), body) + [K"if" _...] -> let + inner_vcx = vcx.toplevel ? with(vcx; inner_cond=true) : vcx + @stm st begin + [K"if" [K"generated"] t f] -> + vst1(inner_vcx, t) & vst1(inner_vcx, f) + [K"if" [K"generated"] _...] -> + @fail(st, "if-generated requires both true and false cases") + [K"if" cond t] -> + vst1(vcx, cond) & vst1(inner_vcx, t) + [K"if" cond t f] -> + vst1(vcx, cond) & vst1(inner_vcx, t) & vst1(inner_vcx, f) + _ -> @fail(st, "expected (if cond body) or (if cond body else)") + end + end + [K"elseif" cond t] -> vst1(vcx, cond) & vst1(vcx, t) + [K"elseif" cond t f] -> vst1(vcx, cond) & vst1(vcx, t) & vst1(vcx, f) + [K"try" _...] -> vst1_try(vcx, st) + [K"function" _...] -> vst1_function(vcx, st) + [K"call" _...] -> vst1_call(vcx, st) + [K"'" x] -> vst1(vcx, x) + [K"." f [K"tuple" _...]] -> vst1_dotcall(vcx, st) + [K"." l r] -> vst1(vcx, l) & vst1_dot_getproperty_rhs(vcx, r) + [K"." x] -> vst1(vcx, x) # BroadcastFunction(x) + [K"do" call lam] -> + (vst1_call(vcx, call) | vst1_dotcall(vcx, call) | vst0_macrocall(vcx, call)) & + vst1_lam(vcx, lam) + [K"=" _...] -> vst1_assign(vcx, st) + (_, when=(vr=vst1_dotted_or_op_assign(vcx, st); is_known(vr))) -> vr + [K"return" val] -> vcx.return_ok ? + vst1(vcx, val) : + @fail(st, "`return` not allowed inside comprehension or generator") + ([K"continue"], when=vcx.in_loop) -> pass() + ([K"continue" lab], when=vcx.in_loop) -> vst1_ident(vcx, lab; lhs=true) + # An unlabeled break is also allowed inside anonymous `@label` blocks; + # breaking through a named block is rejected with a precise error during + # linearization. + ([K"break"], when=vcx.in_loop||vcx.in_symblock) -> pass() + ([K"break" lab], when=vcx.in_loop||vcx.in_symblock) -> + vst1_ident(vcx, lab; lhs=true) + ([K"break" lab x], when=vcx.in_loop||vcx.in_symblock) -> + vst1_ident(vcx, lab; lhs=true) & vst1(vcx, x) + [K"for" [K"block" is...] body] -> + all(vst1_iter, vcx, is) & + vst1(with(vcx; in_loop=true, in_gscope=false), body) + [K"for" iter1 body] -> + vst1_iter(vcx, iter1) & + vst1(with(vcx; in_loop=true, in_gscope=false), body) + [K"while" cond body] -> + vst1(vcx, cond) & + vst1(with(vcx; in_loop=true, in_gscope=false), body) + [K"tuple" _...] -> + vst1_tuple(vcx, st) + [K"curly" t tvs...] -> + vst1(vcx, t) & no_assignment(tvs, "type parameter list") & + all(vst1_curly_typevar, vcx, tvs) + [K"where" t tds...] -> + vst1(vcx, t) & all(vst1_typevar_decl, vcx, tds) + [K"string" xs...] -> + all(vst1_splat_or_val, vcx, xs) + [K"->" _...] -> + vst1_lam(vcx, st) + [K"flatten" g] -> vst1_generator(vcx, g) + [K"generator" _...] -> vst1_generator(vcx, st) + [K"comprehension" [K"flatten" g]] -> vst1(vcx, g) + [K"comprehension" g] -> vst1(vcx, g) + [K"comprehension" xs...] -> + # HACK: We shouldn't be creating trees here, but this is extremely rare + # (deprecated even in 2016) + vst1_generator(vcx, @ast _ st [K"generator" xs...]) + [K"typed_comprehension" t [K"flatten" g]] -> + vst1(vcx, t) & vst1(vcx, g) + [K"typed_comprehension" t g] -> + vst1(vcx, t) & vst1(vcx, g) + [K"comparison" xs...] -> + length(xs) < 3 || iseven(length(xs)) ? + @fail(st, "`comparison` expects n>=3 args and odd n") : + # TODO: can we restrict xs[2:2:end] to identifier or .identifier? + all(vst1, vcx, xs[2:2:end]) & + all(vst1, vcx, xs[1:2:end]) + [K"<:" xs...] -> all(vst1_call_arg, vcx, xs) + [K">:" xs...] -> all(vst1_call_arg, vcx, xs) + [K"-->" xs...] -> all(vst1_call_arg, vcx, xs) + [K"::" x y] -> vst1(vcx, x) & vst1(vcx, y) + # TODO: inner_cond on args[2:end] + [K"&&" xs...] -> all(vst1, vcx, xs) + [K"||" xs...] -> all(vst1, vcx, xs) + [K".&&" x y] -> vst1(vcx, x) & vst1(vcx, y) + [K".||" x y] -> vst1(vcx, x) & vst1(vcx, y) + (_, when=(vr=vst1_arraylike(vcx, st); is_known(vr))) -> vr + # syntax TODO: disallow pre-desugared const, broken with complex rhs + [K"const" l r] -> vst1_ident(vcx, l; lhs=true) & vst1(vcx, r) + [K"const" [K"global" x]] -> !vcx.toplevel ? + @fail(st, "unsupported `const` inside function") : + vst1_const_assign(vcx, x) + [K"const" x] -> !vcx.toplevel ? + @fail(st, "unsupported `const` inside function") : + vst1_const_assign(vcx, x) + [K"global" xs...] -> minlen(st, xs, 1) & all(vst1_global_arg, vcx, xs) + [K"local" xs...] -> minlen(st, xs, 1) & all(vst1_local_arg, vcx, xs) + [K"macrocall" _...] -> vst0_macrocall(vcx, st) + [K"quote" x] -> vcx.unexpanded ? vst0_quoted(vcx, x; quote_level=1) : + @fail(st, "interpolating quote not valid syntax after macro expansion") + + #--------------------------------------------------------------------------- + # Forms not produced by the parser + [K"ssavalue" [K"Value"]] -> pass() + [K"static_parameter" [K"Value"]] -> pass() + [K"inert" _] -> pass() + [K"syntaxinert" _] -> pass() + [K"core" [K"Identifier"]] -> pass() + [K"top" [K"Identifier"]] -> pass() + [K"meta" _...] -> pass() # TODO + [K"toplevel" xs...] -> pass() # this will be validated when we lower it + [K"opaque_closure" argt lb ub bool lam] -> + all(vst1, vcx, [argt, lb, ub, bool]) & vst1_lam(vcx, lam) + [K"symboliclabel" lab] -> vst1_ident(vcx, lab; lhs=true) + [K"symbolicgoto" lab] -> vst1_ident(vcx, lab; lhs=true) + [K"symbolicblock" lab body] -> + vst1_ident(vcx, lab; lhs=true) & vst1(with(vcx; in_symblock=true), body) + [K"gc_preserve" x ids...] -> vst1(vcx, x) & all(vst1_ident, vcx, ids) + # lowering TODO: 0 args segfaults + [K"gc_preserve_begin" ids...] -> all(vst1_ident, vcx, ids) + [K"gc_preserve_end" ids...] -> all(vst1_ident, vcx, ids) + [K"isdefined" [K"Identifier"]] -> pass() + [K"isdefined" [K"static_parameter" [K"Value"]]] -> pass() + [K"lambda" _...] -> vst1_raw_lambda(vcx, st) + [K"with-static-parameters" lam sps...] -> + vst1_raw_lambda(vcx, lam) & all(vst1_ident, vcx, sps; lhs=true) + [K"softscope" _] -> pass() + [K"softscope"] -> pass() + [K"generated"] -> pass() + [K"foreigncall" fname rt at cconv roots_args...] -> + # TODO: could be stricter + vst1(vcx, fname) & + vst1(vcx, rt) & + vst1(vcx, at) & + vst1(vcx, cconv) & + all(vst1, vcx, roots_args) + [K"foreignglobal" fname] -> vst1(vcx, fname) # TODO: could be stricter + [K"cfunction" [K"Value"] f rt at [K"inert" [K"Identifier"]]] -> + vst1(vcx, f) & vst1(vcx, rt) & vst1(vcx, at) + [K"cconv" tup nreq] -> (tup.value isa Tuple && nreq.value isa Int) ? pass() : + @fail(st, "expected (cconv convention_tuple n_req_args)") + [K"tryfinally" t f] -> vst1(vcx, t) & vst1(vcx, f) + [K"tryfinally" t f scope] -> vst1(vcx, t) & vst1(vcx, f) & vst1(vcx, scope) + [K"loopinfo" _...] -> pass() # TODO + [K"boundscheck"] -> pass() # optional bool arg does nothing + ([K"boundscheck" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + ([K"inbounds" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + ([K"inbounds" [K"Identifier"]], when=(syntax_name(st[1]) == "pop")) -> pass() + ([K"inline" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + ([K"noinline" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + [K"purity"] -> pass() + [K"purity" _ _...] -> numchildren(st) == fieldcount(Base.EffectsOverride) ? + pass() : @fail(st, "wrong number of args to `purity` expression") + [K"locals"] -> pass() + [K"islocal" _] -> pass() + [K"isglobal" _] -> pass() + [K"copyast" [K"inert" _]] -> pass() + [K"new" t args...] -> vst1(vcx, t) & all(vst1, vcx, args) + [K"splatnew" t arg] -> vst1(vcx, t) & vst1(vcx, arg) + [K"thisfunction"] -> vcx.toplevel ? + @fail(st, "can only be used inside a function") : + !vcx.return_ok ? + @fail(st, "current function not defined in comprehension or generator") : pass() + [K"unknown_head" xs...] -> let head = syntax_name(st) + if head === "latestworld-if-toplevel" + maxlen(st, xs, 0) + elseif head === "scope-block" + minlen(st, xs, 1) & maxlen(st, xs, 1) & + all(vst1, with(vcx; in_gscope=false), xs) + else + @fail(st, string("unknown expr head: ", head)) + end + end + [K"aliasscope"] -> pass() + [K"popaliasscope"] -> pass() + + #--------------------------------------------------------------------------- + # Invalid forms for which we want to produce detailed errors + [K"..." _...] -> + @fail(st, "unexpected `...`\nsplatting can only be done into a `call`, `tuple`, `curly`, or array-like expression") + [K"parameters" _...] -> + @fail(st, "unexpected semicolon") + [K"braces" _...] -> + @fail(st, "`{ }` outside of `where` is reserved for future use") + [K"bracescat" _...] -> + @fail(st, "`{ }` outside of `where` is reserved for future use") + [K"atomic" _...] -> + @fail(st, "unimplemented or unsupported `atomic` declaration") + [K"::" x] -> + @fail(st, "`::` must be written `value::type` outside function argument lists") + # internal + [K"Symbol"] -> + @fail(st, "`Symbol` kind not valid until desugaring") + [K"Placeholder"] -> + @fail(st, "`Placeholder` kind not valid until desugaring") + [K"$" x] -> @fail(st, raw"`$` expression outside string or quote") + [K"continue" _...] -> + @fail(st, "`continue` outside of a `while` or `for` loop") + [K"break"] -> + @fail(st, "unlabeled `break` outside of a `while` or `for` loop") + [K"break" _...] -> + @fail(st, "labeled `break` outside of loop or symbolic block") + _ -> let top_vr = vst1_toplevel_only(vcx, st) + if vcx.toplevel + top_vr + else + !top_vr.ok ? unknown() : + @fail(st, "this syntax is only allowed at top level") + end + end | @fail(st, string( + "invalid syntax: unknown form `", kind(st), + "` or number of arguments ", numchildren(st))) +end + +vst1_toplevel_only(vcx, st) = @stm st begin + # body will be validated when lowered + [K"module" [K"Value"] [K"Value"] [K"Identifier"] [K"block" xs...]] -> + !(st[1].value isa VersionNumber) ? @fail(st[1], "expected version") : + !(st[2].value isa Bool) ? @fail(st[2], "expected boolean bare flag") : + pass() + [K"module" [K"Value"] [K"Identifier"] [K"block" xs...]] -> + !(st[1].value isa Bool) ? @fail(st[1], "expected boolean bare flag") : + pass() + [K"macro" _...] -> + vst1_macro(vcx, st) + [K"struct" [K"Value"] sig [K"block" body...]] -> + vst1_typesig(vcx, sig) & ( + !(st[1].value isa Bool) ? @fail(st[1], "expected mutable flag") : + _struct_noassign(vcx, body) & all(vst1_struct_arg, vcx, body)) + [K"abstract" sig] -> + vst1_typesig(vcx, sig) + [K"primitive" sig n] -> + vst1_typesig(vcx, sig) & vst1(vcx, n) + [K"import" [K":" p1 ps...]] -> + (vst1_importpath(vcx, p1; dots_ok=true) & + all(vst1_importpath, vcx, ps; dots_ok=false)) + [K"using" [K":" p1 ps...]] -> + (vst1_importpath(vcx, p1; dots_ok=true) & + all(vst1_importpath, vcx, ps; dots_ok=false)) + [K"import" ps...] -> + minlen(st, ps, 1) & all(vst1_importpath, vcx, ps; dots_ok=true) + [K"using" ps...] -> + minlen(st, ps, 1) & all(vst1_importpath, vcx, ps; dots_ok=true) + [K"public" xs...] -> all(vst1_ident, vcx, xs) + [K"export" xs...] -> all(vst1_ident, vcx, xs) + [K"latestworld"] -> pass() + [K"typegroup" [K"block" xs...]] -> + all(vst1, vcx, xs) + _ -> unknown() +end + +#------------------------------------------------------------------------------- + +vst1_local_arg(vcx, st) = @stm st begin + [K"function" _...] -> vst1_function(vcx, st) + _ -> vst1_symdecl_or_assign(vcx, st) | vst1_dotted_or_op_assign(vcx, st) | + @fail(st, "invalid local declaration: expected identifier or assignment") +end + +vst1_global_arg(vcx, st) = @stm st begin + [K"function" _...] -> vcx.toplevel ? + vst1_function(vcx, st) : + @fail(st, "global function needs to be placed at top level, or use eval") + _ -> vst1_symdecl_or_assign(vcx, st) | vst1_dotted_or_op_assign(vcx, st) | + @fail(st, "invalid global declaration: expected identifier or assignment") +end + +# @stm doesn't work so well with n dots and m identifiers +# one of: +# (as (importpath . . . x y z) ident) +# (importpath . . . x y z) +# where y, z may be quoted (syntax TODO: require var"" for odd identifiers?) +function vst1_importpath(vcx, st; dots_ok) + ok = pass() + path_components = @stm st begin + [K"as" [K"." xs...] [K"Identifier"]] -> xs + [K"as" [K"." xs...] x] -> (ok &= @fail(x, "expected identifier"); xs) + [K"." xs...] -> xs + _ -> return @fail(st, "malformed import path") + end + seen_first = false + for c in path_components + if kind(c) === K"Identifier" && syntax_name(c) === "." + if !dots_ok || seen_first + ok &= @fail(c, "unexpected `.` in import path") + end + continue + end + if kind(c) === K"inert" && numchildren(c) == 1 + c = c[1] + end + # syntax todo: lhs should probably not be true here + ok = ok & (vst1_ident(vcx, c).ok ? pass() : vst1_ident(vcx, c; lhs=true)) + seen_first = true + end + return !seen_first ? @fail(st, "expected identifier in `importpath`") : ok +end + +vst1_tuple(vcx, st) = @stm st begin + [K"tuple" [K"parameters" kws...]] -> all(vst1_call_kwarg, vcx, kws) + [K"tuple" [K"parameters" _ _...] _ _...] -> @fail( + st[1], "cannot mix tuple `(a,b,c)` and named tuple `(;a,b,c)` syntax") + ([K"tuple" args...], when=any(x->kind(x)===K"=", args)) -> + all(vst1_call_arg, vcx, args) + [K"tuple" xs...] -> all(vst1_splat_or_val, vcx, xs) + _ -> @fail(st, "malformed tuple") +end + +# TODO: disallow (has-unmatched-symbolic-goto? tryb) +vst1_try(vcx, st) = @stm st begin + [K"try" _] -> @fail(st, "try without catch or finally") + [K"try" tryb cvar catchb] -> + vst1(vcx, tryb) & + vst1_try_catchvar(vcx, cvar) & + vst1(vcx, catchb) + [K"try" tryb cvar catchb finallyb] -> + vst1(vcx, tryb) & + vst1_try_catchvar(vcx, cvar) & + vst1(vcx, catchb) & + vst1(vcx, finallyb) + [K"try" tryb cvar catchb finallyb elseb] -> + vst1(vcx, tryb) & + vst1_try_catchvar(vcx, cvar) & + vst1(vcx, catchb) & + vst1(vcx, finallyb) & + vst1(vcx, elseb) + _ -> @fail(st, "malformed `try` expression") +end + +vst1_try_catchvar(_vcx, st) = @stm st begin + [K"Identifier"] -> pass() + ([K"Value"], when=st.value===false) -> pass() +end + +# syntax TODO: +# - const is inoperative in the function case +# - single-arg const with no value (presumably to poison this name) was likely +# not intended to work, and can only be produced by macros +vst1_const_assign(vcx, st) = @stm st begin + [K"=" _ _] -> vst1_assign(vcx, st; in_const=true) + [K"Identifier"] -> pass() + [K"local" _...] -> @fail(st, "unsupported `const local` declaration") + _ -> @fail(st, "expected assignment after `const`") +end + +# syntax TODO: all-underscore variables may be read from with dot syntax +vst1_dot_getproperty_rhs(vcx, st) = @stm st begin + [K"inert" x] -> pass() + [K"syntaxinert" x] -> pass() + [K"Identifier"] -> pass() + (_, when=is_expr_value(st)) -> pass() + _ -> @fail(st, "invalid `.` syntax") +end + +# We can't validate A.B in general (usually lowers to getproperty), but it shows +# up in a number of syntax special cases where we can. (flisp: sym-ref?) +vst1_calldecl_dot_name(vcx, st) = @stm st begin + [K"." l r] -> + vst1_calldecl_dot_name(vcx, l) & + vst1_calldecl_dot_name_rhs(vcx, r) | + @fail(st, "invalid `.` form") + [K"Value"] -> pass() + i -> vst1_ident(vcx, i) +end + +vst1_calldecl_dot_name_rhs(vcx, st) = @stm st begin + [K"inert" x] -> vst1_calldecl_dot_name_rhs(vcx, x) + [K"syntaxinert" x] -> vst1_calldecl_dot_name_rhs(vcx, x) + [K"Identifier"] -> vst1_ident(vcx, st; lhs=true) + ([K"Value"], when=st.value isa String) -> _ident_str(vcx, st, st.value; lhs=true) + [K"String"] -> _ident_str(vcx, st, st.value; lhs=true) + [K"tuple" _...] -> @fail(st, "dotcall syntax not valid here") + _ -> @fail(st, "invalid `.` syntax") +end + +vst1_symdecl_or_assign(vcx, st) = + @fail(st, "expected identifier or assignment") | + vst1_symdecl(vcx, st) | vst1_assign(vcx, st) + +vst1_symdecl(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"::" [K"Identifier"] t] -> vst1(vcx, t) + _ -> @fail(st, "expected identifier or `identifier::type`") +end + +# TODO: globalref (identifier with .mod) might not be valid everywhere; check +# usage of this function +vst1_ident(vcx, st; lhs=false) = @stm st begin + [K"Identifier"] -> _ident_str(vcx, st, syntax_name(st); lhs) + _ -> @fail(st, "expected identifier") +end +function _ident_str(vcx, st, s::String; lhs=false) + if !lhs && (!vcx.readable_underscore || !is_flisp_compat(st)) && + is_writeonly_est_name(s) + @fail(st, "all-underscore identifiers are write-only and their values cannot be used in expressions") + elseif lhs && s in ("ccall", "cglobal") + @fail(st, string(s, " is a reserved identifier")) + else + pass() + end +end + +"N.B. this shouldn't be used after `est_to_dst`, as JuliaLowering uses the +Placeholder kind when we have write-only identifiers" +function is_writeonly_est_name(s::String) + (all(==('_'), s) || s == UNUSED) && length(s) > 0 +end + +vst1_call(vcx, st) = @stm st begin + ([K"call" [K"Identifier"] args...], when=syntax_name(st[1])==="cglobal") -> + (1 <= length(args) <= 2 ? pass() : + @fail(st, "cglobal must have one or two arguments")) & + all(vst1_call_arg, vcx, args) + [K"call" f [K"parameters" kwargs...] args...] -> + (vst1_ident(vcx, f) | vst1(vcx, f)) & + all(vst1_call_arg, vcx, args) & + all(vst1_call_kwarg, vcx, kwargs) + [K"call" f args...] -> + (vst1_ident(vcx, f) | vst1(vcx, f)) & + all(vst1_call_arg, vcx, args) + [K"call" _...] -> @fail(st, "malformed `call`") + _ -> unknown() +end + +vst1_dotcall(vcx, st) = @stm st begin + [K"." f [K"tuple" [K"parameters" kwargs...] args...]] -> + vst1(vcx, f) & all(vst1_call_kwarg, vcx, kwargs) & + all(vst1_call_arg, vcx, args) + [K"." f [K"tuple" args...]] -> + vst1(vcx, f) & all(vst1_call_arg, vcx, args) + _ -> unknown() +end + +# Arg to call (not function decl), pre-semicolon. This can be anything, but +# additionally allow `kw` and `...` forms. +vst1_call_arg(vcx, st) = @stm st begin + [K"kw" id val] -> vst1_ident(vcx, id; lhs=true) & vst1(vcx, val) + _ -> vst1_splat_or_val(vcx, st) +end + +# Arg to `parameters` (post-semicolon) in a call (not function decl). Stricter +# than `vst1_call_arg`. `=` desugars to `kw`. +vst1_call_kwarg(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"kw" id val] -> vst1_ident(vcx, id; lhs=true) & vst1(vcx, val) + [K"=" id val] -> vst1_ident(vcx, id; lhs=true) & vst1(vcx, val) + [K"..." x] -> vst1(vcx, x) + [K"." x [K"inert" id]] -> vst1(vcx, x) & vst1_ident(vcx, id; lhs=true) + [K"." x [K"syntaxinert" id]] -> vst1(vcx, x) & vst1_ident(vcx, id; lhs=true) + ([K"call" [K"Identifier"] symval v], when=(syntax_name(st[1])==="=>")) -> + vst1(vcx, symval) & vst1(vcx, v) + _ -> @fail(st, "expected identifier, `=`, or `...` after semicolon") +end + +vst1_lam(vcx, st) = let + f_vcx = with(vcx; return_ok=true, toplevel=false, in_gscope=false) + @stm st begin + [K"->" l r] -> + vst1_lam_lhs(with(f_vcx; return_ok=false), l) & vst1(f_vcx, r) + _ -> @fail(st, "expected `->` expression") + end +end + +vst1_lam_lhs(vcx, st) = @stm st begin + [K"tuple" [K"parameters" _...] ps...] -> + _calldecl_positionals(vcx, ps, true) & vst1_calldecl_kws(vcx, st[1]) + [K"tuple" ps...] -> + _calldecl_positionals(vcx, ps, true) + [K"where" ps tds...] -> + vst1_lam_lhs(vcx, ps) & + all(vst1_typevar_decl, with(vcx; readable_underscore=true), tds) + # syntax TODO: This is handled badly in the parser + [K"block"] -> pass() + [K"block" x] -> _calldecl_positionals(vcx, SyntaxList(x), true) + [K"block" x p] -> _calldecl_positionals(vcx, SyntaxList(x), true) & + @stm p begin + [K"=" kw v] -> vst1_param(vcx, kw) & vst1(vcx, v) + [K"kw" kw v] -> vst1_param(vcx, kw) & vst1(vcx, v) + [K"..." kw] -> vst1_param_varkw(vcx, kw) + _ -> vst1_param(vcx, p) + end + [K"block" _ _ _ _...] -> @fail(st, "more than one semicolon in signature") + # unwrapped single arg + _ -> let ps = SyntaxList(st) + _calldecl_positionals(vcx, ps, true) + end +end + +vst1_function(vcx, st) = let + f_vcx = with(vcx; return_ok=true, toplevel=false, in_gscope=false) + # lowering TODO: conditional nested function definitions are known to be + # broken, but are not disallowed, and can be found in stdlibs. + # vcx.inner_cond && @fail(st, "conditional inner method definitions\ + # are not supported; use `()->()` syntax instead") + @stm st begin + [K"function" name] -> vst1_ident(vcx, name) + [K"function" callex body] -> + vst1_function_calldecl(with(vcx; return_ok=false), callex) & + vst1(f_vcx, body) + [K"=" callex body] -> + vst1_function_calldecl(with(vcx; return_ok=false), callex) & + vst1(f_vcx, body) + _ -> @fail(st, "malformed `function`") + end +end + +# Note that we consistently refer to children of a declaring call as +# "parameters" rather than arguments (and children of a K"parameters" block as +# "keyword args/params") so we don't mix them up with children to a real call, +# whose valid forms are subtly different. + +vst1_function_calldecl(vcx, st) = @stm st begin + [K"where" callex tds...] -> + vst1_function_calldecl(vcx, callex) & + all(vst1_typevar_decl, with(vcx; readable_underscore=true), tds) + [K"::" callex rt] -> + vst1_simple_calldecl(vcx, callex) & vst1(vcx, rt) + _ -> vst1_simple_calldecl(vcx, st) +end + +vst1_simple_calldecl(vcx, st) = @stm st begin + [K"call" f [K"parameters" _...] ps...] -> + vst1_calldecl_name(vcx, f) & + _calldecl_positionals(vcx, ps, false) & + vst1_calldecl_kws(vcx, st[2]) + [K"call" f ps...] -> vst1_calldecl_name(vcx, f) & + _calldecl_positionals(vcx, ps, false) + # anonymous function syntax `function (x); end` or `function (x...); end` is + # subject to bad-arglist rules (block, etc.) + _ -> vst1_lam_lhs(vcx, st) | @fail(st, "malformed `call` in function decl") +end + +vst1_macro(vcx, st) = @stm st begin + [K"macro" m] -> vst1_ident(vcx, m; lhs=true) | vst1_ident(vcx, m; lhs=false) + [K"macro" [K"call" _ [K"parameters" _...] _...] _...] -> + @fail(st[1][end], "macros cannot accept keyword arguments") + [K"macro" [K"call" m ps...] body] -> + let vcx = with(vcx; return_ok=false, toplevel=false, in_gscope=false) + vst1_macro_calldecl_name(vcx, m) & + _calldecl_positionals(vcx, ps, false) & + vst1(with(vcx; return_ok=true), body) + end + [K"macro" [K"where" _...] _...] -> + @fail(st[1], "`where` not allowed in macro signatures") + [K"macro" _...] -> @fail(st, "malformed `macro`") + _ -> unknown() +end + +# Macros may have either underscore or reserved (ccall, cglobal) names +vst1_macro_calldecl_name(vcx, st) = @stm st begin + [K"." _ _] -> vst1_calldecl_dot_name(vcx, st) + m -> @fail(st, "invalid macro name") | + vst1_ident(vcx, m; lhs=true) | vst1_ident(vcx, m; lhs=false) +end + +vst1_calldecl_name(vcx, st) = @stm (st=strip_arg_meta(st)) begin + [K"Identifier"] -> vst1_ident(vcx, st; lhs=true) & + (!is_dotted_operator(syntax_name(st)) ? pass() : + @fail(st, "dotted operator is not a valid function name")) + [K"." _ _] -> + vst1_calldecl_dot_name(vcx, st) + [K"curly" t tvs...] -> + vst1_calldecl_name(vcx, t) & all(vst1, vcx, tvs) + [K"Value"] -> + pass() # GlobalRef works. Function? Type? + ([K"::" _...], when=!vcx.toplevel) -> + @fail(st, "adding methods to callable type only allowed at top level") + [K"::" t] -> vst1(vcx, t) + [K"::" x t] -> vst1_pparam_simple_tuple(vcx, x) & vst1(vcx, t) + # TODO: @overlay broken in many cases, should be stricter + [K"overlay" mt x] -> + vst1(vcx, mt) & vst1_calldecl_name(vcx, x) + + [K"where" t tds...] -> + vst1_calldecl_name(vcx, t) & all(vst1_typevar_decl, vcx, tds) + _ -> @fail(st, "invalid function name") +end + +strip_arg_meta(st) = @stm st begin + [K"meta" s arg] -> let meta_s = est_syntax_name(s, "") + meta_s isa String || return st + kind(arg) === K"meta" ? st : + !(meta_s in ("specialize", "nospecialize")) ? st : arg + end + _ -> st +end + +# Check mandatory and optional positional params: +# `[pparam* pparam_and_default* pparam_and_splatdefault? pparam_va?]` +# TODO: add list matching to @stm +function _calldecl_positionals(vcx, params_meta, eq_is_kw) + isempty(params_meta) && return pass() + ok = Ref(pass()) + params = map(strip_arg_meta, params_meta) + va_ok = vst1_pparam_va(vcx, params[end]; eq_is_kw) + if is_known(va_ok) + params = params[1:end-1] + ok[] &= va_ok + end + require_assign = false + for (i, p) in enumerate(params) + if kind(p) === K"kw" || kind(p) === K"=" && eq_is_kw + require_assign = true + allow_val_splat = i == lastindex(params) + ok[] &= vst1_pparam_and_default(vcx, p; eq_is_kw, allow_val_splat) + elseif kind(p) === K"..." + ok[] &= @fail(p, "`...` may only be used on the final parameter") + elseif require_assign # TODO: multi-syntaxtree error + ok[] &= @fail(p, "all function parameters after an optional parameter must also be optional") + else + ok[] &= vst1_pparam_typed_tuple(vcx, p) + end + end + return ok[] +end + +# TODO: flisp optional-positional-defs counts a wrapped `...` as an optional +# arg, not a vararg, but it appears to work the same +vst1_pparam_va(vcx, st; eq_is_kw) = @stm st begin + [K"kw" [K"..." va] val] -> + vst1_pparam_typed_tuple(vcx, va) & vst1_splat_or_val(vcx, val) + ([K"=" [K"..." va] val], when=eq_is_kw) -> + vst1_pparam_typed_tuple(vcx, va) & vst1_splat_or_val(vcx, val) + [K"..." va] -> vst1_pparam_typed_tuple(vcx, va) + _ -> unknown() +end + +# destructuring args: function f(a, (x, y)) ... TODO: the strip_arg_meta call +# here corresponds to no-op nospecialize, and should ideally be removed. +vst1_pparam_typed_tuple(vcx, st) = @stm (st=strip_arg_meta(st)) begin + [K"::" [K"tuple" _...] t] -> + vst1_pparam_simple_tuple(vcx, st[1]) & + vst1(with(vcx; readable_underscore=true), t) + [K"tuple" _...] -> vst1_pparam_simple_tuple(vcx, st) + _ -> vst1_param(vcx, st) +end +vst1_pparam_simple_tuple_or_splat(vcx, st) = @stm st begin + [K"..." t] -> vst1_pparam_simple_tuple(vcx, t) + t -> vst1_pparam_simple_tuple(vcx, t) +end +# Similar to an assignment to a tuple LHS, but does not allow `::`. Also should +# not allow ref, curly, or call, but flisp does, so we may need to change this. +vst1_pparam_simple_tuple(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"tuple" [K"parameters" _ _...] _ _...] -> @fail( + st[1], "cannot mix tuple `(a,b,c)` and named tuple `(;a,b,c)` syntax") + [K"tuple" [K"parameters" kws...]] -> all(vst1_ident, vcx, kws; lhs=true) + [K"tuple" xs...] -> + all(vst1_pparam_simple_tuple_or_splat, vcx, xs) & + (count(kind(x)===K"..." for x in xs) <= 1 ? pass() : + @fail(st, "multiple `...` in destructured parameter is ambiguous")) + [K"::" _...] -> @fail(st, "cannot have type in destructured argument") + _ -> @fail(st, "expected identifier or tuple") +end + +vst1_param(vcx, st) = @stm st begin + [K"Identifier"] -> vst1_ident(vcx, st; lhs=true) + [K"::" id t] -> vst1_ident(vcx, id; lhs=true) & + vst1(with(vcx; readable_underscore=true), t) + [K"::" t] -> vst1(with(vcx; readable_underscore=true), t) + _ -> @fail(st, "expected identifier or `identifier::type`") +end + +# allow_val_splat=true when this is the final optional param (even if there are +# varargs after it). See #50563 +vst1_pparam_and_default(vcx, st; eq_is_kw, allow_val_splat) = @stm st begin + [K"kw" id val] -> + vst1_pparam_typed_tuple(vcx, id) & @stm val begin + [K"..." v] -> allow_val_splat ? vst1(vcx, v) : + @fail(val, "splat only allowed on final positional default arg") + _ -> vst1(with(vcx; return_ok=true, toplevel=false, in_gscope=false), val) + end + ([K"=" id val], when=eq_is_kw) -> + vst1_pparam_typed_tuple(vcx, id) & @stm val begin + [K"..." v] -> allow_val_splat ? vst1(vcx, v) : + @fail(val, "splat only allowed on final positional default arg") + _ -> vst1(with(vcx; return_ok=true, toplevel=false, in_gscope=false), val) + end + _ -> @fail(st, "malformed optional positional parameter; expected `=`") +end + +vst1_calldecl_kws(vcx, st) = @stm st begin + ([K"parameters" kws... last], + when=(varkw = strip_arg_meta(last); + kind(varkw) === K"..." && numchildren(varkw) == 1)) -> + all(vst1_param_kw, vcx, kws) & vst1_param_varkw(vcx, varkw[1]) + [K"parameters" kws...] -> all(vst1_param_kw, vcx, kws) + _ -> @fail(st, "malformed keyword parameters") +end + +vst1_param_varkw(vcx, st) = @stm st begin + [K"Identifier"] -> vst1_ident(vcx, st; lhs=true) + [K"::" _...] -> + @fail(st, "keyword parameter with `...` may not be given a type") + _ -> @fail(st, "expected identifier") +end + +# note no return_ok in default val, unlike positional defaults, due to bugs +vst1_param_kw(vcx, st) = @stm (st=strip_arg_meta(st)) begin + [K"kw" id val] -> + vst1_param(vcx, id) & vst1(with(vcx; toplevel=false, in_gscope=false), val) + [K"..." _...] -> + @fail(st, "`...` may only be used for the final keyword parameter") + _ -> vst1_param(vcx, st) | + @fail(st, "malformed keyword parameter; expected identifier, `=`, or `::`") +end + +vst1_typevar_decl(vcx, st) = @stm st begin + [K"Identifier"] -> vst1_ident(vcx, st; lhs=true) + [K"<:" t old] -> + vst1_ident(vcx, t; lhs=true) & vst1(vcx, old) + [K">:" t old] -> + vst1_ident(vcx, t; lhs=true) & vst1(vcx, old) + ([K"comparison" val_l [K"Identifier"] t [K"Identifier"] val_r], + when=(syntax_name(st[2])===syntax_name(st[4]) && syntax_name(st[2]) in ("<:", ">:"))) -> + vst1(vcx, val_l) & + vst1_ident(vcx, t; lhs=true) & + vst1(vcx, val_r) + [K"<:" x _] -> + @fail(x, "expected type name") + [K">:" x _] -> + @fail(x, "expected type name") + [K"comparison" _...] -> + @fail(st, "expected `lb <: type_name <: ub` or `ub >: type_name >: lb`") + _ -> @fail(st, "expected type name or type bounds") +end + +vst1_typesig(vcx, st) = @stm st begin + [K"Identifier"] -> + vst1_ident(vcx, st) + [K"curly" t tvs...] -> + vst1_ident(vcx, t) & all(vst1_typevar_decl, vcx, tvs) + [K"<:" [K"curly" t tvs...] super] -> + vst1_ident(vcx, t) & vst1(vcx, super) & + all(vst1_typevar_decl, vcx, tvs) + [K"<:" t super] -> + vst1_ident(vcx, t) & vst1(vcx, super) + _ -> @fail(st, "invalid type signature") +end + +# normal, non-lhs curly may have implicit `(<: t)` +vst1_curly_typevar(vcx, st) = @stm st begin + [K"<:" t] -> vst1_splat_or_val(vcx, t) + [K">:" t] -> vst1_splat_or_val(vcx, t) + _ -> vst1_splat_or_val(vcx, st) +end + +# assignment should never be allowed, but flisp fails to check inside blocks or +# after anything that isn't a field. See #62075. +function _struct_noassign(vcx, body) + for st in body + if kind(st) === K"=" && vst1_struct_field(vcx, st[1]).ok + return @fail(st, "assignment syntax in structure fields is reserved") + elseif !vst1_struct_field(vcx, st).ok + return pass() + end + end + return pass() +end + +vst1_struct_arg(vcx, st) = @stm st begin + [K"block" xs...] -> all(vst1_struct_arg, vcx, xs) + _ -> vst1_struct_field(vcx, st) | vst1(vcx, st) +end + +vst1_struct_field(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"::" x t] -> vst1_struct_field(vcx, x) & vst1(vcx, t) + [K"const" x] -> vst1_struct_field(vcx, x) + [K"atomic" x] -> vst1_struct_field(vcx, x) + _ -> unknown() +end + +# Messy: expr uses a different head for every op `(a op= b)` and `(a .op= b)`. +# RawGreenNode uses K"op=" and K".op=" with an extra argument specifying `op`. +# The tree we're matching stays one-to-one with Expr by using `K"unknown_head"`. +# +# Note simple `op` and `.op` are calls to (dotted) identifiers, so this special +# handling isn't necessary. +vst1_dotted_or_op_assign(vcx, st) = let op_s = est_syntax_name(st, "") + @stm st begin + [K".=" l r] -> vst1_dotassign_lhs(vcx, l) & vst1(vcx, r) + (_, when=(!Base.isoperator(op_s))) -> unknown() + (_, when=(isempty(op_s) || op_s[end] !== '=')) -> + unknown() + ([K"unknown_head" l r], when=((op_s::String)[1] === '.')) -> + vst1_dotassign_lhs(vcx, l) & vst1(vcx, r) + ([K"unknown_head" l r]) -> + vst1_assign_lhs(vcx, l) & vst1(vcx, r) + _ -> unknown() + end +end + +vst1_assign(vcx, st; in_const = false) = @stm st begin + # This case handles a proper function declaration (= (call ...) ...) form. + # `vst1_assign_lhs_nontuple` also accepts call forms, but that is a lowering + # bug where the "function body" is evaluated immediately + ([K"=" l r], when=is_eventually_call(l)) -> vst1_function(vcx, st) + [K"=" l r] -> vst1_assign_lhs(vcx, l; in_const) & vst1(vcx, r) + [K"=" _...] -> @fail(st, "malformed assignment") + _ -> unknown() +end + +# TODO: We could do some destructuring checks here (e.g. fail `(a,b,c) = (1,2)`) +# +# syntax TODO: +# - call (only within a tuple using JuliaSyntax) can declare a function with +# arguments, but can't use them on the rhs if in a tuple +# - in curly, typevars are checked for structure, but not used. +# - (local/global (= lhs rhs)) forms should probably reject the same +# lhss as const (ref and .) +vst1_assign_lhs(vcx, st; in_const=false, in_tuple=false) = @stm st begin + [K"tuple" [K"parameters" xs...]] -> all(vst1_symdecl, vcx, xs) + [K"tuple" xs...] -> + all(vst1_assign_lhs, vcx, xs; in_const, in_tuple=true) & + (count(kind(x)===K"..." for x in xs) <= 1 ? pass() : + @fail(st, "multiple `...` in destructuring assignment are ambiguous")) + # type-annotated tuple segfaults, haha + # [K"::" [K"tuple" _...] t] -> ??? + [K"..." x] -> !in_tuple ? + @fail(st, "splat on left side of assignment must be in a tuple") : + vst1_assign_lhs_nontuple(vcx, x; in_const) + ([K"parameters" _...], when=in_tuple) -> @fail(st, """ + property destructuring must use a single `;` before the property \ + names, e.g. `(; a, b) = rhs`""") + _ -> vst1_assign_lhs_nontuple(vcx, st; in_const) +end +vst1_assign_lhs_nontuple(vcx, st; in_const=false, in_tuple=false) = @stm st begin + [K"ssavalue" [K"Value"]] -> in_const ? @fail(st, "cannot declare ssavalue const") : pass() + (_, when=(is_eventually_call(st))) -> + vst1_function_calldecl(vcx, st) + [K"::" x t] -> + vst1_assign_lhs(vcx, x; in_const, in_tuple) & vst1(vcx, t) + [K"." x y] -> + in_const ? @fail(st, "cannot declare this form constant") : + kind(y) === K"tuple" ? @fail(st, "dotcall syntax not valid here") : + vst1(vcx, x) & vst1(vcx, y) + [K"ref" x is...] -> + in_const ? @fail(st, "cannot declare this form constant") : + vst1(vcx, x) & all(vst1_call_arg, vcx, is) + [K"curly" x tvs...] -> + vst1_ident(vcx, x; lhs=true) & all(vst1_typevar_decl, vcx, tvs) + + [K"typed_hcat" _...] -> + @fail(st, "invalid spacing in left side of indexed assignment") + [K"typed_vcat" _...] -> + @fail(st, "unexpected `;` in left side of indexed assignment") + [K"typed_ncat" _...] -> + @fail(st, "unexpected `;` in left side of indexed assignment") + (_, when=(kind(st) in KSet"vect hcat vcat ncat")) -> + @fail(st, "use `(a, b) = ...` to assign multiple values") + _ -> @fail(st, "invalid syntax in left-hand side of assignment") | + vst1_ident(vcx, st; lhs=true) +end + +vst1_dotassign_lhs(vcx, st) = vst1_assign_lhs(vcx, st) | vst1(vcx, st) + +# TODO: more validation is possible here, e.g. when row/nrow can show up in ncat +vst1_arraylike(vcx, st) = @stm st begin + [K"vect" xs...] -> + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"hcat" xs...] -> + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"vcat" xs...] -> + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"ncat" [K"Value"] xs...] -> + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"ref" x is...] -> vst1(vcx, x) & all(vst1_call_arg, vcx, is) + [K"row" xs...] -> + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"nrow" [K"Value"] xs...] -> + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"typed_hcat" t xs...] -> vst1(vcx, t) & + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"typed_vcat" t xs...] -> vst1(vcx, t) & + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + [K"typed_ncat" t xs...] -> vst1(vcx, t) & + no_assignment(xs, "array expression") & all(vst1_splat_or_val, vcx, xs) + _ -> unknown() +end + +function no_assignment(sl, hint="this expression") + for st in sl + if kind(st) === K"=" + return @fail(st, string( + "assignment is not allowed in ", hint)) + end + end + return pass() +end + +# If there is both a min and a max, prefer a finite number of match cases +function minlen(err_st::SyntaxTree, sl, n::Int) + length(sl) >= n ? pass() : + @fail(err_st, string( + "expected at least ", n, " argument", (n === 1 ? "" : "s"))) +end +function maxlen(err_st::SyntaxTree, sl, n::Int) + length(sl) <= n ? pass() : + @fail(err_st, string( + "expected at most ", n, " argument", (n === 1 ? "" : "s"))) +end + +vst1_splat_or_val(vcx, st) = @stm st begin + [K"..." x] -> vst1_splat_or_val(vcx, x) + [K"..." _...] -> @fail(st, "expected one argument to `...`") + _ -> vst1(vcx, st) +end + +vst1_generator(vcx, st) = let + vcx = with(vcx; return_ok=false, toplevel=false, in_gscope=false) + @stm st begin + [K"generator" _] -> @fail(st, "`generator` requires >=2 args") + [K"generator" val [K"filter" cond is...]] -> + vst1(vcx, val) & + vst1(vcx, cond) & + all(vst1_iter, vcx, is) + [K"generator" val is...] -> + vst1(with(vcx; readable_underscore=true), val) & + all(vst1_iter, vcx, is) + [K"generator" _...] -> @fail(st, "malformed `generator`") + _ -> @fail(st, "expected `generator`") + end +end + +vst1_iter(vcx, st) = @stm st begin + [K"=" [K"outer" i] v] -> vst1_assign_lhs(vcx, i) & vst1(vcx, v) + # rare, malformed, happens to work in desugaring + [K"=" i [K"..." v]] -> vst1_assign_lhs(vcx, i) & vst1(vcx, v) + [K"=" i v] -> vst1_assign_lhs(vcx, i) & vst1(vcx, v) + _ -> @fail(st, "expected one of `=`, `in`, `∈`") +end + +vst1_raw_lambda(vcx, st) = @stm st begin + [K"lambda" [K"Value"] body] -> let args = st[1].value + (args isa Vector && all(a->a isa Symbol, args) ? pass() : + @fail(st[1], "expected Vector of Symbol")) & + vst1(with(vcx; return_ok=true, toplevel=false, in_gscope=false), body) + end + [K"lambda" _...] -> @fail(st, "malformed `lambda`") + _ -> @fail(st, "expected `lambda`") +end + +#------------------------------------------------------------------------------- +# Pre-macro-expansion (st0) is mostly a subset of st1, except with `macrocall` +# and `quote`. + +""" +Assumes `st` is parsed from surface syntax, and not a partially-expanded tree. +""" +function valid_st0(st::SyntaxTree) + DEBUG && assert_syntaxtree(st) + vr = vst1(with(Validation1Context(), unexpanded=true), st) + # hack: A macrocall can show up almost anywhere, so filter errors pointing + # at macrocalls instead of adding cases to every function above. + isnothing(vr.errors) && return vr.ok + vr2_errors = filter(vr.errors) do err + isempty(err.sts) || !(kind(err.sts[1]) === K"macrocall") + end + vr2 = ValidationResult(isempty(vr2_errors), vr2_errors) + return vr2.ok +end + +vst0(_, st) = vst1(with(Validation1Context(), unexpanded=true), st) + +""" +TODO: While we can't validate any arguments to a macrocall in general, it would +make sense to check usage for things like @ccall and @doc. +""" +vst0_macrocall(vcx, st) = @stm st begin + (_, when=!vcx.unexpanded) -> + @fail(st, "macrocall not valid in AST after macro expansion") + ([K"macrocall" name [K"Value"] args...], + when=(typeof(st[2].value) in (LineNumberNode, MacroSource))) -> + pass() + [K"macrocall" _...] -> + @fail(st, "expected (macrocall name linenode args...)") + _ -> @fail(st, "invalid macrocall syntax") +end + +vst0_quoted(vcx, st; quote_level) = @stm st begin + ([K"$" x], when=quote_level===1) -> + vst1_splat_or_val(vcx, x) + [K"$" x] -> + vst0_quoted(vcx, x; quote_level=quote_level-1) + [K"quote" x] -> + vst0_quoted(vcx, x; quote_level=quote_level+1) + _ -> all(vst0_quoted, vcx, children(st); quote_level) +end + +#------------------------------------------------------------------------------- +# Tree invariants assumed everywhere, including `show`, so fallback printing +# should be used on failure. (These checks really belong in the type system.) +# Failure should only be possible working on AST-internal functions. + +function assert_syntaxtree(st::SyntaxTree, recursive=true) + vr = recursive ? _assert_syntaxtree(st, SyntaxTree[], pass()) : + _assert_syntaxtree_node(st) + @jl_assert is_known(vr) st + if !vr.ok + msg = string("assert_syntaxtree failed: ", node_string(st), "\n") + for err in vr.errors + msg *= "node: " * node_string(only(err.sts)) * + "\nreason: " * string(err.msgs) + end + throw(error(msg)) + end + nothing +end + +function _assert_syntaxtree_node(st::SyntaxTree) + vr = pass() + # TODO: assert st has context (parser doesn't add any) + if is_leaf(st) + if kind(st) === K"globalref" && st.mod === nothing + vr &= @fail(st, "leaf globalref requires module in .mod") + end + (needs_val, valtype) = @stm st begin + [K"Identifier"] -> (true,String) + [K"core"] -> (true,String) + [K"top"] -> (true,String) + [K"Symbol"] -> (true,String) + [K"globalref"] -> (true,String) + [K"Placeholder"] -> (false, Any) + [K"BindingId"] -> (true,IdTag) + [K"label"] -> (true,Int) + [K"symboliclabel"] -> (true,String) + [K"symbolicgoto"] -> (true,String) + [K"Value"] -> (true,Any) + [K"slot"] -> (true,Int) + [K"static_parameter"] -> (true,Int) + [K"SSAValue"] -> (true,Int) + [K"nothing"] -> (false, Any) + [K"TOMBSTONE"] -> (false, Any) + [K"SourceLocation"] -> (false, Any) + [K"latestworld"] -> (false, Any) + [K"latestworld_if_toplevel"] -> (false, Any) + (_, when=JuliaSyntax.is_literal(st)) -> (true,Any) + (_, when=JuliaSyntax.is_trivia(st)) -> (false, Any) # green tree only + (_, when=JuliaSyntax.is_operator(st)) -> (true,String) # TODO: remove + [K"StrMacroName"] -> (true,String) + [K"CmdMacroName"] -> (true,String) + [K"LambdaBindings"] -> (true,LambdaBindings) + [K"Slots"] -> (true,Vector{Slot}) + [K"VERSION"] -> (true,VersionNumber) + _ -> return vr & @fail(st, "unrecognized leaf kind $(kind(st))") + end + if needs_val + if !(st.value isa valtype) + vr &= @fail(st, "needs value ::"*string(valtype)) + end + end + else + # Note some kinds can show up as non-leaves too (mostly from Expr) + if kind(st) in KSet"""Identifier Value Placeholder BindingId label + Symbol SSAValue nothing TOMBSTONE SourceLocation LambdaBindings Slots""" + vr &= @fail(st, "Found leaf-only kind with children") + end + if kind(st) === K"unknown_head" + if !(st.value isa String) + vr &= @fail(st, string("needs value ::String")) + end + end + end + vr +end + +function _assert_syntaxtree(st::SyntaxTree, parents::Vector{SyntaxTree}, vr) + if st in parents + err = "cycle detected: [" + for p in parents + err *= "\n" * node_string(p) + end + return vr & @fail(st, err*"]") + end + vr &= _assert_syntaxtree_node(st) + # TODO: Proper traversal along .source and macro prov (need to cache results + # to avoid exponential repeated lookups, and figure out how these edges may + # form cycles with child edges) + st.source === st && (vr &= @fail(st, ".source equal to self ID")) + sc = st.context + sc isa SyntaxContext && + sc.unexpanded === st && (vr &= @fail(st, "unexpanded equal to self")) + + push!(parents, st) + is_leaf(st) || for c in children(st) + vr &= _assert_syntaxtree(c, parents, vr) + end + pop!(parents) + vr +end + +#------------------------------------------------------------------------------- +# The post-desugaring tree "st2". Failure shouldn't be reachable by user code; +# this is just for internal documentation and debugging purposes. + +Base.@kwdef struct Validation2Context <: ValidationContext + in_method_defs::Bool=false +end + +function with(vcx::Validation2Context; + in_method_defs = vcx.in_method_defs) + Validation2Context(in_method_defs) +end + +function valid_st2(st::SyntaxTree) + assert_syntaxtree(st) + vr = vst2(Validation2Context(), st) + @jl_assert is_known(vr) st + return vr +end + +vst2(vcx::Validation2Context, st::SyntaxTree) = @stm st begin + (_, when=is_leaf(st)) -> kind(st) in KSet""" + Identifier BindingId Placeholder nothing static_parameter + Bool Char Float Float32 BinInt OctInt HexInt Integer + SourceLocation String Symbol Value core top + latestworld latestworld_if_toplevel symbolicgoto symboliclabel TOMBSTONE + """ ? pass() : @fail(st, "unrecognized leaf kind $(kind(st))") + + [K"call" [K"static_eval" cg] xs...] -> est_syntax_name(cg, "") === "cglobal" ? + all(vst2, vcx, xs) : @fail(st, "expected (call (static_eval cglobal) _...)") + [K"call" xs...] -> all(vst2, vcx, xs) + [K"block" xs...] -> all(vst2, vcx, xs) + [K"scope_block" [K"neutral_scope"] xs...] -> all(vst2, vcx, xs) + [K"scope_block" [K"hard_scope"] xs...] -> all(vst2, vcx, xs) + [K"=" l r] -> vst2_ident_lhs(vcx, l) & vst2(vcx, r) + [K"assign_or_constdecl_if_global" l r] -> vst2_ident_lhs(vcx, l) & vst2(vcx, r) + [K"global_if_global" x] -> vst2_ident_lhs(vcx, x) + [K"constdecl" l] -> vst2_ident_lhs(vcx, l) + [K"constdecl" l r] -> vst2_ident_lhs(vcx, l) & vst2(vcx, r) + [K"global" x] -> vst2_ident_lhs(vcx, x) + [K"local" x] -> vst2_ident_lhs(vcx, x) + [K"decl" x t] -> vst2_ident(vcx, x) & vst2(vcx, t) + [K"if" cond t] -> vst2(vcx, cond) & vst2(vcx, t) + [K"if" cond t f] -> vst2(vcx, cond) & vst2(vcx, t) & vst2(vcx, f) + [K"elseif" cond t] -> vst2(vcx, cond) & vst2(vcx, t) + [K"elseif" cond t f] -> vst2(vcx, cond) & vst2(vcx, t) & vst2(vcx, f) + [K"&&" xs...] -> all(vst2, vcx, xs) + [K"||" xs...] -> all(vst2, vcx, xs) + [K"symbolicblock" [K"symboliclabel"] body] -> vst2(vcx, body) + [K"break" [K"symboliclabel"]] -> pass() + [K"break" [K"symboliclabel"] x] -> vst2(vcx, x) + [K"return" x] -> vst2(vcx, x) + [K"trycatchelse" t c] -> vst2(vcx, t) & vst2(vcx, c) + [K"trycatchelse" t c e] -> vst2(vcx, t) & vst2(vcx, c) & vst2(vcx, e) + [K"tryfinally" t f] -> vst2(vcx, t) & vst2(vcx, f) + [K"tryfinally" t f scope] -> vst2(vcx, t) & vst2(vcx, f) & vst2(vcx, scope) + [K"_opaque_closure" id argt lb ub partial nargs isva src lam] -> + vst2_ident(vcx, id) & + all(vst2, vcx, children(st)[2:end-1]) & + vst2_lam(vcx, lam) + [K"_do_while" body cond] -> vst2(vcx, body) & vst2(vcx, cond) + [K"_while" cond body] -> vst2(vcx, cond) & vst2(vcx, body) + [K"inert" _] -> pass() + [K"syntaxinert" _] -> pass() + [K"lambda" _...] -> vst2_lam(vcx, st) + [K"function_decl" x] -> vst2_ident(vcx, x) + [K"function_type" x] -> vst2(vcx, x) + [K"method" mtable argtypes lam] -> !vcx.in_method_defs ? + @fail(st, "method outside of method_defs") : + (kind(mtable) === K"nothing" ? pass() : vst2(vcx, mtable)) & + vst2(vcx, argtypes) & vst2_lam(vcx, lam) + [K"method_defs" id [K"block" sps...] body] -> + (kind(id) === K"nothing" ? pass() : vst2_ident_val(vcx, id)) & + all(vst2_typevar, vcx, sps) & vst2(with(vcx; in_method_defs=true), body) + [K"no_method_defs" id] -> vst2_ident_val(vcx, id) + [K"new" t args...] -> vst2(vcx, t) & all(vst2, vcx, args) + [K"splatnew" t arg] -> vst2(vcx, t) & vst2(vcx, arg) + [K"softscope"] -> pass() + [K"softscope" _] -> pass() + [K"thisfunction"] -> pass() + [K"gc_preserve_begin" xs...] -> all(vst2_ident, vcx, xs) + [K"gc_preserve_end" xs...] -> minlen(st, xs, 1) & all(vst2_ident, vcx, xs) + + [K"meta" xs...] -> all(vst2, vcx, xs) # TODO + [K"loopinfo" xs...] -> all(vst2, vcx, xs) # TODO + [K"boundscheck"] -> pass() + [K"inbounds_pop"] -> pass() + ([K"inbounds" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + ([K"inline" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + ([K"noinline" [K"Value"]], when=(st[1].value isa Bool)) -> pass() + [K"purity"] -> pass() + [K"purity" _ _...] -> numchildren(st) == fieldcount(Base.EffectsOverride) ? + pass() : @fail(st, "wrong number of args to `purity` expression") + [K"aliasscope"] -> pass() + [K"popaliasscope"] -> pass() + + [K"always_defined" x] -> vst2_ident(vcx, x) + [K"assert" [K"Symbol"] x] -> vst2(vcx, x) + [K"removable" x] -> vst2(vcx, x) + [K"relayered_global" [K"Identifier"]] -> pass() + + # Could be made stricter + [K"foreigncall" _ [K"static_eval" rt] [K"static_eval" at] cconv roots_args...] -> + vst2(vcx, rt) & + vst2(vcx, at) & + vst2(vcx, cconv) & + all(vst2, vcx, roots_args) + [K"foreignglobal" _] -> pass() + [K"cfunction" [K"Value"] [K"static_eval" fptr] [K"static_eval" rt] [K"static_eval" at] [K"Symbol"]] -> + vst2(vcx, fptr) & vst2(vcx, rt) & vst2(vcx, at) + [K"cfunction" [K"Value"] fptr [K"static_eval" rt] [K"static_eval" at] [K"Symbol"]] -> + vst2(vcx, fptr) & vst2(vcx, rt) & vst2(vcx, at) + + [K"isdefined" x] -> vst2_ident_val(vcx, x) + [K"isglobal" [K"Placeholder"]] -> pass() + [K"islocal" [K"Placeholder"]] -> pass() + [K"isglobal" x] -> vst2_ident_val(vcx, x) + [K"islocal" x] -> vst2_ident_val(vcx, x) + [K"locals"] -> pass() + _ -> @fail(st, "unrecognized form out of desugaring") +end + +vst2_ident_lhs(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"BindingId"] -> pass() + [K"Placeholder"] -> pass() + _ -> @fail(st, "expected identifier (lhs)") +end + +vst2_ident(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"BindingId"] -> pass() + _ -> @fail(st, "expected identifier or BindingId") +end + +vst2_ident_val(vcx, st) = @stm st begin + [K"Identifier"] -> pass() + [K"BindingId"] -> pass() + [K"core"] -> pass() + [K"top"] -> pass() + [K"thisfunction"] -> pass() + [K"static_parameter"] -> pass() + _ -> @fail(st, "expected identifier (val)") +end + +vst2_lam(vcx, st) = @stm st begin + [K"lambda" [K"block" args...] [K"block" sps...] body] -> + all(vst2_ident_lhs, vcx, args) & + all(vst2_ident_lhs, vcx, sps) & + vst2(vcx, body) + [K"lambda" [K"block" args...] [K"block" sps...] body rett] -> + all(vst2_ident_lhs, vcx, args) & + all(vst2_ident_lhs, vcx, sps) & + vst2(vcx, body) & + vst2(vcx, rett) + _ -> @fail(st, "malformed lambda") +end + +vst2_typevar(vcx, st) = @stm st begin + [K"typevar" tv val] -> vst2_ident_lhs(vcx, tv) & vst2(vcx, val) + _ -> @fail(st, "malformed sparam") +end diff --git a/JuliaLowering/test/arrays.jl b/JuliaLowering/test/arrays.jl new file mode 100644 index 0000000000000..22073c9c5c0a1 --- /dev/null +++ b/JuliaLowering/test/arrays.jl @@ -0,0 +1,158 @@ +using Test, JuliaLowering + +test_mod = Module() + +# Test that two array element types are equal and that they are also equal +# elementwise +function ≅(a, b) + eltype(a) == eltype(b) && a == b +end + +# vect +@test JuliaLowering.include_string(test_mod, """ +[1,2,3] +""") ≅ [1,2,3] + +# hcat +@test JuliaLowering.include_string(test_mod, """ +[1 2 3] +""") ≅ [1 2 3] + +# typed_hcat +@test JuliaLowering.include_string(test_mod, """ +Int[1.0 2.0 3.0] +""") ≅ [1 2 3] + +# splat with vect/hcat/typed_hcat +@test JuliaLowering.include_string(test_mod, """ +let xs = [1,2,3] + [0, xs...] +end +""") ≅ [0,1,2,3] +@test JuliaLowering.include_string(test_mod, """ +let xs = [1,2,3] + [0 xs...] +end +""") ≅ [0 1 2 3] +@test JuliaLowering.include_string(test_mod, """ +let xs = [1,2,3] + Int[0 xs...] +end +""") ≅ Int[0 1 2 3] + +# vcat +@test JuliaLowering.include_string(test_mod, """ +[1;2;3] +""") ≅ [1; 2; 3] + +@test JuliaLowering.include_string(test_mod, """ +let + xs = (1,2) + [xs...; xs...] +end +""") ≅ [1,2,1,2] + +# hvcat +@test JuliaLowering.include_string(test_mod, """ +[1 2 3; 4 5 6] +""") ≅ [1 2 3; + 4 5 6] + +# hvcat_rows +@test JuliaLowering.include_string(test_mod, """ +let + xs = (1,2) + [xs... 3; 4 xs...] +end +""") ≅ [1 2 3; + 4 1 2] + +# typed_vcat +@test JuliaLowering.include_string(test_mod, """ +Int[1.0; 2.0; 3.0] +""") ≅ [1; 2; 3] + +# typed_hvcat +@test JuliaLowering.include_string(test_mod, """ +Int[1.0 2.0 3.0; 4.0 5.0 6.0] +""") ≅ [1 2 3; + 4 5 6] + +# typed_hvcat_rows +@test JuliaLowering.include_string(test_mod, """ +let + xs = (1.0,2.0) + Int[xs... 3; 4 xs...] +end +""") ≅ [1 2 3; + 4 1 2] + +# ncat with a single dimension +@test JuliaLowering.include_string(test_mod, """ +[1 ;;; 2 ;;; 3] +""") ≅ [1 ;;; 2 ;;; 3] + +@test JuliaLowering.include_string(test_mod, """ +Int[1.0 ;;; 2.0 ;;; 3.0] +""") ≅ [1 ;;; 2 ;;; 3] + +# Lowering of ref to setindex +@test JuliaLowering.include_string(test_mod, """ +let + as = [0,0,0,0] + as[begin] = 1 + as[2] = 2 + as[end] = 4 + as +end +""") == [1, 2, 0, 4] + +@test JuliaLowering.include_string(test_mod, """ +let + as = zeros(Int, 2,3) + as[begin, end] = 1 + as[end, begin] = 2 + js = (2,) + as[js..., end] = 3 + as +end +""") == [0 0 1; + 2 0 3] + +# getindex +@test JuliaLowering.include_string(test_mod, """ +let + x = [1 2; + 3 4] + (x[end,begin], x[begin,end]) +end +""") == (3, 2) + +# getindex with splats +@test JuliaLowering.include_string(test_mod, """ +let + x = [1 2; + 3 4 + ;;; + 5 6; + 7 8] + inds = (2,1) + ind1 = (1,) + (x[inds..., begin], x[inds..., end], x[1, inds...], + x[ind1..., ind1..., end]) +end +""") == (3, 7, 2, 5) + +# begin/end not replaced in some cases +JuliaLowering.include_string(test_mod, "f(args...;kws...) = 2") +@test JuliaLowering.include_string(test_mod, """ + [7,8,9][f(;var"end"=123, var"begin"=456)] +""") === 8 +@test JuliaLowering.include_string(test_mod, """ + [7,8,9][f(quote var"end" end)] +""") === 8 +@test JuliaLowering.include_string(test_mod, """ +let var"end" = [1,2,3], y = [7,8,9] + y[var"end"[var"end"]] +end +""") === 9 diff --git a/JuliaLowering/test/arrays_ir.jl b/JuliaLowering/test/arrays_ir.jl new file mode 100644 index 0000000000000..0d5ebaacd0dd0 --- /dev/null +++ b/JuliaLowering/test/arrays_ir.jl @@ -0,0 +1,510 @@ +######################################## +# vect syntax +[10, 20, 30] +#--------------------- +1 (call top.vect 10 20 30) +2 (return %₁) + +######################################## +# vect with splat +[x, xs...] +#--------------------- +1 TestMod.x +2 (call core.tuple %₁) +3 TestMod.xs +4 (call core._apply_iterate top.iterate top.vect %₂ %₃) +5 (return %₄) + +######################################## +# vect with splats +[x, xs..., y, ys...] +#--------------------- +1 TestMod.x +2 (call core.tuple %₁) +3 TestMod.xs +4 TestMod.y +5 (call core.tuple %₄) +6 TestMod.ys +7 (call core._apply_iterate top.iterate top.vect %₂ %₃ %₅ %₆) +8 (return %₇) + +######################################## +# Error: vect syntax with parameters +[10, 20; 30] +#--------------------- +LoweringError: +[10, 20; 30] +# └──┘ ── unexpected semicolon + +######################################## +# Error: vect syntax with embedded assignments +[a=20, 30] +#--------------------- +LoweringError: +[a=20, 30] +#└──┘ ── assignment is not allowed in array expression + +######################################## +# hcat syntax +[10 20 30] +#--------------------- +1 (call top.hcat 10 20 30) +2 (return %₁) + +######################################## +# hcat with splat +[x xs...] +#--------------------- +1 TestMod.x +2 (call core.tuple %₁) +3 TestMod.xs +4 (call core._apply_iterate top.iterate top.hcat %₂ %₃) +5 (return %₄) + +######################################## +# typed hcat syntax +T[10 20 30] +#--------------------- +1 TestMod.T +2 (call top.typed_hcat %₁ 10 20 30) +3 (return %₂) + +######################################## +# typed hcat syntax with splat +T[x xs...] +#--------------------- +1 TestMod.T +2 TestMod.x +3 (call core.tuple %₁ %₂) +4 TestMod.xs +5 (call core._apply_iterate top.iterate top.typed_hcat %₃ %₄) +6 (return %₅) + +######################################## +# Error: hcat syntax with embedded assignments +[10 20 a=40] +#--------------------- +LoweringError: +[10 20 a=40] +# └───┘ ── assignment is not allowed in array expression + +######################################## +# vcat syntax +[10; 20; 30] +#--------------------- +1 (call top.vcat 10 20 30) +2 (return %₁) + +######################################## +# vcat with splats +[a...; 20; 30] +#--------------------- +1 TestMod.a +2 (call core.tuple 20 30) +3 (call core._apply_iterate top.iterate top.vcat %₁ %₂) +4 (return %₃) + +######################################## +# hvcat syntax +[10; 20 30; 40 e f] +#--------------------- +1 (call core.tuple 1 2 3) +2 TestMod.e +3 TestMod.f +4 (call top.hvcat %₁ 10 20 30 40 %₂ %₃) +5 (return %₄) + +######################################## +# hvcat with splats nested within rows +[10; 20 a...] +#--------------------- +1 (call core.tuple 10) +2 (call core.tuple 20) +3 TestMod.a +4 (call core._apply_iterate top.iterate core.tuple %₂ %₃) +5 (call top.hvcat_rows %₁ %₄) +6 (return %₅) + +######################################## +# Error: vcat syntax with assignments +[a=20; 30] +#--------------------- +LoweringError: +[a=20; 30] +#└──┘ ── assignment is not allowed in array expression + +######################################## +# typed_vcat syntax +T[10; 20; 30] +#--------------------- +1 TestMod.T +2 (call top.typed_vcat %₁ 10 20 30) +3 (return %₂) + +######################################## +# typed_hvcat syntax +T[10; 20 30; 40 50 60] +#--------------------- +1 TestMod.T +2 (call core.tuple 1 2 3) +3 (call top.typed_hvcat %₁ %₂ 10 20 30 40 50 60) +4 (return %₃) + +######################################## +# typed_hvcat with splats nested within rows +T[10; 20 a...] +#--------------------- +1 TestMod.T +2 (call core.tuple 10) +3 (call core.tuple 20) +4 TestMod.a +5 (call core._apply_iterate top.iterate core.tuple %₃ %₄) +6 (call top.typed_hvcat_rows %₁ %₂ %₅) +7 (return %₆) + +######################################## +# ncat with a single dimension +[10 ;;; 20 ;;; 30] +#--------------------- +1 (call top.hvncat 3 10 20 30) +2 (return %₁) + +######################################## +# typed_ncat with a single dimension +T[10 ;;; 20 ;;; 30] +#--------------------- +1 TestMod.T +2 (call top.typed_hvncat %₁ 3 10 20 30) +3 (return %₂) + +######################################## +# ncat with balanced column major element layout +[10 ; 20 ; 30 ;;; 40 ; 50 ; 60] +#--------------------- +1 (call core.tuple 3 1 2) +2 (call top.hvncat %₁ false 10 20 30 40 50 60) +3 (return %₂) + +######################################## +# typed multidimensional ncat +T[10 ; 20 ; 30 ;;; 40 ; 50 ; 60] +#--------------------- +1 TestMod.T +2 (call core.tuple 3 1 2) +3 (call top.typed_hvncat %₁ %₂ false 10 20 30 40 50 60) +4 (return %₃) + +######################################## +# ncat with balanced row major element layout +[10 20 30 ; 40 50 60 ;;;] +#--------------------- +1 (call core.tuple 2 3 1) +2 (call top.hvncat %₁ true 10 20 30 40 50 60) +3 (return %₂) + +######################################## +# ncat of 3D array with balanced layout +[10 ; 20 ;; 30 ; 40 ;;; 50 ; 60 ;; 70 ; 80] +#--------------------- +1 (call core.tuple 2 2 2) +2 (call top.hvncat %₁ false 10 20 30 40 50 60 70 80) +3 (return %₂) + +######################################## +# ncat with unbalanced column major layout +[10 ; 20 ;; 30 ;;; 40 ;;;;] +#--------------------- +1 (call core.tuple 2 1 1) +2 (call core.tuple 3 1) +3 (call core.tuple 4) +4 (call core.tuple 4) +5 (call core.tuple %₁ %₂ %₃ %₄) +6 (call top.hvncat %₅ false 10 20 30 40) +7 (return %₆) + +######################################## +# ncat with unbalanced row major layout +[10 20 ; 30 40 ; 50 60 ;;; 70 ;;; 80 ;;;;] +#--------------------- +1 (call core.tuple 2 2 2 1 1) +2 (call core.tuple 6 1 1) +3 (call core.tuple 8) +4 (call core.tuple 8) +5 (call core.tuple %₁ %₂ %₃ %₄) +6 (call top.hvncat %₅ true 10 20 30 40 50 60 70 80) +7 (return %₆) + +######################################## +# Splatting with 1D ncat +[xs ;;; ys... ;;; zs] +#--------------------- +1 TestMod.xs +2 (call core.tuple 3 %₁) +3 TestMod.ys +4 TestMod.zs +5 (call core.tuple %₄) +6 (call core._apply_iterate top.iterate top.hvncat %₂ %₃ %₅) +7 (return %₆) + +######################################## +# Error: splatting with multi-dimensional ncat +[xs ; ys ;;; zs...] +#--------------------- +LoweringError: +[xs ; ys ;;; zs...] +# └───┘ ── Splatting ... in an `ncat` with multiple dimensions is not supported + +######################################## +# Error: bad nrow nesting +@ast_ [K"ncat" + 3::K"Value" + [K"nrow" + 1::K"Value" + [K"nrow" + 1::K"Integer" + 1::K"Integer" + ] + ] +] +#--------------------- +LoweringError: +#= line 1 =# - invalid syntax: unknown form `nrow` or number of arguments 2 +Expression: + (nrow 1 1) + +######################################## +# Error: bad nrow nesting +@ast_ [K"ncat" + 3::K"Value" + [K"nrow" + 2::K"Value" + [K"row" + 1::K"Integer" + ] + ] +] +#--------------------- +LoweringError: +#= line 1 =# - 2D `nrow` cannot be mixed with `row` in `ncat` +Expression: + (nrow 2 (row 1)) + +######################################## +# Error: bad nrow nesting +@ast_ [K"ncat" + 3::K"Value" + [K"row" + [K"row" + 1::K"Integer" + ] + ] +] +#--------------------- +LoweringError: +#= line 1 =# - Badly nested rows in `ncat` +Expression: + (row 1) + +######################################## +# Simple getindex +a[i] +#--------------------- +1 TestMod.a +2 TestMod.i +3 (call top.getindex %₁ %₂) +4 (return %₃) + +######################################## +# simple 1D getindex with begin +a[begin] +#--------------------- +1 TestMod.a +2 (call top.firstindex %₁) +3 (call top.getindex %₁ %₂) +4 (return %₃) + +######################################## +# simple 1D getindex with end +a[end] +#--------------------- +1 TestMod.a +2 (call top.lastindex %₁) +3 (call top.getindex %₁ %₂) +4 (return %₃) + +######################################## +# multidimensional getindex with begin +a[i, begin] +#--------------------- +1 TestMod.a +2 TestMod.i +3 (call top.firstindex %₁ 2) +4 (call top.getindex %₁ %₂ %₃) +5 (return %₄) + +######################################## +# multidimensional getindex with end +a[i, end] +#--------------------- +1 TestMod.a +2 TestMod.i +3 (call top.lastindex %₁ 2) +4 (call top.getindex %₁ %₂ %₃) +5 (return %₄) + +######################################## +# multidimensional getindex with begin/end and splats +a[is..., end, js..., begin] +#--------------------- +1 TestMod.a +2 TestMod.is +3 (call top.length %₂) +4 (call top.+ 1 %₃) +5 (call top.lastindex %₁ %₄) +6 TestMod.js +7 (call top.length %₂) +8 (call top.length %₆) +9 (call top.+ 2 %₇ %₈) +10 (call top.firstindex %₁ %₉) +11 (call core.tuple %₁) +12 (call core.tuple %₅) +13 (call core.tuple %₁₀) +14 (call core._apply_iterate top.iterate top.getindex %₁₁ %₂ %₁₂ %₆ %₁₃) +15 (return %₁₄) + +######################################## +# getindex with nontrivial array expression and begin/end +f()[end] +#--------------------- +1 TestMod.f +2 (call %₁) +3 (call top.lastindex %₂) +4 (call top.getindex %₂ %₃) +5 (return %₄) + +######################################## +# nested refs with getindex and begin/end +b[a[begin, end], begin, end] +#--------------------- +1 TestMod.b +2 TestMod.a +3 (call top.firstindex %₂ 1) +4 (call top.lastindex %₂ 2) +5 (call top.getindex %₂ %₃ %₄) +6 (call top.firstindex %₁ 2) +7 (call top.lastindex %₁ 3) +8 (call top.getindex %₁ %₅ %₆ %₇) +9 (return %₈) + +######################################## +# Error: parameters in array ref +a[i, j; w=1] +#--------------------- +LoweringError: +a[i, j; w=1] +# └───┘ ── unexpected semicolon + +######################################## +# simple setindex! +a[i] = x +#--------------------- +1 TestMod.x +2 TestMod.a +3 TestMod.i +4 (call top.setindex! %₂ %₁ %₃) +5 (return %₁) + +######################################## +# simple setindex! with begin +a[begin] = x +#--------------------- +1 TestMod.a +2 TestMod.x +3 (call top.firstindex %₁) +4 (call top.setindex! %₁ %₂ %₃) +5 (return %₂) + +######################################## +# simple setindex! with end +a[end] = x +#--------------------- +1 TestMod.a +2 TestMod.x +3 (call top.lastindex %₁) +4 (call top.setindex! %₁ %₂ %₃) +5 (return %₂) + +######################################## +# multidimensional setindex! with begin +a[i, begin] = x +#--------------------- +1 TestMod.a +2 TestMod.x +3 TestMod.i +4 (call top.firstindex %₁ 2) +5 (call top.setindex! %₁ %₂ %₃ %₄) +6 (return %₂) + +######################################## +# multidimensional setindex! with end +a[i, end] = x +#--------------------- +1 TestMod.a +2 TestMod.x +3 TestMod.i +4 (call top.lastindex %₁ 2) +5 (call top.setindex! %₁ %₂ %₃ %₄) +6 (return %₂) + +######################################## +# multidimensional setindex! with begin/end and splats +a[is..., end, js..., begin] = x +#--------------------- +1 TestMod.a +2 TestMod.is +3 (call top.length %₂) +4 (call top.+ 1 %₃) +5 (call top.lastindex %₁ %₄) +6 TestMod.js +7 (call top.length %₂) +8 (call top.length %₆) +9 (call top.+ 2 %₇ %₈) +10 (call top.firstindex %₁ %₉) +11 TestMod.x +12 (call core.tuple %₁ %₁₁) +13 (call core.tuple %₅) +14 (call core.tuple %₁₀) +15 (call core._apply_iterate top.iterate top.setindex! %₁₂ %₂ %₁₃ %₆ %₁₄) +16 (return %₁₁) + +######################################## +# setindex! with nontrivial array expression and begin/end +f()[end] = x +#--------------------- +1 TestMod.f +2 (call %₁) +3 TestMod.x +4 (call top.lastindex %₂) +5 (call top.setindex! %₂ %₃ %₄) +6 (return %₃) + +######################################## +# nested refs +b[a[begin]] = x +#--------------------- +1 TestMod.b +2 TestMod.x +3 TestMod.a +4 (call top.firstindex %₃) +5 (call top.getindex %₃ %₄) +6 (call top.setindex! %₁ %₂ %₅) +7 (return %₂) + +######################################## +# empty ref and setindex! +a[] = rhs +#--------------------- +1 TestMod.rhs +2 TestMod.a +3 (call top.setindex! %₂ %₁) +4 (return %₁) diff --git a/JuliaLowering/test/assignments.jl b/JuliaLowering/test/assignments.jl new file mode 100644 index 0000000000000..a0279538291be --- /dev/null +++ b/JuliaLowering/test/assignments.jl @@ -0,0 +1,613 @@ +test_mod = Module() + +Base.include_string(test_mod, +""" +mutable struct X + a + b +end +""") + +# TODO: Desugaring of assignment done, but needs `where` lowering +JuliaLowering.include_string(test_mod, """ +MyVector{T} = Array{1,T} +""") +@test test_mod.MyVector{Int} == Array{1,Int} + +# Chained assignment +@test JuliaLowering.include_string(test_mod, """ +let + a = b = 42 +end +""") == 42 + +# Assignment in value but not tail position +@test JuliaLowering.include_string(test_mod, """ +let + x = begin + y = 42 + end + x +end +""") == 42 + +@test JuliaLowering.include_string(test_mod, """ +let + x = [] + a = b = (push!(x, 1); 42) + (a,b,x) +end +""") == (42,42,[1]) + +@testset "setproperty" begin + @test JuliaLowering.include_string(test_mod, """ + let + x = X(1,2) + x.a = 10 + (x.a, x.b) + end + """) == (10,2) + + # RHS of the dot is not restricted like getproperty, and can be anything but + # a syntactic tuple (tested as "no assignment to dotcall"). + JuliaLowering.include_string(test_mod, """ + mutable struct AnyDotSetProperty; x; end + global anydotsetproperty = AnyDotSetProperty(1) + function Base.setproperty!(asp::AnyDotSetProperty, y, z) + setfield!(asp, :x, (y, z)) + end + """) + @test jl_eval(test_mod, Expr(:(=), Expr(:., :anydotsetproperty, 1), 2)) == 2 + @test test_mod.anydotsetproperty.x == (1,2) + @test jl_eval(test_mod, + Expr(:(=), Expr(:., :anydotsetproperty, + Expr(:call, :identity, 1)), 2)) == 2 + @test test_mod.anydotsetproperty.x == (1,2) + @test jl_eval(test_mod, + Expr(:(=), Expr(:., :anydotsetproperty, + QuoteNode(Expr(:call, :identity, 1))), 2)) == 2 + @test test_mod.anydotsetproperty.x == (Expr(:call, :identity, 1),2) +end + +# Declarations +@test JuliaLowering.include_string(test_mod, """ +let + x::Int = 1 + x = 10.0 + x +end +""") === 10 + +# Updating assignments +@test JuliaLowering.include_string(test_mod, """ +let x = "hi" + x *= " ho" + x +end +""") == "hi ho" + +@test JuliaLowering.include_string(test_mod, """ +let x = [1,3] + x .-= [0,1] + x +end +""") == [1,2] + +@test JuliaLowering.include_string(test_mod, """ +let x = [1 2; 3 4] + x[begin, 1:end] .-= 1 + x +end +""") == [0 1 ; 3 4] + +@testset "in-place broadcast returns the lhs" begin + mod = @newmod() + JL.include_string(mod, """ + f50794(Y) = Y .+= Any[zeros(size(Y))][1] + g50794(A, x) = A[1, :] .+= x + """) + @test only(Base.return_types(mod.f50794, (Vector{Float64},))) === Vector{Float64} + rt = only(Base.return_types(mod.g50794, (Matrix{Float64}, Any))) + @test rt <: SubArray{Float64, 1, Matrix{Float64}} +end + +# Tuple-destructuring updating assignment `x, y += a, b` +@test JuliaLowering.include_string(@newmod(), """ +struct Vec1; v::Int; end +Base.:+(a::Tuple{Vec1,Vec1}, b::Tuple{Vec1,Vec1}) = (Vec1(a[1].v+b[1].v), Vec1(a[2].v+b[2].v)) +let x = Vec1(1), y = Vec1(2) + x, y += Vec1(3), Vec1(4) + (x.v, y.v) +end +""") == (4, 6) + +@testset "lhs forms" begin + @test JuliaLowering.include_string(test_mod, """ + mutable struct with_mutable_x; x; end + let x1 = 1, x2 = [2], x3 = Ref(3), x4 = with_mutable_x(4) + val = (x1, x2[1], x3[], x4.x) = 10,20,30,40 + (val, x1, x2[1], x3[], x4.x) + end + """) == ((10,20,30,40), 10,20,30,40) + + # TODO: both flisp and JL drop the final conversion + @test JuliaLowering.include_string(test_mod, """ + let x1 = 1, x2 = [2], x3 = Ref(3), x4 = with_mutable_x(4) + val = (x1::Int, x2[1]::Int, x3[]::Int, x4.x::Int) = 10.0,20.0,30.0,40.0 + (val, x1, x2[1], x3[], x4.x) + end + """) == ((10.0,20.0,30.0,40.0), 10,20,30,40.0) + + # add local/global + @test JuliaLowering.include_string(test_mod, """ + let x1 = 1, x2 = [2], x3 = Ref(3), x4 = with_mutable_x(4) + val = local (x1, x2[1], x3[], x4.x) = 10,20,30,40 + (val, x1, x2[1], x3[], x4.x) + end + """) == ((10,20,30,40), 10,20,30,40) + @test JuliaLowering.include_string(@newmod(), """ + mutable struct with_mutable_x; x; end + let x1 = 1, x2 = [2], x3 = Ref(3), x4 = with_mutable_x(4) + val = global (x1::Int, x2[1]::Int, x3[]::Int, x4.x::Int) = 10.0,20.0,30.0,40.0 + (val, x1, x2[1], x3[], x4.x) + end + """) == ((10,20,30,40), 10,20,30,40.0) +end + +@testset "chaining assignments (robot-generated)" begin + @test_throws MethodError JuliaLowering.include_string(test_mod, """ + let vec = [1,2,3] + vec[1] = vec = 0 + end + """) + + # same, but with an identifier rhs (assigned directly, no temporary) + @test JuliaLowering.include_string(test_mod, """ + let x = Any[1,2,3], w = Any[0,0] + x[1] = x = w + (x === w, w[1] === w) + end + """) == (true, true) + + # setproperty! also sees the newly assigned value + @test_throws FieldError JuliaLowering.include_string(test_mod, """ + let x = X(1,2) + x.a = x = 0 + end + """) + + # nesting the other way around assigns the element first + @test JuliaLowering.include_string(test_mod, """ + let x = [1,2,3] + x = x[1] = 2 + x + end + """) === 2 + + # side effects run right to left: innermost rhs, then each lhs in turn + @test JuliaLowering.include_string(test_mod, """ + let order = Symbol[] + x = [0, 0] + obj = X(nothing, nothing) + getarr() = (push!(order, :arr); x) + getidx() = (push!(order, :idx); 2) + getobj() = (push!(order, :obj); obj) + getarr()[getidx()] = getobj().a = v = (push!(order, :rhs); 42) + (order, x[2], obj.a, v) + end + """) == ([:rhs, :obj, :arr, :idx], 42, 42, 42) + + # destructuring of an outer lhs happens after the inner assignment + @test JuliaLowering.include_string(test_mod, """ + let a = 0, b = 0 + (a, b) = a = (1, 2) + (a, b) + end + """) == (1, 2) + + # the value of the chain is the rhs itself; conversions performed by + # setindex!, setproperty! or decls in the chain don't leak into other lhss + @test JuliaLowering.include_string(test_mod, """ + let x = [1.0, 2.0] + a = x[2] = 3 + (a === 3, x[2] === 3.0) + end + """) == (true, true) + @test JuliaLowering.include_string(test_mod, """ + let r = Ref{Float64}(0.0) + a = r.x = 3 + (a === 3, r.x === 3.0) + end + """) == (true, true) + @test JuliaLowering.include_string(test_mod, """ + let + a = b::Int = 1.0 + (a, b) + end + """) === (1.0, 1) + @test JuliaLowering.include_string(test_mod, """ + let + a::Int = b = 1.0 + (a, b) + end + """) === (1, 1.0) + + # destructuring middles yield the unmodified rhs, not a new container + @test JuliaLowering.include_string(test_mod, """ + let t = (1, 2) + a = (b, c) = t + (a === t, b, c) + end + """) == (true, 1, 2) + @test JuliaLowering.include_string(test_mod, """ + let nt = (p = 1, q = 2) + a = (; p) = nt + (a === nt, p) + end + """) == (true, 1) + @test JuliaLowering.include_string(test_mod, """ + let + a = (b, c...) = (1, 2, 3) + (a, b, c) + end + """) == ((1, 2, 3), 1, (2, 3)) + @test JuliaLowering.include_string(test_mod, """ + let a = 0, b = 0, c = 0 + r = (a, b) = (c, d) = (1, 2) + (r, a, b, c, d) + end + """) == ((1, 2), 1, 2, 1, 2) + + # underscores may be assigned anywhere in a chain, but read never + @test JuliaLowering.include_string(test_mod, "let; a = _ = 3; a; end") === 3 + @test JuliaLowering.include_string(test_mod, "let; _ = a = 3; a; end") === 3 + @test JuliaLowering.include_string(test_mod, "let; _ = _ = 5; end") === 5 + @test_throws LoweringError JuliaLowering.include_string(test_mod, "let; a = b = _; end") + + # repeated variable + @test JuliaLowering.include_string(test_mod, "let x = 1; x = x = 2; x; end") === 2 + + # an identifier rhs is read, so must be defined + @test_throws UndefVarError JuliaLowering.include_string(test_mod, """ + let + local p, q, r + p = q = r + end + """) + + # globals and locals can be mixed in one chain, including via a + # value-position `global` declaration + @test JuliaLowering.include_string(test_mod, """ + let + global chain_gmid + lmix = chain_gmid = 8 + (lmix, chain_gmid) + end + """) == (8, 8) + @test JuliaLowering.include_string(test_mod, """ + let + a = global chain_gval = 2 + (a, chain_gval) + end + """) == (2, 2) + + # decl forms wrapping a chain declare (and convert for) only the first lhs + @test JuliaLowering.include_string(test_mod, """ + let + local lt::Int = ltb = 1.0 + (lt, ltb) + end + """) === (1, 1.0) + @test JuliaLowering.include_string(test_mod, """ + global chain_tg::Int = chain_tgb = 2.0 + (chain_tg, chain_tgb) + """) === (2, 2.0) + + # chains stop at `+=` and `.=` + @test JuliaLowering.include_string(test_mod, """ + let c = 1 + a = b = c += 1 + (a, b, c) + end + """) == (2, 2, 2) + @test JuliaLowering.include_string(test_mod, """ + let y = [1, 2] + a = b = y .= 0 + (a === y, b === y) + end + """) == (true, true) + + # ... and at short form function definitions, also when written as `=` + # in pre-parsed ASTs (see "short form function def" in assignments_ir.jl) + @testset let ex = Expr(:block, + Expr(:local, :a), + Expr(:(=), :a, + Expr(:(=), Expr(:call, :chain_f), Expr(:(=), :c, 1))), + Expr(:call, :(===), :a, :chain_f)) + @test fl_eval(test_mod, ex) == true + @test jl_eval(test_mod, ex) == true + end + + # curly (type alias definition) in the middle of a chain + @test JuliaLowering.include_string(test_mod, """ + chain_ga = ChainAlias{T} = Vector + (chain_ga, ChainAlias) + """) == (Vector, Vector) + # but type parameters aren't in scope in the hoisted rhs (same in flisp) + @test_throws UndefVarError JuliaLowering.include_string(test_mod, """ + chain_gb = ChainAlias2{T} = AbstractVector{T} + """) + + # invalid lhss are rejected anywhere in a chain + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + let b = 0, c = 0 + a = [b, c] = (1, 2) + end + """) + + # const applies to the first lhs only + @test JuliaLowering.include_string(test_mod, """ + global chain_arr = [0, 0] + const chain_c1 = chain_arr[1] = 99 + (chain_c1, chain_arr[1]) + """) == (99, 99) + @test JuliaLowering.include_string(test_mod, """ + const chain_ca2 = (chain_cb2, chain_cc2) = (3, 4) + (chain_ca2, chain_cb2, chain_cc2) + """) == ((3, 4), 3, 4) + Core.@latestworld + @test Base.isconst(test_mod, :chain_c1) + @test !Base.isconst(test_mod, :chain_arr) + @test Base.isconst(test_mod, :chain_ca2) + @test !Base.isconst(test_mod, :chain_cb2) + + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + const chain_arr[1] = b = 1 + """) + + # FIXME: flisp accepts non-identifier first lhss in const chains; these + # currently throw an internal lowering error + @test_broken JuliaLowering.include_string(test_mod, """ + const (chain_ca, chain_cb) = chain_cc = (1, 2) + """) == (1, 2) + @test_broken JuliaLowering.include_string(test_mod, """ + const chain_ct::Int = chain_ctb = 1.0 + """) == 1.0 +end # assignment chaining + +# Test that side effects of computing indices in left hand side only occur +# once. +@test JuliaLowering.include_string(test_mod, """ +let + x = [1, 2] + n_calls = 0 + the_index() = (n_calls = n_calls + 1; 1) + x[the_index()] += 1 + x[the_index()]::Int += 1 + x[the_index():end] .+= 1 + n_calls +end +""") == 3 + +# removing argument side effect in kwcall lhs +@eval test_mod f60152(v, pa; kw) = copy(v) +@test JuliaLowering.include_string(test_mod, """ + f60152([1, 2, 3], 0; kw=0) .*= 2 +""") == [2,4,6] +@test JuliaLowering.include_string(test_mod, """ +let + pa_execs = 0 + kw_execs = 0 + out = f60152([1, 2, 3], (pa_execs+=1); kw=(kw_execs+=1)) .*= 2 + (out, pa_execs, kw_execs) +end +""") == ([2,4,6], 1, 1) + +@testset "distinction between `=`` and `kw`" begin + eq = Expr(:(=), :a, 1) + peq = Expr(:parameters, eq) + kw = Expr(:kw, :b, 2) + pkw = Expr(:parameters, kw) + + function outer_ab(ex::Expr) + Expr(:let, Expr(:block, :(a = 0), :(b = 0)), + Expr(:block, ex, + Expr(:tuple, :a, :b))) + end + + @eval test_mod function collect_args(args...; kws...) + (args..., :semicolon, kws...) + end + + @testset "in :call" begin + # call + @testset let ex = Expr(:call, :collect_args, eq) + @test fl_eval(test_mod, ex) == (1, :semicolon) + @test jl_eval(test_mod, ex) == (1, :semicolon) + # `=` in a call assigns the value + @test fl_eval(test_mod, outer_ab(ex)) == (1, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (1, 0) + end + @testset let ex = Expr(:call, :collect_args, peq) + @test fl_eval(test_mod, ex) == (:semicolon, :a=>1) + @test jl_eval(test_mod, ex) == (:semicolon, :a=>1) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + # `kw` always passes a kwarg and does not assign a value + @testset let ex = Expr(:call, :collect_args, kw) + @test fl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test jl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + @testset let ex = Expr(:call, :collect_args, pkw) + @test fl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test jl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + end + + @testset "in dotcall" begin + let eq = Expr(:(=), :a, [1]), + peq = Expr(:parameters, eq), + kw = Expr(:kw, :b, 2), + pkw = Expr(:parameters, kw) + + @testset let ex = Expr(:(.), :collect_args, Expr(:tuple, eq)) + @test fl_eval(test_mod, ex) == [(1, :semicolon)] + @test jl_eval(test_mod, ex) == [(1, :semicolon)] + @test fl_eval(test_mod, outer_ab(ex)) == ([1], 0) + @test jl_eval(test_mod, outer_ab(ex)) == ([1], 0) + end + @testset let ex = Expr(:(.), :collect_args, Expr(:tuple, peq)) + @test fl_eval(test_mod, ex) == (:semicolon, :a=>[1]) + @test jl_eval(test_mod, ex) == (:semicolon, :a=>[1]) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + @testset let ex = Expr(:(.), :collect_args, Expr(:tuple, kw)) + @test fl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test jl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + @testset let ex = Expr(:(.), :collect_args, Expr(:tuple, pkw)) + @test fl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test jl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + end + end + + @testset "in :ref" begin + @eval test_mod struct DummyGetIndex; field; end + @eval test_mod function Base.getindex(s::DummyGetIndex, args...; kws...) + (args..., :semicolon, kws...) + end + @testset let ex = Expr(:ref, test_mod.DummyGetIndex(1), eq) + @test fl_eval(test_mod, ex) == (1, :semicolon) + @test jl_eval(test_mod, ex) == (1, :semicolon) + @test fl_eval(test_mod, outer_ab(ex)) == (1, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (1, 0) + end + @testset let ex = Expr(:ref, test_mod.DummyGetIndex(1), peq) + @test_throws "unexpected semicolon" fl_eval(test_mod, ex) + @test_throws "unexpected semicolon" jl_eval(test_mod, ex) + end + @testset let ex = Expr(:ref, test_mod.DummyGetIndex(1), kw) + @test fl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test jl_eval(test_mod, ex) == (:semicolon, :b=>2) + @test fl_eval(test_mod, outer_ab(ex)) == (0, 0) + @test jl_eval(test_mod, outer_ab(ex)) == (0, 0) + end + @testset let ex = Expr(:ref, test_mod.DummyGetIndex(1), pkw) + @test_throws "unexpected semicolon" fl_eval(test_mod, ex) + @test_throws "unexpected semicolon" jl_eval(test_mod, ex) + end + end + + @testset "in :tuple" begin + @testset let ex = Expr(:tuple, eq) + @test fl_eval(test_mod, ex) == (a=1,) + @test jl_eval(test_mod, ex) == (a=1,) + end + @testset let ex = Expr(:tuple, peq) + @test fl_eval(test_mod, ex) == (a=1,) + @test jl_eval(test_mod, ex) == (a=1,) + end + @testset let ex = Expr(:tuple, kw) # calls tuple constructor with kw + @test_throws MethodError fl_eval(test_mod, ex) + @test_throws MethodError jl_eval(test_mod, ex) broken=true + end + @testset let ex = Expr(:tuple, pkw) + @test fl_eval(test_mod, ex) == (b=2,) + @test jl_eval(test_mod, ex) == (b=2,) + end + end + + @testset "in :curly" begin + @testset let ex = Expr(:curly, Array, Int, eq) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:curly, Array, Int, peq) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:curly, Array, Int, kw) # calls constructor with kw + @test_throws MethodError fl_eval(test_mod, ex) + @test_throws MethodError jl_eval(test_mod, ex) broken=true + end + @testset let ex = Expr(:curly, Array, Int, pkw) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + end + @testset "in :vect" begin + @testset let ex = Expr(:vect, eq) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:vect, peq) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:vect, kw) # calls vect constructor with kw + @test_throws MethodError fl_eval(test_mod, ex) + @test_throws MethodError jl_eval(test_mod, ex) broken=true + end + @testset let ex = Expr(:vect, pkw) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + end + @testset "in :braces" begin + @testset let ex = Expr(:braces, eq) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:braces, peq) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:braces, kw) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + @testset let ex = Expr(:braces, pkw) + @test_throws ErrorException fl_eval(test_mod, ex) + @test_throws LoweringError jl_eval(test_mod, ex) + end + end +end + +@testset "macros can have lhs-reserved or underscore names" begin + local m = Module() + + @test JuliaLowering.include_string(m, """ + module ShortForm + macro ccall end + macro cglobal end + macro _ end + end + """) isa Module + @test Base.isdefinedglobal(m.ShortForm, Symbol("@ccall")) + @test Base.isdefinedglobal(m.ShortForm, Symbol("@cglobal")) + @test Base.isdefinedglobal(m.ShortForm, Symbol("@_")) + + @test JuliaLowering.include_string(m, """ macro ccall(x); x; end """) isa Function + @test JuliaLowering.include_string(m, "@ccall(1)") == 1 + + @test JuliaLowering.include_string(m, """ macro cglobal(x); x; end """) isa Function + @test JuliaLowering.include_string(m, "@cglobal(1)") == 1 + + @test JuliaLowering.include_string(m, """ macro _(x); x; end """) isa Function + @test JuliaLowering.include_string(m, "@_(3)") == 3 + + # empty name is usable (though won't parse) + @test jl_eval(m, Expr(:macro, Expr(:call, Symbol(""), :x), :x)) isa Function + @test jl_eval(m, Expr(:macrocall, Symbol("@"), LineNumberNode(1), 123)) == 123 +end diff --git a/JuliaLowering/test/assignments_ir.jl b/JuliaLowering/test/assignments_ir.jl new file mode 100644 index 0000000000000..02691225f1c9d --- /dev/null +++ b/JuliaLowering/test/assignments_ir.jl @@ -0,0 +1,378 @@ +######################################## +# chain of assignments +let + a = b = c = 1 +end +#--------------------- +1 1 +2 (= slot₃/c %₁) +3 (= slot₂/b %₁) +4 (= slot₁/a %₁) +5 (return %₁) + +######################################## +# chain of assignments with nontrivial rhs +let + a = b = c = f() +end +#--------------------- +1 TestMod.f +2 (call %₁) +3 (= slot₃/c %₂) +4 (= slot₂/b %₂) +5 (= slot₁/a %₂) +6 (return %₂) + +######################################## +# Assignment in value but not tail position +let + x = begin + y = 42 + end + x +end +#--------------------- +1 42 +2 (= slot₂/y %₁) +3 (= slot₁/x %₁) +4 slot₁/x +5 (return %₄) + +######################################## +# short form function def, not chain of assignments +begin + local a + a = b() = c = d +end +#--------------------- +1 (call core.define_method TestMod :b) +2 latestworld +3 TestMod.b +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::3:9 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.b %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/c(!read,single_assign)] + 1 TestMod.d + 2 (= slot₂/c %₁) + 3 (return %₁) +10 latestworld +11 TestMod.b +12 (= slot₁/a %₁₁) +13 (return %₁₁) + +######################################## +# a.b = ... => setproperty! assignment +let + a.b = c +end +#--------------------- +1 TestMod.a +2 TestMod.c +3 (call top.setproperty! %₁ :b %₂) +4 TestMod.c +5 (return %₄) + +######################################## +# a.b.c = f() => setproperty! assignment, complex case +let + a.b.c = f() +end +#--------------------- +1 TestMod.a +2 (call top.getproperty %₁ :b) +3 TestMod.f +4 (call %₃) +5 (call top.setproperty! %₂ :c %₄) +6 (return %₄) + +######################################## +# declarations of typed locals +let + x::T = f() + x +end +#--------------------- +1 (newvar slot₁/x) +2 TestMod.f +3 (call %₂) +4 TestMod.T +5 (= slot₂/tmp %₃) +6 (call core.isa slot₂/tmp %₄) +7 (gotoifnot %₆ label₉) +8 (goto label₁₁) +9 (call top.convert %₄ slot₂/tmp) +10 (= slot₂/tmp (call core.typeassert %₉ %₄)) +11 slot₂/tmp +12 (= slot₁/x %₁₁) +13 slot₁/x +14 (return %₁₃) + +######################################## +# "complex lhs" of `::T` => type-assert, not decl +let + a.b::T = f() + x +end +#--------------------- +1 TestMod.a +2 (call top.getproperty %₁ :b) +3 TestMod.T +4 (call core.typeassert %₂ %₃) +5 TestMod.f +6 (call %₅) +7 TestMod.a +8 (call top.setproperty! %₇ :b %₆) +9 TestMod.x +10 (return %₉) + +######################################## +# UnionAll expansion at global scope results in const decl +X{T} = Y{T,T} +#--------------------- +1 (call core.TypeVar :T) +2 (= slot₁/T %₁) +3 TestMod.Y +4 (call core.apply_type %₃ slot₁/T slot₁/T) +5 (call core.UnionAll slot₁/T %₄) +6 (call core.declare_const TestMod :X %₅) +7 latestworld +8 (return %₅) + +######################################## +# UnionAll expansion in local scope +let + X{T} = Y{T,T} +end +#--------------------- +1 (call core.TypeVar :T) +2 (= slot₂/T %₁) +3 TestMod.Y +4 (call core.apply_type %₃ slot₂/T slot₂/T) +5 (call core.UnionAll slot₂/T %₄) +6 (= slot₁/X %₅) +7 (return %₅) + +######################################## +# Error: Invalid lhs in `=` +a.(b) = rhs +#--------------------- +LoweringError: +a.(b) = rhs +└───┘ ── dotcall syntax not valid here + +######################################## +# Error: Invalid lhs in `=` +a.(b,c) = rhs +#--------------------- +LoweringError: +a.(b,c) = rhs +└─────┘ ── dotcall syntax not valid here + +######################################## +# Error: Invalid lhs in `=` +T[x y] = rhs +#--------------------- +LoweringError: +T[x y] = rhs +└────┘ ── invalid spacing in left side of indexed assignment + +######################################## +# Error: Invalid lhs in `=` +T[x; y] = rhs +#--------------------- +LoweringError: +T[x; y] = rhs +└─────┘ ── unexpected `;` in left side of indexed assignment + +######################################## +# Error: Invalid lhs in `=` +T[x ;;; y] = rhs +#--------------------- +LoweringError: +T[x ;;; y] = rhs +└────────┘ ── unexpected `;` in left side of indexed assignment + +######################################## +# Error: Invalid lhs in `=` +[x, y] = rhs +#--------------------- +LoweringError: +[x, y] = rhs +└────┘ ── use `(a, b) = ...` to assign multiple values + +######################################## +# Error: Invalid lhs in `=` +[x y] = rhs +#--------------------- +LoweringError: +[x y] = rhs +└───┘ ── use `(a, b) = ...` to assign multiple values + +######################################## +# Error: Invalid lhs in `=` +[x; y] = rhs +#--------------------- +LoweringError: +[x; y] = rhs +└────┘ ── use `(a, b) = ...` to assign multiple values + +######################################## +# Error: Invalid lhs in `=` +[x ;;; y] = rhs +#--------------------- +LoweringError: +[x ;;; y] = rhs +└───────┘ ── use `(a, b) = ...` to assign multiple values + +######################################## +# Error: Invalid lhs in `=` +1 = rhs +#--------------------- +LoweringError: +1 = rhs +╙ ── invalid syntax in left-hand side of assignment + +######################################## +# Basic updating assignment +begin + local x + x += y +end +#--------------------- +1 TestMod.+ +2 slot₁/x +3 TestMod.y +4 (call %₁ %₂ %₃) +5 (= slot₁/x %₄) +6 (return %₄) + +######################################## +# Broadcasted updating assignment +begin + local x + x .+= y +end +#--------------------- +1 (newvar slot₁/x) +2 slot₁/x +3 TestMod.+ +4 TestMod.y +5 (call top.broadcasted %₃ %₂ %₄) +6 (call top.materialize! %₂ %₅) +7 (return %₂) + +######################################## +# Broadcasted updating assignment with general left hand side permitted +f() .+= y +#--------------------- +1 TestMod.f +2 (call %₁) +3 TestMod.+ +4 TestMod.y +5 (call top.broadcasted %₃ %₂ %₄) +6 (call top.materialize! %₂ %₅) +7 (return %₂) + +######################################## +# Updating assignment with basic ref as left hand side +x[i] += y +#--------------------- +1 TestMod.+ +2 TestMod.x +3 TestMod.i +4 (call top.getindex %₂ %₃) +5 TestMod.y +6 (call %₁ %₄ %₅) +7 TestMod.x +8 TestMod.i +9 (call top.setindex! %₇ %₆ %₈) +10 (return %₆) + +######################################## +# Updating assignment with complex ref as left hand side +g()[f(), end] += y +#--------------------- +1 TestMod.g +2 (call %₁) +3 TestMod.f +4 (call %₃) +5 (call top.lastindex %₂ 2) +6 TestMod.+ +7 (call top.getindex %₂ %₄ %₅) +8 TestMod.y +9 (call %₆ %₇ %₈) +10 (call top.setindex! %₂ %₉ %₄ %₅) +11 (return %₉) + +######################################## +# Updating assignment with type assert on left hand side +begin + local x + x::T += y +end +#--------------------- +1 TestMod.+ +2 slot₁/x +3 TestMod.T +4 (call core.typeassert %₂ %₃) +5 TestMod.y +6 (call %₁ %₄ %₅) +7 (= slot₁/x %₆) +8 (return %₆) + +######################################## +# Updating assignment with ref and type assert on left hand side +begin + local x + x[f()]::T += y +end +#--------------------- +1 (newvar slot₁/x) +2 TestMod.f +3 (call %₂) +4 TestMod.+ +5 slot₁/x +6 (call top.getindex %₅ %₃) +7 TestMod.T +8 (call core.typeassert %₆ %₇) +9 TestMod.y +10 (call %₄ %₈ %₉) +11 slot₁/x +12 (call top.setindex! %₁₁ %₁₀ %₃) +13 (return %₁₀) + +######################################## +# Error: invalid LHS: block +begin; x; end = 1 +#--------------------- +LoweringError: +begin; x; end = 1 +└───────────┘ ── invalid syntax in left-hand side of assignment + +######################################## +# Error: Updating assignment with invalid left hand side +f() += y +#--------------------- +LoweringError: +f() += y +└─┘ ── invalid assignment location + +######################################## +# Error: Updating assignment with invalid tuple destructuring on left hand side +(if false end, b) += 2 +#--------------------- +LoweringError: +(if false end, b) += 2 +#└──────────┘ ── invalid syntax in left-hand side of assignment + +######################################## +# Error: Updating assignment to ssavalue (JuliaLang/julia#30062) +f(), x += 10, 20 +#--------------------- +LoweringError: +f(), x += 10, 20 +└────┘ ── invalid multiple assignment location diff --git a/JuliaLowering/test/ast.jl b/JuliaLowering/test/ast.jl new file mode 100644 index 0000000000000..7e25586e8cabc --- /dev/null +++ b/JuliaLowering/test/ast.jl @@ -0,0 +1,85 @@ +let node = JS.newleaf(LineNumberNode(1), K"Value", nothing) + @test node.value === nothing +end + +@testset "assert_syntaxtree" begin + st = parsestmt(SyntaxTree, "function foo end") + @test JuliaLowering.assert_syntaxtree(st) === nothing + + bad_st = JuliaSyntax.newleaf(st, K"Identifier") + @test_throws "needs value" JuliaLowering.assert_syntaxtree(bad_st) + @test_throws "needs value" show(bad_st) + + bad_st = JuliaSyntax.newleaf(st, K"code_info") + @test_throws "unrecognized leaf kind" JuliaLowering.assert_syntaxtree(bad_st) + + setfield!(bad_st, :children, SyntaxList(bad_st)) + @test_throws "cycle detected" JuliaLowering.assert_syntaxtree(bad_st) + + cyc_1 = JuliaSyntax.newnode(st, K"block", SyntaxList()) + cyc_2 = JuliaSyntax.newnode(st, K"block", SyntaxList(cyc_1)) + setfield!(cyc_1, :children, SyntaxList(cyc_2)) + @test_throws "cycle detected" JuliaLowering.assert_syntaxtree(cyc_1) + @test_throws "cycle detected" JuliaLowering.assert_syntaxtree(cyc_2) +end + +@testset "flatten_blocks" begin + let + st = @ast_ [K"block"] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"block"] + + st = @ast_ [K"block" 1::K"Value"] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"block" 1::K"Value"] + + st = @ast_ [K"block" 1::K"Value" [K"block" 1::K"Value"]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"block" 1::K"Value" 1::K"Value"] + + st = @ast_ [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]] + + st = @ast_ [K"block" 1::K"Value" [K"block"]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"block" 1::K"Value" (::K"nothing")] + + st = @ast_ [K"block" 1::K"Value" [K"block"] 1::K"Value"] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"block" 1::K"Value" 1::K"Value"] + + st = @ast_ [K"block" [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"block" [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]]] + + # repeat with call wrapper + st = @ast_ [K"call" [K"block"]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"block"]] + + st = @ast_ [K"call" [K"block" 1::K"Value"]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"block" 1::K"Value"]] + + st = @ast_ [K"call" [K"block" 1::K"Value" [K"block" 1::K"Value"]]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"block" 1::K"Value" 1::K"Value"]] + + st = @ast_ [K"call" [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]]] + + st = @ast_ [K"call" [K"block" 1::K"Value" [K"block"]]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"block" 1::K"Value" (::K"nothing")]] + + st = @ast_ [K"call" [K"block" 1::K"Value" [K"block"] 1::K"Value"]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"block" 1::K"Value" 1::K"Value"]] + + st = @ast_ [K"call" [K"block" [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]]]] + @test JuliaLowering.flatten_blocks(st) ≈ + @ast_ [K"call" [K"block" [K"inert" [K"block" 1::K"Value" [K"block" 1::K"Value"]]]]] + end +end diff --git a/JuliaLowering/test/branching.jl b/JuliaLowering/test/branching.jl new file mode 100644 index 0000000000000..24d27813d2fc7 --- /dev/null +++ b/JuliaLowering/test/branching.jl @@ -0,0 +1,559 @@ +# Branching + +test_mod = Module() + +#------------------------------------------------------------------------------- +@testset "Tail position" begin + +@test JuliaLowering.include_string(test_mod, """ +let a = true + if a + 1 + end +end +""") === 1 + +@test JuliaLowering.include_string(test_mod, """ +let a = false + if a + 1 + end +end +""") === nothing + +@test JuliaLowering.include_string(test_mod, """ +let a = true + if a + 1 + else + 2 + end +end +""") === 1 + +@test JuliaLowering.include_string(test_mod, """ +let a = false + if a + 1 + else + 2 + end +end +""") === 2 + +@test JuliaLowering.include_string(test_mod, """ +let a = false, b = true + if a + 1 + elseif b + 2 + else + 3 + end +end +""") === 2 + +@test JuliaLowering.include_string(test_mod, """ +let a = false, b = false + if a + 1 + elseif b + 2 + else + 3 + end +end +""") === 3 + +end + +#------------------------------------------------------------------------------- +@testset "Value required but not tail position" begin + +@test JuliaLowering.include_string(test_mod, """ +let a = true + x = if a + 1 + end + x +end +""") === 1 + +@test JuliaLowering.include_string(test_mod, """ +let a = false + x = if a + 1 + end + x +end +""") === nothing + +@test JuliaLowering.include_string(test_mod, """ +let a = true + x = if a + 1 + else + 2 + end + x +end +""") === 1 + +@test JuliaLowering.include_string(test_mod, """ +let a = false + x = if a + 1 + else + 2 + end + x +end +""") === 2 + +@test JuliaLowering.include_string(test_mod, """ +let a = false, b = true + x = if a + 1 + elseif b + 2 + else + 3 + end + x +end +""") === 2 + +@test JuliaLowering.include_string(test_mod, """ +let a = false, b = false + x = if a + 1 + elseif b + 2 + else + 3 + end + x +end +""") === 3 + +end + +#------------------------------------------------------------------------------- +@testset "Side effects (not value or tail position)" begin + +@test JuliaLowering.include_string(test_mod, """ +let a = true + x = nothing + if a + x = 1 + end + x +end +""") === 1 + +@test JuliaLowering.include_string(test_mod, """ +let a = false + x = nothing + if a + x = 1 + end + x +end +""") === nothing + +@test JuliaLowering.include_string(test_mod, """ +let a = true + x = nothing + if a + x = 1 + else + x = 2 + end + x +end +""") === 1 + +@test JuliaLowering.include_string(test_mod, """ +let a = false + x = nothing + if a + x = 1 + else + x = 2 + end + x +end +""") === 2 + +@test JuliaLowering.include_string(test_mod, """ +let a = false, b = true + x = nothing + if a + x = 1 + elseif b + x = 2 + else + x = 3 + end + x +end +""") === 2 + +@test JuliaLowering.include_string(test_mod, """ +let a = false, b = false + x = nothing + if a + x = 1 + elseif b + x = 2 + else + x = 3 + end + x +end +""") === 3 + +end + +@testset "elseif is valid outside of if" begin + @test jl_eval(test_mod, Expr(:elseif, true, 1)) == 1 + @test jl_eval(test_mod, Expr(:elseif, false, 1)) == nothing + @test jl_eval(test_mod, Expr(:elseif, true, 1, 2)) == 1 + @test jl_eval(test_mod, Expr(:elseif, false, 1, 2)) == 2 +end + +@testset "comparison" begin + @test jl_eval( + test_mod, + Expr(:comparison, 1, :(Base.FastMath.le_fast), 2, + :(Base.FastMath.le_fast), 3)) +end + +#------------------------------------------------------------------------------- +# Block condition +@test JuliaLowering.include_string(test_mod, """ +let a = true + if begin; x = 2; a; end + x + end +end +""") === 2 + +@testset "(AI) block conditions" begin + # An empty block as a condition evaluates to `nothing` and must throw a + # runtime TypeError (as the flisp lowering does for `if begin end`), not + # crash lowering. + @test_throws TypeError jl_eval(test_mod, Expr(:if, Expr(:block), 1, 2)) + @test_throws TypeError jl_eval(test_mod, Expr(:while, Expr(:block), 1)) + @test_throws TypeError JuliaLowering.include_string(test_mod, """ + (begin end) ? 1 : 2 + """) + # ... including as a term of a `&&` chain + @test_throws TypeError JuliaLowering.include_string(test_mod, """ + if (begin end) && true + 1 + else + 2 + end + """) + + # do not ignore block[1:end-1] + @test JuliaLowering.include_string(test_mod, """ + let a = true + if (nothing; x = 2; a) + x + end + end + """) === 2 + + # In a `while`, the whole block re-runs on every iteration, including the + # final iteration that exits the loop. + @test JuliaLowering.include_string(test_mod, """ + let s = 0, i = 0 + while (i = i + 1; i <= 3) + s = s + i + end + (s, i) + end + """) === (6, 4) + @test JuliaLowering.include_string(test_mod, """ + let log = [] + i = 0 + while (push!(log, :cond); i = i + 1; i <= 2 && true) + push!(log, :body) + end + (i, log) + end + """) == (3, [:cond, :body, :cond, :body, :cond]) + + # A non-final non-Bool term of a `&&` ending a multi-statement block + # condition still throws TypeError (whether or not the chain gets the + # direct-jump lowering). + fnb = JuliaLowering.include_string(test_mod, """ + function () + if (nothing; 1 && true) + 1 + else + 2 + end + end + """) + @test_throws TypeError fnb() + + # A multi-statement block as a *term* of a `&&` chain is not flattened and + # must stay lazy: its statements run only if earlier terms pass. + @test JuliaLowering.include_string(test_mod, """ + let log = [] + f(v) = (push!(log, v); v) + r = if f(true) && (f(:pre); f(false)) + 1 + else + 2 + end + (r, log) + end + """) == (2, [true, :pre, false]) + @test JuliaLowering.include_string(test_mod, """ + let log = [] + f(v) = (push!(log, v); v) + r = if f(false) && (f(:pre); f(true)) + 1 + else + 2 + end + (r, log) + end + """) == (2, [false]) + + # Degenerate 0-arg `&&`/`||` as the final statement of a multi-statement + # block condition + @test jl_eval(test_mod, Expr(:if, Expr(:block, :(1 + 1), Expr(:&&)), 1, 2)) === 1 + @test jl_eval(test_mod, Expr(:if, Expr(:block, :(1 + 1), Expr(:||)), 1, 2)) === 2 + + @test JuliaLowering.include_string(test_mod, """ + if begin begin true end end + 1 + else + 2 + end + """) === 1 +end + +#------------------------------------------------------------------------------- +@testset "`&&` and `||` chains" begin + +# 0-1 arguments +@test jl_eval(test_mod, Expr(:&&)) == true +@test jl_eval(test_mod, Expr(:&&, true)) == true +@test jl_eval(test_mod, Expr(:&&, false)) == false +@test jl_eval(test_mod, Expr(:||)) == false +@test jl_eval(test_mod, Expr(:||, true)) == true +@test jl_eval(test_mod, Expr(:||, false)) == false + +# 0-1 arguments in condition position (`expand_condition`, used by `if`/`while`) +# have their own desugaring path separate from the value-position case above. +@test jl_eval(test_mod, Expr(:if, Expr(:&&), 1, 2)) == 1 +@test jl_eval(test_mod, Expr(:if, Expr(:&&, true), 1, 2)) == 1 +@test jl_eval(test_mod, Expr(:if, Expr(:&&, false), 1, 2)) == 2 +@test jl_eval(test_mod, Expr(:if, Expr(:||), 1, 2)) == 2 +@test jl_eval(test_mod, Expr(:if, Expr(:||, true), 1, 2)) == 1 +@test jl_eval(test_mod, Expr(:if, Expr(:||, false), 1, 2)) == 2 + +# Same, but with the condition inside a block (the `isblock` branch of +# `expand_condition`) +@test jl_eval(test_mod, Expr(:if, Expr(:block, Expr(:&&, true)), 1, 2)) == 1 +@test jl_eval(test_mod, Expr(:if, Expr(:block, Expr(:||, false)), 1, 2)) == 2 +@test jl_eval(test_mod, Expr(:if, Expr(:block, Expr(:&&)), 1, 2)) == 1 + +# Degenerate arities nested inside another `&&`/`||` (flattened away by +# `expand_cond_children`) +@test jl_eval(test_mod, Expr(:if, Expr(:&&, Expr(:&&)), 1, 2)) == 1 +@test jl_eval(test_mod, Expr(:if, Expr(:&&, Expr(:&&, false), true), 1, 2)) == 2 + +# `while` conditions share `expand_condition` with `if` +@test jl_eval(test_mod, Expr(:while, Expr(:||), 1)) === nothing +@test jl_eval(test_mod, Expr(:while, Expr(:&&), Expr(:break))) === nothing + +@test JuliaLowering.include_string(test_mod, """ +true && "hi" +""") == "hi" + +@test JuliaLowering.include_string(test_mod, """ +true && true && "hi" +""") == "hi" + +@test JuliaLowering.include_string(test_mod, """ +false && "hi" +""") == false + +@test JuliaLowering.include_string(test_mod, """ +true && false && "hi" +""") == false + +@test JuliaLowering.include_string(test_mod, """ +begin + z = true && "hi" + z +end +""") == "hi" + +@test JuliaLowering.include_string(test_mod, """ +begin + z = false && "hi" + z +end +""") == false + + +@test JuliaLowering.include_string(test_mod, """ +true || "hi" +""") == true + +@test JuliaLowering.include_string(test_mod, """ +true || true || "hi" +""") == true + +@test JuliaLowering.include_string(test_mod, """ +false || "hi" +""") == "hi" + +@test JuliaLowering.include_string(test_mod, """ +false || true || "hi" +""") == true + +@test JuliaLowering.include_string(test_mod, """ +false || false || "hi" +""") == "hi" + +@test JuliaLowering.include_string(test_mod, """ +begin + z = false || "hi" + z +end +""") == "hi" + +@test JuliaLowering.include_string(test_mod, """ +begin + z = true || "hi" + z +end +""") == true + +end + +@testset "(AI) diverging (`return`) condition terms" begin + # A `&&`/`||` chain used directly as an `if`/`elseif` test may contain a + # `return` as a (flattened) operand, eg PSSFSS's `a && b && return true` used as + # a bare `elseif` test with an empty body. The `return` term compiles to a + # divergence (no reachable value); `compile_conditional` must skip the gate for + # it rather than crash. All expected values below are the flisp lowering's. + + # Trailing `return` in a `&&` test with an empty `elseif` body (the MWE) + fa = JuliaLowering.include_string(test_mod, """ + function (x, y) + if y == 1 + return 10 + elseif y == 2 && x == 2 && return 20 + elseif y == 3 + return 30 + end + return 99 + end + """) + @test fa(2, 1) === 10 # first branch + @test fa(2, 2) === 20 # `&&` chain reaches the `return` + @test fa(3, 2) === 99 # `x == 2` false -> chain false -> fall through + @test fa(2, 3) === 30 # later branch still reachable + @test fa(0, 0) === 99 + + # Trailing `return` in a `||` test + fd = JuliaLowering.include_string(test_mod, """ + function (x, y) + if y == 1 + return 10 + elseif x == 2 || return 20 + elseif y == 3 + return 30 + end + return 99 + end + """) + @test fd(2, 1) === 10 + @test fd(0, 2) === 20 # `x == 2` false -> `return 20` + @test fd(2, 2) === 99 # `x == 2` true -> empty body -> fall through + @test fd(0, 3) === 20 # reached before the `y == 3` branch + + # `return` in a non-final position of the flattened chain (via parens): the + # terms after it are unreachable and must be dropped. + fh = JuliaLowering.include_string(test_mod, """ + function (a, b) + if (a && return 20) && b + return 111 + end + return 99 + end + """) + @test fh(true, false) === 20 # `a` true -> `return 20`; `b` never evaluated + @test fh(false, true) === 99 # `a` false -> chain false -> skip body + + # `||` with a non-final diverging term + fj = JuliaLowering.include_string(test_mod, """ + function (a, b) + if (a || return 20) || b + return 111 + end + return 99 + end + """) + @test fj(true, false) === 111 # `a` true -> short circuit -> body + @test fj(false, true) === 20 # `a` false -> `return 20`; `b` unreachable + + # The body of an `if` whose test always diverges when true is unreachable. + fb = JuliaLowering.include_string(test_mod, """ + function (x, y) + if y == 2 && x == 2 && return 20 + return 111 + end + return 99 + end + """) + @test fb(2, 2) === 20 + @test fb(3, 2) === 99 + @test fb(2, 3) === 99 + + # Diverging test of a ternary (`?:` lowers to `if`) + ff = JuliaLowering.include_string(test_mod, """ + function (a, y, z) + (a && return 20) ? y : z + end + """) + @test ff(true, 1, 2) === 20 # diverges before the ternary result + @test ff(false, 1, 2) === 2 # chain false -> select `z` + + # `return` as a `&&` term in *value* position (already handled via the + # `if`-desugaring path) must keep working. + fg = JuliaLowering.include_string(test_mod, """ + function (a) + x = a && return 1 + x + end + """) + @test fg(true) === 1 + @test fg(false) === false +end + +@testset "symbolic goto/label" begin + @test JuliaLowering.include_string(test_mod, """ + let + a = [] + i = 1 + @label foo + push!(a, i) + i = i + 1 + if i <= 2 + @goto foo + end + a + end + """) == [1,2] +end diff --git a/JuliaLowering/test/branching_ir.jl b/JuliaLowering/test/branching_ir.jl new file mode 100644 index 0000000000000..46561db786cc2 --- /dev/null +++ b/JuliaLowering/test/branching_ir.jl @@ -0,0 +1,258 @@ +######################################## +# Basic branching tail && value +begin + local a, b + if a + b + end +end +#--------------------- +1 (newvar slot₁/a) +2 (newvar slot₂/b) +3 slot₁/a +4 (gotoifnot %₃ label₇) +5 slot₂/b +6 (return %₅) +7 (return core.nothing) + +######################################## +# Branching, !tail && !value +begin + local a, b, c + if a + b + end + c +end +#--------------------- +1 (newvar slot₁/a) +2 (newvar slot₂/b) +3 (newvar slot₃/c) +4 slot₁/a +5 (gotoifnot %₄ label₇) +6 slot₂/b +7 slot₃/c +8 (return %₇) + +######################################## +# Branching with else +begin + local a, b, c + if a + b + else + c + end +end +#--------------------- +1 (newvar slot₁/a) +2 (newvar slot₂/b) +3 (newvar slot₃/c) +4 slot₁/a +5 (gotoifnot %₄ label₈) +6 slot₂/b +7 (return %₆) +8 slot₃/c +9 (return %₈) + +######################################## +# Branching with else, !tail && !value +begin + local a, b, c, d + if a + b + else + c + end + d +end +#--------------------- +1 (newvar slot₁/a) +2 (newvar slot₂/b) +3 (newvar slot₃/c) +4 (newvar slot₄/d) +5 slot₁/a +6 (gotoifnot %₅ label₉) +7 slot₂/b +8 (goto label₁₀) +9 slot₃/c +10 slot₄/d +11 (return %₁₀) + +######################################## +# Blocks compile directly to branches +begin + local a, b, c, d + if (a; b && c) + d + end +end +#--------------------- +1 (newvar slot₁/a) +2 (newvar slot₂/b) +3 (newvar slot₃/c) +4 (newvar slot₄/d) +5 slot₁/a +6 slot₂/b +7 (gotoifnot %₆ label₁₂) +8 slot₃/c +9 (gotoifnot %₈ label₁₂) +10 slot₄/d +11 (return %₁₀) +12 (return core.nothing) + +######################################## +# symbolic goto forward jump +begin + a + @goto foo + b + @label foo +end +#--------------------- +1 TestMod.a +2 (goto label₄) +3 TestMod.b +4 (return core.nothing) + +######################################## +# symbolic goto backward jump +begin + a + @label foo + b + @goto foo +end +#--------------------- +1 TestMod.a +2 TestMod.b +3 (goto label₂) + +######################################## +# Jumping out of try and catch blocks using @goto +begin + try + a + @goto lab + b + catch + c + @goto lab + d + end + @label lab +end +#--------------------- +1 (enter label₈) +2 TestMod.a +3 (leave %₁) +4 (goto label₁₃) +5 TestMod.b +6 (leave %₁) +7 (goto label₁₃) +8 TestMod.c +9 (pop_exception %₁) +10 (goto label₁₃) +11 TestMod.d +12 (pop_exception %₁) +13 (return core.nothing) + +######################################## +# Jumping out of nested try/catch and catch/try +begin + try + try + a + catch + b + @goto lab + c + end + catch + try + d + @goto lab + e + catch + end + end + @label lab +end +#--------------------- +1 (enter label₁₄) +2 (enter label₆) +3 TestMod.a +4 (leave %₂) +5 (goto label₁₂) +6 TestMod.b +7 (pop_exception %₂) +8 (leave %₁) +9 (goto label₂₄) +10 TestMod.c +11 (pop_exception %₂) +12 (leave %₁) +13 (goto label₂₄) +14 (enter label₂₂) +15 TestMod.d +16 (pop_exception %₁) +17 (leave %₁₄) +18 (goto label₂₄) +19 TestMod.e +20 (leave %₁₄) +21 (goto label₂₃) +22 (pop_exception %₁₄) +23 (pop_exception %₁) +24 (return core.nothing) + +######################################## +# Error: no symbolic label +begin + @goto foo +end +#--------------------- +LoweringError: +begin + @goto foo +# └─┘ ── label `foo` referenced but not defined +end + +######################################## +# Error: duplicate symbolic label +begin + @label foo + @label foo +end +#--------------------- +LoweringError: +#= none:3 =# - Label `foo` defined multiple times +Expression: + label:foo + +######################################## +# Error: using value of symbolic label +x = @label foo +#--------------------- +LoweringError: +#= none:1 =# - misplaced label in value position +Expression: + label:foo + +######################################## +# Anonymous labeled block with valued break +@label begin + a + break _ 42 + b +end +#--------------------- +1 TestMod.a +2 (= slot₁/loop-exit_result 42) +3 (goto label₆) +4 TestMod.b +5 (= slot₁/loop-exit_result %₄) +6 (isdefined slot₁/loop-exit_result) +7 (gotoifnot %₆ label₉) +8 (goto label₁₀) +9 (= slot₁/loop-exit_result core.nothing) +10 slot₁/loop-exit_result +11 (return %₁₀) diff --git a/JuliaLowering/test/closures.jl b/JuliaLowering/test/closures.jl new file mode 100644 index 0000000000000..1c7533b4977d0 --- /dev/null +++ b/JuliaLowering/test/closures.jl @@ -0,0 +1,1659 @@ +test_mod = Module() + +# Capture assigned before closure +@test JuliaLowering.include_string(test_mod, """ +let + x = 1 + f(y) = x+y + f(2), f(3) +end +""") == (3,4) + +# Capture assigned after closure +@test JuliaLowering.include_string(test_mod, """ +let + f(y) = x+y + x = 1 + f(2) +end +""") == 3 + +# Capture assigned inside closure +@test JuliaLowering.include_string(test_mod, """ +let + x = 1 + function f(y) + x = y + end + f(100) + x +end +""") == 100 + +Base.eval(test_mod, :(call_it(f, args...) = f(args...))) + +# Closure where an argument `x` is captured but not boxed +@test JuliaLowering.include_string(test_mod, """ +begin + function f_unboxed_test(x) + z = 0 + function g() + y = x # x will not be boxed + (y + 1, z) + end + z = 2 # will be boxed + (x, g()) + end + f_unboxed_test(10) +end +""") == (10,(11,2)) + +# Use of isdefined +@test JuliaLowering.include_string(test_mod, """ +begin + function f_isdefined(x) + local w + function g() + z = 3 + (@isdefined(x), # unboxed, always defined capture + @isdefined(y), # boxed capture + @isdefined(z), # normal local var + @isdefined(w)) # boxed undefined var + end + y = 2 + (@isdefined(y), @isdefined(w), g()) + end + f_isdefined(1) +end +""") == (true, false, (true, true, true, false)) + +# Mutually recursive closures (closure capturing a closure) +@test JuliaLowering.include_string(test_mod, """ +let + function recursive_a(n) + here = (:a, n) + n <= 0 ? here : (here, recursive_b(n-1)) + end + function recursive_b(n) + ((:b, n), recursive_a(n-1)) + end + recursive_a(2) +end +""") == ((:a, 2), ((:b, 1), (:a, 0))) + +# Global method capturing local variables +JuliaLowering.include_string(test_mod, """ +begin + local x = 1 + function f_global_method_capturing_local() + x = x + 1 + end +end +""") +@test test_mod.f_global_method_capturing_local() == 2 +@test test_mod.f_global_method_capturing_local() == 3 + +# quote interpolated AST +@test JuliaLowering.include_string(test_mod, """ +let x = Symbol("foo"), xq = QuoteNode(x) + global function f_global_method_capturing_sym() + x, xq + end + f_global_method_capturing_sym() +end +""") == (:foo, QuoteNode(:foo)) +@test JuliaLowering.include_string(test_mod, """ +global dont_resolve = 1 +let x = GlobalRef(@__MODULE__, :dont_resolve), xq = QuoteNode(x) + global function f_global_method_capturing_gr() + x, xq + end + f_global_method_capturing_gr() +end +""") == (GlobalRef(test_mod, :dont_resolve), QuoteNode(GlobalRef(test_mod, :dont_resolve))) + +# Closure with multiple methods depending on local variables +f_closure_local_var_types = JuliaLowering.include_string(test_mod, """ +let T=Int, S=Float64 + function f_closure_local_var_types(::T) + 1 + end + function f_closure_local_var_types(::S) + 1.0 + end +end +""") +@test f_closure_local_var_types(2) == 1 +@test f_closure_local_var_types(2.0) == 1.0 +@test_throws MethodError f_closure_local_var_types("hi") + +# Multiply nested closures. In this case g_nest needs to capture `x` in order +# to construct an instance of `h_nest()` inside it. +@test JuliaLowering.include_string(test_mod, """ +begin + function f_nest(x) + function g_nest(y) + function h_nest(z) + (x,y,z) + end + end + end + + f_nest(1)(2)(3) +end +""") === (1,2,3) + +# Closure with return type must capture the return type +@test JuliaLowering.include_string(test_mod, """ +let T = Int + function f_captured_return_type()::T + 2.0 + end + f_captured_return_type() +end +""") === 2 + +# Capturing a typed local +@test JuliaLowering.include_string(test_mod, """ +let T = Int + x::T = 1.0 + function f_captured_typed_local() + x = 2.0 + end + f_captured_typed_local() + x +end +""") === 2 + +# Capturing a typed local where the type is a nontrivial expression +@test begin + res = JuliaLowering.include_string(test_mod, """ + let T = Int, V=Vector + x::V{T} = [1,2] + function f_captured_typed_local_composite() + x = [100.0, 200.0] + end + f_captured_typed_local_composite() + x + end + """) + res == [100, 200] && eltype(res) == Int +end + +# Evil case where we mutate `T` which is the type of `x`, such that x is +# eventually set to a Float64. +# +# Completely dynamic types for variables should be disallowed somehow?? For +# example, by emitting the expression computing the type of `x` alongside the +# newvar node. However, for now we verify that this potentially evil behavior +# is compatible with the existing implementation :) +@test JuliaLowering.include_string(test_mod, """ +let T = Int + x::T = 1.0 + function f_captured_mutating_typed_local() + x = 2 + end + T = Float64 + f_captured_mutating_typed_local() + x +end +""") === 2.0 + +# Anon function syntax +@test JuliaLowering.include_string(test_mod, """ +begin + local y = 2 + call_it(x->x+y, 3) +end +""") == 5 + +# Anon function syntax with `where` +@test JuliaLowering.include_string(test_mod, """ +begin + local y = 2 + call_it((x::T where {T<:Integer})->x+y, 3) +end +""") == 5 + +# Do block syntax +@test JuliaLowering.include_string(test_mod, """ +begin + local y = 2 + call_it(3) do x + x + y + end +end +""") == 5 + +# Attempt to reference capture which is not assigned +@test_throws UndefVarError(:x, :local) JuliaLowering.include_string(test_mod, """ +let + function f() + x + end + f() + x = 1 +end +""") + +# Opaque closure +@test JuliaLowering.include_string(test_mod, """ +let y = 1 + oc = Base.Experimental.@opaque x->2x + y + oc(3) +end +""") == 7 + +# Opaque closure with `...` +@test JuliaLowering.include_string(test_mod, """ +let + oc = Base.Experimental.@opaque (xs...)->xs + oc(3,4,5) +end +""") == (3,4,5) + +# Opaque closure inside a closure can capture the enclosing closure's captures +@test JuliaLowering.include_string(test_mod, """ +let y = [1] + outer = () -> begin + inner = Base.Experimental.@opaque n -> n in y + inner(1) + end + outer() +end +""") === true + +# Nested opaque closure capture preserves boxed variable sharing +@test JuliaLowering.include_string(test_mod, """ +let y = 1 + outer = () -> begin + inner = Base.Experimental.@opaque () -> begin + y = y + 1 + end + inner() + end + outer() + y +end +""") === 2 + +# Opaque closure nested in another opaque closure can capture the outer OC environment +@test JuliaLowering.include_string(test_mod, """ +let y = [1] + outer = Base.Experimental.@opaque () -> begin + inner = Base.Experimental.@opaque n -> n in y + inner(1) + end + outer() +end +""") === true + +# Opaque closure type-bound expressions can capture enclosing closure captures +@test JuliaLowering.include_string(test_mod, """ +let T = Tuple{Int} + outer = () -> begin + inner = Base.Experimental.@opaque T -> _ (n) -> n + inner(1) + end + outer() +end +""") === 1 +@test JuliaLowering.include_string(test_mod, """ +let RT = Float64 + outer = () -> begin + inner = Base.Experimental.@opaque _ -> RT () -> 1.0 + inner() + end + outer() +end +""") === 1.0 + +# OC in lambda +@test JuliaLowering.include_string(test_mod, """ +(x->(y->(z->(Base.Experimental.@opaque ()->"opaque"))('z'))('y'))('x')() +""") == "opaque" +@test JuliaLowering.include_string(test_mod, """ +(x->(y->(z->(Base.Experimental.@opaque ()->(x,y,z)))('z'))('y'))('x')() +""") == ('x','y','z') + +# opaque_closure_method internals +method_ex = lower_str(test_mod, "Base.Experimental.@opaque x -> 2x").args[1].code[3] +@test method_ex.head === :opaque_closure_method +@test method_ex.args[1] === nothing +@test method_ex.args[4] isa LineNumberNode + +# Argument reassigned in outer scope - no Box needed +@test JuliaLowering.include_string(test_mod, """ +begin + function f_arg_reassign(x) + x = 1 + return ()->x + end + f_arg_reassign(100)() +end +""") == 1 + +# Argument reassigned in if-branch before capture - no Box needed (PR #60567 review) +@test JuliaLowering.include_string(test_mod, """ +begin + function f_arg_if_branch(x, cond) + if cond + x = 5 + return ()->x + end + return x + end + # When closure is returned, it captures the reassigned value + f_arg_if_branch(100, true)() +end +""") == 5 + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_arg_if_branch2(x, cond) + if cond + x = 5 + return ()->x + end + return x + end + # When original value is returned + f_arg_if_branch2(100, false) +end +""") == 100 + +# Variable declared outside loop, assigned inside - needs Box (issue #37690) +@test JuliaLowering.include_string(test_mod, """ +begin + function f_loop_capture() + local f + local x + for k = 1 : 2 + x = k + if k == 1 + f = () -> x + end + end + f() + end + f_loop_capture() +end +""") == 2 + +# Typed local declared outside loop, assigned inside - needs Box (issue #37690) +@test JuliaLowering.include_string(test_mod, """ +begin + function f_typed_local_loop() + local f + local x::Int + for k = 1:2 + x = k + if k == 1 + f = () -> x + end + end + f() + end + f_typed_local_loop() +end +""") == 2 + +# Label can be jumped to, bypassing assignment - needs Box +@test JuliaLowering.include_string(test_mod, """ +let + @goto L + y = 1 + @label L + f = ()->y + f.y +end +""") isa Core.Box + +# Argument reassigned inside loop - needs Box (argument is implicitly declared outside loop) (issue #37690) +@test JuliaLowering.include_string(test_mod, """ +begin + function f_arg_loop(x) + local f + for i in 1:2 + x = i + i == 1 && (f = ()->x;) + end + f() + end + f_arg_loop(0) +end +""") == 2 + +# Variable in while-true loop with break - needs Box (issue #37690) +let x = JuliaLowering.include_string(test_mod, """ + begin + function f_break_loop() + local f + local x + i = 1 + while true + x = i + if i == 1 + f = ()->x + end + i >= 3 && break + i += 1 + end + f.x + end + f_break_loop() + end + """) + @test x isa Core.Box + @test x.contents == 3 +end + +# Variable in while-true loop with post-dominated capture (not captured in a branch) - no Box +let x = JuliaLowering.include_string(test_mod, """ + begin + function f_break_loop2() + local f + local x + i = 1 + while true + x = i + f = ()->x + i >= 3 && break + i += 1 + end + f.x + end + f_break_loop2() + end + """) + @test x isa Int + @test x === 3 +end + +let keep = JuliaLowering.include_string(test_mod, """ + begin + function f_for_after_capture(cond) + if cond + keep = Set{Base.PkgId}() + return ()->keep + end + for x in 1:3; end + end + f_for_after_capture(true).keep + end + """) + @test keep isa Set{Base.PkgId} + @test keep == Set{Base.PkgId}() +end + +# Function where arguments are captured into closure and assigned (boxed) +@test JuliaLowering.include_string(test_mod, """ +begin + function f_arg_captured_assigned(x) + function g() + x = 10 + end + g() + x + end + f_arg_captured_assigned(1) +end +""") == 10 + +# Closure declaration with no methods +@test JuliaLowering.include_string(test_mod, """ +begin + local no_method_f + function no_method_f + end + no_method_f +end +""") isa Function + +# Closure with keyword arguments +@test JuliaLowering.include_string(test_mod, """ +let y = 10 + function f_kw_closure(; x=1) + x + y + end + (f_kw_closure(), f_kw_closure(x=5)) +end +""") == (11, 15) + +# Adding kw methods to kw let-function +@test JuliaLowering.include_string(test_mod, """ +let f(a; kw1 = nothing, kw2 = nothing) = "outer" + f(::Integer; kwargs...) = "call me" + f(1; kw1 = 1, kw2 = 2) +end +""") == "call me" + +# Currently an error in both lowering implementations (closure-conversion ordering) +@test_broken JuliaLowering.include_string(test_mod, """ +let f(a; kw1 = nothing, kw2 = nothing) = "outer" + let + f(::Integer; kwargs...) = error("call me") + end + f(1; kw1 = 1, kw2 = 2) +end +""") == "outer" + +# Self-reference in let-function +@test JuliaLowering.include_string(test_mod, """ +let f(x) = x <= 0 ? x : f(x-1) + f(5) +end +""") == 0 +@test_throws UndefVarError JuliaLowering.include_string(test_mod, """ +let f(x::typeof(f)) = x + f(f) +end +""") isa Function # desirable? + +# Self-reference in let-function default args +@test JuliaLowering.include_string(test_mod, """ +let f(x=f) = x + f() +end +""") isa Function +@test_throws UndefVarError JuliaLowering.include_string(test_mod, """ +let f(x::typeof(f)) = x + f(f) +end +""") isa Function +@test JuliaLowering.include_string(test_mod, """ +let f(;x=f) = x + f() +end +""") isa Function +@test_throws UndefVarError JuliaLowering.include_string(test_mod, """ +let f(;x::typeof(f)) = x + f(x=f) +end +""") isa Function + +# Anonymous function syntax with `function` +@test JuliaLowering.include_string(test_mod, """ +begin + local y = 2 + call_it(function (x) x + y end, 3) +end +""") == 5 + +# Closure where static parameter is captured +@test JuliaLowering.include_string(test_mod, """ +begin + function f_static_param_capture(::T) where T + function g() + T + end + g() + end + f_static_param_capture(1) +end +""") == Int + +# Closure with static parameter that may be undefined +JuliaLowering.include_string(test_mod, """ +function f_undef_static_param(x::Union{T,Nothing}) where T + function inner() + return T + end + inner +end +""") +@test_throws UndefVarError test_mod.f_undef_static_param(nothing)() +@test test_mod.f_undef_static_param(42)() == Int + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_inner_sp(x::T) where T + function inner(y::U) where U + (T, U) + end + (T, inner("foo")) + end + f_inner_sp(1) +end +""") == (Int, (Int, String)) + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_complex_arg_sp(a) + function inner(x::(let z = T; Vector{z} end)) where {T <: Integer} + T, typeof(x), @isdefined(z) + end + inner(a) + end + f_complex_arg_sp([1,2,3]) +end +""") == (Int, Vector{Int}, false) + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_inner_rt_sp(a) + function inner(x::T)::Tuple{T, Vector{T}} where T + (x,T[x]) + end + inner(a) + end + f_inner_rt_sp(1), f_inner_rt_sp("foo") +end +""") == ((1, [1]), ("foo", ["foo"])) + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_many_closure_sp() + function (); function (); function (x::T) where T; (x, T) end; end; end + end + f_many_closure_sp()()()(1) +end +""") == (1, Int) + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_many_closure_sp_capt(x::T) where T + function (); function (); function (); (x, T) end; end; end + end + f_many_closure_sp_capt(1)()()() +end +""") == (1, Int) + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_argcapt_sp(x::T) where T + (inner_x::T)->(x, inner_x, T) + end + f_argcapt_sp(1)(2) +end +""") == (1, 2, Int) + +# Inner method typevar `U` depending on a static parameter `T`: the hoisted +# method def for `inner` captures `T` as a closure type parameter, making +# `inner` parametric on `T`. This doesn't work in flisp (UndefVarError). +@test JuliaLowering.include_string(test_mod, """ +begin + function f_typevarcapt_sp(x::T) where T + function inner(y::U) where {U<:T} + (x,y,T,U) + end + end + f_typevarcapt_sp(1)(2) +end +""") == (1,2,Int,Int) + +# https://github.com/JuliaLang/JuliaLowering.jl/issues/134#issuecomment-3739626003 +JuliaLowering.include_string(test_mod, """ +function f_update_outer_capture() + local response # declare outside closure + f = ()->begin + response = 1 + end + f() + return (f, response) +end +""") +let (f, response) = test_mod.f_update_outer_capture() + @test f.response isa Core.Box + @test response == 1 +end + +# https://github.com/JuliaLang/JuliaLowering.jl/issues/147 +JuliaLowering.include_string(test_mod, """ +function f_box_regression147() + function foo() + return true + end + return (()->foo, foo) +end +""") +let (f, foo) = test_mod.f_box_regression147() + @test !(f.foo isa Core.Box) + @test f.foo === foo +end + +# The internal "helper" of an (inner) kwargs function should not be boxed. +JuliaLowering.include_string(test_mod, """ +function f_kwbody_box() + function inner(x; verbose=false) + return verbose ? x : nothing + end + return (inner, inner(1; verbose=true)) +end +""") +let (inner, result) = test_mod.f_kwbody_box() + @test result == 1 + # The kw body closure should be captured directly, not through a Box + kw_body_field = only(filter(f -> contains(string(f), "#kw_body#"), fieldnames(typeof(inner)))) + @test !(getfield(inner, kw_body_field) isa Core.Box) +end + +# Any `let` variables marked always-defined && assigned-once are known to +# dominate their scope, so they should not be boxed even in the presence +# of `@label` +JuliaLowering.include_string(test_mod, """ +function f_let_capture_with_label() + for x in [1,2,3] + let x = x + if false + @goto done + @label done # force the binding analysis to give up + else + return (() -> x,) + end + end + end +end +""") +let (f,) = test_mod.f_let_capture_with_label() + @test !(f.x isa Core.Box) + @test f.x == 1 +end + +JuliaLowering.include_string(test_mod, """ +function f_arg_reassign_with_label(x) + g() = x + if false + @goto done + @label done + end + x = 1 + return (g, x) +end +""") +let (g, x) = test_mod.f_arg_reassign_with_label(42) + @test g.x isa Core.Box + @test g() == 1 + @test x == 1 +end + +@test JuliaLowering.include_string(test_mod, """ +func_in_own_sig(x::typeof(func_in_own_sig)) = (x, 1) +""") isa Function +@test JuliaLowering.include_string(test_mod, """ +func_in_own_sig(func_in_own_sig) +""") == (test_mod.func_in_own_sig, 1) +@test JuliaLowering.include_string(test_mod, """ +function func_in_own_sp(x::T) where {T<:typeof(func_in_own_sp)} +(x, T) +end +""") isa Function +@test JuliaLowering.include_string(test_mod, """ +func_in_own_sp(func_in_own_sp) +""") == (test_mod.func_in_own_sp, typeof(test_mod.func_in_own_sp)) + +@testset "(AI) Captured static parameters" begin + # Captured static parameter used in a closure signature: dispatch must pin + # `T` to the value captured in the closure's type parameter, not re-derive + # it from the argument. + JuliaLowering.include_string(test_mod, """ + function f_sigcapt_sp(x::T) where T + g_sigcapt(y::T) = (y, T) + g_sigcapt + end + """) + @test test_mod.f_sigcapt_sp(1)(2) == (2, Int) + @test_throws MethodError test_mod.f_sigcapt_sp(1)(2.5) + + # All methods of a closure share its captured static parameters, even + # methods that don't reference them. + @test JuliaLowering.include_string(test_mod, """ + begin + function f_multimeth_sp(x::T) where T + g() = T + g(y) = (y, T) + g + end + (f_multimeth_sp(1.5)(), f_multimeth_sp(1.5)(2)) + end + """) == (Float64, (2, Float64)) + + # Static parameter captured by an opaque closure (as a field, since opaque + # closures have no closure type to parameterize) + @test JuliaLowering.include_string(test_mod, """ + begin + function f_oc_sp(x::T) where T + Base.Experimental.@opaque y -> (x, y, T) + end + f_oc_sp(1)(2) + end + """) == (1, 2, Int) + + # A captured sparam's typevar bound may reference another sparam the closure + # doesn't mention at all: `S` must be captured transitively or the hoisted + # method signature contains a free TypeVar. (flisp instead drops the bound.) + JuliaLowering.include_string(test_mod, """ + function f_captsp_bound_dep(x::S, y::T) where {S, T<:AbstractVector{S}} + g_bound_dep(z::T) = (z, T) + g_bound_dep + end + """) + let g = test_mod.f_captsp_bound_dep(1, [1, 2]) + @test g([3, 4]) == ([3, 4], Vector{Int}) + @test_throws MethodError g("nope") + @test_throws MethodError g(Any[1, 2]) # T is pinned to Vector{Int} + end + + # As above, but the dependency `S` is also captured normally by the body + @test JuliaLowering.include_string(test_mod, """ + begin + function f_captsp_bound_dep2(x::S, y::T) where {S, T<:AbstractVector{S}} + g_bound_dep2(z::T) = (z, T, S) + g_bound_dep2 + end + f_captsp_bound_dep2(1, [1, 2])([3, 4]) + end + """) == ([3, 4], Vector{Int}, Int) + + # Same, via a lower bound + JuliaLowering.include_string(test_mod, """ + function f_captsp_lb_dep(x::T, ys::Vector{S}) where {T, S>:T} + g_lb_dep(z::S) = (z, S) + g_lb_dep + end + """) + @test test_mod.f_captsp_lb_dep(1, Any[1.0])("anything") == ("anything", Any) + + # (broken in flisp, JL may or may not have the desired behaviour) Typevar + # bound referencing a static parameter two closure levels up: `g` must + # capture `T` in passing so its value reaches `h`'s creation site. + @test JuliaLowering.include_string(test_mod, """ + begin + function f_typevarcapt_sp_deep(::T) where T + function g_deep() + h_deep(y::U) where {U<:T} = (y, T, U) + h_deep + end + g_deep + end + f_typevarcapt_sp_deep(1.5)()(2.0) + end + """) == (2.0, Float64, Float64) + + # Runtime state of enclosing functions can't be used in hoisted method + # signatures or typevar bounds (flisp errors identically). + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + function f_local_in_closure_sig(x) + g(y::typeof(x)) = y + g + end + """) + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + function f_local_in_closure_spbound(x) + g(y::T) where {T<:typeof(x)} = y + g + end + """) + + # Global method definitions can't be nested inside functions (flisp errors + # identically). + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + function f_nested_global_methdef() + global g_nested_global_methdef + g_nested_global_methdef(x::T) where T = x + end + """) + + #------------------------------------------------------------------------------- + # Static parameter capture: combinations with other closure features + + # do-block closure capturing a static parameter + @test JuliaLowering.include_string(test_mod, """ + begin + function f_do_sp(y::T) where T + call_it(3) do x + (x, y, T) + end + end + f_do_sp(1.5) + end + """) == (3, 1.5, Float64) + + # comprehensions and generators capturing a static parameter + @test JuliaLowering.include_string(test_mod, """ + begin + function f_generator_sp(::T) where T + ([T for _ in 1:2], first(T for _ in 1:1)) + end + f_generator_sp(1) + end + """) == ([Int, Int], Int) + + # Closure defined inside an opaque closure, capturing a static parameter + # through it. This is broken in flisp ("Found raw symbol T in code returned + # from lowering"). + @test JuliaLowering.include_string(test_mod, """ + begin + function f_closure_in_oc_sp(x::T) where T + Base.Experimental.@opaque () -> begin + g() = (x, T) + g() + end + end + f_closure_in_oc_sp(1)() + end + """) == (1, Int) + + # Opaque closure nested inside a regular closure, capturing a static parameter + # through it + @test JuliaLowering.include_string(test_mod, """ + begin + function f_oc_in_closure_sp(x::T) where T + function mid() + Base.Experimental.@opaque () -> (x, T) + end + mid + end + f_oc_in_closure_sp(2)()() + end + """) == (2, Int) + + # Local function overloaded with keyword and plain methods; the captured + # static parameter appears only in a keyword default, which is evaluated in + # the kwcall method, while the body method's signature mentions the closure's + # type unparameterized. + @test JuliaLowering.include_string(test_mod, """ + begin + function f_kw_overload_sp(x::T) where T + g(y; k=T) = (:kw, y, k) + g(y::Int) = (:plain, y) + (g(1), g(1.5), g(1.5; k=2)) + end + f_kw_overload_sp(1) + end + """) == ((:plain, 1), (:kw, 1.5, Int), (:kw, 1.5, 2)) + + # Keyword closure with the captured sparam in both signature and kw default: + # dispatch through the kw sorter still pins `T`. + JuliaLowering.include_string(test_mod, """ + function f_kwsig_sp(x::T) where T + g_kwsig(y::T; k=T) = (y, k, T) + g_kwsig + end + """) + let g = test_mod.f_kwsig_sp(1) + @test g(2) == (2, Int, Int) + @test g(2; k=Int8) == (2, Int8, Int) + @test_throws MethodError g(2.5) + end + + # Self-recursive closure capturing a static parameter + @test JuliaLowering.include_string(test_mod, """ + begin + function f_recursive_sp(x::T) where T + g(n) = n <= 0 ? T : g(n - 1) + g(3) + end + f_recursive_sp(1.5) + end + """) == Float64 + + # Boxed (assigned) captured local and captured static parameter in one closure + @test JuliaLowering.include_string(test_mod, """ + begin + function f_box_plus_sp(x::T) where T + c = 0 + g() = (c += 1; (c, T)) + (g(), g()) + end + f_box_plus_sp(1im) + end + """) == ((1, Complex{Int}), (2, Complex{Int})) + + # @isdefined of a captured static parameter inside a closure; an undefined + # sparam throws at closure creation (as in flisp), not at the @isdefined + JuliaLowering.include_string(test_mod, """ + function f_isdefined_sp(x::Union{T,Nothing}) where T + g_isdefined_sp() = @isdefined(T) + g_isdefined_sp + end + """) + @test test_mod.f_isdefined_sp(1)() + @test_throws UndefVarError test_mod.f_isdefined_sp(nothing) + + # Closure with an anonymous static parameter of its own plus a captured one + @test JuliaLowering.include_string(test_mod, """ + begin + function f_anon_sp_closure(x::T) where T + g(y) where _ = (y, T) + g(2) + end + f_anon_sp_closure(1) + end + """) == (2, Int) + + # `let` inside a closure signature referencing a captured static parameter + @test JuliaLowering.include_string(test_mod, """ + begin + function f_let_sig_sp(::T) where T + g(x::(let v = T; Vector{v} end)) = x + g + end + f_let_sig_sp(1)([1, 2]) + end + """) == [1, 2] + + # Undetermined static parameter throws at closure creation (flisp parity) + @test_throws UndefVarError JuliaLowering.include_string(test_mod, """ + begin + function f_undet_sp_creation(x::T, y::S) where {T, S>:T} + g(z::S) = z + g + end + f_undet_sp_creation(1, 2.5) + end + """) + + # Signature capture through four levels of nested closures + @test JuliaLowering.include_string(test_mod, """ + begin + function f_sig_capture_depth4(x::T) where T + function a() + function b() + function c() + d(y::T) = (y, T) + d(x) + end + c() + end + b() + end + a() + end + f_sig_capture_depth4(42) + end + """) == (42, Int) + + # The closure's own sparam shadows the captured one, with the outer as bound + JuliaLowering.include_string(test_mod, """ + function f_shadow_sp_bound(x::T) where T + g_shadow(y::T) where {T<:T} = (y, T) + g_shadow + end + """) + let g = test_mod.f_shadow_sp_bound(1) + @test g(2) == (2, Int) + @test_throws MethodError g(2.5) # inner T <: outer T == Int + end + + # Vararg length pinned by a captured static parameter + JuliaLowering.include_string(test_mod, """ + function f_vararg_n_sp(::Val{N}) where N + g_vararg_n(xs::Vararg{Int,N}) = xs + g_vararg_n + end + """) + let g = test_mod.f_vararg_n_sp(Val(2)) + @test g(1, 2) == (1, 2) + @test_throws MethodError g(1, 2, 3) # N is pinned to 2 + end + + # Return-type annotation is the only use of the captured static parameter + JuliaLowering.include_string(test_mod, """ + function f_rett_only_sp(x::T) where T + g_rett(y)::T = y + g_rett + end + """) + @test test_mod.f_rett_only_sp(1)(2) === 2 + @test_throws MethodError test_mod.f_rett_only_sp(1)("s") # convert(Int, "s") + + # Two sibling closures capturing the same static parameter (and sharing its + # typevar object in their hoisted method signatures) + @test JuliaLowering.include_string(test_mod, """ + begin + function f_two_closures_sp(x::T) where T + g(y::T) = (y, :g) + h(y::T) = (y, :h) + (g(x), h(x)) + end + f_two_closures_sp(2) + end + """) == ((2, :g), (2, :h)) + + # A closure which captures nothing must be creatable even when the enclosing + # method's sparams are undetermined: closures must capture only the sparams + # they (transitively) reference, not all lexically enclosing ones. + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nocapt_undet_sp(x::Union{T,Nothing}) where T + g_nocapt() = 1 + g_nocapt + end + f_nocapt_undet_sp(nothing)() + end + """) == 1 + + # The static parameter's owner is itself a nested closure: its hoisted typevar + # assignments must be emitted before the inner closure's hoisted methods, + # which reference them (issue found as a segfault from a forward SSA ref). + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nested_owner_body() + function g(x::T) where T + () -> T + end + g + end + f_nested_owner_body()(1)() + end + """) == Int + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nested_owner_sig() + function g(x::T) where T + h(y::T) = (y, T) + h(x) + end + g + end + f_nested_owner_sig()(2.5) + end + """) == (2.5, Float64) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nested_owner_bound() + function g(x::T) where T + h(y::U) where {U<:T} = (y, T, U) + h(x) + end + g + end + f_nested_owner_bound()(3) + end + """) == (3, Int, Int) + + # Sparams owned by two different nesting levels, captured together + @test JuliaLowering.include_string(test_mod, """ + begin + function f_two_level_sps(a::A) where A + function g(x::T) where T + () -> (A, T) + end + g + end + f_two_level_sps(1im)(2.5)() + end + """) == (Complex{Int}, Float64) + + # method_defs in value position (not a top-level sequence point), with a + # nested closure capturing the sparam + @test JuliaLowering.include_string(test_mod, """ + begin + x_value_pos_sp = (f_value_pos_sp(y::T) where T = () -> T) + f_value_pos_sp(42)() + end + """) == Int + + # CC must not lift typevar T above assignment to x + @test JuliaLowering.include_string(test_mod, """ + for x in (Int, Float64) + global f_local_in_tvbounds + f_local_in_tvbounds(y::T) where {T<:x} = T + end + """) == nothing + @test JuliaLowering.include_string(test_mod, """ + f_local_in_tvbounds(1) + """) == Int + @test JuliaLowering.include_string(test_mod, """ + f_local_in_tvbounds(1.0) + """) == Float64 + + @test JuliaLowering.include_string(test_mod, """ + global g_tvbound = Int + for i in 1:2 + global f_global_in_tvbounds, g_tvbound + f_global_in_tvbounds(y::T) where {T<:g_tvbound} = (i, T) + g_tvbound = Float64 + end + f_global_in_tvbounds(1), f_global_in_tvbounds(1.5) + """) == ((1, Int), (2, Float64)) +end + +# questionable test: g_shadowed_by_sparam is not an sparam of the single-arg +# version of `f`, so capture into the single-arg method's body resolves to the +# outer typevar instead of an sparam, which scope resolution re-resolves in +# global scope (typevars are only visible in the same lambda). However, it +# would probably make more sense to keep the sparam in both methods, and have +# the inner lambda's sig refer to the sparam instead of the global. +@test_warn r"declares type variable g_shadowed_by_sparam but does not use it" @test JuliaLowering.include_string(test_mod, """ +global g_shadowed_by_sparam = Int +function f_sp_in_sig_in_lam_in_optarg( + x, y=((z::g_shadowed_by_sparam)->z)) where g_shadowed_by_sparam + y +end +f_sp_in_sig_in_lam_in_optarg(1.)(2) +""") == 2 + +@testset "sparam captured into static_eval" begin + @test JuliaLowering.include_string(test_mod, """ + function f_capt_sp_in_ccall_rett(v::Vector{T}) where {T} + g = () -> ccall(:memset, Ptr{T}, (Ptr{Cvoid}, Cint, Csize_t), v, 0, 0) + g() isa Ptr + end + f_capt_sp_in_ccall_rett([1, 2]) + """) == true + @test JuliaLowering.include_string(test_mod, """ + function f_capt_sp_in_ccall_argtype(v::Vector{T}) where {T} + g = () -> ccall(:memset, Ptr{Cvoid}, (Ptr{T}, Cint, Csize_t), v, 0, 0) + g() isa Ptr + end + f_capt_sp_in_ccall_argtype([1, 2]) + """) == true + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + function f_local_in_ccall_in_closure(v) + T = typeof(v) + g = () -> ccall(:memset, Ptr{T}, (Ptr{Cvoid}, Cint, Csize_t), v, 0, 0) + g() + end + """) +end + +# Global methods are not lifted out of control flow. Local methods also aren't, +# except for in loops +@testset "local and global functions in control flow" begin + @test JuliaLowering.include_string(@newmod(), """ + begin + local cltypes = [] + local out = [] + for i in 1:2 + local function f(); 1; end + push!(cltypes, typeof(f)) + push!(out, f()) + end + (cltypes[1] == cltypes[2], out) + end + """) == (true, [1,1]) + @test JuliaLowering.include_string(@newmod(), """ + begin + local cltypes = [] + local out = [] + for i in 1:2 + local function f(); i; end # captures `i` + push!(cltypes, typeof(f)) + push!(out, f()) + end + (cltypes[1] === cltypes[2], out) + end + """) == (true, [1,2]) + # i not defined + @test_throws UndefVarError JuliaLowering.include_string(@newmod(), """begin + local cltypes = [] + local out = [] + for i in 1:2 + local function f(x::T) where {T<:typeof(i)}; x; end + push!(cltypes, typeof(f)) + push!(out, f()) + end + (cltypes[1] === cltypes[2], out) + end""") + + @test JuliaLowering.include_string(@newmod(), """ + if false + global function f(); 1; end + end + (isdefined(@__MODULE__, :f)) + """) == false + @test JuliaLowering.include_string(@newmod(), """ + for i in 1:0 + global function f(); 1; end + end + (isdefined(@__MODULE__, :f)) + """) == false # flisp: true but with 0 methods + @test JuliaLowering.include_string(@newmod(), """ + for i in 1:1 + global function f(); 1; end + end + (isdefined(@__MODULE__, :f)) + """) == true + + # short-form (broken in flisp) + @test JuliaLowering.include_string(@newmod(), """ + if false + global function f end + end + (isdefined(@__MODULE__, :f)) + """) == false + @test JuliaLowering.include_string(@newmod(), """ + for i in 1:0 + global function f end + end + (isdefined(@__MODULE__, :f)) + """) == false + @test JuliaLowering.include_string(@newmod(), """ + for i in 1:1 + global function f end + end + (isdefined(@__MODULE__, :f)) + """) == true +end + +# A function definition is treated as an assignment to a name +@testset "inner functions sharing names" begin + @test JuliaLowering.include_string(test_mod, """ + begin + function f_midfunc_redefines_inner(exec_mid) + function inner(); 1; end + function mid() + function inner(); 2; end + end + exec_mid && mid() + inner + end + (f_midfunc_redefines_inner(false)(), f_midfunc_redefines_inner(true)()) + end + """) == (1,2) + + # Inner methods in control flow are known to be buggy/discouraged + @test_warn r"Method definition inner\(\).*overwritten" @test JuliaLowering.include_string(test_mod, """ + begin + function f_if_redefines_inner(cond) + function inner(); 1; end + if cond + function inner(); 2; end + end + inner + end + (f_if_redefines_inner(false)(), f_if_redefines_inner(true)()) + end + """) == (2,2) + + @test_warn r"Method definition inner\(\).*overwritten" @test JuliaLowering.include_string(test_mod, """ + begin + function f_let_redefines_inner() + local a,b + a = function inner(); 1; end + let + b = function inner(); 2; end + end + a(), b(), inner() + end + f_let_redefines_inner() + end + """) == (2,2,2) + + # like normal locals, reassign doesn't happen if you add a local decl + @test JuliaLowering.include_string(test_mod, """ + begin + function f_midfunc_redefines_inner(exec_mid) + function inner(); 1; end + function mid() + local inner + function inner(); 2; end + end + exec_mid && mid() + inner + end + (f_midfunc_redefines_inner(false)(), f_midfunc_redefines_inner(true)()) + end + """) == (1,1) + @test JuliaLowering.include_string(test_mod, """ + begin + function f_let_local_shadows_inner() + local a,b + a = function inner(); 1; end + let + local inner + b = function inner(); 2; end + end + a(), b(), inner() + end + f_let_local_shadows_inner() + end + """) == (1,2,1) + + @test JuliaLowering.include_string(test_mod, """ + begin + function multimethod_inner(arg) + function inner(x::Int); x; end + function inner(x::Float64); x; end + (length(methods(inner)), inner(arg)) + end + multimethod_inner(1), multimethod_inner(1.0) + end + """) == ((2, 1), (2, 1.0)) + @test JuliaLowering.include_string(test_mod, """ + begin + function multimethod_let_inner(arg) + function inner(x::Int); x; end + let + function inner(x::Float64); x; end + end + (length(methods(inner)), inner(arg)) + end + multimethod_let_inner(1), multimethod_let_inner(1.0) + end + """) == ((2, 1), (2, 1.0)) + # reassign does not add method + @test JuliaLowering.include_string(test_mod, """ + begin + function f_mid_does_not_add_method(exec_mid) + function inner(x::Int); x; end + function mid() + function inner(x::Float64); x; end + end + exec_mid && mid() + (length(methods(inner)), inner(exec_mid ? 123.0 : 123)) + end + (f_mid_does_not_add_method(false), f_mid_does_not_add_method(true)) + end + """) == ((1, 123), (1, 123.0)) + + # Closure info is keyed on pair (binding_id, lambda_id) + @testset "(AI) inner functions with differing captures across lambdas" begin + @test JuliaLowering.include_string(test_mod, """ + begin + function f_diffcap(a, b) + inner() = a + v1 = inner + function mid() + inner() = a + b + end + mid() + v2 = inner + (v1(), v2()) + end + f_diffcap(1, 10) + end + """) == (1, 11) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_diffcap_types(a, b) + inner() = a + t1 = typeof(inner) + function mid() + inner() = a + b + end + mid() + t1 === typeof(inner) + end + f_diffcap_types(1, 10) + end + """) == false + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_sp_redef(::Type{T}) where {T} + g() = T + v1 = g + function mid(::Type{S}) where {S} + g() = (T, S) + end + mid(Float64) + (v1(), g()) + end + f_sp_redef(Int) + end + """) == (Int, (Int, Float64)) + + @test JuliaLowering.include_string(test_mod, """ + let + function closure2(::Type{T}) where {T} + function f() + T + end + f + end + closure2(Int)() + + x2 = 2 + function closure3(a) + b = "whatever" + f(c) = (a, b, c) + b = x2 + return f + end + f = closure3(1) + f(3) + end + """) == (1,2,3) + end +end + +@testset "captured type declarations" begin + JuliaLowering.include_string(test_mod, """ + function f_boxed_typed_capture(v, k) + if k == -1 + k::Int = maximum(v) + end + findall(x -> x >= k, v) + end + """) + @test test_mod.f_boxed_typed_capture([3, 1, 2], -1) == [1] + @test only(Base.return_types(test_mod.f_boxed_typed_capture, (Vector{Int}, Int))) === + Vector{Int} + + # declared types may reference locals in the outer function + JuliaLowering.include_string(test_mod, """ + function f_boxed_sparam_typed(v::Vector{T}, k) where T + if k == -1 + k::T = maximum(v) + end + findall(x -> x >= k, v) + end + function f_boxed_localvar_typed(v, k) + T = Int + if k == -1 + k::T = maximum(v) + end + findall(x -> x >= k, v) + end + """) + @test test_mod.f_boxed_sparam_typed([3, 1, 2], -1) == [1] + @test only(Base.return_types(test_mod.f_boxed_sparam_typed, (Vector{Int}, Int))) === + Vector{Int} + @test test_mod.f_boxed_localvar_typed([3, 1, 2], -1) == [1] + @test only(Base.return_types(test_mod.f_boxed_localvar_typed, (Vector{Int}, Int))) === + Vector{Int} +end + +@testset "(AI) regression tests: inner function gets its value after method def" begin + ex_local_alias = :(let + A{T} = Vector{T} + A(x::A) = 1 + A([2]) + end) + @test fl_eval(Module(), ex_local_alias) == 1 + @test jl_eval(Module(), ex_local_alias) == 1 + + ex_alias_semantics = :(let + A{T} = Vector{T} + A(x::A) = 1 + r2 = try; A(2); catch e; nameof(typeof(e)); end + (A([2]), r2, A isa Function) + end) + @test fl_eval(Module(), ex_alias_semantics) == (1, :MethodError, true) + @test jl_eval(Module(), ex_alias_semantics) == (1, :MethodError, true) + + ex_typeof_value_sig = :(let + f = 2 + f(x::typeof(f)) = 1 + f(2) + end) + @test fl_eval(Module(), ex_typeof_value_sig) == 1 + @test jl_eval(Module(), ex_typeof_value_sig) == 1 + + ex_isdefined_sig = :(let + r = try + f(x::((@isdefined f) ? Int : String)) = x + f(1) + catch e + nameof(typeof(e)) + end + r + end) + @test fl_eval(Module(), ex_isdefined_sig) == :MethodError + @test jl_eval(Module(), ex_isdefined_sig) == :MethodError + + ex_where_self = :(let + r = try + f(x::T) where {T <: typeof(f)} = 1 + f(f) + catch e + nameof(typeof(e)) + end + r + end) + @test fl_eval(Module(), ex_where_self) == :UndefVarError + @test jl_eval(Module(), ex_where_self) == :UndefVarError + + ex_kwarg_self = :(let + r = try + A = Vector + A(; x::A = [7]) = x + A(x = [2]) + catch e + nameof(typeof(e)) + end + r + end) + @test fl_eval(Module(), ex_kwarg_self) == :TypeError + @test jl_eval(Module(), ex_kwarg_self) == :TypeError + + ex_error_leaves_name = :(let + A = Vector + ok = try + A(x::3) = x + :defined + catch e + nameof(typeof(e)) + end + (ok, A === Vector, A isa Function) + end) + @test fl_eval(Module(), ex_error_leaves_name) == (:ArgumentError, true, false) + @test jl_eval(Module(), ex_error_leaves_name) == (:ArgumentError, true, false) + + ex_sig_assigns_capture = :(let + local y + f(x::(begin; y = 2; Int; end)) = x + y + f(1) + end) + @test fl_eval(Module(), ex_sig_assigns_capture) == 3 + @test jl_eval(Module(), ex_sig_assigns_capture) == 3 + + # broken in the same way for both, but shouldn't crash + ex_undef_inner = :(function f() + for i in 1:0 + g() = 1 + end + g() = 2 + g() + end; f()) + @test_warn "Method definition g() in module anonymous at" @test_throws UndefVarError fl_eval(Module(), ex_undef_inner) + @test_warn "Method definition g() in module anonymous at" @test_throws UndefVarError jl_eval(Module(), ex_undef_inner) + + # flisp's "scan the thunk for non-short method defs upon seeing a short + # method def" behaviour is not currently implemented + ex_undef_inner_short = :(function f() + for i in 1:0 + function g end + end + g() = 2 + g() + end; f()) + @test fl_eval(Module(), ex_undef_inner_short) == 2 + @test_broken jl_eval(Module(), ex_undef_inner_short) == 2 +end diff --git a/JuliaLowering/test/closures_ir.jl b/JuliaLowering/test/closures_ir.jl new file mode 100644 index 0000000000000..0005279636e9c --- /dev/null +++ b/JuliaLowering/test/closures_ir.jl @@ -0,0 +1,1174 @@ +######################################## +# Simple closure - single-assigned capture before control flow doesn't need Box +let + x = 1 + function f(y) + x + y + end +end +#--------------------- +1 (= slot₂/x 1) +2 (call core.svec) +3 (call core.svec :x) +4 (call core.svec false) +5 (call JuliaLowering.eval_closure_type TestMod :#f#f##0 %₂ %₃ %₄) +6 latestworld +7 TestMod.#f#f##0 +8 (call core._typeof_captured_variable slot₂/x) +9 (call core.apply_type %₇ %₈) +10 (new %₉ slot₂/x) +11 TestMod.#f#f##0 +12 (call core.svec %₁₁ core.Any) +13 (call core.svec) +14 SourceLocation::3:5 +15 (call core.svec %₁₂ %₁₃ %₁₄) +16 (call core.define_method TestMod core.nothing %₁₅ + --- code_info + slots: [slot₁/#self#(!read) slot₂/y] + 1 TestMod.+ + 2 (call core.getfield slot₁/#self# :x) + 3 (call %₁ %₂ slot₂/y) + 4 (return %₃) +17 latestworld +18 (= slot₁/f %₁₀) +19 slot₁/f +20 (return %₁₉) + +######################################## +# Closure declaration with no methods +begin + local no_method_f + function no_method_f + end +end +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#no_method_f#no_method_f##0 %₁ %₂ %₃) +5 latestworld +6 TestMod.#no_method_f#no_method_f##0 +7 (new %₆) +8 (= slot₁/no_method_f %₇) +9 slot₁/no_method_f +10 (return %₉) + +######################################## +# Closure which sets the value of a captured variable +let + x = 1 + function f(y) + x = 2 + end +end +#--------------------- +1 (= slot₂/x (call core.Box)) +2 1 +3 slot₂/x +4 (call core.setfield! %₃ :contents %₂) +5 (call core.svec) +6 (call core.svec :x) +7 (call core.svec true) +8 (call JuliaLowering.eval_closure_type TestMod :#f#f##1 %₅ %₆ %₇) +9 latestworld +10 TestMod.#f#f##1 +11 slot₂/x +12 (new %₁₀ %₁₁) +13 TestMod.#f#f##1 +14 (call core.svec %₁₃ core.Any) +15 (call core.svec) +16 SourceLocation::3:5 +17 (call core.svec %₁₄ %₁₅ %₁₆) +18 (call core.define_method TestMod core.nothing %₁₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/y(!read)] + 1 2 + 2 (call core.getfield slot₁/#self# :x) + 3 (call core.setfield! %₂ :contents %₁) + 4 (return %₁) +19 latestworld +20 (= slot₁/f %₁₂) +21 slot₁/f +22 (return %₂₁) + +######################################## +# Function where arguments are captured into a closure and assigned +function f(x) + function g() + x = 10 + end + g() + x +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.svec) +4 (call core.svec :x) +5 (call core.svec true) +6 (call JuliaLowering.eval_closure_type TestMod :#g#f##2 %₃ %₄ %₅) +7 latestworld +8 TestMod.#g#f##2 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::2:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 10 + 2 (call core.getfield slot₁/#self# :x) + 3 (call core.setfield! %₂ :contents %₁) + 4 (return %₁) +14 latestworld +15 TestMod.f +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(single_assign) slot₃/g(single_assign,called) slot₄/x(!read,maybe_undef) slot₅/x(!read)] + 1 (= slot₅/x slot₂/x) + 2 slot₅/x + 3 (= slot₅/x (call core.Box %₂)) + 4 TestMod.#g#f##2 + 5 slot₅/x + 6 (new %₄ %₅) + 7 (= slot₃/g %₆) + 8 slot₃/g + 9 (call %₈) + 10 slot₅/x + 11 (call core.isdefined %₁₀ :contents) + 12 (gotoifnot %₁₁ label₁₄) + 13 (goto label₁₆) + 14 (newvar slot₄/x) + 15 slot₄/x + 16 (call core.getfield %₁₀ :contents) + 17 (return %₁₆) +22 latestworld +23 TestMod.f +24 (return %₂₃) + +######################################## +# Argument reassigned in outer scope then captured - no Box needed +# (from PR #60567 review) +function foo(x) + if rand(Bool) + x = 5 + return ()->x + end + return x +end +#--------------------- +1 (call core.define_method TestMod :foo) +2 latestworld +3 (call core.svec) +4 (call core.svec :x) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#foo##0#foo##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#foo##0#foo##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::4:16 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :x) + 2 (return %₁) +14 latestworld +15 TestMod.foo +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.foo %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(single_assign) slot₃/#->#(single_assign) slot₄/x(!read)] + 1 (= slot₄/x slot₂/x) + 2 (newvar slot₃/#->#) + 3 TestMod.rand + 4 TestMod.Bool + 5 (call %₃ %₄) + 6 (gotoifnot %₅ label₁₇) + 7 (= slot₄/x 5) + 8 TestMod.#foo##0#foo##1 + 9 slot₄/x + 10 (call core._typeof_captured_variable %₉) + 11 (call core.apply_type %₈ %₁₀) + 12 slot₄/x + 13 (new %₁₁ %₁₂) + 14 (= slot₃/#-># %₁₃) + 15 slot₃/#-># + 16 (return %₁₅) + 17 slot₄/x + 18 (return %₁₇) +22 latestworld +23 TestMod.foo +24 (return %₂₃) + +######################################## +# Closure where a local `x` is captured but not boxed +function f(x) + function g() + y = x + end + z = x +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.svec) +4 (call core.svec :x) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#g#f##3 %₃ %₄ %₅) +7 latestworld +8 TestMod.#g#f##3 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::2:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/y(!read,single_assign)] + 1 (call core.getfield slot₁/#self# :x) + 2 (= slot₂/y %₁) + 3 (return %₁) +14 latestworld +15 TestMod.f +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/g(single_assign) slot₄/z(!read,single_assign)] + 1 TestMod.#g#f##3 + 2 (call core._typeof_captured_variable slot₂/x) + 3 (call core.apply_type %₁ %₂) + 4 (new %₃ slot₂/x) + 5 (= slot₃/g %₄) + 6 slot₂/x + 7 (= slot₄/z %₆) + 8 (return %₆) +22 latestworld +23 TestMod.f +24 (return %₂₃) + +######################################## +# Closure where a static parameter of an outer function is captured +function f(::T) where T + function g() + use(T) + end +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.TypeVar :T) +4 (call core.svec :T) +5 (call core.svec) +6 (call core.svec) +7 (call JuliaLowering.eval_closure_type TestMod :#g#f##4 %₄ %₅ %₆) +8 latestworld +9 TestMod.#g#f##4 +10 (call core.apply_type %₉ %₃) +11 (call core.svec %₁₀) +12 (call core.svec %₃) +13 SourceLocation::2:5 +14 (call core.svec %₁₁ %₁₂ %₁₃) +15 (call core.define_method TestMod core.nothing %₁₄ + --- code_info + slots: [slot₁/#self#(!read)] + 1 TestMod.use + 2 static_parameter₁ + 3 (call %₁ %₂) + 4 (return %₃) +16 latestworld +17 TestMod.f +18 (call core.TypeEqOf %₁₇) +19 (call core.svec %₁₈ %₃) +20 (call core.svec %₃) +21 SourceLocation::1:1 +22 (call core.svec %₁₉ %₂₀ %₂₁) +23 (call core.define_method TestMod TestMod.f %₂₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read) slot₃/g(single_assign)] + 1 TestMod.#g#f##4 + 2 static_parameter₁ + 3 (call core.apply_type %₁ %₂) + 4 (new %₃) + 5 (= slot₃/g %₄) + 6 slot₃/g + 7 (return %₆) +24 latestworld +25 TestMod.f +26 (return %₂₅) + +######################################## +# Closure captures with `isdefined` +function f(x) + function g() + z = 3 + (@isdefined(x), # unboxed, always defined capture + @isdefined(y), # boxed capture + @isdefined(z)) # normal local var + end + y = 2 + (@isdefined(y), # boxed local + @isdefined(x)) # always defined local (function arg) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.svec) +4 (call core.svec :x :y) +5 (call core.svec false true) +6 (call JuliaLowering.eval_closure_type TestMod :#g#f##5 %₃ %₄ %₅) +7 latestworld +8 TestMod.#g#f##5 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::2:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/z(single_assign)] + 1 (= slot₂/z 3) + 2 (call core.getfield slot₁/#self# :y) + 3 (call core.isdefined %₂ :contents) + 4 (isdefined slot₂/z) + 5 (call core.tuple true %₃ %₄) + 6 (return %₅) +14 latestworld +15 TestMod.f +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/g(single_assign) slot₄/y(single_assign)] + 1 (= slot₄/y (call core.Box)) + 2 TestMod.#g#f##5 + 3 (call core._typeof_captured_variable slot₂/x) + 4 (call core.apply_type %₂ %₃) + 5 slot₄/y + 6 (new %₄ slot₂/x %₅) + 7 (= slot₃/g %₆) + 8 2 + 9 slot₄/y + 10 (call core.setfield! %₉ :contents %₈) + 11 slot₄/y + 12 (call core.isdefined %₁₁ :contents) + 13 (call core.tuple %₁₂ true) + 14 (return %₁₃) +22 latestworld +23 TestMod.f +24 (return %₂₃) + +######################################## +# Nested captures - here `g` captures `x` because it is needed to initialize +# the closure `h` which captures both `x` and `y`. +# [method_filter: #g_nest#f_nest##0] +function f_nest(x) + function g_nest(y) + function h_nest(z) + (x,y,z) + end + end +end +#--------------------- +slots: [slot₁/#self#(!read) slot₂/y slot₃/h_nest(single_assign)] +1 TestMod.#h_nest#f_nest##1 +2 (call core.getfield slot₁/#self# :x) +3 (call core._typeof_captured_variable %₂) +4 (call core._typeof_captured_variable slot₂/y) +5 (call core.apply_type %₁ %₃ %₄) +6 (call core.getfield slot₁/#self# :x) +7 (new %₅ %₆ slot₂/y) +8 (= slot₃/h_nest %₇) +9 slot₃/h_nest +10 (return %₉) + +######################################## +# Global method capturing local variables +begin + local x = 1 + function f() + x = x + 1 + end +end +#--------------------- +1 (= slot₁/x (call core.Box)) +2 1 +3 slot₁/x +4 (call core.setfield! %₃ :contents %₂) +5 (call core.define_method TestMod :f) +6 latestworld +7 TestMod.f +8 (call core.TypeEqOf %₇) +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::3:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read,maybe_undef)] + 1 TestMod.+ + 2 (captured_local 1) + 3 (call core.isdefined %₂ :contents) + 4 (gotoifnot %₃ label₆) + 5 (goto label₈) + 6 (newvar slot₂/x) + 7 slot₂/x + 8 (call core.getfield %₂ :contents) + 9 (call %₁ %₈ 1) + 10 (captured_local 1) + 11 (call core.setfield! %₁₀ :contents %₉) + 12 (return %₉) +14 slot₁/x +15 (call core.svec %₁₄) +16 (call JuliaLowering.replace_captured_locals %₁₃ %₁₅) +17 (call core.define_method TestMod TestMod.f %₁₂ %₁₆) +18 latestworld +19 TestMod.f +20 (return %₁₉) + +######################################## +# Anonymous function syntax with -> +x -> x*x +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#1#2 %₁ %₂ %₃) +5 latestworld +6 TestMod.#1#2 +7 (new %₆) +8 TestMod.#1#2 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:1 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x] + 1 TestMod.* + 2 (call %₁ slot₂/x slot₂/x) + 3 (return %₂) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 (return %₁₆) + +######################################## +# Anonymous function syntax with `function` +function (x) + x*x +end +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#3#4 %₁ %₂ %₃) +5 latestworld +6 TestMod.#3#4 +7 (new %₆) +8 TestMod.#3#4 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:1 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x] + 1 TestMod.* + 2 (call %₁ slot₂/x slot₂/x) + 3 (return %₂) +14 latestworld +15 (= slot₁/#anon# %₇) +16 slot₁/#anon# +17 (return %₁₆) + +######################################## +# `do` blocks +f(x; a=1) do y + y + 2 +end +#--------------------- +1 TestMod.f +2 (call core.tuple :a) +3 (call core.apply_type core.NamedTuple %₂) +4 (call core.tuple 1) +5 (call %₃ %₄) +6 (call core.svec) +7 (call core.svec) +8 (call core.svec) +9 (call JuliaLowering.eval_closure_type TestMod :#5#6 %₆ %₇ %₈) +10 latestworld +11 TestMod.#5#6 +12 (new %₁₁) +13 TestMod.#5#6 +14 (call core.svec %₁₃ core.Any) +15 (call core.svec) +16 SourceLocation::1:10 +17 (call core.svec %₁₄ %₁₅ %₁₆) +18 (call core.define_method TestMod core.nothing %₁₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/y] + 1 TestMod.+ + 2 (call %₁ slot₂/y 2) + 3 (return %₂) +19 latestworld +20 (= slot₁/#-># %₁₂) +21 slot₁/#-># +22 TestMod.x +23 (call core.kwcall %₅ %₁ %₂₁ %₂₂) +24 (return %₂₃) + +######################################## +# Error: Static parameter clashing with closure name +function f(::g) where {g} + function g() + end +end +#--------------------- +LoweringError: +function f(::g) where {g} + function g() +# ╙ ── cannot overwrite a static parameter + end +end + +######################################## +# Opaque closure (y is single-assigned before capture, no Box needed) +let y = 1 + Base.Experimental.@opaque (x, z::T)->2x + y - z +end +#--------------------- +1 1 +2 (= slot₁/y %₁) +3 TestMod.T +4 (call core.apply_type core.Tuple core.Any %₃) +5 (call core.apply_type core.Union) +6 --- opaque_closure_method core.nothing 2 false SourceLocation::2:31 + slots: [slot₁/#self#(!read) slot₂/x slot₃/z] + 1 TestMod.- + 2 TestMod.+ + 3 TestMod.* + 4 (call %₃ 2 slot₂/x) + 5 (call core.getfield slot₁/#self# 1) + 6 (call %₂ %₄ %₅) + 7 (call %₁ %₆ slot₃/z) + 8 (return %₇) +7 (new_opaque_closure %₄ %₅ core.Any true %₆ slot₁/y) +8 (return %₇) + +######################################## +# Opaque closure with `...` +let + Base.Experimental.@opaque (x, ys...)->ys +end +#--------------------- +1 (call core.apply_type core.Vararg core.Any) +2 (call core.apply_type core.Tuple core.Any %₁) +3 (call core.apply_type core.Union) +4 --- opaque_closure_method core.nothing 2 true SourceLocation::2:31 + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/ys] + 1 slot₃/ys + 2 (return %₁) +5 (new_opaque_closure %₂ %₃ core.Any true %₄) +6 (return %₅) + +######################################## +# Error: Opaque closure with default args +Base.Experimental.@opaque (x=1)->2x +#--------------------- +LoweringError: +Base.Experimental.@opaque (x=1)->2x +# └─┘ ── opaque closure cannot have optional or keyword arguments + +######################################## +# Mutually recursive closures +let + function recursive_a() + recursive_b() + end + function recursive_b() + recursive_a() + end +end +#--------------------- +1 (= slot₂/recursive_b (call core.Box)) +2 (call core.svec) +3 (call core.svec :recursive_b) +4 (call core.svec true) +5 (call JuliaLowering.eval_closure_type TestMod :#recursive_a#recursive_a##0 %₂ %₃ %₄) +6 latestworld +7 TestMod.#recursive_a#recursive_a##0 +8 slot₂/recursive_b +9 (new %₇ %₈) +10 TestMod.#recursive_a#recursive_a##0 +11 (call core.svec %₁₀) +12 (call core.svec) +13 SourceLocation::2:5 +14 (call core.svec %₁₁ %₁₂ %₁₃) +15 (call core.define_method TestMod core.nothing %₁₄ + --- code_info + slots: [slot₁/#self#(!read) slot₂/recursive_b(!read,maybe_undef)] + 1 (call core.getfield slot₁/#self# :recursive_b) + 2 (call core.isdefined %₁ :contents) + 3 (gotoifnot %₂ label₅) + 4 (goto label₇) + 5 (newvar slot₂/recursive_b) + 6 slot₂/recursive_b + 7 (call core.getfield %₁ :contents) + 8 (call %₇) + 9 (return %₈) +16 latestworld +17 (= slot₁/recursive_a %₉) +18 (call core.svec) +19 (call core.svec :recursive_a) +20 (call core.svec false) +21 (call JuliaLowering.eval_closure_type TestMod :#recursive_b#recursive_b##0 %₁₈ %₁₉ %₂₀) +22 latestworld +23 TestMod.#recursive_b#recursive_b##0 +24 (call core._typeof_captured_variable slot₁/recursive_a) +25 (call core.apply_type %₂₃ %₂₄) +26 (new %₂₅ slot₁/recursive_a) +27 TestMod.#recursive_b#recursive_b##0 +28 (call core.svec %₂₇) +29 (call core.svec) +30 SourceLocation::5:5 +31 (call core.svec %₂₈ %₂₉ %₃₀) +32 (call core.define_method TestMod core.nothing %₃₁ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :recursive_a) + 2 (call %₁) + 3 (return %₂) +33 latestworld +34 slot₂/recursive_b +35 (call core.setfield! %₃₄ :contents %₂₆) +36 slot₂/recursive_b +37 (call core.isdefined %₃₆ :contents) +38 (gotoifnot %₃₇ label₄₀) +39 (goto label₄₂) +40 (newvar slot₃/recursive_b) +41 slot₃/recursive_b +42 (call core.getfield %₃₆ :contents) +43 (return %₄₂) + +######################################## +# Closure with keywords +let y = y_init + function f_kw_closure(; x::X=x_default) + x + y + end +end +#--------------------- +1 TestMod.y_init +2 (= slot₁/y %₁) +3 (call core.svec) +4 (call core.svec :y) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_kw_closure#kw_body#7#8 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_kw_closure#kw_body#7#8 +9 (call core._typeof_captured_variable slot₁/y) +10 (call core.apply_type %₈ %₉) +11 (new %₁₀ slot₁/y) +12 (= slot₂/#f_kw_closure#kw_body#7 %₁₁) +13 (call core.svec) +14 (call core.svec :#f_kw_closure#kw_body#7) +15 (call core.svec false) +16 (call JuliaLowering.eval_closure_type TestMod :#f_kw_closure#f_kw_closure##0 %₁₃ %₁₄ %₁₅) +17 latestworld +18 TestMod.#f_kw_closure#f_kw_closure##0 +19 (call core._typeof_captured_variable slot₂/#f_kw_closure#kw_body#7) +20 (call core.apply_type %₁₈ %₁₉) +21 (new %₂₀ slot₂/#f_kw_closure#kw_body#7) +22 TestMod.#f_kw_closure#kw_body#7#8 +23 TestMod.X +24 TestMod.#f_kw_closure#f_kw_closure##0 +25 (call core.svec %₂₂ %₂₃ %₂₄) +26 (call core.svec) +27 SourceLocation::2:5 +28 (call core.svec %₂₅ %₂₆ %₂₇) +29 (call core.define_method TestMod core.nothing %₂₈ + --- code_info + slots: [slot₁/#f_kw_closure#kw_body#7(!read) slot₂/x slot₃/#self#(!read)] + 1 (meta :nkw 1) + 2 TestMod.+ + 3 (call core.getfield slot₁/#f_kw_closure#kw_body#7 :y) + 4 (call %₂ slot₂/x %₃) + 5 (return %₄) +30 latestworld +31 TestMod.#f_kw_closure#f_kw_closure##0 +32 (call core.svec %₃₁) +33 (call core.svec) +34 SourceLocation::2:5 +35 (call core.svec %₃₂ %₃₃ %₃₄) +36 (call core.define_method TestMod core.nothing %₃₅ + --- code_info + slots: [slot₁/#self#] + 1 (call core.getfield slot₁/#self# :#f_kw_closure#kw_body#7) + 2 TestMod.x_default + 3 (call %₁ %₂ slot₁/#self#) + 4 (return %₃) +37 latestworld +38 (= slot₃/f_kw_closure %₂₁) +39 (call core.typeof core.kwcall) +40 TestMod.#f_kw_closure#f_kw_closure##0 +41 (call core.svec %₃₉ core.NamedTuple %₄₀) +42 (call core.svec) +43 SourceLocation::2:5 +44 (call core.svec %₄₁ %₄₂ %₄₃) +45 (call core.define_method TestMod core.nothing %₄₄ + --- code_info + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self# slot₄/x(!read) slot₅/#kwtmp#] + 1 (newvar slot₄/x) + 2 (newvar slot₅/#kwtmp#) + 3 (call core.isdefined slot₂/kws :x) + 4 (gotoifnot %₃ label₁₅) + 5 (call core.getfield slot₂/kws :x) + 6 TestMod.X + 7 (call core.isa %₅ %₆) + 8 (gotoifnot %₇ label₁₀) + 9 (goto label₁₃) + 10 TestMod.X + 11 (new core.TypeError :keyword argument :x %₁₀ %₅) + 12 (call core.throw %₁₁) + 13 (= slot₅/#kwtmp# %₅) + 14 (goto label₁₇) + 15 TestMod.x_default + 16 (= slot₅/#kwtmp# %₁₅) + 17 slot₅/#kwtmp# + 18 (call top.keys slot₂/kws) + 19 (call core.tuple :x) + 20 (call top.diff_names %₁₈ %₁₉) + 21 (call top.isempty %₂₀) + 22 (gotoifnot %₂₁ label₂₄) + 23 (goto label₂₅) + 24 (call top.kwerr slot₂/kws slot₃/#self#) + 25 (call core.getfield slot₃/#self# :#f_kw_closure#kw_body#7) + 26 (call %₂₅ %₁₇ slot₃/#self#) + 27 (return %₂₆) +46 latestworld +47 slot₃/f_kw_closure +48 (return %₄₇) + +######################################## +# Closure capturing a typed local must also capture the type expression +# [method_filter: #f_captured_typed_local##0] +let T=Blah + x::T = 1.0 + function f_captured_typed_local() + x = 2.0 + end + f_captured_typed_local() + x +end +#--------------------- +slots: [slot₁/#self#(!read) slot₂/tmp(!read)] +1 2.0 +2 (call core.getfield slot₁/#self# :x) +3 (call core.getfield slot₁/#self# :T) +4 (= slot₂/tmp %₁) +5 (call core.isa slot₂/tmp %₃) +6 (gotoifnot %₅ label₈) +7 (goto label₁₀) +8 (call top.convert %₃ slot₂/tmp) +9 (= slot₂/tmp (call core.typeassert %₈ %₃)) +10 slot₂/tmp +11 (call core.setfield! %₂ :contents %₁₀) +12 (return %₁) + +######################################## +# Assignment after if statement doesn't need Box (flisp-compatible save/restore) +function f_after_if(cond) + if cond + println("hello") + end + y = 1 + () -> y +end +#--------------------- +1 (call core.define_method TestMod :f_after_if) +2 latestworld +3 (call core.svec) +4 (call core.svec :y) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_after_if##0#f_after_if##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_after_if##0#f_after_if##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::6:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :y) + 2 (return %₁) +14 latestworld +15 TestMod.f_after_if +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_after_if %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/cond slot₃/#->#(single_assign) slot₄/y(single_assign)] + 1 (newvar slot₃/#->#) + 2 (gotoifnot slot₂/cond label₅) + 3 TestMod.println + 4 (call %₃ "hello") + 5 (= slot₄/y 1) + 6 TestMod.#f_after_if##0#f_after_if##1 + 7 (call core._typeof_captured_variable slot₄/y) + 8 (call core.apply_type %₆ %₇) + 9 (new %₈ slot₄/y) + 10 (= slot₃/#-># %₉) + 11 slot₃/#-># + 12 (return %₁₁) +22 latestworld +23 TestMod.f_after_if +24 (return %₂₃) + +######################################## +# Ternary operator (if expression in value position) doesn't need Box +function f_ternary(x) + y = x > 0 ? x : 0 + () -> y +end +#--------------------- +1 (call core.define_method TestMod :f_ternary) +2 latestworld +3 (call core.svec) +4 (call core.svec :y) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_ternary##0#f_ternary##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_ternary##0#f_ternary##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::3:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :y) + 2 (return %₁) +14 latestworld +15 TestMod.f_ternary +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_ternary %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/#->#(single_assign) slot₄/y(single_assign) slot₅/if_val(!read)] + 1 (newvar slot₃/#->#) + 2 TestMod.> + 3 (call %₂ slot₂/x 0) + 4 (gotoifnot %₃ label₈) + 5 slot₂/x + 6 (= slot₅/if_val %₅) + 7 (goto label₉) + 8 (= slot₅/if_val 0) + 9 slot₅/if_val + 10 (= slot₄/y %₉) + 11 TestMod.#f_ternary##0#f_ternary##1 + 12 (call core._typeof_captured_variable slot₄/y) + 13 (call core.apply_type %₁₁ %₁₂) + 14 (new %₁₃ slot₄/y) + 15 (= slot₃/#-># %₁₄) + 16 slot₃/#-># + 17 (return %₁₆) +22 latestworld +23 TestMod.f_ternary +24 (return %₂₃) + +######################################## +# || guard pattern (value position with early exit) doesn't need Box +function f_or_guard(x) + (x === nothing || x === missing) && return nothing + y = x + () -> y +end +#--------------------- +1 (call core.define_method TestMod :f_or_guard) +2 latestworld +3 (call core.svec) +4 (call core.svec :y) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_or_guard##0#f_or_guard##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_or_guard##0#f_or_guard##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::4:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :y) + 2 (return %₁) +14 latestworld +15 TestMod.f_or_guard +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_or_guard %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/#->#(single_assign) slot₄/y(single_assign) slot₅/if_val(!read)] + 1 (newvar slot₃/#->#) + 2 TestMod.=== + 3 TestMod.nothing + 4 (call %₂ slot₂/x %₃) + 5 (gotoifnot %₄ label₈) + 6 (= slot₅/if_val true) + 7 (goto label₁₁) + 8 TestMod.=== + 9 TestMod.missing + 10 (= slot₅/if_val (call %₈ slot₂/x %₉)) + 11 slot₅/if_val + 12 (gotoifnot %₁₁ label₁₆) + 13 TestMod.nothing + 14 (return %₁₃) + 15 (goto label₁₆) + 16 slot₂/x + 17 (= slot₄/y %₁₆) + 18 TestMod.#f_or_guard##0#f_or_guard##1 + 19 (call core._typeof_captured_variable slot₄/y) + 20 (call core.apply_type %₁₈ %₁₉) + 21 (new %₂₀ slot₄/y) + 22 (= slot₃/#-># %₂₁) + 23 slot₃/#-># + 24 (return %₂₃) +22 latestworld +23 TestMod.f_or_guard +24 (return %₂₃) + +######################################## +# Argument reassigned in outer scope - no Box needed +function f_arg_reassign(x) + x = 1 + return ()->x +end +#--------------------- +1 (call core.define_method TestMod :f_arg_reassign) +2 latestworld +3 (call core.svec) +4 (call core.svec :x) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_arg_reassign##0#f_arg_reassign##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_arg_reassign##0#f_arg_reassign##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::3:12 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :x) + 2 (return %₁) +14 latestworld +15 TestMod.f_arg_reassign +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_arg_reassign %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(single_assign) slot₃/#->#(single_assign) slot₄/x(!read)] + 1 (= slot₄/x slot₂/x) + 2 (= slot₄/x 1) + 3 TestMod.#f_arg_reassign##0#f_arg_reassign##1 + 4 slot₄/x + 5 (call core._typeof_captured_variable %₄) + 6 (call core.apply_type %₃ %₅) + 7 slot₄/x + 8 (new %₆ %₇) + 9 (= slot₃/#-># %₈) + 10 slot₃/#-># + 11 (return %₁₀) +22 latestworld +23 TestMod.f_arg_reassign +24 (return %₂₃) + +######################################## +# Label can be jumped to, bypassing assignment - needs Box +let + @goto L + y = 1 + @label L + ()->y +end +#--------------------- +1 (newvar slot₁/#->#) +2 (= slot₂/y (call core.Box)) +3 (goto label₇) +4 1 +5 slot₂/y +6 (call core.setfield! %₅ :contents %₄) +7 (call core.svec) +8 (call core.svec :y) +9 (call core.svec true) +10 (call JuliaLowering.eval_closure_type TestMod :#9#10 %₇ %₈ %₉) +11 latestworld +12 TestMod.#9#10 +13 slot₂/y +14 (new %₁₂ %₁₃) +15 TestMod.#9#10 +16 (call core.svec %₁₅) +17 (call core.svec) +18 SourceLocation::5:5 +19 (call core.svec %₁₆ %₁₇ %₁₈) +20 (call core.define_method TestMod core.nothing %₁₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/y(!read,maybe_undef)] + 1 (call core.getfield slot₁/#self# :y) + 2 (call core.isdefined %₁ :contents) + 3 (gotoifnot %₂ label₅) + 4 (goto label₇) + 5 (newvar slot₂/y) + 6 slot₂/y + 7 (call core.getfield %₁ :contents) + 8 (return %₇) +21 latestworld +22 (= slot₁/#-># %₁₄) +23 slot₁/#-># +24 (return %₂₃) + +######################################## +# Local single-assigned after declaration - no Box needed +function f_local_no_box() + local x + x = 1 + ()->x +end +#--------------------- +1 (call core.define_method TestMod :f_local_no_box) +2 latestworld +3 (call core.svec) +4 (call core.svec :x) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_local_no_box##0#f_local_no_box##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_local_no_box##0#f_local_no_box##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::4:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :x) + 2 (return %₁) +14 latestworld +15 TestMod.f_local_no_box +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_local_no_box %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(single_assign) slot₃/#->#(single_assign)] + 1 (= slot₂/x 1) + 2 TestMod.#f_local_no_box##0#f_local_no_box##1 + 3 (call core._typeof_captured_variable slot₂/x) + 4 (call core.apply_type %₂ %₃) + 5 (new %₄ slot₂/x) + 6 (= slot₃/#-># %₅) + 7 slot₃/#-># + 8 (return %₇) +22 latestworld +23 TestMod.f_local_no_box +24 (return %₂₃) + +######################################## +# Typed local single-assigned after declaration - no Box needed +function f_typed_local_no_box() + local x::Int + x = 1 + ()->x +end +#--------------------- +1 (call core.define_method TestMod :f_typed_local_no_box) +2 latestworld +3 (call core.svec) +4 (call core.svec :x) +5 (call core.svec false) +6 (call JuliaLowering.eval_closure_type TestMod :#f_typed_local_no_box##0#f_typed_local_no_box##1 %₃ %₄ %₅) +7 latestworld +8 TestMod.#f_typed_local_no_box##0#f_typed_local_no_box##1 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::4:5 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (call core.getfield slot₁/#self# :x) + 2 (return %₁) +14 latestworld +15 TestMod.f_typed_local_no_box +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_typed_local_no_box %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(single_assign) slot₃/#->#(single_assign) slot₄/tmp(!read)] + 1 (newvar slot₃/#->#) + 2 1 + 3 TestMod.Int + 4 (= slot₄/tmp %₂) + 5 (call core.isa slot₄/tmp %₃) + 6 (gotoifnot %₅ label₈) + 7 (goto label₁₀) + 8 (call top.convert %₃ slot₄/tmp) + 9 (= slot₄/tmp (call core.typeassert %₈ %₃)) + 10 slot₄/tmp + 11 (= slot₂/x %₁₀) + 12 TestMod.#f_typed_local_no_box##0#f_typed_local_no_box##1 + 13 (call core._typeof_captured_variable slot₂/x) + 14 (call core.apply_type %₁₂ %₁₃) + 15 (new %₁₄ slot₂/x) + 16 (= slot₃/#-># %₁₅) + 17 slot₃/#-># + 18 (return %₁₇) +22 latestworld +23 TestMod.f_typed_local_no_box +24 (return %₂₃) + +######################################## +# Error: Closure outside any top level context +# (Should only happen in a user-visible way when lowering code emitted +# from a `@generated` function code generator.) +@ast_ [K"generated_lambda" + [K"block"] + [K"block"] + [K"->" [K"tuple"] [K"block"]] +] +#--------------------- +LoweringError: +#= line 1 =# - invalid syntax: unknown form `generated_lambda` or number of arguments 3 +Expression: + (generated_lambda (block) (block) (-> (tuple) (block))) diff --git a/JuliaLowering/test/compat.jl b/JuliaLowering/test/compat.jl new file mode 100644 index 0000000000000..3ea3eff7fa0c5 --- /dev/null +++ b/JuliaLowering/test/compat.jl @@ -0,0 +1,706 @@ +test_mod = Module() + +const JL_DIR = joinpath(@__DIR__, "..") + +# copied from JuliaSyntax/test/parse_packages.jl +function find_source_in_path(basedir) + src_list = String[] + for (root, _dirs, files) in walkdir(basedir) + append!(src_list, (joinpath(root, f) for f in files + if endswith(f, ".jl") && (p = joinpath(root,f); !islink(p) && isfile(p)))) + end + src_list +end + +function find_diff(e1, e2, loc=Ref(LineNumberNode(0))) + if expr_equal_forgiving(e1, e2) + return nothing, nothing + elseif !(e1 isa Expr && e2 isa Expr) || + e1.head !== e2.head || + length(e1.args) !== length(e2.args) + return (e1, e2), (loc[]) + else + for i in 1:length(e1.args) + e1.args[i] isa LineNumberNode && (loc[] = e1.args[i]) + (diff, path) = find_diff(e1.args[i], e2.args[i], loc) + isnothing(diff) || return (diff, (e1.head, i, path)) + end + end +end + +function test_each_in_path(test_f::Function, basedir) + ran = 0 + for filepath in find_source_in_path(basedir) + @testset "$(relpath(filepath, basedir))" begin + str = try + read(filepath, String) + catch + continue + end + ran += test_f(str) + end + end + @test ran > 0 + nothing +end + +# ignore_linenums=false is good for checking, but too noisy to use much +function expr_equal_forgiving(e1, e2; ignore_linenums=true) + if e1 isa QuoteNode && e2 isa QuoteNode + return expr_equal_forgiving(e1.value, e2.value; ignore_linenums) + end + !(e1 isa Expr && e2 isa Expr) && return e1 == e2 + if ignore_linenums + e1, e2 = let e1b = Expr(e1.head), e2b = Expr(e2.head) + e1b.args = filter(x->!(x isa LineNumberNode), e1.args) + e2b.args = filter(x->!(x isa LineNumberNode), e2.args) + e1b, e2b + end + end + + e1.head === e2.head && length(e1.args) === length(e2.args) && + all(expr_equal_forgiving(a1, a2; ignore_linenums) for (a1, a2) in + zip(e1.args, e2.args)) +end + +@testset "Expr<->EST" begin + function roundtrip(e) + JuliaLowering.est_to_expr(JuliaLowering.expr_to_est(e)) + end + function roundtrip_eq(str) + e_ref = try + JuliaSyntax.parseall(Expr, str) + catch _ + nothing + end + isnothing(e_ref) && return 0 + e_test = roundtrip(e_ref) + pass = expr_equal_forgiving(e_test, e_ref) + @test pass + if !pass + ((e_ref_min, e_test_min), indices) = find_diff(e_ref, e_test) + @info "diff:" e_ref_min e_test_min indices # e_ref e_test + end + return 1 + end + + local expr_syntax = Any[ + LineNumberNode(1) + :foo + Expr(:foo, 1) + GlobalRef(Core, :nothing) + nothing + ] + + local expr_wrappers = Function[ + identity + x->QuoteNode(x) + x->Expr(:function, x) + x->Expr(:dummy, x) + ] + + # TODO: `@ast_` escaping is broken + unused = JuliaSyntax.parsestmt(JuliaSyntax.SyntaxTree, "foo") + local st_wrappers = Function[ + x->(@ast _ unused (x::K"Value")) + x->(@ast _ unused [K"inert" x::K"Value"]) + x->(@ast _ unused [K"function" x::K"Value"]) + ] + + @testset "every basic case" begin + for e in expr_syntax, w1 in expr_wrappers, w2 in expr_wrappers + e_wrapped = w2(w1(e)) + @test roundtrip(e_wrapped) == e_wrapped + end + + for e in expr_syntax, st_w in st_wrappers, e_w in expr_wrappers + e_wrapped = st_w(e_w(e)) + @test roundtrip(e_wrapped) == e_wrapped + e_wrapped = e_w(st_w(e)) + @test roundtrip(e_wrapped) == e_wrapped + end + end + + @testset "special cases: Value implicitly quotes AST nodes" begin + @test JL.est_to_expr(@ast_ :foo::K"Value") == + JL.est_to_expr(@ast_ [K"inert" "foo"::K"Identifier"]) == + QuoteNode(:foo) + @test JL.est_to_expr(@ast_ Expr(:call, 1)::K"Value") == + JL.est_to_expr(@ast_ [K"inert" [K"call" 1::K"Value"]]) == + QuoteNode(Expr(:call, 1)) + @test JL.est_to_expr(@ast_ QuoteNode(Expr(:call, 1))::K"Value") == + JL.est_to_expr(@ast_ [K"inert" [K"inert" [K"call" 1::K"Value"]]]) == + QuoteNode(QuoteNode(Expr(:call, 1))) + end + + @testset "provenance via scavenging for LineNumberNodes" begin + # Provenance of a node should generally be the last seen + # LineNumberNode in the depth-first traversal of the Expr, or the + # initial line given if none have been seen yet. If none have been seen + # and no initial line was given, .source should still be defined on all + # nodes (of unspecified value, but hopefully a helpful value for the + # user.) + + ex = Expr(:block, + LineNumberNode(123), + Expr(:block, + Expr(:block, LineNumberNode(456)), + Expr(:block)), + Expr(:block, + Expr(:block), + Expr(:block))) + + # No initial line provided + st = JuliaLowering.expr_to_est(ex) + @test let lnn = st[1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][1].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][2].source; lnn isa LineNumberNode && lnn.line === 456; end + + # Same tree, but provide an initial line + st = JuliaLowering.expr_to_est(ex, LineNumberNode(789)) + @test let lnn = st.source; lnn isa LineNumberNode && lnn.line === 789; end + @test let lnn = st[1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][1].source; lnn isa LineNumberNode && lnn.line === 123; end + @test let lnn = st[1][2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][1].source; lnn isa LineNumberNode && lnn.line === 456; end + @test let lnn = st[2][2].source; lnn isa LineNumberNode && lnn.line === 456; end + + ex = parsestmt(Expr, """ + begin + try + maybe + lots + of + lines + catch exc + y + end + end""") + st = JuliaLowering.expr_to_est(ex, LineNumberNode(1)) + + # sanity: ensure we're testing the tree we expect + @test st ≈ @ast_ [K"block" + [K"try" + [K"block" + "maybe"::K"Identifier" + "lots"::K"Identifier" + "of"::K"Identifier" + "lines"::K"Identifier" + ] + "exc"::K"Identifier" + [K"block" "y"::K"Identifier"] + ] + ] + + @test let lnn = st.source; lnn isa LineNumberNode && lnn.line === 1; end + @test let lnn = st[1].source; lnn isa LineNumberNode && lnn.line === 2; end + @test let lnn = st[1][1].source; lnn isa LineNumberNode && lnn.line === 2; end + @test let lnn = st[1][1][1].source; lnn isa LineNumberNode && lnn.line === 3; end + @test let lnn = st[1][1][2].source; lnn isa LineNumberNode && lnn.line === 4; end + @test let lnn = st[1][1][3].source; lnn isa LineNumberNode && lnn.line === 5; end + @test let lnn = st[1][1][4].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][2].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][3].source; lnn isa LineNumberNode && lnn.line === 6; end + @test let lnn = st[1][3][1].source; lnn isa LineNumberNode && lnn.line === 8; end + + st_shortfunc = JuliaLowering.expr_to_est( + Expr(:block, + LineNumberNode(11), + Expr(:(=), + Expr(:call, :f), + :body)) + ) + @test st_shortfunc ≈ @ast_ [K"block" + [K"=" + [K"call" "f"::K"Identifier"] + "body"::K"Identifier" + ] + ] + @test let lnn = st_shortfunc[1][1].source; lnn isa LineNumberNode && lnn.line === 11; end + + st_shortfunc_2 = JuliaLowering.expr_to_est( + Expr(:block, + LineNumberNode(11), + Expr(:(=), + Expr(:call, :f), + Expr(:block, + LineNumberNode(22), + :body))) + ) + @test st_shortfunc_2 ≈ @ast_ [K"block" + [K"=" + [K"call" "f"::K"Identifier"] + [K"block" "body"::K"Identifier"] + ] + ] + @test let lnn = st_shortfunc_2[1][1].source; lnn isa LineNumberNode && lnn.line === 22; end + end + + @testset "linenodes equal (modules and functions have extra)" begin + e = JuliaSyntax.parseall(Expr, """ + module M + function f() + if x + j + elseif y + let + y + end + end + end + begin + 1 + end + end + """; filename="foo") + @test e == roundtrip(e) + end + + @testset "bulk parsed code, no linenodes" begin + test_each_in_path(roundtrip_eq, JL_DIR) + end +end + +# taken from JuliaSyntax expr.jl +test_programs = [ + "begin a\nb\n\nc\nend", + "(a;b;c)", + "begin end", + "(;;)", + "a;b", + "module A\n\nbody\nend", + "function f()\na\n\nb\nend", + "f() = 1", + "macro f()\na\nend", + "function f end", + "macro f end", + "function (f() where {T}) end", + "function (f()::S) end", + "a -> b", + "(a,) -> b", + "(a where {T}) -> b", + "a -> (\nb;c)", + "a -> begin\nb\nc\nend", + "(a;b=1) -> c", + "(a...;b...) -> c", + "(;) -> c", + "a::T -> b", + "let i=is, j=js\nbody\nend", + "for x=xs\n\nend", + "for x=xs\ny\nend", + "while cond\n\nend", + "while cond\ny\nend", + "f() = xs", + "f() =\n(a;b)", + "f() =\nbegin\na\nb\nend", + "let f(x) =\ng(x)=1\nend", + "f() .= xs", + "for i=is body end", + "for i=is, j=js\nbody\nend", + "f(x) do y\n body end", + "@f(x) do y body end", + "f(x; a=1) do y body end", + "g(f(x) do y\n body end)", + "f(a=1)", + "f(; b=2)", + "f(a=1; b=2)", + "f(a; b; c)", + "+(a=1,)", + "(a=1)()", + "(x=1) != 2", + "+(a=1)", + "(a=1)'", + "f.(a=1; b=2)", + "(a=1,)", + "(a=1,; b=2)", + "(a=1,; b=2; c=3)", + "x[i=j]", + "(i=j)[x]", + "x[a, b; i=j]", + "(i=j){x}", + "x{a, b; i=j}", + "[a=1,; b=2]", + "{a=1,; b=2}", + "f(a .= 1)", + "f(((a = 1)))", + "(((a = 1)),)", + "(;((a = 1)),)", + "(a = 1) |> f", + "(a = 1)'", + "a.b", + "a.@b x", + "f.(x,y)", + "f.(x=1)", + "f.(a=1; b=2)", + "(a=1).()", + "x .+ y", + "(x=1) .+ y", + "a .< b .< c", + "a .< (.<) .< c", + "quote .+ end", + ".+(x)", + ".+x", + "f(.+)", + "(a, .+)", + "x += y", + "x .+= y", + "x \u2212= y", + "let x=1\n end", + "let x=1 ; end", + "let x ; end", + "let x::1 ; end", + "let x=1,y=2 end", + "let x+=1 ; end", + "let ; end", + "let ; body end", + "let\na\nb\nend", + "A where {T}", + "A where {S, T}", + "A where {X, Y; Z}", + "@m\n", + "\n@m", + "@m(x; a)", + "@m(a=1; b=2)", + "@S[a,b]", + "@S[a b]", + "@S[a; b]", + "@S[a ;; b]", + "[x,y ; z]", + "[a ;;; b ;;;; c]", + "[a b ; c d]", + "[a\nb]", + "[a b]", + "[a b ; c d]", + "T[a ;;; b ;;;; c]", + "T[a b ; c d]", + "T[a\nb]", + "T[a b]", + "T[a b ; c d]", + "(x for a in as for b in bs)", + "(x for a in as, b in bs)", + "(x for a in as, b in bs if z)", + "(x for a in as, b in bs for c in cs, d in ds)", + "(x for a in as for b in bs if z)", + "(x for a in as if z for b in bs)", + "[x for a = as for b = bs if cond1 for c = cs if cond2]" , + "[x for a = as if begin cond2 end]" , + "(x for a in as if z)", + "return x", + "struct A end", + "mutable struct A end", + "struct A <: B \n a::X \n end", + "struct A \n a \n b \n end", + "struct A const a end", + "export a", + "export +, ==", + "export \n a", + "global x", + "local x", + "global x,y", + "const x,y = 1,2", + "const x = 1", + "global x ~ 1", + "global x += 1", + "(;)", + "(; a=1)", + "(; a=1; b=2)", + "(a; b; c,d)", + "module A end", + "baremodule A end", + "import A", + "A.x", + "A.\$x", + "try x catch e; y end", + "try x finally y end", + "try x catch e; y finally z end", + "try x catch e; y else z end", + "try x catch e; y else z finally w end", + "..", + "a..b", + "..(a)", + "..(..,..)", + "@.", + "@..", + "@..." +] +test_toplevel_programs = [ + "\"docstr\"\nthing_to_be_documented", +] + +@testset "Test RawGreenNode->EST->Expr against RawGreenNode->Expr" begin + function test_est(str; rule=:all, test_validator=true) + parse = rule === :all ? JS.parseall : JS.parsestmt + e_ref = try + parse(Expr, str) + catch _ + nothing + end + isnothing(e_ref) && return 0 + est_test = parse(SyntaxTree, str) + e_test = JL.est_to_expr(est_test) + pass = expr_equal_forgiving(e_test, e_ref) + @test pass + if !pass + ((e_ref_min, e_test_min), indices) = find_diff(e_ref, e_test) + @info "diff:" e_ref_min e_test_min indices # e_ref e_test + end + + # test the validator + test_validator && @test JL.valid_st0(est_test) + return 1 + end + + @testset "snippets" begin + for p in test_programs + test_est(p; rule=:statement, test_validator=false) + end + for p in test_toplevel_programs + test_est(p; test_validator=false) + end + end + + @testset "bulk parsed code, no linenodes" begin + test_each_in_path(test_est, JL_DIR) + + basedir = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "base") + test_each_in_path(test_est, basedir) + + base_testdir = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "test") + test_each_in_path(test_est, base_testdir) + + @testset "Parse Julia stdlib at $(Sys.STDLIB)" begin + for stdlib in readdir(Sys.STDLIB) + fulldir = joinpath(Sys.STDLIB, stdlib) + if isdir(fulldir) + test_each_in_path(test_est, joinpath(Sys.STDLIB, fulldir)) + end + end + end + + end + + @testset "test exceptions to blocks containing linenodes" begin + # Macro authors are otherwise expected to handle LineNumberNode in + # blocks, but since they were never emitted in `let` or `for` assignment + # blocks, test that we have the same behaviour. + @testset "linenodes equal in `let`" begin + s = """ + let a=1, b=2, c=3 + a,b,c + end + """ + @test JL.est_to_expr(JS.parsestmt(SyntaxTree, s)) == JS.parsestmt(Expr, s) + end + @testset "linenodes equal in `for`" begin + s = """ + for a in 1:2, b in 3:4, c in 5:6 + a,b,c + end + """ + @test JL.est_to_expr(JS.parsestmt(SyntaxTree, s)) == JS.parsestmt(Expr, s) + end + end + + # empty let block linenumbernode is accepted by lowering + fl_eval(test_mod, Expr(:let, Expr(:block, LineNumberNode(1)), Expr(:block, 1))) == 1 + jl_eval(test_mod, Expr(:let, Expr(:block, LineNumberNode(1)), Expr(:block, 1))) == 1 +end + +@testset "non-ASCII operator handling" begin + # regression test for invalid string index + @test JuliaLowering.include_string(test_mod, raw""" + @noinline (x = 0xF; x ⊻= 1; x) + """; expr_compat_mode=true) == 0xE +end + +@testset "Expr(:ssavalue) conversion" begin + # Expr(:ssavalue, N) should be converted to [K"ssavalue" N::K"Value"] + st = JuliaLowering.expr_to_est(Expr(:ssavalue, 0)) + @test kind(st) === K"ssavalue" + @test st[1].value == 0 + + st = JuliaLowering.expr_to_est(Expr(:ssavalue, 42)) + @test kind(st) === K"ssavalue" + @test st[1].value == 42 + + # Roundtrip: ssavalue should convert back to Expr(:ssavalue, N) + @test JL.est_to_expr(JuliaLowering.expr_to_est(Expr(:ssavalue, 5))) == + Expr(:ssavalue, 5) + + # ssavalue references inside a lambda body should lower successfully + lambda = Expr(:lambda, Any[:x], + Expr(:block, + Expr(:(=), Expr(:ssavalue, 0), Expr(:call, GlobalRef(Core, :typeof), :x)), + Expr(:return, Expr(:ssavalue, 0)))) + out = JL.core_lowering_hook(lambda, test_mod) + @test out isa Core.SimpleVector && out[1] isa Core.CodeInfo +end + +@testset "expr compat: #self# becomes `thisfunction`" begin + @test JuliaLowering.include_string(test_mod, raw""" + (function var_self() + var"#self#" + end)() + """; expr_compat_mode=true) isa Function + + @test JuliaLowering.include_string(test_mod, raw""" + (let + ()->(var"#self#") + end)() + """; expr_compat_mode=true) isa Function + + @test JuliaLowering.include_string(test_mod, raw""" + var_self_short() = var"#self#" + var_self_short() + """; expr_compat_mode=true) isa Function + + @test JuliaLowering.include_string(test_mod, raw""" + macro var_self_macro(); var"#self#"; end + @var_self_macro + """; expr_compat_mode=true) isa Function + + @test JuliaLowering.include_string(test_mod, raw""" + (function var_self_kw(; k=1) + var"#self#" + end)() + """; expr_compat_mode=true) isa Function + + @test JuliaLowering.include_string(test_mod, raw""" + (function var_self_quoted() + :(var"#self#") + end)() + """; expr_compat_mode=true) === Symbol("#self#") + + @test JuliaLowering.include_string(test_mod, raw""" + @isdefined(var"#self#") ? var"#self#" : nothing + """; expr_compat_mode=true) == nothing + @test JuliaLowering.include_string(test_mod, raw""" + (function var_self_isdefined() + @isdefined(var"#self#") ? var"#self#" : nothing + end)() + """; expr_compat_mode=true) isa Function + + # we don't bother with generators + @test_broken JuliaLowering.include_string(test_mod, raw""" + collect(var"#self#" for i in 1:1)[1] + """; expr_compat_mode=true) isa Function + + # we assume the user doesn't create vars with this name + @test_broken JuliaLowering.include_string(test_mod, raw""" + (function var_self_assign() + var"#self#" = 1 + var"#self#" + end)() + """; expr_compat_mode=true) == 1 +end + +@testset "scope-block" begin + lam = Expr(:lambda, Symbol[Symbol("#self#"), :f], + Expr(Symbol("scope-block"), + Expr(:block, + Expr(:return, 1)))) + @test fl_eval(test_mod, lam) isa Core.CodeInfo + @test jl_eval(test_mod, lam) isa Core.CodeInfo +end + +@testset "with-static-parameters" begin + lam = Expr(Symbol("with-static-parameters"), + Expr(:lambda, [Symbol("#self#"), :x], + Expr(:block, Expr(:return, :T))), :T) + @test fl_eval(test_mod, lam) isa Core.CodeInfo + @test jl_eval(test_mod, lam) isa Core.CodeInfo +end + +# `x^n` is rewritten to `literal_pow(^, x, Val(n))` if n is an Int +@testset "(AI) literal_pow" begin + pow_mod = @newmod(:LiteralPowTest) + Core.eval(pow_mod, quote + struct P end + Base.:^(::P, n) = (:call, n) + Base.literal_pow(::typeof(^), ::P, ::Val{n}) where {n} = (:literal, n) + Base.Broadcast.broadcastable(x::P) = Ref(x) + const p = P() + end) + + cases = [ + "p^2" => (:literal, 2) + "p^0" => (:literal, 0) + "p^1" => (:literal, 1) + "p^-2" => (:literal, -2) + "p^(2)" => (:literal, 2) + "^(p, 2)" => (:literal, 2) + "p^$(typemax(Int))" => (:literal, typemax(Int)) + "p^$(BigInt(typemax(Int)) + 1)" => (:call, BigInt(typemax(Int)) + 1) + "p^true" => (:call, true) + "p^0x02" => (:call, 0x02) + "p^0x0002" => (:call, 0x0002) + "p^0x00000002" => (:call, 0x00000002) + "p^0x0000000000000002" => (:call, 0x0000000000000002) + "p^0x00000000000000000000000000000002" => (:call, UInt128(2)) + "p^big\"2\"" => (:call, big(2)) + "p^(1 + 1)" => (:call, 2) + "Base.:^(p, 2)" => (:call, 2) + "p .^ 2" => (:literal, 2) + "p .^ -1" => (:literal, -1) + "(.^)(p, 2)" => (:literal, 2) + "[p] .^ 2" => [(:literal, 2)] + "identity.([p] .^ 2)" => [(:literal, 2)] + "p .^ 0x02" => (:call, 0x02) + "p .^ true" => (:call, true) + "p .^ 2.0" => (:call, 2.0) + "(.^)(p, 0x02)" => (:call, 0x02) + "[p] .^ 0x02" => [(:call, 0x02)] + ] + + for (str, expected) in cases + ex = parsestmt(SyntaxTree, str) + fl = fl_eval(pow_mod, ex) + jl = jl_eval(pow_mod, ex; expr_compat_mode=true) + @test (str, fl) == (str, expected) context=str + @test (str, jl) == (str, fl) context=str + end + + let ex = parsestmt(SyntaxTree, "let q = p; q ^= 2; q end") + @test fl_eval(pow_mod, ex) == (:literal, 2) + @test_broken jl_eval(pow_mod, ex; expr_compat_mode=true) == (:literal, 2) + end +end + +@testset "quoted import path components" begin + local run(s) = JuliaLowering.include_string(test_mod, s) + + @test run("baremodule M; end; isdefined(M, :+)") == false + @test run("baremodule M; import Base: Base.:(+); end; getglobal(M,:+) === Base.:+") + @test run("baremodule M; import Base.:+; end; getglobal(M,:+) === Base.:+") + @test run("baremodule M; import Base: (+); end; getglobal(M,:+) === Base.:+") + @test run("baremodule M; import Base.var\"+\"; end; getglobal(M,:+) === Base.:+") + @test run("baremodule M; import Base: var\"+\"; end; getglobal(M,:+) === Base.:+") + @test run("baremodule M; import Base.:sin; end; getglobal(M,:sin) === Base.sin") + @test run("baremodule M; import Base.:(sin); end; getglobal(M,:sin) === Base.sin") + @test run("baremodule M; import Base.:Iterators; end; getglobal(M,:Iterators) === Base.Iterators") + @test run("baremodule M; import Base.:Iterators.:take; end; getglobal(M,:take) === Base.Iterators.take") + @test run("baremodule M; import Base.:Iterators: take; end; getglobal(M,:take) === Base.Iterators.take") + @test run("baremodule M; import Base.:+ as plus; end; getglobal(M,:plus) === Base.:+") + @test run("baremodule M; using Base.:Iterators; end; isdefined(M, :take)") + @test run("module Outer; f() = 1; baremodule In; import ..Outer.:f; end; end; getglobal(Outer.In,:f) === Outer.f") + @test run("module Outer; f() = 1; baremodule In; import ..Outer: f; end; end; getglobal(Outer.In,:f) === Outer.f") +end + +@testset "validation of macro-expansion-specific forms" begin + @test_throws LoweringError jl_eval( + test_mod, Expr(:escape)) + @test_throws LoweringError jl_eval( + test_mod, Expr(Symbol("hygienic-scope"), Expr(:escape), @__MODULE__)) + @test_throws LoweringError jl_eval( + test_mod, Expr(Symbol("hygienic-scope"), Expr(:escape, :x, :y), @__MODULE__)) + @test_throws LoweringError jl_eval( + test_mod, Expr(Symbol("hygienic-scope"))) + @test_throws LoweringError jl_eval( + test_mod, Expr(Symbol("hygienic-scope"), :x)) + @test_throws LoweringError jl_eval( + test_mod, Expr(Symbol("hygienic-scope"), :x, :y, :z)) +end diff --git a/JuliaLowering/test/decls.jl b/JuliaLowering/test/decls.jl new file mode 100644 index 0000000000000..f03541b7d5097 --- /dev/null +++ b/JuliaLowering/test/decls.jl @@ -0,0 +1,1231 @@ +test_mod = Module() + +@test JuliaLowering.include_string(test_mod, """ +begin + local x::Int = 1.0 + x +end +""") === 1 + +# In value position, yield the right hand side, not `x` +@test JuliaLowering.include_string(test_mod, """ +begin + local x::Int = 1.0 +end +""") === 1.0 + +@testset "global decl in value position" begin + # Global decl in value position without assignment returns nothing + @test JuliaLowering.include_string(test_mod, "global x_no_assign") === nothing + + # In tail position in a function is OK; returns nothing + @test JuliaLowering.include_string(test_mod, """ + function f_tail_global_decl() + global x_tail_decl + end + f_tail_global_decl() + """) === nothing + @test Base.binding_kind(test_mod, :x_tail_decl) == Base.PARTITION_KIND_DECLARED + @test JuliaLowering.include_string(test_mod, """ + function f_tail_global_decl_2(b) + if b + global x_tail_decl2, x_tail_decl3 + end + end + (f_tail_global_decl_2(true), f_tail_global_decl_2(false)) + """) === (nothing, nothing) + @test JuliaLowering.include_string(test_mod, "(() -> (global x_tail_decl4))()") === nothing + + # non-simple is fine to read from for some reason + @test JuliaLowering.include_string( + test_mod, "_ = global tail_decl_typed::Int") === nothing + @test Base.binding_kind(test_mod, :tail_decl_typed) == Base.PARTITION_KIND_GLOBAL + @test JuliaLowering.include_string( + test_mod, "_ = global _______________::Int") === nothing + + # disallowed in value position otherwise + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + function f_value_global_decl() + y = (global x_value_decl) + end + """) +end + +# Unadorned declarations +@test JuliaLowering.include_string(test_mod, """ +let + a = 0.0 + x::Int = a + x +end +""") === 0 + +@test JuliaLowering.include_string(test_mod, """ +let + local x::Int = 1 + x1 = x + x = 20.0 + x2 = x + (x1,x2) +end +""") === (1, 20) + +@testset "Global const mixes" for (mod, run) in [(Module(), fl_eval), + (Module(), jl_eval)] + + @gensym sym + @test run(mod, :(const $sym = 1)) === 1 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @gensym sym + @test_broken run(mod, :(const $sym::Int = 1.0)) === 1.0 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @testset for wrap in [identity, x->Expr(:let, Expr(:block), Expr(:block, x))] + @gensym sym + @test run(mod, wrap(:(global $sym = 1))) === 1 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test !Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @gensym sym + @test run(mod, wrap(:(global $sym::Int = 1.0))) === 1.0 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test !Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @gensym sym + @test run(mod, wrap(:(global const $sym = 1))) === 1 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @gensym sym + @test_broken run(mod, wrap(:(global const $sym::Int = 1.0))) === 1.0 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @gensym sym + @test run(mod, wrap(:(const global $sym = 1))) === 1 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + + @gensym sym + @test_broken run(mod, wrap(:(const global $sym::Int = 1.0))) === 1.0 + Core.@latestworld + @test Base.isdefinedglobal(mod, sym) + @test Base.isconst(mod, sym) + @test getproperty(mod, sym) === 1 + end +end + +# lowering is strict about the nesting order where parsing is not +@test_throws LoweringError jl_eval(test_mod, Expr(:global, Expr(:const, Expr(:(=), :a, 1)))) +@test_throws ErrorException fl_eval(test_mod, Expr(:global, Expr(:const, Expr(:(=), :a, 1)))) + +# Possibly worth testing excessive global/const keywords or invalid combinations +# (local + global/const) once we decide whether that's a parse error or a +# lowering error + +# Global decls with types +@test JuliaLowering.include_string(test_mod, """ +global a_typed_global::Int = 10.0 +""") === 10.0 +@test Core.get_binding_type(test_mod, :a_typed_global) === Int +@test test_mod.a_typed_global === 10 +@test JuliaLowering.include_string(test_mod, """ +global a_curly_typed_global::Union{Int, Float64} = 10.0 +""") === 10.0 +@test Core.get_binding_type(test_mod, :a_curly_typed_global) === Union{Int, Float64} +@test test_mod.a_curly_typed_global === 10.0 +@test JuliaLowering.include_string(test_mod, """ +begin + global opassign_global = 1 + global opassign_global += 1 +end +""") === 2 +@test test_mod.opassign_global === 2 +@test JuliaLowering.include_string(test_mod, """ +begin + global dotopassign_global = [1,2,3] + global dotopassign_global .+= 1 +end +""") == [2,3,4] +@test test_mod.dotopassign_global == [2,3,4] + +# Also allowed in nontrivial scopes in a top level thunk +@test JuliaLowering.include_string(test_mod, """ +let + global a_typed_global_2::Int = 10.0 +end +""") === 10.0 +@test Core.get_binding_type(test_mod, :a_typed_global_2) === Int +@test test_mod.a_typed_global_2 === 10 +@test JuliaLowering.include_string(test_mod, """ +let + global a_curly_typed_global_2::Union{Int, Float64} = 10.0 +end +""") === 10.0 +@test Core.get_binding_type(test_mod, :a_curly_typed_global_2) === Union{Int, Float64} +@test test_mod.a_curly_typed_global_2 === 10.0 +@test JuliaLowering.include_string(test_mod, """ +begin + global opassign_global_t::Int = 1 + global opassign_global_t::Int += 1.0 +end +""") === 2.0 +@test Core.get_binding_type(test_mod, :opassign_global_t) === Int +@test test_mod.opassign_global_t === 2 +@test JuliaLowering.include_string(test_mod, """ +begin + global dotopassign_global_t::Vector{Int} = [1,2,3] + global dotopassign_global_t::Vector{Int} .+= [1.0,2.0,3.0] +end +""") == [2.0,4.0,6.0] +@test Core.get_binding_type(test_mod, :dotopassign_global_t) === Vector{Int} +@test test_mod.dotopassign_global_t == [2,4,6] + +@test JuliaLowering.include_string(test_mod, "const x_c_T::Int = 9") === 9 +@test Base.isdefinedglobal(test_mod, :x_c_T) +@test Base.isconst(test_mod, :x_c_T) + +@testset "typed const redeclaration" begin + # redeclaration of the same value used to be allowed + @test_throws ErrorException JuliaLowering.include_string(test_mod, "x_c_T = 9") + @test_throws ErrorException JuliaLowering.include_string(test_mod, "x_c_T = 10") + # redeclaration with const should be OK + @test JuliaLowering.include_string(test_mod, "const x_c_T::Int = 0") === 0 +end + +# Tuple/destructuring assignments +@test JuliaLowering.include_string(test_mod, "(a0, a1, a2) = [1,2,3]") == [1,2,3] +@test JuliaLowering.include_string(test_mod, "const a,b,c = 1,2,3") === (1, 2, 3) + +@testset "Placeholder decls" begin + @test JuliaLowering.include_string(test_mod, "global _ = 1") === 1 + @test JuliaLowering.include_string(test_mod, "global _::Int = 1") === 1 + @test JuliaLowering.include_string(test_mod, "let; local _; _ = 1; end") === 1 + @test JuliaLowering.include_string(test_mod, "let; local _::Int = 1; end") === 1 + @test JuliaLowering.include_string(test_mod, "let; local (a0, _, a2) = [1,2,3]; end") == [1,2,3] + @test JuliaLowering.include_string(test_mod, "let; local (a0, _::Int, a2) = [1,2,3]; end") == [1,2,3] + @test JuliaLowering.include_string(test_mod, "_{x} = Vector{x}") == Vector + @test !Base.isdefinedglobal(test_mod, :_) +end + +test_mod_2 = Module() +@testset "toplevel-preserving syntax" begin + JuliaLowering.include_string(test_mod_2, "if true; global v1::Bool; else const v1 = 1; end") + @test !isdefined(test_mod_2, :v1) + @test Base.binding_kind(test_mod_2, :v1) == Base.PARTITION_KIND_GLOBAL + @test Core.get_binding_type(test_mod_2, :v1) == Bool + + JuliaLowering.include_string(test_mod_2, "if false; global v2::Bool; else const v2 = 2; end") + @test test_mod_2.v2 === 2 + @test Base.binding_kind(test_mod_2, :v2) == Base.PARTITION_KIND_CONST + + JuliaLowering.include_string(test_mod_2, "v3 = if true; global v4::Bool; 4 else const v4 = 5; 6; end") + @test test_mod_2.v3 == 4 + @test !isdefined(test_mod_2, :v4) + @test Base.binding_kind(test_mod_2, :v4) == Base.PARTITION_KIND_GLOBAL + @test Core.get_binding_type(test_mod_2, :v4) == Bool + + JuliaLowering.include_string(test_mod_2, "v5 = if false; global v6::Bool; 4 else const v6 = 5; 6; end") + @test test_mod_2.v5 === 6 + @test test_mod_2.v6 === 5 + @test Base.binding_kind(test_mod_2, :v6) == Base.PARTITION_KIND_CONST +end + +@testset "decls on functions" begin + # local + @gensym func func2 + @testset let ex = Expr(:let, Expr(:block), + Expr(:block, + Expr(:local, + Expr(:(=), Expr(:call, func, :x), :x), + Expr(:(=), Expr(:call, func2, :y), :(y+1))), + Expr(:tuple, + Expr(:call, func, 1), + Expr(:call, func2, 1)))) + @test jl_eval(test_mod, ex) == (1, 2) + @test !isdefined(test_mod, func) + @test !isdefined(test_mod, func2) + end + + # const + @gensym func func2 + @testset let ex = Expr(:const, Expr(:(=), Expr(:call, func, :x), :x)) + @test jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test getproperty(test_mod, func)(1) == 1 + end + + # global + @gensym func func2 + @testset let ex = Expr(:global, + Expr(:(=), Expr(:call, func, :x), :x), + Expr(:(=), Expr(:call, func2, :y), :(y+1))) + @test jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test getproperty(test_mod, func)(1) == 1 + @test getproperty(test_mod, func2)(1) == 2 + end + + # const global 1-arg + @gensym func func2 + @testset let ex = Expr(:const, + Expr(:global, + Expr(:(=), Expr(:call, func, :x), :x))) + @test jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test getproperty(test_mod, func)(1) == 1 + end + + # const global 2-arg (should probably disallow) + @gensym func func2 + @testset let ex = Expr(:const, + Expr(:global, + Expr(:(=), Expr(:call, func, :x), :x), + Expr(:(=), Expr(:call, func2, :y), :(y+1)))) + @test_broken jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test_broken getproperty(test_mod, func)(1) == 1 + # also broken in flisp (func2 doesn't get defined) + @test_broken getproperty(test_mod, func2)(1) == 2 + end + + # global in local scope + @gensym func func2 + @testset let ex = Expr(:let, Expr(:block), + Expr(:global, + Expr(:(=), Expr(:call, func, :x), :x), + Expr(:(=), Expr(:call, func2, :y), :(y+1)))) + @test jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test getproperty(test_mod, func)(1) == 1 + @test getproperty(test_mod, func2)(1) == 2 + end + + @gensym func func2 + @testset let ex = Expr(:let, Expr(:block), + Expr(:const, + Expr(:global, + Expr(:(=), Expr(:call, func, :x), :x)))) + @test jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test getproperty(test_mod, func)(1) == 1 + end + # const global in local scope 2-arg (should probably disallow) + @gensym func func2 + @testset let ex = Expr(:let, Expr(:block), + Expr(:const, + Expr(:global, + Expr(:(=), Expr(:call, func, :x), :x), + Expr(:(=), Expr(:call, func2, :y), :(y+1))))) + @test_broken jl_eval(test_mod, ex) isa Function + Core.@latestworld + @test_broken getproperty(test_mod, func)(1) == 1 + # also broken in flisp (func2 doesn't get defined) + @test_broken getproperty(test_mod, func2)(1) == 2 + end +end + +@testset "all non-call assignment forms within global, local" for declkind in (:local, :global) + # basic form + @gensym sym + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(declkind, Expr(:(=), sym, 1)), + Expr(:tuple, sym))) + @test jl_eval(test_mod, ex) == (1,) + Core.@latestworld + if declkind === :global + @test getproperty(test_mod, sym) == 1 + else + @test !isdefined(test_mod, sym) + end + end + + # setproperty form: decl is ignored (this is misleading, syntax TODO) + @gensym sym + @eval test_mod mutable struct with_mutable_a; a; end + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(:(=), sym, :(with_mutable_a(1))), + Expr(declkind, Expr(:(=), Expr(:., sym, QuoteNode(:a)), 2)), + sym)) + @test jl_eval(test_mod, ex).a == 2 + Core.@latestworld + @test !isdefined(test_mod, sym) + end + + # ref form: decl is ignored, but assignment works (syntax TODO) + @gensym sym + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(declkind, Expr(:(=), Expr(:ref, sym), 0)))) + @test_throws UndefVarError jl_eval(test_mod, ex) + Core.@latestworld + @test !isdefined(test_mod, sym) + end + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(:(=), sym, [1,2,3]), + Expr(declkind, Expr(:(=), Expr(:ref, sym, 2), 0)), + Expr(:tuple, sym))) + @test jl_eval(test_mod, ex) == ([1,0,3],) + Core.@latestworld + @test !isdefined(test_mod, sym) + end + + # chained, decl on first + @gensym sym1 sym2 sym3 + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(declkind, + Expr(:(=), sym1, + Expr(:(=), sym2, + Expr(:(=), sym3, :(gensym()))))), + Expr(:tuple, sym1, sym2, sym3))) + res = jl_eval(test_mod, ex) + Core.@latestworld + @test res isa Tuple + @test res[1] == res[2] == res[3] + if declkind === :global + @test isdefined(test_mod, sym1) + else + @test !isdefined(test_mod, sym1) + end + @test !isdefined(test_mod, sym2) + @test !isdefined(test_mod, sym3) + end + + # decl sym += val + @gensym sym + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(declkind, Expr(:(=), sym, 1)), + Expr(declkind, Expr(:(+=), sym, 2)), + Expr(:tuple, sym))) + @test jl_eval(test_mod, ex) == (3,) + Core.@latestworld + if declkind === :global + @test getproperty(test_mod, sym) == 3 + else + @test !isdefined(test_mod, sym) + end + end + + # decl sym .= val + @gensym sym + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(declkind, Expr(:(=), sym, [1,2,3])), + Expr(declkind, Expr(:(.=), sym, 0)), + Expr(:tuple, sym))) + @test jl_eval(test_mod, ex) == ([0,0,0],) + Core.@latestworld + if declkind === :global + @test getproperty(test_mod, sym) == [0,0,0] + else + @test !isdefined(test_mod, sym) + end + end + + # decl sym .+= val + @gensym sym + @testset let ex = + Expr(:let, Expr(:block), + Expr(:block, + Expr(declkind, Expr(:(=), sym, [1,2,3])), + Expr(declkind, Expr(:(.+=), sym, [4,5,6])), + Expr(:tuple, sym))) + @test jl_eval(test_mod, ex) == ([5,7,9],) + Core.@latestworld + if declkind === :global + @test getproperty(test_mod, sym) == [5,7,9] + else + @test !isdefined(test_mod, sym) + end + end +end + +@testset "all non-call non-globalref assignment forms within `const`" begin + # prohibited by parsing as of writing this, so hard to make into an IR test + ex = Expr(:const, Expr(:(.=), :x, 1)) + @test_throws LoweringError jl_lower(test_mod, ex) + ex = Expr(:const, Expr(:(+=), :x, 1)) + @test_throws LoweringError jl_lower(test_mod, ex) + ex = Expr(:const, Expr(:(.+=), :x, 1)) + @test_throws LoweringError jl_lower(test_mod, ex) + + # placeholder + @test jl_eval(test_mod, :(const _ = 1)) === 1 + @test jl_eval(test_mod, :(const _ = _ = __ = ___ = 1)) === 1 + @test jl_eval(test_mod, :(const _::Int = 1.0)) === 1.0 + @test jl_eval(test_mod, :(const _{x} = Vector{x})) == Vector + @test jl_eval(test_mod, :(const (_, _::Int, _{x}) = 1, 2, Vector)) == (1, 2, Vector) + Core.@latestworld + @test !Base.isdefinedglobal(Main, :_) + @test !Base.isdefinedglobal(Main, :__) + @test !Base.isdefinedglobal(Main, :___) + + # pre-desugared const + @gensym sym + ex = Expr(:const, sym, 1) + @test jl_eval(test_mod, ex) == 1 + @test Base.binding_kind(test_mod, sym) == Base.PARTITION_KIND_CONST + + # chained, const first + @gensym sym1 sym2 sym3 + @testset let ex = Expr(:const, + Expr(:(=), sym1, + Expr(:(=), sym2, + Expr(:(=), sym3, :(gensym()))))) + @test jl_eval(test_mod, ex) isa Symbol + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_GLOBAL + @test Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_GLOBAL + @test getproperty(test_mod, sym1) == + getproperty(test_mod, sym2) == + getproperty(test_mod, sym3) + end + + # chained, const first, with types + @gensym sym1 sym2 sym3 + @testset let ex = Expr(:const, + Expr(:(=), Expr(:(::), sym1, :Symbol), + Expr(:(=), Expr(:(::), sym2, :Symbol), + Expr(:(=), Expr(:(::), sym3, :Symbol), :(gensym()))))) + @test_broken jl_eval(test_mod, ex) isa Symbol + Core.@latestworld + + @test_broken Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test_broken Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_GLOBAL + @test_broken Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_GLOBAL + # also broken in flisp (sym1 has type Any, others are Symbol) + @test_broken Core.get_binding_type(test_mod, sym1) == Symbol + @test_broken Core.get_binding_type(test_mod, sym2) == Symbol + @test_broken Core.get_binding_type(test_mod, sym3) == Symbol + @test_broken getproperty(test_mod, sym1) == + getproperty(test_mod, sym2) == + getproperty(test_mod, sym3) + end + + # chained, const all + @gensym sym1 sym2 sym3 + @testset let ex = Expr(:const, + Expr(:(=), sym1, + Expr(:const, + Expr(:(=), sym2, + Expr(:const, + Expr(:(=), sym3, :(gensym()))))))) + @test jl_eval(test_mod, ex) isa Symbol + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_CONST + @test getproperty(test_mod, sym1) == + getproperty(test_mod, sym2) == + getproperty(test_mod, sym3) + end + + # destructured + @gensym sym1 sym2 sym3 + @testset let ex = :(const ($sym1, ($sym2, $sym3)) = (1, (2, 3))) + @test jl_eval(test_mod, ex) == (1, (2, 3)) + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_CONST + @test getproperty(test_mod, sym1) == 1 + @test getproperty(test_mod, sym2) == 2 + @test getproperty(test_mod, sym3) == 3 + end + + # destructured, with types + @gensym sym1 sym2 sym3 + @testset let ex = :(const ($sym1::Int, ($sym2::Int, $sym3::Int)) = (1, (2, 3))) + @test jl_eval(test_mod, ex) == (1, (2, 3)) + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_CONST + @test getproperty(test_mod, sym1) == 1 + @test getproperty(test_mod, sym2) == 2 + @test getproperty(test_mod, sym3) == 3 + + # note flisp also doesn't set binding types, though it does for globals + @test Core.get_binding_type(test_mod, sym1) == Any + @test Core.get_binding_type(test_mod, sym2) == Any + @test Core.get_binding_type(test_mod, sym3) == Any + end + + # destructured, nested NamedTuple + @gensym sym1 sym2 sym3 + @testset let ex = :(const ($sym1, (;$sym2, $sym3)) = (1, (;$sym2=2, $sym3=3))) + @test jl_eval(test_mod, ex) == (1, (;sym2=>2, sym3=>3)) + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test_broken Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_CONST + @test_broken Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_CONST + @test getproperty(test_mod, sym1) == 1 + @test getproperty(test_mod, sym2) == 2 + @test getproperty(test_mod, sym3) == 3 + end + + # destructured, slurp + @gensym sym1 sym2 sym3 + @testset let ex = :(const ($sym1, $sym2..., $sym3) = (1, 2, 22, 222, 3)) + @test jl_eval(test_mod, ex) == (1, 2, 22, 222, 3) + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym2) == Base.PARTITION_KIND_CONST + @test Base.binding_kind(test_mod, sym3) == Base.PARTITION_KIND_CONST + @test getproperty(test_mod, sym1) == 1 + @test getproperty(test_mod, sym2) == (2, 22, 222) + @test getproperty(test_mod, sym3) == 3 + end + + # curly + @gensym sym1 sym2 + @testset let ex = Expr(:const, + Expr(:(=), Expr(:curly, sym1, sym2), + Expr(:curly, Vector, sym2))) + @test jl_eval(test_mod, ex) == Vector + Core.@latestworld + + @test Base.binding_kind(test_mod, sym1) == Base.PARTITION_KIND_CONST + @test !Base.isdefinedglobal(test_mod, sym2) + @test getproperty(test_mod, sym1){Int} == Vector{Int} + end +end + +gr_mod = Module() + +@testset "GlobalRef as an identifier" begin + # gr = 1 + @gensym sym + @test 1 == jl_eval(test_mod, Expr(:(=), GlobalRef(gr_mod, sym), 1)) + @test Base.isdefinedglobal(gr_mod, sym) + @test getproperty(gr_mod, sym) == 1 + @test !Base.isdefinedglobal(test_mod, sym) + # test gr as a value + @test 1 == jl_eval(test_mod, Expr(:block, GlobalRef(gr_mod, sym))) + + # gr resolves when a similar local is in scope + @gensym sym + Base.eval(gr_mod, Expr(:(=), sym, "gr")) + @test ("let-local", "gr") == jl_eval( + test_mod, Expr(:let, Expr(:block, Expr(:(=), sym, "let-local")), + Expr(:tuple, sym, GlobalRef(gr_mod, sym)))) + @test !Base.isdefinedglobal(test_mod, sym) + + @test ("let-local", "gr reassigned") == jl_eval( + test_mod, Expr(:let, Expr(:block, Expr(:(=), sym, "let-local")), + Expr(:block, + Expr(:(=), GlobalRef(gr_mod, sym), "gr reassigned"), + Expr(:tuple, sym, GlobalRef(gr_mod, sym))))) + @test !Base.isdefinedglobal(test_mod, sym) + @test getproperty(gr_mod, sym) == "gr reassigned" + + @test ("let-local", "gr reassigned twice") == jl_eval( + test_mod, Expr(:let, Expr(:block, Expr(:(=), sym, "let-local")), + Expr(:block, + Expr(:(*=), GlobalRef(gr_mod, sym), " twice"), + Expr(:tuple, sym, GlobalRef(gr_mod, sym))))) + @test !Base.isdefinedglobal(test_mod, sym) + @test getproperty(gr_mod, sym) == "gr reassigned twice" + + @test ("lambda-local", "gr reassigned twice") == jl_eval( + test_mod, Expr(:let, Expr(:block, Expr(:(=), sym, "let-local")), + Expr(:call, + Expr(:->, Expr(:tuple, Expr(:kw, sym, "lambda-local")), + Expr(:block, Expr(:tuple, sym, GlobalRef(gr_mod, sym))))))) + @test !Base.isdefinedglobal(test_mod, sym) + + # gr1 = gr2 = gr3 = gr4 = 1 + @gensym sym1 sym2 sym3 sym4 + @test 1 == jl_eval( + test_mod, + Expr(:(=), GlobalRef(gr_mod, sym1), + Expr(:(=), GlobalRef(gr_mod, sym2), + Expr(:(=), GlobalRef(gr_mod, sym3), + Expr(:(=), GlobalRef(gr_mod, sym4), 1))))) + @test Base.isdefinedglobal(gr_mod, sym1) + @test Base.isdefinedglobal(gr_mod, sym2) + @test Base.isdefinedglobal(gr_mod, sym3) + @test Base.isdefinedglobal(gr_mod, sym4) + @test getproperty(gr_mod, sym1) == 1 + @test getproperty(gr_mod, sym2) == 1 + @test getproperty(gr_mod, sym3) == 1 + @test getproperty(gr_mod, sym4) == 1 + @test !Base.isdefinedglobal(test_mod, sym1) + @test !Base.isdefinedglobal(test_mod, sym2) + @test !Base.isdefinedglobal(test_mod, sym3) + @test !Base.isdefinedglobal(test_mod, sym4) + + # gr += 5 + @gensym sym + jl_eval(test_mod, Expr(:(=), GlobalRef(gr_mod, sym), 10)) + @test 15 == jl_eval( + test_mod, Expr(:(+=), GlobalRef(gr_mod, sym), 5)) + @test getproperty(gr_mod, sym) == 15 + + # (gr1, gr2) = (1, 2) + @gensym sym1 sym2 + @test (1, 2) == jl_eval( + test_mod, Expr(:(=), + Expr(:tuple, GlobalRef(gr_mod, sym1), GlobalRef(gr_mod, sym2)), + Expr(:call, :tuple, 1, 2))) + @test getproperty(gr_mod, sym1) == 1 + @test getproperty(gr_mod, sym2) == 2 + @test !Base.isdefinedglobal(test_mod, sym1) + + # global gr::Int = 1 + @gensym sym + @test 1 == jl_eval( + test_mod, Expr(:global, + Expr(:(=), + Expr(:(::), GlobalRef(gr_mod, sym), Int), + 1))) + @test Base.isdefinedglobal(gr_mod, sym) + @test Core.get_binding_type(gr_mod, sym) == Int + @test getproperty(gr_mod, sym) == 1 + @test !Base.isdefinedglobal(test_mod, sym) + + # global gr::Int + @gensym sym + @test nothing == jl_eval( + test_mod, Expr(:global, Expr(:(::), GlobalRef(gr_mod, sym), Int))) + @test Core.get_binding_type(gr_mod, sym) == Int + + # const gr = 1 + @gensym sym + @test 1 == jl_eval( + test_mod, Expr(:const, Expr(:(=), GlobalRef(gr_mod, sym), 1))) + @test Base.isdefinedglobal(gr_mod, sym) + @test getproperty(gr_mod, sym) == 1 + @test Base.binding_kind(gr_mod, sym) == Base.PARTITION_KIND_CONST + @test !Base.isdefinedglobal(test_mod, sym) + + # const gr::Int = 42 + @gensym sym + @test 42 == jl_eval( + test_mod, Expr(:const, + Expr(:(=), + Expr(:(::), GlobalRef(gr_mod, sym), Int), + 42))) + @test Base.isdefinedglobal(gr_mod, sym) + @test getproperty(gr_mod, sym) == 42 + @test Base.binding_kind(gr_mod, sym) == Base.PARTITION_KIND_CONST + @test !Base.isdefinedglobal(test_mod, sym) + + # local gr (error) + @gensym sym + @test_throws LoweringError jl_eval( + test_mod, Expr(:local, GlobalRef(gr_mod, sym))) + @test_throws LoweringError jl_eval( + test_mod, Expr(:let, Expr(:block, Expr(:(=), GlobalRef(gr_mod, sym), 1)))) + @test !Base.isdefinedglobal(test_mod, sym) + + # function gr end + @gensym sym + @test jl_eval(test_mod, Expr(:function, GlobalRef(gr_mod, sym))) isa Function + @test Base.isdefinedglobal(gr_mod, sym) + @test getproperty(gr_mod, sym) isa Function + @test !Base.isdefinedglobal(test_mod, sym) + + # function gr(x); x; end + @gensym sym + @test jl_eval(test_mod, Expr(:function, + Expr(:call, GlobalRef(gr_mod, sym), :x), + Expr(:block, :x))) isa Function + @test Base.isdefinedglobal(gr_mod, sym) + @test getproperty(gr_mod, sym)(1) == 1 + @test !Base.isdefinedglobal(test_mod, sym) + + # function gr(x;kw1,kw2=2); x; end + @gensym sym + @test jl_eval(test_mod, Expr(:function, + Expr(:call, + GlobalRef(gr_mod, sym), + Expr(:parameters, :kw1, Expr(:kw, :kw2, 2)), + :x), + Expr(:block, + Expr(:tuple, :x, :kw1, :kw2)))) isa Function + @test Base.isdefinedglobal(gr_mod, sym) + @test getproperty(gr_mod, sym)(0;kw1=1) == (0,1,2) + @test getproperty(gr_mod, sym)(0;kw1=1,kw2=20) == (0,1,20) + @test !Base.isdefinedglobal(test_mod, sym) + + # gr inner function (let) should act like global inner function + @gensym sym + @test jl_eval( + test_mod, + Expr(:let, + Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, 2)), + Expr(:block, + Expr(:function, Expr(:call, GlobalRef(gr_mod, sym), :c), + Expr(:block, Expr(:tuple, :a, :b, :c)))))) isa Function + @test Base.isdefinedglobal(gr_mod, sym) + @test !Base.isdefinedglobal(test_mod, sym) + @test getproperty(gr_mod, sym)(3) == (1,2,3) + + # error: gr inner function (function) should act like global inner function + @gensym sym outer_f + @test_throws LoweringError jl_eval( + test_mod, + Expr(:function, Expr(:call, outer_f), + Expr(:block, + Expr(:function, Expr(:call, GlobalRef(gr_mod, sym)), + Expr(:block))))) + + # macro gr end + @gensym sym + mac_sym = Symbol("@"*string(sym)) + @test jl_eval(test_mod, Expr(:macro, GlobalRef(gr_mod, sym))) isa Function + @test Base.isdefinedglobal(gr_mod, mac_sym) + @test !Base.isdefinedglobal(test_mod, mac_sym) + + # macro gr(x); (x, @__MODULE__); end + # + # should define the symbol in gr_mod, but the method (and expansion) are + # attributed to test_mod, where the macro expression was evaluated. + @gensym sym + mac_sym = Symbol("@"*string(sym)) + @test jl_eval(test_mod, Expr(:macro, Expr(:call, GlobalRef(gr_mod, sym), :x), + Expr(:block, + Expr(:tuple, :x, :(@__MODULE__())))); + expr_compat_mode=true) isa Function + @test Base.isdefinedglobal(gr_mod, mac_sym) + @test !Base.isdefinedglobal(test_mod, mac_sym) + @test jl_eval(gr_mod, :(@($mac_sym)(1))) == (1, test_mod) + @testset "globalref as macrocall name" begin + @test (1, test_mod) == jl_eval( + test_mod, + Expr(:macrocall, GlobalRef(gr_mod, mac_sym), LineNumberNode(1, :none), 1)) + @test (1, test_mod) == jl_eval( + gr_mod, + Expr(:macrocall, GlobalRef(gr_mod, mac_sym), LineNumberNode(1, :none), 1)) + # globalref(test_mod, mac_sym) should fail + @test_throws MacroExpansionError jl_eval( + test_mod, + Expr(:macrocall, GlobalRef(test_mod, mac_sym), LineNumberNode(1, :none), 1)) + @test_throws MacroExpansionError jl_eval( + gr_mod, + Expr(:macrocall, GlobalRef(test_mod, mac_sym), LineNumberNode(1, :none), 1)) + end + + # error: begin; local gr = 1; end + # (note: flisp allows this) + @gensym sym + @test_throws "cannot use GlobalRef as local identifier" jl_eval( + test_mod, Expr(:block, + Expr(:local, Expr(:(=), GlobalRef(gr_mod, sym), 1)))) + @test !Base.isdefinedglobal(test_mod, sym) + @test !Base.isdefinedglobal(gr_mod, sym) + + # error: let gr = 1; end + # (note: flisp allows this) + @gensym sym + @test_throws "cannot use GlobalRef as local identifier" jl_eval( + test_mod, Expr(:let, + Expr(:block, Expr(:(=), GlobalRef(gr_mod, sym), 1)), + Expr(:block))) + + # error: for gr = 1:3 + # (note: flisp allows this) + @gensym sym + @test_throws "cannot use GlobalRef as local identifier" jl_eval( + test_mod, Expr(:for, + Expr(:(=), GlobalRef(gr_mod, sym), + Expr(:call, :(:), 1, 3)), + Expr(:block))) + + # error: function f(gr); end + @gensym sym + @test_throws "cannot use GlobalRef as local identifier" jl_eval( + test_mod, Expr(:function, + Expr(:call, :fname, GlobalRef(gr_mod, sym)), + Expr(:block))) + + + # error: try/catch with GlobalRef catch var + @gensym sym + @test_throws ErrorException fl_eval( + test_mod, Expr(:try, + Expr(:block, Expr(:call, :error, "oops")), + GlobalRef(gr_mod, sym), + Expr(:block, 1))) broken=true +end + +@testset "All possible `let` forms" for run in [fl_eval, jl_eval], + maybe_int in [identity, x->Expr(:(::), x, :Int)] + # no-assignment forms + @test run(test_mod, + Expr(:let, maybe_int(:a), + Expr(:tuple, + Expr(:islocal, :a), + Expr(:isdefined, :a)))) == (true, false) + @test run(test_mod, + Expr(:let, Expr(:block, maybe_int(:a)), + Expr(:tuple, + Expr(:islocal, :a), + Expr(:isdefined, :a)))) == (true, false) + @test run(test_mod, + Expr(:let, Expr(:block, maybe_int(:a), maybe_int(:b), maybe_int(:c)), + Expr(:tuple, + Expr(:islocal, :a), + Expr(:isdefined, :a), + Expr(:islocal, :b), + Expr(:isdefined, :b), + Expr(:islocal, :c), + Expr(:isdefined, :c)))) == + (true, false, true, false, true, false) + + # placeholder should at least pass lowering + # flisp bug: isdefined throws because `_` is assumed global + @testset "placeholder" for p_inner in [maybe_int(:_), Expr(:(=), maybe_int(:_), 1)], + p_block in [p_inner, Expr(:block, p_inner)] + ex = Expr(:let, p_block, Expr(:block, Expr(:islocal, :_))) + @test run(test_mod, ex) == false context=ex + end + # placeholder should still run rhs + @test run(test_mod, :(let v = Int[] + let _ = push!(v, 1); end + length(v) + end)) == 1 + @test run(test_mod, :(let v::Vector{Int} = Int[] + let _ = push!(v, 1); end + length(v) + end)) == 1 + + # assignment forms + @test run(test_mod, + Expr(:let, + Expr(:(=), maybe_int(:a), 1), + Expr(:tuple, Expr(:islocal, :a), :a))) == (true, 1) + @test run(test_mod, + Expr(:let, + Expr(:block, Expr(:(=), maybe_int(:a), 1)), + Expr(:tuple, Expr(:islocal, :a), :a))) == (true, 1) + @test run(test_mod, + Expr(:let, + Expr(:block, + Expr(:(=), maybe_int(:a), 10), + Expr(:(=), maybe_int(:b), 20), + Expr(:(=), maybe_int(:c), 30)), + Expr(:tuple, + Expr(:islocal, :a), :a, + Expr(:islocal, :b), :b, + Expr(:islocal, :c), :c))) == (true, 10, true, 20, true, 30) + + @test run(test_mod, + Expr(:let, + Expr(:block, + Expr(:(=), + Expr(:tuple, :a1, maybe_int(:a2), :a3), + Expr(:tuple, 11, 12, 13)), + Expr(:(=), + Expr(:tuple, :b1, :b2, :b3, :_), + Expr(:tuple, 21, 22, 23, 0)), + Expr(:(=), + Expr(:tuple, Expr(:parameters, :c1, maybe_int(:c2), :c3)), + :((;c1=31, c2=32, c3=33)))), + Expr(:tuple, + Expr(:islocal, :a1), :a1, + Expr(:islocal, :a2), :a2, + Expr(:islocal, :a3), :a3, + Expr(:islocal, :b1), :b1, + Expr(:islocal, :b2), :b2, + Expr(:islocal, :b3), :b3, + Expr(:islocal, :c1), :c1, + Expr(:islocal, :c2), :c2, + Expr(:islocal, :c3), :c3, + ))) == + (true, 11, true, 12, true, 13, + true, 21, true, 22, true, 23, + true, 31, true, 32, true, 33) + + @test run(test_mod, + Expr(:let, + Expr(:block, + Expr(:(=), + Expr(:tuple, :a1, maybe_int(:a2), Expr(:..., :a3)), + Expr(:tuple, 11, 12, 13, 14, 15)), + Expr(:(=), + Expr(:tuple, :b1, :b2, :b3, Expr(:..., :_)), + Expr(:tuple, 21, 22, 23, 0, 0, 0))), + Expr(:tuple, + Expr(:islocal, :a1), :a1, + Expr(:islocal, :a2), :a2, + Expr(:islocal, :a3), :a3, + Expr(:islocal, :b1), :b1, + Expr(:islocal, :b2), :b2, + Expr(:islocal, :b3), :b3, + ))) == + (true, 11, true, 12, true, (13, 14, 15), + true, 21, true, 22, true, 23) + + # functions + @test run(test_mod, + Expr(:let, + Expr(:(=), maybe_int(Expr(:call, :f)), 1), + Expr(:tuple, + Expr(:call, :f), + Expr(:islocal, :f)))) == (1, true) + + @test run(test_mod, + Expr(:let, + Expr(:block, + Expr(:(=), maybe_int(Expr(:call, :f)), 1), + Expr(:(=), maybe_int(Expr(:call, :g)), 2)), + Expr(:tuple, + Expr(:call, :f), + Expr(:islocal, :f), + Expr(:call, :g), + Expr(:islocal, :g)))) == (1, true, 2, true) + + @test run(test_mod, + Expr(:let, + Expr(:(=), + Expr(:where, maybe_int(Expr(:call, :f, :(x::Int))), :Int), + :x), + Expr(:tuple, + Expr(:call, :f, "foo"), + Expr(:islocal, :f)))) == ("foo", true) + + @test run(test_mod, + Expr(:let, + Expr(:(=), Expr(:where, + Expr(:where, + maybe_int(Expr(:call, :f, :(x::Int), :(y::T))), + :T), + :Int), :(x*y)), + Expr(:tuple, + Expr(:call, :f, "x", "y"), + Expr(:islocal, :f)))) == ("xy", true) + +end + +@testset "(AI) reassigned local with `<:`-bounded declared type" begin + # A declared type containing a `<:` bound desugars to a `TypeVar(...)` + # construction bound to an SSA value. The declared type is re-evaluated at + # each assignment (like flisp), so each emission needs fresh SSA values. + @test JuliaLowering.include_string(test_mod, """ + function f() + local x::Type{<:Real} = Int + x = Float64 + x + end + f() + """) === Float64 + + # Bare declaration followed by two assignments + @test JuliaLowering.include_string(test_mod, """ + function f() + local x::Type{<:Real} + x = Int + x = Float64 + x + end + f() + """) === Float64 + + # `Vector{<:Real}` variant + @test JuliaLowering.include_string(test_mod, """ + function f() + local x::Vector{<:Real} = [1, 2, 3] + x = [1.0] + x + end + f() + """) == [1.0] + + # Declaration + reassignment inside a `let` + @test JuliaLowering.include_string(test_mod, """ + let + local y::Type{<:Real} = Int + y = Float64 + y + end + """) === Float64 + + # Captured typed local: the closure sees the reassigned value + @test JuliaLowering.include_string(test_mod, """ + function f() + local x::Type{<:Real} = Int + g = () -> x + x = Float64 + g() + end + f() + """) === Float64 + + # Three-plus assignments + @test JuliaLowering.include_string(test_mod, """ + function f() + local x::Type{<:Real} = Int + x = Float32 + x = Float64 + x = Int8 + x + end + f() + """) === Int8 + + # The convert/typeassert is still enforced on every assignment + @test_throws Exception JuliaLowering.include_string(test_mod, """ + function f() + local x::Type{<:Real} = Int + x = String + x + end + f() + """) + + # A side-effecting declared-type expression is re-evaluated once per + # assignment, exactly as flisp does + @test JuliaLowering.include_string(test_mod, """ + let side = Ref(0) + sidetype() = (side[] += 1; Type{<:Real}) + function f() + local x::(sidetype()) = Int + x = Float64 + x + end + (f(), side[]) + end + """) === (Float64, 2) +end + +@testset "Bodyless `function Name end` declares a fresh generic function" begin + # Shadowing a type visible via `using Mod` (Globtim / Optim.Sphere): the + # following method builds a real Function, not the imported constructor. + m = @newmod() + @test JuliaLowering.include_string(m, """ + module Provider + export Sphere + struct Sphere end + end + module Consumer + using ..Provider + function Sphere end + Sphere(x::AbstractVector) = sum(x) + end + (Consumer.Sphere isa Function, parentmodule(Consumer.Sphere) === Consumer, + Consumer.Sphere([1,2,3])) + """) == (true, true, 6) + + # Shadowing a type visible only via the implicit `using Core, Base` + # (CImGui / Base.Docs.Text): `Base.Text`'s own constructor is untouched. + m = @newmod() + @test JuliaLowering.include_string(m, """ + module Consumer + function Text end + Text(fmt) = "text: \$fmt" + end + (Consumer.Text isa Function, Consumer.Text !== Base.Text, + Base.Text("x") isa Base.Docs.Text, Consumer.Text("hi")) + """) == (true, true, true, "text: hi") + + # A fresh (unimported) name works unchanged. + m = @newmod() + @test JuliaLowering.include_string(m, """ + module Consumer + function Foo end + Foo(x::Int) = x + 1 + end + (Consumer.Foo isa Function, parentmodule(Consumer.Foo) === Consumer, + Consumer.Foo(3)) + """) == (true, true, 4) + + # A function visible via `using Mod` is likewise shadowed by a fresh one. + m = @newmod() + @test JuliaLowering.include_string(m, """ + module Provider + export myfun + function myfun end + myfun(x::Int) = x * 10 + end + module Consumer + using ..Provider + function myfun end + myfun(x::AbstractVector) = length(x) + end + (Consumer.myfun !== Provider.myfun, Consumer.myfun([1,2,3])) + """) == (true, 3) + + # An *explicit* selective `using Mod: Name` binding is extended, not + # shadowed -- flisp defers to the existing binding (the runtime `global` + # refers to it), so the bodyless decl leaves `Name` bound to the imported + # type. The bare `(method Name)` on that type prints the same "extended + # without qualification" warning flisp does, so we silence it here. + m = @newmod() + @test redirect_stderr(devnull) do + JuliaLowering.include_string(m, """ + module Provider + export Sphere + struct Sphere end + end + module Consumer + using ..Provider: Sphere + function Sphere end + end + (Consumer.Sphere === Provider.Sphere, Consumer.Sphere isa Function) + """) + end == (true, false) + + # Likewise for an explicit `import Mod: Name`. + m = @newmod() + @test JuliaLowering.include_string(m, """ + module Provider + export Sphere + struct Sphere end + end + module Consumer + import ..Provider: Sphere + function Sphere end + end + (Consumer.Sphere === Provider.Sphere, Consumer.Sphere isa Function) + """) == (true, false) + + # A bodyless decl whose name is a local stays a local closure and leaks no + # module global (flisp's `global-if-global` local exemption). + m = @newmod() + @test JuliaLowering.include_string(m, """ + begin + local no_method_f + function no_method_f end + no_method_f + end + """) isa Function + @test !isdefined(m, :no_method_f) +end diff --git a/JuliaLowering/test/decls_ir.jl b/JuliaLowering/test/decls_ir.jl new file mode 100644 index 0000000000000..61305c7559546 --- /dev/null +++ b/JuliaLowering/test/decls_ir.jl @@ -0,0 +1,558 @@ +######################################## +# Local declaration with type +begin + local x::T = 1 +end +#--------------------- +1 (newvar slot₁/x) +2 1 +3 TestMod.T +4 (= slot₂/tmp %₂) +5 (call core.isa slot₂/tmp %₃) +6 (gotoifnot %₅ label₈) +7 (goto label₁₀) +8 (call top.convert %₃ slot₂/tmp) +9 (= slot₂/tmp (call core.typeassert %₈ %₃)) +10 slot₂/tmp +11 (= slot₁/x %₁₀) +12 (return %₂) + +######################################## +# Local declarations outside a scope could be disallowed +# See https://github.com/JuliaLang/julia/issues/57483 +local x +#--------------------- +1 (newvar slot₁/x) +2 (return core.nothing) + +######################################## +# Local declaration allowed in tail position +begin + local x +end +#--------------------- +1 (newvar slot₁/x) +2 (return core.nothing) + +######################################## +# Local declaration allowed in value position +# TODO: This may be a bug in flisp lowering - should we reconsider this? +let + y = local x +end +#--------------------- +1 (newvar slot₁/x) +2 core.nothing +3 (= slot₂/y %₂) +4 (return %₂) + +######################################## +# Global declaration allowed in tail position +global x +#--------------------- +1 (call core.declare_global TestMod :x false) +2 latestworld +3 (return core.nothing) + +######################################## +# Global declaration allowed in tail position, nested +begin + global x +end +#--------------------- +1 (call core.declare_global TestMod :x false) +2 latestworld +3 (return core.nothing) + +######################################## +# Error: Global declaration not allowed in value position +y = global x +#--------------------- +LoweringError: +y = global x +# ╙ ── global declaration doesn't read the variable and can't return a value + +######################################## +# const +const xx = 10 +#--------------------- +1 10 +2 (call core.declare_const TestMod :xx %₁) +3 latestworld +4 (return %₁) + +######################################## +# Typed const +const xx::T = 10 +#--------------------- +1 TestMod.T +2 (= slot₁/tmp 10) +3 (call core.isa slot₁/tmp %₁) +4 (gotoifnot %₃ label₆) +5 (goto label₈) +6 (call top.convert %₁ slot₁/tmp) +7 (= slot₁/tmp (call core.typeassert %₆ %₁)) +8 slot₁/tmp +9 (call core.declare_const TestMod :xx %₈) +10 latestworld +11 (return %₈) + +######################################## +# Const tuple +const xxx,xxxx,xxxxx = 10,20,30 +#--------------------- +1 10 +2 (call core.declare_const TestMod :xxx %₁) +3 latestworld +4 20 +5 (call core.declare_const TestMod :xxxx %₄) +6 latestworld +7 30 +8 (call core.declare_const TestMod :xxxxx %₇) +9 latestworld +10 (call core.tuple 10 20 30) +11 (return %₁₀) + +######################################## +# Const in chain: only first is const +const c0 = v0 = v1 = 123 +#--------------------- +1 123 +2 (call core.declare_global TestMod :v1 true) +3 latestworld +4 (call core.get_binding_type TestMod :v1) +5 (= slot₁/tmp %₁) +6 (call core.isa slot₁/tmp %₄) +7 (gotoifnot %₆ label₉) +8 (goto label₁₀) +9 (= slot₁/tmp (call top.convert %₄ slot₁/tmp)) +10 slot₁/tmp +11 (call core.setglobal! TestMod :v1 %₁₀) +12 (call core.declare_global TestMod :v0 true) +13 latestworld +14 (call core.get_binding_type TestMod :v0) +15 (= slot₂/tmp %₁) +16 (call core.isa slot₂/tmp %₁₄) +17 (gotoifnot %₁₆ label₁₉) +18 (goto label₂₀) +19 (= slot₂/tmp (call top.convert %₁₄ slot₂/tmp)) +20 slot₂/tmp +21 (call core.setglobal! TestMod :v0 %₂₀) +22 (call core.declare_const TestMod :c0 %₁) +23 latestworld +24 (return %₁) + +######################################## +# Global assignment +xx = 10 +#--------------------- +1 (call core.declare_global TestMod :xx true) +2 latestworld +3 (call core.get_binding_type TestMod :xx) +4 (= slot₁/tmp 10) +5 (call core.isa slot₁/tmp %₃) +6 (gotoifnot %₅ label₈) +7 (goto label₉) +8 (= slot₁/tmp (call top.convert %₃ slot₁/tmp)) +9 slot₁/tmp +10 (call core.setglobal! TestMod :xx %₉) +11 (return 10) + +######################################## +# Typed global assignment +global xx::T = 10 +#--------------------- +1 (call core.declare_global TestMod :xx false) +2 latestworld +3 TestMod.T +4 (call core.declare_global TestMod :xx true %₃) +5 latestworld +6 (call core.declare_global TestMod :xx true) +7 latestworld +8 (call core.get_binding_type TestMod :xx) +9 (= slot₁/tmp 10) +10 (call core.isa slot₁/tmp %₈) +11 (gotoifnot %₁₀ label₁₃) +12 (goto label₁₄) +13 (= slot₁/tmp (call top.convert %₈ slot₁/tmp)) +14 slot₁/tmp +15 (call core.setglobal! TestMod :xx %₁₄) +16 (return 10) + +######################################## +# Error: local with two type declarations +begin + local x::T = 1 + local x::S = 1 +end +#--------------------- +LoweringError: +begin + local x::T = 1 + local x::S = 1 +# └───────┘ ── multiple type declarations found for `x` +end + +######################################## +# Error: local with two type declarations, requiring scope resolution +begin + local x::Int = 1 + x::Int = 1 +end +#--------------------- +LoweringError: +begin + local x::Int = 1 + x::Int = 1 +# └────────┘ ── multiple type declarations found for `x` +end + +######################################## +# multiple type declarations is OK for globals +begin + global x::Int + x::Int = 1 +end +#--------------------- +1 TestMod.Int +2 (call core.declare_global TestMod :x true %₁) +3 latestworld +4 (call core.declare_global TestMod :x false) +5 latestworld +6 TestMod.Int +7 (call core.declare_global TestMod :x true %₆) +8 latestworld +9 (call core.declare_global TestMod :x true) +10 latestworld +11 (call core.get_binding_type TestMod :x) +12 (= slot₁/tmp 1) +13 (call core.isa slot₁/tmp %₁₁) +14 (gotoifnot %₁₃ label₁₆) +15 (goto label₁₇) +16 (= slot₁/tmp (call top.convert %₁₁ slot₁/tmp)) +17 slot₁/tmp +18 (call core.setglobal! TestMod :x %₁₇) +19 (return 1) + +######################################## +# Error: const ref +const x[] = 1 +#--------------------- +LoweringError: +const x[] = 1 +# └─┘ ── cannot declare this form constant + +######################################## +# Error: const ref +const x[2] = 1 +#--------------------- +LoweringError: +const x[2] = 1 +# └──┘ ── cannot declare this form constant + +######################################## +# Error: const setproperty +const Main.x = 1 +#--------------------- +LoweringError: +const Main.x = 1 +# └────┘ ── cannot declare this form constant + +######################################## +# Error: const ref, tuple-nested +const (a, (b, x[])) = (1, (2, 3)) +#--------------------- +LoweringError: +const (a, (b, x[])) = (1, (2, 3)) +# └─┘ ── cannot declare this form constant + +######################################## +# Error: const setproperty, tuple-nested +const (a, (b, x.y)) = (1, (2, 3)) +#--------------------- +LoweringError: +const (a, (b, x.y)) = (1, (2, 3)) +# └─┘ ── cannot declare this form constant + +######################################## +# Error: Const not supported on locals +const local x = 1 +#--------------------- +LoweringError: +const local x = 1 +# └──────────┘ ── unsupported `const local` declaration + +######################################## +# Error: Const not supported on locals +let + const x = 1 +end +#--------------------- +LoweringError: +let + const x = 1 +# └────┘ ── unsupported `const` declaration on local variable +end + +######################################## +# Error: Const not supported on locals +begin + local x + const x = 1 +end +#--------------------- +LoweringError: +begin + local x + const x = 1 +# └────┘ ── unsupported `const` declaration on local variable +end + +######################################## +# Error: local in let first arg +let local a = 1 +end +#--------------------- +LoweringError: +let local a = 1 +# └─────────┘ ── expected identifier or assignment +end + +######################################## +# Error: global in let first arg +let global a = 1 +end +#--------------------- +LoweringError: +let global a = 1 +# └──────────┘ ── expected identifier or assignment +end + +######################################## +# Error: const in let first arg +let const a = 1 +end +#--------------------- +LoweringError: +let const a = 1 +# └─────────┘ ── expected identifier or assignment +end + +######################################## +# Error: function-stub in let first arg +let function foo end +end +#--------------------- +LoweringError: +let function foo end +# └──────────────┘ ── expected identifier or assignment +end + +######################################## +# Error: function in let first arg +let function foo(x); x; end +end +#--------------------- +LoweringError: +let function foo(x); x; end +# └─────────────────────┘ ── expected identifier or assignment +end + +######################################## +# Error: Const not supported in function scope +function (); global g; const g = 1; end +#--------------------- +LoweringError: +function (); global g; const g = 1; end +# └─────────┘ ── unsupported `const` inside function + +######################################## +# Type decl on function argument +function f(x) + x::Int = 1 + x = 2.0 + x +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/tmp(!read) slot₄/tmp(!read) slot₅/x(!read)] + 1 (= slot₅/x slot₂/x) + 2 1 + 3 TestMod.Int + 4 (= slot₃/tmp %₂) + 5 (call core.isa slot₃/tmp %₃) + 6 (gotoifnot %₅ label₈) + 7 (goto label₁₀) + 8 (call top.convert %₃ slot₃/tmp) + 9 (= slot₃/tmp (call core.typeassert %₈ %₃)) + 10 slot₃/tmp + 11 (= slot₅/x %₁₀) + 12 2.0 + 13 TestMod.Int + 14 (= slot₄/tmp %₁₂) + 15 (call core.isa slot₄/tmp %₁₃) + 16 (gotoifnot %₁₅ label₁₈) + 17 (goto label₂₀) + 18 (call top.convert %₁₃ slot₄/tmp) + 19 (= slot₄/tmp (call core.typeassert %₁₈ %₁₃)) + 20 slot₄/tmp + 21 (= slot₅/x %₂₀) + 22 slot₅/x + 23 (return %₂₂) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Error: global type decls only allowed at top level +function f() + global x::Int +end +#--------------------- +LoweringError: +function f() + global x::Int +# └────┘ ── type declarations for global variables must be at top level, not inside a function +end + +######################################## +# Error: global type decls only allowed at top level (=) +function f() + global x::Int = 1 +end +#--------------------- +LoweringError: +function f() + global x::Int = 1 +# └─────────┘ ── type declarations for global variables must be at top level, not inside a function +end + +######################################## +# Error: global type decls only allowed at top level, requiring scope resolution +function f() + global x + x::Int = 1 +end +#--------------------- +LoweringError: +function f() + global x + x::Int = 1 +# └────────┘ ── type declarations for global variables must be at top level, not inside a function +end + +######################################## +# FIXME: Error: global type decls only allowed at top level (.=) +function f() + global x::Int .= 1 +end +#--------------------- +1 (method TestMod.f) +2 latestworld +3 (call core.declare_global TestMod :x false) +4 latestworld +5 TestMod.f +6 (call core.TypeEqOf %₅) +7 (call core.svec %₆) +8 (call core.svec) +9 SourceLocation::1:10 +10 (call core.svec %₇ %₈ %₉) +11 --- method TestMod.f %₁₀ + slots: [slot₁/#self#(!read)] + 1 TestMod.x + 2 TestMod.Int + 3 (call core.typeassert %₁ %₂) + 4 (call top.broadcasted top.identity 1) + 5 (call top.materialize! %₃ %₄) + 6 (return %₅) +12 latestworld +13 TestMod.f +14 (return %₁₃) + +######################################## +# FIXME: Error: global type decls only allowed at top level (+=) +function f() + global x::Int += 1 +end +#--------------------- +1 (method TestMod.f) +2 latestworld +3 (call core.declare_global TestMod :x false) +4 latestworld +5 (call core.declare_global TestMod :x true) +6 latestworld +7 TestMod.f +8 (call core.TypeEqOf %₇) +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::1:10 +12 (call core.svec %₉ %₁₀ %₁₁) +13 --- method TestMod.f %₁₂ + slots: [slot₁/#self#(!read) slot₂/tmp(!read)] + 1 TestMod.+ + 2 TestMod.x + 3 TestMod.Int + 4 (call core.typeassert %₂ %₃) + 5 (call %₁ %₄ 1) + 6 (call core.get_binding_type TestMod :x) + 7 (= slot₂/tmp %₅) + 8 (call core.isa slot₂/tmp %₆) + 9 (gotoifnot %₈ label₁₁) + 10 (goto label₁₂) + 11 (= slot₂/tmp (call top.convert %₆ slot₂/tmp)) + 12 slot₂/tmp + 13 (call core.setglobal! TestMod :x %₁₂) + 14 (return %₅) +14 latestworld +15 TestMod.f +16 (return %₁₅) + +######################################## +# FIXME: Error: global type decls only allowed at top level (.+=) +function f() + global x::Int .+= 1 +end +#--------------------- +1 (method TestMod.f) +2 latestworld +3 (call core.declare_global TestMod :x false) +4 latestworld +5 TestMod.f +6 (call core.TypeEqOf %₅) +7 (call core.svec %₆) +8 (call core.svec) +9 SourceLocation::1:10 +10 (call core.svec %₇ %₈ %₉) +11 --- method TestMod.f %₁₀ + slots: [slot₁/#self#(!read)] + 1 TestMod.x + 2 TestMod.+ + 3 TestMod.Int + 4 (call core.typeassert %₁ %₃) + 5 (call top.broadcasted %₂ %₄ 1) + 6 (call top.materialize! %₁ %₅) + 7 (return %₆) +12 latestworld +13 TestMod.f +14 (return %₁₃) + +######################################## +# Error: global type decls only allowed at top level (tuple) +function f() + global (x::Int, y) = 1,2 +end +#--------------------- +LoweringError: +function f() + global (x::Int, y) = 1,2 +# └────────────────┘ ── type declarations for global variables must be at top level, not inside a function +end diff --git a/JuliaLowering/test/destructuring.jl b/JuliaLowering/test/destructuring.jl new file mode 100644 index 0000000000000..8ba51138474fe --- /dev/null +++ b/JuliaLowering/test/destructuring.jl @@ -0,0 +1,221 @@ +test_mod = Module() + +@testset "Destructuring via iteration" begin + +@test JuliaLowering.include_string(test_mod, """ +let + as = [1,2,3] + (x,y) = as + (x,y) +end +""") == (1,2) + +@test JuliaLowering.include_string(test_mod, """ +let + as = [1,2,3] + (x,ys...) = as + (x,ys) +end +""") == (1, [2,3]) + +@test JuliaLowering.include_string(test_mod, """ +let + as = [1,2,3,4] + (x,ys...,z) = as + (x,ys,z) +end +""") == (1, [2, 3], 4) + +@test JuliaLowering.include_string(test_mod, """ +let + as = [1,2,3,4] + (xs...,y) = as + (xs,y) +end +""") == ([1, 2, 3], 4) + +# Case where indexed_iterate is just iteration +@test JuliaLowering.include_string(test_mod, """ +let + (x,ys...,z) = "aβcδe" + (x,ys,z) +end +""") == ('a', "βcδ", 'e') + + +# Use in value position yields rhs +@test JuliaLowering.include_string(test_mod, """ +let + as = [1,2] + zs = begin + (x,y) = as + end + (x,y, as === zs) +end +""") == (1, 2, true) + +# lhs variable name in rhs +@test JuliaLowering.include_string(test_mod, """ +let + x = (1,2) + (x,y) = x + (x,y) +end +""") == (1, 2) + +@test JuliaLowering.include_string(test_mod, """ +let + x = (1,2) + (x...,y) = x + (x,y) +end +""") == ((1,), 2) + +@test JuliaLowering.include_string(test_mod, """ +let + zs = [(1,2), (3,(4,5))] + ((a,b), (c,(d,e))) = zs + (a,b,c,d,e) +end +""") == (1,2,3,4,5) + +@test JuliaLowering.include_string(test_mod, """ +let + zs = [[1,2,3], 4] + ((a,bs...), c) = zs + (a, bs, c) +end +""") == (1, [2,3], 4) + +end + + +@testset "Tuple elimination with tuples on both sides" begin + +# Simple case +@test JuliaLowering.include_string(test_mod, """ +let a = 1, b = 2 + (x,y) = (a,b) + (x,y) +end +""") == (1, 2) + +# lhs variable name in rhs +@test JuliaLowering.include_string(test_mod, """ +let x = 1, y = 2 + (x,y) = (y,x) + (x,y) +end +""") == (2, 1) + +# Slurps and splats + +@test JuliaLowering.include_string(test_mod, """ +let a = 1, b = 2, c = 3 + (x, ys..., z) = (a, b, c) + (x, ys, z) +end +""") == (1, (2,), 3) + +@test JuliaLowering.include_string(test_mod, """ +let a = 1, b = 2, cs = (3,4) + (x, ys...) = (a, b, cs...) + (x, ys) +end +""") == (1, (2,3,4)) + +@test JuliaLowering.include_string(test_mod, """ +let a = 1, bs = (2,3), c = 4 + (x, ys...) = (a, bs..., c) + (x, ys) +end +""") == (1, (2,3,4)) + +@test JuliaLowering.include_string(test_mod, """ +let a = 1, b = 2, cs = (3,4) + (x, ys..., z) = (a, b, cs...) + (x, ys, z) +end +""") == (1, (2,3), 4) + +@test JuliaLowering.include_string(test_mod, """ +let a = 1 + (x, ys...) = (a,) + (x, ys) +end +""") == (1, ()) + +# dotted rhs in last place +@test JuliaLowering.include_string(test_mod, """ +let + rh = (2, 3) + (x,y,z) = (1,rh...) + (x,y,z) +end +""") == (1, 2, 3) + +# in value position +@test JuliaLowering.include_string(test_mod, """ +let + rh = (2, 3) + (x,y) = (1,rh...) +end +""") == (1, 2, 3) + +# Side effects in the right hand tuple can affect the previous left hand side +# bindings, for example, `x`, below. In this case we need to ensure `f()` is +# called before `x` is assigned the value from the right hand side. +# (the flisp implementation fails this test.) +@test JuliaLowering.include_string(test_mod, """ +let + function f() + x=100 + 2 + end + (x,y) = (1,f()) + x,y +end +""") == (1,2) + +# `x` is not assigned and no side effect from `f()` happens when the right hand +# side throws an UndefVarError +@test JuliaLowering.include_string(test_mod, """ +let x=1, y=2, z=3, side_effect=false, a + exc = try + function f() + side_effect=true + end + (x,y,z) = (100, a, f()) + catch e + e + end + (x, y, z, side_effect, exc.var) +end +""") == (1, 2, 3, false, :a) + +# Require that rhs is evaluated before any assignments, thus `x` is not defined +# here because accessing `a` first throws an UndefVarError +@test JuliaLowering.include_string(test_mod, """ +let x, y, a + try + (x, y) = (1, a) + catch + end + @isdefined(x) +end +""") == false + +end + + +@testset "Property destructuring" begin + +@test JuliaLowering.include_string(test_mod, """ +let + ab = (a=1, b=2) + (; a, b) = ab + (a, b) +end +""") == (1, 2) + +end diff --git a/JuliaLowering/test/destructuring_ir.jl b/JuliaLowering/test/destructuring_ir.jl new file mode 100644 index 0000000000000..d319c7d701fcf --- /dev/null +++ b/JuliaLowering/test/destructuring_ir.jl @@ -0,0 +1,385 @@ +######################################## +# Simple destructuring +let + (x,y) = as +end +#--------------------- +1 TestMod.as +2 (call top.indexed_iterate %₁ 1) +3 (= slot₂/x (call core.getfield %₂ 1)) +4 (= slot₁/iterstate (call core.getfield %₂ 2)) +5 TestMod.as +6 slot₁/iterstate +7 (call top.indexed_iterate %₅ 2 %₆) +8 (= slot₃/y (call core.getfield %₇ 1)) +9 TestMod.as +10 (return %₉) + +######################################## +# Trivial slurping +let + (xs...,) = as +end +#--------------------- +1 TestMod.as +2 (= slot₁/xs (call top.rest %₁)) +3 TestMod.as +4 (return %₃) + +######################################## +# Slurping last arg +let + (x, ys...) = as +end +#--------------------- +1 TestMod.as +2 (call top.indexed_iterate %₁ 1) +3 (= slot₂/x (call core.getfield %₂ 1)) +4 (= slot₁/iterstate (call core.getfield %₂ 2)) +5 TestMod.as +6 slot₁/iterstate +7 (= slot₃/ys (call top.rest %₅ %₆)) +8 TestMod.as +9 (return %₈) + +######################################## +# Slurping, first arg +let + (xs..., y, z) = as +end +#--------------------- +1 TestMod.as +2 (call top.split_rest %₁ 2) +3 (= slot₂/xs (call core.getfield %₂ 1)) +4 (call core.getfield %₂ 2) +5 (call top.indexed_iterate %₄ 1) +6 (= slot₃/y (call core.getfield %₅ 1)) +7 (= slot₁/iterstate (call core.getfield %₅ 2)) +8 slot₁/iterstate +9 (call top.indexed_iterate %₄ 2 %₈) +10 (= slot₄/z (call core.getfield %₉ 1)) +11 TestMod.as +12 (return %₁₁) + +######################################## +# Slurping, middle arg +let + (x, ys..., z) = as +end +#--------------------- +1 TestMod.as +2 (call top.indexed_iterate %₁ 1) +3 (= slot₂/x (call core.getfield %₂ 1)) +4 (= slot₁/iterstate (call core.getfield %₂ 2)) +5 TestMod.as +6 slot₁/iterstate +7 (call top.split_rest %₅ 1 %₆) +8 (= slot₃/ys (call core.getfield %₇ 1)) +9 (call core.getfield %₇ 2) +10 (call top.indexed_iterate %₉ 1) +11 (= slot₄/z (call core.getfield %₁₀ 1)) +12 TestMod.as +13 (return %₁₂) + +######################################## +# Error: Slurping multiple args +(xs..., ys...) = x +#--------------------- +LoweringError: +(xs..., ys...) = x +└────────────┘ ── multiple `...` in destructuring assignment are ambiguous + +######################################## +# Recursive destructuring +let + ((x,y), (z,w)) = as +end +#--------------------- +1 TestMod.as +2 (call top.indexed_iterate %₁ 1) +3 (call core.getfield %₂ 1) +4 (= slot₁/iterstate (call core.getfield %₂ 2)) +5 TestMod.as +6 slot₁/iterstate +7 (call top.indexed_iterate %₅ 2 %₆) +8 (call core.getfield %₇ 1) +9 (call top.indexed_iterate %₃ 1) +10 (= slot₅/x (call core.getfield %₉ 1)) +11 (= slot₂/iterstate (call core.getfield %₉ 2)) +12 slot₂/iterstate +13 (call top.indexed_iterate %₃ 2 %₁₂) +14 (= slot₆/y (call core.getfield %₁₃ 1)) +15 (call top.indexed_iterate %₈ 1) +16 (= slot₇/z (call core.getfield %₁₅ 1)) +17 (= slot₃/iterstate (call core.getfield %₁₅ 2)) +18 slot₃/iterstate +19 (call top.indexed_iterate %₈ 2 %₁₈) +20 (= slot₄/w (call core.getfield %₁₉ 1)) +21 TestMod.as +22 (return %₂₁) + +######################################## +# Recursive destructuring with slurping +let + ((x,ys...), z) = as +end +#--------------------- +1 TestMod.as +2 (call top.indexed_iterate %₁ 1) +3 (call core.getfield %₂ 1) +4 (= slot₁/iterstate (call core.getfield %₂ 2)) +5 TestMod.as +6 slot₁/iterstate +7 (call top.indexed_iterate %₅ 2 %₆) +8 (= slot₅/z (call core.getfield %₇ 1)) +9 (call top.indexed_iterate %₃ 1) +10 (= slot₃/x (call core.getfield %₉ 1)) +11 (= slot₂/iterstate (call core.getfield %₉ 2)) +12 slot₂/iterstate +13 (= slot₄/ys (call top.rest %₃ %₁₂)) +14 TestMod.as +15 (return %₁₄) + +######################################## +# Destructuring with simple tuple elimination +let + (x, y) = (a, b) +end +#--------------------- +1 TestMod.a +2 TestMod.b +3 (= slot₁/x %₁) +4 (= slot₂/y %₂) +5 (call core.tuple %₁ %₂) +6 (return %₅) + +######################################## +# Destructuring with tuple elimination where variables are repeated +let + (x, y, z) = (y, a, x) +end +#--------------------- +1 slot₂/y +2 TestMod.a +3 slot₁/x +4 (= slot₁/x %₁) +5 (= slot₂/y %₂) +6 (= slot₃/z %₃) +7 (call core.tuple %₁ %₂ %₃) +8 (return %₇) + +######################################## +# Destructuring with simple tuple elimination and rhs with side effects +let + (x, y) = (f(), b) +end +#--------------------- +1 TestMod.f +2 (call %₁) +3 TestMod.b +4 (= slot₁/x %₂) +5 (= slot₂/y %₃) +6 (call core.tuple %₂ %₃) +7 (return %₆) + +######################################## +# Destructuring with simple tuple elimination and lhs with side effects +let + (x[10], y[20]) = (1,2) +end +#--------------------- +1 1 +2 TestMod.x +3 (call top.setindex! %₂ %₁ 10) +4 2 +5 TestMod.y +6 (call top.setindex! %₅ %₄ 20) +7 (call core.tuple 1 2) +8 (return %₇) + +######################################## +# Destructuring with tuple elimination and trailing rhs ... +let + (x, y) = (a, rhs...) +end +#--------------------- +1 TestMod.a +2 TestMod.rhs +3 (= slot₁/x %₁) +4 (call top.indexed_iterate %₂ 1) +5 (= slot₂/y (call core.getfield %₄ 1)) +6 (call core.tuple %₁) +7 (call core._apply_iterate top.iterate core.tuple %₆ %₂) +8 (return %₇) + +######################################## +# Destructuring with non-trailing rhs `...` does not use tuple elimination +# (though we could do it for the `x = a` part here) +let + (x, y, z) = (a, rhs..., b) +end +#--------------------- +1 TestMod.a +2 (call core.tuple %₁) +3 TestMod.rhs +4 TestMod.b +5 (call core.tuple %₄) +6 (call core._apply_iterate top.iterate core.tuple %₂ %₃ %₅) +7 (call top.indexed_iterate %₆ 1) +8 (= slot₂/x (call core.getfield %₇ 1)) +9 (= slot₁/iterstate (call core.getfield %₇ 2)) +10 slot₁/iterstate +11 (call top.indexed_iterate %₆ 2 %₁₀) +12 (= slot₃/y (call core.getfield %₁₁ 1)) +13 (= slot₁/iterstate (call core.getfield %₁₁ 2)) +14 slot₁/iterstate +15 (call top.indexed_iterate %₆ 3 %₁₄) +16 (= slot₄/z (call core.getfield %₁₅ 1)) +17 (return %₆) + +######################################## +# Destructuring with tuple elimination and final ... on lhs +let + (x, ys...) = (a,b,c) +end +#--------------------- +1 TestMod.a +2 TestMod.b +3 TestMod.c +4 (= slot₁/x %₁) +5 (call core.tuple %₂ %₃) +6 (= slot₂/ys %₅) +7 (call core.tuple %₁ %₂ %₃) +8 (return %₇) + +######################################## +# Destructuring with tuple elimination, slurping, and completely effect free right hand sides +let + (x, ys...) = (1,2,3) +end +#--------------------- +1 (= slot₁/x 1) +2 (call core.tuple 2 3) +3 (= slot₂/ys %₂) +4 (call core.tuple 1 2 3) +5 (return %₄) + +######################################## +# Destructuring with tuple elimination and non-final ... on lhs +let + (x, ys..., z) = (a,b,c) +end +#--------------------- +1 TestMod.a +2 TestMod.b +3 TestMod.c +4 (= slot₁/x %₁) +5 (call core.tuple %₂) +6 (= slot₂/ys %₅) +7 (= slot₃/z %₃) +8 (call core.tuple %₁ %₂ %₃) +9 (return %₈) + +######################################## +# Error: Destructuring with tuple elimination and too few RHS elements +(x,) = () +#--------------------- +LoweringError: +(x,) = () +└───────┘ ── More variables on left hand side than right hand in tuple assignment + +######################################## +# Error: Destructuring with tuple elimination, slurping, and too few RHS elements +(x,y,ys...) = (1,) +#--------------------- +LoweringError: +(x,y,ys...) = (1,) +└────────────────┘ ── More variables on left hand side than right hand in tuple assignment + +######################################## +# Destructuring with tuple elimination but not in value position never creates +# the tuple +let + (x, ys...) = (a,b,c) + nothing +end +#--------------------- +1 TestMod.a +2 TestMod.b +3 TestMod.c +4 (= slot₁/x %₁) +5 (call core.tuple %₂ %₃) +6 (= slot₂/ys %₅) +7 TestMod.nothing +8 (return %₇) + +######################################## +# Property destructuring +let + (; x, y) = rhs +end +#--------------------- +1 TestMod.rhs +2 (= slot₁/x (call top.getproperty %₁ :x)) +3 (= slot₂/y (call top.getproperty %₁ :y)) +4 (return %₁) + +######################################## +# Property destructuring with colliding symbolic lhs/rhs +let + local x + (; x, y) = x +end +#--------------------- +1 slot₁/x +2 (= slot₁/x (call top.getproperty %₁ :x)) +3 (= slot₂/y (call top.getproperty %₁ :y)) +4 (return %₁) + +######################################## +# Property destructuring with nontrivial rhs +let + (; x, y) = f() +end +#--------------------- +1 TestMod.f +2 (call %₁) +3 (= slot₁/x (call top.getproperty %₂ :x)) +4 (= slot₂/y (call top.getproperty %₂ :y)) +5 (return %₂) + +######################################## +# Property destructuring with type decl +let + (; x::T) = rhs +end +#--------------------- +1 (newvar slot₁/x) +2 TestMod.rhs +3 (call top.getproperty %₂ :x) +4 TestMod.T +5 (= slot₂/tmp %₃) +6 (call core.isa slot₂/tmp %₄) +7 (gotoifnot %₆ label₉) +8 (goto label₁₁) +9 (call top.convert %₄ slot₂/tmp) +10 (= slot₂/tmp (call core.typeassert %₉ %₄)) +11 slot₂/tmp +12 (= slot₁/x %₁₁) +13 (return %₂) + +######################################## +# Error: Property destructuring with frankentuple +(x ; a, b) = rhs +#--------------------- +LoweringError: +(x ; a, b) = rhs +# └─────┘ ── property destructuring must use a single `;` before the property names, e.g. `(; a, b) = rhs` + +######################################## +# Error: Property destructuring with values for properties +(; a=1, b) = rhs +#--------------------- +LoweringError: +(; a=1, b) = rhs +# └─┘ ── expected identifier or `identifier::type` diff --git a/JuliaLowering/test/exceptions.jl b/JuliaLowering/test/exceptions.jl new file mode 100644 index 0000000000000..c2c5c3170d0c3 --- /dev/null +++ b/JuliaLowering/test/exceptions.jl @@ -0,0 +1,648 @@ +test_mod = Module() + +@test isempty(current_exceptions()) + +@testset "tail position" begin + + @test JuliaLowering.include_string(test_mod, """ + try + 1 + catch + 2 + end + """) == 1 + + @test JuliaLowering.include_string(test_mod, """ + try + error("hi") + 1 + catch + 2 + end + """) == 2 + + @test JuliaLowering.include_string(test_mod, """ + try + error("hi") + catch exc + exc + end + """) == ErrorException("hi") + + + @test JuliaLowering.include_string(test_mod, """ + try + 1 + catch + 2 + else + 3 + end + """) == 3 + + @test JuliaLowering.include_string(test_mod, """ + try + error("hi") + 1 + catch + 2 + else + 3 + end + """) == 2 + + @test JuliaLowering.include_string(test_mod, """ + begin + function f() + try + return 1 + catch + end + return 2 + end + f() + end + """) == 1 + + @test JuliaLowering.include_string(test_mod, """ + begin + function g() + try + return 1 + catch + end + end + g() + end + """) == 1 + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + while true + try + error("hi") + catch + x = 2 + break + end + end + x + end + """) == 2 + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + while true + try + x = 2 + break + catch + end + end + x + end + """) == 2 +end + +@testset "value position" begin + + @test JuliaLowering.include_string(test_mod, """ + let + x = try + 1 + catch + 2 + end + x + end + """) == 1 + + @test JuliaLowering.include_string(test_mod, """ + let + x = try + error("hi") + 1 + catch + 2 + end + x + end + """) == 2 + + @test JuliaLowering.include_string(test_mod, """ + let + x = try + error("hi") + catch exc + exc + end + x + end + """) == ErrorException("hi") + + + @test JuliaLowering.include_string(test_mod, """ + let + x = try + 1 + catch + 2 + else + 3 + end + x + end + """) == 3 + + @test JuliaLowering.include_string(test_mod, """ + let + x = try + error("hi") + 1 + catch + 2 + else + 3 + end + x + end + """) == 2 + +end + +@testset "not value/tail position" begin + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + try + x = 1 + catch + x = 2 + end + x + end + """) == 1 + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + try + error("hi") + x = 1 + catch + x = 2 + end + x + end + """) == 2 + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + try + x = error("hi") + catch exc + x = exc + end + x + end + """) == ErrorException("hi") + + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + try + x = 1 + catch + x = 2 + else + x = 3 + end + x + end + """) == 3 + + @test JuliaLowering.include_string(test_mod, """ + let x = -1 + try + error("hi") + x = 1 + catch + x = 2 + else + x = 3 + end + x + end + """) == 2 + +end + +@testset "exception stack" begin + + @test JuliaLowering.include_string(test_mod, """ + try + try + error("hi") + catch + error("ho") + end + catch + a = [] + for x in current_exceptions() + push!(a, x.exception) + end + a + end + """) == [ErrorException("hi"), ErrorException("ho")] + +end + +@test isempty(current_exceptions()) + +#------------------------------------------------------------------------------- +@testset "try/finally" begin + +test_mod = Module() + +@test JuliaLowering.include_string(test_mod, """ +let x = -1 + try + x = 1 + finally + x = 2 + end + x +end +""") == 2 + +@test JuliaLowering.include_string(test_mod, """ +let x = -1 + try + try + error("hi") + x = 1 + finally + x = 2 + end + catch + end + x +end +""") == 2 + +JuliaLowering.include_string(test_mod, """ +begin + function nested_finally(a, x, b, c) + try + try + if x + return b + end + c + finally + push!(a, 1) + end + finally + push!(a, 2) + end + end +end +""") +@test (a = []; res = test_mod.nested_finally(a, true, 100, 200); (a, res)) == ([1,2], 100) +@test (a = []; res = test_mod.nested_finally(a, false, 100, 200); (a, res)) == ([1,2], 200) + +@test JuliaLowering.include_string(test_mod, """ +try + 1 +catch + 2 +finally + 3 +end +""") == 1 + +@test JuliaLowering.include_string(test_mod, """ +try + error("hi") + 1 +catch + 2 +finally + 3 +end +""") == 2 + +@test JuliaLowering.include_string(test_mod, """ +begin + function f_try_catch_nospecialize(@nospecialize(cond)) + try + cond && throw(ArgumentError("")) + catch + return 1 + end + return 2 + end + ( + f_try_catch_nospecialize(true), + f_try_catch_nospecialize(false), + ) +end +""") == (1,2) + +@testset "continue/break and finally" for maybe_catch in ("", "catch _", "catch _\nelse") + # continue in try -> finally block -> loop-cont + @test JuliaLowering.include_string(test_mod, """ + let out = [] + for x in [1,2,3] + try + push!(out, ("try", x)) + continue + $maybe_catch + finally + push!(out, ("finally", x)) + end + push!(out, ("bad", x)) + end + out + end + """) == [("try", 1), ("finally", 1), ("try", 2), ("finally", 2), ("try", 3), ("finally", 3)] + @test isempty(current_exceptions()) + + # break in try -> finally block -> loop-exit + @test JuliaLowering.include_string(test_mod, """ + let out = [] + for x in [1,2,3] + try + push!(out, ("try", x)) + break + $maybe_catch + finally + push!(out, ("finally", x)) + end + push!(out, ("bad", x)) + end + out + end + """) == [("try", 1), ("finally", 1)] + @test isempty(current_exceptions()) + + # break/continue in loop in try -> loop-cont, loop-exit -> finally + @test JuliaLowering.include_string(test_mod, """ + let out = [], x = 1 + try + push!(out, ("try", x)) + for outer x in 1:100 + push!(out, ("loop", x)) + x > 2 && break + continue + push!(out, ("bad", x)) + end + $maybe_catch + finally + push!(out, ("finally", x)) + end + out + end + """) == [("try", 1), ("loop", 1), ("loop", 2), ("loop", 3), ("finally", 3)] + @test isempty(current_exceptions()) + + # break/continue in loop in finally -> loop-cont, loop-exit + @test JuliaLowering.include_string(test_mod, """ + let out = [], x = 1 + try + push!(out, ("try", x)) + $maybe_catch + finally + for outer x in 1:100 + push!(out, ("loop", x)) + x > 2 && break + continue + push!(out, ("bad", x)) + end + push!(out, ("finally", x)) + end + out + end + """) == [("try", 1), ("loop", 1), ("loop", 2), ("loop", 3), ("finally", 3)] + @test isempty(current_exceptions()) + + # continue in finally -> loop-cont + @test JuliaLowering.include_string(test_mod, """ + let out = [] + for x in [1,2,3] + try + push!(out, ("try", x)) + $maybe_catch + finally + continue + push!(out, ("finally", x)) + end + push!(out, ("bad", x)) + end + out + end + """) == [("try", 1), ("try", 2), ("try", 3)] + @test isempty(current_exceptions()) + + # break in finally -> loop-exit + @test JuliaLowering.include_string(test_mod, """ + let out = [] + for x in [1,2,3] + try + push!(out, ("try", x)) + $maybe_catch + finally + break + push!(out, ("finally", x)) + end + push!(out, ("bad", x)) + end + out + end + """) == [("try", 1)] + @test isempty(current_exceptions()) + + # TODO: commented out to avoid polluting the exception stack; need to port + # https://github.com/JuliaLang/julia/pull/55876 + + # # error in try -> continue in finally -> loop-cont + # @test JuliaLowering.include_string(test_mod, """ + # let out = [] + # for x in [1,2,3] + # try + # push!(out, ("try", x)) + # error() + # $maybe_catch + # finally + # continue + # push!(out, ("finally", x)) + # end + # push!(out, ("bad", x)) + # end + # out + # end + # """) == [("try", 1), ("try", 2), ("try", 3)] + # @test isempty(current_exceptions()) + # + # # error in try -> break in finally -> loop-exit + # @test JuliaLowering.include_string(test_mod, """ + # let out = [] + # for x in [1,2,3] + # try + # push!(out, ("try", x)) + # error() + # $maybe_catch + # finally + # break + # push!(out, ("finally", x)) + # end + # push!(out, ("bad", x)) + # end + # out + # end + # """) == [("try", 1)] + # @test isempty(current_exceptions()) + + # (nested) continue in try -> finally block -> loop-cont + @test JuliaLowering.include_string(test_mod, """ + let out = [] + for x in [1,2] + try + push!(out, ("try", x)) + continue + $maybe_catch + finally + push!(out, ("finally", x)) + try + push!(out, ("try2", x)) + continue + finally + push!(out, ("finally2", x)) + end + push!(out, ("bad_finally", x)) + end + push!(out, ("bad", x)) + end + out + end + """) == [("try", 1), ("finally", 1), ("try2", 1), ("finally2", 1), + ("try", 2), ("finally", 2), ("try2", 2), ("finally2", 2)] + @test isempty(current_exceptions()) + + # (nested) break in try -> finally block -> loop-exit + @test JuliaLowering.include_string(test_mod, """ + let out = [] + for x in [1,2] + try + push!(out, ("try", x)) + break + $maybe_catch + finally + push!(out, ("finally", x)) + try + push!(out, ("try2", x)) + break + finally + push!(out, ("finally2", x)) + end + push!(out, ("bad_finally", x)) + end + push!(out, ("bad", x)) + end + out + end + """) == [("try", 1), ("finally", 1), ("try2", 1), ("finally2", 1)] + @test isempty(current_exceptions()) +end + + @testset "valid uses of @goto" for run in (JuliaLowering.include_string, + Base.include_string) + # no finally block + @test run(test_mod, """let v = [] + try + @goto lab + push!(v, 1) + catch e + else + end + @label lab + v + end + """) == [] + @test run(test_mod, """let v = [] + try + error() + catch e + @goto lab + push!(v, 1) + else + end + @label lab + v, length(current_exceptions()) + end + """) == ([], 0) + @test run(test_mod, """let v = [] + try + catch e + else + @goto lab + push!(v, 1) + end + @label lab + v + end + """) == [] + + # with finally, but clearly not skipping it + @test run(test_mod, """let v = [] + try + @goto lab + push!(v, 1) + @label lab + catch e + else + finally + end + v + end + """) == [] + @test run(test_mod, """let v = [] + try + error() + catch e + @goto lab + push!(v, 1) + @label lab + else + finally + end + v + end + """) == [] + @test run(test_mod, """let v = [] + try + catch e + else + @goto lab + push!(v, 1) + @label lab + finally + end + v + end + """) == [] + # broken in flisp, will be broken in JL: finally blocks are cloned + # @test run(test_mod, """let v = [] + # try + # catch e + # else + # finally + # @goto lab + # push!(v, 1) + # @label lab + # end + # v + # end + # """) == [] + end + +end diff --git a/JuliaLowering/test/exceptions_ir.jl b/JuliaLowering/test/exceptions_ir.jl new file mode 100644 index 0000000000000..fecbfc3471fa8 --- /dev/null +++ b/JuliaLowering/test/exceptions_ir.jl @@ -0,0 +1,424 @@ +######################################## +# Return from inside try/catch +try + f + return x +catch + g + return y +end +#--------------------- +1 (enter label₆) +2 TestMod.f +3 TestMod.x +4 (leave %₁) +5 (return %₃) +6 TestMod.g +7 TestMod.y +8 (pop_exception %₁) +9 (return %₇) + +######################################## +# Return from inside try/catch with simple return vals +try + f + return 10 +catch + g + return 20 +end +#--------------------- +1 (enter label₅) +2 TestMod.f +3 (leave %₁) +4 (return 10) +5 TestMod.g +6 (pop_exception %₁) +7 (return 20) + +######################################## +# Return from multiple try + try/catch +try + try + return 10 + catch + return 20 + end +catch +end +#--------------------- +1 (enter label₁₄) +2 (enter label₇) +3 (leave %₁ %₂) +4 (return 10) +5 (leave %₂) +6 (goto label₁₁) +7 (leave %₁) +8 (pop_exception %₂) +9 (return 20) +10 (pop_exception %₂) +11 slot₁/try_result +12 (leave %₁) +13 (return %₁₁) +14 (pop_exception %₁) +15 (return core.nothing) + +######################################## +# Return from multiple catch + try/catch +try +catch + try + return 10 + catch + return 20 + end +end +#--------------------- +1 (enter label₄) +2 (leave %₁) +3 (return core.nothing) +4 (enter label₈) +5 (leave %₄) +6 (pop_exception %₁) +7 (return 10) +8 (pop_exception %₁) +9 (return 20) + +######################################## +# try/catch/else, tail position +try + a +catch + b +else + c +end +#--------------------- +1 (enter label₆) +2 TestMod.a +3 (leave %₁) +4 TestMod.c +5 (return %₄) +6 TestMod.b +7 (pop_exception %₁) +8 (return %₆) + +######################################## +# try/catch/else, value position +let + z = try + a + catch + b + else + c + end +end +#--------------------- +1 (newvar slot₁/z) +2 (enter label₈) +3 TestMod.a +4 (leave %₂) +5 TestMod.c +6 (= slot₂/try_result %₅) +7 (goto label₁₁) +8 TestMod.b +9 (= slot₂/try_result %₈) +10 (pop_exception %₂) +11 slot₂/try_result +12 (= slot₁/z %₁₁) +13 (return %₁₁) + +######################################## +# try/catch/else, not value/tail +begin + try + a + catch + b + else + c + end + z +end +#--------------------- +1 (enter label₆) +2 TestMod.a +3 (leave %₁) +4 TestMod.c +5 (goto label₈) +6 TestMod.b +7 (pop_exception %₁) +8 TestMod.z +9 (return %₈) + +######################################## +# basic try/finally, tail position +try + a +finally + b +end +#--------------------- +1 (enter label₇) +2 (= slot₁/finally_tag -1) +3 (= slot₂/returnval_via_finally TestMod.a) +4 (= slot₁/finally_tag 1) +5 (leave %₁) +6 (goto label₈) +7 (= slot₁/finally_tag 2) +8 TestMod.b +9 (call core.=== slot₁/finally_tag 2) +10 (gotoifnot %₉ label₁₂) +11 (call top.rethrow) +12 slot₂/returnval_via_finally +13 (return %₁₂) + +######################################## +# basic try/finally, value position +let + z = try + a + finally + b + end +end +#--------------------- +1 (newvar slot₁/z) +2 (enter label₈) +3 (= slot₃/finally_tag -1) +4 TestMod.a +5 (= slot₂/try_result %₄) +6 (leave %₂) +7 (goto label₉) +8 (= slot₃/finally_tag 1) +9 TestMod.b +10 (call core.=== slot₃/finally_tag 1) +11 (gotoifnot %₁₀ label₁₃) +12 (call top.rethrow) +13 slot₂/try_result +14 (= slot₁/z %₁₃) +15 (return %₁₃) + +######################################## +# basic try/finally, not value/tail +begin + try + a + finally + b + end + z +end +#--------------------- +1 (enter label₆) +2 (= slot₁/finally_tag -1) +3 TestMod.a +4 (leave %₁) +5 (goto label₇) +6 (= slot₁/finally_tag 1) +7 TestMod.b +8 (call core.=== slot₁/finally_tag 1) +9 (gotoifnot %₈ label₁₁) +10 (call top.rethrow) +11 TestMod.z +12 (return %₁₁) + +######################################## +# try/finally + break +while true + try + a + break + finally + b + end +end +#--------------------- +1 (gotoifnot true label₁₉) +2 (enter label₁₀) +3 (= slot₂/finally_tag -1) +4 TestMod.a +5 (= slot₂/finally_tag 1) +6 (leave %₂) +7 (goto label₁₁) +8 (leave %₂) +9 (goto label₁₁) +10 (= slot₂/finally_tag 2) +11 TestMod.b +12 (call core.=== slot₂/finally_tag 2) +13 (gotoifnot %₁₂ label₁₅) +14 (call top.rethrow) +15 (call core.=== slot₂/finally_tag 1) +16 (gotoifnot %₁₅ label₁₈) +17 (goto label₂₀) +18 (goto label₁) +19 (= slot₁/loop-exit_result core.nothing) +20 (isdefined slot₁/loop-exit_result) +21 (gotoifnot %₂₀ label₂₃) +22 (goto label₂₄) +23 (= slot₁/loop-exit_result core.nothing) +24 slot₁/loop-exit_result +25 (return %₂₄) + +######################################## +# try/catch/finally +try + a +catch + b +finally + c +end +#--------------------- +1 (enter label₁₅) +2 (= slot₁/finally_tag -1) +3 (enter label₈) +4 TestMod.a +5 (= slot₂/try_result %₄) +6 (leave %₃) +7 (goto label₁₁) +8 TestMod.b +9 (= slot₂/try_result %₈) +10 (pop_exception %₃) +11 (= slot₃/returnval_via_finally slot₂/try_result) +12 (= slot₁/finally_tag 1) +13 (leave %₁) +14 (goto label₁₆) +15 (= slot₁/finally_tag 2) +16 TestMod.c +17 (call core.=== slot₁/finally_tag 2) +18 (gotoifnot %₁₇ label₂₀) +19 (call top.rethrow) +20 slot₃/returnval_via_finally +21 (return %₂₀) + +######################################## +# Nested finally blocks +try + try + if x + return a + end + b + finally + c + end +finally + d +end +#--------------------- +1 (enter label₃₀) +2 (= slot₁/finally_tag -1) +3 (enter label₁₅) +4 (= slot₃/finally_tag -1) +5 TestMod.x +6 (gotoifnot %₅ label₁₁) +7 (= slot₄/returnval_via_finally TestMod.a) +8 (= slot₃/finally_tag 1) +9 (leave %₃) +10 (goto label₁₆) +11 TestMod.b +12 (= slot₂/try_result %₁₁) +13 (leave %₃) +14 (goto label₁₆) +15 (= slot₃/finally_tag 2) +16 TestMod.c +17 (call core.=== slot₃/finally_tag 2) +18 (gotoifnot %₁₇ label₂₀) +19 (call top.rethrow) +20 (call core.=== slot₃/finally_tag 1) +21 (gotoifnot %₂₀ label₂₆) +22 (= slot₅/returnval_via_finally slot₄/returnval_via_finally) +23 (= slot₁/finally_tag 1) +24 (leave %₁) +25 (goto label₃₁) +26 (= slot₆/returnval_via_finally slot₂/try_result) +27 (= slot₁/finally_tag 2) +28 (leave %₁) +29 (goto label₃₁) +30 (= slot₁/finally_tag 3) +31 TestMod.d +32 (call core.=== slot₁/finally_tag 3) +33 (gotoifnot %₃₂ label₃₅) +34 (call top.rethrow) +35 (call core.=== slot₁/finally_tag 2) +36 (gotoifnot %₃₅ label₃₉) +37 slot₆/returnval_via_finally +38 (return %₃₇) +39 slot₅/returnval_via_finally +40 (return %₃₉) + +######################################## +# Access to the exception object +try + a +catch exc + b +end +#--------------------- +1 (enter label₅) +2 TestMod.a +3 (leave %₁) +4 (return %₂) +5 (= slot₁/exc (call JuliaLowering.current_exception)) +6 TestMod.b +7 (pop_exception %₁) +8 (return %₆) + +######################################## +# Error: unmatched goto from try +begin + try + @goto lab + finally + end + @label lab +end +#--------------------- +LoweringError: +begin + try + @goto lab +# └─┘ ── `goto` out of a `try` block is not permitted with `finally` + finally + end + +######################################## +# Error: unmatched goto from catch +begin + try + catch + @goto lab + finally + end + @label lab +end +#--------------------- +LoweringError: + try + catch + @goto lab +# └─┘ ── `goto` out of a `catch` block is not permitted with `finally` + finally + end + +######################################## +# Error: unmatched goto from else +begin + try + catch + else + @goto lab + finally + end + @label lab +end +#--------------------- +LoweringError: + catch + else + @goto lab +# └─┘ ── `goto` out of an `else` block is not permitted with `finally` + finally + end diff --git a/JuliaLowering/test/function_calls_ir.jl b/JuliaLowering/test/function_calls_ir.jl new file mode 100644 index 0000000000000..f2c7079d005c9 --- /dev/null +++ b/JuliaLowering/test/function_calls_ir.jl @@ -0,0 +1,643 @@ +######################################## +# Simple call +f(x, y) +#--------------------- +1 TestMod.f +2 TestMod.x +3 TestMod.y +4 (call %₁ %₂ %₃) +5 (return %₄) + +######################################## +# Keyword calls +f(x; a=1, b=2) +#--------------------- +1 TestMod.f +2 (call core.tuple :a :b) +3 (call core.apply_type core.NamedTuple %₂) +4 (call core.tuple 1 2) +5 (call %₃ %₄) +6 TestMod.x +7 (call core.kwcall %₅ %₁ %₆) +8 (return %₇) + +######################################## +# Keyword call with only splats for kws +f(; ks1..., ks2...) +#--------------------- +1 TestMod.f +2 (call core.NamedTuple) +3 TestMod.ks1 +4 (call top.merge %₂ %₃) +5 TestMod.ks2 +6 (call top.merge %₄ %₅) +7 (call top.isempty %₆) +8 (gotoifnot %₇ label₁₁) +9 (call %₁) +10 (return %₉) +11 (call core.kwcall %₆ %₁) +12 (return %₁₁) + +######################################## +# Error: Call with repeated keywords +f(x; a=1, a=2) +#--------------------- +LoweringError: +f(x; a=1, a=2) +# ╙ ── Repeated keyword argument name + +######################################## +# literal_pow lowering +x^42 +#--------------------- +1 TestMod.^ +2 TestMod.x +3 (call core.apply_type top.Val 42) +4 (call %₃) +5 (call top.literal_pow %₁ %₂ %₄) +6 (return %₅) + +######################################## +# almost but not quite literal_pow lowering :) +x^42.0 +#--------------------- +1 TestMod.^ +2 TestMod.x +3 (call %₁ %₂ 42.0) +4 (return %₃) + +######################################## +# Error: Call with no function name +@ast_ [K"call"] +#--------------------- +LoweringError: +#= line 1 =# - malformed `call` +Expression: + (call) + +######################################## +# Simple broadcast +x .* y .+ f.(z) +#--------------------- +1 TestMod.+ +2 TestMod.* +3 TestMod.x +4 TestMod.y +5 (call top.broadcasted %₂ %₃ %₄) +6 TestMod.f +7 TestMod.z +8 (call top.broadcasted %₆ %₇) +9 (call top.broadcasted %₁ %₅ %₈) +10 (call top.materialize %₉) +11 (return %₁₀) + +######################################## +# Broadcast with unary function calls +.+x +#--------------------- +1 TestMod.+ +2 TestMod.x +3 (call top.broadcasted %₁ %₂) +4 (call top.materialize %₃) +5 (return %₄) + +######################################## +# Broadcast with short circuit operators +x .&& y .|| z +#--------------------- +1 TestMod.x +2 TestMod.y +3 (call top.broadcasted top.andand %₁ %₂) +4 TestMod.z +5 (call top.broadcasted top.oror %₃ %₄) +6 (call top.materialize %₅) +7 (return %₆) + +######################################## +# Scalar comparison chain +x < y < z +#--------------------- +1 TestMod.< +2 TestMod.x +3 TestMod.y +4 (call %₁ %₂ %₃) +5 (gotoifnot %₄ label₁₁) +6 TestMod.< +7 TestMod.y +8 TestMod.z +9 (call %₆ %₇ %₈) +10 (return %₉) +11 (return false) + +######################################## +# Broadcasted comparison chain +x .< y .< z +#--------------------- +1 TestMod.< +2 TestMod.x +3 TestMod.y +4 (call top.broadcasted %₁ %₂ %₃) +5 TestMod.< +6 TestMod.y +7 TestMod.z +8 (call top.broadcasted %₅ %₆ %₇) +9 (call top.broadcasted top.& %₄ %₈) +10 (call top.materialize %₉) +11 (return %₁₀) + +######################################## +# Mixed scalar / broadcasted comparison chain +a < b < c .< d .< e +#--------------------- +1 TestMod.< +2 TestMod.a +3 TestMod.b +4 (call %₁ %₂ %₃) +5 (gotoifnot %₄ label₁₁) +6 TestMod.< +7 TestMod.b +8 TestMod.c +9 (= slot₁/if_val (call %₆ %₇ %₈)) +10 (goto label₁₂) +11 (= slot₁/if_val false) +12 slot₁/if_val +13 TestMod.< +14 TestMod.c +15 TestMod.d +16 (call top.broadcasted %₁₃ %₁₄ %₁₅) +17 (call top.broadcasted top.& %₁₂ %₁₆) +18 TestMod.< +19 TestMod.d +20 TestMod.e +21 (call top.broadcasted %₁₈ %₁₉ %₂₀) +22 (call top.broadcasted top.& %₁₇ %₂₁) +23 (call top.materialize %₂₂) +24 (return %₂₃) + +######################################## +# Mixed scalar / broadcasted comparison chain +a .< b .< c < d < e +#--------------------- +1 TestMod.< +2 TestMod.a +3 TestMod.b +4 (call top.broadcasted %₁ %₂ %₃) +5 TestMod.< +6 TestMod.b +7 TestMod.c +8 (call top.broadcasted %₅ %₆ %₇) +9 (call top.broadcasted top.& %₄ %₈) +10 TestMod.< +11 TestMod.c +12 TestMod.d +13 (call %₁₀ %₁₁ %₁₂) +14 (gotoifnot %₁₃ label₂₀) +15 TestMod.< +16 TestMod.d +17 TestMod.e +18 (= slot₁/if_val (call %₁₅ %₁₆ %₁₇)) +19 (goto label₂₁) +20 (= slot₁/if_val false) +21 slot₁/if_val +22 (call top.broadcasted top.& %₉ %₂₁) +23 (call top.materialize %₂₂) +24 (return %₂₃) + +######################################## +# Comparison chain fused with other broadcasting +x .+ (a .< b .< c) +#--------------------- +1 TestMod.+ +2 TestMod.x +3 TestMod.< +4 TestMod.a +5 TestMod.b +6 (call top.broadcasted %₃ %₄ %₅) +7 TestMod.< +8 TestMod.b +9 TestMod.c +10 (call top.broadcasted %₇ %₈ %₉) +11 (call top.broadcasted top.& %₆ %₁₀) +12 (call top.broadcasted %₁ %₂ %₁₁) +13 (call top.materialize %₁₂) +14 (return %₁₃) + +######################################## +# Broadcast with literal_pow +x.^3 +#--------------------- +1 TestMod.^ +2 TestMod.x +3 (call core.apply_type top.Val 3) +4 (call %₃) +5 (call top.broadcasted top.literal_pow %₁ %₂ %₄) +6 (call top.materialize %₅) +7 (return %₆) + +######################################## +# Broadcast with keywords +f.(x, y, z = 1; w = 2) +#--------------------- +1 top.broadcasted_kwsyntax +2 (call core.tuple :z :w) +3 (call core.apply_type core.NamedTuple %₂) +4 (call core.tuple 1 2) +5 (call %₃ %₄) +6 TestMod.f +7 TestMod.x +8 TestMod.y +9 (call core.kwcall %₅ %₁ %₆ %₇ %₈) +10 (call top.materialize %₉) +11 (return %₁₀) + +######################################## +# Broadcast with unary dot syntax +(.+)(x,y) +#--------------------- +1 TestMod.+ +2 TestMod.x +3 TestMod.y +4 (call top.broadcasted %₁ %₂ %₃) +5 (call top.materialize %₄) +6 (return %₅) + +######################################## +# Trivial in-place broadcast update +x .= y +#--------------------- +1 TestMod.x +2 TestMod.y +3 (call top.broadcasted top.identity %₂) +4 (call top.materialize! %₁ %₃) +5 (return %₁) + +######################################## +# Fused in-place broadcast update +x .= y .+ z +#--------------------- +1 TestMod.x +2 TestMod.+ +3 TestMod.y +4 TestMod.z +5 (call top.broadcasted %₂ %₃ %₄) +6 (call top.materialize! %₁ %₅) +7 (return %₁) + +######################################## +# In-place broadcast update with property assignment on left hand side +x.prop .= y +#--------------------- +1 TestMod.x +2 (call top.dotgetproperty %₁ :prop) +3 TestMod.y +4 (call top.broadcasted top.identity %₃) +5 (call top.materialize! %₂ %₄) +6 (return %₂) + +######################################## +# In-place broadcast update with ref on left hand side +x[i,end] .= y +#--------------------- +1 TestMod.x +2 TestMod.i +3 (call top.lastindex %₁ 2) +4 (call top.dotview %₁ %₂ %₃) +5 TestMod.y +6 (call top.broadcasted top.identity %₅) +7 (call top.materialize! %₄ %₆) +8 (return %₄) + +######################################## +# <: as a function call +x <: y +#--------------------- +1 TestMod.<: +2 TestMod.x +3 TestMod.y +4 (call %₁ %₂ %₃) +5 (return %₄) + +######################################## +# >: as a function call +x >: y +#--------------------- +1 TestMod.>: +2 TestMod.x +3 TestMod.y +4 (call %₁ %₂ %₃) +5 (return %₄) + +######################################## +# --> as a function call +x --> y +#--------------------- +1 TestMod.--> +2 TestMod.x +3 TestMod.y +4 (call %₁ %₂ %₃) +5 (return %₄) + +######################################## +# basic ccall +ccall(:strlen, Csize_t, (Cstring,), "asdfg") +#--------------------- +1 TestMod.Cstring +2 (call top.cconvert %₁ "asdfg") +3 (call top.unsafe_convert %₁ %₂) +4 (foreigncall :strlen (static_eval TestMod.Csize_t) (static_eval (call core.svec TestMod.Cstring)) 0 :ccall %₃ %₂) +5 (return %₄) + +######################################## +# ccall with library name as a global var +ccall((:strlen, libc), Csize_t, (Cstring,), "asdfg") +#--------------------- +1 TestMod.Cstring +2 (call top.cconvert %₁ "asdfg") +3 (call top.unsafe_convert %₁ %₂) +4 (foreigncall (foreignsymbol (tuple (inert strlen) TestMod.libc)) (static_eval TestMod.Csize_t) (static_eval (call core.svec TestMod.Cstring)) 0 :ccall %₃ %₂) +5 (return %₄) + +######################################## +# ccall with a calling convention +ccall(:foo, stdcall, Csize_t, ()) +#--------------------- +1 (foreigncall :foo (static_eval TestMod.Csize_t) (static_eval (call core.svec)) 0 :stdcall) +2 (return %₁) + +######################################## +# ccall with Any args become core.Any and don't need conversion or GC roots +ccall(:foo, stdcall, Csize_t, (Any,), x) +#--------------------- +1 core.Any +2 TestMod.x +3 (foreigncall :foo (static_eval TestMod.Csize_t) (static_eval (call core.svec core.Any)) 0 :stdcall %₂) +4 (return %₃) + +######################################## +# ccall with variable as function name (must eval to a pointer) +ccall(ptr, Csize_t, (Cstring,), "asdfg") +#--------------------- +1 TestMod.Cstring +2 (call top.cconvert %₁ "asdfg") +3 TestMod.ptr +4 (call top.unsafe_convert %₁ %₂) +5 (foreigncall %₃ (static_eval TestMod.Csize_t) (static_eval (call core.svec TestMod.Cstring)) 0 :ccall %₄ %₂) +6 (return %₅) + +######################################## +# ccall with varargs +ccall(:printf, Cint, (Cstring, Cstring...), "%s = %s\n", "2 + 2", "5") +#--------------------- +1 TestMod.Cstring +2 TestMod.Cstring +3 TestMod.Cstring +4 (call top.cconvert %₁ "%s = %s\n") +5 (call top.cconvert %₂ "2 + 2") +6 (call top.cconvert %₃ "5") +7 (call top.unsafe_convert %₁ %₄) +8 (call top.unsafe_convert %₂ %₅) +9 (call top.unsafe_convert %₃ %₆) +10 (foreigncall :printf (static_eval TestMod.Cint) (static_eval (call core.svec TestMod.Cstring TestMod.Cstring TestMod.Cstring)) 1 :ccall %₇ %₈ %₉ %₄ %₅ %₆) +11 (return %₁₀) + +######################################## +# Error: ccall with too few arguments +ccall(:foo, Csize_t) +#--------------------- +LoweringError: +ccall(:foo, Csize_t) +└──────────────────┘ ── too few arguments to ccall + +######################################## +# Error: ccall with calling conv and too few arguments +ccall(:foo, thiscall, Csize_t) +#--------------------- +LoweringError: +ccall(:foo, thiscall, Csize_t) +└────────────────────────────┘ ── too few arguments to ccall with calling convention specified + +######################################## +# Error: ccall without tuple for argument types +ccall(:foo, Csize_t, Cstring) +#--------------------- +LoweringError: +ccall(:foo, Csize_t, Cstring) +# └─────┘ ── ccall argument types must be a tuple; try `(T,)` + +######################################## +# Error: ccall without tuple for argument types +ccall(:foo, (Csize_t,), "arg") +#--------------------- +LoweringError: +ccall(:foo, (Csize_t,), "arg") +# └────────┘ ── ccall argument types must be a tuple; try `(T,)` and check if you specified a correct return type + +######################################## +# Error: ccall with library name which is a local variable +let libc = "libc" + ccall((:strlen, libc), Csize_t, (Cstring,), "asdfg") +end +#--------------------- +LoweringError: +let libc = "libc" + ccall((:strlen, libc), Csize_t, (Cstring,), "asdfg") +# └──┘ ── function name and library expression cannot reference local variable +end + +######################################## +# Error: ccall with return type which is a local variable +let Csize_t = 1 + ccall(:strlen, Csize_t, (Cstring,), "asdfg") +end +#--------------------- +LoweringError: +let Csize_t = 1 + ccall(:strlen, Csize_t, (Cstring,), "asdfg") +# └─────┘ ── ccall return type cannot reference local variable +end + +######################################## +# Error: ccall with argument type which is a local variable +let Cstring = 1 + ccall(:strlen, Csize_t, (Cstring,), "asdfg") +end +#--------------------- +LoweringError: +let Cstring = 1 + ccall(:strlen, Csize_t, (Cstring,), "asdfg") +# └─────┘ ── ccall argument type cannot reference local variable +end + +######################################## +# Error: ccall with too few arguments +ccall(:strlen, Csize_t, (Cstring,)) +#--------------------- +LoweringError: +ccall(:strlen, Csize_t, (Cstring,)) +└─────────────────────────────────┘ ── Too few arguments in ccall compared to argument types + +######################################## +# Error: ccall with too many arguments +ccall(:strlen, Csize_t, (Cstring,), "asdfg", "blah") +#--------------------- +LoweringError: +ccall(:strlen, Csize_t, (Cstring,), "asdfg", "blah") +└──────────────────────────────────────────────────┘ ── More arguments than types in ccall + +######################################## +# Error: ccall varargs with too few args +ccall(:foo, Csize_t, (Cstring...,), "asdfg") +#--------------------- +LoweringError: +ccall(:foo, Csize_t, (Cstring...,), "asdfg") +# └────────┘ ── C ABI prohibits vararg without one required argument + +######################################## +# Error: ccall with multiple varargs +ccall(:foo, Csize_t, (Cstring..., Cstring...), "asdfg", "blah") +#--------------------- +LoweringError: +ccall(:foo, Csize_t, (Cstring..., Cstring...), "asdfg", "blah") +# └────────┘ ── only the trailing ccall argument type should have `...` + +######################################## +# cglobal special support for (sym, lib) tuple +# unlike flisp we outline the tuple and allow constant propagation to put it +# back before codegen generates code for `cglobal` +cglobal((:sym, lib), Int) +#--------------------- +1 TestMod.Int +2 (call core.apply_type top.Ptr %₁) +3 (foreignglobal (foreignsymbol (tuple (inert sym) TestMod.lib))) +4 (call top.bitcast %₂ %₃) +5 (return %₄) + +######################################## +# cglobal - non-tuple expressions in first arg are lowered as normal +cglobal(f(), Int) +#--------------------- +1 TestMod.Int +2 (call core.apply_type top.Ptr %₁) +3 TestMod.f +4 (call %₃) +5 (foreignglobal %₄) +6 (call top.bitcast %₂ %₅) +7 (return %₆) + +######################################## +# Error: cglobal too many arguments +cglobal(:sym, Int, blah) +#--------------------- +LoweringError: +cglobal(:sym, Int, blah) +└──────────────────────┘ ── cglobal must have one or two arguments + +######################################## +# Error: assigning to `cglobal` +cglobal = 10 +#--------------------- +LoweringError: +cglobal = 10 +└─────┘ ── invalid syntax in left-hand side of assignment + +######################################## +# Error: assigning to `ccall` +ccall = 10 +#--------------------- +LoweringError: +ccall = 10 +└───┘ ── invalid syntax in left-hand side of assignment + +######################################## +# Error: assigning to `var"ccall"` +var"ccall" = 10 +#--------------------- +LoweringError: +var"ccall" = 10 +# └───┘ ── invalid syntax in left-hand side of assignment + +######################################## +# Error: Invalid function name ccall +function ccall() +end +#--------------------- +LoweringError: +function ccall() +# └───┘ ── ccall is a reserved identifier +end + +######################################## +# Error: Invalid function name ccall +function A.ccall() +end +#--------------------- +LoweringError: +function A.ccall() +# └───┘ ── ccall is a reserved identifier +end + +######################################## +# Error: Invalid function name ccall +function ccall{<:T}() +end +#--------------------- +LoweringError: +function ccall{<:T}() +# └───┘ ── ccall is a reserved identifier +end + +######################################## +# Nested splat: simple case +tuple((xs...)...) +#--------------------- +1 TestMod.tuple +2 (call core.tuple top.iterate %₁) +3 TestMod.xs +4 (call core._apply_iterate top.iterate core._apply_iterate %₂ %₃) +5 (return %₄) + +######################################## +# Nested splat: with mixed arguments +tuple(a, (xs...)..., b) +#--------------------- +1 TestMod.tuple +2 TestMod.a +3 (call core.tuple %₂) +4 (call core.tuple top.iterate %₁ %₃) +5 TestMod.xs +6 TestMod.b +7 (call core.tuple %₆) +8 (call core.tuple %₇) +9 (call core._apply_iterate top.iterate core._apply_iterate %₄ %₅ %₈) +10 (return %₉) + +######################################## +# Nested splat: multiple nested splats +tuple((xs...)..., (ys...)...) +#--------------------- +1 TestMod.tuple +2 (call core.tuple top.iterate %₁) +3 TestMod.xs +4 TestMod.ys +5 (call core._apply_iterate top.iterate core._apply_iterate %₂ %₃ %₄) +6 (return %₅) + +######################################## +# Nested splat: triple nesting +tuple(((xs...)...)...) +#--------------------- +1 TestMod.tuple +2 (call core.tuple top.iterate %₁) +3 (call core.tuple top.iterate core._apply_iterate %₂) +4 TestMod.xs +5 (call core._apply_iterate top.iterate core._apply_iterate %₃ %₄) +6 (return %₅) + +######################################## +# Error: Standalone splat expression +(xs...) +#--------------------- +LoweringError: +(xs...) +#└───┘ ── unexpected `...` +splatting can only be done into a `call`, `tuple`, `curly`, or array-like expression diff --git a/JuliaLowering/test/functions.jl b/JuliaLowering/test/functions.jl new file mode 100644 index 0000000000000..3aa87395e5590 --- /dev/null +++ b/JuliaLowering/test/functions.jl @@ -0,0 +1,2373 @@ +test_mod = Module() + +# Function calls +# Splatting +@test JuliaLowering.include_string(test_mod, """ +let + x = 1 + y = 2 + zs = (3,4) + w = 5 + (tuple(zs...), + tuple(zs..., w), + tuple(y, zs...), + tuple(x, y, zs..., w)) +end +""") == ((3,4), + (3,4,5), + (2,3,4), + (1,2,3,4,5)) + +# Nested splatting +@test JuliaLowering.include_string(test_mod, """ +let + xs = [[1, 2], [3, 4]] + tuple((xs...)...) +end +""") == (1, 2, 3, 4) + +@test JuliaLowering.include_string(test_mod, """ +let + xs = [[1, 2]] + ys = [[3, 4]] + tuple((xs...)..., (ys...)...) +end +""") == (1, 2, 3, 4) + +# Multiple (>2) nested splat +@test JuliaLowering.include_string(test_mod, """ +let + xs = [[[1, 2]]] + tuple(((xs...)...)...) +end +""") == (1, 2) +@test JuliaLowering.include_string(test_mod, """ +let + xs = [[[1, 2]]] + ys = [[[3, 4]]] + tuple(((xs...)...)..., ((ys...)...)...) +end +""") == (1, 2, 3, 4) +@test JuliaLowering.include_string(test_mod, """ +let + xs = [[[1, 2]]] + ys = [[[3, 4]]] + tuple(((xs...)...)..., ((ys...)...)) +end +""") == (1, 2, [3, 4]) + +# Trailing comma case should still work (different semantics) +@test JuliaLowering.include_string(test_mod, """ +let + xs = [[1, 2], [3, 4]] + tuple((xs...,)...) +end +""") == ([1, 2], [3, 4]) + +# Keyword calls +Base.eval(test_mod, :( +begin + function kwtest(; kws...) + values(kws) + end + + # Note this definition generates an arguably-spurious warning when run via + # `Pkg.test()` due to the use of `--warn-override=true` in the test + # harness. + function kwtest() + "non-kw version of kwtest" + end +end +)) + +@test JuliaLowering.include_string(test_mod, """ +let + kws = (c=3,d=4) + kwtest(; kws..., a=1, d=0, e=5) +end +""") == (c=3, d=0, a=1, e=5) + +@test JuliaLowering.include_string(test_mod, """ +let + kws = (;) + kwtest(; kws..., kws...) +end +""") == "non-kw version of kwtest" + +# literal_pow +@test JuliaLowering.include_string(test_mod, """ +2^4 +""") == 16 + +#------------------------------------------------------------------------------- +# Arrow syntax +@test JuliaLowering.include_string(test_mod, """ +let + f = ((x::T, y::T) where T) -> x + y + f(1, 2) +end +""") === 3 + +@test JuliaLowering.include_string(test_mod, """ +let + f = ((x::T; y=2) where T) -> x + y + f(1) +end +""") === 3 + +# Passes desugaring, but T is detected as unused and throws an error. +# Is it clear whether this should be `f(x::T) where T` or `f(x::T where T)`? +@test JuliaLowering.include_string(test_mod, """ +let + f = ((x::T) where T) -> x + f(1) +end +""") === 1 + +#------------------------------------------------------------------------------- +# Function definitions +@test JuliaLowering.include_string(test_mod, """ +begin + function f(x) + y = x + 1 + "hi", x, y + end + + f(1) +end +""") == ("hi", 1, 2) + +@test JuliaLowering.include_string(test_mod, """ +begin + function unused_arg(x, _, y) + x + y + end + unused_arg(1,2,3) +end +""") == 4 + +@test JuliaLowering.include_string(test_mod, """ +begin + function g(x)::Int + if x == 1 + return 42.0 + end + 0xff + end + (g(1), g(2)) +end +""") === (42, 255) + +# static parameters +@test JuliaLowering.include_string(test_mod, """ +begin + function h(x, y) + "fallback" + end + function h(::Vector{T}, ::S) where {T, S <: T} + T, S + end + (h(1, 2), h(Number[0xff], 1.0), h(Int[1], 1), h(Int[1], 1.0)) +end +""") === ("fallback", (Number, Float64), (Int, Int), "fallback") + +# Static parameter may be undefined +@test JuliaLowering.include_string(test_mod, """ +begin + func_undef_static_param(x::Union{T,Nothing}) where T = @isdefined(T) + (func_undef_static_param(nothing), func_undef_static_param(42)) +end +""") === (false, true) + +Base.eval(test_mod, +:(struct X1{T} end) +) + +# `where` params used in function obj type +@test JuliaLowering.include_string(test_mod, """ +begin + function (x::X1{T})() where T + T + end + X1{Int}()() +end +""") === Int +# or anywhere +@test JuliaLowering.include_string(test_mod, """ +let f = function foo(y::X1{T})::X1{T} where T + y + end + f(X1{Int}()) +end +""") == test_mod.X1{Int}() +@test JuliaLowering.include_string(test_mod, """ +let f = function foo(y::X1{<:T})::X1{<:T} where T + y + end + f(X1{Int}()) +end +""") == test_mod.X1{Int}() + +Base.include_string(test_mod, +""" + struct X end + + # Erroneous `convert` to test type assert in function return values + Base.convert(::Type{X}, y) = y +""") + +@test_throws TypeError JuliaLowering.include_string(test_mod, """ +begin + function h()::X + return nothing + end + h() +end +""") + +@test JuliaLowering.include_string(test_mod, """ +x = 0 +function f_return_in_value_pos() + global x + x = return 42 +end + +(f_return_in_value_pos(), x) +""") === (42, 0) + +@test JuliaLowering.include_string(test_mod, """ +function f_return_in_call() + f_return_in_call(return 123) +end + +f_return_in_call() +""") === 123 + +@test JuliaLowering.include_string(test_mod, raw""" +function f_return_in_interpolation() + :(1 + $(return 123)) +end + +f_return_in_interpolation() +""") === 123 + +@test JuliaLowering.include_string(test_mod, raw""" +function f_return_in_pparam_default(x, y=(return x), z=10) + (x, y, z) +end +(f_return_in_pparam_default(1), + f_return_in_pparam_default(1,2), + f_return_in_pparam_default(1,2,3)) +""") === (1, (1,2,10), (1,2,3)) + +@test JuliaLowering.include_string(test_mod, raw""" +function f_return_in_pparam_default2(x, y=(return x), z=(return y); kw=1) + (x, y, z, kw) +end +(f_return_in_pparam_default2(1), + f_return_in_pparam_default2(1,2), + f_return_in_pparam_default2(1,2,3), + f_return_in_pparam_default2(1;kw=0), + f_return_in_pparam_default2(1,2;kw=0), + f_return_in_pparam_default2(1,2,3;kw=0)) +""") === (1, 2, (1,2,3,1), 1, 2, (1,2,3,0)) + +@testset "Optional positional arguments" begin + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_simple(x=1, y=2, z=x) + (x,y,z) + end + + (f_def_simple(), f_def_simple(10), f_def_simple(10,20), f_def_simple(10,20,30)) + end + """) == ((1,2,1), (10,2,10), (10,20,10), (10,20,30)) + + # anon forms + @test JL.include_string(test_mod, "((x=1,y=2,z=3,va...)->(x,y,z,va))()") == (1,2,3,()) + @test JL.include_string(test_mod, "((x=1,y=2,z=3,va...)->(x,y,z,va))(0)") == (0,2,3,()) + @test JL.include_string(test_mod, "((x=1,y=2,z=3,va...)->(x,y,z,va))(0,0)") == (0,0,3,()) + @test JL.include_string(test_mod, "((x=1,y=2,z=3,va...)->(x,y,z,va))(0,0,0)") == (0,0,0,()) + @test JL.include_string(test_mod, "((x=1,y=2,z=3,va...)->(x,y,z,va))(0,0,0,0)") == (0,0,0,(0,)) + @test JL.include_string(test_mod, "(function (x=1,y=2,z=3,va...); (x,y,z,va); end)()") == (1,2,3,()) + @test JL.include_string(test_mod, "(function (x=1,y=2,z=3,va...); (x,y,z,va); end)(0)") == (0,2,3,()) + @test JL.include_string(test_mod, "(function (x=1,y=2,z=3,va...); (x,y,z,va); end)(0,0)") == (0,0,3,()) + @test JL.include_string(test_mod, "(function (x=1,y=2,z=3,va...); (x,y,z,va); end)(0,0,0)") == (0,0,0,()) + @test JL.include_string(test_mod, "(function (x=1,y=2,z=3,va...); (x,y,z,va); end)(0,0,0,0)") == (0,0,0,(0,)) + + # defaults containing previous args + @test JL.include_string(test_mod, "((x=1,y=x,z=x,va...=x)->(x,y,z,va))()") == (1,1,1,(1,)) + @test JL.include_string(test_mod, "((x=1,y=x,z=x,va...=x)->(x,y,z,va))(2)") == (2,2,2,(2,)) + @test JL.include_string(test_mod, "((x=1,y=x,z=y+x,va...=z+y+x)->(x,y,z,va))()") == (1,1,2,(4,)) + @test JL.include_string(test_mod, "((x=1,y=x,z=y+x,va...=z+y+x)->(x,y,z,va))(2)") == (2,2,4,(8,)) + # defaults shadowed by later args (resolution should not pick the arg) + @test JL.include_string(test_mod, "let x = 1; ((x=x)->(x,))(); end") == (1,) + @test JL.include_string(test_mod, "let x = 1; ((x=x)->(x,))(0); end") == (0,) + @test JL.include_string(test_mod, "let x = 1; ((x...=x)->(x,))(); end") == ((1,),) + @test JL.include_string(test_mod, "let x = 1; ((x...=x)->(x,))(0); end") == ((0,),) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y=y)->(x,y))(); end") == (2,2,) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y=y)->(x,y))(0); end") == (0,2,) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y=y)->(x,y))(0,0); end") == (0,0,) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y...=y)->(x,y))(); end") == (2,(2,),) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y...=y)->(x,y))(0); end") == (0,(2,),) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y...=y)->(x,y))(0,0); end") == (0,(0,),) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y=y,z=y)->(x,y,z))(); end") == (2,2,2) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y=y,z=y)->(x,y,z))(0); end") == (0,2,2) + @test JL.include_string(test_mod, "let y = 2; ((x=y,y=y,z=y)->(x,y,z))(0,0); end") == (0,0,0) + # defaults containing previous sparams + @test JL.include_string(test_mod, "(((x::T=1,y=T) where T)->(x,y,T))()") == (1, Int, Int) + @test JL.include_string(test_mod, "(((x::T=1,y=T) where T)->(x,y,T))(true)") == (true, Bool, Bool) + @test JL.include_string(test_mod, "(((x::Type{T}=Vector{Int},y=T) where T)->(x,y,T))()") == + (Vector{Int}, Vector{Int}, Vector{Int}) + @test JL.include_string(test_mod, "(((x::Type{T}=Vector{Int},y=T) where T)->(x,y,T))(Bool)") == + (Bool, Bool, Bool) + # https://github.com/JuliaLang/JuliaLowering.jl/issues/158 + @test JL.include_string( + test_mod, "(((::Type{T}=Vector{UInt8}, sz=Base.aligned_sizeof(eltype(T))) where T)->sz)()") == + 1 + @test JL.include_string( + test_mod, "(((::Type{T}=Vector{UInt8}, sz=Base.aligned_sizeof(eltype(T))) where T)->sz)(Int32)") == + 4 + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_placeholders(::T=1, _::S=1.0) where {T,S} + (T,S) + end + + (f_def_placeholders(), f_def_placeholders(1.0), f_def_placeholders(1.0, 1)) + end + """) == ((Int,Float64), (Float64,Float64), (Float64,Int)) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_typevars(x, y::S=[1], z::U=2) where {T, S<:AbstractVector{T}, U} + (x, y, z, T, S, U) + end + + (f_def_typevars(1), f_def_typevars(1,[1.0]), f_def_typevars(1,[1.0],-1.0)) + end + """) == ((1, [1], 2, Int, Vector{Int}, Int), + (1, [1.0], 2, Float64, Vector{Float64}, Int), + (1, [1.0], -1.0, Float64, Vector{Float64}, Float64)) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_typevar_vararg_undef(x::T, y::Vararg{S}) where {T,S} + (x, y, @isdefined S) + end + + (f_def_typevar_vararg_undef(1), f_def_typevar_vararg_undef(1,2), f_def_typevar_vararg_undef(1,2,3)) + end + """) === ((1, (), false), (1, (2,), true), (1, (2, 3), true)) + + @test JuliaLowering.include_string(test_mod, """ + begin + f_def_typevar_with_lowerbound(x::T) where {T>:Int} = + (x, @isdefined(T)) + (f_def_typevar_with_lowerbound(1), f_def_typevar_with_lowerbound(1.0)) + end + """) == ((1, true), (1.0, false)) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_slurp(x=1, ys...) + (x, ys) + end + + (f_def_slurp(), f_def_slurp(2), f_def_slurp(2,3)) + end + """) == ((1, ()), + (2, ()), + (2, (3,))) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_ret_type(x=1.0)::Int + x + end + + (f_def_ret_type(), f_def_ret_type(10.0)) + end + """) === (1,10) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_slurp_splat(ys...=(1,2)...) + ys + end + + (f_def_slurp_splat(), f_def_slurp_splat(10,20)) + end + """) == ((1,2), + (10,20)) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_def_destructure(x, (y,z)::Tuple{Int,Int}, (w,)...=(4,)...) + (x,y,z,w) + end + + f_def_destructure(1, (2,3)) + end + """) == (1,2,3,4) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_optarg_complex_spbounds(p::T, o=1) where {T<:Complex{<:Real}} + T, p, o + end + f_optarg_complex_spbounds(Complex(1)), f_optarg_complex_spbounds(Complex(2), Complex(3)) + end + """) == ((Complex{Int},Complex(1),1), (Complex{Int},Complex(2),Complex(3))) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_optarg_complex_spbounds2(p::T, o::T=Complex(0)) where {T<:Complex{<:Real}} + T, p, o + end + f_optarg_complex_spbounds2(Complex(1)), f_optarg_complex_spbounds2(Complex(2)) + end + """) == ((Complex{Int},Complex(1),Complex(0)), (Complex{Int},Complex(2),Complex(0))) + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_optarg_complex_spbounds_rett(p::T, o=1)::T where {T<:Complex{<:Real}} + p + end + f_optarg_complex_spbounds_rett(Complex(1)), f_optarg_complex_spbounds_rett(Complex(2)) + end + """) == (Complex(1), Complex(2)) + + # flisp will evaluate the sparam bound multiple times + let res = JuliaLowering.include_string(test_mod, """ + let eval_spbounds_counter = 0 + global function f_optarg_eval_spbounds_counter( + p::T, o=1,_=2,_=3) where { + T<:Complex{<:(eval_spbounds_counter += 1; Real)}} + (p, eval_spbounds_counter) + end + f_optarg_eval_spbounds_counter(Complex(1)) + end + """) + @test res == (Complex(1), 1) + end +end + +@testset "slotflags" begin + # Direct and broadcast callees should be marked called for specialization. + JuliaLowering.include_string(test_mod, """ + function f_slotflags(x, y, f, z) + f() + x + y + end + + f_slotflags_broadcast(f, x) = f.(x) + f_slotflags_broadcast_kw(f, x) = f.(x; init=0) + """) + @test only(methods(test_mod.f_slotflags)).called == 0b0100 + @test only(methods(test_mod.f_slotflags_broadcast)).called == 0b0001 + @test only(methods(test_mod.f_slotflags_broadcast_kw)).called == 0b0001 +end + +@testset "nospecialize" begin + # note f(a,b,c) means a is arg 1, not f + function test_arg_unspecialized(f::Function, arg_i::Int) + for m in methods(f) + arg_i > m.nargs-1 && return nothing + @test m.nospecialize & (1 << (arg_i-1)) != 0 + end + end + function test_arg_specialized(f::Function, arg_i::Int) + for m in methods(f) + arg_i > m.nargs-1 && return nothing + @test m.nospecialize & (1 << (arg_i-1)) == 0 + end + end + + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nospecialize(u, v, @nospecialize(x), y, @nospecialize(z)) + (u, v, x, y, z) + end + + f_nospecialize(1,2,3,4,5) + end + """) == (1,2,3,4,5) + # We dig into the internal of `Method` here to check which slots have been + # flagged as nospecialize. + @test only(methods(test_mod.f_nospecialize)).nospecialize == 0b10100 + + # Branching combined with nospecialize meta in CodeInfo + @test JuliaLowering.include_string(test_mod, """ + begin + function f_branch_meta(@nospecialize(x), cond) + if cond + x + 1 + else + x + 2 + end + end + + (f_branch_meta(10, false), f_branch_meta(20, true)) + end + """) == (12, 21) + + # @nospecialize with multiple args in function body + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nospecialize_multi_body(a, b, c, d) + @nospecialize a c d + (a, b, c, d) + end + + f_nospecialize_multi_body(1, 2, 3, 4) + end + """) == (1, 2, 3, 4) + @test only(methods(test_mod.f_nospecialize_multi_body)).nospecialize == 0b1101 + + # @nospecialize with single arg in function body + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nospecialize_single_body(a, b) + :nonmeta_should_not_interfere + @nospecialize b + (a, b) + end + + f_nospecialize_single_body(1, 2) + end + """) == (1, 2) + @test only(methods(test_mod.f_nospecialize_single_body)).nospecialize == 0b10 + + # @nospecialize with zero args in function body (blanket nospecialize) + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nospecialize_zero_body(a, b, c) + :nonmeta_should_not_interfere + @nospecialize + (a, b, c) + end + + f_nospecialize_zero_body(1, 2, 3) + end + """) == (1, 2, 3) + # 0-arg @nospecialize sets all bits (-1 == typemax(Int32) for nospecialize) + @test only(methods(test_mod.f_nospecialize_zero_body)).nospecialize == -1 + + # @nospecialize with exceptions: what should this do? + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nospecialize_body_exceptions(a, @specialize(b), c) + @nospecialize + @specialize c + (a,b,c) + end + f_nospecialize_body_exceptions(1, 2, 3) + end + """) == (1, 2, 3) + @test_broken only(methods(test_mod.f_nospecialize_body_exceptions)).nospecialize == 0b100 + + # @nospecialize with default value in signature + @test JuliaLowering.include_string(test_mod, """ + begin + function f_nospecialize_default(x, @nospecialize(y=1)) + (x, y) + end + + (f_nospecialize_default(10, 20), f_nospecialize_default(30)) + end + """) == ((10, 20), (30, 1)) + # The 2-arg method has nospecialize on y (bit 2), the 1-arg forwarding method has no y + ms = collect(methods(test_mod.f_nospecialize_default)) + @test any(m -> m.nargs == 3 && m.nospecialize == 0b10, ms) + @test any(m -> m.nargs == 2 && m.nospecialize == 0b00, ms) + + # Body-level @nospecialize with default value in signature + @test JuliaLowering.include_string(test_mod, """ + begin + function f_body_nospecialize_default(x, y=1) + :nonmeta_should_not_interfere + @nospecialize + (x, y) + end + (f_body_nospecialize_default(10, 20), f_body_nospecialize_default(30)) + end + """) == ((10, 20), (30, 1)) + # The 2-arg method has nospecialize on y (bit 2), the 1-arg forwarding method has no y + ms = collect(methods(test_mod.f_body_nospecialize_default)) + @test count(m -> m.nargs == 3 && m.nospecialize == -1, ms) == 1 + @test count(m -> m.nargs == 2 && m.nospecialize == -1, ms) == 1 + + # body nospecialize into complex sig: all + @test JuliaLowering.include_string(test_mod, """ + begin + function f_body_nospecialize_nontrivial_sig(x::T, y::Vector{<:U}=[])::Any where T where U + :nonmeta_should_not_interfere + @nospecialize + (x, y) + end + (f_body_nospecialize_nontrivial_sig(10, [20]), f_body_nospecialize_nontrivial_sig(30)) + end + """) == ((10, [20]), (30, [])) + test_arg_unspecialized(test_mod.f_body_nospecialize_nontrivial_sig, 1) + test_arg_unspecialized(test_mod.f_body_nospecialize_nontrivial_sig, 2) + # should be blanket-nospecialized + ms = collect(methods(test_mod.f_body_nospecialize_nontrivial_sig)) + @test count(m -> m.nargs == 3 && m.nospecialize == -1, ms) == 1 + @test count(m -> m.nargs == 2 && m.nospecialize == -1, ms) == 1 + + # body nospecialize into complex sig: by name + @test JuliaLowering.include_string(test_mod, """ + begin + function f_body_nospecialize_nontrivial_sig2(x::T, y::Vector{<:U}=[])::Any where T where U + :nonmeta_should_not_interfere + @nospecialize x + (x, y) + end + (f_body_nospecialize_nontrivial_sig2(10, [20]), f_body_nospecialize_nontrivial_sig2(30)) + end + """) == ((10, [20]), (30, [])) + test_arg_unspecialized(test_mod.f_body_nospecialize_nontrivial_sig2, 1) + test_arg_specialized(test_mod.f_body_nospecialize_nontrivial_sig2, 2) + + # callable type: should compile, but nospecialize doesn't do anything + @test JuliaLowering.include_string(test_mod, """ + struct nospecialize_callable_type; field; end + (@nospecialize(x::nospecialize_callable_type))() = (x.field,) + nospecialize_callable_type(0)() + """) == (0,) + @test JuliaLowering.include_string(test_mod, """ + (@nospecialize(::nospecialize_callable_type))(x::Int) = (x,) + nospecialize_callable_type(0)(1) + """) == (1,) + @test JuliaLowering.include_string(test_mod, """ + (@nospecialize(_::nospecialize_callable_type))(x::Int, y::Int) = (x,y) + nospecialize_callable_type(0)(1,2) + """) == (1,2) + @test JuliaLowering.include_string(test_mod, """ + function (self::nospecialize_callable_type)(x::Int, y::Int, z::Int) + @nospecialize self + (self.field,x,y,z) + end + nospecialize_callable_type(0)(1,2,3) + """) == (0,1,2,3) + @test JuliaLowering.include_string(test_mod, """ + (@nospecialize((;field)::nospecialize_callable_type))(x::Int, y::Int, z::Int, a::Int) = (field,x,y,z,a) + nospecialize_callable_type(0)(1,2,3,4) + """) == (0,1,2,3,4) + @test_throws LoweringError JuliaLowering.include_string(test_mod, """ + (@nospecialize(x)::nospecialize_callable_type)() = 1 + """) + + # function name: should compile, but nospecialize doesn't do anything + @test_broken JuliaLowering.include_string(test_mod, """ + (@nospecialize(_))(x::Int) = () + func_nospecialize_self(1) + """) == () + @test JuliaLowering.include_string(test_mod, """ + (@nospecialize(func_nospecialize_self))(x::Int) = (x,) + func_nospecialize_self(1) + """) == (1,) + + # all positional arg forms + @testset for arg0 in [:x, :(x::Type), :(::Type), :(_), :(_::Type)], + arg1 in [arg0, Expr(:..., arg0)], + arg2 in [arg1, Expr(:kw, arg1, :Int)], + expander in [fl_macroexpand, jl_macroexpand] + + @testset let expanded = expander( + test_mod, :(function (specialized, @nospecialize($arg2)) + specialized + end)) + f = jl_eval(test_mod, expanded) + test_arg_specialized(f, 1) + test_arg_unspecialized(f, 2) + end + + @testset let expanded = expander( + test_mod, :(function ($arg2,) + @nospecialize + end)) + f = jl_eval(test_mod, expanded) + test_arg_unspecialized(f, 1) + end + end + + # nospecialize should still compile where flisp drops it + @test jl_eval( + test_mod, + :(let bad(@nospecialize(x) = 1) = x + (bad(0), bad()) + end)) == (0, 1) + @test jl_eval( + test_mod, + :(let bad(@nospecialize(x::Int) = 1) = x + (bad(0), bad()) + end)) == (0, 1) + + @testset "kwargs" for expander in [fl_macroexpand, (_,x)->x] + local f + @test (f = jl_eval(test_mod, expander( + test_mod, quote + function (@nospecialize(a); kw=1) + (a, kw) + end + end))) isa Function + Core.@latestworld + @test f(1, kw=2) == (1,2) && f(3) == (3,1) + test_arg_unspecialized(f, 1) + @test only(methods(Core.kwcall, (NamedTuple,typeof(f),Any))).nospecialize == 1 << 2 + + # Body-level @nospecialize + @test (f = jl_eval(test_mod, expander( + test_mod, quote + function (a; kw=1) + @nospecialize a + (a, kw) + end + end))) isa Function + Core.@latestworld + @test f(1, kw=2) == (1,2) && f(3) == (3,1) + test_arg_unspecialized(f, 1) + @test only(methods(Core.kwcall, (NamedTuple,typeof(f),Any))).nospecialize == 1 << 2 + + # kw nospecialize. TODO: The body method is local; how do we get it out + # for testing? + @test (f = jl_eval(test_mod, expander( + test_mod, quote + function (a; @nospecialize(kw=1)) + (a, kw) + end + end))) isa Function + Core.@latestworld + @test f(1, kw=2) == (1,2) && f(3) == (3,1) + test_arg_specialized(f, 1) + + # kw... nospecialize (same TODO) + @test (f = jl_eval(test_mod, expander( + test_mod, quote + function (a; @nospecialize(kw...)) + (a, kw...) + end + end))) isa Function + Core.@latestworld + @test f(1, kw=2, a=3) == (1,:kw=>2,:a=>3) + test_arg_specialized(f, 1) + end + + # macros already mark all non-internal args nospecialize + @testset "macro definitions" begin + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(x)); end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(x::Int)); end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(x=1)); end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(x::Int=1)); end)) isa Function + + @gensym sym + @test jl_eval(test_mod, :(macro $sym(x); @nospecialize(); end)) isa Function + + @gensym sym + @test jl_eval(test_mod, :(macro $sym(x); @nospecialize(x); x; end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(x::Int); @nospecialize(x); x; end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(x=1); @nospecialize(x); x; end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(x::Int=1); @nospecialize(x); x; end)) isa Function + + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(_)); end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(_::Int)); end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(_=1)); end)) isa Function + @gensym sym + @test jl_eval(test_mod, :(macro $sym(@nospecialize(_::Int=1)); end)) isa Function + end +end + +@testset "Keyword functions" begin + JuliaLowering.include_string(test_mod, """ + function f_kw_simple(a::Int=1, b::Float64=1.0; x::Char='a', y::Bool=true) + (a, b, x, y) + end + """) + + @test test_mod.f_kw_simple() === (1, 1.0, 'a', true) + @test test_mod.f_kw_simple(x='b') === (1, 1.0, 'b', true) + @test test_mod.f_kw_simple(y=false) === (1, 1.0, 'a', false) + @test test_mod.f_kw_simple(x='b', y=false) === (1, 1.0, 'b', false) + + @test test_mod.f_kw_simple(20) === (20, 1.0, 'a', true) + @test test_mod.f_kw_simple(20; x='b') === (20, 1.0, 'b', true) + @test test_mod.f_kw_simple(20; y=false) === (20, 1.0, 'a', false) + @test test_mod.f_kw_simple(20; x='b', y=false) === (20, 1.0, 'b', false) + + @test test_mod.f_kw_simple(20, 2.0) === (20, 2.0, 'a', true) + @test test_mod.f_kw_simple(20, 2.0; x='b') === (20, 2.0, 'b', true) + @test test_mod.f_kw_simple(20, 2.0; y=false) === (20, 2.0, 'a', false) + @test test_mod.f_kw_simple(20, 2.0; x='b', y=false) === (20, 2.0, 'b', false) + + # Bad types for keyword args throw a type error + @test_throws(TypeError(Symbol("keyword argument"), :x, Char, 100), + test_mod.f_kw_simple(x=100)) + @test_throws(TypeError(Symbol("keyword argument"), :y, Bool, 100), + test_mod.f_kw_simple(y=100)) + + # Keywords which aren't present throw an error + try + test_mod.f_kw_simple(20; not_present=100) + @test false + catch exc + @test exc isa MethodError + @test exc.f == Core.kwcall + @test exc.args == ((; not_present=100), test_mod.f_kw_simple, 20, 1.0) + end + + @testset "anonymous forms" begin + f = JL.include_string(test_mod, "function (;kw); kw; end") + @test f(;kw=1) == 1 + @test_throws UndefKeywordError f(;) + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "function (;kw::Int); kw; end") + @test f(;kw=1) == 1 + @test_throws TypeError f(;kw=1.1) + @test_throws UndefKeywordError f(;) + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "function (;kw=2); kw; end") + @test f(;kw=1) == 1 + @test f(;) == 2 + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "function (;kw::Int=2); kw; end") + @test f(;kw=1) == 1 + @test f(;) == 2 + @test_throws TypeError f(;kw=1.1) + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "function (;kw...); kw; end") + @test f(;kw=1) isa Base.Pairs + @test (f(;kw=1))[:kw] == 1 + exkw = (;k3=3) + @test f(;k1=1, k2=2, exkw...) isa Base.Pairs + @test (f(;k1=1, k2=2, exkw...))[:k1] == 1 + @test (f(;k1=1, k2=2, exkw...))[:k2] == 2 + @test (f(;k1=1, k2=2, exkw...))[:k3] == 3 + # -> + f = JL.include_string(test_mod, "(;kw)->kw") + @test f(;kw=1) == 1 + @test_throws UndefKeywordError f(;) + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "(;kw::Int)->kw") + @test f(;kw=1) == 1 + @test_throws TypeError f(;kw=1.1) + @test_throws UndefKeywordError f(;) + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "(;kw=2)->kw") + @test f(;kw=1) == 1 + @test f(;) == 2 + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "(;kw::Int=2)->kw") + @test f(;kw=1) == 1 + @test f(;) == 2 + @test_throws TypeError f(;kw=1.1) + @test_throws MethodError f(1) + f = JL.include_string(test_mod, "(;kw...)->kw") + @test f(;kw=1) isa Base.Pairs + @test (f(;kw=1))[:kw] == 1 + exkw = (;k3=3) + @test f(;k1=1, k2=2, exkw...) isa Base.Pairs + @test (f(;k1=1, k2=2, exkw...))[:k1] == 1 + @test (f(;k1=1, k2=2, exkw...))[:k2] == 2 + @test (f(;k1=1, k2=2, exkw...))[:k3] == 3 + end + + # Slurping of positional args with keywords + JuliaLowering.include_string(test_mod, """ + function f_pos_slurp_with_kws(z, args...; x=1,y=2) + args + end + """) + @test test_mod.f_pos_slurp_with_kws(3, 2, 1; x = 100) === (2,1) + @test test_mod.f_pos_slurp_with_kws(3, 2, 1) === (2,1) + + # Slurping of keyword args + JuliaLowering.include_string(test_mod, """ + function f_kw_slurp_all(; kws...) + kws + end + """) + @test values(test_mod.f_kw_slurp_all(x = 1, y = 2)) === (x=1, y=2) + @test values(test_mod.f_kw_slurp_all()) === (;) + + # Slurping of keyword args + JuliaLowering.include_string(test_mod, """ + function f_kw_slurp_some(; x=1, y=2, kws...) + kws + end + """) + @test values(test_mod.f_kw_slurp_some(z=3, x = 1, y = 2, w=4)) === (z=3, w=4) + @test values(test_mod.f_kw_slurp_some(x = 1)) === (;) + @test values(test_mod.f_kw_slurp_some()) === (;) + + # Slurping with defaults depending on keyword names + JuliaLowering.include_string(test_mod, """ + function f_kw_slurp_dep(; a=1, b=a, kws...) + (a, b, length(kws)) + end + """) + @test test_mod.f_kw_slurp_dep(; a=1) == (1, 1, 0) + @test test_mod.f_kw_slurp_dep(; a=2, c=3) == (2, 2, 1) + + # Keyword defaults which depend on other keywords. + JuliaLowering.include_string(test_mod, """ + begin + aaa = :outer + function f_kw_default_dependencies(; x=1, y=x, bbb=aaa, aaa=:aaa_kw, ccc=aaa) + (x, y, bbb, aaa, ccc) + end + end + """) + @test values(test_mod.f_kw_default_dependencies()) === (1, 1, :outer, :aaa_kw, :aaa_kw) + @test values(test_mod.f_kw_default_dependencies(x = 10)) === (10, 10, :outer, :aaa_kw, :aaa_kw) + @test values(test_mod.f_kw_default_dependencies(x = 10, aaa=:blah)) === (10, 10, :outer, :blah, :blah) + + # depend on positional args + @test JuliaLowering.include_string(test_mod, """ + function f_kw_pos_dependencies(p1, o1=1, va...; kw1=p1, kw2=o1, kw3=va) + (p1, o1, va..., kw1, kw2, kw3...) + end + """) isa Function + @test test_mod.f_kw_pos_dependencies('p', 'o', 'v', 'v') == + ('p', 'o', 'v', 'v', 'p', 'o', 'v', 'v') + @test test_mod.f_kw_pos_dependencies('p', 'o', 'v') == + ('p', 'o', 'v', 'p', 'o', 'v') + @test test_mod.f_kw_pos_dependencies('p', 'o') == + ('p', 'o', 'p', 'o') + @test test_mod.f_kw_pos_dependencies('p') == + ('p', 1, 'p', 1) + + # Keywords with static parameters + JuliaLowering.include_string(test_mod, """ + function f_kw_sparams(x::X, y::Y; a::A, b::B) where {X,Y,A,B} + (X,Y,A,B) + end + """) + @test values(test_mod.f_kw_sparams(1, 1.0; a="a", b='b')) === (Int, Float64, String, Char) + + # Keywords with static parameters, where some keyword types can be inferred + # based on the positional parameters and others cannot. + JuliaLowering.include_string(test_mod, """ + function f_kw_type_errors(x::X; a::F, b::X) where {X<:Integer,F<:AbstractFloat} + (X,F) + end + """) + @test values(test_mod.f_kw_type_errors(1; a=1.0, b=10)) === (Int, Float64) + # The following is a keyword TypeError because we can infer `X` based on + # the positional parameters and use that to check the type of `b`. + @test_throws TypeError values(test_mod.f_kw_type_errors(1; a=1.0, b="str")) + # The following is only a method error as we can't infer `F` prior to + # dispatching to the body function. + @test_throws MethodError values(test_mod.f_kw_type_errors(1; a="str", b=10)) + + # Return type annotation using default argument names + # The return type must be evaluated in a scope where keyword args are bound. + JuliaLowering.include_string(test_mod, """ + function f_default_rett(T::Type=Int)::Vector{T} + T[1,2,3] + end + """) + @test test_mod.f_default_rett() isa Vector{Int} + @test test_mod.f_default_rett(Float64) isa Vector{Float64} + + # Return type annotation using keyword argument names + # The return type must be evaluated in a scope where keyword args are bound. + JuliaLowering.include_string(test_mod, """ + function f_kw_rett(; T::Type=Int)::Vector{T} + T[1,2,3] + end + """) + @test test_mod.f_kw_rett() isa Vector{Int} + @test test_mod.f_kw_rett(T=Float64) isa Vector{Float64} + + JuliaLowering.include_string(test_mod, """ + function f_kw_rett2(; T::Type=Int)::Union{Vector{<:T}, Vector{<:AbstractVector{<:T}}} + false && return T[] + T == Int ? T[1,2,3] : [T[1,2],T[3,4]] + end + """) + @test test_mod.f_kw_rett2() isa Vector{Int} + @test test_mod.f_kw_rett2() == Int[1,2,3] + @test test_mod.f_kw_rett2(T=Float64) isa Vector{Vector{Float64}} + @test test_mod.f_kw_rett2(T=Float64) == [Float64[1.0,2.0],Float64[3.0,4.0]] + + # Throwing of UndefKeywordError + JuliaLowering.include_string(test_mod, """ + function f_kw_no_default(; x) + x + end + """) + @test test_mod.f_kw_no_default(x = 10) == 10 + @test_throws UndefKeywordError(:x) test_mod.f_kw_no_default() == 10 + + # Closure with keywords + cl = JuliaLowering.include_string(test_mod, """ + let y = 1 + function f_kw_closure(; x=10) + x + y + end + end + """) + @test cl() == 11 + @test cl(x = 20) == 21 + f = JuliaLowering.include_string(test_mod, """ + function f_kw_closure_outer(; x=1) + function f_kw_closure(; y=2) + (x, y) + end + end + """) + @test f() isa Function + @test f()() == (1, 2) + @test f()(y = 3) == (1, 3) + @test f(x = 10) isa Function + @test f(x = 10)(y = 10) == (10, 10) + f = JuliaLowering.include_string(test_mod, """ + function f_kw_closure_capt_default(; x=1) + function f_kw_closure(; y=x) + (x, y) + end + end + """) + @test f() isa Function + @test f()() == (1, 1) + @test f(x=2)(y=3) == (2, 3) + f = JuliaLowering.include_string(test_mod, """ + let outer_capt = 0 + function f_kw_closure_capt_default(; x=1) + function f_kw_closure(; y=x) + (outer_capt, x, y) + end + end + end + """) + @test f() isa Function + @test f()() == (0, 1, 1) + @test f(x=2)(y=3) == (0, 2, 3) + + f = JuliaLowering.include_string(test_mod, """ + function f_kw_anon(outervar) + (a,;kw=1)->a+kw+outervar + end + """) + + @test f(100) isa Function + @test f(100)(2) == 103 + @test f(100)(2;kw=2) == 104 + + @testset "complex arg types requiring temporaries" begin + @test JL.include_string( + test_mod, """ + let f = function (x::Vector{<:Number};kw=[2]) + (x,kw) + end + f([1]), f([1], kw=[0]) + end + """) == (([1], [2]), ([1], [0])) + @test JL.include_string( + test_mod, """ + let f = function (x::Vector{<:Number}, o1=10, o2=20;kw=[2]) + (x,kw,o1,o2) + end + f([1]), f([1], kw=[0]) + end + """) == (([1], [2], 10, 20), ([1], [0], 10, 20)) + @test JL.include_string( + test_mod, """ + let f = function (x;kw::Vector{<:Number}=x) + (x,kw) + end + f([1]), f([1], kw=[0]) + end + """) == (([1], [1]), ([1], [0])) + @test JL.include_string( + test_mod, """ + let f = function (x, o1=10, o2=20;kw::Vector{<:Number}=x) + (x,kw,o1,o2) + end + f([1]), f([1], kw=[0]) + end + """) == (([1], [1], 10, 20), ([1], [0], 10, 20)) + @test JL.include_string( + test_mod, """ + let f = function (o1::Vector{<:Number}=[10];kw=1) + (kw,o1) + end + f(), f([1]), f(;kw=2), f([1]; kw=2) + end + """) == ((1, [10]), (1, [1]), (2, [10]), (2, [1])) + end + @testset "complex sparam bounds requiring temporaries" begin + @test JL.include_string( + test_mod, """ + let f = function (x::T;kw=[2]) where {T<:Vector{<:Number}} + (T,x,kw) + end + f([1]), f([1], kw=[0]) + end + """) == ((Vector{Int}, [1], [2]), (Vector{Int}, [1], [0])) + @test JL.include_string( + test_mod, """ + let f = function (x::T, o1=10, o2=20;kw=[2]) where {T<:Vector{<:Number}} + (T,x,kw,o1,o2) + end + f([1]), f([1], kw=[0]) + end + """) == ((Vector{Int}, [1], [2], 10, 20), (Vector{Int}, [1], [0], 10, 20)) + @test JL.include_string( + test_mod, """ + let f = function (x;kw::T=x) where {T<:Vector{<:Number}} + (T,x,kw) + end + f([1]), f([1], kw=[0]) + end + """) == ((Vector{Int}, [1], [1]), (Vector{Int}, [1], [0])) + @test JL.include_string( + test_mod, """ + let f = function (x, o1=10, o2=20;kw::T=x) where {T<:Vector{<:Number}} + (T,x,kw,o1,o2) + end + f([1]), f([1], kw=[0]) + end + """) == ((Vector{Int}, [1], [1], 10, 20), (Vector{Int}, [1], [0], 10, 20)) + @test JL.include_string( + test_mod, """ + let f = function (o1::T=[10];kw=1) where {T<:Vector{<:Number}} + (T,kw,o1) + end + f(), f([1]), f(;kw=2), f([1]; kw=2) + end + """) == ((Vector{Int}, 1, [10]), (Vector{Int}, 1, [1]), + (Vector{Int}, 2, [10]), (Vector{Int}, 2, [1])) + end + + @testset "destructured args" begin + @test JL.include_string( + test_mod, "(function ((d1,d2);kw); [d1,d2,kw]; end)((1,2);kw=3)") == [1,2,3] + # with kw default + @test JL.include_string( + test_mod, "(function ((d1,d2);kw=4); [d1,d2,kw]; end)((1,2);kw=3)") == [1,2,3] + @test JL.include_string( + test_mod, "(function ((d1,d2);kw=4); [d1,d2,kw]; end)((1,2))") == [1,2,4] + # flisp doesn't do this either + @test_broken JL.include_string( + test_mod, "(function ((d1,d2);kw1=d1); [d1,d2,kw1]; end)((1,2))") == [1,2,1] + # with kw deps + @test JL.include_string( + test_mod, "(function ((d1,d2);kw1=1,kw2=kw1); [d1,d2,kw1,kw2]; end)((1,2);kw1=9)") == [1,2,9,9] + @test JL.include_string( + test_mod, "(function ((d1,d2);kw1=1,kw2=kw1); [d1,d2,kw1,kw2]; end)((1,2);kw1=9,kw2=10)") == [1,2,9,10] + end +end + +# Brittle test, needs fixing if kw_body naming or kwarg implementation changes +@testset "(AI) kw function helper is declared in the correct module" begin + # Extending another module's keyword function with a new keyword method must + # reserve the hidden `#kw_body#...` global in the *extending* module (the + # call/eval site), never in the extended function's home module -- reserving + # it in a foreign (possibly precompiled/closed) module breaks incremental + # compilation. This is most easily broken when the method name arrives as an + # interpolated `GlobalRef` *value* (the StatsBase/TracedSample shape), whose + # `:mod` attribute would otherwise steer the reservation to the owner module. + kwbodies(m) = filter(s -> occursin("kw_body", String(s)), names(m; all=true)) + + # (a) interpolated `GlobalRef` value as the method name (the regressing case) + OwnerA = Module() + JL.include_string(OwnerA, "sample(x; y=1) = x + y") + a_before = Set(kwbodies(OwnerA)) + ExtA = Module() + @eval ExtA const OwnerA = $OwnerA + JL.include_string(ExtA, """ + let fn = GlobalRef(OwnerA, :sample) + @eval \$fn(x::Symbol; y=1) = y + end + """) + @test isempty(setdiff(Set(kwbodies(OwnerA)), a_before)) # no new global in owner + @test !isempty(kwbodies(ExtA)) # reserved in extender + @test OwnerA.sample(3; y=10) == 13 # original method intact + @test OwnerA.sample(:s; y=7) == 7 # new method dispatches + @test OwnerA.sample(:s) == 1 # ...with its own default + + # (b) syntactic dotted name reaches the same conclusion (guards the common path) + OwnerB = Module() + JL.include_string(OwnerB, "sample(x; y=1) = x + y") + b_before = Set(kwbodies(OwnerB)) + ExtB = Module() + @eval ExtB const OwnerB = $OwnerB + JL.include_string(ExtB, "function OwnerB.sample(x::Symbol; y=1); y; end") + @test isempty(setdiff(Set(kwbodies(OwnerB)), b_before)) + @test !isempty(kwbodies(ExtB)) + @test OwnerB.sample(:s; y=7) == 7 +end + +@testset "pre-desugared arg::Vararg" begin + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_nosplat = function (x::Vararg{Int}) + x + end + f_vararg_nosplat(1,2,3) + end + """) == (1, 2, 3) + + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_nosplat = function ((a,b,c)::Vararg{Int}) + (a,b,c) + end + f_vararg_nosplat(1,2,3) + end + """) == (1, 2, 3) + + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_nosplat = function (((a,b)...,c)::Vararg{Int}) + (a,b,c) + end + f_vararg_nosplat(1,2,3) + end + """) == (1, 2, 3) + + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_nosplat = function (((a,b)...,c)::Vararg{Tuple{Vararg{Int}}}) + (a,b,c) + end + f_vararg_nosplat((1,2),(3,),(4,)) + end + """) == ((1, 2), (3,), (4,)) + + @testset "(AI) in keyword functions" begin + # A trailing positional `Vararg{T,N}` written with an explicit `::Vararg` + # annotation (rather than `...`) must still be splatted when the keyword + # wrappers forward it to the body method. The zero-keyword path (defaulting + # sorter), the explicit-keyword path, and splatted keywords must all work, + # and `N` may be referenced in the signature and body. + JuliaLowering.include_string(test_mod, """ + function f_vararg_N_kws(y::Integer, args::Vararg{Integer,N}; kwargs...) where {N} + (y, args, N, kwargs) + end + """) + @test test_mod.f_vararg_N_kws(1, 2, 3) === (1, (2, 3), 2, Base.pairs(NamedTuple())) + let r = test_mod.f_vararg_N_kws(1, 2, 3; foo=1) + @test (r[1], r[2], r[3]) === (1, (2, 3), 2) + @test r[4][:foo] == 1 + end + let ekw = (a=1, b=2), r = test_mod.f_vararg_N_kws(1, 2; ekw...) + @test (r[1], r[2], r[3]) === (1, (2,), 1) + @test (r[4][:a], r[4][:b]) == (1, 2) + end + + # `N` used as a keyword default (exercises the sorter/body kw forwarding too). + JuliaLowering.include_string(test_mod, """ + function f_vararg_N_kwdefault(y::Integer, args::Vararg{Integer,N}; scale::Int=N) where {N} + (y, args, scale) + end + """) + @test test_mod.f_vararg_N_kwdefault(1, 2, 3) === (1, (2, 3), 2) + @test test_mod.f_vararg_N_kwdefault(1, 2, 3; scale=10) === (1, (2, 3), 10) + + # `Vararg{T}` with no count, and a bare `Vararg`, plus `Vararg` on an + # anonymous (unnamed) positional argument. + JuliaLowering.include_string(test_mod, """ + function f_vararg_T_kws(y, args::Vararg{Integer}; kwargs...) + (y, args, kwargs) + end + """) + @test test_mod.f_vararg_T_kws(1, 2, 3) === (1, (2, 3), Base.pairs(NamedTuple())) + @test test_mod.f_vararg_T_kws(1, 2, 3; foo=1)[3][:foo] == 1 + + JuliaLowering.include_string(test_mod, """ + function f_vararg_bare_kws(y, args::Vararg; kwargs...) + (y, args, kwargs) + end + """) + @test test_mod.f_vararg_bare_kws(1, 2, 3) === (1, (2, 3), Base.pairs(NamedTuple())) + @test test_mod.f_vararg_bare_kws(1; z=9)[3][:z] == 9 + + JuliaLowering.include_string(test_mod, """ + function f_vararg_anon_kws(y, ::Vararg{Integer,N}; kwargs...) where {N} + (y, N, kwargs) + end + """) + @test test_mod.f_vararg_anon_kws(1, 2, 3) === (1, 2, Base.pairs(NamedTuple())) + @test test_mod.f_vararg_anon_kws(1, 2, 3; k=1)[3][:k] == 1 + + # Equivalent `args::T...` and plain `args...` forms with kwargs (already + # handled, covered here for parity). + JuliaLowering.include_string(test_mod, """ + function f_vararg_dots_typed_kws(y, args::Integer...; kwargs...) + (y, args, kwargs) + end + """) + @test test_mod.f_vararg_dots_typed_kws(1, 2, 3) === (1, (2, 3), Base.pairs(NamedTuple())) + @test test_mod.f_vararg_dots_typed_kws(1, 2, 3; foo=1)[3][:foo] == 1 + + JuliaLowering.include_string(test_mod, """ + function f_vararg_dots_kws(y, args...; kwargs...) + (y, args, kwargs) + end + """) + @test test_mod.f_vararg_dots_kws(1, 2, 3) === (1, (2, 3), Base.pairs(NamedTuple())) + @test test_mod.f_vararg_dots_kws(1, 2, 3; foo=1)[3][:foo] == 1 + + # Vararg-annotated positional args carrying a default value (`K"kw"`-wrapped + # in the AST), both named and anonymous. + JuliaLowering.include_string(test_mod, """ + function f_vararg_default_kws(y, args::Vararg{Int,N}=1; k=1) where {N} + (y, args, N, k) + end + """) + @test test_mod.f_vararg_default_kws(1, 2, 3) === (1, (2, 3), 2, 1) + @test test_mod.f_vararg_default_kws(1, 2, 3; k=9) === (1, (2, 3), 2, 9) + + JuliaLowering.include_string(test_mod, """ + function f_vararg_anon_default_kws(y, ::Vararg{Int,N}=1; k=1) where {N} + (y, N, k) + end + """) + @test test_mod.f_vararg_anon_default_kws(1, 2, 3) === (1, 2, 1) + @test test_mod.f_vararg_anon_default_kws(1, 2, 3; k=9) === (1, 2, 9) + + # Callable-type method with a trailing Vararg and keywords (shape from + # SerializedElementArrays.jl). + JuliaLowering.include_string(test_mod, """ + struct VKS{T,N} + dims::NTuple{N,Int} + end + function (A::Type{VKS{<:Any,N}})(::UndefInitializer, dims::Vararg{Integer,N}; kw=1) where {N} + (N, dims, kw) + end + """) + @test test_mod.VKS{<:Any,2}(undef, 3, 4) === (2, (3, 4), 1) + @test test_mod.VKS{<:Any,1}(undef, 7; kw=9) === (1, (7,), 9) + end +end + +@testset "all known valid positional argument forms" begin + make_defaults(x) = let (ps, vals) = x + # (p1,p2,p3) => (v1,v2,v3) to + # ((kw p1 v1),(kw p2 v2),(kw p3 v3)) => (v1,v2,v3) + map(zip(ps, vals)) do pv + Expr(:kw, pv[1], pv[2]) + end => vals + end + make_typed(pv) = let (ps, vals) = pv + new_ps = map(ps) do p + # types go under `...` + if Meta.isexpr(p, :...) + Expr(:..., Expr(:(::), p.args[1], Any)) + else + Expr(:(::), p, Any) + end + end + new_ps => vals + end + + pparams_req = let + # tuple of params => tuple of acceptable values + pparams_untyped = [ + # x,y,z must be defined for testing + (:x, + :y, + :z) => + (1,2,3), + (:x, + Expr(:tuple, :y, :z)) => + (1,(2,3)), + (:x, + Expr(:tuple, Expr(:parameters, :y, :z))) => + (1,(;y=2,z=3)), + (:x, + Expr(:tuple, Expr(:..., :y), :z)) => + (1,(2,3,4)), + (Expr(:tuple, Expr(:tuple, :x, :y), :z),) => + (((1,2),3),), + (Expr(:tuple, Expr(:..., Expr(:tuple, :x, :y)), :z),) => + ((1,2,3),), + (Expr(:tuple, Expr(:..., Expr(:tuple, :x, :y)), :z),) => + ((1,2,3,4,5),), + (:x, + :y, + Expr(:..., :z)) => + (1,2,3), + ] + pparams_typed = map(make_typed, pparams_untyped) + vcat(pparams_untyped, pparams_typed) + end + + @testset "required args" for (params_i, args_i) in pparams_req + @testset let f_expr = Expr(:function, + Expr(:call, gensym(), params_i...), + Expr(:tuple, :x, :y, :z)), + f_st = JuliaLowering.expr_to_est(f_expr) + + local func_ref, func_test + @test ((func_ref = fl_eval(test_mod, f_expr)) isa Function) + @test ((func_test = jl_eval(test_mod, f_st)) isa Function) + Core.@latestworld + @test func_ref(args_i...) == func_test(args_i...) + end + end + + pparams_default = map(make_defaults, pparams_req) + + @testset "default args" for (params_i, args_i) in pparams_default + @testset let f_expr = Expr(:function, + Expr(:call, gensym(), params_i...), + Expr(:tuple, :x, :y, :z)), + f_st = JuliaLowering.expr_to_est(f_expr) + + local func_ref, func_test + @test ((func_ref = fl_eval(test_mod, f_expr)) isa Function) + @test ((func_test = jl_eval(test_mod, f_st)) isa Function) + Core.@latestworld + @test func_ref(args_i...) == func_test(args_i...) + @test func_ref() == func_test() + end + end + + # test vararg-tuples and splatted defaults separately, as providing defaults + # must be done with a syntactic splat, and some variants are valid syntax + # but not callable (may later be disallowed) + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((x,y,z)...) + (x,y,z) + end + f_vararg_tuple(1,2,3), f_vararg_tuple(1,2,3,4,5) + end + """) === ((1,2,3), (1,2,3)) + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((x,y,z)...=(1,2,3)...) + (x,y,z) + end + f_vararg_tuple(4,5,6,7), f_vararg_tuple() + end + """) === ((4,5,6), (1,2,3)) + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((x,(y,z))...=(1,(2,3))...) + (x,y,z) + end + f_vararg_tuple(4,(5,6),7), f_vararg_tuple() + end + """) === ((4,5,6), (1,2,3)) + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((x,(y,z)...)...=(1,(2,3)...)...) + (x,y,z) + end + f_vararg_tuple(4,5,6,7), f_vararg_tuple() + end + """) === ((4,5,6), (1,2,3)) + + # uncallable(?) + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((x,y,z)::Tuple...) + (x,y,z) + end + end + """) isa Function + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((;x,y,z)...) + (x,y,z) + end + end + """) isa Function + @test JuliaLowering.include_string(test_mod, """ + let + f_vararg_tuple = function ((;x,y,z)::NamedTuple...) + (x,y,z) + end + end + """) isa Function + + # final default arg may always be splatted, even if no-op or followed by va + @test JuliaLowering.include_string(test_mod, """ + let + f = function (x=1...) + x + end + f(), f(2), try; f(9,9); catch e; "fail"; end + end + """) === (1, 2, "fail") + @test JuliaLowering.include_string(test_mod, """ + let + f = function (x=1..., args...) + x, args + end + f(), f(2), f(3,4,5) + end + """) === ((1, ()), + (2, ()), + (3, (4,5))) +end + +@testset "first-arg destructuring" begin + @eval test_mod struct XY; x; y; end + Core.@latestworld + xy = test_mod.XY(1,2) + @test JL.include_string(test_mod, "((;x,y)::XY)(arg) = (x,y,arg)") === nothing + Core.@latestworld + @test xy(3) == (1,2,3) + + @eval test_mod begin + struct XYVec; val::Core.SimpleVector; end + Base.iterate(x::XYVec) = Base.iterate(x.val) + Base.iterate(x::XYVec, i) = Base.iterate(x.val, i) + end + xy = test_mod.XYVec(Core.svec(1,2,999)) + @test JL.include_string(test_mod, "((x,y)::XYVec)(arg) = (x,y,arg)") === nothing + Core.@latestworld + @test xy(3) == (1,2,3) + + # `...` + xy = test_mod.XYVec(Core.svec(1,2,9,9,9)) + @test JL.include_string(test_mod, "((x,y,rest...)::XYVec)() = (x,y,rest...,)") === nothing + Core.@latestworld + @test xy() == (1,2,9,9,9) + + xy = test_mod.XYVec(Core.svec(1,9,9,9,2)) + @test JL.include_string(test_mod, "((x,rest...,y)::XYVec)(a1,a2) = (x,rest...,y,a1,a2)") === nothing + Core.@latestworld + @test xy(0,0) == (1,9,9,9,2,0,0) +end + +@testset "sparam in keyword default" begin + # The keyword default is evaluated in the body method, which carries all + # of the function's static parameters + @test JL.include_string(test_mod, + "f_kwdef_sp(y::T; k=T) where T = (y, k); f_kwdef_sp(1)") == (1, Int) + # ... but an sparam unused in the signature is undetermined at dispatch + @test_warn r"declares type variable T but does not use it" JL.include_string( + test_mod, "f_kwdef_sp_undet(y; k=T) where T = (y, k)") + @test_throws UndefVarError test_mod.f_kwdef_sp_undet(1) +end + +@testset "anonymous static parameters" begin + # `where _` declares a static parameter which can never be referenced + @test JL.include_string(test_mod, "f_anon_sp(x) where _ = x; f_anon_sp(42)") == 42 + @test JL.include_string( + test_mod, "f_anon_sp2(x::T) where {T, _} = (x, T); f_anon_sp2(1.5)") == (1.5, Float64) + @test_throws LoweringError JL.include_string( + test_mod, "f_anon_sp3(x) where {_, _} = x"; expr_compat_mode=true) + # Currently allowed (like arguments). Could error like flisp. + @test_throws LoweringError JL.include_string( + test_mod, "f_anon_sp3(x) where {_, _} = x") broken=true +end + +@testset "first arg `where`" begin + @eval test_mod struct A12238{T} end + Core.@latestworld + @test JL.include_string(test_mod, "(A12238{T} where T<:Real)(x) = 0") === nothing + @test test_mod.A12238{<:Real}(0) == 0 + @test_throws MethodError test_mod.A12238{<:Integer}(0) + + # Nested where + @eval test_mod struct A12238_2{T, U}; x::T; y::U; end + Core.@latestworld + @test JL.include_string( + test_mod, + "(A12238_2{T, U} where T<:U where U<:Real)(x) = A12238_2(x,x)") === nothing + @test (test_mod.A12238_2{T, U} where {U<:Real, T<:U})(0) === + test_mod.A12238_2{Int, Int}(0, 0) + + # Implicit whereparams + @eval test_mod struct A12238_3{T, U}; x::T; y::U; end + Core.@latestworld + @test JL.include_string( + test_mod, + "(A12238_3{<:Real, <:AbstractVector{<:Real}})() = A12238_3(1,Int[1])") === nothing + @test (test_mod.A12238_3{<:Real, <:AbstractVector{<:Real}})() isa + test_mod.A12238_3{Int, Vector{Int}} + @test (test_mod.A12238_3{<:Real, <:AbstractVector{<:Real}})().x == 1 + @test (test_mod.A12238_3{<:Real, <:AbstractVector{<:Real}})().y == [1] + + # >: + @eval test_mod struct A12238_4{T} end + Core.@latestworld + @test JL.include_string( + test_mod, + "(A12238_4{T} where T>:Int)(x) = x") === nothing + @test test_mod.A12238_4{>:Int}(1) == 1 + @test_throws MethodError test_mod.A12238_4{<:Int}(1) +end + +@testset "Write-only placeholder function arguments" begin + # positional arguments may be duplicate placeholders. keyword arguments can + # contain placeholders, but they must be unique + params_req = ["" + "_" + "::Int" + "_, _" + "(_, _)"] + params_opt = ["" + "::Int=2" + "_=2"] + params_va = ["", "_..."] + params_kw = ["" + "; _" + "; _::Int" + "; _::Int=1" + "; _=1, __=2" + "; _..." + "; _=1, __..."] + for req in params_req, opt in params_opt, va in params_va, kw in params_kw + arg_str = join(filter(!isempty, (req, opt, va, kw)), ", ") + f_str = "function ($arg_str); end" + @testset "$f_str" begin + @test JuliaLowering.include_string(test_mod, f_str) isa Function + end + f_lam_str = "($arg_str)->nothing" + @testset "$f_lam_str" begin + @test JuliaLowering.include_string(test_mod, f_lam_str) isa Function + end + end +end + +@testset "Badly-parsed anonymous forms (fix_arglist)" begin + @test JL.include_string(test_mod, "(()->nothing)()") == nothing + @test JL.include_string(test_mod, "((a...)->(a...,))(1,2,3)") == (1,2,3) + @test JL.include_string(test_mod, "((a::Int)->(a))(1)") == 1 + @test JL.include_string(test_mod, "((a::Int...)->(a...,))(1,2,3)") == (1,2,3) + @test JL.include_string(test_mod, "((;)->nothing)()") == nothing + @test JL.include_string(test_mod, "((a;)->a)(1)") == 1 + @test JL.include_string(test_mod, "((a;b=2)->(a,b))(1)") == (1,2) + @test JL.include_string(test_mod, "((a;b=2)->(a,b))(1;b=3)") == (1,3) + @test JL.include_string(test_mod, "((a=0;b=2)->(a,b))()") == (0,2) + @test JL.include_string(test_mod, "((a=0;b=2)->(a,b))(1)") == (1,2) + @test JL.include_string(test_mod, "((a=0;b=2)->(a,b))(;b=3)") == (0,3) + @test JL.include_string(test_mod, "((a=0;b=2)->(a,b))(1;b=3)") == (1,3) + @test_throws LoweringError JL.include_string(test_mod, "(a=0;b=2;c=3)->nothing") + + # try again with `where` + @test JL.include_string(test_mod, "(((a::T...) where T<:U where U<:Any) ->(a...,))(1,2,3)") == (1,2,3) + @test JL.include_string(test_mod, "(((a::T;) where T<:U where U<:Any) ->a)(1)") == 1 + @test JL.include_string(test_mod, "(((a::T;b=2) where T<:U where U<:Any) ->(a,b))(1)") == (1,2) + @test JL.include_string(test_mod, "(((a::T;b=2) where T<:U where U<:Any) ->(a,b))(1;b=3)") == (1,3) + @test JL.include_string(test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) ->(a,b))()") == (0,2) + @test JL.include_string(test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) ->(a,b))(1)") == (1,2) + @test JL.include_string(test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) ->(a,b))(;b=3)") == (0,3) + @test JL.include_string(test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) ->(a,b))(1;b=3)") == (1,3) + @test_throws LoweringError JL.include_string(test_mod, "(a=0;b=2;c=3)->nothing") + + # `...` is the only parser-reachable bad form with (function notcall _) forms + @test JL.include_string(test_mod, "(function (a...); (a...,); end)(1,2,3)") == (1,2,3) + @test JL.include_string(test_mod, "(function (a::Int...); (a...,); end)(1,2,3)") == (1,2,3) + # test with where: need empty tv list to avoid unused sparam warning + @test jl_eval(test_mod, + Expr(:call, + Expr(:function, Expr(:where, Expr(:where, Expr(:..., :a))), + Expr(:block, Expr(:tuple, Expr(:..., :a)))), + 1,2,3)) == (1,2,3) + @test JL.include_string(test_mod, "(function (a::T) where T<:U where U<:Any; a; end)(1)") == 1 + @test JL.include_string(test_mod, "(function (a::T...) where T<:U where U<:Any; a; end)(1,2,3)") == (1,2,3) +end + +@testset "Consequences of accepting badly-parsed anonymous forms" begin + # kw + @test jl_eval(test_mod, + Expr(:call, + Expr(:function, Expr(:kw, :a, 1), + Expr(:block, Expr(:tuple, :a))), + )) == (1,) + + # empty block + @test jl_eval(test_mod, + Expr(:call, + Expr(:function, Expr(:block), + Expr(:block, Expr(:tuple))), + )) == () + + # unwrapped or block-wrapped arg + @testset for a1 in [:a, Expr(:(::), :a, :Int)], + a2 in [a1, Expr(:(=), :a, 0), Expr(:kw, :a, 0)], + wrap_where in [identity, x->Expr(:where, x), x->Expr(:where, Expr(:where, x))] + + @test jl_eval(test_mod, + Expr(:call, + Expr(:function, wrap_where(a2), + Expr(:block, Expr(:tuple, :a))), + 1)) == (1,) + @test jl_eval(test_mod, + Expr(:call, + Expr(:function, wrap_where(Expr(:block, a2)), + Expr(:block, Expr(:tuple, :a))), + 1)) == (1,) + end + + # two-arg block + @test jl_eval(test_mod, + Expr(:call, + Expr(:function, Expr(:block, :a, :b), + Expr(:block, Expr(:tuple, :a, :b))), + 1, Expr(:kw, :b, 2))) == (1,2) +end + +@testset "assignment to where-wrapped-tuple" begin + @test_throws LoweringError JL.include_string( + test_mod, "(((a,b,c::T) where T<:U where U<:Any) = (a,b,c))(1,2,3)") == (1,2,3) + @test_throws LoweringError JL.include_string( + test_mod, "(((a,b=0,c::T=0) where T<:U where U<:Any) = (a,b,c))(1)") == (1,0,0) + @test_throws LoweringError JL.include_string( + test_mod, "(((a,b=0,c::T=0) where T<:U where U<:Any) = (a,b,c))(1,2)") == (1,2,0) + @test_throws LoweringError JL.include_string( + test_mod, "(((a,b=0,c::T=0) where T<:U where U<:Any) = (a,b,c))(1,2,3)") == (1,2,3) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T...) where T<:U where U<:Any) = (a...,))(1,2,3)") == (1,2,3) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T;) where T<:U where U<:Any) = a)(1)") == 1 + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T;b=2) where T<:U where U<:Any) = (a,b))(1)") == (1,2) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T;b=2) where T<:U where U<:Any) = (a,b))(1;b=3)") == (1,3) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) = (a,b))()") == (0,2) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) = (a,b))(1)") == (1,2) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) = (a,b))(;b=3)") == (0,3) + @test_throws LoweringError JL.include_string( + test_mod, "(((a::T=0;b=2) where T<:U where U<:Any) = (a,b))(1;b=3)") == (1,3) + @test_throws LoweringError JL.include_string( + test_mod, "(a=0;b=2;c=3) where T = nothing") + @test_throws LoweringError jl_eval( + test_mod, + Expr(:call, + Expr(:(=), Expr(:where, Expr(:where, Expr(:..., :a))), + Expr(:block, Expr(:tuple, Expr(:..., :a)))), + 1,2,3)) == (1,2,3) +end + +@testset "Assigned-to arguments" begin + # These examples are all macros, since they have specialized de-optimization + # behavior that sends un-optimized code straight to codegen. Normal compiled + # functions essentially always pass through SSA conversion on the way to the + # optimizer, erasing these slots (potentially hiding bugs in slot handling) + + @test JuliaLowering.include_string(test_mod, raw""" + macro m_assigned_args_1(x) + x = x + 1 + return x + end + var"@m_assigned_args_1"(LineNumberNode(0, nothing), Main, 2) + """; expr_compat_mode=true) == 3 + + @test JuliaLowering.include_string(test_mod, raw""" + macro m_assigned_args_2(x, y = 1) + (y, x) = (x + 1, y + 1) + return y - x + end + ( + var"@m_assigned_args_2"(LineNumberNode(0, nothing), Main, 2), + var"@m_assigned_args_2"(LineNumberNode(0, nothing), Main, 1, 2), + ) + """; expr_compat_mode=true) == (1, -1) + + for expr_compat_mode in (false, true) + @test JuliaLowering.include_string(test_mod, raw""" + macro m_assigned_args(ex) + ex = Base.remove_linenums!(ex) + return ex + end + ((@m_assigned_args 1 + 1), @m_assigned_args 1) + """; expr_compat_mode) == (2, 1) + end +end + +@testset "Generated functions" begin; for expr_compat_mode in (false, true) + local genfunc_s, genfunc_f + @eval test_mod import JuliaLowering.@legacy_quote_to_syntax + + @testset "returning special syntax forms" begin + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_nothing() = nothing + f_gen_nothing() + end + """; expr_compat_mode) == nothing + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_quotenothing() = :(nothing) + f_gen_quotenothing() + end + """; expr_compat_mode) == nothing + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_quotenodenothing() = QuoteNode(nothing) + f_gen_quotenodenothing() + end + """; expr_compat_mode) == nothing + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_quotenodeexpr() = QuoteNode(Expr(:begin, nothing)) + f_gen_quotenodeexpr() + end + """; expr_compat_mode) == Expr(:begin, nothing) + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_gr_nothing() = GlobalRef(Core, :nothing) + f_gen_gr_nothing() + end + """; expr_compat_mode) == nothing + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_quotegr_nothing() = :(GlobalRef(Core, :nothing)) + f_gen_quotegr_nothing() + end + """; expr_compat_mode) == GlobalRef(Core, :nothing) + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated f_gen_quotenodegr_nothing() = QuoteNode(GlobalRef(Core, :nothing)) + f_gen_quotenodegr_nothing() + end + """; expr_compat_mode) == GlobalRef(Core, :nothing) + end + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function f_gen_trivial(x) + x + end + + f_gen_trivial(1), f_gen_trivial(Int[1]) + end + """; expr_compat_mode) == (Int, Vector{Int}) + + @test JuliaLowering.include_string(test_mod, raw""" + begin + function f_gen_trivial_if(x) + if @generated + x + else + :($x, "nongen") + end + end + + f_gen_trivial_if(1), f_gen_trivial_if(Int[1]) + end + """; expr_compat_mode) == (Int, Vector{Int}) + + @testset "anonymous forms" begin + @test JuliaLowering.include_string(test_mod, """ + let + f = @generated function (x); x; end + f(1), f(Int[1]) + end + """; expr_compat_mode) == (Int, Vector{Int}) + @test JuliaLowering.include_string(test_mod, """ + let + f = (x)->(if @generated(); x; else; "nongen"; end) + f(1), f(Int[1]) + end + """; expr_compat_mode) == (Int, Vector{Int}) + end + + @testset "destructured args" begin + genfunc_s = raw""" + function ((d1,d2)::T) where {T} + if @generated + :($T, "gen") + else + :($T, "nongen") + end + end + """ + @test (genfunc_f = JL.include_string(test_mod, genfunc_s; expr_compat_mode)) isa Function + @test genfunc_f((1,2)) == (Tuple{Int, Int}, "gen") + end + + @testset "destructured args: values" begin + genfunc_s = raw""" + function ((d1,d2)::T) where {T} + if @generated + :(d1, d2, $T, "gen") + else + :($T, "nongen") + end + end + """ + @test (genfunc_f = JL.include_string(test_mod, genfunc_s; expr_compat_mode)) isa Function + @test genfunc_f((1,2)) == (1, 2, Tuple{Int, Int}, "gen") + end + + @testset "(AI) destructured args: shapes" begin + # A destructured-tuple argument in a fully-`@generated` function whose + # body is a `quote`/`Expr(:block)` (not a bare single expression): the + # implicit `(names...) = ` prologue must reach the generated code. + @test JL.include_string(test_mod, raw""" + @generated function fds_named(x, (a, b)); quote a + b end; end + fds_named(1, (2, 3)) + """; expr_compat_mode) == 5 + # Same, with the generated body built as an explicit `Expr(:block, ...)`. + @test JL.include_string(test_mod, raw""" + @generated function fds_exprblock(x, (a, b)); Expr(:block, :(a + b)); end + fds_exprblock(1, (2, 3)) + """; expr_compat_mode) == 5 + # Nested destructuring. + @test JL.include_string(test_mod, raw""" + @generated function fds_nested(x, (a, (b, c))); quote a + b + c end; end + fds_nested(1, (2, (3, 4))) + """; expr_compat_mode) == 9 + # Destructured first argument. + @test JL.include_string(test_mod, raw""" + @generated function fds_first((a, b)); quote a + b end; end + fds_first((2, 3)) + """; expr_compat_mode) == 5 + # Positional vararg after a destructured argument. + @test JL.include_string(test_mod, raw""" + @generated function fds_va((a, b), xs...); quote a + b + length(xs) end; end + fds_va((2, 3), 10, 20) + """; expr_compat_mode) == 7 + # Destructured argument alongside keyword arguments. + @test JL.include_string(test_mod, raw""" + @generated function fds_kw((a, b); k=0); quote a + b + k end; end + fds_kw((2, 3); k=10) + """; expr_compat_mode) == 15 + # Multiple destructured args + @test JL.include_string(test_mod, raw""" + @generated function fds_multi((a, b), (c, d)); quote a + b + c + d end; end + fds_multi((1, 2), (3, 4)) + """; expr_compat_mode) == 10 + end + + @testset "keyword args" begin + genfunc_f = JL.include_string(test_mod, raw""" + function (parg::Tuple{T}; kw) where {T} + if @generated + :($parg, $T, $kw, "gen") + else + :($parg, $T, $kw, "nongen") + end + end + """; expr_compat_mode) + + @test genfunc_f((1,); kw=1) == + (Tuple{Int}, Int, Int, "gen") + @test_throws UndefKeywordError genfunc_f((1,)) + + genfunc_f = JL.include_string(test_mod, raw""" + function (parg::Tuple{T}; kw::Vector{T}) where {T} + if @generated + :($parg, $T, $kw, "gen") + else + :($parg, $T, $kw, "nongen") + end + end + """; expr_compat_mode) + + @test genfunc_f((1,); kw=[1]) == (Tuple{Int}, Int, Vector{Int}, "gen") + @test_throws UndefKeywordError genfunc_f((1,)) + @test_throws TypeError genfunc_f((1,); kw=1) + + genfunc_f = JL.include_string(test_mod, raw""" + function (; kw::T, rkw...) where {T} + if @generated + :($T, $kw, $rkw, "gen") + else + :($T, $kw, $rkw, "nongen") + end + end + """; expr_compat_mode) + + @test genfunc_f(; kw=1) == + (Int, Int, Base.Pairs{Symbol, Union{}, Nothing, @NamedTuple{}}, "gen") + @test genfunc_f(; kw=1, kw2=2) == + (Int, Int, Base.Pairs{Symbol, Int, Nothing, @NamedTuple{kw2::Int}}, "gen") + @test_throws UndefKeywordError genfunc_f() + end + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function f_gen(x::NTuple{N,T}) where {N,T} + quote + ($x, $N, $T) + end + end + + f_gen((1,2,3,4,5)) + end + """; expr_compat_mode) == (NTuple{5,Int}, 5, Int) + + @test JuliaLowering.include_string(test_mod, """ + begin + @generated function f_gen_unnamed_args(::Type{T}, y, ::Type{U}) where {T, U} + return (T, y, U) + end + + f_gen_unnamed_args(Int, UInt8(3), Float64) + end + """; expr_compat_mode) == (Int, UInt8, Float64) + + @test JuliaLowering.include_string(test_mod, raw""" + begin + function f_partially_gen(x::NTuple{N,T}) where {N,T} + shared = :shared_stuff + if @generated + if N == 2 + error("intentionally broken codegen (will trigger nongen branch)") + end + quote + unshared = (:gen, ($x, $N, $T)) + end + else + unshared = (:nongen, (typeof(x), N, T)) + end + (shared, unshared) + end + + (f_partially_gen((1,2)), f_partially_gen((1,2,3,4,5))) + end + """; expr_compat_mode) == + ((:shared_stuff, (:nongen, (NTuple{2,Int}, 2, Int))), + (:shared_stuff, (:gen, (NTuple{5,Int}, 5, Int)))) + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function f_gen_calls_macros(x::T) where {T} + s = @raw_str "foo" + :(@raw_str $s) + end + f_gen_calls_macros(1) + end + """; expr_compat_mode) === "foo" + @test JuliaLowering.include_string(test_mod, raw"""begin + @generated function calls_versioned_macro(::Type{T}, ::Val{i}) where {T, i} + i isa Integer || @goto err + return i + @label err + return 0 + end + + calls_versioned_macro(Tuple{Int}, Val(1)) + end """; expr_compat_mode) == 1 + + # (AI) pkgeval reduction: `(. value macroname)` should work + @test (let m = Module() + jl_eval(m, :(module RM5b; macro mm(ex); esc(ex); end; end); expr_compat_mode=true) + rm = Core.eval(m, :RM5b) + jl_eval(m, quote + @generated function fr5b(x) + Expr(:macrocall, Expr(:., $rm, QuoteNode(Symbol("@mm"))), + LineNumberNode(1), :(x[1])) + end + end; expr_compat_mode=true) + jl_eval(m, :(fr5b([7, 8])); expr_compat_mode=true) + end) == 7 + + @testset "(AI) anonymous args promoted by optional/keyword args" begin + # A `@generated` method with >=2 anonymous args (`::T` or `_`) whose + # placeholder slots get promoted to `#arg#` identifiers because the + # method also has an optional positional or keyword arg used to fail at + # first call with "function argument name not unique". + + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function g_anon_opt(x, + ::Val{A}=Val(false), + ::Val{B}=Val(false)) where {A,B} + :( (x, A, B) ) + end + g_anon_opt(1) + end + """; expr_compat_mode) === (1, false, false) + + # calling the same function at two different type instantiations + @test JuliaLowering.include_string(test_mod, raw""" + (g_anon_opt(1, Val(:a), Val(:b)), g_anon_opt(2.0, Val(3))) + """; expr_compat_mode) === ((1, :a, :b), (2.0, 3, false)) + + # 2 anonymous required args forced by an unrelated keyword arg + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function g_anon_kw(::Val{A}, ::Val{B}; kw=1) where {A,B} + :( (A, B, kw) ) + end + g_anon_kw(Val(1), Val(2)) + end + """; expr_compat_mode) === (1, 2, 1) + @test JuliaLowering.include_string(test_mod, + "g_anon_kw(Val(1), Val(2); kw=5)"; expr_compat_mode) === (1, 2, 5) + + # underscore args (also anonymous), forced by a default + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function g_anon_underscore(_, _, z=10) + :( z ) + end + g_anon_underscore(:a, :b) + end + """; expr_compat_mode) === 10 + + # named + anonymous mix, with the body reading the named arg while the + # generator also uses the where-params + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function g_named_anon(a, ::Val{A}, ::Val{B}=Val(0); k=7) where {A,B} + :( (a, A, B, k) ) + end + g_named_anon("hi", Val(1)) + end + """; expr_compat_mode) === ("hi", 1, 0, 7) + + # single anonymous arg (no collision possible) still works with a kwarg + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function g_one_anon(x, ::Val{A}; k=3) where {A} + :( (x, A, k) ) + end + g_one_anon(1, Val(2)) + end + """; expr_compat_mode) === (1, 2, 3) + + # Pathological: a user arg literally named `#arg#` (the promotion name) + # must remain a real, body-referenceable slot -- the discriminator is a + # metadata tag on promoted anonymous args, not a name match. + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function g_user_hasharg(var"#arg#", ::Val{A}=Val(0)) where {A} + :( (var"#arg#", A) ) + end + g_user_hasharg(5) + end + """; expr_compat_mode) === (5, 0) + end + + @testset "hygiene in generated functions" begin + # (AI) A generator whose returned body is a bare macrocall to an + # old-style macro that re-wraps `esc`'d fragments in a freshly-built, + # unescaped `Expr` (e.g. `Base.Cartesian.@nif`), referencing the + # generated function's own arguments and static parameters. Those + # escaped references unwind to the generator's base layer, so the + # synthesized argument/sparam names of the staged method must live in + # that same layer -- otherwise they resolve as bogus module globals + # (`UndefVarError`). + @test JuliaLowering.include_string(test_mod, raw""" + begin + @generated function find_first_eq(x, itr::I) where { + N, I <: Tuple{Vararg{Any, N}} + } + return :(Base.Cartesian.@nif $(N + 1) d -> (x == getfield(itr, d)) d -> (d) d -> (nothing)) + end + (find_first_eq(20, (10, 20, 30)), find_first_eq(99, (10, 20, 30))) + end"""; expr_compat_mode) === (2, nothing) + @test JuliaLowering.include_string(test_mod, raw"""begin + @generated function nif_uses_sparam(x, ::Type{T}) where {T} + return :(Base.Cartesian.@nif 2 d -> (x isa T) d -> (T) d -> (nothing)) + end + nif_uses_sparam(1, Int) + end"""; expr_compat_mode) === Int + end +end + + genfunc_quote_s = """ + begin + function f_gen_quote_1(::Tuple{T}) where {T} + out = @legacy_quote_to_syntax :(:x1,first) + if @generated + else + end + return out + end + + f_gen_quote_1((1,)) + end + """ + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=true) == :(:x1,first) + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=false) ≈ + @ast_ [K"tuple" [K"inert" "x1"::K"Identifier"] "first"::K"Identifier"] + + genfunc_quote_s = """ + begin + function f_gen_quote_2(::Tuple{T}) where {T} + out = nothing + if @generated + @legacy_quote_to_syntax :(out = @legacy_quote_to_syntax :(:x2,generated)) + else + out = (:x2,nongen) + end + return out + end + + f_gen_quote_2((1,)) + end + """ + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=true) == :(:x2,generated) + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=false) ≈ + @ast_ [K"tuple" [K"inert" "x2"::K"Identifier"] "generated"::K"Identifier"] + + genfunc_quote_s = """ + begin + function f_gen_quote_3(::Tuple{T}) where {T} + if @generated + else + end + return @legacy_quote_to_syntax :(:x4,after) + end + + f_gen_quote_3((1,)) + end + """ + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=true) == :(:x4,after) + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=false) ≈ + @ast_ [K"tuple" [K"inert" "x4"::K"Identifier"] "after"::K"Identifier"] + + genfunc_quote_s = raw""" + begin + function f_gen_interpolate(::Tuple{T}) where {T} + out = :(:x1,first) + if @generated + out = @legacy_quote_to_syntax :($out, generated) + else + out = @legacy_quote_to_syntax :($out, nongen) + end + return out + end + + f_gen_interpolate((1,)) + end + """ + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=true) == :((:x1,first),nongen) + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=false) ≈ + @ast_ [K"tuple" [K"tuple" + [K"inert" "x1"::K"Identifier"] + "first"::K"Identifier"] + "nongen"::K"Identifier"] + + genfunc_quote_s = raw""" + begin + @eval function f_gen_eval_quote_1(::Tuple{T}) where {T} + out = $(Expr(:quote, Expr(:call, :+, 1, Expr(:if, Expr(:generated), 1, 2)))) + if @generated + else + end + return out + end + f_gen_eval_quote_1((1,)) + end + """ + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=true) == + :(1 + $(Expr(:if, Expr(:generated), 1, 2))) + @test JuliaLowering.include_string( + test_mod, genfunc_quote_s; expr_compat_mode=false) == + :(1 + $(Expr(:if, Expr(:generated), 1, 2))) + + # Test generated function edges to bindings + # (see also https://github.com/JuliaLang/julia/pull/57230) + JuliaLowering.include_string(test_mod, raw""" + const delete_me = 4 + @generated f_generated_return_delete_me() = return quote; delete_me; end + """) + @test test_mod.f_generated_return_delete_me() == 4 + Base.delete_binding(test_mod, :delete_me) + @test_throws UndefVarError test_mod.f_generated_return_delete_me() +end + +@testset "pre-desugared meta-generated" begin + @test JuliaLowering.include_string(test_mod, raw""" + @eval function meta_generated_form() + $(Expr(:meta, :generated, Base.identity)) + $(Expr(:meta, :generated_only)) + end + """, expr_compat_mode=true) isa Function +end + +@testset "Broadcast" begin + @test JuliaLowering.include_string(test_mod, """ + let x = [1,2], y = [3,4], z = [5,6] + x .* y .+ z + end + """) == [8, 14] + + @test JuliaLowering.include_string(test_mod, """ + let nums = [1, 2, 3] + string.(nums, base=2; pad=2) + end + """) == ["01", "10", "11"] + + @test JuliaLowering.include_string(test_mod, """ + let lhs = [0,0], x = [1,2], y = [3,4], z = [5,6] + lhs .= x .* y .+ z + lhs + end + """) == [8, 14] + + @test JuliaLowering.include_string(test_mod, """ + [1,2] .+ ([3,4] .< [5,6] .< [7,1]) + """) == [2, 2] + + @test JuliaLowering.include_string(test_mod, """ + let + x = [0,0,0,0] + x[begin+1:end-1] .= [1,2] .+ [3,4] + x + end + """) == [0,4,6,0] +end + +@testset "method table overlays" begin + OverlayModule = @newmod() + + @eval OverlayModule Base.Experimental.@MethodTable mt + @test JL.include_string(OverlayModule, """ + Base.Experimental.@overlay mt function sin(x::Float64); 1; end + """) isa Method + @test JL.include_string(OverlayModule, """ + Base.Experimental.@overlay mt cos(x::Float64) = 2 + """) isa Method + @test JL.include_string(OverlayModule, """ + Base.Experimental.@overlay mt tan(x::T) where {T} = 3 + """) isa Method + + let ms = Base._methods_by_ftype( + Tuple{typeof(sin), Float64}, nothing, 1, Base.get_world_counter()) + @test only(ms).method.module === Base.Math + end + let ms = Base._methods_by_ftype( + Tuple{typeof(sin), Float64}, OverlayModule.mt, 1, Base.get_world_counter()) + @test only(ms).method.module === OverlayModule + end + let ms = Base._methods_by_ftype( + Tuple{typeof(sin), Int}, OverlayModule.mt, 1, Base.get_world_counter()) + @test isempty(ms) + end + + # anything may go in first arg + @test JL.include_string(OverlayModule, """ + Base.Experimental.@overlay @__MODULE__().mt identity(x::Int) = x + """) isa Method + let ms = Base._methods_by_ftype( + Tuple{typeof(identity), Int}, nothing, 1, Base.get_world_counter()) + @test only(ms).method.module === Base + end +end diff --git a/JuliaLowering/test/functions_ir.jl b/JuliaLowering/test/functions_ir.jl new file mode 100644 index 0000000000000..3790dba2ad5ff --- /dev/null +++ b/JuliaLowering/test/functions_ir.jl @@ -0,0 +1,2236 @@ +######################################## +# Function declaration with no methods +function f +end +#--------------------- +1 (call core.declare_global TestMod :f false) +2 latestworld +3 (call core.define_method TestMod :f) +4 latestworld +5 TestMod.f +6 (return %₅) + +######################################## +# Functions with placeholder arg +function f(x, _, y) + x + y +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/#unused#(!read) slot₄/y] + 1 TestMod.+ + 2 (call %₁ slot₂/x slot₄/y) + 3 (return %₂) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Functions with argument types only, no name +function f(::T, x) + x +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.T +6 (call core.svec %₄ %₅ core.Any) +7 (call core.svec) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read) slot₃/x] + 1 slot₃/x + 2 (return %₁) +11 latestworld +12 TestMod.f +13 (return %₁₂) + +######################################## +# Functions argument types +function f(x, y::T) + body +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.T +6 (call core.svec %₄ core.Any %₅) +7 (call core.svec) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/y(!read)] + 1 TestMod.body + 2 (return %₁) +11 latestworld +12 TestMod.f +13 (return %₁₂) + +######################################## +# Functions with slurp of Any +function f(x, ys...) + body +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.apply_type core.Vararg core.Any) +6 (call core.svec %₄ core.Any %₅) +7 (call core.svec) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/ys(!read)] + 1 TestMod.body + 2 (return %₁) +11 latestworld +12 TestMod.f +13 (return %₁₂) + +######################################## +# Functions with slurp of T +function f(x, ys::T...) + body +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.T +6 (call core.apply_type core.Vararg %₅) +7 (call core.svec %₄ core.Any %₆) +8 (call core.svec) +9 SourceLocation::1:1 +10 (call core.svec %₇ %₈ %₉) +11 (call core.define_method TestMod TestMod.f %₁₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/ys(!read)] + 1 TestMod.body + 2 (return %₁) +12 latestworld +13 TestMod.f +14 (return %₁₃) + +######################################## +# Error: Function with slurp not in last position arg +function f(xs..., y) + body +end +#--------------------- +LoweringError: +function f(xs..., y) +# └───┘ ── `...` may only be used on the final parameter + body +end + +######################################## +# Basic static parameters +function f(::T, ::U, ::V) where T where {U,V} + (T,U,V) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.TypeVar :U) +4 (call core.TypeVar :V) +5 (call core.TypeVar :T) +6 TestMod.f +7 (call core.TypeEqOf %₆) +8 (call core.svec %₇ %₅ %₃ %₄) +9 (call core.svec %₃ %₄ %₅) +10 SourceLocation::1:1 +11 (call core.svec %₈ %₉ %₁₀) +12 (call core.define_method TestMod TestMod.f %₁₁ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read) slot₃/#unused#(!read) slot₄/#unused#(!read)] + 1 static_parameter₃ + 2 static_parameter₁ + 3 static_parameter₂ + 4 (call core.tuple %₁ %₂ %₃) + 5 (return %₄) +13 latestworld +14 TestMod.f +15 (return %₁₄) + +######################################## +# Static parameter with bounds and used with apply_type in argument +function f(::S{T}) where X <: T <: Y + T +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.X +4 TestMod.Y +5 (call core.TypeVar :T %₃ %₄) +6 TestMod.f +7 (call core.TypeEqOf %₆) +8 TestMod.S +9 (call core.apply_type %₈ %₅) +10 (call core.svec %₇ %₉) +11 (call core.svec %₅) +12 SourceLocation::1:1 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 (call core.define_method TestMod TestMod.f %₁₃ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read)] + 1 static_parameter₁ + 2 (return %₁) +15 latestworld +16 TestMod.f +17 (return %₁₆) + +######################################## +# Static parameter with lower bound +function f(::S{T}) where T >: X + T +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.X +4 (call core.TypeVar :T %₃ core.Any) +5 TestMod.f +6 (call core.TypeEqOf %₅) +7 TestMod.S +8 (call core.apply_type %₇ %₄) +9 (call core.svec %₆ %₈) +10 (call core.svec %₄) +11 SourceLocation::1:1 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod TestMod.f %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read)] + 1 static_parameter₁ + 2 (return %₁) +14 latestworld +15 TestMod.f +16 (return %₁₅) + +######################################## +# Static parameter which is used only in the bounds of another static parameter +# See https://github.com/JuliaLang/julia/issues/49275 +function f(x, y::S) where {T, S<:AbstractVector{T}} + (T,S) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.TypeVar :T) +4 TestMod.AbstractVector +5 (call core.apply_type %₄ %₃) +6 (call core.TypeVar :S %₅) +7 TestMod.f +8 (call core.TypeEqOf %₇) +9 (call core.svec %₈ core.Any %₆) +10 (call core.svec %₃ %₆) +11 SourceLocation::1:1 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod TestMod.f %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/y(!read)] + 1 static_parameter₁ + 2 static_parameter₂ + 3 (call core.tuple %₁ %₂) + 4 (return %₃) +14 latestworld +15 TestMod.f +16 (return %₁₅) + +######################################## +# Error: underscore sparam is not readable in the function body +function f(x::_) where _ + return _ +end +#--------------------- +LoweringError: +function f(x::_) where _ +# ╙ ── all-underscore identifiers are write-only and their values cannot be used in expressions + return _ +end + +######################################## +# Error: underscore sparam is not readable in a default arg value +f(x::_, y=_) where _ = x +#--------------------- +LoweringError: +f(x::_, y=_) where _ = x +# ╙ ── all-underscore identifiers are write-only and their values cannot be used in expressions + +######################################## +# Error: underscore sparam is not readable in a default kwarg value +f(x::_; y=_) where _ = x +#--------------------- +LoweringError: +f(x::_; y=_) where _ = x +# ╙ ── all-underscore identifiers are write-only and their values cannot be used in expressions + +######################################## +# Error: underscore sparam is not readable in the return type +(f(x)::_) where _ = x +#--------------------- +LoweringError: +(f(x)::_) where _ = x +# ╙ ── all-underscore identifiers are write-only and their values cannot be used in expressions + +######################################## +# Return types +function f(x)::Int + if x + 42.0 + end + 0xff +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/tmp(!read)] + 1 TestMod.Int + 2 (gotoifnot slot₂/x label₃) + 3 (= slot₃/tmp 0xff) + 4 (call core.isa slot₃/tmp %₁) + 5 (gotoifnot %₄ label₇) + 6 (goto label₉) + 7 (call top.convert %₁ slot₃/tmp) + 8 (= slot₃/tmp (call core.typeassert %₇ %₁)) + 9 slot₃/tmp + 10 (return %₉) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Complex return types +function f(c,b1,b2)::Union{Vector{<:Integer}, Int} + if b1 + return 1 + elseif b2 + 2 + else + 3 + end +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/c(!read) slot₃/b1 slot₄/b2 slot₅/tmp(!read) slot₆/tmp(!read) slot₇/tmp(!read)] + 1 TestMod.Union + 2 TestMod.Integer + 3 (call core.TypeVar :#T1 %₂) + 4 TestMod.Vector + 5 (call core.apply_type %₄ %₃) + 6 (call core.UnionAll %₃ %₅) + 7 TestMod.Int + 8 (call core.apply_type %₁ %₆ %₇) + 9 (gotoifnot slot₃/b1 label₁₈) + 10 (= slot₅/tmp 1) + 11 (call core.isa slot₅/tmp %₈) + 12 (gotoifnot %₁₁ label₁₄) + 13 (goto label₁₆) + 14 (call top.convert %₈ slot₅/tmp) + 15 (= slot₅/tmp (call core.typeassert %₁₄ %₈)) + 16 slot₅/tmp + 17 (return %₁₆) + 18 (gotoifnot slot₄/b2 label₂₇) + 19 (= slot₆/tmp 2) + 20 (call core.isa slot₆/tmp %₈) + 21 (gotoifnot %₂₀ label₂₃) + 22 (goto label₂₅) + 23 (call top.convert %₈ slot₆/tmp) + 24 (= slot₆/tmp (call core.typeassert %₂₃ %₈)) + 25 slot₆/tmp + 26 (return %₂₅) + 27 (= slot₇/tmp 3) + 28 (call core.isa slot₇/tmp %₈) + 29 (gotoifnot %₂₈ label₃₁) + 30 (goto label₃₃) + 31 (call top.convert %₈ slot₇/tmp) + 32 (= slot₇/tmp (call core.typeassert %₃₁ %₈)) + 33 slot₇/tmp + 34 (return %₃₃) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Callable type +function (::T)(x) + x +end +#--------------------- +1 TestMod.T +2 (call core.svec %₁ core.Any) +3 (call core.svec) +4 SourceLocation::1:1 +5 (call core.svec %₂ %₃ %₄) +6 (call core.define_method TestMod core.nothing %₅ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x] + 1 slot₂/x + 2 (return %₁) +7 latestworld +8 (return core.nothing) + +######################################## +# Callable type with instance +function (y::T)(x) + (y, x) +end +#--------------------- +1 TestMod.T +2 (call core.svec %₁ core.Any) +3 (call core.svec) +4 SourceLocation::1:1 +5 (call core.svec %₂ %₃ %₄) +6 (call core.define_method TestMod core.nothing %₅ + --- code_info + slots: [slot₁/y slot₂/x] + 1 (call core.tuple slot₁/y slot₂/x) + 2 (return %₁) +7 latestworld +8 (return core.nothing) + +######################################## +# Error: Callable type not at top level +function f(x) + (::T)(y) = x + y +end +#--------------------- +LoweringError: +function f(x) + (::T)(y) = x + y +# └─┘ ── adding methods to callable type only allowed at top level +end + +######################################## +# `where` params used in callable object type +function (x::X1{T})() where T + T +end +#--------------------- +1 (call core.TypeVar :T) +2 TestMod.X1 +3 (call core.apply_type %₂ %₁) +4 (call core.svec %₃) +5 (call core.svec %₁) +6 SourceLocation::1:1 +7 (call core.svec %₄ %₅ %₆) +8 (call core.define_method TestMod core.nothing %₇ + --- code_info + slots: [slot₁/x(!read)] + 1 static_parameter₁ + 2 (return %₁) +9 latestworld +10 (return core.nothing) + +######################################## +# Function with module ref in name +function A.f() +end +#--------------------- +1 TestMod.A +2 (call top.getproperty %₁ :f) +3 (call core.TypeEqOf %₂) +4 (call core.svec %₃) +5 (call core.svec) +6 SourceLocation::1:1 +7 (call core.svec %₄ %₅ %₆) +8 (call core.define_method TestMod core.nothing %₇ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (return core.nothing) +9 latestworld +10 (return core.nothing) + +######################################## +# Error: Invalid dotop function name +function (.+)(x,y) +end +#--------------------- +LoweringError: +function (.+)(x,y) +# └┘ ── invalid function name +end + +######################################## +# Error: Invalid dotop function name +function var".+"(x,y) +end +#--------------------- +LoweringError: +function var".+"(x,y) +# └┘ ── dotted operator is not a valid function name +end + +######################################## +# dotted normal name is fine +function var".f"(); end +#--------------------- +1 (call core.define_method TestMod :.f) +2 latestworld +3 TestMod..f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod..f %₈ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (return core.nothing) +10 latestworld +11 TestMod..f +12 (return %₁₁) + +######################################## +# Error: Invalid function name +function f[](x,y) +end +#--------------------- +LoweringError: +function f[](x,y) +# └─┘ ── invalid function name +end + +######################################## +# Simple positional args with defaults +function f(x::T, y::S=1, z::U=2) + (x,y) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.T +6 (call core.svec %₄ %₅) +7 (call core.svec) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(called) slot₂/x slot₃/y(single_assign)] + 1 1 + 2 (= slot₃/y %₁) + 3 (call slot₁/#self# slot₂/x slot₃/y 2) + 4 (return %₃) +11 latestworld +12 TestMod.f +13 (call core.TypeEqOf %₁₂) +14 TestMod.T +15 TestMod.S +16 (call core.svec %₁₃ %₁₄ %₁₅) +17 (call core.svec) +18 SourceLocation::1:1 +19 (call core.svec %₁₆ %₁₇ %₁₈) +20 (call core.define_method TestMod TestMod.f %₁₉ + --- code_info + slots: [slot₁/#self#(called) slot₂/x slot₃/y] + 1 (call slot₁/#self# slot₂/x slot₃/y 2) + 2 (return %₁) +21 latestworld +22 TestMod.f +23 (call core.TypeEqOf %₂₂) +24 TestMod.T +25 TestMod.S +26 TestMod.U +27 (call core.svec %₂₃ %₂₄ %₂₅ %₂₆) +28 (call core.svec) +29 SourceLocation::1:1 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod TestMod.f %₃₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/y slot₄/z(!read)] + 1 (call core.tuple slot₂/x slot₃/y) + 2 (return %₁) +32 latestworld +33 TestMod.f +34 (return %₃₃) + +######################################## +# Default positional args which depend on other args +function f(x=1, y=x) + (x,y) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(called) slot₂/x(single_assign)] + 1 1 + 2 (= slot₂/x %₁) + 3 (call slot₁/#self# slot₂/x slot₂/x) + 4 (return %₃) +10 latestworld +11 TestMod.f +12 (call core.TypeEqOf %₁₁) +13 (call core.svec %₁₂ core.Any) +14 (call core.svec) +15 SourceLocation::1:1 +16 (call core.svec %₁₃ %₁₄ %₁₅) +17 (call core.define_method TestMod TestMod.f %₁₆ + --- code_info + slots: [slot₁/#self#(called) slot₂/x] + 1 (call slot₁/#self# slot₂/x slot₂/x) + 2 (return %₁) +18 latestworld +19 TestMod.f +20 (call core.TypeEqOf %₁₉) +21 (call core.svec %₂₀ core.Any core.Any) +22 (call core.svec) +23 SourceLocation::1:1 +24 (call core.svec %₂₁ %₂₂ %₂₃) +25 (call core.define_method TestMod TestMod.f %₂₄ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/y] + 1 (call core.tuple slot₂/x slot₃/y) + 2 (return %₁) +26 latestworld +27 TestMod.f +28 (return %₂₇) + +######################################## +# Default positional args with missing arg names (implicit placeholders) +function f(::Int, y=1, z=2) + (y, z) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.Int +6 (call core.svec %₄ %₅) +7 (call core.svec) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(called) slot₂/#arg# slot₃/y(single_assign)] + 1 1 + 2 (= slot₃/y %₁) + 3 (call slot₁/#self# slot₂/#arg# slot₃/y 2) + 4 (return %₃) +11 latestworld +12 TestMod.f +13 (call core.TypeEqOf %₁₂) +14 TestMod.Int +15 (call core.svec %₁₃ %₁₄ core.Any) +16 (call core.svec) +17 SourceLocation::1:1 +18 (call core.svec %₁₅ %₁₆ %₁₇) +19 (call core.define_method TestMod TestMod.f %₁₈ + --- code_info + slots: [slot₁/#self#(called) slot₂/#arg# slot₃/y] + 1 (call slot₁/#self# slot₂/#arg# slot₃/y 2) + 2 (return %₁) +20 latestworld +21 TestMod.f +22 (call core.TypeEqOf %₂₁) +23 TestMod.Int +24 (call core.svec %₂₂ %₂₃ core.Any core.Any) +25 (call core.svec) +26 SourceLocation::1:1 +27 (call core.svec %₂₄ %₂₅ %₂₆) +28 (call core.define_method TestMod TestMod.f %₂₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#arg#(!read) slot₃/y slot₄/z] + 1 (call core.tuple slot₃/y slot₄/z) + 2 (return %₁) +29 latestworld +30 TestMod.f +31 (return %₃₀) + +######################################## +# Default positional args with placeholders +function f(_::Int, x=1) + x +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.Int +6 (call core.svec %₄ %₅) +7 (call core.svec) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(called) slot₂/#arg#] + 1 (call slot₁/#self# slot₂/#arg# 1) + 2 (return %₁) +11 latestworld +12 TestMod.f +13 (call core.TypeEqOf %₁₂) +14 TestMod.Int +15 (call core.svec %₁₃ %₁₄ core.Any) +16 (call core.svec) +17 SourceLocation::1:1 +18 (call core.svec %₁₅ %₁₆ %₁₇) +19 (call core.define_method TestMod TestMod.f %₁₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#arg#(!read) slot₃/x] + 1 slot₃/x + 2 (return %₁) +20 latestworld +21 TestMod.f +22 (return %₂₁) + +######################################## +# Positional args with defaults and `where` clauses +function f(x::T, y::S=1, z::U=2) where {T,S<:T,U<:S} + (x,y,z) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.TypeVar :T) +4 (call core.TypeVar :S %₃) +5 (call core.TypeVar :U %₄) +6 TestMod.f +7 (call core.TypeEqOf %₆) +8 (call core.svec %₇ %₃) +9 (call core.svec %₃) +10 SourceLocation::1:1 +11 (call core.svec %₈ %₉ %₁₀) +12 (call core.define_method TestMod TestMod.f %₁₁ + --- code_info + slots: [slot₁/#self#(called) slot₂/x slot₃/y(single_assign)] + 1 1 + 2 (= slot₃/y %₁) + 3 (call slot₁/#self# slot₂/x slot₃/y 2) + 4 (return %₃) +13 latestworld +14 TestMod.f +15 (call core.TypeEqOf %₁₄) +16 (call core.svec %₁₅ %₃ %₄) +17 (call core.svec %₃ %₄) +18 SourceLocation::1:1 +19 (call core.svec %₁₆ %₁₇ %₁₈) +20 (call core.define_method TestMod TestMod.f %₁₉ + --- code_info + slots: [slot₁/#self#(called) slot₂/x slot₃/y] + 1 (call slot₁/#self# slot₂/x slot₃/y 2) + 2 (return %₁) +21 latestworld +22 TestMod.f +23 (call core.TypeEqOf %₂₂) +24 (call core.svec %₂₃ %₃ %₄ %₅) +25 (call core.svec %₃ %₄ %₅) +26 SourceLocation::1:1 +27 (call core.svec %₂₄ %₂₅ %₂₆) +28 (call core.define_method TestMod TestMod.f %₂₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/y slot₄/z] + 1 (call core.tuple slot₂/x slot₃/y slot₄/z) + 2 (return %₁) +29 latestworld +30 TestMod.f +31 (return %₃₀) + +######################################## +# Positional args and type parameters with transitive dependencies +# See https://github.com/JuliaLang/julia/issues/49275 - the first method +# generated here for only `x` should contain zero type parameters. +function f(x, y::S=[1], z::U=2) where {T, S<:AbstractVector{T}, U} + (x, y, z, T, S, U) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.TypeVar :T) +4 TestMod.AbstractVector +5 (call core.apply_type %₄ %₃) +6 (call core.TypeVar :S %₅) +7 (call core.TypeVar :U) +8 TestMod.f +9 (call core.TypeEqOf %₈) +10 (call core.svec %₉ core.Any) +11 (call core.svec) +12 SourceLocation::1:1 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 (call core.define_method TestMod TestMod.f %₁₃ + --- code_info + slots: [slot₁/#self#(called) slot₂/x slot₃/y(single_assign)] + 1 (call top.vect 1) + 2 (= slot₃/y %₁) + 3 (call slot₁/#self# slot₂/x slot₃/y 2) + 4 (return %₃) +15 latestworld +16 TestMod.f +17 (call core.TypeEqOf %₁₆) +18 (call core.svec %₁₇ core.Any %₆) +19 (call core.svec %₃ %₆) +20 SourceLocation::1:1 +21 (call core.svec %₁₈ %₁₉ %₂₀) +22 (call core.define_method TestMod TestMod.f %₂₁ + --- code_info + slots: [slot₁/#self#(called) slot₂/x slot₃/y] + 1 (call slot₁/#self# slot₂/x slot₃/y 2) + 2 (return %₁) +23 latestworld +24 TestMod.f +25 (call core.TypeEqOf %₂₄) +26 (call core.svec %₂₅ core.Any %₆ %₇) +27 (call core.svec %₃ %₆ %₇) +28 SourceLocation::1:1 +29 (call core.svec %₂₆ %₂₇ %₂₈) +30 (call core.define_method TestMod TestMod.f %₂₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/y slot₄/z] + 1 static_parameter₁ + 2 static_parameter₂ + 3 static_parameter₃ + 4 (call core.tuple slot₂/x slot₃/y slot₄/z %₁ %₂ %₃) + 5 (return %₄) +31 latestworld +32 TestMod.f +33 (return %₃₂) + +######################################## +# Default positional args are allowed before trailing slurp with no default +function f(x=1, ys...) + ys +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(called)] + 1 (call slot₁/#self# 1) + 2 (return %₁) +10 latestworld +11 TestMod.f +12 (call core.TypeEqOf %₁₁) +13 (call core.apply_type core.Vararg core.Any) +14 (call core.svec %₁₂ core.Any %₁₃) +15 (call core.svec) +16 SourceLocation::1:1 +17 (call core.svec %₁₄ %₁₅ %₁₆) +18 (call core.define_method TestMod TestMod.f %₁₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/ys] + 1 slot₃/ys + 2 (return %₁) +19 latestworld +20 TestMod.f +21 (return %₂₀) + +######################################## +# Error: Default positional args before non-default arg +function f(x=1, ys, z=2) + ys +end +#--------------------- +LoweringError: +function f(x=1, ys, z=2) +# └┘ ── all function parameters after an optional parameter must also be optional + ys +end + +######################################## +# Positional arg with slurp and default +function f(xs...=1) + xs +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(called)] + 1 (call slot₁/#self# 1) + 2 (return %₁) +10 latestworld +11 TestMod.f +12 (call core.TypeEqOf %₁₁) +13 (call core.apply_type core.Vararg core.Any) +14 (call core.svec %₁₂ %₁₃) +15 (call core.svec) +16 SourceLocation::1:1 +17 (call core.svec %₁₄ %₁₅ %₁₆) +18 (call core.define_method TestMod TestMod.f %₁₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/xs] + 1 slot₂/xs + 2 (return %₁) +19 latestworld +20 TestMod.f +21 (return %₂₀) + +######################################## +# Positional arg with slurp and splatted default value +function f(xs...=(1,2)...) + xs +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#] + 1 (call core.tuple 1 2) + 2 (call core._apply_iterate top.iterate slot₁/#self# %₁) + 3 (return %₂) +10 latestworld +11 TestMod.f +12 (call core.TypeEqOf %₁₁) +13 (call core.apply_type core.Vararg core.Any) +14 (call core.svec %₁₂ %₁₃) +15 (call core.svec) +16 SourceLocation::1:1 +17 (call core.svec %₁₄ %₁₅ %₁₆) +18 (call core.define_method TestMod TestMod.f %₁₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/xs] + 1 slot₂/xs + 2 (return %₁) +19 latestworld +20 TestMod.f +21 (return %₂₀) + +######################################## +# Trivial function argument destructuring +function f(x, (y,z), w) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/destructured#3 slot₄/w(!read) slot₅/iterstate(single_assign) slot₆/y(!read,single_assign) slot₇/z(!read,single_assign)] + 1 (call top.indexed_iterate slot₃/destructured#3 1) + 2 (= slot₆/y (call core.getfield %₁ 1)) + 3 (= slot₅/iterstate (call core.getfield %₁ 2)) + 4 slot₅/iterstate + 5 (call top.indexed_iterate slot₃/destructured#3 2 %₄) + 6 (= slot₇/z (call core.getfield %₅ 1)) + 7 (return core.nothing) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Function argument destructuring combined with splats, types and defaults +function f((x,)::T...=rhs) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(called)] + 1 TestMod.rhs + 2 (call slot₁/#self# %₁) + 3 (return %₂) +10 latestworld +11 TestMod.f +12 (call core.TypeEqOf %₁₁) +13 TestMod.T +14 (call core.apply_type core.Vararg %₁₃) +15 (call core.svec %₁₂ %₁₄) +16 (call core.svec) +17 SourceLocation::1:1 +18 (call core.svec %₁₅ %₁₆ %₁₇) +19 (call core.define_method TestMod TestMod.f %₁₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/destructured#2 slot₃/x(!read,single_assign)] + 1 (call top.indexed_iterate slot₂/destructured#2 1) + 2 (= slot₃/x (call core.getfield %₁ 1)) + 3 (return core.nothing) +20 latestworld +21 TestMod.f +22 (return %₂₁) + +######################################## +# Error: multiple destructuring in destructured arg +function f(_,(x...,y...)); end +#--------------------- +LoweringError: +function f(_,(x...,y...)); end +# └─────────┘ ── multiple `...` in destructured parameter is ambiguous + +######################################## +# Error: multiple destructuring in var-destructured arg +function f(_,(x...,y...)...); end +#--------------------- +LoweringError: +function f(_,(x...,y...)...); end +# └─────────┘ ── multiple `...` in destructured parameter is ambiguous + +######################################## +# Error: type in destructured arg +function f(_,(x,y::Int)); end +#--------------------- +LoweringError: +function f(_,(x,y::Int)); end +# └────┘ ── cannot have type in destructured argument + +######################################## +# Error: type in destructured arg, nested +function f(_,(x,(y,(z::Int,)))...); end +#--------------------- +LoweringError: +function f(_,(x,(y,(z::Int,)))...); end +# └────┘ ── cannot have type in destructured argument + +######################################## +# Error: type on splat +function (_,((a,b)...)::Int); end +#--------------------- +LoweringError: +function (_,((a,b)...)::Int); end +# └──────┘ ── expected identifier + +######################################## +# Error: destructured arg with kw +function f(_,(x,y=1)); end +#--------------------- +LoweringError: +function f(_,(x,y=1)); end +# └─┘ ── expected identifier or tuple + +######################################## +# Error: destructured arg with ;kw +function f(_,(;x,y=1)); end +#--------------------- +LoweringError: +function f(_,(;x,y=1)); end +# └─┘ ── expected identifier + +######################################## +# Error: destructured arg with other lhs-likes after ; (call) +function f(_,(;x,y())); end +#--------------------- +LoweringError: +function f(_,(;x,y())); end +# └─┘ ── expected identifier + +######################################## +# Error: destructured arg with other lhs-likes after ; (ref) +function f(_,(;x,y())); end +#--------------------- +LoweringError: +function f(_,(;x,y())); end +# └─┘ ── expected identifier + +######################################## +# Error: destructured arg with other lhs-likes after ; (tuple) +function f(_,(;x,(y,z))); end +#--------------------- +LoweringError: +function f(_,(;x,(y,z))); end +# └───┘ ── expected identifier + +######################################## +# Error: destructured arg with other lhs-likes after ; (...) +function f(_,(;x,y...)); end +#--------------------- +LoweringError: +function f(_,(;x,y...)); end +# └──┘ ── expected identifier + +######################################## +# Error: destructuring mixed tuple +function f(_,(x,;y=1)); end +#--------------------- +LoweringError: +function f(_,(x,;y=1)); end +# └──┘ ── cannot mix tuple `(a,b,c)` and named tuple `(;a,b,c)` syntax + +######################################## +# Error: ref in arg tuple (flisp allows this) +function f(_,(_,x[])); end +#--------------------- +LoweringError: +function f(_,(_,x[])); end +# └─┘ ── expected identifier or tuple + +######################################## +# Error: call in arg tuple (flisp allows this; args ignored) +function f(_,(_,x(y))); end +#--------------------- +LoweringError: +function f(_,(_,x(y))); end +# └──┘ ── expected identifier or tuple + +######################################## +# Error: curly in arg tuple (flisp allows this) +function f(_,(_,x{y})); end +#--------------------- +LoweringError: +function f(_,(_,x{y})); end +# └──┘ ── expected identifier or tuple + +######################################## +# Error: splat on non-final default positional arg +function f(x=1...,y=2); end +#--------------------- +LoweringError: +function f(x=1...,y=2); end +# └──┘ ── splat only allowed on final positional default arg + +######################################## +# Error: splat on non-final default positional arg 2 +function f(x=(1,2)...,y=(3,4)...); end +#--------------------- +LoweringError: +function f(x=(1,2)...,y=(3,4)...); end +# └──────┘ ── splat only allowed on final positional default arg + +######################################## +# Function argument destructuring combined with splats, types and defaults +function f(x=default_x)::T +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(called)] + 1 TestMod.default_x + 2 (call slot₁/#self# %₁) + 3 (return %₂) +10 latestworld +11 TestMod.f +12 (call core.TypeEqOf %₁₁) +13 (call core.svec %₁₂ core.Any) +14 (call core.svec) +15 SourceLocation::1:1 +16 (call core.svec %₁₃ %₁₄ %₁₅) +17 (call core.define_method TestMod TestMod.f %₁₆ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/tmp(!read)] + 1 TestMod.T + 2 (= slot₃/tmp core.nothing) + 3 (call core.isa slot₃/tmp %₁) + 4 (gotoifnot %₃ label₆) + 5 (goto label₈) + 6 (call top.convert %₁ slot₃/tmp) + 7 (= slot₃/tmp (call core.typeassert %₆ %₁)) + 8 slot₃/tmp + 9 (return %₈) +18 latestworld +19 TestMod.f +20 (return %₁₉) + +######################################## +# Duplicate positional placeholders ok +function f(_, _); end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read) slot₃/#unused#(!read)] + 1 (return core.nothing) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Duplicate destructured placeholders ok +function f((_,), (_,)) +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/destructured#2 slot₃/destructured#3] + 1 (call top.indexed_iterate slot₂/destructured#2 1) + 2 (call core.getfield %₁ 1) + 3 (call top.indexed_iterate slot₃/destructured#3 1) + 4 (call core.getfield %₃ 1) + 5 (return core.nothing) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Slot flags +function f(@nospecialize(x), g, y) + g() + y +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(nospecialize,!read) slot₃/g(called) slot₄/y] + 1 (meta :nospecialize slot₂/x) + 2 TestMod.+ + 3 (call slot₃/g) + 4 (call %₂ %₃ slot₄/y) + 5 (return %₄) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# FIXME: Error: nospecialize should be outermost expr in arg +function f_bad_nospecialize(@nospecialize(x)=1); end +#--------------------- +LoweringError: +function f_bad_nospecialize(@nospecialize(x)=1); end +# └──────────────┘ ── expected identifier or `identifier::type` + +######################################## +# Error: nospecialize should be outermost expr in arg +function f_bad_nospecialize(;@nospecialize(x)=1); end +#--------------------- +LoweringError: +function f_bad_nospecialize(;@nospecialize(x)=1); end +# └──────────────┘ ── expected identifier or `identifier::type` + +######################################## +# FIXME: Error: nospecialize should be outermost expr in arg +function f_bad_nospecialize(@nospecialize(x)...); end +#--------------------- +LoweringError: +function f_bad_nospecialize(@nospecialize(x)...); end +# └──────────────┘ ── expected identifier or `identifier::type` + +######################################## +# Error: nospecialize should be outermost expr in arg +function f_bad_nospecialize(;@nospecialize(x)...); end +#--------------------- +LoweringError: +function f_bad_nospecialize(;@nospecialize(x)...); end +# └──────────────┘ ── expected identifier + +######################################## +# Error: nospecialize should be outermost expr in arg +function f_bad_nospecialize(@nospecialize(x)::T); end +#--------------------- +LoweringError: +function f_bad_nospecialize(@nospecialize(x)::T); end +# └──────────────┘ ── expected identifier + +######################################## +# Error: nospecialize should be outermost expr in arg +function f_bad_nospecialize(;@nospecialize(x)::T); end +#--------------------- +LoweringError: +function f_bad_nospecialize(;@nospecialize(x)::T); end +# └──────────────┘ ── expected identifier + +######################################## +# Function return without arguments +function f() + return + after_return # <- distinguish output from implicit return +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (return core.nothing) + 2 TestMod.after_return + 3 (return %₂) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Function return in value position is allowed +function f() + x = return 1 +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.f %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read,single_assign)] + 1 (return 1) + 2 (= slot₂/x core.nothing) +10 latestworld +11 TestMod.f +12 (return %₁₁) + +######################################## +# Binding docs to functions +""" +some docs +""" +function f() +end +#--------------------- +1 (newvar slot₁/val) +2 (gotoifnot true label₁₅) +3 (call core.define_method TestMod :f) +4 latestworld +5 TestMod.f +6 (call core.TypeEqOf %₅) +7 (call core.svec %₆) +8 (call core.svec) +9 SourceLocation:nothing:4:0 +10 (call core.svec %₇ %₈ %₉) +11 (call core.define_method TestMod TestMod.f %₁₀ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (return core.nothing) +12 latestworld +13 TestMod.f +14 (= slot₁/val %₁₃) +15 (call Base.Docs.Binding TestMod :f) +16 (call Core.svec "some docs\n") +17 (call Dict{Symbol, Any} :path => "none" :linenumber => 1 :module => TestMod) +18 (call Base.Docs.docstr %₁₆ %₁₇) +19 TestMod.Union +20 TestMod.Tuple +21 (call core.apply_type %₂₀) +22 (call core.apply_type %₁₉ %₂₁) +23 (call Base.Docs.doc! TestMod %₁₅ %₁₈ %₂₂) +24 (gotoifnot true label₂₇) +25 slot₁/val +26 (return %₂₅) +27 (return core.nothing) + +######################################## +# Binding docs to callable type +""" +some docs +""" +function (x::T)() +end +#--------------------- +1 (newvar slot₁/val) +2 (gotoifnot true label₁₁) +3 TestMod.T +4 (call core.svec %₃) +5 (call core.svec) +6 SourceLocation:nothing:4:0 +7 (call core.svec %₄ %₅ %₆) +8 (call core.define_method TestMod core.nothing %₇ + --- code_info + slots: [slot₁/x(!read)] + 1 (return core.nothing) +9 latestworld +10 (= slot₁/val core.nothing) +11 (call Base.Docs.Binding TestMod :T) +12 (call Core.svec "some docs\n") +13 (call Dict{Symbol, Any} :path => "none" :linenumber => 1 :module => TestMod) +14 (call Base.Docs.docstr %₁₂ %₁₃) +15 TestMod.Union +16 TestMod.Tuple +17 (call core.apply_type %₁₆) +18 (call core.apply_type %₁₅ %₁₇) +19 (call Base.Docs.doc! TestMod %₁₁ %₁₄ %₁₈) +20 (gotoifnot true label₂₃) +21 slot₁/val +22 (return %₂₁) +23 (return core.nothing) + +######################################## +# Keyword function with defaults. +# Order of methods +# 1. #f_kw_simple#0(x, y, ::typeof(f_kw_simple), a, b) (body) +# 2. Core.kwcall(kws, ::typeof(f_kw_simple)) +# 3. Core.kwcall(kws, ::typeof(f_kw_simple), a) +# 4. Core.kwcall(kws, ::typeof(f_kw_simple), a, b) (kwcall body) +# 5. f_kw_simple() +# 6. f_kw_simple(a) +# 7. f_kw_simple(a, b) +function f_kw_simple(a::Int=1, b::Float64=1.0; x::Char='a', y::Bool=true) + (a, b, x, y) +end +#--------------------- +1 (call core.define_method TestMod :#f_kw_simple#kw_body#1) +2 latestworld +3 (call core.define_method TestMod :f_kw_simple) +4 latestworld +5 TestMod.#f_kw_simple#kw_body#1 +6 (call core.TypeEqOf %₅) +7 TestMod.Char +8 TestMod.Bool +9 TestMod.f_kw_simple +10 (call core.TypeEqOf %₉) +11 TestMod.Int +12 TestMod.Float64 +13 (call core.svec %₆ %₇ %₈ %₁₀ %₁₁ %₁₂) +14 (call core.svec) +15 SourceLocation::1:1 +16 (call core.svec %₁₃ %₁₄ %₁₅) +17 (call core.define_method TestMod TestMod.#f_kw_simple#kw_body#1 %₁₆ + --- code_info + slots: [slot₁/#f_kw_simple#kw_body#1(!read) slot₂/x slot₃/y slot₄/#self#(!read) slot₅/a slot₆/b] + 1 (meta :nkw 2) + 2 (call core.tuple slot₅/a slot₆/b slot₂/x slot₃/y) + 3 (return %₂) +18 latestworld +19 TestMod.f_kw_simple +20 (call core.TypeEqOf %₁₉) +21 (call core.svec %₂₀) +22 (call core.svec) +23 SourceLocation::1:1 +24 (call core.svec %₂₁ %₂₂ %₂₃) +25 (call core.define_method TestMod TestMod.f_kw_simple %₂₄ + --- code_info + slots: [slot₁/#self#(called) slot₂/a(single_assign)] + 1 1 + 2 (= slot₂/a %₁) + 3 (call slot₁/#self# slot₂/a 1.0) + 4 (return %₃) +26 latestworld +27 TestMod.f_kw_simple +28 (call core.TypeEqOf %₂₇) +29 TestMod.Int +30 (call core.svec %₂₈ %₂₉) +31 (call core.svec) +32 SourceLocation::1:1 +33 (call core.svec %₃₀ %₃₁ %₃₂) +34 (call core.define_method TestMod TestMod.f_kw_simple %₃₃ + --- code_info + slots: [slot₁/#self#(called) slot₂/a] + 1 (call slot₁/#self# slot₂/a 1.0) + 2 (return %₁) +35 latestworld +36 TestMod.f_kw_simple +37 (call core.TypeEqOf %₃₆) +38 TestMod.Int +39 TestMod.Float64 +40 (call core.svec %₃₇ %₃₈ %₃₉) +41 (call core.svec) +42 SourceLocation::1:1 +43 (call core.svec %₄₀ %₄₁ %₄₂) +44 (call core.define_method TestMod TestMod.f_kw_simple %₄₃ + --- code_info + slots: [slot₁/#self# slot₂/a slot₃/b] + 1 TestMod.#f_kw_simple#kw_body#1 + 2 (call %₁ 'a' true slot₁/#self# slot₂/a slot₃/b) + 3 (return %₂) +45 latestworld +46 (call core.typeof core.kwcall) +47 TestMod.f_kw_simple +48 (call core.TypeEqOf %₄₇) +49 (call core.svec %₄₆ core.NamedTuple %₄₈) +50 (call core.svec) +51 SourceLocation::1:1 +52 (call core.svec %₄₉ %₅₀ %₅₁) +53 (call core.define_method TestMod TestMod.f_kw_simple %₅₂ + --- code_info + slots: [slot₁/#kwcall_self#(called) slot₂/kws slot₃/#self# slot₄/a(single_assign)] + 1 1 + 2 (= slot₄/a %₁) + 3 (call slot₁/#kwcall_self# slot₂/kws slot₃/#self# slot₄/a 1.0) + 4 (return %₃) +54 latestworld +55 (call core.typeof core.kwcall) +56 TestMod.f_kw_simple +57 (call core.TypeEqOf %₅₆) +58 TestMod.Int +59 (call core.svec %₅₅ core.NamedTuple %₅₇ %₅₈) +60 (call core.svec) +61 SourceLocation::1:1 +62 (call core.svec %₅₉ %₆₀ %₆₁) +63 (call core.define_method TestMod TestMod.f_kw_simple %₆₂ + --- code_info + slots: [slot₁/#kwcall_self#(called) slot₂/kws slot₃/#self# slot₄/a] + 1 (call slot₁/#kwcall_self# slot₂/kws slot₃/#self# slot₄/a 1.0) + 2 (return %₁) +64 latestworld +65 (call core.typeof core.kwcall) +66 TestMod.f_kw_simple +67 (call core.TypeEqOf %₆₆) +68 TestMod.Int +69 TestMod.Float64 +70 (call core.svec %₆₅ core.NamedTuple %₆₇ %₆₈ %₆₉) +71 (call core.svec) +72 SourceLocation::1:1 +73 (call core.svec %₇₀ %₇₁ %₇₂) +74 (call core.define_method TestMod TestMod.f_kw_simple %₇₃ + --- code_info + slots: [slot₁/#kwcall_self#(!read) slot₂/kws slot₃/#self# slot₄/a slot₅/b slot₆/x(!read) slot₇/y(!read) slot₈/#kwtmp#] + 1 (newvar slot₆/x) + 2 (newvar slot₇/y) + 3 (newvar slot₈/#kwtmp#) + 4 (call core.isdefined slot₂/kws :x) + 5 (gotoifnot %₄ label₁₆) + 6 (call core.getfield slot₂/kws :x) + 7 TestMod.Char + 8 (call core.isa %₆ %₇) + 9 (gotoifnot %₈ label₁₁) + 10 (goto label₁₄) + 11 TestMod.Char + 12 (new core.TypeError :keyword argument :x %₁₁ %₆) + 13 (call core.throw %₁₂) + 14 (= slot₈/#kwtmp# %₆) + 15 (goto label₁₇) + 16 (= slot₈/#kwtmp# 'a') + 17 slot₈/#kwtmp# + 18 (call core.isdefined slot₂/kws :y) + 19 (gotoifnot %₁₈ label₃₀) + 20 (call core.getfield slot₂/kws :y) + 21 TestMod.Bool + 22 (call core.isa %₂₀ %₂₁) + 23 (gotoifnot %₂₂ label₂₅) + 24 (goto label₂₈) + 25 TestMod.Bool + 26 (new core.TypeError :keyword argument :y %₂₅ %₂₀) + 27 (call core.throw %₂₆) + 28 (= slot₈/#kwtmp# %₂₀) + 29 (goto label₃₁) + 30 (= slot₈/#kwtmp# true) + 31 slot₈/#kwtmp# + 32 (call top.keys slot₂/kws) + 33 (call core.tuple :x :y) + 34 (call top.diff_names %₃₂ %₃₃) + 35 (call top.isempty %₃₄) + 36 (gotoifnot %₃₅ label₃₈) + 37 (goto label₃₉) + 38 (call top.kwerr slot₂/kws slot₃/#self# slot₄/a slot₅/b) + 39 TestMod.#f_kw_simple#kw_body#1 + 40 (call %₃₉ %₁₇ %₃₁ slot₃/#self# slot₄/a slot₅/b) + 41 (return %₄₀) +75 latestworld +76 TestMod.f_kw_simple +77 (return %₇₆) + +######################################## +# FIXME: Error: Duplicate keyword placeholder name +# underscore kwargs apart from `_...` are probably not intended to work anyway +function f_kw_placeholders(; _=1, _=2); end +#--------------------- +1 (method TestMod.#kw_body#f_kw_placeholders#0) +2 latestworld +3 (method TestMod.f_kw_placeholders) +4 latestworld +5 TestMod.#kw_body#f_kw_placeholders#0 +6 (call core.TypeEqOf %₅) +7 TestMod.f_kw_placeholders +8 (call core.TypeEqOf %₇) +9 (call core.svec %₆ core.Any core.Any %₈) +10 (call core.svec) +11 SourceLocation::1:10 +12 (call core.svec %₉ %₁₀ %₁₁) +13 --- method TestMod.#kw_body#f_kw_placeholders#0 %₁₂ + slots: [slot₁/#kw_body#f_kw_placeholders#0(!read) slot₂/#unused#(!read) slot₃/#unused#(!read) slot₄/#self#(!read)] + 1 (meta :nkw 2) + 2 (return core.nothing) +14 latestworld +15 TestMod.f_kw_placeholders +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆) +18 (call core.svec) +19 SourceLocation::1:10 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 --- method TestMod.f_kw_placeholders %₂₀ + slots: [slot₁/#self#] + 1 TestMod.#kw_body#f_kw_placeholders#0 + 2 (call %₁ 1 2 slot₁/#self#) + 3 (return %₂) +22 latestworld +23 (call core.typeof core.kwcall) +24 TestMod.f_kw_placeholders +25 (call core.TypeEqOf %₂₄) +26 (call core.svec %₂₃ core.NamedTuple %₂₅) +27 (call core.svec) +28 SourceLocation::1:10 +29 (call core.svec %₂₆ %₂₇ %₂₈) +30 --- method TestMod.f_kw_placeholders %₂₉ + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self# slot₄/kwtmp] + 1 (newvar slot₄/kwtmp) + 2 (call core.isdefined slot₂/kws :_) + 3 (gotoifnot %₂ label₆) + 4 (= slot₄/kwtmp (call core.getfield slot₂/kws :_)) + 5 (goto label₇) + 6 (= slot₄/kwtmp 1) + 7 slot₄/kwtmp + 8 (call core.isdefined slot₂/kws :_) + 9 (gotoifnot %₈ label₁₂) + 10 (= slot₄/kwtmp (call core.getfield slot₂/kws :_)) + 11 (goto label₁₃) + 12 (= slot₄/kwtmp 2) + 13 slot₄/kwtmp + 14 (call top.keys slot₂/kws) + 15 (call core.tuple :_ :_) + 16 (call top.diff_names %₁₄ %₁₅) + 17 (call top.isempty %₁₆) + 18 (gotoifnot %₁₇ label₂₀) + 19 (goto label₂₁) + 20 (call top.kwerr slot₂/kws slot₃/#self#) + 21 TestMod.#kw_body#f_kw_placeholders#0 + 22 (call %₂₁ %₇ %₁₃ slot₃/#self#) + 23 (return %₂₂) +31 latestworld +32 TestMod.f_kw_placeholders +33 (return %₃₂) + +######################################## +# Keyword slurping - simple forwarding of all kws +function f_kw_slurp_simple(; all_kws...) + all_kws +end +#--------------------- +1 (call core.define_method TestMod :#f_kw_slurp_simple#kw_body#2) +2 latestworld +3 (call core.define_method TestMod :f_kw_slurp_simple) +4 latestworld +5 TestMod.#f_kw_slurp_simple#kw_body#2 +6 (call core.TypeEqOf %₅) +7 (call top.pairs core.NamedTuple) +8 TestMod.f_kw_slurp_simple +9 (call core.TypeEqOf %₈) +10 (call core.svec %₆ %₇ %₉) +11 (call core.svec) +12 SourceLocation::1:1 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 (call core.define_method TestMod TestMod.#f_kw_slurp_simple#kw_body#2 %₁₃ + --- code_info + slots: [slot₁/#f_kw_slurp_simple#kw_body#2(!read) slot₂/all_kws slot₃/#self#(!read)] + 1 (meta :nkw 1) + 2 slot₂/all_kws + 3 (return %₂) +15 latestworld +16 TestMod.f_kw_slurp_simple +17 (call core.TypeEqOf %₁₆) +18 (call core.svec %₁₇) +19 (call core.svec) +20 SourceLocation::1:1 +21 (call core.svec %₁₈ %₁₉ %₂₀) +22 (call core.define_method TestMod TestMod.f_kw_slurp_simple %₂₁ + --- code_info + slots: [slot₁/#self#] + 1 TestMod.#f_kw_slurp_simple#kw_body#2 + 2 (call core.NamedTuple) + 3 (call top.pairs %₂) + 4 (call %₁ %₃ slot₁/#self#) + 5 (return %₄) +23 latestworld +24 (call core.typeof core.kwcall) +25 TestMod.f_kw_slurp_simple +26 (call core.TypeEqOf %₂₅) +27 (call core.svec %₂₄ core.NamedTuple %₂₆) +28 (call core.svec) +29 SourceLocation::1:1 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod TestMod.f_kw_slurp_simple %₃₀ + --- code_info + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self#] + 1 (call top.pairs slot₂/kws) + 2 TestMod.#f_kw_slurp_simple#kw_body#2 + 3 (call %₂ %₁ slot₃/#self#) + 4 (return %₃) +32 latestworld +33 TestMod.f_kw_slurp_simple +34 (return %₃₃) + +######################################## +# Keyword slurping +function f_kw_slurp(; x=x_default, non_x_kws...) + all_kws +end +#--------------------- +1 (call core.define_method TestMod :#f_kw_slurp#kw_body#3) +2 latestworld +3 (call core.define_method TestMod :f_kw_slurp) +4 latestworld +5 TestMod.#f_kw_slurp#kw_body#3 +6 (call core.TypeEqOf %₅) +7 (call top.pairs core.NamedTuple) +8 TestMod.f_kw_slurp +9 (call core.TypeEqOf %₈) +10 (call core.svec %₆ core.Any %₇ %₉) +11 (call core.svec) +12 SourceLocation::1:1 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 (call core.define_method TestMod TestMod.#f_kw_slurp#kw_body#3 %₁₃ + --- code_info + slots: [slot₁/#f_kw_slurp#kw_body#3(!read) slot₂/x(!read) slot₃/non_x_kws(!read) slot₄/#self#(!read)] + 1 (meta :nkw 2) + 2 TestMod.all_kws + 3 (return %₂) +15 latestworld +16 TestMod.f_kw_slurp +17 (call core.TypeEqOf %₁₆) +18 (call core.svec %₁₇) +19 (call core.svec) +20 SourceLocation::1:1 +21 (call core.svec %₁₈ %₁₉ %₂₀) +22 (call core.define_method TestMod TestMod.f_kw_slurp %₂₁ + --- code_info + slots: [slot₁/#self#] + 1 TestMod.#f_kw_slurp#kw_body#3 + 2 TestMod.x_default + 3 (call core.NamedTuple) + 4 (call top.pairs %₃) + 5 (call %₁ %₂ %₄ slot₁/#self#) + 6 (return %₅) +23 latestworld +24 (call core.typeof core.kwcall) +25 TestMod.f_kw_slurp +26 (call core.TypeEqOf %₂₅) +27 (call core.svec %₂₄ core.NamedTuple %₂₆) +28 (call core.svec) +29 SourceLocation::1:1 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod TestMod.f_kw_slurp %₃₀ + --- code_info + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self# slot₄/x(!read) slot₅/#kwtmp#] + 1 (newvar slot₄/x) + 2 (newvar slot₅/#kwtmp#) + 3 (call core.isdefined slot₂/kws :x) + 4 (gotoifnot %₃ label₇) + 5 (= slot₅/#kwtmp# (call core.getfield slot₂/kws :x)) + 6 (goto label₉) + 7 TestMod.x_default + 8 (= slot₅/#kwtmp# %₇) + 9 slot₅/#kwtmp# + 10 (call core.tuple :x) + 11 (call core.apply_type core.NamedTuple %₁₀) + 12 (call top.structdiff slot₂/kws %₁₁) + 13 (call top.pairs %₁₂) + 14 TestMod.#f_kw_slurp#kw_body#3 + 15 (call %₁₄ %₉ %₁₃ slot₃/#self#) + 16 (return %₁₅) +32 latestworld +33 TestMod.f_kw_slurp +34 (return %₃₃) + +######################################## +# Keyword slurping with defaults depending on keyword names +# This tests the case where use_ssa_kw_temps=false because a keyword default +# depends on another keyword name. The slurp argument should not be included +# in kw_val_vars to avoid creating an unwanted global binding. +function f_kw_slurp_dep(; a=1, b=a, kws...) + (a, b, kws) +end +#--------------------- +1 (call core.define_method TestMod :#f_kw_slurp_dep#kw_body#4) +2 latestworld +3 (call core.define_method TestMod :f_kw_slurp_dep) +4 latestworld +5 TestMod.#f_kw_slurp_dep#kw_body#4 +6 (call core.TypeEqOf %₅) +7 (call top.pairs core.NamedTuple) +8 TestMod.f_kw_slurp_dep +9 (call core.TypeEqOf %₈) +10 (call core.svec %₆ core.Any core.Any %₇ %₉) +11 (call core.svec) +12 SourceLocation::1:1 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 (call core.define_method TestMod TestMod.#f_kw_slurp_dep#kw_body#4 %₁₃ + --- code_info + slots: [slot₁/#f_kw_slurp_dep#kw_body#4(!read) slot₂/a slot₃/b slot₄/kws slot₅/#self#(!read)] + 1 (meta :nkw 3) + 2 (call core.tuple slot₂/a slot₃/b slot₄/kws) + 3 (return %₂) +15 latestworld +16 TestMod.f_kw_slurp_dep +17 (call core.TypeEqOf %₁₆) +18 (call core.svec %₁₇) +19 (call core.svec) +20 SourceLocation::1:1 +21 (call core.svec %₁₈ %₁₉ %₂₀) +22 (call core.define_method TestMod TestMod.f_kw_slurp_dep %₂₁ + --- code_info + slots: [slot₁/#self# slot₂/a(single_assign) slot₃/b(single_assign)] + 1 1 + 2 (= slot₂/a %₁) + 3 slot₂/a + 4 (= slot₃/b %₃) + 5 TestMod.#f_kw_slurp_dep#kw_body#4 + 6 (call core.NamedTuple) + 7 (call top.pairs %₆) + 8 (call %₅ slot₂/a slot₃/b %₇ slot₁/#self#) + 9 (return %₈) +23 latestworld +24 (call core.typeof core.kwcall) +25 TestMod.f_kw_slurp_dep +26 (call core.TypeEqOf %₂₅) +27 (call core.svec %₂₄ core.NamedTuple %₂₆) +28 (call core.svec) +29 SourceLocation::1:1 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod TestMod.f_kw_slurp_dep %₃₀ + --- code_info + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self# slot₄/#kwtmp# slot₅/a(single_assign) slot₆/b(single_assign)] + 1 (newvar slot₄/#kwtmp#) + 2 (call core.isdefined slot₂/kws :a) + 3 (gotoifnot %₂ label₆) + 4 (= slot₄/#kwtmp# (call core.getfield slot₂/kws :a)) + 5 (goto label₇) + 6 (= slot₄/#kwtmp# 1) + 7 slot₄/#kwtmp# + 8 (= slot₅/a %₇) + 9 (call core.isdefined slot₂/kws :b) + 10 (gotoifnot %₉ label₁₃) + 11 (= slot₄/#kwtmp# (call core.getfield slot₂/kws :b)) + 12 (goto label₁₅) + 13 slot₅/a + 14 (= slot₄/#kwtmp# %₁₃) + 15 slot₄/#kwtmp# + 16 (= slot₆/b %₁₅) + 17 (call core.tuple :a :b) + 18 (call core.apply_type core.NamedTuple %₁₇) + 19 (call top.structdiff slot₂/kws %₁₈) + 20 (call top.pairs %₁₉) + 21 TestMod.#f_kw_slurp_dep#kw_body#4 + 22 (call %₂₁ slot₅/a slot₆/b %₂₀ slot₃/#self#) + 23 (return %₂₂) +32 latestworld +33 TestMod.f_kw_slurp_dep +34 (return %₃₃) + +######################################## +# Static parameters used in keywords, with and without the static parameter +# being present in positional argument types. +# +# Here the wrong type for `b` will get a `TypeError` but `A` will need to rely +# on a MethodError. +function f_kw_sparams(x::X; a::A=a_def, b::X=b_def) where {X,A} + (X,A) +end +#--------------------- +1 (call core.define_method TestMod :#f_kw_sparams#kw_body#5) +2 latestworld +3 (call core.define_method TestMod :f_kw_sparams) +4 latestworld +5 (call core.TypeVar :X) +6 (call core.TypeVar :A) +7 TestMod.#f_kw_sparams#kw_body#5 +8 (call core.TypeEqOf %₇) +9 TestMod.f_kw_sparams +10 (call core.TypeEqOf %₉) +11 (call core.svec %₈ %₆ %₅ %₁₀ %₅) +12 (call core.svec %₅ %₆) +13 SourceLocation::1:1 +14 (call core.svec %₁₁ %₁₂ %₁₃) +15 (call core.define_method TestMod TestMod.#f_kw_sparams#kw_body#5 %₁₄ + --- code_info + slots: [slot₁/#f_kw_sparams#kw_body#5(!read) slot₂/a(!read) slot₃/b(!read) slot₄/#self#(!read) slot₅/x(!read)] + 1 (meta :nkw 2) + 2 static_parameter₁ + 3 static_parameter₂ + 4 (call core.tuple %₂ %₃) + 5 (return %₄) +16 latestworld +17 (call core.TypeVar :X) +18 TestMod.f_kw_sparams +19 (call core.TypeEqOf %₁₈) +20 (call core.svec %₁₉ %₁₇) +21 (call core.svec %₁₇) +22 SourceLocation::1:1 +23 (call core.svec %₂₀ %₂₁ %₂₂) +24 (call core.define_method TestMod TestMod.f_kw_sparams %₂₃ + --- code_info + slots: [slot₁/#self# slot₂/x] + 1 TestMod.#f_kw_sparams#kw_body#5 + 2 TestMod.a_def + 3 TestMod.b_def + 4 (call %₁ %₂ %₃ slot₁/#self# slot₂/x) + 5 (return %₄) +25 latestworld +26 (call core.TypeVar :X) +27 (call core.typeof core.kwcall) +28 TestMod.f_kw_sparams +29 (call core.TypeEqOf %₂₈) +30 (call core.svec %₂₇ core.NamedTuple %₂₉ %₂₆) +31 (call core.svec %₂₆) +32 SourceLocation::1:1 +33 (call core.svec %₃₀ %₃₁ %₃₂) +34 (call core.define_method TestMod TestMod.f_kw_sparams %₃₃ + --- code_info + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self# slot₄/x slot₅/a(!read) slot₆/b(!read) slot₇/#kwtmp#] + 1 (newvar slot₅/a) + 2 (newvar slot₆/b) + 3 (newvar slot₇/#kwtmp#) + 4 (call core.isdefined slot₂/kws :a) + 5 (gotoifnot %₄ label₈) + 6 (= slot₇/#kwtmp# (call core.getfield slot₂/kws :a)) + 7 (goto label₁₀) + 8 TestMod.a_def + 9 (= slot₇/#kwtmp# %₈) + 10 slot₇/#kwtmp# + 11 (call core.isdefined slot₂/kws :b) + 12 (gotoifnot %₁₁ label₂₃) + 13 (call core.getfield slot₂/kws :b) + 14 static_parameter₁ + 15 (call core.isa %₁₃ %₁₄) + 16 (gotoifnot %₁₅ label₁₈) + 17 (goto label₂₁) + 18 static_parameter₁ + 19 (new core.TypeError :keyword argument :b %₁₈ %₁₃) + 20 (call core.throw %₁₉) + 21 (= slot₇/#kwtmp# %₁₃) + 22 (goto label₂₅) + 23 TestMod.b_def + 24 (= slot₇/#kwtmp# %₂₃) + 25 slot₇/#kwtmp# + 26 (call top.keys slot₂/kws) + 27 (call core.tuple :a :b) + 28 (call top.diff_names %₂₆ %₂₇) + 29 (call top.isempty %₂₈) + 30 (gotoifnot %₂₉ label₃₂) + 31 (goto label₃₃) + 32 (call top.kwerr slot₂/kws slot₃/#self# slot₄/x) + 33 TestMod.#f_kw_sparams#kw_body#5 + 34 (call %₃₃ %₁₀ %₂₅ slot₃/#self# slot₄/x) + 35 (return %₃₄) +35 latestworld +36 TestMod.f_kw_sparams +37 (return %₃₆) + +######################################## +# Keyword @nospecialize +function f_kw_slurp(a,;kw1,kw2=2,restkw...) + @nospecialize +end +#--------------------- +1 (call core.define_method TestMod :#f_kw_slurp#kw_body#6) +2 latestworld +3 (call core.define_method TestMod :f_kw_slurp) +4 latestworld +5 TestMod.#f_kw_slurp#kw_body#6 +6 (call core.TypeEqOf %₅) +7 (call top.pairs core.NamedTuple) +8 TestMod.f_kw_slurp +9 (call core.TypeEqOf %₈) +10 (call core.svec %₆ core.Any core.Any %₇ %₉ core.Any) +11 (call core.svec) +12 SourceLocation::1:1 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 (call core.define_method TestMod TestMod.#f_kw_slurp#kw_body#6 %₁₃ + --- code_info + slots: [slot₁/#f_kw_slurp#kw_body#6(!read) slot₂/kw1(nospecialize,!read) slot₃/kw2(nospecialize,!read) slot₄/restkw(nospecialize,!read) slot₅/#self#(!read) slot₆/a(nospecialize,!read)] + 1 (meta :nospecialize slot₂/kw1 slot₃/kw2 slot₄/restkw slot₆/a) + 2 (meta :nkw 3) + 3 (return core.nothing) +15 latestworld +16 TestMod.f_kw_slurp +17 (call core.TypeEqOf %₁₆) +18 (call core.svec %₁₇ core.Any) +19 (call core.svec) +20 SourceLocation::1:1 +21 (call core.svec %₁₈ %₁₉ %₂₀) +22 (call core.define_method TestMod TestMod.f_kw_slurp %₂₁ + --- code_info + slots: [slot₁/#self# slot₂/a(nospecialize)] + 1 (meta :nospecialize) + 2 TestMod.#f_kw_slurp#kw_body#6 + 3 (call core.UndefKeywordError :kw1) + 4 (call core.throw %₃) + 5 (call core.NamedTuple) + 6 (call top.pairs %₅) + 7 (call %₂ %₄ 2 %₆ slot₁/#self# slot₂/a) + 8 (return %₇) +23 latestworld +24 (call core.typeof core.kwcall) +25 TestMod.f_kw_slurp +26 (call core.TypeEqOf %₂₅) +27 (call core.svec %₂₄ core.NamedTuple %₂₆ core.Any) +28 (call core.svec) +29 SourceLocation::1:1 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod TestMod.f_kw_slurp %₃₀ + --- code_info + slots: [slot₁/#unused#(!read) slot₂/kws slot₃/#self# slot₄/a(nospecialize) slot₅/kw1(!read) slot₆/kw2(!read) slot₇/#kwtmp#] + 1 (meta :nospecialize slot₄/a) + 2 (newvar slot₅/kw1) + 3 (newvar slot₆/kw2) + 4 (newvar slot₇/#kwtmp#) + 5 (call core.isdefined slot₂/kws :kw1) + 6 (gotoifnot %₅ label₉) + 7 (= slot₇/#kwtmp# (call core.getfield slot₂/kws :kw1)) + 8 (goto label₁₁) + 9 (call core.UndefKeywordError :kw1) + 10 (= slot₇/#kwtmp# (call core.throw %₉)) + 11 slot₇/#kwtmp# + 12 (call core.isdefined slot₂/kws :kw2) + 13 (gotoifnot %₁₂ label₁₆) + 14 (= slot₇/#kwtmp# (call core.getfield slot₂/kws :kw2)) + 15 (goto label₁₇) + 16 (= slot₇/#kwtmp# 2) + 17 slot₇/#kwtmp# + 18 (call core.tuple :kw1 :kw2) + 19 (call core.apply_type core.NamedTuple %₁₈) + 20 (call top.structdiff slot₂/kws %₁₉) + 21 (call top.pairs %₂₀) + 22 TestMod.#f_kw_slurp#kw_body#6 + 23 (call %₂₂ %₁₁ %₁₇ %₂₁ slot₃/#self# slot₄/a) + 24 (return %₂₃) +32 latestworld +33 TestMod.f_kw_slurp +34 (return %₃₃) + +######################################## +# Error: argument unpacking in keywords +function f_kw_destruct(; (x,y)=10) +end +#--------------------- +LoweringError: +function f_kw_destruct(; (x,y)=10) +# └───┘ ── expected identifier or `identifier::type` +end + +######################################## +# Error: keyword slurping combined with a default +function f_kw_slurp_default(; kws...=def) +end +#--------------------- +LoweringError: +function f_kw_slurp_default(; kws...=def) +# └────┘ ── expected identifier or `identifier::type` +end + +######################################## +# Error: keyword slurping combined with type +function f_kw_slurp_type(; kws::T...) +end +#--------------------- +LoweringError: +function f_kw_slurp_type(; kws::T...) +# └────┘ ── keyword parameter with `...` may not be given a type +end + +######################################## +# Error: keyword slurping on non-final argument +function f_kw_slurp_not_last(; kws..., x=1) +end +#--------------------- +LoweringError: +function f_kw_slurp_not_last(; kws..., x=1) +# └────┘ ── `...` may only be used for the final keyword parameter +end + +######################################## +# Error: if-generated without else +function foo() + if @generated + 1 + end +end +#--------------------- +LoweringError: +function foo() +# ┌──────────── + if @generated + 1 + end +#─────┘ ── if-generated requires both true and false cases +end + +######################################## +# Fully generated function +@generated function f_only_generated(x, y) + generator_code(x,y) +end +#--------------------- +1 (call core.define_method TestMod :f_only_generated) +2 latestworld +3 (call core.declare_global TestMod :#f_only_generated@generator##0 false) +4 latestworld +5 (call core.define_method TestMod :#f_only_generated@generator##0) +6 latestworld +7 TestMod.#f_only_generated@generator##0 +8 (call core.TypeEqOf %₇) +9 (call core.svec %₈ JuliaSyntax.SyntaxContext core.Any core.Any core.Any) +10 (call core.svec) +11 SourceLocation::1:1 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod TestMod.#f_only_generated@generator##0 %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/#self#(nospecialize,!read) slot₄/x(nospecialize) slot₅/y(nospecialize)] + 1 (meta :nospecialize slot₃/#self# slot₄/x slot₅/y) + 2 TestMod.generator_code + 3 (call %₂ slot₄/x slot₅/y) + 4 (call core.tuple %₃) + 5 (call JuliaLowering.interpolate_syntax (syntaxinert (block (syntaxunquote (block (call generator_code x y))))) %₄) + 6 (return %₅) +14 latestworld +15 TestMod.f_only_generated +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_only_generated %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/y(!read)] + 1 (meta :generated (new JuliaLowering.GeneratedFunctionStub SyntaxContext(#=omitted=#) TestMod.#f_only_generated@generator##0 SourceRef::1:1 (call core.svec :#self# :x :y) (call core.svec))) + 2 (meta :generated_only) + 3 (return core.nothing) +22 latestworld +23 TestMod.f_only_generated +24 (return %₂₃) + +######################################## +# Partially generated function with `if @generated` +function f_partially_generated(x, y) + nongen_stuff = bothgen(x, y) + if @generated + quote + maybe_gen_stuff = some_gen_stuff(x, y) + end + else + maybe_gen_stuff = some_nongen_stuff(x, y) + end + (nongen_stuff, maybe_gen_stuff) +end +#--------------------- +1 (call core.define_method TestMod :f_partially_generated) +2 latestworld +3 (call core.declare_global TestMod :#f_partially_generated@generator##0 false) +4 latestworld +5 (call core.define_method TestMod :#f_partially_generated@generator##0) +6 latestworld +7 TestMod.#f_partially_generated@generator##0 +8 (call core.TypeEqOf %₇) +9 (call core.svec %₈ JuliaSyntax.SyntaxContext core.Any core.Any core.Any) +10 (call core.svec) +11 SourceLocation::1:1 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod TestMod.#f_partially_generated@generator##0 %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/#self#(nospecialize,!read) slot₄/x(nospecialize,!read) slot₅/y(nospecialize,!read)] + 1 (meta :nospecialize slot₃/#self# slot₄/x slot₅/y) + 2 (call JuliaLowering.interpolate_expr (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))) + 3 (call core.tuple %₂) + 4 (call JuliaLowering.interpolate_syntax (syntaxinert (block (= nongen_stuff (call bothgen x y)) (syntaxunquote (block (call JuliaLowering.interpolate_expr (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple nongen_stuff maybe_gen_stuff))) %₃) + 5 (return %₄) +14 latestworld +15 TestMod.f_partially_generated +16 (call core.TypeEqOf %₁₅) +17 (call core.svec %₁₆ core.Any core.Any) +18 (call core.svec) +19 SourceLocation::1:1 +20 (call core.svec %₁₇ %₁₈ %₁₉) +21 (call core.define_method TestMod TestMod.f_partially_generated %₂₀ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/y slot₄/maybe_gen_stuff(single_assign) slot₅/nongen_stuff(single_assign)] + 1 (meta :generated (new JuliaLowering.GeneratedFunctionStub SyntaxContext(#=omitted=#) TestMod.#f_partially_generated@generator##0 SourceRef::1:37 (call core.svec :#self# :x :y) (call core.svec))) + 2 TestMod.bothgen + 3 (= slot₅/nongen_stuff (call %₂ slot₂/x slot₃/y)) + 4 TestMod.some_nongen_stuff + 5 (= slot₄/maybe_gen_stuff (call %₄ slot₂/x slot₃/y)) + 6 slot₅/nongen_stuff + 7 slot₄/maybe_gen_stuff + 8 (call core.tuple %₆ %₇) + 9 (return %₈) +22 latestworld +23 TestMod.f_partially_generated +24 (return %₂₃) + +######################################## +# Error: juxtapose-assignment +4a = 1 +#--------------------- +LoweringError: +4a = 1 +╙ ── expected identifier or `identifier::type` diff --git a/JuliaLowering/test/generators.jl b/JuliaLowering/test/generators.jl new file mode 100644 index 0000000000000..bce04bde7d7ea --- /dev/null +++ b/JuliaLowering/test/generators.jl @@ -0,0 +1,305 @@ +test_mod = Module() + +@test JuliaLowering.include_string(test_mod, """ +collect(x^2 for x in 1:3) +""") == [1,4,9] + +@test JuliaLowering.include_string(test_mod, """ +collect(x for x in 1:5 if isodd(x)) +""") == [1,3,5] + +@test JuliaLowering.include_string(test_mod, """ +collect((y,x) for (x,y) in zip(1:3, 2:4) if y != 3) +""") == [(2,1), (4,3)] + +# product iterator +@test JuliaLowering.include_string(test_mod, """ +collect((x,y) for x in 1:3, y in 1:2) +""") == [(1,1) (1,2) + (2,1) (2,2) + (3,1) (3,2)] + +# flattened iterator +@test JuliaLowering.include_string(test_mod, """ +collect((x,y,z) for x in 1:3, y in 4:5 for z in 6:7) +""") == [ + (1,4,6) + (1,4,7) + (2,4,6) + (2,4,7) + (3,4,6) + (3,4,7) + (1,5,6) + (1,5,7) + (2,5,6) + (2,5,7) + (3,5,6) + (3,5,7) +] + +@testset "duplicate vars" for jeval_s in [ + s->JuliaLowering.include_string(test_mod, "[ $s ]"), + s->JuliaLowering.include_string(test_mod, "collect( $s )")] + + # Duplicate iteration variables - body sees only innermost + @test jeval_s("x for x in 1:3 for x in 10:14") == + [10,11,12,13,14,10,11,12,13,14,10,11,12,13,14,] + # Duplicate in same "for" + @test jeval_s("x for x = 1:3, x = 10:14") == + [10 11 12 13 14 ; 10 11 12 13 14 ; 10 11 12 13 14 ; ] + + # Same as above, but with filter + @test jeval_s("x for x in 1:3 for x in 10:14 if iseven(x)") == + [10,12,14,10,12,14,10,12,14,] + @test jeval_s("x for x = 1:3, x = 10:14 if iseven(x)") == + [10,10,10,12,12,12,14,14,14] + + @test jeval_s("x for x in 1:3 if iseven(x) for x in 10:14") == + [10,11,12,13,14,] +end + +# Outer iteration variables are protected from mutation +@test JuliaLowering.include_string(test_mod, """ +collect((z=y; y=100; z) for y in 1:3 for x in 1:2) +""") == [1, 1, 2, 2, 3, 3] + +# Simple typed comprehension lowered to for loops +@test JuliaLowering.include_string(test_mod, """ +Tuple{Int,Int}[(x,y) for x in 1:2, y in 1:3] +""") == [(1,1) (1,2) (1,3) + (2,1) (2,2) (2,3)] + +# Triply nested comprehension +@test JuliaLowering.include_string(test_mod, """ +[(x,y,z) for x in 1:3 for y in 4:5 for z in 6:7] +""") == [ + (1, 4, 6) + (1, 4, 7) + (1, 5, 6) + (1, 5, 7) + (2, 4, 6) + (2, 4, 7) + (2, 5, 6) + (2, 5, 7) + (3, 4, 6) + (3, 4, 7) + (3, 5, 6) + (3, 5, 7) +] + +# splat in lhs +@test JuliaLowering.include_string(test_mod, """ +[(h, i, j) for (h, i..., j) in ((1,2,3,4),(5,6,7,8))] +""") == [(1,(2,3),4) ; (5,(6,7),8)] +@test JuliaLowering.include_string(test_mod, """ +collect((h, i, j) for (h, i..., j) in ((1,2,3,4),(5,6,7,8))) +""") == [(1,(2,3),4) ; (5,(6,7),8)] + +# bad splat in iterable expression (pkgeval Lerche, MultidimensionalTools): a +# splat happens to work because of the way `xs...` is placed directly into +# `(call Generator ...)` in desugaring, which throws a MethodError if `xs` has +# more than one element. +@test JuliaLowering.include_string(test_mod, """ + func_generator_splat(xs...) = [2i for i in xs...] + func_generator_splat([1, 2]) +""") == [2,4] +@test JuliaLowering.include_string(test_mod, """ + func_comprehension_splat(xs...) = collect(2i for i in xs...) + func_comprehension_splat([1, 2]) +""") == [2,4] + +@testset "generators: issue 18621" begin + @test JuliaLowering.include_string(test_mod, """ + function g18621() + f = function(i) + f = (i)->2 + 1 + end + [f(i) for i in 1:3] + end + g18621() + """) == [1,2,2] + @test JuliaLowering.include_string(test_mod, """ + global f18621 = function(i) + global f18621 = (i) -> 2 + 1 + end + [f18621(i) for i in 1:3] + """) == [1,2,2] + @test JuliaLowering.include_string(test_mod, """ + function h18621() + g = (k(i) for i in 1:5) + k = identity + return collect(g) + end + h18621() + """) == 1:5 +end + +@testset "generator iteration variables are local" begin + @test JuliaLowering.include_string(test_mod, """ + let x = 0 + ys = [x for x::Int in 1:3] + (x, ys) + end + """) == (0, [1,2,3]) + @test JuliaLowering.include_string(test_mod, """ + let x = collect(1:3) + ys = [x for (i, x) in enumerate(x)] + (x, ys) + end + """) == (collect(1:3), [1,2,3]) +end + +@testset "compat: comprehension with non-generator arg" begin + let ex = Expr(:comprehension, :i, Expr(:(=), :i, Expr(:call, :(:), 1, 3))) + @test jl_eval(test_mod, ex) == fl_eval(test_mod, ex) + end +end + +@testset "(AI) correct types" begin + local jeval(str) = jl_eval(test_mod, parsestmt(SyntaxTree, str)) + local feval(str) = fl_eval(test_mod, parsestmt(Expr, str)) + local same_type(str) = jeval(str) == feval(str) + let s = "[(a,b) for (a,b) in [[1,2],[3,4]]]" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "[(k,v) for (k,v) in Dict(1=>10)]" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "collect((k,v) for (k,v) in pairs([10,20]))" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "[(a,) for (a,) in 1:3]" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "[(a,) for (a,) in \"ab\"]" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "collect(a+1 for a::Int in \"ab\")" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "[x for x::Float64 in [1,2,3]]" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "collect(x+1 for x::Int in Real[1.0, 2.0])" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "(r=Int[]; for a::Int in \"ab\"; push!(r, a+1); end; r)" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "Int[a+1 for a::Int in \"ab\"]" + @test same_type(s) + @test jeval(s) == feval(s) + end + let s = "collect(b for (a,b,c) in [(1,2,3)], c::Float64 in 10:11)" + @test same_type(s) + @test jeval(s) == feval(s) + end +end + +@testset "placeholders" begin + local jeval(str) = jl_eval(test_mod, parsestmt(SyntaxTree, str)) + local feval(str) = fl_eval(test_mod, parsestmt(Expr, str)) + + @test jeval(""" + let arr = [1,2,3] + [1 for _ in push!(arr, 4)] + end + """) == [1,1,1,1] + @test jeval(""" + let arr = [1,2,3] + [1 for _::Float64 in push!(arr, 4)] + end + """) == [1,1,1,1] + @test jeval(""" + let arr = [1,2,3] + [1 for _ in push!(arr, 4), _ in push!(arr, 5)] + end + """) == ones(Int, 5, 5) + # Pushes forever in both lowering implementations + # @test jeval(""" + # let arr = [1,2,3] + # [1 for _ in push!(arr, 4) for _ in push!(arr, 5)] + # end + # """) + @test jeval(""" + let arr = [(1,1),(2,2),(3,3)] + [1 for (_,_) in push!(arr, (4,4)), (_,_) in push!(arr, (5,5))] + end + """) == ones(Int, 5, 5) + @test jeval(""" + let arr = [(1,1),(2,2),(3,3,3,3)] + [1 for (_,_...) in push!(arr, (4,4)), (_...,_) in push!(arr, (5,5))] + end + """) == ones(Int, 5, 5) +end + +@testset "generators: `[_ for _ in rhs]`, `[f(_) for _ in rhs]` compat (#18621)" begin + local test_mod = @newmod() + local jeval(str) = jl_eval(test_mod, parsestmt(SyntaxTree, str); expr_compat_mode=true) + local jnew(str) = jl_eval(test_mod, parsestmt(SyntaxTree, str)) + local feval(str) = fl_eval(test_mod, parsestmt(Expr, str)) + JuliaLowering.include_string(test_mod, "f(x) = x; f(x, y) = x; g(x) = x") + + for s in ("[_ for _ in 1:3]", + "collect(_ for _ in 1:3)", + "[f(_) for _ in 1:3]", + "[(f)(__) for __ in 1:3]", + "[identity(_) for _ in 1:3]", + ) + @test jeval(s) == feval(s) context=s + @test_throws LoweringError jnew(s) # this behaviour is deprecated + end + + # Intentionally left out for now + for s in ("[f(_) for _ in 1:3 if isodd(_)]", + "[f(_) for _ in 1:2 for _ in 1:2]", + "[_ for _ in 1:2 for _ in 1:2]", + "[f(_) for a in 1:2 for _ in 1:2]", + "[f(_) for _ in 1:3 if isodd(_) for _ in 1:2]") + @test_broken jeval(s) == feval(s) context=s + @test_throws LoweringError jnew(s) + end + + # Rejected by both + for s in ("[f(_, _) for _ in 1:3]", # >1 argument + "[f(identity(_)) for _ in 1:3]", # arg is not exactly `_` + "[f(_)(_) for _ in 1:3]", # callee contains `_` + "[_(_) for _ in 1:3]", # callee *is* `_` + "[.+(_) for _ in 1:3]", # dot-operator callee + "[sqrt.(_) for _ in 1:3]", # broadcast, not a `call` + "[f(_; k=_) for _ in 1:3]", # keyword arg reads `_` + "[f(_) for _ in 1:2, _ in 1:2]", # comma product (two `_`) + "[f(_) for _ in 1:3 if _ > 0]", # non-reducible filter read + "[f(_) for _ in 1:2 for b in 1:2]") # body reads an outer `_` + @test_throws LoweringError jeval(s) context=s + @test_throws LoweringError jnew(s) + end + + @test (feval("(_ for _ in 1:3)")).f === Base.identity + @test (jeval("(_ for _ in 1:3)")).f === Base.identity + @test (feval("(f(_) for _ in 1:3)")).f === test_mod.f + @test (jeval("(f(_) for _ in 1:3)")).f === test_mod.f +end + +@testset "a comprehension may contain a non-syntactic generator" begin + ex = Expr(:comprehension, + Expr(:call, GlobalRef(Base, :Generator), :(i -> 2i), :(1:3))) + @test fl_eval(test_mod, ex) == [2,4,6] + @test jl_eval(test_mod, ex) == [2,4,6] + + ex = Expr(:typed_comprehension, Int, + Expr(:call, GlobalRef(Base, :Generator), :(i -> 2i), :(1:3))) + @test fl_eval(test_mod, ex) == Int[2,4,6] + @test jl_eval(test_mod, ex) == Int[2,4,6] +end diff --git a/JuliaLowering/test/generators_ir.jl b/JuliaLowering/test/generators_ir.jl new file mode 100644 index 0000000000000..6d680a91cc48d --- /dev/null +++ b/JuliaLowering/test/generators_ir.jl @@ -0,0 +1,358 @@ +######################################## +# Simple 1D generator +(x+1 for x in xs) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#1#2 %₁ %₂ %₃) +5 latestworld +6 TestMod.#1#2 +7 (new %₆) +8 TestMod.#1#2 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:2 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x] + 1 TestMod.+ + 2 (call %₁ slot₂/x 1) + 3 (return %₂) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 TestMod.xs +18 (call top.Generator %₁₆ %₁₇) +19 (return %₁₈) + +######################################## +# Product iteration +(x+y for x in xs, y in ys) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#3#4 %₁ %₂ %₃) +5 latestworld +6 TestMod.#3#4 +7 (new %₆) +8 TestMod.#3#4 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:2 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#generator# slot₃/iterstate(single_assign) slot₄/x(single_assign) slot₅/y(single_assign)] + 1 (call top.indexed_iterate slot₂/#generator# 1) + 2 (= slot₄/x (call core.getfield %₁ 1)) + 3 (= slot₃/iterstate (call core.getfield %₁ 2)) + 4 slot₃/iterstate + 5 (call top.indexed_iterate slot₂/#generator# 2 %₄) + 6 (= slot₅/y (call core.getfield %₅ 1)) + 7 TestMod.+ + 8 slot₄/x + 9 slot₅/y + 10 (call %₇ %₈ %₉) + 11 (return %₁₀) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 TestMod.xs +18 TestMod.ys +19 (call top.product %₁₇ %₁₈) +20 (call top.Generator %₁₆ %₁₉) +21 (return %₂₀) + +######################################## +# Use `identity` as the Generator function when possible eg in filters +((x,y) for (x,y) in iter if f(x)) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#5#6 %₁ %₂ %₃) +5 latestworld +6 TestMod.#5#6 +7 (new %₆) +8 TestMod.#5#6 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:2 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#generator# slot₃/iterstate(single_assign) slot₄/x(single_assign) slot₅/y(single_assign)] + 1 (call top.indexed_iterate slot₂/#generator# 1) + 2 (= slot₄/x (call core.getfield %₁ 1)) + 3 (= slot₃/iterstate (call core.getfield %₁ 2)) + 4 slot₃/iterstate + 5 (call top.indexed_iterate slot₂/#generator# 2 %₄) + 6 (= slot₅/y (call core.getfield %₅ 1)) + 7 slot₄/x + 8 slot₅/y + 9 (call core.tuple %₇ %₈) + 10 (return %₉) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 (call core.svec) +18 (call core.svec) +19 (call core.svec) +20 (call JuliaLowering.eval_closure_type TestMod :#7#8 %₁₇ %₁₈ %₁₉) +21 latestworld +22 TestMod.#7#8 +23 (new %₂₂) +24 TestMod.#7#8 +25 (call core.svec %₂₄ core.Any) +26 (call core.svec) +27 SourceLocation::1:29 +28 (call core.svec %₂₅ %₂₆ %₂₇) +29 (call core.define_method TestMod core.nothing %₂₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#generator# slot₃/iterstate(single_assign) slot₄/x(single_assign) slot₅/y(!read,single_assign)] + 1 (call top.indexed_iterate slot₂/#generator# 1) + 2 (= slot₄/x (call core.getfield %₁ 1)) + 3 (= slot₃/iterstate (call core.getfield %₁ 2)) + 4 slot₃/iterstate + 5 (call top.indexed_iterate slot₂/#generator# 2 %₄) + 6 (= slot₅/y (call core.getfield %₅ 1)) + 7 TestMod.f + 8 slot₄/x + 9 (call %₇ %₈) + 10 (return %₉) +30 latestworld +31 (= slot₂/#-># %₂₃) +32 slot₂/#-># +33 TestMod.iter +34 (call top.Filter %₃₂ %₃₃) +35 (call top.Generator %₁₆ %₃₄) +36 (return %₃₅) + +######################################## +# Use of placeholders in iteration vars +(1 for _ in xs) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#9#10 %₁ %₂ %₃) +5 latestworld +6 TestMod.#9#10 +7 (new %₆) +8 TestMod.#9#10 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:2 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read)] + 1 (return 1) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 TestMod.xs +18 (call top.Generator %₁₆ %₁₇) +19 (return %₁₈) + +######################################## +# Error: Use of placeholders in body +(_ for _ in xs) +#--------------------- +LoweringError: +(_ for _ in xs) +#╙ ── all-underscore identifiers are write-only and their values cannot be used in expressions + +######################################## +# 1D generator with destructuring +(body for (x,_,y) in iter) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#11#12 %₁ %₂ %₃) +5 latestworld +6 TestMod.#11#12 +7 (new %₆) +8 TestMod.#11#12 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:2 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#generator# slot₃/iterstate slot₄/x(!read,single_assign) slot₅/y(!read,single_assign)] + 1 (call top.indexed_iterate slot₂/#generator# 1) + 2 (= slot₄/x (call core.getfield %₁ 1)) + 3 (= slot₃/iterstate (call core.getfield %₁ 2)) + 4 slot₃/iterstate + 5 (call top.indexed_iterate slot₂/#generator# 2 %₄) + 6 (call core.getfield %₅ 1) + 7 (= slot₃/iterstate (call core.getfield %₅ 2)) + 8 slot₃/iterstate + 9 (call top.indexed_iterate slot₂/#generator# 3 %₈) + 10 (= slot₅/y (call core.getfield %₉ 1)) + 11 TestMod.body + 12 (return %₁₁) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 TestMod.iter +18 (call top.Generator %₁₆ %₁₇) +19 (return %₁₈) + +######################################## +# return permitted in quoted syntax in generator +(:(return x) for _ in iter) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#13#14 %₁ %₂ %₃) +5 latestworld +6 TestMod.#13#14 +7 (new %₆) +8 TestMod.#13#14 +9 (call core.svec %₈ core.Any) +10 (call core.svec) +11 SourceLocation::1:4 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read) slot₂/#unused#(!read)] + 1 (call JuliaLowering.interpolate_expr (inert (return x))) + 2 (return %₁) +14 latestworld +15 (= slot₁/#-># %₇) +16 slot₁/#-># +17 TestMod.iter +18 (call top.Generator %₁₆ %₁₇) +19 (return %₁₈) + +######################################## +# Error: `return` not permitted in generator body +((return x) + y for x in iter) +#--------------------- +LoweringError: +((return x) + y for x in iter) +# └──────┘ ── `return` not allowed inside comprehension or generator + +######################################## +# Nested case with duplicate iteration variables +(x for x in 1:3 for x in 1:2) +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#15#16 %₁ %₂ %₃) +5 latestworld +6 TestMod.#15#16 +7 (new %₆) +8 (call core.svec) +9 (call core.svec) +10 (call core.svec) +11 (call JuliaLowering.eval_closure_type TestMod :#17#18 %₈ %₉ %₁₀) +12 latestworld +13 TestMod.#17#18 +14 (call core.svec %₁₃ core.Any) +15 (call core.svec) +16 SourceLocation::1:2 +17 (call core.svec %₁₄ %₁₅ %₁₆) +18 (call core.define_method TestMod core.nothing %₁₇ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/x(single_assign)] + 1 slot₂/x + 2 (= slot₃/x %₁) + 3 slot₃/x + 4 (return %₃) +19 latestworld +20 TestMod.#15#16 +21 (call core.svec %₂₀ core.Any) +22 (call core.svec) +23 SourceLocation::1:2 +24 (call core.svec %₂₁ %₂₂ %₂₃) +25 (call core.define_method TestMod core.nothing %₂₄ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read) slot₃/#->#(single_assign)] + 1 TestMod.#17#18 + 2 (new %₁) + 3 (= slot₃/#-># %₂) + 4 slot₃/#-># + 5 TestMod.: + 6 (call %₅ 1 2) + 7 (call top.Generator %₄ %₆) + 8 (return %₇) +26 latestworld +27 (= slot₁/#-># %₇) +28 slot₁/#-># +29 TestMod.: +30 (call %₂₉ 1 3) +31 (call top.Generator %₂₈ %₃₀) +32 (call top.Flatten %₃₁) +33 (return %₃₂) + +######################################## +# Comprehension lowers to generator with collect +[x for x in xs] +#--------------------- +1 TestMod.xs +2 (call top.Generator top.identity %₁) +3 (call top.collect %₂) +4 (return %₃) + +######################################## +# Simple typed comprehension lowers to for loop +T[(x,y) for x in xs, y in ys] +#--------------------- +1 TestMod.xs +2 TestMod.ys +3 (call top.product %₁ %₂) +4 (call top.IteratorSize %₃) +5 (call core.isa %₄ top.SizeUnknown) +6 TestMod.T +7 (call top._array_for %₆ %₃ %₄) +8 (call top.LinearIndices %₇) +9 (= slot₁/idx (call top.first %₈)) +10 (= slot₃/next (call top.iterate %₂)) +11 slot₃/next +12 (call core.=== %₁₁ core.nothing) +13 (call top.not_int %₁₂) +14 (gotoifnot %₁₃ label₄₉) +15 slot₃/next +16 (= slot₄/y (call core.getfield %₁₅ 1)) +17 (call core.getfield %₁₅ 2) +18 (= slot₂/next (call top.iterate %₁)) +19 slot₂/next +20 (call core.=== %₁₉ core.nothing) +21 (call top.not_int %₂₀) +22 (gotoifnot %₂₁ label₄₃) +23 slot₄/y +24 (= slot₅/y %₂₃) +25 slot₂/next +26 (= slot₆/x (call core.getfield %₂₅ 1)) +27 (call core.getfield %₂₅ 2) +28 slot₆/x +29 (call core.tuple %₂₈ slot₅/y) +30 (gotoifnot %₅ label₃₃) +31 (call top.push! %₇ %₂₉) +32 (goto label₃₅) +33 slot₁/idx +34 (call top.setindex! %₇ %₂₉ %₃₃) +35 slot₁/idx +36 (= slot₁/idx (call top.add_int %₃₅ 1)) +37 (= slot₂/next (call top.iterate %₁ %₂₇)) +38 slot₂/next +39 (call core.=== %₃₈ core.nothing) +40 (call top.not_int %₃₉) +41 (gotoifnot %₄₀ label₄₃) +42 (goto label₂₃) +43 (= slot₃/next (call top.iterate %₂ %₁₇)) +44 slot₃/next +45 (call core.=== %₄₄ core.nothing) +46 (call top.not_int %₄₅) +47 (gotoifnot %₄₆ label₄₉) +48 (goto label₁₅) +49 (return %₇) diff --git a/JuliaLowering/test/hooks.jl b/JuliaLowering/test/hooks.jl new file mode 100644 index 0000000000000..c7a28e53d4ae5 --- /dev/null +++ b/JuliaLowering/test/hooks.jl @@ -0,0 +1,159 @@ +@testset "hooks" begin + test_mod = Module() + + @testset "`core_lowering_hook`" begin + # Non-AST types are often sent through lowering + stuff = Any[LineNumberNode(1), 123, 123.123, true, "foo", test_mod] + for s in stuff + @test JL.core_lowering_hook(s, test_mod) == Core.svec(s) + end + + for ast_type in (Expr, JL.SyntaxTree) + ex = parsestmt(ast_type, "[1,2,3] .+= 1") + out = JL.core_lowering_hook(ex, test_mod) + @test out isa Core.SimpleVector && out[1] isa Expr + val = Core.eval(test_mod, out[1]) + @test val == [2,3,4] + end + + # file argument mismatch with embedded linenumbernodes shouldn't crash + ex = Expr(:block, LineNumberNode(111), :(x = 1), LineNumberNode(222), :(x + 1)) + lwr = JuliaLowering.core_lowering_hook(ex, test_mod, "foo.jl", 333)[1] + @test Core.eval(test_mod, lwr) === 2 + end + + function jeval(str) + prog = parseall(Expr, str) + try + JL.activate!() + return Core.eval(test_mod, prog) + finally + JL.activate!(false) + end + end + @testset "integration: `JuliaLowering.activate!`" begin + out = jeval("global asdf = 1") + @test out === 1 + @test isdefined(test_mod, :asdf) + + out = jeval("module M; x = 1; end") + @test out isa Module + @test isdefined(test_mod, :M) + @test isdefined(test_mod.M, :x) + + @test jeval("@ccall jl_value_ptr(nothing::Any)::Ptr{Cvoid}") isa Ptr{Cvoid} + + # Tricky cases with symbols + out = jeval("""module M2 + Base.@constprop :aggressive function f(x); x; end + const what = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), Core.nothing) + end""") + @test out isa Module + @test isdefined(test_mod, :M2) + @test isdefined(test_mod.M2, :f) + @test isdefined(test_mod.M2, :what) + + out = jeval(""" "docstring" module M3 end """) + @test out isa Module + @test isdefined(test_mod, :M3) + + # Macros may produce toplevel expressions. Note that julia handles + # this case badly (macro expansion replaces M5_inner with a + # globalref) and we handle esc(:M5_inner) badly + out = jeval("""module M5 + macro newmod() + return quote + let a = 1 + $(Expr(:toplevel, + Expr(:module, true, :M5_inner, + Expr(:block, :(global asdf = 1))))) + end + end + end + @newmod() + end""") + @test out isa Module + @test isdefined(test_mod, :M5) + @test isdefined(test_mod.M5, :M5_inner) + @test isdefined(test_mod.M5.M5_inner, :asdf) + + @test jeval("Base.@propagate_inbounds @inline meta_double_quote_issue(x) = x") isa Function + end + + @testset "(AI) `include_string` with `mapexpr`" begin + seen = Any[] + local test_mod = Module(:MapexprSeen) + out = JL.include_string(ex -> (push!(seen, ex); ex), test_mod, + "aa = 1\n\nbb = aa + 1\nbb*10", "none") + @test seen == [Meta.parse("aa = 1"; mod=test_mod), + Meta.parse("bb = aa + 1"; mod=test_mod), + Meta.parse("bb*10"; mod=test_mod)] + @test out === 20 + + local test_mod = Module(:MapexprCmp) + function mapexpr_sees(code) + seen = Any[] + JL.include_string(ex -> (push!(seen, ex); nothing), test_mod, + code, "none") + only(seen) + end + for code in ("xx += 1", "for i in 1:2; end", "function g(a); a; end", + "@inline h(a) = a", "\"doc\" k(a) = a", "using Base.Threads", + "const cc = 1", "a.b = 2", "x[1] = 2", "if p; q; else; r; end", + "macro mmm(); end") + @test mapexpr_sees(code) == Meta.parse(code; mod=test_mod) + end + + # Three vs four arg module form (may later be deprecated, but macros + # shouldn't get confused by this while it's here) + @test mapexpr_sees("module MMM; end") == + Meta.parse("module MMM; end"; mod=test_mod) + + out = JL.include_string(ex -> Expr(:module, true, :Renamed, ex.args[end]), + Module(:MapexprModule), "module Orig; yy = 1; end", "none") + @test out isa Module && nameof(out) === :Renamed + @test Base.invokelatest(() -> isdefined(out, :yy)) + + # Statements are mapped and evaluated one at a time (rather than all + # mapped up front), so `mapexpr` may depend on earlier evaluation + log = Symbol[] + logmod = Module(:MapexprOrder) + Core.eval(logmod, :(const log = $log)) + JL.include_string(ex -> (push!(log, :map); ex), logmod, + "push!(log, :eval)\npush!(log, :eval)", "none") + @test log == [:map, :eval, :map, :eval] + + # return value + @test JL.include_string(ex -> Expr(:call, :+, ex, 100), Module(:MapexprAdd), + "1+1\n2+2", "none") === 104 + # returning `nothing` drops a statement + @test JL.include_string(ex -> nothing, Module(:MapexprDrop), + "error(\"not evaluated\")", "none") === nothing + @test JL.include_string(ex -> error("never called"), Module(:MapexprEmpty), + "# just a comment\n", "none") === nothing + + noop(x) = x + # Definitions survive the round trip (world age) + m = Module(:MapexprDefs) + @test JL.include_string(noop, m, "module Inner; zz = 5; end", "none") isa Module + @test Base.invokelatest(() -> isdefined(m.Inner, :zz)) + @test JL.include_string(noop, m, "f(x) = x + 1\nf(2)", "none") === 3 + @test JL.include_string(noop, m, "macro mm(); 7; end\n@mm", "none") === 7 + + # `expr_compat_mode` still applies + @test JL.include_string(noop, Module(:MapexprCompat), + """ + macro plus1(ex) + :(\$(esc(ex)) + 1) + end + qq = 10 + @plus1 qq + """, "none"; expr_compat_mode=true) === 11 + + # Errors in the included code and in `mapexpr` itself both propagate + @test_throws "boom" JL.include_string(noop, Module(:MapexprErr), + "error(\"boom\")", "none") + @test_throws "in mapexpr" JL.include_string(ex -> error("in mapexpr"), + Module(:MapexprErr2), "1+1", "none") + end +end diff --git a/JuliaLowering/test/import.jl b/JuliaLowering/test/import.jl new file mode 100644 index 0000000000000..9651dfcb29781 --- /dev/null +++ b/JuliaLowering/test/import.jl @@ -0,0 +1,340 @@ +test_mod = Module() + +# Test attributes are correctly set for export/public +JuliaLowering.include_string(test_mod, """ +x = 1 +y = 2 +export x +public y +""") +@test Base.isexported(test_mod, :x) +@test Base.ispublic(test_mod, :x) +@test Base.ispublic(test_mod, :y) +@test !Base.isexported(test_mod, :y) + +# Test various forms of `using` +C = JuliaLowering.include_string(test_mod, """ +module C + module D + export x + public y, f + x = [101] + y = [202] + + function f() + "hi" + end + end + module E + using ..D: f + using ..D + using .D: y as D_y + using .D: x as D_x_2, y as D_y_2 + import .D.y as D_y_3 + end +end +""") +@test C.D.f === C.E.f +@test C.D.x === C.E.x +@test C.D.y === C.E.D_y +@test C.D.x === C.E.D_x_2 +@test C.D.y === C.E.D_y_2 +@test C.D.y === C.E.D_y_3 + +# Test that using F brings in the exported symbol G immediately and that it can +# be used next in the import list. +F = JuliaLowering.include_string(test_mod, """ +module F + export G + module G + export G_global + G_global = "exported from G" + end +end +""") +JuliaLowering.include_string(test_mod, """ +using .F, .G +""") +@test test_mod.F === F +@test test_mod.G === F.G +@test test_mod.G_global === "exported from G" + +# Similarly, that import makes symbols available immediately +H = JuliaLowering.include_string(test_mod, """ +module H + module I + module J + end + end +end +""") +JuliaLowering.include_string(test_mod, """ +import .H.I, .I.J +""") +@test test_mod.I === H.I +@test test_mod.J === H.I.J +@test test_mod.G_global === "exported from G" + +@testset "(AI) from macro expansion" for expr_compat_mode in (true, false) + macrocall_mod = Module() + @eval macrocall_mod import JuliaLowering, JuliaLowering.@legacy_quote_to_syntax + JuliaLowering.include_string(macrocall_mod, raw""" + module Exporter + export val + val = [123] + other = [456] + end + macro imp_names() + @legacy_quote_to_syntax :(import .Exporter: val, other as o) + end + macro use_target() + @legacy_quote_to_syntax :(using .Exporter) + end + """; expr_compat_mode) + Core.@latestworld + JuliaLowering.include_string(macrocall_mod, "@imp_names"; expr_compat_mode) + JuliaLowering.include_string(macrocall_mod, "@use_target"; expr_compat_mode) + Core.@latestworld + @test macrocall_mod.val === macrocall_mod.Exporter.val + @test macrocall_mod.o === macrocall_mod.Exporter.other + @test !isdefined(macrocall_mod, :other) # imported only under the name `o` +end + +@testset "Imported macrocalls" for expr_compat_mode in (true, false) + # Test importing macros by their @-name + macname_mod = Module() + JuliaLowering.include_string(macname_mod, raw""" + module Macros + macro mac1(); "mac1"; end + macro mac2(); "mac2"; end + module Inner + macro mac3(); "mac3"; end + macro mac4(); "mac4"; end + end + end + """; expr_compat_mode) + JuliaLowering.include_string(macname_mod, raw""" + module UseMacros + import ..Macros: + @mac1, + @mac2 as @mac2_renamed, + Inner.@mac3, + Inner.@mac4 as @mac4_renamed + end + """; expr_compat_mode) + Core.@latestworld + @test JuliaLowering.include_string(macname_mod.UseMacros, + "@mac1()"; expr_compat_mode) == "mac1" + @test JuliaLowering.include_string(macname_mod.UseMacros, + "@mac2_renamed()"; expr_compat_mode) == "mac2" + @test JuliaLowering.include_string(macname_mod.UseMacros, + "@mac3()"; expr_compat_mode) == "mac3" + @test JuliaLowering.include_string(macname_mod.UseMacros, + "@mac4_renamed()"; expr_compat_mode) == "mac4" +end + +fl_eval(test_mod, :( + module mod_p_e_n + public_var = 1 + public p + exported_var = 2 + export e + neither_var = 3 + end)) +@testset "colon followed by only from-path" begin + jl_eval(test_mod, Expr(:import, Expr(:(:), Expr(:., :., :mod_p_e_n)))) + @test !isdefined(test_mod, :public_var) + @test !isdefined(test_mod, :exported_var) + @test !isdefined(test_mod, :neither_var) + @test test_mod.mod_p_e_n isa Module + jl_eval(test_mod, Expr(:using, Expr(:(:), Expr(:., :., :mod_p_e_n)))) + @test !isdefined(test_mod, :public_var) + @test !isdefined(test_mod, :exported_var) + @test !isdefined(test_mod, :neither_var) + @test test_mod.mod_p_e_n isa Module +end + +@testset "(AI) public/export module resolution from macros" for (is_new, run) in [ + (false, (mod, x)->fl_eval(mod,JuliaSyntax.parsestmt(Expr, x))), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=true)), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=false)) + ] + + defs_mod = Module(:Defs) + call_mod = Module(:CallSite) + Core.eval(call_mod, :(const Defs = $defs_mod)) + Core.eval(defs_mod, :(import JuliaLowering, JuliaLowering.@legacy_quote_to_syntax)) + Core.eval(defs_mod, :(const var"@ast" = $(JuliaLowering.var"@ast"))) + Core.eval(defs_mod, :(const var"@K_str" = $(JuliaSyntax.var"@K_str"))) + + # old-style macros: hygienic plain name vs escaped argument + fl_eval(defs_mod, :(macro old_pub_plain(); Expr(:public, :op_hyg); end)) + fl_eval(defs_mod, :(macro old_exp_plain(); Expr(:export, :oe_hyg); end)) + fl_eval(defs_mod, :(macro old_pub_arg(name); Expr(:public, esc(name)); end)) + fl_eval(defs_mod, :(macro old_exp_arg(name); Expr(:export, esc(name)); end)) + + # new-style macros: hygienic name (from syntaxquote) vs argument + JuliaLowering.include_string(defs_mod, raw""" + macro new_exp_plain(); @legacy_quote_to_syntax quote export ne_hyg end; end + macro new_pub_arg(name); @ast __context__ __context__.macrocall [K"public" name]; end + macro new_exp_arg(name); @ast __context__ __context__.macrocall [K"export" name]; end + """) + Core.@latestworld + + # Define the names being marked so `isexported`/`ispublic` are meaningful. + # Hygienic names live in defs_mod; argument/escaped names live in call_mod. + Core.eval(defs_mod, :(global op_hyg=1; global oe_hyg=1; global ne_hyg=1)) + Core.eval(call_mod, :(global cp=1; global ce=1; global np=1; global ne=1)) + + # old macros, hygienic plain name -> macro-definition module (defs_mod) + # flisp error: globalref means malformed public + # @test run(call_mod, "Defs.@old_pub_plain()") + # @test Base.ispublic(defs_mod, :op_hyg) + # @test !Base.ispublic(call_mod, :op_hyg) + run(call_mod, "Defs.@old_exp_plain()"); Core.@latestworld + @test !Base.isexported(defs_mod, :oe_hyg) + @test Base.isexported(call_mod, :oe_hyg) + + # old macros, escaped argument -> call-site module (call_mod) + run(call_mod, "Defs.@old_pub_arg(cp)"); Core.@latestworld + @test Base.ispublic(call_mod, :cp) + @test !Base.ispublic(defs_mod, :cp) + run(call_mod, "Defs.@old_exp_arg(ce)"); Core.@latestworld + @test Base.isexported(call_mod, :ce) + @test !Base.isexported(defs_mod, :ce) + + if is_new + # new macros, hygienic name -> macro-definition module (defs_mod) + run(call_mod, "Defs.@new_exp_plain()"); Core.@latestworld + @test Base.isexported(defs_mod, :ne_hyg) + @test !Base.isexported(call_mod, :ne_hyg) + + # new macros, argument -> call-site module (call_mod) + run(call_mod, "Defs.@new_pub_arg(np)"); Core.@latestworld + @test Base.ispublic(call_mod, :np) + @test !Base.ispublic(defs_mod, :np) + run(call_mod, "Defs.@new_exp_arg(ne)"); Core.@latestworld + @test Base.isexported(call_mod, :ne) + @test !Base.isexported(defs_mod, :ne) + end +end + +@testset "(AI) using/import module resolution from macros" for (is_new, run) in [ + (false, (mod, x)->fl_eval(mod, JuliaSyntax.parsestmt(Expr, x))), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=true)), + (true, (mod, x)->JuliaLowering.include_string(mod, x; expr_compat_mode=false)) + ] + + # Unlike `public`/`export`, `using`/`import` are never hygienic + defs_mod = Module(:Defs) + call_mod = Module(:CallSite) + Core.eval(call_mod, :(const Defs = $defs_mod)) + Core.eval(defs_mod, :(import JuliaLowering, JuliaLowering.@legacy_quote_to_syntax)) + + # Distinguishable `Exporter` submodules in *both* modules. The imported + # values reveal which `.Exporter` a relative path resolved to. + Core.eval(defs_mod, :( + module Exporter + export val + val = [:defs_val] + other = [:defs_other] + end)) + Core.eval(call_mod, :( + module Exporter + export val + val = [:call_val] + other = [:call_other] + end)) + # A submodule reachable only from defs_mod: if a path were resolved against + # the macro-definition module, `.OnlyInDefs` would be found there. + Core.eval(defs_mod, :( + module OnlyInDefs + secret = [:secret] + end)) + + fl_eval(defs_mod, :(macro old_use(); :(using .Exporter); end)) + fl_eval(defs_mod, :(macro old_imp(); :(import .Exporter: other as o); end)) + fl_eval(defs_mod, :(macro old_priv(); :(using .OnlyInDefs); end)) + + JuliaLowering.include_string(defs_mod, raw""" + macro new_use(); @legacy_quote_to_syntax quote using .Exporter end; end + macro new_imp(); @legacy_quote_to_syntax quote import .Exporter: other as o end; end + macro new_priv(); @legacy_quote_to_syntax quote using .OnlyInDefs end; end + """) + Core.@latestworld + + # `using .Exporter` -> exported `val` becomes visible in call_mod, resolved + # against call_mod.Exporter, not defs_mod.Exporter; defs_mod is untouched. + run(call_mod, "Defs.@old_use()"); Core.@latestworld + @test call_mod.val === call_mod.Exporter.val + @test call_mod.val !== defs_mod.Exporter.val + @test !isdefined(defs_mod, :val) + + # `import .Exporter: other as o` -> the (plain, non-hygienic) rename target + # `o` is bound in call_mod, again resolved against call_mod.Exporter. + run(call_mod, "Defs.@old_imp()"); Core.@latestworld + @test call_mod.o === call_mod.Exporter.other + @test call_mod.o !== defs_mod.Exporter.other + @test !isdefined(defs_mod, :o) + + # The path is resolved in call_mod, where `.OnlyInDefs` does not exist, so + # the statement errors rather than reaching defs_mod.OnlyInDefs. + @test_throws UndefVarError run(call_mod, "Defs.@old_priv()") + @test !isdefined(call_mod, :OnlyInDefs) + + if is_new + run(call_mod, "Defs.@new_use()"); Core.@latestworld + @test call_mod.val === call_mod.Exporter.val + @test call_mod.val !== defs_mod.Exporter.val + @test !isdefined(defs_mod, :val) + + run(call_mod, "Defs.@new_imp()"); Core.@latestworld + @test call_mod.o === call_mod.Exporter.other + @test call_mod.o !== defs_mod.Exporter.other + @test !isdefined(defs_mod, :o) + + @test_throws UndefVarError run(call_mod, "Defs.@new_priv()") + @test !isdefined(call_mod, :OnlyInDefs) + end +end + +@testset "(AI) all-underscore (`_`) import names" begin + # Names in an import/using path are symbolic references, not value reads, so + # an all-underscore name is a genuine (write-only) binding here rather than a + # discard -- matching flisp, which binds `_` for `import X as _`, + # `using X: a as _` and `using X: _`. Previously the desugaring assert + # `kind(spec[2]) == K"Identifier"` crashed on the `K"Placeholder"` rename + # target (e.g. SymbolicRegression's `using ConstructionBase: ... as _`). + U = JuliaLowering.include_string(test_mod, """ + module Uroot + module Src + export sx + sx = [1] + sy = [2] + end + module AsTarget; using ..Src: sx as _; end # rename target `_` + module ImportColon; import ..Src: sy as _; end # `import X: name as _` + module ImportAs; import ..Src as _; end # whole-module rename + module VarSpelling; using ..Src: sx as var"_"; end # var"_" is the same name + module DoubleUnder; import ..Src as __; end # `__` is also write-only + end + """) + for m in (U.AsTarget, U.ImportColon, U.ImportAs, U.VarSpelling) + @test isdefined(m, :_) + end + @test isdefined(U.DoubleUnder, :__) + # The created `_` stays write-only: reading it is a parser-level error, not a + # usable value (so `as _` really is an import-for-side-effect idiom). + @test_throws JuliaLowering.LoweringError JuliaLowering.include_string(U.AsTarget, "_") + + # Two `as _` targets and a bare `_` source name are accepted (flisp only + # warns -- conflicting import / undeclared binding); they must not error. + redirect_stderr(devnull) do + @test (JuliaLowering.include_string(test_mod, """ + module TwoUnders; using Base: sum as _, prod as _; end + """); true) + @test (JuliaLowering.include_string(test_mod, """ + module SrcUnder; module Inner; end; using .Inner: _; end + """); true) + end +end diff --git a/JuliaLowering/test/import_ir.jl b/JuliaLowering/test/import_ir.jl new file mode 100644 index 0000000000000..a8c028dc70792 --- /dev/null +++ b/JuliaLowering/test/import_ir.jl @@ -0,0 +1,77 @@ +######################################## +# Basic import +import A: b +#--------------------- +1 (call JuliaLowering.eval_import true TestMod (inert (. A)) (inert (. b))) +2 latestworld +3 (return core.nothing) + +######################################## +# Import with paths and `as` +import A.B.C: b, c.d as e +#--------------------- +1 (call JuliaLowering.eval_import true TestMod (inert (. A B C)) (inert (. b)) (inert (as (. c d) e))) +2 latestworld +3 (return core.nothing) + +######################################## +# Import macrocall +import A: B.@mac as @mac2 +#--------------------- +1 (call JuliaLowering.eval_import true TestMod (inert (. A)) (inert (as (. B @mac) @mac2))) +2 latestworld +3 (return core.nothing) + +######################################## +# Imports without `from` module need separating with latestworld +import A, B +#--------------------- +1 (call JuliaLowering.eval_import true TestMod core.nothing (inert (. A))) +2 latestworld +3 (call JuliaLowering.eval_import true TestMod core.nothing (inert (. B))) +4 latestworld +5 (return core.nothing) + +######################################## +# Multiple usings need separating with latestworld +using A, B +#--------------------- +1 (call JuliaLowering.eval_using TestMod (inert (. A))) +2 latestworld +3 (call JuliaLowering.eval_using TestMod (inert (. B))) +4 latestworld +5 (return core.nothing) + +######################################## +# Using with paths and `as` +using A.B.C: b, c.d as e +#--------------------- +1 (call JuliaLowering.eval_import false TestMod (inert (. A B C)) (inert (. b)) (inert (as (. c d) e))) +2 latestworld +3 (return core.nothing) + +######################################## +# Error: Import not at top level +function f() + import A: b +end +#--------------------- +LoweringError: +function f() + import A: b +# └─────────┘ ── this syntax is only allowed at top level +end + +######################################## +# Export +export a, b, c +#--------------------- +1 (call JuliaLowering.eval_public TestMod true ["a", "b", "c"]) +2 (return %₁) + +######################################## +# Public +public a, b, c +#--------------------- +1 (call JuliaLowering.eval_public TestMod false ["a", "b", "c"]) +2 (return %₁) diff --git a/JuliaLowering/test/ir_tests.jl b/JuliaLowering/test/ir_tests.jl new file mode 100644 index 0000000000000..3035a6f3a7bc4 --- /dev/null +++ b/JuliaLowering/test/ir_tests.jl @@ -0,0 +1,10 @@ +@testset "IR tests" begin + testdir = @__DIR__ + for filename in readdir(testdir) + if endswith(filename, "_ir.jl") + @testset "$filename" begin + test_ir_cases(joinpath(testdir, filename)) + end + end + end +end diff --git a/JuliaLowering/test/loops.jl b/JuliaLowering/test/loops.jl new file mode 100644 index 0000000000000..3b0bfb0beab39 --- /dev/null +++ b/JuliaLowering/test/loops.jl @@ -0,0 +1,342 @@ +@testset "while loops" begin + +test_mod = Module() +Base.set_syntax_version(test_mod, v"1.14") + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + i = 0 + while i < 5 + i = i + 1 + push!(a, i) + end + a +end +""") == [1,2,3,4,5] + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + i = 0 + while i < 5 + i = i + 1 + if i == 3 + break + end + push!(a, i) + end + a +end +""") == [1,2] + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + i = 0 + while i < 5 + i = i + 1 + if isodd(i) + continue + end + push!(a, i) + end + a +end +""") == [2,4] + +end + +@testset "for loops" begin + +test_mod = Module() +Base.set_syntax_version(test_mod, v"1.14") + +# iteration +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:3 + push!(a, i) + end + a +end +""") == [1,2,3] + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:0 + push!(a, i) + end + a +end +""") == [] + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for _ = 1:3 + push!(a, 1) + end + a +end +""") == [1, 1, 1] + +# break +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:6 + if i == 3 + break + end + push!(a, i) + end + a +end +""") == [1, 2] +# Break from inner nested loop +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i in 1:2 + for j in 3:4 + push!(a, (i, j)) + j == 6 && break + end + end + a +end +""") == [(1, 3), (1, 4), (2, 3), (2, 4)] + +# continue +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:6 + if isodd(i) + continue + end + push!(a, i) + end + a +end +""") == [2, 4, 6] + +# Loop variable scope +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:3 + push!(a, i) + i = 100 + end + a +end +""") == [1,2,3] + +@test JuliaLowering.include_string(test_mod, """ +let + i = 100 + for i = 1:3 + end + i +end +""") == 100 + +@test JuliaLowering.include_string(test_mod, """ +let + i = 100 + for outer i = 1:2 + nothing + end + i +end +""") == 2 + +# Fancy for loop left hand side - unpacking and scoping +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + i = 100 + j = 200 + for (i,j) in [('a', 'b'), (1,2)] + push!(a, (i,j)) + end + (a, i, j) +end +""") == ([('a', 'b'), (1,2)], 100, 200) + +end + + +@testset "multidimensional for loops" begin + +test_mod = Module() +Base.set_syntax_version(test_mod, v"1.14") + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:2, j = 3:4 + push!(a, (i,j)) + end + a +end +""") == [(1,3), (1,4), (2,3), (2,4)] + +@testset "break/continue" begin + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:2, j = 3:4 + push!(a, (i,j)) + break + end + a +end +""") == [(1,3)] + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:4, j = 3:4 + if isodd(i) + continue + end + push!(a, (i,j)) + end + a +end +""") == [(2,3), (2,4), (4,3), (4,4)] + +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:2, j = 1:4 + if isodd(j) + continue + end + push!(a, (i,j)) + end + a +end +""") == [(1,2), (1,4), (2,2), (2,4)] + +# Labeled continue skips to next iteration of the named outer loop +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + @label outer for i = 1:3 + for j = 1:3 + if j == 2 + continue outer + end + push!(a, (i,j)) + end + end + a +end +""") == [(1,1), (2,1), (3,1)] + +# An unlabeled break exits an anonymous `@label` block +@test JuliaLowering.include_string(test_mod, """ +@label begin + break + error("unreached") +end +""") === nothing + +# ... but breaking through a named block is still an error +@test_throws LoweringError JuliaLowering.include_string(test_mod, """ +@label named begin + break +end +""") + +# Values of labeled breaks are scope-resolved: variables (not just +# literals) work as break values, including from inside nested scopes +@test JuliaLowering.include_string(test_mod, """ +@label begin + let + local t = 1 + break _ t + end + 0 +end +""") == 1 + +@test JuliaLowering.include_string(test_mod, """ +@label myblock begin + let v = 21 + break myblock 2v + end + 0 +end +""") == 42 + +@test JuliaLowering.include_string(test_mod, """ +let a = [] + @label outer for i = 1:10 + x = i * 2 + i == 3 && break outer x + push!(a, i) + end +end +""") == 6 + +end + + +@testset "Loop variable scope" begin + +# Test that `i` is copied in the inner loop +@test JuliaLowering.include_string(test_mod, """ +let + a = [] + for i = 1:2, j = 3:4 + push!(a, (i,j)) + i = 100 + end + a +end +""") == [(1,3), (1,4), (2,3), (2,4)] + +@test JuliaLowering.include_string(test_mod, """ +let + i = 100 + j = 200 + for i = 1:2, j = 3:4 + nothing + end + (i,j) +end +""") == (100,200) + +@test JuliaLowering.include_string(test_mod, """ +let + i = 100 + j = 200 + for outer i = 1:2, j = 3:4 + nothing + end + (i,j) +end +""") == (2,200) + +@test JuliaLowering.include_string(test_mod, """ +let + i = 100 + j = 200 + for i = 1:2, outer j = 3:4 + nothing + end + (i,j) +end +""") == (100,4) + +end + +end diff --git a/JuliaLowering/test/loops_ir.jl b/JuliaLowering/test/loops_ir.jl new file mode 100644 index 0000000000000..bd08378f5bd3f --- /dev/null +++ b/JuliaLowering/test/loops_ir.jl @@ -0,0 +1,184 @@ +######################################## +# Basic while loop +while f(a) + body1 + body2 +end +#--------------------- +1 TestMod.f +2 TestMod.a +3 (call %₁ %₂) +4 (gotoifnot %₃ label₈) +5 TestMod.body1 +6 TestMod.body2 +7 (goto label₁) +8 (= slot₁/loop-exit_result core.nothing) +9 (isdefined slot₁/loop-exit_result) +10 (gotoifnot %₉ label₁₂) +11 (goto label₁₃) +12 (= slot₁/loop-exit_result core.nothing) +13 slot₁/loop-exit_result +14 (return %₁₃) + +######################################## +# While loop with short circuit condition +while a && b + body +end +#--------------------- +1 TestMod.a +2 (gotoifnot %₁ label₇) +3 TestMod.b +4 (gotoifnot %₃ label₇) +5 TestMod.body +6 (goto label₁) +7 (= slot₁/loop-exit_result core.nothing) +8 (isdefined slot₁/loop-exit_result) +9 (gotoifnot %₈ label₁₁) +10 (goto label₁₂) +11 (= slot₁/loop-exit_result core.nothing) +12 slot₁/loop-exit_result +13 (return %₁₂) + +######################################## +# While loop with break and continue +while cond + body1 + break + body2 + continue + body3 +end +#--------------------- +1 TestMod.cond +2 (gotoifnot %₁ label₉) +3 TestMod.body1 +4 (goto label₁₀) +5 TestMod.body2 +6 (goto label₈) +7 TestMod.body3 +8 (goto label₁) +9 (= slot₁/loop-exit_result core.nothing) +10 (isdefined slot₁/loop-exit_result) +11 (gotoifnot %₁₀ label₁₃) +12 (goto label₁₄) +13 (= slot₁/loop-exit_result core.nothing) +14 slot₁/loop-exit_result +15 (return %₁₄) + +######################################## +# Basic for loop +for x in xs + body +end +#--------------------- +1 TestMod.xs +2 (= slot₁/next (call top.iterate %₁)) +3 slot₁/next +4 (call core.=== %₃ core.nothing) +5 (call top.not_int %₄) +6 (gotoifnot %₅ label₁₉) +7 slot₁/next +8 (= slot₂/x (call core.getfield %₇ 1)) +9 (call core.getfield %₇ 2) +10 TestMod.body +11 (= slot₁/next (call top.iterate %₁ %₉)) +12 slot₁/next +13 (call core.=== %₁₂ core.nothing) +14 (call top.not_int %₁₃) +15 (gotoifnot %₁₄ label₁₇) +16 (goto label₇) +17 (= slot₄/if_val core.nothing) +18 (goto label₂₀) +19 (= slot₄/if_val core.nothing) +20 slot₄/if_val +21 (= slot₃/loop-exit_result %₂₀) +22 (isdefined slot₃/loop-exit_result) +23 (gotoifnot %₂₂ label₂₅) +24 (goto label₂₆) +25 (= slot₃/loop-exit_result core.nothing) +26 slot₃/loop-exit_result +27 (return %₂₆) + +######################################## +# Syntax sugar for nested for loop +for x in xs, y in ys + x = 10 # Copy of x; does not overwrite x iteration var +end +#--------------------- +1 TestMod.xs +2 (= slot₂/next (call top.iterate %₁)) +3 slot₂/next +4 (call core.=== %₃ core.nothing) +5 (call top.not_int %₄) +6 (gotoifnot %₅ label₃₆) +7 slot₂/next +8 (= slot₃/x (call core.getfield %₇ 1)) +9 (call core.getfield %₇ 2) +10 TestMod.ys +11 (= slot₁/next (call top.iterate %₁₀)) +12 slot₁/next +13 (call core.=== %₁₂ core.nothing) +14 (call top.not_int %₁₃) +15 (gotoifnot %₁₄ label₂₈) +16 slot₃/x +17 (= slot₄/x %₁₆) +18 slot₁/next +19 (= slot₅/y (call core.getfield %₁₈ 1)) +20 (call core.getfield %₁₈ 2) +21 (= slot₄/x 10) +22 (= slot₁/next (call top.iterate %₁₀ %₂₀)) +23 slot₁/next +24 (call core.=== %₂₃ core.nothing) +25 (call top.not_int %₂₄) +26 (gotoifnot %₂₅ label₂₈) +27 (goto label₁₆) +28 (= slot₂/next (call top.iterate %₁ %₉)) +29 slot₂/next +30 (call core.=== %₂₉ core.nothing) +31 (call top.not_int %₃₀) +32 (gotoifnot %₃₁ label₃₄) +33 (goto label₇) +34 (= slot₇/if_val core.nothing) +35 (goto label₃₇) +36 (= slot₇/if_val core.nothing) +37 slot₇/if_val +38 (= slot₆/loop-exit_result %₃₇) +39 (isdefined slot₆/loop-exit_result) +40 (gotoifnot %₃₉ label₄₂) +41 (goto label₄₃) +42 (= slot₆/loop-exit_result core.nothing) +43 slot₆/loop-exit_result +44 (return %₄₃) + +######################################## +# Error: break outside for/while +break +#--------------------- +LoweringError: +break +└───┘ ── unlabeled `break` outside of a `while` or `for` loop + +######################################## +# Error: continue outside for/while +continue +#--------------------- +LoweringError: +continue +└──────┘ ── `continue` outside of a `while` or `for` loop + +######################################## +# Error: `outer` without outer local variable +let + for outer i = 1:2 + nothing + end + i +end +#--------------------- +LoweringError: +let + for outer i = 1:2 +# ╙ ── `outer` annotations must match with a local variable in an outer scope but no such variable was found + nothing + end diff --git a/JuliaLowering/test/macros.jl b/JuliaLowering/test/macros.jl new file mode 100644 index 0000000000000..a91aeac786d6c --- /dev/null +++ b/JuliaLowering/test/macros.jl @@ -0,0 +1,2098 @@ +test_mod = @newmod(macro_test) +@eval test_mod import JuliaLowering +Base.eval(test_mod, :(const var"@ast" = $(JuliaLowering.var"@ast"))) + +# Set up identity macros for use in this file +# - `old_e`, escaping its whole output, should do nothing to an expression +# - `new_m`, introducing no new syntax, should behave exactly as `old_e` does +# - `old_*` should behave the same across JL and flisp +# - `old_h` should not be specified too hard here (buggy renaming pass) +fl_eval(test_mod, :(macro old_e(x); esc(x); end)) +fl_eval(test_mod, :(macro old_h(x); x; end)) +JuliaLowering.include_string(test_mod, "macro new_m(x); x; end") +fl_eval(test_mod, :(global mvar = "global mvar")) + +@testset "syntax versioning sanity-check" begin + @test JuliaLowering.include_string( + test_mod, "JuliaLowering.@syntax_version") == + JuliaSyntax.JL_NEW_SYNTAX_VERSION + @test JuliaLowering.include_string( + test_mod, "JuliaLowering.@syntax_version"; expr_compat_mode=false) == + JuliaSyntax.JL_NEW_SYNTAX_VERSION + @test JuliaLowering.include_string( + test_mod, "JuliaLowering.@syntax_version"; expr_compat_mode=true) == + JuliaSyntax.JL_OLD_SYNTAX_VERSION + + # TODO: test the version of returned syntax + @test JuliaLowering.include_string(@newmod(), """ + JuliaLowering.@syntax_version JuliaSyntax.JL_NEW_SYNTAX_VERSION macro m(); end + """; expr_compat_mode=false) isa Function + @test JuliaLowering.include_string(@newmod(), """ + JuliaLowering.@syntax_version JuliaSyntax.JL_NEW_SYNTAX_VERSION macro m(); end + """; expr_compat_mode=true) isa Function + + @test JuliaLowering.include_string(@newmod(), """ + JuliaLowering.@syntax_version JuliaSyntax.JL_OLD_SYNTAX_VERSION macro m(); end + """; expr_compat_mode=false) isa Function + @test JuliaLowering.include_string(@newmod(), """ + JuliaLowering.@syntax_version JuliaSyntax.JL_OLD_SYNTAX_VERSION macro m(); end + """; expr_compat_mode=true) isa Function +end + +# Basic checks that arbitrary nesting of transparent macros (no new syntax in new +# macros, escaped/unhygienic in old macros) doesn't introduce opaque layers +@testset "basic transparent macros: old macros" for run in [ + (x::String)->Base.include_string( + test_mod, "#=FLISP SANITY-CHECK=# "*x), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + @test run("@old_e let mvar = 0; mvar; end") == 0 + @test run("@old_e let @old_e(mvar = 0); mvar; end") == 0 + @test run("@old_e let @old_e(@old_e(mvar = 0)); mvar; end") == 0 + @test run("@old_e let @old_e(mvar) = 0; mvar; end") == 0 + @test run("@old_e let @old_e(@old_e(mvar)) = 0; mvar; end") == 0 + @test run("@old_e let mvar = 0; @old_e(mvar); end") == 0 + @test run("@old_e let mvar = 0; @old_e(@old_e(mvar)); end") == 0 + @test run("@old_e let @old_e(@old_e(mvar) = 0); @old_e(mvar); end") == 0 + + @test run("@old_h let mvar = 0; mvar; end") == 0 + @test run("@old_h let @old_e(mvar = 0); mvar; end") == 0 + @test run("@old_h let @old_e(@old_e(mvar = 0)); mvar; end") == 0 + @test run("@old_h let @old_e(mvar) = 0; mvar; end") == 0 + @test run("@old_h let @old_e(@old_e(mvar)) = 0; mvar; end") == 0 + @test run("@old_h let mvar = 0; @old_e(mvar); end") == 0 + @test run("@old_h let mvar = 0; @old_e(@old_e(mvar)); end") == 0 + @test run("@old_h let @old_e(@old_e(mvar) = 0); @old_e(mvar); end") == 0 + + @test run("@old_h @old_h let mvar = 0; mvar; end") == 0 + @test run("@old_h @old_h let @old_e(mvar = 0); mvar; end") == 0 + @test run("@old_h @old_h let @old_e(@old_e(mvar = 0)); mvar; end") == 0 + @test run("@old_h @old_h let @old_e(mvar) = 0; mvar; end") == 0 + @test run("@old_h @old_h let @old_e(@old_e(mvar)) = 0; mvar; end") == 0 + @test run("@old_h @old_h let mvar = 0; @old_e(mvar); end") == 0 + @test run("@old_h @old_h let mvar = 0; @old_e(@old_e(mvar)); end") == 0 + @test run("@old_h @old_h let @old_e(@old_e(mvar) = 0); @old_e(mvar); end") == 0 +end +@testset "basic transparent macros: new macros only" for expr_compat_mode in [true, false] + local run = (x::String)->JuliaLowering.include_string(test_mod, x; expr_compat_mode) + + @test run("@new_m let mvar = 0; mvar; end") == 0 + @test run("@new_m let @new_m(mvar = 0); mvar; end") == 0 + @test run("@new_m let @new_m(@new_m(mvar = 0)); mvar; end") == 0 + @test run("@new_m let @new_m(mvar) = 0; mvar; end") == 0 + @test run("@new_m let @new_m(@new_m(mvar)) = 0; mvar; end") == 0 + @test run("@new_m let mvar = 0; @new_m(mvar); end") == 0 + @test run("@new_m let mvar = 0; @new_m(@new_m(mvar)); end") == 0 + @test run("@new_m let @new_m(@new_m(mvar) = 0); @new_m(mvar); end") == 0 +end +@testset "basic transparent macros: new+old interop" for expr_compat_mode in [true, false], + mcall in ["@old_e ", "@new_m ", "@old_e @new_m ", "@new_m @old_e "], + old_h in ["", "@old_h "] + + local run = (x::String)->JuliaLowering.include_string(test_mod, x; expr_compat_mode) + + @test run(old_h*mcall*"let mvar = 0; mvar; end") == 0 + @test run(old_h*"let ("*mcall*"mvar = 0); mvar; end") == 0 + @test run(old_h*"let ("*mcall*"mvar) = 0; mvar; end") == 0 + @test run(old_h*"let mvar = 0; ("*mcall*"mvar); end") == 0 + + @testset for mcall2 in ["@old_e ", "@new_m ", "@old_e @new_m ", "@new_m @old_e "] + @test run(old_h*mcall*"let ("*mcall2*"mvar) = 0; mvar; end") == 0 + @test run(old_h*mcall*"let mvar = 0; ("*mcall2*"mvar); end") == 0 + @test run(old_h*"let ("*mcall*"mvar = 0); ("*mcall2*"mvar); end") == 0 + end +end + +# More simple checks with no difference between macro module and macrocall module +isdefined(test_mod, :x) && Base.delete_binding(test_mod, :x) +fl_eval(test_mod, :(macro old_read_x(); :x; end)) +fl_eval(test_mod, :(macro old_suggest_x(arg) + quote + let x = "suggested (old)" + $(esc(arg)) + end + end + end)) +JuliaLowering.include_string(test_mod, raw""" + macro new_read_x(); @legacy_quote_to_syntax :x; end +""") +JuliaLowering.include_string(test_mod, raw""" + macro new_suggest_x(arg) + @legacy_quote_to_syntax quote + let x = "suggested (new)" + $arg + end + end + end +""") +@testset "basic hygiene: check that name resolution fails where it should (flisp)" for run in [ + (x::String)->fl_eval(test_mod,JuliaSyntax.parsestmt(Expr, "#=FLISP SANITY-CHECK=# "*x)), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + @test_throws UndefVarError run("@old_read_x()") + @test_throws UndefVarError run("let x = 0; @old_read_x(); end") + @test_throws UndefVarError run("@old_suggest_x(x)") + @test_throws UndefVarError run("@old_suggest_x(@old_read_x())") + @test run("let x = 1; @old_suggest_x(x); end") == 1 + @test run("@old_suggest_x(let x = 1; x; end)") == 1 + @test_throws UndefVarError run("@old_suggest_x(let x = 1; @old_read_x(); end)") == 1 +end +@testset "basic hygiene: check that name resolution fails where it should (new)" for run in [ + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + @test_throws UndefVarError run("@new_read_x()") + @test_throws UndefVarError run("let x = 0; @new_read_x(); end") + @test_throws UndefVarError run("@new_suggest_x(x)") + @test_throws UndefVarError run("@new_suggest_x(@new_read_x())") + @test run("let x = 1; @new_suggest_x(x); end") == 1 + @test run("@new_suggest_x(let x = 1; x; end)") == 1 + @test_throws UndefVarError run("@new_suggest_x(let x = 1; @new_read_x(); end)") == 1 + + @testset "old/new interop" begin + @testset for wrapper in ["", "@old_e ", "@old_h ", "@new_m "] + @test_throws UndefVarError run(wrapper*"@old_suggest_x(@new_read_x())") + @test_throws UndefVarError run(wrapper*"@new_suggest_x(@old_read_x())") + @test run(wrapper*"let x = 1; @old_suggest_x(x); end") == 1 + @test run(wrapper*"let x = 1; @new_suggest_x(x); end") == 1 + @test run(wrapper*"@old_suggest_x(let x = 1; x; end)") == 1 + @test run(wrapper*"@new_suggest_x(let x = 1; x; end)") == 1 + end + end +end + +@eval test_mod (global test_mod_global = "test_mod_global") +@newmod(EvalMod, test_mod) +@testset "@eval" for run in [ + (x::String)->fl_eval(test_mod,JuliaSyntax.parsestmt(Expr, "#=FLISP SANITY-CHECK=# "*x)), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + has_syntax = run(raw"@legacy_quote_to_syntax :x") isa SyntaxTree + treetype = has_syntax ? SyntaxTree : Expr + symtype = has_syntax ? SyntaxTree : Symbol + valtype = has_syntax ? SyntaxTree : Any + + @test run(raw"@eval nothing") == nothing + @test run(raw"@eval :sym") == :sym + @test run(raw"@eval QuoteNode(:sym)") == QuoteNode(:sym) + @test run(raw"@eval Expr(:call, :identity, 1)") == Expr(:call, :identity, 1) + @test run(raw"@eval :(identity(1))") == Expr(:call, :identity, 1) + # syntax version of the caller should be propagated to JL.eval + @test run(raw"@eval @legacy_quote_to_syntax(:sym)") isa symtype + @test run(raw"@eval @legacy_quote_to_syntax(:(identity(1)))") isa treetype + @test run(raw"@eval @eval @legacy_quote_to_syntax(:(identity(1)))") isa treetype + + # quoting behaves the same as outside of eval + @test run(raw"@eval(:(1 + 2))") == Expr(:call, :+, 1, 2) + @test run(raw"@eval(:true)") == true + @test run(raw"@eval(:x)") == :x + + # interpolation + @test run(raw"let x = nothing; @eval $x; end") == nothing + @test run(raw"let x = :identity; @eval $x; end") == Base.identity + @test run(raw"let x = QuoteNode(:sym); @eval $x; end") == :sym + @test run(raw"let x = Expr(:call, :identity, 1); @eval $x; end") == 1 + @test run(raw"let x = :(identity(1)); @eval $x; end") == 1 + @test run(raw"let x = @legacy_quote_to_syntax(:identity); @eval $x; end") == Base.identity + @test run(raw"let x = @legacy_quote_to_syntax(:(identity(1))); @eval $x; end") == 1 + + # interpolate into quote + @test run(raw"let test_mod_global = 0xbad + @eval (@legacy_quote_to_syntax :($test_mod_global)) + end") isa valtype + @test run(raw"let test_mod_global = 0xbad + @eval @legacy_quote_to_syntax(:(1,$test_mod_global)) + end") isa treetype + + # interpolate into quote, double-unquote (mixes of syntax and expr may not + # need to work) + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval (:($($x))) + end") == Base.identity + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval (@legacy_quote_to_syntax :($($x))) + end") isa valtype + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval @legacy_quote_to_syntax(:(1,$$x)) + end") isa treetype + @test run(raw"let x = @legacy_quote_to_syntax(:identity) + @eval $(@eval (:(1,$$x))) + end") == (1, Base.identity) + + # module eval-ed into + @test run(raw"@eval @__MODULE__") == test_mod + @test run(raw"@eval @eval @__MODULE__") == test_mod + # two-arg eval should not obey typical hygiene: decls go to specified module + @test run(raw"@eval EvalMod @__MODULE__") == test_mod.EvalMod + run(raw"@eval EvalMod global eval_mod_global = 1"); Core.@latestworld + @test test_mod.EvalMod.eval_mod_global == 1 + run(raw"@eval EvalMod eval_mod_global_implicit = 1"); Core.@latestworld + @test test_mod.EvalMod.eval_mod_global_implicit == 1 + # standard hygiene atop two-arg eval + fl_eval(test_mod, :(module MacroMod + module MacroModInner; end + macro m_setglobal(); esc(:(mmglobal0 = 0)); end + macro m_eval_inner(x); :(@eval $MacroModInner $x) ; end + end)) + Core.@latestworld + @eval test_mod.EvalMod (const MacroMod2 = $(test_mod.MacroMod)) + @eval test_mod.EvalMod (const var"@legacy_quote_to_syntax" = + $(test_mod.var"@legacy_quote_to_syntax")) + @test run(raw"@eval EvalMod @legacy_quote_to_syntax(:(1 + 1))") isa treetype + @test run(raw"@eval @eval EvalMod @legacy_quote_to_syntax(:(1 + 1))") isa treetype + + @test run(raw"@eval EvalMod MacroMod2.@m_setglobal") == 0 + Core.@latestworld + @test isdefined(test_mod.EvalMod, :mmglobal0) + @test !isdefined(test_mod, :mmglobal0) + @test !isdefined(test_mod.MacroMod, :mmglobal0) + + @test run(raw"@eval EvalMod MacroMod2.@m_eval_inner(global mmglobal1 = 1)") == 1 + Core.@latestworld + @test isdefined(test_mod.MacroMod.MacroModInner, :mmglobal1) + @test !isdefined(test_mod, :mmglobal1) + @test !isdefined(test_mod.MacroMod, :mmglobal1) + @test !isdefined(test_mod.EvalMod, :mmglobal1) + + # interpolation into top-level: symbol declared in the new module + run(raw"let x = @legacy_quote_to_syntax(:sym) + @eval EvalMod module tmp; module inner_eval_mod; global $x = 123; end; end + end") isa Module + Core.@latestworld + @test test_mod.EvalMod.tmp.inner_eval_mod isa Module + @test test_mod.EvalMod.tmp.inner_eval_mod.sym == 123 + + # hygiene + run("let eval_result = 0; @eval 1+1; eval_result; end") == 0 + + @testset "(AI) single-arg @eval does not over-preserve hygiene" for expr_compat_mode in (true, false) + root = @newmod(root) + JuliaLowering.include_string(root, raw""" + module MacB + import JuliaLowering.@legacy_quote_to_syntax + macro do_eval() + @legacy_quote_to_syntax quote + @eval (@__MODULE__) + end + end + end + module MacA + import JuliaLowering.@legacy_quote_to_syntax + import ..MacB + macro wrap() + @legacy_quote_to_syntax quote + @eval (@__MODULE__) + end + end + macro via_b() + @legacy_quote_to_syntax quote + MacB.@do_eval() + end + end + macro wrap_ee() + @legacy_quote_to_syntax quote + @eval @eval (@__MODULE__) + end + end + macro wrap_two_arg() + # two-arg control: explicit target module; the payload's + # `@__MODULE__` must still see the *target* module + @legacy_quote_to_syntax quote + @eval MacB (@__MODULE__) + end + end + macro wrap_arg(ex) + # caller-provided payload (caller's hygiene layer) + @legacy_quote_to_syntax quote + @eval $ex + end + end + macro wrap_fn() + # `@eval` captures the module current when the enclosing function + # *definition* is expanded, like flisp + @legacy_quote_to_syntax quote + () -> @eval (@__MODULE__) + end + end + macro mkmod() + mod = gensym("EvalMod") + @legacy_quote_to_syntax quote + @eval module $mod + const inside = (@__MODULE__) + end + end + end + macro mkmod_payload(ex) + mod = gensym("EvalMod2") + @legacy_quote_to_syntax quote + @eval module $mod + $ex + end + end + end + end + module Sub + import ..MacA + end + """; expr_compat_mode) + Core.@latestworld + + run(str) = JuliaLowering.include_string(root, str; expr_compat_mode) + + # `@eval` inside another macro's unescaped expansion evaluates in the + # caller's module, not the macro's + @test run("MacA.@wrap()") === root + # ... even when the `@eval`-ing macro is called by another macro's expansion + # (flisp: still the dynamic module, not either macro's module) + @test run("MacA.@via_b()") === root + # `@eval` nested in `@eval` re-expands against the outer target + @test run("MacA.@wrap_ee()") === root + # two-arg control: explicit module wins; payload `@__MODULE__` follows it + @test run("MacA.@wrap_two_arg()") === root.MacB + # macro-generated closure: `@eval` binds the definition-time module + @test Base.invokelatest(run("MacA.@wrap_fn()")) === root + # the same macro evaluated into a different module follows the live module + @test JuliaLowering.include_string( + root.Sub, "MacA.@wrap()"; expr_compat_mode) === root.Sub + + # Caller-provided payloads evaluate in the caller's module + @test run("MacA.@wrap_arg(arg_marker = (@__MODULE__))") === root + if !expr_compat_mode + # With SyntaxTree-passed arguments the payload keeps the caller's + # hygiene: the global lands in `root` and is visible there. (In + # expr_compat_mode the old-style Expr round-trip re-layers the payload + # with the macro's hygiene and the assignment becomes a hygienic + # toplevel local -- a pre-existing divergence from flisp tracked by + # the "hygienic toplevel assignments" TODO in scope_analysis.jl.) + @test Base.invokelatest(isdefined, root, :arg_marker) + @test Base.invokelatest(getfield, root, :arg_marker) === root + end + + # The SafeTestsets shape: a macro-generated `@eval module $mod ... end` + # creates the module under the dynamic (caller) module + m = run("MacA.@mkmod()") + @test m isa Module + @test parentmodule(m) === root + @test Base.invokelatest(getfield, m, :inside) === m + # ... and user payload interpolated into the module body sees the fresh + # module as its dynamic module (a user's own `@eval` inside a + # `@safetestset` acts on the anonymous test module) + m2 = run("MacA.@mkmod_payload(@eval user_marker = (@__MODULE__))") + @test m2 isa Module + @test parentmodule(m2) === root + @test Base.invokelatest(getfield, m2, :user_marker) === m2 + + if expr_compat_mode + # Escaped expansions (old-style macros only): same dynamic target + JuliaLowering.include_string(root, raw""" + module MacEsc + macro wrap_esc() + esc(quote + @eval esc_marker = (@__MODULE__) + end) + end + end + """; expr_compat_mode) + Core.@latestworld + @test run("MacEsc.@wrap_esc()") === root + @test Base.invokelatest(isdefined, root, :esc_marker) + @test Base.invokelatest(getfield, root, :esc_marker) === root + end + + # An old-style (flisp-defined and -lowered) macro whose expansion calls + # `@eval` gets the same treatment when invoked under JuliaLowering + fl_eval(root, :(module MacFl + macro flwrap() + quote + @eval (@__MODULE__) + end + end + end)) + Core.@latestworld + @test run("MacFl.@flwrap()") === root + end + + @testset "(AI) const shows up in caller mod" begin + Core.eval(test_mod, :(module MacHome2 + macro make_const() + :( @eval const CMARKER = 42 ) + end + end)) + Core.eval(test_mod, :(MacHome2.@make_const())) + @test isdefined(test_mod, :CMARKER) + + JuliaLowering.eval(test_mod, :(module MacHome2 + macro make_const() + :( @eval const CMARKER = 42 ) + end + end); expr_compat_mode=true) + JuliaLowering.eval(test_mod, :(MacHome2.@make_const())) + @test isdefined(test_mod, :CMARKER) + end +end + +@eval test_mod module hscope_mod; global hscope_g = 123; end +@eval test_mod module nothing_mod; end +@eval test_mod global hscope_g = 234 +@testset "hygienic scope should be usable without macros" begin + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), + 1, test_mod); expr_compat_mode=true) == 1 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), + :hscope_g, + test_mod.hscope_mod); expr_compat_mode=true) == 123 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), + Expr(:escape, :hscope_g), + test_mod.nothing_mod); expr_compat_mode=true) == 234 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), Expr( + Symbol("hygienic-scope"), + Expr(:escape, Expr(:escape, :hscope_g)), + test_mod.nothing_mod), + test_mod.nothing_mod); expr_compat_mode=true) == 234 + @test JuliaLowering.eval( + test_mod, Expr( + Symbol("hygienic-scope"), Expr( + Symbol("hygienic-scope"), + Expr(:escape, :hscope_g), + test_mod.nothing_mod), + test_mod.hscope_mod); expr_compat_mode=true) == 123 +end + +Base.eval(test_mod, :( + test_hscope(x, mod=$test_mod) = Expr(Symbol("hygienic-scope"), x, mod) +)) +Base.eval(test_mod, :( + # +3 new scopes and -4 escapes = normal unhygienic macro + macro oldstyle_silly_scopes(x, y) + stmt1 = test_hscope(test_hscope(test_hscope(esc(esc(esc(esc(:($x = 123)))))))) + stmt2 = esc(test_hscope(esc(test_hscope(esc(test_hscope(esc(:($y = 456)))))))) + Expr(:block, stmt1, stmt2) + end)) +@testset "escape and hygienic-scope forms" for run in [ + (x::String)->Base.include_string( + test_mod, "#=FLISP SANITY-CHECK=# "*x), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL COMPAT=# "*x; expr_compat_mode=true), + (x::String)->JuliaLowering.include_string( + test_mod, "#=JL=# "*x; expr_compat_mode=false)] + + @test run(raw""" + let (x, y) = (0, 0); @oldstyle_silly_scopes(x, y); (x, y); end + """) === (123, 456) + @test run(raw"""begin + global_x, global_y = 0, 0 + @oldstyle_silly_scopes(global_x, global_y) + global_x, global_y + end""") === (123, 456) +end + +@testset "apply_expansion_layer mutation testing" begin + local test_mod = @newmod(apply_expansion_layer) + # recursion can't stop at module/toplevel/inert without tweaks, because a + # macro can pull random stuff out of it. This also tests calling into macro + # expansion from macros, mostly because re-using macros I've already written + # is the easiest way to create non-surface-syntax SyntaxTree as of writing. + JuliaLowering.include_string(test_mod, raw""" + macro undo_inert(x) + x2 = JuliaLowering.macroexpand(x) + x2[1] + end + """) + @test JuliaLowering.include_string(test_mod, raw""" + let foo = 1; @undo_inert(:foo); end + """) == 1 + @test JuliaLowering.include_string(test_mod, raw""" + let foo = 1; @undo_inert(@legacy_quote_to_syntax(:foo)); end + """) == 1 + + JuliaLowering.include_string(test_mod, raw""" + macro mk_toplevel(x, y, z) + JuliaSyntax.newnode( + __context__.macrocall, K"toplevel", + JuliaSyntax.SyntaxList(x, y, z)) + end + macro toplevel_first_child(x) + x2 = JuliaLowering.macroexpand(x) + x2[1] + end + """) + JuliaLowering.include_string(test_mod, raw""" + macro mk_module(x, y, z) + @legacy_quote_to_syntax :(module mk_module_mod; $x; $y; $z; end) + end + macro module_first_child(x) + x2 = JuliaLowering.macroexpand(x) + x2[end][1] + end + """) + # sanity + @test JuliaLowering.include_string(test_mod, """ + @mk_toplevel(1, :y, "z") + """) == "z" + @test JuliaLowering.include_string(test_mod, """ + @mk_module(1, :y, "z") + """) isa Module + @test JuliaLowering.include_string(test_mod, """ + let (x, y, z) = (1, :y, "z") + @toplevel_first_child(@mk_toplevel(x, y, z)) + end + """) == 1 + @test JuliaLowering.include_string(test_mod, """ + let (x, y, z) = (1, :y, "z") + @toplevel_first_child(@mk_toplevel(x, y, z)) + end + """) == 1 + + # escape should obey quote/unquote + JuliaLowering.include_string(test_mod, raw""" + macro esc_in_quote(); Expr(:quote, Expr(:escape, :x)); end + """; expr_compat_mode=true) + @test JuliaLowering.include_string(test_mod, raw""" + @esc_in_quote + """; expr_compat_mode=true) == Expr(:escape, :x) + @test JuliaLowering.include_string(test_mod, raw""" + @esc_in_quote + """) == Expr(:escape, :x) + + JuliaLowering.include_string(test_mod, raw""" + macro esc_in_unquote(); Expr(:quote, Expr(:$, Expr(:escape, :x))); end + """; expr_compat_mode=true) + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1; @esc_in_unquote(); end + """; expr_compat_mode=true) == 1 + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1; @esc_in_unquote(); end + """) == 1 +end + +JuliaLowering.include_string(test_mod, raw""" +module M + using ..JuliaLowering: JuliaLowering, adopt_scope, @legacy_quote_to_syntax + using ..JuliaSyntax + + # Introspection + macro __MODULE__() + JuliaLowering.syntax_module(__context__.macrocall) + end + + macro __FILE__() + JuliaLowering.filename(__context__.macrocall) + end + + macro __LINE__() + JuliaLowering.source_location(__context__.macrocall)[1] + end + + someglobal = "global in module M" + + # Macro with local variables + macro foo(ex) + @legacy_quote_to_syntax :(begin + x = "`x` from @foo" + (x, someglobal, $ex) + end) + end + + # Set `a_global` in M + macro set_a_global(val) + @legacy_quote_to_syntax :(begin + global a_global = $val + end) + end + + macro set_other_global(ex, val) + @legacy_quote_to_syntax :(begin + global $ex = $val + end) + end + + macro set_global_in_parent(ex) + sym_ex = @legacy_quote_to_syntax quote; sym_introduced_from_M; end + e1 = adopt_scope(__context__.macrocall, sym_ex[1]) + @legacy_quote_to_syntax quote + $e1 = $ex + nothing + end + end + + macro inner() + @legacy_quote_to_syntax :(y, z) + end + + macro outer() + @legacy_quote_to_syntax :((x, @inner)) + end + + macro recursive(N) + Nval = N.value::Int + if Nval < 1 + return N + end + @legacy_quote_to_syntax quote + x = $N + (x, @recursive $(Nval-1)) + end + end +end +""") + +@test JuliaLowering.include_string(test_mod, """ +let + x = "`x` from outer scope" + M.@foo x +end +""") == ("`x` from @foo", "global in module M", "`x` from outer scope") +@test !isdefined(test_mod.M, :x) + + +@test JuliaLowering.include_string(test_mod, """ +#line1 +(M.@__MODULE__(), M.@__FILE__(), M.@__LINE__()) +""", "foo.jl") == (test_mod, "foo.jl", 2) + +@test !isdefined(test_mod.M, :a_global) +@test JuliaLowering.include_string(test_mod, """ +begin + M.@set_a_global 42 + M.a_global +end +""") == 42 + +JuliaLowering.include_string(test_mod, """ +M.@set_global_in_parent "bent hygiene!" +""") +@test test_mod.sym_introduced_from_M == "bent hygiene!" + +JuliaLowering.include_string(test_mod, "M.@set_other_global global_in_test_mod 100") +@test !isdefined(test_mod.M, :global_in_test_mod) +@test test_mod.global_in_test_mod == 100 + +@test JuliaLowering.include_string(test_mod, """ +M.@recursive 3 +""") == (3, (2, (1, 0))) + +ex = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "M.@outer()", filename="foo.jl") +expanded = JuliaLowering.macroexpand(test_mod, ex) +@test JuliaSyntax.sourcetext.(JuliaLowering.flattened_provenance(expanded[2])) == [ + "M.@outer()" + "@inner" + "(y, z)" +] + +@testset "expansion special case: macrocall in do expression" for expr_compat_mode in [true, false] + @test JuliaLowering.include_string(test_mod, raw""" + macro mac_called_in_do_expression(dofunc, arg) + @legacy_quote_to_syntax :($dofunc($arg)) + end + """; expr_compat_mode) isa Function + @test JuliaLowering.include_string(test_mod, raw""" + @mac_called_in_do_expression(9) do x + x * 10 + end + """; expr_compat_mode) == 90 + @test JuliaLowering.include_string(test_mod, raw""" + let fp = @cfunction(Cint, (Cint,)) do x + x + Cint(1) + end + ccall(fp isa Ptr ? fp : fp.ptr, Cint, (Cint,), 2) + end + """; expr_compat_mode) == 3 +end + +@test JuliaLowering.include_string(test_mod, raw""" +v"1.14" +""") isa VersionNumber +@test JuliaLowering.include_string(test_mod, raw""" +v"1.14" +""";expr_compat_mode=true) isa VersionNumber +@test JuliaLowering.include_string(test_mod, raw""" +Base.Experimental.@VERSION +""") isa NamedTuple +@test JuliaLowering.include_string(test_mod, raw""" +Base.Experimental.@VERSION +""";expr_compat_mode=true) isa NamedTuple + +# World age support for macro expansion +JuliaLowering.include_string(test_mod, raw""" +macro world_age_test() + 1 +end +""") +world1 = Base.get_world_counter() +JuliaLowering.include_string(test_mod, raw""" +macro world_age_test() + 2 +end +""") +world2 = Base.get_world_counter() + +call_world_arg_test = JuliaLowering.rebase_layers(JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "@world_age_test()"), test_mod, JuliaSyntax.JL_NEW_SYNTAX_VERSION) + @test JuliaLowering.expand_forms_1(call_world_arg_test, world1, true) ≈ + @ast_ 1::K"Value" + @test JuliaLowering.expand_forms_1(call_world_arg_test, world2, true) ≈ + @ast_ 2::K"Value" + +JuliaLowering.include_string(test_mod, """ +f_throw(x) = throw(x) +macro m_throw(x) + :(\$(f_throw(x))) +end +""") +let (err, st) = try + JuliaLowering.include_string(test_mod, "_never_exist = @m_throw 42") + catch e + e, stacktrace(catch_backtrace()) + end + @test err isa JuliaLowering.MacroExpansionError + @test !isnothing(err.err) + # Check that `catch_backtrace` can capture the stacktrace of the macro functions + @test any(sf->sf.func===:f_throw, st) + # TODO: store this in DebugInfo + @test_broken any(sf->sf.func===Symbol("@m_throw"), st) + @test any(sf->sf.func===Symbol("macro expansion"), st) +end + +let err = try + JuliaLowering.include_string(test_mod, "_never_exist = @m_not_exist 42") + catch e + e + end + @test err isa JuliaLowering.MacroExpansionError + @test err.msg == "Macro not found" + @test err.err isa UndefVarError +end + +@testset "(AI) macro name resolution" begin + lib = @newmod(macname_lib, test_mod) + def = @newmod(macname_def, test_mod) + use = @newmod(macname_use, test_mod) + for (m, s) in ((lib, :lib_inner), (def, :def_inner), (use, :use_inner)) + fl_eval(m, :(macro inner(); QuoteNode($(QuoteNode(s))); end)) + end + fl_eval(lib, :(macro name(); Symbol("@inner"); end)) + fl_eval(lib, :(macro name_esc(); esc(Symbol("@inner")); end)) + fl_eval(def, :(import ..macname_lib: @name, @name_esc)) + fl_eval(def, :(const N = $lib)) + fl_eval(use, :(const M = $lib)) + fl_eval(def, quote + macro only_in_def(); QuoteNode(:only_in_def); end + macro outer_esc(); Expr(:macrocall, esc(Symbol("@inner")), __source__); end + macro outer_esc_missing(); Expr(:macrocall, esc(Symbol("@only_in_def")), __source__); end + macro outer_hyg(); Expr(:macrocall, Symbol("@inner"), __source__); end + macro outer_esc_dot(); Expr(:macrocall, esc(Expr(:., :M, QuoteNode(Symbol("@inner")))), __source__); end + macro outer_escl_dot(); Expr(:macrocall, Expr(:., esc(:M), QuoteNode(Symbol("@inner"))), __source__); end + macro outer_hyg_dot(); Expr(:macrocall, Expr(:., :N, QuoteNode(Symbol("@inner"))), __source__); end + macro outer_mc(); Expr(:macrocall, Expr(:macrocall, Symbol("@name"), __source__), __source__); end + macro outer_mc_esc(); Expr(:macrocall, Expr(:macrocall, Symbol("@name_esc"), __source__), __source__); end + macro outer_hs(); Expr(:macrocall, Expr(Symbol("hygienic-scope"), Symbol("@inner"), N), __source__); end + macro outer_gr(); Expr(:macrocall, GlobalRef(N, Symbol("@inner")), __source__); end + macro outer_val(); Expr(:macrocall, N.var"@inner", __source__); end + end) + + # New-style expansions: caller-supplied identifier keeps the caller's layer + JuliaLowering.include_string(def, raw""" + macro outer_new_hyg(); @legacy_quote_to_syntax(:(@inner())); end + macro outer_new_hyg_dot(); @legacy_quote_to_syntax(:(N.@inner())); end + macro outer_new_dot(m); @legacy_quote_to_syntax(:($m.@inner())); end + """) + fl_eval(use, :(import ..macname_def: @outer_esc, @outer_esc_missing, + @outer_hyg, @outer_esc_dot, @outer_escl_dot, @outer_hyg_dot, + @outer_mc, @outer_mc_esc, @outer_hs, @outer_gr, @outer_val, + @outer_new_hyg, @outer_new_hyg_dot, @outer_new_dot)) + Core.@latestworld + + for expr_compat_mode in (true, false) + jl(ex) = jl_eval(use, ex; expr_compat_mode) + @test jl(:(@outer_esc())) === :use_inner + err = try jl(:(@outer_esc_missing())); nothing; catch e; e; end + @test err isa MacroExpansionError + @test err.err isa UndefVarError && err.err.var === Symbol("@only_in_def") + @test err.err.scope === use + # An unescaped name resolves in the macro's module (hygiene) + @test jl(:(@outer_hyg())) === :def_inner + # Dotted names: the leftmost identifier resolves in its own layer, + # whether that differs from the `.`'s layer (`escl_dot`) or not + @test jl(:(@outer_esc_dot())) === :lib_inner + @test jl(:(@outer_escl_dot())) === :lib_inner + @test jl(:(@outer_hyg_dot())) === :lib_inner + # A macrocall in name position: its output is hygienic in its macro's + # module (`lib`), unless the output escapes the name (then `def`) + @test jl(:(@outer_mc())) === :lib_inner + @test jl(:(@outer_mc_esc())) === :def_inner + # Explicit hygienic-scope, GlobalRef and function values in name position + @test jl(:(@outer_hs())) === :lib_inner + @test jl(:(@outer_gr())) === :lib_inner + @test jl(:(@outer_val())) === :lib_inner + # New-style macros + @test jl(:(@outer_new_hyg())) === :def_inner + @test jl(:(@outer_new_hyg_dot())) === :lib_inner + @test jl(:(@outer_new_dot(M))) === :lib_inner + end + + # flisp agrees on every case it can express + fl(ex) = fl_eval(use, ex) + @test fl(:(@outer_hyg())) === :def_inner + @test fl(:(@outer_hyg_dot())) === :lib_inner + @test fl(:(@outer_mc())) === :lib_inner + @test fl(:(@outer_mc_esc())) === :def_inner + @test fl(:(@outer_hs())) === :lib_inner + @test fl(:(@outer_gr())) === :lib_inner + @test fl(:(@outer_val())) === :lib_inner + for ex in (:(@outer_esc()), :(@outer_esc_dot()), :(@outer_escl_dot())) + err = try fl(ex); nothing; catch e; e; end + err isa LoadError && (err = err.error) + @test err isa ErrorException && occursin("used outside of macro expansion", err.msg) + end +end + +@test JuliaLowering.include_string(test_mod, "@ccall strlen(\"foo\"::Cstring)::Csize_t") == 3 +@test JuliaLowering.include_string(test_mod, "@ccall gc_safe=true strlen(\"asdf\"::Cstring)::Csize_t") == 4 +@test JuliaLowering.include_string(test_mod, """ +begin + buf = zeros(UInt8, 20) + @ccall sprintf(buf::Ptr{UInt8}, "num:%d str:%s"::Cstring; 42::Cint, "hello"::Cstring)::Cint + String(buf) +end +""") == "num:42 str:hello\0\0\0\0" + +let (err, st) = try + JuliaLowering.include_string(test_mod, "@ccall strlen(\"foo\"::Cstring)") + catch e + e, stacktrace(catch_backtrace()) + end + @test err isa JuliaLowering.MacroExpansionError + @test err.msg == "expected a return type annotation `::SomeType`" + @test isnothing(err.err) + # Check that `catch_backtrace` can capture the stacktrace of the macro function + @test any(sf->sf.func===:ccall_macro_parse, st) +end + +# Tests for interop between old and new-style macros + +# Hygiene interop: +# call_oldstyle_macro -> oldstyle -> newstyle3 +JuliaLowering.include_string(test_mod, raw""" + macro call_oldstyle_macro(a) + @legacy_quote_to_syntax quote + x = "x in call_oldstyle_macro" + @oldstyle $a x + end + end + + macro newstyle3(a, b, c) + @legacy_quote_to_syntax quote + x = "x in @newstyle3" + ($a, $b, $c, x) + end + end +""") +# TODO: Make this macro lowering go via JuliaSyntax rather than the flisp code +# (JuliaSyntax needs support for old-style quasiquote processing) +Base.eval(test_mod, :( +macro oldstyle(a, b) + quote + x = "x in @oldstyle" + @newstyle3 $(esc(a)) $(esc(b)) x + end +end +)) +@test JuliaLowering.include_string(test_mod, """ +let x = "x in outer scope" + @call_oldstyle_macro x +end +""") == ("x in call_oldstyle_macro", + "x in call_oldstyle_macro", + "x in @oldstyle", + "x in @newstyle3") +# # would be ideal, but we can't get hygiene through oldstyle +# ("x in outer scope", +# "x in call_oldstyle_macro", +# "x in @oldstyle", +# "x in @newstyle3") + +# Old style unhygienic escaping with esc() +Base.eval(test_mod, :( +macro oldstyle_unhygienic() + esc(:x) +end +)) +@test JuliaLowering.include_string(test_mod, """ +let x = "x in outer scope" + @oldstyle_unhygienic +end +""") == "x in outer scope" + +# Exceptions in old style macros +Base.eval(test_mod, :( +macro oldstyle_error() + error("Some error in old style macro") +end +)) +@test try + JuliaLowering.include_string(test_mod, """ + @oldstyle_error + """) +catch exc + sprint(showerror, exc) +end == """ +LoadError: MacroExpansionError while expanding @oldstyle_error in module Main.macro_test: +@oldstyle_error +└─────────────┘ ── Error expanding macro +Caused by: +Some error in old style macro +in expression starting at string:1""" + +# Old-style macros returning non-Expr values +Base.eval(test_mod, :( +macro oldstyle_non_Expr() + 42 +end +)) +@test JuliaLowering.include_string(test_mod, """ +@oldstyle_non_Expr +""") === 42 + +# New-style macros called with the wrong arguments +JuliaLowering.include_string(test_mod, raw""" +macro method_error_test(a) +end +""") +Base.eval(test_mod, :( +macro method_error_test() +end +)) +try + JuliaLowering.include_string(test_mod, raw""" + @method_error_test x y + """) + @test false +catch exc + @test exc isa LoadError + mexc = exc.error.err + @test mexc isa MethodError + @test mexc.args isa Tuple{JuliaLowering.MacroContext, JuliaLowering.SyntaxTree, JuliaLowering.SyntaxTree} +end + +@testset "calling with old/new macro signatures" begin + # Old defined with 1 arg, new with 2 args, both with 3 (but with different values) + Base.eval(test_mod, :(macro sig_mismatch(x); x; end)) + Base.eval(test_mod, :(macro sig_mismatch(x, y, z); z; end)) + JuliaLowering.include_string(test_mod, "macro sig_mismatch(x, y); x; end") + JuliaLowering.include_string(test_mod, "macro sig_mismatch(x, y, z); x; end") + + @test JuliaLowering.include_string(test_mod, "@sig_mismatch(1)") === 1 + @test JuliaLowering.include_string(test_mod, "@sig_mismatch(1, 2)") === 1 + @test JuliaLowering.include_string(test_mod, "@sig_mismatch(1, 2, 3)") === 1 # 3 if we prioritize old sig + err = try + JuliaLowering.include_string(test_mod, "@sig_mismatch(1, 2, 3, 4)") === 1 + catch exc + sprint(showerror, exc, context=:module=>test_mod) + end + @test startswith(err, """ + LoadError: MacroExpansionError while expanding @sig_mismatch in module Main.macro_test: + @sig_mismatch(1, 2, 3, 4) + └───────────────────────┘ ── Error expanding macro + Caused by: + MethodError: no method matching var"@sig_mismatch"(""") +end + +@testset "old macros producing exotic expr heads (or are otherwise complex)" for expr_compat_mode in [true, false] + @test JuliaLowering.include_string(test_mod, """ + let # example from @preserve docstring + x = Ref{Int}(101) + p = Base.unsafe_convert(Ptr{Int}, x) + GC.@preserve x unsafe_load(p) + end"""; expr_compat_mode) === 101 # Expr(:gc_preserve) + + # JuliaLowering.jl/issues/121 + @test JuliaLowering.include_string(test_mod, """ + GC.@preserve @static if true @__MODULE__ else end + """) isa Module + @test JuliaLowering.include_string(test_mod, """ + GC.@preserve @static if true v"1.14" else end + """; expr_compat_mode) isa VersionNumber + + # JuliaLowering.jl/issues/144 + @test JuliaLowering.include_string(test_mod, """ + f_preserve144() = let + val = Any[] + GC.@preserve val begin; end + end + f_preserve144() + """; expr_compat_mode) == nothing + + # JuliaLowering.jl/issues/145 + @test JuliaLowering.include_string(test_mod, """ + f_preserve145() = let + debug_buffer = IOBuffer() + # inside function to force compilation + GC.@preserve debug_buffer 1 + end + f_preserve145() + """; expr_compat_mode) == 1 + + # only invokelatest produces :isglobal now, so MWE here + Base.eval(test_mod, :(macro isglobal(x); esc(Expr(:isglobal, x)); end)) + @test JuliaLowering.include_string(test_mod, """ + some_global = 1 + function isglobal_chk(some_arg) + local some_local = 1 + (@isglobal(some_undefined), @isglobal(some_global), @isglobal(some_arg), @isglobal(some_local)) + end + isglobal_chk(1) + """; expr_compat_mode) === (true, true, false, false) + # with K"Placeholder"s + @test JuliaLowering.include_string(test_mod, """ + __ = 1 + function isglobal_chk(___) + local ____ = 1 + (@isglobal(_), @isglobal(__), @isglobal(___), @isglobal(____)) + end + isglobal_chk(1) + """; expr_compat_mode) === (false, false, false, false) + + # @test appears to be the only macro in base to use :inert + test_result = JuliaLowering.include_string(test_mod, """ + using Test + @test identity(123) === 123 + """; expr_compat_mode) + @test test_result.value === true + + # @enum produces Expr(:toplevel) + JuliaLowering.include_string(test_mod, """ + @enum SOME_ENUM X1 X2 X3 + """; expr_compat_mode) + Core.@latestworld + @test test_mod.SOME_ENUM <: Enum + @test test_mod.X1 isa Enum + + # @deprecate also produces Expr(:toplevel), and :public with expression + # hygiene different from the contained names. + @testset "@deprecate" begin + @test JuliaLowering.include_string(test_mod, """ + module DeprecateMod + d2(x) = x+1 + @deprecate d1(x) d2(0) + end + """; expr_compat_mode) isa Module + Core.@latestworld + @test isdefined(test_mod.DeprecateMod, :d2) + @test isdefined(test_mod.DeprecateMod, :d1) + @test Base.isexported(test_mod.DeprecateMod, :d1) + @test !Base.isexported(test_mod, :d1) + end + + # @testset produces :tryfinally with secret third arg + @eval test_mod :(using Test) + @test JuliaLowering.include_string(test_mod, "@test true") isa Test.Pass + @testset let jltestset = JuliaLowering.include_string(test_mod, """ + @testset begin + @test true + end + """; expr_compat_mode) + @test jltestset isa Test.AbstractTestSet + @test jltestset.n_passed == 1 + end + + # aliasscope + @test jl_eval( + test_mod, + :(function simple_aliasscope(A, B) + Base.Experimental.@aliasscope @inbounds for I in eachindex(A, B) + A[I] = Base.Experimental.Const(B)[I] + end + return 0 + end); expr_compat_mode) isa Function + @test jl_eval( + test_mod, + :(let A = [1,2,3], B = [4,5,6] + simple_aliasscope(A,B), A, B + end); expr_compat_mode) == (0, [4,5,6], [4,5,6]) + + @test JuliaLowering.include_string(test_mod, """@fastmath 1 <= 2 <= 3""") + + # top-level nospecialize + @test JuliaLowering.include_string(test_mod, """ + module NospecializeMod + @nospecialize + f(x) = x + end + first(methods(NospecializeMod.f)).nospecialize + """) == -1 +end + +@testset "empty meta" begin + @test fl_eval(test_mod, Expr(:meta)) == nothing + @test fl_eval(test_mod, Expr(:block, Expr(:meta))) == nothing + @test fl_eval(test_mod, Expr(:call, + Expr(:function, Expr(:call, :func_empty_meta), + Expr(:block, Expr(:meta))))) == nothing + @test jl_eval(test_mod, Expr(:meta)) == nothing + @test jl_eval(test_mod, Expr(:block, Expr(:meta))) == nothing + @test jl_eval(test_mod, Expr(:call, + Expr(:function, Expr(:call, :func_empty_meta), + Expr(:block, Expr(:meta))))) == nothing +end + +@testset "macros producing meta forms" for expr_compat_mode in [true, false] + function find_method_ci(thunk) + ci = thunk.args[1]::Core.CodeInfo + m = findfirst(ci.code) do x + x isa Expr && x.head === :call && length(x.args) >= 5 && + x.args[1] isa GlobalRef && x.args[1].name === :define_method + end + ci.code[m].args[5] + end + jlower_e(s) = JuliaLowering.to_lowered_expr( + JuliaLowering.lower( + test_mod, JuliaLowering.parsestmt( + JuliaLowering.SyntaxTree, s); + expr_compat_mode)) + + prog = "Base.@assume_effects :foldable function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).purity === find_method_ci(our).purity + + prog = "Base.@inline function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).inlining === find_method_ci(our).inlining + + prog = "Base.@noinline function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).inlining === find_method_ci(our).inlining + + prog = "Base.@constprop :none function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).constprop === find_method_ci(our).constprop + + prog = "Base.@nospecializeinfer function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).nospecializeinfer === find_method_ci(our).nospecializeinfer + + prog = "Base.@propagate_inbounds function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).propagate_inbounds === find_method_ci(our).propagate_inbounds + + prog = "Base.@assume_effects :total @inline function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).inlining === find_method_ci(our).inlining + @test find_method_ci(ref).purity === find_method_ci(our).purity + + prog = "Base.@assume_effects :consistent Base.@assume_effects :nothrow function foo(); end" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).purity === find_method_ci(our).purity + + prog = "Base.@pure @inline foo(x) = x + 1" + ref = Meta.lower(test_mod, Meta.parse(prog)) + our = jlower_e(prog) + @test find_method_ci(ref).purity === find_method_ci(our).purity + @test find_method_ci(ref).inlining === find_method_ci(our).inlining + + # TODO: no api for option retrieval, just check that it compiles + let options_mod = Module() + @test fl_eval(options_mod, :(Base.Experimental.@optlevel 1)) == nothing + @test jl_eval(options_mod, :(Base.Experimental.@optlevel 1)) == nothing + @test fl_eval(options_mod, :(Base.Experimental.@max_methods 1)) == nothing + @test jl_eval(options_mod, :(Base.Experimental.@max_methods 1)) == nothing + end +end + +# partially robot-generated +@testset "meta-like forms not using the `meta` expression" for expr_compat_mode in (true,false) + @testset "in value position" begin + @test fl_eval(test_mod, Expr(:boundscheck)) isa Bool + @test jl_eval(test_mod, Expr(:boundscheck); expr_compat_mode) isa Bool + + @test fl_eval(test_mod, Expr(:inbounds, true)) === nothing + @test fl_eval(test_mod, Expr(:inbounds, false)) === nothing + @test fl_eval(test_mod, Expr(:inbounds, :pop)) === nothing + @test jl_eval(test_mod, Expr(:inbounds, true); expr_compat_mode) === nothing + @test jl_eval(test_mod, Expr(:inbounds, false); expr_compat_mode) === nothing + @test jl_eval(test_mod, Expr(:inbounds, :pop); expr_compat_mode) === nothing + + @testset for inline in (:inline, :noinline) + @testset let ex = Expr(:block, + Expr(inline, true), + Expr(inline, false)) + @test fl_eval(test_mod, ex) === nothing + @test jl_eval(test_mod, ex; expr_compat_mode) === nothing + end + @testset let ex = Expr(:function, Expr(:tuple), + Expr(:block, + Expr(inline, true), + Expr(inline, false))) + local f + f = fl_eval(test_mod, ex) + Core.@latestworld + @test f() === nothing + + f = jl_eval(test_mod, ex; expr_compat_mode) + Core.@latestworld + @test f() === nothing + end + end + end + + function find_method_ci(thunk) + ci = thunk.args[1]::Core.CodeInfo + m = findfirst(ci.code) do x + x isa Expr && x.head === :call && length(x.args) >= 5 && + x.args[1] isa GlobalRef && x.args[1].name === :define_method + end + ci.code[m].args[5] + end + jlower_e(s) = JuliaLowering.to_lowered_expr( + JuliaLowering.lower( + test_mod, JuliaLowering.parsestmt( + JuliaLowering.SyntaxTree, s); + expr_compat_mode)) + our_ssaflags(prog) = find_method_ci(jlower_e(prog)).ssaflags + + local INBOUNDS = Core.Compiler.IR_FLAG_INBOUNDS + local INLINE = Core.Compiler.IR_FLAG_INLINE + local NOINLINE = Core.Compiler.IR_FLAG_NOINLINE + + # `compute_ssaflags` shifts the encoded purity overrides up by NUM_IR_FLAGS. + purity_mask(eo::Base.EffectsOverride) = + UInt32(Base.encode_effects_override(eo)) << Core.Compiler.NUM_IR_FLAGS + + # check any IR statement in `prog` has `flags` + has_any(prog, flags) = any(f -> (f & flags) == flags, our_ssaflags(prog)) + has_none(prog, flags) = all(f -> (f & flags) == 0, our_ssaflags(prog)) + + @testset "boundscheck" begin + JuliaLowering.include_string(test_mod, """ + @inline function g_boundscheck(A, i) + @boundscheck checkbounds(A, i) + return A[i] + end + """; expr_compat_mode) + @test test_mod.g_boundscheck(1:2, 2) == 2 + @test_throws BoundsError test_mod.g_boundscheck(1:2, 3) + + # The boundscheck marker itself does not set IR_FLAG_INBOUNDS — it is + # a separate runtime predicate, not an annotation. + @test has_none("function f(A,i); @boundscheck checkbounds(A,i); A[i]; end", + INBOUNDS) + # `Expr(:boundscheck)` should survive lowering as a top-level + # statement (it gets rewritten by inlining/codegen, not lowering). + let our = find_method_ci(jlower_e( + "function f(A,i); @boundscheck checkbounds(A,i); A[i]; end")) + @test any(s -> s isa Expr && s.head === :boundscheck, our.code) + end + end + + @testset "inbounds" begin + JuliaLowering.include_string(test_mod, """ + function sum_inbounds(A::AbstractArray) + r = zero(eltype(A)) + for i in eachindex(A) + @inbounds r += A[i] + end + return r + end + """; expr_compat_mode) + @test test_mod.sum_inbounds([1,2,3]) == 6 + + @test has_none("function f(A,i); A[i]; end", INBOUNDS) + @test has_any("function f(A,i); @inbounds A[i]; end", INBOUNDS) + @test has_any(""" + function f(A) + s = zero(eltype(A)) + @inbounds for i in eachindex(A) + s += A[i] + end + s + end + """, INBOUNDS) + let flags = our_ssaflags(""" + function f(A, i, j) + z = @inbounds A[i] + A[j] + end + """) + @test any(f -> (f & INBOUNDS) != 0, flags) # inside @inbounds + @test any(f -> (f & INBOUNDS) == 0, flags) # outside + end + end + + @testset "inline" begin + @test has_any("function f(g,x); @inline g(x); end", INLINE) + @test has_none("function f(g,x); g(x); end", INLINE) + @test has_none("function f(g,x); @inline g(x); end", NOINLINE) + @test has_any("function f(g,x); @inline g(x) + g(x); end", INLINE) + + # Bare `@inline` inside a function body (1.8+) emits + # `Expr(:meta, :inline)`; no statement gets a call-site IR_FLAG_INLINE. + JuliaLowering.include_string(test_mod, """ + function bare_inline(x) + @inline + x * 2 + end + """; expr_compat_mode) + @test test_mod.bare_inline(3) == 6 + @test has_none("function f(x); @inline; x * 2; end", INLINE) + + # `@inline` on a definition is handled by the meta-expression path + # (covered in "macros producing meta forms"); confirm it still runs + # and that no call-site INLINE bit leaks into the body. + JuliaLowering.include_string(test_mod, """ + @inline f_inline_def(x) = x + 1 + """; expr_compat_mode) + @test test_mod.f_inline_def(2) == 3 + @test has_none("@inline f(x) = x + 1", INLINE) + end + + @testset "noinline" begin + # Analogous to `@inline` but pushes IR_FLAG_NOINLINE. + @test has_any("function f(g,x); @noinline g(x); end", NOINLINE) + @test has_none("function f(g,x); g(x); end", NOINLINE) + @test has_none("function f(g,x); @noinline g(x); end", INLINE) + @test has_any("function f(g,x); @noinline g(x) + g(x); end", NOINLINE) + + JuliaLowering.include_string(test_mod, """ + function bare_noinline(x) + @noinline + x * 2 + end + """; expr_compat_mode) + @test test_mod.bare_noinline(3) == 6 + @test has_none("function f(x); @noinline; x * 2; end", NOINLINE) + + JuliaLowering.include_string(test_mod, """ + @noinline f_noinline_def(x) = x + 1 + """; expr_compat_mode) + @test test_mod.f_noinline_def(2) == 3 + + # Innermost annotation wins when @inline / @noinline nest: the inner + # call gets IR_FLAG_INLINE; the outer @noinline still applies to + # statements outside the inner region. + let flags = our_ssaflags(""" + function f(g, x) + @noinline let + a = @inline g(x) + b = g(x) + (a, b) + end + end + """) + @test any(f -> (f & INLINE) != 0, flags) + @test any(f -> (f & NOINLINE) != 0, flags) + end + end + + @testset "purity" begin + # Sanity: plain function with no purity annotation has no purity bits set. + @test has_none("function f(g,x); g(x); end", + UInt32(0xFFFF) << Core.Compiler.NUM_IR_FLAGS) + # `@assume_effects :foo expr` at a call site expands to + # (block (purity ...11 bool args...) (local (= val expr)) (purity) val) + # where the trailing zero-arg `(purity)` is the region-end token. + @test has_any("function f(g,x); Base.@assume_effects :nothrow g(x); end", + purity_mask(Base.EffectsOverride(nothrow=true))) + # Multiple atomic settings combine to set both bits at once. + @test has_any( + "function f(g,x); Base.@assume_effects :consistent :effect_free g(x); end", + purity_mask(Base.EffectsOverride(consistent=true, effect_free=true))) + + # Function form goes through a different path: `(meta (purity args...))` + JuliaLowering.include_string(test_mod, """ + Base.@assume_effects :total f_assume_def(x) = x + """; expr_compat_mode) + @test test_mod.f_assume_def(5) == 5 + prog_def = "Base.@assume_effects :total function f_assume_total(x); x; end" + ref_ci = find_method_ci(Meta.lower(test_mod, Meta.parse(prog_def))) + our_ci = find_method_ci(jlower_e(prog_def)) + @test ref_ci.purity === our_ci.purity + + prog = """ + Base.@assume_effects :total function f_assume_nospecialize(x) + @nospecialize x + x + end + """ + ref_ci = find_method_ci(fl_lower(test_mod, Meta.parse(prog))) + our_ci = find_method_ci(jlower_e(prog)) + @test ref_ci.purity === our_ci.purity + end +end + +@testset "scope layers for normally-inert ASTs" begin + # Right hand side of `.` + @test JuliaLowering.include_string(test_mod, raw""" + let x = @legacy_quote_to_syntax :(hi) + @legacy_quote_to_syntax :(A.$x) + end + """) ≈ @ast_ [K"." + "A"::K"Identifier" + [K"inert" "hi"::K"Identifier"] + ] + # module + @test JuliaLowering.include_string(test_mod, raw""" + let x = @legacy_quote_to_syntax :(AA) + @legacy_quote_to_syntax :(module $x end) + end + """) ≈ @ast_ [K"module" + v"1.14.0"::K"Value" + true::K"Value" + "AA"::K"Identifier" + [K"block"] + ] + + # In macro expansion, require that expressions passed in as macro + # *arguments* get the lexical scope of the calling context, even for the + # `x` in `M.$x` where the right hand side of `.` is normally quoted. + @test JuliaLowering.include_string(test_mod, raw""" + let x = @legacy_quote_to_syntax :(someglobal) + @eval M.$x + end + """; expr_compat_mode=false) == "global in module M" + @test JuliaLowering.include_string(test_mod, raw""" + let x = @legacy_quote_to_syntax :(someglobal) + @eval M.$x + end + """; expr_compat_mode=true) == "global in module M" + + # @eval quoting should embed the value, not the syntax + @test JuliaLowering.include_string(test_mod, raw""" + let some_local = 101 + @eval module AA + x = $some_local + end + end + """; expr_compat_mode=false) isa Module + @test test_mod.AA.x == 101 + @test JuliaLowering.include_string(test_mod, raw""" + let some_local = 101 + @eval module AA + x = $some_local + end + end + """; expr_compat_mode=true) isa Module + @test test_mod.AA.x == 101 + + # "Deferred hygiene" in macros which emit quoted code. OK to break + # + # The old macro system doesn't handle this - here's the equivalent + # implementation + # macro make_quoted_code(init, y) + # QuoteNode(:(let + # x = "inner x" + # $(esc(init)) + # ($(esc(y)), x) + # end)) + # end + JuliaLowering.include_string(test_mod, raw""" + macro make_quoted_code(init, y) + q = @legacy_quote_to_syntax :(let + x = "inner x" + $init + ($y, x) + end) + @ast _ q [K"syntaxinert" q] + end + """) + code = JuliaLowering.include_string(test_mod, """@make_quoted_code(x="outer x", x)""") + @test JuliaLowering.eval(test_mod, code) == ("outer x", "inner x") +end + +@testset "toplevel macro hygiene" for run in [JuliaLowering.include_string, + Base.include_string] + @eval test_mod global mod = $test_mod + @eval test_mod module MacroMod + global mod = MacroMod + macro escaped_toplevel() + esc(Expr(:toplevel, :(mod))) + end + macro inner_escaped_toplevel() + Expr(:toplevel, esc(:(mod))) + end + macro unescaped_toplevel() + Expr(:toplevel, :(mod)) + end + end + Core.@latestworld + @test run(test_mod, "MacroMod.@escaped_toplevel") === test_mod + @test run(test_mod, "MacroMod.@inner_escaped_toplevel") === test_mod + @test run(test_mod, "MacroMod.@unescaped_toplevel") === test_mod.MacroMod + + unrelated = @newmod(unrelated) + @eval unrelated const MacroMod = $(test_mod.MacroMod) + @eval unrelated global mod = 123 + + @test run(unrelated, "MacroMod.@escaped_toplevel") == 123 + @test run(unrelated, "MacroMod.@inner_escaped_toplevel") == 123 + @test run(unrelated, "MacroMod.@unescaped_toplevel") === test_mod.MacroMod +end + +@testset "toplevel macro hygiene: @__MODULE__" for run in [JuliaLowering.include_string, + Base.include_string] + @eval test_mod module MacroMod + macro atmodule_in_toplevel() + Expr(:toplevel, :(@__MODULE__)) + end + macro atmodule_in_module() + Expr(:toplevel, Expr(:module, true, esc(:atmod_mod), Expr( + :block, :(global global_mod = @__MODULE__)))) + end + end + Core.@latestworld + @test run(test_mod, "MacroMod.@atmodule_in_toplevel") === test_mod + @test run(test_mod, "MacroMod.@atmodule_in_module") isa Module + Core.@latestworld + @test isdefined(test_mod, :atmod_mod) + @test test_mod.atmod_mod.global_mod == test_mod.atmod_mod +end + +# JuliaLang/JuliaLowering.jl#120 +# +# `__module__` should be expanded as the lexical module containing the expanded +# code, not the module corresponding to the current hygienic scope +JuliaLowering.include_string(test_mod, raw""" +module Mod1 +import ..JuliaLowering.@legacy_quote_to_syntax +macro indirect_MODULE() + return @legacy_quote_to_syntax :(@__MODULE__()) +end +end +""") +code = JuliaLowering.include_string(test_mod, """Mod1.@indirect_MODULE()""") +@test JuliaLowering.eval(test_mod, code) === test_mod # !== test_mod.Mod1 +# the lowering/eval iterator needs to expand in the correct world age (currently +# the only way to hit this from user code is macros producing toplevel) + +@testset "old macros defining modules" begin + # escaped module nested in tmpmod_1 + jl_eval(test_mod, :( + module MacMod + macro makemod(name) + Expr(:toplevel, + esc(Expr(:module, false, :tmpmod_1, + Expr(:block, + Expr(:module, false, name, + Expr(:block, Expr(:const, Expr(:(=), :c, 1)))))))) + end + end); expr_compat_mode=true) + + @testset for expr_compat_mode in [true, false] + @test JuliaLowering.include_string( + test_mod, "MacMod.@makemod(newmod)") isa Module + Core.@latestworld + # module name should escape macmod->test_mod + @test test_mod.tmpmod_1.newmod isa Module + @test !isdefined(test_mod.MacMod, :newmod) + @test !isdefined(test_mod.MacMod, :tmpmod_1) + # const in mod body should work + @test test_mod.tmpmod_1.newmod.c == 1 + end + + # escaped module name + jl_eval(test_mod, :( + module MacMod + macro makemod(name) + Expr(:toplevel, + Expr(:module, false, esc(name), + Expr(:block, + Expr(:const, Expr(:(=), esc(:c), 1))))) + end + end); expr_compat_mode=true) + + @testset for expr_compat_mode in [true, false] + @test JuliaLowering.include_string( + test_mod, "MacMod.@makemod(newmod)") isa Module + Core.@latestworld + # module name should escape macmod->test_mod + @test test_mod.newmod isa Module + @test !isdefined(test_mod.MacMod, :newmod) + @test test_mod.newmod.c == 1 + end +end + +@testset "(AI) old macro attribution survives a nested eval in its body (#32)" begin + Base.eval(test_mod, :(module MacDefMod + const secret = 99 + macro getsecret() + __module__.eval(:(nested_eval_side_effect = 1 + 1)) + return :(secret) # bare name -> resolves in the defining module + end + end)) + Core.@latestworld + # `secret` must resolve in MacDefMod (== mod_for_ast), matching flisp. + @test JuliaLowering.include_string(test_mod, "MacDefMod.@getsecret()") == 99 + @test test_mod.nested_eval_side_effect == 2 + @test fl_eval(test_mod, :(MacDefMod.@getsecret())) == 99 +end + +@testset "macros defining macros" begin + @eval test_mod macro make_and_use_macro_toplevel() + Expr(:toplevel, + esc(:(macro from_toplevel_expansion() + :(123) + end)), + esc(:(@from_toplevel_expansion()))) + end + + @test JuliaLowering.include_string( + test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=true) === 123 + + if isdefined(test_mod, Symbol("@from_toplevel_expansion")) + Base.delete_binding(test_mod, Symbol("@from_toplevel_expansion")) + end + + @test JuliaLowering.include_string( + test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=false) === 123 + + # unescaped top-level macro should be visible in the old system + @test JuliaLowering.include_string(test_mod, raw""" + module MacrosDefiningMacros + macro make_old_unescaped_macro(name) + :(macro $name(x) + x + end) + end + macro make_old_escaped_macro(name) + :(macro $(esc(name))(x) + x + end) + end + end + """; expr_compat_mode=true) isa Module + @test JuliaLowering.include_string(test_mod, raw""" + MacrosDefiningMacros.@make_old_unescaped_macro make_old_unescaped_macro_out + """; expr_compat_mode=true) isa Function + @test JuliaLowering.include_string(test_mod, raw""" + MacrosDefiningMacros.@make_old_escaped_macro make_old_escaped_macro_out + """; expr_compat_mode=true) isa Function + @test JuliaLowering.include_string(test_mod, raw""" + MacrosDefiningMacros.@make_old_unescaped_macro_out 1 + """; expr_compat_mode=true) == 1 + @test JuliaLowering.include_string(test_mod, raw""" + @make_old_escaped_macro_out 1 + """; expr_compat_mode=true) == 1 + + # standard scope-layer-carrying macro arg `name` should work in new system + @test JuliaLowering.include_string(test_mod, raw""" + macro make_new_nameprovided_macro(name) + @legacy_quote_to_syntax :(macro $name(x) + x + end) + end + @make_new_nameprovided_macro make_new_nameprovided_macro_out + @make_new_nameprovided_macro_out 1 + """; expr_compat_mode=false) == 1 + + # unhygienic new macro def shouldn't be visible (may change) + @test JuliaLowering.include_string(test_mod, raw""" + macro make_new_anaphoric_macro_fail() + @legacy_quote_to_syntax :(macro make_new_anaphoric_macro_fail_out(x) + x + end) + end + @make_new_anaphoric_macro_fail + """; expr_compat_mode=false) isa Function + @test_throws MacroExpansionError JuliaLowering.include_string( + test_mod, "@make_new_anaphoric_macro_fail_out"; expr_compat_mode=false) +end + +@testset "SIMD loopinfo" begin + @test JuliaLowering.include_string(test_mod, raw""" + @eval let + n = 10 + x = zeros(n) + i = 1 + while i ≤ n + x[i] += 1 + i += 1 + $(Expr(:loopinfo, Symbol("julia.simdloop"), nothing)) # Mark loop as SIMD loop + end + sum(x) + end + """; expr_compat_mode=true) == 10.0 + + @test JuliaLowering.include_string(test_mod, raw""" + @eval let + n = 10 + x = zeros(n) + i = 1 + while i ≤ n + x[i] += 1 + i += 1 + $(Expr(:loopinfo, Symbol("julia.simdloop"), Symbol("julia.ivdep"))) # Mark loop as SIMD loop + end + sum(x) + end + """; expr_compat_mode=true) == 10.0 + + JuliaLowering.include_string(test_mod, """ + @noinline function inner(x, y) + s = zero(eltype(x)) + for i in eachindex(x, y) + @inbounds s += x[i]*y[i] + end + return s + end + """) + + JuliaLowering.include_string(test_mod, """ + @noinline function innersimd(x, y) + s = zero(eltype(x)) + @simd for i in eachindex(x, y) + @inbounds s += x[i] * y[i] + end + return s + end + """) + + @test test_mod.inner([1,2,3], [1,2,3]) == 14 + @test test_mod.innersimd([1,2,3], [1,2,3]) == 14 +end + +@testset "@__FUNCTION__ and Expr(:thisfunction)" begin + @testset "Basic usage" begin + # @__FUNCTION__ in regular functions + JuliaLowering.include_string(test_mod, raw""" + test_function_basic() = @__FUNCTION__ + """; expr_compat_mode=true) + @test test_mod.test_function_basic() === test_mod.test_function_basic + + # Expr(:thisfunction) in regular functions + JuliaLowering.include_string(test_mod, raw""" + @eval regular_func() = @__FUNCTION__ + """; expr_compat_mode=true) + @test test_mod.regular_func() === test_mod.regular_func + end + + @testset "Recursion" begin + # Factorial with @__FUNCTION__ + JuliaLowering.include_string(test_mod, raw""" + factorial_function(n) = n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1) + """; expr_compat_mode=true) + @test test_mod.factorial_function(5) == 120 + + # Fibonacci with Expr(:thisfunction) + JuliaLowering.include_string(test_mod, raw""" + struct RecursiveCallableStruct; end + (::RecursiveCallableStruct)(n) = n <= 1 ? n : @__FUNCTION__()(n-1) + @__FUNCTION__()(n-2) + """; expr_compat_mode=true) + @test test_mod.RecursiveCallableStruct()(10) === 55 + + # Anonymous function recursion + @test JuliaLowering.include_string(test_mod, raw""" + (n -> n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1))(5) + """; expr_compat_mode=true) == 120 + end + + @testset "Closures and nested functions" begin + # Prevents boxed closures + JuliaLowering.include_string(test_mod, raw""" + function make_closure() + fib(n) = n <= 1 ? 1 : (@__FUNCTION__)(n - 1) + (@__FUNCTION__)(n - 2) + return fib + end + """; expr_compat_mode=true) + Test.@inferred test_mod.make_closure() + closure = test_mod.make_closure() + @test closure(5) == 8 + Test.@inferred closure(5) + + # Complex closure of closures + JuliaLowering.include_string(test_mod, raw""" + function f1() + function f2() + function f3() + return @__FUNCTION__ + end + return (@__FUNCTION__), f3() + end + return (@__FUNCTION__), f2()... + end + """; expr_compat_mode=true) + Test.@inferred test_mod.f1() + @test test_mod.f1()[1] === test_mod.f1 + @test test_mod.f1()[2] !== test_mod.f1 + @test test_mod.f1()[3] !== test_mod.f1 + @test test_mod.f1()[3]() === test_mod.f1()[3] + @test test_mod.f1()[2]()[2]() === test_mod.f1()[3] + end + + @testset "Do blocks" begin + function test_do_block() + result = JuliaLowering.include_string(test_mod, raw""" + map([1, 2, 3]) do x + return (@__FUNCTION__, x) + end + """; expr_compat_mode=true) + # All should refer to the same do-block function + @test all(r -> r[1] === result[1][1], result) + # Values should be different + @test [r[2] for r in result] == [1, 2, 3] + # It should be different than `test_do_block` + @test result[1][1] !== test_do_block + end + test_do_block() + end + + @testset "Keyword arguments" begin + # @__FUNCTION__ with kwargs + JuliaLowering.include_string(test_mod, raw""" + f_thisfunction_kw(; n) = n <= 1 ? 1 : n * (@__FUNCTION__)(; n = n - 1) + """; expr_compat_mode=true) + @test test_mod.f_thisfunction_kw(n = 5) == 120 + + # Expr(:thisfunction) with kwargs + JuliaLowering.include_string(test_mod, raw""" + f_thisfunction_kw2(; n=1) = n <= 1 ? n : n * @__FUNCTION__()(; n=n-1) + """; expr_compat_mode=true) + result = test_mod.f_thisfunction_kw2(n=5) + @test result == 120 + end + + @testset "Callable structs" begin + # @__FUNCTION__ in callable structs + JuliaLowering.include_string(test_mod, raw""" + module A + struct CallableStruct{T}; val::T; end + (c::CallableStruct)() = @__FUNCTION__ + end + """; expr_compat_mode=true) + JuliaLowering.include_string(test_mod, raw""" + using .A: CallableStruct + """; expr_compat_mode=true) + c = test_mod.CallableStruct(5) + @test c() === c + + # In closures, var"#self#" should refer to the enclosing function, + # NOT the enclosing struct instance + JuliaLowering.include_string(test_mod, raw""" + struct CallableStruct2; end + @eval function (obj::CallableStruct2)() + function inner_func() + @__FUNCTION__ + end + inner_func + end + """; expr_compat_mode=true) + + let cs = test_mod.CallableStruct2() + @test cs()() === cs() + @test cs()() !== cs + end + + # Accessing values via self-reference + JuliaLowering.include_string(test_mod, raw""" + struct CallableStruct3 + value::Int + end + (obj::CallableStruct3)() = @__FUNCTION__() + (obj::CallableStruct3)(x) = @__FUNCTION__().value + x + """; expr_compat_mode=true) + + let cs = test_mod.CallableStruct3(42) + @test cs() === cs + @test cs(10) === 52 + end + + # Callable struct with args and kwargs + JuliaLowering.include_string(test_mod, raw""" + struct CallableStruct4 + end + @eval function (obj::CallableStruct4)(x, args...; y=2, kws...) + return (; func=(@__FUNCTION__), x, args, y, kws) + end + """; expr_compat_mode=true) + c = test_mod.CallableStruct4() + @test c(1).func === c + @test c(2, 3).args == (3,) + @test c(2; y=4).y == 4 + @test c(2; y=4, a=5, b=6, c=7).kws[:c] == 7 + end + + @testset "Special cases" begin + # Generated functions + JuliaLowering.include_string(test_mod, raw""" + let + @generated foo2() = @__FUNCTION__ + foo2() === foo2 + end + """; expr_compat_mode=true) + + # Struct constructors + let + JuliaLowering.include_string(test_mod, raw""" + struct Cols{T<:Tuple} + cols::T + operator + Cols(args...; operator=union) = (new{typeof(args)}(args, operator); string(@__FUNCTION__)) + end + """; expr_compat_mode=true) + result = @invokelatest test_mod.Cols(1, 2, 3) + @test occursin("Cols", result) + end + + # Should not access arg-map for local variables + # TODO: worth the special case? + JuliaLowering.include_string(test_mod, raw""" + function f_thisfunction_argmap end + function (f_thisfunction_argmap::typeof(f_thisfunction_argmap))() + f_thisfunction_argmap = 1 + @__FUNCTION__ + end + """; expr_compat_mode=true) + @test_broken test_mod.f_thisfunction_argmap() === + test_mod.f_thisfunction_argmap + end + + @test JuliaLowering.include_string(test_mod, """ + @eval let f=[ ()->$(Expr(:thisfunction)) for i = 1:1 ][1]; f() === f; end + """; expr_compat_mode=true) +end + +@testset "macro source LineNumberNode" begin + Base.include_string(test_mod, raw""" + macro srcfile() + string(__source__.file) + end + """) + + mac_ex = Expr(:macrocall, Symbol("@srcfile"), LineNumberNode(1, "goodfile")) + mac_st = JuliaLowering.expr_to_est(mac_ex, LineNumberNode(1, "badfile")) + + @test JuliaLowering.eval(test_mod, mac_st) === "goodfile" + + # tolerate nothing + mac_ex = Expr(:macrocall, Symbol("@srcfile"), nothing) + mac_st = JuliaLowering.expr_to_est(mac_ex, LineNumberNode(1, "badfile")) + @test JuliaLowering.eval(test_mod, mac_st) == "none" +end + +@testset "macro QuoteNode + inert behavior" begin + Base.include_string(test_mod, raw""" + macro quoted_gr() + QuoteNode(GlobalRef(Base, :dontresolveme)) + end + """) + let gr = JuliaLowering.include_string(test_mod, "@quoted_gr") + @test gr.mod === Base + @test gr.name === :dontresolveme + end +end + +@testset "Base macros" begin + jl_eval(test_mod, + :(function test_invokelatest() + @eval invokelatest_target(x, y) = x + y + out = @invokelatest(invokelatest_target(1, 2)) + Base.delete_binding(@__MODULE__, :invokelatest_target) + out + end)) + # the following test needs to define this to be effective + @test_throws UndefVarError JuliaLowering.include_string(test_mod, "invokelatest_target(1,2)") + @test JuliaLowering.include_string(test_mod, "test_invokelatest()") === 3 + + for expr_compat_mode in (false, true), + version in (v"1.13", v"1.14") + + _version = JuliaLowering.include_string(test_mod, + "Base.Experimental.@VERSION"; + expr_compat_mode, version + ) + @test _version isa NamedTuple + @test _version.syntax == version + end +end + +# produces import/using in module that is `@eval`ed. +@testset "safetestset" begin + macro_mod = @newmod(macro_mod, test_mod) + JuliaLowering.include_string(macro_mod, raw""" + macro safetestset(testname, expr) + quote + @eval module $(gensym("safetestset_mod")) + using Test + @testset $testname $expr + end + nothing + end + end + """; expr_compat_mode=true) + + JuliaLowering.include_string(test_mod, """ + macro_mod.@safetestset "Tests" begin + a = 1; b = 2; c = a + b; @test c == 3 + @isdefined(a) == true + end + """; expr_compat_mode=true) + @test !isdefined(test_mod, :a) + @test !isdefined(macro_mod, :a) +end + +# Method annotations propagate from the body to positional-default wrappers +# and the `Core.kwcall` sorter (matching flisp's `propagate-method-meta`). +@testset "method meta propagation" begin + JuliaLowering.include_string(test_mod, raw""" + @inline function _test_opts_kw(x::Int, y::Int=1; k::Int=2) + x + y + k + end + """) + fkw = test_mod._test_opts_kw + @test fkw(1) == 4 + for m in (which(fkw, (Int,)), which(fkw, (Int, Int)), + which(Core.kwcall, (NamedTuple{(:k,), Tuple{Int}}, typeof(fkw), Int))) + @test Base.uncompressed_ast(m).inlining == 0x01 + end + + # Destructuring prepends assignments but must retain the metadata. + JuliaLowering.include_string(test_mod, raw""" + @inline function _test_opts_destr((a, b)::Tuple{Int,Int}, y::Int=1; k::Int=2) + a + b + y + k + end + """) + fd = test_mod._test_opts_destr + @test fd((1, 2)) == 6 + for m in (which(fd, (Tuple{Int,Int},)), which(fd, (Tuple{Int,Int}, Int)), + which(Core.kwcall, (NamedTuple{(:k,), Tuple{Int}}, typeof(fd), Tuple{Int,Int}))) + @test Base.uncompressed_ast(m).inlining == 0x01 + end + + # @nospecializeinfer + local f = JuliaLowering.include_string(@newmod(), raw""" + Base.@nospecializeinfer function f(@nospecialize(x), y::Int=1) + (x, y) + end + """) + @test which(f, (Any, Int)).nospecializeinfer == true + @test which(f, (Any,)).nospecializeinfer == true + + local f = JuliaLowering.include_string(@newmod(), raw""" + Base.@nospecializeinfer function f(@nospecialize(x); k::Int=1) + (x, k) + end + """) + local sorter = x->which(Core.kwcall, (NamedTuple{(:k,), Tuple{Int}}, typeof(x), Any)) + @test sorter(f).nospecializeinfer == true +end diff --git a/JuliaLowering/test/macros_ir.jl b/JuliaLowering/test/macros_ir.jl new file mode 100644 index 0000000000000..8a42f6a9a6aae --- /dev/null +++ b/JuliaLowering/test/macros_ir.jl @@ -0,0 +1,329 @@ +module MacroMethods + using ..JuliaLowering + macro some_macro() + JuliaLowering.@legacy_quote_to_syntax quote + some_global + end + end + + module ExtraMacroMethods + using ..MacroMethods + using ..JuliaLowering + macro MacroMethods.some_macro(ex) + JuliaLowering.@legacy_quote_to_syntax quote + some_global + end + end + end +end + +macro strmac_str(ex, suff=nothing) + s = "$(ex.value) from strmac" + if !isnothing(suff) + s = "$s with suffix $(suff.value)" + end + s +end + +macro cmdmac_cmd(ex, suff=nothing) + s = "$(ex.value) from cmdmac" + if !isnothing(suff) + s = "$s with suffix $(suff.value)" + end + s +end + +#******************************************************************************* +######################################## +# Simple macro +macro add_one(ex) + JuliaLowering.@legacy_quote_to_syntax quote + $ex + 1 + end +end +#--------------------- +1 (call core.define_method TestMod :@add_one) +2 latestworld +3 TestMod.@add_one +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ JuliaLowering.MacroContext core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.@add_one %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/ex] + 1 (call core.tuple slot₃/ex) + 2 (call JuliaLowering.interpolate_syntax (syntaxinert (block (call + (syntaxunquote ex) 1))) %₁) + 3 (return %₂) +10 latestworld +11 TestMod.@add_one +12 (return %₁₁) + +######################################## +# Macro using `__context__` +macro foo(ex) + ctx = __context__ +end +#--------------------- +1 (call core.define_method TestMod :@foo) +2 latestworld +3 TestMod.@foo +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ JuliaLowering.MacroContext core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.@foo %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/__context__ slot₃/ex(!read) slot₄/ctx(!read,single_assign)] + 1 slot₂/__context__ + 2 (= slot₄/ctx %₁) + 3 (return %₁) +10 latestworld +11 TestMod.@foo +12 (return %₁₁) + +######################################## +# Scope for symbols emitted by macros is the module where the method was +# defined, thus two different modules in this case, even though `@some_macro` +# belongs to the MacroMethods module. +(MacroMethods.@some_macro(), MacroMethods.@some_macro(unused)) +#--------------------- +1 TestMod.MacroMethods.some_global +2 TestMod.MacroMethods.ExtraMacroMethods.some_global +3 (call core.tuple %₁ %₂) +4 (return %₃) + +######################################## +# Error: Macro with kw args +macro mmm(a; b=2) +end +#--------------------- +LoweringError: +macro mmm(a; b=2) +# ╙ ── macros cannot accept keyword arguments +end + +######################################## +# Error: Macro with `where` +macro mmm(a::T) where T +end +#--------------------- +LoweringError: +macro mmm(a::T) where T +# └───────────────┘ ── `where` not allowed in macro signatures +end + +######################################## +# Error: Bad macro name +macro mmm[](ex) +end +#--------------------- +LoweringError: +macro mmm[](ex) +# └───┘ ── invalid macro name +end + +######################################## +# Error: Macros not allowed in local scope +let + macro foo(ex) + end +end +#--------------------- +LoweringError: +let +# ┌──────────── + macro foo(ex) + end +#─────┘ ── macro is only allowed in global scope +end + +######################################## +# Error: Macros not allowed in local scope +function f() + macro foo() + end +end +#--------------------- +LoweringError: +function f() +# ┌────────── + macro foo() + end +#─────┘ ── this syntax is only allowed at top level +end + +######################################## +# Error: Macros not found +_never_exist = @m_not_exist 42 +#--------------------- +MacroExpansionError while expanding @m_not_exist in module Main.TestMod: +_never_exist = @m_not_exist 42 +# └──────────┘ ── Macro not found +Caused by: +UndefVarError: `@m_not_exist` not defined in `Main.TestMod` +Suggestion: check for spelling errors or missing imports. + +######################################## +# Simple cmdstring +`echo 1` +#--------------------- +1 Base.cmd_gen +2 (call core.tuple "echo") +3 (call core.tuple "1") +4 (call core.tuple %₂ %₃) +5 (call %₁ %₄) +6 (return %₅) + +######################################## +# Simple string macro +strmac"hello" +#--------------------- +1 (return "hello from strmac") + +######################################## +# String macro with suffix +strmac"hello"blah +#--------------------- +1 (return "hello from strmac with suffix blah") + +######################################## +# Simple cmd macro +cmdmac`hello` +#--------------------- +1 (return "hello from cmdmac") + +######################################## +# Cmd macro with suffix +cmdmac`hello`12345 +#--------------------- +1 (return "hello from cmdmac with suffix 12345") + +######################################## +# @nospecialize at top-level +@nospecialize +#--------------------- +1 (meta :nospecialize) +2 (return core.nothing) + +######################################## +# @nospecialize (zero args) +function foo(a) + @nospecialize +end +#--------------------- +1 (call core.define_method TestMod :foo) +2 latestworld +3 TestMod.foo +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.foo %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/a(nospecialize,!read)] + 1 (meta :nospecialize) + 2 (return core.nothing) +10 latestworld +11 TestMod.foo +12 (return %₁₁) + +######################################## +# @nospecialize (single arg in body) +function foo(a, b) + @nospecialize a + a + b +end +#--------------------- +1 (call core.define_method TestMod :foo) +2 latestworld +3 TestMod.foo +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.foo %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/a(nospecialize) slot₃/b] + 1 (meta :nospecialize slot₂/a) + 2 TestMod.+ + 3 (call %₂ slot₂/a slot₃/b) + 4 (return %₃) +10 latestworld +11 TestMod.foo +12 (return %₁₁) + +######################################## +# @nospecialize (multi-arg in body) +function foo(x, y, z) + @nospecialize x z + x + y + z +end +#--------------------- +1 (call core.define_method TestMod :foo) +2 latestworld +3 TestMod.foo +4 (call core.TypeEqOf %₃) +5 (call core.svec %₄ core.Any core.Any core.Any) +6 (call core.svec) +7 SourceLocation::1:1 +8 (call core.svec %₅ %₆ %₇) +9 (call core.define_method TestMod TestMod.foo %₈ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(nospecialize) slot₃/y slot₄/z(nospecialize)] + 1 (meta :nospecialize slot₂/x slot₄/z) + 2 TestMod.+ + 3 (call %₂ slot₂/x slot₃/y slot₄/z) + 4 (return %₃) +10 latestworld +11 TestMod.foo +12 (return %₁₁) + +######################################## +# Error: thisfunction disallowed in comprehension/generator +[@__FUNCTION__() for x in 1:2] +#--------------------- +LoweringError: +[@__FUNCTION__() for x in 1:2] +#└─────────────┘ ── current function not defined in comprehension or generator + +######################################## +# Error: thisfunction disallowed in comprehension/generator +f(@__FUNCTION__() for x in 1:2) +#--------------------- +LoweringError: +f(@__FUNCTION__() for x in 1:2) +# └─────────────┘ ── current function not defined in comprehension or generator + +######################################## +# Error: thisfunction disallowed outside of function +let + @__FUNCTION__() +end +#--------------------- +LoweringError: +let + @__FUNCTION__() +# └─────────────┘ ── can only be used inside a function +end + +######################################## +# @overlay lowering +Base.Experimental.@overlay mt f() = 1 +#--------------------- +1 TestMod.f +2 (call core.TypeEqOf %₁) +3 (call core.svec %₂) +4 (call core.svec) +5 SourceLocation:nothing:1:0 +6 (call core.svec %₃ %₄ %₅) +7 (call core.define_method TestMod TestMod.mt %₆ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (return 1) +8 latestworld +9 (return %₇) diff --git a/JuliaLowering/test/misc.jl b/JuliaLowering/test/misc.jl new file mode 100644 index 0000000000000..a179535b41240 --- /dev/null +++ b/JuliaLowering/test/misc.jl @@ -0,0 +1,1005 @@ +test_mod = @newmod(misc) + +# Blocks +@test JuliaLowering.include_string(test_mod, """ +begin +end +""") == nothing + +# Placeholders +@test JuliaLowering.include_string(test_mod, """_ = 10""") == 10 + +# GC.@preserve +@test JuliaLowering.include_string(test_mod, """ +let x = [1,2] + GC.@preserve x begin + x + end +end +""") == [1,2] + +# empty strings +@test JuliaLowering.include_string(test_mod, raw""" +\"\"\" +\"\"\" +""") == "" +@test JuliaLowering.include_string(test_mod, raw""" +"" +""") == "" +@test JuliaLowering.include_string(test_mod, raw""" +"$("")" +""") == "" +@test JuliaLowering.include_string(test_mod, raw""" +\"\"\"$("")\"\"\" +""") == "" + +@test JuliaLowering.include_string(test_mod, raw""" +let + x = 10 + @eval $x + 2 +end +""") == 12 + +@test JuliaLowering.include_string(test_mod, raw""" +module EvalTest + _some_var = 1 +end +let + x = 10 + @eval EvalTest $x + _some_var +end +""") == 11 +@test JuliaLowering.include_string(test_mod, raw""" +module EvalTest2 + _some_var = 2 +end +let + x = 10 + @eval EvalTest2 $x + _some_var +end +"""; expr_compat_mode=true) == 12 + +@test JuliaLowering.include_string(test_mod, """ +let x=11 + 20x +end +""") == 220 + +@testset "syntactic --> <: >:" begin + @test jl_eval(test_mod, Expr(:<:, Int, Number)) + @test jl_eval(test_mod, Expr(:<:, Expr(:..., Expr(:tuple, Int, Number)))) + @test jl_eval(test_mod, Expr(:>:, Number, Int)) + @test jl_eval(test_mod, Expr(:>:, Expr(:..., Expr(:tuple, Number, Int)))) + + JuliaLowering.include_string(test_mod, """ + function var"-->"(args...; kws...) + (args, values(kws)) + end + """) + @test jl_eval(test_mod, Expr(:-->, 1, 2)) == ((1,2),(;)) + @test jl_eval(test_mod, Expr(:-->, 1, Expr(:kw, :foo, 2))) == ((1,),(;foo=2)) + @test jl_eval(test_mod, Expr(:-->, Expr(:..., (1,)))) == ((1,),(;)) +end + +@testset "empty symbol" begin + @test JuliaLowering.include_string(test_mod, """ + let var\"\"=1; 1; end + """) == 1 + + # Note function name fails in flisp + @test jl_eval(test_mod, + Expr(:let, Expr(:block), + Expr(:block, + Expr(:function, Expr(:call, Symbol(""), :x), + Expr(:block, :x)), + Expr(:call, Symbol(""), 1)))) == 1 + + # function arg + @test jl_eval(test_mod, + Expr(:let, Expr(:block), + Expr(:block, + Expr(:function, Expr(:call, :func, Symbol("")), + Expr(:block, Symbol(""))), + Expr(:call, :func, 2)))) == 2 + # kwarg + @test jl_eval(test_mod, + Expr(:let, Expr(:block), + Expr(:block, + Expr(:function, Expr(:call, :func, Expr(:parameters, Symbol(""))), + Expr(:block, Symbol(""))), + Expr(:call, :func, Expr(:kw, Symbol(""), 2))))) == 2 + + # empty label + @test jl_eval(test_mod, + Expr(:symbolicblock, Symbol(""), + Expr(:break, Symbol(""), 1))) == 1 + + # read empty local + @test jl_eval(test_mod, + Expr(:let, Expr(:block), + Expr(:block, Expr(:local, Symbol("")), + Expr(:(=), Symbol(""), 17), + Symbol("")))) == 17 + + # read empty global + @test jl_eval(test_mod, + Expr(:block, Expr(:global, Symbol("")), + Expr(:(=), Symbol(""), 7), + Symbol(""))) == 7 + + # typed read of empty + @test jl_eval(test_mod, + Expr(:let, Expr(:block, Expr(:(=), Symbol(""), 5)), + Expr(:block, Expr(:(::), Symbol(""), :Int)))) == 5 + + # empty in curly (read position) + @test jl_eval(test_mod, + Expr(:let, Expr(:block, Expr(:(=), Symbol(""), Int)), + Expr(:block, Expr(:curly, :Vector, Symbol(""))))) == Vector{Int} + + # empty in tuple (read position) + @test jl_eval(test_mod, + Expr(:let, Expr(:block, Expr(:(=), Symbol(""), 99)), + Expr(:block, Expr(:tuple, Symbol(""))))) == (99,) + + # isdefined on empty + @test jl_eval(test_mod, + Expr(:let, Expr(:block, Expr(:(=), Symbol(""), 1)), + Expr(:block, Expr(:isdefined, Symbol(""))))) + + # for-loop empty iter var referenced in body + @test jl_eval(test_mod, + Expr(:let, Expr(:block, Expr(:(=), :s, 0)), + Expr(:block, + Expr(:for, Expr(:(=), Symbol(""), Expr(:tuple, 10, 20, 30)), + Expr(:block, Expr(:(=), :s, Expr(:call, :+, :s, Symbol(""))))), + :s))) == 60 + + # tuple destructure with empty lhs + @test jl_eval(test_mod, + Expr(:let, Expr(:block), + Expr(:block, Expr(:local, Symbol("")), Expr(:local, :a), + Expr(:(=), Expr(:tuple, Symbol(""), :a), Expr(:tuple, 1, 2)), + Expr(:tuple, Symbol(""), :a)))) == (1, 2) + + # quote of empty + @test jl_eval(test_mod, Expr(:quote, Symbol("")); + expr_compat_mode=true) === Symbol("") +end + +@eval test_mod libccalltest_var = "libccalltest" + +@testset "cglobal" begin + cg = JuliaLowering.include_string(test_mod, """ + cglobal(:jl_, Any) + """) + @test cg isa Ptr{Any} + @test cg !== C_NULL + + cg = JuliaLowering.include_string(test_mod, """ + cglobal((:global_var, libccalltest_var), Cint) + """) + @test cg isa Ptr{Cint} + @test cg !== C_NULL + @test unsafe_load(cg) == 1 + + # the pointer-vs-name choice is syntactic, not value-based: a runtime + # variable holding a tuple takes the pointer form, which errors + @eval test_mod global cglobal_tuple = (:global_var, libccalltest_var) + @test_throws TypeError JuliaLowering.include_string(test_mod, """ + cglobal(cglobal_tuple, Cint) + """) + @test_throws TypeError JuliaLowering.include_string(test_mod, """ + let local_tuple = (:global_var, libccalltest_var) + cglobal(local_tuple, Cint) + end + """) + + # unlike the argtypes / rettype of a ccall, cglobal(name, T) should allow + # rettype T to be any runtime expression + cg = JuliaLowering.include_string(test_mod, """ + function cglobal_runtime_type(T) + cglobal((:global_var, libccalltest_var), T) + end + cglobal_runtime_type(Cint) + """) + @test cg isa Ptr{Cint} + @test unsafe_load(cg) == 1 + + # invalid foreignsymbol (tuple) forms should error for cglobal + @test_throws ErrorException JuliaLowering.include_string(test_mod, "cglobal((:a, :b, :c))") + @test_throws ErrorException JuliaLowering.include_string(test_mod, "cglobal(())") + @test_throws TypeError JuliaLowering.include_string(test_mod, "cglobal((1,))") + + # cglobal(name) with a non-static name errors, just like ccall + @test_throws TypeError JuliaLowering.include_string(test_mod, """ + function cglobal_non_static1() + sym = (:global_var, libccalltest_var) + cglobal(sym) + end + cglobal_non_static1() + """) + @eval test_mod global the_sym = (:global_var, libccalltest_var) + @test_throws TypeError JuliaLowering.include_string(test_mod, """ + cglobal_non_static2() = cglobal(the_sym) + cglobal_non_static2() + """) +end + +# ccall +@test JuliaLowering.include_string(test_mod, """ +ccall(:strlen, Csize_t, (Cstring,), "asdfg") +""") == 5 +@test JuliaLowering.include_string(test_mod, """ +function cvarargs_0() + strp = Ref{Ptr{Cchar}}(0) + fmt = "hi" + len = ccall(:asprintf, Cint, (Ptr{Ptr{Cchar}}, Cstring, Cfloat...), strp, fmt) + str = unsafe_string(strp[], len) + Libc.free(strp[]) + return str +end +""") isa Function +@test test_mod.cvarargs_0() == "hi" +@test JuliaLowering.include_string(test_mod, """ +function cvarargs_2(arg1::Float64, arg2::Float64) + strp = Ref{Ptr{Cchar}}(0) + fmt = "%3.1f %3.1f" + len = ccall(:asprintf, Cint, (Ptr{Ptr{Cchar}}, Cstring, Cfloat...), strp, fmt, arg1, arg2) + str = unsafe_string(strp[], len) + Libc.free(strp[]) + return str +end +""") isa Function +@test test_mod.cvarargs_2(1.1, 2.2) == "1.1 2.2" +# (function, library) syntax +@test JuliaLowering.include_string(test_mod, """ + ccall((:ctest, :libccalltest), Complex{Int}, (Complex{Int},), 10 + 20im) +""") === 11 + 18im +# (function, library): library is a global +@test JuliaLowering.include_string(test_mod, """ + ccall((:ctest, libccalltest_var), Complex{Int}, (Complex{Int},), 10 + 20im) +""") === 11 + 18im + +@testset "(robot-generated) ccall (sym, lib) tuple: globals and hygiene" for expr_compat_mode in [true, false] + # library is a module-qualified global + JuliaLowering.include_string(test_mod, """ + module CCallLibMod + const the_lib = "libccalltest" + end + """; expr_compat_mode) + Core.@latestworld + @test JuliaLowering.include_string(test_mod, """ + ccall((:ctest, CCallLibMod.the_lib), Complex{Int}, (Complex{Int},), 10 + 20im) + """) === 11 + 18im + + # macro in a nested module produces ccall with lib from that module (hygiene) + JuliaLowering.include_string(test_mod, raw""" + module CCallHygieneMod + const mylib = "libccalltest" + import ..JuliaLowering.@legacy_quote_to_syntax + macro do_ccall() + @legacy_quote_to_syntax( + :(ccall((:ctest, mylib), Complex{Int}, (Complex{Int},), 10 + 20im))) + end + end + """; expr_compat_mode) + Core.@latestworld + @test JuliaLowering.include_string(test_mod, """ + CCallHygieneMod.@do_ccall() + """) === 11 + 18im + + # hygiene: `mylib` in the macro body should resolve in CCallHygieneMod, not + # the caller, even when the caller defines a different `mylib` + @test JuliaLowering.include_string(test_mod, """ + mylib = "this_lib_does_not_exist" + CCallHygieneMod.@do_ccall() + """; expr_compat_mode) === 11 + 18im + + # macro that interpolates the lib value at expansion time + JuliaLowering.include_string(test_mod, raw""" + module CCallHygieneMod2 + import ..JuliaLowering.@legacy_quote_to_syntax + const mylib2 = "libccalltest" + macro do_ccall_interp() + lib = mylib2 + @legacy_quote_to_syntax( + :(ccall((:ctest, $lib), Complex{Int}, (Complex{Int},), 10 + 20im))) + end + end + """; expr_compat_mode) + Core.@latestworld + @test JuliaLowering.include_string(test_mod, """ + CCallHygieneMod2.@do_ccall_interp() + """; expr_compat_mode) === 11 + 18im + + # ccall with plain symbol name still works inside a function + @test JuliaLowering.include_string(test_mod, """ + function ccall_plain_sym() + ccall(:strlen, Csize_t, (Cstring,), "abc") + end + """; expr_compat_mode) isa Function + Core.@latestworld + @test test_mod.ccall_plain_sym() == 3 + + # ccall with (sym, lib) tuple where lib is a global, inside a function + @test JuliaLowering.include_string(test_mod, """ + function ccall_global_lib() + ccall((:ctest, libccalltest_var), Complex{Int}, (Complex{Int},), 10 + 20im) + end + """; expr_compat_mode) isa Function + Core.@latestworld + @test test_mod.ccall_global_lib() === 11 + 18im + + # ccall with module-qualified lib inside a function + @test JuliaLowering.include_string(test_mod, """ + function ccall_qualified_lib() + ccall((:ctest, CCallLibMod.the_lib), Complex{Int}, (Complex{Int},), 10 + 20im) + end + """; expr_compat_mode) isa Function + Core.@latestworld + @test test_mod.ccall_qualified_lib() === 11 + 18im +end + +# cfunction +JuliaLowering.include_string(test_mod, """ +function f_ccallable(x, y) + x + y * 10 +end +""") +cf_int = JuliaLowering.include_string(test_mod, """ +@cfunction(f_ccallable, Int, (Int,Int)) +""") +@test @ccall($cf_int(2::Int, 3::Int)::Int) == 32 +cf_float = JuliaLowering.include_string(test_mod, """ +@cfunction(f_ccallable, Float64, (Float64,Float64)) +""") +@test @ccall($cf_float(2::Float64, 3::Float64)::Float64) == 32.0 + +# Test that hygiene works with @ccallable function names +JuliaLowering.include_string(test_mod, raw""" +f_ccallable_hygiene() = 1 + +module Nested + import ..JuliaLowering.@legacy_quote_to_syntax + f_ccallable_hygiene() = 2 + macro cfunction_hygiene() + @legacy_quote_to_syntax :(@cfunction($f_ccallable_hygiene, Int, ())) + end +end +""") +cf_hygiene = JuliaLowering.include_string(test_mod, """ +Nested.@cfunction_hygiene +""") +@test @ccall($cf_hygiene()::Int) == 2 +# Same as above, but non-interpolated symbol. Arguably this could return 20, +# but if it should, this is a bug in the macro implementation, not lowering. +# Match Base for now. +JuliaLowering.include_string(test_mod, raw""" +f_ccallable_hygiene() = 10 + +module Nested + import ..JuliaLowering.@legacy_quote_to_syntax + f_ccallable_hygiene() = 20 + macro cfunction_hygiene() + @legacy_quote_to_syntax :(@cfunction(f_ccallable_hygiene, Int, ())) + end +end +""") +cf_hygiene = JuliaLowering.include_string(test_mod, """ +Nested.@cfunction_hygiene +""") +@test @ccall($cf_hygiene()::Int) == 10 + +@test JuliaLowering.include_string(test_mod, """ +cfunction_ignores_locals() = Int32(1) +function getptr() + cfunction_ignores_locals = @cfunction(cfunction_ignores_locals, Int32, ()) + cfunction_ignores_locals +end +getptr() isa Ptr{Cvoid} +""") + +# quoted function in cfunction +quoted_cfn_anon = JuliaLowering.include_string(test_mod, raw""" + @cfunction((function(x); x; end), Int, (Int,)) +""") +@test ccall(quoted_cfn_anon, Int, (Int,), 1) == 1 + +quoted_cfn_named = JuliaLowering.include_string(test_mod, raw""" + @cfunction((function fname_unused(x); x; end), Int, (Int,)) +""") +@test ccall(quoted_cfn_named, Int, (Int,), 1) == 1 + +# flisp-expanded ccall, cfunction should be lowerable by us +let fl_ex = macroexpand( + test_mod, + :(cfun_flisp_thunk() = @cfunction(+, Cint, (Cint, Cint)))) + + fl_st = JuliaLowering.expr_to_est(fl_ex) + fl_fn = JuliaLowering.eval(test_mod, fl_st) + fl_cfn = @invokelatest fl_fn() + @test fl_fn isa Function + @test fl_cfn isa Ptr + @test ccall(fl_cfn, Int, (Int,Int), 1, 2) == 3 +end +let fl_ex = macroexpand( + test_mod, + :(cfun_flisp_thunk() = @cfunction(function some_cfunc_add(x,y); x+y; end, + Cint, (Cint, Cint)))) + + fl_st = JuliaLowering.expr_to_est(fl_ex) + fl_fn = JuliaLowering.eval(test_mod, fl_st) + fl_cfn = @invokelatest fl_fn() + @test fl_fn isa Function + @test fl_cfn isa Ptr + @test ccall(fl_cfn, Int, (Int,Int), 1, 2) == 3 +end +let fl_ex = macroexpand( + test_mod, + :(@ccall(libccalltest_var.ctest((10+20im)::Complex{Int})::Complex{Int}))) + fl_st = JuliaLowering.expr_to_est(fl_ex) + @test JuliaLowering.eval(test_mod, fl_st) == 11 + 18im +end + +# raw :foreigncall should also be lowerable +let raw_foreigncall_ex = Expr( + :foreigncall, Expr( + :tuple, QuoteNode(:ctest), "libccalltest"), + :(Complex{Int}), + :(Core.svec(Complex{Int})), + 0, + QuoteNode((:ccall, 0x0000, false)), + 10+20im, + Complex{Int}) + + # test flisp does this: it's unclear how much desugaring the user is + # responsible for here + @test fl_eval(test_mod, raw_foreigncall_ex) == 11 + 18im + + @test JuliaLowering.eval(test_mod, JuliaLowering.expr_to_est(raw_foreigncall_ex)) == 11 + 18im +end + +# Test that ccall can be passed static parameters in type signatures. +# +# Note that the cases where this works are extremely limited and tend to look +# like `Ptr{T}` or `Ref{T}` (`T` doesn't work!?) because of the compilation +# order in which the runtime inspects the arguments to ccall (`Ptr{T}` has a +# well defined C ABI even when `T` is not yet determined). See also +# https://github.com/JuliaLang/julia/issues/29400 +# https://github.com/JuliaLang/julia/pull/40947 +JuliaLowering.include_string(test_mod, raw""" +function sparam_ccallable(x::Ptr{T}) where {T} + unsafe_store!(x, one(T)) + nothing +end + +function ccall_with_sparams(::Type{T}) where {T} + x = T[zero(T)] + cf = @cfunction(sparam_ccallable, Cvoid, (Ptr{T},)) + @ccall $cf(x::Ptr{T})::Cvoid + x[1] +end +""") +@test test_mod.ccall_with_sparams(Int) === 1 +@test test_mod.ccall_with_sparams(Float64) === 1.0 + +# (AI) has_fcall: sparam-dependent @cfunction must not inline into a caller +# knowing T only abstractly. The abstract field keeps T non-constant there. +@test JuliaLowering.include_string(test_mod, """ +cfunc_has_fcall(buf::Ptr{UInt8}, len::UInt32) = Int32(0) +get_cf(::Type{T}) where {T} = @cfunction cfunc_has_fcall Int32 (Ref{T}, Ptr{UInt8}, UInt32) +do_cf(stream::T) where {T <: IO} = get_cf(T) +mutable struct Box3 + x::IO +end +call_it(b::Box3) = do_cf(b.x) +call_it(Box3(IOBuffer())) +""") isa Ptr{Cvoid} + +# Test that ccall can be passed static parameters in the function name +# Note that this only works with `@generated` functions from 1.13 onwards, +# where the function name can be evaluated at code generation time. +JuliaLowering.include_string(test_mod, raw""" +# In principle, may add other strlen-like functions here for different string +# types +ccallable_sptest_name(::Type{String}) = :strlen + +@generated function ccall_with_sparams_in_name(s::T) where {T} + name = QuoteNode(ccallable_sptest_name(T)) + @legacy_quote_to_syntax :(ccall($name, Csize_t, (Cstring,), s)) +end +""") +@test test_mod.ccall_with_sparams_in_name("hii") == 3 + +# Where local variables may appear in ccall / cfunction type and name positions +@testset "(AI) local variables in ccall/cfunction type/name positions" begin + # fresh module per case: many of these define global methods + cc(code) = JuliaLowering.include_string(Module(:cc_locals), code) + + # ---- ALLOWED: a top-level local interpolated into a global method ---- + # return type + @test cc(""" + begin + local t = Csize_t + g() = ccall(:strlen, t, (Cstring,), "asdfg") + g() + end""") == 5 + # argument type + @test cc(""" + begin + local a = Cstring + g(s) = ccall(:strlen, Csize_t, (a,), s) + g("asdfg") + end""") == 5 + # library (2nd element of the name tuple): a constant string is fine + @test cc(""" + begin + local lib = "libccalltest" + g() = ccall((:ctest, lib), Complex{Int}, (Complex{Int},), 10+20im) + g() + end""") == 11 + 18im + # still a global method (its capture still top-level) when nested in a `let` + @test cc(""" + begin + local t = Csize_t + let + global g + g() = ccall(:strlen, t, (Cstring,), "asdfg") + end + g() + end""") == 5 + # cfunction mirrors ccall for the return and argument types + @test cc("""fcb(x) = x + 1; @cfunction(fcb, Int, (Int,)) isa Ptr""") + @test cc(""" + begin + local RT = Int + fcb(x) = x + 1 + g() = @cfunction(fcb, RT, (Int,)) + g() isa Ptr + end""") + @test cc(""" + begin + local AT = Int + fcb(x) = x + 1 + g() = @cfunction(fcb, Int, (AT,)) + g() isa Ptr + end""") + + # ---- REJECTED (`static_eval_disallowed_binding`) ---- + @test_broken false == """ + Note these should ideally all be LoweringError, but that's just so the error + prints nicely. flisp doesn't do much validation, and leaves it to the ccall + machinery to reject everything it can't handle. + """ + + # A same-frame local (declared in the method that runs the ccall) is a slot. + @test_throws LoweringError cc(""" + function g() + local rt = Csize_t + ccall(:strlen, rt, (Cstring,), "asdfg") + end + g()""") + # A genuine (local-name / escaping) closure captures the type as a *field*. + @test_throws ErrorException cc(""" + let t = Csize_t + g() = ccall(:strlen, t, (Cstring,), "asdfg") + g() + end""") + @test_throws ErrorException cc(""" + let a = Cstring + g(s) = ccall(:strlen, Csize_t, (a,), s) + g("asdfg") + end""") + # A *reassigned* (boxed) top-level local can't be baked in as a constant. + @test_throws LoweringError cc(""" + begin + local t = Csize_t + t = Csize_t + g() = ccall(:strlen, t, (Cstring,), "asdfg") + g() + end""") + # cfunction rejects the same shapes. + @test_throws LoweringError cc(""" + function g() + fcb(x) = x + 1 + local RT = Int + @cfunction(fcb, RT, (Int,)) + end + g()""") + @test_throws UndefVarError cc(""" + let RT = Int + fcb(x) = x + 1 + g() = @cfunction(fcb, RT, (Int,)) + g() + end""") + # ...and so does the library position. + @test_throws LoweringError cc(""" + function g() + local lib = "libccalltest" + ccall((:ctest, lib), Complex{Int}, (Complex{Int},), 10+20im) + end + g()""") + @test_throws TypeError cc(""" + let lib = "libccalltest" + g() = ccall((:ctest, lib), Complex{Int}, (Complex{Int},), 10+20im) + g() + end""") + + # A top-level ccall (no enclosing method) referencing a same-frame local: the + # static_eval can't be evaluated with no locals available. + @test_throws LoweringError cc(""" + let rt = Csize_t + ccall(:strlen, rt, (Cstring,), "asdfg") + end""") + # The function *name* must be a literal symbol/string; a local holding a + # Symbol is only rejected later, at codegen. + @test_throws TypeError cc(""" + function g() + local nm = :strlen + ccall(nm, Csize_t, (Cstring,), "asdfg") + end + g()""") + # A top-level local *library* likewise slips past lowering and fails later. + @test_throws LoweringError cc(""" + let lib = "libccalltest" + ccall((:ctest, lib), Complex{Int}, (Complex{Int},), 10+20im) + end""") + + @test_throws TypeError cc(""" + begin + local nm = :strlen + g() = ccall(nm, Csize_t, (Cstring,), "asdfg") + g() + end""") +end + +@testset "CodeInfo: has_image_globalref" begin + @test lower_str(test_mod, "x + y").args[1].has_image_globalref === false + @test lower_str(Main, "x + y").args[1].has_image_globalref === true +end + +baremodule baremod +macro int128_str(x); error("baremod macro; expected call to Core macro"); end +macro uint128_str(x); error("baremod macro; expected call to Core macro"); end +macro big_str(x); error("baremod macro; expected call to Core macro"); end +macro cmd(x); error("baremod macro; expected call to Core macro"); end +macro doc(x, y); error("baremod macro; expected call to Core macro"); end +global nothing = "baremod.nothing; expected core nothing" +end +@testset "globalrefs inserted by parsing" begin + local jl_s_eval = x->JuliaLowering.include_string(baremod, x; expr_compat_mode=true) + local fl_s_eval = x->fl_eval(baremod, JuliaSyntax.parsestmt(Expr, x; filename="file")) + + let s = "100000000000000000000000000000" + @test jl_s_eval(s) == fl_s_eval(s) + end + let s = "0x100000000000000000000000000000" + @test jl_s_eval(s) == fl_s_eval(s) + end + let s = "10000000000000000000000000000000000000000000000000000000000000000" + @test jl_s_eval(s) == fl_s_eval(s) + end + let s = "`ls`" + @test jl_s_eval(s) == fl_s_eval(s) + end + + let s = """ + "foo" function fl_documented_function(); fl_documented_function; end + """ + @test fl_s_eval(s) isa Function + end + @test baremod.fl_documented_function() == baremod.fl_documented_function + let s = """ + "foo" function jl_documented_function(); jl_documented_function; end + """ + @test jl_s_eval(s) isa Function + end + @test baremod.jl_documented_function() == baremod.jl_documented_function + + let s = """ + function fl_ret_nothing(); return; end + """ + @test fl_s_eval(s) isa Function + end + @test baremod.fl_ret_nothing() == Core.nothing + let s = """ + function jl_ret_nothing(); return; end + """ + @test fl_s_eval(s) isa Function + end + @test baremod.jl_ret_nothing() == Core.nothing +end + +@testset "docstrings: doc-only expressions" begin + local jeval(mod, str) = JuliaLowering.include_string(mod, str; expr_compat_mode=true) + jeval(test_mod, "function fun_exists(x); x; end") + jeval(test_mod, "module M end; module M2 end") + # TODO: return values are to be determined, currently Base.Docs.Binding for + # both lowering implementations. We can't return the value of the + # expression in these special cases. + jeval(test_mod, "\"docstr1\" sym_noexist") + jeval(test_mod, "\"docstr2\" fun_noexist()") + jeval(test_mod, "\"docstr3\" fun_exists(sym_noexist)") + jeval(test_mod, "\"docstr4\" M.sym_noexist") + jeval(test_mod, "\"docstr5\" M.fun_noexist()") + jeval(test_mod, "\"docstr6\" M.fun_exists(sym_noexist)") + @test jeval(test_mod, "@doc sym_noexist") |> string === "docstr1\n" + @test jeval(test_mod, "@doc fun_noexist()") |> string === "docstr2\n" + @test jeval(test_mod, "@doc fun_exists(sym_noexist)") |> string === "docstr3\n" + @test jeval(test_mod, "@doc M.sym_noexist") |> string === "docstr4\n" + @test jeval(test_mod, "@doc M.fun_noexist()") |> string === "docstr5\n" + @test jeval(test_mod, "@doc M.fun_exists(sym_noexist)") |> string === "docstr6\n" + @test jeval(test_mod.M, "@doc M.sym_noexist") |> string === "docstr4\n" + @test jeval(test_mod.M, "@doc M.fun_noexist()") |> string === "docstr5\n" + @test jeval(test_mod.M, "@doc M.fun_exists(sym_noexist)") |> string === "docstr6\n" + + jeval(test_mod.M2, "\"docstr7\" M2.M2.sym_noexist") + jeval(test_mod.M2, "\"docstr8\" M2.M2.fun_noexist()") + jeval(test_mod.M2, "\"docstr9\" M2.M2.fun_exists(sym_noexist)") + @test jeval(test_mod, "@doc M2.M2.sym_noexist") |> string === "docstr7\n" + @test jeval(test_mod, "@doc M2.M2.fun_noexist()") |> string === "docstr8\n" + @test jeval(test_mod, "@doc M2.M2.fun_exists(sym_noexist)") |> string === "docstr9\n" + @test jeval(test_mod.M2, "@doc M2.M2.sym_noexist") |> string === "docstr7\n" + @test jeval(test_mod.M2, "@doc M2.M2.fun_noexist()") |> string === "docstr8\n" + @test jeval(test_mod.M2, "@doc M2.M2.fun_exists(sym_noexist)") |> string === "docstr9\n" + + # Try with signatures and type variables + jeval(test_mod, "abstract type T_exists end") + + jeval(test_mod, "\"docstr10\" f10(x::Int, y, z::T_exists)") + d = jeval(test_mod, "@doc f10") + @test d |> string === "docstr10\n" + + jeval(test_mod, "\"docstr11\" f11(x::T_exists, y::U, z::T) where {T, U<:Number}") + d = jeval(test_mod, "@doc f11") + @test d |> string === "docstr11\n" + + jeval(test_mod, "\"docstr12\" f12(x::Int, y::U, z::T=1) where {T, U<:Number}") + d = jeval(test_mod, "@doc f12") + @test d |> string === "docstr12\n" + + # doc-strings on macrocalls (punned on quoted macrocall) + # TODO: implement and test `doc!` support for this + @test_broken jeval(test_mod, """ + "doc string" + :@test + """) isa Expr +end + +# SyntaxTree @eval should pass along expr_compat_mode +@test JuliaLowering.include_string(test_mod, raw""" + let T = gensym("documented_sym_no_logspam") + @eval @doc $"This is a $T" $T = 1 + end +"""; expr_compat_mode=true) === 1 +@test JuliaLowering.include_string(test_mod, raw""" + let T = gensym("documented_sym_no_logspam") + @eval @doc $"This is a $T" $T = 1 + end +"""; expr_compat_mode=false) === 1 + +@testset "tryfinally with scopedvalues" begin + @eval test_mod scopedval = Base.ScopedValues.ScopedValue(1) + @eval test_mod val_history = [] + ex = Expr(:tryfinally, + :(push!(val_history, scopedval[])), + :(push!(val_history, scopedval[])), + :(Base.ScopedValues.Scope(Core.current_scope(), + $test_mod.scopedval => 2))) + JuliaLowering.eval(test_mod, JuliaLowering.expr_to_est(ex); expr_compat_mode=true) + # try block uses "inner" dynamic scope, finally does not + @test test_mod.val_history == [2, 1] + JuliaLowering.eval(test_mod, JuliaLowering.expr_to_est(ex)) + @test test_mod.val_history == [2, 1, 2, 1] +end + +# JuliaLowering/issues/144 +emptyblock_result = JuliaLowering.eval(test_mod, Expr(:(=), :emptyblock_144, Expr(:block))) +@test emptyblock_result == nothing + +@testset "string forms" begin + @test JuliaLowering.include_string(test_mod, raw""" + "str" + """) == "str" + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1 + "str$x" + end + """) == "str1" + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1 + "str$(x)" + end + """) == "str1" + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + "str$(x...)" + end + """) == "str123" + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + "str$(x)" + end + """) == "str[1, 2, 3]" + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + "str$("innerstr$(x...)")" + end + """) == "strinnerstr123" + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + "str$(["innerstr$(x...)"]...)" + end + """) == "strinnerstr123" + + # cmds + @test JuliaLowering.include_string(test_mod, raw""" + `cmdstr` + """) == `cmdstr` + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1 + `cmdstr$x` + end + """) == `cmdstr1` + @test JuliaLowering.include_string(test_mod, raw""" + let x = 1 + `cmdstr$(x)` + end + """) == `cmdstr1` + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + `cmdstr$(x...)` + end + """) == `cmdstr123` + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + `cmdstr$(x)` + end + """) == `cmdstr1 cmdstr2 cmdstr3` + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + `cmdstr$("innerstr$(x...)")` + end + """) == `cmdstrinnerstr123` + @test JuliaLowering.include_string(test_mod, raw""" + let x = [1,2,3] + `cmdstr$(["innerstr$(x...)"]...)` + end + """) == `cmdstrinnerstr123` +end + +let op_mod = Module(:opmod, false) + @testset "operators" for run in [ + s->fl_eval(op_mod, JuliaSyntax.parseall(Expr, s)), + s->JuliaLowering.include_string(op_mod, s; expr_compat_mode=true), + s->JuliaLowering.include_string(op_mod, s; expr_compat_mode=false)] + + @testset "unary prefix (no parens needed)" for op in String["⋆", "±", "∓", "~", "!", "¬", "√", "∛", "∜"] + @test run("$(op)x = (x,)") isa Function + Core.@latestworld + @test run(op) isa Function + @test run("$(op)1") == (1,) + @test run(""" + let $(op)x = (x,x) + $(op)1 + end """) == (1,1) + end + @testset "prefix" for op in String["..", "+", "-", "⋆", "±", "∓", "~", "!", "¬", "√", "∛", "∜"] + @test run("$op(a,b,c) = 3") isa Function + Core.@latestworld + @test run(op) isa Function + @test run("$op(1,2,3)") == 3 + @test run(""" + let $op(a,b,c) = (c,b,a) + $op(1,2,3) + end """) == (3,2,1) + end + @testset "infix" for op in String["&", "|", "+", "-", ":", ".."] + @test run("a$(op)b = (a,b)") isa Function + Core.@latestworld + @test run(op) isa Function + @test run("var\"$op\"(1,2)") == (1,2) + @test run("1$(op)2") == (1,2) + @test run(""" + let a$(op)b = (b,a) + 1$(op)2 + end """) == (2,1) + end + end +end + +@testset "jl_assert" begin + st = @ast_ [K"function" "foo"::K"Identifier"] + if JL.DEBUG + err = try + JuliaLowering.@jl_assert(1 == 2, (st, "error message 1"), (st, "error message 2")) + nothing + catch err + err + end + @test err isa LoweringError + @test err.internal === true + @test length(err.sts) == 2 + @test length(err.msgs) == 2 + shown = sprint(show, err) + @test contains(shown, "error message 1") + @test contains(shown, "error message 2") + err = try + new_st_name = st + JuliaLowering.@jl_assert(1 == 2, (st, "error message 1"), new_st_name, (st, "error message 2")) + nothing + catch err + err + end + @test err isa LoweringError + @test err.internal === true + shown = sprint(show, err) + @test contains(shown, "new_st_name") + else + @test nothing !== try + JuliaLowering.@jl_assert false st + catch err + err + else + nothing + end + end +end + +@testset "(static_parameter n) form as lowering input" begin + # function (::Type{T},) where T; return (sp 1); end + ex = Expr(:function, + Expr(:where, + Expr(:tuple, Expr(:(::), Expr(:curly, :Type, :T))), + :T), + Expr(:block, Expr(:return, Expr(:static_parameter, 1)))) + local f + @test (f = jl_eval(test_mod, ex)) isa Function + @test f(String) == String + @test (f = jl_eval(test_mod, ex; expr_compat_mode=true)) isa Function + @test f(String) == String + @test (f = jl_eval(test_mod, ex; expr_compat_mode=false)) isa Function + @test f(String) == String + + # function (x::T, y::U) where {T, U}; (x, y, (sp 1), (sp 2)); end + ex = Expr(:function, + Expr(:where, + Expr(:tuple, Expr(:(::), :x, :T), Expr(:(::), :y, :U)), + :T, :U), + Expr(:block, + Expr(:return, + Expr(:tuple, :x, :y, + Expr(:static_parameter, 1), + Expr(:static_parameter, 2))))) + @test (f = jl_eval(test_mod, ex)) isa Function + @test f(1, 'a') == (1, 'a', Int, Char) + @test (f = jl_eval(test_mod, ex; expr_compat_mode=true)) isa Function + @test f(1, 'a') == (1, 'a', Int, Char) + @test (f = jl_eval(test_mod, ex; expr_compat_mode=false)) isa Function + @test f(1, 'a') == (1, 'a', Int, Char) + + ex = Expr(:function, + Expr(:where, + Expr(:tuple, Expr(:(::), Expr(:curly, :Type, :T))), + :T), + Expr(:block, Expr(:isdefined, Expr(:static_parameter, 1)))) + + @test (f = jl_eval(test_mod, ex)) isa Function + @test f(String) == true +end + +# duplicated in base tests +JuliaLowering.include_string(@__MODULE__, """ +@testset "interaction of @. with generators" begin + @test [(x,y,a,b) for x in 1:2, y in 3:4 for a in 5:6, b in 7:8 if true] == + @. [(x,y,a,b) for x in 1:2, y in 3:4 for a in 5:6, b in 7:8 if true] + # + in iterspec gets dotted + @test [[11, 22]] == @. [x for x in [[1, 2] + [10, 20]] if true] + # + in body gets dotted + let m = @. [(x+y) for x in [[1,2],[10,20]], y in [100]] + @test m[1] == [101,102] + @test m[2] == [110,120] + end +end +""") diff --git a/JuliaLowering/test/misc_ir.jl b/JuliaLowering/test/misc_ir.jl new file mode 100644 index 0000000000000..cbb83b2329a82 --- /dev/null +++ b/JuliaLowering/test/misc_ir.jl @@ -0,0 +1,590 @@ +module JuxtuposeTest + using ..JuliaLowering + macro emit_juxtupose() + JuliaLowering.@legacy_quote_to_syntax :(10x) + end +end + +#******************************************************************************* +######################################## +# Getproperty syntax +x.a +#--------------------- +1 TestMod.x +2 (call top.getproperty %₁ :a) +3 (return %₂) + +######################################## +# Getproperty syntax with a string on right hand side +x."b" +#--------------------- +1 TestMod.x +2 (call top.getproperty %₁ (inert "b")) +3 (return %₂) + +######################################## +# Standalone dot syntax +.* +#--------------------- +1 TestMod.* +2 (call top.BroadcastFunction %₁) +3 (return %₂) + +######################################## +# Error: Wrong number of children in `.` +@ast_ [K"." "x"::K"Identifier" "a"::K"Identifier" 3::K"Integer"] +#--------------------- +LoweringError: +#= line 1 =# - invalid syntax: unknown form `.` or number of arguments 3 +Expression: + (. x a 3) + +######################################## +# Error: Placeholder value used +_ + 1 +#--------------------- +LoweringError: +_ + 1 +╙ ── all-underscore identifiers are write-only and their values cannot be used in expressions + +######################################## +# Named tuple +(a=1, b=2) +#--------------------- +1 (call core.tuple :a :b) +2 (call core.apply_type core.NamedTuple %₁) +3 (call core.tuple 1 2) +4 (call %₂ %₃) +5 (return %₄) + +######################################## +# Named tuple with parameters +(; a=1, b=2) +#--------------------- +1 (call core.tuple :a :b) +2 (call core.apply_type core.NamedTuple %₁) +3 (call core.tuple 1 2) +4 (call %₂ %₃) +5 (return %₄) + +######################################## +# Empty named tuple +(;) +#--------------------- +1 (call core.NamedTuple) +2 (return %₁) + +######################################## +# Named tuple with implicit field names +(;x, a.b.c, y._) +#--------------------- +1 (call core.tuple :x :c :_) +2 (call core.apply_type core.NamedTuple %₁) +3 TestMod.x +4 TestMod.a +5 (call top.getproperty %₄ :b) +6 (call top.getproperty %₅ :c) +7 TestMod.y +8 (call top.getproperty %₇ :_) +9 (call core.tuple %₃ %₆ %₈) +10 (call %₂ %₉) +11 (return %₁₀) + +######################################## +# Named tuple with splats +(; a=1, b=2, bs..., c=3, ds...) +#--------------------- +1 (call core.tuple :a :b) +2 (call core.apply_type core.NamedTuple %₁) +3 (call core.tuple 1 2) +4 (call %₂ %₃) +5 TestMod.bs +6 (call top.merge %₄ %₅) +7 (call core.tuple :c) +8 (call core.apply_type core.NamedTuple %₇) +9 (call core.tuple 3) +10 (call %₈ %₉) +11 (call top.merge %₆ %₁₀) +12 TestMod.ds +13 (call top.merge %₁₁ %₁₂) +14 (return %₁₃) + +######################################## +# Named tuple with only splats +(; as..., bs...) +#--------------------- +1 (call core.NamedTuple) +2 TestMod.as +3 (call top.merge %₁ %₂) +4 TestMod.bs +5 (call top.merge %₃ %₄) +6 (return %₅) + +######################################## +# Named tuple with dynamic names +(; a=1, b=2, c=>d) +#--------------------- +1 (call core.tuple :a :b) +2 (call core.apply_type core.NamedTuple %₁) +3 (call core.tuple 1 2) +4 (call %₂ %₃) +5 TestMod.c +6 (call core.tuple %₅) +7 (call core.apply_type core.NamedTuple %₆) +8 TestMod.d +9 (call core.tuple %₈) +10 (call %₇ %₉) +11 (call top.merge %₄ %₁₀) +12 (return %₁₁) + +######################################## +# Error: Named tuple with repeated fields +(; a=1, bs..., c=3, a=2) +#--------------------- +LoweringError: +(; a=1, bs..., c=3, a=2) +# ╙ ── Repeated named tuple field name + +######################################## +# Error: Named tuple frankentuple +(a=1; b=2, c=3) +#--------------------- +LoweringError: +(a=1; b=2, c=3) +# └────────┘ ── cannot mix tuple `(a,b,c)` and named tuple `(;a,b,c)` syntax + +######################################## +# Error: Named tuple field dots in rhs +(; a=xs...) +#--------------------- +LoweringError: +(; a=xs...) +# └───┘ ── unexpected `...` +splatting can only be done into a `call`, `tuple`, `curly`, or array-like expression + +######################################## +# Error: Named tuple field invalid lhs +(; a[]=1) +#--------------------- +LoweringError: +(; a[]=1) +# └─┘ ── expected identifier + +######################################## +# Error: Named tuple element with weird dot syntax +(; a."b") +#--------------------- +LoweringError: +(; a."b") +# └─┘ ── expected identifier + +######################################## +# Error: Named tuple element without valid name +(; a=1, f()) +#--------------------- +LoweringError: +(; a=1, f()) +# └─┘ ── expected identifier, `=`, or `...` after semicolon + +######################################## +# Error: Modules not allowed inside blocks +begin + module C + end +end +#--------------------- +LoweringError: +begin +# ┌─────── + module C + end +#─────┘ ── `module` is only allowed at top level +end + +######################################## +# Error: Modules not allowed in local scope +function f() + module C + end +end +#--------------------- +LoweringError: +function f() +# ┌─────── + module C + end +#─────┘ ── this syntax is only allowed at top level +end + +######################################## +# Basic type assert +x::T +#--------------------- +1 TestMod.x +2 TestMod.T +3 (call core.typeassert %₁ %₂) +4 (return %₃) + +######################################## +# Error: Invalid :: syntax outside function arg list +::T +#--------------------- +LoweringError: +::T +└─┘ ── `::` must be written `value::type` outside function argument lists + +######################################## +# Error: braces vector syntax +{x, y} +#--------------------- +LoweringError: +{x, y} +└────┘ ── `{ }` outside of `where` is reserved for future use + +######################################## +# Error: braces matrix syntax +{x y; y z} +#--------------------- +LoweringError: +{x y; y z} +└────────┘ ── `{ }` outside of `where` is reserved for future use + +######################################## +# Error: Test AST which has no source form and thus must have been constructed +# programmatically (eg, a malformed if) +@ast_ [K"if"] +#--------------------- +LoweringError: +#= line 1 =# - expected (if cond body) or (if cond body else) +Expression: + (if) + +######################################## +# Error: @atomic in wrong position +let + @atomic x +end +#--------------------- +LoweringError: +#= none:2 =# - unimplemented or unsupported `atomic` declaration +Expression: + (atomic x) + +######################################## +# GC.@preserve support +GC.@preserve a b begin + f(a,b) +end +#--------------------- +1 TestMod.a +2 TestMod.b +3 (gc_preserve_begin %₁ %₂) +4 TestMod.f +5 TestMod.a +6 TestMod.b +7 (call %₄ %₅ %₆) +8 (gc_preserve_end %₃) +9 (return %₇) + +######################################## +# Error: GC.@preserve bad args +GC.@preserve a b g() begin + body +end +#--------------------- +MacroExpansionError while expanding GC.@preserve in module Main.TestMod: +GC.@preserve a b g() begin +# └─┘ ── Preserved variable must be a symbol + body +end + +######################################## +# @eval without module +@eval $f(x, y) +#--------------------- +1 TestMod.f +2 (call core.tuple %₁) +3 (call JuliaLowering.interpolate_syntax (syntaxinert (call (syntaxunquote f) x y)) %₂) +4 (call JuliaSyntax.fill_context %₃ SyntaxContext(#=omitted=#)) +5 (= slot₁/eval_result (call JuliaLowering.eval TestMod %₄)) +6 latestworld +7 slot₁/eval_result +8 (return %₇) + +######################################## +# @eval with module +@eval mod $f(x, y) +#--------------------- +1 TestMod.mod +2 TestMod.f +3 (call core.tuple %₂) +4 (call JuliaLowering.interpolate_syntax (syntaxinert (call (syntaxunquote f) x y)) %₃) +5 (call JuliaSyntax.fill_context %₄ SyntaxContext(#=omitted=#)) +6 (= slot₁/eval_result (call JuliaLowering.eval %₁ %₅)) +7 latestworld +8 slot₁/eval_result +9 (return %₈) + +######################################## +# Juxtaposition +20x +#--------------------- +1 TestMod.* +2 TestMod.x +3 (call %₁ 20 %₂) +4 (return %₃) + +######################################## +# Juxtaposition - resolve to macro's mod's `JuxtuposeTest.*` +JuxtuposeTest.@emit_juxtupose +#--------------------- +1 TestMod.JuxtuposeTest.* +2 TestMod.JuxtuposeTest.x +3 (call %₁ 10 %₂) +4 (return %₃) + +######################################## +# @cfunction expansion with global generic function as function argument +@cfunction(callable, Int, (Int, Float64)) +#--------------------- +1 (cfunction Ptr{Nothing} (static_eval TestMod.callable) (static_eval TestMod.Int) (static_eval (call core.svec TestMod.Int TestMod.Float64)) :ccall) +2 (return %₁) + +######################################## +# @cfunction expansion with closed-over callable argument +@cfunction($close_over, Int, (Int, Float64)) +#--------------------- +1 TestMod.close_over +2 (cfunction Base.CFunction %₁ (static_eval TestMod.Int) (static_eval (call core.svec TestMod.Int TestMod.Float64)) :ccall) +3 (return %₂) + +######################################## +# Error: Bad arg types to @cfunction +@cfunction(f, Int, NotATuple) +#--------------------- +MacroExpansionError while expanding @cfunction in module Main.TestMod: +@cfunction(f, Int, NotATuple) +# └───────┘ ── @cfunction argument types must be a literal tuple + +######################################## +# Error: Locals used in @cfunction return type +let T=Float64 + @cfunction(f, T, (Float64,)) +end +#--------------------- +LoweringError: +let T=Float64 + @cfunction(f, T, (Float64,)) +# ╙ ── cfunction return type cannot reference local variable +end + +######################################## +# Error: Locals used in @cfunction arg type +let T=Float64 + @cfunction(f, Float64, (Float64,T)) +end +#--------------------- +LoweringError: +let T=Float64 + @cfunction(f, Float64, (Float64,T)) +# ╙ ── cfunction argument type cannot reference local variable +end + +######################################## +# Basic @ccall lowering +@ccall foo(x::X, y::Y)::R +#--------------------- +1 TestMod.X +2 TestMod.Y +3 TestMod.x +4 (call top.cconvert %₁ %₃) +5 TestMod.y +6 (call top.cconvert %₂ %₅) +7 (call top.unsafe_convert %₁ %₄) +8 (call top.unsafe_convert %₂ %₆) +9 (foreigncall (foreignsymbol (tuple (inert foo))) (static_eval TestMod.R) (static_eval (call core.svec TestMod.X TestMod.Y)) 0 (inert (:ccall, 0x0000, false)) %₇ %₈ %₄ %₆) +10 (return %₉) + +######################################## +# @ccall lowering with gc_safe +@ccall gc_safe=true foo(x::X; y::Y)::R +#--------------------- +1 TestMod.X +2 TestMod.Y +3 TestMod.x +4 (call top.cconvert %₁ %₃) +5 TestMod.y +6 (call top.cconvert %₂ %₅) +7 (call top.unsafe_convert %₁ %₄) +8 (call top.unsafe_convert %₂ %₆) +9 (foreigncall (foreignsymbol (tuple (inert foo))) (static_eval TestMod.R) (static_eval (call core.svec TestMod.X TestMod.Y)) 1 (inert (:ccall, 0x0000, true)) %₇ %₈ %₄ %₆) +10 (return %₉) + +######################################## +# non-macro ccall with vararg in signature, but none provided +ccall(:fcntl, Cint, (RawFD, Cint, Cint...), s, F_GETFL) +#--------------------- +1 TestMod.RawFD +2 TestMod.Cint +3 TestMod.s +4 (call top.cconvert %₁ %₃) +5 TestMod.F_GETFL +6 (call top.cconvert %₂ %₅) +7 (call top.unsafe_convert %₁ %₄) +8 (call top.unsafe_convert %₂ %₆) +9 (foreigncall :fcntl (static_eval TestMod.Cint) (static_eval (call core.svec TestMod.RawFD TestMod.Cint)) 2 :ccall %₇ %₈ %₄ %₆) +10 (return %₉) + +######################################## +# Error: No return annotation on @ccall +@ccall strlen("foo"::Cstring) +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall strlen("foo"::Cstring) +# └ ── expected a return type annotation `::SomeType` + +######################################## +# Error: No argument type on @ccall +@ccall foo("blah"::Cstring, "bad")::Int +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall foo("blah"::Cstring, "bad")::Int +# └───┘ ── argument needs a type annotation + +######################################## +# Error: @ccall varargs without one fixed argument +@ccall foo(; x::Int)::Int +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall foo(; x::Int)::Int +# └──────┘ ── C ABI prohibits varargs without one required argument + +######################################## +# Error: Multiple varargs blocks +@ccall foo(; x::Int; y::Float64)::Int +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall foo(; x::Int; y::Float64)::Int +# └──────┘ ── C ABI prohibits varargs without one required argument + +######################################## +# Error: Bad @ccall first arg +@ccall $(:(foo))(1::Cint)::Cint +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall $(:(foo))(1::Cint)::Cint +# └───────┘ ── interpolated value should be a variable or expression, not a literal name or tuple + +######################################## +# Error: Bad @ccall option +@ccall bad_opt foo(x::Int)::Int +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall bad_opt foo(x::Int)::Int +# └─────┘ ── bad option to ccall + +######################################## +# Error: Unknown @ccall option name +@ccall bad_opt=true foo(x::Int)::Int +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall bad_opt=true foo(x::Int)::Int +# └─────┘ ── unknown option name for ccall + +######################################## +# Error: Unknown option type +@ccall gc_safe="hi" foo(x::Int)::Int +#--------------------- +MacroExpansionError while expanding @ccall in module Main.TestMod: +@ccall gc_safe="hi" foo(x::Int)::Int +# └──┘ ── gc_safe must be true or false + +######################################## +# Error: unary & syntax +&x +#--------------------- +LoweringError: +&x +└┘ ── invalid syntax: unknown form `&` or number of arguments 1 + +######################################## +# Error: $ outside quote/string +$x +#--------------------- +LoweringError: +$x +└┘ ── `$` expression outside string or quote + +######################################## +# Error: splat outside call +x... +#--------------------- +LoweringError: +x... +└──┘ ── unexpected `...` +splatting can only be done into a `call`, `tuple`, `curly`, or array-like expression + +######################################## +# `include` should increment world age +include("hi.jl") +#--------------------- +1 TestMod.include +2 (call %₁ "hi.jl") +3 latestworld +4 (return %₂) + +######################################## +# Const function assignment syntax (legacy) +const f(x::Int)::Int = x+1 +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 TestMod.f +4 (call core.TypeEqOf %₃) +5 TestMod.Int +6 (call core.svec %₄ %₅) +7 (call core.svec) +8 SourceLocation::1:6 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x slot₃/tmp(!read)] + 1 TestMod.Int + 2 TestMod.+ + 3 (= slot₃/tmp (call %₂ slot₂/x 1)) + 4 (call core.isa slot₃/tmp %₁) + 5 (gotoifnot %₄ label₇) + 6 (goto label₉) + 7 (call top.convert %₁ slot₃/tmp) + 8 (= slot₃/tmp (call core.typeassert %₇ %₁)) + 9 slot₃/tmp + 10 (return %₉) +11 latestworld +12 TestMod.f +13 (return %₁₂) + +######################################## +# Error: Destructuring assignment method definitions (broken, legacy) +f(x)::Int, g() = [1.0, 2.0] +#--------------------- +LoweringError: +f(x)::Int, g() = [1.0, 2.0] +└──┘ ── invalid assignment location + +######################################## +# Error: Destructuring assignment typedef, variable, and function (broken, legacy) +T{U}, (x::Float64, g()) = [Bool, (1, 2)] +#--------------------- +LoweringError: +T{U}, (x::Float64, g()) = [Bool, (1, 2)] +# └─┘ ── invalid assignment location + +######################################## +# aliasscope form: should be passed through unless implementation changes +Base.Experimental.@aliasscope 1 +#--------------------- +1 (aliasscope) +2 (= slot₁/aliasscope_result 1) +3 (popaliasscope) +4 slot₁/aliasscope_result +5 (return %₄) diff --git a/JuliaLowering/test/modules.jl b/JuliaLowering/test/modules.jl new file mode 100644 index 0000000000000..d5cc7a9eb295b --- /dev/null +++ b/JuliaLowering/test/modules.jl @@ -0,0 +1,50 @@ +test_mod = Module() + +A = JuliaLowering.include_string(test_mod, """ +module A + function g() + return "hi" + end +end +""", "module_test") +@test A isa Module +@test A.g() == "hi" +@test A.include isa Base.IncludeInto +@test A.eval isa Core.EvalInto +@test A.Base === Base +@test A.eval(:(x = -2)) == -2 +@test A.x == -2 + +B = JuliaLowering.include_string(test_mod, """ +baremodule B +end +""", "baremodule_test") +@test B.Core === Core +@test !isdefined(B, :include) +@test !isdefined(B, :eval) +@test !isdefined(B, :Base) + +# Module init order +Amod = JuliaLowering.include_string(test_mod, """ +module A + init_order = [] + __init__() = push!(init_order, "A") + module B + using ..A + __init__() = push!(A.init_order, "B") + end + module C + using ..A + __init__() = push!(A.init_order, "C") + module D + using ...A + __init__() = push!(A.init_order, "D") + end + module E + using ...A + __init__() = push!(A.init_order, "E") + end + end +end +""") +@test Amod.init_order == ["B", "D", "E", "C", "A"] diff --git a/JuliaLowering/test/provenance.jl b/JuliaLowering/test/provenance.jl new file mode 100644 index 0000000000000..6c8dd732cb1a3 --- /dev/null +++ b/JuliaLowering/test/provenance.jl @@ -0,0 +1,70 @@ +using JuliaLowering: compress_sbt, uncompress_sbt, SourceByteTable, + add_debuginfo! + +test_mod = Module() + +function test_byte_precise(di::Core.DebugInfo, nstmts::Int) + for i in 1:nstmts + sl = Base.Compiler.source_location(di, i) + @test sl.byte > 0 context=di + @test sl.byte_end > 0 context=di + @test sl.col > 0 context=di + @test sl.col_end > 0 context=di + @test sl.line > 0 context=di + @test sl.line_end > 0 context=di + end +end + +@testset "SourceByteTable roundtrip" begin + st_thunk = jl_lower(test_mod, JuliaSyntax.parsestmt(SyntaxTree, "1 + 2 - \n 3")) + st = st_thunk[1] + JuliaLowering.@jl_assert kind(st_thunk) === K"thunk" && + kind(st) === K"code_info" (st_thunk, "fix this brittle test") + add_debuginfo!(st) + csbt = st.meta[:debuginfo] + usbt = uncompress_sbt(csbt) + cusbt = compress_sbt(usbt) + @test csbt isa Core.DebugInfo + @test usbt isa SourceByteTable + @test cusbt isa String + @test cusbt === csbt.linetable + + @test length(usbt.spans) <= numchildren(st[2]) # lhs is unique locs + test_byte_precise(csbt, numchildren(st[2])) +end + +@testset "Attaching DebugInfo to methods" begin + local f = JuliaLowering.include_string(test_mod, """ + function f_with_debuginfo(x) + show(x) + map(a->a+1, x) + end + """) + di = methods(f)[1].debuginfo + @test di.linetable isa String + let nstmts = length(Base._uncompressed_ir(methods(f)[1]).code) + test_byte_precise(di, nstmts) + end +end + +@testset "Stack traces can read our debuginfo" begin + f_errors = JuliaLowering.eval( + test_mod, + Expr(:function, Expr(:call, :func_throws_at_myfile_200), + Expr(:block, + LineNumberNode(100, :myfile), + LineNumberNode(200, :myfile), + Expr(:call, :error, "foo"), + LineNumberNode(300, :myfile), + :(return y * 2)))) + frames = try + f_errors() + @test false=="expected error" + catch err + stacktrace(catch_backtrace()) + end + myframe_i = findfirst(x->x.func==:func_throws_at_myfile_200, frames) + @test myframe_i == 2 + # TODO: line field may disappear + @test frames[myframe_i].line == 200 +end diff --git a/JuliaLowering/test/quoting.jl b/JuliaLowering/test/quoting.jl new file mode 100644 index 0000000000000..3618604979b00 --- /dev/null +++ b/JuliaLowering/test/quoting.jl @@ -0,0 +1,516 @@ +test_mod = @newmod(quoting) + +@testset "basic quoting and dollar-interpolation" begin + @eval test_mod global interpolated_var + + @testset for run in [ + (x::String)->fl_eval(test_mod, Expr(:block, JuliaSyntax.parsestmt(Expr, x))), + (x::String)->jl_eval(test_mod, JuliaSyntax.parsestmt(SyntaxTree, x); expr_compat_mode=true), + (x::String)->jl_eval(test_mod, JuliaSyntax.parsestmt(SyntaxTree, x); expr_compat_mode=false), + ] + @test run(raw":x") == :x + @test run(raw":(:x)") == QuoteNode(:x) + @test run(raw":(:(:x))") == Expr(:quote, (QuoteNode(:x))) + @test run(raw":(:($x))") == Expr(:quote, Expr(:$, :x)) + @test run(raw":($(:($(:x))))") == :x + @test run(raw":($(:(:($x))))") == Expr(:quote, Expr(:$, :x)) + @test run(raw":(:($(:($x))))") == Expr(:quote, Expr(:$, Expr(:quote, Expr(:$, :x)))) + + @testset for ivar_val in [:y, Symbol(""), GlobalRef(Base, :push!), Expr(:call, :identity, 2), 1, nothing], + ivar in [ivar_val, Expr(:quote, ivar_val), Expr(:inert, ivar_val), QuoteNode(ivar_val)] + + Base.setglobal!(test_mod, :interpolated_var, ivar) + + @test run(raw"interpolated_var") == ivar + @test run(raw":($interpolated_var)") == ivar + @test run(raw":($(:($interpolated_var)))") == ivar + @test run(raw":(:($$interpolated_var))") == Expr(:quote, Expr(:$, ivar)) + @test run(raw":(:($($interpolated_var)))") == Expr(:quote, Expr(:$, ivar)) + @test run(raw":(identity($interpolated_var))") == Expr(:call, :identity, ivar) + end + end +end + +@testset "self-quoting forms" for + form in [1, true, "string", [], nothing,], + quoted in [Expr(:quote, form), Expr(:inert, form), QuoteNode(form)] + + @test fl_eval(test_mod, Expr(:block, quoted)) == form + @test jl_eval(test_mod, Expr(:block, quoted); expr_compat_mode=true) == form + @test jl_eval(test_mod, Expr(:block, quoted); expr_compat_mode=false) == form +end +@testset "self-quoting forms, interpolated into quote" for + form in [1, true, "string", [], nothing,], + quoted in [form, Expr(:quote, form), Expr(:inert, form), QuoteNode(form)] + + @test fl_eval(test_mod, Expr(:quote, Expr(:$, quoted))) == form + @test jl_eval(test_mod, Expr(:quote, Expr(:$, quoted)); expr_compat_mode=true) == form + @test jl_eval(test_mod, Expr(:quote, Expr(:$, quoted)); expr_compat_mode=false) == form +end + +@eval test_mod global quotesplatvar = [1,[2,[3,[4]]]] +@testset "unquote-splicing `...`" for run in [ + (x)->fl_eval(test_mod, x), + (x)->jl_eval(test_mod, x; expr_compat_mode=true), + (x)->jl_eval(test_mod, x; expr_compat_mode=false), + ] + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, + Expr(:..., :quotesplatvar))))), + Expr(:call, Base.vect, 1, [2, [3, [4]]])) + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:quote, + Expr(:$, + Expr(:..., # A quoted `...` is left unchanged + Expr(:$, + Expr(:..., :quotesplatvar))))))), + Expr(:quote, Expr(:$, Expr(:..., 1, [2, [3, [4]]])))) + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:quote, + Expr(:$, + Expr(:$, + Expr(:..., + Expr(:..., :quotesplatvar))))))), + Expr(:quote, Expr(:$, 1, 2, [3, [4]]))) + @test expr_structure_eq( + run( + Expr(:quote, + Expr(:quote, + Expr(:$, + Expr(:$, + Expr(:..., + Expr(:..., + Expr(:..., :quotesplatvar)))))))), + Expr(:quote, Expr(:$, 1, 2, 3, [4]))) +end + +@testset "@legacy_quote_to_syntax" begin + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :x") isa SyntaxTree + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :x") |> kind === K"Identifier" + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :($1)") isa SyntaxTree + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :($1)") |> kind === K"Value" + + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :(x+1)") isa SyntaxTree + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :(x+1)") |> kind === K"call" + + # compat mode makes standard quote + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :x"; expr_compat_mode=true) == :x + @test JuliaLowering.include_string( + test_mod, raw"@legacy_quote_to_syntax :(x+1)"; expr_compat_mode=true) == + Expr(:call, :+, :x, 1) + + # syntaxunquote does not support the equivalent of Expr(:$, :a, :b), but + # legacy_quote_to_syntax can convert it + @test expr_structure_eq( + jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, :b))))) + ; expr_compat_mode=true), + Expr(:call, Base.vect, 1, [2, [3, [4]]])) + # splat :b + @test expr_structure_eq( + jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., :b)))))) + ; expr_compat_mode=true), + Expr(:call, Base.vect, 1, 2, [3, [4]])) + # double-splat :b + @test expr_structure_eq( + jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., Expr(:..., :b))))))) + ; expr_compat_mode=true), + Expr(:call, Base.vect, 1, 2, 3, [4])) + + # with compat=false + st = jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, :b))))) + ; expr_compat_mode=false) + @test st isa SyntaxTree + @test JuliaSyntax.numchildren(st) == 3 + st = jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., :b)))))) + ; expr_compat_mode=false) + @test st isa SyntaxTree + @test JuliaSyntax.numchildren(st) == 4 + st = jl_eval( + test_mod, + Expr(:let, Expr(:block, Expr(:(=), :a, 1), Expr(:(=), :b, [2, [3, [4]]])), + Expr(:macrocall, + Symbol("@legacy_quote_to_syntax"), + LineNumberNode(1), + Expr(:quote, + Expr(:call, Base.vect, + Expr(:$, :a, Expr(:..., Expr(:..., :b))))))) + ; expr_compat_mode=false) + @test st isa SyntaxTree + @test JuliaSyntax.numchildren(st) == 5 +end + +ex = JuliaLowering.include_string(test_mod, """ +begin + x = 10 + y = @legacy_quote_to_syntax :(g(z)) + @legacy_quote_to_syntax quote + f(\$(x+1), \$y) + end +end +""") +@test ex ≈ @ast_ [K"block" + [K"call" + "f"::K"Identifier" + 11::K"Value" + [K"call" + "g"::K"Identifier" + "z"::K"Identifier" + ] + ] +] +@test sourcetext(ex[1]) == "f(\$(x+1), \$y)" +@test sourcetext(ex[1][2]) == "\$(x+1)" + +# Test that interpolation with field access works +# (the field name can be interpolated after the dot). +@test JuliaLowering.include_string(test_mod, """ +let + field_name = @legacy_quote_to_syntax :(a) + @legacy_quote_to_syntax :(x.\$field_name) +end +""") ≈ @ast_ [K"." "x"::K"Identifier" [K"inert" "a"::K"Identifier"]] +@test JuliaLowering.include_string(test_mod, """ +let + field_name = @legacy_quote_to_syntax :(a) + @legacy_quote_to_syntax :(x.\$field_name) +end +"""; expr_compat_mode=true) == Expr(:., :x, QuoteNode(:a)) + +# Test quoted property access syntax like `Core.:(foo)` and `Core.:(!==)` +@test JuliaLowering.include_string(test_mod, """ + x = (a=1, b=2) + x.:(a) +""") == 1 +@test JuliaLowering.include_string(test_mod, """ + Core.:(!==) +""") === (!==) + +# Test quoted operator function definitions (issue #20) +@test JuliaLowering.include_string(test_mod, """ +begin + struct Issue20 + x::Int + end + Base.:(==)(a::Issue20, b::Issue20) = a.x == b.x + Issue20(1) == Issue20(1) +end +""") === true + +@test JuliaLowering.include_string(test_mod, """ +begin + Base.:(<)(a::Issue20, b::Issue20) = a.x < b.x + Issue20(1) < Issue20(2) +end +""") === true + +# interpolations at multiple depths +ex = JuliaLowering.include_string(test_mod, raw""" +let + args = (:(x,x),:(y,y)) + quote + x = 1 + y = 2 + quote + f($$(args...)) + end + end +end +""") +@test Base.remove_linenums!(ex) == + Expr(:block, + Expr(:(=), :x, 1), + Expr(:(=), :y, 2), + Expr(:quote, + Expr(:block, + Expr(:call, :f, Expr(:$, Expr(:tuple, :x, :x), + Expr(:tuple, :y, :y)))))) + +# Double interpolation +double_interp_ex = JuliaLowering.include_string(test_mod, raw""" +let + args = (:(xxx),) + :(:($$(args...))) +end +""") +Base.eval(test_mod, :(xxx = 111)) +dinterp_eval = JuliaLowering.eval(test_mod, double_interp_ex) +@test dinterp_eval == 111 + +multi_interp_ex = JuliaLowering.include_string(test_mod, raw""" +let + args = (:(x), :(y)) + :(:($$(args...))) +end +""") + +err = try + JuliaLowering.eval(test_mod, multi_interp_ex) + nothing +catch exc + @test exc isa LoweringError + sprint(io->Base.showerror(io, exc, show_detail=false)) +end +@test contains(err, raw"More than one value in bare `$` expression") + +err = try + JuliaLowering.eval(test_mod, multi_interp_ex, expr_compat_mode=true) + nothing +catch exc + @test exc isa LoweringError + sprint(io->Base.showerror(io, exc, show_detail=false)) +end +@test contains(err, raw"More than one value in bare `$` expression") + +# Symbol should be interpolated (converted from expr) +@eval test_mod using JuliaLowering +symbol_interp = JuliaLowering.include_string(test_mod, """ +let + x = :xx + y = @legacy_quote_to_syntax :yy + @legacy_quote_to_syntax :(f(\$x, \$y, z)) +end +""") +@test symbol_interp ≈ @ast_ [K"call" + "f"::K"Identifier" + "xx"::K"Identifier" + "yy"::K"Identifier" + "z"::K"Identifier" +] +@test sourcetext(symbol_interp[2]) == raw"$x" +@test sourcetext(symbol_interp[3]) == "yy" + +# (may change) Expr interpolated into SyntaxTree +@test_throws LoweringError JuliaLowering.include_string(test_mod, raw""" +let + x = Expr(:call, :f, :x) + @legacy_quote_to_syntax :(g($x)) +end +""") broken=true + +@testset "Interpolation in Expr compat mode" begin + expr_interp = JuliaLowering.include_string(test_mod, raw""" + let + x = :xx + :(f($x, z)) + end + """, expr_compat_mode=true) + @test expr_interp == Expr(:call, :f, :xx, :z) + + double_interp_expr = JuliaLowering.include_string(test_mod, raw""" + let + x = :xx + :(:(f($$x, $y))) + end + """, expr_compat_mode=true) + @test double_interp_expr == Expr(:quote, Expr(:call, :f, Expr(:$, :xx), Expr(:$, :y))) + + # Test that ASTs are copied before they're seen by the user + @test JuliaLowering.include_string(test_mod, raw""" + exs = [] + for i = 1:2 + push!(exs, :(f(x,y))) + push!(exs[end].args, :z) + end + exs + """, expr_compat_mode=true) == Any[Expr(:call, :f, :x, :y, :z), Expr(:call, :f, :x, :y, :z)] + + # Test interpolation into QuoteNode + @test JuliaLowering.include_string(test_mod, raw""" + let x = :push! + @eval Base.$x + end + """; expr_compat_mode=true) == Base.push! +end + +# (. l r) should pass lowering only when r is one of: +# - simple identifier (resolved variable) +# - any simple atom, bare, inert, or in quote +# - anything else if inert (not evaluated) +# - any valid `r` wrapped in unquote, then quote +# +# note Expr(:block) is to avoid the special top-level evaluation of Expr(:.) in +# flisp, which skips handling :quote +@eval test_mod begin + struct GetProperty; gs_field; end + Base.getproperty(::GetProperty, x) = ("got", x) + Base.getproperty(::GetProperty, x::Symbol) = ("got", x) # avoid ambiguity + + global gs = GetProperty([]) + global outer_field = :gs_field +end +@testset "getproperty quoting" for wrap_quote in [ + identity, + x->Expr(:quote, Expr(:$, x)), + x->Expr(:quote, Expr(:$, Expr(:quote, Expr(:$, x))))] + + @testset "arg2 unquoted identifier" for expr_compat_mode in [true, false] + local field = :outer_field + + @test fl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field))) == + ("got", :gs_field) + @test jl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field)); + expr_compat_mode) == + ("got", :gs_field) + end + @testset "arg2 quoted identifier" for expr_compat_mode in [true, false], + field in [Expr(:quote, :gs_field), + Expr(:inert, :gs_field), + QuoteNode(:gs_field)] + + @test fl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field))) == + ("got", :gs_field) + @test jl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field)); + expr_compat_mode) == + ("got", :gs_field) + end + @testset "arg2 maybe-quoted non-identifier atom" for expr_compat_mode in [true, false], + field_atom in ["str", 1], + field in [field_atom, + Expr(:quote, field_atom), + Expr(:inert, field_atom), + QuoteNode(field_atom)] + + @test fl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field))) == + ("got", field_atom) + @test jl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field)); + expr_compat_mode) == + ("got", field_atom) + end + + @testset "arg2 inert AST" for expr_compat_mode in [true, false], + # oddly, bool and nothing don't work unquoted in flisp + field_inner in [true, + nothing, + GlobalRef(Core, :Type), + Expr(:string, "s", "tr"), + Expr(:string, "s", Expr(:call, string, :tr))], + field in [Expr(:inert, field_inner), + QuoteNode(field_inner)] + + @test fl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field))) == + ("got", field_inner) + @test jl_eval(test_mod, Expr(:block, Expr(:., test_mod.gs, field)); + expr_compat_mode) == + ("got", field_inner) + end + + @testset "arg2 non-inert non-atom should throw" for expr_compat_mode in [true, false], + field in [Expr(:string, "s", "tr"), + Expr(:string, "s", Expr(:call, string, :tr)), + Expr(:quote, Expr(:string, "s", "tr")), + Expr(:quote, Expr(:string, "s", Expr(:$, :outer_field)))] + + @test_throws "invalid syntax" fl_eval( + test_mod, Expr(:block, Expr(:., test_mod.gs, field))) + @test_throws LoweringError jl_eval( + test_mod, Expr(:block, Expr(:., test_mod.gs, field))) + end +end + +@testset "syntax context in macro body should be discarded" begin + # Both `x`s should have argument context in the macrocall, not new-syntax + # context (resulting in (99, 2)) + @test JuliaLowering.include_string(test_mod, raw""" + macro set_value(name, body) + @legacy_quote_to_syntax(quote + $name = 1 + $body + end) + end + (function () + x = 99 + r = @set_value x x + 1 + (x, r) + end)() + """) == (1,2) + + @test JuliaLowering.include_string(test_mod, raw""" + macro addone_value(name, body) + @legacy_quote_to_syntax(quote + x = 99 # hygienic + $name += 1 + $body + end) + end + (function () + x = 0 + out = [] + push!(out, (x, @addone_value x x + 1)) + push!(out, (x, @addone_value x x + 1)) + push!(out, (x, @addone_value x x + 1)) + out + end)() + """) == [(0, 2), (1, 3), (2, 4)] + + @test JuliaLowering.include_string(test_mod, raw""" + macro addone_value_quote2(name, body) + @legacy_quote_to_syntax(quote + x = 99 # hygienic + $(:($name)) += 1 + $(:($body)) + end) + end + (function () + x = 0 + out = [] + push!(out, (x, @addone_value_quote2 x x + 1)) + push!(out, (x, @addone_value_quote2 x x + 1)) + push!(out, (x, @addone_value_quote2 x x + 1)) + out + end)() + """) == [(0, 2), (1, 3), (2, 4)] +end diff --git a/JuliaLowering/test/quoting_ir.jl b/JuliaLowering/test/quoting_ir.jl new file mode 100644 index 0000000000000..ab70ba28042b7 --- /dev/null +++ b/JuliaLowering/test/quoting_ir.jl @@ -0,0 +1,88 @@ +######################################## +# Simple interpolation +quote + $x + 1 +end +#--------------------- +1 TestMod.x +2 (call core.tuple %₁) +3 (call JuliaLowering.interpolate_expr (inert (block (call + ($ x) 1))) %₂) +4 (return %₃) + +######################################## +# Trivial interpolation +:($x) +#--------------------- +1 TestMod.x +2 (return %₁) + +######################################## +# Double escape +quote + quote + $$x + 1 + end +end +#--------------------- +1 TestMod.x +2 (call core.tuple %₁) +3 (call JuliaLowering.interpolate_expr (inert (block (quote (block (call + ($ ($ x)) 1))))) %₂) +4 (return %₃) + +######################################## +# Symbols on `.` right hand side need to be scoped correctly +let x = 1 + :(A.$x) +end +#--------------------- +1 1 +2 (= slot₁/x %₁) +3 (call core.tuple slot₁/x) +4 (call JuliaLowering.interpolate_expr (inert (. A (inert ($ x)))) %₃) +5 (return %₄) + +######################################## +# Error: Double escape +quote + $$x + 1 +end +#--------------------- +LoweringError: +quote + $$x + 1 +# └┘ ── `$` expression outside string or quote +end + +######################################## +# Quoted property access with identifier +Core.:(foo) +#--------------------- +1 TestMod.Core +2 (call top.getproperty %₁ :foo) +3 (return %₂) + +######################################## +# Quoted property access with operator +Core.:(!==) +#--------------------- +1 TestMod.Core +2 (call top.getproperty %₁ :!==) +3 (return %₂) + +######################################## +# Quoted operator function definition (issue #20) +function Base.:(==)() end +#--------------------- +1 TestMod.Base +2 (call top.getproperty %₁ :==) +3 (call core.TypeEqOf %₂) +4 (call core.svec %₃) +5 (call core.svec) +6 SourceLocation::1:1 +7 (call core.svec %₄ %₅ %₆) +8 (call core.define_method TestMod core.nothing %₇ + --- code_info + slots: [slot₁/#self#(!read)] + 1 (return core.nothing) +9 latestworld +10 (return core.nothing) diff --git a/JuliaLowering/test/repl_mode.jl b/JuliaLowering/test/repl_mode.jl new file mode 100644 index 0000000000000..cf85717c03cbf --- /dev/null +++ b/JuliaLowering/test/repl_mode.jl @@ -0,0 +1,84 @@ +# JuliaLowering REPL mode: an interactive test utility for lowering code (not +# part of the unit tests) + +module JuliaLoweringREPL + +import ReplMaker +import REPL + +using JuliaLowering: JuliaLowering, SyntaxTree, children +using JuliaSyntax + +function is_incomplete(prompt_state) + str = String(take!(copy(REPL.LineEdit.buffer(prompt_state)))) + stream = JuliaSyntax.ParseStream(str) + JuliaSyntax.parse!(stream, rule=:all) + if JuliaSyntax.any_error(stream) + tree = JuliaSyntax.build_tree(SyntaxNode, stream) + tag = JuliaSyntax._incomplete_tag(tree, 1) + return tag != :none + else + return false + end +end + +function eval_ish(mod::Module, ex::SyntaxTree, do_eval::Bool, do_print_ir::Bool) + k = kind(ex) + if k == K"toplevel" + x = nothing + for e in children(ex) + x = eval_ish(mod, e, do_eval, do_print_ir) + end + return x + end + linear_ir = JuliaLowering.lower(mod, ex) + if do_print_ir + JuliaLowering.print_ir(stdout, linear_ir) + end + if do_eval + println(stdout, "#----------------------") + expr_form = JuliaLowering.to_lowered_expr(linear_ir) + Base.eval(mod, expr_form) + end +end + +PRINT_IR::Bool = true +DO_EVAL::Bool = false +function opts(; do_eval=false, print_ir=false) + global DO_EVAL = do_eval + global PRINT_IR = print_ir +end + +function handle_input(str) + global DO_EVAL, PRINT_IR + if str == "DO_EVAL" + DO_EVAL = true + return + elseif str == "!DO_EVAL" + DO_EVAL = false + return + elseif str == "PRINT_IR" + PRINT_IR = true + return + elseif str == "!PRINT_IR" + PRINT_IR = false + return + end + ex = parseall(SyntaxTree, str; filename="REPL") + eval_ish(Main, ex, DO_EVAL, PRINT_IR) +end + +function init() + ReplMaker.initrepl(handle_input, + valid_input_checker = !is_incomplete, + prompt_text="Lowering> ", + prompt_color = :blue, + start_key=")", + mode_name=:JuliaLowering) +end + +function __init__() + init() +end + +end diff --git a/JuliaLowering/test/runtests.jl b/JuliaLowering/test/runtests.jl new file mode 100644 index 0000000000000..b0fd4817cd5b2 --- /dev/null +++ b/JuliaLowering/test/runtests.jl @@ -0,0 +1,35 @@ +using Test, JuliaLowering + +@test isempty(Test.detect_closure_boxes(JuliaLowering)) + +include("utils.jl") + +@testset "JuliaLowering.jl" begin + include("ir_tests.jl") + + @testset "ast" include("ast.jl") + + @testset "validation" include("validation.jl") + + @testset "arrays" include("arrays.jl") + @testset "assignments" include("assignments.jl") + @testset "branching" include("branching.jl") + @testset "closures" include("closures.jl") + @testset "decls" include("decls.jl") + @testset "destructuring" include("destructuring.jl") + @testset "exceptions" include("exceptions.jl") + @testset "generators" include("generators.jl") + @testset "import" include("import.jl") + @testset "loops" include("loops.jl") + @testset "macros" include("macros.jl") + @testset "misc" include("misc.jl") + @testset "modules" include("modules.jl") + @testset "quoting" include("quoting.jl") + @testset "scopes" include("scopes.jl") + @testset "typedefs" include("typedefs.jl") + @testset "functions" include("functions.jl") + + @testset "provenance" include("provenance.jl") + @testset "compat" include("compat.jl") + @testset "hooks" include("hooks.jl") +end diff --git a/JuliaLowering/test/runtests_vendored.jl b/JuliaLowering/test/runtests_vendored.jl new file mode 100644 index 0000000000000..fbb1b0b0c1dbd --- /dev/null +++ b/JuliaLowering/test/runtests_vendored.jl @@ -0,0 +1,42 @@ +using Pkg + +function with_output_on_failure(f) + output = IOBuffer() + try + return f(output) + catch + seekstart(output) + write(stderr, read(output)) + rethrow() + end +end + +let old_active_project = Base.active_project() + try + # test local (dev) copy of JuliaLowering, not yet vendored into Base + Base.set_active_project(joinpath(@__DIR__, "..", "Project.toml")) + + # `Manifest.toml` is not checked in (see `JuliaLowering/.gitignore`), so + # resolve the environment before loading. `Project.toml` records the path to + # the in-tree `JuliaSyntax` in its `[sources]` section, so instantiating + # picks up that dev copy without needing a committed manifest. Skip the + # registry update: the only dependencies are path-based / stdlib, so no + # network access is required (and tests may run with networking disabled). + with_output_on_failure() do output + Pkg.instantiate(; update_registry=false, io=output) + end + + # restore error hints (emptied by `testdefs.jl`) so that errors print as + # JuliaLowering expects them to + Base.Experimental.register_error_hint(Base.UndefVarError_hint, UndefVarError) + + # n.b.: these must be run in `Main`, so that type-printing is equivalent + # when running via Pkg.test() (e.g. "SomeType" should be printed instead + # of "JuliaLowering.SomeType") + @eval Main using JuliaLowering + Core.include(Main, joinpath(@__DIR__, "runtests.jl")) # run the actual tests + finally + # Restore original load path and active project + Base.set_active_project(old_active_project) + end +end diff --git a/JuliaLowering/test/scopes.jl b/JuliaLowering/test/scopes.jl new file mode 100644 index 0000000000000..ba6f1b9b6ac33 --- /dev/null +++ b/JuliaLowering/test/scopes.jl @@ -0,0 +1,1164 @@ +test_mod = Module() + +#------------------------------------------------------------------------------- +# Scopes +@test JuliaLowering.include_string(test_mod, +""" +let + y = 0 + x = 1 + let x = x + 1 + y = x + end + (x, y) +end +""") == (1, 2) + +JuliaLowering.include_string(test_mod, """ +x = 101 +y = 202 +""") +@test test_mod.x == 101 +@test test_mod.y == 202 +@test JuliaLowering.include_string(test_mod, "x + y") == 303 + +@test JuliaLowering.include_string(test_mod, """ +begin + local x = 1 + local x = 2 + let (x,y) = (:x,:y) + (y,x) + end +end +""") === (:y,:x) + +# Types on left hand side of type decls refer to the outer scope +# (In the flisp implementation they refer to the inner scope, but this seems +# like a bug.) +# edit: Using flisp semantics for now; see test below. +@test_broken JuliaLowering.include_string(test_mod, """ +let x::Int = 10.0 + local Int = Float64 + x +end +""") === 10 + +# The type in a let type decl must apply (and be re-evaluated) on assignments +# to the variable from closures capturing it; hoisting the type into a +# temporary outside the closure broke lowering of the closure body. +@test JuliaLowering.include_string(test_mod, """ +let x::Int = 1.0 + f = function () + x = 2.0 + end + f() + x +end +""") === 2 + +# Closures in let syntax can only capture values from the outside +# (In the flisp implementation it captures from inner scope, but this is +# inconsistent with let assignment where the rhs refers to the outer scope and +# thus seems like a bug.) +@test_broken JuliaLowering.include_string(test_mod, """ +begin + local y = :outer_y + let f() = y + local y = :inner_y + f() + end +end +""") === :outer_y + +#= +| old\new: || global | local | arg | sparam | +|----------++------------+-------+--------------------+--------------------| +| global || no-op | (*) | | | +| local || error (*) | no-op | | | +| arg || shadow(??) | error | error (not unique) | | +| sparam || shadow(??) | error | error (sparam/arg) | error (not unique) | +=# +@testset "Conflicts in the same local scope" begin + + # no-op cases. It would probably be clearer (but breaking) if these were + # errors like the conflict cases (two of the same decl should never do + # anything, and the user might be expecting two variables). + @testset "global,global" begin + s = "function (); global g; global g; 1; end" + @test JuliaLowering.include_string(test_mod, s) isa Function + end + @testset "local,local" begin + s = "function (); local l; local l; end" + @test JuliaLowering.include_string(test_mod, s) isa Function + end + + # locals may not overlap args/sparams/globals + @testset "global,local/local,global" begin + s = "function (); global g; local g; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + end + @testset "arg,local" begin + s = "function (x); local x; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + end + @testset "sparam,local" begin + s = "function (a::s) where {s}; local s; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + end + + # globals may overlap args or sparams (buggy?) TODO: decide whether it's + # worth replicating this behaviour. We would likely need to copy the way + # flisp nests an extra scope block in every lambda. + @testset "arg,global" begin + local f + s = "function (a); global a = 1; a; end" + @test_broken f = JuliaLowering.include_string(test_mod, s) + @test_broken f isa Function + @test_broken f(999) === 1 + @test_broken isdefinedglobal(test_mod, :a) + end + @testset "sparam,global" begin + local f + s = "function (a::s) where {s}; global s = 1; s; end" + @test_broken f = JuliaLowering.include_string(test_mod, s) + @test_broken f isa Function + @test_broken f(999) === 1 + @test_broken isdefinedglobal(test_mod, :s) + end + + # sp/arg conflict + @testset "arg,sparam" begin + s = "function (a) where {a}; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + end + @testset "arg,arg" begin + s = "function (a,a); end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + end + @testset "sparam,sparam" begin + s = "function () where {s,s}; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + end + + # (not in table) destructured args are handled internally like locals, but + # should have similar conflict rules to arguments + @testset "destructured-arg,destructured-arg/arg/local/sp/global" begin + s = "function ((x,x)); end" + # this works in flisp; should it? + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + s = "function ((x,y),x); end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + s = "function ((x,y)) where {x}; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + s = "function ((x,y)); global x; x; end" + @test_throws LoweringError JuliaLowering.include_string(test_mod, s) + # quirk: flisp is OK with this + s = "function ((x,y)); local x; end" + @test JuliaLowering.include_string(test_mod, s) isa Function + end + +end + +# Switch to Core.eval for sanity-checking +expr_eval(mod, ex) = JuliaLowering.eval(mod, ex) + +enable_softscope(e...) = Expr(:block, Expr(:softscope, true), e...) +wrap_none(e...) = Expr(:block, e...) +wrap_neutral(e...) = Expr(:try, # use try so that a value is returned + Expr(:block, e...), + :catchvar, Expr(:block, + Expr(:call, rethrow, :catchvar))) +wrap_func(e...) = Expr(:call, + Expr(:function, Expr(:tuple), + Expr(:block, e...))) +wrap_hard(e...) = Expr(:let, Expr(:block), Expr(:block, e...)) + +decls(e...) = Expr(:block, + :(name = false), + :(local lname = false), + :(global gname = false), + e...) + +decls_none(e...) = decls(e...) +decls_neutral(e...) = wrap_neutral(decls(), e...) +decls_hard(e...) = :(let lname = false # takes a different code path in flisp + name = false + global gname = false + $(e...) + end) +decls_func(e...) = :((function (argname::spname = false) where spname + name = false + local lname = false + global gname = false + $(e...) + end)(#=called=#)) + +lhs_names = (:name, :lname, :gname, :argname, :spname) + +#= +simple test that +``` +distraction_scope_begin + local_scope_begin + lhs = "resolve me" + lhs *= '!' + lhs + local_scope_end +distraction_scope_end === "resolve me!" +``` +=# +@testset "explicit locals and globals in local scope shadowing outer vars" begin + local jl_mod = Module() + local fl_mod = Module() + + @testset for soft_mode in (false, true), + decls_s in (decls_func, decls_hard, decls_neutral, decls_none), + local_s in (wrap_func, wrap_hard, wrap_neutral), + lhs in lhs_names, + assign_ex in (:(local $lhs = "resolve me"; $lhs *= '!'; $lhs), + :(global $lhs = "resolve me"; $lhs *= '!'; $lhs)) + + ex = decls_s(local_s(assign_ex)) + soft_mode && (ex = enable_softscope(ex)) + + if lhs == :spname && decls_s == decls_func && assign_ex.args[1].head === :local + # flisp specifically disallows locals shadowing sparams; why? + @test_broken fl_eval(fl_mod, ex) === "resolve me!" + elseif lhs in (:spname, :argname) && decls_s != decls_func + continue + else + reference_ok = fl_eval(fl_mod, ex) === "resolve me!" + !reference_ok && + @error("shadow test failed: flisp produced unexpected result; fix that or JL scope tests:\n", ex) + @test reference_ok + end + + ok = expr_eval(jl_mod, ex) === "resolve me!" + !ok && @error("shadow test failed:\n", ex) + @test ok + end + +end + +@test JuliaLowering.include_string(test_mod, """ +global g_shadow_sp_bound = Number +function f_g_shadow_sp_bound(x::g_shadow_sp_bound) where { + g_shadow_sp_bound<:g_shadow_sp_bound + } + (x, g_shadow_sp_bound) +end +f_g_shadow_sp_bound(1) +""") == (1, Int) + +# For each distinct outer scope, declaration scope, and assignment scope, and +# each kind of variable (lhs_names) in the declaration scope, set the same name +# to true from the inner scope +@testset "Behaviour of `=` in local scope (shadow or assign-existing)" begin + expected_outer_vals = Dict{Tuple{Bool, Function, Function, Function}, Tuple}( + (false, decls_func, wrap_hard, wrap_func ) => (true,true,true,true), + (false, decls_func, wrap_hard, wrap_hard ) => (true,true,true,true), + (false, decls_func, wrap_hard, wrap_neutral) => (true,true,true,true), + (false, decls_func, wrap_neutral, wrap_func ) => (true,true,true,true), + (false, decls_func, wrap_neutral, wrap_hard ) => (true,true,true,true), + (false, decls_func, wrap_neutral, wrap_neutral) => (true,true,true,true), + (false, decls_func, wrap_none, wrap_func ) => (true,true,true,true), + (false, decls_func, wrap_none, wrap_hard ) => (true,true,true,true), + (false, decls_func, wrap_none, wrap_neutral) => (true,true,true,true), + (false, decls_hard, wrap_hard, wrap_func ) => (true,true,true), + (false, decls_hard, wrap_hard, wrap_hard ) => (true,true,true), + (false, decls_hard, wrap_hard, wrap_neutral) => (true,true,true), + (false, decls_hard, wrap_neutral, wrap_func ) => (true,true,true), + (false, decls_hard, wrap_neutral, wrap_hard ) => (true,true,true), + (false, decls_hard, wrap_neutral, wrap_neutral) => (true,true,true), + (false, decls_hard, wrap_none, wrap_func ) => (true,true,true), + (false, decls_hard, wrap_none, wrap_hard ) => (true,true,true), + (false, decls_hard, wrap_none, wrap_neutral) => (true,true,true), + (false, decls_neutral, wrap_hard, wrap_func ) => (true,true,true), + (false, decls_neutral, wrap_hard, wrap_hard ) => (true,true,true), + (false, decls_neutral, wrap_hard, wrap_neutral) => (true,true,true), + (false, decls_neutral, wrap_neutral, wrap_func ) => (true,true,true), + (false, decls_neutral, wrap_neutral, wrap_hard ) => (true,true,true), + (false, decls_neutral, wrap_neutral, wrap_neutral) => (true,true,true), + (false, decls_neutral, wrap_none, wrap_func ) => (true,true,true), + (false, decls_neutral, wrap_none, wrap_hard ) => (true,true,true), + (false, decls_neutral, wrap_none, wrap_neutral) => (true,true,true), + (false, decls_none, wrap_hard, wrap_func ) => (false,true,false), + (false, decls_none, wrap_hard, wrap_hard ) => (false,true,false), + (false, decls_none, wrap_hard, wrap_neutral) => (false,true,false), + (false, decls_none, wrap_neutral, wrap_func ) => (false,true,false), + (false, decls_none, wrap_neutral, wrap_hard ) => (false,true,false), + (false, decls_none, wrap_neutral, wrap_neutral) => (false,true,false), + (false, decls_none, wrap_none, wrap_func ) => (false,true,false), + (false, decls_none, wrap_none, wrap_hard ) => (false,true,false), + (false, decls_none, wrap_none, wrap_neutral) => (false,true,false), + (true, decls_func, wrap_hard, wrap_func ) => (true,true,true,true), + (true, decls_func, wrap_hard, wrap_hard ) => (true,true,true,true), + (true, decls_func, wrap_hard, wrap_neutral) => (true,true,true,true), + (true, decls_func, wrap_neutral, wrap_func ) => (true,true,true,true), + (true, decls_func, wrap_neutral, wrap_hard ) => (true,true,true,true), + (true, decls_func, wrap_neutral, wrap_neutral) => (true,true,true,true), + (true, decls_func, wrap_none, wrap_func ) => (true,true,true,true), + (true, decls_func, wrap_none, wrap_hard ) => (true,true,true,true), + (true, decls_func, wrap_none, wrap_neutral) => (true,true,true,true), + (true, decls_hard, wrap_hard, wrap_func ) => (true,true,true), + (true, decls_hard, wrap_hard, wrap_hard ) => (true,true,true), + (true, decls_hard, wrap_hard, wrap_neutral) => (true,true,true), + (true, decls_hard, wrap_neutral, wrap_func ) => (true,true,true), + (true, decls_hard, wrap_neutral, wrap_hard ) => (true,true,true), + (true, decls_hard, wrap_neutral, wrap_neutral) => (true,true,true), + (true, decls_hard, wrap_none, wrap_func ) => (true,true,true), + (true, decls_hard, wrap_none, wrap_hard ) => (true,true,true), + (true, decls_hard, wrap_none, wrap_neutral) => (true,true,true), + (true, decls_neutral, wrap_hard, wrap_func ) => (true,true,true), + (true, decls_neutral, wrap_hard, wrap_hard ) => (true,true,true), + (true, decls_neutral, wrap_hard, wrap_neutral) => (true,true,true), + (true, decls_neutral, wrap_neutral, wrap_func ) => (true,true,true), + (true, decls_neutral, wrap_neutral, wrap_hard ) => (true,true,true), + (true, decls_neutral, wrap_neutral, wrap_neutral) => (true,true,true), + (true, decls_neutral, wrap_none, wrap_func ) => (true,true,true), + (true, decls_neutral, wrap_none, wrap_hard ) => (true,true,true), + (true, decls_neutral, wrap_none, wrap_neutral) => (true,true,true), + (true, decls_none, wrap_hard, wrap_func ) => (false,true,false), + (true, decls_none, wrap_hard, wrap_hard ) => (false,true,false), + (true, decls_none, wrap_hard, wrap_neutral) => (false,true,false), + (true, decls_none, wrap_neutral, wrap_func ) => (false,true,false), + (true, decls_none, wrap_neutral, wrap_hard ) => (false,true,false), + (true, decls_none, wrap_neutral, wrap_neutral) => (true,true,true), + (true, decls_none, wrap_none, wrap_func ) => (false,true,false), + (true, decls_none, wrap_none, wrap_hard ) => (false,true,false), + (true, decls_none, wrap_none, wrap_neutral) => (true,true,true), + ) + expected_s(b::Bool) = b ? "assignment to outer var" : "brand-new var" + + local jl_mod = Module() + local fl_mod = Module() + + @testset for ((soft_mode, decls_s, middle_s, assign_s), results) in expected_outer_vals, + (lhs_i, lhs) in enumerate(lhs_names) + + ex = decls_s(middle_s(assign_s(:($lhs = true))), lhs) + soft_mode && (ex = enable_softscope(ex)) + + if lhs in (:argname, :spname) && decls_s !== decls_func + continue + elseif lhs === :spname + @test_throws LoweringError expr_eval(jl_mod, ex) + else + @assert !isdefined(jl_mod, lhs) && !isdefined(fl_mod, lhs) + expected = results[lhs_i] + reference_ok = fl_eval(fl_mod, ex) === expected + !reference_ok && @error("flisp produced unexpected result; fix that or JL scope tests:\n", + "expected $(expected_s(expected)), got $(expected_s(!expected))\n", ex) + @test reference_ok + ok = expr_eval(jl_mod, ex) === expected + !ok && @error("expected $(expected_s(expected)), got $(expected_s(!expected))\n", ex) + @test ok + end + + Core.@latestworld + for mod in (jl_mod, fl_mod), n in (:gname, :name) + isdefined(mod, n) && Base.delete_binding(mod, n) + end + Core.@latestworld + end +end + +@testset "global declarations at top level are ignored in assignment resolution" begin + suggest_global(e) = :(begin; global declared_unassigned_global; $e; end) + for soft_mode in (true, false), scope in (wrap_func, wrap_hard, wrap_neutral) + ex = scope(:(declared_unassigned_global = true)) + soft_mode && (ex = enable_softscope(ex)) + expr_eval(test_mod, ex) + global_assigned = @invokelatest isdefined(test_mod, :declared_unassigned_global) + global_assigned && error("global should not be assigned. settings: $soft_mode $scope\n") + @test !global_assigned + end + + @testset "soft scope isn't top level" begin + ex = quote + begin + for i in 1:1; global soft_assigned_explicit_global = 1; end + for i in 1:1; soft_assigned_explicit_global = 2; end + end + end + expr_eval(test_mod, enable_softscope(ex)) + @test test_mod.soft_assigned_explicit_global === 1 + end +end + +# Distinct from the stateful "existing global" check (probably to get around the +# case where the global only becomes existing within the expression being +# lowered) +@testset "assignments at top level can influence assignment resolution in soft scopes" begin + for soft_mode in (true, false), + s1 in (wrap_neutral, (e)->wrap_neutral(wrap_neutral(e))), + g_assign in (:(assigned_global = false), :(global assigned_global = false)) + + inner_assign_islocal = s1(Expr( + :block, + :(assigned_global = true), + Expr(:(=), :out, Expr(:islocal, :assigned_global)))) + + for ex in (Expr(:block, :(local out), inner_assign_islocal, g_assign, :out), + Expr(:block, :(local out), g_assign, inner_assign_islocal, :out)) + + if soft_mode + ex = enable_softscope(ex) + ok = expr_eval(test_mod, ex) === false + !ok && error("expected assignment to global\n", ex) + @test ok + else + # some of these produce warning in flisp + ok = expr_eval(test_mod, ex) === true + !ok && error("expected assignment to local\n", ex) + @test ok + end + Base.delete_binding(test_mod, :assigned_global) + end + end +end + +module ambiguous_local + global x::Int = 0 +end + +function resolve_and_get_bindings( + mod::Module, ex; + world::UInt = Base.get_world_counter(), + soft_scope::Union{Nothing,Bool} = nothing, + ) + est = JuliaLowering.expr_to_est(ex) + ex0 = JuliaLowering.rebase_layers(est, mod, JuliaLowering.JL_NEW_SYNTAX_VERSION) + ex1 = JuliaLowering.expand_forms_1(ex0, world, true) + ctx2, ex2 = JuliaLowering.expand_forms_2(ex1, world) + ctx3, _ = JuliaLowering.resolve_scopes(ctx2, ex2; soft_scope) + return ctx3.bindings.info +end + +@testset "internal keyword body bindings" begin + bindings = resolve_and_get_bindings(Module(), :(f(; x=1) = x)) + kw_body_bindings = filter(b -> contains(b.name, "#kw_body#"), bindings) + @test !isempty(kw_body_bindings) + @test all(b -> b.is_internal, kw_body_bindings) +end + +@testset "is_ambiguous_local" begin + # Assignment in for loop within begin block after toplevel assignment + let bindings = resolve_and_get_bindings(ambiguous_local, :(for _ = 1:10; x = 1; end)) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :local + @test binfo.is_ambiguous_local + end + + # while loop + let bindings = resolve_and_get_bindings(ambiguous_local, :(while x < 5; x += 1; break; end)) + binfos = filter(b->b.name=="x", bindings) + @test length(binfos) == 2 + binfo = only(filter(b->b.kind==:local, binfos)) + @test binfo.is_ambiguous_local + @test count(b->b.kind==:global, binfos) == 1 + end + + # No ambiguity inside a function (hard scope) + let bindings = resolve_and_get_bindings(ambiguous_local, :(function f() + for _ = 1:10 + x = 1 + end + end)) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :local + @test !binfo.is_ambiguous_local + end + + # No ambiguity when shadowing global variable does not exist + let bindings = resolve_and_get_bindings(ambiguous_local, :(for _ = 1:10; y = 1; end)) + binfo = only(filter(b->b.name=="y", bindings)) + @test binfo.kind === :local + @test !binfo.is_ambiguous_local + end + + # Explicit `global` should not produce ambiguous local + let bindings = resolve_and_get_bindings(ambiguous_local, :(for _ = 1:10; global x = 1; end)) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :global + end + + # Block containing a toplevel assignment preceding a permeable scope + let bindings = resolve_and_get_bindings(Module(), quote + x = 0 + for _ = 1:10 + x = 1 + end + end) + binfos = filter(b->b.name=="x", bindings) + @test length(binfos) == 2 + binfo = only(filter(b->b.kind==:local, binfos)) + @test binfo.is_ambiguous_local + @test count(b->b.kind==:global, binfos) == 1 + end + # Block containing a permeable scope followed by a toplevel assignment + let bindings = resolve_and_get_bindings(Module(), quote + for _ = 1:10 + x = 1 + end + x = 0 + end) + binfos = filter(b->b.name=="x", bindings) + @test length(binfos) == 2 + binfo = only(filter(b->b.kind==:local, binfos)) + @test binfo.is_ambiguous_local + @test count(b->b.kind==:global, binfos) == 1 + end + + # For some reason, flisp can avoid ambiguity when there is an additional `global` annotation. + # JuliaLowering may want to follow suit, but it would be better to first decide on the details of this behaviour. + let bindings = resolve_and_get_bindings(Module(), quote + global x = 0 + for _ = 1:10 + x = 1 + end + end) + binfos = filter(b->b.name=="x", bindings) + @test length(binfos) == 2 + binfo = only(filter(b->b.kind==:local, binfos)) + @test_broken !binfo.is_ambiguous_local + @test count(b->b.kind==:global, binfos) == 1 + end + let bindings = resolve_and_get_bindings(Module(), quote + for _ = 1:10 + x = 1 + end + global x = 0 + end) + binfos = filter(b->b.name=="x", bindings) + @test length(binfos) == 2 + binfo = only(filter(b->b.kind==:local, binfos)) + @test_broken !binfo.is_ambiguous_local + @test count(b->b.kind==:global, binfos) == 1 + end + + @testset "soft_scope kwarg override" begin + # Without soft_scope, x becomes an ambiguous local + let bindings = resolve_and_get_bindings(ambiguous_local, :(for _ = 1:10; x = 1; end)) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :local + @test binfo.is_ambiguous_local + end + # With soft_scope=true, x stays global (no local created) + let bindings = resolve_and_get_bindings(ambiguous_local, :(for _ = 1:10; x = 1; end); soft_scope=true) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :global + end + end + + @testset "world-age propagation" begin + let m = Module() + Core.eval(m, :(global x = 0)) + bindings = resolve_and_get_bindings(m, :(for _ = 1:10; x = 1; end); world=Base.get_world_counter()) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :local + @test binfo.is_ambiguous_local + end + let m = Module() + Core.eval(m, :(global x = 0)) + bindings = resolve_and_get_bindings(m, :(for _ = 1:10; x = 1; end); world=Base.get_world_counter(), soft_scope=true) + binfo = only(filter(b->b.name=="x", bindings)) + @test binfo.kind === :global + end + end +end + +@testset "unescaped macro expansions introduce a hygienic scope" begin + @eval test_mod module macro_mod + macro m(x); x; end + macro mesc(x); esc(x); end + end + + # A function not wrapped in anything is made a macro-module global (#32026) + JuliaLowering.include_string(test_mod, "macro_mod.@m function f_bug_1(); 1; end") + @test isdefined(test_mod.macro_mod, :f_bug_1) + JuliaLowering.include_string(test_mod, "macro_mod.@m function f_bug_2 end") + @test isdefined(test_mod.macro_mod, :f_bug_2) + JuliaLowering.include_string(test_mod, "macro_mod.@m f_bug_3(x) = 1") + @test isdefined(test_mod.macro_mod, :f_bug_3) + JuliaLowering.include_string(test_mod, "macro_mod.@m f_bug_4(x)::Int = 1") + @test isdefined(test_mod.macro_mod, :f_bug_4) + # (wrapped def is fine) + JuliaLowering.include_string(test_mod, "macro_mod.@m begin; f_local_1(x) = 1; end") + @test !isdefined(test_mod.macro_mod, :f_local_1) + @test !isdefined(test_mod, :f_local_1) + + JuliaLowering.include_string(test_mod, "macro_mod.@mesc function f_nonlocal_2(); 1; end") + @test isdefined(test_mod, :f_nonlocal_2) + # An unescaped const should not error coming from an old-style macro + @test JuliaLowering.include_string(test_mod, "macro_mod.@m const c_local_1 = 1") == 1 + # it should be unhygienic module-wise + @test !isdefined(test_mod.macro_mod, :c_local_1) + # flisp mangles it, we make a local + @test !isdefined(test_mod, :c_local_1) + + # The const may be escaped into test_mod + JuliaLowering.include_string(test_mod, "macro_mod.@mesc const c_nonlocal_2 = 1") + @test isdefined(test_mod, :c_nonlocal_2) + JuliaLowering.include_string(test_mod, "macro_mod.@mesc const c_nonlocal_3 = 1"; expr_compat_mode=true) + @test isdefined(test_mod, :c_nonlocal_3) +end + +fl_eval(test_mod, :(macro old_hyg(x); x; end)) +fl_eval(test_mod, :(macro old_esc(x); Expr(:escape, x); end)) + +# caller modules, where test_mod is the macro module +module jl_mod +import ..test_mod.@old_hyg +import ..test_mod.@old_esc +end +module fl_mod +import ..test_mod.@old_hyg +import ..test_mod.@old_esc +end + +# In flisp, with no escaping: +# - Top level globals are unhygienic and declared in the calling module +# - this includes abstract, primitive, and struct types + +# Not yet explicitly handled or tested: +# - Top-level functions are unhygienic and declared in the macro's module +# - Top-level `x=y` implicitly declares hygienic globals (but it is not breaking +# to make them local) +# +# See https://github.com/JuliaLang/julia/issues/53667 for more quirks +@testset "compat: macro hygiene exemptions for explicit globals" begin + # desirable side of this behaviour where global decls passed as arguments to + # non-escaping macros appear to have automatic hygiene (it's assumed that + # declaring a global in the body of a macro is usually intended to produce a + # global in the calling module). + @testset "passed as an argument" for (ctx, mod, run) in [ + ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x)), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true)), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false))] + + @testset for str in [ + # "@old_hyg const GENSYM = 1; GENSYM == 1" # flisp mangles, JL counts local + "@old_hyg(global GENSYM = 1); GENSYM == 1" + "@old_hyg(global GENSYM::Int = 1); GENSYM == 1" + "@old_hyg(global (((GENSYM,),),) = (((1,),),)); GENSYM == 1" + "(()->(@old_hyg global GENSYM = 1))() == 1" + "@old_hyg(global const GENSYM = 1); GENSYM == 1" + "@old_hyg(const global GENSYM = 1); GENSYM == 1" + "@old_hyg(struct GENSYM end); GENSYM isa Type" + "@old_hyg(struct GENSYM; x::Int; GENSYM(x) = new(x); end); GENSYM(42).x == 42" + "@old_hyg(struct GENSYM{T}; x::T; end); GENSYM(42).x == 42" + "@old_hyg(abstract type GENSYM end); GENSYM isa Type" + "@old_hyg(primitive type GENSYM 8 end); GENSYM isa Type" + + # global functions: flisp encounters errors + # "@old_hyg(global GENSYM(x) = x); GENSYM isa Function" + # "@old_hyg(global function GENSYM(x)\nx\nend); GENSYM isa Function" + # "@old_hyg(let\n global GENSYM(x) = x\nend); GENSYM isa Function" + # "@old_hyg(let\n global function GENSYM(x)\nx\nend \nend); GENSYM isa Function" + ] + @gensym gen_global_sym + prog_str = "#="*ctx*"=# "*replace( + str, "GENSYM"=>"var\""*(string(gen_global_sym))*"\"") + prog = JuliaSyntax.parseall(SyntaxTree, prog_str) + + @test run(prog) context=prog_str + Core.@latestworld + @test Base.isdefinedglobal(mod, gen_global_sym) context=prog_str + @test !Base.isdefinedglobal(test_mod, gen_global_sym) context=prog_str + end + + # these shouldn't resolve (JL implementation shouldn't be too lax) + @testset for str in [ + "begin; global GENSYM=1; @old_hyg(GENSYM) == 1; end" + "begin; global GENSYM::Int=1; @old_hyg(GENSYM) == 1; end" + "begin; global (((GENSYM,),),) = (((1,),),); @old_hyg(GENSYM) == 1; end" + "begin; (()->(global GENSYM = 1; @old_hyg(GENSYM)))() == 1; end" + "begin; global const GENSYM = 1; @old_hyg(GENSYM) == 1; end" + "begin; const global GENSYM = 1; @old_hyg(GENSYM) == 1; end" + "begin; struct GENSYM end; @old_hyg(GENSYM) isa Type; end" + "begin; abstract type GENSYM end; @old_hyg(GENSYM) isa Type; end" + "begin; primitive type GENSYM 8 end; @old_hyg(GENSYM) isa Type; end" + ] + @gensym gen_global_sym + prog_str = "#="*ctx*"=# "*replace( + str, "GENSYM"=>"var\""*(string(gen_global_sym))*"\"") + prog = JuliaSyntax.parseall(SyntaxTree, prog_str) + + @test_throws string(gen_global_sym) run(prog) context=prog_str + Core.@latestworld + @test Base.isdefinedglobal(mod, gen_global_sym) context=prog_str + @test !Base.isdefinedglobal(test_mod, gen_global_sym) context=prog_str + end + end + + # bad side: the same global decl coming from the body of the macro behaves + # as if it was passed as a macro argument. It would have been more + # consistent to require escaping here, since now it's impossible to + # represent a global declaration in the macro module. + @testset "from the macro body" for (ctx, mod, run) in [ + ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x)), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true)), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false))] + + # interpolating global name here appears to hit a bug + fl_eval(test_mod, :(macro old_hyg_globalvar(str); + quote + global old_hyg_globalvar_G = $str + end + end)) + run(:(import ..test_mod.@old_hyg_globalvar)) + Core.@latestworld + let s = "ran old_hyg_globalvar" + @test run(:(@old_hyg_globalvar $s)) == s context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_globalvar_G) context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_globalvar_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_globalvar_G) + Base.delete_binding(test_mod, :old_hyg_globalvar_G) + + fl_eval(test_mod, :(macro old_hyg_globalvar_typed(str); + quote + global old_hyg_globalvar_typed_G::String = $str + end + end)) + run(:(import ..test_mod.@old_hyg_globalvar_typed)) + Core.@latestworld + let s = "ran old_hyg_globalvar_typed" + @test run(:(@old_hyg_globalvar_typed $s)) == s context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_globalvar_typed_G) context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_globalvar_typed_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_globalvar_typed_G) + Base.delete_binding(test_mod, :old_hyg_globalvar_typed_G) + + fl_eval(test_mod, :(macro old_hyg_globalvar_tuple(str); + quote + global (((old_hyg_globalvar_tuple_G,),),) = ((($str,),),) + end + end)) + run(:(import ..test_mod.@old_hyg_globalvar_tuple)) + Core.@latestworld + let s = "ran old_hyg_globalvar_tuple" + @test run(:(@old_hyg_globalvar_tuple $s)) == (((s,),),) context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_globalvar_tuple_G) context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_globalvar_tuple_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_globalvar_tuple_G) + Base.delete_binding(test_mod, :old_hyg_globalvar_tuple_G) + + fl_eval(test_mod, :(macro old_hyg_globalvar_in_lam(str); + quote + (()->(global old_hyg_globalvar_in_lam_G = $str))() + end + end)) + run(:(import ..test_mod.@old_hyg_globalvar_in_lam)) + Core.@latestworld + let s = "ran old_hyg_globalvar_in_lam" + @test run(:(@old_hyg_globalvar_in_lam $s)) == s context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_globalvar_in_lam_G) context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_globalvar_in_lam_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_globalvar_in_lam_G) + Base.delete_binding(test_mod, :old_hyg_globalvar_in_lam_G) + + fl_eval(test_mod, :(macro old_hyg_globalvar_const(str); + quote + global const old_hyg_globalvar_const_G = $str + end + end)) + run(:(import ..test_mod.@old_hyg_globalvar_const)) + Core.@latestworld + let s = "ran old_hyg_globalvar_const" + @test run(:(@old_hyg_globalvar_const $s)) == s context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_globalvar_const_G) context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_globalvar_const_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_globalvar_const_G) + Base.delete_binding(test_mod, :old_hyg_globalvar_const_G) + + fl_eval(test_mod, :(macro old_hyg_const_globalvar(str); + quote + const global old_hyg_const_globalvar_G = $str + end + end)) + run(:(import ..test_mod.@old_hyg_const_globalvar)) + Core.@latestworld + let s = "ran old_hyg_const_globalvar" + @test run(:(@old_hyg_const_globalvar $s)) == s context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_const_globalvar_G) context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_const_globalvar_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_const_globalvar_G) + Base.delete_binding(test_mod, :old_hyg_const_globalvar_G) + + fl_eval(test_mod, :(macro old_hyg_struct(str); + quote + struct old_hyg_struct_G end + end + end)) + run(:(import ..test_mod.@old_hyg_struct)) + Core.@latestworld + let s = "ran old_hyg_struct" + @test run(:(@old_hyg_struct $s)) === nothing context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_struct_G) context=ctx + @test mod.old_hyg_struct_G isa Type context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_struct_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_struct_G) + Base.delete_binding(test_mod, :old_hyg_struct_G) + + fl_eval(test_mod, :(macro old_hyg_abstract_type(str); + quote + abstract type old_hyg_abstract_type_G end + end + end)) + run(:(import ..test_mod.@old_hyg_abstract_type)) + Core.@latestworld + let s = "ran old_hyg_abstract_type" + @test run(:(@old_hyg_abstract_type $s)) === nothing context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_abstract_type_G) context=ctx + @test mod.old_hyg_abstract_type_G isa Type context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_abstract_type_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_abstract_type_G) + Base.delete_binding(test_mod, :old_hyg_abstract_type_G) + + fl_eval(test_mod, :(macro old_hyg_primitive_type(str); + quote + primitive type old_hyg_primitive_type_G 8 end + end + end)) + run(:(import ..test_mod.@old_hyg_primitive_type)) + Core.@latestworld + let s = "ran old_hyg_primitive_type" + @test run(:(@old_hyg_primitive_type $s)) === nothing context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_primitive_type_G) context=ctx + @test mod.old_hyg_primitive_type_G isa Type context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_primitive_type_G) context=ctx + end + Base.delete_binding(mod, :old_hyg_primitive_type_G) + Base.delete_binding(test_mod, :old_hyg_primitive_type_G) + + end +end + +# The hygiene exemption above applies only within the scope containing the +# `global` declaration: the declaration creates a global in the calling module, +# but other references to the same name elsewhere in the expansion are still +# hygienic, resolving in the macro's module. +@testset "(AI) compat: hygiene exemption is confined to the declaring scope" begin + # f() should throw an UndefVarError for the macro-module (hygienic) global + undef_in_test_mod(f, name) = begin + err = try; f(); catch e; e; end + err isa UndefVarError && err.var === name && err.scope === test_mod + end + + @testset "references outside the declaring scope" for (ctx, mod, run) in [ + ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x)), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true)), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false))] + + fl_eval(test_mod, :(macro old_hyg_g_in_let(); + quote + let + global old_hyg_g_in_let_G = 1 + end + old_hyg_g_in_let_G + end + end)) + run(:(import ..test_mod.@old_hyg_g_in_let)) + Core.@latestworld + @test undef_in_test_mod(()->run(:(@old_hyg_g_in_let)), + :old_hyg_g_in_let_G) context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_g_in_let_G) && + getglobal(mod, :old_hyg_g_in_let_G) == 1 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_g_in_let_G) context=ctx + Base.delete_binding(mod, :old_hyg_g_in_let_G) + Base.delete_binding(test_mod, :old_hyg_g_in_let_G) + + fl_eval(test_mod, :(macro old_hyg_g_in_lam(); + quote + (()->(global old_hyg_g_in_lam_G = 2))() + old_hyg_g_in_lam_G + end + end)) + run(:(import ..test_mod.@old_hyg_g_in_lam)) + Core.@latestworld + @test undef_in_test_mod(()->run(:(@old_hyg_g_in_lam)), + :old_hyg_g_in_lam_G) context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_g_in_lam_G) && + getglobal(mod, :old_hyg_g_in_lam_G) == 2 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_g_in_lam_G) context=ctx + Base.delete_binding(mod, :old_hyg_g_in_lam_G) + Base.delete_binding(test_mod, :old_hyg_g_in_lam_G) + + fl_eval(test_mod, :(macro old_hyg_g_sibling(); + quote + let + global old_hyg_g_sibling_G = 3 + end + let + old_hyg_g_sibling_G + end + end + end)) + run(:(import ..test_mod.@old_hyg_g_sibling)) + Core.@latestworld + @test undef_in_test_mod(()->run(:(@old_hyg_g_sibling)), + :old_hyg_g_sibling_G) context=ctx + Core.@latestworld + @test Base.isdefinedglobal(mod, :old_hyg_g_sibling_G) && + getglobal(mod, :old_hyg_g_sibling_G) == 3 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_g_sibling_G) context=ctx + Base.delete_binding(mod, :old_hyg_g_sibling_G) + Base.delete_binding(test_mod, :old_hyg_g_sibling_G) + end + + @testset "rescoping conflicts" for (ctx, mod, run) in [ + # flisp is quite unpredictable here: segfaults, local-form-assigns-global + # ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x)), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true)), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false))] + + fl_eval(test_mod, :(macro old_hyg_g_rescope_conflict_old(); + quote + global bad = 1 + end + end)) + run(:(import ..test_mod.@old_hyg_g_rescope_conflict_old)) + Core.@latestworld + @test_throws "conflicts with an existing" run(:( + let bad = 1; @old_hyg_g_rescope_conflict_old(); end)) context=ctx + @test_throws "conflicts with an existing" run(:( + ((bad)->@old_hyg_g_rescope_conflict_old())(1))) context=ctx + @test_throws "conflicts with an existing" run(:( + (function old_hyg_g_rescope_conflict_old_F() where bad + @old_hyg_g_rescope_conflict_old() + end)())) context=ctx + + fl_eval(test_mod, :(macro old_hyg_g_rescope_conflict_new1(); + :(let + local bad = 1 + global bad = 2 + end) + end)) + run(:(import ..test_mod.@old_hyg_g_rescope_conflict_new1)) + Core.@latestworld + @test_throws "unhygienic global" run(:(@old_hyg_g_rescope_conflict_new1())) context=ctx + + fl_eval(test_mod, :(macro old_hyg_g_rescope_conflict_new2(); + :(let + global bad = 2 + local bad = 1 + end) + end)) + run(:(import ..test_mod.@old_hyg_g_rescope_conflict_new2)) + Core.@latestworld + @test_throws "unhygienic global" run(:(@old_hyg_g_rescope_conflict_new2())) context=ctx + + fl_eval(test_mod, :(macro old_hyg_g_rescope_conflict_new3(); + :(let + local bad + global bad + end) + end)) + run(:(import ..test_mod.@old_hyg_g_rescope_conflict_new3)) + Core.@latestworld + @test_throws "conflicts with an existing local variable" run(:(@old_hyg_g_rescope_conflict_new3())) context=ctx + + fl_eval(test_mod, :(macro old_hyg_g_rescope_conflict_new4(); + :(let + global bad + local bad + end) + end)) + run(:(import ..test_mod.@old_hyg_g_rescope_conflict_new4)) + Core.@latestworld + @test_throws "conflicts with an existing local variable" run(:(@old_hyg_g_rescope_conflict_new4())) context=ctx + end + + # A reference in the same scope as the declaration: flisp keeps the + # reference hygienic (hitting the undefined macro-module global), but + # JuliaLowering deliberately resolves it to the global the declaration just + # created in the calling module, which is more consistent. + @testset "references in the declaring scope" for (ctx, mod, run, ref_resolves) in [ + ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x), false), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true), false), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false), false)] + + fl_eval(test_mod, :(macro old_hyg_g_ref_top(); + quote + global old_hyg_g_ref_top_G = 4 + old_hyg_g_ref_top_G + 10 + end + end)) + run(:(import ..test_mod.@old_hyg_g_ref_top)) + Core.@latestworld + if ref_resolves + @test run(:(@old_hyg_g_ref_top)) == 14 context=ctx + else + @test undef_in_test_mod(()->run(:(@old_hyg_g_ref_top)), + :old_hyg_g_ref_top_G) context=ctx + end + Core.@latestworld + @test getglobal(mod, :old_hyg_g_ref_top_G) == 4 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_g_ref_top_G) context=ctx + Base.delete_binding(mod, :old_hyg_g_ref_top_G) + Base.delete_binding(test_mod, :old_hyg_g_ref_top_G) + + fl_eval(test_mod, :(macro old_hyg_g_ref_let(); + quote + let + global old_hyg_g_ref_let_G = 5 + old_hyg_g_ref_let_G + 10 + end + end + end)) + run(:(import ..test_mod.@old_hyg_g_ref_let)) + Core.@latestworld + if ref_resolves + @test run(:(@old_hyg_g_ref_let)) == 15 context=ctx + else + @test undef_in_test_mod(()->run(:(@old_hyg_g_ref_let)), + :old_hyg_g_ref_let_G) context=ctx + end + Core.@latestworld + @test getglobal(mod, :old_hyg_g_ref_let_G) == 5 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_g_ref_let_G) context=ctx + Base.delete_binding(mod, :old_hyg_g_ref_let_G) + Base.delete_binding(test_mod, :old_hyg_g_ref_let_G) + + # repeated `global` declarations of the same name in one scope + fl_eval(test_mod, :(macro old_hyg_g_dup(); + quote + global old_hyg_g_dup_G + global old_hyg_g_dup_G = 6 + old_hyg_g_dup_G + end + end)) + run(:(import ..test_mod.@old_hyg_g_dup)) + Core.@latestworld + if ref_resolves + @test run(:(@old_hyg_g_dup)) == 6 context=ctx + else + @test undef_in_test_mod(()->run(:(@old_hyg_g_dup)), + :old_hyg_g_dup_G) context=ctx + end + Core.@latestworld + @test getglobal(mod, :old_hyg_g_dup_G) == 6 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_g_dup_G) context=ctx + Base.delete_binding(mod, :old_hyg_g_dup_G) + Base.delete_binding(test_mod, :old_hyg_g_dup_G) + end + + # The soft scope exemption (assignment to an existing global from a + # top-level loop) is for globals visible to the user at the macrocall site; + # a hygienic assignment in an expansion must not hit an existing global of + # the same name in the macro's module. + @testset "soft scope assignments stay hygienic" for (ctx, mod, run) in [ + ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x)), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true)), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false))] + + fl_eval(test_mod, :(macro old_hyg_soft(); + quote + for i = 1:1 + old_hyg_soft_G = 99 + end + end + end)) + Core.eval(test_mod, :(global old_hyg_soft_G = 0)) + run(:(import ..test_mod.@old_hyg_soft)) + Core.@latestworld + @test run(enable_softscope(:(@old_hyg_soft))) === nothing context=ctx + Core.@latestworld + @test getglobal(test_mod, :old_hyg_soft_G) == 0 context=ctx + @test !Base.isdefinedglobal(mod, :old_hyg_soft_G) context=ctx + Base.delete_binding(mod, :old_hyg_soft_G) + Base.delete_binding(test_mod, :old_hyg_soft_G) + + # escaped version assigns the caller's global as usual + fl_eval(test_mod, :(macro old_esc_soft(); + Expr(:escape, quote + for i = 1:1 + old_esc_soft_G = 99 + end + end) + end)) + Core.eval(mod, :(global old_esc_soft_G = 0)) + run(:(import ..test_mod.@old_esc_soft)) + Core.@latestworld + @test run(enable_softscope(:(@old_esc_soft))) === nothing context=ctx + Core.@latestworld + @test getglobal(mod, :old_esc_soft_G) == 99 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_esc_soft_G) context=ctx + Base.delete_binding(mod, :old_esc_soft_G) + Base.delete_binding(test_mod, :old_esc_soft_G) + end + + # Structs defined in the body of an old macro are unhygienic. Constructor + # lowering (inner constructors and the runtime default constructors) + # references the global struct name, which must resolve to the rescoped + # global from within the struct's scope. + @testset "struct from the macro body" for (ctx, mod, run) in [ + ("flisp reference (delete if fail)", fl_mod, x->fl_eval(fl_mod, x)), + ("jl expr_compat_mode=true", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=true)), + ("jl expr_compat_mode=false", jl_mod, x->jl_eval(jl_mod, x; expr_compat_mode=false))] + + fl_eval(test_mod, :(macro old_hyg_struct_ctor(); + quote + struct old_hyg_struct_ctor_G + x::Int + old_hyg_struct_ctor_G(x) = new(x) + end + end + end)) + run(:(import ..test_mod.@old_hyg_struct_ctor)) + Core.@latestworld + @test run(:(@old_hyg_struct_ctor)) === nothing context=ctx + Core.@latestworld + @test run(:(old_hyg_struct_ctor_G(42).x)) == 42 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_struct_ctor_G) context=ctx + Base.delete_binding(mod, :old_hyg_struct_ctor_G) + Base.delete_binding(test_mod, :old_hyg_struct_ctor_G) + + fl_eval(test_mod, :(macro old_hyg_struct_tv(); + quote + struct old_hyg_struct_tv_G{T} + x::T + end + end + end)) + run(:(import ..test_mod.@old_hyg_struct_tv)) + Core.@latestworld + @test run(:(@old_hyg_struct_tv)) === nothing context=ctx + Core.@latestworld + @test run(:(old_hyg_struct_tv_G(42).x)) == 42 context=ctx + @test !Base.isdefinedglobal(test_mod, :old_hyg_struct_tv_G) context=ctx + Base.delete_binding(mod, :old_hyg_struct_tv_G) + Base.delete_binding(test_mod, :old_hyg_struct_tv_G) + end +end + +@testset "@isdefined sees imported globals" begin + # implicit Core/Base visibility and `using`-provided names count as defined + # at module scope + m = Module(:IsdefM) + @test JuliaLowering.include_string(m, "@isdefined Core") === true + @test JuliaLowering.include_string(m, "@isdefined Base") === true + @test JuliaLowering.include_string(m, "@isdefined sin") === true + @test JuliaLowering.include_string(m, "@isdefined not_a_thing_anywhere") === false + @test JuliaLowering.include_string(m, "f() = @isdefined(Core); f()") === true +end diff --git a/JuliaLowering/test/scopes_ir.jl b/JuliaLowering/test/scopes_ir.jl new file mode 100644 index 0000000000000..6274a210ad49c --- /dev/null +++ b/JuliaLowering/test/scopes_ir.jl @@ -0,0 +1,499 @@ +using .JuliaLowering: @islocal +using Base: @locals + +#******************************************************************************* +######################################## +# let syntax with decl in binding list +let x::T = rhs + local T = 1 + T # <- This is a different `T` from the T in `x::T` +end +#--------------------- +1 TestMod.rhs +2 (newvar slot₂/T) +3 slot₂/T +4 (= slot₃/tmp %₁) +5 (call core.isa slot₃/tmp %₃) +6 (gotoifnot %₅ label₈) +7 (goto label₁₀) +8 (call top.convert %₃ slot₃/tmp) +9 (= slot₃/tmp (call core.typeassert %₈ %₃)) +10 slot₃/tmp +11 (= slot₁/x %₁₀) +12 (= slot₂/T 1) +13 slot₂/T +14 (return %₁₃) + +######################################## +# let syntax with tuple on lhs +let (x,y) = rhs +end +#--------------------- +1 TestMod.rhs +2 (call top.indexed_iterate %₁ 1) +3 (= slot₂/x (call core.getfield %₂ 1)) +4 (= slot₁/iterstate (call core.getfield %₂ 2)) +5 slot₁/iterstate +6 (call top.indexed_iterate %₁ 2 %₅) +7 (= slot₃/y (call core.getfield %₆ 1)) +8 (return core.nothing) + +######################################## +# let syntax with named tuple on lhs creates locals for the unpacked vars +let (; x,y) = rhs +end +#--------------------- +1 TestMod.rhs +2 (= slot₁/x (call top.getproperty %₁ :x)) +3 (= slot₂/y (call top.getproperty %₁ :y)) +4 (return core.nothing) + +######################################## +# Let syntax with the same name creates nested bindings +let x = f(x), x = g(x) +end +#--------------------- +1 TestMod.f +2 TestMod.x +3 (call %₁ %₂) +4 (= slot₁/x %₃) +5 TestMod.g +6 (call %₅ slot₁/x) +7 (= slot₂/x %₆) +8 (return core.nothing) + +######################################## +# let syntax with a function definition in the binding list creates a closure +let f() = body +end +#--------------------- +1 (call core.svec) +2 (call core.svec) +3 (call core.svec) +4 (call JuliaLowering.eval_closure_type TestMod :#f#f##0 %₁ %₂ %₃) +5 latestworld +6 TestMod.#f#f##0 +7 (new %₆) +8 TestMod.#f#f##0 +9 (call core.svec %₈) +10 (call core.svec) +11 SourceLocation::1:4 +12 (call core.svec %₉ %₁₀ %₁₁) +13 (call core.define_method TestMod core.nothing %₁₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 TestMod.body + 2 (return %₁) +14 latestworld +15 (= slot₁/f %₇) +16 (return core.nothing) + +######################################## +# Error: Invalid `let` var with K"::" +let f[]::T = rhs +end +#--------------------- +LoweringError: +let f[]::T = rhs +# └─┘ ── Invalid assignment location in let syntax +end + +######################################## +# Error: Invalid `let` var +let f[] = rhs +end +#--------------------- +LoweringError: +let f[] = rhs +# └─┘ ── Invalid assignment location in let syntax +end + +######################################## +# Error: Invalid function def in `let` +let (obj::Callable)() = rhs +end +#--------------------- +LoweringError: +let (obj::Callable)() = rhs +# └───────────────┘ ── Function signature does not define a local function name +end + +######################################## +# @islocal with locals and undefined vars +let x = 1 + @islocal(a), @islocal(x) +end +#--------------------- +1 1 +2 (= slot₁/x %₁) +3 (call core.tuple false true) +4 (return %₃) + +######################################## +# @islocal with function arguments +# (y is single-assigned before capture, so no Box needed) +begin + local y = 2 + function f(x) + @islocal(a), @islocal(x), @islocal(y) + end +end +#--------------------- +1 (= slot₁/y 2) +2 (call core.define_method TestMod :f) +3 latestworld +4 TestMod.f +5 (call core.TypeEqOf %₄) +6 (call core.svec %₅ core.Any) +7 (call core.svec) +8 SourceLocation::3:5 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read)] + 1 (call core.tuple false true true) + 2 (return %₁) +11 latestworld +12 TestMod.f +13 (return %₁₂) + +######################################## +# @islocal with global +begin + global x + @islocal(x) +end +#--------------------- +1 (call core.declare_global TestMod :x false) +2 latestworld +3 (return false) + +######################################## +# @locals with local and global +begin + global x + local y + @locals +end +#--------------------- +1 (newvar slot₁/y) +2 (call core.declare_global TestMod :x false) +3 latestworld +4 (call core.apply_type top.Dict core.Symbol core.Any) +5 (call %₄) +6 (isdefined slot₁/y) +7 (gotoifnot %₆ label₁₀) +8 slot₁/y +9 (call top.setindex! %₅ %₈ :y) +10 (return %₅) + +######################################## +# @locals with function args, sparams +function f(z::T) where T + @locals +end +#--------------------- +1 (call core.define_method TestMod :f) +2 latestworld +3 (call core.TypeVar :T) +4 TestMod.f +5 (call core.TypeEqOf %₄) +6 (call core.svec %₅ %₃) +7 (call core.svec %₃) +8 SourceLocation::1:1 +9 (call core.svec %₆ %₇ %₈) +10 (call core.define_method TestMod TestMod.f %₉ + --- code_info + slots: [slot₁/#self#(!read) slot₂/z] + 1 (call core.apply_type top.Dict core.Symbol core.Any) + 2 (call %₁) + 3 (gotoifnot true label₅) + 4 (call top.setindex! %₂ slot₂/z :z) + 5 (isdefined static_parameter₁) + 6 (gotoifnot %₅ label₉) + 7 static_parameter₁ + 8 (call top.setindex! %₂ %₇ :T) + 9 (return %₂) +11 latestworld +12 TestMod.f +13 (return %₁₂) + +######################################## +# Error: Duplicate function argument names +function f(x, x) +end +#--------------------- +LoweringError: +function f(x, x) +# ╙ ── function argument name not unique +end + +######################################## +# Error: Duplicate function argument with destructured arg +function f(x, (x,)) +end +#--------------------- +LoweringError: +function f(x, (x,)) +# ╙ ── destructured argument name `x` conflicts with an existing argument from the same scope +end + +######################################## +# Error: Static parameter name not unique +function f(::T) where T where T +end +#--------------------- +LoweringError: +function f(::T) where T where T +# ╙ ── function static parameter name not unique +end + +######################################## +# Error: static parameter colliding with argument names +function f(x::x) where x +end +#--------------------- +LoweringError: +function f(x::x) where x +# ╙ ── static parameter name `x` conflicts with an existing argument from the same scope +end + +######################################## +# Error: duplicate destructure args +function f((x,), (x,)) +end +#--------------------- +LoweringError: +function f((x,), (x,)) +# ╙ ── destructured argument name `x` conflicts with an existing local variable from the same scope +end + +######################################## +# Error: Conflicting local and global decls +let + local x + global x +end +#--------------------- +LoweringError: +let + local x + global x +# ╙ ── global variable name `x` conflicts with an existing local variable from the same scope +end + +######################################## +# Error: Conflicting argument and local +function f(x) + local x +end +#--------------------- +LoweringError: +function f(x) + local x +# ╙ ── local variable name `x` conflicts with an existing argument from the same scope +end + +######################################## +# Error: Conflicting argument and global +function f(x) + global x +end +#--------------------- +LoweringError: +function f(x) + global x +# ╙ ── global variable name `x` conflicts with an existing argument from the same scope +end + +######################################## +# Error: Conflicting destructured argument and global +# TODO: The error could probably be a bit better here +function f((x,)) + global x +end +#--------------------- +LoweringError: +function f((x,)) + global x +# ╙ ── global variable name `x` conflicts with an existing local variable from the same scope +end + +######################################## +# Error: Conflicting static parameter and local +function f(::T) where T + local T +end +#--------------------- +LoweringError: +function f(::T) where T + local T +# ╙ ── local variable name `T` conflicts with an existing static parameter from the same scope +end + +######################################## +# Error: Conflicting static parameter and global +function f(::T) where T + global T +end +#--------------------- +LoweringError: +function f(::T) where T + global T +# ╙ ── global variable name `T` conflicts with an existing static parameter from the same scope +end + +######################################## +# Error: Conflicting static parameter and implicit local +function f(::T) where T + let + T = rhs + end +end +#--------------------- +LoweringError: +function f(::T) where T + let + T = rhs +# ╙ ── cannot overwrite a static parameter + end +end + +######################################## +# Error: Attempt to add methods to a function argument +function f(g) + function g() + end +end +#--------------------- +LoweringError: +function f(g) + function g() +# ╙ ── Cannot add method to a function argument + end +end + +######################################## +# Error: Global method definition inside function scope +function f() + global global_method + function global_method() + end +end +#--------------------- +LoweringError: +function f() + global global_method + function global_method() +# └───────────┘ ── Global method definition needs to be placed at the top level, or use `eval()` + end +end + +######################################## +# @isdefined with local and global variables +let x = 1 + @isdefined(x), @isdefined(y) +end +#--------------------- +1 1 +2 (= slot₁/x %₁) +3 (call core.isdefinedglobal TestMod :y) +4 (call core.tuple true %₃) +5 (return %₄) + +######################################## +# Global function defined inside let (let over lambda) +let x = 1 + global f(y) = x = y + global g() = x +end +#--------------------- +1 1 +2 (= slot₁/x (call core.Box)) +3 (call core.setfield! slot₁/x :contents %₁) +4 (call core.declare_global TestMod :f false) +5 latestworld +6 (call core.define_method TestMod :f) +7 latestworld +8 TestMod.f +9 (call core.TypeEqOf %₈) +10 (call core.svec %₉ core.Any) +11 (call core.svec) +12 SourceLocation::2:11 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 --- code_info + slots: [slot₁/#self#(!read) slot₂/y] + 1 slot₂/y + 2 (captured_local 1) + 3 (call core.setfield! %₂ :contents %₁) + 4 (return %₁) +15 (call core.svec slot₁/x) +16 (call JuliaLowering.replace_captured_locals %₁₄ %₁₅) +17 (call core.define_method TestMod TestMod.f %₁₃ %₁₆) +18 latestworld +19 (call core.declare_global TestMod :g false) +20 latestworld +21 (call core.define_method TestMod :g) +22 latestworld +23 TestMod.g +24 (call core.TypeEqOf %₂₃) +25 (call core.svec %₂₄) +26 (call core.svec) +27 SourceLocation::3:11 +28 (call core.svec %₂₅ %₂₆ %₂₇) +29 --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read,maybe_undef)] + 1 (captured_local 1) + 2 (call core.isdefined %₁ :contents) + 3 (gotoifnot %₂ label₅) + 4 (goto label₇) + 5 (newvar slot₂/x) + 6 slot₂/x + 7 (call core.getfield %₁ :contents) + 8 (return %₇) +30 (call core.svec slot₁/x) +31 (call JuliaLowering.replace_captured_locals %₂₉ %₃₀) +32 (call core.define_method TestMod TestMod.g %₂₈ %₃₁) +33 latestworld +34 TestMod.g +35 (return %₃₄) + +######################################## +# Modify assignment operator on closure variable +let x = 1 + global f() = x += 1 +end +#--------------------- +1 1 +2 (= slot₁/x (call core.Box)) +3 (call core.setfield! slot₁/x :contents %₁) +4 (call core.declare_global TestMod :f false) +5 latestworld +6 (call core.define_method TestMod :f) +7 latestworld +8 TestMod.f +9 (call core.TypeEqOf %₈) +10 (call core.svec %₉) +11 (call core.svec) +12 SourceLocation::2:11 +13 (call core.svec %₁₀ %₁₁ %₁₂) +14 --- code_info + slots: [slot₁/#self#(!read) slot₂/x(!read,maybe_undef)] + 1 TestMod.+ + 2 (captured_local 1) + 3 (call core.isdefined %₂ :contents) + 4 (gotoifnot %₃ label₆) + 5 (goto label₈) + 6 (newvar slot₂/x) + 7 slot₂/x + 8 (call core.getfield %₂ :contents) + 9 (call %₁ %₈ 1) + 10 (captured_local 1) + 11 (call core.setfield! %₁₀ :contents %₉) + 12 (return %₉) +15 (call core.svec slot₁/x) +16 (call JuliaLowering.replace_captured_locals %₁₄ %₁₅) +17 (call core.define_method TestMod TestMod.f %₁₃ %₁₆) +18 latestworld +19 TestMod.f +20 (return %₁₉) diff --git a/JuliaLowering/test/typedefs.jl b/JuliaLowering/test/typedefs.jl new file mode 100644 index 0000000000000..90ee6dfa871bb --- /dev/null +++ b/JuliaLowering/test/typedefs.jl @@ -0,0 +1,1074 @@ +test_mod = Module(:TestMod) + +Base.eval(test_mod, :(struct XX{S,T,U,W} end)) + +@testset "where" begin + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where T<:Number + """) == Vector{T} where T<:Number + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where T>:Int + """) == Vector{T} where T>:Int + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where Int<:T<:Number + """) == Vector{T} where Int<:T<:Number + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where Number>:T>:Int + """) == Vector{T} where Int<:T<:Number + + # with nontrivial type bounds + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where {T<:(U where U<:(V where V<:Number))} + """) == Vector{T} where T<:Number + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where T<:(()->Number)() + """) == Vector{T} where T<:Number + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where (()->Int)()<:T<:(()->Number)() + """) == Vector{T} where Int<:T<:Number + + # multi-layer + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where {A<:Any, B<:A, C>:B, C<:T<:C} + """) == Vector{T} where {A, B<:A, C>:B, C<:T<:C} + @test JuliaLowering.include_string(test_mod, """ + Pair{T, A} where {A<:E<:D, A<:T<:E} where {A<:C<:B, A<:D<:C} where {A, B<:A} + """) == Pair{T, A} where {A, B<:A, A<:C<:B, A<:D<:C, A<:E<:D, A<:T<:E} + @test JuliaLowering.include_string(test_mod, """ + Vector{T} where (()->U)()<:T<:(()->U)() where (()->Int)()<:U<:(()->Number)() + """) == Vector{T} where {Int<:U<:Number, U<:T<:U} + + @testset "implicit whereparams" begin + @test JuliaLowering.include_string(test_mod, """ + Vector{<:Number} + """) == Vector{<:Number} + @test JuliaLowering.include_string(test_mod, """ + Vector{>:Number} + """) == Vector{>:Number} + @test JuliaLowering.include_string(test_mod, """ + Vector{<:(()->Number)()} + """) == Vector{<:Number} + end +end + +@test JuliaLowering.include_string(test_mod, """ +XX{Int, <:Integer, Float64, >:AbstractChar} +""") == (test_mod.XX{Int, T, Float64, S} where {T <: Integer, S >: AbstractChar}) + +@test JuliaLowering.include_string(test_mod, """ +abstract type A end +""") === nothing +@test supertype(test_mod.A) === Any +@test isabstracttype(test_mod.A) + +@test JuliaLowering.include_string(test_mod, """ +abstract type B <: A end +""") === nothing +@test supertype(test_mod.B) === test_mod.A + +@test JuliaLowering.include_string(test_mod, """ +abstract type C{X} end +""") === nothing + +@test JuliaLowering.include_string(test_mod, """ +abstract type D{X<:A} end +""") === nothing +@test test_mod.D{test_mod.B} isa Type +@test_throws Exception test_mod.D{Int} + +@test JuliaLowering.include_string(test_mod, """ +abstract type E <: C{E} end +""") === nothing +@test test_mod.E isa Type + +@test JuliaLowering.include_string(test_mod, """ +primitive type P <: A 16 end +""") === nothing +@test isconcretetype(test_mod.P) +@test supertype(test_mod.P) === test_mod.A +@test reinterpret(test_mod.P, 0x0001) isa test_mod.P +@test reinterpret(UInt16, reinterpret(test_mod.P, 0x1337)) === 0x1337 + +@test JuliaLowering.include_string(test_mod, """ +struct S1{X,Y} <: A + x::X + y::Y + z +end +""") === nothing +@test !isconcretetype(test_mod.S1) +@test fieldnames(test_mod.S1) == (:x, :y, :z) +@test fieldtypes(test_mod.S1) == (Any, Any, Any) +@test isconcretetype(test_mod.S1{Int,String}) +@test fieldtypes(test_mod.S1{Int,String}) == (Int, String, Any) +@test supertype(test_mod.S1) == test_mod.A + +@testset "atomic/const fields" begin + # The parser rejects wrapped const struct fields ("expected assignment after + # const") which probably shouldn't be the parser's job. + fl_eval(test_mod, :(macro var"const"(x); esc(Expr(:const, x)); end)) + + @gensym S + @test jl_eval( + test_mod, + :(begin + mutable struct $S + @atomic x::Bool + @const y::Bool + end + $S(true,false).x + end)) == true + @test fieldnames(getproperty(test_mod, S)) == (:x, :y) + @test_throws "cannot be changed" jl_eval( + test_mod, :($S(true,false).y = true)) + + @gensym S + @test jl_eval( + test_mod, + :(begin + mutable struct $S + begin + begin + @atomic x::Bool + end + end + begin + begin + @const y::Bool + end + end + end + $S(true,false).x + end)) == true + @test fieldnames(getproperty(test_mod, S)) == (:x, :y) + @test_throws "cannot be changed" jl_eval( + test_mod, :($S(true,false).y = true)) + + @gensym S + @test jl_eval( + test_mod, + :(begin + mutable struct $S + @static if true + # Blocks are expected to be splatted into the struct body + begin + @atomic x::Bool + @const y::Bool + end + else + x::Bool + y::Bool + end + end + $S(true,false).x + end)) == true + @test fieldnames(getproperty(test_mod, S)) == (:x, :y) + @test_throws "cannot be changed" jl_eval(test_mod, :($S(true,false).y = true)) +end + +@testset "placeholder fields" begin + @test JuliaLowering.include_string(test_mod, """ + mutable struct PlaceholderFields + _::Int + __ + const ___ + x + end + fieldnames(PlaceholderFields) + """) === (:_, :__, :___, :x) + @test fieldtypes(test_mod.PlaceholderFields) == (Int, Any, Any, Any) + let p = test_mod.PlaceholderFields(1.0, 2, 3, 4) + @test getfield(p, 1) === 1 + @test getfield(p, 2) === 2 + @test getfield(p, 3) === 3 + @test getfield(p, 4) === 4 + @test_throws "const field" setfield!(p, 3, 0) + end + + # placeholder field types still apply in `new` + @test JuliaLowering.include_string(test_mod, """ + struct PlaceholderNew + _::Int + PlaceholderNew(x) = new(x) + end + getfield(PlaceholderNew(3.0), 1) + """) === 3 +end + +# Inner constructors: one field non-Any +@test JuliaLowering.include_string(test_mod, """ +struct S2 + x::Int + y +end +""") === nothing +@test length(methods(test_mod.S2)) == 2 +let s = test_mod.S2(42, "hi") + # exact types + @test s.x === 42 + @test s.y == "hi" +end +let s = test_mod.S2(42.0, "hi") + # converted types + @test s.x === 42 + @test s.y == "hi" +end + +# Constructors: All fields Any +@test JuliaLowering.include_string(test_mod, """ +struct S3 + x + y +end +""") === nothing +@test length(methods(test_mod.S3)) == 1 +let s = test_mod.S3(42, "hi") + @test s.x === 42 + @test s.y == "hi" +end + +# Inner constructors: All fields Any; dynamically tested against whatever +# S4_Field resolves to +@test JuliaLowering.include_string(test_mod, """ +S4_Field = Any # actually Any! + +struct S4 + x::S4_Field + y +end +""") === nothing +@test length(methods(test_mod.S4)) == 1 +let s = test_mod.S4(42, "hi") + @test s.x === 42 + @test s.y == "hi" +end + +# Inner & outer constructors; parameterized types +@test JuliaLowering.include_string(test_mod, """ +struct S5{U} + x::U + y +end +""") === nothing +@test length(methods(test_mod.S5)) == 1 +let s = test_mod.S5(42, "hi") + @test s isa test_mod.S5{Int} + @test s.x === 42 + @test s.y == "hi" +end +@test length(methods(test_mod.S5{Int})) == 1 +let s = test_mod.S5{Int}(42.0, "hi") + @test s isa test_mod.S5{Int} + @test s.x === 42 + @test s.y == "hi" +end +let s = test_mod.S5{Any}(42.0, "hi") + @test s isa test_mod.S5{Any} + @test s.x === 42.0 + @test s.y == "hi" +end +@test JuliaLowering.include_string(test_mod, """ +function S5{Int}(x::Int) + S5(x, x) +end +""") === nothing +let s = test_mod.S5{Int}(1) + @test s.x === 1 + @test s.y === 1 + @test s isa test_mod.S5{Int} +end +@test_throws MethodError test_mod.S5{Int}(1.1) +@test JuliaLowering.include_string(test_mod, """ +function S5{T}(x, y, z) where {T<:AbstractFloat} + S5(x, x) +end +""") === nothing +let s = test_mod.S5{Float64}(Float64(1.1), 0, 0) + @test s.x === 1.1 + @test s.y === 1.1 + @test s isa test_mod.S5{Float64} +end +@test JuliaLowering.include_string(test_mod, """ +S5{<:AbstractFloat}(x) = S5(x, x) +""") === nothing +let s = test_mod.S5{<:AbstractFloat}(Float64(1.1)) + @test s.x === 1.1 + @test s.y === 1.1 + @test s isa test_mod.S5{Float64} +end +@test JuliaLowering.include_string(test_mod, """ +S5{T}(x::T) where {T<:Real} = S5(x, x) +""") === nothing +let s = test_mod.S5{Real}(pi) + @test s.x === pi + @test s.y === pi + @test s isa test_mod.S5{<:Real} +end +outer_mod = Module() +@test JuliaLowering.include_string(test_mod, """ +Base.Vector{T}(x::T) where {S5<:T<:S5} = T[x] +""") === nothing +let v = Base.Vector{test_mod.S5}(test_mod.S5(1,1)) + @test v isa Vector{test_mod.S5} + @test v[1] === test_mod.S5(1,1) +end + +@testset "inner ctor should always include (latestworld)" begin + @test fl_eval( + test_mod, + :(begin + struct fl_DefaultInnerCtorWorld; x::Int; end + fl_DefaultInnerCtorWorld(1) + end)).x == 1 + @test jl_eval( + test_mod, + :(begin + struct jl_DefaultInnerCtorWorld; x::Int; end + jl_DefaultInnerCtorWorld(1) + end)).x == 1 + @test fl_eval( + test_mod, + :(begin + struct fl_InnerCtorWorld; x::Int; fl_InnerCtorWorld() = new(1); end + fl_InnerCtorWorld() + end)).x == 1 + @test jl_eval( + test_mod, + :(begin + struct jl_InnerCtorWorld; x::Int; jl_InnerCtorWorld() = new(1); end + jl_InnerCtorWorld() + end)).x == 1 +end + +# Likely not set in stone (the behaviour is odd here), but test to detect changes +@testset "when default inner ctors are generated" begin + local function test_has_inner_ctor(t) + @test t(0) isa t + @test t(0).field == 0 + @test_throws MethodError t(nothing) + end + + local function test_no_inner_ctor(t) + @test_throws MethodError t(0) + end + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors1 + field::Int + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors1) + @test fieldnames(test_mod.DefaultInnerCtors1) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors2 + "docs" + field::Int + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors2) + @test fieldnames(test_mod.DefaultInnerCtors2) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors3 + "docs with interpolation $(999)" + field::Int + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors3) + @test fieldnames(test_mod.DefaultInnerCtors3) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors4 + "docs with interpolation $(999)" + "docs with interpolation $(999)" + field::Int + "docs with interpolation $(999)" + "docs with interpolation $(999)" + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors4) + @test fieldnames(test_mod.DefaultInnerCtors4) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors5 + field::Int + 0 + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors5) + @test fieldnames(test_mod.DefaultInnerCtors5) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors6 + field::Int + :inert_sym + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors6) + @test fieldnames(test_mod.DefaultInnerCtors6) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct DefaultInnerCtors7 + field::Int + begin; end + begin; begin; end; end + end + """) === nothing + test_has_inner_ctor(test_mod.DefaultInnerCtors7) + @test fieldnames(test_mod.DefaultInnerCtors7) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct NoDefaultInnerCtors1 + field::Int + identity(1) + end + """) === nothing + test_no_inner_ctor(test_mod.NoDefaultInnerCtors1) + @test fieldnames(test_mod.NoDefaultInnerCtors1) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct NoDefaultInnerCtors2 + field::Int + ()->() + end + """) === nothing + test_no_inner_ctor(test_mod.NoDefaultInnerCtors2) + @test fieldnames(test_mod.NoDefaultInnerCtors2) == (:field,) + + @test JL.include_string(test_mod, raw""" + struct NoDefaultInnerCtors3 + field::Int + NoDefaultInnerCtors3(not,default,too,many,args) = new(1) + end + """) === nothing + test_no_inner_ctor(test_mod.NoDefaultInnerCtors3) + @test fieldnames(test_mod.NoDefaultInnerCtors3) == (:field,) + @test length(methods(test_mod.NoDefaultInnerCtors3)) == 1 + + @test JL.include_string(test_mod, raw""" + struct NoDefaultInnerCtors4 + field::Int + :(inert + code) + end + """) === nothing + test_no_inner_ctor(test_mod.NoDefaultInnerCtors4) + @test fieldnames(test_mod.NoDefaultInnerCtors4) == (:field,) +end + +# User defined inner constructors and helper functions for structs without type params +@test JuliaLowering.include_string(test_mod, """ +"struct docs" +struct S6 + x + S6_f() = new(42) + + "some docs" + S6() = S6_f() + S6(x) = new(x) +end +"""; expr_compat_mode=true) === nothing +let s = test_mod.S6() + @test s isa test_mod.S6 + @test s.x === 42 +end +let s = test_mod.S6(2) + @test s isa test_mod.S6 + @test s.x === 2 +end +@test docstrings_equal(@doc(test_mod.S6), Markdown.doc"struct docs") + +# User defined inner constructors and helper functions for structs with type params +@test JuliaLowering.include_string(test_mod, """ +struct S7{S,T} + x::S + y + + # Cases where full struct type may be deduced and used in body + S7{Int,String}() = new(10.0, "y1") + S7{S,T}() where {S,T} = new(10.0, "y2") + S7{Int,T}() where {T} = new(10.0, "y3") + (::Type{S7{Int,UInt8}})() = new{Int,UInt8}(10.0, "y4") + + # Cases where new{...} is called + S7() = new{Int,Int}(10.0, "y5") + S7{UInt8}() = S7_f() + S7_f() = new{UInt8,UInt8}(10.0, "y6") +end +""") === nothing +let s = test_mod.S7{Int,String}() + @test s isa test_mod.S7{Int,String} + @test s.x === 10 + @test s.y === "y1" +end +let s = test_mod.S7{UInt16,UInt16}() + @test s isa test_mod.S7{UInt16,UInt16} + @test s.x === UInt16(10) + @test s.y === "y2" +end +let s = test_mod.S7{Int,UInt16}() + @test s isa test_mod.S7{Int,UInt16} + @test s.x === 10 + @test s.y === "y3" +end +let s = test_mod.S7{Int,UInt8}() + @test s isa test_mod.S7{Int,UInt8} + @test s.x === 10 + @test s.y === "y4" +end +let s = test_mod.S7() + @test s isa test_mod.S7{Int,Int} + @test s.x === 10 + @test s.y === "y5" +end +let s = test_mod.S7{UInt8}() + @test s isa test_mod.S7{UInt8,UInt8} + @test s.x === UInt8(10) + @test s.y === "y6" +end + +# new() with splats and typed fields +@test JuliaLowering.include_string(test_mod, """ +struct S8 + x::Int + y::Float64 + + S8(xs, ys) = new(xs..., ys...) +end +""") === nothing +let s = test_mod.S8((10.0,), (20,)) + @test s isa test_mod.S8 + @test s.x === 10 + @test s.y === 20.0 +end +# Wrong number of args checked by lowering +@test_throws ArgumentError test_mod.S8((1,), ()) +@test_throws ArgumentError test_mod.S8((1,2,3), ()) + +# empty-curly `new{}()` +@test JuliaLowering.include_string(test_mod, """ +struct S_empty_new + x::Int + S_empty_new() = new{}(5) +end +""") === nothing +let s = test_mod.S_empty_new() + @test s isa test_mod.S_empty_new + @test s.x === 5 +end + +# flisp doesn't error with kwargs after `;` in `new` +@test JuliaLowering.include_string(test_mod, """ +struct S_new_kwargs1 + x + y + S_new_kwargs1(args...; kwargs...) = new(args...; kwargs...) +end +S_new_kwargs1(1,2).x +"""; expr_compat_mode=true) == 1 + +# new() with splats and untyped fields +@test JuliaLowering.include_string(test_mod, """ +struct S9 + x + y + + S9(xs) = new(xs...) +end +""") === nothing +let s = test_mod.S9((10.0,20)) + @test s isa test_mod.S9 + @test s.x === 10.0 + @test s.y === 20 +end +# Wrong number of args checked by the runtime +@test_throws ArgumentError test_mod.S9((1,)) +@test_throws ArgumentError test_mod.S9((1,2,3)) + +# Test cases from +# https://github.com/JuliaLang/julia/issues/36104 +# https://github.com/JuliaLang/julia/pull/36121 +JuliaLowering.include_string(test_mod, """ +# issue #36104 +module M36104 +struct T36104 + v::Vector{M36104.T36104} +end +struct T36104 # check that redefining it works, issue #21816 + v::Vector{T36104} +end +end +""") +@test fieldtypes(test_mod.M36104.T36104) == (Vector{test_mod.M36104.T36104},) +@test_throws ErrorException("expected") JuliaLowering.include_string(test_mod, """struct X36104; x::error("expected"); end""") +@test !isdefined(test_mod, :X36104) +JuliaLowering.include_string(test_mod, "struct X36104; x::Int; end") +@test fieldtypes(test_mod.X36104) == (Int,) +JuliaLowering.include_string(test_mod, "primitive type P36104 8 end") +JuliaLowering.include_string(test_mod, "const orig_P36104 = P36104") +JuliaLowering.include_string(test_mod, "primitive type P36104 16 end") +@test test_mod.P36104 !== test_mod.orig_P36104 + +# Duplicate field names should be rejected +@test_throws LoweringError JuliaLowering.include_string(test_mod, "struct DupField; x; x; end") +@test_throws LoweringError JuliaLowering.include_string(test_mod, "struct DupField2; x::Int; x::String; end") +@test_throws LoweringError JuliaLowering.include_string(test_mod, "mutable struct DupField3; x; y; x; end") + +# Struct with outer constructor where one typevar is constrained by the other +# See https://github.com/JuliaLang/julia/issues/27269) +@test JuliaLowering.include_string(test_mod, """ +struct X27269{T, S <: Vector{T}} + v::Vector{S} +end +""") === nothing +@test test_mod.X27269([[1,2]]) isa test_mod.X27269{Int, Vector{Int}} + +# Definition of type-alias with 2+ type-variables +# See https://github.com/JuliaLang/JuliaLowering.jl/issues/123 +JuliaLowering.include_string(test_mod, """ +struct Foo{T,V,N}; end +const Bar{T,V} = Foo{T,V,1} +""") +@test test_mod.Bar == (test_mod.Foo{T,V,1} where {T,V}) + +# Global function with new() inside struct +# See https://github.com/JuliaLang/JuliaLowering.jl/issues/131 +JuliaLowering.include_string(test_mod, """ +struct S131 + x + global function make_s131() + new(42) + end +end +""") +@test test_mod.make_s131() isa test_mod.S131 +@test test_mod.make_s131().x == 42 + +JuliaLowering.include_string(test_mod, """ +struct S131b{T} + x::T + "documented global function" + global function make_s131b() + new{Int}(100) + end +end +""") +@test test_mod.make_s131b() isa test_mod.S131b{Int} +@test test_mod.make_s131b().x == 100 + +# Inner constructor with local variable shadowing type parameter +# See https://github.com/aviatesk/JETLS.jl/issues/508 +@test JuliaLowering.include_string(test_mod, """ +struct ShadowTypeParam{T} + x::T + function ShadowTypeParam(x) + T = typeof(x) # This should be a new local variable, not capture the type param + return new{T}(x) + end +end +""") === nothing +let s = test_mod.ShadowTypeParam(42) + @test s isa test_mod.ShadowTypeParam{Int} + @test s.x === 42 +end +let s = test_mod.ShadowTypeParam("hello") + @test s isa test_mod.ShadowTypeParam{String} + @test s.x === "hello" +end + +# Inner kwarg constructor +@test JuliaLowering.include_string(test_mod, """ +struct CheckConfig + field::Int + function CheckConfig(parg::String; kw::Int = 4) + new(length(parg) + kw) + end + function CheckConfig(optarg1=1, optarg2=10; kw1, kw2 = 1000) + new(optarg1+optarg2+kw1+kw2) + end + function CheckConfig(optarg1=1, optarg2=10; kw1, kw2 = 1000) + new(optarg1+optarg2+kw1+kw2) + end +end +""") === nothing + +@test isdefined(test_mod, :CheckConfig) + +let s = test_mod.CheckConfig("") + @test s isa test_mod.CheckConfig + @test s.field == 4 +end +let s = test_mod.CheckConfig("";kw=5) + @test s isa test_mod.CheckConfig + @test s.field == 5 +end +let s = test_mod.CheckConfig(;kw1=100) + @test s isa test_mod.CheckConfig + @test s.field == 1111 +end +let s = test_mod.CheckConfig(;kw1=100, kw2=0) + @test s isa test_mod.CheckConfig + @test s.field == 111 +end +let s = test_mod.CheckConfig(0;kw1=100, kw2=0) + @test s isa test_mod.CheckConfig + @test s.field == 110 +end +let s = test_mod.CheckConfig(0,0;kw1=100) + @test s isa test_mod.CheckConfig + @test s.field == 1100 +end + +let s = test_mod.CheckConfig(;kw1=100) + @test s isa test_mod.CheckConfig + @test s.field == 1111 +end +let s = test_mod.CheckConfig(;kw1=100, kw2=0) + @test s isa test_mod.CheckConfig + @test s.field == 111 +end +let s = test_mod.CheckConfig(0;kw1=100, kw2=0) + @test s isa test_mod.CheckConfig + @test s.field == 110 +end +let s = test_mod.CheckConfig(0,0;kw1=100) + @test s isa test_mod.CheckConfig + @test s.field == 1100 +end + +# typegroup: basic mutual recursion +@test JuliaLowering.include_string(test_mod, """ +typegroup + struct TG_Node + edges::Vector{TG_Edge} + end + struct TG_Edge + from::TG_Node + to::TG_Node + end +end +"""; version=v"1.14") === nothing +@test fieldtype(test_mod.TG_Node, :edges) == Vector{test_mod.TG_Edge} +@test fieldtype(test_mod.TG_Edge, :from) == test_mod.TG_Node +@test fieldtype(test_mod.TG_Edge, :to) == test_mod.TG_Node +let n1 = test_mod.TG_Node(test_mod.TG_Edge[]), + n2 = test_mod.TG_Node(test_mod.TG_Edge[]), + e = test_mod.TG_Edge(n1, n2) + push!(n1.edges, e) + @test n1.edges[1].to === n2 +end + +# typegroup: parametric mutual recursion +@test JuliaLowering.include_string(test_mod, """ +typegroup + struct TG_PNode{T} + data::T + edges::Vector{TG_PEdge{T}} + end + struct TG_PEdge{T} + from::TG_PNode{T} + to::TG_PNode{T} + end +end +"""; version=v"1.14") === nothing +@test fieldtype(test_mod.TG_PNode{Int}, :edges) == Vector{test_mod.TG_PEdge{Int}} +@test fieldtype(test_mod.TG_PEdge{String}, :from) == test_mod.TG_PNode{String} + +# typegroup: mutable structs +@test JuliaLowering.include_string(test_mod, """ +typegroup + mutable struct TG_MNode + edges::Vector{TG_MEdge} + end + mutable struct TG_MEdge + from::TG_MNode + to::TG_MNode + end +end +"""; version=v"1.14") === nothing +@test ismutabletype(test_mod.TG_MNode) +@test ismutabletype(test_mod.TG_MEdge) + +# typegroup: supertype referencing incomplete type +@test JuliaLowering.include_string(test_mod, """ +typegroup + struct TG_SuperA <: AbstractVector{TG_SuperB} + data::Vector{TG_SuperB} + end + struct TG_SuperB + a::TG_SuperA + end +end +"""; version=v"1.14") === nothing +@test test_mod.TG_SuperA <: AbstractVector{test_mod.TG_SuperB} + +# bad test: flisp will scan the struct for assigned fields to throw errors, but +# runs a `flatten-blocks` pass before collecting the real fields, so +# assignment-like fields/ctors can sneak through. As of #59882, the @doc form +# emits an assignment inside of a block that sneaks through this way. JL found +# assignments in the actual field list, so `@doc` hit that case. +@test jl_eval(test_mod, Expr(:block, + Expr(:struct, true, :struct_doc_test, Expr(:block, + Expr(:(::), :status, :Int), + Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(0), "doc", + Expr(:function, + Expr(:call, :struct_doc_test, Expr(:(::), :status, :Integer), + Expr(:kw, :headers, Expr(:vect))), + Expr(:block, Expr(:call, :new, :status)))))), + Expr(:., Expr(:call, :struct_doc_test, 5), QuoteNode(:status)))) == 5 + +# bad test: flisp does not continue scanning past anything it doesn't recognize +# as a field +@test jl_eval( + test_mod, + :(begin + struct struct_eq_assigns_test + x::Int + struct_eq_assigns_test(x) = new(x) + default = 1 + struct_eq_assigns_test() = new(default) + end + (struct_eq_assigns_test(5).x, struct_eq_assigns_test().x) + end)) == (5, 1) + +@testset "(AI) inner ctor with return-type annotation and `where`" begin + # Regression: an inner constructor with BOTH an explicit return-type + # annotation and a `where` clause used to fail lowering ("No match found"), + # because `rewrite_ctor_sig` re-wrapped the `where` a second time — nesting + # it inside the `::` — when it recursed through the return-type branch. + m = Module() + JuliaLowering.include_string(m, """ + struct Wrap{T} + x::T + function Wrap{T}(x)::Wrap{T} where {T} + return new{T}(x) + end + end + """) + w = JuliaLowering.include_string(m, "Wrap{Int}(5)") + @test w isa m.Wrap{Int} + @test w.x === 5 + + # The return-type annotation performs `convert` (matching flisp): a + # mismatched annotation only fails at construction time, as a MethodError + # from the missing `convert`, not as a lowering/type-assert error. + JuliaLowering.include_string(m, """ + struct W2{T} + x::T + function W2{T}(x)::W2{Int} where {T} + return new{T}(x) + end + end + """) + @test JuliaLowering.include_string(m, "W2{Int}(3)") isa m.W2{Int} + @test_throws MethodError JuliaLowering.include_string(m, "W2{String}(\"hi\")") + + # The `where`-bound typevar scopes over the return-type annotation (`::T`). + JuliaLowering.include_string(m, """ + struct W4{T} + x::T + function W4{T}(x)::T where {T} + return new{T}(x).x + end + end + """) + @test JuliaLowering.include_string(m, "W4{Int}(9)") === 9 + + # Constructor name without curlies, plus return annotation and `where`. + JuliaLowering.include_string(m, """ + struct B2{T} + x::T + function B2(x::T)::B2{T} where {T} + return new{T}(x) + end + end + """) + @test JuliaLowering.include_string(m, "B2(7)") isa m.B2{Int} + + # Multiple `where`-bound typevars with subtype bounds (the shape found in + # FlexiChains that first surfaced this bug). + JuliaLowering.include_string(m, """ + struct FS{TKey, TIIdx<:Union{Int,Nothing}, TCIdx<:Union{Int,Nothing}, TSIdx<:Real} + _iter::TIIdx + _chain::TCIdx + _stat::TSIdx + function FS{TKey}(iter::TIIdx, chain::TCIdx, stat::TSIdx)::FS{TKey,TIIdx,TCIdx,TSIdx} where { + TKey, TIIdx<:Union{Int,Nothing}, TCIdx<:Union{Int,Nothing}, TSIdx<:Real} + return new{TKey,TIIdx,TCIdx,TSIdx}(iter, chain, stat) + end + end + """) + @test JuliaLowering.include_string(m, "FS{Symbol}(1, nothing, 2.0)") isa + m.FS{Symbol, Int, Nothing, Float64} + + # Success path of the return-type `convert` (complements the W2 failure case + # above): an `::Any` annotation converts by identity, so the constructor + # still returns the freshly-built struct. Combined with `where`. + JuliaLowering.include_string(m, """ + struct WA{T} + x::T + function WA{T}(x)::Any where {T} + return new{T}(x) + end + end + """) + @test JuliaLowering.include_string(m, "WA{Int}(5)") isa m.WA{Int} +end + +@testset "(AI) struct fields named with underscores" begin + # Bug in _defaultctors giving lowering a K"lambda" + m = Module() + JuliaLowering.include_string(m, """ + struct TextItem + _::String + end + struct TwoScores + _::Int + __::Int + end + """) + @test JuliaLowering.include_string(m, """TextItem("hi")._""") == "hi" + t = JuliaLowering.include_string(m, "TwoScores(1, 2)") + @test (t._, t.__) == (1, 2) +end + +# A global should be declared in scope. the local name should not be +# user-visible (the user may use it in a function arg type) +@testset "struct def in local scope" for expr_compat_mode in (true, false) + @test JuliaLowering.include_string(@newmod(), """ + let + struct S; x::Int; end + Core.isdefinedglobal(@__MODULE__, :S) + end + """; expr_compat_mode) == true + + @test JuliaLowering.include_string(@newmod(), """ + let + struct S; x::Int; end + S(y::UInt) = S(-1) + S(UInt(1)).x + end + """; expr_compat_mode) == -1 + + @test JuliaLowering.include_string(@newmod(), """ + let + struct S; x::Int; end + S(y::UInt) = S(-1) + f(arg::S) = arg.x # S should be the global + f(S(UInt(1))) + end + """; expr_compat_mode) == -1 + + @test JuliaLowering.include_string(@newmod(), """ + let + struct S{T}; x::T; end + S(;y=0) = S{typeof(y)}(y) + f(::Type{<:S}) = 1 + (f(S{Int}), fieldcount(S{Int}), nameof(typeof(S(;y=2)))) + end + """; expr_compat_mode) == (1, 1, :S) + + # local captures + @test JuliaLowering.include_string(@newmod(), """ + let loc = 1 + struct S; x::Int; end + S(y::Float64) = S(loc += 1) + S(1).x, S(1.0).x, S(2).x, S(2.0).x, S(3).x, S(3.0).x + end + """; expr_compat_mode) == (1, 2, 2, 3, 3, 4) + @test JuliaLowering.include_string(@newmod(), """ + let loc = 1 + struct S + x::Int + S(y::Float64) = new(loc += 1) + end + S(1.0).x, S(2.0).x, S(3.0).x + end + """; expr_compat_mode) == (2, 3, 4) +end +@testset "nonstruct type def in local scope" for expr_compat_mode in (true, false) + @test JuliaLowering.include_string(test_mod, """ + let + abstract type AbstractTypeInLocalScope end + Core.isdefinedglobal(@__MODULE__, :AbstractTypeInLocalScope) + end + """; expr_compat_mode) == true + + @test JuliaLowering.include_string(test_mod, """ + let + primitive type PrimitiveTypeInLocalScope 8 end + Core.isdefinedglobal(@__MODULE__, :PrimitiveTypeInLocalScope) + end + """; expr_compat_mode) == true +end + +@testset "(AI) typegroup def in local scope" for expr_compat_mode in (true, false) + @test JuliaLowering.include_string(@newmod(), """ + let + typegroup + struct N; x::Int; end + struct E; f::N; end + end + Core.isdefinedglobal(@__MODULE__, :N) + end + """; expr_compat_mode, version=v"1.14") == true + + # A later method whose body refers to the type name. + @test JuliaLowering.include_string(@newmod(), """ + let + typegroup + struct N; x::Int; end + struct E; f::N; end + end + N(y::UInt) = N(-1) + N(UInt(1)).x + end + """; expr_compat_mode, version=v"1.14") == -1 + + # A later method whose signature refers to the type name. + @test JuliaLowering.include_string(@newmod(), """ + let + typegroup + struct N; x::Int; end + struct E; f::N; end + end + N(y::UInt) = N(-1) + f(arg::N) = arg.x + f(N(UInt(1))) + end + """; expr_compat_mode, version=v"1.14") == -1 +end + +@testset "(AI) type def in local scope: name conflicts" for expr_compat_mode in (true, false) + for src in ("""let S = 1 + struct S; x::Int; end + end + """, + """let A = 1 + abstract type A end + end + """, + """let P = 1 + primitive type P 8 end + end + """, + """let N = 1 + typegroup + struct N; x::Int; end + struct E; f::N; end + end + end""") + @test_throws JuliaLowering.LoweringError JuliaLowering.include_string( + test_mod, src; expr_compat_mode, version=v"1.14") + end + + # shadowing is OK + @test JuliaLowering.include_string(@newmod(), """ + let S = 1 + let + struct S; x::Int; end + end + S + end + """; expr_compat_mode) == 1 +end diff --git a/JuliaLowering/test/typedefs_ir.jl b/JuliaLowering/test/typedefs_ir.jl new file mode 100644 index 0000000000000..a31864ec8f62e --- /dev/null +++ b/JuliaLowering/test/typedefs_ir.jl @@ -0,0 +1,1366 @@ +######################################## +# where expression without type bounds +A where X +#--------------------- +1 (call core.TypeVar :X) +2 (= slot₁/X %₁) +3 TestMod.A +4 (call core.UnionAll slot₁/X %₃) +5 (return %₄) + +######################################## +# where expression with upper bound +A where X <: UB +#--------------------- +1 TestMod.UB +2 (call core.TypeVar :X %₁) +3 (= slot₁/X %₂) +4 TestMod.A +5 (call core.UnionAll slot₁/X %₄) +6 (return %₅) + +######################################## +# where expression with lower bound +A where X >: LB +#--------------------- +1 TestMod.LB +2 (call core.TypeVar :X %₁ core.Any) +3 (= slot₁/X %₂) +4 TestMod.A +5 (call core.UnionAll slot₁/X %₄) +6 (return %₅) + +######################################## +# where expression with both bounds +A where LB <: X <: UB +#--------------------- +1 TestMod.LB +2 TestMod.UB +3 (call core.TypeVar :X %₁ %₂) +4 (= slot₁/X %₃) +5 TestMod.A +6 (call core.UnionAll slot₁/X %₅) +7 (return %₆) + +######################################## +# where expression with braces +A where {X, Y<:X} +#--------------------- +1 (call core.TypeVar :X) +2 (= slot₁/X %₁) +3 (call core.TypeVar :Y slot₁/X) +4 (= slot₂/Y %₃) +5 TestMod.A +6 (call core.UnionAll slot₂/Y %₅) +7 (call core.UnionAll slot₁/X %₆) +8 (return %₇) + +######################################## +# Equivalent nested where expression without braces +A where Y<:X where X +#--------------------- +1 (call core.TypeVar :X) +2 (= slot₁/X %₁) +3 (call core.TypeVar :Y slot₁/X) +4 (= slot₂/Y %₃) +5 TestMod.A +6 (call core.UnionAll slot₂/Y %₅) +7 (call core.UnionAll slot₁/X %₆) +8 (return %₇) + +######################################## +# Error: bad type bounds +A where f() +#--------------------- +LoweringError: +A where f() +# └─┘ ── expected type name or type bounds + +######################################## +# Error: bad type bounds +A where X < Y < Z +#--------------------- +LoweringError: +A where X < Y < Z +# └───────┘ ── expected `lb <: type_name <: ub` or `ub >: type_name >: lb` + +######################################## +# Error: bad type bounds +A where X <: f() <: Z +#--------------------- +LoweringError: +A where X <: f() <: Z +# └─┘ ── expected identifier + +######################################## +# Error: bad type bounds +A where f() <: Y +#--------------------- +LoweringError: +A where f() <: Y +# └─┘ ── expected identifier + +######################################## +# Simple type application +X{A,B,C} +#--------------------- +1 TestMod.X +2 TestMod.A +3 TestMod.B +4 TestMod.C +5 (call core.apply_type %₁ %₂ %₃ %₄) +6 (return %₅) + +######################################## +# Type with implicit where param upper bound +X{<:A} +#--------------------- +1 TestMod.A +2 (call core.TypeVar :#T1 %₁) +3 TestMod.X +4 (call core.apply_type %₃ %₂) +5 (call core.UnionAll %₂ %₄) +6 (return %₅) + +######################################## +# Type with implicit where param lower bound +X{>:A} +#--------------------- +1 TestMod.A +2 (call core.TypeVar :#T1 %₁ core.Any) +3 TestMod.X +4 (call core.apply_type %₃ %₂) +5 (call core.UnionAll %₂ %₄) +6 (return %₅) + +######################################## +# Type with several implicit where params +X{S, <:A, T, >:B} +#--------------------- +1 TestMod.A +2 (call core.TypeVar :#T1 %₁) +3 TestMod.B +4 (call core.TypeVar :#T2 %₃ core.Any) +5 TestMod.X +6 TestMod.S +7 TestMod.T +8 (call core.apply_type %₅ %₆ %₂ %₇ %₄) +9 (call core.UnionAll %₄ %₈) +10 (call core.UnionAll %₂ %₉) +11 (return %₁₀) + +######################################## +# Error: parameters in type application +X{S, T; W} +#--------------------- +LoweringError: +X{S, T; W} +# └─┘ ── unexpected semicolon + +######################################## +# Error: assignment in type application +X{S, T=w} +#--------------------- +LoweringError: +X{S, T=w} +# └──┘ ── assignment is not allowed in type parameter list + +######################################## +# Simple abstract type definition +abstract type A end +#--------------------- +1 (call core.svec) +2 (call core._abstracttype TestMod :A %₁) +3 (= slot₁/A %₂) +4 (call core._setsuper! %₂ core.Any) +5 (call core._typebody! slot₁/A) +6 (call core.declare_global TestMod :A false) +7 latestworld +8 (call core.isdefinedglobal TestMod :A false) +9 (gotoifnot %₈ label₁₄) +10 TestMod.A +11 (call core._equiv_typedef %₁₀ %₂) +12 (gotoifnot %₁₁ label₁₄) +13 (goto label₁₆) +14 (call core.declare_const TestMod :A %₂) +15 latestworld +16 (return core.nothing) + +######################################## +# Abstract type definition with supertype +abstract type A <: B end +#--------------------- +1 (call core.svec) +2 (call core._abstracttype TestMod :A %₁) +3 (= slot₁/A %₂) +4 TestMod.B +5 (call core._setsuper! %₂ %₄) +6 (call core._typebody! slot₁/A) +7 (call core.declare_global TestMod :A false) +8 latestworld +9 (call core.isdefinedglobal TestMod :A false) +10 (gotoifnot %₉ label₁₅) +11 TestMod.A +12 (call core._equiv_typedef %₁₁ %₂) +13 (gotoifnot %₁₂ label₁₅) +14 (goto label₁₇) +15 (call core.declare_const TestMod :A %₂) +16 latestworld +17 (return core.nothing) + +######################################## +# Abstract type definition with multiple typevars +abstract type A{X, Y <: X} end +#--------------------- +1 (= slot₂/X (call core.TypeVar :X)) +2 slot₂/X +3 (= slot₃/Y (call core.TypeVar :Y %₂)) +4 slot₂/X +5 slot₃/Y +6 (call core.svec %₄ %₅) +7 (call core._abstracttype TestMod :A %₆) +8 (= slot₁/A %₇) +9 (call core._setsuper! %₇ core.Any) +10 (call core._typebody! slot₁/A) +11 (call core.declare_global TestMod :A false) +12 latestworld +13 (call core.isdefinedglobal TestMod :A false) +14 (gotoifnot %₁₃ label₁₉) +15 TestMod.A +16 (call core._equiv_typedef %₁₅ %₇) +17 (gotoifnot %₁₆ label₁₉) +18 (goto label₂₁) +19 (call core.declare_const TestMod :A %₇) +20 latestworld +21 (return core.nothing) + +######################################## +# Error: Abstract type definition with bad signature +abstract type A() end +#--------------------- +LoweringError: +abstract type A() end +# └─┘ ── invalid type signature + +######################################## +# Error: Abstract type definition with bad signature +abstract type A(){T} end +#--------------------- +LoweringError: +abstract type A(){T} end +# └─┘ ── expected identifier + +######################################## +# Error: Abstract type definition with bad signature +abstract type A() <: B end +#--------------------- +LoweringError: +abstract type A() <: B end +# └─┘ ── expected identifier + +######################################## +# Error: Abstract type definition in function scope +function f() + abstract type A end +end +#--------------------- +LoweringError: +function f() + abstract type A end +# └─────────────────┘ ── this syntax is only allowed at top level +end + +######################################## +# Simple primitive type definition +primitive type P 8 end +#--------------------- +1 (call core.svec) +2 (call core._primitivetype TestMod :P %₁ 8) +3 (= slot₁/P %₂) +4 (call core._setsuper! %₂ core.Any) +5 (call core._typebody! slot₁/P) +6 (call core.declare_global TestMod :P false) +7 latestworld +8 (call core.isdefinedglobal TestMod :P false) +9 (gotoifnot %₈ label₁₄) +10 TestMod.P +11 (call core._equiv_typedef %₁₀ %₂) +12 (gotoifnot %₁₁ label₁₄) +13 (goto label₁₆) +14 (call core.declare_const TestMod :P %₂) +15 latestworld +16 (return core.nothing) + +######################################## +# Complex primitive type definition +primitive type P{X,Y} <: Z 32 end +#--------------------- +1 (= slot₂/X (call core.TypeVar :X)) +2 (= slot₃/Y (call core.TypeVar :Y)) +3 slot₂/X +4 slot₃/Y +5 (call core.svec %₃ %₄) +6 (call core._primitivetype TestMod :P %₅ 32) +7 (= slot₁/P %₆) +8 TestMod.Z +9 (call core._setsuper! %₆ %₈) +10 (call core._typebody! slot₁/P) +11 (call core.declare_global TestMod :P false) +12 latestworld +13 (call core.isdefinedglobal TestMod :P false) +14 (gotoifnot %₁₃ label₁₉) +15 TestMod.P +16 (call core._equiv_typedef %₁₅ %₆) +17 (gotoifnot %₁₆ label₁₉) +18 (goto label₂₁) +19 (call core.declare_const TestMod :P %₆) +20 latestworld +21 (return core.nothing) + +######################################## +# Primitive type definition with computed size (should this be allowed??) +primitive type P P_nbits() end +#--------------------- +1 (call core.svec) +2 TestMod.P_nbits +3 (call %₂) +4 (call core._primitivetype TestMod :P %₁ %₃) +5 (= slot₁/P %₄) +6 (call core._setsuper! %₄ core.Any) +7 (call core._typebody! slot₁/P) +8 (call core.declare_global TestMod :P false) +9 latestworld +10 (call core.isdefinedglobal TestMod :P false) +11 (gotoifnot %₁₀ label₁₆) +12 TestMod.P +13 (call core._equiv_typedef %₁₂ %₄) +14 (gotoifnot %₁₃ label₁₆) +15 (goto label₁₈) +16 (call core.declare_const TestMod :P %₄) +17 latestworld +18 (return core.nothing) + +######################################## +# Empty struct +struct X +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (call core.svec) +5 (call core.svec) +6 (call core.svec) +7 (call core.svec) +8 (call core.svec %₄ %₅ %₆ false 0 core.Any %₇) +9 (call core.isdefinedglobal TestMod :X false) +10 (gotoifnot %₉ label₁₄) +11 TestMod.X +12 (= slot₂/if_val %₁₁) +13 (goto label₁₅) +14 (= slot₂/if_val core.nothing) +15 slot₂/if_val +16 slot₁/X +17 (call core.svec %₁₆) +18 (call core.svec %₈) +19 (call core.svec %₁₅) +20 (call core.resolve_typegroup TestMod %₁₇ %₁₈ %₁₉) +21 (= slot₁/X (call core.getfield %₂₀ 1)) +22 slot₁/X +23 (call core.declare_const TestMod :X %₂₂) +24 latestworld +25 TestMod.X +26 SourceLocation::1:1 +27 (call top._defaultctors %₂₅ %₂₆) +28 latestworld +29 (return core.nothing) + +######################################## +# Empty struct with empty ctor +struct X + X() = new() +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (call core.svec) +5 (call core.svec) +6 (call core.svec) +7 (call core.svec) +8 (call core.svec %₄ %₅ %₆ false 0 core.Any %₇) +9 (call core.isdefinedglobal TestMod :X false) +10 (gotoifnot %₉ label₁₄) +11 TestMod.X +12 (= slot₂/if_val %₁₁) +13 (goto label₁₅) +14 (= slot₂/if_val core.nothing) +15 slot₂/if_val +16 slot₁/X +17 (call core.svec %₁₆) +18 (call core.svec %₈) +19 (call core.svec %₁₅) +20 (call core.resolve_typegroup TestMod %₁₇ %₁₈ %₁₉) +21 (= slot₁/X (call core.getfield %₂₀ 1)) +22 slot₁/X +23 (call core.declare_const TestMod :X %₂₂) +24 latestworld +25 TestMod.X +26 (call core.apply_type core.Type %₂₅) +27 (call core.svec %₂₆) +28 (call core.svec) +29 SourceLocation::2:5 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod core.nothing %₃₀ + --- code_info + slots: [slot₁/#ctor-self#] + 1 slot₁/#ctor-self# + 2 (new %₁) + 3 (return %₂) +32 latestworld +33 (return core.nothing) + +######################################## +# Basic struct +struct X + a + b::T + c +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (call core.svec) +5 (call core.svec :a :b :c) +6 (call core.svec) +7 TestMod.T +8 (call core.svec core.Any %₇ core.Any) +9 (call core.svec %₄ %₅ %₆ false 3 core.Any %₈) +10 (call core.isdefinedglobal TestMod :X false) +11 (gotoifnot %₁₀ label₁₅) +12 TestMod.X +13 (= slot₂/if_val %₁₂) +14 (goto label₁₆) +15 (= slot₂/if_val core.nothing) +16 slot₂/if_val +17 slot₁/X +18 (call core.svec %₁₇) +19 (call core.svec %₉) +20 (call core.svec %₁₆) +21 (call core.resolve_typegroup TestMod %₁₈ %₁₉ %₂₀) +22 (= slot₁/X (call core.getfield %₂₁ 1)) +23 slot₁/X +24 (call core.declare_const TestMod :X %₂₃) +25 latestworld +26 TestMod.X +27 SourceLocation::1:1 +28 (call top._defaultctors %₂₆ %₂₇) +29 latestworld +30 (return core.nothing) + +######################################## +# Struct with supertype and type params +struct X{U, S <: V <: T} <: Z +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (= slot₂/U (call core.TypeVar :U)) +5 TestMod.S +6 TestMod.T +7 (= slot₃/V (call core.TypeVar :V %₅ %₆)) +8 slot₂/U +9 slot₃/V +10 (call core.svec %₈ %₉) +11 (call core.svec) +12 (call core.svec) +13 TestMod.Z +14 (call core.svec) +15 (call core.svec %₁₀ %₁₁ %₁₂ false 0 %₁₃ %₁₄) +16 (call core.isdefinedglobal TestMod :X false) +17 (gotoifnot %₁₆ label₂₁) +18 TestMod.X +19 (= slot₄/if_val %₁₈) +20 (goto label₂₂) +21 (= slot₄/if_val core.nothing) +22 slot₄/if_val +23 slot₁/X +24 (call core.svec %₂₃) +25 (call core.svec %₁₅) +26 (call core.svec %₂₂) +27 (call core.resolve_typegroup TestMod %₂₄ %₂₅ %₂₆) +28 (= slot₁/X (call core.getfield %₂₇ 1)) +29 slot₁/X +30 (call core.declare_const TestMod :X %₂₉) +31 latestworld +32 TestMod.X +33 SourceLocation::1:1 +34 (call top._defaultctors %₃₂ %₃₃) +35 latestworld +36 (return core.nothing) + +######################################## +# Struct with const and atomic fields +struct X + const a + @atomic b + const @atomic c +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (call core.svec) +5 (call core.svec :a :b :c) +6 (call core.svec 1 :const 2 :atomic 3 :atomic 3 :const) +7 (call core.svec core.Any core.Any core.Any) +8 (call core.svec %₄ %₅ %₆ false 3 core.Any %₇) +9 (call core.isdefinedglobal TestMod :X false) +10 (gotoifnot %₉ label₁₄) +11 TestMod.X +12 (= slot₂/if_val %₁₁) +13 (goto label₁₅) +14 (= slot₂/if_val core.nothing) +15 slot₂/if_val +16 slot₁/X +17 (call core.svec %₁₆) +18 (call core.svec %₈) +19 (call core.svec %₁₅) +20 (call core.resolve_typegroup TestMod %₁₇ %₁₈ %₁₉) +21 (= slot₁/X (call core.getfield %₂₀ 1)) +22 slot₁/X +23 (call core.declare_const TestMod :X %₂₂) +24 latestworld +25 TestMod.X +26 SourceLocation::1:1 +27 (call top._defaultctors %₂₅ %₂₆) +28 latestworld +29 (return core.nothing) + +######################################## +# Documented struct +""" +X docs +""" +struct X + "field a docs" + a + "field b docs" + b +end +#--------------------- +1 (newvar slot₁/val) +2 (gotoifnot true label₃₂) +3 (call core.declare_global TestMod :X false) +4 latestworld +5 (= slot₂/X (call core.TypeVar :X)) +6 (call core.svec) +7 (call core.svec :a :b) +8 (call core.svec) +9 (call core.svec core.Any core.Any) +10 (call core.svec %₆ %₇ %₈ false 2 core.Any %₉) +11 (call core.isdefinedglobal TestMod :X false) +12 (gotoifnot %₁₁ label₁₆) +13 TestMod.X +14 (= slot₃/if_val %₁₃) +15 (goto label₁₇) +16 (= slot₃/if_val core.nothing) +17 slot₃/if_val +18 slot₂/X +19 (call core.svec %₁₈) +20 (call core.svec %₁₀) +21 (call core.svec %₁₇) +22 (call core.resolve_typegroup TestMod %₁₉ %₂₀ %₂₁) +23 (= slot₂/X (call core.getfield %₂₂ 1)) +24 slot₂/X +25 (call core.declare_const TestMod :X %₂₄) +26 latestworld +27 TestMod.X +28 SourceLocation:none:1:0 +29 (call top._defaultctors %₂₇ %₂₈) +30 latestworld +31 (= slot₁/val core.nothing) +32 (call Base.Docs.Binding TestMod :X) +33 (call Core.svec "X docs\n") +34 (call Pair{Symbol, Any} :a "field a docs") +35 (call Pair{Symbol, Any} :b "field b docs") +36 (call Dict{Symbol, Any} %₃₄ %₃₅) +37 (call Pair :fields %₃₆) +38 (call Dict{Symbol, Any} :path => "none" :linenumber => 1 :module => TestMod %₃₇) +39 (call Base.Docs.docstr %₃₃ %₃₈) +40 TestMod.Union +41 (call core.apply_type %₄₀) +42 (call Base.Docs.doc! TestMod %₃₂ %₃₉ %₄₁) +43 (gotoifnot true label₄₆) +44 slot₁/val +45 (return %₄₄) +46 (return core.nothing) + +######################################## +# Struct with outer constructor +struct X{U} + x::U +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (= slot₂/U (call core.TypeVar :U)) +5 slot₂/U +6 (call core.svec %₅) +7 (call core.svec :x) +8 (call core.svec) +9 slot₂/U +10 (call core.svec %₉) +11 (call core.svec %₆ %₇ %₈ false 1 core.Any %₁₀) +12 (call core.isdefinedglobal TestMod :X false) +13 (gotoifnot %₁₂ label₁₇) +14 TestMod.X +15 (= slot₃/if_val %₁₄) +16 (goto label₁₈) +17 (= slot₃/if_val core.nothing) +18 slot₃/if_val +19 slot₁/X +20 (call core.svec %₁₉) +21 (call core.svec %₁₁) +22 (call core.svec %₁₈) +23 (call core.resolve_typegroup TestMod %₂₀ %₂₁ %₂₂) +24 (= slot₁/X (call core.getfield %₂₃ 1)) +25 slot₁/X +26 (call core.declare_const TestMod :X %₂₅) +27 latestworld +28 TestMod.X +29 SourceLocation::1:1 +30 (call top._defaultctors %₂₈ %₂₉) +31 latestworld +32 (return core.nothing) + +######################################## +# Struct with outer constructor where one typevar is constrained by the other +# See https://github.com/JuliaLang/julia/issues/27269) +struct X{T, S <: Vector{T}} + v::Vector{S} +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (= slot₂/T (call core.TypeVar :T)) +5 TestMod.Vector +6 slot₂/T +7 (call core.apply_type_or_typeapp %₅ %₆) +8 (= slot₃/S (call core.TypeVar :S %₇)) +9 slot₂/T +10 slot₃/S +11 (call core.svec %₉ %₁₀) +12 (call core.svec :v) +13 (call core.svec) +14 TestMod.Vector +15 slot₃/S +16 (call core.apply_type_or_typeapp %₁₄ %₁₅) +17 (call core.svec %₁₆) +18 (call core.svec %₁₁ %₁₂ %₁₃ false 1 core.Any %₁₇) +19 (call core.isdefinedglobal TestMod :X false) +20 (gotoifnot %₁₉ label₂₄) +21 TestMod.X +22 (= slot₄/if_val %₂₁) +23 (goto label₂₅) +24 (= slot₄/if_val core.nothing) +25 slot₄/if_val +26 slot₁/X +27 (call core.svec %₂₆) +28 (call core.svec %₁₈) +29 (call core.svec %₂₅) +30 (call core.resolve_typegroup TestMod %₂₇ %₂₈ %₂₉) +31 (= slot₁/X (call core.getfield %₃₀ 1)) +32 slot₁/X +33 (call core.declare_const TestMod :X %₃₂) +34 latestworld +35 TestMod.X +36 SourceLocation::1:1 +37 (call top._defaultctors %₃₅ %₃₆) +38 latestworld +39 (return core.nothing) + +######################################## +# User defined inner constructors and helper functions for structs without type params +struct X + x + f() = new(1) + X() = f() # this X() captures `f` (in flisp, as a Box :-/ ) + X(x) = new(x) + X(y,z)::ReallyXIPromise = new(y+z) + """ + Docs for X constructor + """ + X(a,b,c) = new(a) +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (call core.svec) +5 (call core.svec :x) +6 (call core.svec) +7 (call core.svec core.Any) +8 (call core.svec %₄ %₅ %₆ false 1 core.Any %₇) +9 (call core.isdefinedglobal TestMod :X false) +10 (gotoifnot %₉ label₁₄) +11 TestMod.X +12 (= slot₃/if_val %₁₁) +13 (goto label₁₅) +14 (= slot₃/if_val core.nothing) +15 slot₃/if_val +16 slot₁/X +17 (call core.svec %₁₆) +18 (call core.svec %₈) +19 (call core.svec %₁₅) +20 (call core.resolve_typegroup TestMod %₁₇ %₁₈ %₁₉) +21 (= slot₁/X (call core.getfield %₂₀ 1)) +22 slot₁/X +23 (call core.declare_const TestMod :X %₂₂) +24 latestworld +25 (call core.svec) +26 (call core.svec) +27 (call core.svec) +28 (call JuliaLowering.eval_closure_type TestMod :#f#f##0 %₂₅ %₂₆ %₂₇) +29 latestworld +30 TestMod.#f#f##0 +31 (new %₃₀) +32 TestMod.#f#f##0 +33 (call core.svec %₃₂) +34 (call core.svec) +35 SourceLocation::3:5 +36 (call core.svec %₃₃ %₃₄ %₃₅) +37 (call core.define_method TestMod core.nothing %₃₆ + --- code_info + slots: [slot₁/#self#(!read)] + 1 TestMod.X + 2 (new %₁ 1) + 3 (return %₂) +38 latestworld +39 (= slot₂/f %₃₁) +40 TestMod.X +41 (call core.apply_type core.Type %₄₀) +42 (call core.svec %₄₁) +43 (call core.svec) +44 SourceLocation::4:5 +45 (call core.svec %₄₂ %₄₃ %₄₄) +46 --- code_info + slots: [slot₁/#ctor-self#(!read)] + 1 (captured_local 1) + 2 (call %₁) + 3 (return %₂) +47 (call core.svec slot₂/f) +48 (call JuliaLowering.replace_captured_locals %₄₆ %₄₇) +49 (call core.define_method TestMod core.nothing %₄₅ %₄₈) +50 latestworld +51 TestMod.X +52 (call core.apply_type core.Type %₅₁) +53 (call core.svec %₅₂ core.Any) +54 (call core.svec) +55 SourceLocation::5:5 +56 (call core.svec %₅₃ %₅₄ %₅₅) +57 (call core.define_method TestMod core.nothing %₅₆ + --- code_info + slots: [slot₁/#ctor-self# slot₂/x] + 1 slot₁/#ctor-self# + 2 (new %₁ slot₂/x) + 3 (return %₂) +58 latestworld +59 TestMod.X +60 (call core.apply_type core.Type %₅₉) +61 (call core.svec %₆₀ core.Any core.Any) +62 (call core.svec) +63 SourceLocation::6:5 +64 (call core.svec %₆₁ %₆₂ %₆₃) +65 (call core.define_method TestMod core.nothing %₆₄ + --- code_info + slots: [slot₁/#ctor-self# slot₂/y slot₃/z slot₄/tmp(!read)] + 1 TestMod.ReallyXIPromise + 2 slot₁/#ctor-self# + 3 TestMod.+ + 4 (call %₃ slot₂/y slot₃/z) + 5 (= slot₄/tmp (new %₂ %₄)) + 6 (call core.isa slot₄/tmp %₁) + 7 (gotoifnot %₆ label₉) + 8 (goto label₁₁) + 9 (call top.convert %₁ slot₄/tmp) + 10 (= slot₄/tmp (call core.typeassert %₉ %₁)) + 11 slot₄/tmp + 12 (return %₁₁) +66 latestworld +67 TestMod.X +68 (call core.apply_type core.Type %₆₇) +69 (call core.svec %₆₈ core.Any core.Any core.Any) +70 (call core.svec) +71 SourceLocation::10:5 +72 (call core.svec %₆₉ %₇₀ %₇₁) +73 (call core.define_method TestMod core.nothing %₇₂ + --- code_info + slots: [slot₁/#ctor-self# slot₂/a slot₃/b(!read) slot₄/c(!read)] + 1 slot₁/#ctor-self# + 2 (new %₁ slot₂/a) + 3 (return %₂) +74 latestworld +75 (return core.nothing) + +######################################## +# User defined inner constructors and helper functions for structs with type params +struct X{S,T} + x + X{A,B}() = new(1) + X{U,V}() where {U,V} = new(1) + f() = new{A,B}(1) +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (= slot₂/S (call core.TypeVar :S)) +5 (= slot₃/T (call core.TypeVar :T)) +6 slot₂/S +7 slot₃/T +8 (call core.svec %₆ %₇) +9 (call core.svec :x) +10 (call core.svec) +11 (call core.svec core.Any) +12 (call core.svec %₈ %₉ %₁₀ false 1 core.Any %₁₁) +13 (call core.isdefinedglobal TestMod :X false) +14 (gotoifnot %₁₃ label₁₈) +15 TestMod.X +16 (= slot₅/if_val %₁₅) +17 (goto label₁₉) +18 (= slot₅/if_val core.nothing) +19 slot₅/if_val +20 slot₁/X +21 (call core.svec %₂₀) +22 (call core.svec %₁₂) +23 (call core.svec %₁₉) +24 (call core.resolve_typegroup TestMod %₂₁ %₂₂ %₂₃) +25 (= slot₁/X (call core.getfield %₂₄ 1)) +26 slot₁/X +27 (call core.declare_const TestMod :X %₂₆) +28 latestworld +29 TestMod.X +30 TestMod.A +31 TestMod.B +32 (call core.apply_type %₂₉ %₃₀ %₃₁) +33 (call core.apply_type core.Type %₃₂) +34 (call core.svec %₃₃) +35 (call core.svec) +36 SourceLocation::3:5 +37 (call core.svec %₃₄ %₃₅ %₃₆) +38 (call core.define_method TestMod core.nothing %₃₇ + --- code_info + slots: [slot₁/#ctor-self#] + 1 slot₁/#ctor-self# + 2 (new %₁ 1) + 3 (return %₂) +39 latestworld +40 (call core.TypeVar :U) +41 (call core.TypeVar :V) +42 TestMod.X +43 (call core.apply_type %₄₂ %₄₀ %₄₁) +44 (call core.apply_type core.Type %₄₃) +45 (call core.svec %₄₄) +46 (call core.svec %₄₀ %₄₁) +47 SourceLocation::4:5 +48 (call core.svec %₄₅ %₄₆ %₄₇) +49 (call core.define_method TestMod core.nothing %₄₈ + --- code_info + slots: [slot₁/#ctor-self#] + 1 slot₁/#ctor-self# + 2 (new %₁ 1) + 3 (return %₂) +50 latestworld +51 (call core.svec) +52 (call core.svec) +53 (call core.svec) +54 (call JuliaLowering.eval_closure_type TestMod :#f#f##1 %₅₁ %₅₂ %₅₃) +55 latestworld +56 TestMod.#f#f##1 +57 (new %₅₆) +58 TestMod.#f#f##1 +59 (call core.svec %₅₈) +60 (call core.svec) +61 SourceLocation::5:5 +62 (call core.svec %₅₉ %₆₀ %₆₁) +63 (call core.define_method TestMod core.nothing %₆₂ + --- code_info + slots: [slot₁/#self#(!read)] + 1 TestMod.X + 2 TestMod.A + 3 TestMod.B + 4 (call core.apply_type %₁ %₂ %₃) + 5 (new %₄ 1) + 6 (return %₅) +64 latestworld +65 (= slot₄/f %₅₇) +66 latestworld +67 (return core.nothing) + +######################################## +# new() calls with splats; `Any` fields +struct X + x + y + X(xs) = new(xs...) +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (call core.svec) +5 (call core.svec :x :y) +6 (call core.svec) +7 (call core.svec core.Any core.Any) +8 (call core.svec %₄ %₅ %₆ false 2 core.Any %₇) +9 (call core.isdefinedglobal TestMod :X false) +10 (gotoifnot %₉ label₁₄) +11 TestMod.X +12 (= slot₂/if_val %₁₁) +13 (goto label₁₅) +14 (= slot₂/if_val core.nothing) +15 slot₂/if_val +16 slot₁/X +17 (call core.svec %₁₆) +18 (call core.svec %₈) +19 (call core.svec %₁₅) +20 (call core.resolve_typegroup TestMod %₁₇ %₁₈ %₁₉) +21 (= slot₁/X (call core.getfield %₂₀ 1)) +22 slot₁/X +23 (call core.declare_const TestMod :X %₂₂) +24 latestworld +25 TestMod.X +26 (call core.apply_type core.Type %₂₅) +27 (call core.svec %₂₆ core.Any) +28 (call core.svec) +29 SourceLocation::4:5 +30 (call core.svec %₂₇ %₂₈ %₂₉) +31 (call core.define_method TestMod core.nothing %₃₀ + --- code_info + slots: [slot₁/#ctor-self# slot₂/xs] + 1 slot₁/#ctor-self# + 2 (call core._apply_iterate top.iterate core.tuple slot₂/xs) + 3 (splatnew %₁ %₂) + 4 (return %₃) +32 latestworld +33 (return core.nothing) + +######################################## +# new() calls with splats; typed fields +struct X{T} + x::T + y::A + X{T}(xs) where {T} = new(xs...) +end +#--------------------- +1 (call core.declare_global TestMod :X false) +2 latestworld +3 (= slot₁/X (call core.TypeVar :X)) +4 (= slot₂/T (call core.TypeVar :T)) +5 slot₂/T +6 (call core.svec %₅) +7 (call core.svec :x :y) +8 (call core.svec) +9 slot₂/T +10 TestMod.A +11 (call core.svec %₉ %₁₀) +12 (call core.svec %₆ %₇ %₈ false 2 core.Any %₁₁) +13 (call core.isdefinedglobal TestMod :X false) +14 (gotoifnot %₁₃ label₁₈) +15 TestMod.X +16 (= slot₃/if_val %₁₅) +17 (goto label₁₉) +18 (= slot₃/if_val core.nothing) +19 slot₃/if_val +20 slot₁/X +21 (call core.svec %₂₀) +22 (call core.svec %₁₂) +23 (call core.svec %₁₉) +24 (call core.resolve_typegroup TestMod %₂₁ %₂₂ %₂₃) +25 (= slot₁/X (call core.getfield %₂₄ 1)) +26 slot₁/X +27 (call core.declare_const TestMod :X %₂₆) +28 latestworld +29 (call core.TypeVar :T) +30 TestMod.X +31 (call core.apply_type %₃₀ %₂₉) +32 (call core.apply_type core.Type %₃₁) +33 (call core.svec %₃₂ core.Any) +34 (call core.svec %₂₉) +35 SourceLocation::4:5 +36 (call core.svec %₃₃ %₃₄ %₃₅) +37 (call core.define_method TestMod core.nothing %₃₆ + --- code_info + slots: [slot₁/#ctor-self# slot₂/xs slot₃/tmp slot₄/tmp] + 1 (call core._apply_iterate top.iterate core.tuple slot₂/xs) + 2 (call core.nfields %₁) + 3 (call top.ult_int %₂ 2) + 4 (gotoifnot %₃ label₇) + 5 (call top.ArgumentError "too few arguments in `new` (expected 2)") + 6 (call core.throw %₅) + 7 (call top.ult_int 2 %₂) + 8 (gotoifnot %₇ label₁₁) + 9 (call top.ArgumentError "too many arguments in `new` (expected 2)") + 10 (call core.throw %₉) + 11 slot₁/#ctor-self# + 12 (call core.fieldtype %₁₁ 1) + 13 (= slot₃/tmp (call core.getfield %₁ 1)) + 14 (call core.isa slot₃/tmp %₁₂) + 15 (gotoifnot %₁₄ label₁₇) + 16 (goto label₁₈) + 17 (= slot₃/tmp (call top.convert %₁₂ slot₃/tmp)) + 18 slot₃/tmp + 19 (call core.fieldtype %₁₁ 2) + 20 (= slot₄/tmp (call core.getfield %₁ 2)) + 21 (call core.isa slot₄/tmp %₁₉) + 22 (gotoifnot %₂₁ label₂₄) + 23 (goto label₂₅) + 24 (= slot₄/tmp (call top.convert %₁₉ slot₄/tmp)) + 25 slot₄/tmp + 26 (new %₁₁ %₁₈ %₂₅) + 27 (return %₂₆) +38 latestworld +39 (return core.nothing) + +######################################## +# Error: new doesn't accept keywords +struct X + X() = new(a=1) +end +#--------------------- +LoweringError: +struct X + X() = new(a=1) +# └─┘ ── `new` does not accept keyword arguments +end + +######################################## +# Error: new doesn't accept keywords (params block) +struct X + X() = new(; a=1) +end +#--------------------- +LoweringError: +struct X + X() = new(; a=1) +# └───┘ ── `new` does not accept keyword arguments +end + +######################################## +# Error: User defined inner constructors without enough type params {A} +struct X{S,T} + X() = new{A}() +end +#--------------------- +LoweringError: +struct X{S,T} + X() = new{A}() +# └────┘ ── too few type parameters specified in `new{...}` +end + +######################################## +# Error: User defined inner constructors without enough type params {} +struct X{S,T} + X() = new{}() +end +#--------------------- +LoweringError: +struct X{S,T} + X() = new{}() +# └───┘ ── too few type parameters specified in `new{...}` +end + +######################################## +# Error: User defined inner constructors without enough type params +struct X{S,T} + X{A}() = new() +end +#--------------------- +LoweringError: +struct X{S,T} + X{A}() = new() +# └─┘ ── too few type parameters specified in `new` +end + +######################################## +# Error: User defined inner constructors with too many type params +struct X{S,T} + X() = new{A,B,C}() +end +#--------------------- +LoweringError: +struct X{S,T} + X() = new{A,B,C}() +# └────────┘ ── too many type parameters specified in `new{...}` +end + +######################################## +# Error: Struct not at top level +function f() + struct X + end +end +#--------------------- +LoweringError: +function f() +# ┌─────── + struct X + end +#─────┘ ── this syntax is only allowed at top level +end + +######################################## +# Constructor with type parameter +A{<:Real}() = A(1) +#--------------------- +1 TestMod.Real +2 (call core.TypeVar :#T1 %₁) +3 TestMod.A +4 (call core.apply_type %₃ %₂) +5 (call core.UnionAll %₂ %₄) +6 (call core.TypeEqOf %₅) +7 (call core.svec %₆) +8 (call core.svec) +9 SourceLocation::1:1 +10 (call core.svec %₇ %₈ %₉) +11 (call core.define_method TestMod core.nothing %₁₀ + --- code_info + slots: [slot₁/#self#(!read)] + 1 TestMod.A + 2 (call %₁ 1) + 3 (return %₂) +12 latestworld +13 (return core.nothing) + +######################################## +# Inner constructor with local variable shadowing type parameter +# Type parameter T should NOT be boxed or captured. +# See https://github.com/aviatesk/JETLS.jl/issues/508 +struct ShadowTypeParam{T} + x::T + function ShadowTypeParam(x) + T = typeof(x) + return new{T}(x) + end +end +#--------------------- +1 (call core.declare_global TestMod :ShadowTypeParam false) +2 latestworld +3 (= slot₁/ShadowTypeParam (call core.TypeVar :ShadowTypeParam)) +4 (= slot₂/T (call core.TypeVar :T)) +5 slot₂/T +6 (call core.svec %₅) +7 (call core.svec :x) +8 (call core.svec) +9 slot₂/T +10 (call core.svec %₉) +11 (call core.svec %₆ %₇ %₈ false 1 core.Any %₁₀) +12 (call core.isdefinedglobal TestMod :ShadowTypeParam false) +13 (gotoifnot %₁₂ label₁₇) +14 TestMod.ShadowTypeParam +15 (= slot₃/if_val %₁₄) +16 (goto label₁₈) +17 (= slot₃/if_val core.nothing) +18 slot₃/if_val +19 slot₁/ShadowTypeParam +20 (call core.svec %₁₉) +21 (call core.svec %₁₁) +22 (call core.svec %₁₈) +23 (call core.resolve_typegroup TestMod %₂₀ %₂₁ %₂₂) +24 (= slot₁/ShadowTypeParam (call core.getfield %₂₃ 1)) +25 slot₁/ShadowTypeParam +26 (call core.declare_const TestMod :ShadowTypeParam %₂₅) +27 latestworld +28 TestMod.ShadowTypeParam +29 (call core.apply_type core.Type %₂₈) +30 (call core.svec %₂₉ core.Any) +31 (call core.svec) +32 SourceLocation::3:5 +33 (call core.svec %₃₀ %₃₁ %₃₂) +34 (call core.define_method TestMod core.nothing %₃₃ + --- code_info + slots: [slot₁/#ctor-self#(!read) slot₂/x slot₃/tmp slot₄/T(single_assign)] + 1 TestMod.typeof + 2 (= slot₄/T (call %₁ slot₂/x)) + 3 TestMod.ShadowTypeParam + 4 slot₄/T + 5 (call core.apply_type %₃ %₄) + 6 (call core.fieldtype %₅ 1) + 7 slot₂/x + 8 (= slot₃/tmp %₇) + 9 (call core.isa slot₃/tmp %₆) + 10 (gotoifnot %₉ label₁₂) + 11 (goto label₁₃) + 12 (= slot₃/tmp (call top.convert %₆ slot₃/tmp)) + 13 slot₃/tmp + 14 (new %₅ %₁₃) + 15 (return %₁₄) +35 latestworld +36 (return core.nothing) + +######################################## +# Basic typegroup +typegroup + struct A + b::B + end + struct B + a::A + end +end +#--------------------- +1 (call core.declare_global TestMod :A false) +2 latestworld +3 (call core.declare_global TestMod :B false) +4 latestworld +5 (= slot₁/A (call core.TypeVar :A)) +6 (= slot₂/B (call core.TypeVar :B)) +7 (call core.svec) +8 (call core.svec :b) +9 (call core.svec) +10 slot₂/B +11 (call core.svec %₁₀) +12 (call core.svec %₇ %₈ %₉ false 1 core.Any %₁₁) +13 (call core.svec) +14 (call core.svec :a) +15 (call core.svec) +16 slot₁/A +17 (call core.svec %₁₆) +18 (call core.svec %₁₃ %₁₄ %₁₅ false 1 core.Any %₁₇) +19 (call core.isdefinedglobal TestMod :A false) +20 (gotoifnot %₁₉ label₂₄) +21 TestMod.A +22 (= slot₃/if_val %₂₁) +23 (goto label₂₅) +24 (= slot₃/if_val core.nothing) +25 slot₃/if_val +26 (call core.isdefinedglobal TestMod :B false) +27 (gotoifnot %₂₆ label₃₁) +28 TestMod.B +29 (= slot₄/if_val %₂₈) +30 (goto label₃₂) +31 (= slot₄/if_val core.nothing) +32 slot₄/if_val +33 slot₁/A +34 slot₂/B +35 (call core.svec %₃₃ %₃₄) +36 (call core.svec %₁₂ %₁₈) +37 (call core.svec %₂₅ %₃₂) +38 (call core.resolve_typegroup TestMod %₃₅ %₃₆ %₃₇) +39 (= slot₁/A (call core.getfield %₃₈ 1)) +40 slot₁/A +41 (call core.declare_const TestMod :A %₄₀) +42 latestworld +43 (= slot₂/B (call core.getfield %₃₈ 2)) +44 slot₂/B +45 (call core.declare_const TestMod :B %₄₄) +46 latestworld +47 TestMod.A +48 SourceLocation::2:5 +49 (call top._defaultctors %₄₇ %₄₈) +50 TestMod.B +51 SourceLocation::5:5 +52 (call top._defaultctors %₅₀ %₅₁) +53 latestworld +54 (return core.nothing) + +######################################## +# Typegroup with supertype and apply_type_or_typeapp replacement +typegroup + struct A <: AbstractVector{B} + b::Union{Nothing, B} + end + struct B + a::A + end +end +#--------------------- +1 (call core.declare_global TestMod :A false) +2 latestworld +3 (call core.declare_global TestMod :B false) +4 latestworld +5 (= slot₁/A (call core.TypeVar :A)) +6 (= slot₂/B (call core.TypeVar :B)) +7 (call core.svec) +8 (call core.svec :b) +9 (call core.svec) +10 TestMod.AbstractVector +11 slot₂/B +12 (call core.apply_type_or_typeapp %₁₀ %₁₁) +13 TestMod.Union +14 TestMod.Nothing +15 slot₂/B +16 (call core.apply_type_or_typeapp %₁₃ %₁₄ %₁₅) +17 (call core.svec %₁₆) +18 (call core.svec %₇ %₈ %₉ false 1 %₁₂ %₁₇) +19 (call core.svec) +20 (call core.svec :a) +21 (call core.svec) +22 slot₁/A +23 (call core.svec %₂₂) +24 (call core.svec %₁₉ %₂₀ %₂₁ false 1 core.Any %₂₃) +25 (call core.isdefinedglobal TestMod :A false) +26 (gotoifnot %₂₅ label₃₀) +27 TestMod.A +28 (= slot₃/if_val %₂₇) +29 (goto label₃₁) +30 (= slot₃/if_val core.nothing) +31 slot₃/if_val +32 (call core.isdefinedglobal TestMod :B false) +33 (gotoifnot %₃₂ label₃₇) +34 TestMod.B +35 (= slot₄/if_val %₃₄) +36 (goto label₃₈) +37 (= slot₄/if_val core.nothing) +38 slot₄/if_val +39 slot₁/A +40 slot₂/B +41 (call core.svec %₃₉ %₄₀) +42 (call core.svec %₁₈ %₂₄) +43 (call core.svec %₃₁ %₃₈) +44 (call core.resolve_typegroup TestMod %₄₁ %₄₂ %₄₃) +45 (= slot₁/A (call core.getfield %₄₄ 1)) +46 slot₁/A +47 (call core.declare_const TestMod :A %₄₆) +48 latestworld +49 (= slot₂/B (call core.getfield %₄₄ 2)) +50 slot₂/B +51 (call core.declare_const TestMod :B %₅₀) +52 latestworld +53 TestMod.A +54 SourceLocation::2:5 +55 (call top._defaultctors %₅₃ %₅₄) +56 TestMod.B +57 SourceLocation::5:5 +58 (call top._defaultctors %₅₆ %₅₇) +59 latestworld +60 (return core.nothing) + +######################################## +# Error: Duplicate field name in struct +struct A; x; x; end +#--------------------- +LoweringError: +struct A; x; x; end +# ╙ ── duplicate field name + +######################################## +# Error: Duplicate field name with different types +struct A; x::Int; x::String; end +#--------------------- +LoweringError: +struct A; x::Int; x::String; end +# ╙ ── duplicate field name + +######################################## +# Error: Duplicate field name in mutable struct +mutable struct A; x; y; x; end +#--------------------- +LoweringError: +mutable struct A; x; y; x; end +# ╙ ── duplicate field name diff --git a/JuliaLowering/test/utils.jl b/JuliaLowering/test/utils.jl new file mode 100644 index 0000000000000..5b2a87e2d61ee --- /dev/null +++ b/JuliaLowering/test/utils.jl @@ -0,0 +1,420 @@ +# Shared testing code which should be included before running individual test files. +using Test + +using JuliaLowering +using JuliaSyntax + +const JS = JuliaSyntax +const JL = JuliaLowering + +import FileWatching + +# The following are for docstrings testing. We need to load the REPL module +# here for `Base.@doc` lookup to work at all. Yes this does seem really, +# really, REALLY messed up. +using Markdown +import REPL + +using .JuliaSyntax: SourceAttrType, sourcetext, SyntaxList + +using .JuliaLowering: @ast, Bindings, Kind, LoweringError, MacroExpansionError, + ScopeLayer, SourceRef, SyntaxTree, children, flattened_provenance, + is_leaf, mapchildren, numchildren, showprov, syntax_name, syntax_id + +function _source_node(src) + SyntaxTree(K"TOMBSTONE", nothing, nothing, src, nothing) +end + +macro ast_(tree) + quote + srcref = _source_node($(QuoteNode(__source__))) + @ast _ srcref $tree + end +end + +#------------------------------------------------------------------------------- +function _format_as_ast_macro(io, ex, indent) + k = kind(ex) + kind_str = repr(k) + if !is_leaf(ex) + println(io, indent, "[", kind_str) + ind2 = indent*" " + for c in children(ex) + _format_as_ast_macro(io, c, ind2) + end + println(io, indent, "]") + else + val_str = if k == K"Identifier" || k == K"core" || k == K"top" + repr(syntax_name(ex)) + elseif k == K"BindingId" + repr(syntax_id(ex)) + else + repr(get(ex, :value, nothing)) + end + println(io, indent, val_str, "::", kind_str) + end +end + +function format_as_ast_macro(io::IO, ex) + print(io, "@ast_ ") + _format_as_ast_macro(io, ex, "") +end + +""" + format_as_ast_macro(ex) + +Format AST `ex` as a Julia source code call to the `@ast_` macro for generating +test case comparisons with the `≈` function. +""" +format_as_ast_macro(ex) = format_as_ast_macro(stdout, ex) + +#------------------------------------------------------------------------------- + +# Test tools + +function uncomment_description(desc) + replace(desc, r"^# ?"m=>"") +end + +function comment_description(desc) + lines = replace(split(desc, '\n')) do line + strip("# " * line) + end + join(lines, '\n') +end + +function match_ir_test_case(case_str) + m = match(r"(^#(?:.|\n)*?)^([^#](?:.|\n)*)"m, strip(case_str)) + if isnothing(m) + error("Malformatted IR test case:\n$(repr(case_str))") + end + description = uncomment_description(m[1]) + inout = split(m[2], r"#----*") + input, output = length(inout) == 2 ? inout : + length(inout) == 1 ? (inout[1], "") : + error("Too many sections in IR test case") + expect_error = startswith(description, "Error") + is_broken = startswith(description, "FIXME") + method_filter = begin + mf = match(r"\[method_filter: *(.*)\]", description) + isnothing(mf) ? nothing : strip(mf[1]) + end + (; expect_error=expect_error, is_broken=is_broken, + description=strip(description), + method_filter=method_filter, + input=strip(input), output=strip(output)) +end + +function read_ir_test_cases(filename) + str = read(filename, String) + parts = split(str, r"#\*+") + if length(parts) == 2 + preamble_str = strip(parts[1]) + cases_str = parts[2] + else + preamble_str = "" + cases_str = only(parts) + end + (preamble_str, + [match_ir_test_case(s) for s in split(cases_str, r"######*") if strip(s) != ""]) +end + +function setup_ir_test_module(preamble) + test_mod = Module(:TestMod) + Base.eval(test_mod, :(const JuliaLowering = $JuliaLowering)) + Base.eval(test_mod, :(const var"@ast_" = $(var"@ast_"))) + JuliaLowering.include_string(test_mod, preamble) + test_mod +end + +function format_ir_for_test(mod, case) + @assert !case.is_broken + ex = parsestmt(SyntaxTree, case.input) + try + if (kind(ex) == K"macrocall" && kind(ex[1]) == K"Identifier" && + syntax_name(ex[1]) == "@ast_") + # Total hack, until @ast_ can be implemented in terms of new-style + # macros. + ex = Base.eval(mod, JuliaLowering.est_to_expr(ex)) + end + x = JuliaLowering.lower(mod, ex) + if case.expect_error + error("Expected a lowering error in test case \"$(case.description)\"") + end + ir = strip(sprint(JuliaLowering.print_ir, x, case.method_filter)) + return replace(ir, string(mod)=>"TestMod") + catch exc + if exc isa InterruptException + rethrow() + elseif case.expect_error && (exc isa LoweringError) + return sprint(io->Base.showerror(io, exc, show_detail=false)) + elseif case.expect_error && (exc isa MacroExpansionError) + return sprint(io->Base.showerror(io, exc)) + else + throw("Error in test case \"$(case.description)\"") + end + end +end + +function test_ir_cases(filename::AbstractString) + preamble, cases = read_ir_test_cases(filename) + test_mod = setup_ir_test_module(preamble) + for case in cases + if case.is_broken + continue + end + output = format_ir_for_test(test_mod, case) + @testset "$(case.description)" begin + if output != case.output + # Do additional error dumping, as @test will not format errors in a nice way + @error "Test \"$(case.description)\" failed" output=Text(output) ref=Text(case.output) + end + @test output == case.output + end + end +end + +""" +Update all IR test cases in `filename` when the IR format has changed. + +When `pattern` is supplied, update only those tests where +`occursin(pattern, description)` is true. +""" +function refresh_ir_test_cases(filename, pattern=nothing) + preamble, cases = read_ir_test_cases(filename) + test_mod = setup_ir_test_module(preamble) + io = IOBuffer() + if !isempty(preamble) + println(io, preamble, "\n") + println(io, "#*******************************************************************************") + end + for case in cases + if !case.is_broken && (isnothing(pattern) || occursin(pattern, case.description)) + ir = format_ir_for_test(test_mod, case) + if rstrip(ir) != case.output + @info "Refreshing test case $(repr(case.description)) in $filename" + end + else + ir = case.output + end + (case == cases[end] ? print : println)(io, + """ + ######################################## + $(comment_description(case.description)) + $(strip(case.input)) + #--------------------- + $ir + """ + ) + end + # Write only at the end to ensure we don't write rubbish if we crash! + write(filename, take!(io)) + nothing +end + +function refresh_all_ir_test_cases(test_dir=".") + foreach(refresh_ir_test_cases, filter(fn->endswith(fn, "ir.jl"), readdir(test_dir, join=true))) +end + +function watch_ir_tests(dir, delay=0.5) + dir = abspath(dir) + while true + (name, event) = FileWatching.watch_folder(dir) + if endswith(name, "_ir.jl") && (event.changed || event.renamed) + FileWatching.unwatch_folder(dir) + sleep(delay) + try + refresh_ir_test_cases(joinpath(dir, name)) + catch + @error "Error refreshing test case" exception=current_exceptions() + end + end + end +end + +function lower_str(mod::Module, s::AbstractString) + ex = parsestmt(JuliaLowering.SyntaxTree, s) + return JuliaLowering.to_lowered_expr(JuliaLowering.lower(mod, ex)) +end + +# See Julia Base tests in "test/docs.jl" +function docstrings_equal(d1, d2; debug=true) + io1 = IOBuffer() + io2 = IOBuffer() + show(io1, MIME"text/markdown"(), d1) + show(io2, MIME"text/markdown"(), d2) + s1 = String(take!(io1)) + s2 = String(take!(io2)) + if debug && s1 != s2 + print(s1) + println("--------------------------------------------------------------------------------") + print(s2) + println("================================================================================") + end + return s1 == s2 +end +docstrings_equal(d1::Docs.DocStr, d2) = docstrings_equal(Docs.parsedoc(d1), d2) + +#------------------------------------------------------------------------------- +# Tools for test case reduction + +function block_reduction_1(is_lowering_error::Function, orig_ex::ST, ex::ST, + curr_path = Int[]) where {ST <: SyntaxTree} + if !is_leaf(ex) + if kind(ex) == K"block" + for i in 1:numchildren(ex) + trial_ex = delete_block_child(orig_ex, orig_ex, curr_path, i) + if is_lowering_error(trial_ex) + # @info "Reduced expression" curr_path i + return trial_ex + end + end + end + for (i,e) in enumerate(children(ex)) + push!(curr_path, i) + res = block_reduction_1(is_lowering_error, orig_ex, e, curr_path) + if !isnothing(res) + return res + end + pop!(curr_path) + end + end + return nothing +end + +# Find children of all `K"block"`s in an expression and try deleting them while +# preserving the invariant `is_lowering_error(reduced) == true`. +function block_reduction(is_lowering_error, ex) + reduced = ex + was_reduced = false + while true + r = block_reduction_1(is_lowering_error, reduced, reduced) + if isnothing(r) + return (reduced, was_reduced) + end + reduced = r + was_reduced = true + end +end + +function delete_block_child(ctx, ex, block_path, child_idx, depth=1) + if depth > length(block_path) + cs = copy(children(ex)) + deleteat!(cs, child_idx) + @ast ctx ex [ex cs...] + else + j = block_path[depth] + mapchildren(ctx, ex, j:j) do e + delete_block_child(ctx, e, block_path, child_idx, depth+1) + end + end +end + +function throws_lowering_exc(mod, ex) + try + debug_lower(mod, ex) + return false + catch exc + if exc isa LoweringError + return true + else + rethrow() + end + end +end + +# Parse a file and lower the top level expression one child at a time, finding +# any top level statement that fails lowering and producing a partially reduced +# test case. +function reduce_any_failing_toplevel(mod::Module, filename::AbstractString; do_eval::Bool=false) + text = read(filename, String) + ex0 = parseall(SyntaxTree, text; filename) + for ex in children(ex0) + try + ex_compiled = JuliaLowering.lower(mod, ex) + ex_expr = JuliaLowering.to_lowered_expr(ex_compiled) + if do_eval + Base.eval(mod, ex_expr) + end + catch exc + @error "Failure lowering code" ex + if !(exc isa LoweringError) + rethrow() + end + (reduced,was_reduced) = block_reduction(e->throws_lowering_exc(mod,e), ex) + if !was_reduced + @info "No reduction possible" + return ex + else + @info "Reduced code" reduced + return reduced + end + end + end + nothing +end + +function expr_structure_eq(e1,e2) + @nospecialize + typeof(e1) == typeof(e2) || return false + e1 isa Expr || return e1 == e2 + e1.head === e2.head || return false + length(e1.args) == length(e2.args) || return false + for (a, b) in Iterators.zip(e1.args, e2.args) + expr_structure_eq(a,b) || return false + end + true +end + +macro newmod(name="newmod_$(string(__source__))", parentmod=__module__, body...) + mod_ex = :( + module $(Symbol(name)) + const JuliaLowering = $(JuliaLowering) + const JuliaSyntax = $(JuliaSyntax) + const var"@K_str" = JuliaSyntax.var"@K_str" + const var"@legacy_quote_to_syntax" = JuliaLowering.var"@legacy_quote_to_syntax" + $(body...) + end) + Expr(:block, + :(mod = $(Expr(:escape, :(Core.eval($parentmod, $(QuoteNode(mod_ex))))))), + Expr(Symbol("latestworld-if-toplevel")), :mod) +end + +function fl_macroexpand(mod::Module, x::Expr) + ccall(:jl_macroexpand, Any, (Any, Any, Cint, Cint, Cint), x, mod, true, false, true) +end + +function fl_lower(mod::Module, x::Expr) + Base.fl_lower(x, mod, @__FILE__, @__LINE__, Base.get_world_counter())[1] +end + +function fl_eval(mod::Module, x::Expr) + Core.eval(mod, fl_lower(mod, x)) +end + +function jl_macroexpand(mod::Module, x::SyntaxTree; expr_compat_mode=false) + JuliaLowering.macroexpand(mod, x; expr_compat_mode) +end + +function jl_lower(mod::Module, st::SyntaxTree; expr_compat_mode=false) + JuliaLowering.lower(mod, st; expr_compat_mode) +end + +function jl_eval(mod::Module, st::SyntaxTree; expr_compat_mode=false) + JuliaLowering.eval(mod, st; expr_compat_mode) +end + + +fl_macroexpand(mod::Module, st::SyntaxTree; kws...) = + fl_macroexpand(mod, JuliaLowering.est_to_expr(st); kws...) +fl_lower(mod::Module, st::SyntaxTree; kws...) = + fl_lower(mod, JuliaLowering.est_to_expr(st); kws...) +fl_eval(mod::Module, st::SyntaxTree; kws...) = + fl_eval(mod, JuliaLowering.est_to_expr(st); kws...) + +jl_macroexpand(mod::Module, ex::Expr; kws...) = + jl_macroexpand(mod, JuliaLowering.expr_to_est(ex); kws...) +jl_lower(mod::Module, ex::Expr; kws...) = + jl_lower(mod, JuliaLowering.expr_to_est(ex); kws...) +jl_eval(mod::Module, ex::Expr; kws...) = + jl_eval(mod, JuliaLowering.expr_to_est(ex); kws...) diff --git a/JuliaLowering/test/validation.jl b/JuliaLowering/test/validation.jl new file mode 100644 index 0000000000000..6e3c907f1f692 --- /dev/null +++ b/JuliaLowering/test/validation.jl @@ -0,0 +1,188 @@ +# Basic tests that the validator accepts or rejects a given form. For testing +# error messages, write an IR test instead. + +function vst1_ok(x::Expr) + est = JuliaLowering.expr_to_est(x) + JuliaSyntax.fill_context!(est, JuliaSyntax.SyntaxContext(@__MODULE__, v"1.13")) + JuliaLowering.valid_st1(est).ok +end + +let + nonempty_heads = String[ + "let", + "if", + "try", + "function", + "call", + "'", + ".", + "do", + "=", + "return", + "for", + "while", + "curly", + "where", + "->", + "flatten", + "generator", + "comprehension", + "typed_comprehension", + "comparison", + "::", + ".&&", + ".||", + "const", + "global", + "local", + "macrocall", + "quote", + "inert", + "syntaxinert", + "top", + "opaque_closure", + "symboliclabel", + "symbolicgoto", + "symbolicblock", + "gc_preserve", + "isdefined", + "lambda", + "foreigncall", + "cfunction", + "cconv", + "tryfinally", + "inline", + "noinline", + "inbounds", + "islocal", + "isglobal", + "new", + "splatnew", + "thisfunction", + "copyast", + ":", + "...", + ".+=", + "|=", + ".=", + "braces", + "\$", + "parameters", + "kw", + "outer", + "macro", + "struct", + "abstract", + "primitive", + "module", + "local-def", + "::", + "where", + "curly", + "ref", + "ncat", + "nrow", + "typed_hcat", + "typed_vcat", + "typed_ncat", + "import", + "using", + ] + standalone_heads = String[ + "block", + "tuple", + "public", + "export", + "string", + "&&", + "<:", + ">:", + "-->", + "&&", + "||", + "toplevel", + "locals", + "vect", + "hcat", + "vcat", + "meta", + "boundscheck", + "loopinfo", + ] + @testset for h in nonempty_heads + @test !vst1_ok(Expr(Symbol(h))) + end + @testset for h in standalone_heads + ex = Expr(Symbol(h)) + @test !Meta.isexpr(:error, Meta.lower(@__MODULE__, ex)) + @test vst1_ok(ex) + end +end + +if JL.DEBUG + @test_throws ErrorException vst1_ok(Expr(:nothing)) +else + @test !vst1_ok(Expr(:nothing)) +end +@test vst1_ok(Expr(:block, nothing)) +@test vst1_ok(Expr(:block, GlobalRef(Core, :nothing))) + +@test vst1_ok(Expr(:-->, 1)) +@test vst1_ok(Expr(:-->, 1, 2)) +@test vst1_ok(Expr(:-->, 1, 2, 3)) +@test vst1_ok(Expr(:-->, Expr(:..., Expr(:tuple, 1, 2, 3)))) +@test vst1_ok(Expr(:-->, Expr(:kw, :foo, 1))) + +@test vst1_ok(Expr(:const, :a, 1)) + +# vst1_dot_getproperty_rhs allows usually-invalid forms +@testset "dot rhs forms" for rhs in [:_, :__, Symbol("#unused#"), :ccall, :cglobal, 1] + @test vst1_ok(Expr(:., :Mod, rhs)) + @test vst1_ok(Expr(:., :Mod, Expr(:inert, rhs))) + @test vst1_ok(Expr(:., :Mod, QuoteNode(rhs))) + + @test vst1_ok(Expr(:., :Mod, string(rhs))) + @test vst1_ok(Expr(:., :Mod, Expr(:inert, string(rhs)))) + @test vst1_ok(Expr(:., :Mod, QuoteNode(string(rhs)))) +end + +@test vst1_ok(:(using Mod: cglobal)) +@test vst1_ok(:(Mod.cglobal)) +@test vst1_ok(:(Mod._ = 1)) + +@testset "underscores that should probably not be valid" begin + @test vst1_ok(:(Mod._)) + @test vst1_ok(:(function f(x::_); x; end)) + @test vst1_ok(:(global _)) + @test vst1_ok(:(global _::Int)) + @test vst1_ok(:(local _)) + @test vst1_ok(:(local _::Int)) +end + +@testset "empty symbol is valid" for e in [ + Expr(:block, Symbol("")) + Expr(:inert, Symbol("")) + Expr(:(::), Symbol(""), :Int) + Expr(:const, Expr(:(=), Symbol(""), 1)) + Expr(:global, Expr(:(=), Symbol(""), 1)) + Expr(:local, Expr(:(=), Symbol(""), 1)) + Expr(:let, Expr(:block, Expr(:(=), Symbol(""), 1)), Expr(:block)) + Expr(:function, Expr(:call, Symbol("")), Expr(:block)) + ] + @test vst1_ok(e) +end + +@testset "import/using path" begin + # `.` after identifier + @test !vst1_ok(Expr(:import, Expr(:., :A, :., :B))) + # leading `.` on a name + @test !vst1_ok(Expr(:import, Expr(:(:), Expr(:., :M), Expr(:., :., :a)))) + # non-identifier rename + @test !vst1_ok(Expr(:import, Expr(:(:), Expr(:., :M), + Expr(:as, Expr(:., :a), Expr(:call, :f))))) + # empty path + @test !vst1_ok(Expr(:import, Expr(:.))) + # not an import path + @test !vst1_ok(Expr(:import, Expr(:call, :f))) + @test !vst1_ok(Expr(:import, 42)) +end diff --git a/JuliaSyntax/.gitignore b/JuliaSyntax/.gitignore new file mode 100644 index 0000000000000..e42c550d0720b --- /dev/null +++ b/JuliaSyntax/.gitignore @@ -0,0 +1,9 @@ +/Manifest.toml +/Manifest-v*.toml +/docs/Manifest.toml +/tools/pkgs +/tools/pkg_tars +/tools/logs.txt +/docs/build +*.cov +/.vscode/settings.json diff --git a/JuliaSyntax/.mailmap b/JuliaSyntax/.mailmap new file mode 100644 index 0000000000000..a16a55c644b31 --- /dev/null +++ b/JuliaSyntax/.mailmap @@ -0,0 +1,2 @@ +Claire Foster +Claire Foster diff --git a/JuliaSyntax/LICENSE.md b/JuliaSyntax/LICENSE.md new file mode 100644 index 0000000000000..7efd19088a06f --- /dev/null +++ b/JuliaSyntax/LICENSE.md @@ -0,0 +1,45 @@ +The JuliaSyntax.jl package is licensed under the MIT "Expat" License: + +> Copyright (c) 2021 Julia Computing and contributors +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. + +The code in src/tokenize*.jl and test/tokenize.jl is derived from the Tokenize.jl +package and is also licensed under the MIT "Expat" License: + +> Copyright (c) 2016: Kristoffer Carlsson. +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. +> diff --git a/JuliaSyntax/Project.toml b/JuliaSyntax/Project.toml new file mode 100644 index 0000000000000..6ab84fdfb4733 --- /dev/null +++ b/JuliaSyntax/Project.toml @@ -0,0 +1,16 @@ +name = "JuliaSyntax" +uuid = "70703baa-626e-46a2-a12c-08ffd08c73b4" +authors = ["Claire Foster and contributors"] +version = "2.0.0-DEV" + +[compat] +Serialization = "1.0" +julia = "1.0" + +[extras] +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test", "Serialization", "Logging"] diff --git a/JuliaSyntax/README.md b/JuliaSyntax/README.md new file mode 100644 index 0000000000000..ae9b2b9760b36 --- /dev/null +++ b/JuliaSyntax/README.md @@ -0,0 +1,46 @@ +# JuliaSyntax + +[![Build Status](https://github.com/c42f/JuliaSyntax.jl/workflows/CI/badge.svg)](https://github.com/c42f/JuliaSyntax.jl/actions) +[![codecov.io](https://codecov.io/github/JuliaLang/JuliaSyntax.jl/coverage.svg?branch=main)](https://codecov.io/github/JuliaLang/JuliaSyntax.jl?branch=main) + +A Julia compiler frontend, written in Julia. + +Read the [documentation](https://JuliaLang.github.io/JuliaSyntax.jl/dev) for +more information. + +### Status + +JuliaSyntax.jl is used as the new default Julia parser in Julia 1.10. +It's highly compatible with Julia's older +[femtolisp-based parser](https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm) - +It parses all of Base, the standard libraries and General registry. Some minor +difference remain where we've decided to fix bugs or strange behaviors in the +reference parser. + +The AST and tree data structures are usable but their APIs will evolve as we +try out various use cases. Parsing to the standard `Expr` AST is always +possible and will be stable. + +The intention is to extend this library over time to cover more of the Julia +compiler frontend. + +# Getting involved + +For people who want to help improve Julia's error messages by contributing to +JuliaSyntax, I'd suggest looking through the issue list at +https://github.com/JuliaLang/JuliaSyntax.jl/issues and choosing a small issue +or two to work on to familiarize yourself with the code. Anything marked with +the labels `intro issue` or `bug` might be a good place to start. + +Also watching the [2022 JuliaCon talk](https://www.youtube.com/watch?v=CIiGng9Brrk) +and reading the [design](https://julialang.github.io/JuliaSyntax.jl/dev/design/) and +[reference](https://julialang.github.io/JuliaSyntax.jl/dev/reference/) +documentation should be good for an overview. + +As of May 2023, we've got really good positional tracking within the source, +but JuliaSyntax really needs a better system for parser recovery before the +errors are really nice. This requires some research. For example, you could +read up on how rust-analyzer does recovery, or rslint - both these are +event-based recursive decent parsers with similar structure to JuliaSyntax +(though in Rust). I also want to investigate whether we can do data-driven +parser recovery using an ML technique. But again, this is a research project. diff --git a/JuliaSyntax/docs/Project.toml b/JuliaSyntax/docs/Project.toml new file mode 100644 index 0000000000000..1814eb3304f3c --- /dev/null +++ b/JuliaSyntax/docs/Project.toml @@ -0,0 +1,5 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "1" diff --git a/JuliaSyntax/docs/make.jl b/JuliaSyntax/docs/make.jl new file mode 100644 index 0000000000000..5c3a094ba8691 --- /dev/null +++ b/JuliaSyntax/docs/make.jl @@ -0,0 +1,26 @@ +using Documenter, JuliaSyntax + +makedocs(; + modules=[JuliaSyntax], + format=Documenter.HTML( + repolink="https://github.com/JuliaLang/JuliaSyntax.jl" + ), + pages=[ + "Overview" => "index.md" + "How To" => "howto.md" + "Reference" => [ + "reference.md" + "api.md" + ] + "Design Discussion" => "design.md" + ], + repo="https://github.com/JuliaLang/JuliaSyntax.jl/blob/{commit}{path}#L{line}", + sitename="JuliaSyntax.jl", + authors = "Claire Foster and contributors: https://github.com/JuliaLang/JuliaSyntax.jl/graphs/contributors", + warnonly = true +) + +deploydocs(; + repo="github.com/JuliaLang/JuliaSyntax.jl", + push_preview=true +) diff --git a/JuliaSyntax/docs/src/api.md b/JuliaSyntax/docs/src/api.md new file mode 100644 index 0000000000000..18d439d8b2222 --- /dev/null +++ b/JuliaSyntax/docs/src/api.md @@ -0,0 +1,167 @@ +# API Reference + +## Parsing + +```@docs +JuliaSyntax.parsestmt +JuliaSyntax.parseall +JuliaSyntax.parseatom +``` + +### Low level parsing API + +The `ParseStream` interface which provides a low-level stream-like I/O +interface for writing the parser. The parser does not depend on or produce any +concrete tree data structure as part of the parsing phase but the output spans +can be post-processed into various tree data structures as required using +[`JuliaSyntax.build_tree`](@ref). + +```@docs +JuliaSyntax.parse! +JuliaSyntax.ParseStream +JuliaSyntax.build_tree +``` + +## Tokenization + +```@docs +JuliaSyntax.tokenize +JuliaSyntax.untokenize +JuliaSyntax.Token +``` + +## Source code handling + +This section describes the generic functions for source text, source location +computation and formatting functions. + +Contiguous syntax objects like nodes in the syntax tree should implement the +following where possible: + +```@docs +JuliaSyntax.sourcefile +JuliaSyntax.byte_range +``` + +This will provide implementations of the following which include range +information, line numbers, and fancy highlighting of source ranges: + +```@docs +JuliaSyntax.first_byte +JuliaSyntax.last_byte +JuliaSyntax.filename +JuliaSyntax.source_line +JuliaSyntax.source_location +JuliaSyntax.char_range +JuliaSyntax.sourcetext +JuliaSyntax.highlight +``` + +`SourceFile`-specific functions: + +```@docs +JuliaSyntax.SourceFile +JuliaSyntax.source_line_range +``` + +## Expression predicates, kinds and flags + +Expressions are tagged with a kind - like a type, but represented as an integer +tag rather than a full Julia type for efficiency. (Very like the tag of a "sum +type".) `Kind`s are constructed with the `@K_str` macro. + +```@docs +JuliaSyntax.@K_str +JuliaSyntax.Kind +``` + +The kind of an expression `ex` in a tree should be accessed with `kind(ex)` + +```@docs +JuliaSyntax.kind +``` + +In addition to the `kind`, a small integer set of "flags" is included to +further distinguish details of each expression, accessed with the `flags` +function. The kind and flags can be wrapped into a `SyntaxHead` which is +accessed with the `head` function. + +```@docs +JuliaSyntax.flags +JuliaSyntax.SyntaxHead +JuliaSyntax.head +``` + +Details about the flags may be extracted using various predicates: + +```@docs +JuliaSyntax.is_trivia +JuliaSyntax.is_prefix_call +JuliaSyntax.is_infix_op_call +JuliaSyntax.is_prefix_op_call +JuliaSyntax.is_postfix_op_call +JuliaSyntax.numeric_flags +``` + +Some of the more unusual predicates are accessed merely with `has_flags(x, +flag_bits)`, where any of the following uppercase constants may be used for +`flag_bits` after checking that the `kind` is correct. + +```@docs +JuliaSyntax.has_flags +JuliaSyntax.TRIPLE_STRING_FLAG +JuliaSyntax.RAW_STRING_FLAG +JuliaSyntax.PARENS_FLAG +JuliaSyntax.TRAILING_COMMA_FLAG +JuliaSyntax.COLON_QUOTE +JuliaSyntax.TOPLEVEL_SEMICOLONS_FLAG +JuliaSyntax.MUTABLE_FLAG +JuliaSyntax.BARE_MODULE_FLAG +JuliaSyntax.SHORT_FORM_FUNCTION_FLAG +``` + +## Syntax trees + +Access to the children of a tree node is provided by the functions + +```@docs +JuliaSyntax.is_leaf +JuliaSyntax.numchildren +JuliaSyntax.children +``` + +For convenient access to the children, we also provide `node[i]`, `node[i:j]` +and `node[begin:end]` by implementing `Base.getindex()`, `Base.firstindex()` and +`Base.lastindex()`. We choose to return a view from `node[i:j]` to make it +non-allocating. + +Tree traversal is supported by using these functions along with the predicates +such as [`kind`](@ref) listed above. + +### Trees referencing the source + +```@docs +JuliaSyntax.SyntaxNode +``` + +Functions applicable to `SyntaxNode` include everything in the sections on +heads/kinds as well as the accessor functions in the source code handling +section. + +### Relocatable syntax trees + +[`GreenNode`](@ref) is a special low level syntax tree: it's "relocatable" in +the sense that it doesn't carry an absolute position in the source code or even +a reference to the source text. This allows it to be reused for incremental +parsing, but does make it a pain to work with directly! + +```@docs +JuliaSyntax.GreenNode +``` + +Green nodes only have a relative position so implement `span()` instead of +`byte_range()`: + +```@docs +JuliaSyntax.span +``` diff --git a/JuliaSyntax/docs/src/design.md b/JuliaSyntax/docs/src/design.md new file mode 100644 index 0000000000000..a11d1b64140ed --- /dev/null +++ b/JuliaSyntax/docs/src/design.md @@ -0,0 +1,850 @@ +# Design discussion and developer documentation + +## Goals + +* Lossless parsing of Julia code with precise source mapping +* Production quality error recovery, reporting and unit testing +* Parser structure similar to Julia's flisp-based parser +* Speedy enough for interactive editing +* "Compilation as an API" to support all sorts of tooling +* Grow to encompass the rest of the compiler frontend: macro expansion, + desugaring and other lowering steps. +* Replace Julia's flisp-based reference frontend + +## Design Opinions + +* Parser implementation should be independent from tree data structures. So + we have the `ParseStream` interface. +* Tree data structures should be *layered* to balance losslessness with + abstraction and generality. So we have `SyntaxNode` (an AST) layered on top + of `GreenNode` (a lossless parse tree). We might need other tree types later. +* Fancy parser generators still seem marginal for production compilers. We use + a boring but flexible recursive descent parser. + +# Parser implementation + +Our goal is to losslessly represent the source text with a tree; this may be +called a "lossless syntax tree". (This is sometimes called a "concrete syntax +tree", but that term has also been used for the parse tree of the full formal +grammar for a language including any grammar hacks required to solve +ambiguities, etc. So we avoid this term.) + +`JuliaSyntax` uses a mostly recursive descent parser which closely +follows the high level structure of the flisp reference parser. This makes the +code familiar and reduces porting bugs. It also gives a lot of flexibility for +designing the diagnostics, tree data structures, compatibility with different +Julia versions, etc. I didn't choose a parser generator as they still seem +marginal for production compilers — for the parsing itself they don't seem +*greatly* more expressive and they can be less flexible for the important +"auxiliary" code which needs to be written in either case. + +### Lexing + +We use a hand-written lexer (a heavily modified version of +[Tokenize.jl](https://github.com/JuliaLang/Tokenize.jl)) +* Newline-containing whitespace is emitted as a separate kind +* Tokens inside string interpolations are emitted separately from the string +* Strings delimiters are separate tokens and the actual string always has the + `String` kind +* Additional contextual keywords (`as`, `var`, `doc`) have been added and + moved to a subcategory of keywords. +* Nonterminal kinds were added (though these should probably be factored out again) +* Various bugs fixed and additions for newer Julia versions + +### Parsing with ParseStream + +The main parser innovation is the `ParseStream` interface which provides a +stream-like I/O interface for writing the parser. The parser does not +depend on or produce any concrete tree data structure as part of the parsing +phase but the output nodes can be post-processed into various tree data +structures as required. This is like the design of rust-analyzer though with a +simpler implementation. + +Parsing proceeds by recursive descent; + +* The parser consumes a flat list of lexed tokens as *input* using `peek()` to + examine tokens and `bump()` to consume them. +* The parser produces a flat list of `RawGreenNode`s as *output* using `bump()` to + transfer tokens to the output and `position()`/`emit()` for nonterminal nodes. +* Diagnostics are emitted as separate text spans +* Whitespace and comments are automatically `bump()`ed and don't need to be + handled explicitly. The exception is syntactically relevant newlines in space + sensitive mode. +* Parser modes are passed down the call tree using `ParseState`. + +The output nodes track the byte range, a syntax "kind" stored as an integer +tag, and some flags. Each node also stores either the number of child nodes +(for non-terminals) or the original token kind (for terminals). The kind tag +makes the nodes a [sum type](https://blog.waleedkhan.name/union-vs-sum-types/) +but where the type is tracked explicitly outside of Julia's type system. + +For lossless parsing the output nodes must cover the entire input text. Using +`bump()`, `position()` and `emit()` in a natural way also ensures that: +* Nodes are cleanly nested with children contained entirely within their parents +* Sibling nodes are emitted in source order +* Parent nodes are emitted after all their children. + +These properties make the output nodes a post-order traversal of a +["green tree"](#raw-syntax-tree--green-tree) +in the terminology of C#'s Roslyn compiler, with the tree structure +implicit in the node spans. + +### Tree construction + +The `build_tree` function uses the implicit tree structure in the `ParseStream` +output to assemble concrete tree data structures. Since the output is already +a post-order traversal of `RawGreenNode`s with node spans encoding parent-child +relationships, tree construction is straightforward. We build on top of this to +define `build_tree` for various tree types including `GreenNode`, the AST type +`SyntaxNode`, and for normal Julia `Expr`. + +### Error recovery + +The goal of the parser is to produce well-formed hierarchical structure from +the source text. For interactive tools we need this to work even when the +source text contains errors; it's the job of the parser to include the recovery +heuristics to make this work. + +Concretely, the parser in `JuliaSyntax` should always produce a green tree +which is *well formed* in the sense that `GreenNode`s of a given `Kind` have +well-defined layout of children. This means the `GreenNode` to `SyntaxNode` +transformation is deterministic and tools can assume they're working with a +"mostly valid" AST. + +What does "mostly valid" mean? We allow the tree to contain the following types +of error nodes: + +* Missing tokens or nodes may be **added** as placeholders when they're needed + to complete a piece of syntax. For example, we could parse `a + (b *` as + `(call-i a + (call-i * b XXX))` where `XXX` is a placeholder error node. +* A sequence of unexpected tokens may be **removed** by collecting + them as children of an error node and treating them as syntax trivia during + AST construction. For example, `a + b end * c` could be parsed as the green + tree `(call-i a + b (error-t end * c))`, and turned into the AST `(call + a b)`. + +We want to encode both these cases in a way which is simplest for downstream +tools to use. This is an open question, but for now we use `K"error"` as the +kind, with the `TRIVIA_FLAG` set for unexpected syntax. + +# Syntax trees + +Julia's `Expr` abstract syntax tree can't store precise source locations or +deal with syntax trivia like whitespace or comments. So we need some new tree +types in `JuliaSyntax`. + +JuliaSyntax currently deals in three types of trees: +* `GreenNode` is a minimal *lossless syntax tree* where + - Nodes store a kind and length in bytes, but no text + - Syntax trivia are included in the list of children + - Children are strictly in source order +* `SyntaxNode` is an *abstract syntax tree* which has + - An absolute position and pointer to the source text + - Children strictly in source order + - Leaf nodes store values, not text + - Trivia are ignored, but there is a 1:1 mapping of non-trivia nodes to the + associated `GreenTree` nodes. +* `Expr` is used as a conversion target for compatibility + +## More about syntax kinds + +We generally track the type of syntax nodes with a syntax "kind", stored +explicitly in each node an integer tag. This effectively makes the node type a +[sum type](https://blog.waleedkhan.name/union-vs-sum-types/) in the type system +sense, but with the type tracked explicitly outside of Julia's type system. + +Managing the type explicitly brings a few benefits: +* Code and data structures for manipulating syntax nodes is always concretely + typed from the point of view of the compiler. +* We control the data layout and can pack the kind into very few bits along + with other flags bits, as desired. +* Predicates such as `is_operator` can be extremely efficient, given that we + know the meaning of the kind's bits. +* The kind can be applied to several different tree data structures, or + manipulated by itself. +* Pattern matching code is efficient when the full set of kinds is closed and + known during compilation. + +There's arguably a few downsides: +* Normal Julia dispatch can't express dispatch over syntax kind. Luckily, + a pattern matching macro can provide a very elegant way of expressing such + algorithms over a non-extensible set of kinds, so this is not a big problem. +* Different node kinds could come with different data fields, but a syntax + tree must have generic fields to cater for all kinds. (Consider as an analogy + the normal Julia AST `QuoteNode` with a single field vs `Expr` with generic + `head` and `args` fields.) This could be a disadvantage for code which + processes one specific kind but for generic code processing many kinds + having a generic but *concrete* data layout should be faster. + +# Differences from the flisp parser + +_See also the [§ Comparisons to other packages](#comparisons-to-other-packages) section._ + +Practically the flisp parser is not quite a classic [recursive descent +parser](https://en.wikipedia.org/wiki/Recursive_descent_parser), because it +often looks back and modifies the output tree it has already produced. We've +tried to eliminate this pattern in favor of lookahead where possible because + +* It works poorly when the parser is emitting a stream of node spans with + strict source ordering constraints. +* It's confusing to reason about this kind of code + +However, on occasion it seems to solve genuine ambiguities where Julia code +can't be parsed top-down with finite lookahead. Eg for the `kw` vs `=` +ambiguity within parentheses. In these cases we put up with using the +functions `look_behind` and `reset_node!()`. + +## Code structure + +Large structural changes were generally avoided while porting. In particular, +nearly all function names for parsing productions are the same with `-` +replaced by `_` and predicates prefixed by `is_`. + +Some notable differences: + +* `parse-arglist` and a parts of `parse-paren-` have been combined into a + general function `parse_brackets`. This function deals with all the odd + corner cases of how the AST is emitted when mixing `,` and `;` within + parentheses. In particular regard to: + - Determining whether `;` are block syntax separators or keyword parameters + - Determining whether to emit `parameter` sections based on context + - Emitting key-value pairs either as `kw` or `=` depending on context +* The way that `parse-resword` is entered has been rearranged to avoid parsing + reserved words with `parse-atom` inside `parse-unary-prefix`. Instead, we + detect reserved words and enter `parse_resword` earlier. + +## Flisp parser bugs + +Here's some behaviors which seem to be bugs. (Some of these we replicate in the +name of compatibility, perhaps with a warning.) + +* Macro module paths allow calls which gives weird stateful semantics! + ```julia + b() = rand() > 0.5 ? Base : Core + b().@info "hi" + ``` +* Misplaced `@` in macro module paths like `A.@B.x` is parsed as odd + broken-looking AST like `(macrocall (. A (quote (. B @x))))`. It should + probably be rejected. +* Operator prefix call syntax doesn't work in the cases like `+(a;b,c)` where + keyword parameters are separated by commas. A tuple is produced instead. +* `const` and `global` allow chained assignment, but the right hand side is not + constant. `a` const here but not `b`. + ```julia + const a = b = 1 + ``` +* Parsing the `ncat` array concatenation syntax within braces gives + strange AST: `{a ;; b}` parses to `(bracescat 2 a b)` which is the same as + `{2 ; a ; b}`, but should probably be `(bracescat (nrow 2 a b))` in analogy + to how `{a b}` produces `(bracescat (row a b))`. +* `export a, \n $b` is rejected, but `export a, \n b` parses fine. +* In try-catch-finally, the `finally` clause is allowed before the `catch`, but + always executes afterward. (Presumably was this a mistake? It seems pretty awful!) +* When parsing `"[x \n\n ]"` the flisp parser gets confused, but `"[x \n ]"` is + correctly parsed as `Expr(:vect)` (maybe fixed in 1.7?) +* `f(x for x in in xs)` is accepted, and parsed very strangely. +* Octal escape sequences saturate rather than being reported as errors. Eg, + `"\777"` results in `"\xff"`. This is inconsistent with + `Base.parse(::Type{Int}, ...)` +* Leading dots in import paths with operator-named modules are parsed into + dotted operators rather than a relative path. Ie, we have `import .⋆` parsing + to `(import (. .⋆))` whereas it should be `(import (. . ⋆))` for consistency + with the parsing of `import .A`. +* Looking back on the output disregards grouping parentheses which can lead to + odd results in some cases. For example, `f(((((x=1)))))` parses as a keyword + call to function `f` with the keyword `x=1`, but arguably it should be an + assignment. +* Hexfloat literals can have a trailing `f` for example, `0x1p1f` + but this doesn't do anything. In the `flisp` C code such cases are treated as + Float32 literals and this was intentional https://github.com/JuliaLang/julia/pull/2925 + but this has never been officially supported in Julia. It seems this bug + arises from `(set! pred char-hex?)` in `parse-number` accepting hex exponent + digits, all of which are detected as invalid except for a trailing `f` when + processed by `isnumtok_base`. +* `begin` and `end` are not parsed as keywords when indexing. Typed comprehensions + initially look the same, but can be distinguished from indexing once we handle + a `for` token; it is safe to treat `begin` and `end` as keywords afterwards. The + reference parser *only* handles this well when there's a newline before `for`: + ```julia + Any[foo(i) + for i in x if begin + true + end + ] + ``` + works, while + ```julia + Any[foo(i) for i in x if begin + true + end + ] + ``` + does not. JuliaSyntax handles both cases. + +## Parsing / AST oddities and warts + +### Questionable allowed forms + +There's various allowed syntaxes which are fairly easily detected in the +parser, but which will be rejected later during lowering. To allow building +DSLs this is fine and good but some such allowed syntaxes don't seem very +useful, even for DSLs: + +* `macro (x) end` is allowed but there are no anonymous macros. +* `abstract type A < B end` and other subtype comparisons are allowed, but + only `A <: B` makes sense. +* `x where {S T}` produces `(where x (bracescat (row S T)))`. This seems pretty weird! +* `[x for outer x in xs]` parses, but `outer` makes no real sense in this + context (and using this form is a lowering error) + +### `kw` and `=` inconsistencies + +There's many apparent inconsistencies between how `kw` and `=` are used when +parsing `key=val` pairs inside parentheses. + +* Inconsistent parsing of tuple keyword args inside vs outside of dot calls + ```julia + (a=1,) # (tuple (= a 1)) + f.(a=1) # (tuple (kw a 1)) + ``` +* Mixtures of `,` and `;` in calls give nested parameter AST which parses + strangely, and is kind-of-horrible to use. + ```julia + # (tuple (parameters (parameters e f) c d) a b) + (a,b; c,d; e,f) + ``` +* Long-form anonymous functions have argument lists which are parsed + as tuples (or blocks!) rather than argument lists and this mess appears to be + papered over as part of lowering. For example, in `function (a;b) end` the + `(a;b)` is parsed as a block! This leads to more inconsistency in the use of + `kw` for keywords. + + +### Other oddities + +* Operators with suffixes don't seem to always be parsed consistently as the + same operator without a suffix. Unclear whether this is by design or mistake. + For example, `[x +y] ==> (hcat x (+ y))`, but `[x +₁y] ==> (hcat (call +₁ x y))` + +* `global const x=1` is normalized by the parser into `(const (global (= x 1)))`. + I suppose this is somewhat useful for AST consumers, but reversing the source + order is pretty weird and inconvenient when moving to a lossless parser. + +* `let` bindings might be stored in a block, or they might not be, depending on + special cases: + ```julia + # Special cases not in a block + let x=1 ; end # ==> (let (= x 1) (block)) + let x::1 ; end # ==> (let (:: x 1) (block)) + let x ; end # ==> (let x (block)) + + # In a block + let x=1,y=2 ; end # ==> (let (block (= x 1) (= y 2) (block))) + let x+=1 ; end # ==> (let (block (+= x 1)) (block)) + ``` + +* The `elseif` condition is always in a block but not the `if` condition. + Presumably because of the need to add a line number node in the flisp parser + `if a xx elseif b yy end ==> (if a (block xx) (elseif (block b) (block yy)))` + +* Spaces are allowed between import dots — `import . .A` is allowed, and + parsed the same as `import ..A` + +* `import A..` produces `(import (. A .))` which is arguably nonsensical, as `.` + can't be a normal identifier. + +* The raw string escaping rules are *super* confusing for backslashes near + the end of the string: `raw"\\\\ "` contains four backslashes, whereas + `raw"\\\\"` contains only two. However this was an intentional feature to + allow all strings to be represented and it's unclear whether the situation + can be improved. + +* In braces after macrocall, `@S{a b}` is invalid but both `@S{a,b}` and + `@S {a b}` parse. Conversely, `@S[a b]` parses. + +* Macro names and invocations are post-processed from the output of + `parse-atom` / `parse-call`, which leads to some surprising and questionable + constructs which "work": + - Absurdities like `@(((((a))))) x ==> (macrocall @a x)` + - Infix macros!? `@(x + y) ==> (macrocall @+ x y)` (ok, kinda cute and has + some weird logic to it... but what?) + - Similarly additional parentheses are allowed `@(f(x)) ==> (macrocall @f x)` + +* Allowing `@` first in macro module paths (eg `@A.B.x` instead of `A.B.@x`) + seems like unnecessary variation in syntax. It makes parsing valid macro + module paths more complex and leads to oddities like `@$.x y ==> (macrocall + ($ (quote x)) y` where the `$` is first parsed as a macro name, but turns out + to be the module name after the `.` is parsed. But `$` can never be a valid + module name in normal Julia code so this makes no sense. + +* Triple quoted `var"""##"""` identifiers are allowed. But it's not clear these + are required or desired given that they come with the complex triple-quoted + string deindentation rules. + +* Deindentation of triple quoted strings with mismatched whitespace is weird + when there's nothing but whitespace. For example, we have + `"\"\"\"\n \n \n \"\"\"" ==> "\n \n"` so the middle line of whitespace + here isn't dedented but the other two longer lines are?? Here it seems more + consistent that either (a) the middle line should be deindented completely, + or (b) all lines should be dedented only one character, as that's the + matching prefix. + +* Parsing of anonymous function arguments is somewhat inconsistent. + `function (xs...) \n body end` parses the argument list as `(... xs)`, whereas + `function (x) \n body end` parses the argument list as `(tuple x)`. + +* The difference between multidimensional vs flattened iterators is subtle, and + perhaps too syntactically permissive. For example, + - `[(x,y) for x * in 1:10, y in 1:10]` is a multidimensional iterator + - `[(x,y) for x * in 1:10 for y in 1:10]` is a flattened iterator + - `[(x,y) for x in 1:10, y in 1:10 if y < x]` is a flattened iterator + + It's this last case which seems problematic (why not *require* the second + form as a more explicit way to indicate flattening?). It's not even pretty + printed correctly: + ```julia-repl + julia> :([(x,y) for x in 1:10, y in 1:10 if y < x]) + :([(x, y) for $(Expr(:filter, :(y < x), :(x = 1:10), :(y = 1:10)))]) + ``` + +* The character `'` may be written without escaping as `'''` rather than + requiring the form `'\''`. + +# Comparisons to other packages + +### Official Julia compiler + +_See also the [§ Differences from the flisp parser](#differences-from-the-flisp-parser) section._ + +The official Julia compiler frontend lives in the Julia source tree. It's +mostly contained in just a few files: +* The parser in [src/julia-parser.scm](https://github.com/JuliaLang/julia/blob/9c4b75d7f63d01d12b67aaf7ce8bb4a078825b52/src/julia-parser.scm) +* Macro expansion in [src/ast.c](https://github.com/JuliaLang/julia/blob/9c4b75d7f63d01d12b67aaf7ce8bb4a078825b52/src/ast.c) and [src/macroexpand.scm](https://github.com/JuliaLang/julia/blob/9c4b75d7f63d01d12b67aaf7ce8bb4a078825b52/src/macroexpand.scm) +* Syntax lowering in [src/julia-syntax.scm](https://github.com/JuliaLang/julia/blob/9c4b75d7f63d01d12b67aaf7ce8bb4a078825b52/src/julia-syntax.scm) +* The flisp runtime and C extensions for Julia in [src/flisp](https://github.com/JuliaLang/julia/tree/master/src/flisp) +* Supporting utility functions in a few other `.scm` and `.c` files. + +There's two issues with the official reference frontend which suggest a rewrite. + +First, there's no support for precise source locations and the existing data +structures (bare flisp lists) can't easily be extended to add these. Fixing +this would require changes to nearly all of the code. + +Second, it's written in flisp: an aesthetically pleasing, minimal but obscure +implementation of Scheme. Learning Scheme is actually a good way to appreciate +some of Julia's design inspiration, but it's quite a barrier for developers of +Julia language tooling. (Flisp has no user-level documentation but non-schemers +can refer to the [Racket documentation](https://docs.racket-lang.org) which is +quite compatible for basic things.) In addition to the social factors, having +the embedded flisp interpreter and runtime with its own separate data +structures and FFI is complex and inefficient. + +### JuliaParser.jl + +[JuliaParser.jl](https://github.com/JuliaLang/JuliaParser.jl) +was a direct port of Julia's flisp reference parser, but was abandoned around +Julia 0.5 or so. Furthermore, it doesn't support lossless parsing, and adding +that feature would amount to a full rewrite. Given its divergence with the flisp +reference parser since Julia-0.5, it seemed better just to start anew from the +reference parser instead. + +### Tokenize.jl + +[Tokenize.jl](https://github.com/JuliaLang/Tokenize.jl) +is a fast lexer for Julia code. The code from Tokenize has been +imported and used in JuliaSyntax, with some major modifications as discussed in +the [lexer implementation](#lexing) section. + +### CSTParser.jl + +[CSTParser.jl](https://github.com/julia-vscode/CSTParser.jl) +is a ([mostly?](https://github.com/domluna/JuliaFormatter.jl/issues/52#issuecomment-529945126)) +lossless parser with goals quite similar to JuliaParser. It is used extensively +in the VSCode / LanguageServer / JuliaFormatter ecosystem. CSTParser is very +useful, but I do find the implementation hard to understand, and I wanted to try +a fresh approach with a focus on: + +* "Production readiness": Good docs, tests, diagnostics and maximum similarity + with the flisp parser, with the goal of getting the new parser into `Core`. +* Learning from the latest ideas about composable parsing and data structures + from outside Julia. In particular the implementation of `rust-analyzer` is + very clean, well documented, and was a great source of inspiration. +* Composability of tree data structures — I feel like the trees should be + layered somehow with a really lightweight [green tree](#raw-syntax-tree--green-tree) + at the most basic level, similar to Roslyn or rust-analyzer. In comparison, + CSTParser uses a more heavyweight non-layered data structure. Alternatively or + additionally, have a common tree API with many concrete task-specific + implementations. + +A big benefit of the JuliaSyntax parser is that it separates the parser code +from the tree data structures entirely, which should give a lot of flexibility +in experimenting with various tree representations. + +I also want JuliaSyntax to tackle macro expansion and other lowering steps, and +provide APIs for this which can be used by both the core language and the +editor tooling. + +### tree-sitter-julia + +Using a modern production-ready parser generator like `tree-sitter` is an +interesting option and some progress has already been made in +[tree-sitter-julia](https://github.com/tree-sitter/tree-sitter-julia). +But I feel like the grammars for parser generators are only marginally more +expressive than writing the parser by hand, after accounting for the effort +spent on the weird edge cases of a real language and writing the parser's tests +and "supporting code". + +On the other hand, a hand-written parser is completely flexible and can be +mutually understood with the reference implementation, so I chose that approach +for JuliaSyntax. + +# Resources + +## Julia issues + +Here's a few links to relevant Julia issues. + +#### Macro expansion + +* Automatic hygiene for macros https://github.com/JuliaLang/julia/pull/6910 — + would be interesting to implement this in a new frontend. + +#### Lowering + +* A partial implementation of lowering in Julia https://github.com/JuliaLang/julia/pull/32201 — + some of this should be ported. (Last commit at https://github.com/JuliaLang/julia/tree/df61138fcf97d03dcbbba10e962571af9700db56/ ) +* The closure capture problem https://github.com/JuliaLang/julia/issues/15276 — + would be interesting to see whether we can tackle some of the harder cases in + a new implementation. + +## C# Roslyn + +[Persistence, façades and Roslyn’s red-green trees](https://ericlippert.com/2012/06/08/red-green-trees/) +* [Roslyn optimization overview](https://github.com/KirillOsenkov/Bliki/wiki/Roslyn-Immutable-Trees) +* [Literate C# Usage Example](https://johtela.github.io/LiterateCS/LiterateCS/BlockBuilder.html) + + +## Rust-analyzer + +`rust-analyzer` seems to be very close to what I'm building here, and has come +to the same conclusions on green tree layout with explicit trivia nodes. Their +document on internals +[here](https://github.com/rust-analyzer/rust-analyzer/blob/master/docs/dev/syntax.md) +is great. Points of note: + +* They have *three* trees! + 1. Green trees exactly like mine (pretty much all the same design + decisions, including trivia storage). Though note that the team are still + [toying with](https://github.com/rust-analyzer/rust-analyzer/issues/6584) + the idea of using the Roslyn model of trivia. + 2. Untyped red syntax trees somewhat like mine, but much more minimal. For + example, these don't attempt to reorder children. + 3. A typed AST layer with a type for each expression head. The AST searches + for children by dynamically traversing the child list each time, rather + than having a single canonical ordering or remembering the placement of + children which the parser knew. +* "Parser does not see whitespace nodes. Instead, they are attached to the + tree in the TreeSink layer." This may be relevant to us - it's a pain to + attach whitespace to otherwise significant tokens, and inefficient to + allocate and pass around a dynamic list of whitespace trivia. +* "In practice, incremental reparsing doesn't actually matter much for IDE + use-cases, parsing from scratch seems to be fast enough." (I wonder why + they've implemented incremental parsing then?) +* There's various comments about macros... Rust macro expansion seems quite + different from Julia (it appears it may be interleaved with parsing??) + +In general I think it's unclear whether we want typed ASTs in Julia and we +particularly need to deal with the fact that `Expr` is the existing public +interface. Could we have `Expr2` wrap `SyntaxNode`? + +* A related very useful set of blog posts which discuss using the rust syntax + tree library (rowan) for representing of a non-rust toy language is here + https://dev.to/cad97/lossless-syntax-trees-280c + +Not all the design decisions in `rust-analyzer` are finalized but the +[architecture document](https://github.com/rust-analyzer/rust-analyzer/blob/master/docs/dev/architecture.md) +is a fantastic source of design inspiration. + +Highlights: +* "The parser is independent of the particular tree structure and particular + representation of the tokens. It transforms one flat stream of events into + another flat stream of events." This seems great, let's adopt it! +* TODO + +## RSLint + +[RSLint](https://rslint.org/dev) is a linter for javascript, built in Rust. It +uses the same parsing infrastructure and green tree libraries `rust-analyzer`. +There's an excellent and friendly high level overview of how all this works in +the rslint [parsing devdocs](https://rslint.org/dev/parsing.html). + +Points of note: + +* Backtracking and restarting the parser on error is actually quite simple in + the architecture we (mostly) share with `rust-analyzer`: + > ... events allow us to cheaply backtrack the parser by simply draining + > the events and resetting the token source cursor back to some place. + +* The section on [error + recovery](https://rslint.org/dev/parsing.html#error-recovery) is interesting; + they talk about various error recovery strategies. + +## Diagnostics + +The paper [P2429 - Concepts Error Messages for +Humans](https://wg21.tartanllama.xyz/P2429%20-%20Concepts%20Error%20Messages%20for%20Humans.pdf) +is C++ centric, but has a nice review of quality error reporting in various +compilers including Elm, ReasonML, Flow, D and Rust. + +Some Rust-specific resources: +* [rustc_errors::Diagnostic](https://doc.rust-lang.org/stable/nightly-rustc/rustc_errors/struct.Diagnostic.html) +* The source of the Rust compiler's diagnostics system: + - The [`println!` macro](https://github.com/rust-lang/rust/blob/0b6f079e4987ded15c13a15b734e7cfb8176839f/compiler/rustc_builtin_macros/src/format.rs) + shows how these can be emitted from macros + - The parser's [diagnostics.rs](https://github.com/rust-lang/rust/blob/0b6f079e4987ded15c13a15b734e7cfb8176839f/compiler/rustc_parse/src/parser/diagnostics.rs) + +## General resources about parsing + +* [Modern parser generator](https://matklad.github.io/2018/06/06/modern-parser-generator.html) + has a lot of practical notes on writing parsers. Highlights: + - Encourages writing tests for handwritten parsers as inline comments + - Mentions Pratt parsers for simple operator precedence parsing. Good articles: + - [From Aleksey Kladov (matklad - the main rust-analyzer author, etc)](https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html) + - [From Bob Nystrom (munificent - one of the Dart devs, etc](http://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/) + - Some discussion of error recovery + +* Some notes about stateful lexers for parsing shell-like string interpolations: + http://www.oilshell.org/blog/2017/12/17.html + + +# Design notes + +The following are some fairly disorganized design notes covering a mixture of +things which have already been done and musings about further work. + +## Prototyping approach + +The tree datastructure design here is tricky: + +1. The symbolic part of compilation (the compiler frontend) incrementally + abstracts and transforms the source text, but errors along the way should + refer back to the source. + - The tree must be a lossless representation of the source text + - Some aspects of the source text (comments, most whitespace) are irrelevant + to parsing. + - More aspects of the source text are irrelevant after we have an abstract + syntax tree of the surface syntax. Some good examples here are the + parentheses in `2*(x + y)` and the explicit vs implicit multiplication + symbol in `2*x` vs `2x`. + +2. There's various type of *analyses* +- There's many useful ways to augment a syntax tree depending on use case. +- Analysis algorithms should be able to act on any tree type, ignoring + but carrying augmentations which they don't know about. + +Having so many use cases suggests it might be best to have several different +tree types with a common interface rather than one main abstract syntax tree +type. But it seems useful to figure this out by prototyping several important +work flows: + +* Syntax transformations + - Choose some macros to implement. This is a basic test of mixing source + trees from different files while preserving precise source locations. + (Done in .) +* Formatting + - Re-indent a file. This tests the handling of syntax trivia. +* Refactoring + - A pass to rename local variables. This tests how information from further + down the compilation pipeline can be attached to the syntax tree and used + to modify the source code. +* Precise error reporting in lowering + - Syntax desugaring `[a, b] = (c, d)` should report "invalid assignment + location `[a, b]`". But at a precise source location. + - Try something several layers deeper inside lowering? For example "macro + definition not allowed inside a local scope" +* Incremental reparsing + - Reparse a source file, given a byte range replacement + + +## Tree design + +### Raw syntax tree / Green tree + +Raw syntax tree (or ["Green tree"](https://ericlippert.com/2012/06/08/red-green-trees/) +in the terminology from Roslyn) + +We want GreenNode to be +* *structurally minimal* — For efficiency and generality +* *immutable* — For efficiency (& thread safety) +* *complete* — To preserve parser knowledge +* *token agnostic* — To allow use with any source language + +The simplest idea possible is to have: +* Leaf nodes are a single token +* Children are in source order + +Call represents a challenge for the AST vs Green tree in terms of node +placement / iteration for infix operators vs normal prefix function calls. + +- The normal problem of `a + 1` vs `+(a, 1)` +- Or worse, `a + 1 + 2` vs `+(a, 1, 2)` + +Clearly in the AST's *interface* we need to abstract over this placement. For +example with something like the normal Julia AST's iteration order. + +### Abstract syntax tree + +By pointing to green tree nodes, AST nodes become traceable back to the original +source. + +Unlike most languages, designing a new AST is tricky because the existing +`Expr` is a very public API used in every macro expansion. User-defined +macro expansions interpose between the source text and lowering, and using +`Expr` looses source information in many ways. + +There seems to be a few ways forward: +* Maybe we can give `Expr` some new semi-hidden fields to point back to the + green tree nodes that the `Expr` or its `args` list came from? +* We can use the existing `Expr` during macro expansion and try to recover + source information after macro expansion using heuristics. Likely the + presence of correct hygiene can help with this. +* Introducing a new AST would be possible if it were opt-in for some + hypothetical "new-style macros" only. Fixing hygiene should go along with + this. Design challenge: How do we make manipulating expressions reasonable + when literals need to carry source location? + +One option which may help bridge between locationless ASTs and something new +may be to have wrappers for the small number of literal types we need to cover. +For example: + +```julia +SourceSymbol <: AbstractSymbol +SourceInt <: Integer +SourceString <: AbstractString +``` + +Having source location attached to symbols would potentially solve most of the +hygiene problem. There's still the problem of macro helper functions which use +symbol literals; we can't very well be changing the meaning of `:x`! Perhaps +the trick there is to try capturing the current module at the location of the +interpolation syntax. Eg, if you do `:(y + $x)`, lowering expands this to +`Core._expr(:call, :+, :y, x)`, but it could expand it to something like +`Core._expr(:call, :+, :y, _add_source_symbol(_module_we_are_lowering_into, x))`? + +## Parsing + +### Error recovery + +Some disorganized musings about error recovery + +Different types of errors seem to occur... + +* Disallowed syntax (such as lack of spaces in conditional expressions) + where we can reasonably just continue parsing and emit the node with an error + flag which is otherwise fully formed. In some cases like parsing infix + expressions with a missing tail, emitting a zero width error token can lead + to a fully formed parse tree without the productions up the stack needing to + participate in recovery. +* A token which is disallowed in current context. Eg, `=` in parse_atom, or a + closing token inside an infix expression. Here we can emit a `K"error"`, but + we can't descend further into the parse tree; we must pop several recursive + frames off. Seems tricky! + +A typical structure is as follows: + +```julia +function parse_foo(ps) + mark = position(ps) + parse_bar(ps) # What if this fails? + if peek(ps) == K"some-token" + bump(ps) + parse_baz(ps) # What if this fails? + emit(ps, mark, K"foo") + end +end +``` + +Emitting plain error tokens are good in unfinished infix expressions: + +```julia +begin + a = x + +end +``` + +The "missing end" problem is tricky, as the intermediate syntax is valid; the +problem is often only obvious until we get to EOF. + +Missing end +```julia +function f() + begin + a = 10 +end + +# <-- Indentation would be wrong if g() was an inner function of f. +function g() +end +``` + +It seems like ideal error recovery would need to backtrack in this case. For +example: + +- Pop back to the frame which was parsing `f()` +- Backtrack through the parse events until we find a function with indentation + mismatched to the nesting of the parent. +- Reset ParseStream to a parsing checkpoint before `g()` was called +- Emit error and exit the function parsing `f()` +- Restart parsing +- Somehow make sure all of this can't result in infinite recursion 😅 + +Missing commas or closing brackets in nested structures also present the +existing parser with a problem. + +```julia +f(a, + g(b, + c # -- missing comma? + d), + e) +``` + +Again the local indentation might tell a story + +```julia +f(a, + g(b, + c # -- missing closing `)` ? + d) +``` + +But not always! + +```julia +f(a, + g(b, + c # -- missing closing `,` ? + d)) +``` + +Another particularly difficult problem for diagnostics in the current system is +broken parentheses or double quotes in string interpolations, especially when +nested. + +# Fun research questions + +### Parser Recovery + +Can we learn fast and reasonably accurate recovery heuristics for when the +parser encounters broken syntax, rather than hand-coding these? How would we +set the parser up so that training works and injecting the model is +nonintrusive? If the model is embedded in and works together with the parser, +can it be made compact enough that training is fast and the model itself is +tiny? + +### Formatting + +Given source and syntax tree, can we regress/learn a generative model of +indentation from the syntax tree? Source formatting involves a big pile of +heuristics to get something which "looks nice"... and ML systems have become +very good at heuristics. Also, we've got huge piles of training data — just +choose some high quality, tastefully hand-formatted libraries. diff --git a/JuliaSyntax/docs/src/howto.md b/JuliaSyntax/docs/src/howto.md new file mode 100644 index 0000000000000..c8bd0503591d5 --- /dev/null +++ b/JuliaSyntax/docs/src/howto.md @@ -0,0 +1,37 @@ +# How-To + +This section contains brief recipes for particular tasks + +## Use JuliaSyntax as the default parser + +To use JuliaSyntax as the default Julia parser for the REPL and to `include()` +files, parse code with `Meta.parse()`, etc, put the following in your +startup.jl file: + +```julia +using JuliaSyntax +JuliaSyntax.enable_in_core!() +``` + +This works well in Julia 1.9 but in Julia 1.8 will cause some startup latency. +To reduce that you can create a custom system image by running the code in +`./sysimage/compile.jl` as a Julia script (or directly using the shell, on +unix). Then use `julia -J $resulting_sysimage`. + +Using a custom sysimage has the advantage that package precompilation will also +go through the JuliaSyntax parser. + +### VSCode + +To use JuliaSyntax as the default parser for Julia within VSCode, add the +following to your `startup.jl` file: + +```julia +import JuliaSyntax +JuliaSyntax.enable_in_core!() +``` + +To reduce startup latency you can combine with a custom system as described in +the [Julia VScode docs](https://www.julia-vscode.org/docs/dev/userguide/compilesysimage/#Creating-a-sysimage-for-the-active-environment), +combined with the precompile execution file in `sysimage/precompile_exec.jl` in the source tree. +For additional detail see the discussion in [issue #128](https://github.com/JuliaLang/JuliaSyntax.jl/issues/128). diff --git a/JuliaSyntax/docs/src/index.md b/JuliaSyntax/docs/src/index.md new file mode 100644 index 0000000000000..79b8d83b62e6a --- /dev/null +++ b/JuliaSyntax/docs/src/index.md @@ -0,0 +1,79 @@ +# JuliaSyntax.jl + +A Julia compiler frontend, written in Julia. + +A [talk from JuliaCon 2022](https://youtu.be/CIiGng9Brrk) covered some aspects +of this package. + +## Examples + +Here's what parsing of a small piece of code currently looks like in various +forms. We'll use the `JuliaSyntax.parsestmt` function to demonstrate, there's also +`JuliaSyntax.parse!` offering more fine-grained control. + +First, a source-ordered AST with `SyntaxNode` (`call-i` in the dump here means +the `call` has the infix `-i` flag): + +```julia +julia> using JuliaSyntax + +julia> parsestmt(SyntaxNode, "(x + y)*z", filename="foo.jl") +line:col│ tree │ file_name + 1:1 │[call-i] │foo.jl + 1:1 │ [parens] + 1:2 │ [call-i] + 1:2 │ x + 1:4 │ + + 1:6 │ y + 1:8 │ * + 1:9 │ z +``` + +Internally this has a full representation of all syntax trivia (whitespace and +comments) as can be seen with the more raw ["green tree"](#raw-syntax-tree--green-tree) +representation with `GreenNode`. Here ranges on the left are byte ranges, and +`✔` flags nontrivia tokens. Note that the parentheses are trivia in the tree +representation, despite being important for parsing. + +```julia +julia> text = "(x + y)*z" + greentree = parsestmt(JuliaSyntax.GreenNode, text) + 1:9 │[call] + 1:7 │ [parens] + 1:1 │ ( + 2:6 │ [call] + 2:2 │ Identifier ✔ + 3:3 │ Whitespace + 4:4 │ + ✔ + 5:5 │ Whitespace + 6:6 │ Identifier ✔ + 7:7 │ ) + 8:8 │ * ✔ + 9:9 │ Identifier ✔ +``` + +`GreenNode` stores only byte ranges, but the token strings can be shown by +supplying the source text string: + +```julia +julia> show(stdout, MIME"text/plain"(), greentree, text) + 1:9 │[call] + 1:7 │ [parens] + 1:1 │ ( "(" + 2:6 │ [call] + 2:2 │ Identifier ✔ "x" + 3:3 │ Whitespace " " + 4:4 │ + ✔ "+" + 5:5 │ Whitespace " " + 6:6 │ Identifier ✔ "y" + 7:7 │ ) ")" + 8:8 │ * ✔ "*" + 9:9 │ Identifier ✔ "z" +``` + +Julia `Expr` can also be produced: + +```julia +julia> JuliaSyntax.parsestmt(Expr, "(x + y)*z") +:((x + y) * z) +``` diff --git a/JuliaSyntax/docs/src/reference.md b/JuliaSyntax/docs/src/reference.md new file mode 100644 index 0000000000000..050b5f08d7d88 --- /dev/null +++ b/JuliaSyntax/docs/src/reference.md @@ -0,0 +1,327 @@ +# Syntax Trees + +This section describes the syntax trees produced by JuliaSyntax, mainly in +terms of their similarities and differences with the `Expr` tree data +structures used since Julia 0.1. + +## JuliaSyntax trees vs `Expr` + +The tree structure of `GreenNode`/`SyntaxNode` is similar to Julia's `Expr` +data structure but there are various differences: + +### Source ordered children + +The children of our trees are strictly in source order. This has many +consequences in places where `Expr` reorders child expressions. + +* Infix and postfix operator calls have the operator name in the *second* child position. `a + b` is parsed as `(call-i a + b)` - where the infix `-i` flag indicates infix child position - rather than `Expr(:call, :+, :a, :b)`. +* Generators are represented in source order as a single node rather than multiple nested flatten and generator expressions. + +### No `LineNumberNode`s + +Our syntax nodes inherently stores source position, so there's no need for the +`LineNumberNode`s used by `Expr`. + +### More consistent / less redundant `block`s + +Sometimes `Expr` needs redundant block constructs to store `LineNumberNode`s, +but we don't need these. Also in cases which do use blocks we try to use them +consistently. + +* No block is used on the right hand side of short form function syntax +* No block is used for the conditional in `elseif` +* No block is used for the body of anonymous functions after the `->` +* `let` argument lists always use a block regardless of number or form of bindings + +### Faithful representation of the source text / avoid premature lowering + +Some cases of "premature lowering" have been removed, preferring to represent +the source text more closely. + +* `K"macrocall"` - allow users to easily distinguish macrocalls with parentheses from those without them (#218) +* Grouping parentheses are represented with a node of kind `K"parens"` (#222) +* The right hand side of `x where {T}` retains the `K"braces"` node around the `T` to distinguish it from `x where T`. +* Ternary syntax is not immediately lowered to an `if` node: `a ? b : c` parses as `(? a b c)` rather than `Expr(:if, :a, :b, :c)` (#85) +* `global const` and `const global` are not normalized by the parser. This is done in `Expr` conversion (#130) +* [`do` syntax](#Do-blocks) is nested as the last child of the call which the `do` lambda will be passed to (#98, #322) +* `@.` is not lowered to `@__dot__` inside the parser (#146) +* Docstrings use the `K"doc"` kind, and are not lowered to `Core.@doc` until later (#217) +* Juxtaposition uses the `K"juxtapose"` kind rather than lowering immediately to `*` (#220) +* `return` without a value has zero children, rather than lowering to `return nothing` (#220) +* Command syntax `` `foo` `` parses into a `cmdstring` tree node wrapping the string, as `(cmdstring "foo")` (#438). These are lowered to a macro call later rather than by the parser. + +### Containers for string-like constructs + +String-like constructs always come within a container node, not as a single +token. These are useful for tooling which works with the tokens of the source +text. Also separating the delimiters from the text they delimit removes a whole +class of tokenization errors and lets the parser deal with them. + +* string always use `K"string"` to wrap strings, even when they only contain a single string chunk (#94) +* char literals are wrapped in the `K"char"` kind, containing the character literal string along with their delimiters (#121) +* backticks use the `K"cmdstring"` kind +* `var""` syntax uses `K"var"` as the head (#127) +* The parser splits triple quoted strings into string chunks interspersed with whitespace trivia + +### Improvements for AST inconsistencies + +* Field access syntax like `a.b` is parsed as `(. a b)` rather than `(. a (quote b))` to avoid the inconsistency between this and actual quoted syntax literals like `:(b)` and `quote b end` ([#342](https://github.com/JuliaLang/JuliaSyntax.jl/issues/324)) +* Dotted call syntax like `f.(a,b)` and `a .+ b` has been made consistent with the `K"dotcall"` head (#90) +* Standalone dotted operators are always parsed as `(. op)`. For example `.*(x,y)` is parsed as `(call (. *) x y)` (#240) +* The `K"="` kind is used for keyword syntax rather than `kw`, to avoid various inconsistencies and ambiguities (#103) +* Unadorned postfix adjoint is parsed as `call` rather than as a syntactic operator for consistency with suffixed versions like `x'ᵀ` (#124) +* The argument list in the left hand side of `->` is always a tuple. For example, `x->y` parses as `(-> (tuple x) y)` rather than `(-> x y)` (#522) + +### Improvements to awkward AST forms + +* `FrankenTuple`s with multiple parameter blocks like `(a=1, b=2; c=3; d=4)` are flattened into the parent tuple instead of using nested `K"parameters"` nodes (#133) +* Using `try catch else finally end` is parsed with `K"catch"` `K"else"` and `K"finally"` children to avoid the awkwardness of the optional child nodes in the `Expr` representation (#234) +* The dotted import path syntax as in `import A.b.c` is parsed with a `K"importpath"` kind rather than `K"."`, because a bare `A.b.c` has a very different nested/quoted expression representation (#244) +* We use flags rather than child nodes to represent the difference between `struct` and `mutable struct`, `module` and `baremodule` (#220) +* Iterations are represented with the `iteration` and `in` heads rather than `=` within the header of a `for`. Thus `for i=is ; body end` parses to `(for (iteration (in i is)) (block body))`. Cartesian iteration as in `for a=as, b=bs body end` are represented with a nested `(iteration (in a as) (in b bs))` rather than a `block` containing `=` because these lists of iterators are neither semantically nor syntactically a sequence of statements, unlike other uses of `block`. Generators also use the `iteration` head - see information on that below. +* Short form functions like `f(x) = x + 1` are represented with the `function` head rather than the `=` head. In this case the `SHORT_FORM_FUNCTION_FLAG` flag is set to allow the surface syntactic form to be easily distinguished from long form functions. +* All kinds of updating assignment operators like `+=` are represented with a single `K"op="` head, with the operator itself in infix position. For example, `x += 1` is `(op= x + 1)`, where the plus token is of kind `K"Identifier"`. This greatly reduces the number of distinct forms here from a rather big list (`$=` `%=` `&=` `*=` `+=` `-=` `//=` `/=` `<<=` `>>=` `>>>=` `\=` `^=` `|=` `÷=` `⊻=`) and makes the operator itself appear in the AST as kind `K"Identifier"`, as it should. It also makes it possible to add further unicode updating operators while keeping the AST stable. The broadcasting form `x .+= 1` uses the `K".op="` head, ie `(.op= x + 1)`. When an updating operator appears without operands - for example when quoted as `:(+=)` - it is represented by the one-argument form `(op= +)` (and likewise `(.op= +)` for `:(.+=)`), which converts to the `Symbol` `:+=` (resp. `:.+=`). +* The lexer never emits the multi-dot sequences `..` and `...` as single tokens; it always produces a sequence of `K"."` tokens which the parser groups as required (#573). When `..`/`...` are used as ordinary identifiers (for example the `..` operator in `a .. b`, or `...` quoted as in `:(...)`) the dots are gathered into a single leaf of kind `K"DotsIdentifier"` rather than `K"Identifier"`, with the number of dots stored in the numeric flags. Like `K"var"` and `K"MacroName"`, `K"DotsIdentifier"` is one of the kinds which may appear where an identifier is expected, so consumers matching identifiers should account for it rather than testing for `K"Identifier"` alone. `K"..."` is now used exclusively for splatting/slurping (as in `f(x...)`). + +## More detail on tree differences + +### Generators + +Flattened generators are uniquely problematic because the Julia AST doesn't +respect a key rule we normally expect: that the children of an AST node are a +*contiguous* range in the source text. For example, the `for`s in +`[xy for x in xs for y in ys]` are parsed in the normal order of a for loop to +mean + +``` +for x in xs + for y in ys + push!(xy, collection) + end +end +``` + +so the `xy` prefix is in the *body* of the innermost for loop. Following this, +the standard Julia AST is like so: + +``` +(flatten + (generator + (generator + xy + (= y ys)) + (= x xs))) +``` + +however, note that if this tree were flattened, the order would be +`(xy) (y in ys) (x in xs)` and the `x` and `y` iterations are *opposite* of the +source order. + +However, our green tree is strictly source-ordered, so we must deviate from the +Julia AST. We deal with this by grouping cartesian products of iterators +(separated by commas) within `iteration` blocks as in `for` loops, and +use the length of the `iteration` block rather than the `flatten` head to +distinguish flattened iterators. The nested flattens and generators of `Expr` +forms are reconstructed later. In this form the tree structure resembles the +source much more closely. For example, `(xy for x in xs for y in ys)` is parsed as + +``` +(generator + xy + (iteration (in x xs)) + (iteration (in y ys))) +``` + +And the cartesian iteration `(xy for x in xs, y in ys)` is parsed as + +``` +(generator + xy + (iteration (in x xs) (in y ys))) +``` + +### Whitespace trivia inside strings + +For triple quoted strings, the indentation isn't part of the string data so +should also be excluded from the string content within the green tree. That is, +it should be treated as separate whitespace trivia tokens. With this separation +things like formatting should be much easier. The same reasoning goes for +escaping newlines and following whitespace with backslashes in normal strings. + +Detecting string trivia during parsing means that string content is split over +several tokens. Here we wrap these in the K"string" kind (as is already used +for interpolations). The individual chunks can then be reassembled during Expr +construction. (A possible alternative might be to reuse the K"String" and +K"CmdString" kinds for groups of string chunks (without interpolation).) + +Take as an example the following Julia fragment. + +```julia +x = """ + $a + b""" +``` + +Here this is parsed as `(= x (string-s a "\n" "b"))` (the `-s` flag in +`string-s` means "triple quoted string") + +Looking at the green tree, we see the indentation before the `$a` and `b` are +marked as trivia: + +``` +julia> text = "x = \"\"\"\n \$a\n b\"\"\"" + show(stdout, MIME"text/plain"(), parseall(GreenNode, text, rule=:statement), text) + 1:23 │[=] + 1:1 │ Identifier ✔ "x" + 2:2 │ Whitespace " " + 3:3 │ = "=" + 4:4 │ Whitespace " " + 5:23 │ [string] + 5:7 │ """ "\"\"\"" + 8:8 │ String "\n" + 9:12 │ Whitespace " " + 13:13 │ $ "\$" + 14:14 │ Identifier ✔ "a" + 15:15 │ String ✔ "\n" + 16:19 │ Whitespace " " + 20:20 │ String ✔ "b" + 21:23 │ """ "\"\"\"" +``` + +### String nodes always wrapped in `K"string"` or `K"cmdstring"` + +All strings are surrounded by a node of kind `K"string"`, even non-interpolated +literals, so `"x"` parses as `(string "x")`. This makes string handling simpler +and more systematic because interpolations and triple strings with embedded +trivia don't need to be treated differently. It also gives a container in which +to attach the delimiting quotes. + +The same goes for command strings which are always wrapped in `K"cmdstring"` +regardless of whether they have multiple pieces (due to triple-quoted +dedenting) or otherwise. + +### Do blocks + +`do` syntax is represented in the `Expr` AST with the `do` outside the call. +This makes some sense syntactically (do appears as "an operator" after the +function call). + +However semantically this nesting is awkward because the lambda represented by +the do block is passed to the call. This same problem occurs for the macro form +`@f(x) do \n body end` where the macro expander needs a special rule to expand +nestings of the form `Expr(:do, Expr(:macrocall ...), ...)`, rearranging the +expression which are passed to this macro call rather than passing the +expressions up the tree. + +The implied closure is also lowered to a nested `Expr(:->)` expression, though +it this somewhat premature to do this during parsing. + +To resolve these problems we parse + + @f(x, y) do a, b\n body\n end + f(x, y) do a, b\n body\n end + +by tacking the `do` onto the end of the call argument list: + + (macrocall @f x y (do (tuple a b) body)) + (call f x y (do (tuple a b) body)) + +This achieves the following desirable properties +1. Content of `do` is nested inside the call which improves the match between AST and semantics +2. Macro can be passed the syntax as-is rather than the macro expander rearranging syntax before passing it to the macro +3. In the future, a macro can detect when it's being passed do syntax rather than lambda syntax +4. `do` head is used uniformly for both call and macrocall +5. We preserve the source ordering properties we need for the green tree. + +## Tree structure reference + +This section may eventually contain a full description of the Julia AST. For +now, we describe a few of the more subtle features. + +### Concatenation syntax + +Concatenation syntax comes in two syntax forms: +* The traditional `hcat`/`vcat`/`row` which deal with concatenation or matrix + construction along dimensions one and two. +* The new `ncat`/`nrow` syntax which deals with concatenation or array + construction along arbitrary dimensions. + +We write `ncat-3` for concatenation along the third dimension. (The `3` is +stored in the head flags for `SyntaxNode` trees, and in the first `arg` for +`Expr` trees.) Semantically the new syntax can work like the old: +* `ncat-1` is the same as `vcat` +* `ncat-2` is the same as `hcat` +* `row` is the same as `nrow-2` + +#### Vertical concatenation (dimension 1) + +Vertical concatenation along dimension 1 can be done with semicolons or newlines + +```julia-repl +julia> print_tree(:([a + b])) +Expr(:vcat) +├─ :a +└─ :b + +julia> print_tree(:([a ; b])) +Expr(:vcat) +├─ :a +└─ :b +``` + +#### Horizontal concatenation (dimension 2) + +For horizontal concatenation along dimension 2, use spaces or double semicolons + +```julia-repl +julia> print_tree(:([a b])) +Expr(:hcat) +├─ :a +└─ :b + +julia> print_tree(:([a ;; b])) +Expr(:ncat) +├─ 2 +├─ :a +└─ :b +``` + +#### Mixed concatenation + +Concatenation along dimensions 1 and 2 can be done with spaces and single +semicolons or newlines, producing a mixture of `vcat` and `row` expressions: + +```julia-repl +julia> print_tree(:([a b + c d])) +# OR +julia> print_tree(:([a b ; c d])) +Expr(:vcat) +├─ Expr(:row) +│ ├─ :a +│ └─ :b +└─ Expr(:row) + ├─ :c + └─ :d +``` + +General n-dimensional concatenation results in nested `ncat` and `nrow`, for +example + +```julia-repl +julia> print_tree(:([a ; b ;; c ; d ;;; x])) +Expr(:ncat) +├─ 3 +├─ Expr(:nrow) +│ ├─ 2 +│ ├─ Expr(:nrow) +│ │ ├─ 1 +│ │ ├─ :a +│ │ └─ :b +│ └─ Expr(:nrow) +│ ├─ 1 +│ ├─ :c +│ └─ :d +└─ :x +``` diff --git a/JuliaSyntax/prototypes/simple_parser.jl b/JuliaSyntax/prototypes/simple_parser.jl new file mode 100644 index 0000000000000..06a408a26860a --- /dev/null +++ b/JuliaSyntax/prototypes/simple_parser.jl @@ -0,0 +1,174 @@ +# Example parser for a very basic Julia-like language of expressions, calls and +# function definitions. + +using JuliaSyntax: @K_str, is_literal, is_keyword, is_operator + +function parse_toplevel(st) + mark = position(st) + while true + bump_trivia(st, skip_newlines=true) + if peek(st) == K"EndMarker" + break + end + parse_statement(st) + end + emit(st, mark, K"toplevel") +end + +function parse_statement(st) + mark = position(st) + if peek(st) == K"function" + parse_function_def(st) + else + parse_assignment(st) + end +end + +function parse_function_def(st) + mark = position(st) + @assert peek(st) == K"function" + bump(st, TRIVIA_FLAG) + parse_call(st) + parse_block(st, K"end") + emit(st, mark, K"function") +end + +function parse_block(st, closing_kind, mark=position(st)) + while true + bump_trivia(st, skip_newlines=true) + if peek(st) == closing_kind + bump(st, TRIVIA_FLAG) + break + elseif peek(st) == K"EndMarker" + emit_diagnostic(st, error="Unexpected end of input") + break + end + parse_assignment(st) + end + emit(st, mark, K"block") +end + +function parse_assignment(st) + mark = position(st) + parse_expression(st) + if peek(st) == K"=" + bump(st, TRIVIA_FLAG) + parse_expression(st) + emit(st, mark, K"=") + end +end + +function parse_expression(st) + mark = position(st) + parse_term(st) + while peek(st) in (K"+", K"-") + bump(st) + parse_term(st) + emit(st, mark, K"call", INFIX_FLAG) + end +end + +function parse_term(st) + mark = position(st) + parse_call(st) + while peek(st) in (K"*", K"/") + bump(st) + parse_call(st) + emit(st, mark, K"call", INFIX_FLAG) + end +end + +function parse_call(st) + mark = position(st) + parse_atom(st) + if peek(st) == K"(" + bump(st, TRIVIA_FLAG) + need_comma = false + while true + k = peek(st) + if need_comma && k == K"," + bump(st, TRIVIA_FLAG) + k = peek(st) + need_comma = false + end + if k == K")" + bump(st, TRIVIA_FLAG) + break + elseif k == K"EndMarker" + emit_diagnostic(st, error="Unexpected end of input") + break + elseif need_comma + bump_invisible(st, K"error", TRIVIA_TOKEN, error="Expected a `,`") + end + parse_expression(st) + need_comma = true + end + emit(st, mark, K"call") + end +end + +function parse_atom(st) + bump_trivia(st, skip_newlines=true) + mark = position(st) + k = peek(st) + if k == K"Identifier" || is_literal(k) + bump(st) + elseif k in (K"-", K"+") + bump(st) + parse_atom(st) + emit(st, mark, K"call") + elseif k == K"(" + bump(st, TRIVIA_FLAG) + parse_expression(st) + if peek(st) == K")" + bump(st, TRIVIA_FLAG) + # emit(st, mark, K"(") + else + bump_invisible(st, K"error", TRIVIA_FLAG, + error="Expected `)` following expression") + end + elseif k == K"begin" + bump(st, TRIVIA_FLAG) + parse_block(st, K"end", mark) + else + bump(st) + emit(st, mark, K"error", + error="Expected literal, identifier or opening parenthesis") + end +end + +function parse_and_show(production::Function, code) + st = ParseStream(code) + production(st) + t = JuliaSyntax.build_tree(GreenNode, st) + show(stdout, MIME"text/plain"(), t, code, show_trivia=true) + if !isempty(st.diagnostics) + println() + for d in st.diagnostics + JuliaSyntax.show_diagnostic(stdout, d, code) + end + end + t +end + +println() +println("Example good parse:") +parse_and_show(parse_toplevel, + """ + function f(x, y) + z = x - y + begin + a + b + end + z * z + end + + f(1,2) + """) + +println() +println("Example diagnostics:") +parse_and_show(parse_expression, "(x + a*y) * (b") + +nothing diff --git a/JuliaSyntax/prototypes/syntax_interpolation.jl b/JuliaSyntax/prototypes/syntax_interpolation.jl new file mode 100644 index 0000000000000..eddf6748bd423 --- /dev/null +++ b/JuliaSyntax/prototypes/syntax_interpolation.jl @@ -0,0 +1,56 @@ +# # Macros and expression interpolation + +using JuliaSyntax: SourceFile, SyntaxNode, parseall, child, setchild! + +# The following shows that SyntaxNode works nicely for simple macros which +# just interpolate expressions into one another. In particular it shows how +# precise source information from multiple files can coexist within the same +# syntax tree. + +# First, here's the functionality that we're going to implement as a normal +# Julia macro. It's similar to the standard @show macro. +macro show2(ex) + name = sprint(Base.show_unquoted, ex) + quote + value = $(esc(ex)) + println($name, " = ", value) + value + end +end + +# Now, let's implement the same expression interpolation but using SyntaxNode +# (and with a normal Julia function which we need to use, absent any deeper +# integration with the Julia runtime) +function at_show2(ex::SyntaxNode) + name = sprint(show, MIME"text/x.sexpression"(), ex) + quote + value = $(esc(ex)) + println($name, " = ", value) + value + end + # The following emulates the expression interpolation lowering which is + # usually done by the compiler. + # 1. Extract the expression literal as `block` + tree = parseall(SyntaxNode, String(read(@__FILE__)), filename=@__FILE__) + block = child(tree, 3, 2, 2, 1) + # 2. Interpolate local variables into the block at positions of $'s + # Interpolating a SyntaxNode `ex` is simple: + setchild!(block, (1, 2), ex) + # The interpolation of a Julia *value* should inherit the source location + # of the $ interpolation expression. This is different to when substituting + # in a SyntaxNode which should just be inserted as-is. + setchild!(block, (2, 2), + JuliaSyntax.interpolate_literal(child(block, 2, 2), name)) + block +end + +# Usage of at_show2() + +# Let's have some simple expression to pass to at_show2. This will be +# attributed to a different file foo.jl +s2 = parseall(SyntaxNode, "foo +\n42", filename="foo.jl", rule=:statement) + +# Calling at_show2, we see that the precise source information is preserved for +# both the surrounding expression and the interpolated fragments. +println("\nInterpolation example") +s3 = at_show2(s2) diff --git a/JuliaSyntax/src/JuliaSyntax.jl b/JuliaSyntax/src/JuliaSyntax.jl new file mode 100644 index 0000000000000..23e7586352a81 --- /dev/null +++ b/JuliaSyntax/src/JuliaSyntax.jl @@ -0,0 +1,110 @@ +module JuliaSyntax + +macro _public(syms) + if VERSION >= v"1.11" + names = syms isa Symbol ? [syms] : syms.args + esc(Expr(:public, names...)) + else + nothing + end +end + +# Public API, in the order of docs/src/api.md + +# Parsing. +export parseall, parseatom, parsestmt + +@_public parse!, + ParseStream, + build_tree + +# Tokenization +export Token, tokenize, untokenize + +# Source file handling +@_public sourcefile, + byte_range, + char_range, + first_byte, + last_byte, + filename, + source_line, + source_location, + sourcetext, + highlight + +export SourceFile +@_public source_line_range + +# Expression predicates, kinds and flags +export @K_str, kind +@_public Kind +@_public PrecedenceLevel, PREC_NONE, PREC_ASSIGNMENT, + PREC_PAIRARROW, PREC_CONDITIONAL, PREC_ARROW, PREC_LAZYOR, PREC_LAZYAND, + PREC_COMPARISON, PREC_PIPE_LT, PREC_PIPE_GT, PREC_COLON, PREC_PLUS, + PREC_BITSHIFT, PREC_TIMES, PREC_RATIONAL, PREC_POWER, PREC_DECL, + PREC_WHERE, PREC_DOT, PREC_QUOTE, PREC_UNICODE_OPS, PREC_COMPOUND_ASSIGN, + generic_operators_by_level + +@_public flags, + SyntaxHead, + head, + is_trivia, + is_prefix_call, + is_infix_op_call, + is_prefix_op_call, + is_postfix_op_call, + numeric_flags, + has_flags, + TRIPLE_STRING_FLAG, + RAW_STRING_FLAG, + PARENS_FLAG, + COLON_QUOTE, + TOPLEVEL_SEMICOLONS_FLAG, + MUTABLE_FLAG, + BARE_MODULE_FLAG, + SHORT_FORM_FUNCTION_FLAG + +# Syntax trees +@_public is_leaf, + numchildren, + children + +export SyntaxNode + +@_public GreenNode, RedTreeCursor, GreenTreeCursor, + span + +# Helper utilities +include("utils.jl") + +include("julia/kinds.jl") + +# Lexing uses a significantly modified version of Tokenize.jl +include("julia/tokenize.jl") + +# Source and diagnostics +include("core/source_files.jl") +include("core/diagnostics.jl") + +# Parsing +include("core/parse_stream.jl") +include("core/tree_cursors.jl") +include("julia/julia_parse_stream.jl") +include("julia/parser.jl") +include("julia/parser_api.jl") +include("julia/literal_parsing.jl") + +# Tree data structures +include("porcelain/green_node.jl") +include("porcelain/syntax_node.jl") +include("integration/expr.jl") +if VERSION >= v"1.12" + include("porcelain/syntax.jl") +end + +# Hooks to integrate the parser with Base +include("integration/hooks.jl") +include("precompile.jl") + +end diff --git a/JuliaSyntax/src/core/diagnostics.jl b/JuliaSyntax/src/core/diagnostics.jl new file mode 100644 index 0000000000000..46e580c3ea359 --- /dev/null +++ b/JuliaSyntax/src/core/diagnostics.jl @@ -0,0 +1,110 @@ +""" + Diagnostic(first_byte, last_byte; [error="msg" | warning="msg"]) + +A diagnostic message, referring to the source code byte range +first_byte:last_byte, with a `warning` or `error` message. + +Messages should be concise, matter-of-fact and not include decorations: + +* Concise: "Show don't tell". Where possible, let's show the user what's wrong + by annotating their original source code via the byte range. +* Matter-of-fact: Admonishing the user isn't helpful. Let's gently show them + what's wrong instead, using a neutral tone. +* Decorations: Capitalization, punctuation and diagnostic class ("error" / + "warning") should be omitted. These decorations will be added by the + formatting code. + +TODO: At some point we should enhance Diagnostic to allow multiple sub-ranges +for better annotation. Let's follow the excellent precedent set by Rust's +[rustc_errors::Diagnostic](https://doc.rust-lang.org/stable/nightly-rustc/rustc_errors/struct.Diagnostic.html). + +TODO: We should cater for extended descriptions containing multiple sentences +via a diagnostic code which can be used to look up detailed information. Again, +Rust does this well. +""" +struct Diagnostic + first_byte::Int + last_byte::Int + level::Symbol + message::String +end + +function Diagnostic(first_byte, last_byte; error=nothing, warning=nothing) + message = !isnothing(error) ? error : + !isnothing(warning) ? warning : + Base.error("No message in diagnostic") + level = !isnothing(error) ? :error : :warning + Diagnostic(first_byte, last_byte, level, message) +end + +byte_range(d::Diagnostic) = d.first_byte:d.last_byte +is_error(d::Diagnostic) = d.level === :error + +# Make relative path into a file URL +function _file_url(filename) + try + @static if Sys.iswindows() + # TODO: Test this with windows terminal + path = replace(abspath(filename), '\\'=>'/') + else + path = abspath(filename) + end + return "file://$(path)" + catch + # abspath may fail if working directory doesn't exist + # TODO: It seems rather non-ideal to have the behavior here depend on + # the state of the local filesystem. And yet links in diagnostics seem + # useful. + # + # Ideally it'd be up to the caller to provide some notion of the + # "absolute location" of the source code resource when SourceFile is + # constructed. This is often not related to the local filesystem - it + # could be in memory, a fragment embedded in another file, etc etc. + return nothing + end +end + +function show_diagnostic(io::IO, diagnostic::Diagnostic, source::SourceFile) + color,prefix = diagnostic.level === :error ? (:light_red, "Error") : + diagnostic.level === :warning ? (:light_yellow, "Warning") : + diagnostic.level === :note ? (:light_blue, "Note") : + (:normal, "Info") + line, col = source_location(source, first_byte(diagnostic)) + linecol = "$line:$col" + fname = filename(source) + file_href = nothing + if !isempty(fname) + locstr = "$fname:$linecol" + if !startswith(fname, "REPL[") && get(io, :color, false) + url = _file_url(fname) + if !isnothing(url) + file_href = url*"#$linecol" + end + end + else + locstr = "line $linecol" + end + _printstyled(io, "# $prefix @ ", fgcolor=:light_black) + _printstyled(io, "$locstr", fgcolor=:light_black, href=file_href) + print(io, "\n") + highlight(io, source, byte_range(diagnostic), + note=diagnostic.message, notecolor=color, + context_lines_before=1, context_lines_after=0) +end + +function show_diagnostics(io::IO, diagnostics::AbstractVector{Diagnostic}, source::SourceFile) + first = true + for d in diagnostics + first || println(io) + first = false + show_diagnostic(io, d, source) + end +end + +function show_diagnostics(io::IO, diagnostics::AbstractVector{Diagnostic}, text::AbstractString) + show_diagnostics(io, diagnostics, SourceFile(text)) +end + +function any_error(diagnostics::AbstractVector{Diagnostic}) + any(is_error(d) for d in diagnostics) +end diff --git a/JuliaSyntax/src/core/parse_stream.jl b/JuliaSyntax/src/core/parse_stream.jl new file mode 100644 index 0000000000000..8d8209d111a11 --- /dev/null +++ b/JuliaSyntax/src/core/parse_stream.jl @@ -0,0 +1,1001 @@ +#------------------------------------------------------------------------------- +# Flags hold auxiliary information about tokens/nonterminals which the Kind +# doesn't capture in a nice way. +# +# TODO: Use `primitive type SyntaxFlags 16 end` rather than an alias? +const RawFlags = UInt16 +const EMPTY_FLAGS = RawFlags(0) + +# Set for tokens or ranges which are syntax trivia after parsing +const TRIVIA_FLAG = RawFlags(1<<0) + +""" +Set for nodes that are non-terminals +""" +const NON_TERMINAL_FLAG = RawFlags(1<<7) + +function remove_flags(n::RawFlags, fs...) + RawFlags(n & ~(RawFlags((|)(fs...)))) +end + +""" + has_flags(x, test_flags) + +Return true if any of `test_flags` are set. +""" +has_flags(flags::RawFlags, test_flags) = (flags & test_flags) != 0 + +#------------------------------------------------------------------------------- +""" + SyntaxHead(kind, flags) + +A `SyntaxHead` combines the [`Kind`](@ref) of a syntactic construct with a set +of flags. The kind defines the broad "type" of the syntactic construct, while +the flag bits compactly store more detailed information about the construct. +""" +struct SyntaxHead + kind::Kind + flags::RawFlags +end + +kind(head::SyntaxHead) = head.kind + +""" + flags(x) + +Return the flag bits of a syntactic construct. Prefer to query these with the +predicates `is_trivia`, `is_prefix_call`, `is_infix_op_call`, +`is_prefix_op_call`, `is_postfix_op_call`. + +Or extract numeric portion of the flags with `numeric_flags`. +""" +flags(head::SyntaxHead) = head.flags + +function Base.summary(head::SyntaxHead) + untokenize(head, unique=false, include_flag_suff=false) +end + +#------------------------------------------------------------------------------- +# Generic interface for types `T` which have kind and flags. Either: +# 1. Define kind(::T) and flags(::T), or +# 2. Define head(::T) to return a type like `SyntaxKind` for which `kind` and +# `flags` are defined +kind(x) = kind(head(x)) +flags(x) = flags(head(x)) + +# Predicates based on flags() +has_flags(x, test_flags) = has_flags(flags(x), test_flags) +call_type_flags(x) = call_type_flags(flags(x)) + +""" + is_trivia(x) + +Return true for "syntax trivia": tokens in the tree which are either largely +invisible to the parser (eg, whitespace) or implied by the structure of the AST +(eg, reserved words). +""" +is_trivia(x) = has_flags(x, TRIVIA_FLAG) + +#------------------------------------------------------------------------------- +""" +`SyntaxToken` is a token covering a contiguous byte range in the input text. + +We record only the `next_byte` here (the index of the next byte *after* the +token) to avoid duplication of data between neighbouring tokens. This is more +useful than recording the first byte, as it allows an initial fixed sentinel +token to be used for recording the first byte of the first real token. +""" +struct SyntaxToken + head::SyntaxHead + orig_kind::Kind + preceding_whitespace::Bool + next_byte::UInt32 +end + +function Base.show(io::IO, tok::SyntaxToken) + print(io, rpad(untokenize(tok.head, unique=false), 15), " |", tok.next_byte) +end + +head(tok::SyntaxToken) = tok.head +preceding_whitespace(tok::SyntaxToken) = tok.preceding_whitespace + + +#------------------------------------------------------------------------------- + +""" + RawGreenNode(head::SyntaxHead, byte_span::UInt32, orig_kind::Kind) # Terminal + RawGreenNode(head::SyntaxHead, byte_span::UInt32, nchildren::UInt32) # Non-terminal + +A "green tree" is a lossless syntax tree which overlays all the source text. +The most basic properties of a green tree are that: + +* Nodes cover a contiguous span of bytes in the text +* Sibling nodes are ordered in the same order as the text + +As implementation choices, we choose that: + +* Nodes are immutable and don't know their parents or absolute position, so can + be cached and reused +* Nodes are homogeneously typed at the language level so they can be stored + concretely, with the `head` defining the node type. Normally this would + include a "syntax kind" enumeration, but it can also include flags and record + information the parser knew about the layout of the child nodes. +* For simplicity and uniformity, leaf nodes cover a single token in the source. + This is like rust-analyzer, but different from Roslyn where leaves can + include syntax trivia. +* The parser produces a single buffer of `RawGreenNode` which encodes the tree. + There are higher level accessors, which make working with this tree easier. +""" +struct RawGreenNode + head::SyntaxHead # Kind,flags + byte_span::UInt32 # Number of bytes covered by this range + # If NON_TERMINAL_FLAG is set, this is the total number of child nodes + # Otherwise this is a terminal node (i.e. a token) and this is orig_kind + node_span_or_orig_kind::UInt32 + + # Constructor for terminal nodes (tokens) + function RawGreenNode(head::SyntaxHead, byte_span::Integer, orig_kind::Kind) + @assert (flags(head) & NON_TERMINAL_FLAG) == 0 + new(head, UInt32(byte_span), UInt32(reinterpret(UInt16, orig_kind))) + end + + # Constructor for non-terminal nodes - automatically sets NON_TERMINAL_FLAG + function RawGreenNode(head::SyntaxHead, byte_span::Integer, node_span::Integer) + h = SyntaxHead(kind(head), flags(head) | NON_TERMINAL_FLAG) + new(h, UInt32(byte_span), UInt32(node_span)) + end + + global reset_node + function reset_node(node::RawGreenNode, kind, flags) + new(_reset_node_head(node, kind, flags), + getfield(node, :byte_span), + getfield(node, :node_span_or_orig_kind)) + end +end + +function _reset_node_head(node, k, f) + if !isnothing(f) + f = RawFlags(f) + @assert (f & NON_TERMINAL_FLAG) == 0 + f |= flags(node) & NON_TERMINAL_FLAG + else + f = flags(node) + end + return SyntaxHead(isnothing(k) ? kind(node) : k, f) +end + +Base.summary(node::RawGreenNode) = summary(node.head) +function Base.show(io::IO, node::RawGreenNode) + print(io, summary(node), " (", node.byte_span, " bytes,") + if is_terminal(node) + print(io, " orig_kind=", node.orig_kind, ")") + else + print(io, " ", node.node_span, " children)") + end +end + +function Base.getproperty(rgn::RawGreenNode, name::Symbol) + if name === :node_span + has_flags(getfield(rgn, :head), NON_TERMINAL_FLAG) || return UInt32(0) # Leaf nodes have no children + return getfield(rgn, :node_span_or_orig_kind) + elseif name === :orig_kind + has_flags(getfield(rgn, :head), NON_TERMINAL_FLAG) && error("Cannot access orig_kind for non-terminal node") + return Kind(getfield(rgn, :node_span_or_orig_kind)) + end + getfield(rgn, name) +end + +head(range::RawGreenNode) = range.head + +# Helper functions for unified output +is_terminal(node::RawGreenNode) = !has_flags(node.head, NON_TERMINAL_FLAG) +is_non_terminal(node::RawGreenNode) = has_flags(node.head, NON_TERMINAL_FLAG) + +#------------------------------------------------------------------------------- +struct ParseStreamPosition + """ + The current position in the byte stream, i.e. the byte at `byte_index` is + the first byte of the next token to be parsed. + """ + byte_index::UInt32 + """ + The total number of nodes (terminal + non-terminal) in the output so far. + """ + node_index::UInt32 +end + +const NO_POSITION = ParseStreamPosition(0, 0) + +#------------------------------------------------------------------------------- +""" + ParseStream(text::AbstractString, index::Integer=1; version=VERSION) + ParseStream(text::IO; version=VERSION) + ParseStream(text::Vector{UInt8}, index::Integer=1; version=VERSION) + ParseStream(ptr::Ptr{UInt8}, len::Integer, index::Integer=1; version=VERSION) + +Construct a `ParseStream` from input which may come in various forms: +* An string (zero copy for `String` and `SubString`) +* An `IO` object (zero copy for `IOBuffer`). The `IO` object must be seekable. +* A buffer of bytes (zero copy). The caller is responsible for preserving + buffers passed as `(ptr,len)`. + +A byte `index` may be provided as the position to start parsing. + +ParseStream provides an IO interface for the parser which provides lexing of +the source text input into tokens, manages insignificant whitespace tokens on +behalf of the parser, and stores output tokens and tree nodes in a pair of +output arrays. + +`version` (default `VERSION`) may be used to set the syntax version to +any Julia version `>= v"1.0"`. We aim to parse all Julia syntax which has been +added after v"1.0", emitting an error if it's not compatible with the requested +`version`. +""" +mutable struct ParseStream + # `textbuf` is a buffer of UTF-8 encoded text of the source code. This is a + # natural representation as we desire random access and zero-copy parsing + # of UTF-8 text from various containers, and unsafe_wrap(Vector{UInt8}, + # ...) allows us to use a Vector here. + # + # We want `ParseStream` to be concrete so that all `parse_*` functions only + # need to be compiled once. Thus `textbuf` must not be parameterized here. + textbuf::Vector{UInt8} + # GC root for the object which owns the memory in `textbuf`. `nothing` if + # the `textbuf` owner was unknown (eg, ptr,length was passed) + text_root::Any + # Lexer, transforming the input bytes into a token stream + lexer::Tokenize.Lexer{IOBuffer} + # Lookahead buffer for already lexed tokens + lookahead::Vector{SyntaxToken} + lookahead_index::Int + # Pool of stream positions for use as working space in parsing + position_pool::Vector{Vector{ParseStreamPosition}} + output::Vector{RawGreenNode} + # Current byte position in the output (the next byte to be written) + next_byte::Int + # Parsing diagnostics (errors/warnings etc) + diagnostics::Vector{Diagnostic} + # Counter for number of peek()s we've done without making progress via a bump() + peek_count::Int + # (major,minor) version of Julia we're parsing this code for. + # May be different from VERSION! + version::Tuple{Int,Int} + + function ParseStream(text_buf::Vector{UInt8}, text_root, next_byte::Integer, + version::VersionNumber) + io = IOBuffer(text_buf) + seek(io, next_byte-1) + lexer = Tokenize.Lexer(io) + # To avoid keeping track of the exact Julia development version where new + # features were added or comparing prerelease strings, we treat prereleases + # or dev versions as the release version using only major and minor version + # numbers. This means we're inexact for old dev versions but that seems + # like an acceptable tradeoff. + ver = (version.major, version.minor) + # Initial sentinel node (covering all ignored bytes before the first token) + sentinel = RawGreenNode(SyntaxHead(K"TOMBSTONE", EMPTY_FLAGS), next_byte-1, K"TOMBSTONE") + new(text_buf, + text_root, + lexer, + Vector{SyntaxToken}(), + 1, + Vector{Vector{ParseStreamPosition}}(), + RawGreenNode[sentinel], + next_byte, # Initialize next_byte from the parameter + Vector{Diagnostic}(), + 0, + ver) + end +end + +function ParseStream(text::Vector{UInt8}, index::Integer=1; version=VERSION) + ParseStream(text, text, index, version) +end + +# Buffer with unknown owner. Not exactly recommended, but good for C interop +function ParseStream(ptr::Ptr{UInt8}, len::Integer, index::Integer=1; version=VERSION) + ParseStream(unsafe_wrap(Vector{UInt8}, ptr, len), nothing, index, version) +end + +# Buffers originating from strings +function ParseStream(text::String, index::Integer=1; version=VERSION) + ParseStream(unsafe_wrap(Vector{UInt8}, text), + text, index, version) +end +function ParseStream(text::SubString{String}, index::Integer=1; version=VERSION) + # See also IOBuffer(SubString("x")) + ParseStream(unsafe_wrap(Vector{UInt8}, pointer(text), sizeof(text)), + text, index, version) +end +function ParseStream(text::AbstractString, index::Integer=1; version=VERSION) + ParseStream(String(text), index; version=version) +end + +# IO-based cases +# TODO: switch ParseStream to use a Memory internally on newer versions of Julia +VERSION < v"1.11.0-DEV.753" && function ParseStream(io::IOBuffer; version=VERSION) + ParseStream(io.data, io, position(io)+1, version) +end +function ParseStream(io::Base.GenericIOBuffer; version=VERSION) + textbuf = unsafe_wrap(Vector{UInt8}, pointer(io.data), length(io.data)) + ParseStream(textbuf, io, position(io)+1, version) +end +function ParseStream(io::IO; version=VERSION) + textbuf = read(io) + ParseStream(textbuf, textbuf, 1, version) +end + +function Base.show(io::IO, ::MIME"text/plain", stream::ParseStream) + println(io, "ParseStream at position $(stream.next_byte)") +end + +function show_diagnostics(io::IO, stream::ParseStream) + show_diagnostics(io, stream.diagnostics, SourceFile(stream)) +end + +# We manage a pool of stream positions as parser working space +function acquire_positions(stream) + if isempty(stream.position_pool) + return Vector{ParseStreamPosition}() + end + pop!(stream.position_pool) +end + +function release_positions(stream, positions) + empty!(positions) + push!(stream.position_pool, positions) +end + +#------------------------------------------------------------------------------- +# Return true when a terminal (token) was emitted last at stream position `pos` +function token_is_last(stream, pos) + # In the unified structure, check if the node at pos is a terminal + return pos.node_index > 0 && pos.node_index <= length(stream.output) && + is_terminal(stream.output[pos.node_index]) +end + +function lookahead_token_first_byte(stream, i) + i == 1 ? _next_byte(stream) : stream.lookahead[i-1].next_byte +end + +function lookahead_token_last_byte(stream, i) + stream.lookahead[i].next_byte - 1 +end + +#------------------------------------------------------------------------------- +# Stream input interface - the peek_* family of functions + +# Buffer several tokens ahead +function _buffer_lookahead_tokens(lexer, lookahead) + had_whitespace = false + token_count = 0 + while true + raw = Tokenize.next_token(lexer) + k = kind(raw) + was_whitespace = is_whitespace(k) + had_whitespace |= was_whitespace + f = EMPTY_FLAGS + if (k == K"Operator" || raw.op_precedence == Tokenize.PREC_COMPOUND_ASSIGN) && + raw.op_precedence != Tokenize.PREC_NONE + # Store operator precedence in numeric flags + f |= set_numeric_flags(Int(raw.op_precedence)) + end + push!(lookahead, SyntaxToken(SyntaxHead(k, f), k, + had_whitespace, raw.endbyte + 2)) + token_count += 1 + if k == K"EndMarker" + break + end + if !was_whitespace + # Buffer tokens in batches for lookahead. Generally we want a + # moderate-size buffer to make sure we hit the fast path of peek(), + # but not too large to avoid (a) polluting the processor cache and + # (b) doing unnecessary work when not parsing the whole input. + had_whitespace = false + if token_count > 100 + break + end + end + end +end + +# Return the index of the next byte of the input +function _next_byte(stream) + stream.next_byte +end + +# Find the index of the next nontrivia token +@inline function _lookahead_index(stream::ParseStream, n::Integer, skip_newlines::Bool) + # Much of the time we'll be peeking ahead a single token and have one or + # zero whitespace tokens before the next token. The following code is an + # unrolled optimized version for that fast path. Empirically it seems we + # only hit the slow path about 5% of the time here. + i = stream.lookahead_index + @inbounds if n == 1 && i+2 <= length(stream.lookahead) + if skip_newlines + k = kind(stream.lookahead[i]) + if !(k == K"Whitespace" || k == K"Comment" || k == K"NewlineWs") + return i + end + i += 1 + k = kind(stream.lookahead[i]) + if !(k == K"Whitespace" || k == K"Comment" || k == K"NewlineWs") + return i + end + else + k = kind(stream.lookahead[i]) + if !(k == K"Whitespace" || k == K"Comment") + return i + end + i += 1 + k = kind(stream.lookahead[i]) + if !(k == K"Whitespace" || k == K"Comment") + return i + end + end + end + # Fall through to the general case + return __lookahead_index(stream, n, skip_newlines) +end + +@noinline function __lookahead_index(stream, n, skip_newlines) + i = stream.lookahead_index + while true + if i+1 > length(stream.lookahead) + n_to_delete = stream.lookahead_index-1 + if n_to_delete > 0.9*length(stream.lookahead) + Base._deletebeg!(stream.lookahead, n_to_delete) + i -= n_to_delete + stream.lookahead_index = 1 + end + _buffer_lookahead_tokens(stream.lexer, stream.lookahead) + continue + end + k = @inbounds kind(stream.lookahead[i]) + if !((k == K"Whitespace" || k == K"Comment") || + (k == K"NewlineWs" && skip_newlines)) + if n == 1 + return i + end + n -= 1 + end + i += 1 + end +end + +@noinline function _parser_stuck_error(stream) + # Optimization: emit unlikely errors in a separate function + error("The parser seems stuck at byte $(stream.next_byte)") +end + +""" + peek(stream::ParseStream [, n=1]; skip_newlines=false) + +Look ahead in the stream `n` tokens, returning the token kind. Comments and +non-newline whitespace are skipped automatically. Whitespace containing a +single newline is returned as kind `K"NewlineWs"` unless `skip_newlines` is +true. +""" +function Base.peek(stream::ParseStream, n::Integer=1; + skip_newlines::Bool=false, skip_whitespace=true) + kind(peek_token(stream, n; skip_newlines=skip_newlines, skip_whitespace=skip_whitespace)) +end + +""" + peek_token(stream [, n=1]) + +Like `peek`, but return the full token information rather than just the kind. +""" +function peek_token(stream::ParseStream, n::Integer=1; + skip_newlines=false, skip_whitespace=true) + stream.peek_count += 1 + if stream.peek_count > 100_000 + _parser_stuck_error(stream) + end + i = _lookahead_index(stream, n, skip_newlines) + if !skip_whitespace + i = stream.lookahead_index + end + return @inbounds stream.lookahead[i] +end + + +struct FullToken + head::SyntaxHead + first_byte::UInt32 + last_byte::UInt32 +end + +head(t::FullToken) = t.head +byte_range(t::FullToken) = t.first_byte:t.last_byte +span(t::FullToken) = 1 + last_byte(t) - first_byte(t) + +function peek_full_token(stream::ParseStream, n::Integer=1; + skip_newlines=false, skip_whitespace=true) + stream.peek_count += 1 + if stream.peek_count > 100_000 + _parser_stuck_error(stream) + end + i = _lookahead_index(stream, n, skip_newlines) + if !skip_whitespace + i = stream.lookahead_index + end + t = stream.lookahead[i] + + FullToken(head(t), lookahead_token_first_byte(stream, i), + lookahead_token_last_byte(stream, i)) +end + +""" + peek_behind(ps; skip_trivia=true, skip_parens=true) + peek_behind(ps, pos::ParseStreamPosition) + +Return information about a span which was previously inserted into the output, +defaulting to the most previous nontrivia node when `skip_trivia` is true, or +at the provided position `pos`. + +Retroactively inspecting or modifying the parser's output can be confusing, so +using this function should be avoided where possible. +""" +function peek_behind(stream::ParseStream, pos::ParseStreamPosition) + if pos.node_index > 0 && pos.node_index <= length(stream.output) + node = stream.output[pos.node_index] + if is_terminal(node) + return (kind=kind(node), + flags=flags(node), + orig_kind=node.orig_kind, + is_leaf=true) + else + return (kind=kind(node), + flags=flags(node), + orig_kind=K"None", + is_leaf=false) + end + else + return (kind=K"None", + flags=EMPTY_FLAGS, + orig_kind=K"None", + is_leaf=true) + end +end + +""" + first_child_position(stream::ParseStream, pos::ParseStreamPosition) + +Find the first non-trivia child of this node (in the GreenTree/RedTree sense) and return +its position. +""" +function first_child_position(stream::ParseStream, pos::ParseStreamPosition) + output = stream.output + @assert pos.node_index > 0 + cursor = RedTreeCursor(GreenTreeCursor(output, pos.node_index), pos.byte_index-UInt32(1)) + candidate = nothing + for child in reverse(cursor) + is_trivia(child) && continue + candidate = child + end + + candidate !== nothing && return ParseStreamPosition(candidate.byte_end+UInt32(1), candidate.green.position) + + # No children found - return the first non-trivia *token* (even if it + # is the child of a non-terminal trivia node (e.g. an error)). + byte_end = pos.byte_index + for i in pos.node_index-1:-1:(pos.node_index - treesize(cursor)) + node = output[i] + if is_terminal(node) + if !is_trivia(node) + return ParseStreamPosition(byte_end, i) + end + byte_end -= node.byte_span + end + end + + # Still none found. Return a sentinel value + return ParseStreamPosition(0, 0) +end + +""" + last_child_position(stream::ParseStream, pos::ParseStreamPosition) + + Find the last non-trivia child of this node (in the GreenTree/RedTree sense) and + return its position (i.e. the position as if that child had been the last thing parsed). +""" +function last_child_position(stream::ParseStream, pos::ParseStreamPosition) + output = stream.output + @assert pos.node_index > 0 + cursor = RedTreeCursor(GreenTreeCursor(output, pos.node_index), pos.byte_index-1) + for child in reverse(cursor) + is_trivia(child) && continue + return ParseStreamPosition(child.byte_end+UInt32(1), child.green.position) + end + return ParseStreamPosition(0, 0) +end + +# Get last position in stream "of interest", skipping +# * parens nodes +# * deleted tokens (TOMBSTONE) +# * whitespace (if skip_trivia=true) +function peek_behind_pos(stream::ParseStream; skip_trivia::Bool=true, + skip_parens::Bool=true) + # Work backwards through the output + node_idx = length(stream.output) + byte_idx = stream.next_byte + + # Skip parens nodes if requested + if skip_parens + while node_idx > 0 + node = stream.output[node_idx] + if is_non_terminal(node) && kind(node) == K"parens" + node_idx -= 1 + else + break + end + end + end + + # Skip trivia if requested + while node_idx > 0 + node = stream.output[node_idx] + if kind(node) == K"TOMBSTONE" || (skip_trivia && is_trivia(node)) + byte_idx -= node.byte_span + # If this is a non-terminal node, skip its children without + # subtracting their byte_spans, as they're already included in the parent + if is_non_terminal(node) + node_idx -= (1 + node.node_span) + else + node_idx -= 1 + end + else + break + end + end + + return ParseStreamPosition(byte_idx, node_idx) +end + +function peek_behind(stream::ParseStream; kws...) + peek_behind(stream, peek_behind_pos(stream; kws...)) +end + +#------------------------------------------------------------------------------- +# Stream output interface - the `bump_*` and `emit_*` family of functions +# +# Though note bump() really does both input and output + +# Bump up until the `n`th token +# flags and remap_kind are applied to any non-trivia tokens +function _bump_until_n(stream::ParseStream, n::Integer, new_flags, remap_kind=K"None") + if n < stream.lookahead_index + return + end + for i in stream.lookahead_index:n + tok = stream.lookahead[i] + k = kind(tok) + if k == K"EndMarker" + break + end + f = new_flags | flags(tok) + is_trivia = is_whitespace(k) + is_trivia && (f |= TRIVIA_FLAG) + outk = (is_trivia || remap_kind == K"None") ? k : remap_kind + h = SyntaxHead(outk, f) + + # Calculate byte span for this token + if i == stream.lookahead_index + # First token in this batch - calculate span from current stream position + prev_byte = stream.next_byte + else + # Subsequent tokens - use previous token's next_byte + prev_byte = stream.lookahead[i-1].next_byte + end + byte_span = Int(tok.next_byte) - Int(prev_byte) + + # Create terminal RawGreenNode + node = RawGreenNode(h, byte_span, kind(tok)) + push!(stream.output, node) + + # Update next_byte + stream.next_byte += byte_span + end + stream.lookahead_index = n + 1 + # Defuse the time bomb + stream.peek_count = 0 +end + +""" + bump(stream [, flags=EMPTY_FLAGS]; + skip_newlines=false, error, remap_kind) + +Copy the current token from the input stream to the output. Adds the given +flags to the output token (normally this would be the default `EMPTY_FLAGS` or +`TRIVIA_FLAG`). + +Keyword arguments: +* `skip_newlines` - if `true`, newlines are treated as whitespace. +* `error` - if set, emit an error for this token +* `remap_kind` - the kind of the token in the output token stream if it needs + to be modified. +""" +function bump(stream::ParseStream, flags=EMPTY_FLAGS; skip_newlines=false, + error=nothing, remap_kind::Kind=K"None") + emark = position(stream) + _bump_until_n(stream, _lookahead_index(stream, 1, skip_newlines), flags, remap_kind) + if !isnothing(error) + emit(stream, emark, K"error", flags, error=error) + end + # Return last token location in output if needed for reset_node! + return position(stream) +end + +""" +Bump comments and whitespace tokens preceding the next token + +**Skips newlines** by default. Set skip_newlines=false to avoid that. +""" +function bump_trivia(stream::ParseStream, flags=EMPTY_FLAGS; + skip_newlines=true, error=nothing) + emark = position(stream) + _bump_until_n(stream, _lookahead_index(stream, 1, skip_newlines) - 1, EMPTY_FLAGS) + if !isnothing(error) + emit(stream, emark, K"error", flags, error=error) + end + return position(stream) +end + +""" +Bump an invisible zero-width token into the output + +This is useful when surrounding syntax implies the presence of a token. For +example, `2x` means `2*x` via the juxtaposition rules. +""" +function bump_invisible(stream::ParseStream, kind, flags=EMPTY_FLAGS; + error=nothing) + b = stream.next_byte + h = SyntaxHead(kind, flags) + # Zero-width token + node = RawGreenNode(h, 0, kind) + push!(stream.output, node) + # No need to update next_byte for zero-width token + if !isnothing(error) + emit_diagnostic(stream, b:b-1, error=error) + end + stream.peek_count = 0 + return position(stream) +end + +""" +Bump several tokens, gluing them together into a single token + +This is for use in special circumstances where the parser needs to resolve +lexing ambiguities. There's no special whitespace handling — bump any +whitespace if necessary with bump_trivia. +""" +function bump_glue(stream::ParseStream, kind, flags) + i = stream.lookahead_index + h = SyntaxHead(kind, flags) + # Calculate byte span for glued tokens + start_byte = stream.next_byte + end_byte = stream.lookahead[i+1].next_byte + byte_span = end_byte - start_byte + + node = RawGreenNode(h, byte_span, kind) + push!(stream.output, node) + stream.next_byte += byte_span + stream.lookahead_index += 2 + stream.peek_count = 0 + return position(stream) +end + +""" +Reset kind or flags of an existing node in the output stream + +This is a hack, but in some limited occasions the trailing syntax may change +the kind or flags of a token in a way which would require unbounded lookahead +in a recursive descent parser. Modifying the output with reset_node! is useful +in those cases. +""" +function reset_node!(stream::ParseStream, pos::ParseStreamPosition; + kind=nothing, flags=nothing) + node = stream.output[pos.node_index] + stream.output[pos.node_index] = reset_node(node, kind, flags) +end + +""" +Move `numbytes` from the range at output position `pos+1` to the output +position `pos`. If the donor range becomes empty, mark it dead with +K"TOMBSTONE" and return `true`, otherwise return `false`. + +Hack alert! This is used only for managing the complicated rules related to +dedenting triple quoted strings. +""" +function steal_token_bytes!(stream::ParseStream, pos::ParseStreamPosition, numbytes) + i = pos.node_index + t1 = stream.output[i] + t2 = stream.output[i+1] + @assert is_terminal(t1) && is_terminal(t2) + + stream.output[i] = RawGreenNode(t1.head, t1.byte_span + numbytes, + t1.orig_kind) + + t2_is_empty = t2.byte_span == numbytes + head2 = t2_is_empty ? SyntaxHead(K"TOMBSTONE", EMPTY_FLAGS) : t2.head + stream.output[i+1] = RawGreenNode(head2, t2.byte_span - numbytes, + t2.orig_kind) + return t2_is_empty +end + +# Get position of last item emitted into the output stream +function Base.position(stream::ParseStream) + byte_idx = stream.next_byte + node_idx = length(stream.output) + + ParseStreamPosition(byte_idx, node_idx) +end + +""" + emit(stream, mark, kind, flags = EMPTY_FLAGS; error=nothing) + +Emit a new non-terminal node into the output which covers source bytes from `mark` to +the end of the most recent token which was `bump()`'ed. The starting `mark` +should be a previous return value of `position()`. The emitted node will have +its `node_span` set to the number of nodes emitted since `mark`. +""" +function emit(stream::ParseStream, mark::ParseStreamPosition, kind::Kind, + flags::RawFlags = EMPTY_FLAGS; error=nothing) + # Calculate byte span from mark position to current + mark_byte = mark.byte_index + current_byte = stream.next_byte + byte_span = current_byte - mark_byte + + # Calculate node span (number of children, exclusive of the node itself) + node_span = length(stream.output) - mark.node_index + + # Create non-terminal RawGreenNode + node = RawGreenNode(SyntaxHead(kind, flags), byte_span, node_span) + + if !isnothing(error) + emit_diagnostic(stream, mark_byte:current_byte-1, error=error) + end + + push!(stream.output, node) + # Note: emit() for non-terminals doesn't advance next_byte + # because it's a range over already-emitted tokens + return position(stream) +end + +function emit_diagnostic(stream::ParseStream, byterange::AbstractUnitRange; kws...) + emit_diagnostic(stream.diagnostics, byterange; kws...) + return nothing +end + +""" +Emit a diagnostic at the position of the next token + +If `whitespace` is true, the diagnostic is positioned on the whitespace before +the next token. Otherwise it's positioned at the next token as returned by `peek()`. +""" +function emit_diagnostic(stream::ParseStream; whitespace=false, kws...) + i = _lookahead_index(stream, 1, true) + begin_tok_i = i + end_tok_i = i + if whitespace + # It's the whitespace which is the error. Find the range of the current + # whitespace. + begin_tok_i = stream.lookahead_index + end_tok_i = is_whitespace(stream.lookahead[i]) ? + i : max(stream.lookahead_index, i - 1) + end + fbyte = lookahead_token_first_byte(stream, begin_tok_i) + lbyte = lookahead_token_last_byte(stream, end_tok_i) + emit_diagnostic(stream, fbyte:lbyte; kws...) + return nothing +end + +function emit_diagnostic(stream::ParseStream, mark::ParseStreamPosition; trim_whitespace=true, kws...) + # Find the byte range from mark to current position + start_byte = mark.byte_index + end_byte = stream.next_byte - 1 + + if trim_whitespace + # TODO: Implement whitespace trimming for unified output + # This would require scanning the output array + end + + emit_diagnostic(stream, start_byte:end_byte; kws...) +end + +function emit_diagnostic(stream::ParseStream, mark::ParseStreamPosition, + end_mark::ParseStreamPosition; kws...) + emit_diagnostic(stream, mark.byte_index:end_mark.byte_index-1; kws...) +end + +function emit_diagnostic(diagnostics::AbstractVector{Diagnostic}, + byterange::AbstractUnitRange; kws...) + push!(diagnostics, Diagnostic(first(byterange), last(byterange); kws...)) +end + +# Tree construction from the list of text ranges held by ParseStream + +# API for extracting results from ParseStream + +function sourcetext(stream::ParseStream; steal_textbuf=false) + Base.depwarn("Use of `sourcetext(::ParseStream)` is deprecated. Use `SourceFile(stream)` instead", :sourcetext) + root = stream.text_root + # The following kinda works but makes the return type of this method type + # unstable. (Also codeunit(root) == UInt8 doesn't imply UTF-8 encoding?) + # if root isa AbstractString && codeunit(root) == UInt8 + # return root + str = if root isa String || root isa SubString + root + elseif steal_textbuf + String(stream.textbuf) + else + # Safe default for other cases is to copy the buffer. Technically this + # could possibly be avoided in some situations, but might have side + # effects such as mutating stream.text_root or stealing the storage of + # stream.textbuf + String(copy(stream.textbuf)) + end + SubString(str, first_byte(stream), thisind(str, last_byte(stream))) +end + +function SourceFile(stream::ParseStream; kws...) + fbyte = first_byte(stream) + lbyte = last_byte(stream) + if !isempty(stream.diagnostics) + lbyte = max(lbyte, maximum(last_byte(d) for d in stream.diagnostics)) + end + # See also sourcetext() + srcroot = stream.text_root + str = if srcroot isa String + SubString(srcroot, fbyte, thisind(srcroot, lbyte)) + elseif srcroot isa SubString{String} + SubString(srcroot, fbyte, thisind(srcroot, lbyte)) + else + SubString(String(stream.textbuf[fbyte:lbyte])) + end + return SourceFile(str; first_index=first_byte(stream), kws...) +end + +""" + unsafe_textbuf(stream) + +Return the `Vector{UInt8}` text buffer being parsed by this `ParseStream`. + +!!! warning + The caller must hold a reference to `stream` while using textbuf +""" +unsafe_textbuf(stream) = stream.textbuf + +first_byte(stream::ParseStream) = first(stream.output).byte_span + 1 # After sentinel +last_byte(stream::ParseStream) = stream.next_byte - 1 +any_error(stream::ParseStream) = any_error(stream.diagnostics) + +# Return last non-whitespace byte which was parsed +function last_non_whitespace_byte(stream::ParseStream) + byte_pos = stream.next_byte + for i = length(stream.output):-1:1 + node = stream.output[i] + if is_terminal(node) + if kind(node) in KSet"Comment Whitespace NewlineWs ErrorEofMultiComment" || kind(node) == K"error" && node.byte_span == 0 + byte_pos -= node.byte_span + else + return byte_pos - 1 + end + end + end + return first_byte(stream) - 1 +end + +function Base.empty!(stream::ParseStream) + # Keep only the sentinel + if !isempty(stream.output) && kind(stream.output[1]) == K"TOMBSTONE" + resize!(stream.output, 1) + else + empty!(stream.output) + # Restore sentinel node + push!(stream.output, RawGreenNode(SyntaxHead(K"TOMBSTONE", EMPTY_FLAGS), 0, K"TOMBSTONE")) + end + # Reset next_byte to initial position + stream.next_byte = 1 +end diff --git a/JuliaSyntax/src/core/source_files.jl b/JuliaSyntax/src/core/source_files.jl new file mode 100644 index 0000000000000..0ee9cec8b05e5 --- /dev/null +++ b/JuliaSyntax/src/core/source_files.jl @@ -0,0 +1,391 @@ +#------------------------------------------------------------------------------- +# Generic functions for source text, source location computation and formatting +# functions + +""" + sourcefile(x) + +Get the source file object (usually `SourceFile`) for a given syntax object +`x`. The source file along with a byte range may be used to compute +`source_line()`, `source_location()`, `filename()`, etc. +""" +function sourcefile +end + +""" + byte_range(x) + +Return the range of bytes which `x` covers in the source text. See also +[`char_range`](@ref). +""" +function byte_range +end + +""" + char_range(x) + +Compute the range in *character indices* over the source text for syntax object +`x`. If you want to index the source string you need this, rather than +[`byte_range`](@ref). +""" +function char_range(x) + br = byte_range(x) + first(br):thisind(sourcefile(x), last(br)) +end + +""" + first_byte(x) + +Return the first byte of `x` in the source text. +""" +first_byte(x) = first(byte_range(x)) + +""" + last_byte(x) + +Return the last byte of `x` in the source text. +""" +last_byte(x) = last(byte_range(x)) + +""" + filename(x) + +Get file name associated with `source`, or an empty string if one didn't exist. + +For objects `x` such as syntax trees, defers to `filename(sourcefile(x))` by +default. +""" +function filename(x) + source = sourcefile(x) + isnothing(source) ? "" : filename(source) +end + +""" + source_line(x) + source_line(source::SourceFile, byte_index::Integer) + +Get the line number of the first line on which object `x` appears. In the +second form, get the line number at the given `byte_index` within `source`. +""" +source_line(x) = source_line(sourcefile(x), first_byte(x)) + +""" + source_location(x) + source_location(source::SourceFile, byte_index::Integer) + + source_location(LineNumberNode, x) + source_location(LineNumberNode, source, byte_index) + +Get `(line,column)` of the first byte where object `x` appears in the source. +The second form allows one to be more precise with the `byte_index`, given the +source file. + +Providing `LineNumberNode` as the first argument will return the line and file +name in a line number node object. +""" +source_location(x) = source_location(sourcefile(x), first_byte(x)) + +""" + sourcetext(x) + +Get the full source text of syntax object `x` +""" +function sourcetext(x) + view(sourcefile(x), byte_range(x)) +end + +""" + highlight(io, x; color, note, notecolor, + context_lines_before, context_lines_inner, context_lines_after) + + highlight(io::IO, source::SourceFile, range::UnitRange; kws...) + +Print the lines of source code surrounding `x` which is highlighted with +background `color` and underlined with markers in the text. A `note` in +`notecolor` may be provided as annotation. By default, `x` should be an object +with `sourcefile(x)` and `byte_range(x)` implemented. + +The context arguments `context_lines_before`, etc, refer to the number of +lines of code which will be printed as context before and after, with `inner` +referring to context lines inside a multiline region. + +The second form shares the keywords of the first but allows an explicit source +file and byte range to be supplied. +""" +function highlight(io::IO, x; kws...) + highlight(io, sourcefile(x), byte_range(x); kws...) +end + + +#------------------------------------------------------------------------------- +""" + SourceFile(code [; filename=nothing, first_line=1, first_index=1]) + +UTF-8 source text with associated file name and line number, storing the +character indices of the start of each line. `first_line` and `first_index` +can be used to specify the line number and index of the first character of +`code` within a larger piece of source text. + +`SourceFile` may be indexed via `getindex` or `view` to get a string. Line +information for a byte offset can be looked up via the `source_line`, +`source_location` and `source_line_range` functions. +""" +struct SourceFile + # TODO: Rename SourceFile -> SourceText / SourceChunk / SourceIndex / SourceLineIndex ? + # See https://github.com/JuliaLang/JuliaSyntax.jl/issues/190 + code::SubString{String} + # Offset of `code` within a larger chunk of source text + byte_offset::Int + filename::Union{Nothing,String} + # first_column::Int ?? + first_line::Int + # String index of start of every line + line_starts::Vector{Int} +end + +Base.hash(s::SourceFile, h::UInt) = + hash(s.code, hash(s.byte_offset, hash(s.filename, hash(s.first_line, hash(s.line_starts, h))))) +function Base.:(==)(a::SourceFile, b::SourceFile) + a.code == b.code && a.byte_offset == b.byte_offset && a.filename == b.filename && + a.first_line == b.first_line && a.line_starts == b.line_starts +end + +function SourceFile(code::AbstractString; filename=nothing, first_line=1, + first_index=1) + line_starts = Int[1] + for i in eachindex(code) + # The line is considered to start after the `\n` + code[i] == '\n' && push!(line_starts, i+1) + end + SourceFile(code, first_index-1, filename, first_line, line_starts) +end + +function SourceFile(; filename, kwargs...) + SourceFile(read(filename, String); filename=filename, kwargs...) +end + +# Get line number of the given byte within the code +function _source_line_index(source::SourceFile, byte_index) + searchsortedlast(source.line_starts, byte_index - source.byte_offset) +end +_source_line(source::SourceFile, lineidx) = lineidx + source.first_line - 1 + +function source_location(::Type{LineNumberNode}, x) + source_location(LineNumberNode, sourcefile(x), first_byte(x)) +end + +source_line(source::SourceFile, byte_index::Integer) = + _source_line(source, _source_line_index(source, byte_index)) + +function filename(source::SourceFile) + f = source.filename + !isnothing(f) ? f : "" +end + +function source_location(source::SourceFile, byte_index::Integer) + lineidx = _source_line_index(source, byte_index) + i = source.line_starts[lineidx] + column = 1 + while i < byte_index - source.byte_offset + i = nextind(source.code, i) + column += 1 + end + _source_line(source, lineidx), column +end + +""" +Get byte range of the source line at byte_index, buffered by +`context_lines_before` and `context_lines_after` before and after. +""" +function source_line_range(source::SourceFile, byte_index::Integer; + context_lines_before=0, context_lines_after=0) + lineidx = _source_line_index(source, byte_index) + fbyte = source.line_starts[max(lineidx-context_lines_before, 1)] + lline = lineidx + context_lines_after + lbyte = lline >= lastindex(source.line_starts) ? + ncodeunits(source.code) : source.line_starts[lline + 1] - 1 + + return (fbyte + source.byte_offset, + lbyte + source.byte_offset) +end + +function source_location(::Type{LineNumberNode}, source::SourceFile, byte_index::Integer) + fn = filename(source) + LineNumberNode(source_line(source, byte_index), isempty(fn) ? nothing : Symbol(fn)) +end + +function Base.show(io::IO, ::MIME"text/plain", source::SourceFile) + fn = filename(source) + header = "## SourceFile$(isempty(fn) ? "" : " ")$fn ##" + print(io, header, "\n") + heightlim = displaysize(io)[1] ÷ 2 + if !get(io, :limit, false) || length(source.line_starts) <= heightlim + print(io, source.code) + else + r1 = source_line_range(source, 1, context_lines_after=heightlim-3) + print(io, view(source, r1[1]:r1[2])) + println(io, "⋮") + end +end + +function Base.getindex(source::SourceFile, rng::AbstractUnitRange) + i = first(rng) - source.byte_offset + # Convert byte range into unicode String character range. + # Assumes valid unicode! (SubString doesn't give us a reliable way to opt + # out of the valid unicode check. The SubString{String} inner constructor + # has some @boundscheck, but using @inbounds depends on inlining choices.) + j = prevind(source.code, last(rng) + 1 - source.byte_offset) + source.code[i:j] +end + +# TODO: Change view() here to `sourcetext` ? +function Base.view(source::SourceFile, rng::AbstractUnitRange) + i = first(rng) - source.byte_offset + j = prevind(source.code, last(rng) + 1 - source.byte_offset) + SubString(source.code, i, j) +end + +function Base.getindex(source::SourceFile, i::Integer) + source.code[i - source.byte_offset] +end + +function Base.thisind(source::SourceFile, i::Integer) + thisind(source.code, i - source.byte_offset) + source.byte_offset +end + +function Base.nextind(source::SourceFile, i::Integer) + nextind(source.code, i - source.byte_offset) + source.byte_offset +end + +Base.firstindex(source::SourceFile) = firstindex(source.code) + source.byte_offset +Base.lastindex(source::SourceFile) = lastindex(source.code) + source.byte_offset + +""" + sourcetext(source::SourceFile) + +Get the full source text of a `SourceFile` as a string. +""" +function sourcetext(source::SourceFile) + return source.code +end + + +#------------------------------------------------------------------------------- +# Tools for highlighting source ranges +function _print_marker_line(io, prefix_str, str, underline, singleline, color, + note, notecolor) + # Whitespace equivalent in length to `prefix_str` + # Getting exactly the same width of whitespace as `str` is tricky. + # Especially for mixtures of tabs and spaces. + # tabs are zero width according to textwidth + indent = join(isspace(c) ? c : repeat(' ', textwidth(c)) for c in prefix_str) + + # Assume tabs are 4 wide rather than 0. (fixme: implement tab alignment?) + w = textwidth(str) + 4*count(c->c=='\t', str) + if !isempty(indent) + indent = "#" * (first(indent) == '\t' ? indent : indent[nextind(indent,1):end]) + end + + startstr, endstr, singlestart = underline ? ("└","┘","╙") : ("┌","┐","╓") + + markline = + if singleline + w == 0 ? string(indent, startstr) : + w == 1 ? string(indent, singlestart) : + string(indent, startstr, repeat('─', w-2), endstr) + else + if underline && isempty(indent) && w > 1 + string('#', repeat('─', w-2), endstr) + else + s,e = underline ? ("", endstr) : (startstr, "") + w == 0 ? string(indent, s, e) : + string(indent, s, repeat('─', w-1), e) + end + end + if note isa AbstractString + markline *= " ── " + end + _printstyled(io, markline; fgcolor=color) + if !isnothing(note) + if note isa AbstractString + _printstyled(io, note, fgcolor=notecolor) + else + note(io, indent, w) + end + end +end + +function highlight(io::IO, source::SourceFile, range::UnitRange; + color=(120,70,70), context_lines_before=2, + context_lines_inner=1, context_lines_after=2, + note=nothing, notecolor=nothing) + p = first(range) + q = last(range) + + x,y = source_line_range(source, p; + context_lines_before=context_lines_before, + context_lines_after=context_lines_inner) + a,b = source_line_range(source, p) + q1 = max(q, p) # Ignore q for empty ranges + c,d = source_line_range(source, q1) + z,w = source_line_range(source, q1; + context_lines_before=context_lines_inner, + context_lines_after=context_lines_after) + + p_line = source_line(source, p) + q_line = source_line(source, q) + + marker_line_color = :light_black + + if p_line >= q_line + # x----------------- + # a---p-------q----b + # # └───────┘ ── note + # -----------------w + + hitext = source[p:q] + print(io, source[x:p-1]) + _printstyled(io, hitext; bgcolor=color) + #print(io, source[q+1:d]) + print(io, source[nextind(source,q):d]) + if d >= firstindex(source) && source[thisind(source, d)] != '\n' + print(io, "\n") + end + _print_marker_line(io, source[a:p-1], hitext, true, true, marker_line_color, note, notecolor) + else + # x -------------- + # # ┌───── + # a---p----b + # --------------y + # --------------- + # z-------------- + # c --------q----d + # #───────────┘ ── note + # -----------------w + + prefix1 = source[a:p-1] + print(io, source[x:a-1]) + _print_marker_line(io, prefix1, source[p:b], false, false, marker_line_color, nothing, notecolor) + print(io, '\n') + print(io, prefix1) + if q_line - p_line - 1 <= 2*context_lines_inner + # The diagnostic range is compact and we show the whole thing + _printstyled(io, source[p:q]; bgcolor=color) + else + # Or large and we truncate the code to show only the region around the + # start and end of the error. + _printstyled(io, source[p:y]; bgcolor=color) + print(io, "⋮\n") + _printstyled(io, source[z:q]; bgcolor=color) + end + print(io, source[nextind(source, q):d]) + source[thisind(source, d)] == '\n' || print(io, "\n") + qline = source[c:q] + _print_marker_line(io, "", qline, true, false, marker_line_color, note, notecolor) + end + if context_lines_after > 0 && d+1 <= lastindex(source) + print(io, '\n') + w1 = source[thisind(source, w)] == '\n' ? w - 1 : w + print(io, source[d+1:w1]) + end +end diff --git a/JuliaSyntax/src/core/tree_cursors.jl b/JuliaSyntax/src/core/tree_cursors.jl new file mode 100644 index 0000000000000..75a5c0e44008f --- /dev/null +++ b/JuliaSyntax/src/core/tree_cursors.jl @@ -0,0 +1,175 @@ +using Base.Iterators: Reverse + +""" + prev_sibling_assumed(cursor::GreenTreeCursor)::Union{Nothing, GreenTreeCursor} + prev_sibling_assumed(cursor::RedTreeCursor)::Union{Nothing, RedTreeCursor} + +Gives the previous sibling of the current node, but makes the assumption that +there is one or that we are at the top level. +Without knowing the parent, we cannot otherwise know which the last sibling is, +unless we are at the top level in which case `nothing` is returned. +""" +function prev_sibling_assumed end + +""" + GreenTreeCursor + +Represents a cursors into a ParseStream output buffer that makes it easy to +work with the green tree representation. +""" +struct GreenTreeCursor + parser_output::Vector{RawGreenNode} + position::UInt32 +end +GreenTreeCursor(stream::ParseStream) = GreenTreeCursor(stream.output, length(stream.output)) +this(node::GreenTreeCursor) = node.parser_output[node.position] + +const SENTINEL_INDEX = UInt32(1) +function prev_sibling_assumed(cursor::GreenTreeCursor) + next_idx = cursor.position - this(cursor).node_span - UInt32(1) + next_idx == SENTINEL_INDEX && return nothing + GreenTreeCursor(cursor.parser_output, next_idx) +end + +function Base.in(child::GreenTreeCursor, parent::GreenTreeCursor) + @assert child.parser_output === parent.parser_output + child.position < parent.position || return false + return child.position >= parent.position - this(parent).node_span +end + +# Debug printing +function Base.show(io::IO, node::GreenTreeCursor) + print(io, Base.summary(this(node)), " @", node.position) +end + +# Reverse iterator interface +Base.reverse(node::GreenTreeCursor) = Base.Iterators.Reverse(node) +Base.IteratorSize(::Type{Reverse{GreenTreeCursor}}) = Base.SizeUnknown() +@inline function Base.iterate(node::Reverse{GreenTreeCursor}, + (next_idx, final)::NTuple{2, UInt32} = + (node.itr.position-UInt32(1), node.itr.position - this(node.itr).node_span - UInt32(1)))::Union{Nothing, Tuple{GreenTreeCursor, NTuple{2, UInt32}}} + node = node.itr + while true + next_idx == final && return nothing + next_node = GreenTreeCursor(node.parser_output, next_idx) + nrgn = this(next_node) + if getfield(nrgn, :head).kind == K"TOMBSTONE" + # TOMBSTONED nodes are counted as part of the size of the tree, but + # do not contribute either byte ranges or children. + next_idx -= UInt32(1) + continue + end + # Inlined prev_sibling_assumed + new_next_idx = next_idx - nrgn.node_span - UInt32(1) + return (next_node, (new_next_idx, final)) + end +end + +# Accessors / predicates +is_leaf(node::GreenTreeCursor) = !is_non_terminal(this(node)) +head(node::GreenTreeCursor) = this(node).head +treesize(node::GreenTreeCursor) = this(node).node_span +is_non_terminal(node::GreenTreeCursor) = is_non_terminal(this(node)) + +""" + span(node) + +Get the number of bytes this node covers in the source text. +""" +span(node::GreenTreeCursor) = this(node).byte_span + +""" + RedTreeCursor + +Wraps a `GreenTreeCursor` to keep track of the absolute position of the node +in the original source text. +""" +struct RedTreeCursor + green::GreenTreeCursor + # The last byte that is still part of the node + byte_end::UInt32 +end +RedTreeCursor(stream::ParseStream) = RedTreeCursor( + GreenTreeCursor(stream), stream.next_byte - UInt32(1)) + +function prev_sibling_assumed(cursor::RedTreeCursor) + prevgreen = prev_sibling_assumed(cursor.green) + if prevgreen === nothing + return nothing + end + return RedTreeCursor(prevgreen, cursor.byte_end - span(cursor)) +end + + +Base.reverse(node::RedTreeCursor) = Base.Iterators.Reverse(node) +Base.IteratorSize(::Type{Reverse{RedTreeCursor}}) = Base.SizeUnknown() +@inline function Base.iterate(node::Reverse{RedTreeCursor})::Union{Nothing, Tuple{RedTreeCursor, NTuple{3, UInt32}}} + r = iterate(Reverse(node.itr.green)) + return _iterate_red_cursor(r, node.itr.byte_end) +end + +@inline function Base.iterate(node::Reverse{RedTreeCursor}, state::NTuple{3, UInt32})::Union{Nothing, Tuple{RedTreeCursor, NTuple{3, UInt32}}} + r = iterate(Reverse(node.itr.green), Base.tail(state)) + return _iterate_red_cursor(r, first(state)) +end + +@inline function _iterate_red_cursor(r, byte_end) + r === nothing && return nothing + next_node, next_idx = r + return RedTreeCursor(next_node, byte_end), + (byte_end - span(next_node), next_idx...) +end + +is_leaf(node::RedTreeCursor) = is_leaf(node.green) +head(node::RedTreeCursor) = head(node.green) +span(node::RedTreeCursor) = span(node.green) +byte_range(node::RedTreeCursor) = (node.byte_end - span(node.green) + UInt32(1)):node.byte_end +treesize(node::RedTreeCursor) = treesize(node.green) +is_non_terminal(node::RedTreeCursor) = is_non_terminal(node.green) + +function Base.show(io::IO, node::RedTreeCursor) + print(io, node.green, " [", byte_range(node), "]") +end + +has_toplevel_siblings(cursor::GreenTreeCursor) = + treesize(cursor)+1 != length(cursor.parser_output)-1 +has_toplevel_siblings(cursor::RedTreeCursor) = + has_toplevel_siblings(cursor.green) +struct TopLevelSiblingIterator{C} + cursor::C +end + +function reverse_toplevel_siblings(cursor::RedTreeCursor) + @assert cursor.green.position == length(cursor.green.parser_output) + TopLevelSiblingIterator(cursor) +end + +function reverse_toplevel_siblings(cursor::GreenTreeCursor) + @assert cursor.position == length(cursor.parser_output) + TopLevelSiblingIterator(cursor) +end + +function Base.iterate(tsi::TopLevelSiblingIterator) + return (tsi.cursor, tsi.cursor) +end +function Base.iterate(cursor::TopLevelSiblingIterator{C}, last::C) where {C} + this = prev_sibling_assumed(last) + this === nothing && return nothing + return (this, this) +end + +# HACK: Force inlining of `filter` for our cursors to avoid significant perf +# degradation. +@inline function Base.iterate(f::Iterators.Filter{<:Any, Iterators.Reverse{T}}, state...) where {T<:Union{RedTreeCursor, GreenTreeCursor}} + y = iterate(f.itr, state...) + while y !== nothing + if f.flt(y[1]) + return y + end + y = iterate(f.itr, y[2]) + end + nothing +end + +Base.in(child::GreenTreeCursor, parent::RedTreeCursor) = + in(child, parent.green) diff --git a/JuliaSyntax/src/integration/expr.jl b/JuliaSyntax/src/integration/expr.jl new file mode 100644 index 0000000000000..dd3f42f8612d2 --- /dev/null +++ b/JuliaSyntax/src/integration/expr.jl @@ -0,0 +1,716 @@ +#------------------------------------------------------------------------------- +# Conversion to Base.Expr + +""" + @isexpr(ex, head) + @isexpr(ex, head, nargs) + +Type inference friendly replacement for `Meta.isexpr`. + +When using the pattern +```julia +if @isexpr(ex, headsym) + body +end +``` +Julia's type inference knows `ex isa Expr` inside `body`. But `Meta.isexpr` +hides this information from the compiler, for whatever reason. +""" +macro isexpr(ex, head) + ex isa Symbol || error("First argument to `@isexpr` must be a variable name") + :($(esc(ex)) isa Expr && $(esc(ex)).head == $(esc(head))) +end + +macro isexpr(ex, head, nargs) + ex isa Symbol || error("First argument to `@isexpr` must be a variable name") + :($(esc(ex)) isa Expr && + $(esc(ex)).head == $(esc(head)) && + length($(esc(ex)).args) == $(esc(nargs))) +end + +function _reorder_parameters!(args::Vector{Any}, params_pos::Int) + p = 0 + for i = length(args):-1:1 + ai = args[i] + if !@isexpr(ai, :parameters) + break + end + p = i + end + if p == 0 + return + end + # nest frankentuples parameters sections + for i = length(args)-1:-1:p + pushfirst!((args[i]::Expr).args, pop!(args)) + end + # Move parameters to args[params_pos] + insert!(args, params_pos, pop!(args)) +end + +function _strip_parens(ex::Expr) + while true + if @isexpr(ex, :parens) + if length(ex.args) == 1 + ex = ex.args[1] + else + # Only for error cases + return Expr(:block, ex.args...) + end + else + return ex + end + end +end + + +reverse_nontrivia_children(cursor::RedTreeCursor) = Iterators.filter(should_include_node, Iterators.reverse(cursor)) +reverse_nontrivia_children(cursor) = Iterators.filter(should_include_node, Iterators.reverse(children(cursor))) + +# Julia string literals in a `K"string"` node may be split into several chunks +# interspersed with trivia in two situations: +# 1. Triple quoted string indentation is trivia +# 2. An \ before newline removes the newline and any following indentation +# +# This function concatenates adjacent string chunks together as done in the +# reference parser. +function _string_to_Expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt32) + ret = Expr(:string) + it = reverse_nontrivia_children(cursor) + r = iterate(it) + while r !== nothing + (child, state) = r + ex = node_to_expr(child, source, txtbuf, txtbuf_offset) + if isa(ex, String) + # This branch combines consequent string chunks together. + # It's unrolled once to avoid unnecessary allocations. + r = iterate(it, state) + if r === nothing + pushfirst!(ret.args, ex) + continue + end + (child, state) = r + ex2 = node_to_expr(child, source, txtbuf, txtbuf_offset) + if !isa(ex2, String) + pushfirst!(ret.args, ex) + ex = ex2 + # Fall through to process `ex` (!::String) + else + strings = String[ex2, ex] # Note: reversed order since we're iterating backwards + r = iterate(it, state) + while r !== nothing + (child, state) = r + ex = node_to_expr(child, source, txtbuf, txtbuf_offset) + isa(ex, String) || break + pushfirst!(strings, ex) + r = iterate(it, state) + end + buf = IOBuffer() + for s in strings + write(buf, s) + end + pushfirst!(ret.args, String(take!(buf))) + r === nothing && break + # Fall through to process `ex` (!::String) + end + end + # ex not a string + if @isexpr(ex, :parens, 1) + ex = _strip_parens(ex) + if ex isa String + # Wrap interpolated literal strings in (string) so we can + # distinguish them from the surrounding text (issue #38501) + # Ie, "$("str")" vs "str" + # https://github.com/JuliaLang/julia/pull/38692 + ex = Expr(:string, ex) + end + end + @assert ex !== nothing + pushfirst!(ret.args, ex) + r = iterate(it, state) + end + + if length(ret.args) == 1 && ret.args[1] isa String + # If there's a single string remaining after joining, we unwrap + # to give a string literal. + # """\n a\n b""" ==> "a\nb" + return only(ret.args) + else + # This only happens when the kind is K"string" or when an error has occurred. + return ret + end +end + +# Shared fixups for Expr children in cases where the type of the parent node +# affects the child layout. +function fixup_Expr_child(::Type, head::SyntaxHead, @nospecialize(arg), first::Bool) + isa(arg, Expr) || return arg + k = kind(head) + eq_to_kw_in_call = ((k == K"call" || k == K"dotcall") && + is_prefix_call(head)) || k == K"ref" + eq_to_kw_in_params = k != K"vect" && k != K"curly" && + k != K"braces" && k != K"ref" + coalesce_dot = k in KSet"call dotcall curly" || + (k == K"quote" && has_flags(head, COLON_QUOTE)) + was_parens = @isexpr(arg, :parens) + arg = _strip_parens(arg) + if @isexpr(arg, :(=)) && eq_to_kw_in_call && !first + arg = Expr(:kw, arg.args...) + elseif k != K"parens" && @isexpr(arg, :., 1) && arg.args[1] isa Tuple + # This undoes the "Hack" below" + h, a = arg.args[1]::Tuple{SyntaxHead,Any} + arg = ((!was_parens && coalesce_dot && first) || + is_syntactic_operator(h)) ? + Symbol(".", a) : Expr(:., a) + elseif @isexpr(arg, :parameters) && eq_to_kw_in_params + pargs = arg.args + for j = 1:length(pargs) + pj = pargs[j] + if @isexpr(pj, :(=)) + pargs[j] = Expr(:kw, pj.args...) + end + end + end + return arg +end + +# Remove the `do` block from the final position in a function/macro call arg list +function _extract_do_lambda!(args::Vector{Any}) + if length(args) > 1 && Meta.isexpr(args[end], :do_lambda) + do_ex = pop!(args)::Expr + return Expr(:->, do_ex.args...) + else + return nothing + end +end + +function _append_iterspec!(args::Vector{Any}, @nospecialize(ex)) + if @isexpr(ex, :iteration) + for iter in ex.args::Vector{Any} + push!(args, Expr(:(=), iter.args...)) + end + else + push!(args, ex) + end + return args +end + +function parseargs!(retexpr::Expr, loc::LineNumberNode, cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt32) + args = retexpr.args + firstchildhead = secondchildhead = head(cursor) + firstchildrange::UnitRange{UInt32} = byte_range(cursor) + itr = reverse_nontrivia_children(cursor) + r = iterate(itr) + while r !== nothing + (child, state) = r + r = iterate(itr, state) + expr = node_to_expr(child, source, txtbuf, txtbuf_offset) + @assert expr !== nothing + secondchildhead = firstchildhead + firstchildhead = head(child) + firstchildrange = byte_range(child) + pushfirst!(args, fixup_Expr_child( + typeof(cursor), head(cursor), expr, r === nothing)) + end + return (firstchildhead, secondchildhead, firstchildrange) +end + +function version_to_expr(node) + @assert kind(node) === K"VERSION" + nv = numeric_flags(flags(node)) + return VersionNumber(1, nv ÷ 10, nv % 10) +end + +_expr_leaf_val(node::SyntaxNode, _...) = node.val +_expr_leaf_val(cursor::RedTreeCursor, txtbuf::Vector{UInt8}, txtbuf_offset::UInt32) = + parse_julia_literal(txtbuf, head(cursor), byte_range(cursor) .+ txtbuf_offset) +# Extended in JuliaLowering to support `node_to_expr(::SyntaxTree, ...)` + +# Convert `cursor` (SyntaxNode or RedTreeCursor) to an Expr +# `source` is a SourceFile, or if node was an Expr originally, a LineNumberNode +function node_to_expr(cursor, source, txtbuf::Vector{UInt8}, txtbuf_offset::UInt32=UInt32(0)) + if !should_include_node(cursor) + return nothing + end + + nodehead = head(cursor) + k = kind(cursor) + srcrange::UnitRange{UInt32} = byte_range(cursor) + if is_leaf(cursor) + if is_error(k) + return k == K"error" ? + Expr(:error) : + Expr(:error, "$(_token_error_descriptions[k]): `$(source[srcrange])`") + elseif k == K"VERSION" + return version_to_expr(nodehead) + else + val = _expr_leaf_val(cursor, txtbuf, txtbuf_offset) + if val isa Union{Int128,UInt128,BigInt} + # Ignore the values of large integers and convert them back to + # symbolic/textual form for compatibility with the Expr + # representation of these. + str = replace(source[srcrange], '_'=>"") + macname = val isa Int128 ? Symbol("@int128_str") : + val isa UInt128 ? Symbol("@uint128_str") : + Symbol("@big_str") + return Expr(:macrocall, GlobalRef(Core, macname), nothing, str) + elseif is_identifier(k) + return lower_identifier_name(val, k) + else + return val + end + end + end + + if k == K"string" + return _string_to_Expr(cursor, source, txtbuf, txtbuf_offset) + end + + loc = source_location(LineNumberNode, source, first(srcrange)) + + if k == K"cmdstring" + return Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), loc, + _string_to_Expr(cursor, source, txtbuf, txtbuf_offset)) + end + + headstr = untokenize(nodehead, include_flag_suff=false) + headsym = !isnothing(headstr) ? + Symbol(headstr) : + error("Can't untokenize head of kind $(k)") + retexpr = Expr(headsym) + + # Block gets special handling for extra line number nodes + if k == K"block" || (k == K"toplevel" && !has_flags(nodehead, TOPLEVEL_SEMICOLONS_FLAG)) + args = retexpr.args + for child in reverse_nontrivia_children(cursor) + expr = node_to_expr(child, source, txtbuf, txtbuf_offset) + @assert expr !== nothing + # K"block" does not have special first-child handling, so we do not need to keep track of that here + pushfirst!(args, fixup_Expr_child(typeof(cursor), head(cursor), expr, false)) + pushfirst!(args, source_location(LineNumberNode, source, first(byte_range(child)))) + end + isempty(args) && push!(args, loc) + if k == K"block" && has_flags(nodehead, PARENS_FLAG) + popfirst!(args) + end + return retexpr + end + + # Now recurse to parse all arguments + (firstchildhead, secondchildhead, firstchildrange) = + parseargs!(retexpr, loc, cursor, source, txtbuf, txtbuf_offset) + + return _node_to_expr(retexpr, loc, srcrange, + firstchildhead, secondchildhead, firstchildrange, + nodehead, source) +end + +function adjust_macro_name!(retexpr::Union{Expr, Symbol}) + if retexpr isa Symbol + return lower_identifier_name(retexpr, K"macro_name") + else + retexpr::Expr + if length(retexpr.args) == 2 && retexpr.head == :(.) + arg2 = retexpr.args[2] + if isa(arg2, QuoteNode) && arg2.value isa Symbol + retexpr.args[2] = QuoteNode(lower_identifier_name(arg2.value, K"macro_name")) + end + end + return retexpr + end +end + +# Split out from `node_to_expr` for codesize reasons, to avoid specialization on multiple +# tree types. +@noinline function _node_to_expr(retexpr::Expr, loc::LineNumberNode, + srcrange::UnitRange{UInt32}, + firstchildhead::SyntaxHead, secondchildhead::SyntaxHead, + firstchildrange::UnitRange{UInt32}, + nodehead::SyntaxHead, + source) + args = retexpr.args + k = kind(nodehead) + endloc = source_location(LineNumberNode, source, last(srcrange)) + if (k == K"var" || k == K"char") && length(retexpr.args) == 1 + # `var` and `char` nodes have a single argument which is the value. + # However, errors can add additional errors tokens which we represent + # as e.g. `Expr(:var, ..., Expr(:error))`. + return retexpr.args[1] + elseif k == K"macro_name" + return adjust_macro_name!(retexpr.args[1]) + elseif k == K"?" + retexpr.head = :if + elseif k == K"DotsIdentifier" + n = numeric_flags(flags(nodehead)) + return n == 2 ? :(..) : :(...) + elseif k == K"op=" + if length(args) == 3 + lhs = args[1] + op = args[2] + rhs = args[3] + headstr = string(args[2], '=') + retexpr.head = Symbol(headstr) + retexpr.args = Any[lhs, rhs] + elseif length(args) == 1 + return Symbol(string(args[1], '=')) + end + elseif k == K".op=" + if length(args) == 3 + lhs = args[1] + op = args[2] + rhs = args[3] + headstr = '.' * string(args[2], '=') + retexpr.head = Symbol(headstr) + retexpr.args = Any[lhs, rhs] + else + return Symbol(string('.', args[1], '=')) + end + elseif k == K"macrocall" + if length(args) >= 2 + a2 = args[2] + if @isexpr(a2, :macrocall) && kind(firstchildhead) == K"CmdMacroName" + # Fix up for custom cmd macros like foo`x` + args[2] = a2.args[3] + end + if kind(secondchildhead) == K"VERSION" + # Encode the syntax version into `loc` so that the argument order + # matches what ordinary macros expect. + # Core.MacroSource was added in Julia 1.13+; fall back to plain loc on older versions. + @static if isdefined(Core, :MacroSource) + loc = Core.MacroSource(loc, popat!(args, 2)) + else + popat!(args, 2) # discard the version argument + end + end + end + do_lambda = _extract_do_lambda!(args) + _reorder_parameters!(args, 2) + insert!(args, 2, loc) + if do_lambda isa Expr + return Expr(:do, retexpr, do_lambda) + end + elseif k == K"doc" + retexpr.head = :macrocall + retexpr.args = [GlobalRef(Core, Symbol("@doc")), loc, args...] + elseif k == K"dotcall" || k == K"call" + # Julia's standard `Expr` ASTs have children stored in a canonical + # order which is not always source order. We permute the children + # here as necessary to get the canonical order. + if is_infix_op_call(nodehead) || is_postfix_op_call(nodehead) + args[2], args[1] = args[1], args[2] + end + # Lower (call x ') to special ' head + if is_postfix_op_call(nodehead) && args[1] == Symbol("'") + popfirst!(args) + retexpr.head = Symbol("'") + end + do_lambda = _extract_do_lambda!(args) + # Move parameters blocks to args[2] + _reorder_parameters!(args, 2) + if retexpr.head === :dotcall + funcname = args[1] + if is_prefix_call(nodehead) + retexpr.head = :. + retexpr.args = Any[funcname, Expr(:tuple, args[2:end]...)] + else + # operator calls + retexpr.head = :call + if funcname isa Symbol + args[1] = Symbol(:., funcname) + end # else funcname could be an Expr(:error), just propagate it + end + end + if do_lambda isa Expr + return Expr(:do, retexpr, do_lambda) + end + elseif k == K"." + if length(args) == 2 + a2 = args[2] + if !@isexpr(a2, :quote) && !(a2 isa QuoteNode) + args[2] = QuoteNode(a2) + end + elseif length(args) == 1 + # Hack: Here we preserve the head of the operator to determine whether + # we need to coalesce it with the dot into a single symbol later on. + args[1] = (firstchildhead, args[1]) + end + elseif k == K"ref" || k == K"curly" + # Move parameters blocks to args[2] + _reorder_parameters!(args, 2) + elseif k == K"for" + iters = _append_iterspec!([], args[1]) + args[1] = length(iters) == 1 ? only(iters) : Expr(:block, iters...) + # Add extra line number node for the `end` of the block. This may seem + # useless but it affects code coverage. + push!(args[2].args, endloc) + elseif k == K"while" + # Line number node for the `end` of the block as in `for` loops. + push!(args[2].args, endloc) + elseif k in KSet"tuple vect braces" + # Move parameters blocks to args[1] + _reorder_parameters!(args, 1) + elseif k == K"where" + if length(args) == 2 + a2 = args[2] + if @isexpr(a2, :braces) + a2a = a2.args + _reorder_parameters!(a2a, 2) + retexpr.args = Any[args[1], a2a...] + end + end + elseif k == K"catch" + if kind(firstchildhead) == K"Placeholder" + args[1] = false + end + elseif k == K"try" + # Try children in source order: + # try_block catch_var catch_block else_block finally_block + # Expr ordering: + # try_block catch_var catch_block [finally_block] [else_block] + try_ = args[1] + catch_var = false + catch_ = false + else_ = false + finally_ = false + for i in 2:length(args) + a = args[i] + if @isexpr(a, :catch) + catch_var = a.args[1] + catch_ = a.args[2] + elseif @isexpr(a, :else) + else_ = only(a.args) + elseif @isexpr(a, :finally) + finally_ = only(a.args) + elseif @isexpr(a, :error) + finally_ = Expr(:block, a) # Unclear where to put this but here will do? + else + @assert false "Illegal $a subclause in `try`" + end + end + empty!(args) + push!(args, try_, catch_var, catch_) + if finally_ !== false || else_ !== false + push!(args, finally_) + if else_ !== false + push!(args, else_) + end + end + elseif k == K"generator" + # Reconstruct the nested Expr form for generator from our flatter + # source-ordered `generator` format. + gen = args[1] + for j = length(args):-1:2 + gen = Expr(:generator, gen) + _append_iterspec!(gen.args, args[j]) + if j < length(args) + # Additional `for`s flatten the inner generator + gen = Expr(:flatten, gen) + end + end + return gen + elseif k == K"filter" + @assert length(args) == 2 + retexpr.args = _append_iterspec!(Any[args[2]], args[1]) + elseif k == K"nrow" || k == K"ncat" + # For lack of a better place, the dimension argument to nrow/ncat + # is stored in the flags + pushfirst!(args, numeric_flags(flags(nodehead))) + elseif k == K"typed_ncat" + insert!(args, 2, numeric_flags(flags(nodehead))) + elseif k == K"elseif" + # Block for conditional's source location + args[1] = Expr(:block, loc, args[1]) + elseif k == K"->" + a1 = args[1] + if @isexpr(a1, :tuple) + # TODO: This makes the Expr form objectively worse for the sake of + # compatibility. We should consider deleting this special case in + # the future as a minor change. + if length(a1.args) == 1 && + (!has_flags(firstchildhead, PARENS_FLAG) || + !has_flags(firstchildhead, TRAILING_COMMA_FLAG)) && + !Meta.isexpr(a1.args[1], :parameters) + # `(a) -> c` is parsed without tuple on lhs in Expr form + args[1] = a1.args[1] + elseif length(a1.args) == 2 && (a11 = a1.args[1]; @isexpr(a11, :parameters) && + length(a11.args) <= 1 && !Meta.isexpr(a1.args[2], :(...))) + # `(a; b=1) -> c` parses args as `block` in Expr form :-( + if length(a11.args) == 0 + args[1] = Expr(:block, a1.args[2]) + else + a111 = only(a11.args) + assgn = @isexpr(a111, :kw) ? Expr(:(=), a111.args...) : a111 + argloc = source_location(LineNumberNode, source, last(firstchildrange)) + args[1] = Expr(:block, a1.args[2], argloc, assgn) + end + end + end + a2 = args[2] + # Add function source location to rhs; add block if necessary + if @isexpr(a2, :block) + pushfirst!(a2.args, loc) + else + args[2] = Expr(:block, loc, args[2]) + end + elseif k == K"function" + if length(args) > 1 + if has_flags(nodehead, SHORT_FORM_FUNCTION_FLAG) + a1 = args[1] + a2 = args[2] + if !@isexpr(a2, :block) && !@isexpr(a1, Symbol("'")) + args[2] = Expr(:block, a2) + end + retexpr.head = :(=) + else + a1 = args[1] + if @isexpr(a1, :tuple) && + !has_flags(firstchildhead, TRAILING_COMMA_FLAG) + # Convert to weird Expr forms for long-form anonymous functions. + # + # (function (tuple (... xs)) body) ==> (function (... xs) body) + if length(a1.args) == 1 && (a11 = a1.args[1]; @isexpr(a11, :...)) + # function (xs...) \n body end + args[1] = a11 + end + end + end + arg2 = args[2] + # Add location if not ErrorVal or unwrapped block + @isexpr(arg2, :block) && pushfirst!(arg2.args, loc) + end + elseif k == K"macro" + if length(args) > 1 + pushfirst!((args[2]::Expr).args, loc) + end + elseif k == K"module" + insert!(args, kind(firstchildhead) == K"VERSION" ? 2 : 1, !has_flags(nodehead, BARE_MODULE_FLAG)) + pushfirst!((args[end]::Expr).args, loc) + elseif k == K"quote" + if length(args) == 1 + a1 = only(args) + if !(a1 isa Expr || a1 isa QuoteNode || a1 isa Bool) + # Flisp parser does an optimization here: simple values are stored + # as inert QuoteNode rather than in `Expr(:quote)` quasiquote + return QuoteNode(a1) + end + end + elseif k == K"do" + # Temporary head which is picked up by _extract_do_lambda + retexpr.head = :do_lambda + elseif k == K"let" + a1 = args[1] + if @isexpr(a1, :block) + a1a = (args[1]::Expr).args + filter!(a -> !(a isa LineNumberNode), a1a) + # Ugly logic to strip the Expr(:block) in certain cases for compatibility + if length(a1a) == 1 + a = a1a[1] + if a isa Symbol || @isexpr(a, :(=)) || @isexpr(a, :(::)) + args[1] = a + end + end + end + elseif k == K"local" || k === K"global" + if length(args) == 1 + a1 = args[1] + if @isexpr(a1, :const) + # Normalize `local const` to `const local` + args[1] = Expr(retexpr.head, (a1::Expr).args...) + retexpr.head = :const + elseif @isexpr(a1, :tuple) + # Normalize `global (x, y)` to `global x, y` + retexpr.args = a1.args + end + end + elseif k == K"return" && isempty(args) + push!(args, nothing) + elseif k == K"juxtapose" + retexpr.head = :call + pushfirst!(args, :*) + elseif k == K"struct" + @assert args[2].head == :block + orig_fields = args[2].args + fields = Expr(:block) + for field in orig_fields + if @isexpr(field, :macrocall) && field.args[1] == GlobalRef(Core, Symbol("@doc")) + # @doc macro calls don't occur within structs, in Expr form. + push!(fields.args, field.args[3]) + push!(fields.args, field.args[4]) + else + push!(fields.args, field) + end + end + args[2] = fields + pushfirst!(args, has_flags(nodehead, MUTABLE_FLAG)) + elseif k == K"importpath" + retexpr.head = :. + for i = 1:length(args) + ai = args[i] + if ai isa QuoteNode + # Permit nonsense additional quoting such as + # import A.(:b).:c + args[i] = ai.value + end + end + elseif k == K"wrapper" + # This should only happen for errors wrapped next to what should have + # been single statements or atoms - represent these as blocks. + retexpr.head = :block + elseif k == K"comparison" + for i = 2:2:length(args) + arg = args[i] + if @isexpr(arg, :., 1) + args[i] = Symbol(".", arg.args[1]) + end + end + elseif k == K"meta" + # Expr uses plain identifiers, but JuliaSyntax uses quoted (Symbol) identifiers + for (i, a) in enumerate(args) + if a isa QuoteNode && a.value isa Symbol + args[i] = a.value + end + end + end + + return retexpr +end + +function build_tree(::Type{Expr}, stream::ParseStream; + filename=nothing, first_line=1, + # unused, but required since `_parse` is written generic + keep_parens=false) + source = SourceFile(stream, filename=filename, first_line=first_line) + return build_tree(Expr, stream, source) +end + +function build_tree(::Type{Expr}, stream::ParseStream, source::SourceFile) + txtbuf = unsafe_textbuf(stream) + cursor = RedTreeCursor(stream) + wrapper_head = SyntaxHead(K"wrapper",EMPTY_FLAGS) + if has_toplevel_siblings(cursor) + entry = Expr(:block) + for child in + Iterators.filter(should_include_node, reverse_toplevel_siblings(cursor)) + pushfirst!(entry.args, fixup_Expr_child( + RedTreeCursor, wrapper_head, + node_to_expr(child, source, txtbuf), false)) + end + length(entry.args) == 1 && (entry = only(entry.args)) + else + entry = fixup_Expr_child( + RedTreeCursor, wrapper_head, + node_to_expr(cursor, source, txtbuf), false) + end + return entry +end + +function to_expr(node) + source = sourcefile(node) + txtbuf_offset, txtbuf = _unsafe_wrap_substring(sourcetext(source)) + wrapper_head = SyntaxHead(K"wrapper",EMPTY_FLAGS) + return fixup_Expr_child( + typeof(node), wrapper_head, + node_to_expr(node, source, txtbuf, UInt32(txtbuf_offset)), false) +end + +Base.Expr(node::SyntaxNode) = to_expr(node) diff --git a/JuliaSyntax/src/integration/hooks.jl b/JuliaSyntax/src/integration/hooks.jl new file mode 100644 index 0000000000000..597162949b1a2 --- /dev/null +++ b/JuliaSyntax/src/integration/hooks.jl @@ -0,0 +1,428 @@ +# This file provides an adaptor to match the API expected by the Julia runtime +# code in the binding Core._parse + +const _has_v1_6_hooks = VERSION >= v"1.6" +const _has_v1_10_hooks = isdefined(Core, :_setparser!) +const _has_v1_14_version_hooks = isdefined(Base, :fl_parse_bootstrap) + +struct ErrorSpec + child_idx::Int + node::RedTreeCursor + parent_kind::Kind +end + +function first_error_cursor(stream::ParseStream) + output = stream.output + for i = 2:length(output) + is_error(output[i]) && return GreenTreeCursor(output, i) + end +end + +# Find the first error in a SyntaxNode tree, returning the index of the error +# within its parent and the node itself. +function first_tree_error(c::RedTreeCursor, error_cursor::GreenTreeCursor) + @assert !is_leaf(c) && !is_error(c) + it = reverse_nontrivia_children(c) + r = iterate(it) + local child + while r !== nothing + (child, state) = r + r = iterate(it, state) + (error_cursor in child || error_cursor == child.green) || continue + is_error(child) && break + return first_tree_error(child, error_cursor) + end + i = 1 # count node index + while r !== nothing + i += 1 + (_, state) = r + r = iterate(it, state) + end + return ErrorSpec(i, child, kind(c)) +end + +function first_tree_error(stream::ParseStream) + c = RedTreeCursor(stream) + err = first_error_cursor(stream) + for c in reverse_toplevel_siblings(c) + is_error(c) && return ErrorSpec(0, c, K"wrapper") + is_leaf(c) && continue + return first_tree_error(c, err) + end +end + +# Classify an incomplete expression, returning a Symbol compatible with +# Base.incomplete_tag(). +# +# Roughly, the intention here is to classify which expression head is expected +# next if the incomplete stream was to continue. (Though this is just rough. In +# practice several categories are combined for the purposes of the REPL - +# perhaps we can/should do something more precise in the future.) +function _incomplete_tag(theerror::ErrorSpec, codelen) + i = theerror.child_idx + c = theerror.node + kp = theerror.parent_kind + if isnothing(c) || last_byte(c) < codelen || codelen == 0 + if kind(c) == K"ErrorEofMultiComment" + # This is the one weird case where the token itself is an + # incomplete error + return :comment + else + return :none + end + elseif first_byte(c) <= codelen && kind(c) != K"ErrorInvalidEscapeSequence" + # "ErrorInvalidEscapeSequence" may be incomplete, so we don't include it + # here as a hard error. + return :none + end + if kind(c) == K"error" && is_non_terminal(c) + for cc in reverse_nontrivia_children(c) + if kind(cc) == K"error" + return :other + end + end + end + if kp == K"wrapper" + return :other + elseif kp == K"string" || kp == K"var" + return :string + elseif kp == K"cmdstring" + return :cmd + elseif kp == K"char" + return :char + elseif kp in KSet"block quote let try" + return :block + elseif kp in KSet"for while function if" + return i == 1 ? :other : :block + elseif kp == K"module" + return i <= 2 ? :other : :block + elseif kp == K"struct" + return i == 1 ? :other : :block + elseif kp == K"do" + return i < 3 ? :other : :block + else + return :other + end +end + +#------------------------------------------------------------------------------- +function _set_core_parse_hook(parser) + @static if _has_v1_10_hooks + Core._setparser!(parser) + else + # HACK! Fool the runtime into allowing us to set Core._parse, even during + # incremental compilation. (Ideally we'd just arrange for Core._parse to be + # set to the JuliaSyntax parser. But how do we signal that to the dumping + # code outside of the initial creation of Core?) + i = Base.fieldindex(Base.JLOptions, :incremental) + ptr = convert(Ptr{fieldtype(Base.JLOptions, i)}, + cglobal(:jl_options, Base.JLOptions) + fieldoffset(Base.JLOptions, i)) + incremental = unsafe_load(ptr) + if incremental != 0 + unsafe_store!(ptr, 0) + end + + Base.eval(Core, :(_parse = $parser)) + + if incremental != 0 + unsafe_store!(ptr, incremental) + end + end +end + + +# Wrap the function `f` so that it's always invoked in the given `world_age` +# +# NB: We need an inference barrier of one type or another here to prevent +# invalidations. The invokes provide this currently. +function fix_world_age(f, world_age::UInt) + if world_age == typemax(UInt) + function invoke_latestworld(args...; kws...) + Base.invokelatest(f, args...; kws...) + end + else + function invoke_fixedworld(args...; kws...) + Base.invoke_in_world(world_age, f, args...; kws...) + end + end +end + +function _has_nested_error(ex) + if ex isa Expr + if ex.head == :error + return true + else + return any(_has_nested_error(e) for e in ex.args) + end + elseif ex isa QuoteNode + return _has_nested_error(ex.value) + else + return false + end +end + +# Debug log file for dumping parsed code +const _debug_log = Ref{Union{Nothing,IO}}(nothing) + +function core_parser_hook(code, filename::String, lineno::Int, offset::Int, + options::Symbol, version::VersionNumber) + try + # TODO: Check that we do all this input wrangling without copying the + # code buffer + if code isa Core.SimpleVector + # The C entry points will pass us this form. + (ptr,len) = code + code = String(unsafe_wrap(Array, ptr, len)) + elseif !(code isa String || code isa SubString || code isa Vector{UInt8}) + # For non-Base string types, convert to UTF-8 encoding, using an + # invokelatest to avoid world age issues. + code = Base.invokelatest(String, code) + end + if !isnothing(_debug_log[]) + print(_debug_log[], """ + #-#-#------------------------------- + # ENTER filename=$filename, lineno=$lineno, offset=$offset, options=$options" + #-#-#------------------------------- + """) + write(_debug_log[], code) + end + + stream = ParseStream(code, offset+1; version=version) + if options === :statement || options === :atom + # To copy the flisp parser driver: + # * Parsing atoms consumes leading trivia + # * Parsing statements consumes leading+trailing trivia + bump_trivia(stream) + if peek(stream) == K"EndMarker" + # If we're at the end of stream after skipping whitespace, just + # return `nothing` to indicate this rather than attempting to + # parse a statement or atom and failing. + return Core.svec(nothing, last_byte(stream)) + end + end + parse!(stream; rule=options) + if options === :statement + bump_trivia(stream; skip_newlines=false) + if peek(stream) == K"NewlineWs" + bump(stream) + end + end + + if any_error(stream) + pos_before_comments = last_non_whitespace_byte(stream) + errspec = first_tree_error(stream) + tag = _incomplete_tag(errspec, pos_before_comments) + if _has_v1_10_hooks + exc = ParseError(stream, filename=filename, first_line=lineno, + incomplete_tag=tag) + msg = sprint(showerror, exc) + error_ex = Expr(tag === :none ? :error : :incomplete, + Meta.ParseError(msg, exc)) + elseif tag !== :none + # Hack: For older Julia versions, replicate the messages which + # Base.incomplete_tag() will match + msg = + tag === :string ? "incomplete: invalid string syntax" : + tag === :comment ? "incomplete: unterminated multi-line comment #= ... =#" : + tag === :block ? "incomplete: construct requires end" : + tag === :cmd ? "incomplete: invalid \"`\" syntax" : + tag === :char ? "incomplete: invalid character literal" : + "incomplete: premature end of input" + error_ex = Expr(:incomplete, msg) + else + # In the flisp parser errors are normally `Expr(:error, msg)` where + # `msg` is a String. By using a JuliaSyntax.ParseError for msg + # we can do fancy error reporting instead. + error_ex = Expr(:error, ParseError(stream, filename=filename, first_line=lineno)) + end + ex = if options === :all + # When encountering a toplevel error, the reference parser + # * truncates the top level expression arg list before that error + # * includes the last line number + # * appends the error message + source = SourceFile(stream, filename=filename, first_line=lineno) + topex = build_tree(Expr, stream, source) + @assert topex.head == :toplevel + i = findfirst(_has_nested_error, topex.args) + if i > 1 && topex.args[i-1] isa LineNumberNode + i -= 1 + end + resize!(topex.args, i-1) + push!(topex.args, LineNumberNode(source_line(source, first_byte(errspec.node)), filename)) + push!(topex.args, error_ex) + topex + else + error_ex + end + else + # TODO: Figure out a way to show warnings. Meta.parse() has no API + # to communicate this, and we also can't show them to stdout as + # this is too side-effectful and can result in double-reporting in + # the REPL. + # + # show_diagnostics(stdout, stream.diagnostics, code) + # + ex = build_tree(Expr, stream; filename=filename, first_line=lineno) + end + + # Note the next byte in 1-based indexing is `last_byte(stream) + 1` but + # the Core hook must return an offset (ie, it's 0-based) so the factors + # of one cancel here. + last_offset = last_byte(stream) + + if !isnothing(_debug_log[]) + println(_debug_log[], """ + #-#-#- + # EXIT last_offset=$last_offset + #-#-#- + """) + flush(_debug_log[]) + end + + # Rewrap result in an svec for use by the C code + return Core.svec(ex, last_offset) + catch exc + if !isnothing(_debug_log[]) + println(_debug_log[], """ + #-#-#- + # ERROR EXIT + # $exc + #-#-#- + """) + flush(_debug_log[]) + end + @error("""JuliaSyntax parser failed — falling back to flisp! + This is not your fault. Please submit a bug report to https://github.com/JuliaLang/JuliaSyntax.jl/issues""", + exception=(exc,catch_backtrace()), + offset=offset, + code=code) + + _fl_parse_hook(code, filename, lineno, offset, options) + end +end + +# Core._parse gained a `lineno` argument in +# https://github.com/JuliaLang/julia/pull/43876 +# Prior to this, the following signature was needed: +function core_parser_hook(code, filename, offset, options) + core_parser_hook(code, filename, 1, offset, options) +end +function core_parser_hook(code, filename, lineno, offset, options) + core_parser_hook(code, filename, lineno, offset, options, VERSION) +end + +if _has_v1_10_hooks + Base.incomplete_tag(e::JuliaSyntax.ParseError) = e.incomplete_tag +else + # Hack: Meta.parse() attempts to construct a ParseError from a string if it + # receives `Expr(:error)`. Add an override to the ParseError constructor to + # prevent this. + Base.Meta.ParseError(e::JuliaSyntax.ParseError) = e +end + +_default_system_parser = _has_v1_6_hooks ? Core._parse : nothing + +# hook into InteractiveUtils.@activate +activate!(enable=true) = enable_in_core!(enable) + +""" + enable_in_core!([enable=true; freeze_world_age=true, debug_filename=nothing]) + +Connect the JuliaSyntax parser to the Julia runtime so that it replaces the +flisp parser for all parsing work. That is, JuliaSyntax will be used for +`include()`, `Meta.parse()`, the REPL, etc. To reset to the reference parser, +use `enable_in_core!(false)`. + +Keyword arguments: +* `freeze_world_age` - Use a fixed world age for the parser to prevent + recompilation of the parser due to any user-defined methods (default `true`). +""" +function enable_in_core!(enable=true; freeze_world_age = true) + if !_has_v1_6_hooks + error("Cannot use JuliaSyntax as the main Julia parser in Julia version $VERSION < 1.6") + end + if enable + world_age = freeze_world_age ? Base.get_world_counter() : typemax(UInt) + _set_core_parse_hook(fix_world_age(core_parser_hook, world_age)) + else + @assert !isnothing(_default_system_parser) + _set_core_parse_hook(_default_system_parser) + end + nothing +end + +#------------------------------------------------------------------------------- +# Tools to call the reference flisp parser +# +# Call the flisp parser +function _fl_parse_hook(code, filename, lineno, offset, options) + @static if _has_v1_14_version_hooks + Base.fl_parse(code, filename, lineno, offset, options, nothing) + elseif VERSION >= v"1.8.0-DEV.1370" # https://github.com/JuliaLang/julia/pull/43876 + return Core.Compiler.fl_parse(code, filename, lineno, offset, options) + elseif _has_v1_6_hooks + return Core.Compiler.fl_parse(code, filename, offset, options) + else + if options === :all + ex = Base.parse_input_line(String(code), filename=filename, depwarn=false) + if !@isexpr(ex, :toplevel) + ex = Expr(:toplevel, ex) + end + return ex, sizeof(code) + elseif options === :statement || options === :atom + ex, pos = Meta.parse(code, offset+1, greedy=options==:statement, raise=false) + return ex, pos-1 + else + error("Unknown parse options $options") + end + end +end + +#------------------------------------------------ +# Copy of the Meta.parse() API, but ensuring that we call the flisp parser +# rather than using Meta.parse() which may be using the JuliaSyntax parser. + +""" +Like Meta.parse() but always call the flisp reference parser. +""" +function fl_parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true) + ex, pos = fl_parse(str, 1, greedy=true, raise=raise, depwarn=depwarn) + if isa(ex,Expr) && ex.head === :error + return ex + end + if pos <= ncodeunits(str) + raise && throw(Meta.ParseError("extra token after end of expression")) + return Expr(:error, "extra token after end of expression") + end + return ex +end + +function fl_parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true, + depwarn::Bool=true) + ex, pos = _fl_parse_string(str, "none", 1, pos, greedy ? :statement : :atom) + if raise && isa(ex,Expr) && ex.head === :error + throw(Meta.ParseError(ex.args[1])) + end + return ex, pos +end + +""" +Like Meta.parseall() but always call the flisp reference parser. +""" +function fl_parseall(text::AbstractString; filename="none", lineno=1) + ex,_ = _fl_parse_string(text, String(filename), lineno, 1, :all) + return ex +end + +function _fl_parse_string(text::AbstractString, filename::AbstractString, + lineno::Integer, index::Integer, options) + if index < 1 || index > ncodeunits(text) + 1 + throw(BoundsError(text, index)) + end + ex, offset::Int = _fl_parse_hook(text, filename, lineno, index-1, options) + ex, offset+1 +end + +# Convenience functions to mirror `JuliaSyntax.parsestmt(Expr, ...)` in simple cases. +fl_parse(::Type{Expr}, args...; kws...) = fl_parse(args...; kws...) +fl_parseall(::Type{Expr}, args...; kws...) = fl_parseall(args...; kws...) diff --git a/JuliaSyntax/src/julia/julia_parse_stream.jl b/JuliaSyntax/src/julia/julia_parse_stream.jl new file mode 100644 index 0000000000000..bfffac6980cb1 --- /dev/null +++ b/JuliaSyntax/src/julia/julia_parse_stream.jl @@ -0,0 +1,304 @@ +# Set for K"call", K"dotcall" or any syntactic operator heads +# Distinguish various syntaxes which are mapped to K"call" +const PREFIX_CALL_FLAG = RawFlags(0<<3) +const INFIX_FLAG = RawFlags(1<<3) +const PREFIX_OP_FLAG = RawFlags(2<<3) +const POSTFIX_OP_FLAG = RawFlags(3<<3) + +# The following flags are quite head-specific and may overlap with numeric flags + +""" +Set when K"string" or K"cmdstring" was triple-delimited as with \"\"\" or ``` +""" +const TRIPLE_STRING_FLAG = RawFlags(1<<8) + +""" +Set when a K"string", K"cmdstring" or K"Identifier" needs raw string unescaping +""" +const RAW_STRING_FLAG = RawFlags(1<<9) + +""" +Set for K"tuple", K"block" or K"macrocall" which are delimited by parentheses +""" +const PARENS_FLAG = RawFlags(1<<8) + +""" +Set for various delimited constructs when they contains a trailing comma. For +example, to distinguish `(a,b,)` vs `(a,b)`, and `f(a)` vs `f(a,)`. Kinds where +this applies are: `tuple call dotcall macrocall vect curly braces <: >:`. +""" +const TRAILING_COMMA_FLAG = RawFlags(1<<9) + +""" +Set for K"quote" for the short form `:x` as opposed to long form `quote x end` +""" +const COLON_QUOTE = RawFlags(1<<8) + +""" +Set for K"toplevel" which is delimited by semicolons +""" +const TOPLEVEL_SEMICOLONS_FLAG = RawFlags(1<<8) + +""" +Set for K"function" in short form definitions such as `f() = 1` +""" +const SHORT_FORM_FUNCTION_FLAG = RawFlags(1<<8) + +""" +Set for K"struct" when mutable +""" +const MUTABLE_FLAG = RawFlags(1<<8) + +""" +Set for K"module" when it's not bare (`module`, not `baremodule`) +""" +const BARE_MODULE_FLAG = RawFlags(1<<8) + +# Flags holding the dimension of an nrow or other UInt8 not held in the source +# TODO: Given this is only used for nrow/ncat, we could actually use all the flags? +const NUMERIC_FLAGS = RawFlags(RawFlags(0xff)<<8) + +function set_numeric_flags(n::Integer) + f = RawFlags((n << 8) & NUMERIC_FLAGS) + if numeric_flags(f) != n + error("Numeric flags unable to hold large integer $n") + end + f +end + +function call_type_flags(f::RawFlags) + f & 0b11000 +end + +function numeric_flags(f::RawFlags) + Int((f >> 8) % UInt8) +end + +flags(tok::SyntaxToken) = remove_flags(flags(tok.head), NUMERIC_FLAGS) + +""" + is_prefix_call(x) + +Return true for normal prefix function call syntax such as the `f` call node +parsed from `f(x)`. +""" +is_prefix_call(x) = call_type_flags(x) == PREFIX_CALL_FLAG + +""" + is_infix_op_call(x) + +Return true for infix operator calls such as the `+` call node parsed from +`x + y`. +""" +is_infix_op_call(x) = call_type_flags(x) == INFIX_FLAG + +""" + is_prefix_op_call(x) + +Return true for prefix operator calls such as the `+` call node parsed from `+x`. +""" +is_prefix_op_call(x) = call_type_flags(x) == PREFIX_OP_FLAG + +""" + is_postfix_op_call(x) + +Return true for postfix operator calls such as the `'ᵀ` call node parsed from `x'ᵀ`. +""" +is_postfix_op_call(x) = call_type_flags(x) == POSTFIX_OP_FLAG + +""" + numeric_flags(x) + +Return the number attached to a `SyntaxHead`. This is only for kinds `K"nrow"` +and `K"ncat"`, for now. +""" +numeric_flags(x) = numeric_flags(flags(x)) + +function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true) + str = (is_error(kind(head)) ? untokenize(kind(head); unique=false) : + untokenize(kind(head); unique=unique))::String + if include_flag_suff + is_trivia(head) && (str = str*"-t") + is_infix_op_call(head) && (str = str*"-i") + is_prefix_op_call(head) && (str = str*"-pre") + is_postfix_op_call(head) && (str = str*"-post") + + k = kind(head) + # Handle numeric flags for nodes that take them + if k in KSet"nrow ncat typed_ncat DotsIdentifier" + n = numeric_flags(head) + n != 0 && (str = str*"-"*string(n)) + else + # Handle head-specific flags that overlap with numeric flags + if k in KSet"string cmdstring Identifier" + has_flags(head, TRIPLE_STRING_FLAG) && (str = str*"-s") + has_flags(head, RAW_STRING_FLAG) && (str = str*"-r") + elseif k in KSet"tuple block macrocall" + has_flags(head, PARENS_FLAG) && (str = str*"-p") + elseif k == K"quote" + has_flags(head, COLON_QUOTE) && (str = str*"-:") + elseif k == K"toplevel" + has_flags(head, TOPLEVEL_SEMICOLONS_FLAG) && (str = str*"-;") + elseif k == K"function" + has_flags(head, SHORT_FORM_FUNCTION_FLAG) && (str = str*"-=") + elseif k == K"struct" + has_flags(head, MUTABLE_FLAG) && (str = str*"-mut") + elseif k == K"module" + has_flags(head, BARE_MODULE_FLAG) && (str = str*"-bare") + end + if k in KSet"tuple call dotcall macrocall vect curly braces <: >:" && + has_flags(head, TRAILING_COMMA_FLAG) + str *= "-," + end + end + end + str +end + + +#------------------------------------------------------------------------------- +# ParseStream Post-processing + +function validate_tokens(stream::ParseStream) + txtbuf = unsafe_textbuf(stream) + charbuf = IOBuffer() + + # Process terminal nodes in the output + fbyte = stream.output[1].byte_span+1 # Start after sentinel + for i = 2:length(stream.output) + node = stream.output[i] + if !is_terminal(node) || kind(node) == K"TOMBSTONE" + continue + end + + k = kind(node) + nbyte = fbyte + node.byte_span + tokrange = fbyte:nbyte-1 + error_kind = K"None" + + if k in KSet"Integer BinInt OctInt HexInt" + # The following shouldn't be able to error... + # parse_int_literal + # parse_uint_literal + elseif k == K"Float" || k == K"Float32" + if k == K"Float" + x, code = parse_float_literal(Float64, txtbuf, fbyte, nbyte) + # jl_strtod_c can return "underflow" even for valid cases such + # as `5e-324` where the source is an exact representation of + # `x`. So only warn when underflowing to zero. + underflow0 = code === :underflow && x == 0 + else + x, code = parse_float_literal(Float32, txtbuf, fbyte, nbyte) + underflow0 = code === :underflow && x == 0 + end + if code === :ok + # pass + elseif code === :overflow + emit_diagnostic(stream, tokrange, + error="overflow in floating point literal") + error_kind = K"ErrorNumericOverflow" + elseif underflow0 + emit_diagnostic(stream, tokrange, + warning="underflow to zero in floating point literal") + end + elseif k == K"Char" + @assert fbyte < nbyte # Already handled in the parser + truncate(charbuf, 0) + had_error = unescape_julia_string(charbuf, txtbuf, fbyte, + nbyte, stream.diagnostics) + if had_error + error_kind = K"ErrorInvalidEscapeSequence" + else + seek(charbuf,0) + read(charbuf, Char) + if !eof(charbuf) + error_kind = K"ErrorOverLongCharacter" + emit_diagnostic(stream, tokrange, + error="character literal contains multiple characters") + end + end + elseif k == K"String" && !has_flags(node, RAW_STRING_FLAG) + had_error = unescape_julia_string(devnull, txtbuf, fbyte, + nbyte, stream.diagnostics) + if had_error + error_kind = K"ErrorInvalidEscapeSequence" + end + elseif is_error(k) && k != K"error" + # Emit messages for non-generic token errors + tokstr = String(txtbuf[tokrange]) + msg = if k in KSet"ErrorInvisibleChar ErrorUnknownCharacter ErrorIdentifierStart" + "$(_token_error_descriptions[k]) $(repr(tokstr[1]))" + elseif k in KSet"ErrorInvalidUTF8 ErrorBidiFormatting" + "$(_token_error_descriptions[k]) $(repr(tokstr))" + else + _token_error_descriptions[k] + end + emit_diagnostic(stream, tokrange, error=msg) + end + + if error_kind != K"None" + # Update the node with new error kind + stream.output[i] = RawGreenNode(SyntaxHead(error_kind, EMPTY_FLAGS), + node.byte_span, node.orig_kind) + end + + fbyte = nbyte + end + sort!(stream.diagnostics, by=first_byte) +end + +function peek_dotted_op_token(ps) + # Peek the next token, but if it is a dot, peek the next one as well + t = peek_token(ps) + isdotted = kind(t) == K"." + if isdotted + t2 = peek_token(ps, 2) + if preceding_whitespace(t2) + isdotted = false + elseif !is_operator(t2) + isdotted = false + elseif kind(t2) == K"." && peek(ps, 3) == K"." + # Treat `..` as dotted K".", unless there's another dot after + isdotted = false + else + t = t2 + end + end + # A compound assignment such as `x += y` is lexed as an operator carrying + # the special `PREC_COMPOUND_ASSIGN` precedence (immediately followed by an + # `=` token). Operators which don't form a compound assignment - eg `⋅`, + # `√`, or suffixed operators like `+₁` - are not marked this way, so `x ⋅= y` + # is left to be parsed as the identifier `⋅` followed by `=`, matching the + # reference parser. + isassign = is_prec_compound_assign(t) + return (isdotted, isassign, t) +end + +function bump_dotted(ps, isdot, t, flags=EMPTY_FLAGS; emit_dot_node=false, remap_kind=K"None") + mark = position(ps) + if isdot + dotmark = mark + bump(ps, TRIVIA_FLAG) + if kind(t) == K"." + # .. => DotsIdentifier-2 + bump(ps, TRIVIA_FLAG) + pos = emit(ps, dotmark, K"DotsIdentifier", set_numeric_flags(2)) + nt = peek_token(ps) + # `..` directly followed by another operator (eg `a..+b`) is + # ambiguous and requires a space (the lexer used to glue these into + # an invalid operator token). The quote/interpolation/char operators + # `: :: $ '` begin an operand rather than continuing the operator, + # so `a..:b`, `a..$b` and `'a'..'b'` are allowed. + if is_operator(nt) && !preceding_whitespace(nt) && !(kind(nt) in KSet": :: $ '") + # a..+b => (call-i a .. (error-t) (call + b)) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="`..` here is interpreted as a binary operator. A space is required if followed by another operator.") + end + return pos + end + end + min_supported_wrapping_arithmetic_op(ps, mark, t) + pos = bump(ps, flags, remap_kind=remap_kind) + isdot && emit_dot_node && (pos = emit(ps, dotmark, K".")) + return pos +end diff --git a/JuliaSyntax/src/julia/kinds.jl b/JuliaSyntax/src/julia/kinds.jl new file mode 100644 index 0000000000000..1368b9889972a --- /dev/null +++ b/JuliaSyntax/src/julia/kinds.jl @@ -0,0 +1,699 @@ +# Definition of Kind type - mapping from token string identifiers to +# enumeration values as used in @K_str + +""" + K"name" + Kind(namestr) + +`Kind` is a type tag for specifying the type of tokens and interior nodes of +a syntax tree. Abstractly, this tag is used to define our own *sum types* for +syntax tree nodes. We do this explicitly outside the Julia type system because +(a) Julia doesn't have sum types and (b) we want concrete data structures which +are unityped from the Julia compiler's point of view, for efficiency. + +Naming rules: +* Kinds which correspond to exactly one textual form are represented with that + text. This includes keywords like K"for" and operators like K"*". +* Kinds which represent many textual forms have UpperCamelCase names. This + includes kinds like K"Identifier" and K"Comment". +* Kinds which exist merely as delimiters are all uppercase +""" +primitive type Kind 16 end + +# The implementation of Kind here is basically similar to @enum. However we use +# the K_str macro to self-name these kinds with their literal representation, +# rather than needing to invent a new name for each. + +const _kind_str_to_int = Dict{String,UInt16}() +const _kind_int_to_str = Dict{UInt16,String}() +const _kind_modules = Dict{Int,Union{Symbol,Module}}( + 0=>nameof(@__MODULE__), + 1=>:JuliaLowering, + 2=>:JuliaSyntaxFormatter +) +# Number of bits reserved for kind id's belonging to a single module +const _kind_nbits = 10 +const _kind_module_id_max = typemax(UInt16) >> _kind_nbits + +function Kind(x::Integer) + if x < 0 || x > typemax(UInt16) + throw(ArgumentError("Kind out of range: $x")) + end + return Base.bitcast(Kind, convert(UInt16, x)) +end + +function Kind(s::AbstractString) + i = get(_kind_str_to_int, s) do + error("unknown Kind name $(repr(s))") + end + Kind(i) +end + +Base.string(x::Kind) = get(_kind_int_to_str, reinterpret(UInt16, x), "") +Base.print(io::IO, x::Kind) = print(io, string(x)) + +Base.isless(x::Kind, y::Kind) = reinterpret(UInt16, x) < reinterpret(UInt16, y) + +function Base.show(io::IO, k::Kind) + print(io, "K\"", k, "\"") +end + +# Save the string representation rather than the bit pattern so that kinds +# can be serialized and deserialized across different JuliaSyntax versions. +function Base.write(io::IO, k::Kind) + str = string(k) + write(io, UInt8(sizeof(str))) + write(io, str) +end +function Base.read(io::IO, ::Type{Kind}) + len = read(io, UInt8) + str = String(read(io, len)) + Kind(str) +end + +function Base.parentmodule(k::Kind) + mod_id = reinterpret(UInt16, k) >> _kind_nbits + _kind_modules[mod_id]::Module +end + +function _register_kinds!(kind_modules, int_to_kindstr, kind_str_to_int, mod, module_id, names) + if module_id > _kind_module_id_max + error("Kind module id $module_id is out of range") + elseif length(names) >= 1 << _kind_nbits + error("Too many kind names") + elseif !haskey(kind_modules, module_id) + kind_modules[module_id] = mod + else + m = kind_modules[module_id] + if m == nameof(mod) + # Ok: known kind module, but not loaded until now + kind_modules[module_id] = mod + elseif m == mod + existing_kinds = Union{Nothing, Kind}[(i = get(kind_str_to_int, n, nothing); + isnothing(i) ? nothing : Kind(i)) for n in names] + if any(isnothing, existing_kinds) || + !issorted(existing_kinds) || + any(k->parentmodule(k) != mod, existing_kinds) + error("Error registering kinds for module $mod (register_kinds() called more than once inconsistently, or conflict with existing module kinds?)") + else + # Assume we're re-registering kinds as in top level vs `__init__` + return + end + else + error("Kind module ID $module_id already claimed by module $m") + end + end + _register_kinds_names!(int_to_kindstr, kind_str_to_int, module_id, names) +end + +# This function is separated from `_register_kinds!` to prevent sharing of the variable `i` +# here and in the closure in `_register_kinds!`, which causes boxing and bad inference. +function _register_kinds_names!(int_to_kindstr, kind_str_to_int, module_id, names) + # Process names to conflate category BEGIN/END markers with the first/last + # in the category. + i = 0 + for name in names + normal_kind = false + if startswith(name, "BEGIN_") + j = i + elseif startswith(name, "END_") + j = i - 1 + else + normal_kind = true + j = i + i += 1 + end + kind_int = (module_id << _kind_nbits) | j + push!(kind_str_to_int, name=>kind_int) + if normal_kind + push!(int_to_kindstr, kind_int=>name) + end + end +end + +""" + register_kinds!(mod, module_id, names) + +Register custom `Kind`s with the given `names`, belonging to a module `mod`. +`names` is an array of arbitrary strings. + +In order for kinds to be represented by a small number of bits, some nontrivial +cooperation is required between modules using custom kinds: +* The integer `module_id` is globally unique for each `mod` which will be used + together, and not larger than $_kind_module_id_max. +* No two modules register the same `name`. The semantics of a given `kind` name + should be defined by the module which owns it. + +To allow ranges of kinds to be delimited and quickly tested for, some special +names are allowed: `BEGIN_section` and `END_section` pairs are detected, and +alias the next and previous kind id's respectively so that kinds in `section` +can be tested with `BEGIN_section <= k <= END_section`. +""" +function register_kinds!(mod, module_id, names) + _register_kinds!(_kind_modules, _kind_int_to_str, _kind_str_to_int, mod, module_id, names) +end + +#------------------------------------------------------------------------------- + +""" + K"s" + +The kind of a token or AST internal node with string "s". + +For example +* K")" is the kind of the right parenthesis token +* K"block" is the kind of a block of code (eg, statements within a begin-end). +""" +macro K_str(s) + Kind(s) +end + +""" +A set of kinds which can be used with the `in` operator. For example + + k in KSet"+ - *" +""" +macro KSet_str(str) + kinds = [Kind(s) for s in split(str)] + + quote + ($(kinds...),) + end +end + +""" + kind(x) + +Return the `Kind` of `x`. +""" +kind(k::Kind) = k + +#------------------------------------------------------------------------------- +# Kinds used by JuliaSyntax +register_kinds!(JuliaSyntax, 0, [ + # Whitespace + "Comment" + "Whitespace" + "NewlineWs" # newline-containing whitespace + + # Identifiers + "BEGIN_IDENTIFIERS" + "Identifier" + "Operator" + "Placeholder" # Used for empty catch variables, and all-underscore identifiers in lowering + # String and command macro names are modeled as a special kind of + # identifier as they need to be mangled before lookup. + "StrMacroName" + "CmdMacroName" + "END_IDENTIFIERS" + + "BEGIN_KEYWORDS" + "baremodule" + "begin" + "break" + "const" + "continue" + "do" + "export" + "for" + "function" + "global" + "if" + "import" + "let" + "local" + "macro" + "module" + "quote" + "return" + "struct" + "try" + "typegroup" + "using" + "while" + "BEGIN_BLOCK_CONTINUATION_KEYWORDS" + "catch" + "finally" + "else" + "elseif" + "end" + "END_BLOCK_CONTINUATION_KEYWORDS" + "BEGIN_CONTEXTUAL_KEYWORDS" + # contextual keywords + "abstract" + "as" + "doc" + "mutable" + "outer" + "primitive" + "public" + "type" + "var" + "VERSION" + "END_CONTEXTUAL_KEYWORDS" + "END_KEYWORDS" + + "BEGIN_LITERAL" + "BEGIN_NUMBERS" + "Bool" + "Integer" + "BinInt" + "HexInt" + "OctInt" + "Float" + "Float32" + "END_NUMBERS" + "String" + "Char" + "CmdString" + "END_LITERAL" + + "BEGIN_DELIMITERS" + # Punctuation + "@" + "," + ";" + + # Paired delimiters + "[" + "]" + "{" + "}" + "(" + ")" + "\"" + "\"\"\"" + "`" + "```" + "END_DELIMITERS" + + "BEGIN_OPS" + "ErrorInvalidOperator" + "Error**" + + # Various operators that have special parsing rules and thus get explicit heads. + # All other operators (including suffixed versions of these) are K"Operator". + "BEGIN_ASSIGNMENTS" + "=" + ".=" + ":=" + "~" + "≔" + "⩴" + "≕" + # Compound assignments + "op=" + ".op=" + "END_ASSIGNMENTS" + "?" # ternary operator + "||" # not an operator call + ".||" # dotted of above (not emitted by lexer) + "&&" # not an operator call + ".&&" # dotted of above (not emitted by lexer) + "<:" # subtype syntax + ">:" # supertype syntax + "::" # field type syntax + "." # various dot syntax + ".." # .. operator (not emitted by lexer) + "in" # iteration syntax + "isa" + "where" + "!" # syntactic unary + "'" # special postfix parsing + ".'" # special postfix parsing + "->" # syntactic arrow + "-->" # syntactic arrow + ":" # used for quoting + "+" # used in numeric constants + "+%" # wrapping addition + "++" # special chaining syntax + "*" # special chaining syntax + "*%" # wrapping multiplication + "<" # recovery path for :< + ">" # recovery path for :> + "\$" # interpolation + "-" # negated constants + "-%" # wrapping subtraction + "&" # syntactic unary + "∈" # iteration syntax + # all syntactic unary + "⋆" + "±" + "∓" + "¬" + "√" + "∛" + "∜" + "END_OPS" + + # 2. Nonterminals which are exposed in the AST, but where the surface + # syntax doesn't have a token corresponding to the node type. + "BEGIN_SYNTAX_KINDS" + "block" + "call" + "dotcall" + "comparison" + "curly" + "juxtapose" # Numeric juxtaposition like 2x + "string" # A string interior node (possibly containing interpolations) + "cmdstring" # A cmd string node (containing delimiters plus string) + "char" # A char string node (containing delims + char data) + "macrocall" + "parameters" # the list after ; in f(; a=1) + "kw" + "toplevel" + "tuple" + "ref" + "vect" + "parens" + "importpath" + "meta" + # Concatenation syntax + "braces" + "bracescat" + "hcat" + "vcat" + "ncat" + "typed_hcat" + "typed_vcat" + "typed_ncat" + "row" + "nrow" + # splat/slurp + "..." + # ../... as a identifier + "DotsIdentifier" + # Comprehensions + "generator" + "filter" + "iteration" + "comprehension" + "typed_comprehension" + "macro_name" + # typegroup is a keyword (see above in keywords section) + # Container for a single statement/atom plus any trivia and errors + "wrapper" + "END_SYNTAX_KINDS" + + # Kinds not corresponding to surface syntax in RawGreenNode, but required + # for parsing to a provenance-containing structure that is compatible with + # Expr. May shrink with syntax evolution. + "BEGIN_SYNTAXTREE_KINDS" + # A literal Julia value of any kind, as might be inserted into the + # AST during macro expansion. Only used in parsing to SyntaxTree. + "Value" + "unknown_head" + "flatten" + # QuoteNode; not quasiquote + "inert" + "syntaxinert" + "END_SYNTAXTREE_KINDS" + + # Special tokens + "TOMBSTONE" # Empty placeholder for kind to be filled later + "None" # Never emitted by lexer/parser + "EndMarker" # EOF + + "BEGIN_ERRORS" + # Tokenization errors + "ErrorEofMultiComment" + "ErrorInvalidNumericConstant" + "ErrorHexFloatMustContainP" + "ErrorAmbiguousNumericConstant" + "ErrorAmbiguousNumericDotMultiply" + "ErrorInvalidInterpolationTerminator" + "ErrorNumericOverflow" + "ErrorInvalidEscapeSequence" + "ErrorOverLongCharacter" + "ErrorInvalidUTF8" + "ErrorInvisibleChar" + "ErrorIdentifierStart" + "ErrorUnknownCharacter" + "ErrorBidiFormatting" + # Generic error + "error" + "END_ERRORS" +]) + +@enum PrecedenceLevel begin + PREC_NONE + PREC_ASSIGNMENT + PREC_PAIRARROW + PREC_CONDITIONAL + PREC_ARROW + PREC_LAZYOR + PREC_LAZYAND + PREC_COMPARISON + PREC_PIPE_LT + PREC_PIPE_GT + PREC_COLON + PREC_PLUS + PREC_BITSHIFT + PREC_TIMES + PREC_RATIONAL + PREC_POWER + PREC_DECL + PREC_WHERE + PREC_DOT + PREC_QUOTE + PREC_UNICODE_OPS + # Special precedence to only allow compound assignment for designated operators, for + # compatibility with flisp + PREC_COMPOUND_ASSIGN +end + +const generic_operators_by_level = Dict{PrecedenceLevel, Vector{Char}}( + # Operators which have their own kinds are commented out in these lists + PREC_ASSIGNMENT => Char[#= = .= := ~ ≔ ⩴ ≕ =#], + PREC_PAIRARROW => Char[#= => =#], + PREC_CONDITIONAL => Char[#= ? =#], + PREC_ARROW => + [#= -> --> <-- <--> =# + '←', '→', '↔', '↚', '↛', '↞', '↠', '↢', + '↣', '↤', '↦', '↮', '⇎', '⇍', '⇏', '⇐', '⇒', '⇔', '⇴', + '⇶', '⇷', '⇸', '⇹', '⇺', '⇻', '⇼', '⇽', '⇾', '⇿', '⟵', + '⟶', '⟷', '⟹', '⟺', '⟻', '⟼', '⟽', '⟾', '⟿', '⤀', '⤁', + '⤂', '⤃', '⤄', '⤅', '⤆', '⤇', '⤌', '⤍', '⤎', '⤏', '⤐', '⤑', + '⤔', '⤕', '⤖', '⤗', '⤘', '⤝', '⤞', '⤟', '⤠', '⥄', '⥅', '⥆', + '⥇', '⥈', '⥊', '⥋', '⥎', '⥐', '⥒', '⥓', '⥖', '⥗', '⥚', '⥛', + '⥞', '⥟', '⥢', '⥤', '⥦', '⥧', '⥨', '⥩', '⥪', '⥫', '⥬', '⥭', + '⥰', '⧴', '⬱', '⬰', '⬲', '⬳', '⬴', '⬵', '⬶', '⬷', '⬸', '⬹', + '⬺', '⬻', '⬼', '⬽', '⬾', '⬿', '⭀', '⭁', '⭂', '⭃', '⥷', '⭄', + '⥺', '⭇', '⭈', '⭉', '⭊', '⭋', '⭌', '￩', '￫', '⇜', '⇝', '↜', '↝', + '↩', '↪', '↫', '↬', '↼', '↽', '⇀', '⇁', '⇄', '⇆', '⇇', '⇉', '⇋', + '⇌', '⇚', '⇛', '⇠', '⇢', '↷', '↶', '↺', '↻', '🢲'], + PREC_LAZYOR => Char[#= || =#], + PREC_LAZYAND => Char[#= && =#], + PREC_COMPARISON => + [#= <: >: in isa < > ∈ == != !== =# + '≥', '≤', '≡', '≠', '≢', '∉', '∋', + '∌', '⊆', '⊈', '⊂', '⊄', '⊊', '∝', '∊', '∍', '∥', '∦', + '∷', '∺', '∻', '∽', '∾', '≁', '≃', '≂', '≄', '≅', '≆', + '≇', '≈', '≉', '≊', '≋', '≌', '≍', '≎', '≐', '≑', '≒', + '≓', '≖', '≗', '≘', '≙', '≚', '≛', '≜', '≝', '≞', '≟', + '≣', '≦', '≧', '≨', '≩', '≪', '≫', '≬', '≭', '≮', '≯', + '≰', '≱', '≲', '≳', '≴', '≵', '≶', '≷', '≸', '≹', '≺', + '≻', '≼', '≽', '≾', '≿', '⊀', '⊁', '⊃', '⊅', '⊇', '⊉', + '⊋', '⊏', '⊐', '⊑', '⊒', '⊜', '⊩', '⊬', '⊮', '⊰', '⊱', + '⊲', '⊳', '⊴', '⊵', '⊶', '⊷', '⋍', '⋐', '⋑', '⋕', '⋖', + '⋗', '⋘', '⋙', '⋚', '⋛', '⋜', '⋝', '⋞', '⋟', '⋠', '⋡', + '⋢', '⋣', '⋤', '⋥', '⋦', '⋧', '⋨', '⋩', '⋪', '⋫', '⋬', + '⋭', '⋲', '⋳', '⋴', '⋵', '⋶', '⋷', '⋸', '⋹', '⋺', '⋻', + '⋼', '⋽', '⋾', '⋿', '⟈', '⟉', '⟒', '⦷', '⧀', '⧁', '⧡', + '⧣', '⧤', '⧥', '⩦', '⩧', '⩪', '⩫', '⩬', '⩭', '⩮', '⩯', + '⩰', '⩱', '⩲', '⩳', '⩵', '⩶', '⩷', '⩸', '⩹', '⩺', '⩻', + '⩼', '⩽', '⩾', '⩿', '⪀', '⪁', '⪂', '⪃', '⪄', '⪅', '⪆', '⪇', + '⪈', '⪉', '⪊', '⪋', '⪌', '⪍', '⪎', '⪏', '⪐', '⪑', '⪒', '⪓', + '⪔', '⪕', '⪖', '⪗', '⪘', '⪙', '⪚', '⪛', '⪜', '⪝', '⪞', '⪟', + '⪠', '⪡', '⪢', '⪣', '⪤', '⪥', '⪦', '⪧', '⪨', '⪩', '⪪', + '⪫', '⪬', '⪭', '⪮', '⪯', '⪰', '⪱', '⪲', '⪳', '⪴', '⪵', + '⪶', '⪷', '⪸', '⪹', '⪺', '⪻', '⪼', '⪽', '⪾', '⪿', '⫀', + '⫁', '⫂', '⫃', '⫄', '⫅', '⫆', '⫇', '⫈', '⫉', '⫊', '⫋', + '⫌', '⫍', '⫎', '⫏', '⫐', '⫑', '⫒', '⫓', '⫔', '⫕', '⫖', + '⫗', '⫘', '⫙', '⫷', '⫸', '⫹', '⫺', '⊢', '⊣', '⟂', '⫪', '⫫'], + PREC_PIPE_LT => Char[#= <| =#], + PREC_PIPE_GT => Char[#= |> =#], + PREC_COLON => [ #= : .. =# '…', '⁝', '⋮', '⋱', '⋰', '⋯'], + PREC_PLUS => + [ #= + - ± ∓ ++ =# + '⊕', '⊖', '⊞', '⊟', '|', '∪', '∨', + '⊔', '±', '∓', '∔', '∸', '≏', '⊎', '⊻', '⊽', '⋎', '⋓', '⟇', '⧺', + '⧻', '⨈', '⨢', '⨣', '⨤', '⨥', '⨦', '⨧', '⨨', '⨩', '⨪', '⨫', '⨬', '⨭', + '⨮', '⨹', '⨺', '⩁', '⩂', '⩅', '⩊', '⩌', '⩏', '⩐', '⩒', '⩔', '⩖', '⩗', + '⩛', '⩝', '⩡', '⩢', '⩣', '¦'], + PREC_TIMES => + [ #= * ⋆ & =# + '/', '÷', '%', '⋅', '·', '·', '∘', '×', '\\', '∩', '∧', '⊗', + '⊘', '⊙', '⊚', '⊛', '⊠', '⊡', '⊓', '∗', '∙', '∤', '⅋', '≀', '⊼', '⋄', '⋆', + '⋇', '⋉', '⋊', '⋋', '⋌', '⋏', '⋒', '⟑', '⦸', '⦼', '⦾', '⦿', '⧶', '⧷', + '⨇', '⨰', '⨱', '⨲', '⨳', '⨴', '⨵', '⨶', '⨷', '⨸', '⨻', '⨼', '⨽', '⩀', + '⩃', '⩄', '⩋', '⩍', '⩎', '⩑', '⩓', '⩕', '⩘', '⩚', '⩜', '⩞', '⩟', '⩠', + '⫛', '⊍', '▷', '⨝', '⟕', '⟖', '⟗', '⌿', '⨟', + '\u00b7', # '·' Middle Dot + '\u0387' # '·' Greek Ano Teleia + ], + PREC_RATIONAL => Char[#= // =#], + PREC_BITSHIFT => Char[#= << >> >>> =#], + PREC_POWER => ['^', '↑', '↓', '⇵', '⟰', '⟱', '⤈', '⤉', '⤊', '⤋', '⤒', '⤓', '⥉', + '⥌', '⥍', '⥏', '⥑', '⥔', '⥕', '⥘', '⥙', '⥜', '⥝', '⥠', '⥡', '⥣', '⥥', + '⥮', '⥯', '￪', '￬'], +) + +#------------------------------------------------------------------------------- +const _nonunique_kind_names = Set([ + K"Comment" + K"Whitespace" + K"NewlineWs" + K"Identifier" + K"Placeholder" + + K"ErrorEofMultiComment" + K"ErrorInvalidNumericConstant" + K"ErrorHexFloatMustContainP" + K"ErrorAmbiguousNumericConstant" + K"ErrorAmbiguousNumericDotMultiply" + K"ErrorInvalidInterpolationTerminator" + K"ErrorNumericOverflow" + K"ErrorInvalidEscapeSequence" + K"ErrorOverLongCharacter" + K"ErrorInvalidUTF8" + K"ErrorInvisibleChar" + K"ErrorUnknownCharacter" + K"ErrorBidiFormatting" + K"ErrorInvalidOperator" + + K"Bool" + K"Integer" + K"BinInt" + K"HexInt" + K"OctInt" + K"Float" + K"Float32" + K"String" + K"Char" + K"CmdString" + + K"StrMacroName" + K"CmdMacroName" +]) + +""" +Return the string representation of a token kind, or `nothing` if the kind +represents a class of tokens like K"Identifier". + +When `unique=true` only return a string when the kind uniquely defines the +corresponding input token, otherwise return `nothing`. When `unique=false`, +return the name of the kind. + +TODO: Replace `untokenize()` with `Base.string()`? +""" +function untokenize(k::Kind; unique=true) + if unique && k in _nonunique_kind_names + return nothing + else + return string(k) + end +end + +# Error kind => description +const _token_error_descriptions = Dict{Kind, String}( + K"ErrorEofMultiComment" => "unterminated multi-line comment #= ... =#", + K"ErrorInvalidNumericConstant" => "invalid numeric constant", + K"ErrorHexFloatMustContainP" => "hex float literal must contain `p` or `P`", + K"ErrorAmbiguousNumericConstant" => "ambiguous `.` syntax; add whitespace to clarify (eg `1.+2` might be `1.0+2` or `1 .+ 2`)", + K"ErrorAmbiguousNumericDotMultiply" => "numeric constant cannot be implicitly multiplied because it ends with `.`", + K"ErrorInvalidInterpolationTerminator" => "interpolated variable ends with invalid character; use `\$(...)` instead", + K"ErrorNumericOverflow"=>"overflow in numeric literal", + K"ErrorInvalidEscapeSequence"=>"invalid string escape sequence", + K"ErrorOverLongCharacter"=>"character literal contains multiple characters", + K"ErrorInvalidUTF8"=>"invalid UTF-8 sequence", + K"ErrorInvisibleChar"=>"invisible character", + K"ErrorIdentifierStart" => "identifier cannot begin with character", + K"ErrorUnknownCharacter"=>"unknown unicode character", + K"ErrorBidiFormatting"=>"unbalanced bidirectional unicode formatting", + K"ErrorInvalidOperator" => "invalid operator", + K"Error**" => "use `x^y` instead of `x**y` for exponentiation, and `x...` instead of `**x` for splatting", + K"error" => "unknown error token", +) + +#------------------------------------------------------------------------------- +# Predicates +is_identifier(k::Kind) = K"BEGIN_IDENTIFIERS" <= k <= K"END_IDENTIFIERS" +is_contextual_keyword(k::Kind) = K"BEGIN_CONTEXTUAL_KEYWORDS" <= k <= K"END_CONTEXTUAL_KEYWORDS" +is_error(k::Kind) = K"BEGIN_ERRORS" <= k <= K"END_ERRORS" || k == K"ErrorInvalidOperator" || k == K"Error**" +is_keyword(k::Kind) = K"BEGIN_KEYWORDS" <= k <= K"END_KEYWORDS" +is_block_continuation_keyword(k::Kind) = K"BEGIN_BLOCK_CONTINUATION_KEYWORDS" <= k <= K"END_BLOCK_CONTINUATION_KEYWORDS" +is_literal(k::Kind) = K"BEGIN_LITERAL" <= k <= K"END_LITERAL" +is_number(k::Kind) = K"BEGIN_NUMBERS" <= k <= K"END_NUMBERS" +is_operator(k::Kind) = k == K"Operator" || K"BEGIN_OPS" <= k <= K"END_OPS" +is_word_operator(k::Kind) = (k == K"in" || k == K"isa" || k == K"where") + +is_identifier(x) = is_identifier(kind(x)) +is_contextual_keyword(x) = is_contextual_keyword(kind(x)) +is_error(x) = is_error(kind(x)) +is_keyword(x) = is_keyword(kind(x)) +is_literal(x) = is_literal(kind(x)) +is_number(x) = is_number(kind(x)) +is_operator(x) = is_operator(kind(x)) +is_word_operator(x) = is_word_operator(kind(x)) + +# Predicates for operator precedence +# FIXME: Review how precedence depends on dottedness, eg +# https://github.com/JuliaLang/julia/pull/36725 + + +# Most operators no longer have a dedicated kind - they're represented by +# K"Operator" with the precedence level stored in the numeric flags. A few +# operators are still kept as distinct kinds because they're treated specially +# during parsing, so the precedence predicates below additionally check for them. +_is_op_prec(x, prec) = kind(x) == K"Operator" && numeric_flags(head(x)) == Int(prec) + +is_prec_assignment(x) = K"BEGIN_ASSIGNMENTS" <= kind(x) <= K"END_ASSIGNMENTS" +is_prec_pair(x) = _is_op_prec(x, PREC_PAIRARROW) +is_prec_conditional(x) = kind(x) == K"?" +is_prec_arrow(x) = _is_op_prec(x, PREC_ARROW) || kind(x) == K"-->" +is_prec_lazy_or(x) = _is_op_prec(x, PREC_LAZYOR) || kind(x) in KSet"||" +is_prec_lazy_and(x) = _is_op_prec(x, PREC_LAZYAND) || kind(x) in KSet"&&" +is_prec_comparison(x) = _is_op_prec(x, PREC_COMPARISON) || kind(x) in KSet"<: >: in isa < > ∈" +is_prec_pipe_lt(x) = _is_op_prec(x, PREC_PIPE_LT) +is_prec_pipe_gt(x) = _is_op_prec(x, PREC_PIPE_GT) +is_prec_pipe(x) = is_prec_pipe_lt(x) || is_prec_pipe_gt(x) +is_prec_colon(x) = _is_op_prec(x, PREC_COLON) || kind(x) == K".." +is_prec_plus(x) = _is_op_prec(x, PREC_PLUS) || kind(x) in KSet"+ +% - -% ± ∓ $ ++" +is_prec_bitshift(x) = _is_op_prec(x, PREC_BITSHIFT) +is_prec_times(x) = _is_op_prec(x, PREC_TIMES) || kind(x) in KSet"* *% ⋆ &" +is_prec_rational(x) = _is_op_prec(x, PREC_RATIONAL) +is_prec_power(x) = _is_op_prec(x, PREC_POWER) +is_prec_decl(x) = _is_op_prec(x, PREC_DECL) || kind(x) == K"::" +is_prec_where(x) = _is_op_prec(x, PREC_WHERE) || kind(x) == K"where" +is_prec_dot(x) = _is_op_prec(x, PREC_DOT) || kind(x) == K"." +is_prec_quote(x) = _is_op_prec(x, PREC_QUOTE) || kind(x) == K"'" +is_syntax_kind(x) = K"BEGIN_SYNTAX_KINDS"<= kind(x) <= K"END_SYNTAX_KINDS" +is_prec_compound_assign(x) = is_operator(x) && numeric_flags(head(x)) == Int(PREC_COMPOUND_ASSIGN) + +function is_string_delim(x) + kind(x) in (K"\"", K"\"\"\"") +end + +function is_radical_op(x) + kind(x) in (K"√", K"∛", K"∜") +end + +""" +Return true if `x` has whitespace or comment kind +""" +function is_whitespace(x) + k = kind(x) + return k == K"Whitespace" || k == K"NewlineWs" || k == K"Comment" +end + +function is_syntactic_operator(x) + k = kind(x) + # TODO: Do we need to disallow dotted and suffixed forms when this is used + # in the parser? The lexer itself usually disallows such tokens, so it's + # not clear whether we need to handle them. (Though note `.->` is a + # token...) + # Note the assignment-like kinds `= .= op= .op= :=` are all syntactic, just + # as they were when each had its own kind (before they were collapsed into + # `K"Operator"`). + return k in KSet"&& || . ... -> = := .= op= .op=" +end diff --git a/JuliaSyntax/src/julia/literal_parsing.jl b/JuliaSyntax/src/julia/literal_parsing.jl new file mode 100644 index 0000000000000..4db8537195498 --- /dev/null +++ b/JuliaSyntax/src/julia/literal_parsing.jl @@ -0,0 +1,475 @@ +""" +Nontrivia tokens (leaf nodes / literals) which are malformed are parsed into +ErrorVal when `ignore_errors=true` during parsing. +""" +struct ErrorVal +end + +Base.show(io::IO, ::ErrorVal) = printstyled(io, "✘", color=:light_red) + +#------------------------------------------------------------------------------- +# This file contains utility functions for converting undecorated source +# strings into Julia values. For example, string->number, string unescaping, etc. + +function parse_int_literal(str::AbstractString) + # TODO: A specialized code path here can be a lot faster and also + # allocation free + str = replace(replace(str, '_'=>""), '−'=>'-') + x = Base.tryparse(Int, str) + if Int === Int32 && isnothing(x) + x = Base.tryparse(Int64, str) + end + if isnothing(x) + x = Base.tryparse(Int128, str) + if x === nothing + x = Base.parse(BigInt, str) + end + end + return x +end + +function parse_uint_literal(str::AbstractString, k) + str = replace(str, '_'=>"") + if startswith(str, '+') + str = str[2:end] + end + ndigits = length(str)-2 + if k == K"HexInt" + return ndigits <= 2 ? Base.parse(UInt8, str) : + ndigits <= 4 ? Base.parse(UInt16, str) : + ndigits <= 8 ? Base.parse(UInt32, str) : + ndigits <= 16 ? Base.parse(UInt64, str) : + ndigits <= 32 ? Base.parse(UInt128, str) : + Base.parse(BigInt, str) + elseif k == K"BinInt" + ndigits = length(str)-2 + return ndigits <= 8 ? Base.parse(UInt8, str) : + ndigits <= 16 ? Base.parse(UInt16, str) : + ndigits <= 32 ? Base.parse(UInt32, str) : + ndigits <= 64 ? Base.parse(UInt64, str) : + ndigits <= 128 ? Base.parse(UInt128, str) : + Base.parse(BigInt, str) + elseif k == K"OctInt" + x = Base.tryparse(UInt64, str) + if isnothing(x) + x = Base.tryparse(UInt128, str) + if isnothing(x) + x = Base.parse(BigInt, str) + elseif ndigits > 43 + x = BigInt(x) + end + else + x = ndigits <= 3 && x <= typemax(UInt8) ? UInt8(x) : + ndigits <= 6 && x <= typemax(UInt16) ? UInt16(x) : + ndigits <= 11 && x <= typemax(UInt32) ? UInt32(x) : + ndigits <= 22 ? x : + ndigits <= 43 ? UInt128(x) : + BigInt(x) + end + return x + end +end + +#------------------------------------------------------------------------------- +""" +Like `Base.parse(Union{Float64,Float32}, str)`, but permits float underflow + +Parse a floating point value of type T. str[firstind:lastind] must be a valid floating point literal string. Returns a tuple (value, status) where status is :ok, :overflow, or :underflow. +""" +function parse_float_literal(::Type{T}, str::Union{String,SubString,Vector{UInt8}}, + firstind::Integer, endind::Integer) where {T} # force specialize with where {T} + strsize = endind - firstind + bufsz = 50 + if strsize < bufsz + buf = Ref{NTuple{bufsz, UInt8}}() + ptr = Base.unsafe_convert(Ptr{UInt8}, pointer_from_objref(buf)) + GC.@preserve str buf begin + n = _copy_normalize_number!(ptr, pointer(str, firstind), strsize) + _unsafe_parse_float(T, ptr, n) + end + else + # Slower path with allocation. + buf = Vector{UInt8}(undef, strsize+1) + ptr = pointer(buf) + GC.@preserve str buf begin + n = _copy_normalize_number!(ptr, pointer(str, firstind), strsize) + _unsafe_parse_float(T, ptr, n) + end + end +end + +# Like replace(replace(str, '_'=>""), '−'=>'-') +# dest must be of size at least srcsize+1 +function _copy_normalize_number!(dest, src, srcsize) + i = 0 + j = 0 + while i < srcsize + b = unsafe_load(src + i) + if b == UInt8('_') + i += 1 + continue + elseif b == 0xe2 && i+2 < srcsize && + unsafe_load(src + i + 1) == 0x88 && + unsafe_load(src + i + 2) == 0x92 + # src at i,i+1,i+2 is UTF-8 code for unicode minus sign '−' + b = UInt8('-') + i += 2 + end + unsafe_store!(dest+j, b) + i += 1 + j += 1 + end + unsafe_store!(dest+j, 0x00) + return j +end + +# Internals of parse_float_literal, split into a separate function to avoid some +# apparent codegen issues https://github.com/JuliaLang/julia/issues/46509 +# (perhaps we don't want the `buf` in `GC.@preserve buf` to be stack allocated +# on one branch and heap allocated in another?) +@inline function _unsafe_parse_float(::Type{Float64}, ptr, strsize) + Libc.errno(0) + endptr = Ref{Ptr{UInt8}}(C_NULL) + x = ccall(:jl_strtod_c, Cdouble, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, endptr) + @check endptr[] == ptr + strsize + status = :ok + if Libc.errno() == Libc.ERANGE + # strtod man page: + # * If the correct value would cause overflow, plus or + # minus HUGE_VAL, HUGE_VALF, or HUGE_VALL is returned and + # ERANGE is stored in errno. + # * If the correct value would cause underflow, a value with + # magnitude no larger than DBL_MIN, FLT_MIN, or LDBL_MIN is + # returned and ERANGE is stored in errno. + status = abs(x) < 1 ? :underflow : :overflow + end + return (x, status) +end + +@inline function _unsafe_parse_float(::Type{Float32}, ptr, strsize) + # Convert float exponent 'f' to 'e' for strtof, eg, 1.0f0 => 1.0e0 + # Presumes we can modify the data in ptr! + for p in ptr+strsize-1:-1:ptr + if unsafe_load(p) == UInt8('f') + unsafe_store!(p, UInt8('e')) + break + end + end + Libc.errno(0) + endptr = Ref{Ptr{UInt8}}(C_NULL) + status = :ok + @static if Sys.iswindows() + # Call strtod here and convert to Float32 on the Julia side because + # strtof seems buggy on windows and doesn't set ERANGE correctly on + # overflow. See also + # https://github.com/JuliaLang/julia/issues/46544 + x = Float32(ccall(:jl_strtod_c, Cdouble, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, endptr)) + if isinf(x) + status = :overflow + # Underflow not detected, but that will only be a warning elsewhere. + end + else + x = ccall(:jl_strtof_c, Cfloat, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, endptr) + end + @check endptr[] == ptr + strsize + if Libc.errno() == Libc.ERANGE + status = abs(x) < 1 ? :underflow : :overflow + end + return (x, status) +end + + +#------------------------------------------------------------------------------- +""" +Process Julia source code escape sequences for raw strings +""" +function unescape_raw_string(io::IO, txtbuf::Vector{UInt8}, + firstind, endind, is_cmd::Bool) + delim = is_cmd ? u8"`" : u8"\"" + i = firstind + while i < endind + c = txtbuf[i] + if c != u8"\\" + if c == u8"\r" + # convert literal \r and \r\n in strings to \n (issue #11988) + if i+1 < endind && txtbuf[i+1] == u8"\n" + i += 1 + end + c = u8"\n" + end + write(io, c) + i += 1 + continue + end + # Process \ escape sequences + j = i + while j < endind && txtbuf[j] == u8"\\" + j += 1 + end + nbackslash = j - i + if (j < endind && txtbuf[j] == delim) || j >= endind + # Backslashes before a delimiter must also be escaped + nbackslash = div(nbackslash,2) + end + for _ = 1:nbackslash + write(io, u8"\\") + end + i = j + if i < endind + write(io, txtbuf[i]) + i += 1 + end + end +end + +""" +Process Julia source code escape sequences for non-raw strings. +`txtbuf` should be passed without delimiting quotes. +""" +function unescape_julia_string(io::IO, txtbuf::Vector{UInt8}, + firstind, endind, diagnostics) + had_error = false + i = firstind + while i < endind + c = txtbuf[i] + if c != u8"\\" + if c == u8"\r" + # convert literal \r and \r\n in strings to \n (issue #11988) + if i+1 < endind && txtbuf[i+1] == u8"\n" + i += 1 + end + c = u8"\n" + end + write(io, c) + i = nextind(txtbuf, i) + continue + end + # Process \ escape sequences. See also Base.unescape_string which some + # of this code derives from (but which disallows \` \' \$) + escstart = i + i += 1 + if i >= endind + emit_diagnostic(diagnostics, escstart:endind-1, + error="invalid escape sequence") + had_error = true + break + end + c = txtbuf[i] + if c == u8"x" || c == u8"u" || c == u8"U" + n = k = 0 + m = c == u8"x" ? 2 : + c == u8"u" ? 4 : 8 + while (k += 1) <= m && i+1 < endind + nc = txtbuf[i+1] + n = u8"0" <= nc <= u8"9" ? n<<4 + (nc-u8"0") : + u8"a" <= nc <= u8"f" ? n<<4 + (nc-u8"a"+10) : + u8"A" <= nc <= u8"F" ? n<<4 + (nc-u8"A"+10) : break + i += 1 + end + if k == 1 || n > 0x10ffff + m == 4 ? u8"u" : u8"U" + msg = (m == 2) ? "invalid hex escape sequence" : + "invalid unicode escape sequence" + emit_diagnostic(diagnostics, escstart:i, error=msg) + had_error = true + else + if m == 2 # \x escape sequence + write(io, UInt8(n)) + else + print(io, Char(n)) + end + end + elseif u8"0" <= c <= u8"7" + k = 1 + n = Int(c - u8"0") + while (k += 1) <= 3 && i+1 < endind + c = txtbuf[i+1] + n = (u8"0" <= c <= u8"7") ? n<<3 + c-u8"0" : break + i += 1 + end + if n > 255 + emit_diagnostic(diagnostics, escstart:i, + error="invalid octal escape sequence") + had_error = true + else + write(io, UInt8(n)) + end + else + u = # C escapes + c == u8"n" ? u8"\n" : + c == u8"t" ? u8"\t" : + c == u8"r" ? u8"\r" : + c == u8"e" ? u8"\e" : + c == u8"b" ? u8"\b" : + c == u8"f" ? u8"\f" : + c == u8"v" ? u8"\v" : + c == u8"a" ? u8"\a" : + # Literal escapes allowed in Julia source + c == u8"\\" ? u8"\\" : + c == u8"'" ? u8"'" : + c == u8"\"" ? u8"\"" : + c == u8"$" ? u8"$" : + c == u8"`" ? u8"`" : + nothing + if isnothing(u) + emit_diagnostic(diagnostics, escstart:i, + error="invalid escape sequence") + had_error = true + else + write(io, u) + end + end + # For non-ascii characters we may not be in the middle of the UTF-8 + # encoding for that char, but this doesn't matter because unescaping + # only relies on the ascii subset. + i += 1 + end + return had_error +end + +#------------------------------------------------------------------------------- +# Unicode normalization. As of Julia 1.8, this is part of Base and the Unicode +# stdlib under the name `Unicode.julia_chartransform`. See +# https://github.com/JuliaLang/julia/pull/42561 +# +# To allow use on older Julia versions and to workaround the bug +# https://github.com/JuliaLang/julia/issues/45716 +# we reproduce a specialized version of that logic here. + +# static wrapper around user callback function +function utf8proc_custom_func(codepoint::UInt32, ::Ptr{Cvoid})::UInt32 + (codepoint == 0x025B ? 0x03B5 : # 'ɛ' => 'ε' + codepoint == 0x00B5 ? 0x03BC : # 'µ' => 'μ' + codepoint == 0x00B7 ? 0x22C5 : # '·' => '⋅' + codepoint == 0x0387 ? 0x22C5 : # '·' => '⋅' + codepoint == 0x2212 ? 0x002D : # '−' (\minus) => '-' + codepoint == 0x210F ? 0x0127 : # 'ℏ' (\hslash) => 'ħ' \hbar + codepoint) +end + +function utf8proc_decompose(str, options, buffer, nwords) + ret = ccall(:utf8proc_decompose_custom, Int, (Ptr{UInt8}, Int, Ptr{UInt8}, Int, Cint, Ptr{Cvoid}, Ptr{Cvoid}), + str, sizeof(str), buffer, nwords, options, + @cfunction(utf8proc_custom_func, UInt32, (UInt32, Ptr{Cvoid})), C_NULL) + ret < 0 && Base.Unicode.utf8proc_error(ret) + return ret +end + +function utf8proc_map(str::Union{String,SubString{String}}, options::Integer) + nwords = utf8proc_decompose(str, options, C_NULL, 0) + buffer = Base.StringVector(nwords*4) + nwords = utf8proc_decompose(str, options, buffer, nwords) + nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options) + nbytes < 0 && Base.Unicode.utf8proc_error(nbytes) + return String(resize!(buffer, nbytes)) +end + +function normalize_identifier(str) + flags = Base.Unicode.UTF8PROC_STABLE | Base.Unicode.UTF8PROC_COMPOSE + return isascii(str) ? str : utf8proc_map(str, flags) +end + + +#------------------------------------------------------------------------------- +function parse_julia_literal(txtbuf::Vector{UInt8}, head::SyntaxHead, srcrange) + k = kind(head) + # Any errors parsing literals are represented as ErrorVal() - this can + # happen when the user sets `ignore_errors=true` during parsing. + if k == K"Float" + v, code = parse_float_literal(Float64, txtbuf, first(srcrange), + last(srcrange)+1) + return (code === :ok || code === :underflow) ? v : ErrorVal() + elseif k == K"Float32" + v, code = parse_float_literal(Float32, txtbuf, first(srcrange), + last(srcrange)+1) + return (code === :ok || code === :underflow) ? v : ErrorVal() + elseif k == K"Char" + io = IOBuffer() + had_error = unescape_julia_string(io, txtbuf, first(srcrange), + last(srcrange)+1, Diagnostic[]) + if had_error + return ErrorVal() + else + seek(io, 0) + c = read(io, Char) + return eof(io) ? c : ErrorVal() + end + elseif k in KSet"String CmdString" + io = IOBuffer() + had_error = false + if has_flags(head, RAW_STRING_FLAG) + unescape_raw_string(io, txtbuf, first(srcrange), last(srcrange)+1, + k == K"CmdString") + else + had_error = unescape_julia_string(io, txtbuf, first(srcrange), + last(srcrange)+1, Diagnostic[]) + end + return had_error ? ErrorVal() : String(take!(io)) + elseif k == K"Bool" + return txtbuf[first(srcrange)] == u8"t" + elseif k == K"VERSION" + nv = numeric_flags(head) + return VersionNumber(1, nv ÷ 10, nv % 10) + end + + # TODO: Avoid allocating temporary String here + val_str = String(txtbuf[srcrange]) + if k == K"Integer" + parse_int_literal(val_str) + elseif k in KSet"BinInt OctInt HexInt" + parse_uint_literal(val_str, k) + elseif is_identifier(k) + if has_flags(head, RAW_STRING_FLAG) + io = IOBuffer() + unescape_raw_string(io, txtbuf, first(srcrange), last(srcrange)+1, false) + Symbol(normalize_identifier(String(take!(io)))) + else + Symbol(normalize_identifier(val_str)) + end + elseif is_operator(k) + isempty(srcrange) ? + Symbol(untokenize(k)) : # synthetic invisible tokens + Symbol(normalize_identifier(val_str)) + elseif k == K"error" + ErrorVal() + elseif is_syntax_kind(head) + nothing + elseif is_keyword(k) + # This should only happen for tokens nested inside errors + Symbol(val_str) + else + # Other kinds should only happen for tokens nested inside errors + # TODO: Consolidate this with the is_keyword() above? Something else? + ErrorVal() + end +end + +""" + lower_identifier_name(name, kind) + +Lower a Julia identifier `name` of given `kind` to the name used by the Julia +runtime. (In particular, this handles the name mangling of macros.) + +This is a lowering (rather than parsing) step, but is needed for `Expr` +conversion and is also used for pretty printing. +""" +function lower_identifier_name(name::AbstractString, k::Kind) + # Replicate eager lowering done by the flisp parser + if k == K"macro_name" + name == "." ? "@__dot__" : "@$name" + elseif k == K"StrMacroName" + "@$(name)_str" + elseif k == K"CmdMacroName" + "@$(name)_cmd" + else + name + end +end + +function lower_identifier_name(name::Symbol, k::Kind) + if k == K"Identifier" + name # avoid unnecessary conversion + else + Symbol(lower_identifier_name(string(name), k)) + end +end diff --git a/JuliaSyntax/src/julia/parser.jl b/JuliaSyntax/src/julia/parser.jl new file mode 100644 index 0000000000000..5abfb29d3c8d0 --- /dev/null +++ b/JuliaSyntax/src/julia/parser.jl @@ -0,0 +1,3856 @@ +""" + ParseState(stream::ParseStream) + +ParseState is an internal data structure wrapping `ParseStream` to carry parser +context as we recursively descend into the parse tree. For example, normally +`x -y` means `(x) - (y)`, but when parsing matrix literals we're in +`space_sensitive` mode, and `[x -y]` means [(x) (-y)]. +""" +struct ParseState + stream::ParseStream + + # Disable range colon for parsing ternary conditional operator + range_colon_enabled::Bool + # In space-sensitive mode "x -y" is 2 expressions, not a subtraction + space_sensitive::Bool + # Seeing `for` stops parsing macro arguments and makes a generator + for_generator::Bool + # Treat 'end' like a normal symbol instead of a reserved word + end_symbol::Bool + # Treat newline like ordinary whitespace instead of as a potential separator + whitespace_newline::Bool + # Enable parsing `where` with high precedence + where_enabled::Bool + # First byte of the content of the bare parens currently being parsed + # (0 otherwise). + paren_content_byte_index::UInt32 +end + +# Normal context +function ParseState(stream::ParseStream) + ParseState(stream, true, false, false, false, false, true, UInt32(0)) +end + +function ParseState(ps::ParseState; range_colon_enabled=nothing, + space_sensitive=nothing, for_generator=nothing, + end_symbol=nothing, whitespace_newline=nothing, + where_enabled=nothing, paren_content_byte_index=nothing) + ParseState(ps.stream, + range_colon_enabled === nothing ? ps.range_colon_enabled : range_colon_enabled, + space_sensitive === nothing ? ps.space_sensitive : space_sensitive, + for_generator === nothing ? ps.for_generator : for_generator, + end_symbol === nothing ? ps.end_symbol : end_symbol, + whitespace_newline === nothing ? ps.whitespace_newline : whitespace_newline, + where_enabled === nothing ? ps.where_enabled : where_enabled, + paren_content_byte_index === nothing ? + ps.paren_content_byte_index : paren_content_byte_index) +end + +# Functions to change parse state + +function normal_context(ps::ParseState) + ParseState(ps; + range_colon_enabled=true, + space_sensitive=false, + where_enabled=true, + for_generator=false, + end_symbol=false, + whitespace_newline=false) +end + +function with_space_sensitive(ps::ParseState) + ParseState(ps, + space_sensitive=true, + whitespace_newline=false) +end + +# Convenient wrappers for ParseStream + +function Base.peek(ps::ParseState, n=1; skip_newlines=nothing) + skip_nl = isnothing(skip_newlines) ? ps.whitespace_newline : skip_newlines + peek(ps.stream, n; skip_newlines=skip_nl) +end + +function peek_token(ps::ParseState, n=1; skip_newlines=nothing) + skip_nl = isnothing(skip_newlines) ? ps.whitespace_newline : skip_newlines + peek_token(ps.stream, n, skip_newlines=skip_nl) +end + +function peek_full_token(ps::ParseState, n=1; skip_newlines=nothing, kws...) + skip_nl = isnothing(skip_newlines) ? ps.whitespace_newline : skip_newlines + peek_full_token(ps.stream, n; skip_newlines=skip_nl, kws...) +end + +function peek_behind(ps::ParseState, args...; kws...) + peek_behind(ps.stream, args...; kws...) +end + +function peek_behind_pos(ps::ParseState, args...; kws...) + peek_behind_pos(ps.stream, args...; kws...) +end + +function bump(ps::ParseState, flags=EMPTY_FLAGS; skip_newlines=nothing, kws...) + skip_nl = isnothing(skip_newlines) ? ps.whitespace_newline : skip_newlines + bump(ps.stream, flags; skip_newlines=skip_nl, kws...) +end + +function bump_trivia(ps::ParseState, args...; kws...) + bump_trivia(ps.stream, args...; kws...) +end + +function bump_invisible(ps::ParseState, args...; kws...) + bump_invisible(ps.stream, args...; kws...) +end + +function bump_glue(ps::ParseState, args...; kws...) + bump_glue(ps.stream, args...; kws...) +end + +function reset_node!(ps::ParseState, args...; kws...) + reset_node!(ps.stream, args...; kws...) +end + +function steal_token_bytes!(ps::ParseState, args...) + steal_token_bytes!(ps.stream, args...) +end + +function Base.position(ps::ParseState, args...) + position(ps.stream, args...) +end + +function emit(ps::ParseState, args...; kws...) + emit(ps.stream, args...; kws...) +end + +function emit_diagnostic(ps::ParseState, args...; kws...) + emit_diagnostic(ps.stream, args...; kws...) +end + +function unsafe_textbuf(ps::ParseState) + unsafe_textbuf(ps.stream) +end + +function first_child_position(ps::ParseState, pos::ParseStreamPosition) + first_child_position(ps.stream, pos) +end + +function last_child_position(ps::ParseState, pos::ParseStreamPosition) + last_child_position(ps.stream, pos) +end +#------------------------------------------------------------------------------- +# Parser Utils + +# Bump an expected closing token. If not found, discard unexpected tokens +# until we find it or another closing token. +# +# Crude recovery heuristic: bump any tokens which aren't block or bracket +# closing tokens. +function bump_closing_token(ps, closing_kind, alternative_closer_hint=nothing) + # todo: Refactor with recover() ? + if peek(ps) == closing_kind + bump_trivia(ps) + bump(ps, TRIVIA_FLAG) + return + end + errmsg = "Expected `$(untokenize(closing_kind))`" + if !isnothing(alternative_closer_hint) + errmsg *= alternative_closer_hint + end + # We didn't find the closing token. Read ahead in the stream + mark = position(ps) + emit_diagnostic(ps, mark, mark, error=errmsg) + while true + k = peek(ps) + if is_closing_token(ps, k) && !(k in KSet", ;") + break + end + bump(ps) + end + # mark as trivia => ignore in AST. + emit(ps, mark, K"error", TRIVIA_FLAG) + if peek(ps) == closing_kind + bump(ps, TRIVIA_FLAG) + end +end + +# Read tokens until we find an expected closing token. +# Bump the big pile of resulting tokens as a single nontrivia error token +function recover(is_closer::Function, ps, flags=EMPTY_FLAGS; mark = position(ps), error="unexpected tokens") + while true + k = peek(ps) + if k == K"EndMarker" + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="premature end of input") + break + elseif is_closer(ps, k) + break + end + bump(ps) + end + emit(ps, mark, K"error", flags, error=error) +end + +@noinline function min_supported_version_err(ps, mark, message, min_ver) + major, minor = ps.stream.version + msg = "$message not supported in Julia version $major.$minor < $(min_ver.major).$(min_ver.minor)" + emit(ps, mark, K"error", error=msg) +end + +# Emit an error if the version is less than `min_ver` +function min_supported_version(min_ver, ps, mark, message) + if ps.stream.version < (min_ver.major, min_ver.minor) + min_supported_version_err(ps, mark, message, min_ver) + end +end + +function min_supported_wrapping_arithmetic_op(ps, mark, t) + if kind(t) in KSet"+% -% *%" + min_supported_version(v"1.14", ps, mark, "wrapping arithmetic operators `+%`, `-%`, and `*%`") + end +end + +# flisp: disallow-space +function bump_disallowed_space(ps) + if preceding_whitespace(peek_token(ps)) + bump_trivia(ps, TRIVIA_FLAG, skip_newlines=false, + error="whitespace is not allowed here") + end +end + +function bump_semicolon_trivia(ps) + while peek(ps) in KSet"; NewlineWs" + bump(ps, TRIVIA_FLAG) + end +end + +#------------------------------------------------------------------------------- +# Parsing-specific predicates on tokens/kinds +# +# All these take either a raw kind or a token. + +function is_closing_token(ps::ParseState, k) + k = kind(k) + return k in KSet"else elseif catch finally , ) ] } ; EndMarker" || + (k == K"end" && !ps.end_symbol) +end + +function is_block_continuation_keyword(ps::ParseState, k) + is_block_continuation_keyword(k) && !(ps.end_symbol && k == K"end") +end + +function is_closer_or_newline(ps::ParseState, k) + is_closing_token(ps,k) || k == K"NewlineWs" +end + +# Closer for "Non-delimited reserved words" like `return` which aren't closed with an `end` +function is_nd_resword_closer(ps::ParseState, k) + return is_closer_or_newline(ps, k) || + (k == K":" && !ps.range_colon_enabled) +end + +function is_initial_reserved_word(ps::ParseState, k) + k = kind(k) + is_iresword = k in KSet"begin while if for try return break continue function + macro quote let local global const do struct typegroup module + baremodule using import export" + # `begin` means firstindex(a) inside a[...] + if k == K"begin" && ps.end_symbol + return false + end + # `typegroup` is only a keyword in Julia >= 1.14 + # N.B: In some cases, we parse as typegroup anyway for error recovery - see below + if k == K"typegroup" && ps.stream.version < (1, 14) + return false + end + return is_iresword +end + +function is_reserved_word(k) + k = kind(k) + is_keyword(k) && !is_contextual_keyword(k) +end + +# Return true if the next word (or word pair) is reserved, introducing a +# syntactic structure. +function peek_initial_reserved_words(ps::ParseState) + k = peek(ps) + if is_initial_reserved_word(ps, k) + return true + elseif is_contextual_keyword(k) + k2 = peek(ps, 2, skip_newlines=false) + return (k == K"mutable" && k2 == K"struct") || + (k == K"primitive" && k2 == K"type") || + (k == K"abstract" && k2 == K"type") + elseif k == K"typegroup" && ps.stream.version < (1, 14) + # On older versions, typegroup is an identifier. But if followed by + # a type definition keyword (which would be a syntax error in old + # Julia due to juxtaposition), parse as typegroup for error recovery. + k2 = peek(ps, 2, skip_newlines=false) + return k2 in KSet"struct mutable abstract primitive @ \" \"\"\"" + else + return false + end +end + +function is_block_form(k) + kind(k) in KSet"block quote if for while let function macro + abstract primitive struct typegroup try module" +end + +function is_syntactic_unary_op(k) + kind(k) in KSet"$ & ::" +end + +function is_type_operator(t, isdot) + kind(t) in KSet"<: >:" && !isdot +end + +function is_unary_op(t, isdot) + k = kind(t) + (k in KSet"<: >:" && !isdot) || + k in KSet"+ +% - -% ! ~ ¬ √ ∛ ∜ ⋆ ± ∓" # dotop allowed +end + +# Operators that are both unary and binary +function is_both_unary_and_binary(t, isdot) + k = kind(t) + k in KSet"+ +% - -% ⋆ ± ∓" || (k in KSet"$ & ~" && !isdot) +end + +function is_string_macro_suffix(k) + k == K"Identifier" || is_keyword(k) || is_word_operator(k) || is_number(k) +end + +# flisp: invalid-identifier? +function is_valid_identifier(k) + k = kind(k) + !(is_syntactic_operator(k) || k in KSet"? .'") +end + +# The expression is a call after stripping `where` and `::` +function was_eventually_call(ps::ParseState) + stream = ps.stream + p = peek_behind_pos(ps) + while true + b = peek_behind(stream, p) + if b.kind == K"call" + return true + elseif b.kind == K"where" || b.kind == K"parens" || + (b.kind == K"::" && has_flags(b.flags, INFIX_FLAG)) + if b.kind == K"::" + p_last = last_child_position(ps, p) + if p == p_last + return false + end + end + p = first_child_position(ps, p) + else + return false + end + end +end + + +#------------------------------------------------------------------------------- +# Parser +# +# The definitions and top-level comments here were copied to match the +# structure of Julia's previous flisp-based parser to make both codebases +# mutually understandable and make porting changes simple. +# +# The `parse_*` functions are listed here roughly in order of increasing +# precedence (lowest to highest binding power). A few helper functions are +# interspersed. + +# parse left-to-right binary operator +# produces structures like (+ (+ (+ 2 3) 4) 5) +# +# flisp: parse-LtoR +function parse_LtoR(ps::ParseState, down, is_op) + mark = position(ps) + down(ps) + while true + isdot, isassign, tk = peek_dotted_op_token(ps) + (is_op(tk) && !isassign) || break + isdot && bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG + bump(ps, remap_kind=K"Identifier") + down(ps) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + end +end + +# parse right-to-left binary operator +# produces structures like (=> a (=> b (=> c d))) +# +# flisp: parse-RtoL +function parse_RtoL(ps::ParseState, down, is_op, self) + mark = position(ps) + down(ps) + isdot, isassign, tk = peek_dotted_op_token(ps) + if is_op(tk) && !isassign + bump_dotted(ps, isdot, tk, remap_kind=K"Identifier") + self(ps) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + end +end + +# parse block-like structures +# +# `delimiters` are a set of token kinds acting as delimiters; `closing_tokens` +# stop the parsing. +# +# Returns true if the block was nontrivial and a node needs to be emitted by +# the caller. +# +# flisp: parse-Nary +function parse_Nary(ps::ParseState, down, delimiters, closing_tokens) + bump_trivia(ps) + k = peek(ps) + if k in closing_tokens + return true + end + n_delims = 0 + if k in delimiters + # allow leading delimiters + # ; a ==> (block a) + else + # a ; b ==> (block a b) + down(ps) + end + while peek(ps) in delimiters + bump(ps, TRIVIA_FLAG) + n_delims += 1 + k = peek(ps) + if k == K"EndMarker" || k in closing_tokens + break + elseif k in delimiters + # ignore empty delimited sections + # a;;;b ==> (block a b) + continue + end + down(ps) + end + return n_delims != 0 +end + +# Parse a sequence of top level statements separated by newlines, all wrapped +# in a toplevel node. +# +# a \n b ==> (toplevel a b) +# +# Note that parse_stmts can also emit toplevel nodes for semicolon-separated +# statements, so it's possible for these to be nested one level deep. +# +# a;b \n c;d ==> (toplevel (toplevel a b) (toplevel c d)) +function parse_toplevel(ps::ParseState) + mark = position(ps) + while true + if peek(ps, skip_newlines=true) == K"EndMarker" + # Allow end of input if there is nothing left but whitespace + # a \n \n ==> (toplevel a) + # Empty files + # ==> (toplevel) + bump_trivia(ps) + break + else + bump_trivia(ps) + parse_stmts(ps) + end + end + emit(ps, mark, K"toplevel") + nothing +end + +# Parse a newline or semicolon-delimited list of expressions. +# Repeated delimiters are allowed but ignored +# a;b;c ==> (block a b c) +# a;;;b;; ==> (block a b) +# ;a ==> (block a) +# \n a ==> (block a) +# a \n b ==> (block a b) +# +# flisp: parse-block +function parse_block(ps::ParseState, down=parse_eq, mark=position(ps)) + parse_block_inner(ps::ParseState, down) + emit(ps, mark, K"block") +end + +# Parse a block, but leave emitting the block up to the caller. +function parse_block_inner(ps::ParseState, down::F) where {F <: Function} + parse_Nary(ps, down, KSet"NewlineWs ;", KSet"end else elseif catch finally") +end + +# ";" at the top level produces a sequence of top level expressions +# +# a;b;c ==> (toplevel a b c) +# a;;;b;; ==> (toplevel a b) +# "x" a ; "y" b ==> (toplevel (doc (string "x") a) (doc (string "y") b)) +# +# flisp: parse-stmts +function parse_stmts(ps::ParseState) + mark = position(ps) + do_emit = parse_Nary(ps, parse_public, (K";",), (K"NewlineWs",)) + # check for unparsed junk after an expression + junk_mark = position(ps) + while peek(ps) ∉ KSet"EndMarker NewlineWs" + # Error recovery + bump(ps) + end + if junk_mark != position(ps) + # x y ==> x (error-t y) + emit(ps, junk_mark, K"error", TRIVIA_FLAG, + error="extra tokens after end of expression") + end + if do_emit + emit(ps, mark, K"toplevel", TOPLEVEL_SEMICOLONS_FLAG) + end +end + +# Parse `public foo, bar` +# +# We *only* call this from toplevel contexts (file and module level) for +# compatibility. In the future we should probably make public a full fledged +# keyword like `export`. +function parse_public(ps::ParseState) + if ps.stream.version >= (1, 11) && peek(ps) == K"public" + if peek(ps, 2) ∈ KSet"( = [" + # this branch is for compatibility with use of public as a non-keyword. + # it should be removed at some point. + emit_diagnostic(ps, warning="using public as an identifier is deprecated") + else + return parse_resword(ps) + end + end + parse_docstring(ps) +end + +# Parse docstrings attached by a space or single newline +# +# flisp: parse-docstring +function parse_docstring(ps::ParseState, down=parse_eq) + mark = position(ps) + down(ps) + if peek_behind(ps).kind == K"string" + is_doc = true + k = peek(ps) + if is_closing_token(ps, k) + # "notdoc" ] ==> (string "notdoc") + is_doc = false + elseif k == K"NewlineWs" + k2 = peek(ps, 2) + if is_closing_token(ps, k2) || k2 == K"NewlineWs" + # "notdoc" \n] ==> (string "notdoc") + # "notdoc" \n\n foo ==> (string "notdoc") + is_doc = false + else + # Allow a single newline + # "doc" \n foo ==> (doc (string "doc") foo) + bump(ps, TRIVIA_FLAG) # NewlineWs + end + else + # "doc" foo ==> (doc (string "doc") foo) + # "doc $x" foo ==> (doc (string "doc " x) foo) + # Allow docstrings with embedded trailing whitespace trivia + # """\n doc\n """ foo ==> (doc (string-s "doc\n") foo) + end + if is_doc + down(ps) + emit(ps, mark, K"doc") + end + end +end + +# Parse assignments with comma separated lists on each side +# a = b ==> (= a b) +# a .= b ==> (.= a b) +# a += b ==> (+= a b) +# a .+= b ==> (.+= a b) +# a, b = c, d ==> (= (tuple a b) (tuple c d)) +# x, = xs ==> (= (tuple x) xs) +# +# flisp: parse-eq +function parse_eq(ps::ParseState) + parse_assignment(ps, parse_comma) +end + +# parse_eq_star is used where commas are special, for example in an argument list +# +# flisp: parse-eq* +function parse_eq_star(ps::ParseState) + k = peek(ps) + k2 = peek(ps,2) + if (is_literal(k) || k == K"Identifier") && k2 in KSet", ) } ]" + # optimization: skip checking the whole precedence stack if we have a + # simple token followed by a common closing token + bump(ps) + else + parse_assignment(ps, parse_pair) + end +end + +# a = b ==> (= a b) +# +# flisp: parse-assignment +function parse_assignment(ps::ParseState, down) + mark = position(ps) + down(ps) + parse_assignment_with_initial_ex(ps, mark, down) +end + +function parse_assignment_with_initial_ex(ps::ParseState, mark, down::T) where {T} # where => specialize on `down` + isdot, is_compound_assignment, t = peek_dotted_op_token(ps) + k = kind(t) + + if !is_prec_assignment(t) && !is_compound_assignment + return + end + + if k == K"~" + if ps.space_sensitive && preceding_whitespace(t) && !preceding_whitespace(peek_token(ps, 2)) + # Unary ~ in space sensitive context is not assignment precedence + # [a ~b] ==> (hcat a (call-pre ~ b)) + return + end + # ~ is currently the only assignment-precedence operator which is parsed as a call. + # TODO: Make the other non-syntactic assignments such as `≔ ⩴ ≕` into calls as well? + # a ~ b ==> (call-i a ~ b) + # a .~ b ==> (dotcall-i a ~ b) + # [a ~ b c] ==> (hcat (call-i a ~ b) c) + # [a~b] ==> (vect (call-i a ~ b)) + bump_dotted(ps, isdot, t, remap_kind=K"Identifier") + bump_trivia(ps) + parse_assignment(ps, down) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + else + # f() = 1 ==> (function-= (call f) 1) + # f() .= 1 ==> (.= (call f) 1) + # a .= b ==> (.= a b) + is_short_form_func = k == K"=" && !isdot && was_eventually_call(ps) + if is_compound_assignment + # x += y ==> (op= x + y) + # x .+= y ==> (.op= x + y) + bump_trivia(ps) + bump_dotted(ps, isdot, t, remap_kind=K"Identifier") + bump(ps, TRIVIA_FLAG) # bump the = + k = K"op=" # Set k for the emit below + else + bump_dotted(ps, isdot, t, TRIVIA_FLAG) + end + bump_trivia(ps) + # Syntax Edition TODO: We'd like to call `down` here when + # is_short_form_func is true, to prevent `f() = 1 = 2` from parsing. + parse_assignment(ps, down) + emit(ps, mark, + is_short_form_func ? K"function" : (isdot ? dotted(k) : k), + is_short_form_func ? SHORT_FORM_FUNCTION_FLAG : flags(t)) + end +end + +# parse_comma is needed for commas outside parens, for example a = b,c +# +# flisp: parse-comma +function parse_comma(ps::ParseState, do_emit=true) + mark = position(ps) + n_commas = 0 + parse_pair(ps) + while true + if peek(ps) != K"," + if do_emit && n_commas >= 1 + emit(ps, mark, K"tuple") + end + return n_commas + end + bump(ps, TRIVIA_FLAG) + n_commas += 1 + if kind(peek_token(ps)) == K"=" + # Allow trailing comma before `=` + # x, = xs ==> (tuple x) + continue + end + parse_pair(ps) + end +end + +# flisp: parse-pair +# a => b ==> (call-i a => b) +# a .=> b ==> (dotcall-i a => b) +function parse_pair(ps::ParseState) + parse_RtoL(ps, parse_cond, is_prec_pair, parse_pair) +end + +# Parse short form conditional expression +# a ? b : c ==> (? a b c) +# +# flisp: parse-cond +function parse_cond(ps::ParseState) + mark = position(ps) + parse_arrow(ps) + t = peek_token(ps) + if kind(t) != K"?" + return + end + if !preceding_whitespace(t) + # a? b : c => (? a (error-t) b c) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="space required before `?` operator") + end + bump(ps, TRIVIA_FLAG) # ? + t = peek_token(ps) + if !preceding_whitespace(t) + # a ?b : c ==> (? a (error-t) b c) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="space required after `?` operator") + end + parse_eq_star(ParseState(ps, range_colon_enabled=false)) + t = peek_token(ps) + if !preceding_whitespace(t) + # a ? b: c ==> (? a b (error-t) c) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="space required before `:` in `?` expression") + end + if kind(t) == K":" + bump(ps, TRIVIA_FLAG) + else + # a ? b c ==> (? a b (error-t) c) + bump_invisible(ps, K"error", TRIVIA_FLAG, error="`:` expected in `?` expression") + end + t = peek_token(ps; skip_newlines = true) + if !preceding_whitespace(t) + # a ? b :c ==> (? a b (error-t) c) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="space required after `:` in `?` expression") + end + + # FIXME: This is a very specific case. Error recovery should be handled more + # generally elsewhere. + if is_block_continuation_keyword(ps, kind(t)) + # a "continuation keyword" is likely to belong to the surrounding code, so + # we abort early + + # if true; x ? true elseif true end ==> (if true (block (if x true (error-t) (error-t))) (elseif true (block))) + # if true; x ? true end ==> (if true (block (if x true (error-t) (error-t)))) + # if true; x ? true\n end ==> (if true (block (if x true (error-t) (error-t)))) + # if true; x ? true : elseif true end ==> (if true (block (if x true (error-t))) (elseif true (block))) + bump_invisible(ps, K"error", TRIVIA_FLAG, error="unexpected `$(kind(t))`") + emit(ps, mark, K"if") + return + else + # A[x ? y : end] ==> (ref A (? x y end)) + end + parse_eq_star(ps) + emit(ps, mark, K"?") +end + +# Parse arrows. Like parse_RtoL, but specialized for --> syntactic operator +# +# flisp: parse-arrow +function parse_arrow(ps::ParseState) + mark = position(ps) + parse_or(ps) + isdot, isassign, t = peek_dotted_op_token(ps) + k = kind(t) + if is_prec_arrow(t) + if kind(t) == K"-->" && !isdot + # x --> y ==> (--> x y) # The only syntactic arrow + bump(ps, TRIVIA_FLAG) + parse_arrow(ps) + emit(ps, mark, k, flags(t)) + else + # x → y ==> (call-i x → y) + # x <--> y ==> (call-i x <--> y) + # x .--> y ==> (dotcall-i x --> y) + # x -->₁ y ==> (call-i x -->₁ y) + bump_dotted(ps, isdot, t, remap_kind=K"Identifier") + parse_arrow(ps) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + end + end +end + +function dotted(k) + if k == K"||" + return K".||" + elseif k == K"&&" + return K".&&" + elseif k == K"=" + return K".=" + elseif k == K"op=" + return K".op=" + else + error("Unexpected dotted operator: $k") + end +end + +# Like parse_RtoL, but specialized for the version test of dotted operators. +function parse_lazy_cond(ps::ParseState, down, is_op, self) + mark = position(ps) + down(ps) + (isdot, isassign, t) = peek_dotted_op_token(ps) + k = kind(t) + if is_op(t) + bump_dotted(ps, isdot, t, TRIVIA_FLAG) + self(ps) + emit(ps, mark, isdot ? dotted(k) : k, flags(t)) + if isdot + min_supported_version(v"1.7", ps, mark, "dotted operators `.||` and `.&&`") + end + end +end + +# x || y || z ==> (|| x (|| y z)) +#v1.6: x .|| y ==> (error (.|| x y)) +#v1.7: x .|| y ==> (.|| x y) +# +# flisp: parse-or +function parse_or(ps::ParseState) + parse_lazy_cond(ps, parse_and, is_prec_lazy_or, parse_or) +end + +# x && y && z ==> (&& x (&& y z)) +#v1.6: x .&& y ==> (error (.&& x y)) +#v1.7: x .&& y ==> (.&& x y) +# +# flisp: parse-and +function parse_and(ps::ParseState) + parse_lazy_cond(ps, parse_comparison, is_prec_lazy_and, parse_and) +end + +# Parse binary comparisons and comparison chains +# +# flisp: parse-comparison +function parse_comparison(ps::ParseState, subtype_comparison=false) + mark = position(ps) + if subtype_comparison && is_reserved_word(peek(ps)) + # Recovery + # struct try end ==> (struct (error (try)) (block)) + name = untokenize(peek(ps)) + bump(ps) + emit(ps, mark, K"error", error="Invalid type name `$name`") + else + parse_pipe_lt(ps) + end + n_comparisons = 0 + op_pos = NO_POSITION + op_dotted = false + (initial_dot, initial_isassign, initial_tok) = peek_dotted_op_token(ps) + while ((isdot, isassign, t) = peek_dotted_op_token(ps); is_prec_comparison(t)) + n_comparisons += 1 + op_dotted = isdot + op_pos = bump_dotted(ps, isdot, t, emit_dot_node=true, remap_kind=K"Identifier") + parse_pipe_lt(ps) + end + if n_comparisons == 1 + if is_type_operator(initial_tok, initial_dot) + # Type comparisons are syntactic + # x <: y ==> (<: x y) + # x >: y ==> (>: x y) + reset_node!(ps, op_pos, flags=TRIVIA_FLAG) + emit(ps, mark, kind(initial_tok)) + else + # Normal binary comparisons + # x < y ==> (call-i x < y) + # x .< y ==> (dotcall-i x < y) + if op_dotted + # Reset the extra (non-terminal) K"." (e.g. in `(. <)`) node to just `. <` + reset_node!(ps, op_pos, kind=K"TOMBSTONE", flags=TRIVIA_FLAG) + end + emit(ps, mark, op_dotted ? K"dotcall" : K"call", INFIX_FLAG) + end + elseif n_comparisons > 1 + # Comparison chains + # x < y < z ==> (comparison x < y < z) + # x == y < z ==> (comparison x == y < z) + # x .< y .< z ==> (comparison x (. <) y (. <) z) + # x .< y < z ==> (comparison x (. <) y < z) + emit(ps, mark, K"comparison") + end +end + +# x <| y <| z ==> (call-i x <| (call-i y <| z)) +# flisp: parse-pipe< +function parse_pipe_lt(ps::ParseState) + parse_RtoL(ps, parse_pipe_gt, is_prec_pipe_lt, parse_pipe_lt) +end + +# x |> y |> z ==> (call-i (call-i x |> y) |> z) +# x .|> y ==> (dotcall-i x |> y) +# flisp: parse-pipe> +function parse_pipe_gt(ps::ParseState) + parse_LtoR(ps, parse_range, is_prec_pipe_gt) +end + +# parse ranges and postfix ... +# colon is strange; 3 arguments with 2 colons yields one call: +# 1:2 ==> (call-i 1 : 2) +# 1:2:3 ==> (call-i 1 : 2 3) +# Chaining gives +# a:b:c:d:e ==> (call-i (call-i a : b c) : d e) +# +# flisp: parse-range +function parse_range(ps::ParseState) + mark = position(ps) + parse_invalid_ops(ps) + + # The compound-assignment flag (`a ..= b` etc.) is intentionally ignored + # here; such forms are rejected in `parse_assignment_with_initial_ex`. + (initial_dot, _, initial_tok) = peek_dotted_op_token(ps) + initial_kind = kind(initial_tok) + if initial_kind != K":" && (is_prec_colon(initial_tok) || (initial_dot && initial_kind == K".")) + # a..b ==> (call-i a (DotsIdentifier-2) b) + # a … b ==> (call-i a … b) + # a .… b ==> (dotcall-i a … b) + bump_dotted(ps, initial_dot, initial_tok, remap_kind=K"Identifier") + parse_invalid_ops(ps) + emit(ps, mark, (initial_dot && initial_kind != K".") ? K"dotcall" : K"call", INFIX_FLAG) + elseif initial_kind == K":" && ps.range_colon_enabled + # a ? b : c:d ==> (? a b (call-i c : d)) + n_colons = 0 + while peek(ps) == K":" + if ps.space_sensitive && + preceding_whitespace(peek_token(ps)) && + !preceding_whitespace(peek_token(ps, 2)) + # Tricky cases in space sensitive mode + # [1 :a] ==> (hcat 1 (quote-: a)) + # [1 2:3 :a] ==> (hcat 1 (call-i 2 : 3) (quote-: a)) + break + end + t2 = peek_token(ps,2) + if kind(t2) in KSet"< >" && !preceding_whitespace(t2) + # Error heuristic: we found `:>` or `:<` which are invalid lookalikes + # for `<:` and `>:`. Attempt to recover by treating them as a + # comparison operator. + # a :> b ==> (call-i a (error : >) b) + bump_trivia(ps, skip_newlines=false) + emark = position(ps) + bump(ps, remap_kind=K"Identifier") # K":" + ks = untokenize(peek(ps)) + bump(ps, remap_kind=K"Identifier") # K"<" or K">" + emit(ps, emark, K"error", + error="Invalid `:$ks` found, maybe replace with `$ks:`") + parse_invalid_ops(ps) + emit(ps, mark, K"call", INFIX_FLAG) + break + end + n_colons += 1 + bump(ps, n_colons == 1 ? EMPTY_FLAGS : TRIVIA_FLAG; remap_kind=K"Identifier") + had_newline = peek(ps) == K"NewlineWs" + t = peek_token(ps) + if is_closing_token(ps, kind(t)) + # 1: } ==> (call-i 1 : (error)) + # 1:2: } ==> (call-i 1 : 2 (error)) + bump_invisible(ps, K"error", + error="missing last argument in range expression") + emit(ps, mark, K"call", INFIX_FLAG) + emit_diagnostic(ps, error="found unexpected closing token") + return + end + if had_newline + # Error message for people coming from python + # 1:\n2 ==> (call-i 1 : (error)) + # (1:\n2) ==> (parens (call-i 1 : 2)) + emit_diagnostic(ps, whitespace=true, + error="line break after `:` in range expression") + bump_invisible(ps, K"error") + emit(ps, mark, K"call", INFIX_FLAG) + return + end + parse_invalid_ops(ps) + if n_colons == 2 + emit(ps, mark, K"call", INFIX_FLAG) + n_colons = 0 + end + end + if n_colons > 0 + emit(ps, mark, K"call", INFIX_FLAG) + end + end + + # x... ==> (... x) + # x:y... ==> (... (call-i x : y)) + # x..y... ==> (... (call-i x .. y)) # flisp parser fails here + if peek(ps) == K"." && peek(ps, 2) == K"." && peek(ps, 3) == K"." + bump(ps, TRIVIA_FLAG) + bump(ps, TRIVIA_FLAG) # second dot + bump(ps, TRIVIA_FLAG) # third dot + emit(ps, mark, K"...") + end +end + +# Parse invalid binary operators +# +# Having this is unnecessary, but it improves error messages and the +# error-containing parse tree. +# +# a--b ==> (call-i a (error) b) +function parse_invalid_ops(ps::ParseState) + mark = position(ps) + parse_expr(ps) + while ((isdot, isassign, t) = peek_dotted_op_token(ps); kind(t) in KSet"ErrorInvalidOperator Error**") + bump_trivia(ps) + bump_dotted(ps, isdot, t) + parse_expr(ps) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + end +end + +# a - b - c ==> (call-i (call-i a - b) - c) +# a + b + c ==> (call-i a + b c) +# a .+ b ==> (dotcall-i a + b) +# +# flisp: parse-expr +function parse_expr(ps::ParseState) + parse_with_chains(ps, parse_term, is_prec_plus, KSet"+ +% ++") +end + +# a * b * c ==> (call-i a * b c) +# +# flisp: parse-term +function parse_term(ps::ParseState) + parse_with_chains(ps, parse_rational, is_prec_times, KSet"* *%") +end + +# Parse left to right, combining any of `chain_ops` into one call +# +# flisp: parse-with-chains +function parse_with_chains(ps::ParseState, down, is_op, chain_ops) + mark = position(ps) + down(ps) + while ((isdot, isassign, t) = peek_dotted_op_token(ps); is_op(t) && !isassign) + if ps.space_sensitive && preceding_whitespace(t) && + is_both_unary_and_binary(t, isdot) && + !preceding_whitespace(peek_token(ps, 2)) + # The following is two elements of a hcat + # [x +y] ==> (hcat x (call-pre + y)) + # [x+y +z] ==> (hcat (call-i x + y) (call-pre + z)) + # Conversely the following are infix calls + # [x +₁y] ==> (vect (call-i x +₁ y)) + # [x+y+z] ==> (vect (call-i x + y z)) + # [x+y + z] ==> (vect (call-i x + y z)) + break + end + bump_dotted(ps, isdot, t, remap_kind=K"Identifier") + down(ps) + if kind(t) in chain_ops && !isdot + # a + b + c ==> (call-i a + b c) + # a + b .+ c ==> (dotcall-i (call-i a + b) + c) + parse_chain(ps, down, kind(t)) + end + # a +₁ b +₁ c ==> (call-i (call-i a +₁ b) +₁ c) + # a .+ b .+ c ==> (dotcall-i (dotcall-i a + b) + c) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + end +end + +# parse left to right chains of a given binary operator +# +# flisp: parse-chain +function parse_chain(ps::ParseState, down, op_kind) + while true + isdot, isassign, t = peek_dotted_op_token(ps) + if kind(t) != op_kind || isdot + break + end + if ps.space_sensitive && preceding_whitespace(t) && + is_both_unary_and_binary(t, false) && + !preceding_whitespace(peek_token(ps, 2)) + # [x +y] ==> (hcat x (call-pre + y)) + break + end + bump(ps, TRIVIA_FLAG) + down(ps) + end +end + +# flisp: parse-rational +# x // y // z ==> (call-i (call-i x // y) // z) +function parse_rational(ps::ParseState) + parse_LtoR(ps, parse_shift, is_prec_rational) +end + +# flisp: parse-shift +# x >> y >> z ==> (call-i (call-i x >> y) >> z) +function parse_shift(ps::ParseState) + parse_LtoR(ps, parse_unary_subtype, is_prec_bitshift) +end + +# parse `<: A where B` as `<: (A where B)` (issue #21545) +# +# flisp: parse-unary-subtype +function parse_unary_subtype(ps::ParseState) + t = peek_token(ps) + if is_type_operator(t, false) + k2 = peek(ps, 2) + if is_closing_token(ps, k2) || k2 in KSet"NewlineWs =" + # return operator by itself + # <: ) ==> <: + # <: \n ==> <: + # <: = ==> <: + bump(ps) + elseif k2 in KSet"{ (" + # parse <:{T}(x::T) or <:(x::T) like other unary operators + # <:{T}(x::T) ==> (call (curly <: T) (:: x T)) + # <:(x::T) ==> (<:-pre (parens (:: x T))) + parse_where(ps, parse_juxtapose) + else + # <: x ==> (<:-pre x) + # <: A where B ==> (<:-pre (where A B)) + # <: <: x ==> (<:-pre (<:-pre x)) + mark = position(ps) + bump(ps, TRIVIA_FLAG) + parse_unary_subtype(ps) + emit(ps, mark, kind(t), PREFIX_OP_FLAG) + end + else + parse_where(ps, parse_juxtapose) + end +end + +# flisp: parse-where-chain +function parse_where_chain(ps0::ParseState, mark) + ps = ParseState(ps0, where_enabled=false) + while peek(ps) == K"where" + bump(ps, TRIVIA_FLAG) # where + bump_trivia(ps) + k = peek(ps) + if k == K"{" + # x where \n {T} ==> (where x (braces T)) + # x where {T,S} ==> (where x (braces T S)) + # Also various nonsensical forms permitted + # x where {T S} ==> (where x (bracescat (row T S))) + # x where {y for y in ys} ==> (where x (braces (generator y (iteration (in y ys))))) + m = position(ps) + bump(ps, TRIVIA_FLAG) + ckind, cflags, dim = parse_cat(ps, K"}", ps.end_symbol) + emit_braces(ps, m, ckind, cflags, dim) + emit(ps, mark, K"where") + else + # x where T ==> (where x T) + # x where \n T ==> (where x T) + # x where T<:S ==> (where x (<: T S)) + parse_comparison(ps) + emit(ps, mark, K"where") + end + end +end + +# flisp: parse-where +function parse_where(ps::ParseState, down) + # `where` needs to be below unary for the following to work + # +(x::T,y::T) where {T} = x + mark = position(ps) + down(ps) + if ps.where_enabled && peek(ps) == K"where" + parse_where_chain(ps, mark) + end +end + +# Juxtaposition. Kinda ugh but so useful for units and field identities like `im` +# +# flisp: parse-juxtapose +function parse_juxtapose(ps::ParseState) + mark = position(ps) + parse_unary(ps) + n_terms = 1 + while true + t = peek_token(ps) + k = kind(t) + prev_k = peek_behind(ps).kind + is_juxtapose = false + if !preceding_whitespace(t) && + (is_number(prev_k) || + (!is_number(k) && # disallow "x.3" and "f(2)2" + k != K"@" && # disallow "x@y" + !(is_block_form(prev_k) || + is_syntactic_unary_op(prev_k) || + is_initial_reserved_word(ps, prev_k) ))) && + (!is_operator(k) || is_radical_op(k)) && + !is_closing_token(ps, k) && + k != K"ErrorInvalidOperator" && k != K"Error**" + if prev_k == K"string" || is_string_delim(t) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="cannot juxtapose string literal") + # JuliaLang/julia#20575 + # Error, but assume juxtapose for recovery + # "a""b" ==> (juxtapose (string "a") (error-t) (string "b")) + # "a"x ==> (juxtapose (string "a") (error-t) x) + # "$y"x ==> (juxtapose (string y) (error-t) x) + # "a"begin end ==> (juxtapose (string \"a\") (error-t) (block)) + is_juxtapose = true + elseif !is_initial_reserved_word(ps, k) + # 2x ==> (juxtapose 2 x) + # 2(x) ==> (juxtapose 2 (parens x)) + # (2)(3)x ==> (juxtapose (parens 2) (parens 3) x) + # (x-1)y ==> (juxtapose (parens (call-i x - 1)) y) + # x'y ==> (juxtapose (call-post x ') y) + # 1√x ==> (juxtapose 1 (call-pre √ x)) + is_juxtapose = true + end + end + if !is_juxtapose + # x.3 ==> x + # f(2)2 ==> (call f 2) + # x' y ==> (call-post x ') + # x 'y ==> x + # x@y ==> x + break + end + if is_radical_op(t) + parse_unary(ps) + else + parse_factor(ps) + end + n_terms += 1 + end + if n_terms > 1 + emit(ps, mark, K"juxtapose") + end +end + +# Parse numeric literal prefixes, calls to unary operators and prefix +# calls involving arbitrary operators with bracketed arglists (as opposed to +# infix notation) +# +# flisp: parse-unary, parse-unary-call +function parse_unary(ps::ParseState) + mark = position(ps) + bump_trivia(ps) + (op_dotted, op_isassign, op_t) = peek_dotted_op_token(ps) + op_k = kind(op_t) + if ( + !is_operator(op_k) || + is_word_operator(op_k) || + (op_k in KSet": ' .'") || + (is_syntactic_unary_op(op_k) && !op_dotted) || + is_syntactic_operator(op_k) + ) + # `op_t` is not an initial operator + # :T ==> (quote-: T) + # in::T ==> (:: in T) + # isa::T ==> (:: isa T) + parse_factor(ps) + return + end + t2 = peek_token(ps, 2+op_dotted) + k2 = kind(t2) + if op_k in KSet"- +" && !op_dotted + if !preceding_whitespace(t2) && (k2 in KSet"Integer Float Float32" || + (op_k == K"+" && k2 in KSet"BinInt HexInt OctInt")) + + t3 = peek_token(ps, 3) + if is_prec_power(t3) || kind(t3) in KSet"[ {" + # `[`, `{` (issue #18851) and `^` have higher precedence than + # unary negation + # -2^x ==> (call-pre - (call-i 2 ^ x)) + # -2[1, 3] ==> (call-pre - (ref 2 1 3)) + bump(ps, remap_kind=K"Identifier") + parse_factor(ps) + emit(ps, mark, K"call", PREFIX_OP_FLAG) + else + # We have a signed numeric literal. Glue the operator to the + # next token to create a signed literal: + # -2 ==> -2 + # +2.0 ==> 2.0 + # -1.0f0 ==> -1.0f0 + # -2*x ==> (call-i -2 * x) + # +0xff ==> 0xff + bump_glue(ps, kind(t2), EMPTY_FLAGS) + parse_factor_with_initial_ex(ps, mark) + end + return + end + end + if is_closing_token(ps, k2) || k2 in KSet"NewlineWs =" + # Standalone operators parsed as `op` or `(. op)` + # +) ==> + + # +\n ==> + + # + = ==> + + # .+ ==> (. +) + # .& ==> (. &) + parse_atom(ps) + elseif k2 == K"{" || (!is_unary_op(op_t, op_dotted) && k2 == K"(") + # Call with type parameters or non-unary prefix call + # +{T}(x::T) ==> (call (curly + T) (:: x T)) + # *(x) ==> (call * x) + # .*(x) ==> (call .* x) + parse_factor(ps) + elseif k2 == K"(" + # Cases like +(a;b) are ambiguous: are they prefix calls to + with b as + # a keyword argument, or is `a;b` a block? We resolve this with a + # simple heuristic: if there were any commas (or an initial splat), it + # was a function call. + # + # (The flisp parser only considers commas before `;` and thus gets this + # last case wrong) + op_pos = bump_dotted(ps, op_dotted, op_t, emit_dot_node=true, remap_kind=K"Identifier") + + space_before_paren = preceding_whitespace(t2) + if space_before_paren + # Setup possible whitespace error between operator and ( + ws_mark = position(ps) + bump_trivia(ps) + ws_error_pos = emit(ps, ws_mark, K"TOMBSTONE") + ws_mark_end = position(ps) + end + + mark_before_paren = position(ps) + bump(ps, TRIVIA_FLAG) # ( + initial_semi = peek(ps, skip_newlines=true) == K";" + opts = parse_brackets(ps, K")") do had_commas, had_splat, num_semis, num_subexprs + is_paren_call = had_commas || had_splat || + (initial_semi && num_subexprs > 0) || + (initial_semi && num_semis == 1) || + (num_semis == 0 && num_subexprs == 0) + return (needs_parameters=is_paren_call, + is_paren_call=is_paren_call, + is_block=!is_paren_call && num_semis > 0) + end::NamedTuple{(:needs_parameters, :is_paren_call, :is_block, :delim_flags), Tuple{Bool, Bool, Bool, RawFlags}} + + # The precedence between unary + and any following infix ^ depends on + # whether the parens are a function call or not + if opts.is_paren_call + if space_before_paren + # Whitespace not allowed before prefix function call bracket + # + (a,b) ==> (call + (error) a b) + reset_node!(ps, ws_error_pos, kind=K"error") + emit_diagnostic(ps, ws_mark, ws_mark_end, + error="whitespace not allowed between prefix function call and argument list") + end + # Prefix function calls for operators which are both binary and unary + # +(a,b) ==> (call + a b) + # +(a=1,) ==> (call + (= a 1)) + # +(a...) ==> (call + (... a)) + # +(a;b,c) ==> (call + a (parameters b c)) + # +(;a) ==> (call + (parameters a)) + # +() ==> (call +) + # Prefix calls have higher precedence than ^ + # +(a,b)^2 ==> (call-i (call + a b) ^ 2) + # +(a,b)(x)^2 ==> (call-i (call (call + a b) x) ^ 2) + if is_type_operator(op_t, op_dotted) + # <:(a,) ==> (<: a) + emit(ps, mark, op_k, opts.delim_flags) + reset_node!(ps, op_pos, flags=TRIVIA_FLAG, kind=op_k) + else + emit(ps, mark, K"call", opts.delim_flags) + end + parse_call_chain(ps, mark) + parse_factor_with_initial_ex(ps, mark) + else + # Unary function calls with brackets as grouping, not an arglist + # .+(a) ==> (dotcall-pre + (parens a)) + if opts.is_block + # +(a;b) ==> (call-pre + (block-p a b)) + emit(ps, mark_before_paren, K"block", PARENS_FLAG) + else + emit(ps, mark_before_paren, K"parens") + end + # Not a prefix operator call but a block; `=` is not `kw` + # +(a=1) ==> (call-pre + (parens (= a 1))) + # Unary operators have lower precedence than ^ + # +(a)^2 ==> (call-pre + (call-i (parens a) ^ 2)) + # .+(a)^2 ==> (dotcall-pre + (call-i (parens a) ^ 2)) + # +(a)(x,y)^2 ==> (call-pre + (call-i (call (parens a) x y) ^ 2)) + parse_call_chain(ps, mark_before_paren) + parse_factor_with_initial_ex(ps, mark_before_paren) + if is_type_operator(op_t, op_dotted) + # <:(a) ==> (<:-pre (parens a)) + emit(ps, mark, op_k, PREFIX_OP_FLAG) + reset_node!(ps, op_pos, flags=TRIVIA_FLAG, kind=op_k) + else + if op_dotted + emit(ps, mark, K"dotcall", PREFIX_OP_FLAG) + # Reset the extra (non-terminal) K"." (e.g. in `(. +)`) node to just `. +` + reset_node!(ps, op_pos, kind=K"TOMBSTONE") + else + emit(ps, mark, K"call", PREFIX_OP_FLAG) + end + end + end + else + if is_unary_op(op_t, op_dotted) + # Normal unary calls + # +x ==> (call-pre + x) + # √x ==> (call-pre √ x) + # .~x ==> (dotcall-pre ~ x) + # Things which are not quite negative literals + # -0x1 ==> (call-pre - 0x01) + # - 2 ==> (call-pre - 2) + # .-2 ==> (dotcall-pre - 2) + op_pos = bump_dotted(ps, op_dotted, op_t, remap_kind=K"Identifier") + else + # /x ==> (call-pre (error /) x) + # +₁ x ==> (call-pre (error +₁) x) + # .<: x ==> (dotcall-pre (error (. <:)) x) + bump_dotted(ps, op_dotted, op_t, emit_dot_node=true, remap_kind=K"Identifier") + op_pos = emit(ps, mark, K"error", error="not a unary operator") + end + parse_unary(ps) + if is_type_operator(op_t, op_dotted) + reset_node!(ps, op_pos, flags=TRIVIA_FLAG) + emit(ps, mark, op_k, PREFIX_OP_FLAG) + else + emit(ps, mark, op_dotted ? K"dotcall" : K"call", PREFIX_OP_FLAG) + end + end +end + +# handle ^ and .^ +# +# x^y ==> (call-i x ^ y) +# x^y^z ==> (call-i x ^ (call-i y ^ z)) +# x .^ y ==> (dotcall-i x ^ y) +# begin x end::T ==> (:: (block x) T) +# +# flisp: parse-factor +function parse_factor(ps::ParseState) + mark = position(ps) + parse_call(ps) + parse_factor_with_initial_ex(ps, mark) +end + +# flisp: parse-factor-with-initial-ex +function parse_factor_with_initial_ex(ps::ParseState, mark) + parse_decl_with_initial_ex(ps, mark) + if ((isdot, isassign, t) = peek_dotted_op_token(ps); is_prec_power(t) && !isassign) + bump_dotted(ps, isdot, t, remap_kind=K"Identifier") + parse_factor_after(ps) + emit(ps, mark, isdot ? K"dotcall" : K"call", INFIX_FLAG) + end +end + +# flisp: parse-factor-after +function parse_factor_after(ps::ParseState) + parse_RtoL(ps, parse_juxtapose, is_prec_power, parse_factor_after) +end + +# Parse type declarations and lambda syntax +# a::b ==> (::-i a b) +# a->b ==> (-> a b) +# +# flisp: parse-decl-with-initial-ex +function parse_decl_with_initial_ex(ps::ParseState, mark) + while peek(ps) == K"::" + # a::b::c ==> (::-i (::-i a b) c) + bump(ps, TRIVIA_FLAG) + parse_where(ps, parse_call) + emit(ps, mark, K"::", INFIX_FLAG) + end + if peek(ps) == K"->" + kb = peek_behind(ps).kind + if kb == K"tuple" + # (x,y) -> z + # (x) -> y + # (x; a=1) -> y + elseif kb == K"where" + # `where` and `->` have the "wrong" precedence when writing anon functions. + # So ignore this case to allow use of grouping brackets with `where`. + # This needs to worked around in lowering :-( + # (x where T) -> y ==> (-> (x where T) y) + else + # x -> y ==> (-> (tuple x) y) + # a::b->c ==> (-> (tuple (::-i a b)) c) + emit(ps, mark, K"tuple") + end + bump(ps, TRIVIA_FLAG) + # -> is unusual: it binds tightly on the left and loosely on the right. + parse_eq_star(ps) + emit(ps, mark, K"->") + end +end + +# parse function call, indexing, dot, and transpose expressions +# also handles looking for syntactic reserved words +# +# flisp: parse-call +function parse_call(ps::ParseState) + if peek_initial_reserved_words(ps) + parse_resword(ps) + else + mark = position(ps) + # f(x) ==> (call f x) + # $f(x) ==> (call ($ f) x) + parse_unary_prefix(ps) + parse_call_chain(ps, mark) + end +end + +# parse syntactic unary operators +# +# &a ==> (& a) +# ::a ==> (::-pre a) +# $a ==> ($ a) +# +# flisp: parse-unary-prefix +function parse_unary_prefix(ps::ParseState, has_unary_prefix=false) + mark = position(ps) + (isdot, isassign, t) = peek_dotted_op_token(ps) + k = kind(t) + if is_syntactic_unary_op(k) && !isdot + k2 = peek(ps, 2) + if k in KSet"& $" && (is_closing_token(ps, k2) || k2 == K"NewlineWs") + # &) ==> & + # $\n ==> $ + bump(ps) + else + bump(ps, TRIVIA_FLAG) + if k in KSet"& ::" + # &a ==> (& a) + parse_where(ps, parse_call) + else + # $a ==> ($ a) + # $$a ==> ($ ($ a)) + # $&a ==> ($ (& a)) + parse_unary_prefix(ps, true) + end + # Only need PREFIX_OP_FLAG for :: + f = k == K"::" ? PREFIX_OP_FLAG : EMPTY_FLAGS + emit(ps, mark, k, f) + end + else + # .&(x,y) ==> (call .& x y) + parse_atom(ps, true, has_unary_prefix) + end +end + +function maybe_parsed_macro_name(ps, processing_macro_name, last_identifier_orig_kind, mark) + if processing_macro_name + emit(ps, mark, K"macro_name") + maybe_parsed_special_macro(ps, last_identifier_orig_kind) + end + return false +end + +function maybe_parsed_special_macro(ps, last_identifier_orig_kind) + is_special = last_identifier_orig_kind == K"VERSION" + if is_special && ps.stream.version >= (1, 14) + # Encode the current parser version into an invisible token + bump_invisible(ps, K"VERSION", + set_numeric_flags(ps.stream.version[2] * 10)) + end +end + +# Parses a chain of suffixes at function call precedence, leftmost binding +# tightest. This handles +# * Bracketed calls like a() b[] c{} +# * Field access like a.b.c +# - Various dotted syntax like f.() and f.:x +# * Adjoint suffix like a' +# * String macros like a"str" b"""str""" c`str` d```str``` +# +# f(a).g(b) ==> (call (. (call f a) g) b) +# +# flisp: parse-call-chain, parse-call-with-initial-ex +function parse_call_chain(ps::ParseState, mark, is_macrocall=false) + if is_number(peek_behind(ps).kind) && peek(ps) == K"(" + # juxtaposition with numbers is multiply, not call + # 2(x) ==> (* 2 x) + return + end + processing_macro_name = is_macrocall + saw_misplaced_atsym = false + misplaced_atsym_mark = nothing + # source range of the @-prefixed part of a macro + macro_atname_range = nothing + # $A.@x ==> (macrocall (. ($ A) (macro_name x))) + maybe_strmac = true + last_identifier_pos = peek_behind_pos(ps) + last_identifier_orig_kind = peek_behind(ps, last_identifier_pos).orig_kind + while true + maybe_strmac_1 = false + t = peek_token(ps) + k = kind(t) + if !is_macrocall && ps.space_sensitive && preceding_whitespace(t) && + k in KSet"( [ { \" \"\"\" ` ```" + # [f (x)] ==> (hcat f (parens x)) + # [f x] ==> (hcat f x) + break + elseif is_macrocall && (preceding_whitespace(t) || !(k in KSet"( [ { ' .")) + # Macro calls with space-separated arguments + # @foo a b ==> (macrocall (macro_name foo) a b) + # @foo (x) ==> (macrocall (macro_name foo) (parens x)) + # @foo (x,y) ==> (macrocall (macro_name foo) (tuple-p x y)) + # [@foo x] ==> (vect (macrocall (macro_name foo) x)) + # [@foo] ==> (vect (macrocall (macro_name foo))) + # @var"#" a ==> (macrocall (macro_name (var #)) a) + # A.@x y ==> (macrocall (. A (macro_name x)) y) + # A.@var"#" a ==> (macrocall (. A (macro_name (var #))) a) + # @+x y ==> (macrocall (macro_name +) x y) + # A.@.x ==> (macrocall (. A (macro_name .)) x) + maybe_parsed_macro_name( + ps, processing_macro_name, last_identifier_orig_kind, mark) + let ps = with_space_sensitive(ps) + # Space separated macro arguments + # A.@foo a b ==> (macrocall (. A (macro_name foo)) a b) + # @A.foo a b ==> (macrocall (macro_name (. A foo)) a b) + # + # 1.14: A macro call which is the immediate child of parens + # can span multiple lines + # (@foo x\n y) ==> (parens (macrocall (macro_name foo) x y)) + # (@m @n x y \n z) ==> (parens (macrocall m (macrocall n x y) z)) + # (x, @m a \n b) ==> (tuple-p x (macrocall m a) (error-t b)) + n_args = parse_space_separated_exprs( + ps, ps.paren_content_byte_index == mark.byte_index) + is_doc_macro = last_identifier_orig_kind == K"doc" + if is_doc_macro && n_args == 1 + # Parse extended @doc args on next line + # @doc x\ny ==> (macrocall (macro_name doc) x y) + # A.@doc x\ny ==> (macrocall (. A (macro_name doc)) x y) + # @A.doc x\ny ==> (macrocall (macro_name (. A doc)) x y) + # @doc x y\nz ==> (macrocall (macro_name doc) x y) + # + # Excluded cases + # @doc x\n\ny ==> (macrocall (macro_name doc) x) + # @doc x\nend ==> (macrocall (macro_name doc) x) + k2 = peek(ps, 2) + if peek(ps) == K"NewlineWs" && !is_closing_token(ps, k2) && + k2 != K"NewlineWs" + bump(ps) # newline + parse_eq(ps) + end + end + emit(ps, mark, K"macrocall") + end + break + elseif k == K"(" + # f(a,b) ==> (call f a b) + # f(a=1; b=2) ==> (call f (= a 1) (parameters (= b 2))) + # f(a; b; c) ==> (call f a (parameters b) (parameters c)) + # (a=1)() ==> (call (parens (= a 1))) + # f (a) ==> (call f (error-t) a) + processing_macro_name = maybe_parsed_macro_name( + ps, processing_macro_name, last_identifier_orig_kind, mark) + bump_disallowed_space(ps) + bump(ps, TRIVIA_FLAG) + opts = parse_call_arglist(ps, K")") + if peek(ps) == K"do" + # f(x) do y body end ==> (call f x (do (tuple y) (block body))) + parse_do(ps) + end + emit(ps, mark, is_macrocall ? K"macrocall" : K"call", + # TODO: Add PARENS_FLAG to all calls which use them? + (is_macrocall ? PARENS_FLAG : EMPTY_FLAGS)|opts.delim_flags) + if is_macrocall + # @x(a, b) ==> (macrocall-p (macro_name x) a b) + # A.@x(y) ==> (macrocall-p (. A (macro_name x)) y) + # A.@x(y).z ==> (. (macrocall-p (. A (macro_name x)) y) z) + is_macrocall = false + # @f()() ==> (call (macrocall-p (macro_name f))) + macro_atname_range = nothing + end + elseif k == K"[" + processing_macro_name = maybe_parsed_macro_name( + ps, processing_macro_name, last_identifier_orig_kind, mark) + m = position(ps) + # a [i] ==> (ref a (error-t) i) + bump_disallowed_space(ps) + bump(ps, TRIVIA_FLAG) + ckind, cflags, dim = parse_cat(ParseState(ps, end_symbol=true), + K"]", ps.end_symbol) + if is_macrocall + # @S[a,b] ==> (macrocall (macro_name S) (vect a b)) + # @S[a b] ==> (macrocall (macro_name S) (hcat a b)) + # @S[a; b] ==> (macrocall (macro_name S) (vcat a b)) + # A.@S[a] ==> (macrocall (. A (macro_name S)) (vect a)) + # @S[a].b ==> (. (macrocall (macro_name S) (vect a)) b) + #v1.7: @S[a ;; b] ==> (macrocall (macro_name S) (ncat-2 a b)) + #v1.6: @S[a ;; b] ==> (macrocall (macro_name S) (error (ncat-2 a b))) + emit(ps, m, ckind, cflags | set_numeric_flags(dim)) + check_ncat_compat(ps, m, ckind) + emit(ps, mark, K"macrocall") + is_macrocall = false + macro_atname_range = nothing + else + # a[i] ==> (ref a i) + # a[i,j] ==> (ref a i j) + # (a=1)[] ==> (ref (parens (= a 1))) + # a[end] ==> (ref a end) + # T[x y] ==> (typed_hcat T x y) + # T[x ; y] ==> (typed_vcat T x y) + # T[a b; c d] ==> (typed_vcat T (row a b) (row c d)) + # T[x for x in xs] ==> (typed_comprehension T (generator x (iteration (in x xs)))) + #v1.8: T[a ; b ;; c ; d] ==> (typed_ncat-2 T (nrow-1 a b) (nrow-1 c d)) + outk = ckind == K"vect" ? K"ref" : + ckind == K"hcat" ? K"typed_hcat" : + ckind == K"vcat" ? K"typed_vcat" : + ckind == K"comprehension" ? K"typed_comprehension" : + ckind == K"ncat" ? K"typed_ncat" : + internal_error("unrecognized kind in parse_cat ", string(ckind)) + emit(ps, mark, outk, cflags | set_numeric_flags(dim)) + check_ncat_compat(ps, mark, ckind) + end + elseif k == K"." + # Check if this is a dotted operator, not field access + k2 = peek(ps, 2) + if is_operator(k2) && !is_word_operator(k2) && k2 != K":" && k2 != K"$" && k2 != K"'" && k2 != K"?" + # This is a dotted operator like .=, .+, etc., not field access + # Let the appropriate parser handle it + break + end + # x .y ==> (. x (error-t) y) + bump_disallowed_space(ps) + emark = position(ps) + if !isnothing(macro_atname_range) + # Allow `@` in macrocall only in first and last position + # A.B.@x ==> (macrocall (. (. A B) (macro_name x))) + # @A.B.x ==> (macrocall (macro_name (. (. A B) x))) + # A.@B.x ==> (macrocall (. (. A (error-t) B) (macro_name (error-t) x))) + emit_diagnostic(ps, macro_atname_range..., + error="`@` must appear on first or last macro name component") + # Recover by treating the `@` as if it had been on the last identifier + saw_misplaced_atsym = true + reset_node!(ps, macro_atname_range[2], kind=K"TOMBSTONE") + reset_node!(ps, macro_atname_range[1], kind=K"error") + end + bump(ps, TRIVIA_FLAG) + k = peek(ps) + if k == K"(" + if is_macrocall + # Recover by pretending we do have the syntax + processing_macro_name = maybe_parsed_macro_name( + ps, processing_macro_name, last_identifier_orig_kind, mark) + # @M.(x) ==> (macrocall (dotcall (macro_name M) (error-t) x)) + bump_invisible(ps, K"error", TRIVIA_FLAG) + emit_diagnostic(ps, mark, + error="dot call syntax not supported for macros") + end + # f.(a,b) ==> (dotcall f a b) + # f. (x) ==> (dotcall f (error-t) x) + bump_disallowed_space(ps) + bump(ps, TRIVIA_FLAG) + opts = parse_call_arglist(ps, K")") + emit(ps, mark, K"dotcall", opts.delim_flags) + elseif k == K":" + # A.:+ ==> (. A (quote-: +)) + # A.: + ==> (. A (error-t) (quote-: +)) + m = position(ps) + bump(ps, TRIVIA_FLAG) + bump_disallowed_space(ps) + parse_atom(ps, false) + emit(ps, m, K"quote", COLON_QUOTE) + emit(ps, mark, K".") + elseif k == K"$" + # f.$x ==> (. f ($ x)) + # f.$(x+y) ==> (. f ($ (call + x y))) + # A.$B.@x ==> (macrocall (. (. A ($ B)) (macro_name x))) + # @A.$x a ==> (macrocall (macro_name (. A (error x))) a) + m = position(ps) + bump(ps, TRIVIA_FLAG) + parse_atom(ps) + if is_macrocall + emit(ps, m, K"error", error="invalid macro name") + else + emit(ps, m, K"$") + end + last_identifier_orig_kind = K"$" + emit(ps, mark, K".") + elseif k == K"@" + # A macro call after some prefix A has been consumed + # A.@x ==> (macrocall (. A (macro_name x))) + # A.@x a ==> (macrocall (. A (macro_name x)) a) + m = position(ps) + if is_macrocall + # @A.B.@x a ==> (macrocall (. (. A B) (error-t) (macro_name x)) a) + bump(ps, TRIVIA_FLAG, error="repeated `@` in macro module path") + else + bump(ps, TRIVIA_FLAG) + end + parse_macro_name(ps) + last_identifier_pos = peek_behind_pos(ps) + last_identifier_orig_kind = peek_behind(ps, last_identifier_pos).orig_kind + !is_macrocall && emit(ps, m, K"macro_name") + macro_atname_range = (m, position(ps)) + is_macrocall = true + emit(ps, mark, K".") + maybe_parsed_special_macro(ps, last_identifier_orig_kind) + elseif k == K"'" + # f.' => (dotcall-post f (error ')) + bump(ps, remap_kind=K"Identifier") # bump ' + # TODO: Reclaim dotted postfix operators :-) + emit(ps, emark, K"error", + error="the .' operator for transpose is discontinued") + emit(ps, mark, K"dotcall", POSTFIX_OP_FLAG) + elseif k == K"[" || k == K"{" + # f.[x] ==> (error f x) + # f.{x} ==> (error f x) + # Parse as broadcasted brackets, then wrap in error + close = k == K"[" ? K"]" : K"}" + bump(ps, TRIVIA_FLAG) + parse_cat(ParseState(ps, end_symbol=true), close, ps.end_symbol) + emit(ps, mark, K"error", error="brackets are not allowed after `.`") + else + if saw_misplaced_atsym + # If we saw a misplaced `@` earlier, this might be the place + # where it should have been. Opportunistically bump the + # zero-width error token here. If that's not right, we'll + # reset it later. + if misplaced_atsym_mark !== nothing + reset_node!(ps, misplaced_atsym_mark[1], kind=K"TOMBSTONE") + reset_node!(ps, misplaced_atsym_mark[2], kind=K"TOMBSTONE") + end + macro_name_mark = position(ps) + bump_invisible(ps, K"error", TRIVIA_FLAG) + aterror_mark = position(ps) + end + # Field/property syntax + # f.x.y ==> (. (. f x) y) + parse_atom(ps, false) + if saw_misplaced_atsym + emit(ps, macro_name_mark, K"macro_name") + misplaced_atsym_mark = (aterror_mark, position(ps)) + end + last_identifier_pos = peek_behind_pos(ps) + last_identifier_orig_kind = peek_behind(ps, last_identifier_pos).orig_kind + maybe_strmac_1 = true + emit(ps, mark, K".") + end + elseif is_prec_quote(t) && !preceding_whitespace(t) + # f' ==> (call-post f ') + # f'ᵀ ==> (call-post f 'ᵀ) + bump(ps, remap_kind=K"Identifier") + emit(ps, mark, K"call", POSTFIX_OP_FLAG) + elseif k == K"{" + processing_macro_name = maybe_parsed_macro_name( + ps, processing_macro_name, last_identifier_orig_kind, mark) + # Type parameter curlies and macro calls + m = position(ps) + # S {a} ==> (curly S (error-t) a) + bump_disallowed_space(ps) + bump(ps, TRIVIA_FLAG) + opts = parse_call_arglist(ps, K"}") + if is_macrocall + # @S{a,b} ==> (macrocall (macro_name S) (braces a b)) + # A.@S{a} ==> (macrocall (. A (macro_name S)) (braces a)) + # @S{a}.b ==> (. (macrocall (macro_name S) (braces a)) b) + emit(ps, m, K"braces", opts.delim_flags) + emit(ps, mark, K"macrocall") + min_supported_version(v"1.6", ps, mark, "macro call without space before `{}`") + is_macrocall = false + macro_atname_range = nothing + else + # S{a,b} ==> (curly S a b) + emit(ps, mark, K"curly", opts.delim_flags) + end + elseif k in KSet" \" \"\"\" ` ``` " && + !preceding_whitespace(t) && maybe_strmac && + (# Must mirror the logic in lex_quote() for consistency + origk = last_identifier_orig_kind; + origk == K"Identifier" || is_contextual_keyword(origk) || is_word_operator(origk)) + # Custom string and command literals + # x"str" ==> (macrocall @x_str (string-r "str")) + # x`str` ==> (macrocall @x_cmd (cmdstring-r "str")) + # x"" ==> (macrocall @x_str (string-r "")) + # x`` ==> (macrocall @x_cmd (cmdstring-r "")) + # Triple quoted processing for custom strings + # r"""\nx""" ==> (macrocall @r_str (string-s-r "x")) + # r"""\n x\n y""" ==> (macrocall @r_str (string-s-r "x\n" "y")) + # r"""\n x\\n y""" ==> (macrocall @r_str (string-s-r "x\\\n" "y")) + # + # Use a special token kind for string and cmd macro names so the + # names can be expanded later as necessary. + name_kind = is_string_delim(k) ? K"StrMacroName" : K"CmdMacroName" + reset_node!(ps, last_identifier_pos, kind=name_kind) + parse_string(ps, true) + t = peek_token(ps) + k = kind(t) + if !preceding_whitespace(t) && is_string_macro_suffix(k) + # Macro suffixes can include keywords and numbers + # x"s"y ==> (macrocall @x_str (string-r "s") "y") + # x"s"end ==> (macrocall @x_str (string-r "s") "end") + # x"s"in ==> (macrocall @x_str (string-r "s") "in") + # x"s"2 ==> (macrocall @x_str (string-r "s") 2) + # x"s"10.0 ==> (macrocall @x_str (string-r "s") 10.0) + suffix_kind = (k == K"Identifier" || is_keyword(k) || + is_word_operator(k)) ? K"String" : k + bump(ps, remap_kind=suffix_kind) + end + emit(ps, mark, K"macrocall") + else + break + end + maybe_strmac = maybe_strmac_1 + end +end + +# Parse the `A<:B` part of type definitions like `struct A<:B end` +# +# flisp: parse-subtype-spec +function parse_subtype_spec(ps::ParseState) + # Wart: why isn't the flisp parser more strict here? + # <: is the only operator which isn't a syntax error, but + # parse_comparison allows all sorts of things. + parse_comparison(ps, true) +end + +# flisp: parse-struct-field +function parse_struct_field(ps::ParseState) + mark = position(ps) + const_field = peek(ps) == K"const" + if const_field + bump(ps, TRIVIA_FLAG) + end + parse_eq(ps) + if const_field + # Const fields https://github.com/JuliaLang/julia/pull/43305 + #v1.8: struct A const a end ==> (struct A (block (const x))) + #v1.7: struct A const a end ==> (struct A (block (error (const x)))) + emit(ps, mark, K"const") + min_supported_version(v"1.8", ps, mark, "`const` struct field") + end +end + +# parse expressions or blocks introduced by syntactic reserved words. +# +# The caller should use peek_initial_reserved_words to determine whether +# to call parse_resword, or whether contextual keywords like `mutable` are +# simple identifiers. +# +# flisp: parse-resword +function parse_resword(ps::ParseState) + bump_trivia(ps, skip_newlines=true) + word = peek(ps) + v_1_14_break_cont_ret = ps.stream.version >= (1,14) + if v_1_14_break_cont_ret && (word == K"break" || word == K"continue" || word == K"return") + # "Non-delimited" reserved words don't have an `end` delimiter and + # should preserve end_symbol and range_colon_enabled. + ps = ParseState(ps; + space_sensitive=false, + where_enabled=true, + for_generator=false, + whitespace_newline=false) + else + ps = normal_context(ps) + end + + mark = position(ps) + if word in KSet"begin quote" + # begin end ==> (block) + # begin a ; b end ==> (block a b) + # begin\na\nb\nend ==> (block a b) + bump(ps, TRIVIA_FLAG) + parse_block_inner(ps, parse_docstring) + bump_closing_token(ps, K"end") + emit(ps, mark, K"block") + if word == K"quote" + # quote end ==> (quote (block)) + # quote body end ==> (quote (block body)) + emit(ps, mark, K"quote") + end + elseif word == K"while" + # while cond body end ==> (while cond (block body)) + # while x < y \n a \n b \n end ==> (while (call-i x < y) (block a b)) + bump(ps, TRIVIA_FLAG) + parse_cond(ps) + parse_block(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, K"while") + elseif word == K"for" + # for x in xs end ==> (for (iteration (in x xs)) (block)) + # for x in xs, y in ys \n a \n end ==> (for (iteration (in x xs) (in y ys)) (block a)) + bump(ps, TRIVIA_FLAG) + parse_iteration_specs(ps) + parse_block(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, K"for") + elseif word == K"let" + bump(ps, TRIVIA_FLAG) + m = position(ps) + if peek(ps) in KSet"NewlineWs ;" + # let end ==> (let (block) (block)) + # let ; end ==> (let (block) (block)) + # let ; body end ==> (let (block) (block body)) + else + # let x=1\n end ==> (let (block (= x 1)) (block)) + # let x=1 ; end ==> (let (block (= x 1)) (block)) + # let x::1 ; end ==> (let (block (::-i x 1)) (block)) + # let x ; end ==> (let (block x) (block)) + # let x=1,y=2 ; end ==> (let (block (= x 1) (= y 2) (block))) + # let x+=1 ; end ==> (let (block (op= x + 1)) (block)) + parse_comma_separated(ps, parse_eq_star) + end + emit(ps, m, K"block") + k = peek(ps) + if k in KSet"NewlineWs ;" + bump(ps, TRIVIA_FLAG) + elseif k == K"end" + # pass + else + recover(is_closer_or_newline, ps, TRIVIA_FLAG, + error="let variables should end in `;` or newline") + end + # let\na\nb\nend ==> (let (block) (block a b)) + parse_block(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, K"let") + elseif word == K"if" + parse_if_elseif(ps) + elseif word in KSet"global local" + # global x ==> (global x) + # local x ==> (local x) + bump(ps, TRIVIA_FLAG) + const_mark = nothing + if peek(ps) == K"const" + const_mark = position(ps) + bump(ps, TRIVIA_FLAG) + end + had_assignment = parse_global_local_const_vars(ps) + if !isnothing(const_mark) + # global const x = 1 ==> (global (const (= x 1))) + # local const x = 1 ==> (local (const (= x 1))) + emit(ps, const_mark, K"const") + if !had_assignment + # global const x ==> (global (error (const x))) + emit(ps, mark, K"error", error="expected assignment after `const`") + end + end + emit(ps, mark, word) + elseif word == K"const" + # const x = 1 ==> (const (= x 1)) + bump(ps, TRIVIA_FLAG) + scope_mark = nothing + scope_k = peek(ps) + if scope_k in KSet"local global" + scope_mark = position(ps) + bump(ps, TRIVIA_FLAG) + end + had_assignment = parse_global_local_const_vars(ps) + if !isnothing(scope_mark) + # const global x = 1 ==> (const (global (= x 1))) + # const local x = 1 ==> (const (local (= x 1))) + emit(ps, scope_mark, scope_k) + end + emit(ps, mark, K"const") + if !had_assignment + # const x .= 1 ==> (error (const (.= x 1))) + emit(ps, mark, K"error", error="expected assignment after `const`") + end + elseif word in KSet"function macro" + bump(ps, TRIVIA_FLAG) + bump_trivia(ps) + has_body = parse_function_signature(ps, word == K"function") + if has_body + # The function body + # function f() \n a \n b end ==> (function (call f) (block a b)) + # function f() end ==> (function (call f) (block)) + parse_block(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, word) + else + # Function/macro definition with no methods + # function f end ==> (function f) + # (function f \n end) ==> (parens (function f)) + # function f \n\n end ==> (function f) + # function $f end ==> (function ($ f)) + # macro f end ==> (macro f) + bump(ps, TRIVIA_FLAG, skip_newlines=true) + emit(ps, mark, word) + end + elseif word == K"abstract" + # Abstract type definitions + # abstract type A end ==> (abstract A) + # abstract type A ; end ==> (abstract A) + # abstract type \n\n A \n\n end ==> (abstract A) + # abstract type A <: B end ==> (abstract (<: A B)) + # abstract type A <: B{T,S} end ==> (abstract (<: A (curly B T S))) + # Oddities allowed by parser + # abstract type A < B end ==> (abstract (call-i A < B)) + bump(ps, TRIVIA_FLAG) + @check peek(ps) == K"type" + bump(ps, TRIVIA_FLAG) + parse_subtype_spec(ps) + bump_semicolon_trivia(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, K"abstract") + elseif word in KSet"struct mutable" + # struct A <: B \n a::X \n end ==> (struct (<: A B) (block (::-i a X))) + # struct A \n a \n b \n end ==> (struct A (block a b)) + #v1.7: struct A const a end ==> (struct A (block (error (const a)))) + #v1.8: struct A const a end ==> (struct A (block (const a))) + is_mut = word == K"mutable" + if is_mut + # mutable struct A end ==> (struct-mut A (block)) + bump(ps, TRIVIA_FLAG) + else + # struct A end ==> (struct A (block)) + end + @check peek(ps) == K"struct" + bump(ps, TRIVIA_FLAG) + parse_subtype_spec(ps) + parse_block(ps, ps1->parse_docstring(ps1, parse_struct_field)) + bump_closing_token(ps, K"end") + emit(ps, mark, K"struct", is_mut ? MUTABLE_FLAG : EMPTY_FLAGS) + elseif word == K"primitive" + # primitive type A 32 end ==> (primitive A 32) + # primitive type A 32 ; end ==> (primitive A 32) + # primitive type A $N end ==> (primitive A ($ N)) + # primitive type A <: B \n 8 \n end ==> (primitive (<: A B) 8) + bump(ps, TRIVIA_FLAG) + @check peek(ps) == K"type" + bump(ps, TRIVIA_FLAG) + let ps = with_space_sensitive(ps) + parse_subtype_spec(ps) + parse_cond(ps) + end + bump_semicolon_trivia(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, K"primitive") + elseif word == K"typegroup" + # Grouped type definitions (mutually recursive) + # typegroup struct A ... end end ==> (typegroup (block ...)) + bump(ps, TRIVIA_FLAG) + parse_block(ps, parse_docstring) + bump_closing_token(ps, K"end") + emit(ps, mark, K"typegroup") + min_supported_version(v"1.14", ps, mark, "typegroup") + elseif word == K"try" + parse_try(ps) + elseif word == K"return" + bump(ps, TRIVIA_FLAG) + k = peek(ps) + if v_1_14_break_cont_ret ? is_nd_resword_closer(ps, k) : + (k == K"NewlineWs" || is_closing_token(ps, k)) + # return\nx ==> (return) + # (return) ==> (parens (return)) + else + # return x ==> (return x) + # return x,y ==> (return (tuple x y)) + parse_eq(ps) + end + emit(ps, mark, K"return") + elseif word == K"break" || word == K"continue" + bump(ps, TRIVIA_FLAG) + k = peek(ps) + if v_1_14_break_cont_ret + if is_nd_resword_closer(ps, k) + # break ==> (break) + # continue ==> (continue) + emit(ps, mark, word) + else + # break label ==> (break label) [1.14+] + # continue label ==> (continue label) [1.14+] + bump_trivia(ps) + emark = position(ps) + parse_unary_prefix(ps) + b = peek_behind(ps) + if b.orig_kind == K"Identifier" || is_contextual_keyword(b.orig_kind) || + b.kind == K"$" || b.kind == K"var" + # Label ok. Check for `break label value` syntax + if word == K"break" + t2 = peek_token(ps) + if !is_nd_resword_closer(ps, kind(t2)) + # break label val ==> (break label val) [1.14+] + if !preceding_whitespace(t2) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="expected space after break label") + end + parse_eq(ps) + end + end + else + emit(ps, emark, K"error", TRIVIA_FLAG, + error="expected identifier for break label") + end + emit(ps, mark, word) + if !is_nd_resword_closer(ps, peek(ps)) + recover(is_nd_resword_closer, ps, TRIVIA_FLAG, + error="unexpected token after $(untokenize(word))") + end + end + else + if k in KSet"NewlineWs ; ) : EndMarker" || (k == K"end" && !ps.end_symbol) + # break ==> (break) + emit(ps, mark, word) + else + # break label ==> ERROR + recover(is_closer_or_newline, ps, TRIVIA_FLAG, + error="unexpected token after $(untokenize(word))") + emit(ps, mark, word) + if k == K"Identifier" || is_contextual_keyword(k) || k == K"$" || k == K"var" + min_supported_version(v"1.14", ps, mark, "labeled `break` and `continue`") + end + end + end + elseif word in KSet"module baremodule" + # module A end ==> (module A (block)) + # baremodule A end ==> (module-bare A (block)) + bump(ps, TRIVIA_FLAG) + if is_reserved_word(peek(ps)) + # module do \n end ==> (module (error do) (block)) + bump(ps, error="Invalid module name") + else + if ps.stream.version >= (1, 14) + # Encode the parser version that parsed this module - the runtime + # will use this to set the same parser version for runtime `include` + # etc into this module. + bump_invisible(ps, K"VERSION", + set_numeric_flags(ps.stream.version[2] * 10)) + end + # module $A end ==> (module ($ A) (block)) + parse_unary_prefix(ps) + end + # module A \n a \n b \n end ==> (module A (block a b)) + # module A \n "x"\na \n end ==> (module A (block (doc (string "x") a))) + parse_block(ps, parse_public) + bump_closing_token(ps, K"end") + emit(ps, mark, K"module", + word == K"baremodule" ? BARE_MODULE_FLAG : EMPTY_FLAGS) + elseif word in KSet"export public" + # export a ==> (export a) + # export @a ==> (export (macro_name a)) + # export a, \n @b ==> (export a (macro_name b)) + # export +, == ==> (export + ==) + # export \n a ==> (export a) + # export \$a, \$(a*b) ==> (export (\$ a) (\$ (parens (call-i a * b)))) + bump(ps, TRIVIA_FLAG) + parse_comma_separated(ps, x->parse_import_atsym(x, false)) + emit(ps, mark, word) + elseif word in KSet"import using" + parse_imports(ps) + elseif word == K"do" + bump(ps, TRIVIA_FLAG, error="invalid `do` syntax") + else + internal_error("unhandled reserved word ", string(word)) + end +end + +# Parse if-elseif-else-end expressions +# +# if a xx elseif b yy else zz end ==> (if a (block xx) (elseif b (block yy) (block zz))) +function parse_if_elseif(ps, is_elseif=false, is_elseif_whitespace_err=false) + mark = position(ps) + word = peek(ps) + if is_elseif_whitespace_err + # Only get here on recovery from error case - pretend we're parsing elseif. + word = K"elseif" + else + bump(ps, TRIVIA_FLAG) + end + if peek(ps) in KSet"NewlineWs end" + # if end ==> (if (error) (block)) + # if \n end ==> (if (error) (block)) + bump_trivia(ps, error="missing condition in `$(untokenize(word))`") + else + # if a end ==> (if a (block)) + # if a xx end ==> (if a (block xx)) + parse_cond(ps) + end + # if a \n\n xx \n\n end ==> (if a (block xx)) + parse_block(ps) + bump_trivia(ps) + k = peek(ps) + if k == K"elseif" + # if a xx elseif b yy end ==> (if a (block xx) (elseif b (block yy))) + parse_if_elseif(ps, true) + elseif k == K"else" + emark = position(ps) + bump(ps, TRIVIA_FLAG) + if peek(ps) == K"if" + # Recovery: User wrote `else if` by mistake ? + # if a xx else if b yy end ==> (if a (block xx) (error-t) (elseif b (block yy))) + bump(ps, TRIVIA_FLAG) + emit(ps, emark, K"error", TRIVIA_FLAG, + error="use `elseif` instead of `else if`") + parse_if_elseif(ps, true, true) + else + # if a xx else yy end ==> (if a (block xx) (block yy)) + parse_block(ps) + end + end + if !is_elseif + bump_closing_token(ps, K"end") + end + emit(ps, mark, word) +end + +# Like parse_assignment, but specialized so that we can omit the +# tuple when there's commas but no assignment. +function parse_global_local_const_vars(ps) + mark = position(ps) + n_commas = parse_comma(ps, false) + # `isassign` is true for a (possibly dotted) operator followed by `=`, ie a + # compound assignment like `x += 1` or `x .+= 1`. + (isdot, isassign, t) = peek_dotted_op_token(ps) + + if is_prec_assignment(t) || isassign + if n_commas >= 1 + # const x,y = 1,2 ==> (const (= (tuple x y) (tuple 1 2))) + emit(ps, mark, K"tuple") + end + # const x = 1 ==> (const (= x 1)) + # global x ~ 1 ==> (global (call-i x ~ 1)) + # global x += 1 ==> (global (op= x + 1)) + parse_assignment_with_initial_ex(ps, mark, parse_comma) + else + # global x,y ==> (global x y) + end + return kind(t) == K"=" && !isdot +end + +# Parse function and macro definitions +function parse_function_signature(ps::ParseState, is_function::Bool) + is_anon_func = false + needs_parse_call = true + + mark = position(ps) + if !is_function + # Parse macro name + parse_unary_prefix(ps) + kb = peek_behind(ps).orig_kind + if is_initial_reserved_word(ps, kb) + # macro while(ex) end ==> (macro (call (error while) ex) (block)) + emit(ps, mark, K"error", error="invalid macro name") + else + # macro f() end ==> (macro (call f) (block)) + # macro (:)(ex) end ==> (macro (call (parens :) ex) (block)) + # macro (type)(ex) end ==> (macro (call (parens type) ex) (block)) + # macro $f() end ==> (macro (call ($ f)) (block)) + # macro ($f)() end ==> (macro (call (parens ($ f))) (block)) + end + else + if peek(ps) != K"(" + # function f() end ==> (function (call f)) + parse_unary_prefix(ps) + else + # When an initial parenthesis is present, we need to distinguish + # between + # * The function name in parens, followed by (args...) + # * An anonymous function argument list in parens + # * The whole function declaration, in parens + bump(ps, TRIVIA_FLAG) + is_empty_tuple = peek(ps, skip_newlines=true) == K")" + opts = parse_brackets(ps, K")") do had_commas, had_splat, num_semis, num_subexprs + _parsed_call = was_eventually_call(ps) + _maybe_grouping_parens = !had_commas && !had_splat && num_semis == 0 && num_subexprs == 1 + # Check if there's a newline between `)` and the next `(` or `.`. + # We need to find where `)` is and check what immediately follows it. + # If peek(1, skip_newlines=false) is `)`, we're directly before it. + # Otherwise there's whitespace/newline before `)`. + next_token_pos = if peek(ps, 1, skip_newlines=false) == K")" + # Directly before ), token after ) is at 2 + 2 + else + # There's whitespace before ), so ) is at 2 + # and what follows ) is at 3 + 3 + end + token_after_paren = peek(ps, next_token_pos, skip_newlines=false) + # If token_after_paren is a newline, this is an anonymous function + has_newline_after_paren = _maybe_grouping_parens && token_after_paren == K"NewlineWs" + # Get the next significant token to determine if we need to parse a call + next_kind = peek(ps, 2, skip_newlines=_maybe_grouping_parens && !has_newline_after_paren) + _needs_parse_call = next_kind ∈ KSet"( ." + _is_anon_func = (!_needs_parse_call && !_parsed_call) || had_commas + return (needs_parameters = _is_anon_func, + is_anon_func = _is_anon_func, + parsed_call = _parsed_call, + needs_parse_call = _needs_parse_call, + maybe_grouping_parens = _maybe_grouping_parens) + end::NamedTuple{(:needs_parameters, :is_anon_func, :parsed_call, :needs_parse_call, :maybe_grouping_parens, :delim_flags), + Tuple{Bool, Bool, Bool, Bool, Bool, RawFlags}} + is_anon_func = opts.is_anon_func + needs_parse_call = opts.needs_parse_call + if is_anon_func + # function (x) body end ==> (function (tuple-p x) (block body)) + # function (x::f()) end ==> (function (tuple-p (::-i x (call f))) (block)) + # function (x,y) end ==> (function (tuple-p x y) (block)) + # function (x=1) end ==> (function (tuple-p (= x 1)) (block)) + # function (;x=1) end ==> (function (tuple-p (parameters (= x 1))) (block)) + # function (f(x),) end ==> (function (tuple-p (call f x)) (block)) + ambiguous_parens = opts.maybe_grouping_parens && + peek_behind(ps).kind in KSet"macrocall $" + emit(ps, mark, K"tuple", PARENS_FLAG|opts.delim_flags) + if ambiguous_parens + # Got something like `(@f(x))`. Is it anon `(@f(x),)` or named sig `@f(x)` ?? + emit(ps, mark, K"error", error="Ambiguous signature. Add a trailing comma if this is a 1-argument anonymous function; remove parentheses if this is a macro call acting as function signature.") + end + elseif is_empty_tuple + # Weird case which is consistent with parse_paren but will be + # rejected in lowering + # function ()(x) end ==> (function (call (tuple-p) x) (block)) + emit(ps, mark, K"tuple", PARENS_FLAG) + else + # function (A).f() end ==> (function (call (. (parens A) f)) (block)) + # function (:)() end ==> (function (call (parens :)) (block)) + # function (x::T)() end ==> (function (call (parens (::-i x T))) (block)) + # function (::T)() end ==> (function (call (parens (::-pre T))) (block)) + # function (:*=(f))() end ==> (function (call (parens (call (quote-: (op= *)) f))) (block)) + emit(ps, mark, K"parens", PARENS_FLAG) + end + end + if !is_anon_func + kb = peek_behind(ps).orig_kind + if is_reserved_word(kb) + # function begin() end ==> (function (call (error begin)) (block)) + emit(ps, mark, K"error", error="invalid function name") + else + # function f() end ==> (function (call f) (block)) + # function type() end ==> (function (call type) (block)) + # function \n f() end ==> (function (call f) (block)) + # function $f() end ==> (function (call ($ f)) (block)) + # function (::Type{T})(x) end ==> (function (call (parens (::-pre (curly Type T))) x) (block)) + end + end + end + if needs_parse_call + # Parse function argument list + # function f(x,y) end ==> (function (call f x y) (block)) + # function f{T}() end ==> (function (call (curly f T)) (block)) + # function A.f() end ==> (function (call (. A f)) (block)) + parse_call_chain(ps, mark) + sig_kind = peek_behind(ps).kind + if sig_kind in KSet"Identifier var $" && peek(ps, skip_newlines=true) == K"end" + # function f end ==> (function f) + # function $f end ==> (function $f) + return false + elseif sig_kind == K"macrocall" + min_supported_version(v"1.12", ps, mark, "macro call as function signature") + elseif sig_kind != K"call" + # function f body end ==> (function (error f) (block body)) + emit(ps, mark, K"error", + error="Invalid signature in $(is_function ? "function" : "macro") definition") + end + end + if is_function && peek(ps) == K"::" + # Function return type + # function f()::T end ==> (function (::-i (call f) T) (block)) + # function f()::g(T) end ==> (function (::-i (call f) (call g T)) (block)) + bump(ps, TRIVIA_FLAG) + parse_call(ps) + emit(ps, mark, K"::", INFIX_FLAG) + end + if peek(ps) == K"where" + # Function signature where syntax + # function f() where {T} end ==> (function (where (call f) (braces T)) (block)) + # function f() where T end ==> (function (where (call f) T) (block)) + parse_where_chain(ps, mark) + end + # function f()::S where T end ==> (function (where (::-i (call f) S) T) (block)) + # + # Ugly cases for compat where extra parentheses existed and we've + # already parsed at least the call part of the signature + # + # function (f() where T) end ==> (function (where (call f) T) (block)) + # function (f()) where T end ==> (function (where (call f) T) (block)) + # function (f() where T) where U end ==> (function (where (where (call f) T) U) (block)) + # function (f()::S) end ==> (function (parens (::-i (call f) S)) (block)) + # function ((f()::S) where T) end ==> (function (where (parens (::-i (call f) S)) T) (block)) + # + # TODO: Warn for use of parens? The precedence of `::` and + # `where` don't work inside parens so this is a bit of a syntax + # oddity/aberration. + return true +end + +# Parse a try block +# +# try \n x \n catch e \n y \n finally \n z end ==> (try (block x) (catch e (block y)) (finally (block z))) +#v1.8: try \n x \n catch e \n y \n else z finally \n w end ==> (try (block x) (catch e (block y)) (else (block z)) (finally (block w))) +# +# flisp: embedded in parse_resword +function parse_try(ps) + mark = position(ps) + bump(ps, TRIVIA_FLAG) + parse_block(ps) + has_catch = false + has_finally = false + bump_trivia(ps) + if peek(ps) == K"catch" + has_catch = true + parse_catch(ps) + end + bump_trivia(ps) + if peek(ps) == K"else" + # catch-else syntax: https://github.com/JuliaLang/julia/pull/42211 + # + #v1.8: try catch ; else end ==> (try (block) (catch □ (block)) (else (block))) + else_mark = position(ps) + bump(ps, TRIVIA_FLAG) + parse_block(ps) + if !has_catch + #v1.8: try else x finally y end ==> (try (block) (else (error (block x))) (finally (block y))) + emit(ps, else_mark, K"error", error="Expected `catch` before `else`") + end + #v1.7: try catch ; else end ==> (try (block) (catch □ (block)) (else (error (block)))) + min_supported_version(v"1.8", ps, else_mark, "`else` after `catch`") + emit(ps, else_mark, K"else") + end + bump_trivia(ps) + if peek(ps) == K"finally" + finally_mark = position(ps) + # try x finally y end ==> (try (block x) (finally (block y))) + has_finally = true + bump(ps, TRIVIA_FLAG) + parse_block(ps) + emit(ps, finally_mark, K"finally") + end + # Wart: the flisp parser allows finally before catch, the *opposite* order + # in which these blocks execute. + bump_trivia(ps) + if !has_catch && peek(ps) == K"catch" + # try x finally y catch e z end ==> (try (block x) (finally (block y)) (catch e (block z))) + m = position(ps) + parse_catch(ps) + emit_diagnostic(ps, m, + warning="`catch` after `finally` will execute out of order") + end + missing_recovery = !has_catch && !has_finally + if missing_recovery + # try x end ==> (try (block x) (error-t)) + bump_invisible(ps, K"error", TRIVIA_FLAG) + end + bump_closing_token(ps, K"end") + emit(ps, mark, K"try") + if missing_recovery + emit_diagnostic(ps, mark, error="try without catch or finally") + end +end + +function parse_catch(ps::ParseState) + mark = position(ps) + bump(ps, TRIVIA_FLAG) + k = peek(ps) + if k in KSet"NewlineWs ;" || is_closing_token(ps, k) + # try x catch end ==> (try (block x) (catch □ (block))) + # try x catch ; y end ==> (try (block x) (catch □ (block y))) + # try x catch \n y end ==> (try (block x) (catch □ (block y))) + bump_invisible(ps, K"Placeholder") + else + # try x catch e y end ==> (try (block x) (catch e (block y))) + # try x catch $e y end ==> (try (block x) (catch ($ e) (block y))) + m = position(ps) + parse_eq_star(ps) + if !(peek_behind(ps).kind in KSet"Identifier var $") + # try x catch e+3 y end ==> (try (block x) (catch (error (call-i e + 3)) (block y))) + emit(ps, m, K"error", error="a variable name is expected after `catch`") + end + end + parse_block(ps) + emit(ps, mark, K"catch") +end + +# flisp: parse-do +function parse_do(ps::ParseState) + mark = position(ps) + bump(ps, TRIVIA_FLAG) # do + ps = normal_context(ps) + m = position(ps) + if peek(ps) in KSet"NewlineWs ;" + # f() do\nend ==> (call f (do (tuple) (block))) + # f() do ; body end ==> (call f (do (tuple) (block body))) + # this trivia needs to go into the tuple due to the way position() + # works. + bump(ps, TRIVIA_FLAG) + else + # f() do x, y\n body end ==> (call f (do (tuple x y) (block body))) + parse_comma_separated(ps, parse_range) + end + emit(ps, m, K"tuple") + parse_block(ps) + bump_closing_token(ps, K"end") + emit(ps, mark, K"do") +end + +function _is_valid_macro_name(peektok) + return !is_error(peektok.kind) && (peektok.is_leaf || peektok.kind == K"var") +end + +# flisp: parse-macro-name +function parse_macro_name(ps::ParseState) + # @! x ==> (macrocall @! x) + # @.. x ==> (macrocall (macro_name ..) x) + # @$ x ==> (macrocall (macro_name $) x) + # @var"#" x ==> (macrocall (macro_name (var #)) x) + bump_disallowed_space(ps) + mark = position(ps) + parse_atom(ps, false) + b = peek_behind(ps, skip_parens=false) + if b.kind == K"parens" + emit_diagnostic(ps, mark, + warning="parenthesizing macro names is unnecessary") + elseif !_is_valid_macro_name(b) + # @[x] y z ==> (macrocall (macro_name (error (vect x))) y z) + emit(ps, mark, K"error", error="invalid macro name") + end +end + +# Parse an identifier, interpolation or @-prefixed symbol +# +# flisp: parse-atsym +function parse_import_atsym(ps::ParseState, allow_quotes=true) + bump_trivia(ps) + if peek(ps) == K"@" + mark = position(ps) + # export @a ==> (export (macro_name a)) + # export @var"'" ==> (export (macro_name (var '))) + # export a, \n @b ==> (export a (macro_name b)) + bump(ps, TRIVIA_FLAG) + parse_macro_name(ps) + emit(ps, mark, K"macro_name") + else + # export a ==> (export a) + # export \n a ==> (export a) + # export $a, $(a*b) ==> (export ($ a) (parens ($ (call * a b)))) + # export (x::T) ==> (export (error (parens (::-i x T)))) + # export outer ==> (export outer) + # export ($f) ==> (export ($ f)) + mark = position(ps) + # Syntax Edition TODO: make all the various ways to quote things inside + # import paths an error and require `var""` in the few remaining cases. + if allow_quotes && peek(ps) == K":" && !is_closing_token(ps, peek(ps,2)) + # import A.:+ ==> (import (importpath A (quote-: +))) + emit_diagnostic(ps, warning="quoting with `:` is not required here") + end + parse_unary_prefix(ps) + pos = position(ps) + warn_parens = false + if peek_behind(ps, pos).kind == K"parens" + # import A.(:+) ==> (import (importpath A (parens (quote-: +)))) + pos = first_child_position(ps, pos) + warn_parens = true + end + if allow_quotes && peek_behind(ps, pos).kind == K"quote" + pos = first_child_position(ps, pos) + if peek_behind(ps, pos).kind == K"parens" + # import A.:(+) ==> (import (importpath A (quote-: (parens +)))) + pos = first_child_position(ps, pos) + warn_parens = true + end + end + b = peek_behind(ps, pos) + if warn_parens && b.kind != K"DotsIdentifier" + emit_diagnostic(ps, mark, warning="parentheses are not required here") + end + ok = (b.is_leaf && (b.kind == K"Identifier" || is_operator(b.kind))) || + (!b.is_leaf && (b.kind in KSet"$ var" || b.kind == K"DotsIdentifier")) + if !ok + emit(ps, mark, K"error", error="expected identifier") + end + end +end + +# Parse import and using syntax +# +# flisp: parse-imports +function parse_imports(ps::ParseState) + mark = position(ps) + word = peek(ps) + @check word in KSet"import using" + bump(ps, TRIVIA_FLAG) + emark = position(ps) + initial_as = parse_import(ps, word, false) + t = peek_token(ps) + k = kind(t) + has_import_prefix = false # true if we have `prefix:` in `import prefix: stuff` + has_comma = false + if k == K":" && !preceding_whitespace(t) + bump(ps, TRIVIA_FLAG) + has_import_prefix = true + if initial_as + # import A as B: x ==> (import (: (error (as (importpath A) B)) (importpath x))) + emit(ps, emark, K"error", error="`as` before `:` in import/using") + end + elseif k == K"," + bump(ps, TRIVIA_FLAG) + has_comma = true + end + if has_import_prefix || has_comma + # import A, y ==> (import (importpath A) (importpath y)) + # import A: x, y ==> (import (: (importpath A) (importpath x) (importpath y))) + # import A: +, == ==> (import (: (importpath A) (importpath +) (importpath ==))) + has_import_prefix_ = has_import_prefix + parse_comma_separated(ps, ps1->parse_import(ps1, word, has_import_prefix_)) + if peek(ps) == K":" + # Error recovery + # import A: x, B: y ==> (import (: (importpath A) (importpath x) (importpath B) (error-t (importpath y)))) + emark = position(ps) + bump(ps, TRIVIA_FLAG) + parse_comma_separated(ps, ps1->parse_import(ps1, word, has_import_prefix_)) + emit(ps, emark, K"error", TRIVIA_FLAG, + error="`:` can only be used when importing a single module. Split imports into multiple lines") + end + end + if has_import_prefix + # import A: x ==> (import (: (importpath A) (importpath x))) + emit(ps, mark, K":") + end + # using A ==> (using (importpath A)) + # import A ==> (import (importpath A)) + emit(ps, mark, word) +end + +# Parse individual module path and renaming with `as` +# +# flisp: parse-import +function parse_import(ps::ParseState, word, has_import_prefix) + mark = position(ps) + parse_import_path(ps) + # import A: x, y ==> (import (: (importpath A) (importpath x) (importpath y))) + if peek(ps) == K"as" + # import A as B ==> (import (as (importpath A) B)) + # import A: x as y ==> (import (: (importpath A) (as (importpath x) y))) + # using A: x as y ==> (using (: (importpath A) (as (importpath x) y))) + bump(ps, TRIVIA_FLAG) + parse_import_atsym(ps, false) + emit(ps, mark, K"as") + if word == K"using" && !has_import_prefix + # using A as B ==> (using (error (as (importpath A) B))) + # using A, B as C ==> (using (importpath A) (error (as (importpath B) C))) + emit(ps, mark, K"error", + error="`using` with `as` renaming requires a `:` and context module") + end + #v1.5: import A as B ==> (import (error (as (importpath A) B))) + min_supported_version(v"1.6", ps, mark, "`import ... as`") + return true + else + return false + end +end + +# flisp: parse-import-path +function parse_import_path(ps::ParseState) + mark = position(ps) + bump_trivia(ps) + # The tokenizer produces conjoined dotted tokens .. and ... + # When parsing import we must split these into single dots + # import .A ==> (import (importpath . A)) + # import ..A ==> (import (importpath . . A)) + # import ...A ==> (import (importpath . . . A)) + # import ....A ==> (import (importpath . . . . A)) + # Dots with spaces are allowed (a misfeature?) + # import . .A ==> (import (importpath . . A)) + # Modules with operator symbol names + # import .⋆ ==> (import (importpath . ⋆)) + first_dot = true + while true + t = peek_token(ps) + k = kind(t) + if !first_dot && preceding_whitespace(t) + emit_diagnostic(ps, whitespace=true, + warning="space between dots in import path") + end + if k == K"." + bump(ps) + else + break + end + first_dot = false + end + # import @x ==> (import (importpath (macro_name x))) + # import $A ==> (import (importpath ($ A))) + parse_import_atsym(ps, false) + while true + t = peek_token(ps) + k = kind(t) + if k == K"." + # import A.B ==> (import (importpath A B)) + # import $A.@x ==> (import (importpath ($ A) (macro_name x))) + # import A.B.C ==> (import (importpath A B C)) + # import A.⋆.f ==> (import (importpath A ⋆ f)) + next_tok = peek_token(ps, 2) + if is_operator(kind(next_tok)) + if kind(next_tok) == K"." && peek(ps, 3) == K"." + # Import the .. operator + # import A... ==> (import (importpath A (DotsIdentifier-2))) + bump_disallowed_space(ps) + bump(ps, TRIVIA_FLAG) + dotmark = position(ps) + bump(ps, TRIVIA_FLAG) + bump(ps, TRIVIA_FLAG) + emit(ps, dotmark, K"DotsIdentifier", set_numeric_flags(2)) + continue + end + if preceding_whitespace(t) + # Whitespace in import path allowed but discouraged + # import A .== ==> (import (importpath A ==)) + emit_diagnostic(ps, whitespace=true, + warning="space between dots in import path") + end + bump_trivia(ps) + else + bump_disallowed_space(ps) + end + bump(ps, TRIVIA_FLAG) + parse_import_atsym(ps) + elseif k in KSet"NewlineWs ; , : EndMarker" + # import A; B ==> (import (importpath A)) + break + else + # Could we emit a more comprehensible error here? + break + end + end + emit(ps, mark, K"importpath") +end + +# parse comma-separated assignments, like "i=1:n,j=1:m,..." +# +# flisp: parse-comma-separated +function parse_comma_separated(ps::ParseState, down) + n_subexprs = 0 + while true + down(ps) + n_subexprs += 1 + if peek(ps) == K"," + bump(ps, TRIVIA_FLAG) + else + break + end + end + return n_subexprs +end + +# FIXME(sschaub): for backwards compatibility, allows newline before =/in/∈ +# in generator expressions. See issue #37393 +function peek_skip_newline_in_gen(ps::ParseState, n=1) + k = peek(ps, n) + if ps.for_generator && k == K"NewlineWs" + k = peek(ps, n+1) + end + return k +end + +# parse comma-separated "assignment" but allowing `in` and `∈` as assignment operators +# +# i = rhs ==> (= i rhs) +# i in rhs ==> (= i rhs) +# i ∈ rhs ==> (= i rhs) +# +# i = 1:10 ==> (= i (call : 1 10)) +# (i,j) in iter ==> (= (tuple-p i j) iter) +# +# flisp: parse-iteration-spec +function parse_iteration_spec(ps::ParseState) + mark = position(ps) + # Handle `outer` contextual keyword + parse_pipe_lt(with_space_sensitive(ps)) + if peek_behind(ps).orig_kind == K"outer" + if peek_skip_newline_in_gen(ps) in KSet"= in ∈" + # Not outer keyword + # outer = rhs ==> (iteration (in outer rhs)) + # outer <| x = rhs ==> (iteration (in (call-i outer <| x) rhs)) + else + # outer i = rhs ==> (iteration (in (outer i) rhs)) + # outer (x,y) = rhs ==> (iteration (in (outer (tuple-p x y)) rhs)) + reset_node!(ps, position(ps), kind=K"outer", flags=TRIVIA_FLAG) + parse_pipe_lt(ps) + emit(ps, mark, K"outer") + end + end + if peek_skip_newline_in_gen(ps) in KSet"= in ∈" + bump(ps, TRIVIA_FLAG) + parse_pipe_lt(ps) + else + # Recovery heuristic + recover(ps, error="invalid iteration spec: expected one of `=` `in` or `∈`") do ps, k + k in KSet", NewlineWs" || is_closing_token(ps, k) + end + # Or try parse_pipe_lt ??? + end + emit(ps, mark, K"in") +end + +# Parse an iteration spec, or a comma separate list of such for for loops and +# generators +function parse_iteration_specs(ps::ParseState) + mark = position(ps) + _n_iters = parse_comma_separated(ps, parse_iteration_spec) + emit(ps, mark, K"iteration") +end + +# flisp: parse-space-separated-exprs +function parse_space_separated_exprs(ps::ParseState, macro_eats_newlines::Bool=false) + ps = with_space_sensitive(ps) + n_sep = 0 + while true + k = peek(ps) + if is_closing_token(ps, k) || (ps.for_generator && k == K"for") + break + elseif k == K"NewlineWs" + k2 = peek(ps, skip_newlines=true) + if !macro_eats_newlines || is_closing_token(ps, k2) || k2 == K"for" || + (is_operator(k2) && k2 != K"'") + break + end + bump_trivia(ps) + continue + end + parse_eq(ps) + n_sep += 1 + end + return n_sep +end + +# like parse-arglist, but with `for` parsed as a generator +# +# flisp: parse-call-arglist +function parse_call_arglist(ps::ParseState, closer) + ps = ParseState(ps, for_generator=true) + + parse_brackets(ps, closer, false) do _, _, _, _ + return (needs_parameters=true,) + end::NamedTuple{(:needs_parameters, :delim_flags), Tuple{Bool, RawFlags}} +end + +# Parse the suffix of comma-separated array expressions such as +# [x, suffix]. Consumes `closer`, but does not emit the AST node for the +# surrounding brackets. +# +# flisp: parse-vect +function parse_vect(ps::ParseState, closer, prefix_trailing_comma) + # [x, y] ==> (vect x y) + # [x, y] ==> (vect x y) + # [x,y ; z] ==> (vect x y (parameters z)) + # [x=1, y=2] ==> (vect (= x 1) (= y 2)) + # [x=1, ; y=2] ==> (vect (= x 1) (parameters (= y 2))) + opts = parse_brackets(ps, closer) do _, _, _, num_subexprs + return (needs_parameters=true, + num_subexprs=num_subexprs) + end::NamedTuple{(:needs_parameters, :num_subexprs, :delim_flags), Tuple{Bool, Int, RawFlags}} + delim_flags = opts.delim_flags + if opts.num_subexprs == 0 && prefix_trailing_comma + delim_flags |= TRAILING_COMMA_FLAG + end + return (K"vect", delim_flags) +end + +# Parse generators +# +# We represent generators quite differently from `Expr`: +# * Iteration variables and their iterators are grouped within K"iteration" +# nodes, as in the short form of `for` loops. +# * The `generator` kind is used for both cartesian and flattened generators +# +# (x for a in as for b in bs) ==> (parens (generator x (iteration (in a as)) (iteration (in b bs)))) +# (x for a in as, b in bs) ==> (parens (generator x (iteration (in a as) (in b bs)))) +# (x for a in as, b in bs if z) ==> (parens (generator x (filter (iteration (in a as) (in b bs)) z))) +# +# flisp: parse-generator +function parse_generator(ps::ParseState, mark) + while (t = peek_token(ps); kind(t) == K"for") + if !preceding_whitespace(t) + # ((x)for x in xs) ==> (parens (generator (parens x) (error) (iteration (in x xs)))) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="Expected space before `for` in generator") + end + bump(ps, TRIVIA_FLAG) + iter_mark = position(ps) + parse_iteration_specs(ps) + if peek(ps) == K"if" + # (x for a in as if z) ==> (parens (generator x (filter (iteration (in a as)) z))) + bump(ps, TRIVIA_FLAG) + parse_cond(ps) + emit(ps, iter_mark, K"filter") + end + end + emit(ps, mark, K"generator") +end + +# flisp: parse-comprehension +function parse_comprehension(ps::ParseState, mark, closer) + # [x for a in as] ==> (comprehension (generator x (iteration (in a as)))) + ps = ParseState(ps, whitespace_newline=true, + space_sensitive=false, + end_symbol=false) + parse_generator(ps, mark) + bump_closing_token(ps, closer) + return (K"comprehension", EMPTY_FLAGS) +end + +# Parse array concatenation syntax with multiple semicolons +# +# Normal matrix construction syntax +# [x y ; z w] ==> (vcat (row x y) (row z w)) +# [x y ; z w ; a b] ==> (vcat (row x y) (row z w) (row a b)) +# [x ; y ; z] ==> (vcat x y z) +# [x;] ==> (vcat x) +# [x y] ==> (hcat x y) +# +# Mismatched rows +# [x y ; z] ==> (vcat (row x y) z) +# +# Single elements in rows +#v1.7: [x ; y ;; z ] ==> (ncat-2 (nrow-1 x y) z) +#v1.7: [x y ;;; z ] ==> (ncat-3 (row x y) z) +# +# Higher dimensional ncat +# Row major +#v1.7: [x y ; z w ;;; a b ; c d] ==> +# (ncat-3 (nrow-1 (row x y) (row z w)) (nrow-1 (row a b) (row c d))) +# Column major +#v1.7: [x ; y ;; z ; w ;;; a ; b ;; c ; d] ==> +# (ncat-3 (nrow-2 (nrow-1 x y) (nrow-1 z w)) (nrow-2 (nrow-1 a b) (nrow-1 c d))) +# +# flisp: parse-array +function parse_array(ps::ParseState, mark, closer, end_is_symbol) + ps = ParseState(ps, end_symbol=end_is_symbol) + + array_order = Ref(:unknown) + # Outer array parsing loop - parse chain of separators with descending + # precedence such as + #v1.7: [a ; b ;; c ;;; d ;;;; e] ==> (ncat-4 (ncat-3 (ncat-2 (ncat-1 a b) c) d) e) + # + # Ascending and equal precedence is handled by parse_array_inner. + # + # This is a variant of a Pratt parser, but we have a separate outer loop + # because there's no minimum precedence/binding power - you can always get + # a lower binding power by adding more semicolons. + # + # For an excellent overview of Pratt parsing, see + # https://matklad.github.io/2020/04/13/simple-but-powerful-pratt-parsing.html + (dim, binding_power) = parse_array_separator(ps, array_order) + if binding_power == typemin(Int) + # [x@y ==> (hcat x (error-t ✘ y)) + bump_closing_token(ps, closer) + return (K"hcat", 0) + end + while true + (next_dim, next_bp) = parse_array_inner(ps, binding_power, array_order) + if next_bp == typemin(Int) + break + end + if binding_power == 0 + emit(ps, mark, K"row") + else + emit(ps, mark, K"nrow", set_numeric_flags(dim)) + end + dim = next_dim + binding_power = next_bp + end + bump_closing_token(ps, closer) + return binding_power == -1 ? (K"vcat", 0) : + binding_power == 0 ? (K"hcat", 0) : + (K"ncat", dim) +end + +# Parse equal and ascending precedence chains of array concatenation operators - +# semicolons, newlines and whitespace. Invariants: +# +# * The caller must have already consumed +# - The left hand side +# - The concatenation operator, providing `binding_power`. +# So eg, we're here in the input stream, either at an element or closing token +# | +# [a ;; b ; c ] +# [a ;; ] +# +# * The caller must call emit() to delimit the AST node for this binding power. +# +function parse_array_inner(ps, binding_power, array_order) + mark = NO_POSITION + dim = -1 + bp = binding_power + while true + if bp < binding_power + return (dim, bp) + end + # Allow trailing separators + # [a ;] ==> (vcat a) + # [a ; b;;] ==> (ncat-2 (nrow-1 a b)) + if is_closing_token(ps, peek(ps)) + return (typemin(Int), typemin(Int)) + end + if bp == binding_power + # Parse one expression + mark = position(ps) + parse_eq_star(ps) + (next_dim, next_bp) = parse_array_separator(ps, array_order) + else # bp > binding_power + # Recurse to parse a separator with greater binding power. Eg: + # [a ;; b ; c ] + # | ^------ the next input is here + # '---------- the mark is here + (next_dim, next_bp) = parse_array_inner(ps, bp, array_order) + if bp == 0 + emit(ps, mark, K"row") + else + emit(ps, mark, K"nrow", set_numeric_flags(dim)) + end + end + dim, bp = next_dim, next_bp + end +end + +# Parse a separator in an array concatenation +# +# Here we return a tuple (dim, binding_power) containing +# * Dimension on which the next separator acts +# * Binding power (precedence) of the separator, where whitespace binds +# tightest: ... < `;;;` < `;;` < `;`,`\n` < whitespace. We choose binding +# power of 0 for whitespace and negative numbers for other separators. +# +function parse_array_separator(ps, array_order) + sep_mismatch_err = "cannot mix space and ;; separators in an array expression, except to wrap a line" + mark = position(ps) + t = peek_token(ps, skip_newlines=true) + if kind(t) == K";" + # Newlines before semicolons are not significant + # [a \n ;] ==> (vcat a) + bump_trivia(ps) + n_semis = 1 + while true + bump(ps, TRIVIA_FLAG) + t = peek_token(ps) + if kind(t) != K";" + break + end + if preceding_whitespace(t) + bump_disallowed_space(ps) + end + n_semis += 1 + end + had_newline = peek(ps) == K"NewlineWs" + # Newlines after semicolons are not significant + # [a ; \n] ==> (vcat a) + # [a ; \n\n b] ==> (vcat a b) + #v1.7: [a ;; \n b] ==> (ncat-2 a b) + bump_trivia(ps) + if n_semis == 2 + if array_order[] === :row_major + if had_newline + # In hcat with spaces as separators, `;;` is a line + # continuation character + #v1.7: [a b ;; \n c] ==> (hcat a b c) + #v1.7: [a b \n ;; c] ==> (ncat-2 (row a b (error-t)) c) + return (2, 0) + else + # Can't mix spaces and multiple ;; + #v1.7: [a b ;; c] ==> (ncat-2 (row a b (error-t)) c) + emit(ps, mark, K"error", TRIVIA_FLAG, error=sep_mismatch_err) + end + else + array_order[] = :column_major + end + end + return (n_semis, -n_semis) + end + t = peek_token(ps) + k = kind(t) + if k == K"NewlineWs" + bump_trivia(ps) + if peek(ps) == K"]" + # Linebreaks not significant before closing `]` + # [a b\n\n] ==> (hcat a b) + return (typemin(Int), typemin(Int)) + else + # Treat a linebreak prior to a value as a semicolon (ie, separator + # for the first dimension) if no previous semicolons observed + # [a \n b] ==> (vcat a b) + return (1, -1) + end + elseif k == K"," + # Treat `,` as semicolon for the purposes of recovery + # [a; b, c] ==> (vcat a b (error-t) c) + bump(ps, TRIVIA_FLAG, error="unexpected comma in array expression") + return (1, -1) + else + if preceding_whitespace(t) && !is_closing_token(ps, k) + if array_order[] === :column_major + # Can't mix multiple ;'s and spaces + #v1.7: [a ;; b c] ==> (ncat-2 a (row b (error-t) c)) + bump_trivia(ps, TRIVIA_FLAG, error=sep_mismatch_err) + else + array_order[] = :row_major + end + return (2, 0) + else + # Something else; use typemin to exit array parsing + return (typemin(Int), typemin(Int)) + end + end +end + +# Parse array concatenation/construction/indexing syntax inside of `[]` or `{}`. +# The opening bracket has been consumed. +# +# flisp: parse-cat +function parse_cat(ps::ParseState, closer, end_is_symbol) + ps = ParseState(ps, range_colon_enabled=true, + space_sensitive=true, + where_enabled=true, + whitespace_newline=false, + for_generator=true) + k = peek(ps, skip_newlines=true) + mark = position(ps) + if k == closer + # [] ==> (vect) + ckind, cflags = parse_vect(ps, closer, false) + return (ckind, cflags, 0) + elseif k == K";" + #v1.8: [;] ==> (ncat-1) + #v1.8: [;;] ==> (ncat-2) + #v1.8: [\n ;; \n ] ==> (ncat-2) + #v1.7: [;;] ==> (ncat-2 (error)) + bump_trivia(ps) + dim, _ = parse_array_separator(ps, Ref(:unknown)) + min_supported_version(v"1.8", ps, mark, "empty multidimensional array syntax") + bump_closing_token(ps, closer) + return (K"ncat", EMPTY_FLAGS, dim) + end + parse_eq_star(ps) + k = peek(ps, skip_newlines=true) + if k == K"," || (is_closing_token(ps, k) && k != K";") + prefix_trailing_comma = k == K"," + if prefix_trailing_comma + # [x,] ==> (vect x) + bump(ps, TRIVIA_FLAG; skip_newlines = true) + end + # [x] ==> (vect x) + # [x \n ] ==> (vect x) + # [x ==> (vect x (error-t)) + ckind, cflags = parse_vect(ps, closer, prefix_trailing_comma) + return (ckind, cflags, 0) + elseif k == K"for" + # [x for a in as] ==> (comprehension (generator x (iteration (in a as)))) + # [x \n\n for a in as] ==> (comprehension (generator x (iteration (in a as)))) + ckind, cflags = parse_comprehension(ps, mark, closer) + return (ckind, cflags, 0) + else + # [x y] ==> (hcat x y) + # and other forms; See parse_array. + ckind, dim = parse_array(ps, mark, closer, end_is_symbol) + return (ckind, EMPTY_FLAGS, dim) + end +end + +function check_ncat_compat(ps, mark, k) + # https://github.com/JuliaLang/julia/pull/33697 + if k == K"ncat" + min_supported_version(v"1.7", ps, mark, "multidimensional array syntax") + end +end + +# Parse un-prefixed parenthesized syntax. This is hard because parentheses are +# *very* overloaded! +# +# flisp: parse-paren / parse-paren- +function parse_paren(ps::ParseState, check_identifiers=true, has_unary_prefix=false) + ps = ParseState(ps, range_colon_enabled=true, + space_sensitive=false, + where_enabled=true, + whitespace_newline=true) + mark = position(ps) + @check peek(ps) == K"(" + bump(ps, TRIVIA_FLAG) # K"(" + (_isdot, isassign, tok) = peek_dotted_op_token(ps) + k = kind(tok) + if k == K")" + # () ==> (tuple-p) + bump(ps, TRIVIA_FLAG) + emit(ps, mark, K"tuple", PARENS_FLAG) + elseif is_syntactic_operator(k) || isassign + # allow :(=) etc in unchecked contexts, eg quotes + # :(=) ==> (quote-: (parens =)) + parse_atom(ps, check_identifiers) + bump_closing_token(ps, K")") + emit(ps, mark, K"parens") + elseif !check_identifiers && k == K"::" && + peek(ps, 2, skip_newlines=true) == K")" + # allow :(::) as a special case + # :(::) ==> (quote-: (parens ::)) + bump(ps) + bump(ps, TRIVIA_FLAG, skip_newlines=true) + emit(ps, mark, K"parens") + else + # Deal with all other cases of tuple or block syntax via the generic + # parse_brackets + initial_semi = peek(ps) == K";" + opts = parse_brackets(ps, K")", bare_parens=true) do had_commas, had_splat, num_semis, num_subexprs + is_tuple = had_commas || (had_splat && num_semis >= 1) || + (initial_semi && (num_semis == 1 || num_subexprs > 0)) || + (peek(ps, 2) == K"->" && (peek_behind(ps).kind != K"where" && !has_unary_prefix)) + return (needs_parameters=is_tuple, + is_tuple=is_tuple, + is_block=num_semis > 0) + end::NamedTuple{(:needs_parameters, :is_tuple, :is_block, :delim_flags), Tuple{Bool, Bool, Bool, RawFlags}} + if opts.is_tuple + # Tuple syntax with commas + # (x,) ==> (tuple-p x) + # (x,y) ==> (tuple-p x y) + # (x=1, y=2) ==> (tuple-p (= x 1) (= y 2)) + # + # Named tuple with initial semicolon + # (;) ==> (tuple-p (parameters)) + # (; a=1) ==> (tuple-p (parameters (= a 1))) + # + # Extra credit: nested parameters and frankentuples + # (x...;) ==> (tuple-p (... x) (parameters)) + # (x...; y) ==> (tuple-p (... x) (parameters y)) + # (; a=1; b=2) ==> (tuple-p (parameters (= a 1)) (parameters (= b 2))) + # (a; b; c,d) ==> (tuple-p a (parameters b) (parameters c d)) + # (a=1, b=2; c=3) ==> (tuple-p (= a 1) (= b 2) (parameters (= c 3))) + emit(ps, mark, K"tuple", PARENS_FLAG|opts.delim_flags) + elseif opts.is_block + # Blocks + # (;;) ==> (block-p) + # (a=1;) ==> (block-p (= a 1)) + # (a;b;;c) ==> (block-p a b c) + # (a=1; b=2) ==> (block-p (= a 1) (= b 2)) + emit(ps, mark, K"block", PARENS_FLAG) + else + # Parentheses used for grouping + # (a * b) ==> (parens (call-i * a b)) + # (a=1) ==> (parens (= a 1)) + # (x) ==> (parens x) + # (a...) ==> (parens (... a)) + emit(ps, mark, K"parens") + end + end +end + +# Handle bracketed syntax inside any of () [] or {} where there's a mixture +# of commas and semicolon delimiters. +# +# For parentheses this is tricky because there's various cases to disambiguate, +# depending on outside context and the content of the brackets (number of +# semicolons, presence of commas or splats). The `after_parse` function must be +# provided by the caller to disambiguate these cases. +# +# Expressions (X; Y; Z) with more semicolons are also allowed by the flisp +# parser and generally parse as nested parameters blocks. This is invalid Julia +# syntax so the parse tree is pretty strange in these cases! Some macros +# probably use it though. Example: +# +# (a,b=1; c,d=2; e,f=3) ==> (tuple-p a (= b 1) (parameters c (= d 2)) (parameters e (= f 3))) +# +# flisp: parts of parse-paren- and parse-arglist +function parse_brackets(after_parse::F, + ps::ParseState, closing_kind, generator_is_last=true; + bare_parens::Bool=false) where {F} + ps = ParseState(ps, range_colon_enabled=true, + space_sensitive=false, + where_enabled=true, + whitespace_newline=true) + params_positions = acquire_positions(ps.stream) + num_subexprs = 0 + num_semis = 0 + had_commas = false + had_splat = false + param_start = nothing + trailing_comma = false + while true + k = peek(ps) + if k == closing_kind + break + elseif k == K";" + # Start of parameters list + # a, b; c d ==> a b (parameters c d) + if !isnothing(param_start) + push!(params_positions, emit(ps, param_start, K"TOMBSTONE")) + end + num_semis += 1 + param_start = position(ps) + bump(ps, TRIVIA_FLAG) + bump_trivia(ps) + elseif is_closing_token(ps, k) + trailing_comma = false + # Error; handled below in bump_closing_token + break + else + mark = position(ps) + parse_eq_star(!(bare_parens && num_subexprs == 0 && num_semis == 0 && ps.stream.version >= (1, 14)) ? ps : + ParseState(ps, paren_content_byte_index=first(byte_range(peek_full_token(ps))))) + trailing_comma = false + num_subexprs += 1 + if num_subexprs == 1 + had_splat = peek_behind(ps).kind == K"..." + end + k = peek(ps, skip_newlines=true) + if k == K"for" + # Generator syntax + # (x for a in as) ==> (parens (generator x (iteration (in a as)))) + parse_generator(ps, mark) + if generator_is_last + break + end + k = peek(ps, skip_newlines=true) + end + if k == K"," + had_commas = true + bump(ps, TRIVIA_FLAG) + trailing_comma = true + elseif k == K";" || k == closing_kind + # Handled above + continue + else + # Error - recovery done when consuming closing_kind + break + end + end + end + if !isnothing(param_start) && position(ps) != param_start + push!(params_positions, emit(ps, param_start, K"TOMBSTONE")) + end + opts = after_parse(had_commas, had_splat, num_semis, num_subexprs) + # Emit nested parameter nodes if necessary + if opts.needs_parameters + for pos in params_positions + reset_node!(ps, pos, kind=K"parameters") + end + end + release_positions(ps.stream, params_positions) + bump_closing_token(ps, closing_kind, " or `,`") + return (; opts..., delim_flags=trailing_comma ? TRAILING_COMMA_FLAG : EMPTY_FLAGS) +end + +_is_indentation(b::UInt8) = (b == u8" " || b == u8"\t") + +# Parse a string, embedded interpolations and deindent triple quoted strings +# by marking indentation characters as whitespace trivia. +# +# flisp: parse-string-literal-, parse-interpolate +function parse_string(ps::ParseState, raw::Bool) + mark = position(ps) + delim_k = peek(ps) + triplestr = delim_k in KSet"\"\"\" ```" + string_chunk_kind = delim_k in KSet"\" \"\"\"" ? K"String" : K"CmdString" + indent_ref_i = 0 + indent_ref_len = typemax(Int) + indent_chunks = acquire_positions(ps.stream) + txtbuf = unsafe_textbuf(ps) + chunk_flags = raw ? RAW_STRING_FLAG : EMPTY_FLAGS + bump(ps, TRIVIA_FLAG) + first_chunk = true + n_nontrivia_chunks = 0 + prev_chunk_newline = false + while true + t = peek_full_token(ps) + k = kind(t) + if k == K"$" + if raw + # FIXME: This case is actually a tokenization error: + # The `K"$"` token should not occur when a raw string + # is being parsed, but this would require the lexer to know + # about the parse state. (see also parse_atom) + break + end + if prev_chunk_newline + # """\n$x\n a""" ==> (string-s x "\n" " a") + indent_ref_i = first_byte(t) + indent_ref_len = 0 + end + bump(ps, TRIVIA_FLAG) + k = peek(ps) + if k == K"(" + # "a $(x + y) b" ==> (string "a " (parens (call-i x + y)) " b") + # "hi$("ho")" ==> (string "hi" (parens (string "ho"))) + m = position(ps) + bump(ps, TRIVIA_FLAG) + opts = parse_brackets(ps, K")") do had_commas, _had_splat, num_semis, num_subexprs + return (needs_parameters=false, + simple_interp=!had_commas && num_semis == 0 && num_subexprs == 1) + end::NamedTuple{(:needs_parameters, :simple_interp, :delim_flags), Tuple{Bool, Bool, RawFlags}} + if !opts.simple_interp || peek_behind(ps, skip_parens=false).kind == K"generator" + # "$(x,y)" ==> (string (parens (error x y))) + emit(ps, m, K"error", error="invalid interpolation syntax") + end + emit(ps, m, K"parens") + elseif k == K"var" + # var identifiers disabled in strings + # "$var" ==> (string var) + bump(ps, remap_kind=K"Identifier") + elseif k == K"Identifier" || is_keyword(k) || is_word_operator(k) + # "a $foo b" ==> (string "a " foo " b") + # "$outer" ==> (string outer) + # "$in" ==> (string in) + parse_atom(ps) + else + bump_invisible(ps, K"error", + error="identifier or parenthesized expression expected after \$ in string") + end + first_chunk = false + n_nontrivia_chunks += 1 + prev_chunk_newline = false + elseif k == string_chunk_kind + if triplestr && first_chunk && span(t) <= 2 && + begin + s = span(t) + b = txtbuf[last_byte(t)] + # Test whether the string is a single logical newline + (s == 1 && (b == u8"\n" || b == u8"\r")) || + (s == 2 && (txtbuf[first_byte(t)] == u8"\r" && b == u8"\n")) + end + # First line of triple string is a newline only: mark as trivia. + # """\nx""" ==> (string-s "x") + # """\n\nx""" ==> (string-s "\n" "x") + bump(ps, TRIVIA_FLAG) + first_chunk = false + prev_chunk_newline = true + else + if triplestr + # Triple-quoted dedenting: + # Various newlines (\n \r \r\n) and whitespace (' ' \t) + # """\n x\n y""" ==> (string-s "x\n" "y") + # ```\n x\n y``` ==> (macrocall :(Core.var"@cmd") (cmdstring-s-r "x\n" "y")) + # """\r x\r y""" ==> (string-s "x\n" "y") + # """\r\n x\r\n y""" ==> (string-s "x\n" "y") + # Spaces or tabs or mixtures acceptable + # """\n\tx\n\ty""" ==> (string-s "x\n" "y") + # """\n \tx\n \ty""" ==> (string-s "x\n" "y") + # + # Mismatched tab vs space not deindented + # Find minimum common prefix in mismatched whitespace + # """\n\tx\n y""" ==> (string-s "\tx\n" " y") + # """\n x\n y""" ==> (string-s "x\n" " y") + # """\n x\n y""" ==> (string-s " x\n" "y") + # """\n \tx\n y""" ==> (string-s "\tx\n" " y") + # """\n x\n \ty""" ==> (string-s " x\n" "\ty") + # + # Empty lines don't affect dedenting + # """\n x\n\n y""" ==> (string-s "x\n" "\n" "y") + # Non-empty first line doesn't participate in deindentation + # """ x\n y""" ==> (string-s " x\n" "y") + # + # Dedenting and interpolations + # """\n $a\n $b""" ==> (string-s a "\n" b) + # """\n $a \n $b""" ==> (string-s a " \n" b) + # """\n $a\n $b\n""" ==> (string-s " " a "\n" " " b "\n") + # + if prev_chunk_newline && (b = txtbuf[first_byte(t)]; + b != u8"\n" && b != u8"\r") + # Compute length of longest common prefix of mixed + # spaces and tabs, in bytes + # + # Initial whitespace is never regarded as indentation + # in any triple quoted string chunk, as it's always + # preceded in the source code by a visible token of + # some kind; either a """ delimiter or $() + # interpolation. + if indent_ref_i == 0 + # No indentation found yet. Find indentation we'll + # use as a reference + i = first_byte(t) - 1 + while i < last_byte(t) && _is_indentation(txtbuf[i+1]) + i += 1 + end + indent_ref_i = first_byte(t) + indent_ref_len = i - first_byte(t) + 1 + else + # Matching the current indentation with reference, + # shortening length if necessary. + j = 0 + while j < span(t) && j < indent_ref_len + if txtbuf[j + first_byte(t)] != txtbuf[j + indent_ref_i] + break + end + j += 1 + end + indent_ref_len = min(indent_ref_len, j) + end + # Prepare a place for indentiation trivia, if necessary + push!(indent_chunks, bump_invisible(ps, K"TOMBSTONE")) + end + b = txtbuf[last_byte(t)] + prev_chunk_newline = b == UInt8('\n') || b == UInt8('\r') + end + bump(ps, chunk_flags) + first_chunk = false + n_nontrivia_chunks += 1 + end + elseif k == K"ErrorInvalidInterpolationTerminator" || + k == K"ErrorBidiFormatting" || + k == K"ErrorInvalidUTF8" + # Treat these errors as string chunks + bump(ps) + n_nontrivia_chunks += 1 + else + break + end + end + had_end_delim = peek(ps) == delim_k + if triplestr && prev_chunk_newline && had_end_delim + # Newline at end of string + # """\n x\n y\n""" ==> (string-s " x\n" " y\n") + indent_ref_len = 0 + end + if triplestr && indent_ref_len > 0 + for pos in indent_chunks + reset_node!(ps, pos, kind=K"Whitespace", flags=TRIVIA_FLAG) + rhs_empty = steal_token_bytes!(ps, pos, indent_ref_len) + if rhs_empty + # Empty chunks after dedent are removed + # """\n \n """ ==> (string-s "\n") + n_nontrivia_chunks -= 1 + end + end + end + release_positions(ps.stream, indent_chunks) + if had_end_delim + if n_nontrivia_chunks == 0 + # Empty strings, or empty after triple quoted processing + # "" ==> (string "") + # """\n """ ==> (string-s "") + bump_invisible(ps, string_chunk_kind, chunk_flags) + end + bump(ps, TRIVIA_FLAG) + else + # Missing delimiter recovery + # "str ==> (string "str" (error-t)) + bump_invisible(ps, K"error", TRIVIA_FLAG, error="unterminated string literal") + end + # String interpolations + # "$x$y$z" ==> (string x y z) + # "$(x)" ==> (string (parens x)) + # "$x" ==> (string x) + # """$x""" ==> (string-s x) + # + # Strings with embedded whitespace trivia + # "a\\\nb" ==> (string "a" "b") + # "a\\\rb" ==> (string "a" "b") + # "a\\\r\nb" ==> (string "a" "b") + # "a\\\n \tb" ==> (string "a" "b") + # + # Strings with only a single valid string chunk + # "str" ==> (string "str") + # "a\\\n" ==> (string "a") + # "a\\\r" ==> (string "a") + # "a\\\r\n" ==> (string "a") + string_kind = delim_k in KSet"\" \"\"\"" ? K"string" : K"cmdstring" + str_flags = (triplestr ? TRIPLE_STRING_FLAG : EMPTY_FLAGS) | + (raw ? RAW_STRING_FLAG : EMPTY_FLAGS) + emit(ps, mark, string_kind, str_flags) +end + +function emit_braces(ps, mark, ckind, cflags, dim=0) + if ckind == K"hcat" + # {x y} ==> (bracescat (row x y)) + emit(ps, mark, K"row", cflags & ~TRAILING_COMMA_FLAG) + elseif ckind == K"ncat" + # {x ;;; y} ==> (bracescat (nrow-3 x y)) + emit(ps, mark, K"nrow", set_numeric_flags(dim)) + end + check_ncat_compat(ps, mark, ckind) + outk = ckind in KSet"vect comprehension" ? K"braces" : K"bracescat" + delim_flags = outk == K"braces" ? (cflags & TRAILING_COMMA_FLAG) : EMPTY_FLAGS + emit(ps, mark, outk, delim_flags) +end + +# parse numbers, identifiers, parenthesized expressions, lists, vectors, etc. +# +# If `check_identifiers` is true, identifiers are disallowed from being one of +# the syntactic operators or closing tokens. +# +# flisp: parse-atom +function parse_atom(ps::ParseState, check_identifiers=true, has_unary_prefix=false) + bump_trivia(ps) + mark = position(ps) + (leading_dot, leading_isassign, leading_tok) = peek_dotted_op_token(ps) + leading_kind = kind(leading_tok) + # todo: Reorder to put most likely tokens first? + if leading_dot + is_operator(leading_kind) && @goto is_operator + bump(ps, remap_kind=K"Identifier") + if check_identifiers + # . ==> (error .) + emit(ps, mark, K"error", error="invalid identifier") + end + elseif kind(leading_tok) == K"." && peek(ps, 2) == K"." && peek(ps, 3) == K"." + # ... + bump(ps, TRIVIA_FLAG) + bump(ps, TRIVIA_FLAG) + bump(ps, TRIVIA_FLAG) + emit(ps, mark, K"DotsIdentifier", set_numeric_flags(3)) + if check_identifiers + # ... ==> (error ...) + emit(ps, mark, K"error", error="invalid identifier") + end + elseif is_error(leading_kind) + # Errors for bad tokens are emitted in validate_tokens() rather than + # here. + bump(ps) + elseif leading_kind == K"'" + # char literal + bump(ps, TRIVIA_FLAG) + k = peek(ps) + if k == K"'" + # '' ==> (char (error)) + bump_invisible(ps, K"error", error="empty character literal") + bump(ps, TRIVIA_FLAG) + elseif k == K"EndMarker" + # ' ==> (char (error)) + bump_invisible(ps, K"error", error="unterminated character literal") + else + if k == K"Char" + bump(ps) + elseif is_error(k) + bump(ps) + else + # FIXME: This case is actually a tokenization error. + # Make a best-effort attempt to workaround this for now by + # remapping the kind. This needs to be fixed by rewinding the + # tokenizer's buffer and re-tokenizing the next token as a + # char. (A lot of work for a very obscure edge case) + # + # x in'c' ==> (call-i x in (char 'c')) + bump(ps, remap_kind=K"Char") + end + if peek(ps) == K"'" + # 'a' ==> (char 'a') + # 'α' ==> (char 'α') + # '\xce\xb1' ==> (char 'α') + bump(ps, TRIVIA_FLAG) + else + # 'a ==> (char 'a' (error-t)) + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="unterminated character literal") + end + end + emit(ps, mark, K"char") + elseif leading_kind == K"Char" + # FIXME: This is a tokenization error and should be preceded with + # K"'". However this workaround is better than emitting a bare Char. + bump(ps, remap_kind=K"Identifier") + elseif leading_kind == K":" + # symbol/expression quote + # :foo ==> (quote-: foo) + t = peek_token(ps, 2) + k = kind(t) + if is_closing_token(ps, k) && (!is_keyword(k) || preceding_whitespace(t)) + # : is a literal colon in some circumstances + # :) ==> : + # : end ==> : + bump(ps) # K":" + return + end + bump(ps, TRIVIA_FLAG) # K":" + if preceding_whitespace(t) + # : foo ==> (quote-: (error-t) foo) + # :\nfoo ==> (quote-: (error-t) foo) + bump_trivia(ps, TRIVIA_FLAG, + error="whitespace not allowed after `:` used for quoting") + end + # Being inside quote makes keywords into identifiers at the + # first level of nesting + # :end ==> (quote-: end) + # :(end) ==> (quote-: (parens (error-t))) + # Being inside quote makes end non-special again (issue #27690) + # a[:(end)] ==> (ref a (quote-: (error-t end))) + parse_atom(ParseState(ps, end_symbol=false), false) + emit(ps, mark, K"quote", COLON_QUOTE) + elseif check_identifiers && leading_kind == K"=" && kind(peek_token(ps)) == K"=" && !leading_dot + # = ==> (error =) + bump(ps, error="unexpected `=`") + elseif leading_kind == K"Identifier" + # xx ==> xx + # x₁ ==> x₁ + bump(ps) + elseif is_word_operator(leading_kind) + # where=1 ==> (= where 1) + bump(ps, remap_kind=K"Identifier") + elseif is_operator(leading_kind) +@label is_operator + # + ==> + + # .+ ==> (. +) + is_compound_assignment = !is_prec_assignment(leading_tok) && leading_isassign + bump_dotted(ps, leading_dot, leading_tok, emit_dot_node=!is_compound_assignment, remap_kind= + is_syntactic_operator(leading_kind) ? leading_kind : K"Identifier") + + if is_compound_assignment + bump(ps, TRIVIA_FLAG) # consume the = but mark as trivia + emit(ps, mark, leading_dot ? K".op=" : K"op=") + if check_identifiers + # += ==> (error (op= +)) + # .+= ==> (error (.op= +)) + emit(ps, mark, K"error", error="invalid identifier") + end + # Quoted syntactic operators are allowed + # :+= ==> (quote-: (op= +)) + return + end + + if check_identifiers && !(is_valid_identifier(leading_kind) || (leading_dot && leading_kind == K".")) + # ? ==> (error ?) + emit(ps, mark, K"error", error="invalid identifier") + end + elseif is_keyword(leading_kind) + if leading_kind == K"var" && (t = peek_token(ps,2); + kind(t) == K"\"" && !preceding_whitespace(t)) + # var"x" ==> (var x) + # Raw mode unescaping + # var"" ==> (var ) + # var"\"" ==> (var ") + # var"\\"" ==> (var \") + # var"\\x" ==> (var \\x) + # + # NB: Triple quoted var identifiers are not implemented, but with + # the complex deindentation rules they seem like a misfeature + # anyway, maybe? + # var"""x""" !=> x + bump(ps, TRIVIA_FLAG) + bump(ps, TRIVIA_FLAG) + if peek(ps) == K"String" + bump(ps, RAW_STRING_FLAG; remap_kind=K"Identifier") + else + bump_invisible(ps, K"Identifier", RAW_STRING_FLAG) + end + if peek(ps) == K"\"" + bump(ps, TRIVIA_FLAG) + else + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="unterminated `var\"\"` identifier") + end + t = peek_token(ps) + k = kind(t) + if preceding_whitespace(t) || is_operator(k) || + k in KSet"( ) [ ] { } , ; @ EndMarker" + # var"x"+ ==> x + # var"x") ==> x + # var"x"( ==> x + elseif is_string_macro_suffix(k) + # var"x"end ==> (var x (error-t)) + # var"x"1 ==> (var x (error-t)) + # var"x"y ==> (var x (error-t)) + bump(ps, TRIVIA_FLAG, error="suffix not allowed after `var\"...\"` syntax") + elseif k == K"`" || k == K"\"" || k == K"\"\"\"" || k == K"```" + # Disallow `var"#""str". To allow this we'd need to fix `raw` + # detection in lex_quote to be consistent with the parser. + bump_invisible(ps, K"error", TRIVIA_FLAG, + error="`var\"...\"` syntax not supported as string macro name") + end + emit(ps, mark, K"var") + elseif check_identifiers && is_closing_token(ps, leading_kind) + # :(end) ==> (quote-: (error end)) + bump(ps, error="invalid identifier") + else + # Remap keywords to identifiers. + # :end ==> (quote-: end) + # :<: ==> (quote-: <:) + bump(ps, remap_kind=K"Identifier") + end + elseif leading_kind == K"(" # parens or tuple + parse_paren(ps, check_identifiers, has_unary_prefix) + elseif leading_kind == K"[" # cat expression + bump(ps, TRIVIA_FLAG) + ckind, cflags, dim = parse_cat(ps, K"]", ps.end_symbol) + emit(ps, mark, ckind, cflags | set_numeric_flags(dim)) + check_ncat_compat(ps, mark, ckind) + elseif leading_kind == K"{" # cat expression + bump(ps, TRIVIA_FLAG) + ckind, cflags, dim = parse_cat(ps, K"}", ps.end_symbol) + emit_braces(ps, mark, ckind, cflags, dim) + elseif leading_kind == K"@" # macro call + # Macro names can be keywords + # @end x ==> (macrocall (macro_name end) x) + bump(ps, TRIVIA_FLAG) + parse_macro_name(ps) + parse_call_chain(ps, mark, true) + elseif is_string_delim(leading_kind) + parse_string(ps, false) + elseif leading_kind in KSet"` ```" + # `` ==> (cmdstring-r "") + # `cmd` ==> (cmdstring-r "cmd") + # ```cmd``` ==> (cmdstring-s-r "cmd") + parse_string(ps, true) + elseif is_literal(leading_kind) + # 42 ==> 42 + bump(ps) + elseif is_closing_token(ps, leading_kind) + # Leave closing token in place for other productions to + # recover with + # ) ==> error + msg = leading_kind == K"EndMarker" ? + "premature end of input" : + "unexpected `$(untokenize(leading_kind))`" + emit_diagnostic(ps, error=msg) + bump_invisible(ps, K"error") + else + bump(ps, error="invalid syntax atom") + end +end diff --git a/JuliaSyntax/src/julia/parser_api.jl b/JuliaSyntax/src/julia/parser_api.jl new file mode 100644 index 0000000000000..f00808bb4a040 --- /dev/null +++ b/JuliaSyntax/src/julia/parser_api.jl @@ -0,0 +1,230 @@ +# The main parser API. +# +# This is defined separately from parser.jl so that: +# * parser.jl doesn't need to refer to any tree data structures +# * It's clear which parts are the public API + +struct ParseError <: Exception + source::SourceFile + diagnostics::Vector{Diagnostic} + incomplete_tag::Symbol # Used only for Base Expr(:incomplete) support +end + +function ParseError(stream::ParseStream; incomplete_tag=:none, kws...) + source = SourceFile(stream; kws...) + ParseError(source, stream.diagnostics, incomplete_tag) +end + +function Base.showerror(io::IO, err::ParseError) + # Only show the first parse error for now - later errors are often + # misleading due to the way recovery works + i = findfirst(is_error, err.diagnostics) + if isnothing(i) + i = lastindex(err.diagnostics) + level_info = " some warnings detected:" + else + level_info = "" + end + println(io, "ParseError:", level_info) + show_diagnostics(io, err.diagnostics[1:i], err.source) +end + +sourcefile(err::ParseError) = err.source + +""" + parse!(stream::ParseStream; rule=:all) + +Parse Julia source code from a [`ParseStream`](@ref) object. Output tree data +structures may be extracted from `stream` with the [`build_tree`](@ref) function. + +`rule` may be any of +* `:all` (default) — parse a whole "file" of top level statements. In this + mode, the parser expects to fully consume the input. +* `:statement` — parse a single statement, or statements separated by semicolons. +* `:atom` — parse a single syntax "atom": a literal, identifier, or + parenthesized expression. +""" +function parse!(stream::ParseStream; rule::Symbol=:all) + if rule == :toplevel + Base.depwarn("Use of rule == :toplevel in parse!() is deprecated. use `rule=:all` instead.", :parse!) + rule = :all + end + ps = ParseState(stream) + if rule === :all + parse_toplevel(ps) + elseif rule === :statement + parse_stmts(ps) + elseif rule === :atom + parse_atom(ps) + else + throw(ArgumentError("Unknown grammar rule $rule")) + end + validate_tokens(stream) + stream +end + +""" + parse!(TreeType, io::IO; rule=:all, version=VERSION) + +Parse Julia source code from a seekable `IO` object. The output is a tuple +`(tree, diagnostics)`. When `parse!` returns, the stream `io` is positioned +directly after the last byte which was consumed during parsing. +""" +function parse!(::Type{TreeType}, io::IO; + rule::Symbol=:all, version=VERSION, kws...) where {TreeType} + stream = ParseStream(io; version=version) + parse!(stream; rule=rule) + tree = build_tree(TreeType, stream; kws...) + seek(io, last_byte(stream)) + tree, stream.diagnostics +end + +""" + build_tree(type, ps::ParseStream; kws...) + +Build a tree of type `type` from a raw parsed stream `ps`. This is the final step in parsing. +""" +function build_tree end + +function _parse(rule::Symbol, need_eof::Bool, ::Type{T}, text, index=1; version=VERSION, + ignore_trivia=true, filename=nothing, first_line=1, ignore_errors=false, + ignore_warnings=ignore_errors, kws...) where {T} + stream = ParseStream(text, index; version=version) + if ignore_trivia && rule != :all + bump_trivia(stream, skip_newlines=true) + end + parse!(stream; rule=rule) + if need_eof + if (ignore_trivia && peek(stream, skip_newlines=true) != K"EndMarker") || + (!ignore_trivia && (peek(stream, skip_newlines=false, skip_whitespace=false) != K"EndMarker")) + emit_diagnostic(stream, error="unexpected text after parsing $rule") + end + end + if (!ignore_errors && any_error(stream.diagnostics)) || + (!ignore_warnings && !isempty(stream.diagnostics)) + throw(ParseError(stream, filename=filename, first_line=first_line)) + end + tree = build_tree(T, stream; filename=filename, first_line=first_line, kws...) + tree, last_byte(stream) + 1 +end + +_parse_docs = """ + # Parse a single expression/statement + parsestmt(TreeType, text, [index]; + version=VERSION, + ignore_trivia=true, + filename=nothing, + ignore_errors=false, + ignore_warnings=ignore_errors) + + # Parse all statements at top level (file scope) + parseall(...) + + # Parse a single syntax atom + parseatom(...) + +Parse Julia source code string `text` into a data structure of type `TreeType`. +`parsestmt` parses a single Julia statement, `parseall` parses top level statements +at file scope and `parseatom` parses a single Julia identifier or other "syntax +atom". + +If `text` is passed without `index`, all the input text must be consumed and a +tree data structure is returned. When an integer byte `index` is passed, a +tuple `(tree, next_index)` will be returned containing the next index in `text` +to resume parsing. By default whitespace and comments before and after valid +code are ignored but you can turn this off by setting `ignore_trivia=false`. + +`version` (default `VERSION`) may be used to set the syntax version to +any Julia version `>= v"1.0"`. We aim to parse all Julia syntax which has been +added after v"1.0", emitting an error if it's not compatible with the requested +`version`. + +Pass `filename` to set any file name information embedded within the output +tree, if applicable. This will also annotate errors and warnings with the +source file name. + +A `ParseError` will be thrown if any errors or warnings occurred during +parsing. To avoid exceptions due to warnings, use `ignore_warnings=true`. To +also avoid exceptions due to errors, use `ignore_errors=true`. +""" + +"$_parse_docs" +parsestmt(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:statement, true, T, text; kws...)[1] + +"$_parse_docs" +parseall(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:all, true, T, text; kws...)[1] + +"$_parse_docs" +parseatom(::Type{T}, text::AbstractString; kws...) where {T} = _parse(:atom, true, T, text; kws...)[1] + +parsestmt(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:statement, false, T, text, index; kws...) +parseall(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:all, false, T, text, index; kws...) +parseatom(::Type{T}, text::AbstractString, index::Integer; kws...) where {T} = _parse(:atom, false, T, text, index; kws...) + +#------------------------------------------------------------------------------- +# Tokens interface +""" +Token type resulting from calling `tokenize(text)` + +Use +* `kind(tok)` to get the token kind +* `untokenize(tok, text)` to retrieve the text +* Predicates like `is_error(tok)` to query token categories and flags +""" +struct Token + head::SyntaxHead + range::UnitRange{UInt32} +end + +Token() = Token(SyntaxHead(K"None", EMPTY_FLAGS), 0:0) + +head(t::Token) = t.head + +""" + tokenize(text; operators_as_identifiers=true) + +Returns the tokenized UTF-8 encoded `text` as a vector of `Token`s. The +text for the token can be retrieved by using `untokenize()`. The full text can be +reconstructed with, for example, `join(untokenize.(tokenize(text), text))`. + +This interface works on UTF-8 encoded string or buffer data only. + +The keyword `operators_as_identifiers` specifies whether operators in +identifier-position should have `K"Identifier"` as their kind, or be emitted as +more specific operator kinds. For example, whether the `+` in `a + b` should be +emitted as `K"Identifier"` (the default) or as `K"+"`. +""" +function tokenize(text; operators_as_identifiers=true) + ps = ParseStream(text) + parse!(ps, rule=:all) + ts = ps.output + output_tokens = Token[] + byte_start::UInt32 = ps.output[1].byte_span + 1 + for i = 2:length(ts) + if kind(ts[i]) == K"TOMBSTONE" || is_non_terminal(ts[i]) + continue + end + r = byte_start:(byte_start+ts[i].byte_span - 1) + byte_start = last(r) + 1 + k = kind(ts[i]) + if k == K"Identifier" && !operators_as_identifiers + orig_k = ts[i].orig_kind + if is_operator(orig_k) && !is_word_operator(orig_k) + k = orig_k + end + end + f = flags(ts[i]) + push!(output_tokens, Token(SyntaxHead(k,f), r)) + end + output_tokens +end + +function untokenize(token::Token, text::AbstractString) + text[first(token.range):thisind(text, last(token.range))] +end + +function untokenize(token::Token, text::Vector{UInt8}) + text[token.range] +end + +@deprecate parse parsestmt diff --git a/JuliaSyntax/src/julia/tokenize.jl b/JuliaSyntax/src/julia/tokenize.jl new file mode 100644 index 0000000000000..eb6b36583bf84 --- /dev/null +++ b/JuliaSyntax/src/julia/tokenize.jl @@ -0,0 +1,1319 @@ +module Tokenize + +export tokenize, untokenize + +using ..JuliaSyntax: JuliaSyntax, Kind, @K_str, @KSet_str, @callsite_inline, + generic_operators_by_level, PrecedenceLevel, PREC_NONE, PREC_ASSIGNMENT, + PREC_PAIRARROW, PREC_CONDITIONAL, PREC_ARROW, PREC_LAZYOR, PREC_LAZYAND, + PREC_COMPARISON, PREC_PIPE_LT, PREC_PIPE_GT, PREC_COLON, PREC_PLUS, + PREC_BITSHIFT, PREC_TIMES, PREC_RATIONAL, PREC_POWER, PREC_DECL, + PREC_WHERE, PREC_DOT, PREC_QUOTE, PREC_UNICODE_OPS, PREC_COMPOUND_ASSIGN + +import ..JuliaSyntax: kind, + is_literal, is_contextual_keyword, is_word_operator, is_operator + +#------------------------------------------------------------------------------- +# Character-based predicates for tokenization +import Base.Unicode + +const EOF_CHAR = typemax(Char) + +function is_identifier_char(c::Char) + c == EOF_CHAR && return false + isvalid(c) || return false + return Base.is_id_char(c) +end + +function is_identifier_start_char(c::Char) + c == EOF_CHAR && return false + isvalid(c) || return false + c == '🢲' && return false # First divergence from Base.is_id_start_char + return Base.is_id_start_char(c) +end + +function is_invisible_char(c::Char) + # These are the chars considered invisible by the reference parser. + # TODO: There's others we could add? See for example + # https://invisible-characters.com/ + return c == '\u00ad' || # soft hyphen + c == '\u200b' || # zero width space + c == '\u200c' || # zero width non-joiner + c == '\u200d' || # zero width joiner + c == '\u200e' || # left-to-right mark + c == '\u200f' || # right-to-left mark + c == '\u2060' || # word joiner + c == '\u2061' # function application + # https://github.com/JuliaLang/julia/issues/49850 + # c == '\u115f' || # Hangul Choseong filler +end + +# Chars that we will never allow to be part of a valid non-operator identifier +function is_never_id_char(ch::Char) + isvalid(ch) || return true + cat = Unicode.category_code(ch) + c = UInt32(ch) + return ( + # spaces and control characters: + (cat >= Unicode.UTF8PROC_CATEGORY_ZS && cat <= Unicode.UTF8PROC_CATEGORY_CS) || + + # ASCII and Latin1 non-connector punctuation + (c < 0xff && + cat >= Unicode.UTF8PROC_CATEGORY_PD && cat <= Unicode.UTF8PROC_CATEGORY_PO) || + + c == UInt32('`') || + + # mathematical brackets + (c >= 0x27e6 && c <= 0x27ef) || + # angle, corner, and lenticular brackets + (c >= 0x3008 && c <= 0x3011) || + # tortoise shell, square, and more lenticular brackets + (c >= 0x3014 && c <= 0x301b) || + # fullwidth parens + (c == 0xff08 || c == 0xff09) || + # fullwidth square brackets + (c == 0xff3b || c == 0xff3d) + ) +end + +readchar(io::IO) = eof(io) ? EOF_CHAR : read(io, Char) + +function _char_in_set_expr(varname, firstchars) + codes = sort!(UInt32.(unique(firstchars))) + terms = [] + i = 1 + while i <= length(codes) + j = i + while j < length(codes) && codes[j+1] == codes[j]+1 + j += 1 + end + if i == j + push!(terms, :($varname == $(codes[i]))) + else + push!(terms, :($(codes[i]) <= $varname <= $(codes[j]))) + end + i = j+1 + end + foldr((t1,t2)->:($t1 || $t2), terms) +end + +@eval function is_operator_start_char(c) + if c == EOF_CHAR || !isvalid(c) + return false + end + # Check if character is a known operator char or in our unicode ops + # dictionary. The second tuple lists the unicode operators which don't + # appear in `_unicode_ops` because they have their own kinds (and explicit + # branches in `_next_token`). + return c in ('!', '#', '$', '%', '&', '*', '+', '-', '−', '/', ':', '<', '=', '>', '?', '@', '\\', '^', '|', '~', '÷', '⊻', '\'') || + c in ('∈', '≔', '⩴', '≕', '¬', '√', '∛', '∜') || + haskey(_unicode_ops, c) +end + +# Checks whether a Char is an operator which can be prefixed with a dot `.` +function is_dottable_operator_start_char(c) + return c != '?' && c != '$' && c != ':' && c != '\'' && c != '#' && c != '@' && is_operator_start_char(c) +end + +@eval function isopsuffix(c::Char) + c == EOF_CHAR && return false + isvalid(c) || return false + u = UInt32(c) + if (u < 0xa1 || u > 0x10ffff) + return false + end + cat = Base.Unicode.category_code(u) + if (cat == Base.Unicode.UTF8PROC_CATEGORY_MN || + cat == Base.Unicode.UTF8PROC_CATEGORY_MC || + cat == Base.Unicode.UTF8PROC_CATEGORY_ME) + return true + end + # Additional allowed cases + return $(_char_in_set_expr(:u, + collect("²³¹ʰʲʳʷʸˡˢˣ˱˲ᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵅᵇᵈᵉᵋᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶲᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₔₕₖₗₘₙₚₛₜⱼⱽꜛꜜꜝ"))) +end + +function optakessuffix(k) + # Most operators can take suffix except for specific ones + is_operator(k) && + !( + K"BEGIN_ASSIGNMENTS" <= k <= K"END_ASSIGNMENTS" || + k == K"?" || + k == K"<:" || + k == K">:" || + k == K"&&" || + k == K"||" || + k == K"in" || + k == K"isa" || + k == K":" || + k == K"$" || + k == K"::" || + k == K"where" || + k == K"." || + k == K"!" || + k == K".'" || + k == K"->" || + K"¬" <= k <= K"∜" + ) +end + +const _unicode_ops = let + # Map single-character unicode operators to their precedence levels + ops = Dict{Char, PrecedenceLevel}() + + # Add operators from generic_operators_by_level + for (prec, chars) in generic_operators_by_level + for c in chars + ops[c] = prec + end + end + + ops +end + +#------------------------------------------------------------------------------- +# Tokens + +struct RawToken + kind::Kind + # Offsets into a string or buffer + startbyte::Int # The byte where the token start in the buffer + endbyte::Int # The byte where the token ended in the buffer + op_precedence::PrecedenceLevel # If K"Operator", the operator's precedence level +end +function RawToken(kind::Kind, startbyte::Int, endbyte::Int) + RawToken(kind, startbyte, endbyte, PREC_NONE) +end +RawToken() = RawToken(K"error", 0, 0, PREC_NONE) + +const EMPTY_TOKEN = RawToken() + +kind(t::RawToken) = t.kind + +startbyte(t::RawToken) = t.startbyte +endbyte(t::RawToken) = t.endbyte + + +function untokenize(t::RawToken, str::String) + String(codeunits(str)[1 .+ (t.startbyte:t.endbyte)]) +end + +function Base.show(io::IO, t::RawToken) + print(io, rpad(string(startbyte(t), "-", endbyte(t)), 11, " ")) + print(io, rpad(kind(t), 15, " ")) +end + +#------------------------------------------------------------------------------- +# Lexer + +@inline ishex(c::Char) = isdigit(c) || ('a' <= c <= 'f') || ('A' <= c <= 'F') +@inline isbinary(c::Char) = c == '0' || c == '1' +@inline isoctal(c::Char) = '0' ≤ c ≤ '7' +@inline iswhitespace(c::Char) = (isvalid(c) && Base.isspace(c)) || c === '\ufeff' + +struct StringState + triplestr::Bool + raw::Bool + delim::Char + paren_depth::Int +end + +""" +`Lexer` reads from an input stream and emits a single token each time +`next_token` is called. + +Ideally a lexer is stateless but some state is needed here for: +* Disambiguating cases like x' (adjoint) vs 'x' (character literal) +* Tokenizing code within string interpolations +""" +mutable struct Lexer{IO_t <: IO} + io::IO_t + + token_startpos::Int + + last_token::Kind + string_states::Vector{StringState} + chars::Tuple{Char,Char,Char,Char} + charspos::Tuple{Int,Int,Int,Int} +end + +function Lexer(io::IO) + c1 = ' ' + p1 = position(io) + if eof(io) + c2, p2 = EOF_CHAR, p1 + c3, p3 = EOF_CHAR, p1 + c4, p4 = EOF_CHAR, p1 + else + c2 = read(io, Char) + p2 = position(io) + if eof(io) + c3, p3 = EOF_CHAR, p2 + c4, p4 = EOF_CHAR, p2 + else + c3 = read(io, Char) + p3 = position(io) + if eof(io) + c4, p4 = EOF_CHAR, p3 + else + c4 = read(io, Char) + p4 = position(io) + end + end + end + Lexer(io, position(io), + K"error", Vector{StringState}(), + (c1,c2,c3,c4), (p1,p2,p3,p4)) +end +Lexer(str::AbstractString) = Lexer(IOBuffer(str)) + +""" + tokenize(x) + +Returns an `Iterable` containing the tokenized input. Can be reverted by e.g. +`join(untokenize.(tokenize(x)))`. +""" +tokenize(x) = Lexer(x) + +# Iterator interface +Base.IteratorSize(::Type{<:Lexer}) = Base.SizeUnknown() +Base.IteratorEltype(::Type{<:Lexer}) = Base.HasEltype() +Base.eltype(::Type{<:Lexer}) = RawToken + + +function Base.iterate(l::Lexer) + l.token_startpos = position(l) + + t = next_token(l) + return t, t.kind == K"EndMarker" +end + +function Base.iterate(l::Lexer, isdone::Any) + isdone && return nothing + t = next_token(l) + return t, t.kind == K"EndMarker" +end + +function Base.show(io::IO, l::Lexer) + print(io, typeof(l), " at position: ", position(l)) +end + +""" + startpos(l::Lexer) + +Return the latest `RawToken`'s starting position. +""" +startpos(l::Lexer) = l.token_startpos + +""" + startpos!(l::Lexer, i::Integer) + +Set a new starting position. +""" +startpos!(l::Lexer, i::Integer) = l.token_startpos = i + +""" + peekchar(l::Lexer) + +Returns the next character without changing the lexer's state. +""" +peekchar(l::Lexer) = l.chars[2] + +""" +dpeekchar(l::Lexer) + +Returns the next two characters without changing the lexer's state. +""" +dpeekchar(l::Lexer) = l.chars[2], l.chars[3] + +""" +peekchar3(l::Lexer) + +Returns the next three characters without changing the lexer's state. +""" +peekchar3(l::Lexer) = l.chars[2], l.chars[3], l.chars[4] + +""" + position(l::Lexer) + +Returns the current position. +""" +Base.position(l::Lexer) = l.charspos[1] + +""" + eof(l::Lexer) + +Determine whether the end of the lexer's underlying buffer has been reached. +""" +Base.eof(l::Lexer) = eof(l.io) + +Base.seek(l::Lexer, pos) = seek(l.io, pos) + +""" + start_token!(l::Lexer) + +Updates the lexer's state such that the next `RawToken` will start at the current +position. +""" +function start_token!(l::Lexer) + l.token_startpos = l.charspos[1] +end + +""" + readchar(l::Lexer) + +Returns the next character and increments the current position. +""" +function readchar(l::Lexer) + c = readchar(l.io) + l.chars = (l.chars[2], l.chars[3], l.chars[4], c) + l.charspos = (l.charspos[2], l.charspos[3], l.charspos[4], position(l.io)) + return l.chars[1] +end + +""" + accept(l::Lexer, f::Union{Function, Char, Vector{Char}, String}) + +Consumes the next character `c` if either `f::Function(c)` returns true, `c == f` +for `c::Char` or `c in f` otherwise. Returns `true` if a character has been +consumed and `false` otherwise. +""" +@inline function accept(l::Lexer, f::Union{Function, Char, Vector{Char}, String}) + c = peekchar(l) + if isa(f, Function) + ok = f(c) + elseif isa(f, Char) + ok = c == f + else + ok = c in f + end + ok && readchar(l) + return ok +end + +""" + accept_batch(l::Lexer, f) + +Consumes all following characters until `accept(l, f)` is `false`. +""" +@inline function accept_batch(l::Lexer, f) + ok = false + while accept(l, f) + ok = true + end + return ok +end + +""" + emit(l::Lexer, kind::Kind) + +Returns a `RawToken` of kind `kind` and starts a new `RawToken`. +""" +function emit(l::Lexer, kind::Kind) + tok = RawToken(kind, startpos(l), position(l) - 1, PREC_NONE) + + l.last_token = kind + return tok +end + +function emit_operator(l::Lexer, kind::Kind, precedence::PrecedenceLevel, take_suffix=optakessuffix(kind)) + if take_suffix + while isopsuffix(peekchar(l)) + readchar(l) + kind = K"Operator" + end + end + tok = RawToken(kind, startpos(l), position(l) - 1, precedence) + + l.last_token = kind + return tok +end + +# Check whether the operator just lexed forms a compound assignment like `+=`, +# ie whether the input continues with the single token `=`. Peeking one +# character is not enough: the other tokens starting with `=` (`==`, `===` and +# `=>`) do not form compound assignments, eg `a +== b` is `a + (== b)`. +function compound_assign_follows(l::Lexer) + pc, ppc = dpeekchar(l) + return pc == '=' && ppc != '=' && ppc != '>' +end + +# Emit `kind` with precedence `prec`, unless the operator is immediately followed +# by `=`, in which case it forms a compound assignment like `+=` and is emitted as +# a `K"Operator"` with `PREC_COMPOUND_ASSIGN`. +function emit_operator_or_compound_assign(l::Lexer, kind::Kind, prec::PrecedenceLevel) + if compound_assign_follows(l) + return emit_operator(l, K"Operator", PREC_COMPOUND_ASSIGN) + end + return emit_operator(l, kind, prec) +end + +# Emit wrapping arithmetic operators with their own kind so `+%=`, `-%=`, and +# `*%=` can round-trip to distinct update-assignment heads. +function emit_wrapping_operator_or_compound_assign(l::Lexer, kind::Kind, prec::PrecedenceLevel) + if compound_assign_follows(l) + return emit_operator(l, kind, PREC_COMPOUND_ASSIGN) + end + return emit_operator(l, kind, prec) +end + +function emit_trivia(l::Lexer, kind::Kind) + tok = RawToken(kind, startpos(l), position(l) - 1, PREC_NONE) + l.last_token = kind + return tok +end + +""" + next_token(l::Lexer) + +Returns the next `RawToken`. +""" +function next_token(l::Lexer, start = true) + start && start_token!(l) + if !isempty(l.string_states) + return lex_string_chunk(l) + else + return _next_token(l, readchar(l)) + end +end + +function _next_token(l::Lexer, c) + if c == EOF_CHAR + return emit(l, K"EndMarker") + elseif iswhitespace(c) + return lex_whitespace(l, c) + elseif c == '[' + return emit(l, K"[") + elseif c == ']' + return emit(l, K"]") + elseif c == '{' + return emit(l, K"{") + elseif c == ';' + return emit(l, K";") + elseif c == '}' + return emit(l, K"}") + elseif c == '(' + return emit(l, K"(") + elseif c == ')' + return emit(l, K")") + elseif c == ',' + return emit(l, K",") + elseif c == '*' + return lex_star(l); + elseif c == '^' + return lex_circumflex(l); + elseif c == '@' + return emit(l, K"@") + elseif c == '?' + return emit(l, K"?") + elseif c == '$' + return lex_dollar(l); + elseif c == '⊻' + return lex_xor(l); + elseif c == '~' + return emit(l, K"~") + elseif c == '#' + return lex_comment(l) + elseif c == '=' + return lex_equal(l) + elseif c == '!' + return lex_exclaim(l) + elseif c == '>' + return lex_greater(l) + elseif c == '<' + return lex_less(l) + elseif c == ':' + return lex_colon(l) + elseif c == '|' + return lex_bar(l) + elseif c == '&' + return lex_amper(l) + elseif c == '\'' + return lex_prime(l) + elseif c == '÷' + return lex_division(l) + elseif c == '"' + return lex_quote(l); + elseif c == '%' + return lex_percent(l); + elseif c == '/' + return lex_forwardslash(l); + elseif c == '\\' + return lex_backslash(l); + elseif c == '.' + return lex_dot(l); + elseif c == '+' + return lex_plus(l); + elseif c == '-' + return lex_minus(l); + elseif c == '`' + return lex_backtick(l); + elseif c == '−' # \minus '−' treated as hyphen '-' + return emit_operator_or_compound_assign(l, K"-", PREC_PLUS) + elseif c == '∈' + return emit_operator(l, K"∈", PREC_COMPARISON) + elseif c == '⋆' + return emit_operator(l, K"⋆", PREC_TIMES) + elseif c == '±' + return emit_operator(l, K"±", PREC_PLUS) + elseif c == '∓' + return emit_operator(l, K"∓", PREC_PLUS) + elseif c == '¬' + return emit(l, K"¬") + elseif c == '√' + return emit(l, K"√") + elseif c == '∛' + return emit(l, K"∛") + elseif c == '∜' + return emit(l, K"∜") + elseif c == '≔' + return emit_operator(l, K"≔", PREC_ASSIGNMENT) + elseif c == '⩴' + return emit_operator(l, K"⩴", PREC_ASSIGNMENT) + elseif c == '≕' + return emit_operator(l, K"≕", PREC_ASSIGNMENT) + elseif haskey(_unicode_ops, c) + return emit_operator(l, K"Operator", _unicode_ops[c]) + elseif is_identifier_start_char(c) + return lex_identifier(l, c) + elseif isdigit(c) + return lex_digit(l, K"Integer") + else + return emit(l, + !isvalid(c) ? K"ErrorInvalidUTF8" : + is_invisible_char(c) ? K"ErrorInvisibleChar" : + is_identifier_char(c) ? K"ErrorIdentifierStart" : + K"ErrorUnknownCharacter") + end +end + +# UAX #9: Unicode Bidirectional Algorithm +# https://unicode.org/reports/tr9/ +# Very partial implementation - just enough to check correct nesting in strings +# and multiline comments. +function update_bidi_state((embedding_nesting, isolate_nesting), c) + if c == '\n' + embedding_nesting = 0 + isolate_nesting = 0 + elseif c == '\U202A' || c == '\U202B' || c == '\U202D' || c == '\U202E' # LRE RLE LRO RLO + embedding_nesting += 1 + elseif c == '\U202C' # PDF + embedding_nesting -= 1 + elseif c == '\U2066' || c == '\U2067' || c == '\U2068' # LRI RLI FSI + isolate_nesting += 1 + elseif c == '\U2069' # PDI + isolate_nesting -= 1 + end + return (embedding_nesting, isolate_nesting) +end + +# We're inside a string; possibly reading the string characters, or maybe in +# Julia code within an interpolation. +function lex_string_chunk(l) + state = last(l.string_states) + if state.paren_depth > 0 + # Read normal Julia code inside an interpolation but track nesting of + # parentheses. + # TODO: This stateful tracking should probably, somehow, be done by the + # parser instead? Especially for recovery of unbalanced parens inside + # interpolations? + c = readchar(l) + if c == '(' + l.string_states[end] = StringState(state.triplestr, state.raw, state.delim, + state.paren_depth + 1) + return emit(l, K"(") + elseif c == ')' + l.string_states[end] = StringState(state.triplestr, state.raw, state.delim, + state.paren_depth - 1) + return emit(l, K")") + else + return _next_token(l, c) + end + end + pc = peekchar(l) + if l.last_token == K"$" + pc = peekchar(l) + # Interpolated symbol or expression + if pc == '(' + readchar(l) + l.string_states[end] = StringState(state.triplestr, state.raw, state.delim, + state.paren_depth + 1) + return emit(l, K"(") + elseif is_identifier_start_char(pc) + return lex_identifier(l, readchar(l)) + else + # Getting here is a syntax error - fall through to reading string + # characters and let the parser deal with it. + end + elseif l.last_token == K"Identifier" && + !(pc == EOF_CHAR || is_operator_start_char(pc) || is_never_id_char(pc)) + # Only allow certain characters after interpolated vars + # https://github.com/JuliaLang/julia/pull/25234 + readchar(l) + return emit(l, K"ErrorInvalidInterpolationTerminator") + end + if pc == EOF_CHAR + return emit(l, K"EndMarker") + elseif !state.raw && pc == '$' + # Start interpolation + readchar(l) + return emit(l, K"$") + elseif !state.raw && pc == '\\' && (pc2 = dpeekchar(l)[2]; + pc2 == '\r' || pc2 == '\n') + # Process escaped newline as whitespace + readchar(l) + readchar(l) + if pc2 == '\r' && peekchar(l) == '\n' + readchar(l) + end + while (pc = peekchar(l); pc == ' ' || pc == '\t') + readchar(l) + end + return emit(l, K"Whitespace") + elseif pc == state.delim && string_terminates(l, state.delim, state.triplestr) + if state.delim == '\'' && l.last_token == K"'" && dpeekchar(l)[2] == '\'' + # Handle ''' + readchar(l) + return emit(l, K"Char") + end + # Terminate string + pop!(l.string_states) + readchar(l) + if state.triplestr + readchar(l); readchar(l) + return emit(l, state.delim == '"' ? + K"\"\"\"" : K"```") + else + return emit(l, state.delim == '"' ? K"\"" : + state.delim == '`' ? K"`" : K"'") + end + end + # Read a chunk of string characters + init_bidi_state = (0,0) + bidi_state = init_bidi_state + valid = true + if state.raw + # Raw strings treat all characters as literals with the exception that + # the closing quotes can be escaped with an odd number of \ characters. + while true + pc = peekchar(l) + if string_terminates(l, state.delim, state.triplestr) || pc == EOF_CHAR + break + elseif state.triplestr && (pc == '\n' || pc == '\r') + # triple quoted newline splitting + readchar(l) + if pc == '\n' + bidi_state = init_bidi_state + elseif pc == '\r' && peekchar(l) == '\n' + bidi_state = init_bidi_state + readchar(l) + end + break + end + c = readchar(l) + if c == '\\' + n = 1 + while peekchar(l) == '\\' + readchar(l) + n += 1 + end + if peekchar(l) == state.delim && !iseven(n) + readchar(l) + end + end + bidi_state = update_bidi_state(bidi_state, c) + valid &= isvalid(c) + end + else + while true + pc = peekchar(l) + if pc == '$' || pc == EOF_CHAR + break + elseif state.triplestr && (pc == '\n' || pc == '\r') + # triple quoted newline splitting + readchar(l) + if pc == '\n' + bidi_state = init_bidi_state + elseif pc == '\r' && peekchar(l) == '\n' + readchar(l) + bidi_state = init_bidi_state + end + break + elseif pc == state.delim && string_terminates(l, state.delim, state.triplestr) + break + elseif pc == '\\' + # Escaped newline + _, pc2, pc3 = peekchar3(l) + if pc2 == '\r' || pc2 == '\n' + if pc2 == '\n' || pc3 == '\n' + bidi_state = init_bidi_state + end + break + end + end + c = readchar(l) + if c == '\\' + c = readchar(l) + c == EOF_CHAR && break + end + bidi_state = update_bidi_state(bidi_state, c) + valid &= isvalid(c) + end + end + outk = !valid ? K"ErrorInvalidUTF8" : + state.delim == '\'' ? K"Char" : + bidi_state != init_bidi_state ? K"ErrorBidiFormatting" : + state.delim == '"' ? K"String" : + state.delim == '`' ? K"CmdString" : + (@assert(state.delim in KSet"' \" `"); K"error") + return emit(l, outk) +end + +# Lex whitespace, a whitespace char `c` has been consumed +function lex_whitespace(l::Lexer, c) + k = K"Whitespace" + while true + if c == '\n' + k = K"NewlineWs" + end + pc, ppc = dpeekchar(l) + # stop on non whitespace and limit to a single newline in a token + if !iswhitespace(pc) || + (k == K"NewlineWs" && (pc == '\n' || (pc == '\r' && ppc == '\n'))) + break + end + c = readchar(l) + end + return emit_trivia(l, k) +end + +function lex_comment(l::Lexer) + if peekchar(l) != '=' + valid = true + while true + pc, ppc = dpeekchar(l) + if pc == '\n' || (pc == '\r' && ppc == '\n') || pc == EOF_CHAR + return valid ? emit_trivia(l, K"Comment") : + emit(l, K"ErrorInvalidUTF8") + end + valid &= isvalid(pc) + readchar(l) + end + else + c = readchar(l) # consume the '=' + init_bidi_state = (0,0) + bidi_state = init_bidi_state + skip = true # true => c was part of the prev comment marker pair + nesting = 1 + valid = true + while true + if c == EOF_CHAR + return emit(l, K"ErrorEofMultiComment") + end + nc = readchar(l) + bidi_state = update_bidi_state(bidi_state, nc) + valid &= isvalid(nc) + if skip + skip = false + else + if c == '#' && nc == '=' + nesting += 1 + skip = true + elseif c == '=' && nc == '#' + nesting -= 1 + skip = true + if nesting == 0 + outk = !valid ? K"ErrorInvalidUTF8" : + bidi_state != init_bidi_state ? K"ErrorBidiFormatting" : + K"Comment" + return emit(l, outk) + end + end + end + c = nc + end + end +end + +# Lex a greater char, a '>' has been consumed +function lex_greater(l::Lexer) + if accept(l, '>') + # >> >>> >>= >>>= + accept(l, '>') + return emit_operator_or_compound_assign(l, K"Operator", PREC_BITSHIFT) + elseif accept(l, '=') + return emit_operator(l, K"Operator", PREC_COMPARISON) # >= + elseif accept(l, ':') + return emit(l, K">:") + else + return emit_operator(l, K">", PREC_COMPARISON) + end +end + +# Lex a less char, a '<' has been consumed +function lex_less(l::Lexer) + if accept(l, '<') + # << or <<= + return emit_operator_or_compound_assign(l, K"Operator", PREC_BITSHIFT) + elseif accept(l, '=') + return emit_operator(l, K"Operator", PREC_COMPARISON) # <= + elseif accept(l, ':') + return emit(l, K"<:") + elseif accept(l, '|') + return emit_operator(l, K"Operator", PREC_PIPE_LT) # <| + elseif dpeekchar(l) == ('-', '-') + readchar(l); readchar(l) + if accept(l, '-') + return emit(l, K"ErrorInvalidOperator") + else + if accept(l, '>') + return emit_operator(l, K"Operator", PREC_ARROW) # <--> + elseif accept(l, '-') + return emit(l, K"ErrorInvalidOperator") + else + return emit_operator(l, K"Operator", PREC_ARROW) # <-- + end + end + else + return emit_operator(l, K"<", PREC_COMPARISON) + end +end + +# Lex all tokens that start with an = character. +# An '=' char has been consumed +function lex_equal(l::Lexer) + if accept(l, '=') + accept(l, '=') + return emit_operator(l, K"Operator", PREC_COMPARISON) # ==, === + elseif accept(l, '>') + return emit_operator(l, K"Operator", PREC_PAIRARROW) + else + return emit(l, K"=") + end +end + +# Lex a colon, a ':' has been consumed +function lex_colon(l::Lexer) + if accept(l, ':') + return emit(l, K"::") + elseif accept(l, '=') + return emit(l, K":=") + else + return emit_operator(l, K":", PREC_COLON) + end +end + +function lex_exclaim(l::Lexer) + if accept(l, '=') + if accept(l, '=') + return emit_operator(l, K"Operator", PREC_COMPARISON) # !== + else + return emit_operator(l, K"Operator", PREC_COMPARISON) # != + end + else + return emit(l, K"!") + end +end + +function lex_percent(l::Lexer) + return emit_operator_or_compound_assign(l, K"Operator", PREC_TIMES) +end + +function lex_bar(l::Lexer) + if accept(l, '>') + return emit_operator(l, K"Operator", PREC_PIPE_GT) # |> + elseif accept(l, '|') + return emit(l, K"||") + else + return emit_operator_or_compound_assign(l, K"Operator", PREC_PLUS) + end +end + +function lex_plus(l::Lexer) + if accept(l, '+') + return emit_operator(l, K"++", PREC_PLUS) + elseif accept(l, '%') + return emit_wrapping_operator_or_compound_assign(l, K"+%", PREC_PLUS) + end + return emit_operator_or_compound_assign(l, K"+", PREC_PLUS) +end + +function lex_minus(l::Lexer) + if accept(l, '-') + if accept(l, '>') + return emit_operator(l, K"-->", PREC_ARROW) + else + return emit(l, K"ErrorInvalidOperator") # "--" is an invalid operator + end + elseif l.last_token != K"." && accept(l, '>') + return emit_operator(l, K"->", PREC_ARROW) + elseif accept(l, '%') + return emit_wrapping_operator_or_compound_assign(l, K"-%", PREC_PLUS) + end + return emit_operator_or_compound_assign(l, K"-", PREC_PLUS) +end + +function lex_star(l::Lexer) + if accept(l, '*') + return emit(l, K"Error**") # "**" is an invalid operator; use ^ + elseif accept(l, '%') + return emit_wrapping_operator_or_compound_assign(l, K"*%", PREC_TIMES) + end + return emit_operator_or_compound_assign(l, K"*", PREC_TIMES) +end + +function lex_circumflex(l::Lexer) + return emit_operator_or_compound_assign(l, K"Operator", PREC_POWER) # ^ +end + +function lex_division(l::Lexer) + return emit_operator_or_compound_assign(l, K"Operator", PREC_TIMES) # / +end + +function lex_dollar(l::Lexer) + return emit_operator_or_compound_assign(l, K"$", PREC_PLUS) +end + +function lex_xor(l::Lexer) + return emit_operator_or_compound_assign(l, K"Operator", PREC_PLUS) +end + +function accept_number(l::Lexer, f::F) where {F} + lexed_number = false + while true + pc, ppc = dpeekchar(l) + if pc == '_' && !f(ppc) + return lexed_number + elseif f(pc) || pc == '_' + readchar(l) + else + return lexed_number + end + lexed_number = true + end +end + +# A digit has been consumed +function lex_digit(l::Lexer, kind) + accept_number(l, isdigit) + pc,ppc = dpeekchar(l) + if pc == '.' + if ppc == '.' + # Number followed by K"." + return emit(l, kind) + elseif kind === K"Float" + # If we enter the function with kind == K"Float" then a '.' has been parsed. + readchar(l) + return emit(l, K"ErrorInvalidNumericConstant") + elseif is_dottable_operator_start_char(ppc) + readchar(l) + return emit(l, K"ErrorAmbiguousNumericConstant") # `1.+` + end + readchar(l) + + kind = K"Float" + accept(l, '_') && return emit(l, K"ErrorInvalidNumericConstant") # `1._` + had_fraction_digs = accept_number(l, isdigit) + pc, ppc = dpeekchar(l) + if (pc == 'e' || pc == 'E' || pc == 'f') && (isdigit(ppc) || ppc == '+' || ppc == '-' || ppc == '−') + kind = pc == 'f' ? K"Float32" : K"Float" + readchar(l) + accept(l, "+-−") + if accept_batch(l, isdigit) + pc,ppc = dpeekchar(l) + if pc === '.' && ppc != '.' && !is_dottable_operator_start_char(ppc) + readchar(l) + return emit(l, K"ErrorInvalidNumericConstant") # `1.e1.` + end + else + return emit(l, K"ErrorInvalidNumericConstant") # `1.e` + end + elseif pc == '.' && ppc != '.' && !is_dottable_operator_start_char(ppc) + readchar(l) + return emit(l, K"ErrorInvalidNumericConstant") # `1.1.` + elseif !had_fraction_digs && (is_identifier_start_char(pc) || + pc == '(' || pc == '[' || pc == '{' || + pc == '@' || pc == '`' || pc == '"') + return emit(l, K"ErrorAmbiguousNumericDotMultiply") # `1.(` `1.x` + end + elseif (pc == 'e' || pc == 'E' || pc == 'f') && (isdigit(ppc) || ppc == '+' || ppc == '-' || ppc == '−') + kind = pc == 'f' ? K"Float32" : K"Float" + readchar(l) + accept(l, "+-−") + if accept_batch(l, isdigit) + pc,ppc = dpeekchar(l) + if pc === '.' && ppc != '.' && !is_dottable_operator_start_char(ppc) + accept(l, '.') + return emit(l, K"ErrorInvalidNumericConstant") # `1e1.` + end + else + return emit(l, K"ErrorInvalidNumericConstant") # `1e+` + end + elseif position(l) - startpos(l) == 1 && l.chars[1] == '0' + kind == K"Integer" + is_bin_oct_hex_int = false + if pc == 'x' + kind = K"HexInt" + isfloat = false + readchar(l) + had_digits = accept_number(l, ishex) + pc,ppc = dpeekchar(l) + if pc == '.' && ppc != '.' + readchar(l) + had_digits |= accept_number(l, ishex) + isfloat = true + end + if accept(l, "pP") + kind = K"Float" + accept(l, "+-−") + if !accept_number(l, isdigit) || !had_digits + return emit(l, K"ErrorInvalidNumericConstant") # `0x1p` `0x.p0` + end + # Check for invalid trailing decimal point + # https://github.com/JuliaLang/julia/issues/60189 + pc,ppc = dpeekchar(l) + if pc == '.' && ppc != '.' && !is_dottable_operator_start_char(ppc) + accept_batch(l, c->(c == '.' || isdigit(c))) + # `0x1p3.` `0x1p3.2` `0x1.5p2.3` + return emit(l, K"ErrorInvalidNumericConstant") + end + elseif isfloat + return emit(l, K"ErrorHexFloatMustContainP") # `0x.` `0x1.0` + end + is_bin_oct_hex_int = !isfloat + elseif pc == 'b' + readchar(l) + had_digits = accept_number(l, isbinary) + kind = K"BinInt" + is_bin_oct_hex_int = true + elseif pc == 'o' + readchar(l) + had_digits = accept_number(l, isoctal) + kind = K"OctInt" + is_bin_oct_hex_int = true + end + if is_bin_oct_hex_int + pc = peekchar(l) + @assert @isdefined(had_digits) + if !had_digits || isdigit(pc) || is_identifier_start_char(pc) + accept_batch(l, c->isdigit(c) || is_identifier_start_char(c)) + # `0x` `0xg` `0x_` `0x-` + # `0b123` `0o78p` `0xenomorph` `0xaα` + return emit(l, K"ErrorInvalidNumericConstant") + end + end + end + return emit(l, kind) +end + +function lex_prime(l) + if l.last_token == K"Identifier" || + is_contextual_keyword(l.last_token) || + is_word_operator(l.last_token) || + l.last_token == K"." || + l.last_token == K")" || + l.last_token == K"]" || + l.last_token == K"}" || + l.last_token == K"'" || + l.last_token == K"end" || + is_literal(l.last_token) + # FIXME ^ This doesn't cover all cases - probably needs involvement + # from the parser state. + return emit_operator(l, K"'", PREC_QUOTE) + else + push!(l.string_states, StringState(false, true, '\'', 0)) + return emit(l, K"'") + end +end + +function lex_amper(l::Lexer) + if accept(l, '&') + return emit(l, K"&&") + else + return emit_operator_or_compound_assign(l, K"&", PREC_TIMES) + end +end + +# Parse a token starting with a quote. +# A '"' has been consumed +function lex_quote(l::Lexer) + raw = l.last_token == K"Identifier" || + is_contextual_keyword(l.last_token) || + is_word_operator(l.last_token) + pc, dpc = dpeekchar(l) + triplestr = pc == '"' && dpc == '"' + push!(l.string_states, StringState(triplestr, raw, '"', 0)) + if triplestr + readchar(l) + readchar(l) + emit(l, K"\"\"\"") + else + emit(l, K"\"") + end +end + +function string_terminates(l, delim::Char, triplestr::Bool) + if triplestr + c1, c2, c3 = peekchar3(l) + c1 === delim && c2 === delim && c3 === delim + else + peekchar(l) === delim + end +end + +# Parse a token starting with a forward slash. +# A '/' has been consumed +function lex_forwardslash(l::Lexer) + prec = accept(l, '/') ? PREC_RATIONAL : PREC_TIMES + return emit_operator_or_compound_assign(l, K"Operator", prec) # // or / +end + +function lex_backslash(l::Lexer) + return emit_operator_or_compound_assign(l, K"Operator", PREC_TIMES) +end + +function lex_dot(l::Lexer) + if l.last_token == K"@" + if accept(l, '.') + if !accept(l, '.') && is_dottable_operator_start_char(peekchar(l)) + readchar(l) + return emit(l, K"ErrorInvalidOperator") + end + end + # Emit `.`, `..` and `...` as identifiers after `@` + emit(l, K"Identifier") + elseif l.last_token != K"." && Base.isdigit(peekchar(l)) + # Only start a numeric constant if the previous token wasn't a dot + return lex_digit(l, K"Float") + else + return emit(l, K".") + end +end + +# A ` has been consumed +function lex_backtick(l::Lexer) + pc, dpc = dpeekchar(l) + triplestr = pc == '`' && dpc == '`' + # Backticks always contain raw strings only. See discussion on bug + # https://github.com/JuliaLang/julia/issues/3150 + raw = true + push!(l.string_states, StringState(triplestr, raw, '`', 0)) + if triplestr + readchar(l) + readchar(l) + emit(l, K"```") + else + emit(l, K"`") + end +end + +const MAX_KW_LENGTH = 10 +const ascii_is_identifier_char = Bool[is_identifier_char(Char(b)) for b=0x00:0x7f] +function lex_identifier(l::Lexer, c) + h = simple_hash(c, UInt64(0)) + n = 1 + ascii = isascii(c) + graphemestate = Ref(Int32(ascii)) # all ASCII id chars are UTF8PROC_BOUNDCLASS_OTHER + graphemestate_peek = Ref(zero(Int32)) + while true + pc, ppc = dpeekchar(l) + ascii = ascii && isascii(pc) + if ascii # fast path + pc_byte = pc % UInt8 + @inbounds if (pc_byte == UInt8('!') && ppc == '=') || !ascii_is_identifier_char[pc_byte+1] + break + end + elseif @callsite_inline Unicode.isgraphemebreak!(graphemestate, c, pc) + if (pc == '!' && ppc == '=') || !is_identifier_char(pc) + break + end + elseif pc in ('\u200c','\u200d') # ZWNJ/ZWJ control characters + # ZWJ/ZWNJ only within grapheme sequences, not at end + graphemestate_peek[] = graphemestate[] + if @callsite_inline Unicode.isgraphemebreak!(graphemestate_peek, pc, ppc) + break + end + end + c = readchar(l) + h = simple_hash(c, h) + n += 1 + end + + if n > MAX_KW_LENGTH + return emit(l, K"Identifier") + elseif h == _true_hash || h == _false_hash + return emit(l, K"Bool") + else + k = get(_kw_hash, h, K"Identifier") + return emit(l, k) + end +end + +# This creates a hash for chars in [A-z] using 6 bit per char. +# Requires an additional input-length check somewhere, because +# this only works up to ~10 chars. +@inline function simple_hash(c::Char, h::UInt64) + bytehash = (clamp(c - 'A' + 1, -1, 60) % UInt8) & 0x3f + h << 6 + bytehash +end + +function simple_hash(str) + ind = 1 + h = UInt64(0) + L = min(lastindex(str), MAX_KW_LENGTH) + while ind <= L + h = simple_hash(str[ind], h) + ind = nextind(str, ind) + end + h +end + +kws = [ +K"baremodule", +K"begin", +K"break", +K"catch", +K"const", +K"continue", +K"do", +K"else", +K"elseif", +K"end", +K"export", +K"finally", +K"for", +K"function", +K"global", +K"if", +K"import", +K"let", +K"local", +K"macro", +K"module", +K"public", +K"quote", +K"return", +K"struct", +K"try", +K"typegroup", +K"using", +K"while", +K"in", +K"isa", +K"where", + +K"abstract", +K"as", +K"doc", +K"mutable", +K"outer", +K"primitive", +K"type", +K"var", +K"VERSION" +] + +const _true_hash = simple_hash("true") +const _false_hash = simple_hash("false") +const _kw_hash = Dict(simple_hash(string(kw)) => kw for kw in kws) + +end # module diff --git a/JuliaSyntax/src/porcelain/green_node.jl b/JuliaSyntax/src/porcelain/green_node.jl new file mode 100644 index 0000000000000..f34e97e9bcf67 --- /dev/null +++ b/JuliaSyntax/src/porcelain/green_node.jl @@ -0,0 +1,175 @@ +""" + struct GreenNode + +An explicit pointer-y representation of the green tree produced by the parser. +See `RawGreenNode` for documentation on working with the implicit green +tree directly. However, this representation is useful for introspection as it +provides O(1) access to the children (as well as forward iteration). +""" +struct GreenNode{Head} + head::Head + span::UInt32 + children::Union{Nothing,Vector{GreenNode{Head}}} +end + +function GreenNode(head::Head, span::Integer, children=nothing) where {Head} + GreenNode{Head}(head, span, children) +end + +# Accessors / predicates +is_leaf(node::GreenNode) = isnothing(node.children) +children(node::GreenNode) = node.children +numchildren(node::GreenNode) = isnothing(node.children) ? 0 : length(node.children) +head(node::GreenNode) = node.head + +""" + span(node) + +Get the number of bytes this node covers in the source text. +""" +span(node::GreenNode) = node.span + +Base.getindex(node::GreenNode, i::Int) = children(node)[i] +Base.getindex(node::GreenNode, rng::UnitRange) = view(children(node), rng) +Base.firstindex(::GreenNode) = 1 +Base.lastindex(node::GreenNode) = children(node) === nothing ? 0 : length(children(node)) + +""" +Get absolute position and span of the child of `node` at the given tree `path`. +""" +function child_position_span(node::GreenNode, path::Int...) + n = node + p = 1 + for index in path + cs = children(n) + for i = 1:index-1 + p += span(cs[i]) + end + n = cs[index] + end + return n, p, n.span +end + +function highlight(io::IO, source::SourceFile, node::GreenNode, path::Int...; kws...) + _, p, span = child_position_span(node, path...) + q = p + span - 1 + highlight(io, source, p:q; kws...) +end + +Base.summary(node::GreenNode) = summary(node.head) + +function Base.hash(node::GreenNode, h::UInt) + children = node.children + if children === nothing + h = hash(nothing, h) + else # optimization - avoid extra allocations from `hash(::AbstractVector, ::UInt)` + for child in children + h = hash(child, h) + end + end + hash(node.head, hash(node.span, h)) +end +function Base.:(==)(n1::GreenNode, n2::GreenNode) + n1.head == n2.head && n1.span == n2.span && n1.children == n2.children +end + +# Pretty printing +function _show_green_node(io, node, indent, pos, str, show_trivia) + if !show_trivia && is_trivia(node) + return + end + posstr = "$(lpad(pos, 6)):$(rpad(pos+span(node)-1, 6)) │" + leaf = is_leaf(node) + if leaf + line = string(posstr, indent, summary(node)) + else + line = string(posstr, indent, '[', summary(node), ']') + end + if !is_trivia(node) && leaf + line = rpad(line, 40) * "✔" + end + if is_error(node) + line = rpad(line, 41) * "✘" + end + if leaf && !isnothing(str) + line = string(rpad(line, 43), ' ', repr(str[pos:prevind(str, pos + span(node))])) + end + line = line*"\n" + if is_error(node) + printstyled(io, line, color=:light_red) + else + print(io, line) + end + if !leaf + new_indent = indent*" " + p = pos + for x in children(node) + _show_green_node(io, x, new_indent, p, str, show_trivia) + p += x.span + end + end +end + +function Base.show(io::IO, ::MIME"text/plain", node::GreenNode) + _show_green_node(io, node, "", 1, nothing, true) +end + +function Base.show(io::IO, ::MIME"text/plain", node::GreenNode, str::AbstractString; show_trivia=true) + _show_green_node(io, node, "", 1, str, show_trivia) +end + +function _show_green_node_sexpr(io, node::GreenNode, position) + if is_leaf(node) + print(io, position, "-", position+node.span-1, "::", untokenize(head(node); unique=false)) + else + print(io, "(", untokenize(head(node); unique=false)) + p = position + for n in children(node) + print(io, ' ') + _show_green_node_sexpr(io, n, p) + p += n.span + end + print(io, ')') + end +end + +function Base.show(io::IO, node::GreenNode) + _show_green_node_sexpr(io, node, 1) +end + +function GreenNode(cursor::GreenTreeCursor) + chead = head(cursor) + T = typeof(chead) + if is_leaf(cursor) + return GreenNode{T}(head(cursor), span(cursor), nothing) + else + children = GreenNode{T}[] + for child in reverse(cursor) + pushfirst!(children, GreenNode(child)) + end + return GreenNode{T}(head(cursor), span(cursor), children) + end +end + +function build_tree(::Type{GreenNode}, stream::ParseStream; + # unused, but required since `_parse` is written generically + filename=nothing, first_line=1, keep_parens=false) + cursor = GreenTreeCursor(stream) + if has_toplevel_siblings(cursor) + # There are multiple toplevel nodes, e.g. because we're using this + # to test a partial parse. Wrap everything in K"wrapper" + local cs + for child in reverse_toplevel_siblings(cursor) + c = GreenNode(child) + if @isdefined(cs) + pushfirst!(cs, c) + else + cs = GreenNode{SyntaxHead}[c] + end + end + @assert @isdefined(cs) && length(cs) != 1 + return GreenNode(SyntaxHead(K"wrapper", NON_TERMINAL_FLAG), stream.next_byte-1, cs) + else + return GreenNode(cursor) + end +end diff --git a/JuliaSyntax/src/porcelain/syntax.jl b/JuliaSyntax/src/porcelain/syntax.jl new file mode 100644 index 0000000000000..f7e3dfebb7af8 --- /dev/null +++ b/JuliaSyntax/src/porcelain/syntax.jl @@ -0,0 +1,1437 @@ +# Data structures used by macro-expansion and lowering + +mutable struct ScopeLayer + const mod::Module + const escaped::Union{Nothing, ScopeLayer} +end + +""" +Each node has a SyntaxContext describing its macro expansion and syntax version. +`SyntaxContext` is shared between all nodes of a single macro expansion, and is +one-to-one with ScopeLayer, with a few exceptions (contexts sharing same layer): +- `escape` and adopt_scope +- Desugaring creates internal contexts in its better version of `gensym` + +We may want to move layer out of this struct for easier adopt_scope and +rebase_layer operations, but assuming mostly hygienic macros and few +scope-changing functions, this is most compact. +""" +mutable struct SyntaxContext + const layer::ScopeLayer + # For provenance; is not affected by escaping + const unexpanded::Any # Union{SyntaxTree, Nothing} + const version::VersionNumber + const internal::Bool +end + +# Reference to bytes within a source file +struct SourceRef + file::Base.RefValue{SourceFile} + first_byte::UInt32 + last_byte::UInt32 +end + +mutable struct SyntaxTree + const kind::Kind + # Should be considered immutable + children::Union{Nothing, Vector{SyntaxTree}} + value::Any + source::Union{SyntaxTree,SourceRef,LineNumberNode} + context::Union{Nothing, SyntaxContext} + jl_source::Union{Nothing, LineNumberNode} + meta::Union{Nothing, Base.ImmutableDict{Symbol,Any}} + # TODO: this is rarely used, and should just be part of context + mod::Union{Nothing, Module} + # TODO: this is almost never populated and semantically irrelevant after + # parsing + syntax_flags::UInt16 +end + +function SyntaxTree(kind::Kind, children, @nospecialize(value), source, context) + SyntaxTree(kind, children, value, source, context, + nothing, nothing, nothing, UInt16(0)) +end + +const SourceAttrType = Union{SyntaxTree,SourceRef,LineNumberNode} + +function setchildren!(id::SyntaxTree, children::AbstractVector{SyntaxTree}) + setfield!(id, :children, children) +end + +# fallback printing. TODO: vulnerable to invalidations +function node_string(ex::SyntaxTree, depth=2) + out = "(kind="*string(kind(ex)) + for n in sort!(collect(fieldnames(typeof(ex)))) + val = getproperty(ex, n) + if !isnothing(val) && n !== :kind + val_str = if val isa SyntaxTree && depth > 1 + node_string(val, depth-1) + elseif isbits(val) || val isa + Union{AbstractString, Symbol, Module, LineNumberNode} + repr(val) + else + repr(typeof(val)) + end + out *= ", "*string(n)*"="*val_str + end + end + if is_leaf(ex) + out *= ", leaf" + elseif depth > 1 + out *= ", children=[" + for c in children(ex) + out *= "\n"*node_string(c, depth-1) + end + out *= "]" + end + out *= ")" + return out +end + +function Base.getindex(ex::SyntaxTree, i::Integer) + ex.children[i] +end + +function Base.getindex(ex::SyntaxTree, r::UnitRange) + @view ex.children[r] +end + +Base.firstindex(::SyntaxTree) = 1 +Base.lastindex(ex::SyntaxTree) = numchildren(ex) + +function Base.:≈(ex1::SyntaxTree, ex2::SyntaxTree) + if kind(ex1) != kind(ex2) || is_leaf(ex1) != is_leaf(ex2) + return false + end + if is_leaf(ex1) + return ex1.value == ex2.value + else + if numchildren(ex1) != numchildren(ex2) + return false + end + return all(c1 ≈ c2 for (c1,c2) in zip(children(ex1), children(ex2))) + end +end + +function _setattr!(ex::SyntaxTree, name::Symbol, @nospecialize(val)) + setfield!(ex, name, val) + ex +end +_setattr(ex::SyntaxTree, name::Symbol, @nospecialize(val)) = + _setattr!(is_leaf(ex) ? mkleaf(ex) : mknode(ex, children(ex)), name, val) + +const CompileHints = Base.ImmutableDict{Symbol,Any} +function setmeta!(st::SyntaxTree, key::Symbol, @nospecialize(val)) + meta = let m = st.meta + isnothing(m) ? CompileHints(key, val) : CompileHints(m, key, val) + end + setfield!(st, :meta, meta) + st +end +function setmeta(st::SyntaxTree, key::Symbol, @nospecialize(val)) + setmeta!(is_leaf(st) ? mkleaf(st) : mknode(st, children(st)), key, val) +end +function getmeta(st, name, @nospecialize(default)) + meta = st.meta + isnothing(meta) ? default : get(meta, name, default) +end + +Base.setproperty!(ex::SyntaxTree, name::Symbol, @nospecialize(val)) = + error("SyntaxTree: this can't be mutated") + +# JuliaSyntax tree API + +function is_leaf(ex::SyntaxTree) + ex.children === nothing +end + +function numchildren(ex::SyntaxTree) + cs = ex.children + isnothing(cs) ? 0 : length(cs) +end + +# TODO: Better to make this an error, since it can cause nodes that were +# intended to be leaves `SyntaxTree(kind, children(old), ...)` to be non-leaves +const NO_CHILDREN = SyntaxTree[] + +function children(ex::SyntaxTree) + cs = ex.children + cs === nothing ? NO_CHILDREN : cs +end + +function head(ex::SyntaxTree) + SyntaxHead(kind(ex), flags(ex)) +end + +function kind(ex::SyntaxTree) + ex.kind +end + +function flags(ex::SyntaxTree) + ex.syntax_flags +end + +# A default context corresponding to no expansion +function SyntaxContext(mod::Module, version::VersionNumber) + SyntaxContext(ScopeLayer(mod, nothing), nothing, version, false) +end + +# TODO: switch from bool-based `expr_compat_mode` to `version` +const JL_NEW_SYNTAX_VERSION = v"1.14" +const JL_OLD_SYNTAX_VERSION = v"1.13" + +is_base_layer(sc::SyntaxContext) = sc.layer.escaped === nothing + +# The scope corresponding to no macro expansion. Use with caution: macros may +# expand to top-level forms, so "base layer" !== "this top-level thunk's +# pre-expansion context" (usually ctx.syntax_context) +function base_layer(sc::SyntaxContext) + l = sc.layer + while l.escaped !== nothing + l = l.escaped + end + return l +end + +function escape_layer(sc::SyntaxContext, recursive::Bool) + l2 = recursive ? base_layer(sc) : sc.layer.escaped + SyntaxContext(l2, sc.unexpanded, sc.version, sc.internal) +end + +syntax_module(sc::SyntaxContext) = sc.layer.mod +function syntax_module(st::SyntaxTree) + st_mod = st.mod + st_mod === nothing || return st_mod::Module + syntax_module(st.context::SyntaxContext) +end + +is_flisp_compat(sc::SyntaxContext) = sc.version < JL_NEW_SYNTAX_VERSION +is_flisp_compat(st::SyntaxTree) = is_flisp_compat(st.context) + +# Unconditional; tramples existing scope, and includes quoted forms. Only +# changes layer where it needs changing. +function adopt_scope(sc_in::SyntaxContext, st::SyntaxTree, scmap) + st_sc = st.context + sc2 = st_sc isa SyntaxContext ? get(scmap, st_sc, nothing) : nothing + if isnothing(sc2) && st_sc isa SyntaxContext + sc2 = scmap[st_sc] = st_sc.layer === sc_in.layer ? st_sc : + SyntaxContext( + sc_in.layer, st_sc.unexpanded, st_sc.version, st_sc.internal) + elseif isnothing(sc2) + sc2 = sc_in + end + if is_leaf(st) || numchildren(st) == 0 + sc2 === st_sc ? st : _setattr(st, :context, sc2) + else + out = mapchildren(c->adopt_scope(sc_in, c, scmap), st) + sc2 === st_sc ? out : + out !== st ? _setattr!(out, :context, sc2) : + _setattr(out, :context, sc2) + end +end +function adopt_scope(reference::SyntaxTree, st::SyntaxTree) + adopt_scope(reference.context::SyntaxContext, st, + Dict{SyntaxContext, SyntaxContext}()) +end + +function fill_context!(st::SyntaxTree, sc::SyntaxContext) + _setattr!(st, :context, sc) + !is_leaf(st) && for c in children(st) + fill_context!(c, sc) + end + st +end +fill_context(st, sc) = fill_context!(mktree(st), sc) + +function remove_context!(st::SyntaxTree) + sc = st.context + isnothing(sc) || _setattr!(st, :context, nothing) + for c in children(st) + remove_context!(c) + end + st +end +remove_context(st) = remove_context!(mktree(st)) + +function Base.show(io::IO, ::MIME"text/plain", sl::ScopeLayer) + color = isnothing(sl.escaped) ? :normal : :cyan + printstyled(io, "SL("; color) + print(io, string(sl.mod)) + print(io, ",") + !isnothing(sl.escaped) && print(io, sl.escaped) + print(io, ",") + printstyled(io, string(objectid(sl);base=62); color) + printstyled(io, ")"; color) +end +Base.show(io::IO, sl::ScopeLayer) = Base.show(io::IO, MIME"text/plain"(), sl) + +function Base.show(io::IO, ::MIME"text/plain", sc::SyntaxContext) + color = sc.internal ? :light_black : + sc.version == JL_NEW_SYNTAX_VERSION ? :normal : :blue + printstyled(io, "["; color) + if sc.version != JL_NEW_SYNTAX_VERSION + printstyled(io, "old,"; color) + end + if sc.internal + printstyled(io, "internal,"; color) + end + print(io, sc.layer) + print(io, ",") + if sc.unexpanded isa SyntaxTree + k = kind(sc.unexpanded) + k === K"macrocall" ? print(io, sc.unexpanded[1]) : print(io, k) + end + printstyled(io, "]"; color) +end +Base.show(io::IO, sc::SyntaxContext) = Base.show(io::IO, MIME"text/plain"(), sc) + +sourcefile(src::SourceRef) = src.file[] +first_byte(src::SourceRef) = Int(src.first_byte) +last_byte(src::SourceRef) = Int(src.last_byte) +byte_range(src::SourceRef) = first_byte(src):last_byte(src) + +# TODO: Adding these methods to support LineNumberNode is kind of hacky but we +# can remove these after JuliaLowering becomes self-bootstrapping for macros +# and we a proper SourceRef for @ast's @HERE form. +byte_range(::LineNumberNode) = 0:0 +source_location(src::LineNumberNode) = (src.line, 0) +source_location(::Type{LineNumberNode}, src::LineNumberNode) = src +source_line(src::LineNumberNode) = src.line +# The following somewhat strange cases are for where LineNumberNode is standing in +# for SourceFile because we've only got Expr-based provenance info +sourcefile(src::LineNumberNode) = src +sourcetext(::LineNumberNode) = SubString("") +source_location(src::LineNumberNode, _byte_index::Integer) = (src.line, 0) +source_location(::Type{LineNumberNode}, src::LineNumberNode, _byte_index::Integer) = src +filename(src::LineNumberNode) = string(src.file) + +function highlight(io::IO, src::LineNumberNode; note="") + print(io, src, " - ", note) +end + +function highlight(io::IO, src::SourceRef; kws...) + highlight(io, sourcefile(src), first_byte(src):last_byte(src); kws...) +end + +function Base.show(io::IO, ::MIME"text/plain", src::SourceRef) + highlight(io, src; note="these are the bytes you're looking for 😊", context_lines_inner=20) +end + +""" +Provenance notes: A SyntaxTree `st` has `.source` equal to one of: +- SyntaxTree (of the SyntaxTree `st` was transformed from) +- a reference to source text (either SourceRef or LineNumberNode). + +Let "textref" refer to a SyntaxTree with non-SyntaxTree `.source`. Every SyntaxTree +is either a textref or has one at the end of its `.source` chain. + +All invariants noted in this section are awaiting the design of the "new macro" +API. As of writing this, the user has more freedom than they should have. +""" + +""" +SyntaxList of [st.source, st.source.source, ..., textref] +""" +function provenance(st::SyntaxTree) + prov = SyntaxList() + s = st.source + while s isa SyntaxTree + push!(prov, s) + s = s.source + end + return prov +end + +"`provenance(st)[1]`, or `st` if that's empty" +function prov(st::SyntaxTree) + source = st.source + source isa SyntaxTree ? source : st +end + +"textref of st (possibly == st)" +function prov_end(st::SyntaxTree) + out = st + while out.source isa SyntaxTree + out = prov(out) + end + return out +end + +"`st`'s textref's `.source`, ignoring all expansions" +function sourceref(st::SyntaxTree) + src = prov_end(st) + src.source::Union{LineNumberNode, SourceRef} +end + +"The last macro expansion `st` was involved in, or nothing" +function macro_prov(st::SyntaxTree) + sc = st.context + isnothing(sc) && return nothing + msrc = (sc::SyntaxContext).unexpanded + isnothing(msrc) ? nothing : msrc::typeof(st) +end + +"The first macro expansion `st` was involved in (chronologically), or nothing" +function macro_prov_end(st::SyntaxTree) + lastmp = mp = macro_prov(st) + while !isnothing(mp) + lastmp, mp = mp, macro_prov(mp) + end + return lastmp +end + +"The top-level location of `st`" +function unexpanded_sourceref(st::SyntaxTree) + mp = macro_prov_end(st) + isnothing(mp) ? sourceref(st) : sourceref(mp) +end + +""" +A SyntaxList of textrefs associated with `st`. The number of returned trees +should equal one plus the number of macro expansions `st` "went through": + +- For new macros, this is the number of macro expansions `st` was both an input + and output of, so if `st` was created in a macro body, `flattened_provenance` + returns a list of length 1. + +- For old macros, we can't determine whether expanded syntax is from the + macrocall args or macro body (it will have LineNumberNode .source), so all + expanded syntax counts as having "went through" the macrocall. + +The resulting list should be in the order +`[outermost_macrocall, innermost_macrocall, ..., expression_textref]`. +""" +function flattened_provenance(st::SyntaxTree) + _flattened_provenance(st, SyntaxList()) +end + +# Only recurse on the first macro source in any source chain +function _flattened_provenance(st::SyntaxTree, out) + msrc = macro_prov(st) + # macro source === source means `st` is from the `msrc` macro body + !isnothing(msrc) && msrc != prov(st) && + _flattened_provenance(msrc, out) + push!(out, prov_end(st)) + out +end + +sourcefile(ex::SyntaxTree) = sourcefile(sourceref(ex)) +byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) + +function sourcetext(ex::SyntaxTree) + sf = sourcefile(ex) + sf isa LineNumberNode && return SubString("") + view(sf, byte_range(ex)) +end + +# TODO (refactoring): make SyntaxList an immutable wrapper around node children +const SyntaxList = Vector{SyntaxTree} + +SyntaxList(rest::SyntaxTree...) = SyntaxTree[rest...] + +function mapsyntax(f, exs::AbstractVector{SyntaxTree}) + out = SyntaxList() + for ex in exs + push!(out, f(ex)) + end + out +end + +function mapindex(sl::SyntaxList, i::Int) + out = SyntaxList() + for st in sl + push!(out, getindex(st, i)) + end + out +end + +#------------------------------------------------------------------------------- +# AST creation utilities + +""" + newnode(prov::SourceAttrType, k::Kind, children) + +Create a new node with reference to parsed source text `prov`. +""" +function newnode(prov::SourceAttrType, k::Kind, children) + context = prov isa SyntaxTree ? prov.context : nothing + SyntaxTree(k, children, nothing, prov, context) +end +function newleaf(prov::SourceAttrType, k::Kind) + context = prov isa SyntaxTree ? prov.context : nothing + SyntaxTree(k, nothing, nothing, prov, context) +end + +function mknode(old::SyntaxTree, children) + SyntaxTree(old.kind, children, old.value, old, old.context, + old.jl_source, old.meta, old.mod, old.syntax_flags) +end +function mkleaf(old::SyntaxTree) + SyntaxTree(old.kind, nothing, old.value, old, old.context, + old.jl_source, old.meta, old.mod, old.syntax_flags) +end +function mktree(old::SyntaxTree) + if is_leaf(old) + mkleaf(old) + else + cs = mapsyntax(mktree, children(old)) + mknode(old, cs) + end +end + +#------------------------------------------------------------------------------- +# Mapping and copying of AST nodes + +# This function should be allocation-free if no children were changed +function mapchildren(f::Function, ex::SyntaxTree) + if is_leaf(ex) + return ex + end + orig_children = children(ex) + cs = nothing + for (i,e) in enumerate(orig_children) + newchild = f(e)::SyntaxTree + if isnothing(cs) + if newchild == e + continue + else + cs = SyntaxList(undef, length(orig_children)) + copyto!(cs, orig_children[1:i-1]) + end + end + cs[i] = newchild + end + if isnothing(cs) + return ex + end + cs::SyntaxList + ex2 = mknode(ex, cs) + return ex2 +end + +""" +Recursively copy AST `ex` into `ctx`. Every node in `ex` should be copied at +most once. + +TODO: Likely unnecessary with immutable tree +""" +function copy_ast(ex::SyntaxTree) + _copy_ast(ex, Dict{SyntaxTree, SyntaxTree}()) +end + +function _copy_ast(id1::SyntaxTree, seen) + let copied = get(seen, id1, nothing) + isnothing(copied) || return copied + end + id2 = is_leaf(id1) ? mkleaf(id1) : mknode(id1, children(id1)) + seen[id1] = id2 + if !is_leaf(id1) + cs = SyntaxTree[] + for cid in children(id1) + push!(cs, _copy_ast(cid, seen)) + end + setchildren!(id2, cs) + end + src1 = id1.source + if src1 isa SyntaxTree + src2 = _copy_ast(src1, seen) + _setattr!(id2, :source, src2) + elseif !isnothing(src1) + _setattr!(id2, :source, src1) + else + throw("bad source?") + end + return id2 +end + +""" + unalias_nodes(st::SyntaxTree) + +Return a tree where each descendent of `st` has exactly one parent in `st`. The +returned tree is identical to `st` in all but underlying representation, where +every additional parent to a subtree generates a copy of that subtree. Apart +from achieving this, `unalias_nodes` should not allocate new nodes. + + unalias_nodes(sl::SyntaxList) + +If a `SyntaxList` is given, every resulting tree will be unique with respect to +each other as well as internally. A duplicate entry will produce a copied tree. +""" +unalias_nodes(st::SyntaxTree) = + _unalias_nodes(st, Set{SyntaxTree}(), Base.IdSet{Vector{SyntaxTree}}()) + +function unalias_nodes(sl::SyntaxList) + seen = Set{SyntaxTree}() + seen_children = Base.IdSet{Vector{SyntaxTree}}() + mapsyntax(st->_unalias_nodes(st, seen, seen_children), sl) +end + +function _unalias_copy_tree(old::SyntaxTree) + out = if is_leaf(old) + mkleaf(old) + else + cs = mapsyntax(_unalias_copy_tree, children(old)) + mknode(old, cs) + end + # difference from mktree: don't add to provenance chain + _setattr!(out, :source, old.source) +end + +function _unalias_nodes(st::SyntaxTree, seen::Set{SyntaxTree}, + seen_children::Base.IdSet{Vector{SyntaxTree}}) + if st in seen + return _unalias_copy_tree(st) + end + push!(seen, st) + if !is_leaf(st) + cs = children(st) + if cs in seen_children + cs = copy(cs) + setchildren!(st, cs) + end + push!(seen_children, cs) + for (i, c) in enumerate(cs) + c2 = _unalias_nodes(c, seen, seen_children) + c !== c2 && (cs[i] = c2) + end + end + return st +end + +""" +Give each descendent of `st` a `parent::SyntaxTree` attribute. +""" +function annotate_parent!(st::SyntaxTree) + st = unalias_nodes(st) + mapchildren(t->_annotate_parent!(t, st), st) +end + +function _annotate_parent!(st::SyntaxTree, pid::SyntaxTree) + setmeta!(st, :parent, pid) + mapchildren(t->_annotate_parent!(t, st), st) +end + +#------------------------------------------------------------------------------- +# AST destructuring utilities + +raw""" +Simple `SyntaxTree` pattern matching + +Returns the first result where its corresponding pattern matches `syntax_tree` +and each extra `cond` is true. Throws an error if no match is found. + +## Patterns + +A pattern is used as both a conditional (does this syntax tree have a certain +structure?) and a `let` (bind trees to these names if so). Each pattern uses a +limited version of the @ast syntax: + +``` + = + | [K"" *] + | [K"" * ... *] + +# note "*" is the meta-operator meaning one or more, and "..." is literal +``` + +where a `[K"k" p1 p2 ps...]` form matches any tree with kind `k` and >=2 +children (bound to `p1` and `p2`), and `ps` is bound to the possibly-empty +SyntaxList of children `3:end`. Identifiers (except `_`) can't be re-used, but +may check for some form of tree equivalence in a future implementation. + +## Extra condition: `when` + +Like an escape hatch to the structure-matching mechanism. `when=cond` requires +`cond` to evaluate to `true` for this branch to be taken. `cond` may also bind +variables or printf-debug the matching process, as it runs only when its pattern +matches and no previous branch was taken. `cond` may not mutate the object +being matched. + +## Scope of variables + +Every `(pattern, when=cond) -> result` introduces a local scope. Identifiers in +the pattern are let-bound when evaluating `cond` and `result`. `cond` can +introduce variables for use in `result`. User code in `cond` and `result` (but +not `pattern`) can refer to outer variables. + +## Example + +``` +julia> st = JuliaSyntax.parsestmt( + JuliaSyntax.SyntaxTree, "function foo(x,y,z); x; end") + +julia> JuliaSyntax.@stm st begin + [K"function" [K"call" fname [K"parameters" kws...]] body] -> + "no positional args, only kwargs: $(kws)" + [K"function" fname] -> + "zero-method function $fname" + [K"function" [K"call" fname args...] body] -> + "normal function $fname" + ([K"=" [K"call" _...] _...], when=(args=if_valid_get_args(st[1]); !isnothing(args))) -> + "deprecated call-equals form with args $args" + (_, when=(show("printf debugging is great"); true)) -> "something else" + _ -> "unreachable due to the case above" +end +"normal function foo" +``` + +See [Racket `match`](https://docs.racket-lang.org/reference/match.html) for the +inspiration for this macro and an example of a much more featureful pattern +language. +""" +macro stm(st, pats) + _stm(__source__, st, pats; debug=false) +end + +"Like `@stm`, but prints a trace during matching." +macro stm_debug(st, pats) + _stm(__source__, st, pats; debug=true) +end + +# TODO: SyntaxList pattern matching could take similar syntax and use most of +# the same machinery + +function _stm(line::LineNumberNode, st, pats; debug=false) + _stm_check_usage(pats) + # We leave most code untouched, so the user probably wants esc(output) + st_gs, result_gs, k_gs, nc_gs = gensym.("st", "result", "k", "nc") + out_blk = Expr(:let, Expr(:block, :($st_gs = $st::$SyntaxTree), + :($result_gs), + :($k_gs = $kind($st_gs)), + :($nc_gs = $numchildren($st_gs))), + Expr(:if, false, nothing)) + case_list_tail = out_blk.args[2].args + for pcr in pats.args + pcr isa LineNumberNode && (line = pcr; continue) + p, cond, result = _stm_destruct_pat(pcr) + pat_ok = p isa Symbol ? true : _stm_matches(p, st_gs, k_gs, nc_gs, debug) + # We need to let-bind patvars in both cond and the result, so result + # needs to live in the first argument of :if with the extra conditions. + case = Expr(:elseif, + Expr(:&&, pat_ok, + Expr(:let, _stm_assigns(p, st_gs), + Expr(:&&, cond, + Expr(:block, line, + :($result_gs = $result), true)))), + result_gs) + push!(case_list_tail, case) + case_list_tail = case_list_tail[3].args + end + push!(case_list_tail, + :(throw(ErrorException(string( + "No match found for `", $st_gs, "` at ", $(string(line))))))) + return esc(out_blk) +end + +# recursively flatten `vcat` expressions +function _stm_vcat_to_hcat(p::Expr) + if Meta.isexpr(p, :vcat) + out = Expr(:hcat) + for a in p.args + Meta.isexpr(a, :row) ? append!(out.args, a.args) : push!(out.args, a) + end + else + out = Expr(p.head, p.args...) + end + for i in eachindex(out.args) + out.args[i] = _stm_vcat_to_hcat(out.args[i]) + end + return out +end +_stm_vcat_to_hcat(x) = x + +# return (pat_expr, when_expr|nothing, res_expr) +function _stm_destruct_pat(pcr::Expr) + pc, r = pcr.args[1:2] + Base.remove_linenums!(pc) # errors in lhs of `->` are caught in usage check + (p_vcat, c) = Meta.isexpr(pc, :tuple) ? + (pc.args[1], pc.args[2].args[2]) : (pc, true) + return (_stm_vcat_to_hcat(p_vcat), c, r) +end + +function _stm_matches_wrapper(p::Expr, st_ex, debug) + st_gs, k_gs, nc_gs = gensym.("st", "k", "nc") + Expr(:let, Expr(:block, :($st_gs = $st_ex::$SyntaxTree), + :($k_gs = $kind($st_gs)), + :($nc_gs = $numchildren($st_gs))), + _stm_matches(p, st_gs, k_gs, nc_gs, debug)) +end + +function _stm_matches(p::Expr, st_gs::Symbol, k_gs::Symbol, nc_gs::Symbol, debug) + pat_k = Kind(p.args[1].args[3]) + out = Expr(:&&, :($pat_k === $k_gs)) + debug && push!(out.args, Expr(:block, :(printstyled( + string("[kind]: ", $k_gs, "\n"); color=:yellow)), true)) + + p_args = p.args[2:end] + dots_i = findfirst(x->Meta.isexpr(x, :(...)), p_args) + dots_start = something(dots_i, length(p_args) + 1) + n_after_dots = length(p_args) - dots_start # -1 if no dots + + push!(out.args, isnothing(dots_i) ? + :($nc_gs === $(length(p_args))) : + :($nc_gs >= $(length(p_args) - 1))) + debug && push!(out.args, Expr(:block, :(printstyled( + string("[numc]: ", $nc_gs, "\n"); color=:yellow)), true)) + + for i in 1:dots_start-1 + p_args[i] isa Symbol && continue + push!(out.args, + _stm_matches_wrapper(p_args[i], :($st_gs[$i]), debug)) + end + for i in n_after_dots-1:-1:0 + p_args[end-i] isa Symbol && continue + push!(out.args, + _stm_matches_wrapper(p_args[end-i], :($st_gs[end-$i]), debug)) + end + debug && push!(out.args, Expr(:block, :(printstyled( + string("matched: ", $st_gs, " with ", $(QuoteNode(p)), "\n"); + color=:green)), true)) + return out +end + +# Assuming _stm_matches, construct an Expr that assigns syms to SyntaxTrees. +# Note st_rhs_expr is a ref-expr with a SyntaxTree/List value (in context). +function _stm_assigns(p, st_rhs_expr; assigns=Expr(:block)) + if p isa Symbol + p != :_ && push!(assigns.args, Expr(:(=), p, st_rhs_expr)) + return assigns + elseif p isa Expr + p_args = p.args[2:end] + dots_i = findfirst(x->Meta.isexpr(x, :(...)), p_args) + dots_start = something(dots_i, length(p_args) + 1) + n_after_dots = length(p_args) - dots_start + for i in 1:dots_start-1 + _stm_assigns(p_args[i], :($st_rhs_expr[$i]); assigns) + end + if !isnothing(dots_i) + _stm_assigns(p_args[dots_i].args[1], + :($st_rhs_expr[$dots_i:end-$n_after_dots]); assigns) + for i in n_after_dots-1:-1:0 + _stm_assigns(p_args[end-i], :($st_rhs_expr[end-$i]); assigns) + end + end + return assigns + end + @assert false "unexpected syntax; enable or fix `_stm_check_usage`" +end + +# Check for correct pattern syntax. Not needed outside of development. +function _stm_check_pattern(p, syms::Set{Symbol}) + if Meta.isexpr(p, :(...), 1) + p = p.args[1] + @assert(p isa Symbol, "Expected symbol before `...` in $p") + end + if p isa Symbol + # No support for duplicate syms for now (user is either looking for + # some form of equality we don't implement, or they made a mistake) + dup = p in syms && p !== :_ + push!(syms, p) + @assert(!dup, "invalid duplicate non-underscore identifier $p") + return nothing + elseif Meta.isexpr(p, :vect) + @assert(length(p.args) === 1, + "use spaces, not commas, in @stm []-patterns") + elseif Meta.isexpr(p, :hcat) + @assert(length(p.args) >= 2) + elseif Meta.isexpr(p, :vcat) + p = _stm_vcat_to_hcat(p) + @assert(length(p.args) >= 2) + else + @assert(false, "malformed pattern $p") + end + @assert(count(x->Meta.isexpr(x, :(...)), p.args[2:end]) <= 1, + "Multiple `...` in a pattern is ambiguous") + + # This exact `K"kind"` syntax is not necessary since the kind can't be + # provided by a variable, but requiring [K"kinds"] is consistent with + # `@ast` and allows us to implement list matching later. + @assert(Meta.isexpr(p.args[1], :macrocall, 3) && + p.args[1].args[1] === Symbol("@K_str") && + p.args[1].args[3] isa String, "first pattern elt must be K\"\"") + + for subp in p.args[2:end] + _stm_check_pattern(subp, syms) + end + return nothing +end + +function _stm_check_usage(pats::Expr) + @assert Meta.isexpr(pats, :block) "Usage: @stm st begin; ...; end" + for pcr in pats.args + pcr isa LineNumberNode && continue + @assert(Meta.isexpr(pcr, :(->), 2), "Expected pat -> res, got malformed case: $pcr") + if Meta.isexpr(pcr.args[1], :tuple) + @assert(length(pcr.args[1].args) === 2, + "Expected `pat` or `(pat, when=cond)`, got $(pcr.args[1])") + p = pcr.args[1].args[1] + c = pcr.args[1].args[2] + @assert(Meta.isexpr(c, :(=), 2) && c.args[1] === :when, + "Expected `(when=cond)` in tuple pattern, got $(c)") + else + p = pcr.args[1] + end + _stm_check_pattern(p, Set{Symbol}()) + end +end + +#------------------------------------------------------------------------------- +# RawGreenNode->SyntaxTree1 + +function build_tree(::Type{SyntaxTree}, stream::ParseStream; + filename=nothing, first_line=1) + cursor = RedTreeCursor(stream) + sf = Ref(SourceFile(stream; filename, first_line)) + source = SourceRef(sf, first_byte(stream), last_byte(stream)) + cs = SyntaxList() + for c in reverse_toplevel_siblings(cursor) + is_trivia(c) && !is_error(c) && continue + push!(cs, SyntaxTree(sf, c)) + end + # There may be multiple non-trivia toplevel nodes (e.g. parse error) + length(cs) === 1 && return only(cs) + id = SyntaxTree(K"wrapper", reverse(cs), nothing, source, nothing) + return id +end + +function SyntaxTree(sf::Base.RefValue{SourceFile}, cursor::RedTreeCursor) + green_id = GC.@preserve sf begin + raw_offset, txtbuf = _unsafe_wrap_substring(sf[].code) + offset = raw_offset - sf[].byte_offset + _insert_green(sf, txtbuf, offset, cursor) + end + gst = green_id + out = _green_to_est(gst, 0, gst) + @assert !isnothing(out) "SyntaxTree requires >0 nontrivia nodes" + return out +end + +function _insert_green(sf::Base.RefValue{SourceFile}, + txtbuf::Vector{UInt8}, offset::Int, + cursor::RedTreeCursor) + source = SourceRef(sf, first_byte(cursor), last_byte(cursor)) + id = SyntaxTree(kind(cursor), nothing, nothing, source, nothing) + let f = remove_flags(flags(cursor), NON_TERMINAL_FLAG) + f != 0 && _setattr!(id, :syntax_flags, f) + end + if !is_leaf(cursor) + cs = SyntaxList() + for c in reverse(cursor) + push!(cs, _insert_green(sf, txtbuf, offset, c)) + end + setchildren!(id, reverse!(cs)) + else + v = parse_julia_literal(txtbuf, head(cursor), byte_range(cursor) .+ offset) + if v isa Symbol + # TODO: Fixes in JuliaSyntax to avoid ever converting to Symbol + _setattr!(id, :value, string(v)) + elseif !isnothing(v) + _setattr!(id, :value, v) + end + end + return id +end + +""" +Convert green `st` to a SyntaxTree with Expr structure. `parent_i` is the final +position of `convert(st)` (our return value) within `convert(parent)`. If +`parent_i == 0`, neither it nor our `parent` are known or relevant to this +conversion. + +We can't assume much about `st` since it's anything the parser produces. Our +correctness is defined against existing text->Expr transformations. + +All node rearrangements and head changes are determined before recursing on +children, unlike in `node_to_expr`. This is because knowing our parent's kind +and our position within it ahead-of-time makes conversion simpler. By default, +for each node `st`, we + 1. let `cs` be `children(st)` minus (non-recursively) all trivia and parens + 2. rearrange `cs` based on length(cs), their/our/parent's kind/flags, etc. + 3. let `ret_cs` be `map(convert, cs)` + 4. return our new node `convert(st)` with `ret_cs` as children. +However, we can stop and return an answer between any of these steps. For +example, deleting a child is easy in (2), but new non-leaf children we insert +should be added to `ret_cs` rather than `cs` (unless the new child has +pre-transformation structure and we're OK with step 3 creating it again). +""" +function _green_to_est(parent::SyntaxTree, parent_i::Int, + st::SyntaxTree; kw_in_params=false) + if !should_include_node(st) + @assert kind(parent) === K"None" && parent_i === 0 + return nothing + end + + k = kind(st) + syntax_name(x) = x.value::String + symleaf(s::String) = _setattr!(newleaf(st, K"Identifier"), :value, s) + core_globalref(s::String) = _setattr!(symleaf(s), :mod, Core) + valleaf(@nospecialize(v)) = _setattr!(newleaf(st, K"Value"), :value, v) + + if k === K"DotsIdentifier" + # `..`/`...` used as an ordinary identifier (eg the `..` operator, or + # `...` quoted as in `:(...)`). The dots are held as trivia children, so + # this is not a leaf; represent it as a plain identifier named by the + # dots themselves (the dot count is stored in the numeric flags). + return symleaf(repeat('.', numeric_flags(st))) + end + + if is_leaf(st) + return if k === K"CmdMacroName" || k === K"StrMacroName" + name = lower_identifier_name(syntax_name(st), k) + symleaf(name) + elseif k === K"VERSION" + valleaf(version_to_expr(st)) + elseif (v = st.value; v isa Union{Int128,UInt128,BigInt}) + # syntax TODO: likely unnecessary; this is just to match RGN->Expr, + # which added this to match flisp parsing text->Expr. + macname = v isa Int128 ? "@int128_str" : + v isa UInt128 ? "@uint128_str" : "@big_str" + mac = core_globalref(macname) + arg = valleaf(replace(sourcetext(st), '_'=>"")) + ret_cids = SyntaxList(mac, valleaf(nothing), arg) + newnode(st, K"macrocall", ret_cids) + elseif is_error(k) + mkleaf(st) + elseif st.value isa String && !(k in KSet"Identifier Value" || is_literal(k)) + # certain kinds should really be identifiers. known: &, |, : + symleaf(syntax_name(st)) + else + st + end + end + + # Non-leaf cases: each branch should either set `ret_k` and `cs` or recurse + # manually and return a finished SyntaxTree + ret_k::Kind = k + cs = preprocessed_green_children(st) + n_cs = length(cs) + + if k === K"string" && n_cs > 0 + return _string_to_est(st, cs; unwrap_literal=true) + elseif k === K"cmdstring" && n_cs > 0 + # (cmdstring _...) => (macrocall Core.@cmd lno joined_str) + cmd_arg = _string_to_est(st, cs; unwrap_literal=true) + loc_st = valleaf(source_location(LineNumberNode, st)) + return newnode(st, K"macrocall", SyntaxList( + core_globalref("@cmd"), loc_st, cmd_arg)) + elseif k === K"macro_name" && n_cs === 1 + # "M.@x" => (. M (macro_name x)) => (. M @x) + # "@M.x" => (macro_name (. M x)) => (. M @x) + # (macro_name else) => else + if kind(cs[1]) === K"Identifier" + return symleaf(lower_identifier_name(syntax_name(cs[1]), K"macro_name")) + else + inner_st = cs[1] + inner_cs = preprocessed_green_children(inner_st) + if (length(inner_cs) === 2 && kind(inner_st) === K"." && + kind(inner_cs[2]) === K"Identifier") + (lhs, raw_m) = _green_to_est(cs[1], 1, inner_cs[1]), inner_cs[2] + mname_s = lower_identifier_name(syntax_name(raw_m), K"macro_name") + mname = _setattr!(mkleaf(raw_m), :value, mname_s) + mname_inert = newnode(raw_m, K"inert", SyntaxList(mname)) + return mknode(inner_st, SyntaxList(lhs, mname_inert)) + else + return _green_to_est(parent, 1, inner_st) + end + end + elseif k === K"?" + ret_k = K"if" + elseif k === K"op=" && n_cs === 3 + # (op= a + b) => (+= a b) + # (.op= a + b) => (.+= a b) below + op_s = string(cs[2]) * '=' + lhs = _green_to_est(st, 0, cs[1]) + rhs = _green_to_est(st, 0, cs[3]) + out = newnode(st, K"unknown_head", SyntaxList(lhs, rhs)) + return _setattr!(out, :value, op_s) + elseif k === K".op=" && n_cs === 3 + op_s = '.' * string(cs[2]) * '=' + lhs = _green_to_est(st, 0, cs[1]) + rhs = _green_to_est(st, 0, cs[3]) + out = newnode(st, K"unknown_head", SyntaxList(lhs, rhs)) + return _setattr!(out, :value, op_s) + elseif k === K"op=" && n_cs === 1 + # (op= +) => += (the operator name itself, eg when quoted as `:(+=)`) + return symleaf(string(cs[1]) * '=') + elseif k === K".op=" && n_cs === 1 + # (.op= +) => .+= + return symleaf('.' * string(cs[1]) * '=') + elseif k === K"macrocall" && n_cs > 0 + # LineNumberNodes are not usually added to the tree as they are in Expr, + # but this specifically inserts the macrocall child for compatibility + loc_st = let loc = source_location(LineNumberNode, st) + if n_cs >= 2 && kind(cs[2]) === K"VERSION" + v = version_to_expr(popat!(cs, 2)) + @static if isdefined(Core, :MacroSource) + loc = Core.MacroSource(loc, v) + end + end + valleaf(loc) + end + insert!(cs, 2, loc_st) + # foo`x` parses to (macrocall foo::CmdMacroName (cmdstring ::CmdString)) + # so we need to unwrap the CmdString or else we get two macrocalls + if n_cs >= 2 && kind(cs[1]) === K"CmdMacroName" + ret_cs = _map_green_to_est(st, cs) + ret_cs[3] = ret_cs[3][3] # node leak + return mknode(st, ret_cs) + end + do_ex = kind(cs[end]) === K"do" ? pop!(cs) : nothing + _reorder_parameters!(cs, 3) + !isnothing(do_ex) && return _make_do_expression(st, cs, do_ex) + elseif k === K"doc" + # (doc str obj) => (macrocall Core.@doc lno str obj) + ret_k = K"macrocall" + pushfirst!(cs, valleaf(source_location(LineNumberNode, st))) + pushfirst!(cs, core_globalref("@doc")) + elseif k === K"dotcall" || k === K"call" && n_cs > 0 + if is_infix_op_call(st) || is_postfix_op_call(st) + cs[2], cs[1] = cs[1], cs[2] + end + if is_postfix_op_call(st) && kind(cs[1]) == K"Identifier" && + syntax_name(cs[1]) === "'" + popfirst!(cs) + ret_k = K"'" + end + do_ex = kind(cs[end]) === K"do" ? pop!(cs) : nothing + _reorder_parameters!(cs, 2) + if k === K"dotcall" + if is_prefix_call(st) + # (dotcall f args...) => (. f (tuple args...)) + ret_cs = _map_green_to_est(st, cs) + tuple = newnode(st, K"tuple", ret_cs[2:end]) + return newnode(st, K".", SyntaxList(ret_cs[1], tuple)) + else + # (dotcall + args...) => (call .+ args...) + ret_k = K"call" + if kind(cs[1]) === K"Identifier" + cs[1] = symleaf('.' * syntax_name(cs[1])) + end + end + end + !isnothing(do_ex) && return _make_do_expression(st, cs, do_ex) + elseif k === K"." + if n_cs === 2 + # (. lhs rhs) => (. lhs (inert rhs)) + lhs = _green_to_est(st, 1, cs[1]) + rhs = _green_to_est(st, 2, cs[2]) + inert_rhs = kind(rhs) in KSet"quote inert" ? rhs : + newnode(cs[2], K"inert", SyntaxList(rhs)) + return mknode(st, SyntaxList(lhs, inert_rhs)) + elseif n_cs === 1 + # (. x) => (. x) or .x + # TODO: This is the one place where K"parens" change the result, + # meaning that either Expr is doing something wrong or SyntaxNode is + # deleting semantics. + paren_st = filter(should_include_node, children(parent))[1] + coalesce_dot = !(kind(paren_st) === K"parens") && parent_i === 1 && + kind(parent) in KSet"call dotcall curly quote" + + if (coalesce_dot || is_syntactic_operator(kind(cs[1])) || + kind(parent) === K"comparison" && iseven(parent_i)) + return symleaf('.' * syntax_name(cs[1])) + end + end + elseif k === K"ref" || k === K"curly" + _reorder_parameters!(cs, 2) + elseif k === K"for" && n_cs === 2 + # (for (iteration iter1) body) => (for iter1 body) + iters = preprocessed_green_children(cs[1]) + if length(iters) === 1 + cs[1] = iters[1] + end + elseif k === K"iteration" + # (for (iteration iter1 iters...) body) => (for (block iter1 iters...) body) + @assert kind(parent) === K"for" && parent_i === 1 + ret_k = K"block" + elseif k === K"vect" || k === K"braces" + _reorder_parameters!(cs, 1) + elseif k === K"tuple" + # Unwrap singleton, no-trailing-comma tuple in a couple cases: + # (function (tuple (... xs)) body) => (function (... xs) body) + # (-> (tuple _) body) => (-> _ body), assuming _ not parameters + if n_cs === 1 && parent_i === 1 && + !has_flags(st, TRAILING_COMMA_FLAG) + p_k = kind(parent) + c_k = kind(cs[1]) + if (p_k === K"function" && c_k === K"...") || + (p_k === K"->" && c_k !== K"parameters") + return _green_to_est(parent, parent_i, cs[1]) + end + elseif n_cs === 2 && kind(parent) === K"->" && parent_i === 1 && + kind(cs[2]) === K"parameters" && kind(cs[1]) !== K"..." + # This case should really be deleted. + # (-> (tuple x (parameters y)) _) => (-> (block x y) _) + c2_cs = preprocessed_green_children(cs[2]) + if length(c2_cs) === 0 + ret_k = K"block" + pop!(cs) + elseif length(c2_cs) === 1 + ret_k = K"block" + cs[2] = c2_cs[1] + end + end + _reorder_parameters!(cs, 1) + elseif k === K"where" && n_cs === 2 + # (where lhs (braces a b c)) => (where lhs a b c) + if kind(cs[2]) === K"braces" + rhs = pop!(cs) + append!(cs, preprocessed_green_children(rhs)) + _reorder_parameters!(cs, 2) + end + elseif k === K"try" + # anything => (try try_block e catch_block [finally_block] [else_block]) + try_ = cs[1] + st_false = valleaf(false) + catch_var = catch_ = else_ = finally_ = st_false + for c in cs[2:end] + inner_cs = preprocessed_green_children(c) + if kind(c) === K"catch" + if kind(inner_cs[1]) !== K"Placeholder" + catch_var = inner_cs[1] + end + catch_ = inner_cs[2] + elseif kind(c) === K"else" + else_ = only(inner_cs) + elseif kind(c) === K"finally" + finally_ = only(inner_cs) + elseif is_error(kind(c)) + return mknode(st, cs) # give up + else + @assert false "Illegal subclause in `try`" + end + end + empty!(cs) + push!(cs, try_, catch_var, catch_) + if finally_ != st_false || else_ != st_false + push!(cs, finally_) + if else_ != st_false + push!(cs, else_) + end + end + elseif k === K"generator" && n_cs >= 2 + # let (g2 x iter) mean (generator x iter.children...) + # (generator val iter_1 ... iter_n) => + # (flatten (g2 (... (flatten (g2 (g2 val i_n) i_{n-1})) ...) i_1)) + g_out = _green_to_est(st, 1, popfirst!(cs)) + for c in Iterators.reverse(cs) + gen_cs = let rest = kind(c) === K"iteration" ? + preprocessed_green_children(c) : SyntaxList(c) + rest = _map_green_to_est(st, rest; undef_parent=true) + pushfirst!(rest, g_out) + end + g_out = mknode(st, gen_cs) + if c !== cs[end] + g_out = newnode(c, K"flatten", SyntaxList(g_out)) + end + end + return _setattr!(g_out, :source, st) # outermost provenance + elseif k === K"filter" + @assert n_cs === 2 + # (filter (iteration is...) cond) => (filter cond is...) + cond = pop!(cs) + cs = preprocessed_green_children(cs[1]) + pushfirst!(cs, cond) + elseif k === K"in" + ret_k = K"=" + elseif k === K"nrow" || k === K"ncat" + pushfirst!(cs, valleaf(numeric_flags(flags(st)))) + elseif k === K"typed_ncat" + insert!(cs, 2, valleaf(numeric_flags(flags(st)))) + elseif k === K"elseif" + # (elseif cond body) => (elseif (block cond) body) + # RGN->Expr block-wraps for linenodes; we do it for parity + ret_cs = _map_green_to_est(st, cs) + ret_cs[1] = newnode(cs[1], K"block", SyntaxList(ret_cs[1])) + return mknode(st, ret_cs) + elseif k === K"->" && kind(cs[2]) !== K"block" + ret_cs = _map_green_to_est(st, cs) + ret_cs[2] = newnode(cs[2], K"block", SyntaxList(ret_cs[2])) + return mknode(st, ret_cs) + elseif k === K"function" && n_cs >= 2 && + has_flags(st, SHORT_FORM_FUNCTION_FLAG) + # (function-= callex body) => (= callex (block body)) + # exception: no block on "x' = y", or if body is already a block + if kind(cs[2]) !== K"block" && !is_postfix_op_call(cs[1]) + ret_cs = _map_green_to_est(st, cs) + ret_cs[2] = newnode(cs[2], K"block", SyntaxList(ret_cs[2])) + return newnode(st, K"=", ret_cs) + end + ret_k = K"=" + elseif k === K"module" + not_bare = valleaf(!has_flags(st, BARE_MODULE_FLAG)) + insert!(cs, kind(cs[1]) === K"VERSION" ? 2 : 1, not_bare) + elseif k === K"quote" && n_cs === 1 + # (quote something_simple) => (inert something_simple) + ret_c = _green_to_est(st, 1, cs[1]) + return is_leaf(ret_c) && kind(ret_c) !== K"Bool" ? + newnode(st, K"inert", SyntaxList(ret_c)) : + mknode(st, SyntaxList(ret_c)) + elseif k === K"do" + ret_k = K"->" + elseif k === K"block" + # (let (block x) _...) => (let x _...) + # (let (block (= x y)) _...) => (let (= x y) _...) + # (let (block (:: x y)) _...) => (let (:: x y) _...) + # (struct _ (block (doc "foo" field1) (doc "bar" field2))) => + # (struct _ (block "foo" field1 "bar" field2)) + if kind(parent) === K"let" && parent_i === 1 && n_cs === 1 + out = _green_to_est(st, 1, cs[1]) + return kind(out) in KSet"Identifier = ::" ? out : + mknode(st, SyntaxList(out)) + elseif kind(parent) === K"struct" && parent_i === 3 + cs_tmp = SyntaxList() + for c in cs + kind(c) === K"doc" ? + append!(cs_tmp, preprocessed_green_children(c)) : + push!(cs_tmp, c) + end + cs = cs_tmp + end + elseif (k === K"local" || k === K"global") && n_cs === 1 + # (local (const _)) => (const (local _)) + # (local (tuple a b c)) => (local a b c) + if kind(cs[1]) === K"const" + ret_c1_cs = _map_green_to_est(st, preprocessed_green_children(cs[1])) + ret_cs = SyntaxList(mknode(st, ret_c1_cs)) + return mknode(cs[1], ret_cs) + elseif kind(cs[1]) === K"tuple" + cs = preprocessed_green_children(cs[1]) + end + elseif k === K"return" && n_cs === 0 + push!(cs, valleaf(nothing)) + elseif k === K"juxtapose" + ret_k = K"call" + pushfirst!(cs, symleaf("*")) + elseif k === K"struct" + is_mutable = valleaf(has_flags(st, MUTABLE_FLAG)) + pushfirst!(cs, is_mutable) + elseif k === K"importpath" + ret_k = K"." + for i in eachindex(cs) + if kind(cs[i]) === K"inert" + inner_cs = preprocessed_green_children(cs[i]) + length(inner_cs) === 1 && (cs[i] = only(inner_cs)) + end + end + elseif k === K"wrapper" # parse errors only + ret_k = K"block" + elseif k === K"parameters" + kw_in_params = kind(parent) === K"parameters" && parent_i === 1 ? + kw_in_params : !(kind(parent) in KSet"vect curly braces ref") + elseif k === K"=" + p_k = kind(parent) + because_params = p_k === K"parameters" && parent_i >= 1 && kw_in_params + because_call = parent_i > 1 && (p_k == K"ref" || + p_k in KSet"call dotcall" && is_prefix_call(parent)) + ret_k = because_params || because_call ? K"kw" : K"=" + elseif k in KSet"var char parens" && n_cs === 1 + # Reachable if this is the top node + return _green_to_est(parent, parent_i, cs[1]) + end + + # Recurse on `cs`. If no children change, just return `st`. + ret_cs = _map_green_to_est(st, cs; kw_in_params) + return ret_cs == children(st) && ret_k == kind(st) ? + st : newnode(st, ret_k, ret_cs) +end + +function _map_green_to_est(parent::SyntaxTree, cs; + kw_in_params=false, undef_parent=false) + ret_cs = SyntaxList() + for (i, c) in enumerate(cs) + new_c = _green_to_est(parent, undef_parent ? 0 : i, c; kw_in_params) + @assert should_include_node(new_c) + push!(ret_cs, new_c) + end + ret_cs +end + +# When converting, first delete trivia and wrapper nodes in children so we can +# observe child kinds before recursing, thus creating fewer "temporary" nodes +function preprocessed_green_children(st::SyntaxTree) + cs = filter(should_include_node, children(st)) + for i in eachindex(cs) + while kind(cs[i]) in KSet"var char parens" + inner_cs = preprocessed_green_children(cs[i]) + if length(inner_cs) === 1 + cs[i] = inner_cs[1] + else + break + end + end + end + return cs +end + +# (call f a b (parameters c d) (parameters e)) => +# (call f (parameters (parameters e) c d) a b) +function _reorder_parameters!(cs::SyntaxList, params_pos::Int) + (length(cs) > params_pos && kind(cs[end]) === K"parameters") || return cs + local param_ball = pop!(cs) + while length(cs) >= 1 && kind(cs[end]) === K"parameters" + next_ball_cs = pushfirst!(copy(children(cs[end])), param_ball) + # `mknode` leaks nodes, but having multiple `parameters` blocks is + # extremely rare nonsense syntax (`f(a,b;c=d;e)`) + param_ball = mknode(cs[end], next_ball_cs) + pop!(cs) + end + insert!(cs, params_pos, param_ball) + nothing +end + +# (call args... (do _...)) -> (do (call args...) (-> _...)) +# +# Expects preprocessed and rearranged `args` +function _make_do_expression(st::SyntaxTree, args::SyntaxList, doex::SyntaxTree) + ret_doex = _green_to_est(st, 0, doex) + ret_callex = mknode(st, _map_green_to_est(st, args)) + return newnode(st, K"do", SyntaxList(ret_callex, ret_doex)) +end + +# A `string` or `cmdstring` may have multiple literal strings within (from +# newlines when triple-quoting). A `string` may have interpolated values. +# +# (string "a" "b" "c") => "abc" # unwrap_literal=true +# (string "a" "b" "c" 1) => (string "abc" 1) +# (string "a" "b" (string "c" "d")) => (string "ab" (string "cd")) +# +# (cmdstring "a"::CmdString "b"::CmdString) => "ab"::CmdString +# +# Converting children-first (as _string_to_Expr does) would make this much +# harder by converting literal strings without the parent's knowledge +function _string_to_est(st::SyntaxTree, cs::SyntaxList; unwrap_literal) + ret_cs = SyntaxList() + literal_k = kind(st) === K"cmdstring" ? K"CmdString" : K"String" + cur_str = false + next_str = length(cs) > 0 && kind(cs[1]) === literal_k + buf = IOBuffer() + for i in eachindex(cs) + c = cs[i] + (prev_str, cur_str) = (cur_str, next_str) + next_str = i != lastindex(cs) && kind(cs[i+1]) === literal_k + # optimization: push the current child mostly unchanged if the following + # one isn't a literal string + if !prev_str && cur_str && !next_str + push!(ret_cs, c) + elseif cur_str + write(buf, c.value) + if !next_str + ret_c = newleaf(st, literal_k) + _setattr!(ret_c, :value, String(take!(buf))) + push!(ret_cs, ret_c) + end + else + ret_c = kind(c) === K"string" ? + _string_to_est(c, preprocessed_green_children(c); + unwrap_literal=false) : + _green_to_est(st, i, c) + + push!(ret_cs, ret_c) + end + end + if unwrap_literal && length(ret_cs) === 1 && kind(ret_cs[1]) === literal_k + return _setattr(ret_cs[1], :source, st) + end + return mknode(st, ret_cs) +end diff --git a/JuliaSyntax/src/porcelain/syntax_node.jl b/JuliaSyntax/src/porcelain/syntax_node.jl new file mode 100644 index 0000000000000..30f2371a08622 --- /dev/null +++ b/JuliaSyntax/src/porcelain/syntax_node.jl @@ -0,0 +1,347 @@ +#------------------------------------------------------------------------------- +# AST interface, built on top of raw tree + +abstract type AbstractSyntaxData end + +mutable struct TreeNode{NodeData} # ? prevent others from using this with NodeData <: AbstractSyntaxData? + parent::Union{Nothing,TreeNode{NodeData}} + children::Union{Nothing,Vector{TreeNode{NodeData}}} + data::Union{Nothing,NodeData} + + # Use this constructor rather than the automatically generated one to pass + # Test.detect_unbound_args() test in Base. + function TreeNode{NodeData}(parent::Union{Nothing,TreeNode{NodeData}}, + children::Union{Nothing,Vector{TreeNode{NodeData}}}, + data::Union{Nothing,NodeData}) where {NodeData} + new{NodeData}(parent, children, data) + end +end + +# Exclude parent from hash and equality checks. This means that subtrees can compare equal. +function Base.hash(node::TreeNode, h::UInt) + h = hash(node.data, h) + children = node.children + if children === nothing + return hash(nothing, h) + else # optimization - avoid extra allocations from `hash(::AbstractVector, ::UInt)` + for child in children + h = hash(child, h) + end + return h + end +end +function Base.:(==)(a::TreeNode{T}, b::TreeNode{T}) where T + a.children == b.children && a.data == b.data +end + +# Implement "pass-through" semantics for field access: access fields of `data` +# as if they were part of `TreeNode` +function Base.getproperty(node::TreeNode, name::Symbol) + name === :parent && return getfield(node, :parent) + name === :children && return getfield(node, :children) + d = getfield(node, :data) + name === :data && return d + return getproperty(d, name) +end + +function Base.setproperty!(node::TreeNode, name::Symbol, x) + name === :parent && return setfield!(node, :parent, x) + name === :children && return setfield!(node, :children, x) + name === :data && return setfield!(node, :data, x) + d = getfield(node, :data) + return setfield!(d, name, x) +end + +const AbstractSyntaxNode = TreeNode{<:AbstractSyntaxData} + +struct SyntaxData <: AbstractSyntaxData + source::SourceFile + raw::GreenNode{SyntaxHead} + byte_end::UInt32 + val::Any +end +function Base.getproperty(data::SyntaxData, name::Symbol) + if name === :position + # Previous versions of JuliaSyntax had `position::Int`. + # Allow access for compatibility. It was renamed (with changed semantics) + # to `byte_end::UInt32` to match the rest of the code base, which identified + # nodes, by their last byte. + return Int(getfield(data, :byte_end) - getfield(data, :raw).span + UInt32(1)) + end + return getfield(data, name) +end + +Base.hash(data::SyntaxData, h::UInt) = + hash(data.source, hash(data.raw, hash(data.byte_end, + # Avoid dynamic dispatch: + # This does not support custom `hash` implementation that may be defined for `typeof(data.val)`, + # However, such custom user types should not generally appear in the AST. + Core.invoke(hash, Tuple{Any,UInt}, data.val, h)))) +function Base.:(==)(a::SyntaxData, b::SyntaxData) + a.source == b.source && a.raw == b.raw && a.byte_end == b.byte_end && a.val === b.val +end + +""" + SyntaxNode(source::SourceFile, cursor::RedTreeCursor, + raw::GreenNode{SyntaxHead}; keep_parens=false) + +A pointer-y AST constructed by removing all trivia from the raw parser output +(see `RawGreenNode`). The layout of `SyntaxNode` is different from that +of `Expr` due to the invariant that each node's children are in the order they +appeared in the source text. +""" +const SyntaxNode = TreeNode{SyntaxData} + +function SyntaxNode(source::SourceFile, cursor::RedTreeCursor; + keep_parens=false) + # Build the full GreenNode tree once upfront (including trivia) + green = GreenNode(cursor.green) + + GC.@preserve source begin + raw_offset, txtbuf = _unsafe_wrap_substring(source.code) + offset = raw_offset - source.byte_offset + _to_SyntaxNode(source, txtbuf, offset, cursor, green, keep_parens) + end +end + +function SyntaxNode(source::SourceFile, cursor::RedTreeCursor, green::GreenNode{SyntaxHead}; + keep_parens=false) + GC.@preserve source begin + raw_offset, txtbuf = _unsafe_wrap_substring(source.code) + offset = raw_offset - source.byte_offset + _to_SyntaxNode(source, txtbuf, offset, cursor, green, keep_parens) + end +end + +should_include_node(child) = !is_trivia(child) || is_error(child) + +function _to_SyntaxNode(source::SourceFile, txtbuf::Vector{UInt8}, offset::Int, + cursor::RedTreeCursor, green::GreenNode{SyntaxHead}, keep_parens::Bool) + if is_leaf(cursor) + # Here we parse the values eagerly rather than representing them as + # strings. Maybe this is good. Maybe not. + valrange = byte_range(cursor) + val = parse_julia_literal(txtbuf, head(cursor), valrange .+ offset) + return SyntaxNode(nothing, nothing, SyntaxData(source, green, cursor.byte_end, val)) + else + cs = SyntaxNode[] + green_children = children(green) + + # We need to match up the filtered SyntaxNode children with the unfiltered GreenNode children + # Both cursor and green children need to be traversed in the same order + # Since cursor iterates in reverse, we need to match from the end of green_children + green_children === nothing ? 0 : length(green_children) + + for (i, child_cursor) in enumerate(reverse(cursor)) + if should_include_node(child_cursor) + pushfirst!(cs, _to_SyntaxNode(source, txtbuf, offset, child_cursor, green[end-i+1], keep_parens)) + end + end + + if !keep_parens && kind(cursor) == K"parens" && length(cs) == 1 + return cs[1] + end + node = SyntaxNode(nothing, cs, SyntaxData(source, green, cursor.byte_end, nothing)) + for c in cs + c.parent = node + end + return node + end +end + +""" + is_leaf(node) + +Determine whether the node is a leaf of the tree. In our trees a "leaf" +corresponds to a single token in the source text. +""" +is_leaf(node::TreeNode) = node.children === nothing + +""" + children(node) + +Return an iterable list of children for the node. For leaves, return `nothing`. +""" +children(node::TreeNode) = node.children + +""" + numchildren(node) + +Return `length(children(node))` but possibly computed in a more efficient way. +""" +numchildren(node::TreeNode) = (isnothing(node.children) ? 0 : length(node.children)) + +Base.getindex(node::AbstractSyntaxNode, i::Int) = children(node)[i] +Base.getindex(node::AbstractSyntaxNode, rng::UnitRange) = view(children(node), rng) +Base.firstindex(::AbstractSyntaxNode) = 1 +Base.length(node::AbstractSyntaxNode) = length(children(node)) +Base.lastindex(node::AbstractSyntaxNode) = length(node) + +function Base.setindex!(node::SN, x::SN, i::Int) where {SN<:AbstractSyntaxNode} + children(node)[i] = x +end + +""" + head(x) + +Get the [`SyntaxHead`](@ref) of a node of a tree or other syntax-related data +structure. +""" +head(node::AbstractSyntaxNode) = head(node.raw) + +span(node::AbstractSyntaxNode) = node.raw.span + +byte_range(node::AbstractSyntaxNode) = (node.byte_end - span(node) + 1):node.byte_end + +first_byte(node::AbstractSyntaxNode) = first(byte_range(node)) +last_byte(node::AbstractSyntaxNode) = last(byte_range(node)) + +sourcefile(node::AbstractSyntaxNode) = node.source + +function leaf_string(ex) + if !is_leaf(ex) + throw(ArgumentError("leaf_string should be used for leaf nodes only")) + end + k = kind(ex) + value = ex.val + # TODO: Dispatch on kind extension module (??) + return k == K"Placeholder" ? "□"*string(value) : + is_identifier(k) ? string(value) : + value isa Symbol ? string(value) : # see parse_julia_literal for other cases which go here + repr(value) +end + +function _show_syntax_node(io, current_filename, node::AbstractSyntaxNode, + indent, show_location, show_kind) + line, col = source_location(node) + if show_location + fname = filename(node) + # Add filename if it's changed from the previous node + if fname != current_filename[] + println(io, indent, " -file- │ ", repr(fname)) + current_filename[] = fname + end + posstr = "$(lpad(line, 4)):$(rpad(col,3))│$(lpad(first_byte(node),6)):$(rpad(last_byte(node),6))│" + else + posstr = "" + end + nodestr = is_leaf(node) ? leaf_string(node) : "[$(untokenize(head(node)))]" + treestr = string(indent, nodestr) + if show_kind && is_leaf(node) + treestr = rpad(treestr, 40)*" :: "*string(kind(node)) + end + println(io, posstr, treestr) + if !is_leaf(node) + new_indent = indent*" " + for n in children(node) + _show_syntax_node(io, current_filename, n, new_indent, show_location, show_kind) + end + end +end + +function _show_syntax_node_sexpr(io, node::AbstractSyntaxNode, show_kind) + if is_leaf(node) + if is_error(node) + print(io, "(", untokenize(head(node)), ")") + else + str = leaf_string(node) + k = kind(node) + if is_identifier(k) && !show_kind + str = lower_identifier_name(str, k) + end + print(io, str) + if show_kind + print(io, "::", kind(node)) + end + end + else + print(io, "(", untokenize(head(node))) + for n in children(node) + print(io, ' ') + _show_syntax_node_sexpr(io, n, show_kind) + end + print(io, ')') + end +end + +function Base.show(io::IO, ::MIME"text/plain", node::AbstractSyntaxNode; show_location=false, show_kind=true) + println(io, "SyntaxNode:") + if show_location + println(io, "line:col│ byte_range │ tree") + end + _show_syntax_node(io, Ref(""), node, "", show_location, show_kind) +end + +function Base.show(io::IO, ::MIME"text/x.sexpression", node::AbstractSyntaxNode; show_kind=false) + _show_syntax_node_sexpr(io, node, show_kind) +end + +function Base.show(io::IO, node::AbstractSyntaxNode) + _show_syntax_node_sexpr(io, node, false) +end + +function Base.push!(node::SN, child::SN) where SN<:AbstractSyntaxNode + if is_leaf(node) + error("Cannot add children") + end + args = children(node) + push!(args, child) +end + +function Base.copy(node::TreeNode) + # copy the container but not the data (ie, deep copy the tree, shallow copy the data). copy(::Expr) is similar + # copy "un-parents" the top-level `node` that you're copying + newnode = typeof(node)(nothing, is_leaf(node) ? nothing : typeof(node)[], copy(node.data)) + if !is_leaf(node) + for child in children(node) + newchild = copy(child) + newchild.parent = newnode + push!(newnode, newchild) + end + end + return newnode +end + +# shallow-copy the data +Base.copy(data::SyntaxData) = SyntaxData(data.source, data.raw, data.byte_end, data.val) + +function build_tree(::Type{SyntaxNode}, stream::ParseStream; + filename=nothing, first_line=1, keep_parens=false) + source = SourceFile(stream, filename=filename, first_line=first_line) + cursor = RedTreeCursor(stream) + if has_toplevel_siblings(cursor) + # There are multiple toplevel nodes, e.g. because we're using this + # to test a partial parse. Wrap everything in K"wrapper" + + # First build the full green tree for all children (including trivia) + green_children = GreenNode{SyntaxHead}[] + for child in reverse_toplevel_siblings(cursor) + pushfirst!(green_children, GreenNode(child.green)) + end + + # Create a wrapper GreenNode with children + green = GreenNode(SyntaxHead(K"wrapper", NON_TERMINAL_FLAG), + stream.next_byte-1, green_children) + + # Now build SyntaxNodes, iterating through cursors and green nodes together + cs = SyntaxNode[] + for (i, child) in enumerate(reverse_toplevel_siblings(cursor)) + if should_include_node(child) + pushfirst!(cs, SyntaxNode(source, child, green[end-i+1], keep_parens=keep_parens)) + end + end + + length(cs) == 1 && return only(cs) + + node = SyntaxNode(nothing, cs, SyntaxData(source, green, + stream.next_byte-1, nothing)) + for c in cs + c.parent = node + end + return node + else + return SyntaxNode(source, cursor, keep_parens=keep_parens) + end +end + +@deprecate haschildren(x) !is_leaf(x) false diff --git a/JuliaSyntax/src/precompile.jl b/JuliaSyntax/src/precompile.jl new file mode 100644 index 0000000000000..9fb71c74d7132 --- /dev/null +++ b/JuliaSyntax/src/precompile.jl @@ -0,0 +1,12 @@ +# Just parse some file as a precompile workload +let filename = joinpath(@__DIR__, "julia/literal_parsing.jl") + text = read(filename, String) + parseall(Expr, text) + parseall(SyntaxNode, text) + if _has_v1_6_hooks + enable_in_core!() + Meta.parse("1 + 2") + Meta.parse(SubString("1 + 2")) + enable_in_core!(false) + end +end diff --git a/JuliaSyntax/src/utils.jl b/JuliaSyntax/src/utils.jl new file mode 100644 index 0000000000000..c21c251eb688a --- /dev/null +++ b/JuliaSyntax/src/utils.jl @@ -0,0 +1,144 @@ +# Compatibility hacks for older Julia versions +if VERSION < v"1.1" + isnothing(x) = x === nothing +end +if VERSION < v"1.4" + function only(x::Union{AbstractVector,AbstractString}) + if length(x) != 1 + error("Collection must contain exactly 1 element") + end + return x[1] + end +end +if VERSION < v"1.5" + import Base.peek +end + +@static if VERSION < v"1.8" + macro callsite_inline(call) + esc(call) + end +else + const var"@callsite_inline" = var"@inline" +end + + +_unsafe_wrap_substring(s) = (s.offset, unsafe_wrap(Vector{UInt8}, s.string)) + +#-------------------------------------------------- +# +# Internal error, used as assertion failure for cases we expect can't happen. +@noinline function internal_error(strs::Vararg{String, N}) where {N} + error("Internal error: ", strs...) +end + +# Like @assert, but always enabled and calls internal_error() +macro check(ex, msgs...) + msg = isempty(msgs) ? ex : msgs[1] + if isa(msg, AbstractString) + msg = String(msg) + elseif !isempty(msgs) && (isa(msg, Expr) || isa(msg, Symbol)) + msg = :(string($(esc(msg)))) + else + msg = string(msg) + end + return :($(esc(ex)) ? nothing : internal_error($msg)) +end + +# Really remove line numbers, even from Expr(:toplevel) +remove_linenums!(ex) = ex +function remove_linenums!(ex::Expr) + if ex.head === :block || ex.head === :quote || ex.head === :toplevel + filter!(ex.args) do x + !(isa(x, Expr) && x.head === :line || isa(x, LineNumberNode)) + end + end + for subex in ex.args + subex isa Expr && remove_linenums!(subex) + end + return ex +end + +# String macro to get the UInt8 code of an ascii character +macro u8_str(str) + c = str == "\\" ? '\\' : only(unescape_string(str)) + isascii(c) || error("Non-ascii character in u8_str") + codepoint(c) % UInt8 +end + +#------------------------------------------------------------------------------- +# Text printing/display utils + +const _fg_color_codes = Dict( + :black => 30, + :red => 31, + :green => 32, + :yellow => 33, + :blue => 34, + :magenta => 35, + :cyan => 36, + :white => 37, + :light_black => 90, # gray + :light_red => 91, + :light_green => 92, + :light_yellow => 93, + :light_blue => 94, + :light_magenta => 95, + :light_cyan => 96, + :light_white => 97, +) + +""" + _printstyled(io::IO, text; + fgcolor=nothing, bgcolor=nothing, href=nothing) + +Like Base.printstyled, but allows providing RGB colors for true color +terminals, both foreground and background colors, and hyperlinks. Colors may be +given as one of the standard color names as in `Base.printstyled`, an integer +for 256 color terms, or an (r,g,b) triple with `0 <= r <= 255` etc for true +color terminals. + +* `fgcolor` - set foreground color +* `bgcolor` - set background color +* `href` - set hyperlink reference +""" +function _printstyled(io::IO, text; fgcolor=nothing, bgcolor=nothing, href=nothing) + if (isnothing(fgcolor) && isnothing(bgcolor) && isnothing(href)) || !get(io, :color, false) + print(io, text) + return + end + colcode = "" + if !isnothing(fgcolor) + if fgcolor isa Symbol && haskey(_fg_color_codes, fgcolor) + colcode *= "\e[$(_fg_color_codes[fgcolor])m" + elseif fgcolor isa Integer && 0 <= fgcolor <= 255 + colcode *= "\e[38;5;$(fgcolor)m" + elseif fgcolor isa Tuple && length(fgcolor) == 3 && all(0 .<= fgcolor .<= 255) + colcode *= "\e[38;2;$(fgcolor[1]);$(fgcolor[2]);$(fgcolor[3])m" + else + error("Invalid ansi color $fgcolor") + end + end + if !isnothing(bgcolor) + if bgcolor isa Symbol && haskey(_fg_color_codes, bgcolor) + colcode *= "\e[$(10 + _fg_color_codes[bgcolor])m" + elseif bgcolor isa Integer && 0 <= bgcolor <= 255 + colcode *= "\e[48;5;$(bgcolor)m" + elseif bgcolor isa Tuple && length(bgcolor) == 3 && all(0 .<= bgcolor .<= 255) + colcode *= "\e[48;2;$(bgcolor[1]);$(bgcolor[2]);$(bgcolor[3])m" + else + error("Invalid ansi color $bgcolor") + end + end + colreset = "\e[0;0m" + first = true + for linepart in split(text, '\n') + first || print(io, '\n') + line = string(colcode, linepart, colreset) + if !isnothing(href) + line = "\e]8;;$href\e\\$line\e]8;;\e\\" + end + print(io, line) + first = false + end +end diff --git a/JuliaSyntax/sysimage/.gitignore b/JuliaSyntax/sysimage/.gitignore new file mode 100644 index 0000000000000..344cdc510a371 --- /dev/null +++ b/JuliaSyntax/sysimage/.gitignore @@ -0,0 +1,3 @@ +JuliaSyntax +Project.toml +Manifest.toml diff --git a/JuliaSyntax/sysimage/JuliaSyntaxCore/Project.toml b/JuliaSyntax/sysimage/JuliaSyntaxCore/Project.toml new file mode 100644 index 0000000000000..7e31a0dd5bc26 --- /dev/null +++ b/JuliaSyntax/sysimage/JuliaSyntaxCore/Project.toml @@ -0,0 +1,10 @@ +name = "JuliaSyntaxCore" +uuid = "05e5f68f-ccd0-4d84-a81a-f557a333a331" +authors = ["Chris Foster and contributors"] +version = "0.1.0" + +[compat] +julia = "1.6" + +[deps] +JuliaSyntax = "54354a4c-6cac-4c00-8566-e7c1beb8bfd8" diff --git a/JuliaSyntax/sysimage/JuliaSyntaxCore/src/JuliaSyntaxCore.jl b/JuliaSyntax/sysimage/JuliaSyntaxCore/src/JuliaSyntaxCore.jl new file mode 100644 index 0000000000000..d1804b89053b7 --- /dev/null +++ b/JuliaSyntax/sysimage/JuliaSyntaxCore/src/JuliaSyntaxCore.jl @@ -0,0 +1,12 @@ +module JuliaSyntaxCore + +# A tiny module to hold initialization code for JuliaSyntax.jl integration with +# the runtime. + +using JuliaSyntax + +function __init__() + JuliaSyntax.enable_in_core!() +end + +end diff --git a/JuliaSyntax/sysimage/compile.jl b/JuliaSyntax/sysimage/compile.jl new file mode 100755 index 0000000000000..390901eb56cd1 --- /dev/null +++ b/JuliaSyntax/sysimage/compile.jl @@ -0,0 +1,47 @@ +#!/bin/bash +#= +[[ $1 == +* ]] && juliaup_arg=$1 && shift # release channel for juliaup +exec julia ${juliaup_arg} --startup-file=no -e 'include(popfirst!(ARGS))' "$0" "$@" +=# + +imgs_base_path = joinpath(first(DEPOT_PATH), "sysimages", "v$VERSION") +mkpath(imgs_base_path) + +using Libdl + +cd(@__DIR__) + +# Create a copy of JuliaSyntax so we can change the project UUID. +# This allows us to use an older version of JuliaSyntax for developing +# JuliaSyntax itself. +rm("JuliaSyntax", force=true, recursive=true) +mkdir("JuliaSyntax") +cp("../src", "JuliaSyntax/src") +cp("../test", "JuliaSyntax/test") +projstr = replace(read("../Project.toml", String), + "70703baa-626e-46a2-a12c-08ffd08c73b4"=>"54354a4c-6cac-4c00-8566-e7c1beb8bfd8") +write("JuliaSyntax/Project.toml", projstr) + +using Pkg +rm("Project.toml", force=true) +rm("Manifest.toml", force=true) +Pkg.activate(".") +Pkg.develop(path="./JuliaSyntax") +Pkg.develop(path="./JuliaSyntaxCore") +Pkg.add("PackageCompiler") + +image_path = joinpath(imgs_base_path, "juliasyntax_sysimage."*Libdl.dlext) + +using PackageCompiler +PackageCompiler.create_sysimage( + ["JuliaSyntaxCore"], + project=".", + sysimage_path=image_path, + precompile_execution_file="precompile_exec.jl", + incremental=true, +) + +@info """## System image compiled! + + Use it with `julia -J "$image_path"` + """ diff --git a/JuliaSyntax/sysimage/precompile.jl b/JuliaSyntax/sysimage/precompile.jl new file mode 100644 index 0000000000000..a1ae9555f33a2 --- /dev/null +++ b/JuliaSyntax/sysimage/precompile.jl @@ -0,0 +1,6 @@ +function precompile_JuliaSyntax(mod, juliasyntax_path) + Base.include(mod, joinpath(juliasyntax_path, "test", "test_utils.jl")) + Base.include(mod, joinpath(juliasyntax_path, "test", "parser.jl")) + JuliaSyntax.enable_in_core!() + Meta.parse("x+y+z-w .+ [a b c]") +end diff --git a/JuliaSyntax/sysimage/precompile_exec.jl b/JuliaSyntax/sysimage/precompile_exec.jl new file mode 100644 index 0000000000000..99c8069341275 --- /dev/null +++ b/JuliaSyntax/sysimage/precompile_exec.jl @@ -0,0 +1,3 @@ +import JuliaSyntax +include("precompile.jl") +precompile_JuliaSyntax(@__MODULE__(), pkgdir(JuliaSyntax)) diff --git a/JuliaSyntax/test/benchmark.jl b/JuliaSyntax/test/benchmark.jl new file mode 100644 index 0000000000000..0a4e260e485e9 --- /dev/null +++ b/JuliaSyntax/test/benchmark.jl @@ -0,0 +1,35 @@ +using BenchmarkTools +using JuliaSyntax + +include("test_utils.jl") + +function concat_base() + basedir = joinpath(Sys.BINDIR, "..", "share", "julia", "base") + io = IOBuffer() + for f in find_source_in_path(basedir) + write(io, read(f, String)) + println(io) + end + return String(take!(io)) +end + +all_base_code = concat_base() + +b_ParseStream = @benchmark JuliaSyntax.parse!(JuliaSyntax.ParseStream(all_base_code), rule=:all) +b_GreenNode = @benchmark JuliaSyntax.parseall(JuliaSyntax.GreenNode, all_base_code, ignore_warnings=true) +b_SyntaxNode = @benchmark JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, all_base_code, ignore_warnings=true) +b_Expr = @benchmark JuliaSyntax.parseall(Expr, all_base_code, ignore_warnings=true) +b_flisp = @benchmark JuliaSyntax.fl_parseall(all_base_code) + +@info "Benchmarks" ParseStream=b_ParseStream GreenNode=b_GreenNode SyntaxNode=b_SyntaxNode Expr=b_Expr flisp=b_flisp + + +# Allocation profiling +# +# using Profile.Allocs +# using PProf +# Allocs.clear() +# stream = JuliaSyntax.ParseStream(text); +# JuliaSyntax.peek(stream); +# Allocs.@profile sample_rate=1 JuliaSyntax.parsestmt(stream) +# PProf.Allocs.pprof() diff --git a/JuliaSyntax/test/diagnostics.jl b/JuliaSyntax/test/diagnostics.jl new file mode 100644 index 0000000000000..a4912b665c902 --- /dev/null +++ b/JuliaSyntax/test/diagnostics.jl @@ -0,0 +1,286 @@ +function diagnostic(str; only_first=false, allow_multiple=false, rule=:all, version=v"1.6") + stream = ParseStream(str; version=version) + parse!(stream, rule=rule) + if allow_multiple + stream.diagnostics + else + if !only_first + @test length(stream.diagnostics) == 1 + end + return isempty(stream.diagnostics) ? nothing : stream.diagnostics[1] + end +end + +@testset "token errors" begin + @test diagnostic("a\xf5b") == Diagnostic(2, 2, :error, "invalid UTF-8 sequence \"\\xf5\"") + @test diagnostic("# a\xf5b") == Diagnostic(1, 5, :error, "invalid UTF-8 sequence \"# a\\xf5b\"") + for c in ['\u00ad', '\u200b', '\u200c', '\u200d', + '\u200e', '\u200f', '\u2060', '\u2061'] + @test diagnostic("a$(c)b") == + Diagnostic(2, 1+sizeof(string(c)), :error, "invisible character $(repr(c))") + end + @test diagnostic("₁") == Diagnostic(1, 3, :error, "identifier cannot begin with character '₁'") + @test diagnostic(":⥻") == Diagnostic(2, 4, :error, "unknown unicode character '⥻'") + + @test diagnostic("\"X \u202a X\"") == Diagnostic(2, 8, :error, "unbalanced bidirectional unicode formatting \"X \\u202a X\"") + @test diagnostic("#= \u202a =#") == Diagnostic(1, 9, :error, "unbalanced bidirectional unicode formatting \"#= \\u202a =#\"") + @test diagnostic("\"X \u202a \$xx\u202c\"", allow_multiple=true) == [ + Diagnostic(2, 7, :error, "unbalanced bidirectional unicode formatting \"X \\u202a \"") + Diagnostic(11, 13, :error, "unbalanced bidirectional unicode formatting \"\\u202c\"") + ] + + @test diagnostic("0x") == Diagnostic(1, 2, :error, "invalid numeric constant") + @test diagnostic("0x0.1") == Diagnostic(1, 5, :error, "hex float literal must contain `p` or `P`") +end + +@testset "parser errors" begin + @test diagnostic("+ #==# (a,b)") == + Diagnostic(2, 7, :error, "whitespace not allowed between prefix function call and argument list") + @test diagnostic("1 -+ (a=1, b=2)") == + Diagnostic(5, 5, :error, "whitespace not allowed between prefix function call and argument list") + @test diagnostic("\n+ (x, y)") == + Diagnostic(3, 3, :error, "whitespace not allowed between prefix function call and argument list") + + @test diagnostic("function (\$f) body end") == + Diagnostic(10, 13, :error, "Ambiguous signature. Add a trailing comma if this is a 1-argument anonymous function; remove parentheses if this is a macro call acting as function signature.") + + @test diagnostic("A.@B.x", only_first=true) == + Diagnostic(3, 4, :error, "`@` must appear on first or last macro name component") + @test diagnostic("@M.(x)") == + Diagnostic(1, 3, :error, "dot call syntax not supported for macros") + @test diagnostic("a.[x]") == + Diagnostic(1, 5, :error, "brackets are not allowed after `.`") + @test diagnostic("a.{x}") == + Diagnostic(1, 5, :error, "brackets are not allowed after `.`") + + @test diagnostic("try x end") == + Diagnostic(1, 9, :error, "try without catch or finally") + # TODO: better range + @test diagnostic("@A.\$x a") == + Diagnostic(4, 5, :error, "invalid macro name") + + @test diagnostic("a, , b") == + Diagnostic(4, 4, :error, "unexpected `,`") + @test diagnostic(")", allow_multiple=true) == [ + Diagnostic(1, 1, :error, "unexpected `)`") + Diagnostic(1, 1, :error, "extra tokens after end of expression") + ] + + @test diagnostic("if\nfalse\nend") == + Diagnostic(3, 3, :error, "missing condition in `if`") + @test diagnostic("if false\nelseif\nend") == + Diagnostic(16, 16, :error, "missing condition in `elseif`") + + @test diagnostic("f(x::V) where {V) = x", allow_multiple=true) == [ + Diagnostic(17, 16, :error, "Expected `}` or `,`") + Diagnostic(17, 21, :error, "extra tokens after end of expression") + ] + @test diagnostic("[1)", allow_multiple=true) == [ + Diagnostic(3, 2, :error, "Expected `]` or `,`") + Diagnostic(3, 3, :error, "extra tokens after end of expression") + ] + @test diagnostic("f(x, y #=hi=#\ng(z)") == Diagnostic(7, 6, :error, "Expected `)` or `,`") + @test diagnostic("(x, y \nz") == Diagnostic(6, 5, :error, "Expected `)` or `,`") + @test diagnostic("function f(x, y \nz end") == Diagnostic(16, 15, :error, "Expected `)` or `,`") + + @test diagnostic("sin. (1)") == + Diagnostic(5, 5, :error, "whitespace is not allowed here") + @test diagnostic("x [i]") == + Diagnostic(2, 2, :error, "whitespace is not allowed here") + @test diagnostic("\nf() [i]") == + Diagnostic(5, 5, :error, "whitespace is not allowed here") + @test diagnostic("\nf() (i)") == + Diagnostic(5, 5, :error, "whitespace is not allowed here") + @test diagnostic("\nf() .i") == + Diagnostic(5, 5, :error, "whitespace is not allowed here") + @test diagnostic("\nf() {i}") == + Diagnostic(5, 5, :error, "whitespace is not allowed here") + @test diagnostic("\n@ m") == + Diagnostic(3, 3, :error, "whitespace is not allowed here") + @test diagnostic("\nusing a .b") == + Diagnostic(9, 9, :error, "whitespace is not allowed here") + + @test diagnostic("const x") == + Diagnostic(1, 7, :error, "expected assignment after `const`") + @test diagnostic("global const x") == + Diagnostic(1, 14, :error, "expected assignment after `const`") + + @test diagnostic("(for i=1; println())") == + Diagnostic(20, 19, :error, "Expected `end`") + @test diagnostic("(try i=1; println())", allow_multiple=true) == [ + Diagnostic(2, 19, :error, "try without catch or finally") + Diagnostic(20, 19, :error, "Expected `end`") + ] + + @test diagnostic("\"\$(x,y)\"") == + Diagnostic(3, 7, :error, "invalid interpolation syntax") + + @test diagnostic("", rule=:statement) == + Diagnostic(1, 0, :error, "premature end of input") + @test diagnostic("", rule=:atom) == + Diagnostic(1, 0, :error, "premature end of input") + + # `..` immediately followed by another operator must be space-separated, as + # the lexer no longer glues the dots into a single `..` token (#573) + @test diagnostic("a..+b") == + Diagnostic(4, 3, :error, "`..` here is interpreted as a binary operator. A space is required if followed by another operator.") + @test diagnostic("a..−b") == + Diagnostic(4, 3, :error, "`..` here is interpreted as a binary operator. A space is required if followed by another operator.") +end + +@testset "parser warnings" begin + @test diagnostic("@(A)", only_first=true) == + Diagnostic(2, 4, :warning, "parenthesizing macro names is unnecessary") + @test diagnostic("try finally catch a ; b end") == + Diagnostic(13, 23, :warning, "`catch` after `finally` will execute out of order") + @test diagnostic("import . .A") == + Diagnostic(9, 10, :warning, "space between dots in import path") + @test diagnostic("import A .==") == + Diagnostic(9, 9, :warning, "space between dots in import path") + @test diagnostic("import A.:+") == + Diagnostic(10, 10, :warning, "quoting with `:` is not required here") + # No warnings for imports of `:` and parenthesized `(..)` + @test diagnostic("import A.:, :", allow_multiple=true) == [] + @test diagnostic("import A: (..)", allow_multiple=true) == [] + @test diagnostic("import A.(:+)") == + Diagnostic(10, 13, :warning, "parentheses are not required here") + @test diagnostic("export (x)") == + Diagnostic(8, 10, :warning, "parentheses are not required here") + @test diagnostic("import :A") == + Diagnostic(8, 9, :error, "expected identifier") + @test diagnostic("export :x") == + Diagnostic(8, 9, :error, "expected identifier") + @test diagnostic("public = 4", version=v"1.11") == + diagnostic("public[7] = 5", version=v"1.11") == + diagnostic("public() = 6", version=v"1.11") == + Diagnostic(1, 6, :warning, "using public as an identifier is deprecated") + + @test diagnostic("a +% b", version=v"1.13") == + Diagnostic(2, 1, :error, "wrapping arithmetic operators `+%`, `-%`, and `*%` not supported in Julia version 1.13 < 1.14") + @test diagnostic("a +% b", only_first=true, version=v"1.14") === nothing + + @test diagnostic("break +", only_first=true, version=v"1.13") == + Diagnostic(6, 7, :error, "unexpected token after break") + @test diagnostic("break ()", only_first=true, version=v"1.13") == + Diagnostic(6, 7, :error, "unexpected token after break") + @test diagnostic("continue +", only_first=true, version=v"1.13") == + Diagnostic(9, 10, :error, "unexpected token after continue") + @test diagnostic("break label", only_first=true, version=v"1.13") == + Diagnostic(1, 11, :error, "labeled `break` and `continue` not supported in Julia version 1.13 < 1.14") + + @test diagnostic("break +", only_first=true, version=v"1.14") == + Diagnostic(7, 7, :error, "expected identifier for break label") + @test diagnostic("continue +", only_first=true, version=v"1.14") == + Diagnostic(10, 10, :error, "expected identifier for break label") + + @test diagnostic("break f()", only_first=true, version=v"1.14") == + Diagnostic(8, 7, :error, "expected space after break label") + + @test diagnostic("continue f val", only_first=true, version=v"1.14") == + Diagnostic(11, 14, :error, "unexpected token after continue") +end + +@testset "diagnostics for literal parsing" begin + # Float overflow/underflow + @test diagnostic("x = 10.0e1000;") == + Diagnostic(5, 13, :error, "overflow in floating point literal") + @test diagnostic("x = 10.0f1000;") == + Diagnostic(5, 13, :error, "overflow in floating point literal") + @test diagnostic("x = 10.0e-1000;") == + Diagnostic(5, 14, :warning, "underflow to zero in floating point literal") + @test diagnostic("x = 10.0f-1000;") == + Diagnostic(5, 14, :warning, "underflow to zero in floating point literal") + # Underflow boundary + @test diagnostic("5e-324", allow_multiple=true) == [] + @test diagnostic("2e-324") == + Diagnostic(1, 6, :warning, "underflow to zero in floating point literal") + + # Char + @test diagnostic("x = ''") == + Diagnostic(6, 5, :error, "empty character literal") + @test diagnostic("x = 'abc'") == + Diagnostic(6, 8, :error, "character literal contains multiple characters") + @test diagnostic("x = '\\xq'") == + Diagnostic(6, 7, :error, "invalid hex escape sequence") + @test diagnostic("x = '\\uq'") == + Diagnostic(6, 7, :error, "invalid unicode escape sequence") + @test diagnostic("x = '\\Uq'") == + Diagnostic(6, 7, :error, "invalid unicode escape sequence") + @test diagnostic("x = '\\777'") == + Diagnostic(6, 9, :error, "invalid octal escape sequence") + @test diagnostic("x = '\\k'") == + Diagnostic(6, 7, :error, "invalid escape sequence") + @test diagnostic("'\\", allow_multiple=true) == [ + Diagnostic(2, 2, :error, "invalid escape sequence"), + Diagnostic(3, 2, :error, "unterminated character literal") + ] + # Various cases from Base + @test diagnostic("'\\xff\\xff\\xff\\xff'") == + Diagnostic(2, 17, :error, "character literal contains multiple characters") + @test diagnostic("'\\100\\42'") == + Diagnostic(2, 8, :error, "character literal contains multiple characters") + @test diagnostic("'\\xff\\xff\\xff\\xff\\xff'") == + Diagnostic(2, 21, :error, "character literal contains multiple characters") + @test diagnostic("'abcd'") == + Diagnostic(2, 5, :error, "character literal contains multiple characters") + @test diagnostic("'\\uff\\xff'") == + Diagnostic(2, 9, :error, "character literal contains multiple characters") + @test diagnostic("'\\xffa'") == + Diagnostic(2, 6, :error, "character literal contains multiple characters") + @test diagnostic("'\\uffffa'") == + Diagnostic(2, 8, :error, "character literal contains multiple characters") + @test diagnostic("'\\U00002014a'") == + Diagnostic(2, 12, :error, "character literal contains multiple characters") + @test diagnostic("'\\1000'") == + Diagnostic(2, 6, :error, "character literal contains multiple characters") + + # String + @test diagnostic("x = \"abc\\xq\"") == + Diagnostic(9, 10, :error, "invalid hex escape sequence") + @test diagnostic("x = \"abc\\uq\"") == + Diagnostic(9, 10, :error, "invalid unicode escape sequence") + @test diagnostic("x = \"abc\\Uq\"") == + Diagnostic(9, 10, :error, "invalid unicode escape sequence") + @test diagnostic("x = \"abc\\777\"") == + Diagnostic(9, 12, :error, "invalid octal escape sequence") + @test diagnostic("x = \"abc\\k\"") == + Diagnostic(9, 10, :error, "invalid escape sequence") + @test diagnostic("x = \"abc\\k \\k\"", allow_multiple=true) == [ + Diagnostic(9, 10, :error, "invalid escape sequence"), + Diagnostic(12, 13, :error, "invalid escape sequence") + ] + @test diagnostic("\"\$x෴ \"") == + Diagnostic(4, 6, :error, "interpolated variable ends with invalid character; use `\$(...)` instead") +end + +@testset "diagnostic printing" begin + stream = JuliaSyntax.ParseStream("a -- b -- c") + JuliaSyntax.parse!(stream) + @test sprint(JuliaSyntax.show_diagnostics, stream) == """ + # Error @ line 1:3 + a -- b -- c + # └┘ ── invalid operator + # Error @ line 1:8 + a -- b -- c + # └┘ ── invalid operator""" + + stream = JuliaSyntax.ParseStream("a -- b") + JuliaSyntax.parse!(stream) + fname = "test.jl" + sf = SourceFile(stream, filename=fname) + url = JuliaSyntax._file_url(fname) + @test sprint(JuliaSyntax.show_diagnostics, stream.diagnostics, sf, + context=:color=>true) == """ + \e[90m# Error @ \e[0;0m\e]8;;$url#1:3\e\\\e[90mtest.jl:1:3\e[0;0m\e]8;;\e\\ + a \e[48;2;120;70;70m--\e[0;0m b + \e[90m# └┘ ── \e[0;0m\e[91minvalid operator\e[0;0m""" + + if Sys.isunix() + tempdirname = mktempdir() + cd(tempdirname) do + rm(tempdirname) + # Test _file_url doesn't fail with nonexistent directories + @test isnothing(JuliaSyntax._file_url(joinpath("__nonexistent__", "test.jl"))) + end + end +end diff --git a/JuliaSyntax/test/expr.jl b/JuliaSyntax/test/expr.jl new file mode 100644 index 0000000000000..b391f1115e485 --- /dev/null +++ b/JuliaSyntax/test/expr.jl @@ -0,0 +1,871 @@ +@testset "Expr parsing with $method" for method in ["build_tree", "SyntaxNode conversion"] + parseatom, parsestmt, parseall = + if method == "build_tree" + ((s; kws...) -> JuliaSyntax.parseatom(Expr, s; kws...), + (s; kws...) -> JuliaSyntax.parsestmt(Expr, s; kws...), + (s; kws...) -> JuliaSyntax.parseall(Expr, s; kws...)) + else + ((s; kws...) -> Expr(JuliaSyntax.parseatom(SyntaxNode, s; keep_parens=true, kws...)), + (s; kws...) -> Expr(JuliaSyntax.parsestmt(SyntaxNode, s; keep_parens=true, kws...)), + (s; kws...) -> Expr(JuliaSyntax.parseall(SyntaxNode, s; keep_parens=true, kws...))) + end + + @testset "Quote nodes" begin + @test parseatom(":(a)") == QuoteNode(:a) + @test parseatom(":(:a)") == Expr(:quote, QuoteNode(:a)) + @test parseatom(":(1+2)") == Expr(:quote, Expr(:call, :+, 1, 2)) + @test parseatom(":...") == QuoteNode(Symbol("...")) + @test parseatom(":(...)") == QuoteNode(Symbol("...")) + # Compatibility hack for VERSION >= v"1.4" + # https://github.com/JuliaLang/julia/pull/34077 + @test parseatom(":true") == Expr(:quote, true) + end + + @testset "Line numbers" begin + @testset "Blocks" begin + @test parsestmt("begin a\nb\n\nc\nend") == + Expr(:block, + LineNumberNode(1), + :a, + LineNumberNode(2), + :b, + LineNumberNode(4), + :c, + ) + @test parsestmt("(a;b;c)") == + Expr(:block, + :a, + LineNumberNode(1), + :b, + LineNumberNode(1), + :c, + ) + @test parsestmt("begin end") == + Expr(:block, + LineNumberNode(1) + ) + @test parsestmt("(;;)") == + Expr(:block) + + @test parseall("a\n\nb") == + Expr(:toplevel, + LineNumberNode(1), + :a, + LineNumberNode(3), + :b, + ) + @test parsestmt("a;b") == + Expr(:toplevel, :a, :b) + + @test parsestmt("module A\n\nbody\nend"; version=v"1.13") == + Expr(:module, + true, + :A, + Expr(:block, + LineNumberNode(1), + LineNumberNode(3), + :body, + ), + ) + + @test parseall("a\n\nx") == + Expr(:toplevel, + LineNumberNode(1), + :a, + LineNumberNode(3), + :x + ) + @test parseall("a\n\nx;y") == + Expr(:toplevel, + LineNumberNode(1), + :a, + LineNumberNode(3), + Expr(:toplevel, :x, :y) + ) + end + + @testset "Function definition lines" begin + @test parsestmt("function f()\na\n\nb\nend") == + Expr(:function, + Expr(:call, :f), + Expr(:block, + LineNumberNode(1), + LineNumberNode(2), + :a, + LineNumberNode(4), + :b, + ) + ) + @test parsestmt("f() = 1") == + Expr(:(=), + Expr(:call, :f), + Expr(:block, + LineNumberNode(1), + 1 + ) + ) + @test parsestmt("macro f()\na\nend") == + Expr(:macro, + Expr(:call, :f), + Expr(:block, + LineNumberNode(1), + LineNumberNode(2), + :a, + ) + ) + + # function/macro without methods + @test parsestmt("function f end") == + Expr(:function, :f) + @test parsestmt("macro f end") == + Expr(:macro, :f) + + # weird cases with extra parens + @test parsestmt("function (f() where T) end") == + Expr(:function, Expr(:where, Expr(:call, :f), :T), + Expr(:block, LineNumberNode(1), LineNumberNode(1))) + @test parsestmt("function (f()::S) end") == + Expr(:function, Expr(:(::), Expr(:call, :f), :S), + Expr(:block, LineNumberNode(1), LineNumberNode(1))) + end + + @testset "->" begin + @test parsestmt("a -> b") == + Expr(:->, :a, Expr(:block, LineNumberNode(1), :b)) + @test parsestmt("(a,) -> b") == + Expr(:->, Expr(:tuple, :a), Expr(:block, LineNumberNode(1), :b)) + @test parsestmt("(a where T) -> b") == + Expr(:->, Expr(:where, :a, :T), Expr(:block, LineNumberNode(1), :b)) + # @test parsestmt("a -> (\nb;c)") == + # Expr(:->, :a, Expr(:block, LineNumberNode(1), :b)) + @test parsestmt("a -> begin\nb\nc\nend") == + Expr(:->, :a, Expr(:block, + LineNumberNode(1), + LineNumberNode(2), :b, + LineNumberNode(3), :c)) + @test parsestmt("(a;b=1) -> c") == + Expr(:->, + Expr(:block, :a, LineNumberNode(1), Expr(:(=), :b, 1)), + Expr(:block, LineNumberNode(1), :c)) + @test parsestmt("(a...;b...) -> c") == + Expr(:->, + Expr(:tuple, Expr(:parameters, Expr(:(...), :b)), Expr(:(...), :a)), + Expr(:block, LineNumberNode(1), :c)) + @test parsestmt("(;) -> c") == + Expr(:->, + Expr(:tuple, Expr(:parameters)), + Expr(:block, LineNumberNode(1), :c)) + @test parsestmt("a::T -> b") == + Expr(:->, Expr(:(::), :a, :T), Expr(:block, LineNumberNode(1), :b)) + end + + @testset "elseif" begin + @test parsestmt("if a\nb\nelseif c\n d\nend") == + Expr(:if, + :a, + Expr(:block, + LineNumberNode(2), + :b), + Expr(:elseif, + Expr(:block, + LineNumberNode(3), # Line number for elseif condition + :c), + Expr(:block, + LineNumberNode(4), + :d), + ) + ) + end + + @testset "No line numbers in let bindings" begin + @test parsestmt("let i=is, j=js\nbody\nend") == + Expr(:let, + Expr(:block, + Expr(:(=), :i, :is), + Expr(:(=), :j, :js), + ), + Expr(:block, + LineNumberNode(2), + :body + ) + ) + end + + @testset "Loops" begin + @test parsestmt("for x=xs\n\nend") == + Expr(:for, + Expr(:(=), :x, :xs), + Expr(:block, + LineNumberNode(1), + LineNumberNode(3) + ) + ) + @test parsestmt("for x=xs\ny\nend") == + Expr(:for, + Expr(:(=), :x, :xs), + Expr(:block, + LineNumberNode(2), + :y, + LineNumberNode(3) + ) + ) + @test parsestmt("while cond\n\nend") == + Expr(:while, + :cond, + Expr(:block, + LineNumberNode(1), + LineNumberNode(3) + ) + ) + @test parsestmt("while cond\ny\nend") == + Expr(:while, + :cond, + Expr(:block, + LineNumberNode(2), + :y, + LineNumberNode(3) + ) + ) + end + end + + @testset "Short form function line numbers" begin + # A block is added to hold the line number node + @test parsestmt("f() = xs") == + Expr(:(=), + Expr(:call, :f), + Expr(:block, + LineNumberNode(1), + :xs)) + + @test parsestmt("f() =\n(a;b)") == + Expr(:(=), + Expr(:call, :f), + Expr(:block, + LineNumberNode(1), + :a, + LineNumberNode(2), + :b)) + + @test parsestmt("f() =\nbegin\na\nb\nend") == + Expr(:(=), + Expr(:call, :f), + Expr(:block, + LineNumberNode(1), + LineNumberNode(3), + :a, + LineNumberNode(4), + :b)) + + @test parsestmt("let f(x) =\ng(x)=1\nend") == + Expr(:let, + Expr(:(=), + Expr(:call, :f, :x), + Expr(:block, + LineNumberNode(1), + Expr(:(=), + Expr(:call, :g, :x), + Expr(:block, + LineNumberNode(2), + 1)))), + Expr(:block, + LineNumberNode(3))) + + # short-form postfix function shouldn't introduce a block + @test parsestmt("x' = 1") == + Expr(:(=), + Expr(Symbol("'"), :x), + 1) + @test parsestmt("x' = A * x") == + Expr(:(=), + Expr(Symbol("'"), :x), + Expr(:call, :*, :A, :x)) + + # `.=` doesn't introduce short form functions + @test parsestmt("f() .= xs") == + Expr(:(.=), Expr(:call, :f), :xs) + end + + @testset "for" begin + @test parsestmt("for i=is body end") == + Expr(:for, + Expr(:(=), :i, :is), + Expr(:block, + LineNumberNode(1), + :body, + LineNumberNode(1) + ) + ) + @test parsestmt("for i=is, j=js\nbody\nend") == + Expr(:for, + Expr(:block, + Expr(:(=), :i, :is), + Expr(:(=), :j, :js), + ), + Expr(:block, + LineNumberNode(2), + :body, + LineNumberNode(3), + ) + ) + end + + @testset "Long form anonymous functions" begin + @test parsestmt("function (xs...)\nbody end") == + Expr(:function, + Expr(:..., :xs), + Expr(:block, + LineNumberNode(1), + LineNumberNode(2), + :body)) + end + + @testset "String conversions" begin + # String unwrapping / wrapping + @test parsestmt("\"str\"") == "str" + @test parsestmt("\"\$(\"str\")\"") == + Expr(:string, Expr(:string, "str")) + # Concatenation of string chunks in triple quoted cases + @test parsestmt("```\n a\n b```") == + Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), LineNumberNode(1), + "a\nb") + @test parsestmt("\"\"\"\n a\n \$x\n b\n c\"\"\"") == + Expr(:string, "a\n", :x, "\nb\nc") + # Incomplete cases + @test parsestmt("`x", ignore_errors=true) == + Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), LineNumberNode(1), + Expr(:string, "x", Expr(:error))) + @test parsestmt("`", ignore_errors=true) == + Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), LineNumberNode(1), + Expr(:string, Expr(:error))) + end + + @testset "Char conversions" begin + @test parsestmt("'a'") == 'a' + @test parsestmt("'α'") == 'α' + @test parsestmt("'\\xce\\xb1'") == 'α' + end + + @testset "do block conversion" begin + @test parsestmt("f(x) do y\n body end") == + Expr(:do, + Expr(:call, :f, :x), + Expr(:->, Expr(:tuple, :y), + Expr(:block, + LineNumberNode(2), + :body))) + + @test parsestmt("@f(x) do y body end") == + Expr(:do, + Expr(:macrocall, Symbol("@f"), LineNumberNode(1), :x), + Expr(:->, Expr(:tuple, :y), + Expr(:block, + LineNumberNode(1), + :body))) + + @test parsestmt("f(x; a=1) do y body end") == + Expr(:do, + Expr(:call, :f, Expr(:parameters, Expr(:kw, :a, 1)), :x), + Expr(:->, Expr(:tuple, :y), + Expr(:block, + LineNumberNode(1), + :body))) + + # Test calls with do inside them + @test parsestmt("g(f(x) do y\n body end)") == + Expr(:call, + :g, + Expr(:do, + Expr(:call, :f, :x), + Expr(:->, Expr(:tuple, :y), + Expr(:block, + LineNumberNode(2), + :body)))) + end + + @testset "= to Expr(:kw) conversion" begin + # Call + @test parsestmt("f(a=1)") == + Expr(:call, :f, Expr(:kw, :a, 1)) + @test parsestmt("f(; b=2)") == + Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2))) + @test parsestmt("f(a=1; b=2)") == + Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1)) + @test parsestmt("f(a; b; c)") == + Expr(:call, :f, Expr(:parameters, Expr(:parameters, :c), :b), :a) + @test parsestmt("+(a=1,)") == + Expr(:call, :+, Expr(:kw, :a, 1)) + @test parsestmt("(a=1)()") == + Expr(:call, Expr(:(=), :a, 1)) + + # Operator calls: = is not :kw + @test parsestmt("(x=1) != 2") == + Expr(:call, :!=, Expr(:(=), :x, 1), 2) + @test parsestmt("+(a=1)") == + Expr(:call, :+, Expr(:(=), :a, 1)) + @test parsestmt("(a=1)'") == + Expr(Symbol("'"), Expr(:(=), :a, 1)) + @test parsestmt("(a=1)'ᵀ") == + Expr(:call, Symbol("'ᵀ"), Expr(:(=), :a, 1)) + + # Dotcall + @test parsestmt("f.(a=1; b=2)") == + Expr(:., :f, Expr(:tuple, + Expr(:parameters, Expr(:kw, :b, 2)), + Expr(:kw, :a, 1))) + + # Named tuples + @test parsestmt("(a=1,)") == + Expr(:tuple, Expr(:(=), :a, 1)) + @test parsestmt("(a=1,; b=2)") == + Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:(=), :a, 1)) + @test parsestmt("(a=1,; b=2; c=3)") == + Expr(:tuple, + Expr(:parameters, + Expr(:parameters, Expr(:kw, :c, 3)), + Expr(:kw, :b, 2)), + Expr(:(=), :a, 1)) + + # ref + @test parsestmt("x[i=j]") == + Expr(:ref, :x, Expr(:kw, :i, :j)) + @test parsestmt("(i=j)[x]") == + Expr(:ref, Expr(:(=), :i, :j), :x) + @test parsestmt("x[a, b; i=j]") == + Expr(:ref, :x, Expr(:parameters, Expr(:(=), :i, :j)), :a, :b) + # curly + @test parsestmt("(i=j){x}") == + Expr(:curly, Expr(:(=), :i, :j), :x) + @test parsestmt("x{a, b; i=j}") == + Expr(:curly, :x, Expr(:parameters, Expr(:(=), :i, :j)), :a, :b) + + # vect + @test parsestmt("[a=1,; b=2]") == + Expr(:vect, + Expr(:parameters, Expr(:(=), :b, 2)), + Expr(:(=), :a, 1)) + # braces + @test parsestmt("{a=1,; b=2}") == + Expr(:braces, + Expr(:parameters, Expr(:(=), :b, 2)), + Expr(:(=), :a, 1)) + + # dotted = is not :kw + @test parsestmt("f(a .= 1)") == + Expr(:call, :f, Expr(:.=, :a, 1)) + + # = inside parens in calls and tuples + @test parsestmt("f(((a = 1)))") == + Expr(:call, :f, Expr(:kw, :a, 1)) + @test parsestmt("(((a = 1)),)") == + Expr(:tuple, Expr(:(=), :a, 1)) + @test parsestmt("(;((a = 1)),)") == + Expr(:tuple, Expr(:parameters, Expr(:kw, :a, 1))) + end + + @testset "Field access syntax" begin + @test parsestmt("a.b") == Expr(:., :a, QuoteNode(:b)) + @test parsestmt("a.\$b") == Expr(:., :a, QuoteNode(Expr(:$, :b))) + @test parsestmt("a.:b") == Expr(:., :a, QuoteNode(:b)) + @test parsestmt("a.@b x") == Expr(:macrocall, + Expr(:., :a, QuoteNode(Symbol("@b"))), + LineNumberNode(1), + :x) + end + + @testset "dotcall / dotted operators" begin + @test parsestmt("f.(x,y)") == Expr(:., :f, Expr(:tuple, :x, :y)) + @test parsestmt("f.(x=1)") == Expr(:., :f, Expr(:tuple, Expr(:kw, :x, 1))) + @test parsestmt("f.(a=1; b=2)") == + Expr(:., :f, Expr(:tuple, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))) + @test parsestmt("(a=1).()") == Expr(:., Expr(:(=), :a, 1), Expr(:tuple)) + @test parsestmt("x .+ y") == Expr(:call, Symbol(".+"), :x, :y) + @test parsestmt("(x=1) .+ y") == Expr(:call, Symbol(".+"), Expr(:(=), :x, 1), :y) + @test parsestmt("a .< b .< c") == Expr(:comparison, :a, Symbol(".<"), + :b, Symbol(".<"), :c) + @test parsestmt("a .< (.<) .< c") == Expr(:comparison, :a, Symbol(".<"), + Expr(:., :<), Symbol(".<"), :c) + @test parsestmt(".*(x)") == Expr(:call, Symbol(".*"), :x) + @test parsestmt(".+(x)") == Expr(:call, Symbol(".+"), :x) + @test parsestmt(".+x") == Expr(:call, Symbol(".+"), :x) + @test parsestmt("(.+)(x)") == Expr(:call, Expr(:., :+), :x) + @test parsestmt("(.+).(x)") == Expr(:., Expr(:., :+), Expr(:tuple, :x)) + + @test parsestmt(".+") == Expr(:., :+) + @test parsestmt(":.+") == QuoteNode(Symbol(".+")) + @test parsestmt(":(.+)") == Expr(:quote, (Expr(:., :+))) + @test parsestmt("quote .+ end") == Expr(:quote, + Expr(:block, + LineNumberNode(1), + Expr(:., :+))) + @test parsestmt(".+{x}") == Expr(:curly, Symbol(".+"), :x) + + # Quoted syntactic ops act different when in parens + @test parsestmt(":.=") == QuoteNode(Symbol(".=")) + @test parsestmt(":(.=)") == QuoteNode(Symbol(".=")) + + # A few other cases of bare dotted ops + @test parsestmt("f(.+)") == Expr(:call, :f, Expr(:., :+)) + @test parsestmt("(a, .+)") == Expr(:tuple, :a, Expr(:., :+)) + @test parsestmt("A.:.+") == Expr(:., :A, QuoteNode(Symbol(".+"))) + + # Issue #341 + @test parsestmt("./x", ignore_errors=true) == Expr(:call, Expr(:error, Expr(:., :/)), :x) + end + + @testset "syntactic update-assignment operators" begin + @test parsestmt("x += y") == Expr(:(+=), :x, :y) + @test parsestmt("x .+= y") == Expr(:(.+=), :x, :y) + @test parsestmt("x +%= y"; version=v"1.14") == Expr(Symbol("+%="), :x, :y) + @test parsestmt("x -%= y"; version=v"1.14") == Expr(Symbol("-%="), :x, :y) + @test parsestmt("x *%= y"; version=v"1.14") == Expr(Symbol("*%="), :x, :y) + @test parsestmt("x .+%= y"; version=v"1.14") == Expr(Symbol(".+%="), :x, :y) + @test parsestmt(":+=") == QuoteNode(Symbol("+=")) + @test parsestmt(":+%="; version=v"1.14") == QuoteNode(Symbol("+%=")) + @test parsestmt(":(+=)") == QuoteNode(Symbol("+=")) + @test parsestmt(":(+%=)"; version=v"1.14") == QuoteNode(Symbol("+%=")) + @test parsestmt(":.+=") == QuoteNode(Symbol(".+=")) + @test parsestmt(":.+%="; version=v"1.14") == QuoteNode(Symbol(".+%=")) + @test parsestmt(":(.+=)") == QuoteNode(Symbol(".+=")) + @test parsestmt(":(.+%=)"; version=v"1.14") == QuoteNode(Symbol(".+%=")) + @test parsestmt("x \u2212= y") == Expr(:(-=), :x, :y) + end + + @testset "let" begin + @test parsestmt("let x=1\n end") == + Expr(:let, Expr(:(=), :x, 1), Expr(:block, LineNumberNode(2))) + @test parsestmt("let x=1 ; end") == + Expr(:let, Expr(:(=), :x, 1), Expr(:block, LineNumberNode(1))) + @test parsestmt("let x ; end") == + Expr(:let, :x, Expr(:block, LineNumberNode(1))) + @test parsestmt("let x::1 ; end") == + Expr(:let, Expr(:(::), :x, 1), Expr(:block, LineNumberNode(1))) + @test parsestmt("let x=1,y=2 end") == + Expr(:let, Expr(:block, Expr(:(=), :x, 1), Expr(:(=), :y, 2)), Expr(:block, LineNumberNode(1))) + @test parsestmt("let x+=1 ; end") == + Expr(:let, Expr(:block, Expr(:+=, :x, 1)), Expr(:block, LineNumberNode(1))) + @test parsestmt("let ; end") == + Expr(:let, Expr(:block), Expr(:block, LineNumberNode(1))) + @test parsestmt("let ; body end") == + Expr(:let, Expr(:block), Expr(:block, LineNumberNode(1), :body)) + @test parsestmt("let\na\nb\nend") == + Expr(:let, Expr(:block), Expr(:block, LineNumberNode(2), :a, LineNumberNode(3), :b)) + end + + @testset "where" begin + @test parsestmt("A where T") == Expr(:where, :A, :T) + @test parsestmt("A where {T}") == Expr(:where, :A, :T) + @test parsestmt("A where {S, T}") == Expr(:where, :A, :S, :T) + @test parsestmt("A where {X, Y; Z}") == Expr(:where, :A, Expr(:parameters, :Z), :X, :Y) + end + + @testset "macrocall" begin + # line numbers + @test parsestmt("@m\n") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1)) + @test parsestmt("\n@m") == Expr(:macrocall, Symbol("@m"), LineNumberNode(2)) + # parameters + @test parsestmt("@m(x; a)") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1), + Expr(:parameters, :a), :x) + @test parsestmt("@m(a=1; b=2)") == Expr(:macrocall, Symbol("@m"), LineNumberNode(1), + Expr(:parameters, Expr(:kw, :b, 2)), Expr(:(=), :a, 1)) + # @__dot__ + @test parsestmt("@.") == Expr(:macrocall, Symbol("@__dot__"), LineNumberNode(1)) + @test parsestmt("using A: @.") == Expr(:using, Expr(Symbol(":"), Expr(:., :A), Expr(:., Symbol("@__dot__")))) + + # var"" + @test parsestmt("@var\"#\" a") == Expr(:macrocall, Symbol("@#"), LineNumberNode(1), :a) + @test parsestmt("@var\"\\\"\" a") == Expr(:macrocall, Symbol("@\""), LineNumberNode(1), :a) + @test parsestmt("A.@var\"#\" a") == Expr(:macrocall, Expr(:., :A, QuoteNode(Symbol("@#"))), LineNumberNode(1), :a) + + # Square brackets + @test parsestmt("@S[a,b]") == + Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:vect, :a, :b)) + @test parsestmt("@S[a b]") == + Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:hcat, :a, :b)) + @test parsestmt("@S[a; b]") == + Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:vcat, :a, :b)) + @test parsestmt("@S[a ;; b]", version=v"1.7") == + Expr(:macrocall, Symbol("@S"), LineNumberNode(1), Expr(:ncat, 2, :a, :b)) + end + + @testset "var" begin + @test parsestmt("var\"x\"") == :x + @test parsestmt("var\"\"") == Symbol("") + @test parsestmt("var\"\\\"\"") == Symbol("\"") + @test parsestmt("var\"\\\\\\\"\"") == Symbol("\\\"") + @test parsestmt("var\"\\\\x\"") == Symbol("\\\\x") + @test parsestmt("var\"x\"+y") == Expr(:call, :+, :x, :y) + end + + @testset "vect" begin + @test parsestmt("[x,y ; z]") == Expr(:vect, Expr(:parameters, :z), :x, :y) + end + + @testset "concatenation" begin + @test parsestmt("[a ;;; b ;;;; c]", version=v"1.7") == + Expr(:ncat, 4, Expr(:nrow, 3, :a, :b), :c) + @test parsestmt("[a b ; c d]") == + Expr(:vcat, Expr(:row, :a, :b), Expr(:row, :c, :d)) + @test parsestmt("[a\nb]") == Expr(:vcat, :a, :b) + @test parsestmt("[a b]") == Expr(:hcat, :a, :b) + @test parsestmt("[a b ; c d]") == + Expr(:vcat, Expr(:row, :a, :b), Expr(:row, :c, :d)) + + @test parsestmt("T[a ;;; b ;;;; c]", version=v"1.7") == + Expr(:typed_ncat, :T, 4, Expr(:nrow, 3, :a, :b), :c) + @test parsestmt("T[a b ; c d]") == + Expr(:typed_vcat, :T, Expr(:row, :a, :b), Expr(:row, :c, :d)) + @test parsestmt("T[a\nb]") == Expr(:typed_vcat, :T, :a, :b) + @test parsestmt("T[a b]") == Expr(:typed_hcat, :T, :a, :b) + @test parsestmt("T[a b ; c d]") == + Expr(:typed_vcat, :T, Expr(:row, :a, :b), Expr(:row, :c, :d)) + end + + @testset "generators" begin + @test parsestmt("(x for a in as for b in bs)") == + Expr(:flatten, Expr(:generator, + Expr(:generator, :x, Expr(:(=), :b, :bs)), + Expr(:(=), :a, :as))) + @test parsestmt("(x for a in as, b in bs)") == + Expr(:generator, :x, Expr(:(=), :a, :as), Expr(:(=), :b, :bs)) + @test parsestmt("(x for a in as, b in bs if z)") == + Expr(:generator, :x, + Expr(:filter, :z, Expr(:(=), :a, :as), Expr(:(=), :b, :bs))) + @test parsestmt("(x for a in as, b in bs for c in cs, d in ds)") == + Expr(:flatten, + Expr(:generator, + Expr(:generator, :x, Expr(:(=), :c, :cs), Expr(:(=), :d, :ds)), + Expr(:(=), :a, :as), Expr(:(=), :b, :bs))) + @test parsestmt("(x for a in as for b in bs if z)") == + Expr(:flatten, Expr(:generator, + Expr(:generator, :x, Expr(:filter, :z, Expr(:(=), :b, :bs))), + Expr(:(=), :a, :as))) + @test parsestmt("(x for a in as if z for b in bs)") == + Expr(:flatten, Expr(:generator, + Expr(:generator, :x, Expr(:(=), :b, :bs)), + Expr(:filter, :z, Expr(:(=), :a, :as)))) + @test parsestmt("[x for a = as for b = bs if cond1 for c = cs if cond2]" ) == + Expr(:comprehension, + Expr(:flatten, + Expr(:generator, + Expr(:flatten, + Expr(:generator, + Expr(:generator, + :x, + Expr(:filter, + :cond2, + Expr(:(=), :c, :cs))), + Expr(:filter, + :cond1, + Expr(:(=), :b, :bs)))), + Expr(:(=), :a, :as)))) + @test parsestmt("[x for a = as if begin cond2 end]" ) == + Expr(:comprehension, Expr(:generator, :x, + Expr(:filter, + Expr(:block, LineNumberNode(1), :cond2), + Expr(:(=), :a, :as)))) + @test parsestmt("(x for a in as if z)") == + Expr(:generator, :x, Expr(:filter, :z, Expr(:(=), :a, :as))) + end + + @testset "try" begin + @test parsestmt("try x catch e; y end") == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + :e, + Expr(:block, LineNumberNode(1), :y)) + @test parsestmt("try x finally y end") == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + false, + false, + Expr(:block, LineNumberNode(1), :y)) + @test parsestmt("try x catch e; y finally z end") == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + :e, + Expr(:block, LineNumberNode(1), :y), + Expr(:block, LineNumberNode(1), :z)) + @test parsestmt("try x catch e; y else z end", version=v"1.8") == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + :e, + Expr(:block, LineNumberNode(1), :y), + false, + Expr(:block, LineNumberNode(1), :z)) + @test parsestmt("try x catch e; y else z finally w end", version=v"1.8") == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + :e, + Expr(:block, LineNumberNode(1), :y), + Expr(:block, LineNumberNode(1), :w), + Expr(:block, LineNumberNode(1), :z)) + # finally before catch + @test parsestmt("try x finally y catch e z end", ignore_warnings=true) == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + :e, + Expr(:block, LineNumberNode(1), :z), + Expr(:block, LineNumberNode(1), :y)) + # empty recovery + @test parsestmt("try x end", ignore_errors=true) == + Expr(:try, + Expr(:block, LineNumberNode(1), :x), + false, false, + Expr(:block, Expr(:error))) + end + + @testset "juxtapose" begin + @test parsestmt("2x") == Expr(:call, :*, 2, :x) + @test parsestmt("(2)(3)x") == Expr(:call, :*, 2, 3, :x) + end + + @testset "Core.@doc" begin + @test parsestmt("\"x\" f") == + Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(1), "x", :f) + @test parsestmt("\n\"x\" f") == + Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(2), "x", :f) + end + + @testset "String and cmd macros" begin + # Custom string macros + @test parsestmt("foo\"str\"") == + Expr(:macrocall, Symbol("@foo_str"), LineNumberNode(1), "str") + # Bare @cmd + @test parsestmt("\n`str`") == + Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), LineNumberNode(2), "str") + # Custom cmd macros + @test parsestmt("foo`str`") == + Expr(:macrocall, Symbol("@foo_cmd"), LineNumberNode(1), "str") + @test parsestmt("foo`str`flag") == + Expr(:macrocall, Symbol("@foo_cmd"), LineNumberNode(1), "str", "flag") + @test parsestmt("foo```\n a\n b```") == + Expr(:macrocall, Symbol("@foo_cmd"), LineNumberNode(1), "a\nb") + # Expr conversion distinguishes from explicit calls to a macro of the same name + @test parsestmt("@foo_cmd `str`") == + Expr(:macrocall, Symbol("@foo_cmd"), LineNumberNode(1), + Expr(:macrocall, GlobalRef(Core, Symbol("@cmd")), LineNumberNode(1), "str")) + end + + @testset "return" begin + @test parsestmt("return x") == Expr(:return, :x) + @test parsestmt("return") == Expr(:return, nothing) + end + + @testset "Large integer macros" begin + @test parsestmt("0x00000000000000001") == + Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), + nothing, "0x00000000000000001") + + @test parsestmt("(0x00000000000000001)") == + Expr(:macrocall, GlobalRef(Core, Symbol("@uint128_str")), + nothing, "0x00000000000000001") + end + + @testset "struct" begin + @test parsestmt("struct A end") == + Expr(:struct, false, :A, Expr(:block, LineNumberNode(1))) + @test parsestmt("mutable struct A end") == + Expr(:struct, true, :A, Expr(:block, LineNumberNode(1))) + + @test parsestmt("struct A <: B \n a::X \n end") == + Expr(:struct, false, Expr(:<:, :A, :B), + Expr(:block, LineNumberNode(2), Expr(:(::), :a, :X))) + @test parsestmt("struct A \n a \n b \n end") == + Expr(:struct, false, :A, + Expr(:block, LineNumberNode(2), :a, LineNumberNode(3), :b)) + @test parsestmt("struct A const a end", version=v"1.8") == + Expr(:struct, false, :A, Expr(:block, LineNumberNode(1), Expr(:const, :a))) + + @test parsestmt("struct A \n \"doc\" \n a end") == + Expr(:struct, false, :A, Expr(:block, LineNumberNode(2), "doc", :a)) + end + + @testset "typegroup" begin + @test parsestmt("typegroup\nstruct A\nend\nend", version=v"1.14") == + Expr(:typegroup, Expr(:block, LineNumberNode(2), + Expr(:struct, false, :A, Expr(:block, LineNumberNode(2))))) + end + + @testset "export" begin + @test parsestmt("export a") == Expr(:export, :a) + @test parsestmt("export @a") == Expr(:export, Symbol("@a")) + @test parsestmt("export @var\"'\"") == Expr(:export, Symbol("@'")) + @test parsestmt("export a, \n @b") == Expr(:export, :a, Symbol("@b")) + @test parsestmt("export +, ==") == Expr(:export, :+, :(==)) + @test parsestmt("export \n a") == Expr(:export, :a) + end + + @testset "global/const/local" begin + @test parsestmt("global x") == Expr(:global, :x) + @test parsestmt("local x") == Expr(:local, :x) + @test parsestmt("global x,y") == Expr(:global, :x, :y) + @test parsestmt("global const x = 1") == Expr(:const, Expr(:global, Expr(:(=), :x, 1))) + @test parsestmt("local const x = 1") == Expr(:const, Expr(:local, Expr(:(=), :x, 1))) + @test parsestmt("const global x = 1") == Expr(:const, Expr(:global, Expr(:(=), :x, 1))) + @test parsestmt("const local x = 1") == Expr(:const, Expr(:local, Expr(:(=), :x, 1))) + @test parsestmt("const x,y = 1,2") == Expr(:const, Expr(:(=), Expr(:tuple, :x, :y), Expr(:tuple, 1, 2))) + @test parsestmt("const x = 1") == Expr(:const, Expr(:(=), :x, 1)) + @test parsestmt("global x ~ 1") == Expr(:global, Expr(:call, :~, :x, 1)) + @test parsestmt("global x += 1") == Expr(:global, Expr(:+=, :x, 1)) + + # Parsing of global/local with + @test parsestmt("global (x,y)") == Expr(:global, :x, :y) + @test parsestmt("local (x,y)") == Expr(:local, :x, :y) + end + + @testset "tuples" begin + @test parsestmt("(;)") == Expr(:tuple, Expr(:parameters)) + @test parsestmt("(; a=1)") == Expr(:tuple, Expr(:parameters, Expr(:kw, :a, 1))) + @test parsestmt("(; a=1; b=2)") == + Expr(:tuple, Expr(:parameters, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))) + @test parsestmt("(a; b; c,d)") == + Expr(:tuple, Expr(:parameters, Expr(:parameters, :c, :d), :b), :a) + end + + @testset "module" begin + @test parsestmt("module A end"; version=v"1.13") == + Expr(:module, true, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1))) + @test parsestmt("module A end"; version=v"1.14") == + Expr(:module, v"1.14", true, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1))) + @test parsestmt("baremodule A end"; version=v"1.13") == + Expr(:module, false, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1))) + end + + @testset "errors" begin + @test parsestmt("--", ignore_errors=true) == + Expr(:error, "invalid operator: `--`") + @test parseall("a b", ignore_errors=true) == + Expr(:toplevel, LineNumberNode(1), :a, + LineNumberNode(1), Expr(:error, :b)) + @test parsestmt("(x", ignore_errors=true) == + Expr(:block, :x, Expr(:error)) + @test parsestmt("x do", ignore_errors=true) == + Expr(:block, :x, Expr(:error, :do)) + @test parsestmt("x var\"y\"", ignore_errors=true) == + Expr(:block, :x, Expr(:error, :var, ErrorVal(), "y", ErrorVal())) + @test parsestmt("var\"y", ignore_errors=true) == + Expr(:var, :y, Expr(:error)) + end + + @testset "import" begin + @test parsestmt("import A") == Expr(:import, Expr(:., :A)) + @test parsestmt("import A.(:b).:c: x.:z", ignore_warnings=true) == + Expr(:import, Expr(Symbol(":"), Expr(:., :A, :b, :c), Expr(:., :x, :z))) + # Stupid parens and quotes in import paths + @test parsestmt("import A.:+", ignore_warnings=true) == + Expr(:import, Expr(:., :A, :+)) + @test parsestmt("import A.(:+)", ignore_warnings=true) == + Expr(:import, Expr(:., :A, :+)) + @test parsestmt("import A.:(+)", ignore_warnings=true) == + Expr(:import, Expr(:., :A, :+)) + @test parsestmt("import A.:(+) as y", ignore_warnings=true, version=v"1.6") == + Expr(:import, Expr(:as, Expr(:., :A, :+), :y)) + end +end + +@testset "SyntaxNode->Expr conversion" begin + src = repeat('a', 1000) * '\n' * "@hi" + @test Expr(parsestmt(SyntaxNode, SubString(src, 1001:lastindex(src)))) == + Expr(:macrocall, Symbol("@hi"), LineNumberNode(2)) +end diff --git a/JuliaSyntax/test/fuzz_test.jl b/JuliaSyntax/test/fuzz_test.jl new file mode 100644 index 0000000000000..63ec83ebe1186 --- /dev/null +++ b/JuliaSyntax/test/fuzz_test.jl @@ -0,0 +1,1023 @@ +using .JuliaSyntax: tokenize +import Logging +import Test + +# Parser fuzz testing tools. + +const all_tokens = [ + "#x\n" + "#==#" + " " + "\t" + "\n" + "x" + "@" + "," + ";" + + "baremodule" + "begin" + "break" + "const" + "continue" + "do" + "export" + "for" + "function" + "global" + "if" + "import" + "let" + "local" + "macro" + "module" + "quote" + "return" + "struct" + "try" + "using" + "while" + "catch" + "finally" + "else" + "elseif" + "end" + "abstract" + "as" + "doc" + "mutable" + "outer" + "primitive" + "type" + "var" + + "1" + "0b1" + "0x1" + "0o1" + "1.0" + "1.0f0" + "\"s\"" + "'c'" + "`s`" + "true" + "false" + + "[" + "]" + "{" + "}" + "(" + ")" + "\"" + "\"\"\"" + "`" + "```" + + "=" + "+=" + "-=" # Also used for "−=" + "−=" + "*=" + "/=" + "//=" + "|=" + "^=" + "÷=" + "%=" + "<<=" + ">>=" + ">>>=" + "\\=" + "&=" + ":=" + "~" + "\$=" + "⊻=" + "≔" + "⩴" + "≕" + + "=>" + + "?" + + "-->" + "<--" + "<-->" + "←" + "→" + "↔" + "↚" + "↛" + "↞" + "↠" + "↢" + "↣" + "↤" + "↦" + "↮" + "⇎" + "⇍" + "⇏" + "⇐" + "⇒" + "⇔" + "⇴" + "⇶" + "⇷" + "⇸" + "⇹" + "⇺" + "⇻" + "⇼" + "⇽" + "⇾" + "⇿" + "⟵" + "⟶" + "⟷" + "⟹" + "⟺" + "⟻" + "⟼" + "⟽" + "⟾" + "⟿" + "⤀" + "⤁" + "⤂" + "⤃" + "⤄" + "⤅" + "⤆" + "⤇" + "⤌" + "⤍" + "⤎" + "⤏" + "⤐" + "⤑" + "⤔" + "⤕" + "⤖" + "⤗" + "⤘" + "⤝" + "⤞" + "⤟" + "⤠" + "⥄" + "⥅" + "⥆" + "⥇" + "⥈" + "⥊" + "⥋" + "⥎" + "⥐" + "⥒" + "⥓" + "⥖" + "⥗" + "⥚" + "⥛" + "⥞" + "⥟" + "⥢" + "⥤" + "⥦" + "⥧" + "⥨" + "⥩" + "⥪" + "⥫" + "⥬" + "⥭" + "⥰" + "⧴" + "⬱" + "⬰" + "⬲" + "⬳" + "⬴" + "⬵" + "⬶" + "⬷" + "⬸" + "⬹" + "⬺" + "⬻" + "⬼" + "⬽" + "⬾" + "⬿" + "⭀" + "⭁" + "⭂" + "⭃" + "⭄" + "⭇" + "⭈" + "⭉" + "⭊" + "⭋" + "⭌" + "￩" + "￫" + "⇜" + "⇝" + "↜" + "↝" + "↩" + "↪" + "↫" + "↬" + "↼" + "↽" + "⇀" + "⇁" + "⇄" + "⇆" + "⇇" + "⇉" + "⇋" + "⇌" + "⇚" + "⇛" + "⇠" + "⇢" + "↷" + "↶" + "↺" + "↻" + + "||" + + "&&" + + "<:" + ">:" + ">" + "<" + ">=" + "≥" + "<=" + "≤" + "==" + "===" + "≡" + "!=" + "≠" + "!==" + "≢" + "∈" + "in" + "isa" + "∉" + "∋" + "∌" + "⊆" + "⊈" + "⊂" + "⊄" + "⊊" + "∝" + "∊" + "∍" + "∥" + "∦" + "∷" + "∺" + "∻" + "∽" + "∾" + "≁" + "≃" + "≂" + "≄" + "≅" + "≆" + "≇" + "≈" + "≉" + "≊" + "≋" + "≌" + "≍" + "≎" + "≐" + "≑" + "≒" + "≓" + "≖" + "≗" + "≘" + "≙" + "≚" + "≛" + "≜" + "≝" + "≞" + "≟" + "≣" + "≦" + "≧" + "≨" + "≩" + "≪" + "≫" + "≬" + "≭" + "≮" + "≯" + "≰" + "≱" + "≲" + "≳" + "≴" + "≵" + "≶" + "≷" + "≸" + "≹" + "≺" + "≻" + "≼" + "≽" + "≾" + "≿" + "⊀" + "⊁" + "⊃" + "⊅" + "⊇" + "⊉" + "⊋" + "⊏" + "⊐" + "⊑" + "⊒" + "⊜" + "⊩" + "⊬" + "⊮" + "⊰" + "⊱" + "⊲" + "⊳" + "⊴" + "⊵" + "⊶" + "⊷" + "⋍" + "⋐" + "⋑" + "⋕" + "⋖" + "⋗" + "⋘" + "⋙" + "⋚" + "⋛" + "⋜" + "⋝" + "⋞" + "⋟" + "⋠" + "⋡" + "⋢" + "⋣" + "⋤" + "⋥" + "⋦" + "⋧" + "⋨" + "⋩" + "⋪" + "⋫" + "⋬" + "⋭" + "⋲" + "⋳" + "⋴" + "⋵" + "⋶" + "⋷" + "⋸" + "⋹" + "⋺" + "⋻" + "⋼" + "⋽" + "⋾" + "⋿" + "⟈" + "⟉" + "⟒" + "⦷" + "⧀" + "⧁" + "⧡" + "⧣" + "⧤" + "⧥" + "⩦" + "⩧" + "⩪" + "⩫" + "⩬" + "⩭" + "⩮" + "⩯" + "⩰" + "⩱" + "⩲" + "⩳" + "⩵" + "⩶" + "⩷" + "⩸" + "⩹" + "⩺" + "⩻" + "⩼" + "⩽" + "⩾" + "⩿" + "⪀" + "⪁" + "⪂" + "⪃" + "⪄" + "⪅" + "⪆" + "⪇" + "⪈" + "⪉" + "⪊" + "⪋" + "⪌" + "⪍" + "⪎" + "⪏" + "⪐" + "⪑" + "⪒" + "⪓" + "⪔" + "⪕" + "⪖" + "⪗" + "⪘" + "⪙" + "⪚" + "⪛" + "⪜" + "⪝" + "⪞" + "⪟" + "⪠" + "⪡" + "⪢" + "⪣" + "⪤" + "⪥" + "⪦" + "⪧" + "⪨" + "⪩" + "⪪" + "⪫" + "⪬" + "⪭" + "⪮" + "⪯" + "⪰" + "⪱" + "⪲" + "⪳" + "⪴" + "⪵" + "⪶" + "⪷" + "⪸" + "⪹" + "⪺" + "⪻" + "⪼" + "⪽" + "⪾" + "⪿" + "⫀" + "⫁" + "⫂" + "⫃" + "⫄" + "⫅" + "⫆" + "⫇" + "⫈" + "⫉" + "⫊" + "⫋" + "⫌" + "⫍" + "⫎" + "⫏" + "⫐" + "⫑" + "⫒" + "⫓" + "⫔" + "⫕" + "⫖" + "⫗" + "⫘" + "⫙" + "⫷" + "⫸" + "⫹" + "⫺" + "⊢" + "⊣" + "⟂" + "⫪" + "⫫" + + "<|" + "|>" + + ":" + ".." + "…" + "⁝" + "⋮" + "⋱" + "⋰" + "⋯" + + "\$" + "+" + "-" # also used for "−" + "−" + "++" + "⊕" + "⊖" + "⊞" + "⊟" + "|" + "∪" + "∨" + "⊔" + "±" + "∓" + "∔" + "∸" + "≏" + "⊎" + "⊻" + "⊽" + "⋎" + "⋓" + "⧺" + "⧻" + "⨈" + "⨢" + "⨣" + "⨤" + "⨥" + "⨦" + "⨧" + "⨨" + "⨩" + "⨪" + "⨫" + "⨬" + "⨭" + "⨮" + "⨹" + "⨺" + "⩁" + "⩂" + "⩅" + "⩊" + "⩌" + "⩏" + "⩐" + "⩒" + "⩔" + "⩖" + "⩗" + "⩛" + "⩝" + "⩡" + "⩢" + "⩣" + "¦" + + "*" + "/" + "÷" + "%" + "⋅" # also used for lookalikes "·" and "·" + "·" + "·" + "∘" + "×" + "\\" + "&" + "∩" + "∧" + "⊗" + "⊘" + "⊙" + "⊚" + "⊛" + "⊠" + "⊡" + "⊓" + "∗" + "∙" + "∤" + "⅋" + "≀" + "⊼" + "⋄" + "⋆" + "⋇" + "⋉" + "⋊" + "⋋" + "⋌" + "⋏" + "⋒" + "⟑" + "⦸" + "⦼" + "⦾" + "⦿" + "⧶" + "⧷" + "⨇" + "⨰" + "⨱" + "⨲" + "⨳" + "⨴" + "⨵" + "⨶" + "⨷" + "⨸" + "⨻" + "⨼" + "⨽" + "⩀" + "⩃" + "⩄" + "⩋" + "⩍" + "⩎" + "⩑" + "⩓" + "⩕" + "⩘" + "⩚" + "⩜" + "⩞" + "⩟" + "⩠" + "⫛" + "⊍" + "▷" + "⨝" + "⟕" + "⟖" + "⟗" + "⌿" + "⨟" + + "//" + + "<<" + ">>" + ">>>" + + "^" + "↑" + "↓" + "⇵" + "⟰" + "⟱" + "⤈" + "⤉" + "⤊" + "⤋" + "⤒" + "⤓" + "⥉" + "⥌" + "⥍" + "⥏" + "⥑" + "⥔" + "⥕" + "⥘" + "⥙" + "⥜" + "⥝" + "⥠" + "⥡" + "⥣" + "⥥" + "⥮" + "⥯" + "￪" + "￬" + + "::" + + "where" + + "." + + "!" + "'" + ".'" + "->" + + "¬" + "√" + "∛" + "∜" +] + +const cutdown_tokens = [ + "#x\n" + "#==#" + " " + "\t" + "\n" + "x" + "β" + "@" + "," + ";" + + "baremodule" + "begin" + "break" + "const" + "continue" + "do" + "export" + "for" + "function" + "global" + "if" + "import" + "let" + "local" + "macro" + "module" + "quote" + "return" + "struct" + "try" + "using" + "while" + "catch" + "finally" + "else" + "elseif" + "end" + "abstract" + "as" + "doc" + "mutable" + "outer" + "primitive" + "type" + "var" + + "1" + "0b1" + "0x1" + "0o1" + "1.0" + "1.0f0" + "\"s\"" + "'c'" + "`s`" + "true" + "false" + + "[" + "]" + "{" + "}" + "(" + ")" + "\"" + "\"\"\"" + "`" + "```" + + "=" + "+=" + "~" + + "=>" + + "?" + + "-->" + + "||" + + "&&" + + "<:" + ">:" + ">" + "<" + ">=" + "<=" + "==" + "===" + "!=" + + "<|" + "|>" + + ":" + ".." + "…" + + "\$" + "+" + "−" + "-" + "|" + + "*" + "/" + "⋅" # also used for lookalikes "·" and "·" + "·" + "\\" + + "//" + + "<<" + + "^" + + "::" + + "where" + + "." + + "!" + "'" + "->" + + "√" +] + +#------------------------------------------------------------------------------- +# Parsing functions for use with fuzz_test + +function try_parseall_failure(str) + try + JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, str, ignore_errors=true); + return nothing + catch exc + !(exc isa InterruptException) || rethrow() + rstr = reduce_text(str, parser_throws_exception) + @error "Parser threw exception" rstr exception=current_exceptions() + return rstr + end +end + +function try_hook_failure(str) + try + test_logger = Test.TestLogger() + Logging.with_logger(test_logger) do + try + Meta_parseall(str) + catch exc + exc isa Meta.ParseError || exc isa JuliaSyntax.ParseError || rethrow() + end + end + if !isempty(test_logger.logs) + return str + end + catch exc + return str + end + return nothing +end + +#------------------------------------------------------------------------------- +"""Delete `nlines` adjacent lines from code, at `niters` randomly chosen positions""" +function delete_lines(lines, nlines, niters) + selection = trues(length(lines)) + for j=1:niters + i = rand(1:length(lines)-nlines) + selection[i:i+nlines] .= false + end + join(lines[selection], '\n') +end + +"""Delete `ntokens` adjacent tokens from code, at `niters` randomly chosen positions""" +function delete_tokens(code, tokens, ntokens, niters) + # [ aa bbbb cc d eeeeee ] + # | | | | | | + selection = trues(length(tokens)) + for j=1:niters + i = rand(1:length(tokens)-ntokens) + selection[i:i+ntokens] .= false + end + io = IOBuffer() + i = 1 + while true + while i <= length(selection) && !selection[i] + i += 1 + end + if i > length(selection) + break + end + first_ind = first(tokens[i].range) + while selection[i] && i < length(selection) + i += 1 + end + last_ind = last(tokens[i].range) + write(io, @view code[first_ind:last_ind]) + if i == length(selection) + break + end + end + return String(take!(io)) +end + +#------------------------------------------------------------------------------- +# Generators for "potentially bad input" + +""" +Fuzz test parser against all tuples of length `N` with elements taken from +`tokens`. +""" +function product_token_fuzz(tokens, N) + (join(ts) for ts in Iterators.product([tokens for _ in 1:N]...)) +end + +function random_token_fuzz(tokens, ntokens, ntries) + (join(rand(tokens, ntokens)) for _ in 1:ntries) +end + +""" +Fuzz test parser against randomly generated binary strings +""" +function random_binary_fuzz(nbytes, N) + (String(rand(UInt8, nbytes)) for _ in 1:N) +end + +""" +Fuzz test by deleting random lines of some given source `code` +""" +function deleted_line_fuzz(code, N; nlines=10, niters=10) + lines = split(code, '\n') + (delete_lines(lines, nlines, niters) for _=1:N) +end + +""" +Fuzz test by deleting random tokens from given source `code` +""" +function deleted_token_fuzz(code, N; ntokens=10, niters=10) + ts = tokenize(code) + (delete_tokens(code, ts, ntokens, niters) for _=1:N) +end + +""" +Fuzz test a parsing function by trying it with many "bad" input strings. + +`try_parsefail` should return `nothing` when the parser succeeds, and return a +string (or reduced string) when parsing fails'.. +""" +function fuzz_test(try_parsefail::Function, bad_input_iter) + error_strings = [] + for str in bad_input_iter + res = try_parsefail(str) + if !isnothing(res) + push!(error_strings, res) + end + end + return error_strings +end + + +# Examples +# +# fuzz_test(try_hook_failure, product_token_fuzz(cutdown_tokens, 2)) +# fuzz_test(try_parseall_failure, product_token_fuzz(cutdown_tokens, 2)) diff --git a/JuliaSyntax/test/green_node.jl b/JuliaSyntax/test/green_node.jl new file mode 100644 index 0000000000000..0c3be65873c2e --- /dev/null +++ b/JuliaSyntax/test/green_node.jl @@ -0,0 +1,68 @@ +@testset "GreenNode" begin + t = parsestmt(GreenNode, "aa + b") + + @test span(t) == 6 + @test !is_leaf(t) + @test head(t) == SyntaxHead(K"call", 0x0088) + @test span.(children(t)) == [2,1,1,1,1] + @test head.(children(t)) == [ + SyntaxHead(K"Identifier", 0x0000) + SyntaxHead(K"Whitespace", 0x0001) + SyntaxHead(K"Identifier", 0x0000) + SyntaxHead(K"Whitespace", 0x0001) + SyntaxHead(K"Identifier", 0x0000) + ] + + @test numchildren(t) == 5 + @test !is_leaf(t) + @test is_leaf(t[1]) + + @test t[1] === children(t)[1] + @test t[2:4] == [t[2],t[3],t[4]] + @test firstindex(t) == 1 + @test lastindex(t) == 5 + + t2 = parsestmt(GreenNode, "aa + b") + @test t == t2 + @test t !== t2 + + text = "f(@x(y), z)" + @test sprint(show, MIME("text/plain"), parsestmt(GreenNode, text)) == + """ + 1:11 │[call] + 1:1 │ Identifier ✔ + 2:2 │ ( + 3:7 │ [macrocall] + 3:4 │ [macro_name] + 3:3 │ @ + 4:4 │ Identifier ✔ + 5:5 │ ( + 6:6 │ Identifier ✔ + 7:7 │ ) + 8:8 │ , + 9:9 │ Whitespace + 10:10 │ Identifier ✔ + 11:11 │ ) + """ + + @test sprint(show, MIME("text/plain"), parsestmt(GreenNode, text), text) == + """ + 1:11 │[call] + 1:1 │ Identifier ✔ "f" + 2:2 │ ( "(" + 3:7 │ [macrocall] + 3:4 │ [macro_name] + 3:3 │ @ "@" + 4:4 │ Identifier ✔ "x" + 5:5 │ ( "(" + 6:6 │ Identifier ✔ "y" + 7:7 │ ) ")" + 8:8 │ , "," + 9:9 │ Whitespace " " + 10:10 │ Identifier ✔ "z" + 11:11 │ ) ")" + """ + + @test sprint(show, parsestmt(GreenNode, "a + bb - f(ccc)")) == + "(call-i (call-i 1-1::Identifier 2-2::Whitespace-t 3-3::Identifier 4-4::Whitespace-t 5-6::Identifier) 7-7::Whitespace-t 8-8::Identifier 9-9::Whitespace-t (call 10-10::Identifier 11-11::(-t 12-14::Identifier 15-15::)-t))" +end diff --git a/JuliaSyntax/test/hooks.jl b/JuliaSyntax/test/hooks.jl new file mode 100644 index 0000000000000..6965a7d6efed2 --- /dev/null +++ b/JuliaSyntax/test/hooks.jl @@ -0,0 +1,551 @@ +function _unwrap_parse_error(core_hook_result) + @test Meta.isexpr(core_hook_result[1], :error, 1) + err = core_hook_result[1].args[1] + if JuliaSyntax._has_v1_10_hooks + @test err isa Meta.ParseError + return err.detail + else + @test err isa JuliaSyntax.ParseError + return err + end +end + +@testset "Hooks for Core integration" begin + @testset "whitespace and comment parsing" begin + @test JuliaSyntax.core_parser_hook("", "somefile", 1, 0, :statement) == Core.svec(nothing, 0) + @test JuliaSyntax.core_parser_hook("", "somefile", 1, 0, :statement) == Core.svec(nothing, 0) + + @test JuliaSyntax.core_parser_hook(" ", "somefile", 1, 2, :statement) == Core.svec(nothing,2) + @test JuliaSyntax.core_parser_hook(" #==# ", "somefile", 1, 6, :statement) == Core.svec(nothing,6) + + @test JuliaSyntax.core_parser_hook(" x \n", "somefile", 1, 0, :statement) == Core.svec(:x,4) + @test JuliaSyntax.core_parser_hook(" x \n", "somefile", 1, 0, :atom) == Core.svec(:x,2) + + # https://github.com/JuliaLang/JuliaSyntax.jl/issues/316#issuecomment-1870294857 + stmtstr = + """ + plus(a, b) = a + b + + # Issue #81 + f() = nothing + """ + @test JuliaSyntax.core_parser_hook(stmtstr, "somefile", 1, 0, :statement)[2] == 19 + end + + @testset "filename and lineno" begin + ex = JuliaSyntax.core_parser_hook("@a", "somefile", 1, 0, :statement)[1] + @test Meta.isexpr(ex, :macrocall) + @test ex.args[2] == LineNumberNode(1, "somefile") + + ex = JuliaSyntax.core_parser_hook("@a", "otherfile", 2, 0, :statement)[1] + @test ex.args[2] == LineNumberNode(2, "otherfile") + + # Errors also propagate file & lineno + err = _unwrap_parse_error( + JuliaSyntax.core_parser_hook("[x)", "f1", 1, 0, :statement) + ) + @test err isa JuliaSyntax.ParseError + @test filename(err) == "f1" + @test err.source.first_line == 1 + err = _unwrap_parse_error( + JuliaSyntax.core_parser_hook("[x)", "f2", 2, 0, :statement) + ) + @test err isa JuliaSyntax.ParseError + @test filename(err) == "f2" + @test err.source.first_line == 2 + + # Errors including nontrivial offset indices + err = _unwrap_parse_error( + JuliaSyntax.core_parser_hook("a\nh{x)\nb", "test.jl", 1, 2, :statement) + ) + @test err isa JuliaSyntax.ParseError + @test err.source.first_line == 1 + @test err.diagnostics[1].first_byte == 6 + @test err.diagnostics[1].last_byte == 5 + @test err.diagnostics[1].message == "Expected `}` or `,`" + end + + @testset "toplevel errors" begin + ex = JuliaSyntax.core_parser_hook("a\nb\n[x,\ny)", "somefile", 1, 0, :all)[1] + @test ex.head == :toplevel + @test ex.args[1:5] == [ + LineNumberNode(1, "somefile"), + :a, + LineNumberNode(2, "somefile"), + :b, + LineNumberNode(4, "somefile"), + ] + @test Meta.isexpr(ex.args[6], :error) + + ex = JuliaSyntax.core_parser_hook("x.", "somefile", 0, 0, :all)[1] + @test ex.head == :toplevel + @test ex.args[2].head == :incomplete + end + + @testset "enable_in_core!" begin + JuliaSyntax.enable_in_core!() + + @test Meta.parse("x + 1") == :(x + 1) + @test Meta.parse("x + 1", 1) == (:(x + 1), 6) + + # Test that parsing statements incrementally works and stops after + # whitespace / comment trivia + @test Meta.parse("x + 1\n(y)\n", 1) == (:(x + 1), 7) + @test Meta.parse("x + 1\n(y)\n", 7) == (:y, 11) + @test Meta.parse(" x#==#", 1) == (:x, 7) + @test Meta.parse(" #==# ", 1) == (nothing, 7) + + # Check the exception type that Meta.parse throws + if JuliaSyntax._has_v1_10_hooks + @test_throws Meta.ParseError Meta.parse("[x)") + @test_throws Meta.ParseError eval(Meta.parse("[x)", raise=false)) + @test_throws Meta.ParseError eval(Meta.parse("(x")) # Expr(:incomplete) + else + @test_throws JuliaSyntax.ParseError Meta.parse("[x)") + end + + # Check custom string types defined in a world age later than + # enable_in_core!() can be passed to Meta.parse() + mystr = @eval begin + struct MyString <: AbstractString + x::String + end + Base.String(s::MyString) = s.x + Base.ncodeunits(s::MyString) = ncodeunits(s.x) + + MyString("hi") + end + @test Meta.parse(mystr) == :hi + + err = Meta.parse("\"") + @test Meta.isexpr(err, :incomplete) + if JuliaSyntax._has_v1_10_hooks + @test err.args[1] isa Meta.ParseError + exc = err.args[1] + @test exc.msg == "ParseError:\n# Error @ none:1:2\n\"\n#└ ── unterminated string literal" + @test exc.detail isa JuliaSyntax.ParseError + @test exc.detail.incomplete_tag === :string + else + @test err.args[1] isa String + end + + JuliaSyntax.enable_in_core!(false) + end + + if isdefined(Base, :VersionedParse) && isdefined(Base, :set_syntax_version) + @test VERSION > v"1.13" + @testset "`activate!` and `Base.set_syntax_version` should work together" begin + JuliaSyntax.enable_in_core!() + try + parse_in(mod, str) = Meta.parse(str; mod=mod, raise=false) + m = Module(:Mod) + w0 = Base.get_world_counter() + typegroup_src = "typegroup struct T end end" + @test Meta.isexpr(invokelatest(parse_in, m, typegroup_src), :typegroup) + Base.set_syntax_version(m, v"1.13") + @test Meta.isexpr(invokelatest(parse_in, m, typegroup_src), :error) + Base.set_syntax_version(m, VERSION) + @test Meta.isexpr(invokelatest(parse_in, m, typegroup_src), :typegroup) + + # a parser defined after a Core._parse caller should work + old_parser = Core._parse + try + late_parser = @eval m begin + late_parser(code, args...) = + Core.svec(:late, ncodeunits(code)) + end + JuliaSyntax._set_core_parse_hook(late_parser) + w1 = Base.get_world_counter() + @test Base.invoke_in_world(w0, Meta.parse, "x") === :late + @test Base.invoke_in_world(w1, Meta.parse, "x") === :late + finally + JuliaSyntax._set_core_parse_hook(old_parser) + end + finally + JuliaSyntax.enable_in_core!(false) + end + end + end + + @testset "Expr(:incomplete)" begin + for (str, tag) in [ + "\"" => :string + "\"\$foo" => :string + "#=" => :comment + "'" => :char + "'a" => :char + "`" => :cmd + "(" => :other + "[" => :other + "begin" => :block + "quote" => :block + "let" => :block + "let;" => :block + "for" => :other + "for x=xs" => :block + "function" => :other + "function f()" => :block + "macro" => :other + "macro f()" => :block + "f() do" => :other + "f() do x" => :block + "module" => :other + "module X" => :block + "baremodule" => :other + "baremodule X" => :block + "mutable struct" => :other + "mutable struct X" => :block + "struct" => :other + "struct X" => :block + "if" => :other + "if x" => :block + "while" => :other + "while x" => :block + "try" => :block + # could be `try x catch exc body end` or `try x catch ; body end` + "try x catch" => :block + "using" => :other + "import" => :other + "local" => :other + "global" => :other + + "1 == 2 ?" => :other + "1 == 2 ? 3 :" => :other + "1," => :other + "1, " => :other + "1,\n" => :other + "1, \n" => :other + "f(1, " => :other + "[x " => :other + "( " => :other + + # Reference parser fails to detect incomplete exprs in this case + "(x for y" => :other + + # Syntax which may be an error but is not incomplete + "" => :none + ")" => :none + "1))" => :none + "a b" => :none + "()x" => :none + "." => :none + + # Some error tokens which cannot be made complete by appending more characters + "1.e1." => :none + "\u200b" => :none + "x #=\xf5b\n=#" => :none + "₁" => :none + "0x1.0\n" => :none + "\"\$x෴\"" => :none + "10e1000" => :none + + # Multiline input with comments (#519) + "function f()\nbody #comment" => :block + "a = [\n1,\n2, #comment" => :other + + # Extended set of cases extracted from the REPL stdlib tests. + # There is some redundancy here, but we've mostly left these + # here because incomplete-detection is partly heuristic and + # it's good to have a wide variety of incomplete expressions. + # + # The "desired" incomplete tag here was generated from the + # flisp parser. + "Main.CompletionFoo." => :other + "Base.return_types(getin" => :other + "test7()." => :other + "(3,2)." => :other + "Base.print(\"lol" => :string + "run(`lol" => :cmd + "copy(A')." => :other + "cd(\"path_to_an_empty_folder_should_not_complete_latex\\\\\\alpha" => :string + "\"C:\\\\ \\alpha" => :string + "cd(\"C:\\U" => :string + "max(" => :other + "!(" => :other + "!isnothing(" => :other + "!!isnothing(" => :other + "CompletionFoo.test(1, 1, " => :other + "CompletionFoo.test(CompletionFoo.array," => :other + "CompletionFoo.test(1,1,1," => :other + "CompletionFoo.test1(Int," => :other + "CompletionFoo.test1(Float64," => :other + "prevind(\"θ\",1," => :other + "(1, CompletionFoo.test2(\")\"," => :other + "(1, CompletionFoo.test2(')'," => :other + "(1, CompletionFoo.test2(`')'`," => :other + "CompletionFoo.test3([1, 2] .+ CompletionFoo.varfloat," => :other + "CompletionFoo.test3([1.,2.], 1.," => :other + "CompletionFoo.test4(\"e\",r\" \"," => :other + "CompletionFoo.test5(broadcast((x,y)->x==y, push!(Base.split(\"\",' '),\"\",\"\"), \"\")," => :other + "CompletionFoo.test5(Bool[x==1 for x=1:4]," => :other + "CompletionFoo.test4(CompletionFoo.test_y_array[1]()[1], CompletionFoo.test_y_array[1]()[2], " => :other + "CompletionFoo.test4(\"\\\"\"," => :other + "convert(" => :other + "convert(" => :other + "CompletionFoo.test5(AbstractArray[Bool[]][1]," => :other + "CompletionFoo.test3(@time([1, 2] .+ CompletionFoo.varfloat)," => :other + "CompletionFoo.kwtest( " => :other + "CompletionFoo.kwtest(;" => :other + "CompletionFoo.kwtest(; x=1, " => :other + "CompletionFoo.kwtest(; kw=1, " => :other + "CompletionFoo.kwtest(x=1, " => :other + "CompletionFoo.kwtest(x=1; " => :other + "CompletionFoo.kwtest(x=kw=1, " => :other + "CompletionFoo.kwtest(; x=kw=1, " => :other + "CompletionFoo.kwtest2(1, x=1," => :other + "CompletionFoo.kwtest2(1; x=1, " => :other + "CompletionFoo.kwtest2(1, x=1; " => :other + "CompletionFoo.kwtest2(1, kw=1, " => :other + "CompletionFoo.kwtest2(1; kw=1, " => :other + "CompletionFoo.kwtest2(1, kw=1; " => :other + "CompletionFoo.kwtest2(y=3, 1, " => :other + "CompletionFoo.kwtest2(y=3, 1; " => :other + "CompletionFoo.kwtest2(kw=3, 1, " => :other + "CompletionFoo.kwtest2(kw=3, 1; " => :other + "CompletionFoo.kwtest2(1; " => :other + "CompletionFoo.kwtest2(1, " => :other + "CompletionFoo.kwtest4(x23=18, x; " => :other + "CompletionFoo.kwtest4(x23=18, x, " => :other + "CompletionFoo.kwtest4(x23=18, " => :other + "CompletionFoo.kwtest5(3, somekwarg=6," => :other + "CompletionFoo.kwtest5(3, somekwarg=6, anything, " => :other + "CompletionFoo.?([1,2,3], 2.0" => :other + "CompletionFoo.?('c'" => :other + "CompletionFoo.?(false, \"a\", 3, " => :other + "CompletionFoo.?(false, \"a\", 3, " => :other + "CompletionFoo.?(\"a\", 3, " => :other + "CompletionFoo.?(; " => :other + "CompletionFoo.?(" => :other + "CompletionFoo.test10(z, Integer[]...," => :other + "CompletionFoo.test10(3, Integer[]...," => :other + "CompletionFoo.test10(3, 4," => :other + "CompletionFoo.test10(3, 4, 5," => :other + "CompletionFoo.test10(z, z, 0, " => :other + "CompletionFoo.test10(\"a\", Union{Signed,Bool,String}[3][1], " => :other + "CompletionFoo.test11(Integer[false][1], Integer[14][1], " => :other + "CompletionFoo.test11(Integer[-7][1], Integer[0x6][1], 6," => :other + "CompletionFoo.test11(3, 4," => :other + "CompletionFoo.test11(0x8, 5," => :other + "CompletionFoo.test11(0x8, 'c'," => :other + "CompletionFoo.test11('d', 3," => :other + "CompletionFoo.test!12(" => :other + "CompletionFoo.kwtest(; x=2, y=4; kw=3, " => :other + "CompletionFoo.kwtest(x=2; y=4; " => :other + "CompletionFoo.kwtest((x=y)=4, " => :other + "CompletionFoo.kwtest(; (x=y)=4, " => :other + "CompletionFoo.kwtest(; w...=16, " => :other + "CompletionFoo.kwtest(; 2, " => :other + "CompletionFoo.kwtest(; 2=3, " => :other + "CompletionFoo.kwtest3(im; (true ? length : length), " => :other + "CompletionFoo.kwtest.(x=2; y=4; " => :other + "CompletionFoo.kwtest.(; w...=16, " => :other + "(1+2im)." => :other + "((1+2im))." => :other + "CompletionFoo.test_y_array[1]." => :other + "CompletionFoo.named." => :other + "#=\n\\alpha" => :comment + "#=\nmax" => :comment + "using " => :other + "(max" => :other + "@show \"/dev/nul" => :string + "@show \"/tm" => :string + "@show \"/dev/nul" => :string + "(Iter" => :other + "\"/tmp/jl_4sjOtz/tmpfoob" => :string + "\"~" => :string + "\"~user" => :string + "\"/tmp/jl_Mn9Rbz/selfsym" => :string + "\"~/ka8w5rsz" => :string + "\"foo~bar" => :string + "\"~/Zx6Wa0GkC" => :string + "\"~/Zx6Wa0GkC0" => :string + "\"~/Zx6Wa0GkC0/my_" => :string + "\"~/Zx6Wa0GkC0/my_file" => :string + "cd(\"folder_do_not_exist_77/file" => :string + "CompletionFoo.tuple." => :other + "CompletionFoo.test_dict[\"ab" => :string + "CompletionFoo.test_dict[\"abcd" => :string + "CompletionFoo.test_dict[ \"abcd" => :string + "CompletionFoo.test_dict[\"abcd" => :string + "CompletionFoo.test_dict[:b" => :other + "CompletionFoo.test_dict[:bar2" => :other + "CompletionFoo.test_dict[Ba" => :other + "CompletionFoo.test_dict[occ" => :other + "CompletionFoo.test_dict[`l" => :cmd + "CompletionFoo.test_dict[6" => :other + "CompletionFoo.test_dict[66" => :other + "CompletionFoo.test_dict[(" => :other + "CompletionFoo.test_dict[\"\\alp" => :string + "CompletionFoo.test_dict[\"\\alpha" => :string + "CompletionFoo.test_dict[\"α" => :string + "CompletionFoo.test_dict[:α" => :other + "CompletionFoo.test_dict[" => :other + "CompletionFoo.test_customdict[\"ab" => :string + "CompletionFoo.test_customdict[\"abcd" => :string + "CompletionFoo.test_customdict[ \"abcd" => :string + "CompletionFoo.test_customdict[\"abcd" => :string + "CompletionFoo.test_customdict[:b" => :other + "CompletionFoo.test_customdict[:bar2" => :other + "CompletionFoo.test_customdict[Ba" => :other + "CompletionFoo.test_customdict[occ" => :other + "CompletionFoo.test_customdict[`l" => :cmd + "CompletionFoo.test_customdict[6" => :other + "CompletionFoo.test_customdict[66" => :other + "CompletionFoo.test_customdict[(" => :other + "CompletionFoo.test_customdict[\"\\alp" => :string + "CompletionFoo.test_customdict[\"\\alpha" => :string + "CompletionFoo.test_customdict[\"α" => :string + "CompletionFoo.test_customdict[:α" => :other + "CompletionFoo.test_customdict[" => :other + "test_repl_comp_dict[\"ab" => :string + "test_repl_comp_dict[\"abcd" => :string + "test_repl_comp_dict[ \"abcd" => :string + "test_repl_comp_dict[\"abcd" => :string + "test_repl_comp_dict[:b" => :other + "test_repl_comp_dict[:bar2" => :other + "test_repl_comp_dict[Ba" => :other + "test_repl_comp_dict[occ" => :other + "test_repl_comp_dict[`l" => :cmd + "test_repl_comp_dict[6" => :other + "test_repl_comp_dict[66" => :other + "test_repl_comp_dict[(" => :other + "test_repl_comp_dict[\"\\alp" => :string + "test_repl_comp_dict[\"\\alpha" => :string + "test_repl_comp_dict[\"α" => :string + "test_repl_comp_dict[:α" => :other + "test_repl_comp_dict[" => :other + "test_repl_comp_customdict[\"ab" => :string + "test_repl_comp_customdict[\"abcd" => :string + "test_repl_comp_customdict[ \"abcd" => :string + "test_repl_comp_customdict[\"abcd" => :string + "test_repl_comp_customdict[:b" => :other + "test_repl_comp_customdict[:bar2" => :other + "test_repl_comp_customdict[Ba" => :other + "test_repl_comp_customdict[occ" => :other + "test_repl_comp_customdict[`l" => :cmd + "test_repl_comp_customdict[6" => :other + "test_repl_comp_customdict[66" => :other + "test_repl_comp_customdict[(" => :other + "test_repl_comp_customdict[\"\\alp" => :string + "test_repl_comp_customdict[\"\\alpha" => :string + "test_repl_comp_customdict[\"α" => :string + "test_repl_comp_customdict[:α" => :other + "test_repl_comp_customdict[" => :other + "CompletionFoo.kwtest3(a;foob" => :other + "CompletionFoo.kwtest3(a; le" => :other + "CompletionFoo.kwtest3.(a;\nlength" => :other + "CompletionFoo.kwtest3(a, length=4, l" => :other + "CompletionFoo.kwtest3(a; kwargs..., fo" => :other + "CompletionFoo.kwtest3(a; another!kwarg=0, le" => :other + "CompletionFoo.kwtest3(a; another!" => :other + "CompletionFoo.kwtest3(a; another!kwarg=0, foob" => :other + "CompletionFoo.kwtest3(a; namedarg=0, foob" => :other + "kwtest3(blabla; unknown=4, namedar" => :other + "kwtest3(blabla; named" => :other + "kwtest3(blabla; named." => :other + "kwtest3(blabla; named..., another!" => :other + "kwtest3(blabla; named..., len" => :other + "kwtest3(1+3im; named" => :other + "kwtest3(1+3im; named." => :other + "CompletionFoo.kwtest4(a; x23=0, _" => :other + "CompletionFoo.kwtest4(a; xαβγ=1, _" => :other + "CompletionFoo.kwtest4.(a; xαβγ=1, _" => :other + "CompletionFoo.kwtest4(a; x23=0, x" => :other + "CompletionFoo.kwtest4.(a; x23=0, x" => :other + "CompletionFoo.kwtest4(a; _a1b=1, x" => :other + "CompletionFoo.kwtest5(3, 5; somek" => :other + "CompletionFoo.kwtest5(3, 5, somekwarg=4, somek" => :other + "CompletionFoo.kwtest5(3, 5, 7; somekw" => :other + "CompletionFoo.kwtest5(3, 5, 7, 9; somekw" => :other + "CompletionFoo.kwtest5(3, 5, 7, 9, Any[]...; somek" => :other + "CompletionFoo.kwtest5(unknownsplat...; somekw" => :other + "CompletionFoo.kwtest5(3, 5, 7, 9, somekwarg=4, somek" => :other + "CompletionFoo.kwtest5(String[]..., unknownsplat...; xy" => :other + "CompletionFoo.kwtest5('a', unknownsplat...; xy" => :other + "CompletionFoo.kwtest5('a', 3, String[]...; xy" => :other + "CompletionFoo.kwtest3(" => :other + "CompletionFoo.kwtest3(a;" => :other + "CompletionFoo.kwtest3(a; len2=" => :other + "CompletionFoo.kwtest3(a; len2=le" => :other + "CompletionFoo.kwtest3(a; len2=3 " => :other + "CompletionFoo.kwtest3(a; [le" => :other + "CompletionFoo.kwtest3([length; le" => :other + "CompletionFoo.kwtest3(a; (le" => :other + "CompletionFoo.kwtest3(a; foo(le" => :other + "CompletionFoo.kwtest3(a; (; le" => :other + "CompletionFoo.kwtest3(a; length, " => :other + "CompletionFoo.kwtest3(a; kwargs..., " => :other + ":(function foo(::Int) end).args[1].args[2]." => :other + "log(log.(varfloat)," => :other + "Base.return_types(getin" => :other + "test(1,1, " => :other + "test.(1,1, " => :other + "prevind(\"θ\",1," => :other + "typeof(+)." => :other + "test_dict[\"ab" => :string + "CompletionFoo.x." => :other + "@noexist." => :other + "Main.@noexist." => :none # <- Invalid syntax which adding a suffix can't fix + "@Main.noexist." => :other + "@show." => :other + "@macroexpand." => :other + "CompletionFoo.@foobar()." => :other + "CompletionFoo.@foobar(4)." => :other + "foo(#=#==#=##==#).rs[1]." => :other + "foo().r." => :other + "foo(#=#=# =#= =#).r." => :other + "test_47594." => :other + "Issue36437(42)." => :other + "Some(Issue36437(42)).value." => :other + "some_issue36437.value." => :other + "some_issue36437.value.a, some_issue36437.value." => :other + "@show some_issue36437.value.a; some_issue36437.value." => :other + "()." => :other + "Ref(Issue36437(42))[]." => :other + "global_dict[:r]." => :other + "global_dict_nested[:g][:r]." => :other + "global_dict_nested[" => :other + "global_dict_nested[:g][" => :other + "pop!(global_xs)." => :other + "tcd1." => :other + "tcd1.x." => :other + "tcd1.x.v." => :other + "getkeyelem(mutable_const_prop)." => :other + "getkeyelem(mutable_const_prop).value." => :other + "var\"complicated " => :string + "WeirdNames().var\"oh " => :string + "WeirdNames().var\"" => :string + "\"abc\"." => :other + "(rand(Bool) ? issue51499_2_1 : issue51499_2_2)." => :other + "union_somes(1, 1.0)." => :other + "union_some_ref(1, 1.0)." => :other + "Issue49892(fal" => :other + "-CompletionFoo.Test_y(3)." => :other + "99 ⨷⁻ᵨ⁷ CompletionFoo.type_test." => :other + "CompletionFoo.type_test + CompletionFoo.Test_y(2)." => :other + "(CompletionFoo.type_test + CompletionFoo.Test_y(2))." => :other + "CompletionFoo.type_test + CompletionFoo.unicode_αβγ." => :other + "(CompletionFoo.type_test + CompletionFoo.unicode_αβγ)." => :other + "using Base." => :other + "@time(using .Iss" => :other + "using .Issue52922.Inner1." => :other + "Issue53126()." => :other + "using " => :other + "global xxx::Number = Base." => :other + "let x = 1 # comment" => :other + ] + @testset "$(repr(str))" begin + # Test :statement parsing + ex = JuliaSyntax.core_parser_hook(str, "somefile", 1, 0, :statement)[1] + @test Base.incomplete_tag(ex) == tag + # Test :all parsing - this is what the REPL uses to parse user input. + ex = JuliaSyntax.core_parser_hook(str, "somefile", 1, 0, :all)[1] + @test ex.head == :toplevel + @test Base.incomplete_tag(ex.args[end]) == tag + end + end + + # Should not throw + @test JuliaSyntax.core_parser_hook("+=", "somefile", 1, 0, :statement)[1] isa Expr + end +end diff --git a/JuliaSyntax/test/kinds.jl b/JuliaSyntax/test/kinds.jl new file mode 100644 index 0000000000000..5179544ec15d3 --- /dev/null +++ b/JuliaSyntax/test/kinds.jl @@ -0,0 +1,59 @@ +# Only test this once per session, as kind modules must be unique (ugh) +if !isdefined(@__MODULE__, :FooKinds) +@eval module FooKinds + +using ..JuliaSyntax + +function _init_kinds() + JuliaSyntax.register_kinds!(@__MODULE__, 42, [ + "BEGIN_FOO" + "foo_1" + "foo_2" + "BEGIN_FOOBAR" + "foobar_1" + "foobar_2" + "END_FOOBAR" + "END_FOO" + ]) +end + +_init_kinds() + +k_before_init = K"foo_1" + +function __init__() + _init_kinds() +end + +end + +@eval module BarKinds + # Intentionally empty +end + +end + +@testset "Kinds" begin + @test K"foo_1" != K"foo_2" + + @test FooKinds.k_before_init == K"foo_1" + + @test K"BEGIN_FOO" == K"foo_1" + @test K"foo_2" < K"BEGIN_FOOBAR" + @test K"BEGIN_FOOBAR" == K"foobar_1" + @test K"END_FOOBAR" == K"foobar_2" + @test K"END_FOO" == K"foobar_2" + + @test parentmodule(K"foo_1") == FooKinds + @test sprint(show, K"foo_1") == "K\"foo_1\"" + + # Too many kind modules + @test_throws ErrorException JuliaSyntax.register_kinds!(BarKinds, 64, ["hoo?"]) + # Too many kind names per module + @test_throws ErrorException JuliaSyntax.register_kinds!(BarKinds, 42, string.(1:1024)) + # Re-registering or registering new kinds is not supported + @test_throws ErrorException JuliaSyntax.register_kinds!(FooKinds, 42, ["foo_2", "foo_1"]) + @test_throws ErrorException JuliaSyntax.register_kinds!(FooKinds, 42, ["foo_3"]) + # Module ID already taken by FooKinds + @test_throws ErrorException JuliaSyntax.register_kinds!(BarKinds, 42, ["hii?"]) +end diff --git a/JuliaSyntax/test/literal_parsing.jl b/JuliaSyntax/test/literal_parsing.jl new file mode 100644 index 0000000000000..bfb8e932458ad --- /dev/null +++ b/JuliaSyntax/test/literal_parsing.jl @@ -0,0 +1,256 @@ +using .JuliaSyntax: + parse_int_literal, + parse_uint_literal, + parse_float_literal, + unescape_julia_string + +@testset "Float parsing" begin + # Float64 + @test parse_float_literal(Float64, "123", 1, 4) === (123.0, :ok) + @test parse_float_literal(Float64, "123", 2, 4) === (23.0, :ok) + @test parse_float_literal(Float64, "123", 2, 3) === (2.0, :ok) + @test parse_float_literal(Float64, "1.3", 1, 4) === (1.3, :ok) + @test parse_float_literal(Float64, "1.3e2", 1, 6) === (1.3e2, :ok) + @test parse_float_literal(Float64, "1.3E2", 1, 6) === (1.3e2, :ok) + @test parse_float_literal(Float64, "1.0e-1000", 1, 10) === (0.0, :underflow) + @test parse_float_literal(Float64, "1.0e+1000", 1, 10) === (Inf, :overflow) + # Slow path (exceeds static buffer size) + @test parse_float_literal(Float64, "0.000000000000000000000000000000000000000000000000000000000001", 1, 63) === (1e-60, :ok) + # hexfloat + @test parse_float_literal(Float64, "0x0ap-0", 1, 8) === (Float64(10), :ok) + @test parse_float_literal(Float64, "0xffp-0", 1, 8) === (Float64(255), :ok) + + # Float32 + @test parse_float_literal(Float32, "123", 1, 4) === (123.0f0, :ok) + @test parse_float_literal(Float32, "1.3f2", 1, 6) === (1.3f2, :ok) + if !Sys.iswindows() + @test parse_float_literal(Float32, "1.0f-50", 1, 8) === (0.0f0, :underflow) + end + @test parse_float_literal(Float32, "1.0f+50", 1, 8) === (Inf32, :overflow) + + # Assertions + @test_throws ErrorException parse_float_literal(Float64, "x", 1, 2) + @test_throws ErrorException parse_float_literal(Float64, "1x", 1, 3) + + # Underscore and \minus allowed + @test parse_float_literal(Float64, "10_000.0_0", 1, 9) === (Float64(10000), :ok) + @test parse_float_literal(Float64, "−10.0", 1, 8) === (Float64(-10), :ok) + @test parse_float_literal(Float64, "10e\u22121", 1, 8) === (Float64(1), :ok) +end + +hexint(s) = parse_uint_literal(s, K"HexInt") +binint(s) = parse_uint_literal(s, K"BinInt") +octint(s) = parse_uint_literal(s, K"OctInt") + +@testset "Integer parsing" begin + # Integers + @testset "Signed Integers" begin + @test parse_int_literal("-1") isa Int + @test parse_int_literal("1") isa Int + @test parse_int_literal("2147483647") isa Int + @test parse_int_literal("9223372036854775807") isa Int64 + @test parse_int_literal("9223372036854775808") isa Int128 + @test parse_int_literal("170141183460469231731687303715884105727") isa Int128 + @test parse_int_literal("170141183460469231731687303715884105728") isa BigInt + end + + # HexInt + @testset "HexInt numeric limits for different types" begin + @test hexint("0xff") === UInt8(0xff) + @test hexint("0x100") === UInt16(0x100) + @test hexint("0xffff") === UInt16(0xffff) + @test hexint("0x10000") === UInt32(0x10000) + @test hexint("0xffffffff") === UInt32(0xffffffff) + @test hexint("0x100000000") === UInt64(0x100000000) + @test hexint("0xffffffffffffffff") === UInt64(0xffffffffffffffff) + @test hexint("0x10000000000000000") === UInt128(0x10000000000000000) + @test hexint("0xffffffffffffffffffffffffffffffff") === UInt128(0xffffffffffffffffffffffffffffffff) + @test (n = hexint("0x100000000000000000000000000000000"); + n isa BigInt && n == big"0x100000000000000000000000000000000") + end + @testset "HexInt string length limits for different types" begin + @test hexint("0x00") === UInt8(0) + @test hexint("0x000") === UInt16(0) + @test hexint("0x0000") === UInt16(0) + @test hexint("0x00000") === UInt32(0) + @test hexint("0x00000000") === UInt32(0) + @test hexint("0x000000000") === UInt64(0) + @test hexint("0x0000000000000000") === UInt64(0) + @test hexint("0x00000000000000000") === UInt128(0) + @test hexint("0x00000000000000000000000000000000") === UInt128(0) + @test (n = hexint("0x000000000000000000000000000000000"); + n isa BigInt && n == 0) + end + + # BinInt + @testset "BinInt numeric limits for different types" begin + @test binint("0b11111111") === UInt8(0xff) + @test binint("0b100000000") === UInt16(0x100) + @test binint("0b1111111111111111") === UInt16(0xffff) + @test binint("0b10000000000000000") === UInt32(0x10000) + @test binint("0b11111111111111111111111111111111") === UInt32(0xffffffff) + @test binint("0b100000000000000000000000000000000") === UInt64(0x100000000) + @test binint("0b1111111111111111111111111111111111111111111111111111111111111111") === UInt64(0xffffffffffffffff) + @test binint("0b10000000000000000000000000000000000000000000000000000000000000000") === UInt128(0x10000000000000000) + @test binint("0b11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111") === UInt128(0xffffffffffffffffffffffffffffffff) + @test (n = binint("0b100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + n isa BigInt && n == big"0x100000000000000000000000000000000") + end + @testset "BinInt string length limits for different types" begin + @test binint("0b00000000") === UInt8(0) + @test binint("0b000000000") === UInt16(0) + @test binint("0b0000000000000000") === UInt16(0) + @test binint("0b00000000000000000") === UInt32(0) + @test binint("0b00000000000000000000000000000000") === UInt32(0) + @test binint("0b000000000000000000000000000000000") === UInt64(0) + @test binint("0b0000000000000000000000000000000000000000000000000000000000000000") === UInt64(0) + @test binint("0b00000000000000000000000000000000000000000000000000000000000000000") === UInt128(0) + @test binint("0b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") === UInt128(0) + @test (n = binint("0b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + n isa BigInt && n == 0) + end + + # OctInt + @testset "OctInt numeric limits for different types" begin + @test octint("0o377") === UInt8(0xff) + @test octint("0o400") === UInt16(0x100) + @test octint("0o177777") === UInt16(0xffff) + @test octint("0o200000") === UInt32(0x10000) + @test octint("0o37777777777") === UInt32(0xffffffff) + @test octint("0o40000000000") === UInt64(0x100000000) + @test octint("0o1777777777777777777777") === UInt64(0xffffffffffffffff) + @test octint("0o2000000000000000000000") === UInt128(0x10000000000000000) + @test octint("0o3777777777777777777777777777777777777777777") === UInt128(0xffffffffffffffffffffffffffffffff) + @test (n = octint("0o4000000000000000000000000000000000000000000"); + n isa BigInt && n == big"0x100000000000000000000000000000000") + end + @testset "OctInt string length limits for different types" begin + @test octint("0o000") === UInt8(0) + @test octint("0o0000") === UInt16(0) + @test octint("0o000000") === UInt16(0) + @test octint("0o0000000") === UInt32(0) + @test octint("0o00000000000") === UInt32(0) + @test octint("0o000000000000") === UInt64(0) + @test octint("0o0000000000000000000000") === UInt64(0) + @test octint("0o00000000000000000000000") === UInt128(0) + @test octint("0o0000000000000000000000000000000000000000000") === UInt128(0) + @test (n = octint("0o00000000000000000000000000000000000000000000"); + n isa BigInt && n == 0) + end + + @testset "Underscore separators" begin + @test parse_int_literal("10_000") === 10000 + @test parse_uint_literal("0xff_ff", K"HexInt") === 0xffff + @test parse_uint_literal("0b1111_1111", K"BinInt") === 0xff + @test parse_uint_literal("0o177_777", K"OctInt") === 0xffff + end + + @testset "\\minus ('\\u2212' / '−') allowed in numbers" begin + @test parse_int_literal("−10") === -10 + end +end + +function unesc(str, firstind=firstindex(str), endind=lastindex(str)+1; diagnostics=false) + io = IOBuffer() + ds = JuliaSyntax.Diagnostic[] + unescape_julia_string(io, Vector{UInt8}(str), firstind, endind, ds) + if diagnostics + ds + else + @test isempty(ds) + String(take!(io)) + end +end + +@testset "String unescaping" begin + # offsets + @test unesc("abcd", 1, 3) == "ab" + @test unesc("abcd", 2, 4) == "bc" + @test unesc("abcd", 3, 5) == "cd" + + # Allowed escapes of delimiters and dollar sign + @test unesc("\\\\") == "\\" + @test unesc("\\\"") == "\"" + @test unesc("\\\$") == "\$" + @test unesc("\\'") == "\'" + @test unesc("\\`") == "`" + + # Newline normalization + @test unesc("a\nb\rc\r\nd") == "a\nb\nc\nd" + + # Invalid escapes + @test !isempty(unesc("\\.", diagnostics=true)) + @test !isempty(unesc("\\z", diagnostics=true)) + + # Standard C escape sequences + @test codeunits(unesc("\\n\\t\\r\\e\\b\\f\\v\\a")) == + UInt8[0x0a, 0x09, 0x0d, 0x1b, 0x08, 0x0c, 0x0b, 0x07] + + # Hex and unicode escapes; \x \u and \U + @test unesc("x\\x61x") == "xax" + @test unesc("x\\u03b1x") == "xαx" + @test unesc("x\\U001F604x") == "x😄x" + # Maximum unicode code point + @test unesc("x\\U10ffffx") == "x\U10ffffx" + @test !isempty(unesc("x\\U110000x", diagnostics=true)) + + # variable-length octal + @test unesc("x\\7x") == "x\ax" + @test unesc("x\\77x") == "x?x" + @test unesc("x\\141x") == "xax" + @test unesc("x\\377x") == "x\xffx" + @test !isempty(unesc("x\\400x", diagnostics=true)) +end + +function unesc_raw(str, is_cmd) + io = IOBuffer() + JuliaSyntax.unescape_raw_string(io, Vector{UInt8}(str), + firstindex(str), lastindex(str)+1, is_cmd) + return String(take!(io)) +end + +@testset "Raw string unescaping" begin + # " delimited + # x\"x ==> x"x + @test unesc_raw("x\\\"x", false) == "x\"x" + # x\`x ==> x\`x + @test unesc_raw("x\\`x", false) == "x\\`x" + # x\\\"x ==> x\"x + @test unesc_raw("x\\\\\\\"x", false) == "x\\\"x" + # x\\\`x ==> x\\\`x + @test unesc_raw("x\\\\\\`x", false) == "x\\\\\\`x" + # '\\ ' ==> '\\ ' + @test unesc_raw("\\\\ ", false) == "\\\\ " + # '\\' ==> '\' + @test unesc_raw("\\\\", false) == "\\" + # '\\\\' ==> '\\' + @test unesc_raw("\\\\\\\\", false) == "\\\\" + + # ` delimited + # x\"x ==> x\"x + @test unesc_raw("x\\\"x", true) == "x\\\"x" + # x\`x ==> x`x + @test unesc_raw("x\\`x", true) == "x`x" + # x\\\"x ==> x\"x + @test unesc_raw("x\\\\\\\"x", true) == "x\\\\\\\"x" + # x\\\`x ==> x\`x + @test unesc_raw("x\\\\\\`x", true) == "x\\`x" + # '\\ ' ==> '\\ ' + @test unesc_raw("\\\\ ", true) == "\\\\ " +end + +@testset "Normalization of identifiers" begin + # NFC normalization + # https://github.com/JuliaLang/julia/issues/5434 + # https://github.com/JuliaLang/julia/pull/19464 + @test JuliaSyntax.normalize_identifier("\u0069\u0302") == "\u00ee" + + # Special Julia normalization + # https://github.com/JuliaLang/julia/pull/42561 + @test JuliaSyntax.normalize_identifier("julia\u025B\u00B5\u00B7\u0387\u2212") == + "julia\u03B5\u03BC\u22C5\u22C5\u002D" + + # https://github.com/JuliaLang/julia/issues/48870 + # ℏ -> ħ + @test JuliaSyntax.normalize_identifier("\u210f") == "\u0127" +end diff --git a/JuliaSyntax/test/parse_packages.jl b/JuliaSyntax/test/parse_packages.jl new file mode 100644 index 0000000000000..b5b08f488a918 --- /dev/null +++ b/JuliaSyntax/test/parse_packages.jl @@ -0,0 +1,79 @@ +# Full-scale parsing tests of JuliaSyntax itself, Julia Base, etc. + +juliasyntax_dir = joinpath(@__DIR__, "..") +@testset "Parse JuliaSyntax" begin + test_parse_all_in_path(joinpath(juliasyntax_dir, "src")) +end +@testset "Parse JuliaSyntax tests" begin + test_parse_all_in_path(joinpath(juliasyntax_dir, "test")) +end + +base_path = let + p = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "base") + if !isdir(p) + # For julia 1.9 images. + p = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "src", "base") + if !isdir(p) + error("source for Julia base not found") + end + end + p +end +@testset "Parse Base at $base_path" begin + test_parse_all_in_path(base_path) do f + if endswith(f, "gmp.jl") + # Loose comparison due to `f(::g(w) = z) = a` syntax + return exprs_roughly_equal + end + return exprs_equal_no_linenum + end +end + +base_tests_path = joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "test") +@testset "Parse Base tests at $base_tests_path" begin + test_parse_all_in_path(base_tests_path) do f + # In julia-1.6, test/copy.jl had spurious syntax which became the + # multidimensional array syntax in 1.7. + if endswith(f, "copy.jl") && v"1.6" <= VERSION < v"1.7" + return nothing + end + + # syntax.jl has some intentionally weird syntax which we parse + # differently than the flisp parser, and some cases which we've + # decided are syntax errors. + if endswith(f, "syntax.jl") + return nothing + end + + if endswith(f, "core.jl") + # The test + # @test Union{Tuple{T}, Tuple{T,Int}} where {T} === widen_diagonal(Union{Tuple{T}, Tuple{T,Int}} where {T}) + # depends on a JuliaSyntax bugfix and parses differently (wrong) using + # flisp. This was added in julia#52228 and backported in julia#52045 + if v"1.10.0-rc1.39" <= VERSION + return nothing + else + # Loose comparison due to `for f() = 1:3` syntax + return exprs_roughly_equal + end + end + + # subtype.jl also depends on the where precedence JuliaSyntax bugfix as of julia#53034 + if endswith(f, "subtype.jl") && v"1.11.0-DEV.1382" <= VERSION + return nothing + end + + return exprs_equal_no_linenum + end +end + +@testset "Parse Julia stdlib at $(Sys.STDLIB)" begin + for stdlib in readdir(Sys.STDLIB) + fulldir = joinpath(Sys.STDLIB, stdlib) + if isdir(fulldir) + @testset "Parse $stdlib" begin + test_parse_all_in_path(joinpath(Sys.STDLIB, fulldir)) + end + end + end +end diff --git a/JuliaSyntax/test/parse_stream.jl b/JuliaSyntax/test/parse_stream.jl new file mode 100644 index 0000000000000..cda8443be113a --- /dev/null +++ b/JuliaSyntax/test/parse_stream.jl @@ -0,0 +1,168 @@ +# Prototype ParseStream interface +# +# Here we test the ParseStream interface, by taking input code and checking +# that the correct sequence of emit() and bump() produces a valid parse tree. + +using .JuliaSyntax: ParseStream, + peek, peek_token, + bump, bump_trivia, bump_invisible, + emit, emit_diagnostic, TRIVIA_FLAG, INFIX_FLAG, + ParseStreamPosition, first_child_position, last_child_position, + parsestmt + +# Here we manually issue parse events in the order the Julia parser would issue +# them +@testset "ParseStream" begin + code = """ + for i = 1:10 + xx[i] + 2 + # hi + yy + end + """ + st = ParseStream(code) + + p1 = position(st) + @test peek(st) == K"for" + bump(st, TRIVIA_FLAG) + p2 = position(st) + @test peek(st) == K"Identifier" # 'i' + bump(st) + @test peek(st) == K"=" + bump(st, TRIVIA_FLAG) + p3 = position(st) + @test peek(st) == K"Integer" # 1 + bump(st) + @test peek(st) == K":" + bump(st) # : + @test peek(st) == K"Integer" # 10 + bump(st) # 10 + emit(st, p3, K"call", INFIX_FLAG) + emit(st, p2, K"=") + @test peek(st) == K"NewlineWs" + bump(st, TRIVIA_FLAG) + p4 = position(st) + p5 = position(st) # [call] + p6 = position(st) # [ref] + @test peek(st) == K"Identifier" # 'xx' + bump(st) + @test peek(st) == K"[" + bump(st, TRIVIA_FLAG) + @test peek(st) == K"Identifier" # 'i' + bump(st) + @test peek(st) == K"]" + bump(st, TRIVIA_FLAG) + emit(st, p6, K"ref") + @test peek(st) == K"+" + bump(st) + @test peek(st) == K"Integer" # 2 + bump(st) + emit(st, p5, K"call", INFIX_FLAG) + @test peek(st) == K"NewlineWs" + bump(st, TRIVIA_FLAG) + @test peek(st) == K"NewlineWs" + bump(st, TRIVIA_FLAG) + @test peek(st) == K"Identifier" # 'yy' + bump(st) + emit(st, p4, K"block") + @test peek(st) == K"NewlineWs" + bump(st, TRIVIA_FLAG) + bump(st, TRIVIA_FLAG) # end + emit(st, p1, K"for") + @test peek(st) == K"NewlineWs" + bump(st, TRIVIA_FLAG) + emit(st, p1, K"toplevel") +end + +@testset "ParseStream constructors" begin + @testset "Byte buffer inputs" begin + # Vector{UInt8} + let + st = ParseStream(Vector{UInt8}("x+y")) + bump(st) + @test build_tree(Expr, st) == :x + @test JuliaSyntax.last_byte(st) == 1 + end + let + st = ParseStream(Vector{UInt8}("x+y"), 3) + bump(st) + @test build_tree(Expr, st) == :y + @test JuliaSyntax.last_byte(st) == 3 + end + # Ptr{UInt8}, len + code = "x+y" + GC.@preserve code begin + let + st = ParseStream(pointer(code), 3) + bump(st) + @test build_tree(Expr, st) == :x + @test JuliaSyntax.last_byte(st) == 1 + end + end + end +end + +@testset "ParseStream tree traversal" begin + # NB: ParseStreamPosition.node_index includes an initial sentinel token so + # indices here are one more than "might be expected". Additionally, note that + # the byte index points to the first byte after the token. + st = parse_sexpr("((a b) c)") + child1_pos = first_child_position(st, position(st)) + @test child1_pos == ParseStreamPosition(7, 8) + @test first_child_position(st, child1_pos) == ParseStreamPosition(4, 4) + @test last_child_position(st, position(st)) == ParseStreamPosition(9, 10) + @test last_child_position(st, child1_pos) == ParseStreamPosition(6, 6) + + st = parse_sexpr("( (a b) c)") + child1_pos = first_child_position(st, position(st)) + @test child1_pos == ParseStreamPosition(8, 9) + @test first_child_position(st, child1_pos) == ParseStreamPosition(5, 5) + @test last_child_position(st, position(st)) == ParseStreamPosition(10, 11) + @test last_child_position(st, child1_pos) == ParseStreamPosition(7, 7) + + st = parse_sexpr("(a (b c))") + @test first_child_position(st, position(st)) == ParseStreamPosition(3, 3) + child2_pos = last_child_position(st, position(st)) + @test child2_pos == ParseStreamPosition(9, 10) + @test first_child_position(st, child2_pos) == ParseStreamPosition(6, 6) + @test last_child_position(st, child2_pos) == ParseStreamPosition(8, 8) + + st = parse_sexpr("( a (b c))") + @test first_child_position(st, position(st)) == ParseStreamPosition(4, 4) + child2_pos = last_child_position(st, position(st)) + @test child2_pos == ParseStreamPosition(10, 11) + @test first_child_position(st, child2_pos) == ParseStreamPosition(7, 7) + @test last_child_position(st, child2_pos) == ParseStreamPosition(9, 9) + + st = parse_sexpr("a (b c)") + @test first_child_position(st, position(st)) == ParseStreamPosition(5, 5) + @test last_child_position(st, position(st)) == ParseStreamPosition(7, 7) + + st = parse_sexpr("(a) (b c)") + @test first_child_position(st, position(st)) == ParseStreamPosition(7, 8) + @test last_child_position(st, position(st)) == ParseStreamPosition(9, 10) + + st = parse_sexpr("(() ())") + @test first_child_position(st, position(st)) == ParseStreamPosition(4, 5) + @test last_child_position(st, position(st)) == ParseStreamPosition(7, 9) +end + +@testset "SubString{GenericString} (issue #505)" begin + x = Test.GenericString("1 2") + @test x == "1 2" + y = split(x)[1] + @test y == "1" + @test y isa SubString{GenericString} + @test ParseStream(y) isa ParseStream + @test parsestmt(Expr, y) == parsestmt(Expr, "1") +end + +@testset "peek_behind_pos with negative byte index" begin + # Test that peek_behind_pos doesn't cause InexactError when byte_idx goes negative + # This can happen when parsing certain incomplete keywords like "do" + # where trivia skipping walks back past the beginning of the stream + @test_throws JuliaSyntax.ParseError parseall(GreenNode, "do") + @test_throws JuliaSyntax.ParseError parseall(GreenNode, "do ") + @test_throws JuliaSyntax.ParseError parseall(GreenNode, " do") + @test_throws JuliaSyntax.ParseError parseall(GreenNode, "do\n") +end diff --git a/JuliaSyntax/test/parser.jl b/JuliaSyntax/test/parser.jl new file mode 100644 index 0000000000000..a65c190a3ea18 --- /dev/null +++ b/JuliaSyntax/test/parser.jl @@ -0,0 +1,1463 @@ +""" +Parse string to SyntaxNode tree and show as an sexpression +""" +function parse_to_sexpr_str(production, code::AbstractString; v=v"1.6", show_kws...) + stream = ParseStream(code, version=v) + production(ParseState(stream)) + JuliaSyntax.validate_tokens(stream) + s = build_tree(SyntaxNode, stream, keep_parens=true) + return sprint(io->show(io, MIME("text/x.sexpression"), s; show_kws...)) +end + +function test_parse(production, input, expected; show_kws...) + if !(input isa AbstractString) + opts, input = input + else + opts = NamedTuple() + end + parsed = parse_to_sexpr_str(production, input; show_kws..., opts...) + if expected isa Regex # Could be AbstractPattern, but that type was added in Julia 1.6. + @test match(expected, parsed) !== nothing + else + @test parsed == expected + end +end + +function test_parse(inout::Pair) + test_parse(JuliaSyntax.parse_toplevel, inout...) +end + +PARSE_ERROR = r"\(error" + +with_version(v::VersionNumber, (i,o)::Pair) = ((;v=v), i) => o + +# TODO: +# * Extract the following test cases from the source itself. +# * Use only the green tree to generate the S-expressions +# (add flag annotations to heads) +tests = [ + JuliaSyntax.parse_toplevel => [ + "a \n b" => "(toplevel a b)" + "a;b \n c;d" => "(toplevel (toplevel-; a b) (toplevel-; c d))" + "a \n \n" => "(toplevel a)" + "" => "(toplevel)" + ], + JuliaSyntax.parse_block => [ + "a;b;c" => "(block a b c)" + "a;;;b;;" => "(block a b)" + ";a" => "(block a)" + "\n a" => "(block a)" + "a\nb" => "(block a b)" + ], + JuliaSyntax.parse_stmts => [ + "a;b;c" => "(toplevel-; a b c)" + "a;;;b;;" => "(toplevel-; a b)" + """ "x" a ; "y" b """ => + """(toplevel-; (doc (string "x") a) (doc (string "y") b))""" + "x y" => "(wrapper x (error-t y))" + # Newline-continued macro arguments apply only in bare parens, not in + # string interpolation or call/signature brackets + with_version(v"1.14", "\"\$(@foo x\n y)\"" => "(string (parens (macrocall (macro_name foo) x) (error-t y)))") + with_version(v"1.14", "function f(@nospecialize x\n y) end" => "(function (call f (macrocall (macro_name nospecialize) x) (error-t y)) (block))") + ], + JuliaSyntax.parse_eq => [ + # parse_assignment + "a = b" => "(= a b)" + "a .= b" => "(.= a b)" + "a += b" => "(op= a + b)" + "a .+= b" => "(.op= a + b)" + with_version(v"1.14", "a +%= b" => "(op= a +% b)") + with_version(v"1.14", "a -%= b" => "(op= a -% b)") + with_version(v"1.14", "a *%= b" => "(op= a *% b)") + "a, b = c, d" => "(= (tuple a b) (tuple c d))" + "x, = xs" => "(= (tuple x) xs)" + "[a ~b]" => "(hcat a (call-pre ~ b))" + "a ~ b" => "(call-i a ~ b)" + "a .~ b" => "(dotcall-i a ~ b)" + "[a ~ b c]" => "(hcat (call-i a ~ b) c)" + "[a~b]" => "(vect (call-i a ~ b))" + "f(x) .= 1" => "(.= (call f x) 1)" + "::g() = 1" => "(= (::-pre (call g)) 1)" + "f(x) = 1" => "(function-= (call f x) 1)" + "f(x)::T = 1" => "(function-= (::-i (call f x) T) 1)" + "f(x) where S where U = 1" => "(function-= (where (where (call f x) S) U) 1)" + "(f(x)::T) where S = 1" => "(function-= (where (parens (::-i (call f x) T)) S) 1)" + "f(x) = 1 = 2" => "(function-= (call f x) (= 1 2))" # Should be a warning! + # Suffixed operators don't form compound assignments (matching the + # reference parser): `+₁` is parsed as the operator, leaving a stray `=` + "a +₁= b" => "(call-i a +₁ (error =))" + # ... and likewise operators which simply have no compound-assignment + # form are parsed as an identifier being assigned to + "⋅ = 5" => "(= ⋅ 5)" + "⋅=5" => "(= ⋅ 5)" + # Operators followed by `==`, `===` or `=>` (rather than the single + # token `=`) are not compound assignments + "a +== b" => "(call-i a + (call-pre (error ==) b))" + "a -=> b" => "(call-i a - (call-pre (error =>) b))" + with_version(v"1.14", "a +%== b" => "(call-i a +% (call-pre (error ==) b))") + with_version(v"1.14", "a -%=> b" => "(call-i a -% (call-pre (error =>) b))") + "a >>>== b" => "(call-i a >>> (call-pre (error ==) b))" + "a .+== b" => "(dotcall-i a + (call-pre (error ==) b))" + ], + JuliaSyntax.parse_pair => [ + "a => b" => "(call-i a => b)" + "a .=> b" => "(dotcall-i a => b)" + ], + JuliaSyntax.parse_cond => [ + "a ? b : c" => "(? a b c)" + "a ?\nb : c" => "(? a b c)" + "a ? b :\nc" => "(? a b c)" + "a ? b : c:d" => "(? a b (call-i c : d))" + # Following are errors but should recover + "a? b : c" => "(? a (error-t) b c)" + "a ?b : c" => "(? a (error-t) b c)" + "a ? b: c" => "(? a b (error-t) c)" + "a ? b :c" => "(? a b (error-t) c)" + "a ? b c" => "(? a b (error-t) c)" + "A[x ? y : end]" => "(ref A (? x y end))" + ], + JuliaSyntax.parse_arrow => [ + "x → y" => "(call-i x → y)" + "x <--> y" => "(call-i x <--> y)" + "x --> y" => "(--> x y)" + "x .--> y" => "(dotcall-i x --> y)" + "x -->₁ y" => "(call-i x -->₁ y)" + ], + JuliaSyntax.parse_or => [ + "x || y || z" => "(|| x (|| y z))" + ((v=v"1.6",), "x .|| y") => "(error (.|| x y))" + ((v=v"1.7",), "x .|| y") => "(.|| x y)" + ], + JuliaSyntax.parse_and => [ + "x && y && z" => "(&& x (&& y z))" + ((v=v"1.6",), "x .&& y") => "(error (.&& x y))" + ((v=v"1.7",), "x .&& y") => "(.&& x y)" + ], + JuliaSyntax.parse_comparison => [ + # Type comparisons are syntactic + "x <: y" => "(<: x y)" + "x >: y" => "(>: x y)" + # Normal binary comparisons + "x < y" => "(call-i x < y)" + "x .< y" => "(dotcall-i x < y)" + "x .<: y" => "(dotcall-i x <: y)" + # A dotted operator directly following a float literal + "1.1.∈a" => "(dotcall-i 1.1 ∈ a)" + ":. == :." => "(call-i (quote-: .) == (quote-: .))" + # Comparison chains + "x < y < z" => "(comparison x < y < z)" + "x == y < z" => "(comparison x == y < z)" + "x .< y .< z" => "(comparison x (. <) y (. <) z)" + "x .< y < z" => "(comparison x (. <) y < z)" + ], + JuliaSyntax.parse_pipe_lt => [ + "x <| y <| z" => "(call-i x <| (call-i y <| z))" + ], + JuliaSyntax.parse_pipe_gt => [ + "x |> y |> z" => "(call-i (call-i x |> y) |> z)" + "x .|> y" => "(dotcall-i x |> y)" + ], + JuliaSyntax.parse_range => [ + "1:2" => "(call-i 1 : 2)" + "1:2:3" => "(call-i 1 : 2 3)" + "a:b:c:d:e" => "(call-i (call-i a : b c) : d e)" + "a :< b" => "(call-i a (error : <) b)" + "1:\n2" => "(call-i 1 : (error))" + ], + JuliaSyntax.parse_range => [ + "a..b" => "(call-i a (DotsIdentifier-2) b)" + "-1e10..2" => "(call-i -1.0e10 (DotsIdentifier-2) 2)" + "0x1p3..2" => "(call-i 8.0 (DotsIdentifier-2) 2)" + "a..+b" => "(call-i a (DotsIdentifier-2) (error-t) (call-pre + b))" + # `..` may be directly followed by the operand-starting operators `: :: $ '` + "a..:b" => "(call-i a (DotsIdentifier-2) (quote-: b))" + "'a'..'b'" => "(call-i (char 'a') (DotsIdentifier-2) (char 'b'))" + "a..\$b" => "(call-i a (DotsIdentifier-2) (\$ b))" + "a..::b" => "(call-i a (DotsIdentifier-2) (::-pre b))" + "a … b" => "(call-i a … b)" + "a .… b" => "(dotcall-i a … b)" + "[1 :a]" => "(hcat 1 (quote-: a))" + "[1 2:3 :a]" => "(hcat 1 (call-i 2 : 3) (quote-: a))" + "x..." => "(... x)" + "x:y..." => "(... (call-i x : y))" + "x..y..." => "(... (call-i x (DotsIdentifier-2) y))" + ], + JuliaSyntax.parse_invalid_ops => [ + "a--b" => "(call-i a (ErrorInvalidOperator) b)" + ], + JuliaSyntax.parse_expr => [ + "a - b - c" => "(call-i (call-i a - b) - c)" + "a + b + c" => "(call-i a + b c)" + with_version(v"1.14", "a +% b +% c" => "(call-i a +% b c)") + with_version(v"1.14", "a -% b -% c" => "(call-i (call-i a -% b) -% c)") + "a + b .+ c" => "(dotcall-i (call-i a + b) + c)" + # parse_with_chains: + # The following are two elements of an hcat + "[x +y]" => "(hcat x (call-pre + y))" + "[x+y +z]" => "(hcat (call-i x + y) (call-pre + z))" + # Conversely the following are infix calls + "[x +₁y]" => "(vect (call-i x +₁ y))" + "[x+y+z]" => "(vect (call-i x + y z))" + "[x+y + z]" => "(vect (call-i x + y z))" + # Dotted and normal operators + "a +₁ b +₁ c" => "(call-i (call-i a +₁ b) +₁ c)" + "a .+ b .+ c" => "(dotcall-i (dotcall-i a + b) + c)" + ], + JuliaSyntax.parse_term => [ + "a * b * c" => "(call-i a * b c)" + with_version(v"1.14", "a *% b *% c" => "(call-i a *% b c)") + "a .* b" => "(dotcall-i a * b)" + "-2*x" => "(call-i -2 * x)" + ], + JuliaSyntax.parse_rational => [ + "x // y // z" => "(call-i (call-i x // y) // z)" + ], + JuliaSyntax.parse_shift => [ + "x >> y >> z" => "(call-i (call-i x >> y) >> z)" + ], + JuliaSyntax.parse_juxtapose => [ + "2x" => "(juxtapose 2 x)" + "2x" => "(juxtapose 2 x)" + "2(x)" => "(juxtapose 2 (parens x))" + "(2)(3)x" => "(juxtapose (parens 2) (parens 3) x)" + "(x-1)y" => "(juxtapose (parens (call-i x - 1)) y)" + "x'y" => "(juxtapose (call-post x ') y)" + "1√x" => "(juxtapose 1 (call-pre √ x))" + # errors + "\"a\"\"b\"" => "(juxtapose (string \"a\") (error-t) (string \"b\"))" + "\"a\"x" => "(juxtapose (string \"a\") (error-t) x)" + "\"\$y\"x" => "(juxtapose (string y) (error-t) x)" + "\"a\"begin end" => "(juxtapose (string \"a\") (error-t) (block))" + # Not juxtaposition - parse_juxtapose will consume only the first token. + "x.3" => "x" + "f(2)2" => "(call f 2)" + "x' y" => "(call-post x ')" + "x 'y" => "x" + "x@y" => "x" + "(begin end)x" => "(parens (block))" + # Invalid operators (`**`, `--`) are not juxtaposed + "2**2" => "2" + "2--2" => "2" + ], + JuliaSyntax.parse_unary => [ + ":T" => "(quote-: T)" + "in::T" => "(::-i in T)" + "isa::T" => "(::-i isa T)" + "-2^x" => "(call-pre - (call-i 2 ^ x))" + "-2[1, 3]" => "(call-pre - (ref 2 1 3))" + # signed literals + "-2" => "-2" + "+2.0" => "2.0" + "-1.0f0" => "-1.0f0" + "-0xf.0p0" => "-15.0" + "+0b10010" => "0x12" + "+0o22" => "0x12" + "+0x12" => "0x12" + "-0b10010" => "(call-pre - 0x12)" + "-0o22" => "(call-pre - 0x12)" + "-0x12" => "(call-pre - 0x12)" + "-1::T" => "(::-i -1 T)" + # Standalone dotted operators are parsed as (|.| op) + ".+" => "(. +)" + ".+\n" => "(. +)" + ".+ =" => "(. +)" + ".+)" => "(. +)" + ".&" => "(. &)" + # Standalone non-dotted operators + "+)" => "+" + # Call with type parameters or non-unary prefix call + "+{T}(x::T)" => "(call (curly + T) (::-i x T))" + "*(x)" => "(call * x)" + ".*(x)" => "(call (. *) x)" + # Prefix function calls for operators which are both binary and unary + "+(a,b)" => "(call + a b)" + "+(a,)" => "(call-, + a)" + ".+(a,)" => "(call-, (. +) a)" + "(.+)(a)" => "(call (parens (. +)) a)" + "(.~(a))" => "(parens (dotcall-pre ~ (parens a)))" + "+(a=1,)" => "(call-, + (= a 1))" + "+(a...)" => "(call + (... a))" + "+(a;b,c)" => "(call + a (parameters b c))" + "+(;a)" => "(call + (parameters a))" + "+(;;a)" => "(call + (parameters) (parameters a))" + "+()" => "(call +)" + "+(\n;a)" => "(call + (parameters a))" + "+(;)" => "(call + (parameters))" + "+(\n;\n)" => "(call + (parameters))" + "+(\n)" => "(call +)" + # Whitespace not allowed before prefix function call bracket + "+ (a,b)" => "(call + (error) a b)" + # Prefix calls have higher precedence than ^ + "+(a,b)^2" => "(call-i (call + a b) ^ 2)" + "+(a,b)(x)^2" => "(call-i (call (call + a b) x) ^ 2)" + "<:(a,)" => "(<:-, a)" + # Unary function calls with brackets as grouping, not an arglist + ".+(a)" => "(dotcall-pre + (parens a))" + "+(a;b)" => "(call-pre + (block-p a b))" + "+(;;)" => "(call-pre + (block-p))" + "+(;;)" => "(call-pre + (block-p))" + "+(a;)" => "(call-pre + (block-p a))" + "+(a;;)" => "(call-pre + (block-p a))" + "+(\n;\n;\n)" => "(call-pre + (block-p))" + "+(a=1)" => "(call-pre + (parens (= a 1)))" + # Unary operators have lower precedence than ^ + "+(a)^2" => "(call-pre + (call-i (parens a) ^ 2))" + ".+(a)^2" => "(dotcall-pre + (call-i (parens a) ^ 2))" + "+(a)(x,y)^2" => "(call-pre + (call-i (call (parens a) x y) ^ 2))" + "<:(a)" => "(<:-pre (parens a))" + # Normal unary calls + "+x" => "(call-pre + x)" + "√x" => "(call-pre √ x)" + ".~x" => "(dotcall-pre ~ x)" + # Things which are not quite negative literals + "-0x1"=> "(call-pre - 0x01)" + "- 2" => "(call-pre - 2)" + ".-2" => "(dotcall-pre - 2)" + # Not a unary operator + "/x" => "(call-pre (error /) x)" + "+₁ x" => "(call-pre (error +₁) x)" + ".<: x" => "(dotcall-pre (error (. <:)) x)" + "?\"str\"" => """(call-pre (error ?) (string "str"))""" + ], + JuliaSyntax.parse_factor => [ + "x^y" => "(call-i x ^ y)" + "x^y^z" => "(call-i x ^ (call-i y ^ z))" + "x .^ y" => "(dotcall-i x ^ y)" + "begin x end::T" => "(::-i (block x) T)" + # parse_decl_with_initial_ex + "a::b" => "(::-i a b)" + "a::b::c" => "(::-i (::-i a b) c)" + "a->b" => "(-> (tuple a) b)" + "(a,b)->c" => "(-> (tuple-p a b) c)" + "(a;b=1)->c" => "(-> (tuple-p a (parameters (= b 1))) c)" + "x::T->c" => "(-> (tuple (::-i x T)) c)" + "\$a->b" => "(-> (tuple (\$ a)) b)" + "\$(a)->b" => "(-> (tuple (\$ (parens a))) b)" + # FIXME "&(a)->b" => "(-> (tuple-p (& (parens a))) b)" + # FIXME "::(a)->b" => "(-> (tuple-p (:: (parens a))) b)" + # `where` combined with `->` still parses strangely. However: + # * It's extra hard to add a tuple around the `x` in this syntax corner case. + # * The user already needs to add additional, ugly, parens to get this + # to parse correctly because the precedence of `where` is + # inconsistent with `::` and `->` in this case. + "(x where T)->c" => "(-> (parens (where x T)) c)" + "((x::T) where T)->c" => "(-> (parens (where (parens (::-i x T)) T)) c)" + ], + JuliaSyntax.parse_unary_subtype => [ + "<: )" => "<:" + "<: \n" => "<:" + "<: =" => "<:" + "<:{T}(x::T)" => "(call (curly <: T) (::-i x T))" + "<:(x::T)" => "(<:-pre (parens (::-i x T)))" + "<: x" => "(<:-pre x)" + "<: <: x" => "(<:-pre (<:-pre x))" + "<: A where B" => "(<:-pre (where A B))" + # FIXME: The following bizarre precedence seems broken, but is + # compatible with the reference parser (see #248) + "+ <: A where B" => "(where (call-pre + (<:-pre A)) B)" + # Really for parse_where + "x where \n {T}" => "(where x (braces T))" + "x where {T,S}" => "(where x (braces T S))" + "x where {T,S,}" => "(where x (braces-, T S))" + "x where {T S}" => "(where x (bracescat (row T S)))" + "x where {y for y in ys}" => "(where x (braces (generator y (iteration (in y ys)))))" + "x where T" => "(where x T)" + "x where \n T" => "(where x T)" + "x where T<:S" => "(where x (<: T S))" + # nested unary and unary-syntactic ops + "<: + <: + A" => "(<:-pre (call-pre + (<:-pre (call-pre + A))))" + "* <: A" => "(call-pre (error *) (<:-pre A))" + ], + JuliaSyntax.parse_unary_prefix => [ + "&)" => "&" + "\$\n" => "\$" + "&a" => "(& a)" + "::a" => "(::-pre a)" + "\$a" => "(\$ a)" + "\$\$a" => "(\$ (\$ a))" + ], + JuliaSyntax.parse_call => [ + # parse_call + "f(x)" => "(call f x)" + "\$f(x)" => "(call (\$ f) x)" + ".&(x,y)" => "(call (. &) x y)" + # parse_call_chain + "f(a).g(b)" => "(call (. (call f a) g) b)" + "\$A.@x" => "(macrocall (. (\$ A) (macro_name x)))" + + # non-errors in space sensitive contexts + "[f (x)]" => "(hcat f (parens x))" + "[f x]" => "(hcat f x)" + # space separated macro calls + "@foo a b" => "(macrocall (macro_name foo) a b)" + "@foo (x)" => "(macrocall (macro_name foo) (parens x))" + "@foo (x,y)" => "(macrocall (macro_name foo) (tuple-p x y))" + "A.@foo a b" => "(macrocall (. A (macro_name foo)) a b)" + "@A.foo a b" => "(macrocall (macro_name (. A foo)) a b)" + "[@foo x]" => "(vect (macrocall (macro_name foo) x))" + "[@foo]" => "(vect (macrocall (macro_name foo)))" + "@var\"#\" a" => "(macrocall (macro_name (var #)) a)" + "@(A) x" => "(macrocall (macro_name (parens A)) x)" + "A.@x y" => "(macrocall (. A (macro_name x)) y)" + "A.@var\"#\" a"=> "(macrocall (. A (macro_name (var #))) a)" + "@+x y" => "(macrocall (macro_name +) x y)" + "A.@.x" => "(macrocall (. A (macro_name .)) x)" + # Newlines between macro arguments directly inside parentheses + ((v=v"1.14",), "(@foo a\n b)") => "(parens (macrocall (macro_name foo) a b))" + # Leading trivia in the parentheses is ignored + ((v=v"1.14",), "( @foo a\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(\n@foo a\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(\n\n @foo a\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "( # c\n @foo a\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(#= c\n d =# @foo a\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(@foo a\n\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(@foo a # c\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(@foo a\n # c\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(@foo\n # c\n a)") => "(parens (macrocall (macro_name foo) a))" + ((v=v"1.14",), "(@foo a\r\n b)") => "(parens (macrocall (macro_name foo) a b))" + ((v=v"1.14",), "(@foo\n)") => "(parens (macrocall (macro_name foo)))" + ((v=v"1.14",), "(@foo\n a)") => "(parens (macrocall (macro_name foo) a))" + ((v=v"1.14",), "(@foo a\n)") => "(parens (macrocall (macro_name foo) a))" + ((v=v"1.14",), "(A.@foo a\n b)") => "(parens (macrocall (. A (macro_name foo)) a b))" + ((v=v"1.14",), "(A.B.@foo a\n b)") => "(parens (macrocall (. (. A B) (macro_name foo)) a b))" + ((v=v"1.14",), "(@A.B.foo a\n b)") => "(parens (macrocall (macro_name (. (. A B) foo)) a b))" + ((v=v"1.14",), "(@var\"#\" a\n b)") => "(parens (macrocall (macro_name (var #)) a b))" + # Any prefix expression is part of the macro call + ((v=v"1.14",), "(f(x).@foo a\n b)") => "(parens (macrocall (. (call f x) (macro_name foo)) a b))" + ((v=v"1.14",), "(\$A.@foo a\n b)") => "(parens (macrocall (. (\$ A) (macro_name foo)) a b))" + ((v=v"1.14",), "(@foo(a).@bar x\n y)") => "(parens (macrocall (. (macrocall-p (macro_name foo) a) (macro_name bar)) x y))" + ((v=v"1.14",), "(@(A) x\n y)") => "(parens (macrocall (macro_name (parens A)) x y))" + ((v=v"1.14",), "(@[x] a\n b)") => "(parens (macrocall (macro_name (error (vect x))) a b))" + ((v=v"1.14",), "(@foo a\n; b)") => "(block-p (macrocall (macro_name foo) a) b)" + ((v=v"1.14",), "(@foo x\n y; b)") => "(block-p (macrocall (macro_name foo) x y) b)" + ((v=v"1.14",), "(@foo x\n y, z)") => "(parens (macrocall (macro_name foo) x (tuple y z)))" + # A comma at the start of the following line ends the call, as before + ((v=v"1.14",), "(@foo a\n, b)") => "(tuple-p (macrocall (macro_name foo) a) b)" + ((v=v"1.13",), "(@foo a\n, b)") => "(tuple-p (macrocall (macro_name foo) a) b)" + # Newline-insensitivity only happens for "useless" parens containing + # the macrocall and nothing else + ((v=v"1.14",), "(x, # c\n @foo a\n b)") => "(tuple-p x (macrocall (macro_name foo) a) (error-t b))" + ((v=v"1.14",), "(x;\n @foo a\n b)") => "(block-p x (macrocall (macro_name foo) a) (error-t b))" + ((v=v"1.14",), "(x, @foo a\n b)") => "(tuple-p x (macrocall (macro_name foo) a) (error-t b))" + ((v=v"1.14",), "(x, @foo a\n b, c)") => "(tuple-p x (macrocall (macro_name foo) a) (error-t b ✘ c))" + ((v=v"1.14",), "(x,\n @foo a\n b)") => "(tuple-p x (macrocall (macro_name foo) a) (error-t b))" + ((v=v"1.14",), "(; @foo a\n b)") => "(tuple-p (parameters (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(@foo (x, @bar a\n b))") => "(parens (macrocall (macro_name foo) (tuple-p x (macrocall (macro_name bar) a) (error-t b))))" + ((v=v"1.14",), "(@foo \n (@bar \n x))") => "(parens (macrocall (macro_name foo) (parens (macrocall (macro_name bar) x))))" + ((v=v"1.14",), "(@foo function bar()\n @baz \n x \n end)") => "(parens (macrocall (macro_name foo) (function (call bar) (block (macrocall (macro_name baz)) x))))" + ((v=v"1.14",), "(x = @foo a\n b)") => "(parens (= x (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(x -> @foo a\n b)") => "(parens (-> (tuple x) (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(x + @foo a\n b)") => "(parens (call-i x + (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(1 : @foo x\n y)") => "(parens (call-i 1 : (macrocall (macro_name foo) x)) (error-t y))" + ((v=v"1.14",), "(2@foo x\n y)") => "(parens (juxtapose 2 (macrocall (macro_name foo) x)) (error-t y))" + # ... and the surrounding expression continues past the newline as before + ((v=v"1.14",), "(1 + @foo x\n + 2)") => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" + ((v=v"1.14",), "(1 +\n @foo x\n + 2)") => "(parens (call-i 1 + (macrocall (macro_name foo) x) 2))" + ((v=v"1.14",), "(-@foo a\n b)") => "(parens (call-pre - (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(@foo(a) = @bar b\n c)") => "(parens (= (macrocall-p (macro_name foo) a) (macrocall (macro_name bar) b)) (error-t c))" + ((v=v"1.14",), "(x ? y : @foo a\n b)") => "(parens (? x y (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(:@foo a\n b)") => "(parens (quote-: (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(x where @foo a\n b)") => "(parens (where x (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(return @foo a\n b)") => "(parens (return (macrocall (macro_name foo) a)) (error-t b))" + ((v=v"1.14",), "(@foo(a) + @bar b\n c)") => "(parens (call-i (macrocall-p (macro_name foo) a) + (macrocall (macro_name bar) b)) (error-t c))" + ((v=v"1.14",), "(@foo @bar x y\n z)") => "(parens (macrocall (macro_name foo) (macrocall (macro_name bar) x y) z))" + ((v=v"1.14",), "(@foo x, @bar a\n b)") => "(parens (macrocall (macro_name foo) (tuple x (macrocall (macro_name bar) a)) b))" + ((v=v"1.14",), "(@foo x + @bar a\n b)") => "(parens (macrocall (macro_name foo) (call-i x + (macrocall (macro_name bar) a)) b))" + ((v=v"1.14",), "(@foo [x, @bar a\n b])") => "(parens (macrocall (macro_name foo) (vect x (macrocall (macro_name bar) a) (error-t b))))" + # Blocks inside parentheses use normal newline handling + ((v=v"1.14",), "(begin @foo a\n b end)") => "(parens (block (macrocall (macro_name foo) a) b))" + ((v=v"1.14",), "(f() do x\n @foo a\n b\n end)") => "(parens (call f (do (tuple x) (block (macrocall (macro_name foo) a) b))))" + # A following line which starts with an operator which could apply to + # the macro call ends the call, as before + ((v=v"1.14",), "(@foo x\n - y)") => "(parens (call-i (macrocall (macro_name foo) x) - y))" + ((v=v"1.14",), "(@foo x\n -y)") => "(parens (call-i (macrocall (macro_name foo) x) - y))" + ((v=v"1.14",), "(@foo x\n :y)") => "(parens (call-i (macrocall (macro_name foo) x) : y))" + ((v=v"1.14",), "(@foo x\n \$y)") => "(parens (call-i (macrocall (macro_name foo) x) \$ y))" + ((v=v"1.14",), "(@foo x\n &y)") => "(parens (call-i (macrocall (macro_name foo) x) & y))" + ((v=v"1.14",), "(@foo x\n where T)") => "(parens (where (macrocall (macro_name foo) x) T))" + ((v=v"1.14",), "(@foo x\n in y)") => "(parens (call-i (macrocall (macro_name foo) x) in y))" + ((v=v"1.14",), "(@foo x\n ::T)") => "(parens (::-i (macrocall (macro_name foo) x) T))" + ((v=v"1.14",), "(@foo x\n <: T)") => "(parens (<: (macrocall (macro_name foo) x) T))" + ((v=v"1.14",), "(@foo x\n = y)") => "(parens (= (macrocall (macro_name foo) x) y))" + ((v=v"1.14",), "(@foo x\n ? y : z)") => "(parens (? (macrocall (macro_name foo) x) y z))" + ((v=v"1.14",), "(@foo x\n ...)") => "(parens (... (macrocall (macro_name foo) x)))" + ((v=v"1.14",), "(@foo x\n .y)") => "(parens (. (macrocall (macro_name foo) x) (error-t) y))" + ((v=v"1.14",), "(@foo 1:\n2)") => "(parens (macrocall (macro_name foo) (call-i 1 : (error)) 2))" + # Only a token which can't continue an expression starts a new argument + ((v=v"1.14",), "(@foo x\n 'c')") => "(parens (macrocall (macro_name foo) x (char 'c')))" + ((v=v"1.14",), "(@foo x\n @bar y)") => "(parens (macrocall (macro_name foo) x (macrocall (macro_name bar) y)))" + ((v=v"1.14",), "(@foo x\n [1])") => "(parens (macrocall (macro_name foo) x (vect 1)))" + ((v=v"1.14",), "(@foo x\n \"s\")") => "(parens (macrocall (macro_name foo) x (string \"s\")))" + ((v=v"1.14",), "(@foo +\n x)") => "(parens (macrocall (macro_name foo) + x))" + # `for` on the following line still forms a generator + ((v=v"1.14",), "(@foo x\n for i in xs)") => "(parens (generator (macrocall (macro_name foo) x) (iteration (in i xs))))" + ((v=v"1.14",), "(@doc x\n for i in xs)") => "(parens (macrocall (macro_name doc) x (for (iteration (in i xs)) (block (error)) (error-t))))" + ((v=v"1.14",), "(@doc x\n\ny)") => "(parens (macrocall (macro_name doc) x y))" + ((v=v"1.14",), "(@time\n for i in 1:3 end)") => "(parens (generator (macrocall (macro_name time)) (iteration (in i (call-i 1 : 3)))) (error-t))" + # Before 1.14 a newline always ends the macro arguments + ((v=v"1.13",), "(@foo a\n b)") => "(parens (macrocall (macro_name foo) a) (error-t b))" + ((v=v"1.13",), "(@foo a\n b\n c)") => "(parens (macrocall (macro_name foo) a) (error-t b c))" + ((v=v"1.13",), "(@doc x\n y)") => "(parens (macrocall (macro_name doc) x y))" + # Macro names + "@! x" => "(macrocall (macro_name !) x)" + "@.. x" => "(macrocall (macro_name ..) x)" + "@\$ y" => "(macrocall (macro_name \$) y)" + "@[x] y z" => "(macrocall (macro_name (error (vect x))) y z)" + # Special @doc parsing rules + "@doc x\ny" => "(macrocall (macro_name doc) x y)" + "A.@doc x\ny" => "(macrocall (. A (macro_name doc)) x y)" + "@A.doc x\ny" => "(macrocall (macro_name (. A doc)) x y)" + "@doc x y\nz" => "(macrocall (macro_name doc) x y)" + "@doc x\n\ny" => "(macrocall (macro_name doc) x)" + "@doc x\nend" => "(macrocall (macro_name doc) x)" + + # Special 1.14 @VERSION parsing rules + ((v=v"1.13",), "@VERSION") => "(macrocall (macro_name VERSION))" + ((v=v"1.13",), "@A.B.VERSION") => "(macrocall (macro_name (. (. A B) VERSION)))" + ((v=v"1.13",), "A.B.@VERSION") => "(macrocall (. (. A B) (macro_name VERSION)))" + ((v=v"1.14",), "@VERSION") => "(macrocall (macro_name VERSION) v\"1.14.0\")" + ((v=v"1.14",), "@A.B.VERSION") => "(macrocall (macro_name (. (. A B) VERSION)) v\"1.14.0\")" + ((v=v"1.14",), "A.B.@VERSION") => "(macrocall (. (. A B) (macro_name VERSION)) v\"1.14.0\")" + + # calls with brackets + "f(a,b)" => "(call f a b)" + "f(a,)" => "(call-, f a)" + "f(a=1; b=2)" => "(call f (= a 1) (parameters (= b 2)))" + "f(a; b; c)" => "(call f a (parameters b) (parameters c))" + "(a=1)()" => "(call (parens (= a 1)))" + "f (a)" => "(call f (error-t) a)" + "@x(a, b)" => "(macrocall-p (macro_name x) a b)" + "@x(a, b,)" => "(macrocall-p-, (macro_name x) a b)" + "A.@x(y)" => "(macrocall-p (. A (macro_name x)) y)" + "A.@x(y).z" => "(. (macrocall-p (. A (macro_name x)) y) z)" + "f(y for x = xs; a)" => "(call f (generator y (iteration (in x xs))) (parameters a))" + # do + "f() do\nend" => "(call f (do (tuple) (block)))" + "f() do ; body end" => "(call f (do (tuple) (block body)))" + "f() do x, y\n body end" => "(call f (do (tuple x y) (block body)))" + "f(x) do y body end" => "(call f x (do (tuple y) (block body)))" + "@f(x) do y body end" => "(macrocall-p (macro_name f) x (do (tuple y) (block body)))" + + # square brackets + "@S[a,b]" => "(macrocall (macro_name S) (vect a b))" + "@S[a b]" => "(macrocall (macro_name S) (hcat a b))" + "@S[a; b]" => "(macrocall (macro_name S) (vcat a b))" + "A.@S[a]" => "(macrocall (. A (macro_name S)) (vect a))" + "@S[a].b" => "(. (macrocall (macro_name S) (vect a)) b)" + ((v=v"1.7",), "@S[a ;; b]") => "(macrocall (macro_name S) (ncat-2 a b))" + ((v=v"1.6",), "@S[a ;; b]") => "(macrocall (macro_name S) (error (ncat-2 a b)))" + "a[i]" => "(ref a i)" + "a [i]" => "(ref a (error-t) i)" + "a[i,j]" => "(ref a i j)" + "(a=1)[]" => "(ref (parens (= a 1)))" + "a[end]" => "(ref a end)" + "a[begin]" => "(ref a begin)" + "a[:(end)]" => "(typed_hcat a (quote-: (parens (error-t))) (error-t))" + "T[x y]" => "(typed_hcat T x y)" + "T[x ; y]" => "(typed_vcat T x y)" + "T[a b; c d]" => "(typed_vcat T (row a b) (row c d))" + "T[x for x in xs]" => "(typed_comprehension T (generator x (iteration (in x xs))))" + ((v=v"1.8",), "T[a ; b ;; c ; d]") => "(typed_ncat-2 T (nrow-1 a b) (nrow-1 c d))" + + # Dotted forms + # Allow `@` in macrocall only in first and last position + "A.B.@x" => "(macrocall (. (. A B) (macro_name x)))" + "@A.B.x" => "(macrocall (macro_name (. (. A B) x)))" + "A.@B.x" => "(macrocall (. (. A (error-t) B) (macro_name (error-t) x)))" + "@M.(x)" => "(macrocall (dotcall (macro_name M) (error-t) x))" + "f.(a,b)" => "(dotcall f a b)" + "f.(a,b,)" => "(dotcall-, f a b)" + "f.(a=1; b=2)" => "(dotcall f (= a 1) (parameters (= b 2)))" + "(a=1).()" => "(dotcall (parens (= a 1)))" + "f. (x)" => "(dotcall f (error-t) x)" + # Other dotted syntax + "A.:+" => "(. A (quote-: +))" + "A.:.+" => "(. A (quote-: (. +)))" + "A.: +" => "(. A (quote-: (error-t) +))" + "f.\$x" => "(. f (\$ x))" + "f.\$(x+y)" => "(. f (\$ (parens (call-i x + y))))" + "A.\$B.@x" => "(macrocall (. (. A (\$ B)) (macro_name x)))" + "@A.\$x a" => "(macrocall (macro_name (. A (error x))) a)" + "A.@x" => "(macrocall (. A (macro_name x)))" + "A.@x a" => "(macrocall (. A (macro_name x)) a)" + "@A.B.@x a" => "(macrocall (macro_name (. (. A B) (error-t) x)) a)" + # .[ and .{ disallowed + "f.[x]" => "(error f x)" + "f.{x}" => "(error f x)" + # .' discontinued + "f.'" => "(dotcall-post f (error '))" + # Field/property syntax + "f.x.y" => "(. (. f x) y)" + "x .y" => "(. x (error-t) y)" + "x.?" => "(. x ?)" + "x.in" => "(. x in)" + # Adjoint + "f'" => "(call-post f ')" + "f'ᵀ" => "(call-post f 'ᵀ)" + # Curly calls + "S {a}" => "(curly S (error-t) a)" + "A.@S{a}" => "(macrocall (. A (macro_name S)) (braces a))" + "@S{a,b}" => "(macrocall (macro_name S) (braces a b))" + "A.@S{a}" => "(macrocall (. A (macro_name S)) (braces a))" + "@S{a}.b" => "(. (macrocall (macro_name S) (braces a)) b)" + # Macro calls with chained operations + "@a[b][c]" => "(ref (macrocall (macro_name a) (vect b)) c)" + "@a{b}{c}" => "(curly (macrocall (macro_name a) (braces b)) c)" + "@a[b]{c}" => "(curly (macrocall (macro_name a) (vect b)) c)" + "@a{b}[c]" => "(ref (macrocall (macro_name a) (braces b)) c)" + "S{a,b}" => "(curly S a b)" + "T{y for x = xs; a}" => "(curly T (generator y (iteration (in x xs))) (parameters a))" + # String macros + "x\"str\"" => """(macrocall @x_str (string-r "str"))""" + "x`str`" => """(macrocall @x_cmd (cmdstring-r "str"))""" + "x\"\"" => """(macrocall @x_str (string-r ""))""" + "x``" => """(macrocall @x_cmd (cmdstring-r ""))""" + "in\"str\"" => """(macrocall @in_str (string-r "str"))""" + "outer\"str\"" => """(macrocall @outer_str (string-r "str"))""" + "A.x\"str\"" => """(macrocall (. A @x_str) (string-r "str"))""" + "A.x`str`" => """(macrocall (. A @x_cmd) (cmdstring-r "str"))""" + # Triple quoted processing for custom strings + "r\"\"\"\nx\"\"\"" => raw"""(macrocall @r_str (string-s-r "x"))""" + "r\"\"\"\n x\n y\"\"\"" => raw"""(macrocall @r_str (string-s-r "x\n" "y"))""" + "r\"\"\"\n x\\\n y\"\"\"" => raw"""(macrocall @r_str (string-s-r "x\\\n" "y"))""" + # Macro suffixes can include keywords and numbers + "x\"s\"y" => """(macrocall @x_str (string-r "s") "y")""" + "x\"s\"end" => """(macrocall @x_str (string-r "s") "end")""" + "x\"s\"in" => """(macrocall @x_str (string-r "s") "in")""" + "x\"s\"2" => """(macrocall @x_str (string-r "s") 2)""" + "x\"s\"10.0" => """(macrocall @x_str (string-r "s") 10.0)""" + # Cmd macro suffixes + "x`s`y" => """(macrocall @x_cmd (cmdstring-r "s") "y")""" + "x`s`end" => """(macrocall @x_cmd (cmdstring-r "s") "end")""" + "x`s`in" => """(macrocall @x_cmd (cmdstring-r "s") "in")""" + "x`s`2" => """(macrocall @x_cmd (cmdstring-r "s") 2)""" + "x`s`10.0" => """(macrocall @x_cmd (cmdstring-r "s") 10.0)""" + ], + JuliaSyntax.parse_resword => [ + # In normal_context + "begin f() where T = x end" => "(block (function-= (where (call f) T) x))" + # block + "begin end" => "(block)" + "begin a ; b end" => "(block a b)" + "begin\na\nb\nend" => "(block a b)" + # quote + "quote end" => "(quote (block))" + "quote body end" => "(quote (block body))" + # while + "while cond body end" => "(while cond (block body))" + "while x < y \n a \n b \n end" => "(while (call-i x < y) (block a b))" + # for + "for x in xs end" => "(for (iteration (in x xs)) (block))" + "for x in xs, y in ys \n a \n end" => "(for (iteration (in x xs) (in y ys)) (block a))" + # let + "let x=1\n end" => "(let (block (= x 1)) (block))" + "let x=1 ; end" => "(let (block (= x 1)) (block))" + "let x ; end" => "(let (block x) (block))" + "let x::1 ; end" => "(let (block (::-i x 1)) (block))" + "let x=1,y=2 end" => "(let (block (= x 1) (= y 2)) (block))" + "let x+=1 ; end" => "(let (block (op= x + 1)) (block))" + "let ; end" => "(let (block) (block))" + "let ; body end" => "(let (block) (block body))" + "let\na\nb\nend" => "(let (block) (block a b))" + # abstract type + "abstract type A end" => "(abstract A)" + "abstract type A ; end" => "(abstract A)" + "abstract type \n\n A \n\n end" => "(abstract A)" + "abstract type A <: B end" => "(abstract (<: A B))" + "abstract type A <: B{T,S} end" => "(abstract (<: A (curly B T S)))" + "abstract type A < B end" => "(abstract (call-i A < B))" + # primitive type + "primitive type A 32 end" => "(primitive A 32)" + "primitive type A 32 ; end" => "(primitive A 32)" + "primitive type A \$N end" => "(primitive A (\$ N))" + "primitive type A <: B \n 8 \n end" => "(primitive (<: A B) 8)" + # struct + "struct A <: B \n a::X \n end" => "(struct (<: A B) (block (::-i a X)))" + "struct A \n a \n b \n end" => "(struct A (block a b))" + "struct A \n \"doca\" \n a \n \"docb\" \n b \n end" => "(struct A (block (doc (string \"doca\") a) (doc (string \"docb\") b)))" + "mutable struct A end" => "(struct-mut A (block))" + ((v=v"1.8",), "struct A const a end") => "(struct A (block (const a)))" + ((v=v"1.7",), "struct A const a end") => "(struct A (block (error (const a))))" + "struct A end" => "(struct A (block))" + "struct try end" => "(struct (error try) (block))" + # typegroup (1.14+) + ((v=v"1.14",), "typegroup struct A end end") => "(typegroup (block (struct A (block))))" + ((v=v"1.14",), "typegroup\nstruct A\na::Int\nend\nend") => "(typegroup (block (struct A (block (::-i a Int)))))" + ((v=v"1.14",), "typegroup\nstruct A end\nstruct B end\nend") => "(typegroup (block (struct A (block)) (struct B (block))))" + ((v=v"1.13",), "typegroup struct A end end") => "(error (typegroup (block (struct A (block)))))" + # module/baremodule + "module A end" => "(module A (block))" + "baremodule A end" => "(module-bare A (block))" + "module do \n end" => "(module (error do) (block))" + "module \$A end" => "(module (\$ A) (block))" + "module A \n a \n b \n end" => "(module A (block a b))" + """module A \n "x"\na\n end""" => """(module A (block (doc (string "x") a)))""" + # export + "export a" => "(export a)" + "export @a" => "(export (macro_name a))" + "export @var\"'\"" => "(export (macro_name (var ')))" + "export a, \n @b" => "(export a (macro_name b))" + "export +, ==" => "(export + ==)" + "export \n a" => "(export a)" + "export \$a, \$(a*b)" => "(export (\$ a) (\$ (parens (call-i a * b))))" + "export (x::T)" => "(export (error (parens (::-i x T))))" + "export outer" => "(export outer)" + "export (\$f)" => "(export (parens (\$ f)))" + ], + JuliaSyntax.parse_if_elseif => [ + "if a xx elseif b yy else zz end" => "(if a (block xx) (elseif b (block yy) (block zz)))" + "if end" => "(if (error) (block))" + "if \n end" => "(if (error) (block))" + "if a end" => "(if a (block))" + "if a xx end" => "(if a (block xx))" + "if a \n\n xx \n\n end" => "(if a (block xx))" + "if a xx elseif b yy end" => "(if a (block xx) (elseif b (block yy)))" + "if a xx else if b yy end" => "(if a (block xx) (error-t) (elseif b (block yy)))" + "if a xx else yy end" => "(if a (block xx) (block yy))" + "if true; x ? true elseif true end" => "(if true (block (if x true (error-t) (error-t))) (elseif true (block)))" + "if true; x ? true end" => "(if true (block (if x true (error-t) (error-t))))" + "if true; x ? true\nend" => "(if true (block (if x true (error-t) (error-t))))" + "if true; x ? true : elseif true end" => "(if true (block (if x true (error-t))) (elseif true (block)))" + ], + JuliaSyntax.parse_resword => [ + "global x" => "(global x)" + "local x" => "(local x)" + "global x,y" => "(global x y)" + "global const x = 1" => "(global (const (= x 1)))" + "local const x = 1" => "(local (const (= x 1)))" + "const global x = 1" => "(const (global (= x 1)))" + "const local x = 1" => "(const (local (= x 1)))" + "const x,y = 1,2" => "(const (= (tuple x y) (tuple 1 2)))" + "const x = 1" => "(const (= x 1))" + "const x .= 1" => "(error (const (.= x 1)))" + "global x ~ 1" => "(global (call-i x ~ 1))" + "global x += 1" => "(global (op= x + 1))" + "const x" => "(error (const x))" + "global const x" => "(global (error (const x)))" + "const global x" => "(error (const (global x)))" + ], + JuliaSyntax.parse_resword => [ + # Macros and functions + "macro while(ex) end" => "(macro (call (error while) ex) (block))" + "macro f() end" => "(macro (call f) (block))" + "macro (:)(ex) end" => "(macro (call (parens :) ex) (block))" + "macro (type)(ex) end" => "(macro (call (parens type) ex) (block))" + "macro \$f() end" => "(macro (call (\$ f)) (block))" + "macro (\$f)() end" => "(macro (call (parens (\$ f))) (block))" + "function (x) body end"=> "(function (tuple-p x) (block body))" + "function (x)\n body\nend"=> "(function (tuple-p x) (block body))" + "function (x)\n() end" => "(function (tuple-p x) (block (tuple-p)))" + "function (x,y) end" => "(function (tuple-p x y) (block))" + "function (x,y,) end" => "(function (tuple-p-, x y) (block))" + "function (x=1) end" => "(function (tuple-p (= x 1)) (block))" + "function (;x=1) end" => "(function (tuple-p (parameters (= x 1))) (block))" + "function (f(x),) end" => "(function (tuple-p-, (call f x)) (block))" + "function (@f(x);) end" => "(function (tuple-p (macrocall-p (macro_name f) x) (parameters)) (block))" + "function (@f(x)...) end" => "(function (tuple-p (... (macrocall-p (macro_name f) x))) (block))" + "function (@f(x)) end" => "(function (error (tuple-p (macrocall-p (macro_name f) x))) (block))" + "function (\$f) end" => "(function (error (tuple-p (\$ f))) (block))" + "function ()(x) end" => "(function (call (tuple-p) x) (block))" + "function (A).f() end" => "(function (call (. (parens A) f)) (block))" + "function (:)() end" => "(function (call (parens :)) (block))" + "function (x::T)() end"=> "(function (call (parens (::-i x T))) (block))" + "function (::g(x))() end" => "(function (call (parens (::-pre (call g x)))) (block))" + "function (f::T{g(i)})() end" => "(function (call (parens (::-i f (curly T (call g i))))) (block))" + "function (::T)() end" => "(function (call (parens (::-pre T))) (block))" + "function (\n ::T\n )() end" => "(function (call (parens (::-pre T))) (block))" + "function (\n x::T\n )() end" => "(function (call (parens (::-i x T))) (block))" + "function (\n f\n )() end" => "(function (call (parens f)) (block))" + "function (\n A\n ).f() end" => "(function (call (. (parens A) f)) (block))" + "function (\n ::T\n )(x, y) end" => "(function (call (parens (::-pre T)) x y) (block))" + "function (\n f::T{g(i)}\n )() end" => "(function (call (parens (::-i f (curly T (call g i))))) (block))" + "function (\n x, y\n ) x + y end" => "(function (tuple-p x y) (block (call-i x + y)))" + "function (:*=(f))() end" => "(function (call (parens (call (quote-: (op= *)) f))) (block))" + "function begin() end" => "(function (call (error begin)) (block))" + "function f() end" => "(function (call f) (block))" + "function type() end" => "(function (call type) (block))" + "function \n f() end" => "(function (call f) (block))" + "function \$f() end" => "(function (call (\$ f)) (block))" + "function (::Type{T})(x) end" => "(function (call (parens (::-pre (curly Type T))) x) (block))" + # Function/macro definition with no methods + "function f end" => "(function f)" + "function f \n\n end" => "(function f)" + "function \$f end" => "(function (\$ f))" + "function var\".\" end" => "(function (var .))" + "macro f end" => "(macro f)" + # Function argument list + "function f(x,y) end" => "(function (call f x y) (block))" + "function f{T}() end" => "(function (call (curly f T)) (block))" + "function A.f() end" => "(function (call (. A f)) (block))" + "function f body end" => "(function (error f) (block body))" + "function f()::T end" => "(function (::-i (call f) T) (block))" + "function f()::g(T) end" => "(function (::-i (call f) (call g T)) (block))" + "function f() where {T} end" => "(function (where (call f) (braces T)) (block))" + "function f() where T end" => "(function (where (call f) T) (block))" + "function f()::S where T end" => "(function (where (::-i (call f) S) T) (block))" + # Ugly cases for compat where extra parentheses existed and we've + # already parsed at least the call part of the signature + "function (f() where T) end" => "(function (parens (where (call f) T)) (block))" + "function (f()) where T end" => "(function (where (parens (call f)) T) (block))" + "function (f() where T) where U end" => "(function (where (parens (where (call f) T)) U) (block))" + "function (f()::S) end"=> "(function (parens (::-i (call f) S)) (block))" + "function ((f()::S) where T) end" => "(function (parens (where (parens (::-i (call f) S)) T)) (block))" + "function (x*y ) end" => "(function (parens (call-i x * y)) (block))" + # body + "function f() \n a \n b end" => "(function (call f) (block a b))" + "function f() end" => "(function (call f) (block))" + # Macrocall as sig + ((v=v"1.12",), "function @callmemacro(a::Int) \n 1 \n end") => "(function (macrocall-p (macro_name callmemacro) (::-i a Int)) (block 1))" + ((v=v"1.12",), "function @callmemacro(a::T, b::T) where T <: Int64\n3\nend") => "(function (where (macrocall-p (macro_name callmemacro) (::-i a T) (::-i b T)) (<: T Int64)) (block 3))" + ((v=v"1.12",), "function @callmemacro(a::Int, b::Int, c::Int)::Float64\n4\nend") => "(function (::-i (macrocall-p (macro_name callmemacro) (::-i a Int) (::-i b Int) (::-i c Int)) Float64) (block 4))" + ((v=v"1.12",), "function @f()() end") => "(function (call (macrocall-p (macro_name f))) (block))" + # Errors + "function" => "(function (error (error)) (block (error)) (error-t))" + ], + JuliaSyntax.parse_try => [ + "try \n x \n catch e \n y \n finally \n z end" => + "(try (block x) (catch e (block y)) (finally (block z)))" + ((v=v"1.8",), "try \n x \n catch e \n y \n else z finally \n w end") => + "(try (block x) (catch e (block y)) (else (block z)) (finally (block w)))" + "try x catch end" => "(try (block x) (catch □ (block)))" + "try x catch ; y end" => "(try (block x) (catch □ (block y)))" + "try x catch \n y end" => "(try (block x) (catch □ (block y)))" + "try x catch e y end" => "(try (block x) (catch e (block y)))" + "try x catch \$e y end" => "(try (block x) (catch (\$ e) (block y)))" + "try x catch var\"#\" y end" => "(try (block x) (catch (var #) (block y)))" + "try x catch e+3 y end" => "(try (block x) (catch (error (call-i e + 3)) (block y)))" + "try x finally y end" => "(try (block x) (finally (block y)))" + # v1.8 only + ((v=v"1.8",), "try catch ; else end") => "(try (block) (catch □ (block)) (else (block)))" + ((v=v"1.8",), "try else x finally y end") => "(try (block) (else (error (block x))) (finally (block y)))" + ((v=v"1.7",), "try catch ; else end") => "(try (block) (catch □ (block)) (else (error (block))))" + # finally before catch :-( + "try x finally y catch e z end" => "(try (block x) (finally (block y)) (catch e (block z)))" + "try x end" => "(try (block x) (error-t))" + ], + JuliaSyntax.parse_imports => [ + "import A as B: x" => "(import (: (error (as (importpath A) B)) (importpath x)))" + "import A, y" => "(import (importpath A) (importpath y))" + "import A: +, ==" => "(import (: (importpath A) (importpath +) (importpath ==)))" + "import A: x, y" => "(import (: (importpath A) (importpath x) (importpath y)))" + "import A: x, B: y" => "(import (: (importpath A) (importpath x) (importpath B) (error-t (importpath y))))" + "import A: x" => "(import (: (importpath A) (importpath x)))" + "using A" => "(using (importpath A))" + "import A" => "(import (importpath A))" + # parse_import + "import A: x, y" => "(import (: (importpath A) (importpath x) (importpath y)))" + "import A as B" => "(import (as (importpath A) B))" + "import A: x as y" => "(import (: (importpath A) (as (importpath x) y)))" + "using A: x as y" => "(using (: (importpath A) (as (importpath x) y)))" + ((v=v"1.5",), "import A as B") => "(import (error (as (importpath A) B)))" + "using A as B" => "(using (error (as (importpath A) B)))" + "using A, B as C" => "(using (importpath A) (error (as (importpath B) C)))" + # parse_import_path + # When parsing import we must split initial dots into nontrivial + # leading dots for relative paths + "import .A" => "(import (importpath . A))" + "import ..A" => "(import (importpath . . A))" + "import ...A" => "(import (importpath . . . A))" + "import ....A" => "(import (importpath . . . . A))" + # Dots with spaces are allowed (a misfeature?) + "import . .A" => "(import (importpath . . A))" + # Modules with operator symbol names + "import .⋆" => "(import (importpath . ⋆))" + # Expressions allowed in import paths + "import @x" => "(import (importpath (macro_name x)))" + "import \$A" => "(import (importpath (\$ A)))" + "import \$A.@x" => "(import (importpath (\$ A) (macro_name x)))" + "import A.B" => "(import (importpath A B))" + "import A.B.C" => "(import (importpath A B C))" + "import A.:+" => "(import (importpath A (quote-: +)))" + "import A.(:+)" => "(import (importpath A (parens (quote-: +))))" + "import A.:(+)" => "(import (importpath A (quote-: (parens +))))" + "import A.==" => "(import (importpath A ==))" + "import A.⋆.f" => "(import (importpath A ⋆ f))" + "import A..." => "(import (importpath A (DotsIdentifier-2)))" + "import A; B" => "(import (importpath A))" + # Colons not allowed first in import paths + # but are allowed in trailing components (#473) + "using :A" => "(using (importpath (error (quote-: A))))" + "using A: :b" => "(using (: (importpath A) (importpath (error (quote-: b)))))" + "using A: b.:c" => "(using (: (importpath A) (importpath b (quote-: c))))" + # Syntactic operators not allowed in import + ], + JuliaSyntax.parse_iteration_specs => [ + "i = rhs" => "(iteration (in i rhs))" + "i in rhs" => "(iteration (in i rhs))" + "i ∈ rhs" => "(iteration (in i rhs))" + "i = 1:10" => "(iteration (in i (call-i 1 : 10)))" + "(i,j) in iter" => "(iteration (in (tuple-p i j) iter))" + "outer = rhs" => "(iteration (in outer rhs))" + "outer <| x = rhs" => "(iteration (in (call-i outer <| x) rhs))" + "outer i = rhs" => "(iteration (in (outer i) rhs))" + "outer (x,y) = rhs" => "(iteration (in (outer (tuple-p x y)) rhs))" + ], + JuliaSyntax.parse_paren => [ + # Tuple syntax with commas + "()" => "(tuple-p)" + "(x,)" => "(tuple-p-, x)" + "(x,y)" => "(tuple-p x y)" + "(x=1, y=2)" => "(tuple-p (= x 1) (= y 2))" + # Named tuples with initial semicolon + "(;)" => "(tuple-p (parameters))" + "(; a=1)" => "(tuple-p (parameters (= a 1)))" + # Extra credit: nested parameters and frankentuples + "(x...; y)" => "(tuple-p (... x) (parameters y))" + "(x...;)" => "(tuple-p (... x) (parameters))" + "(; a=1; b=2)" => "(tuple-p (parameters (= a 1)) (parameters (= b 2)))" + "(a; b; c,d)" => "(tuple-p a (parameters b) (parameters c d))" + "(a=1, b=2; c=3)" => "(tuple-p (= a 1) (= b 2) (parameters (= c 3)))" + # Block syntax + "(;;)" => "(block-p)" + "(a=1;)" => "(block-p (= a 1))" + "(a;b;;c)" => "(block-p a b c)" + "(a=1; b=2)" => "(block-p (= a 1) (= b 2))" + # Following is an error for flisp compatibility. But it could be + # allowed as valid block syntax in the future? + "(y for x = xs; a)" => "(parens (generator y (iteration (in x xs))) (error-t ✘ a))" + # Parentheses used for grouping + "(a * b)" => "(parens (call-i a * b))" + "(a=1)" => "(parens (= a 1))" + "(x)" => "(parens x)" + "(a...)" => "(parens (... a))" + # Generators + "(x for a in as)" => "(parens (generator x (iteration (in a as))))" + "(x \n\n for a in as)" => "(parens (generator x (iteration (in a as))))" + # Range parsing in parens + "(1:\n2)" => "(parens (call-i 1 : 2))" + "(1:2)" => "(parens (call-i 1 : 2))" + ], + JuliaSyntax.parse_atom => [ + # char literal + "'a'" => "(char 'a')" + "'α'" => "(char 'α')" + "'\\xce\\xb1'" => "(char 'α')" + "'\\u03b1'" => "(char 'α')" + "'\\U1D7DA'" => "(char '𝟚')" + "'a" => "(char 'a' (error-t))" + "''" => "(char (error))" + "'" => "(char (error))" + # symbol/expression quote + ":foo" => "(quote-: foo)" + # Literal colons + ":)" => ":" + ": end" => ":" + # Whitespace after quoting colon + ": foo" => "(quote-: (error-t) foo)" + ":\nfoo" => "(quote-: (error-t) foo)" + # plain equals + "=" => "(error =)" + # Identifiers + "xx" => "xx" + "x₁" => "x₁" + # var syntax + """var"x" """ => "(var x)" + # var syntax raw string unescaping + "var\"\"" => "(var )" + "var\"\\\"\"" => "(var \")" + "var\"\\\\\\\"\"" => "(var \\\")" + "var\"\\\\x\"" => "(var \\\\x)" + # trailing syntax after var + """var"x"+""" => "(var x)" + """var"x")""" => "(var x)" + """var"x"(""" => "(var x)" + """var"x"end""" => "(var x (error-t))" + """var"x"1""" => "(var x (error-t))" + """var"x"y""" => "(var x (error-t))" + # Standalone syntactic operators are errors + "?" => "(error ?)" + "&&" => "(error &&)" + "||" => "(error ||)" + "." => "(error .)" + "..." => "(error (DotsIdentifier-3))" + "+=" => "(error (op= +))" + "-=" => "(error (op= -))" + "*=" => "(error (op= *))" + "/=" => "(error (op= /))" + "//=" => "(error (op= //))" + "|=" => "(error (op= |))" + "^=" => "(error (op= ^))" + "÷=" => "(error (op= ÷))" + "%=" => "(error (op= %))" + "<<=" => "(error (op= <<))" + ">>=" => "(error (op= >>))" + ">>>="=> "(error (op= >>>))" + "\\=" => "(error (op= \\))" + "&=" => "(error (op= &))" + ":=" => "(error :=)" # Assignment operator, not `:`-update + "\$=" => "(error (op= \$))" + "⊻=" => "(error (op= ⊻))" + ".+=" => "(error (.op= +))" + with_version(v"1.14", "+%=" => "(error (op= +%))") + with_version(v"1.14", "-%=" => "(error (op= -%))") + with_version(v"1.14", "*%=" => "(error (op= *%))") + # Normal operators + "+" => "+" + with_version(v"1.14", "+%" => "+%") + # Assignment-precedence operators which can be used as identifiers + "~" => "~" + "≔" => "≔" + "⩴" => "⩴" + "≕" => "≕" + # Quoted syntactic operators allowed + ":+=" => "(quote-: (op= +))" + ":.+=" => "(quote-: (.op= +))" + ":.=" => "(quote-: (. =))" + ":.&&" => "(quote-: (. &&))" + # Special symbols quoted + ":end" => "(quote-: end)" + ":(end)" => "(quote-: (parens (error-t)))" + ":<:" => "(quote-: <:)" + # unexpected = + "=" => "(error =)" + # parse_cat + "[]" => "(vect)" + "[x,]" => "(vect-, x)" + "[x,y,]" => "(vect-, x y)" + "[x\n,,]" => "(vect-, x (error-t ✘))" + "[x]" => "(vect x)" + "[x \n ]" => "(vect x)" + "[x \n, ]" => "(vect-, x)" + "[x" => "(vect x (error-t))" + "[x \n\n ]" => "(vect x)" + "[x for a in as]" => "(comprehension (generator x (iteration (in a as))))" + "[x \n\n for a in as]" => "(comprehension (generator x (iteration (in a as))))" + # parse_generator + "(x for a in as for b in bs)" => "(parens (generator x (iteration (in a as)) (iteration (in b bs))))" + "(x for a in as, b in bs)" => "(parens (generator x (iteration (in a as) (in b bs))))" + "(x for a in as, b in bs if z)" => "(parens (generator x (filter (iteration (in a as) (in b bs)) z)))" + "(x for a in as, b in bs for c in cs, d in ds)" => "(parens (generator x (iteration (in a as) (in b bs)) (iteration (in c cs) (in d ds))))" + "(x for a in as for b in bs if z)" => "(parens (generator x (iteration (in a as)) (filter (iteration (in b bs)) z)))" + "(x for a in as if z for b in bs)" => "(parens (generator x (filter (iteration (in a as)) z) (iteration (in b bs))))" + "[x for a = as for b = bs if cond1 for c = cs if cond2]" => "(comprehension (generator x (iteration (in a as)) (filter (iteration (in b bs)) cond1) (filter (iteration (in c cs)) cond2)))" + "[x for a = as if begin cond2 end]" => "(comprehension (generator x (filter (iteration (in a as)) (block cond2))))" + "[(x)for x in xs]" => "(comprehension (generator (parens x) (error-t) (iteration (in x xs))))" + "(x for a in as if z)" => "(parens (generator x (filter (iteration (in a as)) z)))" + # parse_vect + "[x, y]" => "(vect x y)" + "[x, y,]" => "(vect-, x y)" + "[x,\n y]" => "(vect x y)" + "[x\n, y]" => "(vect x y)" + "[x\n,, y]" => "(vect-, x (error-t ✘ y))" + "[x,y ; z]" => "(vect x y (parameters z))" + "[x=1, y=2]" => "(vect (= x 1) (= y 2))" + "[x=1, ; y=2]" => "(vect (= x 1) (parameters (= y 2)))" + # parse_paren + ":(=)" => "(quote-: (parens =))" + ":(::)" => "(quote-: (parens ::))" + ":(::\n)" => "(quote-: (parens ::))" + "(function f \n end)" => "(parens (function f))" + # braces + "{x,y}" => "(braces x y)" + "{x,y,}" => "(braces-, x y)" + "{x y}" => "(bracescat (row x y))" + ((v=v"1.7",), "{x ;;; y}") => "(bracescat (nrow-3 x y))" + ((v=v"1.7",), "{a ;; b}") => "(bracescat (nrow-2 a b))" + ((v=v"1.7",), "{a ;;;; b}") => "(bracescat (nrow-4 a b))" + # Macro names can be keywords + "@end x" => "(macrocall (macro_name end) x)" + # __dot__ macro + "@. x" => "(macrocall (macro_name .) x)" + # cmd strings + "``" => "(cmdstring-r \"\")" + "`cmd`" => "(cmdstring-r \"cmd\")" + "```cmd```" => "(cmdstring-s-r \"cmd\")" + # literals + "true" => "true" + "42" => "42" + "1.0e-1000" => "0.0" + "0x123456789abcdefp+0" => "8.19855292164869e16" + # closing tokens + ")" => "(error)" + ], + JuliaSyntax.parse_atom => [ + # Actually parse_array + # Normal matrix construction syntax + "[x y ; z w]" => "(vcat (row x y) (row z w))" + "[x y ; z w ; a b]" => "(vcat (row x y) (row z w) (row a b))" + "[x ; y ; z]" => "(vcat x y z)" + "[x;]" => "(vcat x)" + "[x y]" => "(hcat x y)" + # Early abort in array parsing + "[x@y" => "(hcat x (error-t ✘ y))" + "[x@y]" => "(hcat x (error-t ✘ y))" + # Mismatched rows + "[x y ; z]" => "(vcat (row x y) z)" + # Single elements in rows + ((v=v"1.7",), "[x ; y ;; z ]") => "(ncat-2 (nrow-1 x y) z)" + ((v=v"1.7",), "[x y ;;; z ]") => "(ncat-3 (row x y) z)" + # Higher dimensional ncat + # Row major + ((v=v"1.7",), "[x y ; z w ;;; a b ; c d]") => + "(ncat-3 (nrow-1 (row x y) (row z w)) (nrow-1 (row a b) (row c d)))" + # Column major + ((v=v"1.7",), "[x ; y ;; z ; w ;;; a ; b ;; c ; d]") => + "(ncat-3 (nrow-2 (nrow-1 x y) (nrow-1 z w)) (nrow-2 (nrow-1 a b) (nrow-1 c d)))" + # Dimension 4 ncat + ((v=v"1.7",), "[x ;;;; y]") => "(ncat-4 x y)" + ((v=v"1.7",), "[a ; b ;;;; c ; d]") => "(ncat-4 (nrow-1 a b) (nrow-1 c d))" + ((v=v"1.7",), "[a b ; c d ;;;; e f ; g h]") => + "(ncat-4 (nrow-1 (row a b) (row c d)) (nrow-1 (row e f) (row g h)))" + # Array separators + # Newlines before semicolons are not significant + "[a \n ;]" => "(vcat a)" + # Newlines after semicolons are not significant + "[a ; \n]" => "(vcat a)" + "[a ; \n\n b]" => "(vcat a b)" + ((v=v"1.7",), "[a ;; \n b]") => "(ncat-2 a b)" + # In hcat with spaces as separators, `;;` is a line + # continuation character + ((v=v"1.7",), "[a b ;; \n c]") => "(hcat a b c)" + ((v=v"1.7",), "[a b \n ;; c]") => "(ncat-2 (row a b (error-t)) c)" + # Can't mix spaces and multiple ;'s + ((v=v"1.7",), "[a b ;; c]") => "(ncat-2 (row a b (error-t)) c)" + # Linebreaks not significant before closing `]` + "[a b\n\n]" => "(hcat a b)" + # Treat a linebreak prior to a value as a semicolon (ie, separator for + # the first dimension) if no previous semicolons observed + "[a \n b]" => "(vcat a b)" + # Can't mix multiple ;'s and spaces + ((v=v"1.7",), "[a ;; b c]") => "(ncat-2 a (row b (error-t) c))" + # Empty N-dimensional arrays + ((v=v"1.8",), "[;]") => "(ncat-1)" + ((v=v"1.8",), "[;;]") => "(ncat-2)" + ((v=v"1.8",), "[\n ;; \n ]") => "(ncat-2)" + ((v=v"1.7",), "[;;]") => "(ncat-2 (error))" + # parse_string + "\"\"\"\n\$x\n a\"\"\"" => "(string-s x \"\\n\" \" a\")" + "\"a \$(x + y) b\"" => "(string \"a \" (parens (call-i x + y)) \" b\")" + "\"hi\$(\"ho\")\"" => "(string \"hi\" (parens (string \"ho\")))" + "\"\$(x,y)\"" => "(string (parens (error x y)))" + "\"\$(x;y)\"" => "(string (parens (error x y)))" + "\"\$(x for y in z)\"" => "(string (parens (error (generator x (iteration (in y z))))))" + "\"\$((x for y in z))\"" => "(string (parens (parens (generator x (iteration (in y z))))))" + "\"\$(xs...)\"" => "(string (parens (... xs)))" + "\"a \$foo b\"" => "(string \"a \" foo \" b\")" + "\"\$var\"" => "(string var)" + "\"\$outer\"" => "(string outer)" + "\"\$in\"" => "(string in)" + # Triple-quoted dedenting: + "\"\"\"\nx\"\"\"" => raw"""(string-s "x")""" + "\"\"\"\n\nx\"\"\"" => raw"""(string-s "\n" "x")""" + "```\n x\n y```" => raw"""(cmdstring-s-r "x\n" "y")""" + # Various newlines (\n \r \r\n) and whitespace (' ' \t) + "\"\"\"\n x\n y\"\"\"" => raw"""(string-s "x\n" "y")""" + "\"\"\"\r x\r y\"\"\"" => raw"""(string-s "x\n" "y")""" + "\"\"\"\r\n x\r\n y\"\"\"" => raw"""(string-s "x\n" "y")""" + # Spaces or tabs or mixtures acceptable + "\"\"\"\n\tx\n\ty\"\"\"" => raw"""(string-s "x\n" "y")""" + "\"\"\"\n \tx\n \ty\"\"\"" => raw"""(string-s "x\n" "y")""" + # Mismatched tab vs space not deindented + # Find minimum common prefix in mismatched whitespace + "\"\"\"\n\tx\n y\"\"\"" => raw"""(string-s "\tx\n" " y")""" + "\"\"\"\n x\n y\"\"\"" => raw"""(string-s "x\n" " y")""" + "\"\"\"\n x\n y\"\"\"" => raw"""(string-s " x\n" "y")""" + "\"\"\"\n \tx\n y\"\"\"" => raw"""(string-s "\tx\n" " y")""" + "\"\"\"\n x\n \ty\"\"\"" => raw"""(string-s " x\n" "\ty")""" + # Empty lines don't affect dedenting + "\"\"\"\n x\n\n y\"\"\"" => raw"""(string-s "x\n" "\n" "y")""" + # Non-empty first line doesn't participate in deindentation + "\"\"\" x\n y\"\"\"" => raw"""(string-s " x\n" "y")""" + # Dedenting and interpolations + "\"\"\"\n \$a\n \$b\"\"\"" => raw"""(string-s a "\n" b)""" + "\"\"\"\n \$a \n \$b\"\"\"" => raw"""(string-s a " \n" b)""" + "\"\"\"\n \$a\n \$b\n\"\"\"" => raw"""(string-s " " a "\n" " " b "\n")""" + # Empty chunks after dedent are removed + "\"\"\"\n \n \"\"\"" => "(string-s \"\\n\")" + # Newline at end of string + "\"\"\"\n x\n y\n\"\"\"" => raw"""(string-s " x\n" " y\n")""" + # Empty strings, or empty after triple quoted processing + "\"\"" => "(string \"\")" + "\"\"\"\n \"\"\"" => "(string-s \"\")" + # Missing delimiter + "\"str" => "(string \"str\" (error-t))" + # String interpolations + "\"\$x\$y\$z\"" => "(string x y z)" + "\"\$(x)\"" => "(string (parens x))" + "\"\$x\"" => "(string x)" + # Strings with embedded whitespace trivia + "\"a\\\nb\"" => raw"""(string "a" "b")""" + "\"a\\\rb\"" => raw"""(string "a" "b")""" + "\"a\\\r\nb\"" => raw"""(string "a" "b")""" + "\"a\\\n \tb\"" => raw"""(string "a" "b")""" + # Strings with only a single valid string chunk + "\"str\"" => "(string \"str\")" + "\"a\\\n\"" => "(string \"a\")" + "\"a\\\r\"" => "(string \"a\")" + "\"a\\\r\n\"" => "(string \"a\")" + ], + JuliaSyntax.parse_atom => [ + # errors in literals + "\"\\xqqq\"" => "(string (ErrorInvalidEscapeSequence))" + "'\\xq'" => "(char (ErrorInvalidEscapeSequence))" + "'ab'" => "(char (ErrorOverLongCharacter))" + "\"\xf5\"" => "(string (ErrorInvalidUTF8))" + "'\xf5'" => "(char (ErrorInvalidUTF8))" + "`\xf5`" => "(cmdstring-r (ErrorInvalidUTF8))" + "10.0e1000'" => "(ErrorNumericOverflow)" + "10.0f100'" => "(ErrorNumericOverflow)" + ], + JuliaSyntax.parse_stmts => with_version.(v"1.11", [ + "function f(public)\n public + 3\nend" => "(function (call f public) (block (call-i public + 3)))" + "public A, B" => "(public A B)" + "if true \n public *= 4 \n end" => "(if true (block (op= public * 4)))" + "module Mod\n public A, B \n end" => "(module Mod (block (public A B)))" + "module Mod2\n a = 3; b = 6; public a, b\n end" => "(module Mod2 (block (= a 3) (= b 6) (public a b)))" + "a = 3; b = 6; public a, b" => "(toplevel-; (= a 3) (= b 6) (public a b))" + "begin \n public A, B \n end" => PARSE_ERROR + "if true \n public A, B \n end" => PARSE_ERROR + "public export=true foo, bar" => PARSE_ERROR # but these may be + "public experimental=true foo, bar" => PARSE_ERROR # supported soon ;) + "public(x::String) = false" => "(function-= (call public (::-i x String)) false)" + "module M; export @a; end" => "(module M (block (export (macro_name a))))" + "module M; public @a; end" => "(module M (block (public (macro_name a))))" + "module M; export ⤈; end" => "(module M (block (export ⤈)))" + "module M; public ⤈; end" => "(module M (block (public ⤈)))" + "public = 4" => "(= public 4)" + "public[7] = 5" => "(= (ref public 7) 5)" + "public() = 6" => "(function-= (call public) 6)" + ]), + JuliaSyntax.parse_stmts => [ + ((v = v"1.12",), "@callmemacro(b::Float64) = 2") => "(= (macrocall-p (macro_name callmemacro) (::-i b Float64)) 2)" + ], + JuliaSyntax.parse_docstring => [ + """ "notdoc" ] """ => "(string \"notdoc\")" + """ "notdoc" \n] """ => "(string \"notdoc\")" + """ "notdoc" \n\n foo """ => "(string \"notdoc\")" + """ "doc" \n foo """ => """(doc (string "doc") foo)""" + """ "doc" foo """ => """(doc (string "doc") foo)""" + """ "doc \$x" foo """ => """(doc (string "doc " x) foo)""" + # Allow docstrings with embedded trailing whitespace trivia + "\"\"\"\n doc\n \"\"\" foo" => """(doc (string-s "doc\\n") foo)""" + ], +] + +@testset "Inline test cases" begin + @testset "$production" for (production, test_specs) in tests + @testset "$(repr(input))" for (input, output) in test_specs + test_parse(production, input, output) + end + end +end + +parsestmt_test_specs = [ + # whitespace before keywords in space-insensitive mode + "(y::\nif x z end)" => "(parens (::-i y (if x (block z))))" + # Contextual keyword pairs inside parentheses + "(abstract type X end)" => "(parens (abstract X))" + "(mutable struct X end)" => "(parens (struct-mut X (block)))" + # parsing of tricky primes + "x in'c'" => "(call-i x in (char 'c'))" + "1where'c'" => "(where 1 (char 'c'))" + ":+'y'" => "(juxtapose (call-post (quote-: +) ') (call-post y '))" + # unary subtype ops and newlines + "a +\n\n<:" => "(call-i a + <:)" + "for\n\n<:" => "(for (iteration (in <: (error (error-t)))) (block (error)) (error-t))" + # Empty character consumes trailing ' delimiter (ideally this could be + # tested above but we don't require the input stream to be consumed in the + # unit tests there. + "''" => "(char (error))" + + # return + "return\nx" => "(return)" + "return x" => "(return x)" + "return x,y" => "(return (tuple x y))" + # closing tokens after return + "if x return else end" => "(if x (block (return)) (block))" + "(return)" => "(parens (return))" + "[return]" => "(vect (return))" + "{return}" => "(braces (return))" + # return doesn't require a closing token afterward + "[return x y]" => "(hcat (return x) y)" + # 1.14: return respects end/colon parse state + ((v=v"1.14",), "a[return end]") => "(ref a (return end))" + ((v=v"1.14",), "x ? return : y") => "(? x (return) y)" + ((v=v"1.13",), "a[return end]") => PARSE_ERROR + ((v=v"1.13",), "x ? return : y") => PARSE_ERROR + # break/continue + "break" => "(break)" + "(break)" => "(parens (break))" + "continue" => "(continue)" + # break/continue respect other closing delimiters (>=1.14) + ((v=v"1.14",), "[break]") => "(vect (break))" + ((v=v"1.14",), "{break}") => "(braces (break))" + # break/continue with labels (plain identifiers only, requires >=1.14) + ((v=v"1.14",), "break _") => "(break _)" + ((v=v"1.14",), "break _ x") => "(break _ x)" + ((v=v"1.14",), "break label") => "(break label)" + ((v=v"1.14",), "break var\"label\"") => "(break (var label))" + ((v=v"1.14",), "break \$label") => "(break (\$ label))" + ((v=v"1.14",), "break label x") => "(break label x)" + ((v=v"1.14",), "break f ()") => "(break f (tuple-p))" + ((v=v"1.14",), "break f()") => "(break f (error-t) (tuple-p))" + ((v=v"1.14",), "continue _") => "(continue _)" + ((v=v"1.14",), "continue label") => "(continue label)" + ((v=v"1.14",), "break +") => "(break (error-t +))" + ((v=v"1.14",), "a[break label end]") => "(ref a (break label end))" + ((v=v"1.14",), "x ? break : y") => "(? x (break) y)" + ((v=v"1.14",), "x ? break label z : y") => "(? x (break label z) y)" + # `break label x` must be followed by closing token + ((v=v"1.14",), "[break label x y]") => "(vect (break label x) (error-t y))" + # misfeature disabled in 1.14 (`:` always considered a break closing token) + ((v=v"1.14",), "break : x") => PARSE_ERROR + + ((v=v"1.13",), "break label") => "(error (break (error-t label (error-t))))" + ((v=v"1.13",), "continue label") => "(error (continue (error-t label (error-t))))" + ((v=v"1.13",), "break +") => "(break (error-t + (error-t)))" + ((v=v"1.13",), "x ? break : y") => "(? x (break) y)" + ((v=v"1.13",), "a[break label end]") => PARSE_ERROR + ((v=v"1.13",), "x ? break label z : y") => PARSE_ERROR + ((v=v"1.13",), "break : x") => "(call-i (break) : x)" + + # break / continue with trailing tokens are legal in some cases + "a ? break : c" => "(? a (break) c)" + "begin break end" => "(block (break))" + "a ? continue : c" => "(? a (continue) c)" + "begin continue end" => "(block (continue))" + "break:x" => "(call-i (break) : x)" # range colon allowed + + # The following may not be ideal error recovery! But at least the parser + # shouldn't crash + "@(x y)" => "(macrocall (macro_name (parens x (error-t y))))" + "|(&\nfunction" => "(call | (& (function (error (error)) (block (error)) (error-t))) (error-t))" + "@(" => "(macrocall (macro_name (parens (error-t))))" + "x = @(" => "(= x (macrocall (macro_name (parens (error-t)))))" + "function(where" => "(function (tuple-p where (error-t)) (block (error)) (error-t))" + # Contextual keyword pairs must not be separated by newlines even within parens + "(abstract\ntype X end)" => "(wrapper (parens abstract (error-t type X)) (error-t end ✘))" + "(mutable\nstruct X end)" => "(wrapper (parens mutable (error-t struct X)) (error-t end ✘))" + + # Lexer vs parser: issues detecting which tokens are string delimiters and + # detecting raw vs non-raw strings. The old parser was tightly coupled to + # the lexer and the parser state was used to disambiguate these cases. + "x in' '" => "(call-i x in (char (error)))" + "x in'``\$" => "(call-i x in (call-i (juxtapose (char '`' (error-t)) (cmdstring-r (error-t))) \$ (error)))" + "var\"#\"`str`" => "(juxtapose (var # (error-t)) (cmdstring-r \"str\"))" + "var\"#\"\"str\"" => "(juxtapose (var # (error-t)) (error-t) (string \"str\"))" + + # trailing junk in generators (issue #407) + "(x for x = xs a)" => "(parens (generator x (iteration (in x xs))) (error-t a))" + "(x for x = xs a, b)" => "(parens (generator x (iteration (in x xs))) (error-t a ✘ b))" + "f(x for x = xs a)" => "(call f (generator x (iteration (in x xs))) (error-t a))" + + # typegroup as identifier on older versions + ((v=v"1.12",), "typegroup = 3") => "(= typegroup 3)" + ((v=v"1.12",), "let typegroup = 3 end") => "(let (block (= typegroup 3)) (block))" + # typegroup error recovery on older versions (would be a syntax error anyway) + ((v=v"1.12",), "typegroup struct A end end") => "(error (typegroup (block (struct A (block)))))" +] + +@testset "Parsestmt tests" begin + @testset "$(repr(input))" for (input, output) in parsestmt_test_specs + test_parse(JuliaSyntax.parse_stmts, input, output) + end +end + +parsestmt_with_kind_tests = [ + # Most operators are semantically just normal identifiers after parsing so + # get the Kind K"Identifier" + "+" => "+::Identifier" + "a + b" => "(call-i a::Identifier +::Identifier b::Identifier)" + ((v=v"1.14",), "a +% b") => "(call-i a::Identifier +%::Identifier b::Identifier)" + ((v=v"1.14",), "a -% b") => "(call-i a::Identifier -%::Identifier b::Identifier)" + ((v=v"1.14",), "a *% b") => "(call-i a::Identifier *%::Identifier b::Identifier)" + "a .+ b" => "(dotcall-i a::Identifier +::Identifier b::Identifier)" + "a |> b" => "(call-i a::Identifier |>::Identifier b::Identifier)" + "a => b" => "(call-i a::Identifier =>::Identifier b::Identifier)" + "a → b" => "(call-i a::Identifier →::Identifier b::Identifier)" + "a < b < c" => "(comparison a::Identifier <::Identifier b::Identifier <::Identifier c::Identifier)" + "a .<: b"=> "(dotcall-i a::Identifier <:::Identifier b::Identifier)" + "a .. b" => "(call-i a::Identifier (DotsIdentifier-2) b::Identifier)" + "a : b" => "(call-i a::Identifier :::Identifier b::Identifier)" + "-2^x" => "(call-pre -::Identifier (call-i 2::Integer ^::Identifier x::Identifier))" + "-(2)" => "(call-pre -::Identifier (parens 2::Integer))" + "<:(a,)" => "(<:-, a::Identifier)" + "- 2" => "(call-pre -::Identifier 2::Integer)" + "/x" => "(call-pre (error /::Identifier) x::Identifier)" + "a^b" => "(call-i a::Identifier ^::Identifier b::Identifier)" + "f.'" => "(dotcall-post f::Identifier (error '::Identifier))" + "f'" => "(call-post f::Identifier '::Identifier)" + # break/continue labels (contextual keywords allowed) + ((v=v"1.14",), "break label") => "(break label::Identifier)" + ((v=v"1.14",), "continue label") => "(continue label::Identifier)" + ((v=v"1.14",), "break outer") => "(break outer::Identifier)" + ((v=v"1.14",), "continue outer") => "(continue outer::Identifier)" + # Standalone syntactic ops which keep their kind - they can't really be + # used in a sane way as identifiers or interpolated into expressions + # because they have their own syntactic forms. + ":(::)" => "(quote-: (parens ::::::))" + ":(\$)" => "(quote-: (parens \$::\$))" + ":(<:)" => "(quote-: (parens <:::<:))" + ":(&&)" => "(quote-: (parens &&::&&))" + ":(=)" => "(quote-: (parens =::=))" + "a := b" => "(:= a::Identifier b::Identifier)" + "a += b" => "(op= a::Identifier +::Identifier b::Identifier)" + ((v=v"1.14",), "a +%= b") => "(op= a::Identifier +%::Identifier b::Identifier)" + "a .+= b" => "(.op= a::Identifier +::Identifier b::Identifier)" + ((v=v"1.14",), "a .+%= b") => "(.op= a::Identifier +%::Identifier b::Identifier)" + "a >>= b" => "(op= a::Identifier >>::Identifier b::Identifier)" + ":+=" => "(quote-: (op= +::Identifier))" + ((v=v"1.14",), ":+%=") => "(quote-: (op= +%::Identifier))" + ":.+=" => "(quote-: (.op= +::Identifier))" + ((v=v"1.14",), ":.+%=") => "(quote-: (.op= +%::Identifier))" + # str/cmd macro name kinds + "x\"str\"" => """(macrocall x::StrMacroName (string-r "str"::String))""" + "x`str`" => """(macrocall x::CmdMacroName (cmdstring-r "str"::CmdString))""" +] + +@testset "parser `Kind` remapping" begin + @testset "$(repr(input))" for (input, output) in parsestmt_with_kind_tests + test_parse(JuliaSyntax.parse_stmts, input, output; show_kind=true) + end +end + +@testset "Trivia attachment" begin + # TODO: Need to expand this greatly to cover as many forms as possible! + @test show_green_tree("f(a;b)") == """ + 1:6 │[toplevel] + 1:6 │ [call] + 1:1 │ Identifier ✔ "f" + 2:2 │ ( "(" + 3:3 │ Identifier ✔ "a" + 4:5 │ [parameters] + 4:4 │ ; ";" + 5:5 │ Identifier ✔ "b" + 6:6 │ ) ")" + """ +end + +@testset "Unicode normalization in tree conversion" begin + # ɛµ normalizes to εμ + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5()") == "(call \u03B5\u03BC)" + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "@\u025B\u00B5") == "(macrocall (macro_name \u03B5\u03BC))" + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5\"\"") == "(macrocall @\u03B5\u03BC_str (string-r \"\"))" + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "\u025B\u00B5``") == "(macrocall @\u03B5\u03BC_cmd (cmdstring-r \"\"))" + # · and · normalize to ⋅ + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a \u00B7 b") == "(call-i a \u22C5 b)" + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a \u0387 b") == "(call-i a \u22C5 b)" + # − ('\u2212') normalizes to - ('\u002d') + @test parse_to_sexpr_str(JuliaSyntax.parse_expr, "a \u2212 b") == "(call-i a - b)" + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a \u2212= b") == "(op= a - b)" + @test parse_to_sexpr_str(JuliaSyntax.parse_eq, "a .\u2212= b") == "(.op= a - b)" +end + +@testset "Unbalanced bidirectional unicode" begin + # https://trojansource.codes + @test_throws JuliaSyntax.ParseError parsestmt(GreenNode, """ + function checkUserAccess(u::User) + if u.accessLevel != "user\u202e \u2066# users are not allowed\u2069\u2066" + return true + end + return false + end + """) + + @test_throws JuliaSyntax.ParseError parsestmt(GreenNode, """ + function checkUserAccess(u::User) + #=\u202e \u2066if (u.isAdmin)\u2069 \u2066 begin admins only =# + return true + #= end admin only \u202e \u2066end\u2069 \u2066=# + return false + end + """) +end diff --git a/JuliaSyntax/test/parser_api.jl b/JuliaSyntax/test/parser_api.jl new file mode 100644 index 0000000000000..85f80839fe977 --- /dev/null +++ b/JuliaSyntax/test/parser_api.jl @@ -0,0 +1,237 @@ +@testset "parser API" begin + @testset "parse with String input" begin + @test parsestmt(Expr, " x ") == :x + @test JuliaSyntax.remove_linenums!(parseall(Expr, " x ")) == Expr(:toplevel, :x) + @test parseatom(Expr, " x ") == :x + @test parseatom(Expr, "(x)") == :x + + # SubString + @test parsestmt(Expr, SubString("x+y")) == :(x+y) + @test parsestmt(Expr, SubString("α+x")) == :(α+x) + @test parseatom(Expr, SubString("x+y",3,3)) == :y + + # Exceptions due to extra trailing syntax + @test_throws JuliaSyntax.ParseError parseatom(Expr, "x+y") + @test_throws JuliaSyntax.ParseError parsestmt(Expr, "x+y\nz") + + # ignore_warnings flag + @test_throws JuliaSyntax.ParseError parsestmt(Expr, "import . .A") + @test parsestmt(Expr, "import . .A", ignore_warnings=true) == :(import ..A) + + # version selection + @test_throws JuliaSyntax.ParseError parsestmt(Expr, "[a ;; b]", version=v"1.6") + @test parsestmt(Expr, "[a ;; b]", version=v"1.7") == Expr(:ncat, 2, :a, :b) + + # filename + @test parsestmt(Expr, "begin\na\nend", filename="foo.jl", first_line=55) == + Expr(:block, LineNumberNode(56, Symbol("foo.jl")), :a) + + # ignore_trivia + @test parseatom(Expr, " x ", ignore_trivia=true) == :x + @test_throws JuliaSyntax.ParseError parseatom(Expr, " x ", ignore_trivia=false) + + # Top level parsing + @test parseall(Expr, "a\nb") == + Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(2), :b) + @test parseall(Expr, "a\nb #==#") == + Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(2), :b) + @test parseall(Expr, "#==#\na\nb") == + Expr(:toplevel, LineNumberNode(2), :a, LineNumberNode(3), :b) + @test parseall(Expr, "a\nb\n#==#") == + Expr(:toplevel, LineNumberNode(1), :a, LineNumberNode(2), :b) + end + + @testset "IO input" begin + # IOBuffer + io = IOBuffer("x+y") + @test parse!(Expr, io, rule=:statement) == (:(x+y), []) + @test position(io) == 3 + io = IOBuffer("x+y") + seek(io, 2) + @test parse!(Expr, io, rule=:atom) == (:y, []) + @test position(io) == 3 + # A GenericIOBuffer, not actually IOBuffer + io = IOBuffer(SubString("x+y")) + @test parse!(Expr, io, rule=:statement) == (:(x+y), []) + @test position(io) == 3 + # Another type of GenericIOBuffer + io = IOBuffer(codeunits("x+y")) + @test parse!(Expr, io, rule=:statement) == (:(x+y), []) + @test position(io) == 3 + # IOStream + mktemp() do path, io + write(io, "x+y") + close(io) + + open(path, "r") do io + @test parse!(Expr, io, rule=:statement) == (:(x+y), []) + @test position(io) == 3 + end + end + end + + @testset "parse with String and index input" begin + # String + let + ex,pos = parseall(Expr, "x+y\nz", 1) + @test JuliaSyntax.remove_linenums!(ex) == Expr(:toplevel, :(x+y), :z) + @test pos == 6 + end + @test parsestmt(Expr, "x+y\nz", 1) == (:(x+y), 4) + @test parseatom(Expr, "x+y\nz", 1) == (:x, 2) + @test parseatom(Expr, "x+y\nz", 5) == (:z, 6) + + # SubString + @test parsestmt(Expr, SubString("α+x\ny"), 1) == (:(α+x), 5) + @test parseatom(Expr, SubString("x+y"), 1) == (:x, 2) + @test parseatom(Expr, SubString("x+y"), 3) == (:y, 4) + + @test parseatom(Expr, SubString("x+1.0"), 3) == (1.0, 6) + @test parseatom(Expr, SubString("x+\"\n\""), 3) == ("\n", 6) + + # Line numbers are relative to the start of the string we're currently + # parsing + @test JuliaSyntax.parsestmt(Expr, "begin\na\nend\nbegin\nb\nend", 1) == + (Expr(:block, LineNumberNode(2), :a), 12) + @test JuliaSyntax.parsestmt(Expr, "begin\na\nend\nbegin\nb\nend", 12) == + (Expr(:block, LineNumberNode(3), :b), 24) + end + + @testset "error/warning handling" begin + parseshow(s;kws...) = sprint(show, MIME("text/x.sexpression"), parsestmt(SyntaxNode, s; kws...)) + @test_throws JuliaSyntax.ParseError parseshow("try finally catch ex end") + @test parseshow("try finally catch ex end", ignore_warnings=true) == + "(try (block) (finally (block)) (catch ex (block)))" + # ignore_errors + @test_throws JuliaSyntax.ParseError parseshow("[a; b, c]") + @test_throws JuliaSyntax.ParseError parseshow("[a; b, c]", ignore_warnings=true) + @test parseshow("[a; b, c]", ignore_errors=true) == "(vcat a b (error-t) c)" + # errors in literals + @test parseshow("\"\\z\"", ignore_errors=true) == "(string (ErrorInvalidEscapeSequence))" + @test parseshow("'\\z'", ignore_errors=true) == "(char (ErrorInvalidEscapeSequence))" + @test parseshow("'abc'", ignore_errors=true) == "(char (ErrorOverLongCharacter))" + @test parseshow("1e1000", ignore_errors=true) == "(ErrorNumericOverflow)" + @test parseshow("1f1000", ignore_errors=true) == "(ErrorNumericOverflow)" + end +end + +@testset "ParseError printing" begin + try + parsestmt(SyntaxNode, "a -- b -- c", filename="somefile.jl") + @assert false "error should be thrown" + catch exc + @test exc isa JuliaSyntax.ParseError + @test sprint(showerror, exc) == """ + ParseError: + # Error @ somefile.jl:1:3 + a -- b -- c + # └┘ ── invalid operator""" + @test occursin("Stacktrace:\n", sprint(showerror, exc, catch_backtrace())) + file_url = JuliaSyntax._file_url("somefile.jl") + @test sprint(showerror, exc, context=:color=>true) == """ + ParseError: + \e[90m# Error @ \e[0;0m\e]8;;$file_url#1:3\e\\\e[90msomefile.jl:1:3\e[0;0m\e]8;;\e\\ + a \e[48;2;120;70;70m--\e[0;0m b -- c + \e[90m# └┘ ── \e[0;0m\e[91minvalid operator\e[0;0m""" + end + + try + # Test that warnings are printed first followed by only the first error + parsestmt(SyntaxNode, """ + @(a) + x -- y + z -- y""", filename="somefile.jl") + @assert false "error should be thrown" + catch exc + @test exc isa JuliaSyntax.ParseError + @test sprint(showerror, exc) == """ + ParseError: + # Warning @ somefile.jl:1:2 + @(a) + #└─┘ ── parenthesizing macro names is unnecessary + # Error @ somefile.jl:2:1 + @(a) + x + ╙ ── unexpected text after parsing statement""" + end + + try + # Test that initial warnings are always printed + parsestmt(SyntaxNode, """ + @(a)""", filename="somefile.jl") + @assert false "error should be thrown" + catch exc + @test exc isa JuliaSyntax.ParseError + @test sprint(showerror, exc) == """ + ParseError: some warnings detected: + # Warning @ somefile.jl:1:2 + @(a) + #└─┘ ── parenthesizing macro names is unnecessary""" + end +end + +tokensplit(str; kws...) = [kind(tok) => untokenize(tok, str) for tok in tokenize(str; kws...)] + +@testset "tokenize() API" begin + # tokenize() is eager + @test tokenize("aba") isa Vector{JuliaSyntax.Token} + + # . is a separate token from + in `.+` + @test tokensplit("a .+ β") == [ + K"Identifier" => "a", + K"Whitespace" => " ", + K"." => ".", + K"Identifier" => "+", + K"Whitespace" => " ", + K"Identifier" => "β", + ] + + # + is kind K"+" when operators in identifier position are emitted as + # operator kinds. + @test tokensplit("a .+ β"; operators_as_identifiers=false) == [ + K"Identifier" => "a", + K"Whitespace" => " ", + K"." => ".", + K"+" => "+", + K"Whitespace" => " ", + K"Identifier" => "β", + ] + + # Contextual keywords become identifiers where necessary + @test tokensplit("outer = 1") == [ + K"Identifier" => "outer", + K"Whitespace" => " ", + K"=" => "=", + K"Whitespace" => " ", + K"Integer" => "1", + ] + # Including word operators + @test tokensplit("where = 1"; operators_as_identifiers=false) == [ + K"Identifier" => "where", + K"Whitespace" => " ", + K"=" => "=", + K"Whitespace" => " ", + K"Integer" => "1", + ] + + # +₁ is tokenized as a single identifier token (subscripts are valid in identifiers) + @test tokensplit("+₁") == [K"Identifier"=>"+₁"] + + # Buffer interface + @test tokenize(Vector{UInt8}("a + b")) == tokenize("a + b") + + buf = Vector{UInt8}("a-β") + @test untokenize.(tokenize(buf), Ref(buf,)) == [ + Vector{UInt8}("a"), + Vector{UInt8}("-"), + Vector{UInt8}("β") + ] + + @test kind(JuliaSyntax.Token()) == K"None" + + @test tokensplit("'\\") == [ + K"'" => "'", + K"ErrorInvalidEscapeSequence" => "\\", + K"error" => "" + ] +end diff --git a/JuliaSyntax/test/runtests.jl b/JuliaSyntax/test/runtests.jl new file mode 100644 index 0000000000000..bdc2f0f531db0 --- /dev/null +++ b/JuliaSyntax/test/runtests.jl @@ -0,0 +1,54 @@ +if !(@isdefined JuliaSyntax) + using JuliaSyntax +end + +using Test + +# JuliaSyntax is not only tested on master +if isdefined(Test, :detect_closure_boxes) + @test isempty(Test.detect_closure_boxes(JuliaSyntax)) +end + +include("test_utils.jl") +include("test_utils_tests.jl") +include("fuzz_test.jl") + +include("utils.jl") +include("kinds.jl") + +@testset "Tokenize" begin + include("tokenize.jl") +end + +include("parse_stream.jl") +include("parser.jl") +include("green_node.jl") +include("syntax_node.jl") +if VERSION >= v"1.12" + include("syntax.jl") +end +include("diagnostics.jl") +include("parser_api.jl") +include("expr.jl") +@testset "Parsing literals from strings" begin + include("literal_parsing.jl") +end +include("source_files.jl") + +if VERSION >= v"1.6" + # Tests restricted to 1.6+ due to + # * Core._parse hook doesn't exist on v1.5 and lower + # * Reference parser bugs which would need workarounds for package parse comparisons + include("hooks.jl") + include("parse_packages.jl") +end + +include("serialization.jl") + +@static if isdefined(Base, :infer_return_type) + @testset "Basic inference tests" begin + @test Base.infer_return_type(JuliaSyntax.sourcetext, (JuliaSyntax.SyntaxTree,)) <: AbstractString + @test Base.infer_return_type(JuliaSyntax.byte_range, (JuliaSyntax.SyntaxTree,)) == UnitRange{Int} + @test Base.infer_return_type(JuliaSyntax.hasproperty, (JuliaSyntax.SyntaxTree,Symbol)) == Bool + end +end diff --git a/JuliaSyntax/test/runtests_vendored.jl b/JuliaSyntax/test/runtests_vendored.jl new file mode 100644 index 0000000000000..52980e4917dcf --- /dev/null +++ b/JuliaSyntax/test/runtests_vendored.jl @@ -0,0 +1,4 @@ +# Test copy of JuliaSyntax vendored into Base +using Base.JuliaSyntax: JuliaSyntax + +include("runtests.jl") diff --git a/JuliaSyntax/test/serialization.jl b/JuliaSyntax/test/serialization.jl new file mode 100644 index 0000000000000..325c67f39e0cf --- /dev/null +++ b/JuliaSyntax/test/serialization.jl @@ -0,0 +1,29 @@ +using Serialization + +@testset "Equality $T" for T in [Expr, SyntaxNode, JuliaSyntax.GreenNode] + x = JuliaSyntax.parsestmt(T, "f(x) = x + 2") + y = JuliaSyntax.parsestmt(T, "f(x) = x + 2") + z = JuliaSyntax.parsestmt(T, "f(x) = 2 + x") + @test x == y + @test x != z + @test y != z +end + +@testset "Hashing $T" for T in [Expr, SyntaxNode, JuliaSyntax.GreenNode] + x = hash(JuliaSyntax.parsestmt(T, "f(x) = x + 2"))::UInt + y = hash(JuliaSyntax.parsestmt(T, "f(x) = x + 2"))::UInt + z = hash(JuliaSyntax.parsestmt(T, "f(x) = 2 + x"))::UInt + @test x == y # Correctness + @test x != z # No collision + @test y != z # No collision +end + +@testset "Serialization $T" for T in [Expr, SyntaxNode, JuliaSyntax.GreenNode] + x = JuliaSyntax.parsestmt(T, "f(x) = x ⋅ 2") + f = tempname() + open(f, "w") do io + serialize(io, x) + end + y = open(deserialize, f, "r") + @test x == y +end diff --git a/JuliaSyntax/test/source_files.jl b/JuliaSyntax/test/source_files.jl new file mode 100644 index 0000000000000..d518124f1e6e1 --- /dev/null +++ b/JuliaSyntax/test/source_files.jl @@ -0,0 +1,230 @@ +@testset "SourceFile lines and column indexing" begin + @test source_location(SourceFile("a"), 1) == (1,1) + @test source_location(SourceFile("a"), 2) == (1,2) + + @test source_location(SourceFile("a\n"), 2) == (1,2) + @test source_location(SourceFile("a\n"), 3) == (2,1) + + @test source_location(SourceFile("a\nb\n"), 2) == (1,2) + @test source_location(SourceFile("a\nb\n"), 3) == (2,1) + @test source_location(SourceFile("a\nb\n"), 4) == (2,2) + @test source_location(SourceFile("a\nb\n"), 5) == (3,1) + + @test source_location(SourceFile("\n\n"), 1) == (1,1) + @test source_location(SourceFile("\n\n"), 2) == (2,1) + @test source_location(SourceFile("\n\n"), 3) == (3,1) + + @test source_location(SourceFile("a"; first_line=7), 1) == (7,1) + @test source_location(SourceFile("a"; first_line=7), 2) == (7,2) + + @test source_location(SourceFile("a\n"; first_line=7), 2) == (7,2) + @test source_location(SourceFile("a\n"; first_line=7), 3) == (8,1) + + @test source_location(SourceFile("a\nb\n"; first_line=7), 2) == (7,2) + @test source_location(SourceFile("a\nb\n"; first_line=7), 3) == (8,1) + @test source_location(SourceFile("a\nb\n"; first_line=7), 4) == (8,2) + @test source_location(SourceFile("a\nb\n"; first_line=7), 5) == (9,1) + + + mktemp() do path, io + write(io, "a\n") + @test source_location(SourceFile(; filename=path), 1) == (1,1) + @test source_location(SourceFile(; filename=path, first_line=7), 1) == (7,1) + end + + # byte offset + sf = SourceFile("a\nbb\nccc\ndddd", first_index=10) + @test source_location(sf, 13) == (2,2) + @test source_line(sf, 15) == 3 + @test source_line_range(sf, 10) == (10,11) + @test source_line_range(sf, 11) == (10,11) + @test source_line_range(sf, 12) == (12,14) + @test source_line_range(sf, 14) == (12,14) + @test source_line_range(sf, 15) == (15,18) + + # source_line convenience function + @test source_line(SourceFile("a\nb\n"), 2) == 1 + @test source_line(SourceFile("a\nb\n"), 3) == 2 +end + +@testset "SourceFile position indexing" begin + @test SourceFile("a\nb\n")[1:2] == "a\n" + @test SourceFile("a\nb\n")[3:end] == "b\n" + + # unicode + @test SourceFile("αβ")[1:2] == "α" + @test SourceFile("αβ")[3] == 'β' + + # offsets + sf = SourceFile("abcd", first_index=10) + @test firstindex(sf) == 10 + @test lastindex(sf) == 13 + @test sf[10] == 'a' + @test sf[10:11] == "ab" + @test view(sf, 10:11) == "ab" + + @test thisind(SourceFile("xαx", first_index=10), 10) == 10 + @test thisind(SourceFile("xαx", first_index=10), 11) == 11 + @test thisind(SourceFile("xαx", first_index=10), 12) == 11 + @test thisind(SourceFile("xαx", first_index=10), 13) == 13 + + if Base.VERSION >= v"1.4" + # Protect the `[begin` from being viewed by the parser on older Julia versions + @test eval(Meta.parse("SourceFile(\"a\nb\n\")[begin:end]")) == "a\nb\n" + @test eval(Meta.parse("SourceFile(\"abcd\", first_index=10)[begin+1:end-1]")) == "bc" + end +end + +@testset "SourceFile printing and text extraction" begin + srcf = SourceFile("module Foo\nend") + @test sprint(show, MIME("text/plain"), srcf) == """ + ## SourceFile ## + module Foo + end""" + @test sourcetext(srcf) == "module Foo\nend" +end + + +@testset "highlight()" begin + src = SourceFile(""" + abcd + αβγδ + +-*/""") + + # Empty ranges + @test sprint(highlight, src, 1:0) == "abcd\n└\nαβγδ\n+-*/" + @test sprint(highlight, src, 2:1) == "abcd\n#└\nαβγδ\n+-*/" + @test sprint(highlight, src, 3:2) == "abcd\n# └\nαβγδ\n+-*/" + @test sprint(highlight, src, 4:3) == "abcd\n# └\nαβγδ\n+-*/" + @test sprint(highlight, src, 5:4) == "abcd\n# └\nαβγδ\n+-*/" + @test sprint(highlight, src, 6:5) == "abcd\nαβγδ\n└\n+-*/" + @test sprint(highlight, src, 19:18) == "abcd\nαβγδ\n+-*/\n# └" + @test sprint(io->highlight(io, src, 1:0, context_lines_after=0, note="hi")) == + "abcd\n└ ── hi" + + # Single line ranges + @test sprint(highlight, src, 1:4) == "abcd\n└──┘\nαβγδ\n+-*/" + @test sprint(highlight, src, 2:4) == "abcd\n#└─┘\nαβγδ\n+-*/" + @test sprint(highlight, src, 3:4) == "abcd\n# └┘\nαβγδ\n+-*/" + @test sprint(highlight, src, 4:4) == "abcd\n# ╙\nαβγδ\n+-*/" + @test sprint(highlight, src, 5:5) == "abcd\n# └\nαβγδ\n+-*/" + @test sprint(highlight, src, 6:6) == "abcd\nαβγδ\n╙\n+-*/" + @test sprint(highlight, src, 6:9) == "abcd\nαβγδ\n└┘\n+-*/" + @test sprint(highlight, src, 8:8) == "abcd\nαβγδ\n#╙\n+-*/" + + # multi-byte chars + @test sprint(highlight, src, 8:13) == """ + abcd + αβγδ + #└─┘ + +-*/""" + # multi-byte char at eof + @test sprint(highlight, SourceFile("a α"), 3:4) == "a α\n# ╙" + @test sprint(highlight, SourceFile("a\nα"), 1:4) == "┌\na\nα\n┘" + @test sprint(highlight, SourceFile("a\nb\nα"), 3:3) == "a\nb\n╙\nα" + + # empty files + @test sprint(highlight, SourceFile(""), 1:0) == "└" + + # Multi-line ranges + @test sprint(highlight, src, 1:7) == """ + ┌─── + abcd + αβγδ + ┘ + +-*/""" + @test sprint(highlight, src, 2:7) == """ + #┌── + abcd + αβγδ + ┘ + +-*/""" + @test sprint(highlight, src, 2:9) == """ + #┌── + abcd + αβγδ + #┘ + +-*/""" + @test sprint(highlight, src, 4:9) == """ + # ┌ + abcd + αβγδ + #┘ + +-*/""" + @test sprint(highlight, src, 5:9) == """ + # ┌ + abcd + αβγδ + #┘ + +-*/""" + @test sprint(highlight, src, 6:15) == """ + abcd + ┌─── + αβγδ + +-*/ + ┘""" + @test sprint(highlight, src, 8:15) == """ + abcd + #┌── + αβγδ + +-*/ + ┘""" + @test sprint(highlight, src, 1:18) == """ + ┌─── + abcd + αβγδ + +-*/ + #──┘""" + + # context lines + @test sprint(io->highlight(io, src, 8:13; + context_lines_before=0, + context_lines_after=0)) == """ + αβγδ + #└─┘""" + @test sprint(io->highlight(io, src, 8:13; context_lines_after=0)) == """ + abcd + αβγδ + #└─┘""" + @test sprint(io->highlight(io, src, 8:13; context_lines_before=0)) == """ + αβγδ + #└─┘ + +-*/""" + @test sprint(io->highlight(io, src, 1:18; context_lines_inner=0)) == """ + ┌─── + abcd + ⋮ + +-*/ + #──┘""" + + # annotations + @test sprint(io->highlight(io, src, 8:13; note="hello")) == """ + abcd + αβγδ + #└─┘ ── hello + +-*/""" + @test sprint(io->highlight(io, src, 1:13; note="hello")) == """ + ┌─── + abcd + αβγδ + #──┘ ── hello + +-*/""" + @test sprint(io->highlight(io, src, 8:13; + note=(io,indent,w)->print(io, "\n$indent$('!'^w) hello"))) == """ + abcd + αβγδ + #└─┘ + #!!! hello + +-*/""" + + # colored output + @test sprint(io->highlight(io, src, 8:13; context_lines_after=0, note="hello", notecolor=:light_red), + context=:color=>true) == + "abcd\nα\e[48;2;120;70;70mβγδ\e[0;0m\n\e[90m#└─┘ ── \e[0;0m\e[91mhello\e[0;0m" + @test sprint(io->highlight(io, src, 1:13; context_lines_after=0, note="hello", notecolor=(255,0,0)), + context=:color=>true) == + "\e[90m┌───\e[0;0m\n\e[48;2;120;70;70mabcd\e[0;0m\n\e[48;2;120;70;70mαβγδ\e[0;0m\n\e[90m#──┘ ── \e[0;0m\e[38;2;255;0;0mhello\e[0;0m" + @test sprint(io->highlight(io, src, 1:18, context_lines_inner=0), + context=:color=>true) == + "\e[90m┌───\e[0;0m\n\e[48;2;120;70;70mabcd\e[0;0m\n\e[48;2;120;70;70m\e[0;0m⋮\n\e[48;2;120;70;70m+-*/\e[0;0m\n\e[90m#──┘\e[0;0m" +end diff --git a/JuliaSyntax/test/syntax.jl b/JuliaSyntax/test/syntax.jl new file mode 100644 index 0000000000000..3056176bf4c45 --- /dev/null +++ b/JuliaSyntax/test/syntax.jl @@ -0,0 +1,497 @@ +using .JuliaSyntax: SyntaxTree, SyntaxList, @stm, prov, prov_end, provenance, + macro_prov, macro_prov_end, flattened_provenance, sourceref, + unexpanded_sourceref, newleaf, newnode, mkleaf, mknode, mktree, copy_ast, + unalias_nodes, annotate_parent!, _setattr!, getmeta, SyntaxContext, + ScopeLayer, children + +""" +Build a hand-made tree for the DAG-shaped tests below. Each node carries a +distinct integer in `.value` so nodes copied by `unalias_nodes` and friends can +be traced back to the node they were copied from. +""" +function tnode(tag::Int, cs::SyntaxTree...) + st = isempty(cs) ? + newleaf(LineNumberNode(tag), K"Value") : + newnode(LineNumberNode(tag), K"block", SyntaxList(cs...)) + _setattr!(st, :value, tag) +end + +"All nodes of `st` in preorder, with one entry per occurrence" +function flat_nodes(st::SyntaxTree, out=SyntaxList()) + push!(out, st) + for c in children(st) + flat_nodes(c, out) + end + out +end + +@testset "SyntaxTree parsing" begin + # Errors should fall through + @test parsestmt(SyntaxTree, ""; ignore_errors=true) isa SyntaxTree + @test parsestmt(SyntaxTree, " "; ignore_errors=true) isa SyntaxTree + @test parsestmt(SyntaxTree, "@"; ignore_errors=true) isa SyntaxTree + @test parsestmt(SyntaxTree, "@@@"; ignore_errors=true) isa SyntaxTree + @test parsestmt(SyntaxTree, "(a b c)"; ignore_errors=true) isa SyntaxTree + @test parsestmt(SyntaxTree, "'a b c'"; ignore_errors=true) isa SyntaxTree + # Malformed literals become ErrorVal-valued leaves rather than identifiers + @test parsestmt(SyntaxTree, "1.e"; ignore_errors=true) isa SyntaxTree + @test parsestmt(SyntaxTree, "x = 1._"; ignore_errors=true) isa SyntaxTree +end + +@testset "SyntaxTree type stability" begin + st0 = parsestmt(SyntaxTree, "f(::Int)") + # `children` must not leak the `Union{Nothing}` of the raw field into inference. + @test @inferred(children(st0)) isa Vector{SyntaxTree} + @test @inferred(prov(st0)) isa SyntaxTree +end + +@testset "SyntaxTree provenance accessors" begin + + @testset "prov, prov_end, provenance, sourceref" begin + # st3 <- st2 <- st1, with st3 referring to source text + st3 = tnode(3) + st2 = mkleaf(st3) + st1 = mkleaf(st2) + + @test prov(st1) === st2 + @test prov(prov(st1)) === st3 + @test prov(prov(prov(st1))) === st3 + + @test prov_end(st1) === st3 + @test prov_end(prov_end(st1)) === st3 + + @test sourceref(st1) == LineNumberNode(3) + @test sourceref(prov_end(st1)) == LineNumberNode(3) + + @test provenance(st1) == SyntaxList(st2, st3) + @test provenance(prov_end(st1)) == SyntaxList() + end + + @testset "flattened_provenance" begin + ctx_with_unexpanded(u) = SyntaxContext( + ScopeLayer(JuliaSyntax, nothing), + u, + v"0.0", + false) + + st1 = _setattr!(newleaf(LineNumberNode(1), K"Identifier"), :value, "st1") + st2 = _setattr!(mkleaf(st1), :value, "st2") + st3 = _setattr!(mkleaf(st2), :value, "st3") + + stm1 = _setattr!(newleaf(LineNumberNode(1, :m), K"Identifier"), :value, "stm1") + stm2 = _setattr!(mkleaf(stm1), :value, "stm2") + stm3 = _setattr!(mkleaf(stm1), :value, "stm3") + stm_unused = _setattr!(newleaf(LineNumberNode(0), K"Identifier"), :value, "stm_unused") + + stmm1 = _setattr!(newleaf(LineNumberNode(1, :mm), K"Identifier"), :value, "stmm1") + stmm2 = _setattr!(mkleaf(stmm1), :value, "stmm2") + stmm3 = _setattr!(mkleaf(stmm2), :value, "stmm3") + + _setattr!(st1, :context, ctx_with_unexpanded(stm_unused)) + _setattr!(st2, :context, ctx_with_unexpanded(stm_unused)) + _setattr!(st3, :context, ctx_with_unexpanded(stm3)) + _setattr!(stm3, :context, ctx_with_unexpanded(stmm3)) + + # julia> JL._show_provtree(stdout, st3, "") + # st3 + # ├─ st2 + # │ ├─ st1 + # │ │ ├─ @ nothing:1 + # │ │ └─ stm_unused + # │ │ └─ @ nothing:0 + # │ └─ stm_unused + # │ └─ @ nothing:0 + # └─ stm3 + # ├─ stm1 + # │ └─ @ m:1 + # └─ stmm3 + # └─ stmm2 + # └─ stmm1 + # └─ @ mm:1 + + @test macro_prov(st3) == stm3 + @test macro_prov(st2) == stm_unused + @test macro_prov(st1) == stm_unused + @test macro_prov(stm3) == stmm3 + @test macro_prov(stm2) == nothing + @test macro_prov(stm1) == nothing + @test macro_prov(stmm3) == nothing + @test macro_prov(stmm2) == nothing + @test macro_prov(stmm1) == nothing + @test macro_prov_end(st3) == stmm3 + @test macro_prov_end(st2) == stm_unused + @test macro_prov_end(st1) == stm_unused + @test macro_prov_end(stm3) == stmm3 + @test macro_prov_end(stm2) == nothing + @test macro_prov_end(stm1) == nothing + @test macro_prov_end(stmm3) == nothing + @test macro_prov_end(stmm2) == nothing + @test macro_prov_end(stmm1) == nothing + @test unexpanded_sourceref(st3) == LineNumberNode(1, :mm) + @test unexpanded_sourceref(st2) == LineNumberNode(0) + @test unexpanded_sourceref(st1) == LineNumberNode(0) + @test unexpanded_sourceref(stm3) == LineNumberNode(1, :mm) + @test unexpanded_sourceref(stm2) == LineNumberNode(1, :m) + @test unexpanded_sourceref(stm1) == LineNumberNode(1, :m) + @test unexpanded_sourceref(stmm3) == LineNumberNode(1, :mm) + @test unexpanded_sourceref(stmm2) == LineNumberNode(1, :mm) + @test unexpanded_sourceref(stmm1) == LineNumberNode(1, :mm) + @test flattened_provenance(st3) == SyntaxList(stmm1, stm1, st1) + @test flattened_provenance(st2) == SyntaxList(stm_unused, st1) + @test flattened_provenance(st1) == SyntaxList(stm_unused, st1) + @test flattened_provenance(stm3) == SyntaxList(stmm1, stm1) + @test flattened_provenance(stm2) == SyntaxList(stm1) + @test flattened_provenance(stm1) == SyntaxList(stm1) + @test flattened_provenance(stmm3) == SyntaxList(stmm1) + @test flattened_provenance(stmm2) == SyntaxList(stmm1) + @test flattened_provenance(stmm1) == SyntaxList(stmm1) + end +end + +@testset "SyntaxTree utils" begin + @testset "copy_ast, mktree" begin + # A one-child tree whose root also has a provenance chain of its own + leaf = tnode(3) + st2 = newnode(LineNumberNode(1), K"block", SyntaxList(leaf)) + st = mknode(st2, children(st2)) # st.source === st2 + + stcopy = copy_ast(st) + @test stcopy !== st + @test st ≈ stcopy + @test stcopy[1] !== st[1] + # `.source` chains are copied too + @test prov(stcopy) !== prov(st) + @test prov(st) ≈ prov(stcopy) + + # Every node is copied at most once, so aliasing is preserved + shared = tnode(1) + aliased = newnode(LineNumberNode(0), K"block", SyntaxList(shared, shared)) + acopy = copy_ast(aliased) + @test aliased ≈ acopy + @test acopy[1] !== shared + @test acopy[1] === acopy[2] + + # Unlike copy_ast, mktree extends the provenance chain rather than + # copying it + stcopy2 = mktree(st) + @test stcopy2 !== st + @test st ≈ stcopy2 + @test stcopy2[1] !== st[1] + @test stcopy2.source === st + @test stcopy2[1].source === st[1] + end + + @testset "unalias_nodes" begin + # 1 -+-> 2 -+ + # | +-> 4 + # +-> 3 -+ + build1() = let n4 = tnode(4) + tnode(1, tnode(2, n4), tnode(3, n4)) + end + ref = build1() + st = build1() + src4 = st[1][1].source + stu = unalias_nodes(st) + @test stu === st # unaliases in place + @test ref ≈ stu + @test length(flat_nodes(stu)) == 5 # node 4 copied once + @test allunique(flat_nodes(stu)) + # the copy keeps node 4's attributes, and doesn't extend its provenance + @test 4 == stu[1][1].value == stu[2][1].value + @test src4 === stu[1][1].source === stu[2][1].source + + # +-> 5 + # | + # 1 -+-> 2 -+---->>>-> 6 + # | ||| + # +-> 3 -> 7 -+|| + # | || + # +-> 4 -+-----+| + # | | + # +------+ + build2() = let n6 = tnode(6) + tnode(1, + tnode(2, tnode(5), n6), + tnode(3, tnode(7, n6)), + tnode(4, n6, n6)) + end + ref = build2() + stu = unalias_nodes(build2()) + @test ref ≈ stu + # node 6 occurs four times, so it should be copied three times + @test length(flat_nodes(stu)) == 10 + @test allunique(flat_nodes(stu)) + @test 6 == stu[1][2].value == stu[2][1][1].value == + stu[3][1].value == stu[3][2].value + + # 1 -+-> 2 ->-> 4 -+----> 5 ->-> 7 + # | | | | + # +-> 3 -+ +-->-> 6 -+ + # | | + # +------------+ + build3() = let n7 = tnode(7), + n5 = tnode(5, n7), + n6 = tnode(6, n7), + n4 = tnode(4, n5, n6) + tnode(1, tnode(2, n4), tnode(3, n4, n6)) + end + ref = build3() + stu = unalias_nodes(build3()) + @test ref ≈ stu + @test length(flat_nodes(stu)) == 15 + @test allunique(flat_nodes(stu)) + # attrs of nodes 4-7 survive copying + @test 4 == stu[1][1].value == stu[2][1].value + @test 5 == stu[1][1][1].value == stu[2][1][1].value + @test 6 == stu[1][1][2].value == stu[2][1][2].value == stu[2][2].value + @test 7 == stu[1][1][1][1].value == stu[1][1][2][1].value == + stu[2][1][1][1].value == stu[2][1][2][1].value == stu[2][2][1].value + end + + @testset "annotate_parent" begin + chk_parent(st, parent) = getmeta(st, :parent, nothing) === parent && + all(c->chk_parent(c, st), children(st)) + # 1 -+-> 2 ->-> 4 --> 5 + # | | + # +-> 3 -+ + st = let n4 = tnode(4, tnode(5)) + tnode(1, tnode(2, n4), tnode(3, n4)) + end + st = annotate_parent!(st) + @test chk_parent(st, nothing) + end +end + +@testset "SyntaxList" begin + st = parsestmt(SyntaxTree, "function foo end") + + sl0 = SyntaxList() + @test sl0 isa SyntaxList + @test length(sl0) == 0 + + sl1 = SyntaxList(st) + @test sl1 isa SyntaxList + @test length(sl1) == 1 + @test sl1[1] === st + + sl2 = SyntaxList(st, st) + @test sl2 isa SyntaxList + @test length(sl2) == 2 + @test sl2[2] === st +end + +@testset "@stm SyntaxTree pattern-matching" begin + st = parsestmt(SyntaxTree, "foo(a,b=1,c(d=2))") + # (call foo a (kw b 1) (call c (kw d 2))) + + @testset "basic functionality" begin + @test @stm st begin + _ -> true + end + + @test @stm st begin + x -> x isa SyntaxTree + end + + @test @stm st begin + [K"function" f a b c] -> false + [K"call" f a b c] -> true + end + + @test @stm st begin + [K"function" _ _ _ _] -> false + [K"call" _ _ _ _] -> true + end + + @test @stm st begin + [K"call" f a b] -> false + [K"call" f a b c d] -> false + [K"call" f a b c] -> true + end + + @test @stm st begin + [K"call" f a b c] -> + kind(f) === K"Identifier" && + kind(b) === K"kw" && + kind(c) === K"call" + end + end + + @testset "errors" begin + # no match + @test_throws ErrorException @stm st begin + [K"Identifier"] -> false + end + + # assuming we run this checker by default + @testset "_stm_check_usage" begin + bad = Expr[ + :(@stm st begin + [a] -> false + end) + :(@stm st begin + [K"None",a] -> false + end) + :(@stm st begin + [K"None" a a] -> false + end) + :(@stm st begin + x + end) + :(@stm st begin + x() -> false + end) + :(@stm st begin + (a, b=1) -> false + end) + :(@stm st begin + [K"None" a... b...] -> false + end) + ] + for e in bad + Base.remove_linenums!(e) + @testset "$(string(e))" begin + @test_throws AssertionError macroexpand(@__MODULE__, e) + end + end + end + end + + @testset "nested patterns" begin + @test 1 === @stm st begin + [K"call" [K"Identifier"] [K"Identifier"] [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"Identifier"] k2]]] -> 1 + [K"call" [K"Identifier"] [K"Identifier"] [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" _ k2]]] -> 2 + [K"call" [K"Identifier"] [K"Identifier"] [K"kw" _ k1] [K"call" _ _]] -> 3 + [K"call" [K"Identifier"] [K"Identifier"] _ _ ] -> 4 + [K"call" _ _ _ _] -> 5 + end + @test 1 === @stm st begin + [K"call" _ _ [K"None" [K"Identifier"] k1] [K"None" [K"Identifier"] [K"None" [K"None"] k2]]] -> 5 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"None" [K"Identifier"] [K"None" [K"None"] k2]]] -> 4 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"None" [K"None"] k2]]] -> 3 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"None"] k2]]] -> 2 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"Identifier"] k2]]] -> 1 + end + @test 1 === @stm st begin + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"Identifier"] k2] bad]] -> 4 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"Identifier"] k2 bad]]] -> 3 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"Identifier" bad] k2]]] -> 2 + [K"call" _ _ [K"kw" [K"Identifier"] k1] [K"call" [K"Identifier"] [K"kw" [K"Identifier"] k2]]] -> 1 + end + end + + @testset "vcat form (newlines in pattern)" begin + @test @stm st begin + [K"call" + f + a + b + c] -> true + end + @test @stm st begin + [K"call" + f a b c] -> true + end + @test @stm st begin + [K"call" + + + f a b c] -> true + end + @test @stm st begin + [K"call" + [K"Identifier"] [K"Identifier"] + [K"kw" [K"Identifier"] k1] + [K"call" + [K"Identifier"] + [K"kw" + [K"Identifier"] + k2]]] -> true + end + end + + @testset "SyntaxList splat matching" begin + # NB: a splat binds a view of the parent's children, not a SyntaxList + # trailing splat + @test @stm st begin + [K"call" f _...] -> true + end + @test @stm st begin + [K"call" f args...] -> kind(f) === K"Identifier" + end + @test @stm st begin + [K"call" f args...] -> + args isa AbstractVector{SyntaxTree} && length(args) === 3 + end + @test @stm st begin + [K"call" f args...] -> kind(args[1]) === K"Identifier" && + kind(args[2]) === K"kw" && + kind(args[3]) === K"call" + end + @test @stm st begin + [K"call" f a b c empty...] -> + empty isa AbstractVector{SyntaxTree} && length(empty) === 0 + end + + # binds after splat + @test @stm st begin + [K"call" f args... last] -> + args isa AbstractVector{SyntaxTree} && + length(args) === 2 + end + @test @stm st begin + [K"call" f args... last] -> + kind(f) === K"Identifier" && + kind(args[1]) === K"Identifier" && + kind(args[2]) === K"kw" && + kind(last) === K"call" + end + @test @stm st begin + [K"call" empty... f a b c] -> + empty isa AbstractVector{SyntaxTree} && length(empty) === 0 + end + end + + @testset "`when` clauses affect matching" begin + @test @stm st begin + (_, when=false) -> false + (_, when=true) -> true + end + @test @stm st begin + ([K"call" _...], when=false) -> false + ([K"call" _...], when=true) -> true + end + @test @stm st begin + ([K"call" _ _...], when=kind(st[1])===K"Identifier") -> true + end + @test @stm st begin + ([K"call" f _...], when=kind(f)===K"Identifier") -> true + end + end + + @testset "effects of when=cond" begin + let x = Int[] + @test @stm st begin + (_, when=(push!(x, 1); true)) -> x == [1] + end + empty!(x) + + @test @stm st begin + (_, when=(push!(x, 1); false)) -> false + (_, when=(push!(x, 2); false)) -> false + (_, when=(push!(x, 3); true)) -> x == [1, 2, 3] + end + empty!(x) + + @test @stm st begin + ([K"block"], when=(push!(x, 123); false)) -> false + (_, when=(push!(x, 1); true)) -> x == [1] + end + empty!(x) + + @test @stm st begin + (x_pat, when=((x_when = x_pat); true)) -> x_pat == x_when + end + end + end +end diff --git a/JuliaSyntax/test/syntax_node.jl b/JuliaSyntax/test/syntax_node.jl new file mode 100644 index 0000000000000..3e2361ca56b2f --- /dev/null +++ b/JuliaSyntax/test/syntax_node.jl @@ -0,0 +1,117 @@ +@testset "SyntaxNode" begin + # Child access + tt = "a*b + c" + t = parsestmt(SyntaxNode, tt) + + @test sourcetext(t[1]) == "a*b" + @test sourcetext(t[1][1]) == "a" + @test sourcetext(t[1][2]) == "*" + @test sourcetext(t[1][3]) == "b" + @test sourcetext(t[2]) == "+" + @test sourcetext(t[3]) == "c" + + @test JuliaSyntax.first_byte(t[2]) == findfirst(==('+'), tt) + @test JuliaSyntax.source_line(t[3]) == 1 + @test source_location(t[3]) == (1, 7) + + # Child indexing + @test t[end] === t[3] + @test sourcetext.(t[2:3]) == ["+", "c"] + @test sourcetext.(t[2:end]) == ["+", "c"] + @test firstindex(t) == 1 + @test lastindex(t) == 3 + @test !is_leaf(t) + @test is_leaf(t[3]) + + @test sprint(show, t) == "(call-i (call-i a * b) + c)" + @test sprint(io->show(io, MIME("text/x.sexpression"), t, show_kind=true)) == + "(call-i (call-i a::Identifier *::Identifier b::Identifier) +::Identifier c::Identifier)" + + @test sprint(highlight, t[1][3]) == "a*b + c\n# ╙" + + # Pass-through field access + node = t[1][1] + @test node.val === :a + # The specific error text has evolved over Julia versions. Check that it involves `SyntaxData` and immutability + e = try node.val = :q catch e e end + @test occursin("immutable", e.msg) && occursin("SyntaxData", e.msg) + + # Newline-terminated source + t = parsestmt(SyntaxNode, "a*b + c\n") + @test sprint(highlight, t[1][3]) == "a*b + c\n# ╙" + + # copy + t = parsestmt(SyntaxNode, "a*b + c") + ct = copy(t) + ct.data = nothing + @test ct.data === nothing && t.data !== nothing + @test ct[1].parent === ct + @test ct[1] !== t[1] + + node = parsestmt(SyntaxNode, "f()") + push!(node, parsestmt(SyntaxNode, "x")) + @test length(children(node)) == 2 + node[2] = parsestmt(SyntaxNode, "y") + @test sourcetext(node[2]) == "y" + + # SyntaxNode with offsets + t,_ = parsestmt(SyntaxNode, "begin a end\nbegin b end", 13) + @test first(byte_range(t)) == 13 + @test first(byte_range(t[1])) == 19 + @test t[1].val == :b + + # Unicode character ranges + src = "ab + αβ" + t = parsestmt(SyntaxNode, src) + @test char_range(t[1]) == 1:2 + @test char_range(t[2]) == 4:4 + @test char_range(t[3]) == 6:8 + # conversely, β takes two bytes so char_range(t[3]) != byte_range(t[3]) + @test byte_range(t[3]) == 6:9 +end + +@testset "SyntaxNode pretty printing" begin + t = parsestmt(SyntaxNode, "f(a*b,\n c)", filename="foo.jl") + @test sprint(show, MIME("text/plain"), t) == """ + SyntaxNode: + [call] + f :: Identifier + [call-i] + a :: Identifier + * :: Identifier + b :: Identifier + c :: Identifier + """ + + @test sprint(io->show(io, MIME("text/plain"), t, show_location=true)) == """ + SyntaxNode: + line:col│ byte_range │ tree + -file- │ "foo.jl" + 1:1 │ 1:11 │[call] + 1:1 │ 1:1 │ f :: Identifier + 1:3 │ 3:5 │ [call-i] + 1:3 │ 3:3 │ a :: Identifier + 1:4 │ 4:4 │ * :: Identifier + 1:5 │ 5:5 │ b :: Identifier + 2:3 │ 10:10 │ c :: Identifier + """ + + @test sprint(io->show(io, MIME("text/plain"), t, show_kind=false)) == """ + SyntaxNode: + [call] + f + [call-i] + a + * + b + c + """ + + t,_ = parsestmt(SyntaxNode, "begin a end\nbegin b end", 13, first_line=100) + @test sprint(io->show(io, MIME("text/plain"), t, show_location=true)) == """ + SyntaxNode: + line:col│ byte_range │ tree + 100:1 │ 13:23 │[block] + 100:7 │ 19:19 │ b :: Identifier + """ +end diff --git a/JuliaSyntax/test/test_utils.jl b/JuliaSyntax/test/test_utils.jl new file mode 100644 index 0000000000000..e2a4f478bb3de --- /dev/null +++ b/JuliaSyntax/test/test_utils.jl @@ -0,0 +1,503 @@ +using Test + +# We need a relative include here as JuliaSyntax may come from Base. +using .JuliaSyntax: + # Parsing + ParseStream, + ParseState, + Diagnostic, + SourceFile, + source_location, + source_line, + source_line_range, + parse!, + parsestmt, + parseall, + parseatom, + build_tree, + @K_str, + # Nodes + GreenNode, + RedTreeCursor, + SyntaxNode, + ErrorVal, + # Node inspection + kind, + flags, + EMPTY_FLAGS, TRIVIA_FLAG, INFIX_FLAG, + head, + span, + SyntaxHead, + is_trivia, + sourcetext, + is_leaf, + numchildren, + children, + fl_parseall, + fl_parse, + highlight, + tokenize, + untokenize, + filename, + byte_range, + char_range + +if VERSION < v"1.6" + # Compat stuff which might not be in Base for older versions + using JuliaSyntax: isnothing, only, peek +end + +function toks(str) + ts = [JuliaSyntax.Tokenize.untokenize(t, str)=>kind(t) + for t in JuliaSyntax.Tokenize.tokenize(str)] + @test ts[end] == (""=>K"EndMarker") + pop!(ts) + ts +end + +function remove_macro_linenums!(ex) + if Meta.isexpr(ex, :macrocall) + ex.args[2] = nothing + end + if ex isa Expr + map!(remove_macro_linenums!, ex.args, ex.args) + end + return ex +end + +function remove_module_versions!(ex) + # In v1.14+, JuliaSyntax adds a version as the first argument to module expressions. + # Remove it for comparison with flisp output. + if Meta.isexpr(ex, :module) && length(ex.args) >= 1 && ex.args[1] isa VersionNumber + deleteat!(ex.args, 1) + end + if ex isa Expr + for arg in ex.args + remove_module_versions!(arg) + end + end + return ex +end + +function remove_all_linenums_and_modvers!(ex) + JuliaSyntax.remove_linenums!(ex) + remove_macro_linenums!(ex) + remove_module_versions!(ex) +end + +function kw_to_eq(ex) + return Meta.isexpr(ex, :kw) ? Expr(:(=), ex.args...) : ex +end + +function triple_string_roughly_equal(fl_str, str) + # Allow some leeway for a bug in the reference parser with + # triple quoted strings + lines = split(str, '\n') + fl_lines = split(fl_str, '\n') + if length(lines) != length(fl_lines) + return false + end + has_whitespace_only_line = + any(!isempty(fl_line) && all(c in " \t" for c in fl_line) + for fl_line in fl_lines) + if !has_whitespace_only_line + return str == fl_str + end + for (line, fl_line) in zip(lines, fl_lines) + if !all(c in " \t" for c in fl_line) && !endswith(line, fl_line) + return false + end + end + return true +end + +function exprs_equal_no_linenum(fl_ex, ex) + remove_all_linenums_and_modvers!(deepcopy(ex)) == remove_all_linenums_and_modvers!(deepcopy(fl_ex)) +end + +function is_eventually_call(ex) + return ex isa Expr && (ex.head === :call || + (ex.head === :where || ex.head === :(::)) && is_eventually_call(ex.args[1])) +end + +# Compare Expr from reference parser expression to JuliaSyntax parser, ignoring +# differences due to bugs in the reference parser. +function exprs_roughly_equal(fl_ex, ex) + if fl_ex isa Float64 && Meta.isexpr(ex, :call, 3) && + ex.args[1] == :* && + ex.args[2] == fl_ex && + (ex.args[3] == :f || ex.args[3] == :f0) + # 0x1p0f + return true + elseif !(fl_ex isa Expr) || !(ex isa Expr) + if fl_ex isa String && ex isa String + if fl_ex == ex + return true + else + return triple_string_roughly_equal(fl_ex, ex) + end + else + return fl_ex == ex + end + end + # Ignore differences in line number nodes within block-like constructs + fl_args = fl_ex.head in (:block, :quote, :toplevel) ? + filter(x->!(x isa LineNumberNode), fl_ex.args) : + fl_ex.args + args = ex.head in (:block, :quote, :toplevel) ? + filter(x->!(x isa LineNumberNode), ex.args) : + ex.args + if (fl_ex.head == :block && ex.head == :tuple && + length(fl_args) == 2 && length(args) == 2 && + Meta.isexpr(args[1], :parameters, 1) && + exprs_roughly_equal(fl_args[2], args[1].args[1]) && + exprs_roughly_equal(fl_args[1], args[2])) + # Allow `(a; b,)`: + # * Reference parser produces a block + # * New parser produces a frankentuple + return true + end + if fl_ex.head != ex.head + return false + end + h = ex.head + if h == :function && Meta.isexpr(fl_args[1], :block) + blockargs = filter(x->!(x isa LineNumberNode), fl_args[1].args) + posargs = blockargs[1:max(0, length(blockargs))] + kwargs = blockargs[2:end] + for i = 1:length(kwargs) + if Meta.isexpr(kwargs[i], :(=)) + kwargs[i] = Expr(:kw, kwargs[i].args...) + end + end + fl_args[1] = Expr(:tuple, Expr(:parameters, kwargs...), posargs...) + elseif h == :for + iterspec = args[1] + if is_eventually_call(iterspec.args[1]) && + Meta.isexpr(iterspec.args[2], :block) + blk = iterspec.args[2] + if length(blk.args) == 2 && blk.args[1] isa LineNumberNode + # Ignore short form function location differences in + # `for f() = 1:3 end` + iterspec.args[2] = blk.args[2] + end + end + elseif (h == :(=) || h == :kw) && Meta.isexpr(fl_args[1], :(::), 1) && + Meta.isexpr(fl_args[2], :block, 2) && fl_args[2].args[1] isa LineNumberNode + # The flisp parser adds an extra block around `w` in the following case + # f(::g(z) = w) = 1 + fl_args[2] = fl_args[2].args[2] + elseif h == :module && length(args) == length(fl_args) + 1 && args[1] isa VersionNumber + # In v1.14+, JuliaSyntax adds a version as the first argument to module expressions. + # Skip the version when comparing. + args = args[2:end] + end + if length(fl_args) != length(args) + return false + end + if h == :do && length(args) >= 1 && Meta.isexpr(fl_args[1], :macrocall) + # Macrocalls with do, as in `@f(a=1) do\nend` use :kw in the + # reference parser for the `a=1`, but we regard this as a bug. + fl_args = copy(fl_args) + fl_args[1] = Expr(:macrocall, map(kw_to_eq, args[1].args)...) + end + for i = 1:length(args) + if !exprs_roughly_equal(fl_args[i], args[i]) + return false + end + end + return true +end + +function parsers_agree_on_file(filename; kws...) + text = try + read(filename, String) + catch + # Something went wrong reading the file. This isn't a parser failure so + # ignore this case. + return true + end + parsers_agree_on_file(text, filename; kws...) +end + +function parsers_agree_on_file(text, filename; exprs_equal=exprs_equal_no_linenum) + fl_ex = fl_parseall(text, filename=filename) + if Meta.isexpr(fl_ex, :toplevel) && !isempty(fl_ex.args) && + Meta.isexpr(fl_ex.args[end], (:error, :incomplete)) + # Reference parser failed. This generally indicates a broken file not a + # parser problem, so ignore this case. + return true + end + try + stream = ParseStream(text; version=v"1.14") + parse!(stream) + ex = build_tree(Expr, stream, filename=filename) + return !JuliaSyntax.any_error(stream) && exprs_equal(fl_ex, ex) + catch exc + @error "Parsing failed" filename exception=current_exceptions() + return false + end +end + +function find_source_in_path(basedir) + src_list = String[] + for (root, dirs, files) in walkdir(basedir) + append!(src_list, (joinpath(root, f) for f in files + if endswith(f, ".jl") && (p = joinpath(root,f); !islink(p) && isfile(p)))) + end + src_list +end + +test_parse_all_in_path(basedir) = + test_parse_all_in_path(path->exprs_equal_no_linenum, basedir) + +function test_parse_all_in_path(compare_for_path::Function, basedir) + for filepath in find_source_in_path(basedir) + cmp = compare_for_path(filepath) + if isnothing(cmp) + continue + end + @testset "Parse $(relpath(filepath, basedir))" begin + text = try + read(filepath, String) + catch + # Something went wrong reading the file. This isn't a parser failure so + # ignore this case. + continue + end + parsers_agree = parsers_agree_on_file(text, filepath, exprs_equal=cmp) + @test parsers_agree + if !parsers_agree + reduced_failures = reduce_text.(reduce_tree(text), + parsers_fuzzy_disagree) + @test reduced_failures == [] + end + end + end +end + +#------------------------------------------------------------------------------- +# Test case reduction + +# Check whether a given SyntaxNode converts to the same Expr as the flisp +# parser produces from the source text of the node. +function equals_flisp_parse(exprs_equal, tree) + node_text = sourcetext(tree) + # Reparse with JuliaSyntax. This is a crude way to ensure we're not missing + # some context from the parent node. + fl_ex = fl_parseall(node_text, filename="none") + if Meta.isexpr(fl_ex, :error) || (Meta.isexpr(fl_ex, :toplevel) && + length(fl_ex.args) >= 1 && + Meta.isexpr(fl_ex.args[end], :error)) + return true # Something went wrong in reduction; ignore these cases 😬 + end + ex = parseall(Expr, node_text, filename="none", ignore_errors=true) + exprs_equal(fl_ex, ex) +end + +function _reduce_tree(failing_subtrees, tree; exprs_equal=exprs_equal_no_linenum) + if equals_flisp_parse(exprs_equal, tree) + return false + end + if is_leaf(tree) + push!(failing_subtrees, tree) + return true + end + had_failing_subtrees = false + if !is_leaf(tree) + for child in children(tree) + if is_trivia(child) || is_leaf(child) + continue + end + had_failing_subtrees |= _reduce_tree(failing_subtrees, child; exprs_equal=exprs_equal) + end + end + if !had_failing_subtrees + push!(failing_subtrees, tree) + end + return true +end + +""" + reduce_tree(tree::SyntaxNode; exprs_equal=exprs_equal_no_linenum) + +Select minimal subtrees of `tree` which are inconsistent between flisp and +JuliaSyntax parsers. +""" +function reduce_tree(tree::SyntaxNode; kws...) + subtrees = Vector{typeof(tree)}() + _reduce_tree(subtrees, tree; kws...) + subtrees +end + +""" + reduce_tree(text::AbstractString; exprs_equal=exprs_equal_no_linenum) + +Find the minimal subtrees of the parsed form of `text` which are inconsistent +between flisp and JuliaSyntax parsers and return the source text of those +subtrees. +""" +function reduce_tree(text::AbstractString; kws...) + tree = parseall(SyntaxNode, text, ignore_warnings=true) + sourcetext.(reduce_tree(tree; kws...)) +end + + +#------------------------------------------------------------------------------- +# Text-based test case reduction +function parser_throws_exception(text) + try + JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, text, ignore_errors=true) + false + catch + true + end +end + +function parsers_fuzzy_disagree(text::AbstractString) + fl_ex = fl_parseall(text, filename="none") + if Meta.isexpr(fl_ex, (:error,:incomplete)) || + (Meta.isexpr(fl_ex, :toplevel) && length(fl_ex.args) >= 1 && + Meta.isexpr(fl_ex.args[end], (:error,:incomplete))) + return false + end + try + ex = parseall(Expr, text, filename="none", ignore_errors=true) + return !exprs_roughly_equal(fl_ex, ex) + catch + @error "Reduction failed" text + return false + end +end + + +""" +Reduce text of a test case via combination of bisection and random deletion. + +This is suited to randomly generated strings, but it's surprisingly effective +for code-like strings as well. +""" +function reduce_text(str, parse_differs) + while true + if length(str) <= 1 + return str + end + m1 = thisind(str, length(str)÷2) + m2 = nextind(str, m1) + if parse_differs(str[1:m1]) + str = str[1:m1] + elseif parse_differs(str[m2:end]) + str = str[m2:end] + else + chunklen = clamp(length(str)÷10, 1, 10) + reduced = false + for i = 1:100 + m = thisind(str, rand(1:length(str)-chunklen)) + m3 = nextind(str, m+chunklen) + if m3 == nextind(str, m) + continue + end + s = str[1:m]*str[m3:end] + if parse_differs(s) + str = s + reduced = true + break + end + end + if !reduced + return str + end + end + end +end + +function show_green_tree(code; version::VersionNumber=v"1.6") + t = JuliaSyntax.parseall(GreenNode, code, version=version) + sprint(show, MIME"text/plain"(), t, code) +end + +#------------------------------------------------------------------------------- +# Parse s-expressions +function parse_sexpr(code) + st = ParseStream(code) + pos_stack = ParseStreamPosition[] + while true + k = peek(st) + if k == K"(" + push!(pos_stack, position(st)) + bump(st, TRIVIA_FLAG) + elseif k == K")" + if isempty(pos_stack) + bump(st, error="Mismatched `)` with no opening `(`") + break + else + bump(st, TRIVIA_FLAG) + end + emit(st, pop!(pos_stack), K"parens") + elseif k == K"Identifier" || k == K"Integer" + bump(st) + elseif k == K"NewlineWs" + bump(st, TRIVIA_FLAG) + elseif k == K"EndMarker" + if !isempty(pos_stack) + bump_invisible(st, K"error", error="Mismatched `)`") + end + break + else + bump(st, error="Unexpected token") + end + end + if JuliaSyntax.any_error(st) + throw(JuliaSyntax.ParseError(st)) + end + st +end + + +#------------------------------------------------------------------------------- +# Tools copied from Base.Meta which call core_parser_hook as if called by +# Meta.parse(), but without installing the global hook. + +function _Meta_parse_string(text::AbstractString, filename::AbstractString, + lineno::Integer, index::Integer, options) + if index < 1 || index > ncodeunits(text) + 1 + throw(BoundsError(text, index)) + end + ex, offset::Int = JuliaSyntax.core_parser_hook(text, filename, lineno, index-1, options) + ex, offset+1 +end + +function Meta_parse(str::AbstractString, pos::Integer; + filename="none", greedy::Bool=true, raise::Bool=true, depwarn::Bool=true) + ex, pos = _Meta_parse_string(str, String(filename), 1, pos, greedy ? :statement : :atom) + if raise && Meta.isexpr(ex, :error) + err = ex.args[1] + if err isa String + err = Meta.ParseError(err) # For flisp parser + end + throw(err) + end + return ex, pos +end + +function Meta_parse(str::AbstractString; + filename="none", raise::Bool=true, depwarn::Bool=true) + ex, pos = Meta_parse(str, 1; filename=filename, greedy=true, raise=raise, depwarn=depwarn) + if Meta.isexpr(ex, :error) + return ex + end + if pos <= ncodeunits(str) + raise && throw(Meta.ParseError("extra token after end of expression")) + return Expr(:error, "extra token after end of expression") + end + return ex +end + +function Meta_parseatom(text::AbstractString, pos::Integer; filename="none", lineno=1) + return _Meta_parse_string(text, String(filename), lineno, pos, :atom) +end + +function Meta_parseall(text::AbstractString; filename="none", lineno=1) + ex,_ = _Meta_parse_string(text, String(filename), lineno, 1, :all) + return ex +end diff --git a/JuliaSyntax/test/test_utils_tests.jl b/JuliaSyntax/test/test_utils_tests.jl new file mode 100644 index 0000000000000..51515515a83f5 --- /dev/null +++ b/JuliaSyntax/test/test_utils_tests.jl @@ -0,0 +1,37 @@ +# Tests for the test_utils go here to allow the utils to be included on their +# own without invoking the tests. +@testset "Reference parser bugs" begin + # `0x1.8p0f` + @test exprs_roughly_equal(1.5, + Expr(:call, :*, 1.5, :f)) + @test exprs_roughly_equal(1.5, + Expr(:call, :*, 1.5, :f0)) + # `@f(a=1) do \n end` + @test exprs_roughly_equal(Expr(:do, Expr(:macrocall, Symbol("@f"), LineNumberNode(1), Expr(:kw, :a, 1)), + Expr(:->, Expr(:tuple), Expr(:block, LineNumberNode(1)))), + Expr(:do, Expr(:macrocall, Symbol("@f"), LineNumberNode(1), Expr(:(=), :a, 1)), + Expr(:->, Expr(:tuple), Expr(:block, LineNumberNode(1))))) + # `"""\n a\n \n b"""` + @test exprs_roughly_equal("a\n \nb", " a\n\n b") + @test !exprs_roughly_equal("a\n x\nb", " a\n x\n b") + @test exprs_roughly_equal("a\n x\nb", "a\n x\nb") + # `(a; b,)` + @test exprs_roughly_equal(Expr(:block, :a, LineNumberNode(1), :b), + Expr(:tuple, Expr(:parameters, :b), :a)) + @test !exprs_roughly_equal(Expr(:block, :a, LineNumberNode(1), :b), + Expr(:tuple, Expr(:parameters, :c), :a)) + @test !exprs_roughly_equal(Expr(:block, :a, LineNumberNode(1), :b), + Expr(:tuple, Expr(:parameters, :b), :c)) + @test !exprs_roughly_equal(Expr(:block, :a, LineNumberNode(1), :b, :c), + Expr(:tuple, Expr(:parameters, :b), :a)) + + # Line numbers for short form function defs in `for` :-( + @test exprs_roughly_equal(Expr(:for, Expr(:(=), + Expr(:call, :f), + 1), + Expr(:block, LineNumberNode(1))), + Expr(:for, Expr(:(=), + Expr(:call, :f), + Expr(:block, LineNumberNode(1), 1)), + Expr(:block, LineNumberNode(1)))) +end diff --git a/JuliaSyntax/test/tokenize.jl b/JuliaSyntax/test/tokenize.jl new file mode 100644 index 0000000000000..e8abd87d9658c --- /dev/null +++ b/JuliaSyntax/test/tokenize.jl @@ -0,0 +1,1282 @@ +# Hack: Introduce a module here to isolate some Tokenize internals from JuliaSyntax +module TokenizeTests + +using Test + +using ..JuliaSyntax: + JuliaSyntax, + @K_str, + Kind, + kind, + is_error, + is_operator + +using ..JuliaSyntax.Tokenize: + Tokenize, + tokenize, + untokenize, + RawToken + +import ..toks + +tok(str, i = 1) = collect(tokenize(str))[i] + +strtok(str) = untokenize.(collect(tokenize(str)), str) + +function onlytok(str) + ts = collect(tokenize(str)) + (length(ts) == 2 && ts[2].kind == K"EndMarker") || + error("Expected one token got $(length(ts)-1)") + return ts[1].kind +end + +@testset "tokens" begin + for s in ["a", IOBuffer("a")] + l = tokenize(s) + @test Tokenize.readchar(l) == 'a' + + l_old = l + @test l == l_old + @test Tokenize.eof(l) + @test Tokenize.readchar(l) == Tokenize.EOF_CHAR + + end +end # testset + +@testset "tokenize unicode" begin + # FIXME: rm VERSION check once we implement our own is_identifier_char + emoji = VERSION < v"1.5" ? "😄" : "\U1F3F3\UFE0F\U200D\U1F308" # 🏳️‍🌈 requires newer Unicode + str = "𝘋 =2"*emoji + for s in [str, IOBuffer(str)] + l = tokenize(s) + kinds = [K"Identifier", K"Whitespace", K"=", + K"Integer", K"Identifier", K"EndMarker"] + token_strs = ["𝘋", " ", "=", "2", emoji, ""] + for (i, n) in enumerate(l) + @test kind(n) == kinds[i] + @test untokenize(n, str) == token_strs[i] + end + end +end # testset + +@testset "tokenize complex piece of code" begin + + str = """ + function foo!{T<:Bar}(x::{T}=12) + @time (x+x, x+x); + end + try + foo + catch + bar + end + @time x+x + y[[1 2 3]] + [1*2,2;3,4] + "string"; 'c' + (a&&b)||(a||b) + # comment + #= comment + is done here =# + 2%5 + a'/b' + a.'\\b.' + `command` + 12_sin(12) + {} + ' + """ + + # Generate the following with + # ``` + # for t in kind.(collect(tokenize(str))) + # print(kind(t), ",") + # end + # ``` + # and *check* it afterwards. + + kinds = [K"function",K"Whitespace",K"Identifier",K"{",K"Identifier", + K"<:",K"Identifier",K"}",K"(",K"Identifier",K"::", + K"{",K"Identifier",K"}",K"=",K"Integer",K")", + + K"NewlineWs",K"@",K"Identifier",K"Whitespace",K"(", + K"Identifier",K"+",K"Identifier",K",",K"Whitespace", + K"Identifier",K"+",K"Identifier",K")",K";", + + K"NewlineWs",K"end", + + K"NewlineWs",K"try", + K"NewlineWs",K"Identifier", + K"NewlineWs",K"catch", + K"NewlineWs",K"Identifier", + K"NewlineWs",K"end", + + K"NewlineWs",K"@",K"Identifier",K"Whitespace",K"Identifier", + K"+",K"Identifier", + + K"NewlineWs",K"Identifier",K"[",K"[",K"Integer",K"Whitespace", + K"Integer",K"Whitespace",K"Integer",K"]",K"]", + + K"NewlineWs",K"[",K"Integer",K"*",K"Integer",K",",K"Integer", + K";",K"Integer",K",",K"Integer",K"]", + + K"NewlineWs",K"\"",K"String",K"\"",K";",K"Whitespace",K"'",K"Char",K"'", + + K"NewlineWs",K"(",K"Identifier",K"&&",K"Identifier",K")",K"||", + K"(",K"Identifier",K"||",K"Identifier",K")", + + K"NewlineWs",K"Comment", + + K"NewlineWs",K"Comment", + + K"NewlineWs",K"Integer",K"Operator",K"Integer", + + K"NewlineWs",K"Identifier",K"'",K"Operator",K"Identifier",K"'", + + K"NewlineWs",K"Identifier",K".",K"'",K"Operator",K"Identifier",K".",K"'", + + K"NewlineWs",K"`",K"CmdString",K"`", + + K"NewlineWs",K"Integer",K"Identifier",K"(",K"Integer",K")", + + K"NewlineWs",K"{",K"}", + + K"NewlineWs",K"'",K"Char",K"EndMarker"] + + for (i, n) in enumerate(tokenize(str)) + @test kind(n) == kinds[i] + end + + @testset "roundtrippability" begin + @test join(untokenize.(collect(tokenize(str)), str)) == str + end + + @test all((t.endbyte - t.startbyte + 1)==sizeof(untokenize(t, str)) for t in tokenize(str)) +end # testset + +@testset "issue 5, '..'" begin + @test kind.(collect(tokenize("1.23..3.21"))) == [K"Float",K".",K".",K"Float",K"EndMarker"] +end + +@testset "issue 17, >>" begin + str = ">> " + @test untokenize(tok(str), str)==">>" +end + +@testset "tokenize newlines" begin + n = "\n" + rn = "\r\n" + nl = K"NewlineWs" + for i in 0:5 + j = 5 - i + @test toks(n^i * rn^j) == vcat(fill(n => nl, i), fill(rn => nl, j)) + @test toks(rn^i * n^j) == vcat(fill(rn => nl, i), fill(n => nl, j)) + end +end + +@testset "test added operators" begin + # Compound assignments now emit separate operator and = tokens. The operator + # itself is emitted as K"Operator" when immediately followed by `=`. + @test toks("1+=2")[2:3] == ["+"=>K"Operator", "="=>K"="] + @test toks("1-=2")[2:3] == ["-"=>K"Operator", "="=>K"="] + @test toks("1*=2")[2:3] == ["*"=>K"Operator", "="=>K"="] + @test toks("1+%=2")[2:3] == ["+%"=>K"+%", "="=>K"="] + @test toks("1-%=2")[2:3] == ["-%"=>K"-%", "="=>K"="] + @test toks("1*%=2")[2:3] == ["*%"=>K"*%", "="=>K"="] + @test toks("1^=2")[2:3] == ["^"=>K"Operator", "="=>K"="] + @test toks("1÷=2")[2:3] == ["÷"=>K"Operator", "="=>K"="] + @test toks("1\\=2")[2:3] == ["\\"=>K"Operator", "="=>K"="] + @test toks("1\$=2")[2:3] == ["\$"=>K"Operator", "="=>K"="] + @test toks("1⊻=2")[2:3] == ["⊻"=>K"Operator", "="=>K"="] + @test toks("1:=2")[2] == (":="=>K":=") + @test toks("1-->2")[2] == ("-->"=>K"-->") + @test toks("1<--2")[2] == ("<--"=>K"Operator") + @test toks("1<-->2")[2] == ("<-->"=>K"Operator") + @test toks("1>:2")[2] == (">:"=>K">:") + + # Operators followed by `==`, `===` or `=>` (rather than the single token + # `=`) do not form compound assignments + @test toks("1+==2")[2:3] == ["+"=>K"+", "=="=>K"Operator"] + @test toks("1-=>2")[2:3] == ["-"=>K"-", "=>"=>K"Operator"] + @test toks("1*===2")[2:3] == ["*"=>K"*", "==="=>K"Operator"] + @test toks("1+%==2")[2:3] == ["+%"=>K"+%", "=="=>K"Operator"] + @test toks("1-%=>2")[2:3] == ["-%"=>K"-%", "=>"=>K"Operator"] + @test toks("1*%===2")[2:3] == ["*%"=>K"*%", "==="=>K"Operator"] + @test toks("1&==2")[2:3] == ["&"=>K"&", "=="=>K"Operator"] + @test toks("1−==2")[2:3] == ["−"=>K"-", "=="=>K"Operator"] + @test toks("1^==2")[2:3] == ["^"=>K"Operator", "=="=>K"Operator"] + @test toks("1<<==2")[2:3] == ["<<"=>K"Operator", "=="=>K"Operator"] + @test toks("1>>=>2")[2:3] == [">>"=>K"Operator", "=>"=>K"Operator"] + @test toks("1>>>==2")[2:3] == [">>>"=>K"Operator", "=="=>K"Operator"] +end + +@testset "infix" begin + @test tok("1 in 2", 3).kind == K"in" + @test tok("1 in[1]", 3).kind == K"in" + + @test tok("1 isa 2", 3).kind == K"isa" + @test tok("1 isa[2]", 3).kind == K"isa" +end + +@testset "tokenizing true/false literals" begin + @test tok("somtext true", 3).kind == K"Bool" + @test tok("somtext false", 3).kind == K"Bool" + @test tok("somtext tr", 3).kind == K"Identifier" + @test tok("somtext truething", 3).kind == K"Identifier" + @test tok("somtext falsething", 3).kind == K"Identifier" +end + + +roundtrip(str) = join(untokenize.(collect(tokenize(str)), str)) + +@testset "lexing anon functions '->' " begin + @test tok("a->b", 2).kind==K"->" +end + +@testset "comments" begin + ts = collect(tokenize(""" + # + \"\"\" + f + \"\"\" + 1 + """)) + + kinds = [K"Comment", K"NewlineWs", + K"\"\"\"", K"String", K"String", K"\"\"\"", K"NewlineWs", + K"Integer", K"NewlineWs", + K"EndMarker"] + @test kind.(ts) == kinds + + @test toks("#=# text=#") == ["#=# text=#"=>K"Comment"] + + @test toks("#= #= =#") == ["#= #= =#"=>K"ErrorEofMultiComment"] + @test toks("#=#==#=#") == ["#=#==#=#"=>K"Comment"] + @test toks("#=#==#=") == ["#=#==#="=>K"ErrorEofMultiComment"] + # comment terminated by \r\n + @test toks("#\r\n") == ["#" => K"Comment", "\r\n" => K"NewlineWs"] +end + + +@testset "invalid UTF-8" begin + @test toks("#=\xf5b\n=#") == [ + "#=\xf5b\n=#" => K"ErrorInvalidUTF8", + ] + @test toks("#\xf5b\n") == [ + "#\xf5b" => K"ErrorInvalidUTF8", + "\n" => K"NewlineWs" + ] + @test toks("\"\xf5\"") == [ + "\"" => K"\"" + "\xf5" => K"ErrorInvalidUTF8" + "\"" => K"\"" + ] + @test toks("'\xf5'") == [ + "'" => K"'" + "\xf5" => K"ErrorInvalidUTF8" + "'" => K"'" + ] + @test toks("`\xf5`") == [ + "`" => K"`" + "\xf5" => K"ErrorInvalidUTF8" + "`" => K"`" + ] +end + +@testset "primes" begin + str = """ + ImageMagick.save(fn, reinterpret(ARGB32, [0xf0884422]'')) + D = ImageMagick.load(fn) + """ + tokens = collect(tokenize(str)) + @test string(untokenize(tokens[16], str)) == string(untokenize(tokens[17], str))=="'" + + @test roundtrip("'a'") == "'a'" + @test kind.(collect(tokenize("'a'"))) == [K"'", K"Char", K"'", K"EndMarker"] + + # ' is not an operator here, so doesn't consume the suffix ᵀ + @test roundtrip("'ᵀ'") == "'ᵀ'" + @test kind.(collect(tokenize("'₁'"))) == [K"'", K"Char", K"'", K"EndMarker"] + + @test roundtrip("''") == "''" + @test kind.(collect(tokenize("''"))) == [K"'", K"'", K"EndMarker"] + + @test roundtrip("'''") == "'''" + @test kind.(collect(tokenize("'''"))) == [K"'", K"Char", K"'", K"EndMarker"] + + @test roundtrip("''''") == "''''" + @test kind.(collect(tokenize("''''"))) == [K"'", K"Char", K"'", K"'", K"EndMarker"] + + @test tok("()'", 3).kind == K"'" + @test tok("{}'", 3).kind == K"'" + @test tok("[]'", 3).kind == K"'" + @test tok("outer'", 2).kind == K"'" + @test tok("mutable'", 2).kind == K"'" + @test tok("as'", 2).kind == K"'" + @test tok("isa'", 2).kind == K"'" + @test untokenize.(collect(tokenize("a'ᵀ")), "a'ᵀ") == ["a", "'ᵀ", ""] +end + +@testset "keywords" begin + for kw in ["baremodule", + "begin", + "break", + "catch", + "const", + "continue", + "do", + "else", + "elseif", + "end", + "export", + "finally", + "for", + "function", + "global", + "if", + "import", + "let", + "local", + "macro", + "module", + "quote", + "return", + "struct", + "try", + "using", + "while", + + "abstract", + "as", + "doc", + "mutable", + "outer", + "primitive", + "type", + "var"] + + @test kind(tok(kw)) == Kind(kw) + end +end + +@testset "issue in PR #45" begin + @test length(collect(tokenize("x)"))) == 3 +end + +@testset "lex binary" begin + @test tok("0b0101").kind==K"BinInt" +end + +@testset "show" begin + io = IOBuffer() + show(io, collect(tokenize("\"abc\nd\"ef"))[2]) + @test String(take!(io)) == "1-5 String " +end + +~(tok::RawToken, t::Tuple) = tok.kind == t[1] && untokenize(tok, t[3]) == t[2] + +@testset "raw strings" begin + str = raw""" str"x $ \ y" """ + ts = collect(tokenize(str)) + @test ts[1] ~ (K"Whitespace" , " " , str) + @test ts[2] ~ (K"Identifier" , "str" , str) + @test ts[3] ~ (K"\"" , "\"" , str) + @test ts[4] ~ (K"String" , "x \$ \\ y", str) + @test ts[5] ~ (K"\"" , "\"" , str) + @test ts[6] ~ (K"Whitespace" , " " , str) + @test ts[7] ~ (K"EndMarker" , "" , str) + + str = raw"""`x $ \ y`""" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"`" , "`" , str) + @test ts[2] ~ (K"CmdString" , "x \$ \\ y" , str) + @test ts[3] ~ (K"`" , "`" , str) + @test ts[4] ~ (K"EndMarker" , "" , str) + + # str"\\" + str = "str\"\\\\\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"Identifier" , "str" , str) + @test ts[2] ~ (K"\"" , "\"" , str) + @test ts[3] ~ (K"String" , "\\\\" , str) + @test ts[4] ~ (K"\"" , "\"" , str) + @test ts[5] ~ (K"EndMarker" , "" , str) + + # str"\\\"" + str = "str\"\\\\\\\"\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"Identifier" , "str" , str) + @test ts[2] ~ (K"\"" , "\"" , str) + @test ts[3] ~ (K"String" , "\\\\\\\"" , str) + @test ts[4] ~ (K"\"" , "\"" , str) + @test ts[5] ~ (K"EndMarker" , "" , str) + + # Contextual keywords and operators allowed as raw string prefixes + str = raw""" var"x $ \ y" """ + ts = collect(tokenize(str)) + @test ts[2] ~ (K"var" , "var", str) + @test ts[4] ~ (K"String" , "x \$ \\ y", str) + + str = raw""" outer"x $ \ y" """ + ts = collect(tokenize(str)) + @test ts[2] ~ (K"outer" , "outer", str) + @test ts[4] ~ (K"String" , "x \$ \\ y", str) + + str = raw""" isa"x $ \ y" """ + ts = collect(tokenize(str)) + @test ts[2] ~ (K"isa" , "isa", str) + @test ts[4] ~ (K"String" , "x \$ \\ y", str) +end + +@testset "string escaped newline whitespace" begin + str = "\"x\\\n \ty\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"\"", "\"", str) + @test ts[2] ~ (K"String", "x", str) + @test ts[3] ~ (K"Whitespace", "\\\n \t", str) + @test ts[4] ~ (K"String", "y", str) + @test ts[5] ~ (K"\"", "\"", str) + + # No newline escape for raw strings + str = "r\"x\\\ny\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"Identifier", "r", str) + @test ts[2] ~ (K"\"", "\"", str) + @test ts[3] ~ (K"String", "x\\\ny", str) + @test ts[4] ~ (K"\"", "\"", str) +end + +@testset "triple quoted string line splitting" begin + str = "\"\"\"\nx\r\ny\rz\n\r\"\"\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"\"\"\"" , "\"\"\"", str) + @test ts[2] ~ (K"String" , "\n", str) + @test ts[3] ~ (K"String" , "x\r\n", str) + @test ts[4] ~ (K"String" , "y\r", str) + @test ts[5] ~ (K"String" , "z\n", str) + @test ts[6] ~ (K"String" , "\r", str) + @test ts[7] ~ (K"\"\"\"" , "\"\"\"", str) + + # Also for raw strings + str = "r\"\"\"\nx\ny\"\"\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"Identifier" , "r", str) + @test ts[2] ~ (K"\"\"\"" , "\"\"\"", str) + @test ts[3] ~ (K"String" , "\n", str) + @test ts[4] ~ (K"String" , "x\n", str) + @test ts[5] ~ (K"String" , "y", str) + @test ts[6] ~ (K"\"\"\"" , "\"\"\"", str) +end + +@testset "interpolation" begin + @testset "basic" begin + str = "\"\$x \$y\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"\"" , "\"", str) + @test ts[2] ~ (K"$" , "\$", str) + @test ts[3] ~ (K"Identifier" , "x" , str) + @test ts[4] ~ (K"String" , " " , str) + @test ts[5] ~ (K"$" , "\$", str) + @test ts[6] ~ (K"Identifier" , "y" , str) + @test ts[7] ~ (K"\"" , "\"", str) + @test ts[8] ~ (K"EndMarker" , "" , str) + end + + @testset "nested" begin + str = """"str: \$(g("str: \$(h("str"))"))" """ + ts = collect(tokenize(str)) + @test length(ts) == 23 + @test ts[1] ~ (K"\"" , "\"" , str) + @test ts[2] ~ (K"String" , "str: ", str) + @test ts[3] ~ (K"$" , "\$" , str) + @test ts[4] ~ (K"(" , "(" , str) + @test ts[5] ~ (K"Identifier", "g" , str) + @test ts[6] ~ (K"(" , "(" , str) + @test ts[7] ~ (K"\"" , "\"" , str) + @test ts[8] ~ (K"String" , "str: ", str) + @test ts[9] ~ (K"$" , "\$" , str) + @test ts[10] ~ (K"(" , "(" , str) + @test ts[11] ~ (K"Identifier", "h" , str) + @test ts[12] ~ (K"(" , "(" , str) + @test ts[13] ~ (K"\"" , "\"" , str) + @test ts[14] ~ (K"String" , "str" , str) + @test ts[15] ~ (K"\"" , "\"" , str) + @test ts[16] ~ (K")" , ")" , str) + @test ts[17] ~ (K")" , ")" , str) + @test ts[18] ~ (K"\"" , "\"" , str) + @test ts[19] ~ (K")" , ")" , str) + @test ts[20] ~ (K")" , ")" , str) + @test ts[21] ~ (K"\"" , "\"" , str) + @test ts[22] ~ (K"Whitespace", " " , str) + @test ts[23] ~ (K"EndMarker" , "" , str) + end + + @testset "duplicate \$" begin + str = "\"\$\$\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"\"" , "\"", str) + @test ts[2] ~ (K"$" , "\$", str) + @test ts[3] ~ (K"$" , "\$", str) + @test ts[4] ~ (K"\"" , "\"", str) + @test ts[5] ~ (K"EndMarker" , "" , str) + end + + @testset "Unmatched parens" begin + # issue 73: https://github.com/JuliaLang/Tokenize.jl/issues/73 + str = "\"\$(fdsf\"" + ts = collect(tokenize(str)) + @test ts[1] ~ (K"\"" , "\"" , str) + @test ts[2] ~ (K"$" , "\$" , str) + @test ts[3] ~ (K"(" , "(" , str) + @test ts[4] ~ (K"Identifier" , "fdsf" , str) + @test ts[5] ~ (K"\"" , "\"" , str) + @test ts[6] ~ (K"EndMarker" , "" , str) + end + + @testset "Unicode" begin + # issue 178: https://github.com/JuliaLang/Tokenize.jl/issues/178 + str = """ "\$uₕx \$(uₕx - ux)" """ + ts = collect(tokenize(str)) + @test ts[ 1] ~ (K"Whitespace" , " " , str) + @test ts[ 2] ~ (K"\"" , "\"" , str) + @test ts[ 3] ~ (K"$" , "\$" , str) + @test ts[ 4] ~ (K"Identifier" , "uₕx" , str) + @test ts[ 5] ~ (K"String" , " " , str) + @test ts[ 6] ~ (K"$" , "\$" , str) + @test ts[ 7] ~ (K"(" , "(" , str) + @test ts[ 8] ~ (K"Identifier" , "uₕx" , str) + @test ts[ 9] ~ (K"Whitespace" , " " , str) + @test ts[10] ~ (K"-" , "-" , str) + @test ts[11] ~ (K"Whitespace" , " " , str) + @test ts[12] ~ (K"Identifier" , "ux" , str) + @test ts[13] ~ (K")" , ")" , str) + @test ts[14] ~ (K"\"" , "\"" , str) + @test ts[15] ~ (K"Whitespace" , " " , str) + @test ts[16] ~ (K"EndMarker" , "" , str) + end + + @testset "var\"...\" disabled in interpolations" begin + str = """ "\$var"x" " """ + ts = collect(tokenize(str)) + @test ts[ 1] ~ (K"Whitespace" , " " , str) + @test ts[ 2] ~ (K"\"" , "\"" , str) + @test ts[ 3] ~ (K"$" , "\$" , str) + @test ts[ 4] ~ (K"var" , "var" , str) + @test ts[ 5] ~ (K"\"" , "\"" , str) + @test ts[ 6] ~ (K"Identifier" , "x" , str) + @test ts[ 7] ~ (K"\"" , "\"" , str) + @test ts[ 8] ~ (K"String" , " " , str) + @test ts[ 9] ~ (K"\"" , "\"" , str) + @test ts[10] ~ (K"Whitespace" , " " , str) + @test ts[11] ~ (K"EndMarker" , "" , str) + end + + @testset "chars after interpolation identifier" begin + # Operators allowed + @test toks("\"\$x?\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "?"=>K"String" + "\""=>K"\"" + ] + @test toks("\"\$x⫪\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "⫪"=>K"String" + "\""=>K"\"" + ] + # Operators which have their own kind rather than being in the + # generic operator table are also allowed + @test toks("\"\$x∈y\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "∈y"=>K"String" + "\""=>K"\"" + ] + @test toks("\"\$x√y\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "√y"=>K"String" + "\""=>K"\"" + ] + # Some chars disallowed (eg, U+0DF4) + @test toks("\"\$x෴\"") == [ + "\""=>K"\"" + "\$"=>K"$" + "x"=>K"Identifier" + "෴"=>K"ErrorInvalidInterpolationTerminator" + "\""=>K"\"" + ] + end +end + +@testset "inferred" begin + l = tokenize("abc") + @inferred Tokenize.next_token(l) +end + +@testset "modifying function names (!) followed by operator" begin + @test toks("a!=b") == ["a"=>K"Identifier", "!="=>K"Operator", "b"=>K"Identifier"] + @test toks("a!!=b") == ["a!"=>K"Identifier", "!="=>K"Operator", "b"=>K"Identifier"] + @test toks("!=b") == ["!="=>K"Operator", "b"=>K"Identifier"] +end + +@testset "integer literals" begin + @test onlytok("1234") == K"Integer" + @test onlytok("12_34") == K"Integer" + + @test toks("1234_") == ["1234"=>K"Integer", "_"=>K"Identifier"] + @test toks("1234x") == ["1234"=>K"Integer", "x"=>K"Identifier"] + + @test onlytok("_1234") == K"Identifier" + + @test toks("1__2") == ["1"=>K"Integer", "__2"=>K"Identifier"] +end + +@testset "hex integer literals" begin + @test onlytok("0x0167_032") == K"HexInt" + @test onlytok("0x2_0_2") == K"HexInt" + # trailing junk + # https://github.com/JuliaLang/julia/issues/16356 + @test onlytok("0xenomorph") == K"ErrorInvalidNumericConstant" + @test onlytok("0xaα") == K"ErrorInvalidNumericConstant" + @test toks("0x ") == ["0x"=>K"ErrorInvalidNumericConstant", " "=>K"Whitespace"] + @test onlytok("0x") == K"ErrorInvalidNumericConstant" + @test onlytok("0xg") == K"ErrorInvalidNumericConstant" + @test onlytok("0x_") == K"ErrorInvalidNumericConstant" + @test toks("0x-") == ["0x"=>K"ErrorInvalidNumericConstant", "-"=>K"-"] +end + +@testset "hexfloat literals" begin + @test onlytok("0x.1p1") == K"Float" + @test onlytok("0x00p2") == K"Float" + @test onlytok("0x00P2") == K"Float" + @test onlytok("0x0.00p23") == K"Float" + @test onlytok("0x0.0ap23") == K"Float" + @test onlytok("0x0.0_0p2") == K"Float" + @test onlytok("0x0_0_0.0_0p2") == K"Float" + @test onlytok("0x0p+2") == K"Float" + @test onlytok("0x0p-2") == K"Float" + # errors + @test onlytok("0x") == K"ErrorInvalidNumericConstant" + @test onlytok("0x2__2") == K"ErrorInvalidNumericConstant" + @test onlytok("0x1p") == K"ErrorInvalidNumericConstant" + @test onlytok("0x.p0") == K"ErrorInvalidNumericConstant" + @test onlytok("0x.") == K"ErrorHexFloatMustContainP" + @test onlytok("0x1.0") == K"ErrorHexFloatMustContainP" + # https://github.com/JuliaLang/julia/issues/60189 + @test onlytok("0x1p3.") == K"ErrorInvalidNumericConstant" + @test onlytok("0x1p3.2") == K"ErrorInvalidNumericConstant" + @test onlytok("0x1.5p2.3") == K"ErrorInvalidNumericConstant" +end + +@testset "binary literals" begin + @test onlytok("0b0101001_0100_0101") == K"BinInt" + + @test onlytok("0b") == K"ErrorInvalidNumericConstant" + @test toks("0b ") == ["0b"=>K"ErrorInvalidNumericConstant", " "=>K"Whitespace"] + @test onlytok("0b101__101") == K"ErrorInvalidNumericConstant" + @test onlytok("0b123") == K"ErrorInvalidNumericConstant" +end + +@testset "octal literals" begin + @test onlytok("0o0167") == K"OctInt" + @test onlytok("0o01054001_0100_0101") == K"OctInt" + + @test onlytok("0o") == K"ErrorInvalidNumericConstant" + @test onlytok("0o78p") == K"ErrorInvalidNumericConstant" + @test toks("0o ") == ["0o"=>K"ErrorInvalidNumericConstant", " "=>K"Whitespace"] +end + +@testset "float literals" begin + @test onlytok("1.0") == K"Float" + + @test onlytok("1.0e0") == K"Float" + @test onlytok("1.0e-0") == K"Float" + @test onlytok("1.0E0") == K"Float" + @test onlytok("1.0E-0") == K"Float" + @test onlytok("1.0f0") == K"Float32" + @test onlytok("1.0f-0") == K"Float32" + @test onlytok("1.e0") == K"Float" + @test onlytok("1.f0") == K"Float32" + + @test onlytok("0e0") == K"Float" + @test onlytok("0e+0") == K"Float" + @test onlytok("0E0") == K"Float" + @test onlytok("201E+0") == K"Float" + @test onlytok("2f+0") == K"Float32" + @test onlytok("2048f0") == K"Float32" + + # underscores + @test onlytok("1_1.11") == K"Float" + @test onlytok("11.1_1") == K"Float" + @test onlytok("1_1.1_1") == K"Float" + @test onlytok("1.2_3") == K"Float" + @test onlytok("3_2.5_2") == K"Float" + @test toks("_1.1_1") == ["_1"=>K"Identifier", ".1_1"=>K"Float"] + + # juxtapositions with identifiers + @test toks("3e2_2") == ["3e2"=>K"Float", "_2"=>K"Identifier"] + @test toks("1e") == ["1"=>K"Integer", "e"=>K"Identifier"] + + # Floating point with \minus rather than - + @test onlytok("1.0e−0") == K"Float" + @test onlytok("1.0f−0") == K"Float32" + @test onlytok("0x0p−2") == K"Float" + + # Errors + @test onlytok("1._") == K"ErrorInvalidNumericConstant" + @test onlytok("1.1.") == K"ErrorInvalidNumericConstant" + @test onlytok("1e+") == K"ErrorInvalidNumericConstant" + @test onlytok("1.0e+") == K"ErrorInvalidNumericConstant" + @test onlytok("1.e1.") == K"ErrorInvalidNumericConstant" + @test onlytok("1e1.") == K"ErrorInvalidNumericConstant" + @test toks("1.e") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "e"=>K"Identifier"] + @test toks("3.2e2.2") == ["3.2e2."=>K"ErrorInvalidNumericConstant", "2"=>K"Integer"] + @test toks("3e2.2") == ["3e2."=>K"ErrorInvalidNumericConstant", "2"=>K"Integer"] + @test toks("1.2.f") == ["1.2."=>K"ErrorInvalidNumericConstant", "f"=>K"Identifier"] +end + +@testset "numbers with trailing `.` " begin + @test toks("1.") == ["1."=>K"Float"] + + @test toks("1.)") == ["1."=>K"Float", ")"=>K")"] + @test toks("1.]") == ["1."=>K"Float", "]"=>K"]"] + @test toks("1.}") == ["1."=>K"Float", "}"=>K"}"] + @test toks("1.,") == ["1."=>K"Float", ","=>K","] + @test toks("1.;") == ["1."=>K"Float", ";"=>K";"] + @test toks("1.#") == ["1."=>K"Float", "#"=>K"Comment"] + + # ellipses + @test toks("1..") == ["1"=>K"Integer", "."=>K".", "."=>K"."] + @test toks("1...") == ["1"=>K"Integer", "."=>K".", "."=>K".", "."=>K"."] + @test toks(".1..") == [".1"=>K"Float", "."=>K".", "."=>K"."] + @test toks("0x01..") == ["0x01"=>K"HexInt", "."=>K".", "."=>K"."] + @test toks("1e1..2") == ["1e1"=>K"Float", "."=>K".", "."=>K".", "2"=>K"Integer"] + @test toks("1.1e1..2") == ["1.1e1"=>K"Float", "."=>K".", "."=>K".", "2"=>K"Integer"] + @test toks("0x1p3..2") == ["0x1p3"=>K"Float", "."=>K".", "."=>K".", "2"=>K"Integer"] + + # Dotted operators and other dotted suffixes + @test toks("1234 .+1") == ["1234"=>K"Integer", " "=>K"Whitespace", "."=>K".", "+"=>K"+", "1"=>K"Integer"] + @test toks("1234.0+1") == ["1234.0"=>K"Float", "+"=>K"+", "1"=>K"Integer"] + @test toks("1234.0 .+1") == ["1234.0"=>K"Float", " "=>K"Whitespace", "."=>K".", "+"=>K"+", "1"=>K"Integer"] + @test toks("1234 .f(a)") == ["1234"=>K"Integer", " "=>K"Whitespace", "."=>K".", + "f"=>K"Identifier", "("=>K"(", "a"=>K"Identifier", ")"=>K")"] + @test toks("1234.0 .f(a)") == ["1234.0"=>K"Float", " "=>K"Whitespace", "."=>K".", + "f"=>K"Identifier", "("=>K"(", "a"=>K"Identifier", ")"=>K")"] + @test toks("1f0./1") == ["1f0"=>K"Float32", "."=>K".", "/"=>K"Operator", "1"=>K"Integer"] + + # Dotted operators after numeric constants are ok + @test toks("1e1.⫪") == ["1e1"=>K"Float", "."=>K".", "⫪"=>K"Operator"] + @test toks("1.1.⫪") == ["1.1"=>K"Float", "."=>K".", "⫪"=>K"Operator"] + @test toks("1e1.−") == ["1e1"=>K"Float", "."=>K".", "−"=>K"-"] + @test toks("1.1.−") == ["1.1"=>K"Float", "."=>K".", "−"=>K"-"] + @test toks("0x1p3.−") == ["0x1p3"=>K"Float", "."=>K".", "−"=>K"-"] + # ... including operators which have their own kind rather than being in + # the generic operator table + @test toks("1e1.∈") == ["1e1"=>K"Float", "."=>K".", "∈"=>K"∈"] + @test toks("1.1.∈") == ["1.1"=>K"Float", "."=>K".", "∈"=>K"∈"] + @test toks("1.1.√") == ["1.1"=>K"Float", "."=>K".", "√"=>K"√"] + # Non-dottable operators are not ok + @test toks("1e1.\$") == ["1e1."=>K"ErrorInvalidNumericConstant", "\$"=>K"$"] + @test toks("1.1.\$") == ["1.1."=>K"ErrorInvalidNumericConstant", "\$"=>K"$"] + + # Ambiguous dotted operators + @test toks("1.+") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+"] + @test toks("1.+ ") == ["1."=>K"ErrorAmbiguousNumericConstant", "+"=>K"+", " "=>K"Whitespace"] + @test toks("1.⤋") == ["1."=>K"ErrorAmbiguousNumericConstant", "⤋"=>K"Operator"] + @test toks("1.⫪") == ["1."=>K"ErrorAmbiguousNumericConstant", "⫪"=>K"Operator"] + @test toks("1.∈") == ["1."=>K"ErrorAmbiguousNumericConstant", "∈"=>K"∈"] + @test toks("1.√") == ["1."=>K"ErrorAmbiguousNumericConstant", "√"=>K"√"] + @test toks("1.≔") == ["1."=>K"ErrorAmbiguousNumericConstant", "≔"=>K"≔"] + # non-dottable ops are the exception + @test toks("1.:") == ["1."=>K"Float", ":"=>K":"] + @test toks("1.\$") == ["1."=>K"Float", "\$"=>K"$"] + + # Ambiguous - literal vs multiply by juxtaposition + @test toks("1.x") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "x"=>K"Identifier"] + @test toks("1.(") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "("=>K"("] + @test toks("1.[") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "["=>K"["] + @test toks("1.{") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "{"=>K"{"] + @test toks("1.@") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "@"=>K"@"] + @test toks("1.\"") == ["1."=>K"ErrorAmbiguousNumericDotMultiply", "\""=>K"\""] +end + +@testset "julia 0.6 types" begin + @test onlytok("mutable") == K"mutable" + @test onlytok("primitive") == K"primitive" + @test onlytok("struct") == K"struct" + @test onlytok("where") == K"where" + + @test tok("mutable struct s{T} where T", 1).kind == K"mutable" + @test tok("mutable struct s{T} where T", 3).kind == K"struct" + @test tok("mutable struct s{T} where T", 10).kind == K"where" +end + +@testset "CMDs" begin + @test tok("`cmd`",1).kind == K"`" + @test tok("`cmd`",2).kind == K"CmdString" + @test tok("`cmd`",3).kind == K"`" + @test tok("`cmd`",4).kind == K"EndMarker" + @test tok("```cmd```", 1).kind == K"```" + @test tok("```cmd```", 2).kind == K"CmdString" + @test tok("```cmd```", 3).kind == K"```" + @test tok("```cmd```", 4).kind == K"EndMarker" + @test tok("```cmd````cmd`", 1).kind == K"```" + @test tok("```cmd````cmd`", 2).kind == K"CmdString" + @test tok("```cmd````cmd`", 3).kind == K"```" + @test tok("```cmd````cmd`", 4).kind == K"`" + @test tok("```cmd````cmd`", 5).kind == K"CmdString" + @test tok("```cmd````cmd`", 6).kind == K"`" + @test tok("```cmd````cmd`", 7).kind == K"EndMarker" +end + +@testset "where" begin + @test tok("a where b", 3).kind == K"where" +end + +@testset "IO position" begin + io = IOBuffer("#1+1") + skip(io, 1) + @test length(collect(tokenize(io))) == 4 +end + +# Multi-character operators which share `K"Operator"` rather than having their +# own kind. Their precedence is assigned directly in the `lex_*` functions +# rather than via `generic_operators_by_level`, so they're listed here. +const _LEXER_MULTICHAR_OPERATORS = [ + "==", "===", "!=", "!==", "<=", ">=", + "<<", ">>", ">>>", "//", "|>", "<|", "=>", "<--", "<-->", +] + +# Operators which aren't symbolic infix/prefix operators (or aren't operators at +# all) and so can't be exercised by the dotted/suffixed forms below. +const _NON_SYMBOLIC_OPERATORS = Set([ + "ErrorInvalidOperator", "Error**", "Operator", + ".", "..", "where", "isa", "in", ".'", "op=", +]) + +# The non-dotted symbolic operators in the language. Most operators no longer +# have their own `Kind` (they share `K"Operator"`, distinguished by precedence +# flags), so rather than hard-coding the full list we derive it: single-character +# operators come from the precedence table `generic_operators_by_level`, the +# operators which still have a dedicated kind from the `BEGIN_OPS:END_OPS` range, +# and the multi-character `K"Operator"`s from `_LEXER_MULTICHAR_OPERATORS`. +function _all_symbolic_operators() + ops = String[] + # Normalize the chars from the precedence table the same way the lexer does, + # so eg the `·` variants collapse to `⋅` (matching how they're tokenized). + for chars in values(JuliaSyntax.generic_operators_by_level), c in chars + push!(ops, JuliaSyntax.normalize_identifier(string(c))) + end + op_range = reinterpret(UInt16, K"BEGIN_OPS"):reinterpret(UInt16, K"END_OPS") + for k in reinterpret.(Kind, op_range) + push!(ops, string(k)) + end + append!(ops, _LEXER_MULTICHAR_OPERATORS) + return unique!(filter(s -> !(s in _NON_SYMBOLIC_OPERATORS), ops)) +end + +@testset "dotted and suffixed operators" begin + +for op in _all_symbolic_operators() + strs = [ + 1 => [ # unary + "$(op)b", + ".$(op)b", + ], + 2 => [ # binary + "a $op b", + "a .$op b", + "a $(op)₁ b", + "a $(op)\U0304 b", + "a .$(op)₁ b" + ] + ] + + for (arity, container) in strs + for str in container + expr = JuliaSyntax.fl_parse(str, raise = false) + if VERSION < v"1.7" && str == "a .&& b" + expr = Expr(Symbol(".&&"), :a, :b) + end + if expr isa Expr && (expr.head != :error && expr.head != :incomplete) + tokens = collect(tokenize(str)) + exop = expr.head == :call ? expr.args[1] : expr.head + #println(str) + # For dotted operators, we need to reconstruct the operator from separate tokens + # Note: .. and ... are not dotted operators, they're regular operators + exop_str = string(exop) + is_dotted = occursin(".", exop_str) && exop != :.. && exop != :... + if is_dotted + # Dotted operators are now two tokens: . and the operator + dot_pos = arity == 1 ? 1 : 3 + op_pos = arity == 1 ? 2 : 4 + reconstructed_op = Symbol(Tokenize.untokenize(tokens[dot_pos], str) * + Tokenize.untokenize(tokens[op_pos], str)) + if reconstructed_op != exop + @info "" arity str exop reconstructed_op + end + @test reconstructed_op == exop + else + # Regular operators and suffixed operators + op_pos = arity == 1 ? 1 : 3 + if Symbol(Tokenize.untokenize(tokens[op_pos], str)) != exop + @info "" arity str exop op_pos + end + @test Symbol(Tokenize.untokenize(tokens[op_pos], str)) == exop + end + else + break + end + end + end +end +end + +@testset "Normalization of Unicode symbols" begin + # https://github.com/JuliaLang/julia/pull/25157 + @test tok("\u00b7").kind == K"Operator" + @test tok("\u0387").kind == K"Operator" + @test toks(".\u00b7") == ["."=>K".", "\u00b7"=>K"Operator"] + @test toks(".\u0387") == ["."=>K".", "\u0387"=>K"Operator"] + + # https://github.com/JuliaLang/julia/pull/40948 + @test tok("−").kind == K"-" + # −= now emits separate tokens + @test tok("−=").kind == K"Operator" # − before = + @test tok("−=", 2).kind == K"=" + @test toks(".−") == ["."=>K".", "−"=>K"-"] +end + +@testset "perp" begin + @test tok("1 ⟂ 2", 3).kind==K"Operator" +end + +@testset "outer" begin + @test tok("outer", 1).kind==K"outer" +end + +@testset "invalid operator errors" begin + @test toks("--") == ["--"=>K"ErrorInvalidOperator"] + @test toks("1**2") == ["1"=>K"Integer", "**"=>K"Error**", "2"=>K"Integer"] + @test toks("a<---b") == ["a"=>K"Identifier", "<---"=>K"ErrorInvalidOperator", "b"=>K"Identifier"] + # These used to test for invalid operators ..+ and ..−, but now .. is tokenized as two dots + @test toks("a..+b") == ["a"=>K"Identifier", "."=>K".", "."=>K".", "+"=>K"+", "b"=>K"Identifier"] + @test toks("a..−b") == ["a"=>K"Identifier", "."=>K".", "."=>K".", "−"=>K"-", "b"=>K"Identifier"] +end + +@testset "hat suffix" begin + @test tok("ŝ", 1).kind==K"Identifier" + @test untokenize(collect(tokenize("ŝ"))[1], "ŝ") == "ŝ" +end + +@testset "suffixed op" begin + s = "+¹" + @test is_operator(tok(s, 1).kind) + @test untokenize(collect(tokenize(s))[1], s) == s +end + +@testset "circ arrow right op" begin + s = "↻" + @test collect(tokenize(s))[1].kind == K"Operator" +end + +@testset "invalid float" begin + s = ".0." + @test collect(tokenize(s))[1].kind == K"ErrorInvalidNumericConstant" +end + +@testset "allow prime after end" begin + @test tok("begin end'", 4).kind === K"'" +end + +@testset "new ops" begin + ops = [ + raw"= += -= *= /= //= \= ^= ÷= %= <<= >>= >>>= |= &= ⊻= ≔ ⩴ ≕ ~ := $=" + raw"=>" + raw"?" + raw"← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⭄ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫ ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ -->" + raw"||" + raw"&&" + raw"> < >= ≥ <= ≤ == === ≡ != ≠ !== ≢ ∈ ∉ ∋ ∌ ⊆ ⊈ ⊂ ⊄ ⊊ ∝ ∊ ∍ ∥ ∦ ∷ ∺ ∻ ∽ ∾ ≁ ≃ ≂ ≄ ≅ ≆ ≇ ≈ ≉ ≊ ≋ ≌ ≍ ≎ ≐ ≑ ≒ ≓ ≖ ≗ ≘ ≙ ≚ ≛ ≜ ≝ ≞ ≟ ≣ ≦ ≧ ≨ ≩ ≪ ≫ ≬ ≭ ≮ ≯ ≰ ≱ ≲ ≳ ≴ ≵ ≶ ≷ ≸ ≹ ≺ ≻ ≼ ≽ ≾ ≿ ⊀ ⊁ ⊃ ⊅ ⊇ ⊉ ⊋ ⊏ ⊐ ⊑ ⊒ ⊜ ⊩ ⊬ ⊮ ⊰ ⊱ ⊲ ⊳ ⊴ ⊵ ⊶ ⊷ ⋍ ⋐ ⋑ ⋕ ⋖ ⋗ ⋘ ⋙ ⋚ ⋛ ⋜ ⋝ ⋞ ⋟ ⋠ ⋡ ⋢ ⋣ ⋤ ⋥ ⋦ ⋧ ⋨ ⋩ ⋪ ⋫ ⋬ ⋭ ⋲ ⋳ ⋴ ⋵ ⋶ ⋷ ⋸ ⋹ ⋺ ⋻ ⋼ ⋽ ⋾ ⋿ ⟈ ⟉ ⟒ ⦷ ⧀ ⧁ ⧡ ⧣ ⧤ ⧥ ⩦ ⩧ ⩪ ⩫ ⩬ ⩭ ⩮ ⩯ ⩰ ⩱ ⩲ ⩳ ⩵ ⩶ ⩷ ⩸ ⩹ ⩺ ⩻ ⩼ ⩽ ⩾ ⩿ ⪀ ⪁ ⪂ ⪃ ⪄ ⪅ ⪆ ⪇ ⪈ ⪉ ⪊ ⪋ ⪌ ⪍ ⪎ ⪏ ⪐ ⪑ ⪒ ⪓ ⪔ ⪕ ⪖ ⪗ ⪘ ⪙ ⪚ ⪛ ⪜ ⪝ ⪞ ⪟ ⪠ ⪡ ⪢ ⪣ ⪤ ⪥ ⪦ ⪧ ⪨ ⪩ ⪪ ⪫ ⪬ ⪭ ⪮ ⪯ ⪰ ⪱ ⪲ ⪳ ⪴ ⪵ ⪶ ⪷ ⪸ ⪹ ⪺ ⪻ ⪼ ⪽ ⪾ ⪿ ⫀ ⫁ ⫂ ⫃ ⫄ ⫅ ⫆ ⫇ ⫈ ⫉ ⫊ ⫋ ⫌ ⫍ ⫎ ⫏ ⫐ ⫑ ⫒ ⫓ ⫔ ⫕ ⫖ ⫗ ⫘ ⫙ ⫷ ⫸ ⫹ ⫺ ⊢ ⊣ ⟂ <: >:" + raw"<|" + raw"|>" + raw": .. … ⁝ ⋮ ⋱ ⋰ ⋯" + raw"$ + - | ⊕ ⊖ ⊞ ⊟ ++ ∪ ∨ ⊔ ± ∓ ∔ ∸ ≏ ⊎ ⊻ ⊽ ⋎ ⋓ ⧺ ⧻ ⨈ ⨢ ⨣ ⨤ ⨥ ⨦ ⨧ ⨨ ⨩ ⨪ ⨫ ⨬ ⨭ ⨮ ⨹ ⨺ ⩁ ⩂ ⩅ ⩊ ⩌ ⩏ ⩐ ⩒ ⩔ ⩖ ⩗ ⩛ ⩝ ⩡ ⩢ ⩣" + raw"* / ÷ % & ⋅ ∘ × \ ∩ ∧ ⊗ ⊘ ⊙ ⊚ ⊛ ⊠ ⊡ ⊓ ∗ ∙ ∤ ⅋ ≀ ⊼ ⋄ ⋆ ⋇ ⋉ ⋊ ⋋ ⋌ ⋏ ⋒ ⟑ ⦸ ⦼ ⦾ ⦿ ⧶ ⧷ ⨇ ⨰ ⨱ ⨲ ⨳ ⨴ ⨵ ⨶ ⨷ ⨸ ⨻ ⨼ ⨽ ⩀ ⩃ ⩄ ⩋ ⩍ ⩎ ⩑ ⩓ ⩕ ⩘ ⩚ ⩜ ⩞ ⩟ ⩠ ⫛ ⊍ ▷ ⨝ ⟕ ⟖ ⟗" + raw"//" + raw"<< >> >>>" + raw"^ ↑ ↓ ⇵ ⟰ ⟱ ⤈ ⤉ ⤊ ⤋ ⤒ ⤓ ⥉ ⥌ ⥍ ⥏ ⥑ ⥔ ⥕ ⥘ ⥙ ⥜ ⥝ ⥠ ⥡ ⥣ ⥥ ⥮ ⥯ ￪ ￬" + raw"::" + raw"." + ] + if VERSION >= v"1.6.0" + push!(ops, raw"<-- <--> ¦ ⌿") + end + if VERSION >= v"1.7.0" + append!(ops, [ + "−" + "\u00b7 \u0387" + "⫪ ⫫" + ]) + end + if VERSION >= v"1.10-DEV" + push!(ops, "⥷ ⥺ ⟇") + end + allops = split(join(ops, " "), " ") + @test all(s->Base.isoperator(Symbol(s)) == is_operator(first(collect(tokenize(s))).kind), allops) + + # "\U1f8b2" added in Julia 1.12 + @test is_operator(first(collect(tokenize("🢲")))) +end + +const all_kws = Set([ + # Keywords + "baremodule", + "begin", + "break", + "catch", + "const", + "continue", + "do", + "else", + "elseif", + "end", + "export", + "finally", + "for", + "function", + "global", + "if", + "import", + "let", + "local", + "macro", + "module", + "public", + "quote", + "return", + "struct", + "try", + "typegroup", + "using", + "while", + # Contextual keywords + "abstract", + "as", + "doc", + "mutable", + "outer", + "primitive", + "type", + "var", + "VERSION", + # Word-like operators + "in", + "isa", + "where", +]) + +function check_kw_hashes(iter) + for cs in iter + str = String([cs...]) + if Tokenize.simple_hash(str) in keys(Tokenize._kw_hash) + @test str in all_kws + end + end +end + +@testset "simple_hash" begin + @test length(all_kws) == length(Tokenize._kw_hash) + + @testset "Length $len keywords" for len in 1:5 + check_kw_hashes(String([cs...]) for cs in Iterators.product(['a':'z' for _ in 1:len]...)) + end +end + + +@testset "UTF-8 BOM" begin + @test kind.(collect(tokenize("\ufeff[1\ufeff2]"))) == [ + K"Whitespace", + K"[", + K"Integer", + K"Whitespace", + K"Integer", + K"]", + K"EndMarker" + ] +end + +@testset "lexer initialization" begin + # Ranges of EndMarker + @test (t = last(collect(tokenize("+"))); (t.startbyte, t.endbyte)) == (1,0) + @test (t = last(collect(tokenize("+*"))); (t.startbyte, t.endbyte)) == (2,1) +end + +@testset "invalid UTF-8 characters" begin + @test onlytok("\x00") == K"ErrorUnknownCharacter" + @test onlytok("₁") == K"ErrorIdentifierStart" + + bad_chars = [ + first("\xe2") # malformed + first("\xc0\x9b") # overlong + first("\xf0\x83\x99\xae") # overlong + ] + + @testset "bad char $(repr(c))" for c in bad_chars + @test Tokenize.is_identifier_char(c) == false + @test Tokenize.is_identifier_start_char(c) == false + @test Tokenize.is_never_id_char(c) == true + @test Tokenize.is_dottable_operator_start_char(c) == false + @test Tokenize.isopsuffix(c) == false + @test Tokenize.is_operator_start_char(c) == false + @test Tokenize.iswhitespace(c) == false + @test Tokenize.ishex(c) == false + end +end + +@testset "unbalanced bidirectional unicode" begin + open_embedding = ['\U202A', '\U202B', '\U202D', '\U202E'] + close_embedding = '\U202C' + open_isolate = ['\U2066', '\U2067', '\U2068'] + close_isolate = '\U2069' + close_all = '\n' + + all_bidi_codes = [open_embedding; close_embedding; open_isolate; close_isolate] + + bidi_pairs = [Iterators.product(open_embedding, [close_embedding, close_all])..., + Iterators.product(open_isolate, [close_isolate, close_all])...] + + @testset "delimiter $kd" for (kd, chunk_kind) in [ + (K"\"", K"String"), + (K"\"\"\"", K"String"), + (K"`", K"CmdString"), + (K"```", K"CmdString") + ] + d = string(kd) + @testset "Single unbalanced codes" begin + for c in all_bidi_codes + @test toks("$d$c$d") == + [d=>kd, "$c"=>K"ErrorBidiFormatting", d=>kd] + @test toks("pfx$d$c$d") == + ["pfx"=>K"Identifier", d=>kd, "$c"=>K"ErrorBidiFormatting", d=>kd] + end + end + @testset "Balanced pairs" begin + for (openc, closec) in bidi_pairs + str = "$(openc)##$(closec)" + @test toks("$d$str$d") == + [d=>kd, str=>chunk_kind, d=>kd] + @test toks("pfx$d$str$d") == + ["pfx"=>K"Identifier", d=>kd, str=>chunk_kind, d=>kd] + end + end + end + + @testset "multi line comments" begin + @testset "Single unbalanced codes" begin + for c in all_bidi_codes + comment = "#=$c=#" + @test toks(comment) == [comment=>K"ErrorBidiFormatting"] + end + end + @testset "Balanced pairs" begin + for (openc, closec) in bidi_pairs + str = "#=$(openc)zz$(closec)=#" + @test toks(str) == [str=>K"Comment"] + end + end + end + + @testset "extended balanced/unbalanced bidi state" begin + @testset "delimiter $kd" for (kd, chunk_kind) in [ + (K"\"", K"String"), + (K"\"\"\"", K"String"), + (K"`", K"CmdString"), + (K"```", K"CmdString") + ] + d = string(kd) + for balanced in [# Balanced pairs + "\u202a\u202bzz\u202c\u202c" + "\u2066\u2067zz\u2069\u2069" + # Newline is complete bidi state reset + "\u202a\u2067zz\n" + "\u202a\u202azz\n" + # \r\n and \n terminate a line + "\u202azz\r\n" + ] + @test toks("$d$balanced$d") == [ + d=>kd + balanced=>chunk_kind + d=>kd + ] + end + for unbalanced in ["\u202azz\u202c\u202c" + "\u202a\u202bzz\u202c" + # \r does not terminate a bidi line + "\u202azz\r" + ] + @test toks("$d$unbalanced$d") == [ + d=>kd + unbalanced=>K"ErrorBidiFormatting" + d=>kd + ] + end + end + end + + # Interpolations reset bidi state + @test toks("\"\u202a\$zz\n\"") == [ + "\""=>K"\"" + "\u202a"=>K"ErrorBidiFormatting" + "\$"=>K"$" + "zz"=>K"Identifier" + "\n"=>K"String" + "\""=>K"\"" + ] + @testset "newline escaping" begin + @test toks("\"a\u202a\\\n\"") == [ + "\""=>K"\"" + "a\u202a"=>K"String" + "\\\n"=>K"Whitespace" + "\""=>K"\"" + ] + @test toks("\"a\u202a\\\r\n\"") == [ + "\""=>K"\"" + "a\u202a"=>K"String" + "\\\r\n"=>K"Whitespace" + "\""=>K"\"" + ] + @test toks("\"a\u202a\\\r\"") == [ + "\""=>K"\"" + "a\u202a"=>K"ErrorBidiFormatting" + "\\\r"=>K"Whitespace" + "\""=>K"\"" + ] + end + + @testset "delimiter '" begin + for c in all_bidi_codes + @test toks("'$c'") == ["'"=>K"'", "$c"=>K"Char", "'"=>K"'"] + end + end +end + +@testset "dotop miscellanea" begin + @test strtok("a .-> b") == ["a", " ", ".", "-", ">", " ", "b", ""] + @test strtok(".>: b") == [".", ">:", " ", "b", ""] + @test strtok(".<: b") == [".", "<:", " ", "b", ""] + @test strtok("a ||₁ b") == ["a", " ", "||", "₁", " ", "b", ""] + @test strtok("a ||̄ b") == ["a", " ", "||", "̄", " ", "b", ""] + @test strtok("a .||₁ b") == ["a", " ", ".", "||", "₁", " ", "b", ""] + @test strtok("a &&₁ b") == ["a", " ", "&&", "₁", " ", "b", ""] + @test strtok("a &&̄ b") == ["a", " ", "&&", "̄", " ", "b", ""] + @test strtok("a .&&₁ b") == ["a", " ", ".", "&&", "₁", " ", "b", ""] +end + +end diff --git a/JuliaSyntax/test/utils.jl b/JuliaSyntax/test/utils.jl new file mode 100644 index 0000000000000..371da98c9e174 --- /dev/null +++ b/JuliaSyntax/test/utils.jl @@ -0,0 +1,24 @@ +@testset "_printstyled" begin + ps(str; kws...) = sprint(io->JuliaSyntax._printstyled(IOContext(io, :color=>true), str; kws...)) + + @test ps("XX"; fgcolor=:red) == "\e[31mXX\e[0;0m" + @test ps("XX"; fgcolor=42) == "\e[38;5;42mXX\e[0;0m" + @test ps("XX"; fgcolor=(10,100,200)) == "\e[38;2;10;100;200mXX\e[0;0m" + + ps("XX"; bgcolor=:red) == "\e[41mXX\e[0;0m" + @test ps("XX"; bgcolor=42) == "\e[48;5;42mXX\e[0;0m" + @test ps("XX"; bgcolor=(10,100,200)) == "\e[48;2;10;100;200mXX\e[0;0m" + + @test ps("XX"; href="https://www.example.com") == + "\e]8;;https://www.example.com\e\\XX\e[0;0m\e]8;;\e\\" + + @test ps("XX", fgcolor=:red, bgcolor=:green, href="https://www.example.com") == + "\e]8;;https://www.example.com\e\\\e[31m\e[42mXX\e[0;0m\e]8;;\e\\" +end + +@testset "ambiguities" begin + if VERSION >= v"1.8" + @test detect_ambiguities(JuliaSyntax) == [] + @test detect_unbound_args(JuliaSyntax) == [] + end +end diff --git a/JuliaSyntax/tools/bump_in_Base.jl b/JuliaSyntax/tools/bump_in_Base.jl new file mode 100644 index 0000000000000..aec2876deb645 --- /dev/null +++ b/JuliaSyntax/tools/bump_in_Base.jl @@ -0,0 +1,74 @@ +function find_checksum_files(checksum_dir) + filter(readdir(checksum_dir, join=true)) do path + occursin(r"^JuliaSyntax-", basename(path)) + end +end + +function bump_in_Base(julia_dir, juliasyntax_dir, juliasyntax_branch_or_commit) + julia_git_dir = joinpath(julia_dir, ".git") + JuliaSyntax_git_dir = joinpath(juliasyntax_dir, ".git") + if !isdir(julia_git_dir) + @error "Julia .git directory not found" julia_git_dir + return 1 + end + if !isdir(JuliaSyntax_git_dir) + @error "JuliaSyntax .git directory not found" JuliaSyntax_git_dir + return 1 + end + + @info "Vendoring JuliaSyntax into Base" julia_dir juliasyntax_branch_or_commit + + remote_containing_branches = filter(b->occursin(r"^origin/(main|release-.*)$", b), + strip.(split( + read(`git --git-dir=$JuliaSyntax_git_dir branch -r --contains $juliasyntax_branch_or_commit`, String), + '\n', keepempty=false))) + if isempty(remote_containing_branches) + @warn "No remote main or release branches contain the given commit. This is ok for testing, but is otherwise an error." juliasyntax_branch_or_commit + else + @info "Given commit is accessible on remote branch" remote_containing_branches + end + + commit_sha = strip(String(read(`git --git-dir=$JuliaSyntax_git_dir show -s --pretty=tformat:%H $juliasyntax_branch_or_commit`))) + + cd(julia_dir) do + status = read(`git status --porcelain --untracked-files=no`, String) + if status != "" + @error "Julia git directory contains uncommitted changes" status=Text(status) + return 1 + end + + verfile_path = joinpath("deps", "JuliaSyntax.version") + @info "Updating JuliaSyntax.version" verfile_path + write(verfile_path, replace(read(verfile_path, String), r"JULIASYNTAX_SHA1.*"=>"JULIASYNTAX_SHA1 = "*commit_sha)) + run(`git add $verfile_path`) + + @info "Updating JuliaSyntax checksums" + deps_dir = "deps" + checksum_dir = joinpath(deps_dir, "checksums") + old_checksum_paths = find_checksum_files(checksum_dir) + if !isempty(old_checksum_paths) + run(`git rm -rf $old_checksum_paths`) + end + run(`make -C $deps_dir`) + run(`git add $(find_checksum_files(checksum_dir))`) + + # Force rebuild of Base to include the newly vendored JuliaSyntax next time Julia is built. + # (TODO: fix the Makefile instead?) + touch("base/Base.jl") + + @info "JuliaSyntax version updated. You can now test or commit the following changes" + run(`git diff --cached`) + end + + return 0 +end + +if !isinteractive() + if length(ARGS) != 2 + println("Usage: bump_in_Base.jl \$julia_dir \$juliasyntax_branch_or_commit") + exit(1) + else + juliasyntax_dir = dirname(@__DIR__) + exit(bump_in_Base(ARGS[1], juliasyntax_dir, ARGS[2])) + end +end diff --git a/JuliaSyntax/tools/check_all_packages.jl b/JuliaSyntax/tools/check_all_packages.jl new file mode 100644 index 0000000000000..32f255e0cb6ea --- /dev/null +++ b/JuliaSyntax/tools/check_all_packages.jl @@ -0,0 +1,99 @@ +# hacky script to parse all Julia files in all packages in General +# to Exprs and report errors +# +# Run this after registry_download.jl (so the pkgs directory is populated). + +using JuliaSyntax, Logging, TerminalLoggers, ProgressLogging, Serialization + +include("../test/test_utils.jl") +include("../test/fuzz_test.jl") + +srcpaths = isempty(ARGS) ? [joinpath(@__DIR__, "pkgs")] : abspath.(ARGS) +source_paths = vcat(find_source_in_path.(srcpaths)...) + +file_count = length(source_paths) + +exception_count = 0 +mismatch_count = 0 +t0 = time() +exceptions = [] + +all_reduced_failures = String[] + +Logging.with_logger(TerminalLogger()) do + global exception_count, mismatch_count, t0 + @withprogress for (ifile, fpath) in enumerate(source_paths) + @logprogress ifile/file_count time_ms=round((time() - t0)/ifile*1000, digits = 2) + text = read(fpath, String) + expr_cache = fpath*".Expr" + e2 = if isfile(expr_cache) + open(deserialize, fpath*".Expr") + else + @warn "Expr cache not found, parsing using reference parser" expr_cache maxlog=1 + JuliaSyntax.fl_parseall(text, filename=fpath) + end + @assert Meta.isexpr(e2, :toplevel) + try + e1 = JuliaSyntax.parseall(Expr, text, filename=fpath, ignore_warnings=true) + if !exprs_roughly_equal(e2, e1) + mismatch_count += 1 + failing_source = sprint(context=:color=>true) do io + for c in reduce_tree(parseall(SyntaxNode, text)) + JuliaSyntax.highlight(io, c.source, JuliaSyntax.byte_range(c), context_lines_inner=5) + println(io, "\n") + end + end + reduced_failures = reduce_text.(reduce_tree(text), + parsers_fuzzy_disagree) + append!(all_reduced_failures, reduced_failures) + @error("Parsers succeed but disagree", + fpath, + failing_source=Text(failing_source), + reduced_failures, + ) + end + catch err + err isa InterruptException && rethrow() + ex = (err, catch_backtrace()) + push!(exceptions, ex) + ref_parse = "success" + if length(e2.args) >= 1 && Meta.isexpr(last(e2.args), (:error, :incomplete)) + ref_parse = "fail" + if err isa JuliaSyntax.ParseError + # Both parsers agree that there's an error, and + # JuliaSyntax didn't have an internal error. + continue + end + end + + exception_count += 1 + parse_to_syntax = "success" + try + JuliaSyntax.parseall(JuliaSyntax.SyntaxNode, code) + catch err2 + parse_to_syntax = "fail" + end + @error "Parse failed" fpath exception=ex parse_to_syntax + end + end +end + +t_avg = round((time() - t0)/file_count*1000, digits = 2) + +println() +@info """ + Finished parsing $file_count files. + $(exception_count) failures compared to reference parser + $(mismatch_count) Expr mismatches + $(t_avg)ms per file""" + +open(joinpath(@__DIR__, "reduced_failures.jl"), write=true) do io + for str in all_reduced_failures + println(io, repr(str)) + end + for str in all_reduced_failures + println(io, "#------------------------------") + println(io, str) + println(io) + end +end diff --git a/JuliaSyntax/tools/registry_download.jl b/JuliaSyntax/tools/registry_download.jl new file mode 100644 index 0000000000000..e866a6ee72a94 --- /dev/null +++ b/JuliaSyntax/tools/registry_download.jl @@ -0,0 +1,46 @@ +# Hacky script to download the latest version of all packages registered in the +# General registry for testing the parser. +# +# This uses internal Pkg APIs and seems to work on Julia 1.7 + +using Pkg +using Downloads + +registry = only(filter(r->r.name == "General", Pkg.Registry.reachable_registries())) + +packages = [] + +for (uuid,pkg) in registry + versions = collect(Pkg.Registry.registry_info(pkg).version_info) + latest_ver, ver_info = last(sort(versions, by=first)) + if ver_info.yanked + continue + end + + push!(packages, (; uuid, pkg.name, version=latest_ver, ver_info.git_tree_sha1)) + +end + +server = Pkg.pkg_server() +output_dir = "pkgs" +mkpath(output_dir) + +asyncmap(packages, ntasks=5) do pkg + url = "$server/package/$(pkg.uuid)/$(pkg.git_tree_sha1)" + outfile_path = joinpath(output_dir, "$(pkg.name)_$(pkg.version).tgz") + if isfile(outfile_path) + @info "Skipping package" pkg + return outfile_path + else + @info "Download package" url outfile_path + for i=1:5 + try + Downloads.download(url, outfile_path) + break + catch + @error "Error downloading" pkg exception=current_exceptions() + end + sleep(i) + end + end +end diff --git a/JuliaSyntax/tools/untar_packages.jl b/JuliaSyntax/tools/untar_packages.jl new file mode 100644 index 0000000000000..2c6986890bb63 --- /dev/null +++ b/JuliaSyntax/tools/untar_packages.jl @@ -0,0 +1,68 @@ +using Serialization +using JuliaSyntax + +pkgspath = joinpath(@__DIR__, "pkgs") +tarspath = joinpath(@__DIR__, "pkg_tars") + +mkpath(pkgspath) +mkpath(tarspath) + +tar_info = [(m = match(r"(.*)_(\d+\.\d+\.\d+.*)\.tgz$", f); (f, m[1], VersionNumber(m[2]))) + for f in readdir(tarspath) if endswith(f, ".tgz")] + +tar_maxver = Dict{String,VersionNumber}() +for (_,name,ver) in tar_info + v = get(tar_maxver, name, v"0.0.0") + if v < ver + tar_maxver[name] = ver + end +end + +@info "# Untarring packages" + +for tinfos in Iterators.partition(tar_info, 50) + @sync for (tarname, pkgname, pkgver) in tinfos + @async begin + dir = joinpath(pkgspath, "$(pkgname)_$(pkgver)") + if pkgver != tar_maxver[pkgname] + if isdir(dir) + # Clean up old packages + rm(dir; recursive=true, force=true) + end + elseif !isdir(dir) || !isdir(joinpath(dir, "src")) + rm(dir; recursive=true, force=true) + mkpath(dir) + tar_path = joinpath(tarspath, tarname) + try + run(`tar -xf $tar_path -C $dir`) + catch err + @error "could not untar $tar_path" + end + end + end + end +end + +@info "# Parsing files with reference parser" + +let i = 0, tot_files = 0 + for (r, _, files) in walkdir(pkgspath) + for f in files + tot_files += 1 + endswith(f, ".jl") || continue + fpath = joinpath(r, f) + outpath = joinpath(r, f*".Expr") + if !islink(fpath) && isfile(fpath) && !isfile(outpath) + code = read(fpath, String) + fl_ex = JuliaSyntax.fl_parseall(code, filename=fpath) + i += 1 + if i % 100 == 0 + @info "$i/$tot_files files parsed" + end + open(outpath, "w") do io + serialize(io, fl_ex) + end + end + end + end +end diff --git a/Make.inc b/Make.inc index 049d0bebe05de..38a18cf7ab365 100644 --- a/Make.inc +++ b/Make.inc @@ -14,6 +14,17 @@ # Set to zero to turn off extra precompile (e.g. for the REPL) JULIA_PRECOMPILE ?= 1 +# Set to 1 to compile coverage instrumentation into the sysimage and the +# stdlib pkgimages. The resulting images collect coverage without recompiling +# any image code when run with `--code-coverage` in the default `hit` mode, +# whatever the scope, but carry the instrumentation overhead in every use. +# Intended for coverage CI, not for released builds. +JULIA_COVERAGE_IMAGES ?= 0 +# The generating processes instrument only the images they compile, and the +# pkgimage workers inherit the setting through `Base.CacheFlags`. Recursively +# expanded because Make.user is read below. +JULIA_COVERAGE_IMAGE_FLAGS = $(if $(filter 1,$(JULIA_COVERAGE_IMAGES)),--code-coverage=all) + # Set FORCE_ASSERTIONS to 1 to enable assertions in the C and C++ portions # of the Julia code base. You may also want to set LLVM_ASSERTIONS to 1, # which will enable assertions in LLVM. @@ -60,9 +71,14 @@ USE_SYSTEM_PATCHELF:=0 USE_SYSTEM_LIBWHICH:=0 USE_SYSTEM_ZLIB:=0 USE_SYSTEM_ZSTD:=0 +USE_SYSTEM_LMDB:=0 USE_SYSTEM_P7ZIP:=0 USE_SYSTEM_LLD:=0 +# Link libjulia-internal with static libgcc and libstdc++ +USE_RT_STATIC_LIBGCC:=1 +USE_RT_STATIC_LIBSTDCXX:=1 + # Link to the LLVM shared library USE_LLVM_SHLIB := 1 @@ -94,9 +110,15 @@ WITH_NVTX := 0 WITH_TRACY := 0 WITH_TRACY_CALLSTACKS := 0 +# Enable Apple Instruments support +WITH_APPLE_OSLOG := 0 + # Enable Timing Counts support WITH_TIMING_COUNTS := 0 +# Should --gc-sections/-dead_strip be used to remove unreferenced code? +USE_LINKER_GC:=1 + # Prevent picking up $ARCH from the environment variables ARCH:= @@ -553,26 +575,34 @@ MACOSX_VERSION_MIN := 11.0 endif endif -JCFLAGS_COMMON := -std=gnu11 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 -Wformat -Wformat-security -JCFLAGS_CLANG := $(JCFLAGS_COMMON) -JCFLAGS_GCC := $(JCFLAGS_COMMON) -fno-gnu-unique +# These are lazy expansion variables, so that arguments can be added to them later and they'll affect all of these uses too +JCFLAGS_COMMON = -std=gnu11 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 -Wformat -Wformat-security +JCFLAGS_CLANG = $(JCFLAGS_COMMON) +JCFLAGS_GCC = $(JCFLAGS_COMMON) -fno-gnu-unique + +# Xcode 26 (clang 21) folds the new -Wc++-keyword diagnostic into -Wc++-compat, erroring +# under -Werror on uses of C++ keywords like `wchar_t` as ordinary identifiers in C code. +# Suppress it; -Wno-unknown-warning-option keeps older clang from rejecting the flag. +JCFLAGS_CLANG += -Wno-unknown-warning-option -Wno-c++-keyword + # These flags are needed to generate decent debug info -JCPPFLAGS_COMMON := -fasynchronous-unwind-tables -JCPPFLAGS_CLANG := $(JCPPFLAGS_COMMON) -mllvm -enable-tail-merge=0 -JCPPFLAGS_GCC := $(JCPPFLAGS_COMMON) -fno-tree-tail-merge +JCPPFLAGS_COMMON = -fasynchronous-unwind-tables +JCPPFLAGS_CLANG = $(JCPPFLAGS_COMMON) -mllvm -enable-tail-merge=0 +JCPPFLAGS_GCC = $(JCPPFLAGS_COMMON) -fno-tree-tail-merge + +JCXXFLAGS_COMMON = -pipe $(fPIC) -fno-rtti -std=c++17 -Wformat -Wformat-security -fno-strict-aliasing +JCXXFLAGS_CLANG = $(JCXXFLAGS_COMMON) -pedantic +JCXXFLAGS_GCC = $(JCXXFLAGS_COMMON) -fno-gnu-unique -JCXXFLAGS_COMMON := -pipe $(fPIC) -fno-rtti -std=c++17 -Wformat -Wformat-security -fno-strict-aliasing -JCXXFLAGS_CLANG := $(JCXXFLAGS_COMMON) -pedantic -JCXXFLAGS_GCC := $(JCXXFLAGS_COMMON) -fno-gnu-unique +DEBUGFLAGS_COMMON = -O0 -DJL_DEBUG_BUILD -fstack-protector +DEBUGFLAGS_CLANG = $(DEBUGFLAGS_COMMON) -g +DEBUGFLAGS_GCC = $(DEBUGFLAGS_COMMON) -ggdb2 -DEBUGFLAGS_COMMON := -O0 -DJL_DEBUG_BUILD -fstack-protector -DEBUGFLAGS_CLANG := $(DEBUGFLAGS_COMMON) -g -DEBUGFLAGS_GCC := $(DEBUGFLAGS_COMMON) -ggdb2 +SHIPFLAGS_COMMON = -O3 +SHIPFLAGS_CLANG = $(SHIPFLAGS_COMMON) -g +SHIPFLAGS_GCC = $(SHIPFLAGS_COMMON) -ggdb2 -falign-functions -SHIPFLAGS_COMMON := -O3 -SHIPFLAGS_CLANG := $(SHIPFLAGS_COMMON) -g -SHIPFLAGS_GCC := $(SHIPFLAGS_COMMON) -ggdb2 -falign-functions BOLT_LDFLAGS := @@ -591,22 +621,22 @@ endif ifeq ($(USEGCC),1) CC := $(CROSS_COMPILE)gcc CXX := $(CROSS_COMPILE)g++ -JCFLAGS := $(JCFLAGS_GCC) -JCPPFLAGS := $(JCPPFLAGS_GCC) -JCXXFLAGS := $(JCXXFLAGS_GCC) -DEBUGFLAGS := $(DEBUGFLAGS_GCC) -SHIPFLAGS := $(SHIPFLAGS_GCC) $(BOLT_CFLAGS_GCC) +JCFLAGS = $(JCFLAGS_GCC) +JCPPFLAGS = $(JCPPFLAGS_GCC) +JCXXFLAGS = $(JCXXFLAGS_GCC) +DEBUGFLAGS = $(DEBUGFLAGS_GCC) +SHIPFLAGS = $(SHIPFLAGS_GCC) $(BOLT_CFLAGS_GCC) BOLT_CFLAGS := $(BOLT_CFLAGS_GCC) endif ifeq ($(USECLANG),1) CC := $(CROSS_COMPILE)clang CXX := $(CROSS_COMPILE)clang++ -JCFLAGS := $(JCFLAGS_CLANG) -JCPPFLAGS := $(JCPPFLAGS_CLANG) -JCXXFLAGS := $(JCXXFLAGS_CLANG) -DEBUGFLAGS := $(DEBUGFLAGS_CLANG) -SHIPFLAGS := $(SHIPFLAGS_CLANG) $(BOLT_CFLAGS_CLANG) +JCFLAGS = $(JCFLAGS_CLANG) +JCPPFLAGS = $(JCPPFLAGS_CLANG) +JCXXFLAGS = $(JCXXFLAGS_CLANG) +DEBUGFLAGS = $(DEBUGFLAGS_CLANG) +SHIPFLAGS = $(SHIPFLAGS_CLANG) $(BOLT_CFLAGS_CLANG) BOLT_CFLAGS := $(BOLT_CFLAGS_CLANG) ifeq ($(OS), Darwin) @@ -629,6 +659,14 @@ else endif JLDFLAGS := $(RELRO_FLAG) +# Flags passed to Julia's link rules; Windows invokes the linker directly. +LINK_LDFLAGS = $(LDFLAGS) +JL_VISIBILITY_FLAGS := -fvisibility=hidden +ifeq ($(OS),WINNT) +# PE exports are selected by the linker. Clang turns hidden visibility into +# exclusion directives that override even the export map's global patterns. +JL_VISIBILITY_FLAGS := +endif ifeq ($(USECCACHE), 1) # Expand CC, CXX and FC here already because we want the original definition and not the ccache version. @@ -712,7 +750,7 @@ endif ifeq ($(OS),WINNT) define IMPLIB_FLAGS - -Wl,--out-implib,$(build_libdir)/$(notdir $1).a + --out-implib $(build_libdir)/$(notdir $1).a endef else define IMPLIB_FLAGS @@ -815,7 +853,7 @@ SANITIZE_OPTS += -fsanitize=address SANITIZE_LDFLAGS += -fsanitize=address -shared-libasan endif ifeq ($(SANITIZE_THREAD),1) -SANITIZE_OPTS += -fsanitize=thread +SANITIZE_OPTS += -fsanitize=thread -fsanitize-ignorelist=$(JULIAHOME)/contrib/tsan/ignorelist.txt SANITIZE_LDFLAGS += -fsanitize=thread ifneq ($(CROSS_BOOTSTRAP_JULIA),) bootstrap_julia_flags += --target-sanitize=thread @@ -866,28 +904,35 @@ USE_THIRD_PARTY_GC := $(shell echo $(WITH_THIRD_PARTY_GC) | tr A-Z a-z) # to the variables JCFLAGS and JCXXFLAGS according to the mapping (MMTK=1, #NEW_GC=2) ifeq (${USE_THIRD_PARTY_GC},mmtk) -JCXXFLAGS += -DWITH_THIRD_PARTY_HEAP=1 -JCFLAGS += -DWITH_THIRD_PARTY_HEAP=1 +JCXXFLAGS += -DWITH_THIRD_PARTY_HEAP=1 -DGC_BARRIER_ON_TASKS +JCFLAGS += -DWITH_THIRD_PARTY_HEAP=1 -DGC_BARRIER_ON_TASKS + +# Use StickyImmix as the default MMTk plan. +MMTK_PLAN ?= StickyImmix -# Must specify a supported MMTk Plan: Immix or StickyImmix +# Must specify a supported MMTk plan: Immix, StickyImmix, or ConcurrentImmix ifeq (${MMTK_PLAN},Immix) -JCXXFLAGS += -DMMTK_PLAN_IMMIX -JCFLAGS += -DMMTK_PLAN_IMMIX +JCXXFLAGS += -DMMTK_PLAN_IMMIX -DMMTK_PLAN=\"Immix\" +JCFLAGS += -DMMTK_PLAN_IMMIX -DMMTK_PLAN=\"Immix\" else ifeq (${MMTK_PLAN},StickyImmix) -JCXXFLAGS += -DMMTK_PLAN_STICKYIMMIX -JCFLAGS += -DMMTK_PLAN_STICKYIMMIX +JCXXFLAGS += -DMMTK_PLAN_STICKYIMMIX -DMMTK_PLAN=\"StickyImmix\" +JCFLAGS += -DMMTK_PLAN_STICKYIMMIX -DMMTK_PLAN=\"StickyImmix\" +else ifeq (${MMTK_PLAN}, ConcurrentImmix) +JCXXFLAGS += -DMMTK_PLAN_CONCURRENTIMMIX -DMMTK_PLAN=\"ConcurrentImmix\" -DGC_BARRIER_SNAPSHOT +JCFLAGS += -DMMTK_PLAN_CONCURRENTIMMIX -DMMTK_PLAN=\"ConcurrentImmix\" -DGC_BARRIER_SNAPSHOT else -$(error "Unsupported MMTk plan: $(MMTK_PLAN). Supported plan(s): Immix or StickyImmix.") +$(error "Unsupported MMTk plan: $(MMTK_PLAN). Supported plan(s): Immix, StickyImmix, or ConcurrentImmix.") endif -# Do a release build on the binding by default -MMTK_BUILD ?= release +# Julia's MMTk support does not support moving just yet +MMTK_MOVING ?= 0 +JCXXFLAGS += -DMMTK_MOVING=$(MMTK_MOVING) +JCFLAGS += -DMMTK_MOVING=$(MMTK_MOVING) -# Location of mmtk-julia binding -# (needed for api/*.h and .so file) -MMTK_JULIA_DIR ?= $(BUILDROOT)/usr/lib/mmtk_julia -MMTK_DIR = ${MMTK_JULIA_DIR}/mmtk -MMTK_API_INC = ${MMTK_DIR}/api +# Location of the in-tree MMTk integration. +MMTK_DIR := $(JULIAHOME)/src/gc-mmtk +MMTK_JULIA_DIR := $(MMTK_DIR)/mmtk_julia +MMTK_API_INC = ${MMTK_JULIA_DIR}/api MMTK_LIB := -lmmtk_julia # Must specify a supported third-party GC or use the stock GC. @@ -909,8 +954,8 @@ LIBITTAPI:=-littnotify endif ifeq ($(WITH_TRACY), 1) -JCXXFLAGS += -DUSE_TRACY -DTRACY_ENABLE -DTRACY_FIBERS -JCFLAGS += -DUSE_TRACY -DTRACY_ENABLE -DTRACY_FIBERS +JCXXFLAGS += -DUSE_TRACY -DTRACY_ENABLE -DTRACY_FIBERS -I$(build_includedir)/tracy +JCFLAGS += -DUSE_TRACY -DTRACY_ENABLE -DTRACY_FIBERS -I$(build_includedir)/tracy LIBTRACYCLIENT:=-lTracyClient endif ifeq ($(WITH_TRACY_CALLSTACKS), 1) @@ -919,6 +964,15 @@ JCFLAGS += -DTRACY_CALLSTACK=32 LIBTRACYCLIENT:=-lTracyClient endif +ifeq ($(WITH_APPLE_OSLOG), 1) +ifeq ($(OS), Darwin) +JCXXFLAGS += -DUSE_APPLE_OSLOG +JCFLAGS += -DUSE_APPLE_OSLOG +else +$(warning "WITH_APPLE_OSLOG=1 is only supported on Darwin, ignoring") +endif +endif + ifeq ($(WITH_TIMING_COUNTS), 1) JCXXFLAGS += -DUSE_TIMING_COUNTS JCFLAGS += -DUSE_TIMING_COUNTS @@ -929,6 +983,28 @@ JCXXFLAGS += -DUSE_NVTX JCFLAGS += -DUSE_NVTX endif +ifneq ($(findstring $(OS),WINNT FreeBSD OpenBSD),) + USE_LINKER_GC := 0 + USE_RT_STATIC_LIBGCC := 0 + USE_RT_STATIC_LIBSTDCXX := 0 +endif + +# Linker garbage collection +ifeq ($(USE_LINKER_GC), 1) +ifeq ($(OS), Darwin) + LINKER_GC_LDFLAGS := -Wl,-dead_strip +else + LINKER_GC_LDFLAGS := -Wl,--gc-sections + LINKER_GC_CFLAGS := -ffunction-sections -fdata-sections + LINKER_GC_CXXFLAGS := -ffunction-sections -fdata-sections +endif + +JLDFLAGS += $(LINKER_GC_LDFLAGS) +JCFLAGS += $(LINKER_GC_CFLAGS) +JCXXFLAGS += $(LINKER_GC_CXXFLAGS) + +endif + # =========================================================================== # Select the cpu architecture to target, or automatically detects the user's compiler @@ -946,6 +1022,13 @@ BUILD_MACHINE := $(shell $(HOSTCC) -dumpmachine) # don't recognize that, so canonicalize to mingw32 BUILD_MACHINE := $(subst windows-gnu,mingw32,$(BUILD_MACHINE)) +# Detect a request for ucrt libc +ifeq (,$(findstring MINGW,$(RAW_BUILD_OS))) +ifeq (UCRT64,$(MSYSTEM)) +BUILD_MACHINE := $(subst mingw32,ucrt-mingw32,$(BUILD_MACHINE)) +endif +endif + ifeq ($(ARCH),) override ARCH := $(shell $(CC) -dumpmachine | sed "s/\([^-]*\).*$$/\1/") else @@ -1000,7 +1083,7 @@ endif ifeq (${USE_THIRD_PARTY_GC},mmtk) # MMTk is only available on x86_64 Linux for now ifeq ($(OS),Linux) -MMTK_LIB_NAME := libmmtk_julia.so +MMTK_LIB_NAME := libmmtk_julia.$(SHLIB_EXT) else $(error "Unsupported OS for MMTk") endif @@ -1084,18 +1167,23 @@ BINARY:=64 MARCH= endif -# Allow Clang to use CRC instructions (only applicable on AArch64) + +# If we are running on powerpc64 or ppc64, fail out dramatically +ifneq (,$(filter $(ARCH), powerpc64 ppc64)) +$(error Big-endian PPC64 is not supported, to ignore this error, set ARCH=ppc64le) +endif + +# Architecture and platform-specific compiler flags +# Allow Clang to use CRC instructions (only applicable on AArch64) when no specific march is set ifneq (,$(findstring aarch64,$(ARCH))) -ifeq ($(USECLANG),1) ifeq (,$(MARCH)) -JCFLAGS += -mcrc -endif +JCFLAGS_CLANG += -mcrc endif endif -# If we are running on powerpc64 or ppc64, fail out dramatically -ifneq (,$(filter $(ARCH), powerpc64 ppc64)) -$(error Big-endian PPC64 is not supported, to ignore this error, set ARCH=ppc64le) +# Add platform-specific GCC flags +ifeq ($(ISX86),1) +SHIPFLAGS_GCC += -momit-leaf-frame-pointer endif # File name of make binary-dist result @@ -1224,11 +1312,6 @@ endif endif endif -ifeq ($(USEGCC),1) -ifeq ($(ISX86),1) - SHIPFLAGS += -momit-leaf-frame-pointer -endif -endif ifeq ($(OS),WINNT) LIBUNWIND:= @@ -1237,14 +1320,21 @@ LIBUNWIND:= else LIBUNWIND:=-lunwind ifneq ($(findstring $(OS),Darwin OpenBSD),) -JCPPFLAGS+=-DLLVMLIBUNWIND +JCPPFLAGS_COMMON+=-DLLVMLIBUNWIND else ifeq ($(USE_SYSTEM_LIBUNWIND), 1) # Only for linux and freebsd since we want to use not yet released gnu libunwind features JCFLAGS+=-DSYSTEM_LIBUNWIND -JCPPFLAGS+=-DSYSTEM_LIBUNWIND +JCPPFLAGS_COMMON+=-DSYSTEM_LIBUNWIND endif endif +ifeq ($(USE_SYSTEM_LMDB), 1) +LIBLMDB := -llmdb +else +# statically link the archive we build in deps/lmdb.mk into libjulia-codegen +LIBLMDB := $(build_libdir)/liblmdb.a +endif + ifeq ($(origin LLVM_CONFIG), undefined) ifeq ($(USE_SYSTEM_LLVM), 1) LLVM_CONFIG := llvm-config$(EXE) @@ -1254,7 +1344,7 @@ endif endif # LLVM_CONFIG undefined ifeq ($(USE_SYSTEM_LLVM), 1) -JCPPFLAGS+=-DSYSTEM_LLVM +JCPPFLAGS_COMMON+=-DSYSTEM_LLVM endif # SYSTEM_LLVM # Windows builds need a little help finding the LLVM libraries for llvm-config @@ -1368,6 +1458,8 @@ else UTF8PROC_INC := $(build_includedir) endif +LIBTARGET_PARSING := $(build_libdir)/libtarget_parsing.a + # We need python for things like BB triplet recognition. We don't really care # about version, generally, so just find something that works: PYTHON := $(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo not found) @@ -1408,6 +1500,17 @@ BB_TRIPLET := $(subst $(SPACE),-,$(filter-out cxx%,$(filter-out libgfortran%,$(s LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SPACE),$(BB_TRIPLET_LIBGFORTRAN)))) +# The C library of the target. This only matters on Linux, where it is either glibc +# (`*-linux-gnu*`) or musl (`*-linux-musl*`) as recorded in the normalized triplet; +# it is left empty elsewhere. +ifeq ($(OS),Linux) +ifneq (,$(findstring musl,$(BB_TRIPLET))) +LIBC := musl +else +LIBC := glibc +endif +endif + # CSL_NEXT_GLIBCXX_VERSION is a triple of the symbols representing support for whatever # the next libstdc++ version would be. This is used for two things. # 1. Whether the system libraries are new enough, if we need to use the libs bundled with CSL @@ -1428,7 +1531,7 @@ LIBGFORTRAN_VERSION := $(subst libgfortran,,$(filter libgfortran%,$(subst -,$(SP # shipped with CSL. Although we do not depend on any of the symbols, it is entirely # possible that a user might choose to install a library which depends on symbols provided # by a newer libstdc++. Without runtime detection, those libraries would break. -CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.34|GLIBCXX_3\.5\.|GLIBCXX_4\. +CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.35|GLIBCXX_3\.5\.|GLIBCXX_4\. # This is the set of projects that BinaryBuilder dependencies are hooked up for. @@ -1437,10 +1540,6 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.34|GLIBCXX_3\.5\.|GLIBCXX_4\. # or not. See `deps/csl.mk` for more detail. BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB ZSTD P7ZIP LLD LIBTRACYCLIENT BOLT -ifeq (${USE_THIRD_PARTY_GC},mmtk) -BB_PROJECTS += MMTK_JULIA -endif - define SET_BB_DEFAULT # First, check to see if BB is disabled on a global setting ifeq ($$(USE_BINARYBUILDER),0) @@ -1456,13 +1555,33 @@ endif endef $(foreach proj,$(BB_PROJECTS),$(eval $(call SET_BB_DEFAULT,$(proj)))) +# Set the CSL default here too because it affects top-level source flags in +# addition to dependency installation rules in deps/csl.mk. +ifeq ($(USE_BINARYBUILDER),0) +USE_BINARYBUILDER_CSL ?= 0 +else +ifeq ($(USE_SYSTEM_CSL),1) +USE_BINARYBUILDER_CSL ?= 0 +else +LIBSTDCXX_PATH := $(call pathsearch,$(call versioned_libname,libstdc++,6),$(STD_LIB_PATH)) +ifneq (,$(and $(LIBSTDCXX_PATH),$(shell objdump -p '$(LIBSTDCXX_PATH)' | grep '$(CSL_NEXT_GLIBCXX_VERSION)'))) +USE_BINARYBUILDER_CSL ?= 0 +else +USE_BINARYBUILDER_CSL ?= 1 +endif +endif +endif # Warn if the user tries to build something that requires `gfortran` but they don't have it installed. +# Goals that only download or repackage sources never run the fortran compiler and are exempt. +FC_EXEMPT_GOALS := get getall light-source-dist full-source-dist print-% +ifneq (,$(filter-out $(FC_EXEMPT_GOALS),$(or $(MAKECMDGOALS),x))) ifeq ($(FC_VERSION),) ifneq ($(USE_BINARYBUILDER_OPENBLAS)$(USE_BINARYBUILDER_LIBSUITESPARSE),11) $(error "Attempting to build OpenBLAS or SuiteSparse without a functioning fortran compiler!") endif endif +endif # OS specific stuff @@ -1505,11 +1624,11 @@ RPATH_LIB := $(RPATH_ORIGIN) # --whole-archive ifeq ($(OS), Darwin) - WHOLE_ARCHIVE := -Xlinker -all_load - NO_WHOLE_ARCHIVE := + whole_archive = -Xlinker -force_load $(1) +else ifeq ($(OS), WINNT) + whole_archive = --whole-archive $(1) --no-whole-archive else - WHOLE_ARCHIVE := -Wl,--whole-archive - NO_WHOLE_ARCHIVE := -Wl,--no-whole-archive + whole_archive = -Wl,--whole-archive $(1) -Wl,--no-whole-archive endif # Initialize these once, then add to them in OS-specific blocks @@ -1520,7 +1639,7 @@ OSLIBS += -Wl,--no-as-needed -ldl -lrt -lpthread -latomic -Wl,--export-dynamic,- # Detect if ifunc is supported IFUNC_DETECT_SRC := 'void (*f0(void))(void) { return (void(*)(void))0L; }; void f(void) __attribute__((ifunc("f0")));' ifeq (supported, $(shell echo $(IFUNC_DETECT_SRC) | $(CC) -Werror -x c - -S -o /dev/null > /dev/null 2>&1 && echo supported)) -JCPPFLAGS += -DJULIA_HAS_IFUNC_SUPPORT=1 +JCPPFLAGS_COMMON += -DJULIA_HAS_IFUNC_SUPPORT=1 endif JLDFLAGS += -Wl,-Bdynamic OSLIBS += -Wl,--version-script=$(BUILDROOT)/src/julia.expmap @@ -1541,6 +1660,7 @@ endif ifeq ($(OS), FreeBSD) JLDFLAGS += -Wl,-Bdynamic OSLIBS += -lelf -lkvm -lrt -lpthread -latomic +HAVE_SSP := 1 # Tweak order of libgcc_s in DT_NEEDED, # make it loaded first to @@ -1555,8 +1675,7 @@ ifeq (,$(findstring aarch64,$(ARCH))) OSLIBS += -lgcc_s endif -OSLIBS += -Wl,--export-dynamic -Wl,--version-script=$(BUILDROOT)/src/julia.expmap \ - $(NO_WHOLE_ARCHIVE) +OSLIBS += -Wl,--export-dynamic -Wl,--version-script=$(BUILDROOT)/src/julia.expmap -Wl,--no-whole-archive endif ifeq ($(OS), OpenBSD) @@ -1574,24 +1693,57 @@ endif ifeq ($(OS), Darwin) SHLIB_EXT := dylib -OSLIBS += -framework CoreFoundation -WHOLE_ARCHIVE := -Xlinker -all_load -NO_WHOLE_ARCHIVE := HAVE_SSP := 1 JLIBLDFLAGS += -Wl,-compatibility_version,$(SOMAJOR) -Wl,-current_version,$(JULIA_MAJOR_VERSION).$(JULIA_MINOR_VERSION).$(JULIA_PATCH_VERSION) +# When linking LTO bitcode, the debug map points dsymutil at the objects that ld64 +# compiled from it, which ld64 deletes unless told where to keep them. +LTO_OBJECT_PATH = -Wl,-object_path_lto,$(1).lto endif ifeq ($(OS), WINNT) HAVE_SSP := 1 -OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(BUILDROOT)/src/julia.expmap \ - $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -luserenv -lsecur32 -latomic -lole32 +ifeq ($(BINARY),64) +WIN_LD_EMULATION := i386pep +WIN_LD_DLL_ENTRY := DllMainCRTStartup +WIN_LD_EXE_ENTRY := mainCRTStartup +else +WIN_LD_EMULATION := i386pe +WIN_LD_DLL_ENTRY := _DllMainCRTStartup +WIN_LD_EXE_ENTRY := _mainCRTStartup +endif +WIN_LD_SHARED_FLAGS := -m $(WIN_LD_EMULATION) -shared -Bdynamic -e $(WIN_LD_DLL_ENTRY) --enable-auto-image-base --allow-multiple-definition +WIN_LD_EXE_FLAGS := -m $(WIN_LD_EMULATION) -Bdynamic --subsystem console -e $(WIN_LD_EXE_ENTRY) +WIN_CRT_LIB := -lmsvcrt-os +# libmsvcrt-os.a contains MinGW CRT objects that can refer back to +# libmingw32.a/libmingwex.a; keep the selected CRT last. +WIN_MINGW_CRT_LIBS := -lmingwex $(WIN_CRT_LIB) -lmingw32 -lmingwex $(WIN_CRT_LIB) +WIN_LD_DLL_CRT_START := $(build_private_libdir)/dllcrt2.o $(build_private_libdir)/crtbegin.o +WIN_LD_EXE_CRT_START := $(build_private_libdir)/crt2.o $(build_private_libdir)/crtbegin.o +WIN_LD_CRT_END_OBJ := $(build_private_libdir)/crtend.o +WIN_LD_CRT_END := $(WIN_MINGW_CRT_LIBS) $(WIN_LD_CRT_END_OBJ) +WIN_LD_STDCXX := $(build_shlibdir)/$(call versioned_libname,libstdc++,6) +WIN_LD_ATOMIC := $(build_shlibdir)/$(call versioned_libname,libatomic,1) +WIN_SYSTEM_LIBS := -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -luserenv -lsecur32 -lsynchronization $(WIN_LD_ATOMIC) -lole32 +DLLTOOL := $(CROSS_COMPILE)dlltool +ifeq ($(BINARY),64) +DLLTOOL_MACHINE := i386:x86-64 +else +DLLTOOL_MACHINE := i386 +endif +# Julia links its Windows libraries by calling the linker directly, so whatever +# the compiler driver would have added has to be named here instead, such as the +# profile runtime of an instrumented build. +WIN_LD_EXTRA_LIBS ?= +WIN_MINGW_LIBS := -lmingw32 -lgcc_s -lgcc -lmoldname $(WIN_MINGW_CRT_LIBS) -lkernel32 -lpthread -ladvapi32 -lshell32 -luser32 $(WIN_LD_EXTRA_LIBS) +OSLIBS += --export-all-symbols --version-script=$(BUILDROOT)/src/julia.expmap \ + --no-whole-archive $(WIN_SYSTEM_LIBS) $(WIN_MINGW_LIBS) # N.B.: Unlike in the sysimage, we cannot -Wl,--disable-auto-import -Wl,--disable-runtime-pseudo-reloc here, because libstdc++/LLVM are not fully correct under # enforced visibility at this point. -JLDFLAGS += -Wl,--stack,8388608 +JLDFLAGS += --stack 8388608 ifeq ($(ARCH),i686) -JLDFLAGS += -Wl,--large-address-aware +JLDFLAGS += --large-address-aware endif -JCPPFLAGS += -D_WIN32_WINNT=0x0502 +JCPPFLAGS_COMMON += -D_WIN32_WINNT=0x0602 # (0x0602 = _WIN32_WINNT_WIN8) UNTRUSTED_SYSTEM_LIBM := 1 # Use hard links for files on windows, rather than soft links # https://stackoverflow.com/questions/3648819/how-to-make-a-symbolic-link-with-cygwin-in-windows-7 @@ -1601,32 +1753,52 @@ else WIN_MAKE_HARD_LINK := true -ignore endif # $(OS) == WINNT +# Link commands that use $(LD) directly on Windows and the usual compiler driver +# elsewhere. Arguments are: +# 1: output, 2: Windows-only flags before -o, 3: Windows inputs after output, +# 4: non-Windows command before -o, 5: non-Windows inputs after output. +ifeq ($(OS), WINNT) +WIN_LD_LIBPATHS := -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) +# lld's MinGW driver needs a COFF export definition instead of a version script. +ifeq ($(WIN_LD_USE_DEF),1) +WIN_LD = $(PYTHON) $(JULIAHOME)/contrib/windows/link_with_exports.py $(LD) +else +WIN_LD = $(LD) +endif +link-dll = $(WIN_LD) $(WIN_LD_SHARED_FLAGS) $(2) -o $(1) $(3) +link-exe = $(WIN_LD) $(WIN_LD_EXE_FLAGS) $(2) -o $(1) $(3) +else +WIN_LD_LIBPATHS := +link-dll = $(4) -o $(1) $(5) +link-exe = $(4) -o $(1) $(5) +endif + # Threads ifneq ($(JULIA_THREADS), 0) -JCPPFLAGS += -DJULIA_NUM_THREADS=$(JULIA_THREADS) +JCPPFLAGS_COMMON += -DJULIA_NUM_THREADS=$(JULIA_THREADS) endif # Intel VTune Amplifier ifeq ($(USE_INTEL_JITEVENTS), 1) -JCPPFLAGS += -DJL_USE_INTEL_JITEVENTS +JCPPFLAGS_COMMON += -DJL_USE_INTEL_JITEVENTS endif # OProfile ifeq ($(USE_OPROFILE_JITEVENTS), 1) -JCPPFLAGS += -DJL_USE_OPROFILE_JITEVENTS +JCPPFLAGS_COMMON += -DJL_USE_OPROFILE_JITEVENTS endif ifeq ($(DISABLE_LIBUNWIND), 1) -JCPPFLAGS += -DJL_DISABLE_LIBUNWIND +JCPPFLAGS_COMMON += -DJL_DISABLE_LIBUNWIND endif # perf ifeq ($(USE_PERF_JITEVENTS), 1) -JCPPFLAGS += -DJL_USE_PERF_JITEVENTS +JCPPFLAGS_COMMON += -DJL_USE_PERF_JITEVENTS endif ifeq ($(HAVE_SSP),1) -JCPPFLAGS += -DHAVE_SSP=1 +JCPPFLAGS_COMMON += -DHAVE_SSP=1 ifeq ($(USEGCC),1) OSLIBS += -lssp endif @@ -1681,12 +1853,21 @@ endif CLANGSA_FLAGS := CLANGSA_CXXFLAGS := ifeq ($(OS), Darwin) # on new XCode, the files are hidden - CLANGSA_FLAGS += -isysroot $(shell xcrun --show-sdk-path -sdk macosx) + CLANGSA_FLAGS += -isysroot $(shell xcrun --show-sdk-path --sdk macosx) endif ifeq ($(USEGCC),1) # try to help clang find the c++ files for CC by guessing the value for --prefix # by dropping lib/gcc// from the install directory it reports CLANGSA_CXXFLAGS += --gcc-toolchain="$(abspath $(shell LANG=C $(CC) -print-search-dirs | grep '^install: ' | sed -e "s/^install: //")/../../../..)" +# On Linux without system headers (e.g., BinaryBuilder containers), derive sysroot from GCC +ifeq ($(OS), Linux) +GCC_SYSROOT := $(shell $(CC) --print-sysroot 2>/dev/null) +ifneq ($(GCC_SYSROOT),) +ifeq ($(wildcard /usr/include/stdlib.h),) + CLANGSA_FLAGS += --sysroot=$(GCC_SYSROOT) +endif +endif +endif endif diff --git a/Makefile b/Makefile index 3e4fc1356b08d..f8f97d4b54660 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,10 @@ endif VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` +.PHONY: default default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" + +.PHONY: all all: debug release # sort is used to remove potential duplicates @@ -65,6 +68,7 @@ julia_flisp.boot.inc.phony: julia-deps $(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl -o -name version_git.jl \) -prune -o -type f -print) @$(MAKE) docs +.PHONY: julia-symlink julia-symlink: julia-cli-$(JULIA_BUILD_MODE) ifeq ($(OS),WINNT) printf '@"%%~dp0/%s" %%*\n' "$$(printf "%s\n" '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')" | tr / '\\' > $(BUILDROOT)/julia.bat @@ -75,21 +79,42 @@ ifndef JULIA_VAGRANT_BUILD endif endif -TOP_LEVEL_PKGS := Compiler +TOP_LEVEL_PKGS := Compiler JuliaSyntax JuliaLowering TOP_LEVEL_PKG_LINK_TARGETS := $(addprefix $(build_datarootdir)/julia/,$(TOP_LEVEL_PKGS)) # Generate symlinks for top level pkgs in usr/share/julia/ $(foreach module, $(TOP_LEVEL_PKGS), $(eval $(call symlink_target,$$(JULIAHOME)/$(module),$$(build_datarootdir)/julia,$(module)))) +.PHONY: julia-deps julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps +ifeq (${USE_THIRD_PARTY_GC},mmtk) +.PHONY: julia-mmtk_julia +julia-mmtk_julia: | $(DIRS) + @$(MAKE) $(QUIET_MAKE) -f $(JULIAHOME)/src/gc-mmtk/Makefile MMTK_BUILD_MODE=$(JULIA_BUILD_MODE) compile-mmtk_julia +julia-deps: julia-mmtk_julia + +.PHONY: clean-mmtk_julia +clean-mmtk_julia: + @$(MAKE) $(QUIET_MAKE) -f $(JULIAHOME)/src/gc-mmtk/Makefile clean-mmtk_julia +clean: clean-mmtk_julia +endif + +ifeq ($(OS),Darwin) +julia-deps: $(build_libdir)/libSystem.tbd +$(build_libdir)/libSystem.tbd: $(JULIAHOME)/contrib/mac/libSystem.tbd | $(build_libdir) + $(INSTALL_M) $< $@ +endif + # `julia-stdlib` depends on `julia-deps` so that the fake JLL stdlibs can copy in their Artifacts.toml files. +.PHONY: julia-stdlib julia-stdlib: | $(DIRS) julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib -julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac/juliac.jl $(build_datarootdir)/julia/juliac/juliac-buildscript.jl $(build_datarootdir)/julia/juliac/juliac-trim-base.jl $(build_datarootdir)/julia/juliac/juliac-trim-stdlib.jl $(build_datarootdir)/julia/juliac/Artifacts.toml +.PHONY: julia-base +julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: julia-deps @@ -104,51 +129,65 @@ julia-libccalllazybar: julia-deps julia-libccalllazyfoo julia-libllvmcalltest: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libllvmcalltest +.PHONY: julia-src-release julia-src-debug julia-src-release julia-src-debug : julia-src-% : julia-deps julia_flisp.boot.inc.phony julia-cli-% @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src $* +.PHONY: julia-cli-release julia-cli-debug julia-cli-release julia-cli-debug: julia-cli-% : julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/cli $* +.PHONY: julia-sysimg-release julia-sysimg-debug julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$* +.PHONY: julia-sysimg-JL-release julia-sysimg-JL-debug +julia-sysimg-JL-release julia-sysimg-JL-debug : julia-sysimg-JL-% : julia-sysimg-% julia-stdlib | $(build_private_libdir) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-JL-$* + # Useful for cross-bootstrapping +.PHONY: julia-sysbase-release julia-sysbase-debug julia-sysbase-release julia-sysbase-debug : julia-sysbase-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysbase-$* +.PHONY: julia-debug julia-release julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest \ julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache +.PHONY: stdlibs-cache-release stdlibs-cache-debug stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-% @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk $* +.PHONY: debug release debug release : % : julia-% stdlibs-cache-% +.PHONY: docs docs: julia-sysimg-$(JULIA_BUILD_MODE) stdlibs-cache-$(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no' +.PHONY: docs-revise docs-revise: @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/doc JULIA_EXECUTABLE='$(call spawn,$(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))) --startup-file=no' revise=true +WS_CHECK_PATTERNS = *.1 *.c *.cpp *.h *.inc *.jl *.lsp *.make *.md *.mk *.rst *.scm *.sh *.yml *Makefile + +.PHONY: check-whitespace check-whitespace: ifneq ($(NO_GIT), 1) - @# Append the directory containing the julia we just built to the end of `PATH`, - @# to give us the best chance of being able to run this check. - @PATH="$(PATH):$(dir $(JULIA_EXECUTABLE))" julia $(call cygpath_w,$(JULIAHOME)/contrib/check-whitespace.jl) + @git ls-files -- $(WS_CHECK_PATTERNS:%='%') | PATH="$(PATH):$(dir $(JULIA_EXECUTABLE))" julia $(call cygpath_w,$(JULIAHOME)/contrib/check-whitespace.jl) --stdin else $(warn "Skipping whitespace check because git is unavailable") endif +.PHONY: fix-whitespace fix-whitespace: ifneq ($(NO_GIT), 1) - @# Append the directory containing the julia we just built to the end of `PATH`, - @# to give us the best chance of being able to run this check. - @PATH="$(PATH):$(dir $(JULIA_EXECUTABLE))" julia $(call cygpath_w,$(JULIAHOME)/contrib/check-whitespace.jl) --fix + @git ls-files -- $(WS_CHECK_PATTERNS:%='%') | PATH="$(PATH):$(dir $(JULIA_EXECUTABLE))" julia $(call cygpath_w,$(JULIAHOME)/contrib/check-whitespace.jl) --stdin --fix else $(warn "Skipping whitespace fix because git is unavailable") endif +.PHONY: release-candidate release-candidate: release testall @$(JULIA_EXECUTABLE) $(JULIAHOME)/contrib/add_license_to_files.jl #add license headers @#Check documentation @@ -203,6 +242,7 @@ $(build_datarootdir)/julia/%: $(JULIAHOME)/contrib/% | $(build_datarootdir)/juli $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir) @$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c) +.PHONY: julia-base-cache julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia @JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) JULIA_FALLBACK_REPL=1 WINEPATH="$(call cygpath_w,$(build_bindir));$$WINEPATH" \ $(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/contrib/write_base_cache.jl) \ @@ -219,9 +259,6 @@ endif JL_PRIVATE_LIBS-0 := libccalltest libccalllazyfoo libccalllazybar libllvmcalltest JL_PRIVATE_LIBS-1 := # libraries from USE_SYSTEM=1 JL_PRIVATE_EXES := 7z -ifeq ($(OS),WINNT) -JL_PRIVATE_EXES += 7z.dll -endif JL_PRIVATE_TOOLS := ifeq ($(JULIA_BUILD_MODE),release) JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen @@ -252,7 +289,7 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib else JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz endif -JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libzstd +JL_PRIVATE_LIBS-$(USE_SYSTEM_ZSTD) += libzstd JL_PRIVATE_EXES += zstd$(EXE) zstdmt$(EXE) ifeq ($(USE_LLVM_SHLIB),1) JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME) @@ -300,26 +337,8 @@ endif endif ifeq (${USE_THIRD_PARTY_GC},mmtk) -# Make sure we use the right version of $MMTK_PLAN, $MMTK_MOVING and $MMTK_BUILD -# if we use the BinaryBuilder version of mmtk-julia -ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1) -ifeq (${MMTK_PLAN},Immix) -LIB_PATH_PLAN = immix -else ifeq (${MMTK_PLAN},StickyImmix) -LIB_PATH_PLAN = sticky -endif - -ifeq ($(MMTK_MOVING), 1) -LIB_PATH_MOVING := moving -else -LIB_PATH_MOVING := non_moving -endif - -JL_PRIVATE_LIBS-0 += $(LIB_PATH_PLAN)/$(LIB_PATH_MOVING)/$(MMTK_BUILD)/libmmtk_julia -else JL_PRIVATE_LIBS-0 += libmmtk_julia endif -endif # Note that we disable MSYS2's path munging here, as otherwise # it replaces our `:`-separated list as a `;`-separated one. @@ -328,6 +347,7 @@ define stringreplace endef +.PHONY: install install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html @$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE) @for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(private_libexecdir); do \ @@ -344,10 +364,37 @@ else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ $(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ endif - $(INSTALL_M) $(filter-out %-bc.a %-o.a,$(wildcard $(build_private_libdir)/lib*.a)) $(DESTDIR)$(private_libdir)/ - - $(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ - $(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ +# Copy over C runtime files used by Base.Linking and direct Windows links + $(INSTALL_M) $(build_private_libdir)/libgcc.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libgcc_s.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libmsvcrt.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libmsvcrt-os.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libmingwex.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libkernel32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libmingw32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libmoldname.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libntdll.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libpsapi.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libws2_32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libiphlpapi.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libwinmm.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libdbghelp.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libuserenv.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libsecur32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libole32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libuuid.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libadvapi32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libshell32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libuser32.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libpthread.dll.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/libssp.dll.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/crt2.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/crt2u.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/dllcrt2.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/crtbegin.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_private_libdir)/crtend.o $(DESTDIR)$(private_libdir)/ + + $(INSTALL_M) $(build_shlibdir)/libopenlibm.dll.a $(DESTDIR)$(private_libdir)/ else # Copy over .dSYM directories directly for Darwin @@ -401,6 +448,21 @@ endif fi \ done \ done + +# Copy over C runtime files used by Base.Linking +ifeq ($(OS),Darwin) + $(INSTALL_M) $(build_libdir)/libclang_rt.osx.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_libdir)/libSystem.tbd $(DESTDIR)$(private_libdir)/ +else + $(INSTALL_M) $(build_libdir)/libgcc.a $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_libdir)/crti.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_libdir)/crtn.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_libdir)/crtbeginS.o $(DESTDIR)$(private_libdir)/ + $(INSTALL_M) $(build_libdir)/crtendS.o $(DESTDIR)$(private_libdir)/ +ifeq ($(LIBC),glibc) + $(INSTALL_M) $(build_libdir)/libc_nonshared.a $(DESTDIR)$(private_libdir)/ +endif +endif endif for exe in $(JL_PRIVATE_EXES) ; do \ $(INSTALL_M) $(build_private_libexecdir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \ @@ -441,6 +503,7 @@ endif -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-checked + -rm -rf $(DESTDIR)$(datarootdir)/julia/cache # Copy in beautiful new man page $(INSTALL_F) $(build_man1dir)/julia.1 $(DESTDIR)$(man1dir)/ # Copy .desktop file @@ -530,7 +593,6 @@ ifneq ($(DARWIN_FRAMEWORK),1) endif else ifneq (,$(findstring $(OS),Linux FreeBSD)) for j in $(JL_PRIVATE_EXES) ; do \ - [ $$j = 7z ] && continue; \ [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_private_libexecdir_rel)' $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ done @@ -574,6 +636,7 @@ endif distclean: -rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-$(JULIA_COMMIT) +.PHONY: binary-dist binary-dist: distclean ifeq ($(USE_SYSTEM_BLAS),0) ifeq ($(ISX86),1) @@ -620,6 +683,7 @@ app: darwinframework: $(MAKE) -C $(JULIAHOME)/contrib/mac/framework +.PHONY: light-source-dist.tmp light-source-dist.tmp: $(BUILDROOT)/doc/_build/html/en/index.html ifneq ($(BUILDROOT),$(JULIAHOME)) $(error make light-source-dist does not work in out-of-tree builds) @@ -642,6 +706,7 @@ endif find doc/_build/html >> light-source-dist.tmp # Make tarball with only Julia code + stdlib tarballs +.PHONY: light-source-dist light-source-dist: light-source-dist.tmp # Prefix everything with "julia-$(commit-sha)/" or "julia-$(version)/" and then create tarball # To achieve prefixing, we temporarily create a symlink in the source directory that points back @@ -651,10 +716,12 @@ light-source-dist: light-source-dist.tmp tar -cz --no-recursion -T light-source-dist.tmp1 -f julia-$(JULIA_VERSION)_$(JULIA_COMMIT).tar.gz rm julia-${JULIA_COMMIT} +.PHONY: source-dist source-dist: @echo \'source-dist\' target is deprecated: use \'full-source-dist\' instead. # Make tarball with Julia code plus all dependencies +.PHONY: full-source-dist full-source-dist: light-source-dist.tmp # Get all the dependencies downloaded @$(MAKE) -C deps getall DEPS_GIT=0 USE_BINARYBUILDER=0 @@ -671,6 +738,7 @@ full-source-dist: light-source-dist.tmp tar -cz --no-recursion -T full-source-dist.tmp1 -f julia-$(JULIA_VERSION)_$(JULIA_COMMIT)-full.tar.gz rm julia-${JULIA_COMMIT} +.PHONY: clean clean: | $(CLEAN_TARGETS) @-$(MAKE) -C $(BUILDROOT)/base clean @-$(MAKE) -C $(BUILDROOT)/doc clean @@ -688,36 +756,40 @@ clean: | $(CLEAN_TARGETS) # Teporarily add this line to the Makefile to remove extras -rm -fr $(build_datarootdir)/julia/extras +.PHONY: cleanall cleanall: clean @-$(MAKE) -C $(BUILDROOT)/src clean-flisp clean-support - @-$(MAKE) -C $(BUILDROOT)/deps clean-libuv + @-$(MAKE) -C $(BUILDROOT)/deps clean-libuv clean-utf8proc -rm -fr $(build_prefix) $(build_staging) +.PHONY: distcleanall distcleanall: cleanall @-$(MAKE) -C $(BUILDROOT)/stdlib distclean @-$(MAKE) -C $(BUILDROOT)/deps distcleanall @-$(MAKE) -C $(BUILDROOT)/doc cleanall .FORCE: -.PHONY: .FORCE default debug release check-whitespace fix-whitespace release-candidate \ - julia-debug julia-release julia-stdlib julia-deps julia-deps-libs \ - julia-cli-release julia-cli-debug julia-src-release julia-src-debug \ - julia-symlink julia-base julia-sysimg julia-sysimg-ji julia-sysimg-release julia-sysimg-debug \ - test testall testall1 test \ - clean distcleanall cleanall $(CLEAN_TARGETS) \ - run-julia run-julia-debug run-julia-release run \ - install binary-dist light-source-dist.tmp light-source-dist \ - dist full-source-dist source-dist +.PHONY: .FORCE + +.PHONY: $(CLEAN_TARGETS) + +# Generate compilation database (leverages existing clang tooling setup) +.PHONY: compile-database +compile-database: + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src compile-database +.PHONY: test test: check-whitespace $(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test default JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) +.PHONY: testall testall: check-whitespace $(JULIA_BUILD_MODE) cp $(JULIA_SYSIMG) $(BUILDROOT)/local.$(SHLIB_EXT) $(call spawn,$(JULIA_EXECUTABLE) -J $(call cygpath_w,$(BUILDROOT)/local.$(SHLIB_EXT)) -e 'true') rm $(BUILDROOT)/local.$(SHLIB_EXT) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) +.PHONY: testall1 testall1: check-whitespace $(JULIA_BUILD_MODE) @env JULIA_CPU_THREADS=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) @@ -730,7 +802,7 @@ test-revise-%: .FORCE @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test revise-$* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE) # download target for some hardcoded windows dependencies -.PHONY: win-extras wine_path +.PHONY: win-extras win-extras: @$(MAKE) -C $(BUILDROOT)/deps install-p7zip mkdir -p $(JULIAHOME)/dist-extras diff --git a/NEWS.md b/NEWS.md index ea4d8337beeb6..281db6a5cbe5b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,44 +1,162 @@ -Julia v1.13 Release Notes -======================== +Julia v1.14 Release Notes +========================= New language features --------------------- - - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) - - New `nth` function to access the `n`-th element of a generic iterable. ([#56580]) - - New `@__FUNCTION__` macro to refer to the innermost enclosing function. ([#58940]) - - The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, - is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL. - ([JuliaLang/JuliaSyntax.jl#525], [#57143]) +* It is now possible to control which version of the Julia syntax will be used to parse a package by setting the + `compat.julia` or `syntax.julia_version` key in Project.toml. This feature is similar to the notion of "editions" + in other language ecosystems and will allow non-breaking evolution of Julia syntax in future versions. + See the "Syntax Versioning" section in the code loading documentation ([#60018]). +* `ᵅ` (U+U+1D45), `ᵋ` (U+1D4B), `ᶲ` (U+1DB2), `˱` (U+02F1), `˲` (U+02F2), and `ₔ` (U+2094) can now also be used as + operator suffixes, accessible as `\^alpha`, `\^epsilon`, `\^ltphi`, `\_<`, `\_>`, and `\_schwa` at the REPL + ([#60285]). +* The `@label` macro can now create labeled blocks that can be exited early with `break name [value]`. Use + `@label name expr` for named blocks or `@label expr` for anonymous blocks. Anonymous `@label` blocks + participate in the default break scope: a plain `break` or `break _` exits the innermost breakable scope, + whether it is a loop or an `@label` block. The `continue` statement also supports labels with + `continue name` to continue a labeled loop ([#60481]). +* `typegroup` blocks allow defining mutually recursive struct types that reference each other in their + field types. All types in the group are resolved atomically at the end of the block ([#60569]). +* A macrocall directly inside parens — e.g. `(@info "msg" x=1)` — may now + continue its arguments on subsequent lines, so each argument of a long macro + call can be placed on its own line without switching to the comma separated + call syntax ([#60181]). +* Primitive types with non-byte-multiple logical widths can now be defined ([#61359]). +* Introduced explicitly wrapping arithmetic operators `+%`, `-%`, `*%` to annotate arithmetic operations + that are semantically safe to wrap/overflow. Their behavior is currently identical to the default `+`, `-`, `*` + operators. However, in a future version, there may be opt-in support to detect unannotated wrapping + in the default operators ([#50790]). + +* `@sync`, `Threads.@threads` and `Experimental.@sync` blocks now scope a cancellation source + (see `Base.CancellationTokenSource`) over their children, so cancelling an enclosing scope + reaches everything spawned within, and the blocks' teardown awaits internal tasks per the + requested cancellation severity ([#60281]). +* Task cancellation is now supported, organized around cancellation tokens: + `Base.CancellationTokenSource` is a level-triggered, tree-structured cancellation scope + (cancelling a source cancels its whole subtree, at monotonically escalating severities), and + `Base.CancellationToken` is its observe/wait view. The token governing a computation is carried + as a scoped value (`Base.CANCEL_TOKEN`, established with the standard `ScopedValues` API) that + propagates to child tasks; blocking operations + (`wait`, `lock`, Channel operations, `sleep`, stream and command I/O, Sockets, FileWatching, + ...) accept a `cancel` keyword argument defaulting to the scoped token and throw a + `Base.CancellationRequest` while it is cancelled. Cancellation is uniformly level-triggered: + cleanup code that must block under a cancelled scope shields itself with `cancel = nothing` + (or by scoping `Base.CANCEL_TOKEN => nothing` over a whole block). Compute-bound code can opt into cancellation with the + `Base.@cancel_check` cancellation point. + A long-running foreign call can be made cancellable with + `@ccall cancel_handler=(fn, state) ...`: cancelling the governing token runs the + C-callable `fn(state, severity)` on the thread executing the call, signal-handler-style, + so it can tell the library to return early (the pending cancellation is then thrown at + the next cancellation point). Calls into libraries audited for asynchronous unwinding + can be annotated `@ccall reset_safe=true ...` instead, letting a cancellation unwind + the foreign computation at an arbitrary instruction; `BigInt` (GMP) arithmetic uses + this, so checkless bignum loops now cancel cleanly at the first ^C. + In interactive sessions, ^C now cancels the current evaluation's cancellation scope + (instead of throwing an `InterruptException` into whatever code happened to be running), + and a fresh ^C epoch is re-armed at each prompt; a script that catches a ^C + cancellation continues under the cancelled scope unless it re-arms one itself + (`ScopedValues.@with Base.CANCEL_TOKEN => Base.sigint_new_episode!() ...`) ([#60281]). Language changes ---------------- -* `mod(x::AbstractFloat, -Inf)` now returns `x` (as long as `x` is finite), this aligns with C standard and -is considered a bug fix ([#47102]) - - - The `hash` algorithm and its values have changed. Most `hash` specializations will remain correct and require no action. Types that reimplement the core hashing logic independently, such as some third-party string packages do, may require a migration to the new algorithm. ([#57509]) -* Indexless `getindex` and `setindex!` (i.e. `A[]`) on `ReinterpretArray` now correctly throw a `BoundsError` when there is more than one element. ([#58814]) +* `Type{T} <: S` now holds only if every type `==` to `T` is an instance of `S`, fixing a + long-standing soundness hole where e.g. `Type{Int} <: DataType` held even though types like + `Tuple{S} where S<:Int` are `==` (and `isa`) their canonical spelling without being `DataType`s. + In particular `Type{T}` is no longer a subtype of any single kind: use a union of kinds instead + (e.g. `Type{Int} <: Union{DataType,UnionAll}` holds). `isa` and dispatch of type *values* are + unaffected, and a method on `Type{Int}` remains more specific than one on `DataType` + ([#33136], [#62141]). Compiler/Runtime improvements ----------------------------- +* Type inference now refines field types through conditional checks and call signatures. + For example, after `if !isnothing(x.field)`, inference knows `x.field` is not `nothing` within the branch. + Similarly, after a call like `func(x.field)` where `func(::Int)` is the only matching method, inference + refines `x.field` to `Int`. This works for immutable struct fields and `const` fields of mutable structs. + Mutable (non-`const`) fields are not supported due to the lack of per-object memory effect tracking; + for those, the recommended pattern remains storing the field value in a local variable before the check + (e.g. `val = x.field; if !isnothing(val) ... end`) ([#41199], [#47574]). +* Stack traces now show full method signatures with argument types for inlined frames, matching the display + of non-inlined frames ([#53925]). +* Stack traces of errors raised while loading code no longer show the internals of the + code loading machinery, which are collapsed to the single frame that entered loading. + Frames for user code that runs during loading are unaffected. Set the + `JULIA_STACKTRACE_FULL_LOADING` environment variable to `true` to show them ([#52988]). +* Parallel package precompilation now coordinates CPU usage across both the precompile worker processes and + the LLVM threads each spawns to compile its native image, sharing a single thread budget so idle cores are + filled during the long tail without oversubscribing the machine when many packages compile at once. The total + budget can be set with the new `JULIA_PRECOMPILE_THREADS` environment variable ([#61958]). +* Parallel package precompilation no longer attempts packages whose dependencies failed to precompile; + they are reported as skipped instead, and extensions of a failed package are dropped silently. Pass + `skip_dependents=false` to `Base.Precompilation.precompilepkgs` to attempt the packages anyway. A new `force` + keyword recompiles packages whose cache files are already fresh ([#63122]). +* Coverage reports now include code executed by the interpreter, such as top-level statements and method + bodies run with `--compile=min`. Consequently, LCOV output and `.cov` files may contain source lines that + were absent in earlier releases ([#62514]). +* Coverage and allocation tracking use separate unordered atomic loads and stores. This avoids the atomic + read-modify-write overhead reported in [#62424] while keeping concurrent accesses well-defined; execution + counts may still be inaccurate when the same source line runs on multiple threads ([#62724]). +* `--code-coverage=user` no longer includes inlined Base methods whose module cannot be recovered from debug + information. This prevents coverage from writing `.cov` files for Base sources into the Julia installation + ([#62514]). +* Coverage now records only whether each source line ran by default, and reports a count of 1 for executed + lines in `.cov` files and LCOV tracefiles. Use `--code-coverage-mode=count` to collect execution counts + instead. The default `hit` mode avoids the load and increment at each instrumentation point ([#62724]). +* Coverage runs can reuse instrumented package images across processes. The counter mode is part of + the cache identity; `user`, `all`, and `@path` select the same image variants and filter the counters + reported. Count images can also serve hit requests ([#62724]). +* `--code-coverage=all` no longer invalidates system-image code at startup. To collect coverage from + that code, build Julia with `JULIA_COVERAGE_IMAGES=1`, which instruments the system image and bundled + package images in hit mode. `@path` instruments newly compiled and interpreted code like `user`, + while also reporting compatible image counters under the selected path ([#62724]). +* Resolved global variable accesses now carry the binding partition they act on through lowered code, instead + of code generation re-deriving it by scanning a binding's partitions. After optimization, an access that + previously appeared as a `GlobalRef`, `getglobal` or `setglobal!` may instead appear as a + `Core.BindingPartition`, as the left-hand side of an assignment to one, or as a call to one of the new + `Core.getglobal_partition`, `Core.setglobal_partition`, `Core.swapglobal_partition`, + `Core.modifyglobal_partition`, `Core.replaceglobal_partition`, `Core.setglobalonce_partition`, + `Core.isdefinedglobal_partition` or `Core.depwarn_partition` builtin function. This does not change the + meaning of the program, but packages that inspect optimized IR (e.g. from `code_typed`) will encounter + these new forms. See the "Lowered form" section of the developer documentation for their semantics ([#62452]). + Command-line option changes --------------------------- -* The option `--sysimage-native-code=no` has been deprecated. -* The `JULIA_CPU_TARGET` environment variable now supports a `sysimage` keyword to match (or extend) the CPU target used to build the current system image ([#58970]). +* `-P ` is now a shorthand for `--project ` ([#59867]). +* `--code-coverage=@` and `--track-allocation=@` now restrict tracking to the specified file or + directory tree. For example, `@/src/Foo` tracks `/src/Foo/x.jl`, but not `/src/Foobar/x.jl`. Specifying the + filesystem root as `@/` tracks every absolute path. `Base.is_file_tracked` now returns `false` when Julia was + not started with either `@` option ([#62514]). Multi-threading changes ----------------------- -* A new `AbstractSpinLock` is defined with `SpinLock <: AbstractSpinLock` ([#55944]). -* A new `PaddedSpinLock <: AbstractSpinLock` is defined. It has extra padding to avoid false sharing ([#55944]). -* New types are defined to handle the pattern of code that must run once per process, called - a `OncePerProcess{T}` type, which allows defining a function that should be run exactly once - the first time it is called, and then always return the same result value of type `T` - every subsequent time afterwards. There are also `OncePerThread{T}` and `OncePerTask{T}` types for - similar usage with threads or tasks. ([#TBD]) +* The return type of `fetch(::Task)` is now inferred precisely when inference can determine the code + the task was created to run (for example `fetch(Threads.@spawn f(x))`), instead of always being + `Any`. Correspondingly, assigning to the `result` field of a `Task` via property syntax + (`t.result = v`) now throws an error: the result of a task is determined by the return value of its + code, and the runtime and the compiler now rely on this correspondence. To pass a value to a + suspended task, use `schedule(t, val)` or `yieldto(t, val)` ([#59221]). +* New functions `Threads.atomic_fence_heavy` and `Threads.atomic_fence_light` provide support for + asymmetric atomic fences, speeding up atomic synchronization where one side of the synchronization + runs significantly less often than the other ([#60311]). +* `Threads.@threads` now supports array comprehensions with syntax like `@threads [f(i) for i in 1:n]`, + filtered comprehensions like `@threads [f(i) for i in 1:n if condition(i)]`, typed comprehensions + like `@threads Float64[f(i) for i in 1:n]`, and multi-dimensional comprehensions like + `@threads [f(i,j) for i in 1:n, j in 1:m]` (preserves dimensions). All scheduling options + (`:static`, `:dynamic`, `:greedy`) are supported. Results preserve element order for `:static` + and `:dynamic` scheduling; `:greedy` does not guarantee order. Non-indexable iterators are + also supported ([#59019]). +* The task scheduler now avoids O(nthreads) wake overhead on every `@spawn`, significantly reducing + threading overhead particularly on highly oversubscribed machines. Benchmarks show up to 1000x + reduction in spawn time in such scenarios ([#61826]). +* `Threads.Atomic` now supports the reference form of the `@atomic`, `@atomicswap`, `@atomicreplace`, + and `@atomiconce` macros (e.g. `@atomic a[]`, `@atomic a[] = v`, `@atomic a[] += 1`), which allows + the memory ordering to be specified explicitly and makes atomic read-modify-write operations + syntactically clear ([#62382]). Build system changes -------------------- @@ -46,47 +164,139 @@ Build system changes New library functions --------------------- -* `ispositive(::Real)` and `isnegative(::Real)` are provided for performance and convenience ([#53677]). -* Exporting function `fieldindex` to get the index of a struct's field ([#58119]). -* `Base.donotdelete` is now public. It prevents deadcode elemination of its arguments ([#55774]). -* `Sys.sysimage_target()` returns the CPU target string used to build the current system image ([#58970]). +* `tap(f)` creates a function that calls `f(x)` for side effects and returns `x` ([#61340]). +* `unsplat(f)` creates a function that bundles its arguments into a tuple and passes them to `f`; + it is the inverse of `splat` ([#62714]). +* `Base.set_binding_visibility!` sets the declared visibility (`:none`, `:public`, or `:export`) of a name + in a module, allowing an `export` or `public` declaration to be retracted programmatically ([#62131]). +* `Base.generating_output()` has been made `public` (but not exported) to allow checking whether the current + process is performing compilation for a pkgimage/sysimage ([#61224]). +* `Base.isfieldatomic(t, s)` has been made `public` (but not exported); it reports whether a field `s` of a + type `t` is declared `@atomic`. +* `Base.raw_substring` is an unexported, public constructor to build a `SubString` without checking for + valid string indices. +* `Base.unannotate(::AnnotatedString)` returns the underlying un-annotated string of the input string. +* `Base.include_mapexprs(mod)` is an unexported, public function returning the non-identity + `mapexpr` functions used by `include(mapexpr, …)` calls while loading the package rooted at + `mod`, keyed by `(including_module, absolute_path)`. The table is stored inside the package + image, so it survives precompilation; revision tools (e.g. Revise) use it to re-apply the + original transform when an `include(mapexpr, …)`-ed file is edited. New library features -------------------- -* `fieldoffset` now also accepts the field name as a symbol as `fieldtype` already did ([#58100]). -* `sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). -* `Base.AbstractOneTo` is added as a supertype of one-based axes, with `Base.OneTo` as its subtype ([#56902]). -* `takestring!(::IOBuffer)` removes the content from the buffer, returning the content as a `String`. +* `IOContext` supports a new boolean `hexunsigned` option that allows for printing unsigned integers in + decimal instead of hexadecimal ([#60267]). +* `lazy"..."` strings now support a flag `lazy"..."c` that adds `compact` and `limit` flags to the + `IOContext` for final output-string generation ([#61887]). +* The `StringView` type wraps an `AbstractVector{UInt8}` and interprets it as a UTF-8 encoded string, + superseding the [StringViews.jl](https://github.com/JuliaStrings/StringViews.jl) package ([#60526]). +* Package precompilation now supports running precompilation in + a background task and has new interactive keyboard controls: + `c` to cleanly cancel immediately, `d` to detach, `i` for a profile peek, + `v` to toggle verbose mode showing elapsed time, CPU%, and memory usage, and `?` for help ([#60943]). +* Instances of an `Enum` can now be given their own docstrings within the `@enum` definition ([#61955]). +* New methods `readdir(path, DirEntry)` and `readdir(::DirEntry, DirEntry)` return directory contents + along with the type of the entries in a vector of new `DirEntry` objects to provide more efficient `isfile` + etc. checks. `readdir(::DirEntry)` accepts a `DirEntry` as input and, like `readdir(::AbstractString)`, + returns a `Vector{String}` of names. `DirEntry` is exported from `Base` ([#55358]). +* New public but unexported function `Base.unsetindex!` unsets the reference from an array + or a `MemoryRef` to its value, making it as if it was uninitialized. +* Calls to `wait` on one-shot `Timer`s that have already triggered no longer throw `EOFError`. Previously + only the first `wait` returned and subsequent `wait` calls would throw ([#62539]) +* When the display height is too small to show any array entries, the `text/plain` array display + (used e.g. by the REPL and when logging values with `@info` etc.) now shows as many entries as + fit on a single line, truncated to the display width, instead of showing no data at all ([#62543]). +* The element type of broadcast expressions now uses regular inference machinery rather than an idiosyncratic + heuristic. This can help fused or empty broadcasts infer to more precise element types ([#62564]). Standard library changes ------------------------ +* `codepoint(c)` now succeeds for overlong encodings. `Base.ismalformed`, `Base.isoverlong`, and + `Base.show_invalid` are now `public` and documented (but not exported) ([#55152]). +* The `Precompiling` messages printed while loading name packages without their uuid when the + name is unambiguous in the environment, name extensions by their parent package, and say which + dependency is already loaded at a different version when that is why a cache was not reused ([#63185]). + #### JuliaSyntaxHighlighting #### LinearAlgebra +#### Markdown + +* Support "raw" or "inline" HTML inside Markdown data ([#60629], [#60632], [#60732]). +* Support autolinks for email addresses ([#60570]). +* Many improvements and bugfixes for rendering Markdown lists in a terminal ([#55456], [#60519]). +* Strikethrough text via `~strike~` or `~~through~~` is now supported by the Markdown parser ([#60537]). +* Many, many bug fixes and minor tweaks; overall behavior is now much closer to CommonMark ([#59977], [#60502]). + #### Profile #### Random -* `randperm!` and `randcycle!` now support non-`Array` `AbstractArray` inputs, assuming they are mutable and their indices are one-based ([#58596]). - #### REPL -* The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]). -* Display of repeated frames and cycles in stack traces has been improved by bracketing them in the trace and treating them consistently ([#55841]). +* Tab completion now supports `\escape` for `⎋` and `\xmark` for `✗`. +* The Julia REPL now emits OSC 133 semantic prompt markers for terminal integration. +* A `using`/`import` statement that loads several packages, such as `using A, B, C`, now precompiles + all of them (and the extensions they make loadable) in a single parallel session, rather than one + session per package ([#63185]). + +#### Sockets + +* `getsockname` now also accepts a `UDPSocket`, returning the address and port it is bound to ([#63091]). + +#### SharedArrays + +* `close(::SharedArray)` eagerly releases the shared-memory mappings referenced through the + array on all processes, e.g. so the file backing a file-backed `SharedArray` can be deleted + immediately ([#62488]). #### Test +* Pressing `^C` twice at an empty `julia>` prompt now cancels all still-running + work started by earlier evaluations (e.g. a runaway `@async` task spewing + output): each REPL evaluation runs under its own cancellation source, linked + under one session-level source that the repeated press cancels ([#47839]). -* Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). -* Transparent test sets (`@testset let`) now show context when tests error ([#58727]). -* `@test_throws` now supports a three-argument form `@test_throws ExceptionType pattern expr` to test both exception type and message pattern in one call ([#59117]). +#### Test + +* `@test`, `@test_throws`, and `@test_broken` now support a `context` keyword argument that provides + additional information displayed on test failure. This is useful for debugging which specific case failed + in parameterized tests ([#60501]). +* `@test_throws`, `@test_warn`, `@test_nowarn`, `@test_logs`, and `@test_deprecated` now support + `broken` and `skip` keyword arguments for consistency with `@test` ([#60543]). +* New functions `detect_closure_boxes` and `detect_closure_boxes_all` find methods that allocate `Core.Box` + in their lowered code, which can indicate performance issues from captured variables in closures ([#60478]). + +* `detect_unbound_args` now uses a conservative rule derived from how subtyping assigns values + to static parameters, instead of older heuristics. It detects previously missed + possibly-unbound parameters (such as `f(::Type{<:T}) where {T}`, which leaves `T` + unbound when called with `Union{}`, or `f(::Vector{<:T}) where {T}` with a + `Vector{Union{}}` argument), and no longer reports methods whose problematic calls are + all shadowed by more specific methods (such as a `f(::Type{Union{}})` fallback), or + whose lowered bodies never read the possibly-unbound parameters. Parameters left + unbound only by calls with `Union{}` type parameters are reported only with the new + `ambiguous_bottom=true` keyword argument, as for `detect_ambiguities` ([#62405]). + +#### Dates + +* `unix2datetime` now accepts a keyword argument `localtime=true` to use the host system's local time zone instead of UTC ([#50296]). #### InteractiveUtils -* Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}`. Type-annotated variables as in `f(val::Int; kw::Float64)` are not evaluated if the type annotation provides the necessary information, making this syntax compatible with signatures found in stacktraces ([#57909], [#58222]). -* Code introspection macros such as `@code_lowered` and `@code_typed` now have a much better support for broadcasting expressions, including broadcasting assignments of the form `x .+= f(y)` ([#58349]). +* `less`/`@less` and `edit`/`@edit` are now supported for documented variables ([#53539]). +* A new `@methods` macro lists all methods applicable to a call expression, using the types of + the given arguments, e.g. `@methods isvalid('a', 1)` or `@methods isvalid(::AbstractChar, ::Integer)` ([#62311]). + +#### Dates + +#### TOML + +* The parsing functions (`TOML.parsefile`, `TOML.parse`, and their `try` variants) can now capture + the comments of a document into a `TOML.Comments` object via the new `comments` keyword argument, + and `TOML.print` can write them back out via its new `comments` keyword argument. This allows + modifying a TOML file without losing its comments ([#62672]). External dependencies --------------------- @@ -94,4 +304,12 @@ External dependencies Tooling Improvements -------------------- +Deprecated or removed +--------------------- + +* Storing into a `Threads.Atomic` with the plain `a[] = v` form (i.e. `setindex!`) is deprecated in + favor of `@atomic a[] = v`. The plain form makes read-modify-write expressions such as `a[] += 1` + look atomic even though they expand to a separate, non-atomic load and store; use `@atomic a[] += 1` + or `Threads.atomic_add!` for an atomic update. Reading with `a[]` is unchanged ([#62382]). + diff --git a/README.md b/README.md index cbd1b11e982cb..3dc7f58657613 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Continuous integration + @@ -43,7 +44,7 @@ and installing Julia, below. ## Resources - **Homepage:** -- **Install:** +- **Install:** - **Source code:** - **Documentation:** - **Packages:** @@ -53,9 +54,16 @@ and installing Julia, below. - **YouTube:** - **Code coverage:** -New developers may find the notes in -[CONTRIBUTING](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md) -helpful to start contributing to the Julia codebase. + +## Contributing to Julia + +We welcome contributions from developers of all experience levels, including bug fixes, +documentation improvements, tests, and performance enhancements. + +New contributors are encouraged to start by reading [CONTRIBUTING.md](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md). + +> [!IMPORTANT] +> If your pull request contains substantive contributions from a generative AI tool, please disclose so with details, and review all changes before opening. This also applies to other content, such as issues, discussions, and comments. ### Learning Julia @@ -66,11 +74,11 @@ helpful to start contributing to the Julia codebase. The recommended way of installing Julia is to use `juliaup` which will install the latest stable `julia` for you and help keep it up to date. It can also let you install and run different Julia versions simultaneously. Instructions for -this can be found [here](https://julialang.org/install/). If you want to manually -download specific Julia binaries, you can find those on the [downloads -page](https://julialang.org/downloads/). The downloads page also provides +this can be found [here](https://julialang.org/downloads/). If you want to manually +download specific Julia binaries, you can find those on the [Manual Downloads +page](https://julialang.org/downloads/manual-downloads/). The downloads page also provides details on the [different tiers of -support](https://julialang.org/downloads/#supported_platforms) for OS and +support](https://julialang.org/downloads/support) for OS and platform combinations. If everything works correctly, you will get a `julia` program and when you run @@ -95,9 +103,9 @@ Then, acquire the source code by cloning the git repository: and then use the command prompt to change into the resulting julia directory. By default, you will be building the latest unstable version of Julia. However, most users should use the [most recent stable version](https://github.com/JuliaLang/julia/releases) -of Julia. You can get this version by running: +of Julia. You can get this version by running: (replace `[tag]` with the desired tag) - git checkout v1.11.5 + git checkout [tag] To build the `julia` executable, run `make` from within the julia directory. diff --git a/THIRDPARTY.md b/THIRDPARTY.md index 06973b0163e5e..7e3e5c604c138 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -4,6 +4,7 @@ and some utilities (most of the rest of the files in this repository). See below for exceptions. - [crc32c.c](https://stackoverflow.com/questions/17645167/implementing-sse-4-2s-crc32c-in-software) (CRC-32c checksum code by Mark Adler) [[ZLib](https://opensource.org/licenses/Zlib)]. +- [dl-cache.h](https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html) (for reading ld-cache files on startup) [LGPL2.1+] - [LDC](https://github.com/ldc-developers/ldc/blob/master/LICENSE) (for ccall/cfunction ABI definitions) [BSD-3]. The portion of code that Julia uses from LDC is [BSD-3] licensed. - [LLVM](https://releases.llvm.org/3.9.0/LICENSE.TXT) (for parts of src/disasm.cpp) [UIUC] - [NetBSD](https://www.netbsd.org/about/redistribution.html) (for setjmp, longjmp, and strptime implementations on Windows) [BSD-3] diff --git a/VERSION b/VERSION index 181a714af3a79..cd293038ed3bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.0-DEV +1.14.0-DEV diff --git a/base/.gitignore b/base/.gitignore index 0fab5b41fda08..4d02a98c01000 100644 --- a/base/.gitignore +++ b/base/.gitignore @@ -7,5 +7,6 @@ /uv_constants.jl /version_git.jl /version_git.jl.phony +/version_git_dirty /userimg.jl /JuliaSyntax diff --git a/base/Base.jl b/base/Base.jl index 9d510b5c5d47c..b9ab07d9e5207 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -30,12 +30,46 @@ let os = ccall(:jl_get_UNAME, Any, ()) end end +# metaprogramming +include("meta.jl") +using .Meta +using .Meta: is_id_char, parser_for_module + +# Strings +include("multimedia.jl") +using .Multimedia + +include("char.jl") +function array_new_memory(mem::Memory{UInt8}, newlen::Int) + # add an optimization to array_new_memory for StringVector + if (@assume_effects :total @ccall jl_genericmemory_owner(mem::Any,)::Any) === mem + # TODO: when implemented, this should use a memory growing call + return typeof(mem)(undef, newlen) + else + # If data is in a String, keep it that way. + # When implemented, this could use jl_gc_expand_string(oldstr, newlen) as an optimization + str = _string_n(newlen) + return (@assume_effects :total !:consistent @ccall jl_string_to_genericmemory(str::Any,)::Memory{UInt8}) + end +end +include("strings/basic.jl") +include("strings/string.jl") +include("strings/substring.jl") +include("strings/cstring.jl") + +include("cartesian.jl") +using .Cartesian +include("hashing.jl") +include("osutils.jl") + # subarrays include("subarray.jl") include("views.jl") +# String views +include("strings/stringview.jl") + # numeric operations -include("hashing.jl") include("div.jl") include("twiceprecision.jl") include("complex.jl") @@ -58,35 +92,17 @@ include("reduce.jl") include("reshapedarray.jl") include("reinterpretarray.jl") -include("multimedia.jl") -using .Multimedia - # Some type include("some.jl") include("dict.jl") include("set.jl") -# Strings -include("char.jl") -function array_new_memory(mem::Memory{UInt8}, newlen::Int) - # add an optimization to array_new_memory for StringVector - if (@assume_effects :total @ccall jl_genericmemory_owner(mem::Any,)::Any) === mem - # TODO: when implemented, this should use a memory growing call - return typeof(mem)(undef, newlen) - else - # If data is in a String, keep it that way. - # When implemented, this could use jl_gc_expand_string(oldstr, newlen) as an optimization - str = _string_n(newlen) - return (@assume_effects :total !:consistent @ccall jl_string_to_genericmemory(str::Any,)::Memory{UInt8}) - end -end -include("strings/basic.jl") -include("strings/string.jl") -include("strings/substring.jl") -include("strings/cstring.jl") - -include("osutils.jl") +# Dynamic scopes (types only; the ScopedValues API is included much later) +include("scope.jl") +# Cancellation tokens (the `cancel` keyword-argument machinery is used from +# the I/O layer onwards) +include("cancellation.jl") # Core I/O include("io.jl") @@ -94,6 +110,7 @@ include("iobuffer.jl") # Concurrency (part 1) include("linked_list.jl") +include("park.jl") include("condition.jl") include("threads.jl") include("lock.jl") @@ -101,6 +118,29 @@ include("lock.jl") # strings & printing include("intfuncs.jl") include("strings/strings.jl") + +#= +isdebugbuild is defined here as this is imported in libdl.jl (included in libc.jl) +=# +""" + isdebugbuild() + +Return `true` if julia is a debug version. +""" +function isdebugbuild() + return ccall(:jl_is_debugbuild, Cint, ()) != 0 +end + +# Enable dynamic library loading +module Sys end # Sys is populated in stages during bootstrap +Core.eval(Sys, :(include("osinfo.jl"))) +module Filesystem end # Filesystem is populated in stages during bootstrap +Core.eval(Filesystem, :(include("path.jl"))) +using .Filesystem +include("libc.jl") # Libdl (include in libc.jl) is required for regex.jl +using .Libc: getpid, gethostname, time, memcpy, memset, memmove, memcmp + +# More strings & printing include("regex.jl") include("parse.jl") include("shell.jl") @@ -112,8 +152,6 @@ include("arrayshow.jl") include("methodshow.jl") # multidimensional arrays -include("cartesian.jl") -using .Cartesian include("multidimensional.jl") include("broadcast.jl") @@ -127,16 +165,8 @@ include("missing.jl") # version include("version.jl") -#= -isdebugbuild is defined here as this is imported in libdl.jl (included in libc.jl) -The method is added in util.jl -=# -function isdebugbuild end - # system & environment -include("sysinfo.jl") -include("libc.jl") -using .Libc: getpid, gethostname, time, memcpy, memset, memmove, memcmp +Core.eval(Sys, :(include("sysinfo.jl"))) const USING_STOCK_GC = occursin("stock", GC.gc_active_impl()) @@ -152,14 +182,11 @@ include("channels.jl") include("partr.jl") include("task.jl") include("threads_overloads.jl") -include("weakkeydict.jl") +include("weakdict.jl") # ScopedValues include("scopedvalues.jl") -# metaprogramming -include("meta.jl") - # Logging include("logging/logging.jl") using .CoreLogging @@ -175,11 +202,11 @@ include("libuv.jl") include("asyncevent.jl") include("iostream.jl") include("stream.jl") -include("filesystem.jl") -using .Filesystem +Core.eval(Filesystem, :(include("filesystem.jl"))) include("cmd.jl") include("process.jl") include("terminfo.jl") +include("Terminals.jl") # Moved from REPL to reduce invalidations include("secretbuffer.jl") # core math functions @@ -246,7 +273,7 @@ include("irrationals.jl") include("mathconstants.jl") using .MathConstants: ℯ, π, pi -# experimental API's +# experimental APIs include("experimental.jl") # utilities @@ -257,7 +284,6 @@ include("errorshow.jl") include("util.jl") include("initdefs.jl") -Filesystem.__postinit__() # worker threads include("threadcall.jl") @@ -265,11 +291,8 @@ include("threadcall.jl") # code loading include("uuid.jl") include("pkgid.jl") -include("toml_parser.jl") +include("toml/toml.jl") include("linking.jl") -module StaticData -include("staticdata.jl") -end include("loading.jl") # BinaryPlatforms, used by Artifacts. Needs `Sort`. @@ -317,12 +340,19 @@ a_method_to_overwrite_in_test() = inferencebarrier(1) @eval Core const Compiler = $Base.Compiler @eval Compiler const fl_parse = $Base.fl_parse -# External libraries vendored into Base +# Compiler frontend Core.println("JuliaSyntax/src/JuliaSyntax.jl") -include(@__MODULE__, string(BUILDROOT, "JuliaSyntax/src/JuliaSyntax.jl")) # include($BUILDROOT/base/JuliaSyntax/JuliaSyntax.jl) +include(@__MODULE__, string(DATAROOT, "julia/JuliaSyntax/src/JuliaSyntax.jl")) +JuliaSyntax.enable_in_core!(true; freeze_world_age=false) + +# May be replaced in incremental sysimage build after-the-fact +const JuliaLowering = nothing + +set_syntax_version(Base, VERSION) end_base_include = time_ns() +Filesystem.__postinit__() const _sysimage_modules = PkgId[] in_sysimage(pkgid::PkgId) = pkgid in _sysimage_modules @@ -370,6 +400,188 @@ function start_profile_listener() ccall(:jl_set_peek_cond, Cvoid, (Ptr{Cvoid},), cond.handle) end +# The ^C episode source: the cancellation token source governing the current +# interactive foreground evaluation. A fresh source is installed per +# evaluation (severities are monotonic, so a cancelled source is never +# reused); this Ref roots it while the C side mirrors a raw pointer for the +# signal thread's lock-free reads. +# The current ^C episode: the governing source (rooted here; `nothing` +# between episodes) paired with the C-side episode generation the mirror +# swap returned - the dispatch pass consumes a pending press only when it +# targeted this very generation, so a press for a previous episode can +# never cancel a newly installed one. +const _sigint_episode = Ref{Tuple{Union{Nothing, CancellationTokenSource}, UInt64}}((nothing, 0)) +# The task driving the current foreground evaluation (the caller of +# sigint_new_episode!). Nothing in Base reads it yet: the ^C escalation +# machinery (follow-up PR) targets it directly when the running computation +# has no published token binding, and Distributed points it at the handler +# of the most recent remotely-submitted request. +const _sigint_foreground_task = Ref{Union{Nothing, Task}}(nothing) + +""" + Base.sigint_new_episode!([src::CancellationTokenSource]) -> CancellationToken + +Install `src` (a fresh standalone source by default) as the ^C episode +source and return its token. The owner of an interactive session (e.g. the +REPL backend, or the script driver) calls this before each foreground +evaluation and runs the evaluation in a dynamic scope carrying the returned +token (`@with Base.CANCEL_TOKEN => tok ...`), so that ^C cancels exactly +that evaluation. + +The caller chooses the source's place in the cancellation graph. The REPL +passes a fresh *evaluation* source - a child of the session source (see the +session tree in base/client.jl) - so work an evaluation leaves behind stays +sweepable via [`cancel_session_work!`](@ref) after its episode closes. The +session-covering episode installed by `_start` is deliberately a standalone +root: everything outside per-evaluation scopes (including the REPL's own +machinery) runs under it, and a session sweep must never cancel that. +""" +function sigint_new_episode!(src::CancellationTokenSource=CancellationTokenSource()) + _sigint_foreground_task[] = current_task() + # Publish the C-side mirror. The episode Ref roots the new source; the + # signal thread's use of the mirror is GC-excluded rather than rooted, + # so nothing needs to keep the outgoing source alive (see + # jl_sigint_request_cancellation). + gen = ccall(:jl_set_sigint_source, UInt64, (Any,), src) + _sigint_episode[] = (src, gen) + return CancellationToken(src) +end + +# Close the ^C episode without installing a new source (e.g. when the work +# item completes; the session installs a fresh source before its next +# evaluation). +function sigint_close_episode!() + gen = ccall(:jl_set_sigint_source, UInt64, (Any,), nothing) + _sigint_episode[] = (nothing, gen) + _sigint_foreground_task[] = nothing + nothing +end + +# The active severity of the episode source, or `nothing` if it has not +# been cancelled (an alias kept for the ^C machinery's consumers). +sigint_active_severity(src::CancellationTokenSource) = cancel_severity(src) + +# One listener task runs per nonempty threadpool (a pool's threads cannot run +# another pool's tasks, and any pool may be monopolized by a busy victim). +# The listeners race to claim each notification; the pass itself is +# level-based (it re-reads the episode state), so serializing claims through +# the lock is enough. +const sigint_pass_lock = ReentrantLock() + +# One dispatch pass for a claimed ^C notification. Runs with +# `sigint_pass_lock` held; level-based (it re-reads the episode state), so +# serializing passes through the lock is enough. +function _sigint_dispatch_pass() + src, gen = _sigint_episode[] + # A press is consumed only if it targeted this episode's generation; a + # press for a previous episode self-invalidates (its source was already + # marked by the C fast path), so a late pass can never cancel a newly + # installed episode (issues #58689, #42072). + pending = ccall(:jl_consume_sigint_pending, Cint, (UInt64,), gen) != 0 + if src === nothing + # No episode source is installed. In an interactive session this + # is a between-evaluations window (a fresh source arrives with the + # next prompt) - ignore the press. Without a live interactive + # evaluator nothing can be cancelled or resumed - exit as an + # unhandled ^C would. + pending || return + backend = active_repl_backend + if backend !== nothing && !istaskdone(backend.backend_task::Task) + return + end + istaskdone(roottask) || return + exit(128 + 2) # 128 + SIGINT + end + src = src::CancellationTokenSource + if sigint_active_severity(src) === nothing + # The source is unmarked: only a press that targeted exactly this + # episode warrants delivering to it. + pending || return + cancel!(src, CANCEL_REQUEST_SAFE) + elseif pending + # The press was delivered entirely by the C-side fast path with + # no wake-up walk ever run - or this is a repeat press. Parked + # waiters under the episode source still need their wake; + # redeliver! is level-triggered and idempotent (a no-op for + # tasks already unwinding). + redeliver!(src) + end + # else: a wakeup without a press for this episode does nothing + nothing +end + +# Arbitrate and run pending ^C dispatch. Called from the sigint listener +# tasks on their async notification, and inline from any idle thread's +# scheduler loop (see jl_dispatch_sigint_inline in src/scheduler.c) - the +# pass only needs an ordinary task context, which keeps the event loop out +# of the delivery path even when its owning thread is stuck in a foreign +# call. `trylock` (never a parking `lock`): if another pass is running, it +# drains any claim posted meanwhile, and an unclaimed flag is retried by +# the next idle iteration or listener wakeup. +function maybe_dispatch_sigint() + ccall(:jl_peek_sigint_dispatch, Cint, ()) != 0 || return + trylock(sigint_pass_lock) || return + try + while ccall(:jl_claim_sigint_dispatch, Cint, ()) != 0 + try # an error in one pass must not disable the ^C machinery + _sigint_dispatch_pass() + catch ex + try + @invokelatest showerror(stderr, ex, catch_backtrace()) + println(stderr) + catch + end + end + end + finally + unlock(sigint_pass_lock) + end + nothing +end + +function sigint_listener(cond::AsyncCondition) + while _trywait(cond) + maybe_dispatch_sigint() + end + nothing +end + +# The dispatch path must not hit the JIT on the first ^C: it runs from the +# scheduler's idle loop, and a press on a cold or loaded session would +# stall interactive response behind the compile - bake it into the image. +precompile(Tuple{typeof(maybe_dispatch_sigint)}) +precompile(Tuple{typeof(_sigint_dispatch_pass)}) +precompile(Tuple{typeof(cancel!), CancellationTokenSource, CancellationRequest}) +precompile(Tuple{typeof(redeliver!), CancellationTokenSource}) + +function start_sigint_listener() + cond = AsyncCondition() + # N.B.: The condition is deliberately kept ref'd: pending async events on + # unreferenced handles are not dispatched once the loop has no live + # handles left (as in a headless script), which would make the ^C + # notification undeliverable exactly when it matters. The atexit hook + # below closes the handle before the event loop is drained for exit. + listeners = Task[] + Threads.threadpoolsize(:interactive) > 0 && + push!(listeners, errormonitor(Threads.@spawn :interactive sigint_listener(cond))) + push!(listeners, errormonitor(Threads.@spawn :default sigint_listener(cond))) + atexit() do + # destroy this callback when exiting + ccall(:jl_set_sigint_cond, Cvoid, (Ptr{Cvoid},), C_NULL) + # this will prompt any ongoing or pending event to flush also + close(cond) + # error-propagation is not needed, since the errormonitor will handle printing that better + for t in listeners + t === current_task() || _wait(t) + end + end + finalizer(cond) do c + # if something goes south, still make sure we aren't keeping a reference in C to this + ccall(:jl_set_sigint_cond, Cvoid, (Ptr{Cvoid},), C_NULL) + end + ccall(:jl_set_sigint_cond, Cvoid, (Ptr{Cvoid},), cond.handle) +end + function __init__() # Base library init global _atexit_hooks_finished = false @@ -393,11 +605,20 @@ function __init__() # triggering a profile via signals is not implemented on windows start_profile_listener() end + start_sigint_listener() _require_world_age[] = get_world_counter() # Prevent spawned Julia process from getting stuck waiting on Tracy to connect. delete!(ENV, "JULIA_WAIT_FOR_TRACY") - if get_bool_env("JULIA_USE_FLISP_PARSER", false) === false - JuliaSyntax.enable_in_core!() + if get_bool_env("JULIA_USE_FLISP_PARSER", false) === true + JuliaSyntax.enable_in_core!(false) + else + JuliaSyntax.enable_in_core!(true; freeze_world_age=true) + end + + if JuliaLowering !== nothing && get_bool_env("JULIA_USE_FLISP_LOWERING", true) === false + # This is not available by default, but JuliaLowering can be added to + # Base after-the-fact via an incremental sysimage build. + JuliaLowering.activate!() end CoreLogging.global_logger(CoreLogging.ConsoleLogger()) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index ef448a02a15e9..e1732d83c98fa 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -30,6 +30,7 @@ include(path::String) = include(Base, path) struct IncludeInto <: Function m::Module + IncludeInto(m::Module) = new(m) end (this::IncludeInto)(fname::AbstractString) = include(this.m, fname) @@ -61,7 +62,7 @@ function setproperty!(x, f::Symbol, v) return setfield!(x, f, val) end -typeof(function getproperty end).name.constprop_heuristic = Core.FORCE_CONST_PROP +typeof(function getproperty end).name.constprop_heuristic = or_int(Core.FORCE_CONST_PROP, Core.DISABLE_SEMI_CONCRETE_EVAL) typeof(function setproperty! end).name.constprop_heuristic = Core.FORCE_CONST_PROP dotgetproperty(x, f) = getproperty(x, f) @@ -141,6 +142,25 @@ import Core: @doc, @__doc__, WrappedException, @int128_str, @uint128_str, @big_s # Export list include("exports.jl") +function set_syntax_version end +_topmod(m::Module) = ccall(:jl_base_relative_to, Any, (Any,), m)::Module +function _setup_module!(mod::Module, Core.@nospecialize syntax_ver) + # using Base + Core._using(mod, _topmod(mod), UInt8(0)) + Core.declare_const(mod, :include, IncludeInto(mod)) + Core.declare_const(mod, :eval, Core.EvalInto(mod)) + if syntax_ver === nothing + # two cases: (1) VERSION is assumed in bootstrap, and (2) after + # bootstrap, NON_VERSIONED_SYNTAX module forms have no version + if Core._parse === nothing || Core._parse === Base.fl_parse + return nothing + end + syntax_ver = NON_VERSIONED_SYNTAX + end + set_syntax_version(mod, syntax_ver) + return nothing +end + # core docsystem include("docs/core.jl") Core.atdoc!(CoreDocs.docm) @@ -151,9 +171,9 @@ eval(m::Module, x) = Core.eval(m, x) include("public.jl") if false - # simple print definitions for debugging. enable these if something + # Simple print definitions for debugging. Enable these if something # goes wrong during bootstrap before printing code is available. - # otherwise, they just just eventually get (noisily) overwritten later + # otherwise, they eventually get (noisily) overwritten later global show, print, println show(io::IO, x) = Core.show(io, x) print(io::IO, a...) = Core.print(io, a...) @@ -213,7 +233,7 @@ include("options.jl") # to forward to invoke function Core.kwcall(kwargs::NamedTuple, ::typeof(invoke), f, T, args...) @inline - # prepend kwargs and f to the invoked from the user + # prepend kwargs and f to the invoke from the user T = rewrap_unionall(Tuple{Core.Typeof(kwargs), Core.Typeof(f), (unwrap_unionall(T)::DataType).parameters...}, T) return invoke(Core.kwcall, T, kwargs, f, args...) end @@ -227,6 +247,8 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(applicable), @nospecialize(arg return applicable(Core.kwcall, kwargs, args...) end function Core._hasmethod(@nospecialize(f), @nospecialize(t)) # this function has a special tfunc (TODO: make this a Builtin instead like applicable) + Core.@nospecializeinfer + @noinline tt = rewrap_unionall(Tuple{Core.Typeof(f), (unwrap_unionall(t)::DataType).parameters...}, t) return Core._hasmethod(tt) end @@ -324,16 +346,16 @@ using .Checked include("indices.jl") include("genericmemory.jl") include("array.jl") +include("abstractset.jl") +include("abstractdict.jl") +include("iddict.jl") +include("idset.jl") include("abstractarray.jl") include("baseext.jl") include("c.jl") -include("abstractset.jl") include("bitarray.jl") include("bitset.jl") -include("abstractdict.jl") -include("iddict.jl") -include("idset.jl") include("ntuple.jl") include("iterators.jl") using .Iterators: zip, enumerate, only @@ -348,7 +370,6 @@ using .Order include("coreir.jl") include("module.jl") -include("invalidation.jl") BUILDROOT::String = "" DATAROOT::String = "" @@ -377,6 +398,7 @@ process_sysimg_args!() function isready end include(strcat(DATAROOT, "julia/Compiler/src/Compiler.jl")) +using .Compiler.ReinferUtils: ReinferUtils, invalidate_code_for_globalref! const _return_type = Compiler.return_type diff --git a/base/Enums.jl b/base/Enums.jl index d4094945853ec..e657ac7dfea5f 100644 --- a/base/Enums.jl +++ b/base/Enums.jl @@ -162,12 +162,19 @@ macro enum(T::Union{Symbol,Expr}, syms...) namemap = Dict{basetype,Symbol}() lo = hi = i = zero(basetype) hasexpr = false + docs = Dict{Symbol,Expr}() if length(syms) == 1 && syms[1] isa Expr && syms[1].head === :block syms = syms[1].args end for s in syms s isa LineNumberNode && continue + if isa(s, Expr) && s.head === :macrocall && s.args[1] == GlobalRef(Core, Symbol("@doc")) + doc = s + s = s.args[4] + else + doc = nothing + end if isa(s, Symbol) if i == typemin(basetype) && !isempty(values) throw(ArgumentError(LazyString("overflow in value \"", s, "\" of Enum ", typename))) @@ -203,6 +210,9 @@ macro enum(T::Union{Symbol,Expr}, syms...) else hi = max(hi, i) end + if doc !== nothing + docs[s] = doc + end i += oneunit(i) end blk = quote @@ -228,12 +238,17 @@ macro enum(T::Union{Symbol,Expr}, syms...) end if isa(typename, Symbol) for (i, sym) in namemap - push!(blk.args, :(const $(esc(sym)) = $(esc(typename))($i))) + ex = :(const $(esc(sym)) = $(esc(typename))($i)) + if haskey(docs, sym) + docs[sym].args[4] = ex + ex = docs[sym] + end + push!(blk.args, ex) end end push!(blk.args, :nothing) blk.head = :toplevel - return blk + return Base.replace_linenums!(blk, __source__) end end # module diff --git a/base/Makefile b/base/Makefile index 34791f7b4b0d4..0fa4be14b50c4 100644 --- a/base/Makefile +++ b/base/Makefile @@ -17,17 +17,22 @@ else PCRE_INCL_PATH := $(build_includedir)/pcre2.h endif -define parse_features -@printf "%s\n" "# $(2) features" >> $@ -@$(call PRINT_PERL, cat $(SRCDIR)/../src/features_$(1).h | perl -lne 'print "const JL_$(2)_$$1 = UInt32($$2)" if /^\s*JL_FEATURE_DEF(?:_NAME)?\(\s*(\w+)\s*,\s*([^,]+)\s*,.*\)\s*(?:\/\/.*)?$$/' >> $@) +# Extract feature indices from cpufeatures generated headers. +# The FeatureIndex enum has entries like: FEAT_SSE3 = 108, +# We convert them to: const JL_X86_sse3 = UInt32(108) +CPUFEATURES_GENDIR := $(build_includedir)/cpufeatures + +define parse_cpufeatures +@printf "%s\n" "# $(2) features (from cpufeatures)" >> $@ +@$(call PRINT_PERL, perl -lne 'if (/^\s*FEAT_(\w+)\s*=\s*(\d+)/) { my $$n = lc($$1); print "const JL_$(2)_$$n = UInt32($$2)" }' $(1) >> $@) @printf "\n" >> $@ endef -$(BUILDDIR)/features_h.jl: $(SRCDIR)/../src/features_x86.h $(SRCDIR)/../src/features_aarch32.h $(SRCDIR)/../src/features_aarch64.h +$(BUILDDIR)/features_h.jl: $(wildcard $(CPUFEATURES_GENDIR)/target_tables_*.h) @-rm -f $@ - @$(call parse_features,x86,X86) - @$(call parse_features,aarch32,AArch32) - @$(call parse_features,aarch64,AArch64) + @$(call parse_cpufeatures,$(CPUFEATURES_GENDIR)/target_tables_x86_64.h,X86) + @$(call parse_cpufeatures,$(CPUFEATURES_GENDIR)/target_tables_aarch64.h,AArch64) + @$(call parse_cpufeatures,$(CPUFEATURES_GENDIR)/target_tables_riscv64.h,RISCV) $(BUILDDIR)/pcre_h.jl: $(PCRE_INCL_PATH) @$(call PRINT_PERL, $(CPP) -D PCRE2_CODE_UNIT_WIDTH=8 -dM $< | perl -nle '/^\s*#define\s+PCRE2_(\w*)\s*\(?($(PCRE_CONST))\)?u?\s*$$/ and print index($$1, "ERROR_") == 0 ? "const $$1 = Cint($$2)" : "const $$1 = UInt32($$2)"' | LC_ALL=C sort > $@) @@ -71,6 +76,7 @@ endif @printf "%s\n" "const PRIVATE_LIBDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(private_libdir_rel)))) >> $@ @printf "%s\n" "const PRIVATE_LIBEXECDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(private_libexecdir_rel)))) >> $@ @printf "%s\n" "const INCLUDEDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(includedir_rel)))) >> $@ + @printf "%s\n" "const SOURCEDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(shell echo $(call cygpath_w,$(JULIAHOME)))))) >> $@ ifeq ($(DARWIN_FRAMEWORK), 1) @printf "%s\n" "const DARWIN_FRAMEWORK = true" >> $@ @printf "%s\n" "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ @@ -79,7 +85,7 @@ else endif ifeq ($(OS), Darwin) @printf "%s\n" "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ - @printf "%s\n" "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ + @printf "%s\n" "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --sdk macosx --show-sdk-version)\"" >> $@ endif @printf "%s\n" "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ @@ -160,11 +166,10 @@ endif define symlink_system_library libname_$2 := $$(notdir $(call versioned_libname,$2,$3)) -libpath_$2 := $$(shell $$(call spawn,$$(LIBWHICH)) -p $$(libname_$2) 2>/dev/null) symlink_$2: $$(build_private_libdir)/$$(libname_$2) $$(build_private_libdir)/$$(libname_$2): - @if [ -e "$$(libpath_$2)" ]; then \ - REALPATH=$$(libpath_$2); \ + @REALPATH=`$$(call spawn,$$(LIBWHICH)) -p $$(libname_$2) 2>/dev/null`; \ + if [ -e "$$$$REALPATH" ]; then \ $$(call resolve_path,REALPATH) && \ [ -e "$$$$REALPATH" ] && \ rm -f "$$@" && \ @@ -182,14 +187,14 @@ endif endef # libexec executables -symlink_p7zip: $(build_bindir)/7z$(EXE) +symlink_p7zip: $(build_private_libexecdir)/7z$(EXE) ifneq ($(USE_SYSTEM_P7ZIP),0) SYMLINK_SYSTEM_LIBRARIES += symlink_p7zip 7Z_PATH := $(shell which 7z$(EXE)) endif -$(build_bindir)/7z$(EXE): +$(build_private_libexecdir)/7z$(EXE): [ -e "$(7Z_PATH)" ] && \ rm -f "$@" && \ ln -sf "$(7Z_PATH)" "$@" @@ -306,4 +311,5 @@ clean: -rm -f $(BUILDDIR)/file_constants.jl -rm -f $(BUILDDIR)/version_git.jl -rm -f $(BUILDDIR)/version_git.jl.phony + -rm -f $(BUILDDIR)/version_git_dirty -rm -f $(build_private_libdir)/lib*.$(SHLIB_EXT)* diff --git a/stdlib/REPL/src/Terminals.jl b/base/Terminals.jl similarity index 90% rename from stdlib/REPL/src/Terminals.jl rename to base/Terminals.jl index 14ea6dd3dff77..64835f3c92911 100644 --- a/stdlib/REPL/src/Terminals.jl +++ b/base/Terminals.jl @@ -26,7 +26,6 @@ export raw! import Base: - check_open, # stream.jl displaysize, flush, pipe_reader, @@ -123,9 +122,18 @@ cmove_col(t::UnixTerminal, n) = (write(t.out_stream, '\r'); n > 1 && cmove_right if Sys.iswindows() function raw!(t::TTYTerminal,raw::Bool) if Base.ispty(t.in_stream) - run((raw ? `stty raw -echo onlcr -ocrnl opost` : `stty sane`), - t.in_stream, t.out_stream, t.err_stream) - true + try + run((raw ? `stty raw -echo onlcr -ocrnl opost` : `stty sane`), + t.in_stream, t.out_stream, t.err_stream) + true + catch ex + # Fall back to ccall if stty fails (e.g., in some CI environments) + if ex isa ProcessFailedException + ccall(:jl_tty_set_mode, Int32, (Ptr{Cvoid},Int32), t.in_stream.handle::Ptr{Cvoid}, raw) == 0 + else + rethrow() + end + end else ccall(:jl_tty_set_mode, Int32, (Ptr{Cvoid},Int32), t.in_stream.handle::Ptr{Cvoid}, raw) == 0 end diff --git a/base/abstractarray.jl b/base/abstractarray.jl index adc104dadb90b..aa537c6376d66 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -9,7 +9,7 @@ Supertype for `N`-dimensional arrays (or array-like types) with elements of type [`Array`](@ref) and other types are subtypes of this. See the manual section on the [`AbstractArray` interface](@ref man-interface-array). -See also: [`AbstractVector`](@ref), [`AbstractMatrix`](@ref), [`eltype`](@ref), [`ndims`](@ref). +See also [`AbstractVector`](@ref), [`AbstractMatrix`](@ref), [`eltype`](@ref), [`ndims`](@ref). """ AbstractArray @@ -26,7 +26,7 @@ dimension to just get the length of that dimension. Note that `size` may not be defined for arrays with non-standard indices, in which case [`axes`](@ref) may be useful. See the manual chapter on [arrays with custom indices](@ref man-custom-indices). -See also: [`length`](@ref), [`ndims`](@ref), [`eachindex`](@ref), [`sizeof`](@ref). +See also [`length`](@ref), [`ndims`](@ref), [`eachindex`](@ref), [`sizeof`](@ref). # Examples ```jldoctest @@ -39,7 +39,11 @@ julia> size(A, 2) 3 ``` """ -size(t::AbstractArray{T,N}, d) where {T,N} = d::Integer <= N ? size(t)[d] : 1 +function size(t::AbstractArray, dim) + d = Int(dim)::Int + s = size(t) + d <= length(s) ? s[d] : 1 +end """ axes(A, d) @@ -82,7 +86,7 @@ end Return the tuple of valid indices for array `A`. -See also: [`size`](@ref), [`keys`](@ref), [`eachindex`](@ref). +See also [`size`](@ref), [`keys`](@ref), [`eachindex`](@ref). # Examples @@ -225,7 +229,7 @@ For dictionary types, this will be a `Pair{KeyType,ValType}`. The definition instead of types. However the form that accepts a type argument should be defined for new types. -See also: [`keytype`](@ref), [`typeof`](@ref). +See also [`keytype`](@ref), [`typeof`](@ref). # Examples ```jldoctest @@ -261,7 +265,7 @@ elsize(A::AbstractArray) = elsize(typeof(A)) Return the number of dimensions of `A`. -See also: [`size`](@ref), [`axes`](@ref). +See also [`size`](@ref), [`axes`](@ref). # Examples ```jldoctest @@ -282,7 +286,7 @@ Return the number of elements in the collection. Use [`lastindex`](@ref) to get the last valid index of an indexable collection. -See also: [`size`](@ref), [`ndims`](@ref), [`eachindex`](@ref). +See also [`size`](@ref), [`ndims`](@ref), [`eachindex`](@ref). # Examples ```jldoctest @@ -312,7 +316,7 @@ julia> length([1 2; 3 4]) 4 ``` """ -length(t::AbstractArray) = (@inline; prod(size(t))) +length(t::AbstractArray) # `eachindex` is mostly an optimization of `keys` eachindex(itrs...) = keys(itrs...) @@ -409,9 +413,9 @@ keys(s::IndexStyle, A::AbstractArray, B::AbstractArray...) = eachindex(s, A, B.. Return the last index of `collection`. If `d` is given, return the last index of `collection` along dimension `d`. The syntaxes `A[end]` and `A[end, end]` lower to `A[lastindex(A)]` and -`A[lastindex(A, 1), lastindex(A, 2)]`, respectively. +`A[lastindex(A, 1), lastindex(A, 2)]`, respectively; see [`end`](@ref). -See also: [`axes`](@ref), [`firstindex`](@ref), [`eachindex`](@ref), [`prevind`](@ref). +See also [`axes`](@ref), [`firstindex`](@ref), [`eachindex`](@ref), [`prevind`](@ref). # Examples ```jldoctest @@ -432,9 +436,9 @@ lastindex(a, d) = (@inline; last(axes(a, d))) Return the first index of `collection`. If `d` is given, return the first index of `collection` along dimension `d`. The syntaxes `A[begin]` and `A[1, begin]` lower to `A[firstindex(A)]` and -`A[1, firstindex(A, 2)]`, respectively. +`A[1, firstindex(A, 2)]`, respectively; see [`begin`](@ref). -See also: [`first`](@ref), [`axes`](@ref), [`lastindex`](@ref), [`nextind`](@ref). +See also [`first`](@ref), [`axes`](@ref), [`lastindex`](@ref), [`nextind`](@ref). # Examples ```jldoctest @@ -456,7 +460,7 @@ firstindex(a, d) = (@inline; first(axes(a, d))) Get the first element of an iterable collection. Return the start point of an [`AbstractRange`](@ref) even if it is empty. -See also: [`only`](@ref), [`firstindex`](@ref), [`last`](@ref). +See also [`only`](@ref), [`firstindex`](@ref), [`last`](@ref). # Examples ```jldoctest @@ -479,11 +483,11 @@ end Get the first `n` elements of the iterable collection `itr`, or fewer elements if `itr` is not long enough. -See also: [`startswith`](@ref), [`Iterators.take`](@ref). - !!! compat "Julia 1.6" This method requires at least Julia 1.6. +See also [`startswith`](@ref), [`Iterators.take`](@ref). + # Examples ```jldoctest julia> first(["foo", "bar", "qux"], 2) @@ -560,7 +564,7 @@ end Return a tuple of the memory strides in each dimension. -See also: [`stride`](@ref). +See also [`stride`](@ref). # Examples ```jldoctest @@ -577,7 +581,7 @@ function strides end Return the distance in memory (in number of elements) between adjacent elements in dimension `k`. -See also: [`strides`](@ref). +See also [`strides`](@ref). # Examples ```jldoctest @@ -693,11 +697,7 @@ end Throw an error if the specified indices `I` are not in bounds for the given array `A`. """ -function checkbounds(A::AbstractArray, I...) - @inline - checkbounds(Bool, A, I...) || throw_boundserror(A, I) - nothing -end +checkbounds(A::AbstractArray, I...) """ checkbounds_indices(Bool, IA, I) @@ -721,7 +721,7 @@ See also [`checkbounds`](@ref). """ function checkbounds_indices(::Type{Bool}, inds::Tuple, I::Tuple{Any, Vararg}) @inline - return checkindex(Bool, get(inds, 1, OneTo(1)), I[1])::Bool & + return checkindex(Bool, get(inds, 1, OneTo(1)), I[1])::Bool && checkbounds_indices(Bool, safe_tail(inds), tail(I)) end @@ -750,7 +750,7 @@ false checkindex(::Type{Bool}, inds, i) = throw(ArgumentError(LazyString("unable to check bounds for indices of type ", typeof(i)))) checkindex(::Type{Bool}, inds::AbstractUnitRange, i::Real) = (first(inds) <= i) & (i <= last(inds)) checkindex(::Type{Bool}, inds::IdentityUnitRange, i::Real) = checkindex(Bool, inds.indices, i) -checkindex(::Type{Bool}, inds::OneTo{T}, i::T) where {T<:BitInteger} = unsigned(i - one(i)) < unsigned(last(inds)) +checkindex(::Type{Bool}, inds::OneTo{T}, i::T) where {T<:BitInteger} = unsigned(i -% one(i)) < unsigned(last(inds)) checkindex(::Type{Bool}, inds::AbstractUnitRange, ::Colon) = true checkindex(::Type{Bool}, inds::AbstractUnitRange, ::Slice) = true checkindex(::Type{Bool}, inds::AbstractUnitRange, i::AbstractRange) = @@ -814,7 +814,7 @@ julia> similar(falses(10), Float64, 2, 4) 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 ``` -See also: [`undef`](@ref), [`isassigned`](@ref). +See also [`undef`](@ref), [`isassigned`](@ref). """ similar(a::AbstractArray{T}) where {T} = similar(a, T) similar(a::AbstractArray, ::Type{T}) where {T} = similar(a, T, axes(a)) @@ -877,7 +877,7 @@ similar(::Type{T}, dims::Dims) where {T<:AbstractArray} = T(undef, dims) Create an empty vector similar to `v`, optionally changing the `eltype`. -See also: [`empty!`](@ref), [`isempty`](@ref), [`isassigned`](@ref). +See also [`empty!`](@ref), [`isempty`](@ref), [`isassigned`](@ref). # Examples @@ -953,7 +953,10 @@ end function copyto!(dest::AbstractArray, dstart::Integer, src) i = Int(dstart) if haslength(src) && length(dest) > 0 - @boundscheck checkbounds(dest, i:(i + length(src) - 1)) + n = length(src) + n == 0 && return dest + @boundscheck checkbounds(dest, i) + @boundscheck n <= lastindex(dest) - i + 1 || throw(BoundsError(dest, i)) for x in src @inbounds dest[i] = x i += 1 @@ -967,7 +970,7 @@ function copyto!(dest::AbstractArray, dstart::Integer, src) return dest end -# copy from an some iterable object into an AbstractArray +# copy from an iterable object into an AbstractArray function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer) if (sstart < 1) throw(ArgumentError(LazyString("source start offset (",sstart,") is < 1"))) @@ -1001,13 +1004,13 @@ function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer, n:: n < 0 && throw(ArgumentError(LazyString("tried to copy n=",n, ", elements, but n should be non-negative"))) n == 0 && return dest - dmax = dstart + n - 1 inds = LinearIndices(dest) - if (dstart ∉ inds || dmax ∉ inds) | (sstart < 1) - sstart < 1 && throw(ArgumentError(LazyString("source start offset (", - sstart,") is < 1"))) - throw(BoundsError(dest, dstart:dmax)) - end + sstart < 1 && throw(ArgumentError(LazyString("source start offset (", + sstart,") is < 1"))) + (dstart ∈ inds && n <= last(inds) - dstart + 1) || throw(BoundsError(dest, dstart)) + dstart = Int(dstart) + n = Int(n) + dmax = dstart + n - 1 y = iterate(src) for j = 1:(sstart-1) if y === nothing @@ -1023,7 +1026,7 @@ function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer, n:: "expected at least ",sstart," got ", sstart-1))) end val, st = y - i = Int(dstart) + i = dstart @inbounds dest[i] = val for val in Iterators.take(Iterators.rest(src, st), n-1) i += 1 @@ -1136,8 +1139,11 @@ function copyto!(dest::AbstractArray, dstart::Integer, n < 0 && throw(ArgumentError(LazyString("tried to copy n=", n," elements, but n should be non-negative"))) destinds, srcinds = LinearIndices(dest), LinearIndices(src) - (checkbounds(Bool, destinds, dstart) && checkbounds(Bool, destinds, dstart+n-1)) || throw(BoundsError(dest, dstart:dstart+n-1)) - (checkbounds(Bool, srcinds, sstart) && checkbounds(Bool, srcinds, sstart+n-1)) || throw(BoundsError(src, sstart:sstart+n-1)) + (checkbounds(Bool, destinds, dstart) && n <= last(destinds) - dstart + 1) || throw(BoundsError(dest, dstart)) + (checkbounds(Bool, srcinds, sstart) && n <= last(srcinds) - sstart + 1) || throw(BoundsError(src, sstart)) + dstart = Int(dstart) + sstart = Int(sstart) + n = Int(n) src′ = unalias(dest, src) @inbounds for i = 0:n-1 dest[dstart+i] = src′[sstart+i] @@ -1207,6 +1213,7 @@ function copymutable(a::AbstractArray) copyto!(similar(a), a) end copymutable(itr) = collect(itr) +copymutable(a::Array) = copy(a) zero(x::AbstractArray{T}) where {T<:Number} = fill!(similar(x, typeof(zero(T))), zero(T)) zero(x::AbstractArray{S}) where {S<:Union{Missing, Number}} = fill!(similar(x, typeof(zero(S))), zero(S)) @@ -1238,16 +1245,19 @@ oneunit(x::AbstractMatrix{T}) where {T} = _one(oneunit(T), x) iterate_starting_state(A) = iterate_starting_state(A, IndexStyle(A)) iterate_starting_state(A, ::IndexLinear) = firstindex(A) iterate_starting_state(A, ::IndexStyle) = (eachindex(A),) -@inline iterate(A::AbstractArray, state = iterate_starting_state(A)) = _iterate(A, state) -@inline function _iterate(A::AbstractArray, state::Tuple) - y = iterate(state...) +# avoid fragile edges from union-splitting these helpers for abstract arrays (#61667) +typeof(iterate_starting_state).name.max_methods = UInt8(1) +@inline iterate(A::AbstractArray, state = iterate_starting_state(A)) = _iterate_abstractarray(A, state) +@inline function _iterate_abstractarray(A::AbstractArray, state::Tuple) + y = iterate(state...)::Union{Nothing,Tuple} y === nothing && return nothing A[y[1]], (state[1], tail(y)...) end -@inline function _iterate(A::AbstractArray, state::Integer) +@inline function _iterate_abstractarray(A::AbstractArray, state::Integer) checkbounds(Bool, A, state) || return nothing A[state], state + one(state) end +typeof(_iterate_abstractarray).name.max_methods = UInt8(1) isempty(a::AbstractArray) = (length(a) == 0) @@ -1493,7 +1503,19 @@ function _setindex!(::IndexCartesian, A::AbstractArray, v, I::Vararg{Int,M}) whe r end -_unsetindex!(A::AbstractArray, i::Integer) = _unsetindex!(A, to_index(i)) +""" + unsetindex!(A::AbstractArray, i::Integer) -> A + +Unset the reference from `A` at index `i` to its value and return `A`. +The value left in `A[i]`, if any, is implementation-dependent, but after +calling this function, `A` at index `i` no longer holds any reference to a value, +so it no longer protects any underlying resources from being freed (i.e. garbage collected, +or finalizers run). + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. +""" +function unsetindex! end """ parent(A) @@ -1601,7 +1623,7 @@ parts can specialize this method to return the concatenation of the `dataids` of their component parts. A typical definition for an array that wraps a parent is `Base.dataids(C::CustomArray) = dataids(C.parent)`. """ -dataids(A::AbstractArray) = (UInt(objectid(A)),) +dataids(A::AbstractArray) = (objectid(A),) dataids(A::Memory) = (UInt(A.ptr),) dataids(A::Array) = dataids(A.ref.mem) dataids(::AbstractRange) = () @@ -1680,13 +1702,32 @@ typed_hcat(::Type{T}) where {T} = Vector{T}() ## cat: special cases vcat(X::T...) where {T} = T[ X[i] for i=eachindex(X) ] vcat(X::T...) where {T<:Number} = T[ X[i] for i=eachindex(X) ] -hcat(X::T...) where {T} = T[ X[j] for i=1:1, j=eachindex(X) ] -hcat(X::T...) where {T<:Number} = T[ X[j] for i=1:1, j=eachindex(X) ] +hcat(X::T...) where {T} = T[ X[j] for _=1:1, j=eachindex(X) ] +hcat(X::T...) where {T<:Number} = T[ X[j] for _=1:1, j=eachindex(X) ] + +function vcat(X::Number...) + a = Vector{promote_typeof(X...)}(undef, length(X)) + hvncat_fill!(a, false, X) + return a +end + +function hcat(X::Number...) + a = Matrix{promote_typeof(X...)}(undef, 1, length(X)) + hvncat_fill!(a, false, X) + return a +end -vcat(X::Number...) = hvcat_fill!(Vector{promote_typeof(X...)}(undef, length(X)), X) -hcat(X::Number...) = hvcat_fill!(Matrix{promote_typeof(X...)}(undef, 1,length(X)), X) -typed_vcat(::Type{T}, X::Number...) where {T} = hvcat_fill!(Vector{T}(undef, length(X)), X) -typed_hcat(::Type{T}, X::Number...) where {T} = hvcat_fill!(Matrix{T}(undef, 1,length(X)), X) +function typed_vcat(::Type{T}, X::Number...) where {T} + a = Vector{T}(undef, length(X)) + hvncat_fill!(a, false, X) + return a +end + +function typed_hcat(::Type{T}, X::Number...) where {T} + a = Matrix{T}(undef, 1, length(X)) + hvncat_fill!(a, false, X) + return a +end vcat(V::AbstractVector...) = typed_vcat(promote_eltype(V...), V...) vcat(V::AbstractVector{T}...) where {T} = typed_vcat(T, V...) @@ -1823,7 +1864,7 @@ function cat_shape(dims, shapes::Tuple) return out_shape end # The new way to compute the shape (more inferable than combining cat_size & cat_shape, due to Varargs + issue#36454) -cat_size_shape(dims) = ntuple(zero, Val(length(dims))) +cat_size_shape(dims) = ntuple(Returns(0), Val(length(dims))) @inline cat_size_shape(dims, X, tail...) = _cat_size_shape(dims, _cshp(1, dims, (), cat_size(X)), tail...) _cat_size_shape(dims, shape) = shape @inline _cat_size_shape(dims, shape, X, tail...) = _cat_size_shape(dims, _cshp(1, dims, shape, cat_size(X)), tail...) @@ -1877,7 +1918,7 @@ end @inline cat_t(::Type{T}, X...; dims) where {T} = _cat_t(dims, T, X...) # Why isn't this called `__cat!`? -__cat(A, shape, catdims, X...) = __cat_offset!(A, shape, catdims, ntuple(zero, length(shape)), X...) +__cat(A, shape, catdims, X...) = __cat_offset!(A, shape, catdims, ntuple(Returns(0), length(shape)), X...) function __cat_offset!(A, shape, catdims, offsets, x, X...) # splitting the "work" on x from X... may reduce latency (fewer costly specializations) @@ -2230,22 +2271,6 @@ function hvcat(rows::Tuple{Vararg{Int}}, xs::T...) where T<:Number a end -function hvcat_fill!(a::Array, xs::Tuple) - nr, nc = size(a,1), size(a,2) - len = length(xs) - if nr*nc != len - throw(ArgumentError("argument count $(len) does not match specified shape $((nr,nc))")) - end - k = 1 - for i=1:nr - @inbounds for j=1:nc - a[i,j] = xs[k] - k += 1 - end - end - a -end - hvcat(rows::Tuple{Vararg{Int}}, xs::Number...) = typed_hvcat(promote_typeof(xs...), rows, xs...) hvcat(rows::Tuple{Vararg{Int}}, xs...) = typed_hvcat(promote_eltypeof(xs...), rows, xs...) # the following method is needed to provide a more specific one compared to LinearAlgebra/uniformscaling.jl @@ -2259,7 +2284,9 @@ function typed_hvcat(::Type{T}, rows::Tuple{Vararg{Int}}, xs::Number...) where T throw(DimensionMismatch("row $(i) has mismatched number of columns (expected $nc, got $(rows[i]))")) end end - hvcat_fill!(Matrix{T}(undef, nr, nc), xs) + a = Matrix{T}(undef, nr, nc) + hvncat_fill!(a, true, xs) + return a end typed_hvcat(::Type{T}, rows::Tuple{Vararg{Int}}, as...) where T = typed_hvncat(T, rows_to_dimshape(rows), true, as...) @@ -2384,15 +2411,13 @@ _typed_hvncat_0d_only_one() = function _typed_hvncat(::Type{T}, ::Val{N}) where {T, N} N < 0 && throw(ArgumentError("concatenation dimension must be non-negative")) - return Array{T, N}(undef, ntuple(x -> 0, Val(N))) + return Array{T, N}(undef, ntuple(Returns(0), Val(N))) end function _typed_hvncat(T::Type, ::Val{N}, xs::Number...) where N N < 0 && throw(ArgumentError("concatenation dimension must be non-negative")) - A = cat_similar(xs[1], T, (ntuple(x -> 1, Val(N - 1))..., length(xs))) - hvncat_fill!(A, false, xs) - return A + return reshape(T[xs...], (ntuple(Returns(1), Val(N - 1))..., length(xs))) end function _typed_hvncat(::Type{T}, ::Val{N}, as::AbstractArray...) where {T, N} @@ -2404,7 +2429,7 @@ function _typed_hvncat(::Type{T}, ::Val{N}, as::AbstractArray...) where {T, N} throw(ArgumentError("concatenation dimension must be non-negative")) for a ∈ as ndims(a) <= N || all(x -> size(a, x) == 1, (N + 1):ndims(a)) || - return _typed_hvncat(T, (ntuple(x -> 1, Val(N - 1))..., length(as), 1), false, as...) + return _typed_hvncat(T, (ntuple(Returns(1), Val(N - 1))..., length(as), 1), false, as...) # the extra 1 is to avoid an infinite cycle end @@ -2419,7 +2444,7 @@ function _typed_hvncat(::Type{T}, ::Val{N}, as::AbstractArray...) where {T, N} end end - A = cat_similar(as[1], T, (ntuple(d -> size(as[1], d), N - 1)..., Ndim, ntuple(x -> 1, nd - N)...)) + A = cat_similar(as[1], T, (ntuple(d -> size(as[1], d), N - 1)..., Ndim, ntuple(Returns(1), nd - N)...)) k = 1 for a ∈ as for i ∈ eachindex(a) @@ -2446,7 +2471,7 @@ function _typed_hvncat(::Type{T}, ::Val{N}, as...) where {T, N} end end - A = Array{T, nd}(undef, ntuple(x -> 1, Val(N - 1))..., Ndim, ntuple(x -> 1, nd - N)...) + A = Array{T, nd}(undef, ntuple(Returns(1), Val(N - 1))..., Ndim, ntuple(Returns(1), nd - N)...) k = 1 for a ∈ as @@ -2499,7 +2524,7 @@ function _typed_hvncat(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, xs::Num return A end -function hvncat_fill!(A::Array, row_first::Bool, xs::Tuple) +function _hvncat_fill_loop!(A::Array, row_first::Bool, xs::Tuple) nr, nc = size(A, 1), size(A, 2) na = prod(size(A)[3:end]) len = length(xs) @@ -2514,7 +2539,7 @@ function hvncat_fill!(A::Array, row_first::Bool, xs::Tuple) dd = nrc * (d - 1) for i ∈ 1:nr Ai = dd + i - for j ∈ 1:nc + for _ ∈ 1:nc @inbounds A[Ai] = xs[k] k += 1 Ai += nr @@ -2528,10 +2553,83 @@ function hvncat_fill!(A::Array, row_first::Bool, xs::Tuple) end end +function hvncat_fill!(A::Array, row_first::Bool, xs::Tuple) + if @generated + N = fieldcount(xs) + N > 32 && return :(return _hvncat_fill_loop!(A, row_first, xs)) + nd = ndims(A) + if nd <= 2 + return quote + nr = size(A, 1) + nc = size(A, 2) + if nr*nc != $N + throw(ArgumentError("argument count $($N) does not match specified shape $(size(A))")) + end + if row_first + i::Int = 1 + j::Int = 1 + @nexprs $N k -> begin + @inbounds A[i, j] = xs[k] + j += 1 + if j > nc + i += 1 + j = 1 + end + end + else + @nexprs $N k -> begin + @inbounds A[k] = xs[k] + end + end + nothing + end + else + return quote + nr = size(A, 1) + nc = size(A, 2) + nrc = nr * nc + na = prod(size(A)[3:end]) + if nrc * na != $N + throw(ArgumentError("argument count $($N) does not match specified shape $(size(A))")) + end + if row_first + d::Int = 1 + i::Int = 1 + dd::Int = 0 + Ai::Int = dd + i + j::Int = 1 + @nexprs $N k -> begin + @inbounds A[Ai] = xs[k] + j += 1 + Ai += nr + if j > nc + j = 1 + i += 1 + if i > nr + i = 1 + d += 1 + dd = nrc * (d - 1) + end + Ai = dd + i + end + end + else + @nexprs $N k -> begin + @inbounds A[k] = xs[k] + end + end + nothing + end + end + else + _hvncat_fill_loop!(A, row_first, xs) + end +end + function _typed_hvncat(T::Type, dims::NTuple{N, Int}, row_first::Bool, as...) where {N} # function barrier after calculating the max is necessary for high performance nd = max(maximum(cat_ndims(a) for a ∈ as), N) - return _typed_hvncat_dims(T, (dims..., ntuple(x -> 1, nd - N)...), row_first, as) + return _typed_hvncat_dims(T, (dims..., ntuple(Returns(1), nd - N)...), row_first, as) end function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as::Tuple) where {T, N} @@ -2569,7 +2667,7 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as # discover number of rows or columns # d1 dimension is increased by 1 to appropriately handle 0-length arrays for i ∈ 1:dims[d1] - outdims[d1] += cat_size(as[i], d1) + outdims[d1] = checked_add(outdims[d1], cat_size(as[i], d1)) end # adjustment to handle 0-length arrays @@ -2582,12 +2680,12 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as blockcount = 0 elementcount = 0 for i ∈ eachindex(as) - elementcount += cat_length(as[i]) - currentdims[d1] += first_dim_zero ? 1 : cat_size(as[i], d1) + elementcount = checked_add(elementcount, cat_length(as[i])) + currentdims[d1] = checked_add(currentdims[d1], first_dim_zero ? 1 : cat_size(as[i], d1)) if currentdims[d1] == outdims[d1] currentdims[d1] = 0 for d ∈ (d2, 3:N...) - currentdims[d] += cat_size(as[i], d) + currentdims[d] = checked_add(currentdims[d], cat_size(as[i], d)) if outdims[d] == 0 # unfixed dimension blockcount += 1 if blockcount == dims[d] @@ -2616,7 +2714,7 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as outdims[d1] = 0 end - outlen = prod(outdims) + outlen = Core.checked_dims(ntuple(i -> outdims[i], Val(N))...) elementcount == outlen || throw(DimensionMismatch("mismatched number of elements; expected $(outlen), got $(elementcount)")) @@ -2640,7 +2738,7 @@ end function _typed_hvncat(T::Type, shape::NTuple{N, Tuple}, row_first::Bool, as...) where {N} # function barrier after calculating the max is necessary for high performance nd = max(maximum(cat_ndims(a) for a ∈ as), N) - return _typed_hvncat_shape(T, (shape..., ntuple(x -> shape[end], nd - N)...), row_first, as) + return _typed_hvncat_shape(T, (shape..., ntuple(Returns(shape[end]), nd - N)...), row_first, as) end function _typed_hvncat_shape(::Type{T}, shape::NTuple{N, Tuple}, row_first, as::Tuple) where {T, N} @@ -2669,7 +2767,7 @@ function _typed_hvncat_shape(::Type{T}, shape::NTuple{N, Tuple}, row_first, as:: elementcount = 0 for i ∈ eachindex(as) - elementcount += cat_length(as[i]) + elementcount = checked_add(elementcount, cat_length(as[i])) wasstartblock = false for d ∈ 1:N ad = (d < 3 && row_first) ? (d == 1 ? 2 : 1) : d @@ -2677,7 +2775,7 @@ function _typed_hvncat_shape(::Type{T}, shape::NTuple{N, Tuple}, row_first, as:: blockcounts[d] += 1 if d == 1 || i == 1 || wasstartblock - currentdims[d] += dsize + currentdims[d] = checked_add(currentdims[d], dsize) elseif dsize != cat_size(as[i - 1], ad) throw(DimensionMismatch("argument $i has a mismatched number of elements along axis $ad; \ expected $(cat_size(as[i - 1], ad)), got $dsize")) @@ -2703,7 +2801,7 @@ function _typed_hvncat_shape(::Type{T}, shape::NTuple{N, Tuple}, row_first, as:: end end - outlen = prod(outdims) + outlen = Core.checked_dims(ntuple(i -> outdims[i], Val(N))...) elementcount == outlen || throw(ArgumentError("mismatched number of elements; expected $(outlen), got $(elementcount)")) @@ -2716,12 +2814,13 @@ function _typed_hvncat_shape(::Type{T}, shape::NTuple{N, Tuple}, row_first, as:: # copy into final array A = cat_similar(as[1], T, ntuple(i -> outdims[i], nd)) - hvncat_fill!(A, currentdims, blockcounts, d1, d2, as) + if !any(iszero, outdims) + hvncat_fill!(A, currentdims, blockcounts, d1, d2, as) + end return A end -function hvncat_fill!(A::AbstractArray{T, N}, scratch1::Vector{Int}, scratch2::Vector{Int}, - d1::Int, d2::Int, as::Tuple) where {T, N} +function hvncat_fill!(A::AbstractArray{T, N}, scratch1::Vector{Int}, scratch2::Vector{Int}, d1::Int, d2::Int, as::Tuple) where {T, N} N > 1 || throw(ArgumentError("dimensions of the destination array must be at least 2")) length(scratch1) == length(scratch2) == N || throw(ArgumentError("scratch vectors must have as many elements as the destination array has dimensions")) @@ -2729,42 +2828,46 @@ function hvncat_fill!(A::AbstractArray{T, N}, scratch1::Vector{Int}, scratch2::V 0 < d2 < 3 && d1 != d2 || throw(ArgumentError("d1 and d2 must be either 1 or 2, exclusive.")) - outdims = size(A) + outdimsprod = cumprod(size(A)) offsets = scratch1 inneroffsets = scratch2 for a ∈ as + startindex = CartesianIndex(ntuple(i -> offsets[i] + 1, Val(N))) if isa(a, AbstractArray) - for ai ∈ a - @inbounds Ai = hvncat_calcindex(offsets, inneroffsets, outdims, N) - A[Ai] = ai - - @inbounds for j ∈ 1:N - inneroffsets[j] += 1 - inneroffsets[j] < cat_size(a, j) && break - inneroffsets[j] = 0 + if !isempty(a) + if length(a) > 4 + endindex = CartesianIndex(ntuple(i -> offsets[i] + cat_size(a, i), Val(N))) + @inbounds A[startindex:endindex] = a + else + for ai ∈ a + @inbounds Ai = hvncat_calcindex(offsets, inneroffsets, outdimsprod, N) + @inbounds A[Ai] = ai + @inbounds for j ∈ 1:N + inneroffsets[j] += 1 + inneroffsets[j] < cat_size(a, j) && break + inneroffsets[j] = 0 + end + end end end else - @inbounds Ai = hvncat_calcindex(offsets, inneroffsets, outdims, N) - A[Ai] = a + @inbounds A[startindex] = a end - @inbounds for j ∈ (d1, d2, 3:N...) - offsets[j] += cat_size(a, j) - offsets[j] < outdims[j] && break - offsets[j] = 0 + @inbounds for i ∈ (d1, d2, 3:N...) + offsets[i] += cat_size(a, i) + offsets[i] < cat_size(A, i) && break + offsets[i] = 0 end end end @propagate_inbounds function hvncat_calcindex(offsets::Vector{Int}, inneroffsets::Vector{Int}, - outdims::Tuple{Vararg{Int}}, nd::Int) + outdimsprod::NTuple{N, Int}, nd::Int) where {N} Ai = inneroffsets[1] + offsets[1] + 1 for j ∈ 2:nd increment = inneroffsets[j] + offsets[j] - for k ∈ 1:j-1 - increment *= outdims[k] - end + increment *= outdimsprod[j - 1] Ai += increment end Ai @@ -2861,7 +2964,7 @@ julia> hvcat(5, M...) |> size # hvcat puts matrices next to each other (14, 15) ``` """ -stack(iter; dims=:) = _stack(dims, iter) +stack(iter; dims::D=:) where {D} = _stack(dims, iter) """ stack(f, args...; [dims]) @@ -2890,14 +2993,14 @@ julia> stack(eachrow([1 2 3; 4 5 6]), (10, 100); dims=1) do row, n 4.0 5.0 6.0 400.0 500.0 600.0 0.04 0.05 0.06 ``` """ -stack(f, iter; dims=:) = _stack(dims, f(x) for x in iter) -stack(f, xs, yzs...; dims=:) = _stack(dims, f(xy...) for xy in zip(xs, yzs...)) +stack(f, iter; dims::D=:) where {D} = _stack(dims, f(x) for x in iter) +stack(f, xs, yzs...; dims::D=:) where {D} = _stack(dims, f(xy...) for xy in zip(xs, yzs...)) -_stack(dims::Union{Integer, Colon}, iter) = _stack(dims, IteratorSize(iter), iter) +_stack(dims::D, iter) where {D<:Union{Integer, Colon}} = _stack(dims, IteratorSize(iter), iter) -_stack(dims, ::IteratorSize, iter) = _stack(dims, collect(iter)) +_stack(dims::D, ::IteratorSize, iter) where {D} = _stack(dims, collect(iter)) -function _stack(dims, ::Union{HasShape, HasLength}, iter) +function _stack(dims::D, ::Union{HasShape, HasLength}, iter) where {D} S = @default_eltype iter T = S != Union{} ? eltype(S) : Any # Union{} occurs for e.g. stack(1,2), postpone the error if isconcretetype(T) @@ -2935,8 +3038,6 @@ _iterator_axes(x, ::IteratorSize) = axes(x) # For some dims values, stack(A; dims) == stack(vec(A)), and the : path will be faster _typed_stack(dims::Integer, ::Type{T}, ::Type{S}, A) where {T,S} = _typed_stack(dims, T, S, IteratorSize(S), A) -_typed_stack(dims::Integer, ::Type{T}, ::Type{S}, ::HasLength, A) where {T,S} = - _typed_stack(dims, T, S, HasShape{1}(), A) function _typed_stack(dims::Integer, ::Type{T}, ::Type{S}, ::HasShape{N}, A) where {T,S,N} if dims == N+1 _typed_stack(:, T, S, A, (_vec_axis(A),)) @@ -2972,8 +3073,8 @@ _vec_axis(A, ax=_iterator_axes(A)) = length(ax) == 1 ? only(ax) : OneTo(prod(len end function _dim_stack!(::Val{dims}, B::AbstractArray, x1, xrest) where {dims} - before = ntuple(d -> Colon(), dims - 1) - after = ntuple(d -> Colon(), ndims(B) - dims) + before = ntuple(Returns(Colon()), dims - 1) + after = ntuple(Returns(Colon()), ndims(B) - dims) i = firstindex(B, dims) copyto!(view(B, before..., i, after...), x1) @@ -3016,6 +3117,8 @@ function isequal(A::AbstractArray, B::AbstractArray) end function cmp(A::AbstractVector, B::AbstractVector) + ai1, bi1 = firstindex(A), firstindex(B) + isequal(ai1, bi1) || return cmp(ai1, bi1) for (a, b) in zip(A, B) if !isequal(a, b) return isless(a, b) ? -1 : 1 @@ -3036,10 +3139,13 @@ end """ isless(A::AbstractVector, B::AbstractVector) -Return `true` when `A` is less than `B` in lexicographic order. +Return `true` when `A` is less than `B`. Vectors are first compared by +their starting indices, and then lexicographically by their elements. """ isless(A::AbstractVector, B::AbstractVector) = cmp(A, B) < 0 +OrderStyle(::Type{<:AbstractVector{T}}) where {T} = OrderStyle(T) + function (==)(A::AbstractArray, B::AbstractArray) if axes(A) != axes(B) return false @@ -3116,14 +3222,14 @@ end function _ind2sub_recurse(inds, ind) @inline r1 = inds[1] - indnext, f, l = _div(ind, r1) - (ind-l*indnext+f, _ind2sub_recurse(tail(inds), indnext)...) + indnext, indsub = divrem(ind, _indexlength(r1)) + (_lookup(indsub, r1), _ind2sub_recurse(tail(inds), indnext)...) end +_indexlength(d::Integer) = d +_indexlength(r::AbstractUnitRange) = length(r) _lookup(ind, d::Integer) = ind+1 _lookup(ind, r::AbstractUnitRange) = ind+first(r) -_div(ind, d::Integer) = div(ind, d), 1, d -_div(ind, r::AbstractUnitRange) = (d = length(r); (div(ind, d), first(r), d)) # Vectorized forms function _sub2ind(inds::Indices{1}, I1::AbstractVector{T}, I::AbstractVector{T}...) where T<:Integer @@ -3158,8 +3264,7 @@ _sub2ind_vec(i, I1, I...) = (@inline; (I1[i], _sub2ind_vec(i, I...)...)) _sub2ind_vec(i) = () function _ind2sub(inds::Union{DimsInteger{N},Indices{N}}, ind::AbstractVector{<:Integer}) where N - M = length(ind) - t = ntuple(n->similar(ind),Val(N)) + t = ntuple(_->similar(ind),Val(N)) for (i,idx) in pairs(IndexLinear(), ind) sub = _ind2sub(inds, idx) for j = 1:N @@ -3462,7 +3567,7 @@ collection. `destination` must be at least as large as the smallest collection. $(_DOCS_ALIASING_WARNING) -See also: [`map`](@ref), [`foreach`](@ref), [`zip`](@ref), [`copyto!`](@ref). +See also [`map`](@ref), [`foreach`](@ref), [`zip`](@ref), [`copyto!`](@ref). # Examples ```jldoctest @@ -3486,7 +3591,7 @@ julia> map!(+, zeros(Int, 5), 100:999, 1:3) ``` """ function map!(f::F, dest::AbstractArray, As::AbstractArray...) where {F} - @assert !isempty(As) # should dispatch to map!(f, A) + @assert !isempty(As) "should dispatch to map!(f, A)" map_n!(f, dest, As) end @@ -3576,7 +3681,7 @@ push!(A, a, b, c...) = push!(push!(A, a, b), c...) pushfirst!(A, a, b) = pushfirst!(pushfirst!(A, b), a) pushfirst!(A, a, b, c...) = pushfirst!(pushfirst!(A, c...), a, b) -# sizehint! does not nothing by default +# sizehint! does nothing by default sizehint!(a::AbstractVector, _) = a # The semantics of `collect` are weird. Better to write our own diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 66c75cf977c56..ed7448f742970 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -49,14 +49,17 @@ _sub(t::Tuple, s::Tuple) = _sub(tail(t), tail(s)) dropdims(A; dims) Return an array with the same data as `A`, but with the dimensions specified by -`dims` removed. `size(A,d)` must equal 1 for every `d` in `dims`, -and repeated dimensions or numbers outside `1:ndims(A)` are forbidden. +`dims` removed. + +Repeated dimensions or numbers outside `1:ndims(A)` are forbidden. +Moreover `size(A,d)` must equal 1 for every `d` in `dims`. The result shares the same underlying data as `A`, such that the result is mutable if and only if `A` is mutable, and setting elements of one alters the values of the other. -See also: [`reshape`](@ref), [`vec`](@ref). +Inverse of [`insertdims`](@ref). +See also [`reshape`](@ref), [`vec`](@ref). # Examples ```jldoctest @@ -88,17 +91,22 @@ function _dropdims(A::AbstractArray, dims::Dims) dims[j] == dims[i] && throw(ArgumentError("dropped dims must be unique")) end end + ox = axes(A) ax = _foldoneto((ds, d) -> d in dims ? ds : (ds..., axes(A,d)), (), Val(ndims(A))) - reshape(A, ax::typeof(_sub(axes(A), dims))) + if isconcretetype(eltype(ox)) + # if all the axes are the same type, we can use the tail as the + # axes of the result rather than extracting one at each index + return reshape(A, ax::typeof(_sub(ox, dims))) + else + return reshape(A, ax) + end end _dropdims(A::AbstractArray, dim::Integer) = _dropdims(A, (Int(dim),)) - """ insertdims(A; dims) -Inverse of [`dropdims`](@ref); return an array with new singleton dimensions -at every dimension in `dims`. +Return an array with new singleton dimensions at every dimension in `dims`. Repeated dimensions are forbidden and the largest entry in `dims` must be less than or equal than `ndims(A) + length(dims)`. @@ -107,7 +115,9 @@ The result shares the same underlying data as `A`, such that the result is mutable if and only if `A` is mutable, and setting elements of one alters the values of the other. -See also: [`dropdims`](@ref), [`reshape`](@ref), [`vec`](@ref). +Inverse of [`dropdims`](@ref). +See also [`reshape`](@ref), [`vec`](@ref). + # Examples ```jldoctest julia> x = [1 2 3; 4 5 6] @@ -144,7 +154,7 @@ function _insertdims(A::AbstractArray{T, N}, dims::NTuple{M, Int}) where {T, N, end # acc is a tuple, where the first entry is the final shape - # the second entry off acc is a counter for the axes of A + # the second entry of acc is a counter for the axes of A inds= Base._foldoneto((acc, i) -> i ∈ dims ? ((acc[1]..., Base.OneTo(1)), acc[2]) @@ -294,7 +304,7 @@ Return a view of all the data of `A` where the index for dimension `d` equals `i Equivalent to `view(A,:,:,...,i,:,:,...)` where `i` is in position `d`. -See also: [`eachslice`](@ref). +See also [`eachslice`](@ref). # Examples ```jldoctest @@ -336,7 +346,7 @@ first dimension. The generated code is most efficient when the shift amounts are known at compile-time, i.e., compile-time constants. -See also: [`circshift!`](@ref), [`circcopy!`](@ref), [`bitrotate`](@ref), [`<<`](@ref). +See also [`circshift!`](@ref), [`circcopy!`](@ref), [`bitrotate`](@ref), [`<<`](@ref). # Examples ```jldoctest @@ -409,7 +419,7 @@ end Construct an array by repeating array `A` a given number of times in each dimension, specified by `counts`. -See also: [`fill`](@ref), [`Iterators.repeated`](@ref), [`Iterators.cycle`](@ref). +See also [`fill`](@ref), [`Iterators.repeated`](@ref), [`Iterators.cycle`](@ref). # Examples ```jldoctest @@ -548,7 +558,7 @@ repeat_inner_outer(arr, inner, outer) = repeat_outer(repeat_inner(arr, inner), o function repeat_outer(a::AbstractMatrix, (m,n)::NTuple{2, Any}) o, p = size(a,1), size(a,2) - b = similar(a, o*m, p*n) + b = similar(a, Base.checked_mul(o, m), Base.checked_mul(p, n)) for j=1:n d = (j-1)*p+1 R = d:d+p-1 @@ -562,7 +572,7 @@ end function repeat_outer(a::AbstractVector, (m,)::Tuple{Any}) o = length(a) - b = similar(a, o*m) + b = similar(a, Base.checked_mul(o, m)) for i=1:m c = (i-1)*o+1 @inbounds b[c:c+o-1] = a @@ -572,7 +582,7 @@ end function repeat_outer(arr::AbstractArray{<:Any,N}, dims::NTuple{N,Any}) where {N} insize = size(arr) - outsize = map(*, insize, dims) + outsize = map(Base.checked_mul, insize, dims) out = similar(arr, outsize) for I in CartesianIndices(arr) for J in CartesianIndices(dims) @@ -587,7 +597,7 @@ function repeat_outer(arr::AbstractArray{<:Any,N}, dims::NTuple{N,Any}) where {N end function repeat_inner(arr, inner) - outsize = map(*, size(arr), inner) + outsize = map(Base.checked_mul, size(arr), inner) out = similar(arr, outsize) for I in CartesianIndices(arr) for J in CartesianIndices(inner) diff --git a/base/abstractdict.jl b/base/abstractdict.jl index 31c00c5b4f8d5..7216e0c904e5d 100644 --- a/base/abstractdict.jl +++ b/base/abstractdict.jl @@ -51,7 +51,7 @@ struct ValueIterator{T<:AbstractDict} end function summary(io::IO, iter::T) where {T<:Union{KeySet,ValueIterator}} - print(io, T.name.name, " for a ") + print(io, nameof(T), " for a ") summary(io, iter.dict) end @@ -185,7 +185,7 @@ pairs(a::AbstractDict) = a Create an empty `AbstractDict` container which can accept indices of type `index_type` and values of type `value_type`. The second and third arguments are optional and default to the input's `keytype` and `valtype`, respectively. (If only one of the two types is specified, -it is assumed to be the `value_type`, and the `index_type` we default to `keytype(a)`). +it is assumed to be the `value_type`, and the `index_type` defaults to `keytype(a)`). Custom `AbstractDict` subtypes may choose which specific dictionary type is best suited to return for the given index and value types, by specializing on the three-argument signature. @@ -232,6 +232,7 @@ function merge!(d::AbstractDict, others::AbstractDict...) end return d end +typeof(merge!).name.max_methods = UInt8(1) """ mergewith!(combine, d::AbstractDict, others::AbstractDict...) -> d @@ -282,6 +283,7 @@ Dict{Int64, Int64} with 3 entries: function mergewith!(combine, d::AbstractDict, others::AbstractDict...) foldl(mergewith!(combine), others; init = d) end +typeof(mergewith!).name.max_methods = UInt8(1) function mergewith!(combine, d1::AbstractDict, d2::AbstractDict) for (k, v) in d2 @@ -358,11 +360,11 @@ Dict{String, Float64} with 3 entries: """ merge(d::AbstractDict, others::AbstractDict...) = merge!(_typeddict(d, others...), others...) +typeof(merge).name.max_methods = UInt8(1) """ mergewith(combine, d::AbstractDict, others::AbstractDict...) mergewith(combine) - merge(combine, d::AbstractDict, others::AbstractDict...) Construct a merged collection from the given collections. If necessary, the types of the resulting collection will be promoted to accommodate the types of @@ -370,9 +372,6 @@ the merged collections. Values with the same key will be combined using the combiner function. The curried form `mergewith(combine)` returns the function `(args...) -> mergewith(combine, args...)`. -Method `merge(combine::Union{Function,Type}, args...)` as an alias of -`mergewith(combine, args...)` is still available for backward compatibility. - !!! compat "Julia 1.5" `mergewith` requires Julia 1.5 or later. @@ -405,8 +404,7 @@ Dict{Any, Any} with 1 entry: mergewith(combine, d::AbstractDict, others::AbstractDict...) = mergewith!(combine, _typeddict(d, others...), others...) mergewith(combine) = (args...) -> mergewith(combine, args...) -merge(combine::Callable, d::AbstractDict, others::AbstractDict...) = - merge!(combine, _typeddict(d, others...), others...) +typeof(mergewith).name.max_methods = UInt8(1) promoteK(K) = K promoteV(V) = V diff --git a/base/abstractset.jl b/base/abstractset.jl index 3072373e3c4f4..b83227b969795 100644 --- a/base/abstractset.jl +++ b/base/abstractset.jl @@ -95,7 +95,9 @@ max_values(T::Union{map(X -> Type{X}, BitIntegerSmall_types)...}) = 1 << (8*size function max_values(T::Union) a = max_values(T.a)::Int b = max_values(T.b)::Int - return max(a, b, a + b) + r, o = add_with_overflow(a, b) + o && return typemax(Int) + return r end max_values(::Type{Bool}) = 2 max_values(::Type{Nothing}) = 1 @@ -201,9 +203,9 @@ intersect!(s::AbstractSet, itr) = setdiff(s, itrs...) Construct the set of elements in `s` but not in any of the iterables in `itrs`. -Maintain order with arrays. +Maintain order with arrays. The result will have the same element type as `s`. -See also [`setdiff!`](@ref), [`union`](@ref) and [`intersect`](@ref). +See also [`setdiff!`](@ref), [`union`](@ref), [`intersect`](@ref). # Examples ```jldoctest @@ -211,6 +213,10 @@ julia> setdiff([1,2,3], [3,4,5]) 2-element Vector{Int64}: 1 2 + +julia> setdiff([1,2,3], [1.0, 2.0]) +1-element Vector{Int64}: + 3 ``` """ setdiff(s::AbstractSet, itrs...) = setdiff!(copymutable(s), itrs...) @@ -255,7 +261,7 @@ end Construct the symmetric difference of elements in the passed in sets. When `s` is not an `AbstractSet`, the order is maintained. -See also [`symdiff!`](@ref), [`setdiff`](@ref), [`union`](@ref) and [`intersect`](@ref). +See also [`symdiff!`](@ref), [`setdiff`](@ref), [`union`](@ref), [`intersect`](@ref). # Examples ```jldoctest @@ -397,7 +403,7 @@ function ⊋ end ⊊(a, b)::Bool ⊋(b, a)::Bool -Determines if `a` is a subset of, but not equal to, `b`. +Determine if `a` is a subset of, but not equal to, `b`. See also [`issubset`](@ref) (`⊆`), [`⊈`](@ref). @@ -501,7 +507,7 @@ used to implement specialized methods. Determine whether `a` and `b` have the same elements. Equivalent to `a ⊆ b && b ⊆ a` but more efficient when possible. -See also: [`isdisjoint`](@ref), [`union`](@ref). +See also [`isdisjoint`](@ref), [`union`](@ref). # Examples ```jldoctest @@ -549,7 +555,7 @@ issetequal(a) = Fix2(issetequal, a) Determine whether the collections `a` and `b` are disjoint. Equivalent to `isempty(a ∩ b)` but more efficient when possible. -See also: [`intersect`](@ref), [`isempty`](@ref), [`issetequal`](@ref). +See also [`intersect`](@ref), [`isempty`](@ref), [`issetequal`](@ref). !!! compat "Julia 1.5" This function requires at least Julia 1.5. diff --git a/base/accumulate.jl b/base/accumulate.jl index c155ecfb4f75f..e4c5f05d21b48 100644 --- a/base/accumulate.jl +++ b/base/accumulate.jl @@ -1,9 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# accumulate_pairwise slightly slower then accumulate, but more numerically +# `accumulate_pairwise` runs slightly slower than `accumulate`, but more numerically # stable in certain situations (e.g. sums). -# it does double the number of operations compared to accumulate, -# though for cheap operations like + this does not have much impact (20%) +# It does double the number of operations compared to `accumulate`, +# though for cheap operations like `+` this does not have much impact (20%). function _accumulate_pairwise!(op::Op, c::AbstractVector{T}, v::AbstractVector, s, i1, n)::T where {T,Op} if n < 128 @inbounds s_ = v[i1] @@ -476,7 +476,7 @@ function _accumulate_promote_op(op, v; init=nothing) end # Finally loop again with the two types promoted together - # If the `op` fails and reduce_first was used then then this will still just + # If the `op` fails and reduce_first was used then this will still just # return the initial type, allowing the `op` to error during execution. function h(op, v, r) for val in v diff --git a/base/anyall.jl b/base/anyall.jl index 395666dc6765a..c1bc5088d955d 100644 --- a/base/anyall.jl +++ b/base/anyall.jl @@ -13,7 +13,7 @@ If the input contains [`missing`](@ref) values, return `missing` if all non-miss values are `false` (or equivalently, if the input contains no `true` value), following [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic). -See also: [`all`](@ref), [`count`](@ref), [`sum`](@ref), [`|`](@ref), [`||`](@ref). +See also [`all`](@ref), [`count`](@ref), [`sum`](@ref), [`|`](@ref), [`||`](@ref). # Examples ```jldoctest @@ -51,7 +51,7 @@ If the input contains [`missing`](@ref) values, return `missing` if all non-miss values are `true` (or equivalently, if the input contains no `false` value), following [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic). -See also: [`all!`](@ref), [`any`](@ref), [`count`](@ref), [`&`](@ref), [`&&`](@ref), [`allunique`](@ref). +See also [`all!`](@ref), [`any`](@ref), [`count`](@ref), [`&`](@ref), [`&&`](@ref), [`allunique`](@ref). # Examples ```jldoctest @@ -147,7 +147,7 @@ function _any(::typeof(identity), itr::Tuple{Vararg{Bool}}, ::Colon) end # Specialized versions of any(f, ::Tuple) -# We fall back to the for loop implementation all elements have the same type or +# We fall back to the for loop implementation if all elements have the same type or # if the tuple is too large. function any(f, itr::Tuple) if itr isa NTuple || length(itr) > 32 @@ -156,7 +156,7 @@ function any(f, itr::Tuple) _any_tuple(f, false, itr...) end -@inline function _any_tuple(f, anymissing, x, rest...) +@inline function _any_tuple(f, anymissing::Bool, x, rest...) v = f(x) if ismissing(v) anymissing = true @@ -165,7 +165,7 @@ end end return _any_tuple(f, anymissing, rest...) end -@inline _any_tuple(f, anymissing) = anymissing ? missing : false +@inline _any_tuple(_, anymissing::Bool) = anymissing ? missing : false """ all(p, itr)::Bool @@ -242,7 +242,7 @@ function all(f, itr::Tuple) _all_tuple(f, false, itr...) end -@inline function _all_tuple(f, anymissing, x, rest...) +@inline function _all_tuple(f, anymissing::Bool, x, rest...) v = f(x) if ismissing(v) anymissing = true @@ -254,4 +254,4 @@ end end return _all_tuple(f, anymissing, rest...) end -@inline _all_tuple(f, anymissing) = anymissing ? missing : true +@inline _all_tuple(_, anymissing::Bool) = anymissing ? missing : true diff --git a/base/array.jl b/base/array.jl index d62436ca30497..ced3cfa5dc9b2 100644 --- a/base/array.jl +++ b/base/array.jl @@ -185,14 +185,6 @@ function vect(X...) return T[X...] end -size(a::Array, d::Integer) = size(a, Int(d)::Int) -function size(a::Array, d::Int) - d < 1 && error("arraysize: dimension out of range") - sz = getfield(a, :size) - return d > length(sz) ? 1 : getfield(sz, d, false) # @inbounds -end -size(a::Array) = getfield(a, :size) - asize_from(a::Array, n) = n > ndims(a) ? () : (size(a,n), asize_from(a, n+1)...) allocatedinline(@nospecialize T::Type) = (@_total_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0)) @@ -213,10 +205,11 @@ false """ isbitsunion(u::Type) = u isa Union && allocatedinline(u) -function _unsetindex!(A::Array, i::Int) +unsetindex!(A::Array, i::Integer) = unsetindex!(A, to_index(i)) +function unsetindex!(A::Array, i::Int) @inline @boundscheck checkbounds(A, i) - @inbounds _unsetindex!(memoryref(A.ref, i)) + @inbounds unsetindex!(memoryref(A.ref, i)) return A end @@ -291,18 +284,28 @@ end Copy `n` elements from collection `src` starting at the linear index `soffs`, to array `dest` starting at the index `doffs`. Return `dest`. """ -copyto!(dest::Array, doffs::Integer, src::Array, soffs::Integer, n::Integer) = _copyto_impl!(dest, doffs, src, soffs, n) -copyto!(dest::Array, doffs::Integer, src::Memory, soffs::Integer, n::Integer) = _copyto_impl!(dest, doffs, src, soffs, n) -copyto!(dest::Memory, doffs::Integer, src::Array, soffs::Integer, n::Integer) = _copyto_impl!(dest, doffs, src, soffs, n) +copyto!(dest::Array, doffs::Integer, src::Array, soffs::Integer, n::Integer) = + (@_propagate_inbounds_meta; _copyto_impl!(dest, doffs, src, soffs, n)) +copyto!(dest::Array, doffs::Integer, src::Memory, soffs::Integer, n::Integer) = + (@_propagate_inbounds_meta; _copyto_impl!(dest, doffs, src, soffs, n)) +copyto!(dest::Memory, doffs::Integer, src::Array, soffs::Integer, n::Integer) = + (@_propagate_inbounds_meta; _copyto_impl!(dest, doffs, src, soffs, n)) # this is only needed to avoid possible ambiguities with methods added in some packages -copyto!(dest::Array{T}, doffs::Integer, src::Array{T}, soffs::Integer, n::Integer) where {T} = _copyto_impl!(dest, doffs, src, soffs, n) +copyto!(dest::Array{T}, doffs::Integer, src::Array{T}, soffs::Integer, n::Integer) where {T} = + (@_propagate_inbounds_meta; _copyto_impl!(dest, doffs, src, soffs, n)) function _copyto_impl!(dest::Union{Array,Memory}, doffs::Integer, src::Union{Array,Memory}, soffs::Integer, n::Integer) + @inline n == 0 && return dest n > 0 || _throw_argerror("Number of elements to copy must be non-negative.") - @boundscheck checkbounds(dest, doffs:doffs+n-1) - @boundscheck checkbounds(src, soffs:soffs+n-1) + @boundscheck checkbounds(dest, doffs) + @boundscheck n <= length(dest) - doffs + 1 || throw(BoundsError(dest, length(dest) + 1)) + @boundscheck checkbounds(src, soffs) + @boundscheck n <= length(src) - soffs + 1 || throw(BoundsError(src, length(src) + 1)) + doffs = Int(doffs) + soffs = Int(soffs) + n = Int(n) @inbounds let dest = memoryref(dest isa Array ? getfield(dest, :ref) : dest, doffs), src = memoryref(src isa Array ? getfield(src, :ref) : src, soffs) unsafe_copyto!(dest, src, n) @@ -327,18 +330,50 @@ copyto!(dest::Array{T}, src::Array{T}) where {T} = _copyto2arg!(dest, src) copyto!(dest::Array{T}, src::Memory{T}) where {T} = _copyto2arg!(dest, src) copyto!(dest::Memory{T}, src::Array{T}) where {T} = _copyto2arg!(dest, src) -# N.B: The generic definition in multidimensional.jl covers, this, this is just here -# for bootstrapping purposes. -function fill!(dest::Array{T}, x) where T +# N.B: This generic definition in for multidimensional arrays is here instead of +# `multidimensional.jl` for bootstrapping purposes. +""" + fill!(A, x) + +Fill array `A` with the value `x`. If `x` is an object reference, all elements will refer to +the same object. `fill!(A, Foo())` will return `A` filled with the result of evaluating +`Foo()` once. + +# Examples +```jldoctest +julia> A = zeros(2,3) +2×3 Matrix{Float64}: + 0.0 0.0 0.0 + 0.0 0.0 0.0 + +julia> fill!(A, 2.) +2×3 Matrix{Float64}: + 2.0 2.0 2.0 + 2.0 2.0 2.0 + +julia> a = [1, 1, 1]; A = fill!(Vector{Vector{Int}}(undef, 3), a); a[1] = 2; A +3-element Vector{Vector{Int64}}: + [2, 1, 1] + [2, 1, 1] + [2, 1, 1] + +julia> x = 0; f() = (global x += 1; x); fill!(Vector{Int}(undef, 3), f()) +3-element Vector{Int64}: + 1 + 1 + 1 +``` +""" +function fill!(A::AbstractArray{T}, x) where T @inline - x = x isa T ? x : convert(T, x)::T - return _fill!(dest, x) + xT = x isa T ? x : convert(T, x)::T + return _fill!(A, xT) end -function _fill!(dest::Array{T}, x::T) where T - for i in eachindex(dest) - dest[i] = x +function _fill!(A::AbstractArray{T}, x::T) where T + for i in eachindex(A) + A[i] = x end - return dest + return A end """ @@ -375,13 +410,14 @@ end ## Constructors ## -similar(a::Array{T,1}) where {T} = Vector{T}(undef, size(a,1)) -similar(a::Array{T,2}) where {T} = Matrix{T}(undef, size(a,1), size(a,2)) -similar(a::Array{T,1}, S::Type) where {T} = Vector{S}(undef, size(a,1)) -similar(a::Array{T,2}, S::Type) where {T} = Matrix{S}(undef, size(a,1), size(a,2)) +similar(a::Vector{T}) where {T} = Vector{T}(undef, size(a,1)) +similar(a::Matrix{T}) where {T} = Matrix{T}(undef, size(a,1), size(a,2)) +similar(a::Vector{T}, S::Type) where {T} = Vector{S}(undef, size(a,1)) +similar(a::Matrix{T}, S::Type) where {T} = Matrix{S}(undef, size(a,1), size(a,2)) similar(a::Array{T}, m::Int) where {T} = Vector{T}(undef, m) similar(a::Array, T::Type, dims::Dims{N}) where {N} = Array{T,N}(undef, dims) similar(a::Array{T}, dims::Dims{N}) where {T,N} = Array{T,N}(undef, dims) +similar(::Type{Array{T,N}}, dims::Dims) where {T,N} = similar(Array{T}, dims) # T[x...] constructs Array{T,1} """ @@ -515,7 +551,7 @@ julia> v2 [] ``` -See also: [`fill!`](@ref), [`zeros`](@ref), [`ones`](@ref), [`similar`](@ref). +See also [`fill!`](@ref), [`zeros`](@ref), [`ones`](@ref), [`similar`](@ref). # Examples ```jldoctest @@ -647,7 +683,7 @@ julia> collect(Float64, 1:2:5) collect(::Type{T}, itr) where {T} = _collect(T, itr, IteratorSize(itr)) _collect(::Type{T}, itr, isz::Union{HasLength,HasShape}) where {T} = - copyto!(_array_for(T, isz, _similar_shape(itr, isz)), itr) + copyto!(_array_for_inner(T, isz, _similar_shape(itr, isz)), itr) function _collect(::Type{T}, itr, isz::SizeUnknown) where T a = Vector{T}() for x in itr @@ -671,12 +707,12 @@ _similar_for(c::AbstractArray, ::Type{T}, itr, ::HasShape, axs) where {T} = similar(c, T, axs) # make a collection appropriate for collecting `itr::Generator` -_array_for(::Type{T}, ::SizeUnknown, ::Nothing) where {T} = Vector{T}(undef, 0) -_array_for(::Type{T}, ::HasLength, len::Integer) where {T} = Vector{T}(undef, Int(len)) -_array_for(::Type{T}, ::HasShape{N}, axs) where {T,N} = similar(Array{T,N}, axs) +_array_for_inner(::Type{T}, ::SizeUnknown, ::Nothing) where {T} = Vector{T}(undef, 0) +_array_for_inner(::Type{T}, ::HasLength, len::Integer) where {T} = Vector{T}(undef, Int(len)) +_array_for_inner(::Type{T}, ::HasShape{N}, axs) where {T,N} = similar(Array{T,N}, axs) # used by syntax lowering for simple typed comprehensions -_array_for(::Type{T}, itr, isz) where {T} = _array_for(T, isz, _similar_shape(itr, isz)) +_array_for(::Type{T}, itr, isz) where {T} = _array_for_inner(T, isz, _similar_shape(itr, isz)) """ @@ -802,10 +838,10 @@ function collect(itr::Generator) shp = _similar_shape(itr, isz) y = iterate(itr) if y === nothing - return _array_for(et, isz, shp) + return _array_for_inner(et, isz, shp) end v1, st = y - dest = _array_for(typeof(v1), isz, shp) + dest = _array_for_inner(typeof(v1), isz, shp) # The typeassert gives inference a helping hand on the element type and dimensionality # (work-around for #28382) et′ = et <: Type ? Type : et @@ -853,6 +889,29 @@ function setindex_widen_up_to(dest::AbstractArray{T}, el, i) where T return new end +# Batch-widen an array given (index => value) pairs that don't fit the current element type. +function setindices_widen_up_to(dest::AbstractArray, widen_buffers::Vector{Vector{Pair{Int, Any}}}) + widen_pairs = reduce(vcat, widen_buffers; init=Pair{Int,Any}[]) + isempty(widen_pairs) && return dest + new_T = eltype(dest) + for p in widen_pairs + new_T = promote_typejoin(new_T, typeof(p.second)) + end + new_T === eltype(dest) && return dest + # Function barrier: specializes on new_T so the compiler sees + # concrete element types for both source and destination arrays. + return _setindices_widen_up_to(new_T, dest, widen_pairs) +end + +function _setindices_widen_up_to(::Type{T}, dest::AbstractArray, widen_pairs::Vector{Pair{Int, Any}}) where T + new = similar(dest, T) + copyto!(new, dest) + for (idx, val) in widen_pairs + @inbounds new[idx] = val + end + return new +end + function collect_to!(dest::AbstractArray{T}, itr, offs, st) where T # collect to dest array, checking the type of each result. if a result does not # match, widen the result type and re-dispatch. @@ -996,7 +1055,7 @@ function setindex!(A::Array{T}, x, i::Int) where {T} end function _setindex!(A::Array{T}, x::T, i::Int) where {T} @_noub_if_noinbounds_meta - @boundscheck checkbounds(Bool, A, i) || throw_boundserror(A, (i,)) + @boundscheck checkbounds(A, i) memoryrefset!(memoryrefnew(A.ref, i, false), x, :not_atomic, false) return A end @@ -1061,16 +1120,16 @@ end # TODO: This should know about the size of our GC pools # Specifically we are wasting ~10% of memory for small arrays # by not picking memory sizes that max out a GC pool -function overallocation(maxsize) - maxsize < 8 && return 8; - # compute maxsize = maxsize + 4*maxsize^(7/8) + maxsize/8 +function overallocation(maxsize::Int) + # compute maxsize = maxsize + 3*maxsize^(7/8) + maxsize/8 # for small n, we grow faster than O(n) # for large n, we grow at O(n/8) # and as we reach O(memory) for memory>>1MB, # this means we end by adding about 10% of memory each time + # most commonly, this will take steps of 0-3-9-34 or 1-4-16-66 or 2-8-33 exp2 = sizeof(maxsize) * 8 - Core.Intrinsics.ctlz_int(maxsize) - maxsize += (1 << div(exp2 * 7, 8)) * 4 + div(maxsize, 8) - return maxsize + extra = (1 << div(exp2 * 7, 8)) * 3 + div(maxsize, 8) + return maxsize > typemax(Int) - extra ? typemax(Int) : maxsize + extra end array_new_memory(mem::Memory, newlen::Int) = typeof(mem)(undef, newlen) # when implemented, this should attempt to first expand mem @@ -1080,25 +1139,25 @@ function _growbeg_internal!(a::Vector, delta::Int, len::Int) ref = a.ref mem = ref.mem offset = memoryrefoffset(ref) - newlen = len + delta + newlen = checked_add(len, delta) memlen = length(mem) - if offset + len - 1 > memlen || offset < 1 + if offset < 1 || offset - 1 > memlen || len > memlen - (offset - 1) throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) end # since we will allocate the array in the middle of the memory we need at least 2*delta extra space # the +1 is because I didn't want to have an off by 1 error. - newmemlen = max(overallocation(len), len + 2 * delta + 1) + newmemlen = max(overallocation(len), checked_add(len, checked_mul(2, delta), 1)) newoffset = div(newmemlen - newlen, 2) + 1 # If there is extra data after the end of the array we can use that space so long as there is enough # space at the end that there won't be quadratic behavior with a mix of growth from both ends. # Specifically, we want to ensure that we will only do this operation once before # increasing the size of the array, and that we leave enough space at both the beginning and the end. - if newoffset + newlen < memlen + if newlen < memlen && newoffset < memlen - newlen newoffset = div(memlen - newlen, 2) + 1 newmem = mem unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) for j in offset:newoffset+delta-1 - @inbounds _unsetindex!(mem, j) + @inbounds unsetindex!(mem, j) end else newmem = array_new_memory(mem, newmemlen) @@ -1116,16 +1175,16 @@ function _growbeg!(a::Vector, delta::Integer) delta == 0 && return # avoid attempting to index off the end delta >= 0 || throw(ArgumentError("grow requires delta >= 0")) ref = a.ref - mem = ref.mem len = length(a) offset = memoryrefoffset(ref) - newlen = len + delta - setfield!(a, :size, (newlen,)) + newlen = checked_add(len, delta) # if offset is far enough advanced to fit data in existing memory without copying if delta <= offset - 1 setfield!(a, :ref, @inbounds memoryref(ref, 1 - delta)) + setfield!(a, :size, (newlen,)) else @noinline _growbeg_internal!(a, delta, len) + setfield!(a, :size, (newlen,)) end return end @@ -1134,14 +1193,14 @@ function _growend_internal!(a::Vector, delta::Int, len::Int) ref = a.ref mem = ref.mem memlen = length(mem) - newlen = len + delta + newlen = checked_add(len, delta) offset = memoryrefoffset(ref) - newmemlen = offset + newlen - 1 - if offset + len - 1 > memlen || offset < 1 + if offset < 1 || offset - 1 > memlen || len > memlen - (offset - 1) throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) end + newmemlen = checked_add(offset - 1, newlen) - if offset - 1 > div(5 * newlen, 4) + if offset - 1 > newlen && offset - 1 - newlen > div(newlen, 4) # If the offset is far enough that we can copy without resizing # while maintaining proportional spacing on both ends of the array # note that this branch prevents infinite growth when doing combinations @@ -1173,13 +1232,13 @@ function _growend!(a::Vector, delta::Integer) mem = ref.mem memlen = length(mem) len = length(a) - newlen = len + delta + newlen = checked_add(len, delta) offset = memoryrefoffset(ref) - setfield!(a, :size, (newlen,)) - newmemlen = offset + newlen - 1 + newmemlen = checked_add(offset - 1, newlen) if memlen < newmemlen @noinline _growend_internal!(a, delta, len) end + setfield!(a, :size, (newlen,)) return end @@ -1189,16 +1248,15 @@ function _growat!(a::Vector, i::Integer, delta::Integer) i = Int(i) i == 1 && return _growbeg!(a, delta) len = length(a) - i == len + 1 && return _growend!(a, delta) + len < typemax(Int) && i == len + 1 && return _growend!(a, delta) delta >= 0 || throw(ArgumentError("grow requires delta >= 0")) 1 < i <= len || throw(BoundsError(a, i)) ref = a.ref mem = ref.mem memlen = length(mem) - newlen = len + delta + newlen = checked_add(len, delta) offset = memoryrefoffset(ref) - setfield!(a, :size, (newlen,)) - newmemlen = offset + newlen - 1 + newmemlen = checked_add(offset - 1, newlen) # which side would we rather grow into? prefer_start = i <= div(len, 2) @@ -1207,24 +1265,27 @@ function _growat!(a::Vector, i::Integer, delta::Integer) newref = @inbounds memoryref(mem, offset - delta) unsafe_copyto!(newref, ref, i) setfield!(a, :ref, newref) + setfield!(a, :size, (newlen,)) for j in i:i+delta-1 - @inbounds _unsetindex!(a, j) + @inbounds unsetindex!(a, j) end elseif !prefer_start && memlen >= newmemlen unsafe_copyto!(mem, offset - 1 + delta + i, mem, offset - 1 + i, len - i + 1) + setfield!(a, :size, (newlen,)) for j in i:i+delta-1 - @inbounds _unsetindex!(a, j) + @inbounds unsetindex!(a, j) end else # since we will allocate the array in the middle of the memory we need at least 2*delta extra space # the +1 is because I didn't want to have an off by 1 error. - newmemlen = max(overallocation(memlen), len+2*delta+1) + newmemlen = max(overallocation(memlen), checked_add(len, checked_mul(2, delta), 1)) newoffset = (newmemlen - newlen) ÷ 2 + 1 newmem = array_new_memory(mem, newmemlen) newref = @inbounds memoryref(newmem, newoffset) unsafe_copyto!(newref, ref, i-1) unsafe_copyto!(newmem, newoffset + delta + i - 1, mem, offset + i - 1, len - i + 1) setfield!(a, :ref, newref) + setfield!(a, :size, (newlen,)) end end @@ -1232,25 +1293,33 @@ end function _deletebeg!(a::Vector, delta::Integer) delta = Int(delta) len = length(a) - 0 <= delta <= len || throw(ArgumentError("_deletebeg! requires delta in 0:length(a)")) + # See comment in _deleteend! + if unsigned(delta) > unsigned(len) + throw(ArgumentError("_deletebeg! requires delta in 0:length(a)")) + end for i in 1:delta - @inbounds _unsetindex!(a, i) + @inbounds unsetindex!(a, i) end newlen = len - delta + setfield!(a, :size, (newlen,)) if newlen != 0 # if newlen==0 we could accidentally index past the memory newref = @inbounds memoryref(a.ref, delta + 1) setfield!(a, :ref, newref) end - setfield!(a, :size, (newlen,)) return end function _deleteend!(a::Vector, delta::Integer) delta = Int(delta) len = length(a) - 0 <= delta <= len || throw(ArgumentError("_deleteend! requires delta in 0:length(a)")) + # Do the comparison unsigned, to so the compiler knows `len` cannot be negative. + # This works because if delta is negative, it will overflow and still trigger. + # This enables the compiler to skip the check sometimes. + if unsigned(delta) > unsigned(len) + throw(ArgumentError("_deleteend! requires delta in 0:length(a)")) + end newlen = len - delta for i in newlen+1:len - @inbounds _unsetindex!(a, i) + @inbounds unsetindex!(a, i) end setfield!(a, :size, (newlen,)) return @@ -1304,7 +1373,7 @@ function push! end function push!(a::Vector{T}, item) where T @inline # convert first so we don't grow the array if the assignment won't work - # and also to avoid a dynamic dynamic dispatch in the common case that + # and also to avoid a dynamic dispatch in the common case that # `item` is poorly-typed and `a` is well-typed item = item isa T ? item : convert(T, item)::T return _push!(a, item) @@ -1384,7 +1453,6 @@ append!(a::AbstractVector, iter...) = (foreach(v -> append!(a, v), iter); a) function _append!(a::AbstractVector, ::Union{HasLength,HasShape}, iter) n = Int(length(iter))::Int - i = lastindex(a) sizehint!(a, length(a) + n; shrink=false) for item in iter push!(a, item) @@ -1500,7 +1568,10 @@ function resize!(a::Vector, nl_::Integer) nl = Int(nl_)::Int l = length(a) if nl > l - _growend!(a, nl-l) + # Since l is positive, if nl > l, both are positive, and so nl-l is also + # positive. But the compiler does not know that, so we mask out top bit. + # This allows the compiler to skip the check + _growend!(a, (nl-l) & typemax(Int)) elseif nl != l if nl < 0 _throw_argerror("new length must be ≥ 0") @@ -1580,7 +1651,7 @@ function sizehint!(a::Vector, sz::Integer; first::Bool=false, shrink::Bool=true) end # Fall-back implementation for non-shrinkable collections -# avoid defining this the normal way to avoid avoid infinite recursion +# avoid defining this the normal way to avoid infinite recursion function Core.kwcall(kwargs::NamedTuple{names}, ::typeof(sizehint!), a, sz) where names get(kwargs, :first, false)::Bool get(kwargs, :shrink, true)::Bool @@ -1595,7 +1666,7 @@ Remove an item in `collection` and return it. If `collection` is an ordered container, the last item is returned; for unordered containers, an arbitrary element is returned. -See also: [`popfirst!`](@ref), [`popat!`](@ref), [`delete!`](@ref), [`deleteat!`](@ref), [`splice!`](@ref), and [`push!`](@ref). +See also [`popfirst!`](@ref), [`popat!`](@ref), [`delete!`](@ref), [`deleteat!`](@ref), [`splice!`](@ref), [`push!`](@ref). # Examples ```jldoctest @@ -1646,7 +1717,7 @@ are shifted to fill the resulting gap. When `i` is not a valid index for `a`, return `default`, or throw an error if `default` is not specified. -See also: [`pop!`](@ref), [`popfirst!`](@ref), [`deleteat!`](@ref), [`splice!`](@ref). +See also [`pop!`](@ref), [`popfirst!`](@ref), [`deleteat!`](@ref), [`splice!`](@ref). !!! compat "Julia 1.5" This function is available as of Julia 1.5. @@ -1725,7 +1796,6 @@ function pushfirst!(a::Vector{Any}, @nospecialize x) end function pushfirst!(a::Vector{Any}, @nospecialize x...) @_terminates_locally_meta - na = length(a) nx = length(x) _growbeg!(a, nx) @_safeindex for i = 1:nx @@ -1741,7 +1811,7 @@ Remove the first `item` from `collection`. This function is called `shift` in many other programming languages. -See also: [`pop!`](@ref), [`popat!`](@ref), [`delete!`](@ref). +See also [`pop!`](@ref), [`popat!`](@ref), [`delete!`](@ref). # Examples ```jldoctest @@ -1781,7 +1851,7 @@ end Insert an `item` into `a` at the given `index`. `index` is the index of `item` in the resulting `a`. -See also: [`push!`](@ref), [`replace`](@ref), [`popat!`](@ref), [`splice!`](@ref). +See also [`push!`](@ref), [`replace`](@ref), [`popat!`](@ref), [`splice!`](@ref). # Examples ```jldoctest @@ -1796,12 +1866,12 @@ julia> insert!(Any[1:6;], 3, "here") 6 ``` """ -function insert!(a::Array{T,1}, i::Integer, item) where T +function insert!(a::Vector{T}, i::Integer, item) where T @_propagate_inbounds_meta item = item isa T ? item : convert(T, item)::T return _insert!(a, i, item) end -function _insert!(a::Array{T,1}, i::Integer, item::T) where T +function _insert!(a::Vector{T}, i::Integer, item::T) where T @_noub_meta # Throw convert error before changing the shape of the array _growat!(a, i, 1) @@ -1816,7 +1886,7 @@ end Remove the item at the given `i` and return the modified `a`. Subsequent items are shifted to fill the resulting gap. -See also: [`keepat!`](@ref), [`delete!`](@ref), [`popat!`](@ref), [`splice!`](@ref). +See also [`keepat!`](@ref), [`delete!`](@ref), [`popat!`](@ref), [`splice!`](@ref). # Examples ```jldoctest @@ -1839,7 +1909,6 @@ function deleteat!(a::Vector, r::AbstractUnitRange{<:Integer}) if eltype(r) === Bool return invoke(deleteat!, Tuple{Vector, AbstractVector{Bool}}, a, r) else - n = length(a) f = first(r) f isa Bool && depwarn("passing Bool as an index is deprecated", :deleteat!) isempty(r) || _deleteat!(a, f, length(r)) @@ -1897,7 +1966,7 @@ function _copy_item!(a::Vector, p, q) if isassigned(a, q) a[p] = a[q] else - _unsetindex!(a, p) + unsetindex!(a, p) end end @@ -1958,7 +2027,7 @@ Subsequent items are shifted left to fill the resulting gap. If specified, replacement values from an ordered collection will be spliced in place of the removed item. -See also: [`replace`](@ref), [`delete!`](@ref), [`deleteat!`](@ref), [`pop!`](@ref), [`popat!`](@ref). +See also [`replace`](@ref), [`delete!`](@ref), [`deleteat!`](@ref), [`pop!`](@ref), [`popat!`](@ref). # Examples ```jldoctest @@ -2202,7 +2271,7 @@ end # 1d special cases of reverse(A; dims) and reverse!(A; dims): for (f,_f) in ((:reverse,:_reverse), (:reverse!,:_reverse!)) @eval begin - $f(A::AbstractVector; dims=:) = $_f(A, dims) + $f(A::AbstractVector; dims::D=:) where {D} = $_f(A, dims) $_f(A::AbstractVector, ::Colon) = $f(A, firstindex(A), lastindex(A)) $_f(A::AbstractVector, dim::Tuple{Integer}) = $_f(A, first(dim)) function $_f(A::AbstractVector, dim::Integer) @@ -2219,7 +2288,7 @@ end # This implementation of `midpoint` is performance-optimized but safe # only if `lo <= hi`. -midpoint(lo::T, hi::T) where T<:Integer = lo + ((hi - lo) >>> 0x01) +midpoint(lo::T, hi::T) where T<:Integer = lo +% ((hi -% lo) >>> 0x01) midpoint(lo::Integer, hi::Integer) = midpoint(promote(lo, hi)...) """ @@ -2291,7 +2360,7 @@ function vcat(arrays::Vector{T}...) where T nd = 1 for a in arrays na = length(a) - @assert nd + na <= 1 + length(arr) # Concurrent modification of arrays? + @assert nd + na <= 1 + length(arr) "Concurrent modification of arrays?" unsafe_copyto!(arr, nd, a, 1, na) nd += na end @@ -2347,7 +2416,7 @@ To search for other kinds of values, pass a predicate as the first argument. Indices or keys are of the same type as those returned by [`keys(A)`](@ref) and [`pairs(A)`](@ref). -See also: [`findall`](@ref), [`findnext`](@ref), [`findlast`](@ref), [`searchsortedfirst`](@ref). +See also [`findall`](@ref), [`findnext`](@ref), [`findlast`](@ref), [`searchsortedfirst`](@ref). # Examples ```jldoctest @@ -2497,7 +2566,7 @@ or `nothing` if not found. Indices are of the same type as those returned by [`keys(A)`](@ref) and [`pairs(A)`](@ref). -See also: [`findnext`](@ref), [`findfirst`](@ref), [`findall`](@ref). +See also [`findnext`](@ref), [`findfirst`](@ref), [`findall`](@ref). # Examples ```jldoctest @@ -2533,7 +2602,7 @@ Return `nothing` if there is no `true` value in `A`. Indices or keys are of the same type as those returned by [`keys(A)`](@ref) and [`pairs(A)`](@ref). -See also: [`findfirst`](@ref), [`findprev`](@ref), [`findall`](@ref). +See also [`findfirst`](@ref), [`findprev`](@ref), [`findall`](@ref). # Examples ```jldoctest @@ -2739,7 +2808,7 @@ To search for other kinds of values, pass a predicate as the first argument. Indices or keys are of the same type as those returned by [`keys(A)`](@ref) and [`pairs(A)`](@ref). -See also: [`findfirst`](@ref), [`searchsorted`](@ref). +See also [`findfirst`](@ref), [`searchsorted`](@ref). # Examples ```jldoctest @@ -2799,7 +2868,7 @@ Return an array containing the first index in `b` for each value in `a` that is a member of `b`. The output array contains `nothing` wherever `a` is not a member of `b`. -See also: [`sortperm`](@ref), [`findfirst`](@ref). +See also [`sortperm`](@ref), [`findfirst`](@ref). # Examples ```jldoctest @@ -2914,10 +2983,11 @@ end function indcopy(sz::Dims, I::Tuple{Vararg{RangeIndex}}) n = length(I) - s = sz[n] + _s = sz[n] for i = n+1:length(sz) - s *= sz[i] + _s *= sz[i] end + s = _s dst::typeof(I) = ntuple(i-> _findin(I[i], i < n ? (1:sz[i]) : (1:s)), n)::typeof(I) src::typeof(I) = ntuple(i-> I[i][_findin(I[i], i < n ? (1:sz[i]) : (1:s))], n)::typeof(I) dst, src @@ -2934,7 +3004,7 @@ The function `f` is passed one argument. !!! compat "Julia 1.4" Support for `a` as a tuple requires at least Julia 1.4. -See also: [`filter!`](@ref), [`Iterators.filter`](@ref). +See also [`filter!`](@ref), [`Iterators.filter`](@ref). # Examples ```jldoctest @@ -3133,14 +3203,17 @@ setdiff!( v::AbstractVector, itrs...) = _shrink!(setdiff!, v, itrs) vectorfilter(T::Type, f, v) = T[x for x in v if f(x)] -function _shrink(shrinker!::F, itr, itrs) where F +function intersect(itr, itrs...) T = promote_eltype(itr, itrs...) - keep = shrinker!(Set{T}(itr), itrs...) + keep = intersect!(Set{T}(itr), itrs...) vectorfilter(T, _shrink_filter!(keep), itr) end -intersect(itr, itrs...) = _shrink(intersect!, itr, itrs) -setdiff( itr, itrs...) = _shrink(setdiff!, itr, itrs) +function setdiff(itr, itrs...) + T = eltype(itr) + keep = setdiff!(Set{T}(itr), itrs...) + vectorfilter(T, _shrink_filter!(keep), itr) +end function intersect(v::AbstractVector, r::AbstractRange) T = promote_eltype(v, r) diff --git a/base/arraymath.jl b/base/arraymath.jl index 53a7d132a2c0c..399791a356262 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -2,26 +2,59 @@ ## Binary arithmetic operators ## +function _broadcast_preserving_zero_d(f, A, B) + broadcast_preserving_zero_d(f, A, B) +end + +# Using map over broadcast enables vectorization for wide matrices with few rows. +# This is because we use linear indexing in `map` as opposed to Cartesian indexing in broadcasting. +# https://github.com/JuliaLang/julia/issues/47873#issuecomment-1352472461 +function _broadcast_preserving_zero_d(f, A::Array{<:Any,N}, B::Array{<:Any,N}, Cs::Array{<:Any,N}...) where {N} + map(f, A, B, Cs...) +end + +function _broadcast_preserving_zero_d(f, A::Array, B::Array, Cs::Array...) + # we already know that the shapes are compatible. + # We just need to select the size corresponding to the highest ndims + # and reshape all the arrays to that size + arrays = (A, B, Cs...) + sz = mapreduce(size, (x,y) -> length(x) > length(y) ? x : y, arrays) + # Skip reshaping where possible to avoid the overhead + arrays_sameshape = map(x -> length(sz) == ndims(x) ? x : reshape(x, sz), arrays) + map(f, arrays_sameshape...) +end + +function _broadcast_preserving_zero_d(f, A::Array, B::Number) + map(Fix2(f, B), A) +end + +function _broadcast_preserving_zero_d(f, A::Number, B::Array) + map(Fix1(f, A), B) +end + for f in (:+, :-) @eval function ($f)(A::AbstractArray, B::AbstractArray) promote_shape(A, B) # check size compatibility - broadcast_preserving_zero_d($f, A, B) + _broadcast_preserving_zero_d($f, A, B) end end -function +(A::Array, Bs::Array...) - for B in Bs - promote_shape(A, B) # check size compatibility ++(A::Array) = map(+, A) + +function +(A::Array, B::Array, Cs::Array...) + promote_shape(A, B) + for C in Cs + promote_shape(A, C) # check size compatibility end - broadcast_preserving_zero_d(+, A, Bs...) + _broadcast_preserving_zero_d(+, A, B, Cs...) end for f in (:/, :\, :*) if f !== :/ - @eval ($f)(A::Number, B::AbstractArray) = broadcast_preserving_zero_d($f, A, B) + @eval ($f)(A::Number, B::AbstractArray) = _broadcast_preserving_zero_d($f, A, B) end if f !== :\ - @eval ($f)(A::AbstractArray, B::Number) = broadcast_preserving_zero_d($f, A, B) + @eval ($f)(A::AbstractArray, B::Number) = _broadcast_preserving_zero_d($f, A, B) end end @@ -56,8 +89,8 @@ julia> reverse(b) !!! compat "Julia 1.6" Prior to Julia 1.6, only single-integer `dims` are supported in `reverse`. """ -reverse(A::AbstractArray; dims=:) = _reverse(A, dims) -_reverse(A, dims) = reverse!(copymutable(A); dims) +reverse(A::AbstractArray; dims::D=:) where {D} = _reverse(A, dims) +_reverse(A, dims::D) where {D} = reverse!(copymutable(A); dims) """ reverse!(A; dims=:) @@ -67,7 +100,7 @@ Like [`reverse`](@ref), but operates in-place in `A`. !!! compat "Julia 1.6" Multidimensional `reverse!` requires Julia 1.6. """ -reverse!(A::AbstractArray; dims=:) = _reverse!(A, dims) +reverse!(A::AbstractArray; dims::D=:) where {D} = _reverse!(A, dims) _reverse!(A::AbstractArray{<:Any,N}, ::Colon) where {N} = _reverse!(A, ntuple(identity, Val{N}())) _reverse!(A, dim::Integer) = _reverse!(A, (Int(dim),)) _reverse!(A, dims::NTuple{M,Integer}) where {M} = _reverse!(A, Int.(dims)) diff --git a/base/arrayshow.jl b/base/arrayshow.jl index f792d26d8e9b5..18ad5cb1a6a9f 100644 --- a/base/arrayshow.jl +++ b/base/arrayshow.jl @@ -183,7 +183,7 @@ function _print_matrix(io, @nospecialize(X::AbstractVecOrMat), pre, sep, post, h screenwidth -= length(pre)::Int + length(post)::Int presp = repeat(" ", length(pre)::Int) # indent each row to match pre string postsp = "" - @assert textwidth(hdots) == textwidth(ddots) + @assert textwidth(hdots) == textwidth(ddots) "hdots and ddots must have same textwidth" sepsize = length(sep)::Int m, n = length(rowsA), length(colsA) # To figure out alignments, only need to look at as many rows as could @@ -290,48 +290,56 @@ function _show_nd(io::IO, @nospecialize(a::AbstractArray), print_matrix::Functio reached_last_d = false for I in Is idxs = I.I - if limit - for i = 1:nd - ii = idxs[i] - ind = tailinds[i] - if length(ind) > 10 - if ii == ind[firstindex(ind)+3] && all(d->idxs[d]==first(tailinds[d]),1:i-1) - for j=i+1:nd - szj = length(axs[j+2]) - indj = tailinds[j] - if szj>10 && first(indj)+2 < idxs[j] <= last(indj)-3 - @goto skip + @label entry begin + if limit + for i = 1:nd + ii = idxs[i] + ind = tailinds[i] + if length(ind) > 10 + all_first = true + for d = 1:i-1 + if idxs[d] != first(tailinds[d]) + all_first = false + break end end - print(io, ";"^(i+2)) - print(io, " \u2026 ") - show_full && print(io, "\n\n") - @goto skip - end - if ind[firstindex(ind)+2] < ii <= ind[end-3] - @goto skip + if ii == ind[firstindex(ind)+3] && all_first + for j=i+1:nd + szj = length(axs[j+2]) + indj = tailinds[j] + if szj>10 && first(indj)+2 < idxs[j] <= last(indj)-3 + break entry + end + end + print(io, ";"^(i+2)) + print(io, " \u2026 ") + show_full && print(io, "\n\n") + break entry + end + if ind[firstindex(ind)+2] < ii <= ind[end-3] + break entry + end end end end - end - if show_full - _show_nd_label(io, a, idxs) - end - slice = view(a, axs[1], axs[2], idxs...) - if show_full - print_matrix(io, slice) - print(io, idxs == map(last,tailinds) ? "" : "\n\n") - else - idxdiff = lastidxs .- idxs .< 0 - if any(idxdiff) - lastchangeindex = 2 + findlast(idxdiff) - print(io, ";"^lastchangeindex) - lastchangeindex == ndims(a) && (reached_last_d = true) - print(io, " ") + if show_full + _show_nd_label(io, a, idxs) + end + slice = view(a, axs[1], axs[2], idxs...) + if show_full + print_matrix(io, slice) + print(io, idxs == map(last,tailinds) ? "" : "\n\n") + else + idxdiff = lastidxs .- idxs .< 0 + if any(idxdiff) + lastchangeindex = 2 + findlast(idxdiff) + print(io, ";"^lastchangeindex) + lastchangeindex == ndims(a) && (reached_last_d = true) + print(io, " ") + end + print_matrix(io, slice) end - print_matrix(io, slice) end - @label skip lastidxs = idxs end if !show_full @@ -379,11 +387,17 @@ function show(io::IO, ::MIME"text/plain", X::AbstractArray) io = IOContext(io, :limit => false) end - if get(io, :limit, false)::Bool && displaysize(io)[1]-4 <= 0 - return print(io, " …") - else - println(io) + if get(io, :limit, false)::Bool + # when there is no vertical room to show even one row of entries + # plus a vertical ellipsis, show as many entries as fit on a single + # line, truncated to the terminal width (#58323) + screenheight = displaysize(io)[1] - 4 + if screenheight <= 0 || (screenheight == 1 && (ndims(X) > 2 || size(X, 1) > 1)) + print(io, ' ') + return _show_oneline_truncated(io, X) + end end + println(io) # 3) update typeinfo # @@ -399,6 +413,15 @@ function show(io::IO, ::MIME"text/plain", X::AbstractArray) print_array(recur_io, X) end +function _show_oneline_truncated(io::IO, X::AbstractArray) + cols = displaysize(io)[2] + used = textwidth(sprint(summary, X; context=io)) + 2 + width = max(cols - used, 8) + ctx = IOContext(io, :typeinfo => typeof(X), :compact => true) + str = sprint(show, X; context=ctx, sizehint=min(4width, 4096)) + print(io, _truncate_at_width_or_chars(get(io, :color, false)::Bool, str, width)) +end + ## printing with `show` ### non-Vector arrays @@ -414,7 +437,7 @@ _show_nonempty(io::IO, X::AbstractMatrix, prefix::String) = _show_nonempty(io, inferencebarrier(X), prefix, false, axes(X)) function _show_nonempty(io::IO, @nospecialize(X::AbstractMatrix), prefix::String, drop_brackets::Bool, axs::Tuple{AbstractUnitRange,AbstractUnitRange}) - @assert !isempty(X) + @assert !isempty(X) "X should be non-empty" limit = get(io, :limit, false)::Bool indr, indc = axs nr, nc = length(indr), length(indc) @@ -488,9 +511,12 @@ function show(io::IO, X::AbstractArray) if !implicit io = IOContext(io, :typeinfo => eltype(X)) end - isempty(X) ? - _show_empty(io, X) : - _show_nonempty(io, X, prefix) + if isempty(X) + return _show_empty(io, X) + end + show_circular(io, X) && return + recur_io = IOContext(io, :SHOWN_SET => X) + _show_nonempty(recur_io, X, prefix) end ### 0-dimensional arrays (#31481) diff --git a/base/asyncevent.jl b/base/asyncevent.jl index 13b3a94580fec..0820164cd4738 100644 --- a/base/asyncevent.jl +++ b/base/asyncevent.jl @@ -5,7 +5,7 @@ """ AsyncCondition() -Create a async condition that wakes up tasks waiting for it +Create an async condition that wakes up tasks waiting for it (by calling [`wait`](@ref) on the object) when notified from C by a call to `uv_async_send`. Waiting tasks are woken with an error when the object is closed (by [`close`](@ref)). @@ -41,16 +41,20 @@ end """ AsyncCondition(callback::Function) -Create a async condition that calls the given `callback` function. The `callback` is passed one argument, +Create an async condition that calls the given `callback` function. The `callback` is passed one argument, the async condition object itself. """ function AsyncCondition(cb::Function) async = AsyncCondition() - t = @task begin - unpreserve_handle(async) - while _trywait(async) - cb(async) - isopen(async) || return + # Shielded like the `Timer` callback task below: this task owns the + # handle's lifetime and must survive a cancelled constructing scope. + t = ScopedValues.with(CANCEL_TOKEN => nothing) do + @task begin + unpreserve_handle(async) + while _trywait(async) + cb(async) + isopen(async) || return + end end end # here we are mimicking parts of _trywait, in coordination with task `t` @@ -59,7 +63,7 @@ function AsyncCondition(cb::Function) if async.set schedule(t) else - _wait2(async.cond, t) + schedule_on_notify!(async.cond, t) end end return async @@ -74,8 +78,10 @@ Create a timer that wakes up tasks waiting for it (by calling [`wait`](@ref) on Waiting tasks are woken after an initial delay of at least `delay` seconds, and then repeating after at least `interval` seconds again elapse. If `interval` is equal to `0`, the timer is only triggered -once. When the timer is closed (by [`close`](@ref)) waiting tasks are woken with an error. Use -[`isopen`](@ref) to check whether a timer is still active. An inactive timer will not fire. +once. When closing (by [`close`](@ref)) either a repeating timer or a one-shot timer before it has +triggered, waiting tasks are woken with an error. After a one-shot timer triggers, all subsequent calls +to [`wait`](@ref) return immediately, even if it is closed. +Use [`isopen`](@ref) to check whether a timer is still active. An inactive timer will not fire. Use `t.timeout` and `t.interval` to read the setup conditions of a `Timer` `t`. ```julia-repl @@ -102,9 +108,13 @@ false A `Timer` requires yield points to update its state. For instance, `isopen(t::Timer)` cannot be used to timeout a non-yielding while loop. -!!! compat "Julia 1.12 +!!! compat "Julia 1.12" The `timeout` and `interval` readable properties were added in Julia 1.12. +!!! compat "Julia 1.14" + Prior to Julia 1.14, only the first call to `wait` on a triggered one-shot timer returned, + and subsequent calls threw an `EOFError`. + """ mutable struct Timer @atomic handle::Ptr{Cvoid} @@ -126,13 +136,13 @@ mutable struct Timer associate_julia_struct(this.handle, this) iolock_begin() err = ccall(:uv_timer_init, Cint, (Ptr{Cvoid}, Ptr{Cvoid}), loop, this) - @assert err == 0 + @assert err == 0 "failed to initialize timer" finalizer(uvfinalize, this) ccall(:uv_update_time, Cvoid, (Ptr{Cvoid},), loop) err = ccall(:uv_timer_start, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, UInt64, UInt64), this, @cfunction(uv_timercb, Cvoid, (Ptr{Cvoid},)), timeoutms, intervalms) - @assert err == 0 + @assert err == 0 "failed to start timer" iolock_end() return this end @@ -161,13 +171,19 @@ unsafe_convert(::Type{Ptr{Cvoid}}, async::AsyncCondition) = async.handle # if this returns true, the object has been signaled # if this returns false, the object is closed -function _trywait(t::Union{Timer, AsyncCondition}) +# a cancellation of the governing token is thrown as a CancellationRequest +_trywait(t::Union{Timer, AsyncCondition}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + _trywait(t, resolve_cancel_token(cancel)) +function _trywait(t::Union{Timer, AsyncCondition}, tok::MaybeToken) set = t.set if set # full barrier now for AsyncCondition - t isa Timer || Core.Intrinsics.atomic_fence(:acquire_release) + t isa Timer || Core.Intrinsics.atomic_fence(:acquire_release, :system) else if !isopen(t) + # the :acquire read of isopen pairs with the :release store in uv_timercb, which + # sets `set` beforehand: a waiter observing the trigger-initiated close of a + # one-shot timer cannot miss the trigger on this recheck set = t.set if !set close(t) # wait for the close to complete @@ -179,28 +195,49 @@ function _trywait(t::Union{Timer, AsyncCondition}) if !set preserve_handle(t) lock(t.cond) + locked = true try set = t.set - if !set && t.handle != C_NULL # wait for set or handle, but not the isopen flag + while !set && t.handle != C_NULL # wait for set or handle, but not the isopen flag iolock_end() - set = wait(t.cond) + locked = false + ret = wait(t.cond, tok) + locked = true unlock(t.cond) + locked = false iolock_begin() lock(t.cond) + locked = true + if ret isa Bool + set = ret + break + end + # A wakeup that did not come from this object's notify: + # re-check the state and re-park. + set = t.set end finally - unlock(t.cond) + locked && unlock(t.cond) unpreserve_handle(t) end end iolock_end() end - @atomic :monotonic t.set = false # if there are multiple waiters, an unspecified number may short-circuit past here + if !(t isa Timer && iszero(t.interval_ms)) + # if there are multiple waiters, an unspecified number may short-circuit past here + @atomic :monotonic t.set = false + end return set end -function wait(t::Union{Timer, AsyncCondition}) - _trywait(t) || throw(EOFError()) +waitqueue(t::Union{Timer, AsyncCondition}) = waitqueue(t.cond) + +wait(t::Union{Timer, AsyncCondition}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + wait(t, check_cancel_arg(cancel)) +function wait(t::Union{Timer, AsyncCondition}, tok::MaybeToken) + ok = _trywait(t, tok) + @cancel_check(tok) + ok || throw(EOFError()) nothing end @@ -213,7 +250,7 @@ isopen(t::Union{Timer, AsyncCondition}) = @atomic :acquire t.isopen Close an object `t` and thus mark it as inactive. Once a timer or condition is inactive, it will not produce a new event. -See also: [`isopen`](@ref) +See also [`isopen`](@ref). """ function close(t::Union{Timer, AsyncCondition}) t.handle == C_NULL && !t.isopen && return # short-circuit path, :monotonic @@ -226,16 +263,23 @@ function close(t::Union{Timer, AsyncCondition}) # implement _trywait here without the auto-reset function, just waiting for the final close signal preserve_handle(t) lock(t.cond) + locked = true try while t.handle != C_NULL iolock_end() - wait(t.cond) + # close is the cleanup primitive: its (bounded) completion + # wait is shielded from cancellation + locked = false + wait(t.cond, nothing) + locked = true unlock(t.cond) + locked = false iolock_begin() lock(t.cond) + locked = true end finally - unlock(t.cond) + locked && unlock(t.cond) unpreserve_handle(t) end elseif t.isopen @@ -295,6 +339,8 @@ function uv_timercb(handle::Ptr{Cvoid}) t = @handle_as handle Timer lock(t.cond) try + # this store must stay ordered before the :release store of isopen below, so that + # a waiter observing the close in _trywait is guaranteed to also observe `set` @atomic :monotonic t.set = true if ccall(:uv_timer_get_repeat, UInt64, (Ptr{Cvoid},), t) == 0 # timer is stopped now @@ -311,14 +357,24 @@ function uv_timercb(handle::Ptr{Cvoid}) end """ - sleep(seconds) + sleep(seconds; cancel=Base.DEFAULT_CANCEL) Block the current task for a specified number of seconds. The minimum sleep time is 1 millisecond or input of `0.001`. + +A cancellation of the governing token (by default the scoped token, see +[`CancellationToken`](@ref)) interrupts the sleep by throwing the +[`CancellationRequest`](@ref). """ -function sleep(sec::Real) +function sleep(sec::Real; cancel::CancelTokenArg=DEFAULT_CANCEL) sec ≥ 0 || throw(ArgumentError("cannot sleep for $sec seconds")) - wait(Timer(sec)) + tok = check_cancel_arg(cancel) + t = Timer(sec) + try + wait(t, tok) + finally + close(t) + end nothing end @@ -363,17 +419,23 @@ julia> begin function Timer(cb::Function, timeout; spawn::Union{Nothing,Bool}=nothing, kwargs...) sticky = spawn === nothing ? current_task().sticky : !spawn timer = Timer(timeout; kwargs...) - t = @task begin - unpreserve_handle(timer) - while _trywait(timer) - try - cb(timer) - catch err - write(stderr, "Error in Timer:\n") - showerror(stderr, err, catch_backtrace()) - return + # The callback task carries the timer's lifetime (its preserve is + # balanced in the body): shield it from the constructing scope's + # cancellation token, so a cancelled scope can neither leak the preserve + # nor stop the timer - `close(timer)` is what ends it. + t = ScopedValues.with(CANCEL_TOKEN => nothing) do + @task begin + unpreserve_handle(timer) + while _trywait(timer) + try + cb(timer) + catch err + write(stderr, "Error in Timer:\n") + showerror(stderr, err, catch_backtrace()) + return + end + isopen(timer) || return end - isopen(timer) || return end end t.sticky = sticky @@ -383,7 +445,7 @@ function Timer(cb::Function, timeout; spawn::Union{Nothing,Bool}=nothing, kwargs if timer.set schedule(t) else - _wait2(timer.cond, t) + schedule_on_notify!(timer.cond, t) end end return timer @@ -423,10 +485,58 @@ function timedwait(testcb, timeout::Real; pollint::Real=0.1) if testcb() close(t) return :ok - elseif (time_ns() - start) > ns_timeout + elseif (time_ns() -% start) > ns_timeout close(t) break end end return :timed_out end + +## A deadline as a waitable (see base/park.jl): `TimeoutWait(dt)` in a +## park's waitables wakes the wait with `:timed_out` after `dt` seconds. +## Its enqueue - running inside `park!`, after the arm - starts the timer +## and spawns the claimer task, which wakes the parked task through the +## standard expected-entry claim CAS. That CAS is a *specific-wait* waker: +## it is only sound against a fresh, single-use entry (which the entry +## cache contract supplies automatically - any non-canonical waitable +## shape gets a fresh entry), since entry identity is what scopes the +## claim to this wait and not a later one of the same task. The claimed +## waitq registration is left for the driver's lazy settle; the dequeue +## closes the timer on every exit path. +mutable struct TimeoutWait + const timeout::Float64 + timer::Union{Timer, Nothing} + TimeoutWait(timeout::Real) = new(Float64(timeout), nothing) +end + +function wait_enqueue!(x::TimeoutWait, w::WaitEntry, first::Bool) + ct = current_task() + timer = Timer(x.timeout) + x.timer = timer + t = Task() do + try + # not cancellable: internal mechanism; closing the timer wakes + # this task on every exit path of the governed wait + wait(timer; cancel=nothing) + catch e + # a closed timer means the wait ended first; do nothing + e isa EOFError && return + rethrow() + end + if (@atomicreplace ct.waiting_on w => nothing).success + schedule(ct, :timed_out) + end + end + t.sticky = false + Threads._spawn_set_thrpool(t, :interactive) + schedule(t) + return true +end + +function wait_dequeue!(x::TimeoutWait, w::WaitEntry, why::UInt8) + timer = x.timer + timer === nothing || close(timer) + x.timer = nothing + return nothing +end diff --git a/base/asyncmap.jl b/base/asyncmap.jl index 1914ddc645f31..51cfb32af83c7 100644 --- a/base/asyncmap.jl +++ b/base/asyncmap.jl @@ -74,31 +74,31 @@ julia> asyncmap(batch_func, 1:5; ntasks=2, batch_size=2) ``` """ function asyncmap(f, c...; ntasks=0, batch_size=nothing) - return async_usemap(f, c...; ntasks=ntasks, batch_size=batch_size) -end - -function async_usemap(f, c...; ntasks=0, batch_size=nothing) - ntasks = verify_ntasks(c[1], ntasks) - batch_size = verify_batch_size(batch_size) - - if batch_size !== nothing - exec_func = batch -> begin - # extract the Refs from the input tuple - batch_refs = map(x->x[1], batch) + src = isone(length(c)) ? c[1] : zip(c...) + if haslength(src) + n = length(src) + results = Vector{Any}(undef, n) + else + results = Dict{Int,Any}() + end - # and the args tuple.... - batched_args = map(x->x[2], batch) + foreach(identity, AsyncCollector(f, results, c...; ntasks=ntasks, batch_size=batch_size)) - results = f(batched_args) - foreach(x -> (batch_refs[x[1]].x = x[2]), enumerate(results)) - end + if results isa Dict + v = Any[results[i] for i in 1:length(results)] else - exec_func = (r,args) -> (r.x = f(args...)) + v = results end - chnl, worker_tasks = setup_chnl_and_tasks(exec_func, ntasks, batch_size) - return wrap_n_exec_twice(chnl, worker_tasks, ntasks, exec_func, c...) + return _asyncmap_result(v, src) end +# Narrow element type and reshape to try to match what `map` would return. +_asyncmap_result(v, c1::Tuple) = Tuple(map(identity, v)) +_asyncmap_result(v, c1::AbstractArray) = collect_similar(c1, Generator(identity, reshape(v, axes(c1)))) +_asyncmap_result(v, c1) = _asyncmap_result(v, c1, IteratorSize(c1)) +_asyncmap_result(v, c1, ::HasShape) = collect(Generator(identity, reshape(v, axes(c1)))) +_asyncmap_result(v, _, _) = map(identity, v) + batch_size_err_str(batch_size) = string("batch_size must be specified as a positive integer. batch_size=", batch_size) function verify_batch_size(batch_size) if batch_size === nothing @@ -129,60 +129,6 @@ function verify_ntasks(iterable, ntasks) return ntasks end -function wrap_n_exec_twice(chnl, worker_tasks, ntasks, exec_func, c...) - # The driver task, creates a Ref object and writes it and the args tuple to - # the communication channel for processing by a free worker task. - push_arg_to_channel = (x...) -> (r=Ref{Any}(nothing); put!(chnl,(r,x));r) - - if isa(ntasks, Function) - map_f = (x...) -> begin - # check number of tasks every time, and start one if required. - # number_tasks > optimal_number is fine, the other way around is inefficient. - if length(worker_tasks) < ntasks() - start_worker_task!(worker_tasks, exec_func, chnl) - end - push_arg_to_channel(x...) - end - else - map_f = push_arg_to_channel - end - maptwice(map_f, chnl, worker_tasks, c...) -end - -function maptwice(wrapped_f, chnl, worker_tasks, c...) - # first run, returns a collection of Refs - asyncrun_excp = nothing - local asyncrun - try - asyncrun = map(wrapped_f, c...) - catch ex - if isa(ex,InvalidStateException) - # channel could be closed due to exceptions in the async tasks, - # we propagate those errors, if any, over the `put!` failing - # in asyncrun due to a closed channel. - asyncrun_excp = ex - else - rethrow() - end - end - - # close channel and wait for all worker tasks to finish - close(chnl) - - # check and throw any exceptions from the worker tasks - foreach(x->(v=fetch(x); isa(v, Exception) && throw(v)), worker_tasks) - - # check if there was a genuine problem with asyncrun - (asyncrun_excp !== nothing) && throw(asyncrun_excp) - - if isa(asyncrun, Ref) - # scalar case - return asyncrun.x - else - # second run, extract values from the Refs and return - return map(ref->ref.x, asyncrun) - end -end function setup_chnl_and_tasks(exec_func, ntasks, batch_size=nothing) if isa(ntasks, Function) @@ -199,7 +145,7 @@ function setup_chnl_and_tasks(exec_func, ntasks, batch_size=nothing) # of an error in any of the worker tasks, the channel is closed. This # results in the `put!` in the driver task failing immediately. chnl = Channel(0) - worker_tasks = [] + worker_tasks = Task[] foreach(_ -> start_worker_task!(worker_tasks, exec_func, chnl, batch_size), 1:nt) yield() return (chnl, worker_tasks) @@ -246,15 +192,6 @@ function asyncmap(f, s::AbstractString; kwargs...) return String(s2) end -# map on a single BitArray returns a BitArray if the mapping function is boolean. -function asyncmap(f, b::BitArray; kwargs...) - b2 = async_usemap(f, b; kwargs...) - if eltype(b2) == Bool - return BitArray(b2) - end - return b2 -end - mutable struct AsyncCollector f results @@ -292,21 +229,32 @@ end mutable struct AsyncCollectorState chnl::Channel - worker_tasks::Array{Task,1} + worker_tasks::Vector{Task} + exec_func + batch_size enum_state # enumerator state - AsyncCollectorState(chnl::Channel, worker_tasks::Vector) = - new(chnl, convert(Vector{Task}, worker_tasks)) end function iterate(itr::AsyncCollector) itr.ntasks = verify_ntasks(itr.enumerator, itr.ntasks) itr.batch_size = verify_batch_size(itr.batch_size) - chnl, worker_tasks = setup_chnl_and_tasks((i,args) -> (itr.results[i]=itr.f(args...)), itr.ntasks, itr.batch_size) - return iterate(itr, AsyncCollectorState(chnl, worker_tasks)) + if itr.batch_size !== nothing + exec_func = batch -> begin + indices = map(x -> x[1], batch) + batched_args = map(x -> x[2], batch) + res = itr.f(batched_args) + foreach(((j, v),) -> (itr.results[indices[j]] = v), enumerate(res)) + end + else + exec_func = (i, args) -> (itr.results[i] = itr.f(args...)) + end + + chnl, worker_tasks = setup_chnl_and_tasks(exec_func, itr.ntasks, itr.batch_size) + return iterate(itr, AsyncCollectorState(chnl, worker_tasks, exec_func, itr.batch_size, nothing)) end -function wait_done(itr::AsyncCollector, state::AsyncCollectorState) +function wait_done(state::AsyncCollectorState) close(state.chnl) # wait for all tasks to finish @@ -316,19 +264,28 @@ end function iterate(itr::AsyncCollector, state::AsyncCollectorState) if itr.nt_check && (length(state.worker_tasks) < itr.ntasks()) - start_worker_task!(state.worker_tasks, itr.f, state.chnl) + start_worker_task!(state.worker_tasks, state.exec_func, state.chnl, state.batch_size) end # Get index and mapped function arguments from enumeration iterator. - y = isdefined(state, :enum_state) ? + y = !isnothing(state.enum_state) ? iterate(itr.enumerator, state.enum_state) : iterate(itr.enumerator) - if y === nothing - wait_done(itr, state) + if isnothing(y) + wait_done(state) return nothing end (i, args), state.enum_state = y - put!(state.chnl, (i, args)) + try + put!(state.chnl, (i, args)) + catch + # Prefer throwing a worker exception over the put! failure. + for t in state.worker_tasks + v = fetch(t) + isa(v, Exception) && throw(v) + end + rethrow() + end return (nothing, state) end @@ -357,23 +314,21 @@ end mutable struct AsyncGeneratorState i::Int collector_done::Bool - collector_state::AsyncCollectorState - AsyncGeneratorState(i::Int) = new(i, false) + collector_state::Union{AsyncCollectorState, Nothing} end -function iterate(itr::AsyncGenerator, state::AsyncGeneratorState=AsyncGeneratorState(0)) +function iterate(itr::AsyncGenerator, state::AsyncGeneratorState=AsyncGeneratorState(0, false, nothing)) state.i += 1 results_dict = itr.collector.results while !state.collector_done && !haskey(results_dict, state.i) - y = isdefined(state, :collector_state) ? + y = !isnothing(state.collector_state) ? iterate(itr.collector, state.collector_state) : iterate(itr.collector) - if y === nothing - # `check_done` waits for async tasks to finish. if we do not have the index - # we are looking for, it is an error. + if isnothing(y) + # All input consumed; async tasks may still be in flight. state.collector_done = true - break; + break end _, state.collector_state = y end @@ -384,12 +339,8 @@ function iterate(itr::AsyncGenerator, state::AsyncGeneratorState=AsyncGeneratorS return (r, state) end -# pass-through iterator traits to the iterable -# on which the mapping function is being applied IteratorSize(::Type{AsyncGenerator}) = SizeUnknown() IteratorEltype(::Type{AsyncGenerator}) = EltypeUnknown() -size(itr::AsyncGenerator) = size(itr.collector.enumerator) -length(itr::AsyncGenerator) = length(itr.collector.enumerator) """ asyncmap!(f, results, c...; ntasks=0, batch_size=nothing) diff --git a/base/atomics.jl b/base/atomics.jl index 432c9120939ac..2c92142539aee 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -1,7 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import .Base: setindex!, getindex, unsafe_convert -import .Base.Sys: ARCH, WORD_SIZE +import .Base: getindex +import .Base: getindex_atomic, setindex_atomic!, swapindex_atomic!, modifyindex_atomic!, + replaceindex_atomic!, setindexonce_atomic! export Atomic, @@ -10,7 +11,7 @@ export atomic_add!, atomic_sub!, atomic_and!, atomic_nand!, atomic_or!, atomic_xor!, atomic_max!, atomic_min!, - atomic_fence + atomic_fence, atomic_fence_light, atomic_fence_heavy """ Threads.Atomic{T} @@ -21,22 +22,46 @@ accessed atomically, i.e. in a thread-safe manner. New atomic objects can be created from a non-atomic values; if none is specified, the atomic object is initialized with zero. -Atomic objects can be accessed using the `[]` notation: +`Atomic` is a standalone, [`Ref`](@ref)-like atomic cell. Like `Ref`, it is a +useful building block and is not going away, but an `@atomic` field of a mutable +struct is usually preferable when you have the choice, since it avoids the extra +indirection. + +The value can be loaded with the `[]` notation (`x[]`) and stored with the +reference form of the [`@atomic`](@ref), [`@atomicswap`](@ref), +[`@atomicreplace`](@ref), and [`@atomiconce`](@ref) macros, which also allow the +memory ordering to be specified explicitly: # Examples ```jldoctest julia> x = Threads.Atomic{Int}(3) Base.Threads.Atomic{Int64}(3) -julia> x[] = 1 -1 +julia> @atomic x[] = 4 +4 + +julia> @atomic :monotonic x[] +4 + +julia> @atomicswap x[] = 5 +4 julia> x[] -1 +5 ``` Atomic operations use an `atomic_` prefix, such as [`atomic_add!`](@ref), [`atomic_xchg!`](@ref), etc. + +!!! warning + Storing with the plain `x[] = value` form is deprecated: read-modify-write + expressions such as `x[] += 1` look atomic but expand to a separate, + non-atomic load and store. Use `@atomic x[] = value` (and, for an atomic + read-modify-write, `@atomic x[] += 1` or [`atomic_add!`](@ref)) instead. + +!!! compat "Julia 1.14" + The `@atomic` reference form (`@atomic x[]`) on `Threads.Atomic` requires + at least Julia 1.14. """ mutable struct Atomic{T} @atomic value::T @@ -302,7 +327,28 @@ function atomic_min! end nand(x, y) = ~(x & y) getindex(x::Atomic) = @atomic :acquire x.value -setindex!(x::Atomic, v) = (@atomic :release x.value = v; x) +# NOTE: `setindex!(x::Atomic, v)` (the `x[] = v` form) is deprecated in favor of +# `@atomic x[] = v`; see `base/deprecated.jl`. Reads via `x[]` remain a plain +# atomic (acquire) load. + +# Support the reference form of the `@atomic` family of macros (`@atomic x[]`, +# `@atomicswap x[] = v`, `@atomicreplace x[] o => n`, ...) so that an `Atomic` +# can be used as a single atomic memory location with an explicit memory order. +@inline getindex_atomic(x::Atomic, order::Symbol) = getfield(x, :value, order) +@inline function setindex_atomic!(x::Atomic{T}, order::Symbol, v) where {T} + return setfield!(x, :value, v isa T ? v : convert(T, v)::T, order) +end +@inline function swapindex_atomic!(x::Atomic{T}, order::Symbol, v) where {T} + return swapfield!(x, :value, v isa T ? v : convert(T, v)::T, order) +end +@inline modifyindex_atomic!(x::Atomic, order::Symbol, op, v) = + modifyfield!(x, :value, op, v, order) +@inline function replaceindex_atomic!(x::Atomic{T}, success_order::Symbol, fail_order::Symbol, expected, desired) where {T} + return replacefield!(x, :value, expected, desired isa T ? desired : convert(T, desired)::T, success_order, fail_order) +end +@inline function setindexonce_atomic!(x::Atomic{T}, success_order::Symbol, fail_order::Symbol, v) where {T} + return setfieldonce!(x, :value, v isa T ? v : convert(T, v)::T, success_order, fail_order) +end atomic_cas!(x::Atomic, cmp, new) = (@atomicreplace :acquire_release :acquire x.value cmp => new).old atomic_add!(x::Atomic, v) = (@atomic :acquire_release x.value + v).first atomic_sub!(x::Atomic, v) = (@atomic :acquire_release x.value - v).first @@ -329,4 +375,28 @@ fences should not be necessary in most cases. For further details, see LLVM's `fence` instruction. """ -atomic_fence() = Core.Intrinsics.atomic_fence(:sequentially_consistent) +atomic_fence() = Core.Intrinsics.atomic_fence(:sequentially_consistent, :system) + +""" + Threads.atomic_fence_light() + +Insert the light side of an asymmetric sequential-consistency memory fence. +Asymmetric memory fences are useful in scenarios where one side of the +synchronization runs significantly less often than the other side. Use this +function on the side that runs often and [`atomic_fence_heavy`](@ref) on the +side that runs rarely. + +On supported operating systems and architectures this fence is cheaper than +`Threads.atomic_fence()`, but synchronizes only with [`atomic_fence_heavy`](@ref) +calls from other threads. +""" +atomic_fence_light() = Core.Intrinsics.atomic_fence(:sequentially_consistent, :singlethread) + +""" + Threads.atomic_fence_heavy() + +Insert the heavy side of an asymmetric sequential-consistency memory fence. +Use this function on the side that runs rarely. +See [`atomic_fence_light`](@ref) for more details. +""" +atomic_fence_heavy() = ccall(:jl_membarrier, Cvoid, ()) diff --git a/base/baseext.jl b/base/baseext.jl index 625a82ff29234..8e0ad363961d7 100644 --- a/base/baseext.jl +++ b/base/baseext.jl @@ -19,9 +19,6 @@ convert(::Type{T}, arg::T) where {T<:VecElement} = arg convert(::Type{T}, arg) where {T<:VecElement} = T(arg)::T # ## dims-type-converting Array constructors for convenience -# type and dimensionality specified, accepting dims as series of Integers -Vector{T}(::UndefInitializer, m::Integer) where {T} = Vector{T}(undef, Int(m)) -Matrix{T}(::UndefInitializer, m::Integer, n::Integer) where {T} = Matrix{T}(undef, Int(m), Int(n)) Array{T,N}(::UndefInitializer, d::Vararg{Integer,N}) where {T,N} = Array{T,N}(undef, convert(Tuple{Vararg{Int}}, d)) # type but not dimensionality specified, accepting dims as series of Integers Array{T}(::UndefInitializer, m::Integer) where {T} = Array{T,1}(undef, Int(m)) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index c2f019c4d4eea..0bb7e8e24f50e 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -26,7 +26,8 @@ system, libc implementation, etc... It is, at its heart, a key-value mapping of `"os" => "windows"`, etc...). `Platform` objects are extensible in that the tag mapping is open for users to add their own mappings to, as long as the mappings do not conflict with the set of reserved tags: `arch`, `os`, `os_version`, `libc`, `call_abi`, -`libgfortran_version`, `libstdcxx_version`, `cxxstring_abi` and `julia_version`. +`libgfortran_version`, `libstdcxx_version`, `cxxlib`, `cxxlib_version`, +`cxxstring_abi` and `julia_version`. Valid tags and values are composed of alphanumeric and period characters. All tags and values will be lowercased when stored to reduce variation. @@ -64,23 +65,7 @@ struct Platform <: AbstractPlatform continue end - # Normalize things that are known to be version numbers so that comparisons are easy. - # Note that in our effort to be extremely compatible, we actually allow something that - # doesn't parse nicely into a VersionNumber to persist, but if `validate_strict` is - # set to `true`, it will cause an error later on. - if tag ∈ ("libgfortran_version", "libstdcxx_version", "os_version") - if isa(value, VersionNumber) - value = string(value) - elseif isa(value, String) - v = tryparse(VersionNumber, value) - if isa(v, VersionNumber) - value = string(v) - end - end - end - - # Use `add_tag!()` to add the tag to our collection of tags - add_tag!(tags, tag, string(value)::String) + add_platform_tag!(tags, tag, value) end # Auto-map call_abi and libc where necessary: @@ -88,6 +73,10 @@ struct Platform <: AbstractPlatform # Default to `glibc` on Linux tags["libc"] = "glibc" end + if os == "windows" && !haskey(tags, "libc") + # Default to `msvcrt` on Windows + tags["libc"] = "msvcrt" + end if os == "linux" && arch ∈ ("armv7l", "armv6l") && "call_abi" ∉ keys(tags) # default `call_abi` to `eabihf` on 32-bit ARM tags["call_abi"] = "eabihf" @@ -124,6 +113,43 @@ tagvalue(v::Union{String,VersionNumber,Nothing}) = v tagvalue(v::Symbol) = String(v) tagvalue(v::AbstractString) = convert(String, v)::String +function add_platform_tag!(tags::Dict{String,String}, tag::String, value::Union{String,VersionNumber,Nothing}) + tag = lowercase(tag) + + # Drop `nothing` values; this means feature is not present or use default value. + if value === nothing + return nothing + end + + # For compatibility, libstdcxx_version counts as both cxxlib=libstdcxx and + # cxxlib_version, but don't override an explicit existing cxxlib tag (the + # verifier will check for inconsistencies). + if tag == "libstdcxx_version" + haskey(tags, "cxxlib") || add_tag!(tags, "cxxlib", "libstdcxx") + tag = "cxxlib_version" + elseif tag == "cxxstring_abi" + # Implies cxxlib=libstdcxx for compatibility + haskey(tags, "cxxlib") || add_tag!(tags, "cxxlib", "libstdcxx") + end + + # Normalize things that are known to be version numbers so that comparisons are easy. + # Note that in our effort to be extremely compatible, we actually allow something that + # doesn't parse nicely into a VersionNumber to persist, but if `validate_strict` is + # set to `true`, it will cause an error later on. + if tag ∈ ("libgfortran_version", "cxxlib_version", "os_version") + if isa(value, VersionNumber) + value = string(value) + elseif isa(value, String) + v = tryparse(VersionNumber, value) + if isa(v, VersionNumber) + value = string(v) + end + end + end + + return add_tag!(tags, tag, string(value)::String) +end + # Simple tag insertion that performs a little bit of validation function add_tag!(tags::Dict{String,String}, tag::String, value::String) # I know we said only alphanumeric and dots, but let's be generous so that we can expand @@ -151,7 +177,7 @@ tags(p::Platform) = p.tags Base.getindex(p::AbstractPlatform, k::String) = getindex(tags(p), k) Base.haskey(p::AbstractPlatform, k::String) = haskey(tags(p), k) function Base.setindex!(p::AbstractPlatform, v::String, k::String) - add_tag!(tags(p), k, v) + add_platform_tag!(tags(p), k, v) return p end @@ -176,7 +202,9 @@ function Base.show(io::IO, p::Platform) print(io, ", ") show(io, os(p)) print(io, "; ") - join(io, ("$(k) = $(repr(v))" for (k, v) in tags(p) if k ∉ ("arch", "os")), ", ") + # Sort tags so that the output does not depend on `Dict` iteration order + other_tags = sort!(filter!(kv -> kv[1] ∉ ("arch", "os"), collect(tags(p))); by=first) + join(io, ("$(k) = $(repr(v))" for (k, v) in other_tags), ", ") print(io, ")") end @@ -217,6 +245,10 @@ function validate_tags(tags::Dict) if tags["libc"] ∉ ("glibc", "musl") throw_libc_mismatch() end + elseif tags["os"] == "windows" + if tags["libc"] ∉ ("msvcrt", "ucrt") + throw_libc_mismatch() + end else # Nothing else is allowed to have a `libc` entry if haskey(tags, "libc") @@ -227,7 +259,7 @@ function validate_tags(tags::Dict) # Validate `os`/`arch`/`call_abi` combination throw_call_abi_mismatch() = throw(ArgumentError("Invalid os/arch/call_abi combination: $(tags["os"])/$(tags["arch"])/$(tags["call_abi"])")) if tags["os"] == "linux" && tags["arch"] ∈ ("armv7l", "armv6l") - # If an ARM linux has does not have `call_abi` set to something valid, be sad. + # If an ARM linux does not have `call_abi` set to something valid, be sad. if !haskey(tags, "call_abi") || tags["call_abi"] ∉ ("eabihf", "eabi") throw_call_abi_mismatch() end @@ -244,14 +276,19 @@ function validate_tags(tags::Dict) throw_version_number("libgfortran_version") end - # Validate `cxxstring_abi` is one of the two valid options: - if "cxxstring_abi" in keys(tags) && tags["cxxstring_abi"] ∉ ("cxx03", "cxx11") + # Validate `cxxlib` is one of the valid options. + if haskey(tags, "cxxlib") && tags["cxxlib"] ∉ ("libstdcxx", "libcxx") + throw_invalid_key("cxxlib") + end + + # Validate `cxxstring_abi` is one of the two valid options and only used with libstdc++. + if haskey(tags, "cxxstring_abi") && (tags["cxxstring_abi"] ∉ ("cxx03", "cxx11") || !haskey(tags, "cxxlib") || tags["cxxlib"] != "libstdcxx") throw_invalid_key("cxxstring_abi") end - # Validate `libstdcxx_version` is a parsable `VersionNumber` - if "libstdcxx_version" in keys(tags) && tryparse(VersionNumber, tags["libstdcxx_version"]) === nothing - throw_version_number("libstdcxx_version") + # Validate `cxxlib_version` is a parsable `VersionNumber` + if haskey(tags, "cxxlib_version") && tryparse(VersionNumber, tags["cxxlib_version"]) === nothing + throw_version_number("cxxlib_version") end end @@ -331,8 +368,8 @@ function HostPlatform(p::AbstractPlatform) if haskey(p, "os_version") set_compare_strategy!(p, "os_version", compare_version_cap) end - if haskey(p, "libstdcxx_version") - set_compare_strategy!(p, "libstdcxx_version", compare_version_cap) + if haskey(p, "cxxlib") && p["cxxlib"] == "libstdcxx" && haskey(p, "cxxlib_version") + set_compare_strategy!(p, "cxxlib_version", compare_version_cap) end return p end @@ -399,6 +436,7 @@ julia> libc(Platform("aarch64", "linux"; libc="musl")) "musl" julia> libc(Platform("i686", "Windows")) +"msvcrt" ``` """ libc(p::AbstractPlatform) = get(tags(p), "libc", nothing) @@ -457,9 +495,19 @@ libgfortran_version(p::AbstractPlatform) = VNorNothing(tags(p), "libgfortran_ver libstdcxx_version(p::AbstractPlatform) Get the libstdc++ version dictated by this `Platform` object, or `nothing` if no -compatibility bound is imposed. +compatibility bound is imposed. This is a compatibility accessor for +`cxxlib = "libstdcxx"` platforms with a `cxxlib_version`. """ -libstdcxx_version(p::AbstractPlatform) = VNorNothing(tags(p), "libstdcxx_version") +function libstdcxx_version(p::AbstractPlatform) + platform_tags = tags(p) + if haskey(platform_tags, "libstdcxx_version") + return VNorNothing(platform_tags, "libstdcxx_version") + end + if get(platform_tags, "cxxlib", nothing) == "libstdcxx" + return VNorNothing(platform_tags, "cxxlib_version") + end + return nothing +end """ cxxstring_abi(p::AbstractPlatform) @@ -520,21 +568,32 @@ function triplet(p::AbstractPlatform) ) # Tack on optional compiler ABI flags - if libgfortran_version(p) !== nothing - str = string(str, "-libgfortran", libgfortran_version(p).major) + libgfortran_version_ = libgfortran_version(p) + if libgfortran_version_ !== nothing + str = string(str, "-libgfortran", libgfortran_version_.major) end - if cxxstring_abi(p) !== nothing - str = string(str, "-", cxxstring_abi(p)) + cxxstring_abi_ = cxxstring_abi(p) + if cxxstring_abi_ !== nothing + str = string(str, "-", cxxstring_abi_) end - if libstdcxx_version(p) !== nothing - str = string(str, "-libstdcxx", libstdcxx_version(p).patch) + libstdcxx_version_ = libstdcxx_version(p) + if libstdcxx_version_ !== nothing + str = string(str, "-libstdcxx", libstdcxx_version_.patch) end - # Tack on all extra tags - for (tag, val) in tags(p) + # Tack on all extra tags, sorted so that the output does not depend on `Dict` iteration order + for (tag, val) in sort!(collect(tags(p)); by=first) if tag ∈ ("os", "arch", "libc", "call_abi", "libgfortran_version", "libstdcxx_version", "cxxstring_abi", "os_version") continue end + if tag == "cxxlib" && val == "libstdcxx" && (cxxstring_abi_ !== nothing || libstdcxx_version_ !== nothing) + # Implied by above + continue + end + if tag == "cxxlib_version" && get(tags(p), "cxxlib", nothing) == "libstdcxx" + # Emitted as a libstdcxx compatibility tag above + continue + end str = string(str, "-", tag, "+", val) end return str @@ -551,7 +610,7 @@ function os_str(p::AbstractPlatform) return "-apple-darwin" end elseif os(p) == "windows" - return "-w64-mingw32" + return "-w64" elseif os(p) == "freebsd" osvn = os_version(p) if osvn !== nothing @@ -573,6 +632,10 @@ function libc_str(p::AbstractPlatform) return "" elseif lc === "glibc" return "-gnu" + elseif lc === "msvcrt" + return "-mingw32" + elseif lc === "ucrt" + return "-ucrt-mingw32" else return string("-", lc) end @@ -617,13 +680,6 @@ const arch_march_isa_mapping = let "avx2" => get_set("x86_64", "haswell"), "avx512" => get_set("x86_64", "skylake_avx512"), ], - "armv6l" => [ - "arm1176jzfs" => get_set("armv6l", "arm1176jzfs"), - ], - "armv7l" => [ - "armv7l" => get_set("armv7l", "armv7l"), - "neonvfpv4" => get_set("armv7l", "armv7l+neon+vfpv4"), - ], "aarch64" => [ "armv8_0" => get_set("aarch64", "armv8.0-a"), "armv8_1" => get_set("aarch64", "armv8.1-a"), @@ -631,9 +687,6 @@ const arch_march_isa_mapping = let "a64fx" => get_set("aarch64", "a64fx"), "apple_m1" => get_set("aarch64", "apple_m1"), ], - "powerpc64le" => [ - "power8" => get_set("powerpc64le", "power8"), - ], "riscv64" => [ "riscv64" => get_set("riscv64", "riscv64"), ], @@ -643,11 +696,13 @@ const os_mapping = Dict( "macos" => "-apple-darwin[\\d\\.]*", "freebsd" => "-(.*-)?freebsd[\\d\\.]*", "openbsd" => "-(.*-)?openbsd[\\d\\.]*", - "windows" => "-w64-mingw32", + "windows" => "-w64", "linux" => "-(.*-)?linux", ) const libc_mapping = Dict( "libc_nothing" => "", + "ucrt" => "-ucrt-mingw32", + "msvcrt" => "-mingw32", # We default to msvcrt for plain -mingw32 on Windows "glibc" => "-gnu", "musl" => "-musl", ) @@ -829,7 +884,7 @@ function parse_dl_name_version(path::String, os::String=_this_os_name()) # On OSX, libraries look like `libnettle.6.3.dylib` dlregex = r"^(.*?)((?:\.[\d]+)*)\.dylib$"sa else - # On Linux and others BSD, libraries look like `libnettle.so.6.3.0` + # On Linux and other BSDs, libraries look like `libnettle.so.6.3.0` dlregex = r"^(.*?)\.so((?:\.[\d]+)*)$"sa end @@ -855,7 +910,7 @@ function parse_dl_name_version(path::AbstractString, os::AbstractString=_this_os end function get_csl_member(member::Symbol) - # If CompilerSupportLibraries_jll is an stdlib, we can just grab things from it + # If CompilerSupportLibraries_jll is a stdlib, we can just grab things from it csl_pkgids = filter(pkgid -> pkgid.name == "CompilerSupportLibraries_jll", keys(Base.loaded_modules)) if !isempty(csl_pkgids) CSL_mod = Base.loaded_modules[first(csl_pkgids)] @@ -869,6 +924,42 @@ function get_csl_member(member::Symbol) return nothing end + +function _get_libgfortran_path() + # If CompilerSupportLibraries_jll is a stdlib, we can just directly ask for + # the path here, without checking `dllist()`: + libgfortran_path = get_csl_member(:libgfortran_path) + if libgfortran_path !== nothing + return libgfortran_path::String + end + + # Otherwise, look for it having already been loaded by something + libgfortran_paths = filter!(x -> occursin("libgfortran", x), Libdl.dllist()) + if !isempty(libgfortran_paths) + return first(libgfortran_paths)::String + end + + # One day, I hope to not be linking against libgfortran in base Julia + return nothing +end + +function _get_libstdcxx_handle() + # If CompilerSupportLibraries_jll is a stdlib, we can just directly open it + libstdcxx = get_csl_member(:libstdcxx) + if libstdcxx !== nothing + return nothing + end + + # Otherwise, look for it having already been loaded by something + libstdcxx_paths = filter!(x -> occursin("libstdc++", x), Libdl.dllist()) + if !isempty(libstdcxx_paths) + return Libdl.dlopen(first(libstdcxx_paths), Libdl.RTLD_NOLOAD)::Ptr{Cvoid} + end + + # One day, I hope to not be linking against libstdc++ in base Julia + return nothing +end + """ detect_libgfortran_version() @@ -877,26 +968,8 @@ linked against (if any). Returns `nothing` if no libgfortran version dependence detected. """ function detect_libgfortran_version() - function get_libgfortran_path() - # If CompilerSupportLibraries_jll is an stdlib, we can just directly ask for - # the path here, without checking `dllist()`: - libgfortran_path = get_csl_member(:libgfortran_path) - if libgfortran_path !== nothing - return libgfortran_path::String - end - - # Otherwise, look for it having already been loaded by something - libgfortran_paths = filter!(x -> occursin("libgfortran", x), Libdl.dllist()) - if !isempty(libgfortran_paths) - return first(libgfortran_paths)::String - end - - # One day, I hope to not be linking against libgfortran in base Julia - return nothing - end - - libgfortran_path = get_libgfortran_path() - name, version = parse_dl_name_version(libgfortran_path, os()) + libgfortran_path = _get_libgfortran_path() + _, version = parse_dl_name_version(libgfortran_path, os()) if version === nothing # Even though we complain about this, we allow it to continue in the hopes that # we shall march on to a BRIGHTER TOMORROW. One in which we are not shackled @@ -919,25 +992,8 @@ it is linked against (if any). `max_minor_version` is the latest version in the 3.4 series of GLIBCXX where the search is performed. """ function detect_libstdcxx_version(max_minor_version::Int=30) - function get_libstdcxx_handle() - # If CompilerSupportLibraries_jll is an stdlib, we can just directly open it - libstdcxx = get_csl_member(:libstdcxx) - if libstdcxx !== nothing - return nothing - end - - # Otherwise, look for it having already been loaded by something - libstdcxx_paths = filter!(x -> occursin("libstdc++", x), Libdl.dllist()) - if !isempty(libstdcxx_paths) - return Libdl.dlopen(first(libstdcxx_paths), Libdl.RTLD_NOLOAD)::Ptr{Cvoid} - end - - # One day, I hope to not be linking against libgfortran in base Julia - return nothing - end - # Brute-force our way through GLIBCXX_* symbols to discover which version we're linked against - libstdcxx = get_libstdcxx_handle() + libstdcxx = _get_libstdcxx_handle() if libstdcxx !== nothing # Try all GLIBCXX versions down to GCC v4.8: @@ -1093,7 +1149,7 @@ function platforms_match(a::AbstractPlatform, b::AbstractPlatform) # Call the comparator, passing in which objects requested this comparison (one, the other, or both) # For some comparators this doesn't matter, but for non-symmetrical comparisons, it does. - if !(comparator(ak, bk, a_comp === comparator, b_comp === comparator)::Bool) + if !(@invokelatest(comparator(ak, bk, a_comp === comparator, b_comp === comparator))::Bool) return false end end diff --git a/base/bitarray.jl b/base/bitarray.jl index 5a3469fa7c7a2..edd9057c38c07 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -26,13 +26,12 @@ mutable struct BitArray{N} <: AbstractArray{Bool, N} len::Int dims::NTuple{N,Int} function BitArray{N}(::UndefInitializer, dims::Vararg{Int,N}) where N - n = 1 i = 1 for d in dims d >= 0 || throw(ArgumentError("dimension size must be ≥ 0, got $d for dimension $i")) - n *= d i += 1 end + n = Core.checked_dims(dims...) nc = num_bit_chunks(n) chunks = Vector{UInt64}(undef, nc) nc > 0 && (chunks[end] = UInt64(0)) @@ -104,11 +103,6 @@ length(B::BitArray) = B.len size(B::BitVector) = (B.len,) size(B::BitArray) = B.dims -@inline function size(B::BitVector, d::Integer) - d < 1 && throw_boundserror(size(B), d) - ifelse(d == 1, B.len, 1) -end - isassigned(B::BitArray, i::Int) = 1 <= i <= length(B) IndexStyle(::Type{<:BitArray}) = IndexLinear() @@ -121,7 +115,7 @@ const _msk64 = ~UInt64(0) @inline _blsr(x)= x & (x-1) #zeros the last set bit. Has native instruction on many archs. needed in multidimensional.jl @inline _msk_end(l::Int) = _msk64 >>> _mod64(-l) @inline _msk_end(B::BitArray) = _msk_end(length(B)) -num_bit_chunks(n::Int) = _div64(n+63) +num_bit_chunks(n::Int) = _div64(n) + !iszero(_mod64(n)) @inline get_chunks_id(i::Int) = _div64(i-1)+1, _mod64(i-1) @@ -151,16 +145,14 @@ function copy_chunks!(dest::Vector{UInt64}, pos_d::Int, src::Vector{UInt64}, pos delta_ks = ks1 - ks0 u = _msk64 + msk_d0 = ~(u << ld0) + msk_d1 = (u << (ld1+1)) if delta_kd == 0 - msk_d0 = ~(u << ld0) | (u << (ld1+1)) - else - msk_d0 = ~(u << ld0) - msk_d1 = (u << (ld1+1)) + msk_d0 |= msk_d1 end + msk_s0 = (u << ls0) if delta_ks == 0 - msk_s0 = (u << ls0) & ~(u << (ls1+1)) - else - msk_s0 = (u << ls0) + msk_s0 &= ~(u << (ls1+1)) end chunk_s0 = glue_src_bitchunks(src, ks0, ks1, msk_s0, ls0) @@ -211,16 +203,14 @@ function copy_chunks_rtol!(chunks::Vector{UInt64}, pos_d::Int, pos_s::Int, numbi delta_kd = kd1 - kd0 delta_ks = ks1 - ks0 + msk_d0 = ~(u << ld0) + msk_d1 = (u << (ld1+1)) if delta_kd == 0 - msk_d0 = ~(u << ld0) | (u << (ld1+1)) - else - msk_d0 = ~(u << ld0) - msk_d1 = (u << (ld1+1)) + msk_d0 |= msk_d1 end + msk_s0 = (u << ls0) if delta_ks == 0 - msk_s0 = (u << ls0) & ~(u << (ls1+1)) - else - msk_s0 = (u << ls0) + msk_s0 &= ~(u << (ls1+1)) end chunk_s0 = glue_src_bitchunks(chunks, ks0, ks1, msk_s0, ls0) & ~(u << s) @@ -246,11 +236,10 @@ function fill_chunks!(Bc::Array{UInt64}, x::Bool, pos::Int, numbits::Int) k1, l1 = get_chunks_id(pos+numbits-1) u = _msk64 + msk0 = (u << l0) + msk1 = ~(u << (l1+1)) if k1 == k0 - msk0 = (u << l0) & ~(u << (l1+1)) - else - msk0 = (u << l0) - msk1 = ~(u << (l1+1)) + msk0 &= msk1 end @inbounds if x Bc[k0] |= msk0 @@ -270,7 +259,7 @@ end copy_to_bitarray_chunks!(dest::Vector{UInt64}, pos_d::Int, src::BitArray, pos_s::Int, numbits::Int) = copy_chunks!(dest, pos_d, src.chunks, pos_s, numbits) -# pack 8 Bools encoded as one contiguous UIn64 into a single byte, e.g.: +# pack 8 Bools encoded as one contiguous UInt64 into a single byte, e.g.: # 0000001:0000001:00000000:00000000:00000001:00000000:00000000:00000001 → 11001001 → 0xc9 function pack8bools(z::UInt64) z |= z >>> 7 @@ -313,7 +302,7 @@ function copy_to_bitarray_chunks!(Bc::Vector{UInt64}, pos_d::Int, C::Array{Bool} if nc8 > 0 ind8 = 1 P8 = Ptr{UInt64}(pointer(C, ind)) # unaligned i64 pointer - @inbounds for i = 1:nc8 + @inbounds for _ = 1:nc8 c = UInt64(0) for j = 0:7 # unaligned load @@ -325,7 +314,7 @@ function copy_to_bitarray_chunks!(Bc::Vector{UInt64}, pos_d::Int, C::Array{Bool} end ind += (ind8-1) << 3 end - @inbounds for i = (nc8+1):nc + @inbounds for _ = (nc8+1):nc c = UInt64(0) for j = 0:63 c |= (UInt64(C[ind]) << j) @@ -335,7 +324,7 @@ function copy_to_bitarray_chunks!(Bc::Vector{UInt64}, pos_d::Int, C::Array{Bool} bind += 1 end @inbounds if bind ≤ kd1 - @assert bind == kd1 + @assert bind == kd1 "bind != kd1" c = UInt64(0) for j = 0:ld1 c |= (UInt64(C[ind]) << j) @@ -467,8 +456,8 @@ function _copyto_int!(dest::BitArray, doffs::Int, src::Union{BitArray,Array}, so n < 0 && throw(ArgumentError("Number of elements to copy must be non-negative.")) soffs < 1 && throw(BoundsError(src, soffs)) doffs < 1 && throw(BoundsError(dest, doffs)) - soffs+n-1 > length(src) && throw(BoundsError(src, length(src)+1)) - doffs+n-1 > length(dest) && throw(BoundsError(dest, length(dest)+1)) + n > length(src) - soffs + 1 && throw(BoundsError(src, length(src)+1)) + n > length(dest) - doffs + 1 && throw(BoundsError(dest, length(dest)+1)) return unsafe_copyto!(dest, doffs, src, soffs, n) end @@ -483,11 +472,12 @@ function reshape(B::BitArray{N}, dims::NTuple{N,Int}) where N end reshape(B::BitArray, dims::Tuple{Vararg{Int}}) = _bitreshape(B, dims) function _bitreshape(B::BitArray, dims::NTuple{N,Int}) where N - prod(dims) == length(B) || + len = Core.checked_dims(dims...) + len == length(B) || throw(DimensionMismatch("new dimensions $(dims) must be consistent with array length $(length(B))")) Br = BitArray{N}(undef, ntuple(i->0,Val(N))...) Br.chunks = B.chunks - Br.len = prod(dims) + Br.len = len N != 1 && (Br.dims = dims) return Br end @@ -540,9 +530,6 @@ function _copyto_bitarray!(B::BitArray, A::AbstractArray) return B end -reinterpret(::Type{Bool}, B::BitArray, dims::NTuple{N,Int}) where {N} = reinterpret(B, dims) -reinterpret(B::BitArray, dims::NTuple{N,Int}) where {N} = reshape(B, dims) - (::Type{T})(x::T) where {T<:BitArray} = copy(x)::T BitArray(x::BitArray) = copy(x) @@ -643,7 +630,6 @@ function gen_bitarray_from_itr(itr) end function fill_bitarray_from_itr!(B::BitArray, itr) - n = length(B) C = Vector{Bool}(undef, bitcache_size) Bc = B.chunks ind = 1 @@ -721,7 +707,6 @@ function _unsafe_setindex!(B::BitArray, X::AbstractArray, I::BitArray) Ic = I.chunks length(Bc) == length(Ic) || throw_boundserror(B, I) lc = length(Bc) - lx = length(X) last_chunk_len = _mod64(length(B)-1)+1 Xi = first(eachindex(X)) @@ -730,7 +715,7 @@ function _unsafe_setindex!(B::BitArray, X::AbstractArray, I::BitArray) @inbounds Imsk = Ic[i] @inbounds C = Bc[i] u = UInt64(1) - for j = 1:(i < lc ? 64 : last_chunk_len) + for _ = 1:(i < lc ? 64 : last_chunk_len) if Imsk & u != 0 Xi > lastXi && throw_setindex_mismatch(X, count(I)) @inbounds x = convert(Bool, X[Xi]) @@ -771,14 +756,15 @@ function append!(B::BitVector, items::BitVector) n0 = length(B) n1 = length(items) n1 == 0 && return B + n = checked_add(n0, n1) Bc = B.chunks k0 = length(Bc) - k1 = num_bit_chunks(n0 + n1) + k1 = num_bit_chunks(n) if k1 > k0 _growend!(Bc, k1 - k0) Bc[end] = UInt64(0) end - B.len += n1 + B.len = n copy_chunks!(Bc, n0+1, items.chunks, 1, n1) return B end @@ -790,14 +776,15 @@ function prepend!(B::BitVector, items::BitVector) n0 = length(B) n1 = length(items) n1 == 0 && return B + n = checked_add(n0, n1) Bc = B.chunks k0 = length(Bc) - k1 = num_bit_chunks(n0 + n1) + k1 = num_bit_chunks(n) if k1 > k0 _growend!(Bc, k1 - k0) Bc[end] = UInt64(0) end - B.len += n1 + B.len = n copy_chunks!(Bc, 1 + n1, Bc, 1, n0) copy_chunks!(Bc, 1, items.chunks, 1, n1) return B @@ -1158,7 +1145,7 @@ function (-)(B::BitArray) for i = 1:length(Bc)-1 u = UInt64(1) c = Bc[i] - for j = 1:64 + for _ = 1:64 if c & u != 0 A[ind] = -1 end @@ -1168,7 +1155,7 @@ function (-)(B::BitArray) end u = UInt64(1) c = Bc[end] - for j = 0:_mod64(l-1) + for _ = 0:_mod64(l-1) if c & u != 0 A[ind] = -1 end @@ -1277,7 +1264,7 @@ function _reverse!(B::BitVector, ::Colon) # │000000000000000│ E ││ D │ C ││ B │ A │ # └───────────────┴───────┘└───────────────┴───────┘└───────────────┴───────┘ # k h k h k - # yielding; + # yielding: # ┌───────────────┬───────┐┌───────────────┬───────┐┌───────────────┬───────┐ # │000000000000000│ A' ││ B' │ C' ││ D' │ E' │ # └───────────────┴───────┘└───────────────┴───────┘└───────────────┴───────┘ @@ -1757,8 +1744,8 @@ map(::typeof(one), A::BitArray) = fill!(similar(A), true) map(::typeof(identity), A::BitArray) = copy(A) map!(::Union{typeof(~), typeof(!)}, dest::BitArray, A::BitArray) = bit_map!(~, dest, A) -map!(::typeof(zero), dest::BitArray, A::BitArray) = fill!(dest, false) -map!(::typeof(one), dest::BitArray, A::BitArray) = fill!(dest, true) +map!(::typeof(zero), dest::BitArray, A::BitArray) = bit_map_constant!(dest, A, false) +map!(::typeof(one), dest::BitArray, A::BitArray) = bit_map_constant!(dest, A, true) map!(::typeof(identity), dest::BitArray, A::BitArray) = copyto!(dest, A) for (T, f) in ((:(Union{typeof(&), typeof(*), typeof(min)}), :(&)), @@ -1771,13 +1758,25 @@ for (T, f) in ((:(Union{typeof(&), typeof(*), typeof(min)}), :(&)), (:(typeof(==)), :((p, q) -> ~xor(p, q))), (:(typeof(<)), :((p, q) -> ~p & q)), (:(typeof(>)), :((p, q) -> p & ~q))) - @eval map(::$T, A::BitArray, B::BitArray) = bit_map!($f, similar(A), A, B) + @eval map(::$T, A::BitArray, B::BitArray) = bit_map($f, A, B) @eval map!(::$T, dest::BitArray, A::BitArray, B::BitArray) = bit_map!($f, dest, A, B) end # If we were able to specialize the function to a known bitwise operation, # map across the chunks. Otherwise, fall-back to the AbstractArray method that # iterates bit-by-bit. +function bit_map(f::F, A::BitArray, B::BitArray) where F + AB = zip(A, B) + dest = similar(BitArray, _similar_shape(AB, IteratorSize(AB))) + bit_map!(f, dest, A, B) +end + +function bit_map_constant!(dest::BitArray, A::BitArray, x::Bool) + length(A) <= length(dest) || throw(DimensionMismatch("length of destination must be >= length of collection")) + fill_chunks!(dest.chunks, x, 1, length(A)) + return dest +end + function bit_map!(f::F, dest::BitArray, A::BitArray) where F length(A) <= length(dest) || throw(DimensionMismatch("length of destination must be >= length of collection")) isempty(A) && return dest @@ -1787,12 +1786,12 @@ function bit_map!(f::F, dest::BitArray, A::BitArray) where F for i = 1:(len_Ac-1) destc[i] = f(Ac[i]) end - # the last effected UInt64's original content + # the last affected UInt64's original content dest_last = destc[len_Ac] _msk = _msk_end(A) # first zero out the bits mask is going to change # then update bits by `or`ing with a masked RHS - # DO NOT SEPARATE ONTO TO LINES. + # DO NOT SEPARATE ONTO TWO LINES. # Otherwise there will be bugs when Ac aliases destc destc[len_Ac] = (dest_last & (~_msk)) | f(Ac[len_Ac]) & _msk dest @@ -1809,14 +1808,14 @@ function bit_map!(f::F, dest::BitArray, A::BitArray, B::BitArray) where F for i = 1:len_Ac-1 destc[i] = f(Ac[i], Bc[i]) end - # the last effected UInt64's original content + # the last affected UInt64's original content dest_last = destc[len_Ac] _msk = _msk_end(min_bitlen) # first zero out the bits mask is going to change # then update bits by `or`ing with a masked RHS - # DO NOT SEPARATE ONTO TO LINES. + # DO NOT SEPARATE ONTO TWO LINES. # Otherwise there will be bugs when Ac or Bc aliases destc - destc[len_Ac] = (dest_last & ~(_msk)) | f(Ac[end], Bc[end]) & _msk + destc[len_Ac] = (dest_last & ~(_msk)) | f(Ac[len_Ac], Bc[len_Ac]) & _msk dest end @@ -1861,7 +1860,6 @@ function hcat(A::Union{BitMatrix,BitVector}...) nargs = length(A) nrows = size(A[1], 1) ncols = 0 - dense = true for j = 1:nargs Aj = A[j] nd = ndims(Aj) @@ -1920,19 +1918,8 @@ end # hvcat -> use fallbacks in abstractarray.jl -# BitArray I/O - -write(s::IO, B::BitArray) = write(s, B.chunks) -function read!(s::IO, B::BitArray) - n = length(B) - Bc = B.chunks - nc = length(read!(s, Bc)) - if length(Bc) > 0 && Bc[end] & _msk_end(n) ≠ Bc[end] - Bc[end] &= _msk_end(n) # ensure that the BitArray is not broken - throw(DimensionMismatch("read mismatch, found non-zero bits after BitArray length")) - end - return B -end +# BitArray I/O lives in io.jl (it takes a `cancel` keyword, whose plumbing +# is not yet loaded at this point of bootstrap) sizeof(B::BitArray) = sizeof(B.chunks) diff --git a/base/bitset.jl b/base/bitset.jl index 78d8fc8769de1..aaa5e01878493 100644 --- a/base/bitset.jl +++ b/base/bitset.jl @@ -172,7 +172,7 @@ function _matched_map!(f, s1::BitSet, s2::BitSet) # we must first handle the NO_OFFSET case; we could test for # isempty(s1) but it can be costly, so the user has to call - # empty!(s1) herself before-hand to re-initialize to NO_OFFSET + # empty!(s1) themselves before-hand to re-initialize to NO_OFFSET if s1.offset == NO_OFFSET return left_false_is_false ? s1 : copy!(s1, s2) elseif s2.offset == NO_OFFSET diff --git a/base/bool.jl b/base/bool.jl index 12144756c76c8..ba57bf002edbf 100644 --- a/base/bool.jl +++ b/base/bool.jl @@ -10,32 +10,6 @@ typemax(::Type{Bool}) = true ## boolean operations ## -""" - !(x) - -Boolean not. Implements [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic), -returning [`missing`](@ref) if `x` is `missing`. - -See also [`~`](@ref) for bitwise not. - -# Examples -```jldoctest -julia> !true -false - -julia> !false -true - -julia> !missing -missing - -julia> .![true false true] -1×3 BitMatrix: - 0 1 0 -``` -""" -!(x::Bool) = not_int(x) - (~)(x::Bool) = !x (&)(x::Bool, y::Bool) = and_int(x, y) (|)(x::Bool, y::Bool) = or_int(x, y) @@ -163,12 +137,17 @@ ispositive(x::Bool) = x # could use fallback once #21712 is resolved ## do arithmetic as Int ## -+(x::Bool) = Int(x) --(x::Bool) = -Int(x) ++(x::Bool) = Int(x) ++%(x::Bool) = Int(x) +-(x::Bool) = -%(Int(x)) +-%(x::Bool) = -%(Int(x)) -+(x::Bool, y::Bool) = Int(x) + Int(y) --(x::Bool, y::Bool) = Int(x) - Int(y) ++(x::Bool, y::Bool) = Int(x) +% Int(y) +-(x::Bool, y::Bool) = Int(x) -% Int(y) ++%(x::Bool, y::Bool) = Int(x) +% Int(y) +-%(x::Bool, y::Bool) = Int(x) -% Int(y) *(x::Bool, y::Bool) = x & y +*%(x::Bool, y::Bool) = x & y ^(x::Bool, y::Bool) = x | !y ^(x::Integer, y::Bool) = ifelse(y, x, one(x)) diff --git a/base/boot.jl b/base/boot.jl index d055c47516f91..646fd3a1294d8 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -3,7 +3,17 @@ # commented-out definitions are implemented in C #abstract type Any <: Any end -#abstract type Type{T} end +#abstract type AnyType end +#struct TypeEq <: AnyType +# T +#end +#const Type = TypeEq(T) where T +# TypeEgal{T} is the egality-based dual of TypeEq{T}: its only instance is `T` +# itself (matched by `===`), used internally for dispatch-cache specialization. +# Free typevars are disallowed inside TypeEgal. +#struct TypeEgal <: AnyType +# T +#end #abstract type Vararg{T} end @@ -15,7 +25,7 @@ # name::Symbol #end -#mutable struct DataType <: Type +#mutable struct DataType <: AnyType # name::TypeName # super::Type # parameters::Tuple @@ -29,7 +39,7 @@ # pointerfree::Bool #end -#struct Union <: Type +#struct Union <: AnyType # a # b #end @@ -40,7 +50,7 @@ # ub::Type #end -#struct UnionAll +#struct UnionAll <: AnyType # var::TypeVar # body #end @@ -186,8 +196,8 @@ # sticky::UInt8 # priority::UInt16 # @atomic _isexception::UInt8 -# pad00::UInt8 -# pad01::UInt8 +# @atomic preempt_request::UInt8 +# bound_cancel_default::UInt8 # pad02::UInt8 # rngState0::UInt64 # rngState1::UInt64 @@ -202,12 +212,17 @@ # @atomic last_started_running_at::UInt64 # @atomic running_time_ns::UInt64 # @atomic finished_at::UInt64 +# @atomic waiting_on::Any +# cached_wait_entry::Any +# cached_cancel_entry::Any +# invoked::Any +# @atomic bound_cancel_token::Union{Nothing, CancellationTokenSource} #end export # key types - Any, DataType, Vararg, NTuple, - Tuple, Type, UnionAll, TypeVar, Union, Nothing, Cvoid, + Any, TypeEq, Type, DataType, Vararg, NTuple, + Tuple, UnionAll, TypeVar, Union, Nothing, Cvoid, AbstractArray, DenseArray, NamedTuple, Pair, # special objects Function, Method, Module, Symbol, Task, UndefInitializer, undef, WeakRef, VecElement, @@ -282,13 +297,32 @@ end function Typeof end ccall(:jl_toplevel_eval_in, Any, (Any, Any), Core, quote - (f::typeof(Typeof))(x) = ($(_expr(:meta,:nospecialize,:x)); isa(x,Type) ? Type{x} : typeof(x)) + (f::typeof(Typeof))(x) = begin + $(_expr(:meta,:nospecialize,:x)) + if isa(x,Type) + has_free_typevars(x) ? Type{x} : TypeEgal{x} + else + typeof(x) + end + end + end) + +# like `Typeof`, but yields the equality kind `Type{x}` for type values; used +# by lowering to spell the callee self-type of method definitions, so equal +# UnionAll spellings share the constructor method they define +function TypeEqOf end +ccall(:jl_toplevel_eval_in, Any, (Any, Any), + Core, quote + (f::typeof(TypeEqOf))(x) = begin + $(_expr(:meta,:nospecialize,:x)) + isa(x,Type) ? Type{x} : typeof(x) + end end) function iterate end macro nospecialize(x) - _expr(:meta, :nospecialize, x) + _expr(:escape, _expr(:meta, :nospecialize, x)) end Expr(@nospecialize args...) = _expr(args...) @@ -326,8 +360,9 @@ macro _foldable_meta() #=:nortcall=#true)) end -macro inline() Expr(:meta, :inline) end -macro noinline() Expr(:meta, :noinline) end +macro inline() Expr(:meta, :inline) end +macro noinline() Expr(:meta, :noinline) end +macro nospecializeinfer() Expr(:meta, :nospecializeinfer) end macro _boundscheck() Expr(:boundscheck) end @@ -351,8 +386,6 @@ unsafe_convert(::Type{T}, x::T) where {T} = x # will be inserted by the frontend for closures _typeof_captured_variable(@nospecialize t) = (@_total_meta; t isa Type && has_free_typevars(t) ? typeof(t) : Typeof(t)) -has_free_typevars(@nospecialize t) = (@_total_meta; ccall(:jl_has_free_typevars, Int32, (Any,), t) === Int32(1)) - # dispatch token indicating a kwarg (keyword sorter) call function kwcall end # deprecated internal functions: @@ -365,9 +398,78 @@ kwftype(@nospecialize(t)) = typeof(kwcall) Union{}(a...) = throw(ArgumentError("cannot construct a value of type Union{} for return result")) kwcall(kwargs, ::Type{Union{}}, a...) = Union{}(a...) +# resolve_typegroup must be defined before any struct definition, since all structs +# are now lowered using the typegroup mechanism (for #60919 safety). +function resolve_typegroup(mod::Module, typevars::SimpleVector, struct_infos::SimpleVector, old_types::SimpleVector) + n = _svec_len(typevars) + if n === 0 + return () + end + return ccall(:jl_resolve_typegroup, Any, (Any, Any, Any, Any), mod, typevars, struct_infos, old_types) +end + +# n.b. TypeApp and apply_type_or_typeapp must be defined before the first +# struct definition, whose lowered field-type thunks may call them. +# TypeApp: lazy type application for typegroup blocks. +# Represents a single type application step, like UnionAll represents a single where binding. +# T{P1, P2} is TypeApp(TypeApp(T, P1), P2) -- nested left-to-right. +# Allowed inside UnionAll; rejected by subtyping/intersection (like free typevars). +struct TypeApp + head::Any # Type constructor (TypeVar, Type, or outer TypeApp) + param::Any # Single type parameter + function TypeApp(@nospecialize(head), @nospecialize(param)) + return new(head, param) + end +end + +# Check if a value contains a TypeApp anywhere in its structure +function _contains_typeapp(@nospecialize(x)) + if x isa TypeApp + return true + end + if x isa UnionAll + return _contains_typeapp(x.body) + end + return false +end + +function apply_type_or_typeapp(@nospecialize(tc), @nospecialize params...) + # Head is TypeVar/TypeApp => must defer (apply_type requires UnionAll/DataType head) + if tc isa TypeVar || tc isa TypeApp + # Build nested TypeApp chain: TypeApp(TypeApp(tc, p1), p2), ... + n = nfields(params) + result = tc + i = 1 + while Intrinsics.sle_int(i, n) + result = TypeApp(result, getfield(params, i)) + i = Intrinsics.add_int(i, 1) + end + return result + end + # Any param contains TypeApp => must defer + n = nfields(params) + i = 1 + while Intrinsics.sle_int(i, n) + if _contains_typeapp(getfield(params, i)) + # Build nested TypeApp chain for all params + result = tc + j = 1 + while Intrinsics.sle_int(j, n) + result = TypeApp(result, getfield(params, j)) + j = Intrinsics.add_int(j, 1) + end + return result + end + i = Intrinsics.add_int(i, 1) + end + # All concrete -- real apply_type + return apply_type(tc, params...) +end + abstract type Exception end struct ErrorException <: Exception msg::AbstractString + ErrorException(msg::AbstractString) = new(msg) end struct BoundsError <: Exception @@ -377,12 +479,24 @@ struct BoundsError <: Exception BoundsError(@nospecialize(a)) = (@noinline; new(a)) BoundsError(@nospecialize(a), i) = (@noinline; new(a,i)) end -struct DivideError <: Exception end -struct OutOfMemoryError <: Exception end -struct ReadOnlyMemoryError <: Exception end -struct SegmentationFault <: Exception end -struct StackOverflowError <: Exception end -struct UndefRefError <: Exception end +struct DivideError <: Exception + DivideError() = new() +end +struct OutOfMemoryError <: Exception + OutOfMemoryError() = new() +end +struct ReadOnlyMemoryError <: Exception + ReadOnlyMemoryError() = new() +end +struct SegmentationFault <: Exception + SegmentationFault() = new() +end +struct StackOverflowError <: Exception + StackOverflowError() = new() +end +struct UndefRefError <: Exception + UndefRefError() = new() +end struct UndefVarError <: Exception var::Symbol world::UInt @@ -392,11 +506,15 @@ struct UndefVarError <: Exception end struct ConcurrencyViolationError <: Exception msg::AbstractString + ConcurrencyViolationError(msg::AbstractString) = new(msg) end struct MissingCodeError <: Exception mi::MethodInstance + MissingCodeError(mi::MethodInstance) = new(mi) +end +struct InterruptException <: Exception + InterruptException() = new() end -struct InterruptException <: Exception end struct DomainError <: Exception val msg::AbstractString @@ -410,7 +528,7 @@ struct TypeError <: Exception # `context` optionally adds extra detail, e.g. the name of the type parameter # that got a bad value. func::Symbol - context::Union{AbstractString,Symbol} + context::Union{AbstractString,GlobalRef,Symbol} expected::Type got TypeError(func, context, @nospecialize(expected::Type), @nospecialize(got)) = @@ -425,13 +543,16 @@ struct InexactError <: Exception end struct OverflowError <: Exception msg::AbstractString + OverflowError(msg::AbstractString) = new(msg) end struct ArgumentError <: Exception msg::AbstractString + ArgumentError(msg::AbstractString) = new(msg) end struct UndefKeywordError <: Exception var::Symbol + UndefKeywordError(var::Symbol) = new(var) end const typemax_UInt = Intrinsics.sext_int(UInt, 0xFF) @@ -447,12 +568,14 @@ MethodError(@nospecialize(f), @nospecialize(args)) = MethodError(f, args, typema struct AssertionError <: Exception msg::AbstractString + AssertionError(msg::AbstractString) = (@noinline; new(msg)) end -AssertionError() = AssertionError("") +AssertionError() = (@noinline; AssertionError("")) struct FieldError <: Exception type::DataType field::Symbol + FieldError(type::DataType, field::Symbol) = new(type, field) end abstract type WrappedException <: Exception end @@ -461,11 +584,13 @@ struct LoadError <: WrappedException file::AbstractString line::Int error + LoadError(file::AbstractString, line::Int, @nospecialize(error)) = new(file, line, error) end struct InitError <: WrappedException mod::Symbol error + InitError(mod::Symbol, @nospecialize(error)) = new(mod, error) end struct ABIOverride @@ -474,8 +599,12 @@ struct ABIOverride ABIOverride(@nospecialize(abi::Type), def::MethodInstance) = new(abi, def) end -struct PrecompilableError <: Exception end -struct TrimFailure <: Exception end +struct PrecompilableError <: Exception + PrecompilableError() = new() +end +struct TrimFailure <: Exception + TrimFailure() = new() +end String(s::String) = s # no constructor yet @@ -490,6 +619,7 @@ eval(m::Module, @nospecialize(e)) = (@noinline; ccall(:jl_toplevel_eval_in, Any, struct EvalInto <: Function m::Module + EvalInto(m::Module) = new(m) end (this::EvalInto)(@nospecialize(e)) = eval(this.m, e) @@ -516,6 +646,50 @@ struct VecElement{T} end VecElement(arg::T) where {T} = VecElement{T}(arg) +# inference lattice element types (moved from jltypes.c) +struct Const + val + Const(@nospecialize(v)) = new(v) +end + +struct PartialStruct + typ + undefs::Array{Union{Nothing,Bool}, 1} + fields::Array{Any, 1} + # N.B. The constructor for this struct is intentionally not defined here. + # It is defined in coreir.jl along with some validation logic. + global _PartialStruct + _PartialStruct(@nospecialize(typ), undef::Array{Union{Nothing,Bool}, 1}, fields::Array{Any, 1}) = new(typ, undef, fields) +end + +struct InterConditional + slot::Int + thentype + elsetype + InterConditional(slot::Int, @nospecialize(thentype), @nospecialize(elsetype)) = new(slot, thentype, elsetype) +end + +struct InterMustAlias + slot::Int + vartyp::Any + fldidx::Int + fldtyp::Any + InterMustAlias(slot::Int, @nospecialize(vartyp), fldidx::Int, @nospecialize(fldtyp)) = new(slot, vartyp, fldidx, fldtyp) +end + +struct PartialOpaque + typ::Type + env + parent::MethodInstance + source + PartialOpaque(@nospecialize(typ::Type), @nospecialize(env), parent::MethodInstance, source) = new(typ, env, parent, source) +end + +struct PartialTask + fetch_type + PartialTask(@nospecialize(fetch_type)) = new(fetch_type) +end + eval(Core, quote GotoNode(label::Int) = $(Expr(:new, :GotoNode, :label)) NewvarNode(slot::SlotNumber) = $(Expr(:new, :NewvarNode, :slot)) @@ -532,7 +706,7 @@ eval(Core, quote isa(f, String) && (f = Symbol(f)) return $(Expr(:new, :LineNumberNode, :l, :f)) end - DebugInfo(def::Union{Method,MethodInstance,Symbol}, linetable::Union{Nothing,DebugInfo}, edges::SimpleVector, codelocs::String) = + DebugInfo(def::Union{Method,MethodInstance,Symbol}, linetable::Union{Nothing,DebugInfo,String}, edges::SimpleVector, codelocs::String) = $(Expr(:new, :DebugInfo, :def, :linetable, :edges, :codelocs)) DebugInfo(def::Union{Method,MethodInstance,Symbol}) = $(Expr(:new, :DebugInfo, :def, nothing, Core.svec(), "")) @@ -542,10 +716,6 @@ eval(Core, quote PhiCNode(values::Array{Any, 1}) = $(Expr(:new, :PhiCNode, :values)) UpsilonNode(@nospecialize(val)) = $(Expr(:new, :UpsilonNode, :val)) UpsilonNode() = $(Expr(:new, :UpsilonNode)) - Const(@nospecialize(v)) = $(Expr(:new, :Const, :v)) - _PartialStruct(@nospecialize(typ), undef, fields::Array{Any, 1}) = $(Expr(:new, :PartialStruct, :typ, :undef, :fields)) - PartialOpaque(@nospecialize(typ), @nospecialize(env), parent::MethodInstance, source) = $(Expr(:new, :PartialOpaque, :typ, :env, :parent, :source)) - InterConditional(slot::Int, @nospecialize(thentype), @nospecialize(elsetype)) = $(Expr(:new, :InterConditional, :slot, :thentype, :elsetype)) MethodMatch(@nospecialize(spec_types), sparams::SimpleVector, method::Method, fully_covers::Bool) = $(Expr(:new, :MethodMatch, :spec_types, :sparams, :method, :fully_covers)) end) @@ -574,14 +744,12 @@ end GlobalRef(m::Module, s::Symbol) = ccall(:jl_module_globalref, Ref{GlobalRef}, (Any, Any), m, s) Module(name::Symbol=:anonymous, std_imports::Bool=true, default_names::Bool=true) = ccall(:jl_f_new_module, Ref{Module}, (Any, Bool, Bool), name, std_imports, default_names) -function _Task(@nospecialize(f), reserved_stack::Int, completion_future) - return ccall(:jl_new_task, Ref{Task}, (Any, Any, Int), f, completion_future, reserved_stack) -end - const NTuple{N,T} = Tuple{Vararg{T,N}} ## primitive Array constructors -struct UndefInitializer end +struct UndefInitializer + UndefInitializer() = new() +end const undef = UndefInitializer() # type and dimensionality specified @@ -666,6 +834,8 @@ end)) Array{T}(::UndefInitializer, m::Int) where {T} = Array{T, 1}(undef, m) Array{T}(::UndefInitializer, m::Int, n::Int) where {T} = Array{T, 2}(undef, m, n) Array{T}(::UndefInitializer, m::Int, n::Int, o::Int) where {T} = Array{T, 3}(undef, m, n, o) +# Guard against invalidations due to spurious `Tuple{}` intersections +Array{T}(::UndefInitializer, ::Tuple{}) where {T} = Array{T, 0}(undef) Array{T}(::UndefInitializer, d::NTuple{N, Int}) where {T, N} = Array{T, N}(undef, d) # empty vector constructor (self::Type{Array{T, 1}})() where {T} = self(undef, 0) @@ -746,7 +916,7 @@ end # module providing the IR object model # excluding types already exported by Core (GlobalRef, QuoteNode, Expr, LineNumberNode) -# any type beyond these is self-quoting (see also Base.is_ast_node) +# any type beyond these is self-quoting (see also Base.isa_ast_node) module IR export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode, @@ -794,12 +964,16 @@ macro cmd end # simple stand-alone print definitions for debugging abstract type IO end -struct CoreSTDOUT <: IO end -struct CoreSTDERR <: IO end +struct CoreSTDOUT <: IO + CoreSTDOUT() = new() +end +struct CoreSTDERR <: IO + CoreSTDERR() = new() +end const stdout = CoreSTDOUT() const stderr = CoreSTDERR() -io_pointer(::CoreSTDOUT) = Intrinsics.pointerref(Intrinsics.cglobal(:jl_uv_stdout, Ptr{Cvoid}), 1, 1) -io_pointer(::CoreSTDERR) = Intrinsics.pointerref(Intrinsics.cglobal(:jl_uv_stderr, Ptr{Cvoid}), 1, 1) +io_pointer(::CoreSTDOUT) = Intrinsics.pointerref(cglobal(:jl_uv_stdout, Ptr{Cvoid}), 1, 1) +io_pointer(::CoreSTDERR) = Intrinsics.pointerref(cglobal(:jl_uv_stderr, Ptr{Cvoid}), 1, 1) unsafe_write(io::IO, x::Ptr{UInt8}, nb::UInt) = (ccall(:jl_uv_puts, Cvoid, (Ptr{Cvoid}, Ptr{UInt8}, UInt), io_pointer(io), x, nb); nb) @@ -829,6 +1003,7 @@ struct GeneratedFunctionStub gen argnames::SimpleVector spnames::SimpleVector + GeneratedFunctionStub(@nospecialize(gen), argnames::SimpleVector, spnames::SimpleVector) = new(gen, argnames, spnames) end # If the generator is a subtype of this trait, inference caches the generated unoptimized @@ -899,7 +1074,7 @@ toInt8(x::UInt16) = checked_trunc_sint(Int8, check_sign_bit(Int8, x)) toInt8(x::UInt32) = checked_trunc_sint(Int8, check_sign_bit(Int8, x)) toInt8(x::UInt64) = checked_trunc_sint(Int8, check_sign_bit(Int8, x)) toInt8(x::UInt128) = checked_trunc_sint(Int8, check_sign_bit(Int8, x)) -toInt8(x::Bool) = and_int(bitcast(Int8, x), Int8(1)) +toInt8(x::Bool) = bitcast(Int8, x) toInt16(x::Int8) = sext_int(Int16, x) toInt16(x::Int16) = x toInt16(x::Int32) = checked_trunc_sint(Int16, x) @@ -910,7 +1085,7 @@ toInt16(x::UInt16) = bitcast(Int16, check_sign_bit(Int16, x)) toInt16(x::UInt32) = checked_trunc_sint(Int16, check_sign_bit(Int16, x)) toInt16(x::UInt64) = checked_trunc_sint(Int16, check_sign_bit(Int16, x)) toInt16(x::UInt128) = checked_trunc_sint(Int16, check_sign_bit(Int16, x)) -toInt16(x::Bool) = and_int(zext_int(Int16, x), Int16(1)) +toInt16(x::Bool) = zext_int(Int16, x) toInt32(x::Int8) = sext_int(Int32, x) toInt32(x::Int16) = sext_int(Int32, x) toInt32(x::Int32) = x @@ -921,7 +1096,7 @@ toInt32(x::UInt16) = zext_int(Int32, x) toInt32(x::UInt32) = bitcast(Int32, check_sign_bit(Int32, x)) toInt32(x::UInt64) = checked_trunc_sint(Int32, check_sign_bit(Int32, x)) toInt32(x::UInt128) = checked_trunc_sint(Int32, check_sign_bit(Int32, x)) -toInt32(x::Bool) = and_int(zext_int(Int32, x), Int32(1)) +toInt32(x::Bool) = zext_int(Int32, x) toInt64(x::Int8) = sext_int(Int64, x) toInt64(x::Int16) = sext_int(Int64, x) toInt64(x::Int32) = sext_int(Int64, x) @@ -932,7 +1107,7 @@ toInt64(x::UInt16) = zext_int(Int64, x) toInt64(x::UInt32) = zext_int(Int64, x) toInt64(x::UInt64) = bitcast(Int64, check_sign_bit(Int64, x)) toInt64(x::UInt128) = checked_trunc_sint(Int64, check_sign_bit(Int64, x)) -toInt64(x::Bool) = and_int(zext_int(Int64, x), Int64(1)) +toInt64(x::Bool) = zext_int(Int64, x) toInt128(x::Int8) = sext_int(Int128, x) toInt128(x::Int16) = sext_int(Int128, x) toInt128(x::Int32) = sext_int(Int128, x) @@ -943,7 +1118,7 @@ toInt128(x::UInt16) = zext_int(Int128, x) toInt128(x::UInt32) = zext_int(Int128, x) toInt128(x::UInt64) = zext_int(Int128, x) toInt128(x::UInt128) = bitcast(Int128, check_sign_bit(Int128, x)) -toInt128(x::Bool) = and_int(zext_int(Int128, x), Int128(1)) +toInt128(x::Bool) = zext_int(Int128, x) toUInt8(x::Int8) = bitcast(UInt8, check_sign_bit(UInt8, x)) toUInt8(x::Int16) = checked_trunc_uint(UInt8, x) toUInt8(x::Int32) = checked_trunc_uint(UInt8, x) @@ -954,7 +1129,7 @@ toUInt8(x::UInt16) = checked_trunc_uint(UInt8, x) toUInt8(x::UInt32) = checked_trunc_uint(UInt8, x) toUInt8(x::UInt64) = checked_trunc_uint(UInt8, x) toUInt8(x::UInt128) = checked_trunc_uint(UInt8, x) -toUInt8(x::Bool) = and_int(bitcast(UInt8, x), UInt8(1)) +toUInt8(x::Bool) = bitcast(UInt8, x) toUInt16(x::Int8) = sext_int(UInt16, check_sign_bit(UInt16, x)) toUInt16(x::Int16) = bitcast(UInt16, check_sign_bit(UInt16, x)) toUInt16(x::Int32) = checked_trunc_uint(UInt16, x) @@ -965,7 +1140,7 @@ toUInt16(x::UInt16) = x toUInt16(x::UInt32) = checked_trunc_uint(UInt16, x) toUInt16(x::UInt64) = checked_trunc_uint(UInt16, x) toUInt16(x::UInt128) = checked_trunc_uint(UInt16, x) -toUInt16(x::Bool) = and_int(zext_int(UInt16, x), UInt16(1)) +toUInt16(x::Bool) = zext_int(UInt16, x) toUInt32(x::Int8) = sext_int(UInt32, check_sign_bit(UInt32, x)) toUInt32(x::Int16) = sext_int(UInt32, check_sign_bit(UInt32, x)) toUInt32(x::Int32) = bitcast(UInt32, check_sign_bit(UInt32, x)) @@ -976,7 +1151,7 @@ toUInt32(x::UInt16) = zext_int(UInt32, x) toUInt32(x::UInt32) = x toUInt32(x::UInt64) = checked_trunc_uint(UInt32, x) toUInt32(x::UInt128) = checked_trunc_uint(UInt32, x) -toUInt32(x::Bool) = and_int(zext_int(UInt32, x), UInt32(1)) +toUInt32(x::Bool) = zext_int(UInt32, x) toUInt64(x::Int8) = sext_int(UInt64, check_sign_bit(UInt64, x)) toUInt64(x::Int16) = sext_int(UInt64, check_sign_bit(UInt64, x)) toUInt64(x::Int32) = sext_int(UInt64, check_sign_bit(UInt64, x)) @@ -987,7 +1162,7 @@ toUInt64(x::UInt16) = zext_int(UInt64, x) toUInt64(x::UInt32) = zext_int(UInt64, x) toUInt64(x::UInt64) = x toUInt64(x::UInt128) = checked_trunc_uint(UInt64, x) -toUInt64(x::Bool) = and_int(zext_int(UInt64, x), UInt64(1)) +toUInt64(x::Bool) = zext_int(UInt64, x) toUInt128(x::Int8) = sext_int(UInt128, check_sign_bit(UInt128, x)) toUInt128(x::Int16) = sext_int(UInt128, check_sign_bit(UInt128, x)) toUInt128(x::Int32) = sext_int(UInt128, check_sign_bit(UInt128, x)) @@ -998,7 +1173,7 @@ toUInt128(x::UInt16) = zext_int(UInt128, x) toUInt128(x::UInt32) = zext_int(UInt128, x) toUInt128(x::UInt64) = zext_int(UInt128, x) toUInt128(x::UInt128) = x -toUInt128(x::Bool) = and_int(zext_int(UInt128, x), UInt128(1)) +toUInt128(x::Bool) = zext_int(UInt128, x) # TODO: this is here to work around the 4 method limit in inference (#23210). const BuiltinInts = Union{Int128, Int16, Int32, Int64, Int8, UInt128, UInt16, UInt32, UInt64, UInt8, Bool} @@ -1052,7 +1227,7 @@ end # Bindings for the julia frontend. The internal jl_parse and jl_lower will call # Core._parse and Core._lower respectively (if they are not `nothing`.) -# Core._parse(text, filename, lineno, offset, options) +# Core._parse(text, filename, lineno, offset, options, version) # # Parse Julia code from the buffer `text`, starting at `offset` and attributing # it to `filename`. `text` may be a `String` or `svec(ptr::Ptr{UInt8}, @@ -1094,6 +1269,8 @@ struct Pair{A, B} end function _hasmethod(@nospecialize(tt)) # this function has a special tfunc + @nospecializeinfer + @noinline world = ccall(:jl_get_tls_world_age, UInt, ()) # tls_world_age() return Intrinsics.not_int(ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) === nothing) end @@ -1111,10 +1288,11 @@ EnterNode(old::EnterNode, new_dest::Int) = isdefined(old, :scope) ? EnterNode(new_dest, old.scope) : EnterNode(new_dest) # typename(_).constprop_heuristic -const FORCE_CONST_PROP = 0x1 -const ARRAY_INDEX_HEURISTIC = 0x2 -const ITERATE_HEURISTIC = 0x3 -const SAMETYPE_HEURISTIC = 0x4 +const FORCE_CONST_PROP = 0x01 +const ARRAY_INDEX_HEURISTIC = 0x02 +const ITERATE_HEURISTIC = 0x04 +const SAMETYPE_HEURISTIC = 0x08 +const DISABLE_SEMI_CONCRETE_EVAL = 0x10 # `typename` has special tfunc support in inference to improve # the result for `Type{Union{...}}`. It is defined here, so that the Compiler @@ -1134,11 +1312,18 @@ function typename(a::Union) end typename(union::UnionAll) = typename(union.body) -# Special inference support to avoid execess specialization of these methods. +# Special inference support to avoid excess specialization of these methods. # TODO: Replace this by a generic heuristic. (>:)(@nospecialize(a), @nospecialize(b)) = (b <: a) (!==)(@nospecialize(a), @nospecialize(b)) = Intrinsics.not_int(a === b) include(Core, "optimized_generics.jl") +# Used only by the magic @VERSION macro +struct MacroSource + lno::Any # ::LineNumberNode, but needs to be a pointer + syntax_ver::Any # ::VersionNumber =# + MacroSource(@nospecialize(lno), @nospecialize(syntax_ver)) = new(lno, syntax_ver) +end + ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true) diff --git a/base/broadcast.jl b/base/broadcast.jl index b86baf08ddfe0..9577c439a22f2 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -8,8 +8,7 @@ Module containing the broadcasting implementation. module Broadcast using .Base.Cartesian -using .Base: Indices, OneTo, tail, to_shape, isoperator, promote_typejoin, promote_typejoin_union, - _msk_end, unsafe_bitgetindex, bitcache_chunks, bitcache_size, dumpbitcache, unalias, negate +using .Base: OneTo, tail, isoperator, promote_typejoin, promote_typejoin_union, unalias, negate import .Base: copy, copyto!, axes export broadcast, broadcast!, BroadcastStyle, broadcast_axes, broadcastable, dotview, @__dot__, BroadcastFunction @@ -194,10 +193,12 @@ end struct AndAnd end const andand = AndAnd() broadcasted(::AndAnd, a, b) = broadcasted((a, b) -> a && b, a, b) +typeof(broadcasted).name.concrete_only = true + function broadcasted(::AndAnd, a, bc::Broadcasted) bcf = flatten(bc) # Vararg type signature to specialize on args count. This is necessary for performance - # and innexpensive because this should only ever get called with 1+N = length(bc.args) + # and inexpensive because this should only ever get called with 1+N = length(bc.args) broadcasted(((a, args::Vararg{Any, N}) where {N}) -> a && bcf.f(args...), a, bcf.args...) end struct OrOr end @@ -206,7 +207,7 @@ broadcasted(::OrOr, a, b) = broadcasted((a, b) -> a || b, a, b) function broadcasted(::OrOr, a, bc::Broadcasted) bcf = flatten(bc) # Vararg type signature to specialize on args count. This is necessary for performance - # and innexpensive because this should only ever get called with 1+N = length(bc.args) + # and inexpensive because this should only ever get called with 1+N = length(bc.args) broadcasted(((a, args::Vararg{Any, N}) where {N}) -> a || bcf.f(args...), a, bcf.args...) end @@ -264,6 +265,17 @@ Base.LinearIndices(bc::Broadcasted{<:Any,<:Tuple{Any}}) = LinearIndices(axes(bc) Base.ndims(bc::Broadcasted) = ndims(typeof(bc)) Base.ndims(::Type{<:Broadcasted{<:Any,<:NTuple{N,Any}}}) where {N} = N +Base.ndims(BC::Type{<:Broadcasted{<:Any,Nothing}}) = _maxndims(argtype(BC)) +function Base.ndims(BC::Type{<:Broadcasted{<:AbstractArrayStyle{N},Nothing}}) where {N} + N isa Int ? N : _maxndims(argtype(BC)) +end +_maxndims(::Type{Tuple{}}) = 0 +_maxndims(::Type{Tuple{T}}) where {T} = T <: Tuple ? 1 : Int(ndims(T))::Int +function _maxndims(Args::Type{<:Tuple{T,Vararg}}) where {T} + m = T <: Tuple ? 1 : Int(ndims(T))::Int + n = _maxndims(Base.tuple_type_tail(Args)) + max(m, n) +end Base.size(bc::Broadcasted) = map(length, axes(bc)) Base.length(bc::Broadcasted) = prod(size(bc)) @@ -280,20 +292,6 @@ Base.@propagate_inbounds function Base.iterate(bc::Broadcasted, s) end Base.IteratorSize(::Type{T}) where {T<:Broadcasted} = Base.HasShape{ndims(T)}() -Base.ndims(BC::Type{<:Broadcasted{<:Any,Nothing}}) = _maxndims_broadcasted(BC) -# the `AbstractArrayStyle` type parameter is required to be either equal to `Any` or be an `Int` value -Base.ndims(BC::Type{<:Broadcasted{<:AbstractArrayStyle{Any},Nothing}}) = _maxndims_broadcasted(BC) -Base.ndims(::Type{<:Broadcasted{<:AbstractArrayStyle{N},Nothing}}) where {N} = N::Int - -function _maxndims_broadcasted(BC::Type{<:Broadcasted}) - _maxndims(fieldtype(BC, :args)) -end -_maxndims(::Type{T}) where {T<:Tuple} = reduce(max, ntuple(n -> (F = fieldtype(T, n); F <: Tuple ? 1 : ndims(F)), Base._counttuple(T))) -_maxndims(::Type{<:Tuple{T}}) where {T} = T <: Tuple ? 1 : ndims(T) -function _maxndims(::Type{<:Tuple{T, S}}) where {T, S} - return max(T <: Tuple ? 1 : ndims(T), S <: Tuple ? 1 : ndims(S)) -end - Base.IteratorEltype(::Type{<:Broadcasted}) = Base.EltypeUnknown() ## Instantiation fills in the "missing" fields in Broadcasted. @@ -507,7 +505,7 @@ combine_axes(A) = axes(A) """ broadcast_shape(As...)::Tuple -Determine the result axes for broadcasting across all axes (size Tuples) in `As`. +Determine the result axes for broadcasting across all axes (size `Tuple`s) in `As`. ```jldoctest julia> Broadcast.broadcast_shape((1,2), (2,1)) @@ -577,7 +575,7 @@ Two methods are supported, both allowing for `I` to be specified as either a [`C an `Int`. * `newindex(argument, I)` dynamically constrains `I` based upon the axes of `argument`. -* `newindex(I, keep, default)` constrains `I` using the pre-computed tuples `keeps` and `defaults`. +* `newindex(I, keep, default)` constrains `I` using the pre-computed tuples `keep` and `default`. * `keep` is a tuple of `Bool`s, where `keep[d] == true` means that dimension `d` in `I` should be preserved as is * `default` is a tuple of Integers, specifying what index to use in dimension `d` when `keep[d] == false`. Any remaining indices in `I` beyond the length of the `keep` tuple are truncated. The `keep` and `default` @@ -633,11 +631,15 @@ to_index(::Tuple{}) = CartesianIndex() to_index(Is::Tuple{Any}) = Is[1] to_index(Is::Tuple) = CartesianIndex(Is) -@inline Base.checkbounds(bc::Broadcasted, I::CartesianIndex) = - Base.checkbounds_indices(Bool, axes(bc), (I,)) || Base.throw_boundserror(bc, (I,)) +@inline function Base.checkbounds(bc::Broadcasted, I::CartesianIndex) + Base.checkbounds_indices(Bool, axes(bc), (I,)) || Base.throw_boundserror(bc, I) + nothing +end -@inline Base.checkbounds(bc::Broadcasted, I::Integer) = - Base.checkindex(Bool, eachindex(IndexLinear(), bc), I) || Base.throw_boundserror(bc, (I,)) +@inline function Base.checkbounds(bc::Broadcasted, I::Integer) + Base.checkindex(Bool, eachindex(IndexLinear(), bc), I) || Base.throw_boundserror(bc, I) + nothing +end """ @@ -700,6 +702,8 @@ Base.@propagate_inbounds _getindex(args::Tuple{}, I) = () @inline _broadcast_getindex_evalf(f::Tf, args::Vararg{Any,N}) where {Tf,N} = f(args...) # not propagate_inbounds +const BroadcastScalars = Union{Symbol,AbstractString,Function,UndefInitializer,Nothing,RoundingMode,Missing,Val,Ptr,AbstractPattern,Pair,IO,CartesianIndex} + """ Broadcast.broadcastable(x) @@ -728,7 +732,7 @@ julia> Broadcast.broadcastable("hello") # Strings break convention of matching i Base.RefValue{String}("hello") ``` """ -broadcastable(x::Union{Symbol,AbstractString,Function,UndefInitializer,Nothing,RoundingMode,Missing,Val,Ptr,AbstractPattern,Pair,IO,CartesianIndex}) = Ref(x) +broadcastable(x::BroadcastScalars) = Ref(x) broadcastable(::Type{T}) where {T} = Ref{Type{T}}(T) broadcastable(x::Union{AbstractArray,Number,AbstractChar,Ref,Tuple,Broadcasted}) = x # Default to collecting iterables — which will error for non-iterables @@ -736,6 +740,27 @@ broadcastable(x) = collect(x) broadcastable(::Union{AbstractDict, NamedTuple}) = throw(ArgumentError("broadcasting over dictionaries and `NamedTuple`s is reserved")) ## Computation of inferred result type, for empty and concretely inferred cases only +_bc_eltype(bc::Broadcasted, i) = bc.f(_bc_eltypes(bc.args, i)...) +# Numbers must broadcast as scalars even if they override eltype +# Tuples may have heterogenous eltypes across indices +_bc_eltype(x::Number, i) = _broadcast_getindex(x, i) +_bc_eltype(x::Tuple, i) = _broadcast_getindex(x, i) +# since this method only exists to be inferred, indexing x is never actually executed. the +# inference barrier prevents recursion limiting, and ::eltype gives the desired result +_bc_eltype(x, i) = _broadcast_getindex(Base.inferencebarrier(x), i)::eltype(x) + +_bc_eltypes(args::Tuple, i) = (_bc_eltype(args[1], i), _bc_eltypes(tail(args), i)...) +_bc_eltypes(::Tuple{}, i) = () + +function result_eltype(bc::Broadcasted) + argtypes = Iterators.TupleOrBottom(typeof(bc), eltype(eachindex(bc))) + argtypes === Union{} && return Union{} + rettype = Base._return_type(_bc_eltype, argtypes) + return promote_typejoin_union(rettype) +end + +### this machinery is now dead code in Base, but several packages +### reach into these internals so it's retained for convenience. _broadcast_getindex_eltype(bc::Broadcasted) = combine_eltypes(bc.f, bc.args) _broadcast_getindex_eltype(A) = eltype(A) # Tuple, Array, etc. @@ -743,7 +768,6 @@ eltypes(::Tuple{}) = Tuple{} eltypes(t::Tuple{Any}) = Iterators.TupleOrBottom(_broadcast_getindex_eltype(t[1])) eltypes(t::Tuple{Any,Any}) = Iterators.TupleOrBottom(_broadcast_getindex_eltype(t[1]), _broadcast_getindex_eltype(t[2])) eltypes(t::Tuple) = (TT = eltypes(tail(t)); TT === Union{} ? Union{} : Iterators.TupleOrBottom(_broadcast_getindex_eltype(t[1]), TT.parameters...)) -# eltypes(t::Tuple) = Iterators.TupleOrBottom(ntuple(i -> _broadcast_getindex_eltype(t[i]), Val(length(t)))...) # Inferred eltype of result of broadcast(f, args...) function combine_eltypes(f, args::Tuple) @@ -831,7 +855,7 @@ julia> string.(("one","two","three","four"), ": ", 1:4) ``` """ -broadcast(f::Tf, As...) where {Tf} = materialize(broadcasted(f, As...)) +@inline broadcast(f::Tf, As...) where {Tf} = materialize(broadcasted(f, As...)) # special cases defined for performance @inline broadcast(f, x::Number...) = f(x...) @@ -875,9 +899,9 @@ broadcast!(f::Tf, dest, As::Vararg{Any,N}) where {Tf,N} = (materialize!(dest, br """ broadcast_preserving_zero_d(f, As...) -Like [`broadcast`](@ref), except in the case of a 0-dimensional result where it returns a 0-dimensional container +Like [`broadcast`](@ref), except in the case of a 0-dimensional result where it returns a 0-dimensional container. -Broadcast automatically unwraps zero-dimensional results to be just the element itself, +`broadcast` automatically unwraps zero-dimensional results to be just the element itself, but in some cases it is necessary to always return a container — even in the 0-dimensional case. """ @inline function broadcast_preserving_zero_d(f, As...) @@ -891,7 +915,7 @@ end """ Broadcast.materialize(bc) -Take a lazy `Broadcasted` object and compute the result +Take a lazy `Broadcasted` object and compute the result. """ @inline materialize(bc::Broadcasted) = copy(instantiate(bc)) materialize(x) = x @@ -903,10 +927,16 @@ end @inline function materialize!(dest, bc::Broadcasted{<:Any}) return materialize!(combine_styles(dest, bc), dest, bc) end + @inline function materialize!(::BroadcastStyle, dest, bc::Broadcasted{<:Any}) return copyto!(dest, instantiate(Broadcasted(bc.style, bc.f, bc.args, axes(dest)))) end +materialize!(dest::Union{BroadcastScalars,Number,AbstractChar}, bc::Broadcasted{<:Any}) = _throw_unwritable_dest(dest) +@noinline function _throw_unwritable_dest(@nospecialize dest) + throw(ArgumentError(LazyString("cannot broadcast-assign (`.=`) into a value of type ", typeof(dest)))) +end + ## general `copy` methods @inline copy(bc::Broadcasted{<:AbstractArrayStyle{0}}) = bc[CartesianIndex()] copy(bc::Broadcasted{<:Union{Nothing,Unknown}}) = @@ -915,7 +945,7 @@ copy(bc::Broadcasted{<:Union{Nothing,Unknown}}) = const NonleafHandlingStyles = Union{DefaultArrayStyle,ArrayConflict} @inline function copy(bc::Broadcasted) - ElType = combine_eltypes(bc.f, bc.args) + ElType = result_eltype(bc) if Base.isconcretetype(ElType) # We can trust it and defer to the simpler `copyto!` return copyto!(similar(bc, ElType), bc) @@ -1034,7 +1064,7 @@ end end end @inbounds if bitst != 0 - destc[indc+=1] = remain + destc[indc+1] = remain end return dest end @@ -1284,6 +1314,8 @@ function __dot__(x::Expr) Expr(:let, undot(dotargs[1]), dotargs[2]) elseif x.head === :for # don't add dots to for x=... assignments Expr(:for, undot(dotargs[1]), dotargs[2]) + elseif x.head === :generator || x.head === :filter + Expr(x.head, dotargs[1], map(undot, dotargs[2:end])...) elseif (x.head === :(=) || x.head === :function || x.head === :macro) && Meta.isexpr(x.args[1], :call) # function or macro definition Expr(x.head, x.args[1], dotargs[2]) @@ -1311,6 +1343,10 @@ If you want to *avoid* adding dots for selected function calls in `@. sqrt(abs(\$sort(x)))` is equivalent to `sqrt.(abs.(sort(x)))` (no dot for `sort`). +Note that the postfix `'` and infix `:` operators don't have a broadcasted +version (`.'` and `.:` are not valid operators), and are not affected by `@.`. +The dot operator for property access as in `a.x` is also unaffected. + (`@.` is equivalent to a call to `@__dot__`.) # Examples @@ -1323,6 +1359,18 @@ julia> @. y = x + 3 * sin(x) 4.727892280477045 3.4233600241796016 ``` + +The postfix `'` being unaffected by `@.` can be convenient to broadcast vectors +along different dimensions: + +```jldoctest +julia> @. (10:10:30) + (1:4)' +3×4 Matrix{Int64}: + 11 12 13 14 + 21 22 23 24 + 31 32 33 34 +``` + """ macro __dot__(x) esc(__dot__(x)) diff --git a/base/c.jl b/base/c.jl index 69ea3adf24404..49e72774722b5 100644 --- a/base/c.jl +++ b/base/c.jl @@ -2,20 +2,6 @@ # definitions related to C interface -import .Intrinsics: cglobal - -""" - cglobal((symbol, library) [, type=Cvoid]) - -Obtain a pointer to a global variable in a C-exported shared library, specified exactly as -in [`ccall`](@ref). -Returns a `Ptr{Type}`, defaulting to `Ptr{Cvoid}` if no `Type` argument is -supplied. -The values can be read or written by [`unsafe_load`](@ref) or [`unsafe_store!`](@ref), -respectively. -""" -cglobal - """ CFunction struct @@ -157,12 +143,15 @@ Intended to be called using `do` block syntax as follows: ... end -This is not needed on worker threads (`Threads.threadid() != 1`) since the -`InterruptException` will only be delivered to the master thread. External functions that do not call julia code or julia runtime automatically disable sigint during their execution. """ function disable_sigint(f::Function) + depwarn("`disable_sigint` no longer defers Ctrl-C: SIGINT is delivered as a " * + "cancellation of the current ^C scope and observed at cancellation " * + "points regardless of the sigatomic region this establishes. Shield " * + "a region from cancellation by scoping " * + "`Base.CANCEL_TOKEN => nothing` over it instead.", :disable_sigint) sigatomic_begin() res = f() # Exception unwind sigatomic automatically @@ -188,11 +177,13 @@ end exit_on_sigint(on::Bool) Set `exit_on_sigint` flag of the julia runtime. If `false`, Ctrl-C -(SIGINT) is capturable as [`InterruptException`](@ref) in `try` block. +(SIGINT) cancels the current ^C episode's cancellation scope and is +observed at cancellation points as a [`Base.CancellationRequest`](@ref), +which is capturable in a `try` block. This is the default behavior in REPL, any code run via `-e` and `-E` and in Julia script run with `-i` option. -If `true`, `InterruptException` is not thrown by Ctrl-C. Running code +If `true`, Ctrl-C terminates the process directly. Running code upon such event requires [`atexit`](@ref). This is the default behavior in Julia script run without `-i` option. @@ -226,16 +217,16 @@ function expand_ccallable(name, rt, def) else f = :(typeof($f)) end - at = map(sig.args[2:end]) do a - if isa(a,Expr) && a.head === :(::) - a.args[end] - else - :Any - end - end + at = Any[let a = sig.args[i] + if isa(a,Expr) && a.head === :(::) + a.args[end] + else + :Any + end + end for i in 2:length(sig.args)] return quote @__doc__ $(esc(def)) - _ccallable($name, $(esc(rt)), $(Expr(:curly, :Tuple, esc(f), map(esc, at)...))) + _ccallable($name, $(esc(rt)), $(Expr(:curly, :Tuple, esc(f), map!(esc, at, at)...))) end end end @@ -276,27 +267,39 @@ The above input outputs this: """ function ccall_macro_parse(exprs) gc_safe = false + cancel = nothing expr = nothing if exprs isa Expr expr = exprs - elseif length(exprs) == 1 - expr = exprs[1] - elseif length(exprs) == 2 - gc_expr = exprs[1] - expr = exprs[2] - if gc_expr.head == :(=) && gc_expr.args[1] == :gc_safe - if gc_expr.args[2] == true - gc_safe = true - elseif gc_expr.args[2] == false - gc_safe = false + else + # leading `name = value` options, then the call expression + i = 1 + while i < length(exprs) && isexpr(exprs[i], :(=)) + opt = exprs[i]::Expr + name = opt.args[1] + value = opt.args[2] + if name === :gc_safe + if value === true + gc_safe = true + elseif value === false + gc_safe = false + else + throw(ArgumentError("gc_safe must be true or false")) + end + elseif name === :cancel_handler + if !(isexpr(value, :tuple) && length(value.args) == 2) + throw(ArgumentError("cancel_handler must be a `(handler, state)` tuple")) + end + cancel = (value.args[1], value.args[2]) else - throw(ArgumentError("gc_safe must be true or false")) + throw(ArgumentError("@ccall options are `gc_safe = ` and `cancel_handler = (handler, state)`")) end - else - throw(ArgumentError("@ccall option must be `gc_safe=true` or `gc_safe=false`")) + i += 1 end - else - throw(ArgumentError("@ccall needs a function signature with a return type")) + if i != length(exprs) + throw(ArgumentError("@ccall needs a function signature with a return type")) + end + expr = exprs[i] end # setup and check for errors @@ -313,11 +316,15 @@ function ccall_macro_parse(exprs) # get the function symbols func = let f = call.args[1] if isexpr(f, :.) - :(($(f.args[2]), $(f.args[1]))) + Expr(:tuple, f.args[2], f.args[1]) elseif isexpr(f, :$) - f + func = f.args[1] + if isa(func, String) || (isa(func, QuoteNode) && !isa(func.value, Ptr)) || isa(func, Tuple) || isexpr(func, :tuple) + throw(ArgumentError("interpolated value should be a variable or expression, not a literal name or tuple")) + end + func elseif f isa Symbol - QuoteNode(f) + Expr(:tuple, QuoteNode(f)) else throw(ArgumentError("@ccall function name must be a symbol, a `.` node (e.g. `libc.printf`) or an interpolated function pointer (with `\$`)")) end @@ -349,7 +356,7 @@ function ccall_macro_parse(exprs) end # add any varargs if necessary nreq = 0 - if !isnothing(varargs) + if varargs !== nothing if length(args) == 0 throw(ArgumentError("C ABI prohibits vararg without one required argument")) end @@ -358,38 +365,39 @@ function ccall_macro_parse(exprs) pusharg!(a) end end - return func, rettype, types, args, gc_safe, nreq + return func, rettype, types, args, gc_safe, cancel, nreq end -function ccall_macro_lower(convention, func, rettype, types, args, gc_safe, nreq) - statements = [] - - # if interpolation was used, ensure the value is a function pointer at runtime. - if isexpr(func, :$) - push!(statements, Expr(:(=), :func, esc(func.args[1]))) - name = QuoteNode(func.args[1]) - func = :func - check = quote - if !isa(func, Ptr{Cvoid}) - name = $name - throw(ArgumentError(LazyString("interpolated function `", name, "` was not a Ptr{Cvoid}, but ", typeof(func)))) - end - end - push!(statements, check) - else - func = esc(func) - end - cconv = nothing +function ccall_macro_lower(convention, func, rettype, types, args, gc_safe, cancel, nreq) + have_cancel = cancel !== nothing + # `Base.@assume_effects :reset_safe @ccall ...` arrives as the + # CCALL_EFFECT_RESET_SAFE bit above the standard effects overrides in + # the `@ccall_effects` word (see `@assume_effects` in expr.jl): split it + # back out into its dedicated calling-convention slot here. + reset_safe = false if convention isa Tuple - cconv = Expr(:cconv, (convention..., gc_safe), nreq) + cc_sym, effects = convention + reset_safe = (effects & CCALL_EFFECT_RESET_SAFE) != 0x0000 + base_cconv = (cc_sym, effects & ~CCALL_EFFECT_RESET_SAFE, gc_safe) else - cconv = Expr(:cconv, (convention, UInt16(0), gc_safe), nreq) + base_cconv = (convention, UInt16(0), gc_safe) end - - return Expr(:block, statements..., - Expr(:call, :ccall, func, cconv, esc(rettype), - Expr(:tuple, map(esc, types)...), map(esc, args)...)) + if !have_cancel + cconv = reset_safe ? Expr(:cconv, (base_cconv..., false, true), nreq) : + Expr(:cconv, base_cconv, nreq) + return Expr(:call, :ccall, esc(func), cconv, esc(rettype), + Expr(:tuple, map!(esc, types, types)...), map!(esc, args, args)...) + end + # Our internal ABI for cancellation handlers is f(func, state, args...). Codegen + # inlines the establishment of the handler and the ultimate call to `f` has the + # ordinary ABI without the extra arguments. + fex, sex = cancel + nreq > 0 && (nreq += 2) + cconv = Expr(:cconv, reset_safe ? (base_cconv..., true, true) : (base_cconv..., true), nreq) + return Expr(:call, :ccall, esc(func), cconv, esc(rettype), + Expr(:tuple, :(Ptr{Cvoid}), :(Ptr{Cvoid}), map!(esc, types, types)...), + esc(fex), esc(sex), map!(esc, args, args)...) end """ @@ -441,16 +449,95 @@ The string literal could also be used directly before the function name, if desired `"libglib-2.0".g_uri_escape_string(...` It's possible to declare the ccall as `gc_safe` by using the `gc_safe = true` option: + @ccall gc_safe=true strlen(s::Cstring)::Csize_t + This allows the garbage collector to run concurrently with the ccall, which can be useful whenever the `ccall` may block outside of julia. -WARNING: This option should be used with caution, as it can lead to undefined behavior if the ccall -calls back into the julia runtime. (`@cfunction`/`@ccallables` are safe however) + +!!! warning + This option should be used with caution, as it can lead to undefined behavior if the ccall + calls back into the julia runtime. (`@cfunction`/`@ccallable` are safe however) + +!!! compat "Julia 1.12" + The `gc_safe` argument requires Julia 1.12 or higher. + +# Extended help + +## Cancellation + +### Semantics of reset_safe annotation + +When annotated `@assume_effects :reset_safe`, e.g.: + + Base.@assume_effects :reset_safe @ccall libgmp.__gmpz_mul(x::mpz_t, a::mpz_t, b::mpz_t)::Cvoid + +The compiler has license to extend an earlier reset region through the entire +execution of the called C function. In particular, as a result, the function +may be abandoned at any point (and reset to the appropriate cancellation +point). + +This imposes strict requirements on the foreign code. In particular, it is +generally unsafe to call into most C standard library functions (the Julia +runtime will handle properly protecting the symbol lookup itself), modify +memory other than allocated since the most recent cancellation point (or +which the cancellation will explicitly clean up). Additionally, any memory +access to memory that may be read after a reset must be annotated +`volatile`. This list of conditions is meant to be illustrative, not +exhaustive and may be extended with additional warnings in the future. + +Co-operating foreign code may use the Julia C API to temporarily protect +regions from reset, although this API is not yet stable and requires source +modifications. + +!!! compat "Julia 1.14" + The `:reset_safe` effect requires Julia 1.14 or higher. + +## Cancellation handlers + +Because the `:reset_safe` effect imposes such strict requirements, another +option is provided to allow for cancellation of cooperating foreign code. +In particular, a long-running foreign call can be made cancellable with the +`cancel_handler = (handler, state)` option: + + @ccall cancel_handler=(CANCEL_FPTR, ref) lib.solve(ref::Ptr{Cvoid})::Cvoid + +`handler` is a C-callable function pointer `void (*)(void *state, uint8_t +sev)` (typically from [`@cfunction`](@ref)). If the cancellation token +source (`Base.CancellationTokenSource`) bound to the calling task is +cancelled while the call runs, the runtime invokes `handler(state, sev)` *on +the thread executing the call*, like a signal handler: at an arbitrary point +of the foreign code, on the same stack, resuming the interrupted call when +the handler returns (`sev` is the request state of the cancelled source). +The handler performs the library-specific work to make the foreign call +return early - typically setting a flag or calling the library's own +cancellation entry point. + +Both `CANCEL_FPTR` and `ref` are handled as ordinary ccall arguments (both +of C type `Ptr{Cvoid}`), including for purposes of rooting. Note that since +`CANCEL_FPTR` runs as a signal handler, it must be async-signal-safe in the +ordinary sense, although unlike `:reset_safe`, no requirements are imposed +on the called function itself. + +A function may be annotated with both `:reset_safe` and `cancel_handler`, +in which case the cancel_handler takes precedence. However, in such a +situation, the compiler will attempt to preserve the reset region into the +ccall and it may be read by the cancel_handler (which can then perform some +library specific cleanup before performing the reset as usual). However, +this is not required nor assumed. A cancel handler could instead cause the +function to return early with an error code. Note however, that there is no +automatic cancellation point implied by the `cancel_handler` attribute. It +is the responsibility of the user to set an appropriate cancellation point +either before or after (possibly conditional on an appropriate error return +code) the ccall. + +!!! compat "Julia 1.14" + The `cancel_handler` option requires Julia 1.14 or higher. """ macro ccall(exprs...) return ccall_macro_lower((:ccall), ccall_macro_parse(exprs)...) end -macro ccall_effects(effects::UInt16, expr) - return ccall_macro_lower((:ccall, effects), ccall_macro_parse(expr)...) +macro ccall_effects(effects::UInt16, exprs...) + return ccall_macro_lower((:ccall, effects), ccall_macro_parse(exprs)...) end diff --git a/base/cancellation.jl b/base/cancellation.jl new file mode 100644 index 0000000000000..2234bbb24bf17 --- /dev/null +++ b/base/cancellation.jl @@ -0,0 +1,1128 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## Cancellation tokens +# +# Cancellation is organized around *cancellation token sources* +# (`Core.CancellationTokenSource`): level-triggered condition nodes arranged +# in a DAG (a source may have several parents). Cancelling a source cancels +# all of its descendants; the cancelled state is monotonic (severities only +# escalate, never reset). Following the +# .NET split, a `CancellationToken` is the observe view handed to code that +# may only *react* to cancellation; the source is the capability to *request* +# it. +# +# For convenience, the scoped value `CANCEL_TOKEN` carries the default +# cancellation token governing the current dynamic extent. + +const CancellationTokenSource = Core.CancellationTokenSource + +""" + CancellationToken(src::CancellationTokenSource) + +The observe side of a [`CancellationTokenSource`](@ref): code holding a +token can be interrupted by - and can query ([`iscancelled`](@ref)) - +cancellation of the associated source, but cannot request cancellation +itself. Only the holder of the *source* can call [`cancel!`](@ref). + +A token takes effect by scoping it over a computation via the +[`CANCEL_TOKEN`](@ref) scoped value or by explicit token passing. +Once the associated source is cancelled, the computation's cancellation points (see +[`@cancel_check`](@ref)) throw a [`CancellationRequest`](@ref). +""" +struct CancellationToken + source::CancellationTokenSource +end + +""" + CancellationRequest + +The exception thrown by cancellation points whose governing cancellation +token has been cancelled. The `request` field records the severity +([`CANCEL_REQUEST_SAFE`](@ref), [`CANCEL_REQUEST_ABANDON_EXTERNAL`](@ref) or +[`CANCEL_REQUEST_ABANDON_ALL`](@ref)) as observed at delivery time; the +source may escalate afterwards. +""" +struct CancellationRequest <: Exception + request::UInt8 +end + +""" + CANCEL_REQUEST_SAFE + +Request safe cancellation. Code observing the cancellation will request safe +cancellation of any resources it is waiting for and wait for the cancellation +of such resources to be completed. + +As a result, if either the cancelled code or any of its dependent resources +are currently unable to process cancellation, the request may hang and a more +aggressive cancellation severity may be required. However, in general _SAFE +should be tried first. +""" +const CANCEL_REQUEST_SAFE = CancellationRequest(0x1) + +""" + CANCEL_REQUEST_ABANDON_EXTERNAL + +Request a cancellation that will cease waiting for any external resources +(e.g. I/O objects) without going through a safe cancellation procedure for +such resources. However, internal computational tasks are still awaited. + +This is a middleground between CANCEL_REQUEST_SAFE and +CANCEL_REQUEST_ABANDON_ALL, as external I/O is often engineered for +robustness in case of sudden disappearance of peers. +""" +const CANCEL_REQUEST_ABANDON_EXTERNAL = CancellationRequest(0x3) + +""" + CANCEL_REQUEST_ABANDON_ALL + +Request a cancellation that will cease waiting for all external resources, +and give up on tasks that have not responded to the cancellation: they are +frozen in place and never scheduled again. + +!!! warning + If any cancelled task has acquired locks or other resources that are + contested, this method of cancellation may leak such resources and create + deadlocks in future code. It is intended as a last-resort method to + recover a system, but the necessity of this operation should in general + be considered a bug (e.g. due to insufficient cancellation points in + computationally-heavy code). +""" +const CANCEL_REQUEST_ABANDON_ALL = CancellationRequest(0x4) + +# The status byte reported by a cancellation point (`Core.cancellation_point!`) +# is the governing source's state byte (0 while uncancelled, the severity once +# cancelled) with the 0x40 bit merged in when the task has a pending +# cooperative-yield request. The mask recovers the severity half of such a +# status byte; source state reads themselves need no masking. +const STATUS_PREEMPT_BIT = 0x40 +const SEVERITY_MASK = 0x3f + +# The severity of a request, for the delivery layer's comparisons. +severity(cr::CancellationRequest) = cr.request + +""" + cancel_severity(src::CancellationTokenSource) -> Union{Nothing, CancellationRequest} + cancel_severity(tok::CancellationToken) + +Return `nothing` if the source has not been cancelled, or a +`CancellationRequest` recording the current (monotonically escalating) +severity if it has. +""" +function cancel_severity(src::CancellationTokenSource) + st = @atomic :acquire src.state + st == 0x00 && return nothing + return CancellationRequest(st) +end +cancel_severity(tok::CancellationToken) = cancel_severity(tok.source) + +""" + iscancelled(src::CancellationTokenSource)::Bool + iscancelled(tok::CancellationToken)::Bool + +Whether the source has been cancelled (level-triggered: once cancelled, a +source stays cancelled). +""" +iscancelled(src::CancellationTokenSource) = (@atomic :monotonic src.state) != 0x00 +iscancelled(tok::CancellationToken) = iscancelled(tok.source) + +## Source construction and graph linkage + +""" + CancellationTokenSource() -> CancellationTokenSource + CancellationTokenSource(parents::CancellationToken...) + +Create a new cancellation token source. With no arguments the source is a +standalone root; given one or more parent tokens, the new source is linked +underneath each of them, so that cancellation of *any* parent (or any of its +ancestors) also cancels the new source - at the highest severity requested +among them. Sources therefore form a directed acyclic graph; a source +created under an already-cancelled parent is born cancelled. Linking one +source under several parents is how an operation respects two independent +lifetimes at once (say, a request scope and the connection it arrived on). + +A child source stays linked to its parents for exactly as long as it is +reachable - a held token, or work governed by it, keeps it alive. Once +nothing can observe it any more, it is garbage collected and thereby drops +out of the graph; there is no explicit detach operation. + +Use [`CancellationToken`](@ref)`(src)` for the observe view, and +[`cancel!`](@ref)`(src)` to request cancellation. +""" +CancellationTokenSource(parent::CancellationToken) = + Core._new_cancel_source(parent.source)::CancellationTokenSource +function CancellationTokenSource(parent::CancellationToken, rest::CancellationToken...) + srcs = CancellationTokenSource[parent.source] + for tok in rest + any(s -> s === tok.source, srcs) || push!(srcs, tok.source) + end + return Core._new_cancel_source(srcs...)::CancellationTokenSource +end +CancellationTokenSource(::Nothing) = Core._new_cancel_source()::CancellationTokenSource +CancellationTokenSource() = Core._new_cancel_source()::CancellationTokenSource + +# The i-th (1-based) parent of `src`. Parent links are strong and const, so +# these reads need no synchronization. +_cancel_parent(src::CancellationTokenSource, i::Int) = + ccall(:jl_cancel_source_parent, Any, (Any, Csize_t), src, i - 1)::CancellationTokenSource + +# The sibling after `child` on `parent`'s child list (`nothing` at its end). +# Weak, but safe to traverse from Julia: the returned reference is rooted the +# moment the ccall returns, and the GC's splice pass keeps the lists free of +# collected entries at every safepoint, so a traversal (re-)started from a +# rooted node only ever sees live sources. +_cancel_next_child(parent::CancellationTokenSource, child::CancellationTokenSource) = + ccall(:jl_cancel_source_next_child, Any, (Any, Any), parent, child)::Union{Nothing, CancellationTokenSource} + +# CAS-max the source's state to `sev` (a valid, nonzero severity). Returns +# true if the state was raised, false if it was already at (or above) the +# severity. +# seq_cst: pairs with the (child-list publication; state read) sequence in +# `jl_new_cancel_source` - see the walk in `_cancel_walk_node!`. +function _raise_state!(src::CancellationTokenSource, sev::UInt8) + old = @atomic :monotonic src.state + while true + if old >= sev + return false + end + old, success = @atomicreplace :sequentially_consistent :monotonic src.state old => sev + success && return true + end +end + +## Wait registrations (used by condition.jl and every parked wait) + +# A task's registration on the things it waits for: `task` plus uniform +# {owner, next, aux} slots, one per waitable, with a waitee slot's `owner` +# holding the queue's identity (see `waitqueue`) while enqueued - the "am +# I registered, and on what" witness - and `nothing` otherwise. Field +# protection and atomicity live with the accessors ("Uniform slot access" +# below); the key liberty is that `task` and the slot owners are identity +# witnesses read racily by walkers (the cancellation walk, +# `_waitq_isempty`), whose staleness the protocols tolerate: a claim is +# validated by the `waiting_on` CAS, never by those reads alone. +# +# The wake-claim protocol: a parked task `t` points to its current +# registration through the atomic field `t.waiting_on`. Whoever wants to wake +# it must first claim the wake by atomically clearing that field: +# +# - `notify` (holding the waitee's lock) pops an entry `w` and claims via +# CAS(t.waiting_on, w => nothing). The expected-value CAS makes stale +# entries harmless: if `t` was interrupted and has since registered +# elsewhere, the CAS fails and the popped corpse is simply dropped. +# - an interrupter (`schedule(t, exc, error=true)`) claims via an +# unconditional swap: it is directed at the *task*, not at any particular +# wait, so claiming whatever `t` is currently registered on is correct. +# It then opportunistically unlinks the claimed entry under the waitee's +# lock via `trylock` (see `try_unlink_claimed!`); if the lock is +# unavailable the entry stays linked and is collected lazily, either by +# the interrupted task's own wait cleanup or by the `notify` that pops +# and drops it. The cancellation walk claims like `notify` (expected-entry +# CAS) but leaves the entry linked for the same lazy collection. +# - wake sources directed at one *specific* wait (e.g. the timeout task of +# `Experimental.wait_with_timeout`) must register the wait with a fresh, +# single-use entry: single-use-ness is what guarantees their +# expected-value CAS cannot mistakenly claim a later, unrelated wait. +# +# Entries are heap objects: a task whose interrupted wait left a stale +# registration behind can immediately register anew - e.g. park on a lock +# during its cleanup - with a fresh entry. Entries come in kinds by slot +# count: the 1-slot kind covers plain parks (one waitee slot), the 2-slot +# kind adds the cancellation-source slot for cancellable parks, and the +# variable-slot `WaitEntryN` covers parks on several waitables at once +# (multi-waitable `park!`, and hence waitany/timeouts). The kinds share +# their leading layout; `WaitEntry` is the union of all kinds, and hot +# paths union-split on it. +typegroup + mutable struct WaitEntry1 + @atomic task::Union{Task, Nothing} + @atomic owner1::Any + next1::Union{WaitEntry1, WaitEntry2, Core.WaitEntryN, Nothing} + aux1::UInt64 + WaitEntry1(task::Union{Task, Nothing}) = new(task, nothing, nothing, 0x0) + end + mutable struct WaitEntry2 + @atomic task::Union{Task, Nothing} + @atomic owner1::Any + next1::Union{WaitEntry1, WaitEntry2, Core.WaitEntryN, Nothing} + aux1::UInt64 + @atomic owner2::Any + next2::Union{WaitEntry1, WaitEntry2, Core.WaitEntryN, Nothing} + aux2::UInt64 + WaitEntry2(task::Union{Task, Nothing}) = + new(task, nothing, nothing, 0x0, nothing, nothing, 0x0) + end +end +const WaitEntryN = Core.WaitEntryN +WaitEntryN(task::Union{Task, Nothing}, nslots::Integer) = + ccall(:jl_new_wait_entry, Any, (Any, Csize_t), task, nslots)::WaitEntryN +const WaitEntry = Union{WaitEntry1, WaitEntry2, WaitEntryN} + +## Uniform slot access +# +# Every kind is a `task` plus `_nslots` uniform {owner, next, aux} slots +# (1-based). `task` and the slot `owner`s are atomic, accessed relaxed +# (walkers read them without the owner's locks - see the notes above and +# below); the `owner` doubles as the membership witness (`nothing` = free +# slot). `next` and `aux` are plain, each protected by its owner's +# discipline: waitee slots by the waitee's lock, cancellation-source slots +# by the registration protocol below, and free slots by the owning task. +# Lists link whole entries: a traversal locates its slot in each entry by +# scanning for its own identity (`_find_slot`), so an entry may be +# registered on several waitables at once (wait-any). At most one slot per +# owner per entry. +_nslots(w::WaitEntry1) = 1 +_nslots(w::WaitEntry2) = 2 +_nslots(w::WaitEntryN) = Int(w.nslots) + +# Where an owner read carries ordering obligations, the ordering is +# supplied elsewhere (the waitee's lock, or the seq_cst publish/recheck +# dance of the source registration - `SourceWait` in base/park.jl). +@inline _slot_owner(w::WaitEntry1, i::Int) = @atomic :monotonic w.owner1 +@inline _slot_owner(w::WaitEntry2, i::Int) = + i == 1 ? (@atomic :monotonic w.owner1) : (@atomic :monotonic w.owner2) +@inline _slot_owner(w::WaitEntryN, i::Int) = + ccall(:jl_wait_entry_slot_owner, Any, (Any, Csize_t), w, i - 1) + +@inline function _set_slot_owner!(w::WaitEntry1, i::Int, @nospecialize(v)) + @atomic :monotonic w.owner1 = v + return nothing +end +@inline function _set_slot_owner!(w::WaitEntry2, i::Int, @nospecialize(v)) + if i == 1 + @atomic :monotonic w.owner1 = v + else + @atomic :monotonic w.owner2 = v + end + return nothing +end +@inline _set_slot_owner!(w::WaitEntryN, i::Int, @nospecialize(v)) = + ccall(:jl_wait_entry_set_slot_owner, Cvoid, (Any, Csize_t, Any), w, i - 1, v) + +@inline _slot_next(w::WaitEntry1, i::Int) = w.next1 +@inline _slot_next(w::WaitEntry2, i::Int) = i == 1 ? w.next1 : w.next2 +@inline _slot_next(w::WaitEntryN, i::Int) = + ccall(:jl_wait_entry_slot_next, Any, (Any, Csize_t), w, i - 1)::Union{WaitEntry, Nothing} + +@inline _set_slot_next!(w::WaitEntry1, i::Int, v::Union{WaitEntry, Nothing}) = (w.next1 = v; nothing) +@inline function _set_slot_next!(w::WaitEntry2, i::Int, v::Union{WaitEntry, Nothing}) + i == 1 ? (w.next1 = v) : (w.next2 = v) + return nothing +end +@inline _set_slot_next!(w::WaitEntryN, i::Int, v::Union{WaitEntry, Nothing}) = + ccall(:jl_wait_entry_set_slot_next, Cvoid, (Any, Csize_t, Any), w, i - 1, + v === nothing ? nothing : v) + +@inline _slot_aux(w::WaitEntry1, i::Int) = w.aux1 +@inline _slot_aux(w::WaitEntry2, i::Int) = i == 1 ? w.aux1 : w.aux2 +@inline _slot_aux(w::WaitEntryN, i::Int) = + ccall(:jl_wait_entry_slot_aux, UInt64, (Any, Csize_t), w, i - 1) + +@inline _set_slot_aux!(w::WaitEntry1, i::Int, v::UInt64) = (w.aux1 = v; nothing) +@inline function _set_slot_aux!(w::WaitEntry2, i::Int, v::UInt64) + i == 1 ? (w.aux1 = v) : (w.aux2 = v) + return nothing +end +@inline _set_slot_aux!(w::WaitEntryN, i::Int, v::UInt64) = + ccall(:jl_wait_entry_set_slot_aux, Cvoid, (Any, Csize_t, UInt64), w, i - 1, v) + +# The slot registered on `owner`, or 0. +@inline _find_slot(w::WaitEntry1, @nospecialize(owner)) = + (@atomic :monotonic w.owner1) === owner ? 1 : 0 +@inline _find_slot(w::WaitEntry2, @nospecialize(owner)) = + (@atomic :monotonic w.owner1) === owner ? 1 : + (@atomic :monotonic w.owner2) === owner ? 2 : 0 +function _find_slot(w::WaitEntryN, @nospecialize(owner)) + for i in 1:_nslots(w) + _slot_owner(w, i) === owner && return i + end + return 0 +end + +# The first free slot, or 0. +@inline _free_slot(w::WaitEntry1) = (@atomic :monotonic w.owner1) === nothing ? 1 : 0 +@inline _free_slot(w::WaitEntry2) = + (@atomic :monotonic w.owner1) === nothing ? 1 : + (@atomic :monotonic w.owner2) === nothing ? 2 : 0 +_free_slot(w::WaitEntryN) = _find_slot(w, nothing) + +@noinline _slot_overflow_error() = + throw(ConcurrencyViolationError("wait entry has no free slot for this registration")) + +# Claim a free slot for `owner` (which must not already have one). +@inline function _acquire_slot!(w::WaitEntry, @nospecialize(owner)) + i = _free_slot(w) + i == 0 && _slot_overflow_error() + _set_slot_owner!(w, i, owner) + return i +end + +# A reference to one slot of a wait entry, with `owner`/`next`/`aux` +# property access (owner accesses are relaxed-atomic; see above), and a +# vector view of an entry's slots. `slots(w)[i].owner === src`, +# `slot.next = x`, `for slot in slots(w)` are the intended spellings for +# code that is generic over the slot count. +struct WaitSlotRef{T<:WaitEntry} + entry::T + i::Int +end +@inline function getproperty(s::WaitSlotRef, f::Symbol) + f === :owner && return _slot_owner(getfield(s, :entry), getfield(s, :i)) + f === :next && return _slot_next(getfield(s, :entry), getfield(s, :i)) + f === :aux && return _slot_aux(getfield(s, :entry), getfield(s, :i)) + return getfield(s, f) +end +@inline function setproperty!(s::WaitSlotRef, f::Symbol, @nospecialize(v)) + if f === :owner + _set_slot_owner!(getfield(s, :entry), getfield(s, :i), v) + elseif f === :next + _set_slot_next!(getfield(s, :entry), getfield(s, :i), v::Union{WaitEntry, Nothing}) + elseif f === :aux + _set_slot_aux!(getfield(s, :entry), getfield(s, :i), v::UInt64) + else + throw(FieldError(WaitSlotRef, f)) + end + return v +end + +struct WaitSlots{T<:WaitEntry} <: AbstractVector{WaitSlotRef{T}} + entry::T +end +size(s::WaitSlots) = (_nslots(getfield(s, :entry)),) +@inline getindex(s::WaitSlots, i::Int) = WaitSlotRef(getfield(s, :entry), i) +slots(w::WaitEntry) = WaitSlots(w) + +# Free slot `i` of `w`: `next` and `aux` are cleared first (a freed slot's +# payload must not leak into its next registration - e.g. the uv write +# cancel flag, see base/stream.jl) and the `owner` witness last - its +# clearing is what hands the slot back for reuse. +@inline function _release_slot!(w::WaitEntry, i::Int) + _set_slot_next!(w, i, nothing) + _set_slot_aux!(w, i, UInt64(0)) + _set_slot_owner!(w, i, nothing) + return nothing +end + +# The entry after `w` on `owner`'s list (`nothing` at its end, or - +# defensively - when `w` has no slot for `owner`). +@inline function _next_on(w::WaitEntry, @nospecialize(owner)) + i = _find_slot(w, owner) + return i == 0 ? nothing : _slot_next(w, i) +end + +# Set/clear a bare waitee witness (a registration that marks the entry as +# in-use for `x` without linking it into any list - the libuv request +# waits): the slot's `owner` is the reuse gate, `next` stays free. +_set_wait_witness!(w::WaitEntry, @nospecialize(x)) = (_acquire_slot!(w, x); nothing) +function _clear_wait_witness!(w::WaitEntry, @nospecialize(x)) + i = _find_slot(w, x) + i == 0 || _release_slot!(w, i) + return nothing +end + +# Release the waitee witness of a uv request wait from its completion +# callback, which holds the entry but not the waitee: uv entries carry one +# witness slot alongside an optional cancellation-source slot, which stays +# registered (sticky). +function _clear_uv_witness!(w::WaitEntry) + for slot in slots(w) + o = slot.owner + (o === nothing || o isa CancellationTokenSource) && continue + slot.next = nothing + slot.aux = UInt64(0) # e.g. the uv write cancel flag; see _release_slot! + slot.owner = nothing + break + end + return nothing +end + +# Return `waiter`'s cached wait entry for a *plain* (non-cancellable) park, +# or a fresh (and newly cached) one if the cached one is still in use. +# +# Plain parks arm a *distinct* entry from cancellable parks +# (`_cancel_wait_entry`), and that identity split is what keeps shields +# shielded: the cancellation walk's only sound eligibility gate is its +# expected-entry claim CAS, so an entry registered on a source must never +# be armed for a wait that is not cancellable under it. (Aux data outside +# the claim word - the severity floors - is read racily by the walk and may +# be judged against an adjacent arm of the same entry; that misfire is +# tolerable for a teardown wait's floor, never for a shield: sharing one +# entry would let a walk claim a shielded re-arm using the previous arm's +# eligibility.) +function _cached_wait_entry(waiter::Task) + w = waiter.cached_wait_entry + if w isa WaitEntry1 && (@atomic :monotonic w.owner1) === nothing + @atomic :monotonic w.task = waiter + else + w = WaitEntry1(waiter) + waiter.cached_wait_entry = w + end + return w +end + +# A source slot's aux: the low byte is the minimum delivery severity (the +# "floor"); the watcher bit marks a `wait(::CancellationToken)` slot, whose +# claimed wake the walk *completes* with the request as a value instead of +# interrupting the task - the cancellation is the event that wait is for. +# Staged pre-arm like the floor (see below). +const WAIT_AUX_WATCHER_BIT = UInt64(0x100) + +# Return the entry for a cancellable park of `waiter` governed by `src`, +# with the minimum delivery severity staged on the source slot. Must run +# before the arm: the cancellation walk reads the slot's aux only through +# an armed entry, so it has to be in place when `waiting_on` is published. +# Convention: `min_severity` is the lowest severity that may wake the wait +# (inclusive comparisons on both the registration and walk sides); a +# teardown wait that already acknowledged a delivery at severity `s` stages +# `s + 0x01` so only an escalation wakes it. +# +# The cached entry is reused when its sticky source slot is compatible: +# already on `src` (the common case - a task parking repeatedly under its +# ambient token pays no registry work after the first park), or free. When +# it is bound to a *different* source (the task's governing token changed), +# the owner unregisters it - the one O(list) operation on a park path, paid +# once per token migration - and rebinds. A cache entry stuck on a wait +# queue (stale registration from an interrupted wait, not yet collected) +# leaves a fresh single-use entry whose source registration becomes an +# unarmed corpse for pruning. +function _cancel_wait_entry(waiter::Task, src::CancellationTokenSource, + min_severity::UInt8) + w = waiter.cached_cancel_entry + if w isa WaitEntry2 && (@atomic :monotonic w.owner1) === nothing + o = @atomic :monotonic w.owner2 + if !(o === src || o === nothing) + # rebind: physically drop the stale sticky registration first + unregister_cancellation!(o::CancellationTokenSource, w) + end + @atomic :monotonic w.task = waiter + elseif w isa WaitEntry2 + w = WaitEntry2(waiter) + else + w = WaitEntry2(waiter) + waiter.cached_cancel_entry = w + end + w.aux2 = UInt64(min_severity) + return w +end + +@noinline function _wait_registration_error() + throw(ConcurrencyViolationError("Task is already registered on a wait queue")) +end + +# Publish `w` as `waiter`'s only armed wait registration. +function _arm_wait(waiter::Task, w::WaitEntry) + armed = @atomicreplace :release :monotonic waiter.waiting_on nothing => w + armed.success || _wait_registration_error() + return w +end + +# Claim the wake of the wait that `w` was registered for (returns whether the +# claim succeeded). `w` must be an entry armed for `t` (`_arm_wait`). +function claim_wait(t::Task, w::WaitEntry) + return (@atomicreplace t.waiting_on w => nothing).success +end + +## Waiter registration +# +# The waiter list is lock-free on every hot path (like the source's child +# list): registration is a CAS push of the entry onto `waiters_head`, a +# normal wakeup does no registry work at all (the entry stays registered - +# armed/unarmed is tracked by the task's `waiting_on` claim word), and a +# repeat park under the same source re-arms the already-registered entry +# with no shared-memory operation. Only walks (cancellation delivery, +# pruning, and the owner-side `unregister_cancellation!`) rewrite links, +# serialized by the source's `walk_lock`, which no park or wakeup ever +# takes. + +# Serialize walks on `src` (cancellation delivery, pruning, owner-side +# unregistration). Never taken on park/wake paths; contended walkers sleep. +# The lock object is installed lazily by the first walker, so sources that +# are never walked stay two words smaller than a ReentrantLock. +function _walk_lock(src::CancellationTokenSource) + l = @atomic :acquire src.walk_lock + l === nothing || return l::ReentrantLock + newl = ReentrantLock() + old, ok = @atomicreplace :acquire_release :acquire src.walk_lock nothing => newl + return ok ? newl : old::ReentrantLock +end +# A walk delivers (or collects for) cancellations, possibly of the very +# scope the walking task runs under: the acquire must be shielded. +_lock_walk(src::CancellationTokenSource) = lock(_walk_lock(src); cancel=nothing) +_trylock_walk(src::CancellationTokenSource) = trylock(_walk_lock(src)) +_unlock_walk(src::CancellationTokenSource) = unlock(_walk_lock(src)) + +# Pruning: dead registrations (retired entries, entries of completed tasks) +# are counted where they die - `retire_cancellation_entry!` and the task +# teardown hook - and the incrementer that trips the threshold runs a prune +# walk, so corpses cannot accumulate without bound on a long-lived source +# that never gets cancelled. Sources are mostly scope-lifetime, so in +# practice the dominant reclaimer is the source dying. +const _PRUNE_DEAD_THRESHOLD = UInt32(16) + +# Account a newly dead registration on `src` and prune when enough have +# accumulated. `>=`, not `==`: when the incrementer that crosses the +# threshold loses `_try_prune!`'s trylock to a concurrent walk, corpses that +# walk had already passed remain counted here, and the next death must retry +# the prune rather than let the count sail past the threshold forever. +# The threshold scales with the (approximate) list length: a prune walk is +# O(list), so tripping it every fixed number of deaths makes a mass fan-out +# parking under one source - e.g. any large task tree governed by a shared +# ambient token - quadratic in its waiter count. Requiring the dead to be a +# constant fraction of the list amortizes each walk against the corpses it +# collects. +function _note_dead_registration!(src::CancellationTokenSource) + dc = @atomic :monotonic src.dead_count += UInt32(1) + threshold = max(_PRUNE_DEAD_THRESHOLD, (@atomic :monotonic src.reg_count) >> 2) + dc >= threshold && _try_prune!(src) + return nothing +end + +# Relaxed load of the waiter-list head (an `Any` field whose non-entry +# initial value means the empty list). The walk's seq_cst head read stays +# spelled out at its use site. +@inline function _waiters_head(src::CancellationTokenSource) + h = @atomic :monotonic src.waiters_head + return h isa WaitEntry ? h : nothing +end + +# (Source registration - the sticky lock-free push, the re-arm fence, and +# the post-publication state recheck - lives in base/park.jl as +# `SourceWait`'s `wait_enqueue!`/`wait_recheck` methods; the refusal is +# the driver's fired path.) + +# Owner-side physical unregistration of `w` from `src`'s waiter list: an +# O(list) walk under the walk lock. Used on the rare paths that must drop a +# sticky registration eagerly - rebinding the cached cancel entry after a +# token migration - not on any wakeup path (normal wakeups leave the +# registration in place). `w` must be unarmed and owned by the caller. +function unregister_cancellation!(src::CancellationTokenSource, w::WaitEntry) + wi = _find_slot(w, src) + wi == 0 && return nothing + wslot = slots(w)[wi] + _lock_walk(src) + prev = nothing # the predecessor's slot for `src`, once past the head + x = _waiters_head(src) + while x isa WaitEntry + xi = x === w ? wi : _find_slot(x, src) + # a linked entry always has a slot for this source (see + # _walk_waiters!); bail out without touching the structure otherwise + xi == 0 && break + slot = slots(x)[xi] + xnext = slot.next + if x === w + if prev === nothing + # racing pushes prepend; retrying against the fresh head + # re-finds `w`'s predecessor + if !(@atomicreplace :monotonic :monotonic src.waiters_head x => xnext).success + x = _waiters_head(src) + continue + end + else + prev.next = xnext + end + wslot.next = nothing + break + end + prev = slot + x = xnext + end + wslot.owner = nothing + _unlock_walk(src) + return nothing +end + +# Mark the single-use entry `w` as done with its (sticky) source +# registrations: walks then collect it like an entry of a completed task. +# For per-call entries (`Experimental.wait_with_timeout`, waitany) whose +# task may live - and keep registering - indefinitely. +function retire_cancellation_entry!(w::WaitEntry) + @atomic :monotonic w.task = nothing + for slot in slots(w) + o = slot.owner + o isa CancellationTokenSource && _note_dead_registration!(o) + end + return nothing +end + +# Prune walk: collect unarmed corpses (completed tasks, retired entries). +# Never delivers - delivery is `cancel!`'s walk, and any registrant racing +# a cancellation is covered by its own state recheck. +@noinline function _try_prune!(src::CancellationTokenSource) + _trylock_walk(src) || return nothing # a running walk collects anyway + _walk_waiters!(src, 0x00) + _unlock_walk(src) + return nothing +end + +## Cancellation +# +# Cancellation is uniformly level-triggered: while the governing token is +# cancelled, every cancellation point throws the `CancellationRequest`. + +""" + cancel!(src::CancellationTokenSource, + request::CancellationRequest=CANCEL_REQUEST_SAFE)::Bool + +Cancel `src` and all of its descendants at the given severity. Level-triggered +and monotonic: observers (including future registrants) see the cancellation +until the source is discarded, and repeated calls only have an effect when +they *escalate* the severity ([`CANCEL_REQUEST_SAFE`](@ref) -> +[`CANCEL_REQUEST_ABANDON_EXTERNAL`](@ref) -> +[`CANCEL_REQUEST_ABANDON_ALL`](@ref)). Returns whether the call changed the +state. + +Computations governed by a token of `src` or a descendant observe the cancellation at +their cancellation points (see [`@cancel_check`](@ref)), which throw a +[`CancellationRequest`](@ref). + +When `cancel!` returns, every source currently reachable from `src` has +been advanced to (at least) the requested severity by this very call. +""" +function cancel!(src::CancellationTokenSource, + request::CancellationRequest=CANCEL_REQUEST_SAFE) + sev = request.request + if !(sev == 0x1 || sev == 0x3 || sev == 0x4) + throw(ArgumentError("invalid cancellation severity $(repr(request.request))")) + end + raised = _raise_state!(src, sev) + # Mark the cancelled subgraph (waking parked waiters): each node is + # marked before its children so a concurrent construction of a child + # source is level-triggered. + _cancel_walk!(src, sev) + Threads.atomic_fence_heavy() + # Shoot down any task now bound to a cancelled source: threads inside a + # compiled cancellation region are asynchronously reset to their + # cancellation point, which observes the cancellation and throws. + # Best-effort: a task the walk misses recovers level-triggered at its + # next cancellation point (or reset-safe allocation). + ccall(:jl_shootdown_cancelled_tasks, Cvoid, ()) + return raised +end + +""" + Base.redeliver!(src::CancellationTokenSource) -> Bool + +Re-run the delivery pass for an already-cancelled source at its current +severity: wakes waiters that registered without observing the cancellation +and re-sends the interruption signal to bound running computations (the +signal-based delivery is best-effort and can be missed while a reset point +is unpublished). Used by the ^C machinery when a press finds the episode +source already marked cancelled (e.g. by the C-side fast path in +`jl_sigint_request_cancellation`, which cannot wake parked waiters itself). +Returns whether the source was cancelled at all. +""" +function redeliver!(src::CancellationTokenSource) + st = @atomic :acquire src.state + st == 0x00 && return false + _cancel_walk!(src, st) + Threads.atomic_fence_heavy() + ccall(:jl_shootdown_cancelled_tasks, Cvoid, ()) + return true +end + +function _cancel_walk!(src::CancellationTokenSource, sev::UInt8) + # Iterative worklist (no recursion): a deep source chain must not + # overflow the canceller's stack, and a reconverging ("linked") graph + # must deliver at each node once, not once per path (deduplicated by the + # visited set). + visited = IdSet{CancellationTokenSource}() + push!(visited, src) + pending = CancellationTokenSource[src] + while !isempty(pending) + _cancel_walk_node!(pop!(pending), sev, pending, visited) + end + return nothing +end + +# Walk `node`'s waiter list under the walk lock: physically unlink the +# registrations of completed tasks and retired entries (the only removal in +# the registry - a live task's registration is sticky and stays linked +# between parks) and, when `sev` is nonzero, claim armed waiters eligible +# at that severity. Returns the claimed tasks as a cons list of +# `Pair{Tuple{Task, WaitEntry, UInt64}, Any}` cells (a Pair's declared +# parameters make every cell one type - a tuple cons would mint a fresh +# concrete tuple type per list depth); the caller wakes them after +# releasing the walk lock. +function _walk_waiters!(node::CancellationTokenSource, sev::UInt8) + @atomic :monotonic node.dead_count = UInt32(0) + nlive = UInt32(0) + towake = nothing + prev = nothing # the predecessor's slot for `node`, once past the head + # seq_cst: the S-ordered counterpart of a registrant's seq_cst push - + # a push this read misses is later in the total order, so that + # registrant's state recheck observes the cancellation (see + # `SourceWait`'s registration, base/park.jl); pairs with the seq_cst + # state read the caller performed under the walk lock. + x = @atomic :sequentially_consistent node.waiters_head + w = x isa WaitEntry ? x : nothing + while w isa WaitEntry + wi = _find_slot(w, node) + # a linked entry always has a slot for this source (its owner is + # the membership witness, cleared only on unlink, under this lock); + # bail out without touching the structure if that is ever violated + wi == 0 && break + slot = slots(w)[wi] + wnext = slot.next + t = @atomic :monotonic w.task + if t === nothing || istaskdone(t) + # Unlink. Interior links are rewritten only under the walk lock, + # so they are plain stores; unlinking the head races concurrent + # pushes and simply keeps the entry for the next walk when it + # loses. + unlinked = if prev === nothing + (@atomicreplace :monotonic :monotonic node.waiters_head w => wnext).success + else + prev.next = wnext + true + end + if unlinked + # The slot `owner` is cleared last: it is the membership + # witness whose clearing hands the (dead) slot back. + slot.next = nothing + t === nothing || (@atomic :monotonic w.task = nothing) + slot.owner = nothing + else + prev = slot + end + else + # Claim order: the armed check must come first - the slot's aux + # (the minimum delivery severity) is only meaningful for the arm + # it was staged for, and reading it after observing the arm pins + # it to that arm or a later one. An arm this (seq_cst, pairing + # with the source registration's re-arm fence) read misses + # is covered by its own post-arm state recheck, so skipping it + # here is sound. The claim CAS itself may still land on a + # *later* arm of the same entry than the one whose aux was + # judged - which is why every arm of a source-linked entry must + # be a cancellable park under it (shields arm a distinct + # entry; see _cached_wait_entry): the worst misfire is then a + # spurious below-floor wake into a teardown re-park, which + # handles it like any interruption of its wait + # (conservatively, e.g. by detaching the awaited request). + aux = slot.aux + if sev != 0x00 && (@atomic :sequentially_consistent t.waiting_on) === w && + aux % UInt8 <= sev + if (@atomicreplace t.waiting_on w => nothing).success + towake = Pair{Tuple{Task, WaitEntry, UInt64}, Any}((t, w, aux), towake) + end + # a lost claim: a completion or interrupter won the race; + # the waiter resumes through that wake + end + nlive += UInt32(1) + prev = slot + end + w = wnext + end + # Resync the approximate list length (see _note_dead_registration!'s + # scaled prune threshold) to what this walk left linked. Entries kept + # only because their head-unlink CAS lost count as live: conservative, + # and the next walk resyncs. + @atomic :monotonic node.reg_count = nlive + return towake +end + +function _cancel_walk_node!(node::CancellationTokenSource, sev::UInt8, + pending::Vector{CancellationTokenSource}, + visited::IdSet{CancellationTokenSource}) + _lock_walk(node) + # Deliver at least the node's current severity: a concurrent higher- + # severity cancel! may have raised the state after this walk's own + # transition, and its walk can run before this one takes the walk lock - + # the claims below must then honor the escalated request. (Read under + # the walk lock so the claim section cannot act on a stale, lower + # severity.) seq_cst, matching the C-side propagate: when this walk's + # own raise of the node was lost, this load is the walk's sole operation + # on the state that can order the winner's write before the child_head + # read below; it also pairs with the seq_cst push/arm-fence in + # the source registration (state write before walk on this side, push + # or arm before state recheck on the registrant's). + st = @atomic :sequentially_consistent node.state + sev < st && (sev = st) + creq = CancellationRequest(sev) + towake = _walk_waiters!(node, sev) + # The actual wakes happen after the walk lock is released: holding it + # across `schedule` would only stretch the window against concurrent + # escalation walks. Claimed entries stay registered (sticky); the + # claimed *waitee*-queue entry also stays linked, and the waiter's own + # cleanup (or a later notify) lazily unlinks it. + _unlock_walk(node) + while towake !== nothing + towake = towake::Pair{Tuple{Task, WaitEntry, UInt64}, Any} + (t, w, aux) = towake.first + towake = towake.second + if aux & WAIT_AUX_WATCHER_BIT != 0x00 + # A watcher (`wait(::CancellationToken)`): this cancellation is + # the event its wait completes on, so it is woken with the + # request as a *value* - a watcher observes this source but + # does not run under it. + deliver_claimed_value_wake!(t, w, creq) + continue + end + # N.B.: an ABANDON_ALL request also delivers by interruption for + # now (freezing the task in place is not yet implemented). The + # delivery is claim-scoped - the claim above is the wake ticket, so + # no re-claiming swap - and dropped when the task has re-armed + # meanwhile (see deliver_claimed_wake!). + deliver_claimed_wake!(t, w, creq) + end + # Walk the node's (weak, intrusive) child list, advancing every child to + # this severity and queueing the ones not yet visited (a reconverging + # graph must deliver at each node once, not once per path). The seq_cst + # `child_head` read below (paired with the seq_cst state access above) + # closes the race against a concurrent attach: a child that this read + # misses was published after our state write, so its constructor + # observes that write and the child is born at (at least) this severity. + # Children attached concurrently *during* the walk are prepended before + # the list positions already traversed and are likewise born cancelled. + c = @atomic node.child_head + while c !== nothing + c = c::CancellationTokenSource + _raise_state!(c, sev) + if !(c in visited) + push!(visited, c) + push!(pending, c) + end + c = _cancel_next_child(node, c) + end + return nothing +end + +## Cancellation points + +# The slow path of `@cancel_check`: `st` is the (non-zero) status byte +# reported by the cancellation point. +@noinline function handle_cancellation!(src::Union{Nothing, CancellationTokenSource}, st::UInt8) + ct = current_task() + if st & STATUS_PREEMPT_BIT != 0x00 + # consume the cooperative-yield request + @atomic :monotonic ct.preempt_request = 0x00 + end + if st & SEVERITY_MASK == 0x00 + # preempt-only (a pending yield request, or a preempt shootdown that + # reset this point): let another task run, then resume + yield() + return nothing + end + src = src::CancellationTokenSource + # re-read: deliver the severity current at throw time, not the one the + # fast path happened to observe + st = @atomic :acquire src.state + throw(CancellationRequest(st)) +end + +""" + Core.cancellation_point!(src::Union{Nothing, Core.CancellationTokenSource})::UInt8 + +Check the cancellation state of `src` (see [`@cancel_check`](@ref)), +additionally giving the optimizer license to establish this point as a +cancellation reset point: when compiled, the source is published as the token +binding governing the current computation, and the runtime may asynchronously +unwind execution to the nearest preceding cancellation point when the source +is cancelled. Returns a status byte: `0x00` if nothing is pending, the +(nonzero) severity if `src` is cancelled, with the `0x40` bit set if a +cooperative yield (preemption) was requested. + +!!! warning + `src` must remain otherwise reachable (e.g. through the scope binding + that supplied it, as [`@cancel_check`](@ref) guarantees) for the dynamic + extent of the region this point establishes: copies of the binding saved + by exception handlers are not GC-scanned. +""" +Core.cancellation_point! + +""" + @cancel_check + @cancel_check token + +Explicit cancellation point: checks whether the cancellation token governing +the current computation has been cancelled and, if so, throws the +corresponding [`CancellationRequest`](@ref). Long-running computational code +should place these in its hot loops so that it can be cancelled. + +The one-argument form checks an explicitly provided +`Union{Nothing, CancellationToken}` instead of resolving the scoped default +token; use it to hoist the token lookup out of a tight loop. +""" +macro cancel_check() + quote + # compiled cancellation points are also GC safepoints, so that a tight + # polling loop cannot starve a concurrent stop-the-world. + # The loop re-executes the point whenever the slow path returns (a + # preempt-only status yields and resumes): the slow-path call tears + # down the point's reset region, so passing the point again is what + # re-establishes it before the code the region is meant to cover. + local s = default_cancel_source() + while true + local st = Core.cancellation_point!(s)::UInt8 + st == 0x00 && break + handle_cancellation!(s, st) + end + nothing + end +end + +macro cancel_check(tok) + quote + local t = $(esc(tok)) + local s = t === nothing ? nothing : (t::CancellationToken).source + while true + local st = Core.cancellation_point!(s)::UInt8 + st == 0x00 && break + handle_cancellation!(s, st) + end + nothing + end +end + +# Throw the `CancellationRequest` if `src` is cancelled (level-triggered: +# no per-task state is consulted). Unlike `@cancel_check` this is not a +# compiled cancellation point (it opens no async-interruptible region). +# This is the entry check of every blocking API taking a `cancel` keyword +# argument: it must run *before* the operation has any side effects. +@inline function checkcancel(src::CancellationTokenSource) + st = @atomic :monotonic src.state + st == 0x00 && return nothing + handle_cancellation!(src, st) + return nothing +end +checkcancel(::Nothing) = nothing +checkcancel(tok::CancellationToken) = checkcancel(tok.source) + +## CANCEL_TOKEN +struct CancelTokenKey <: AbstractScopedValue{Union{Nothing, CancellationToken}} end + +""" + CANCEL_TOKEN + +The scoped value carrying the [`CancellationToken`](@ref) that governs the +current dynamic extent, or `nothing` if there is none. [`@cancel_check`](@ref) +checks it, and tasks spawned within a scope inherit it. + +Establish a governing token with the standard scoped-value API +([`ScopedValues.@with`](@ref) / [`ScopedValues.with`](@ref)): + +```julia +using Base.ScopedValues + +src = Base.CancellationTokenSource() +with(Base.CANCEL_TOKEN => Base.CancellationToken(src)) do + ... # cancellation points in here observe `cancel!(src)` +end +``` + +Scoping `Base.CANCEL_TOKEN => nothing` instead *shields* the enclosed code +from an outer (possibly cancelled) token; use this for cleanup that must +complete while the surrounding computation is being cancelled. + +The current value can be read with `Base.CANCEL_TOKEN[]`. +""" +const CANCEL_TOKEN = CancelTokenKey() + +# The cancellation source a task inherited at birth: the CANCEL_TOKEN +# binding of the scope captured at its construction (`nothing` for tasks +# constructed outside any governing scope, or shielded with +# `CANCEL_TOKEN => nothing`). Subscriptions (`schedule_on_notify!`) are +# governed by this source: a task whose birth source is cancelled dies +# instead of starting. +function _birth_cancel_source(t::Task) + # Raw field read: the `t.scope` property guard exists because a + # *running* task swaps its scope as `with` blocks enter and exit - + # racy to read from outside. Subscriptions enforce a never-started + # waiter (see schedule_on_notify!), whose construction-time scope is + # stable until its first schedule. + scope = getfield(t, :scope) + scope isa Scope || return nothing + v = KeyValue.get(scope.values, CANCEL_TOKEN) + v === nothing && return nothing + tok = something(v) + tok === nothing && return nothing + return (tok::CancellationToken).source +end + +# The scoped-default resolution, with a per-task cache: when the task's +# `bound_cancel_default` flag is set, `bound_cancel_token` holds the source +# this lookup would resolve to under the task's current scope (or `nothing`), +# placed there by the slow path below. The flag is dropped wherever a scope +# is installed (`with` enter and its inline exit) and by cancellation points +# publishing a different source, and travels with the token through the +# exception-handler and finalizer save/restore brackets, so a set flag +# always describes the current scope (see bound_cancel_default in +# src/julia_threads.h). Reconstructing the token from the cached source is +# exact: `CancellationToken` is an immutable wrapper, so the copy is egal to +# the token in the scope. +@inline function default_cancel_token() + ct = current_task() + if getfield(ct, :bound_cancel_default) !== 0x00 + # the field's declared type (Union{Nothing, CancellationTokenSource}) + # narrows through the `=== nothing` check without a type-tag load + s = @atomic :monotonic ct.bound_cancel_token + s === nothing && return nothing + return CancellationToken(s) + end + return _default_cancel_token_slow(ct) +end + +@noinline function _default_cancel_token_slow(ct::Task) + tok = nothing + scope = Core.current_scope()::Union{Scope, Nothing} + if scope !== nothing + v = KeyValue.get(scope.values, CANCEL_TOKEN) + if v !== nothing + tok = something(v)::Union{Nothing, CancellationToken} + end + end + # Cache the resolution for the current scope. The store is an untagged + # unsafe point for CancellationLowering, so it cannot break a published + # reset region's (region, token) coherence: any live region is torn down + # before the store and re-established at the next cancellation point. + @atomic :monotonic ct.bound_cancel_token = tok === nothing ? nothing : tok.source + setfield!(ct, :bound_cancel_default, 0x01) + return tok +end + +@inline function default_cancel_source() + tok = default_cancel_token() + tok === nothing && return nothing + return (tok::CancellationToken).source +end + +## `cancel` keyword-argument plumbing + +# The sentinel default for `cancel` keyword arguments: "use the scoped +# default token". Resolution to a concrete token happens once, at the first +# potential-block point of an operation, so fast paths never pay for the +# scope lookup. `cancel = nothing` makes a wait explicitly non-cancellable. +# +# N.B.: a resolved token (`Union{Nothing, CancellationToken}`) is passed +# through *positional* arguments internally: passing the union as a keyword +# argument builds an abstractly-typed NamedTuple whose kwcall the optimizer +# cannot devirtualize (which, among other things, breaks `juliac --trim`). +struct UseDefaultToken end +const DEFAULT_CANCEL = UseDefaultToken() +const CancelTokenArg = Union{UseDefaultToken, CancellationToken, Nothing} +const MaybeToken = Union{Nothing, CancellationToken} + +@inline resolve_cancel_token(::UseDefaultToken) = default_cancel_token() +@inline resolve_cancel_token(tok::Union{CancellationToken, Nothing}) = tok + +# The source of a resolved token (`nothing` stays `nothing`). +cancel_source(tok::CancellationToken) = tok.source +cancel_source(::Nothing) = nothing + +# The entry check of a public API taking a `cancel` keyword argument: +# resolve the token and throw if it is already cancelled (uniformly +# level-triggered for the scoped default and explicit tokens alike). +@inline function check_cancel_arg(cancel::CancelTokenArg) + tok = resolve_cancel_token(cancel) + tok === nothing || checkcancel(tok.source) + return tok +end + +# The lighter entry check for APIs with a non-blocking fast path: an +# explicitly passed token is resolved and checked (throwing when already +# cancelled) right here, while the scoped-default sentinel passes through +# untouched, deferring the scope lookup - and its (level-triggered) check - +# to the operation's first potential-block point, so fast paths pay neither. +# Spelled `cancel = precheck_cancel_arg(cancel)` at the top of such APIs. +@inline precheck_cancel_arg(cancel::CancelTokenArg) = + cancel isa UseDefaultToken ? cancel : check_cancel_arg(cancel) diff --git a/base/cartesian.jl b/base/cartesian.jl index ca0fc0aac0cfc..a04a2040021ce 100644 --- a/base/cartesian.jl +++ b/base/cartesian.jl @@ -36,33 +36,34 @@ If you want just a post-expression, supply [`nothing`](@ref) for the pre-express parentheses and semicolons, you can supply multi-statement expressions. """ macro nloops(N, itersym, rangeexpr, args...) - _nloops(N, itersym, rangeexpr, args...) + _nloops(N, itersym, true, rangeexpr, args...) end -function _nloops(N::Int, itersym::Symbol, arraysym::Symbol, args::Expr...) - @gensym d - _nloops(N, itersym, :($d->Base.axes($arraysym, $d)), args...) +function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, arraysym::Symbol, args::Expr...) + _nloops(N, itersym, false, :(d->axes($(esc(arraysym)), d)), args...) end -function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...) +function _nloops(N::Int, itersym::Symbol, esc_rng::Bool, rangeexpr::Expr, args::Expr...) if rangeexpr.head !== :-> throw(ArgumentError("second argument must be an anonymous function expression to compute the range")) end if !(1 <= length(args) <= 3) - throw(ArgumentError("number of arguments must be 1 ≤ length(args) ≤ 3, got $nargs")) + throw(ArgumentError("number of arguments must be 1 ≤ length(args) ≤ 3, got $(length(args))")) end body = args[end] ex = Expr(:escape, body) for dim = 1:N itervar = inlineanonymous(itersym, dim) + itervar = esc(itervar) rng = inlineanonymous(rangeexpr, dim) - preexpr = length(args) > 1 ? inlineanonymous(args[1], dim) : (:(nothing)) - postexpr = length(args) > 2 ? inlineanonymous(args[2], dim) : (:(nothing)) + esc_rng && (rng = esc(rng)) + preexpr = length(args) > 1 ? esc(inlineanonymous(args[1], dim)) : nothing + postexpr = length(args) > 2 ? esc(inlineanonymous(args[2], dim)) : nothing ex = quote - for $(esc(itervar)) = $(esc(rng)) - $(esc(preexpr)) + for $itervar = $rng + $preexpr $ex - $(esc(postexpr)) + $postexpr end end end @@ -235,7 +236,7 @@ end Generates a sequence of `if ... elseif ... else ... end` statements. For example: - @nif 3 d->(i_d >= size(A,d)) d->(error("Dimension ", d, " too big")) d->println("All OK") + @nif 3 d->(i_d > size(A,d)) d->(error("Dimension ", d, " too big")) d->println("All OK") would generate: @@ -290,14 +291,15 @@ struct LReplace{S<:AbstractString} end LReplace(sym::Symbol, val::Integer) = LReplace(sym, string(sym), val) -lreplace(ex::Expr, sym::Symbol, val) = lreplace!(copy(ex), LReplace(sym, val)) +lreplace(ex::Expr, sym::Symbol, val) = lreplace!(copy(ex), LReplace(sym, val), false, 0) -function lreplace!(sym::Symbol, r::LReplace) +function lreplace!(sym::Symbol, r::LReplace, in_quote_context::Bool, escs::Int) + escs == 0 || return sym sym == r.pat_sym && return r.val - Symbol(lreplace!(string(sym), r)) + Symbol(lreplace_string!(string(sym), r)) end -function lreplace!(str::AbstractString, r::LReplace) +function lreplace_string!(str::String, r::LReplace) i = firstindex(str) pat = r.pat_str j = firstindex(pat) @@ -329,7 +331,7 @@ function lreplace!(str::AbstractString, r::LReplace) if matching && j > lastindex(pat) if i > lastindex(str) || str[i] == '_' # We have a match - return string(str[1:prevind(str, istart)], r.val, lreplace!(str[i:end], r)) + return string(str[1:prevind(str, istart)], r.val, lreplace_string!(str[i:end], r)) end matching = false j = firstindex(pat) @@ -339,24 +341,42 @@ function lreplace!(str::AbstractString, r::LReplace) str end -function lreplace!(ex::Expr, r::LReplace) +function lreplace!(ex::Expr, r::LReplace, in_quote_context::Bool, escs::Int) # Curly-brace notation, which acts like parentheses - if ex.head === :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]::Symbol), "_") - excurly = exprresolve(lreplace!(ex.args[2], r)) + if !in_quote_context && ex.head === :curly && length(ex.args) == 2 && isa(ex.args[1], Symbol) && endswith(string(ex.args[1]::Symbol), "_") + excurly = exprresolve(lreplace!(ex.args[2], r, in_quote_context, escs)) if isa(excurly, Int) return Symbol(ex.args[1]::Symbol, excurly) else ex.args[2] = excurly return ex end + elseif ex.head === :meta || ex.head === :inert + return ex + elseif ex.head === :$ + # no longer an executable expression (handle all equivalent forms of :inert, :quote, and QuoteNode the same way) + in_quote_context = false + elseif ex.head === :quote + # executable again + in_quote_context = true + elseif ex.head === :var"hygienic-scope" + # no longer our expression + escs += 1 + elseif ex.head === :escape + # our expression again once zero + escs == 0 && return ex + escs -= 1 + elseif ex.head === :macrocall + # n.b. blithely go about altering arguments to macros also, assuming that is at all what the user intended + # it is probably the user's fault if they put a macro inside here and didn't mean for it to get rewritten end for i in 1:length(ex.args) - ex.args[i] = lreplace!(ex.args[i], r) + ex.args[i] = lreplace!(ex.args[i], r, in_quote_context, escs) end ex end -lreplace!(arg, r::LReplace) = arg +lreplace!(@nospecialize(arg), r::LReplace, in_quote_context::Bool, escs::Int) = arg poplinenum(arg) = arg @@ -375,9 +395,9 @@ end ## Resolve expressions at parsing time ## -const exprresolve_arith_dict = Dict{Symbol,Function}(:+ => +, +const exprresolve_arith_dict = IdDict{Symbol,Function}(:+ => +, :- => -, :* => *, :/ => /, :^ => ^, :div => div) -const exprresolve_cond_dict = Dict{Symbol,Function}(:(==) => ==, +const exprresolve_cond_dict = IdDict{Symbol,Function}(:(==) => ==, :(<) => <, :(>) => >, :(<=) => <=, :(>=) => >=) function exprresolve_arith(ex::Expr) diff --git a/base/channels.jl b/base/channels.jl index 0bb73e9acba87..81c9ded6849d4 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -13,7 +13,7 @@ popfirst!(c::AbstractChannel) = take!(c) """ Channel{T=Any}(size::Int=0) -Constructs a `Channel` with an internal buffer that can hold a maximum of `size` objects +Construct a `Channel` with an internal buffer that can hold a maximum of `size` objects of type `T`. [`put!`](@ref) calls on a full channel block until an object is removed with [`take!`](@ref). @@ -213,13 +213,14 @@ end """ isopen(c::Channel) -Determines whether a [`Channel`](@ref) is open for new [`put!`](@ref) operations. -Notice that a `Channel`` can be closed and still have -buffered elements which can be consumed with [`take!`](@ref). + +Determine whether a [`Channel`](@ref) is open for new [`put!`](@ref) operations. +Notice that a `Channel` can be closed and still have buffered elements which can be +consumed with [`take!`](@ref). # Examples -Buffered channel with task: +## Buffered channel with task ```jldoctest julia> c = Channel(ch -> put!(ch, 1), 1); @@ -236,7 +237,7 @@ julia> isready(c) false ``` -Unbuffered channel: +## Unbuffered channel ```jldoctest julia> c = Channel{Int}(); @@ -326,9 +327,14 @@ Stacktrace: ``` """ function bind(c::Channel, task::Task) - T = Task(() -> close_chnl_on_taskdone(task, c)) + # the close hook is cleanup: shield it from the constructing scope's + # cancellation, so a bound channel is closed (and its blocked users + # released) even when the scope that bound it is cancelled + T = ScopedValues.with(CANCEL_TOKEN => nothing) do + Task(() -> close_chnl_on_taskdone(task, c)) + end T.sticky = false - _wait2(task, T) + schedule_on_notify!(task, T) return c end @@ -344,8 +350,8 @@ of type `Channel{Any}(0)`. Returns a tuple, `(Array{Channel}, Array{Task})`, of the created channels and tasks. """ function channeled_tasks(n::Int, funcs...; ctypes=fill(Any,n), csizes=fill(0,n)) - @assert length(csizes) == n - @assert length(ctypes) == n + @assert length(csizes) == n "length(csizes) != n" + @assert length(ctypes) == n "length(ctypes) != n" chnls = map(i -> Channel{ctypes[i]}(csizes[i]), 1:n) tasks = Task[ Task(() -> f(chnls...)) for f in funcs ] @@ -391,10 +397,11 @@ task. !!! compat "Julia 1.1" `v` now gets converted to the channel's type with [`convert`](@ref) as `put!` is called. """ -function put!(c::Channel{T}, v) where T +function put!(c::Channel{T}, v; cancel::CancelTokenArg=DEFAULT_CANCEL) where T check_channel_state(c) v = convert(T, v) - return isbuffered(c) ? put_buffered(c, v) : put_unbuffered(c, v) + cancel = precheck_cancel_arg(cancel) + return isbuffered(c) ? put_buffered(c, v, cancel) : put_unbuffered(c, v, cancel) end # Atomically update channel n_avail, *assuming* we hold the channel lock. @@ -407,16 +414,22 @@ function _increment_n_avail(c, inc) @atomic :monotonic c.n_avail_items = newlen end -function put_buffered(c::Channel, v) - lock(c) +function put_buffered(c::Channel, v, cancel::CancelTokenArg=DEFAULT_CANCEL) + lock(c; cancel) + locked = true did_buffer = false try # Increment channel n_avail eagerly (before push!) to count data in the # buffer as well as offers from tasks which are blocked in wait(). _increment_n_avail(c, 1) - while length(c.data) == c.sz_max - check_channel_state(c) - wait(c.cond_put) + if length(c.data) == c.sz_max + tok = resolve_cancel_token(cancel) + while length(c.data) == c.sz_max + check_channel_state(c) + locked = false + wait(c.cond_put, tok) + locked = true + end end check_channel_state(c) push!(c.data, v) @@ -424,27 +437,47 @@ function put_buffered(c::Channel, v) # notify all, since some of the waiters may be on a "fetch" call. notify(c.cond_take, nothing, true, false) finally - # Decrement the available items if this task had an exception before pushing the - # item to the buffer (e.g., during `wait(c.cond_put)`): + # Decrement the available items if this task had an exception before + # pushing the item to the buffer (e.g., during `wait(c.cond_put)`). + # The fixup needs the lock: reacquire (shielded) when a wait-throw + # released this frame's level. + locked || lock(c; cancel=nothing) did_buffer || _increment_n_avail(c, -1) unlock(c) end return v end -function put_unbuffered(c::Channel, v) - lock(c) +function put_unbuffered(c::Channel, v, cancel::CancelTokenArg=DEFAULT_CANCEL) + lock(c; cancel) + locked = true taker = try _increment_n_avail(c, 1) - while isempty(c.cond_take.waitq) + tok = resolve_cancel_token(cancel) + local taker + while true + while isempty(c.cond_take.waitq) + check_channel_state(c) + notify(c.cond_wait) + locked = false + wait(c.cond_put, tok) + locked = true + end check_channel_state(c) - notify(c.cond_wait) - wait(c.cond_put) + # unfair scheduled version of: notify(c.cond_take, v, false, false); yield() + w = popfirst!(waitqueue(c.cond_take)) + t = @atomic :monotonic w.task + if t isa Task && claim_wait(t, w) + taker = t + break + end + # stale registration: this waiter's wake was already claimed + # by an interrupter - drop it and look for the next taker end - check_channel_state(c) - # unfair scheduled version of: notify(c.cond_take, v, false, false); yield() - popfirst!(c.cond_take.waitq) + taker finally + # the decrement needs the lock (see put_buffered's finally) + locked || lock(c; cancel=nothing) _increment_n_avail(c, -1) unlock(c) end @@ -461,7 +494,7 @@ Note: `fetch` is unsupported on an unbuffered (0-size) `Channel`. # Examples -Buffered channel: +## Buffered channel ```jldoctest julia> c = Channel(3) do ch foreach(i -> put!(ch, i), 1:3) @@ -477,22 +510,30 @@ julia> collect(c) # item is not removed 3 ``` """ -fetch(c::Channel) = isbuffered(c) ? fetch_buffered(c) : fetch_unbuffered(c) -function fetch_buffered(c::Channel) - lock(c) +function fetch(c::Channel; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + return isbuffered(c) ? fetch_buffered(c, cancel) : fetch_unbuffered(c) +end +function fetch_buffered(c::Channel, cancel::CancelTokenArg=DEFAULT_CANCEL) + lock(c; cancel) + locked = true try - while isempty(c.data) - check_channel_state(c) - wait(c.cond_take) + if isempty(c.data) + tok = resolve_cancel_token(cancel) + while isempty(c.data) + check_channel_state(c) + locked = false + wait(c.cond_take, tok) + locked = true + end end return c.data[1] finally - unlock(c) + locked && unlock(c) end end fetch_unbuffered(c::Channel) = throw(ErrorException("`fetch` is not supported on an unbuffered Channel.")) - """ take!(c::Channel) @@ -501,7 +542,7 @@ For unbuffered channels, blocks until a [`put!`](@ref) is performed by a differe # Examples -Buffered channel: +## Buffered channel ```jldoctest julia> c = Channel(1); @@ -511,7 +552,7 @@ julia> take!(c) 1 ``` -Unbuffered channel: +## Unbuffered channel ```jldoctest julia> c = Channel(0); @@ -523,46 +564,59 @@ julia> take!(c) 1 ``` """ -take!(c::Channel) = isbuffered(c) ? take_buffered(c) : take_unbuffered(c) -function take_buffered(c::Channel) - lock(c) +function take!(c::Channel; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + return isbuffered(c) ? take_buffered(c, cancel) : take_unbuffered(c, cancel) +end +function take_buffered(c::Channel, cancel::CancelTokenArg=DEFAULT_CANCEL) + lock(c; cancel) + locked = true try - while isempty(c.data) - check_channel_state(c) - wait(c.cond_take) + if isempty(c.data) + tok = resolve_cancel_token(cancel) + while isempty(c.data) + check_channel_state(c) + locked = false + wait(c.cond_take, tok) + locked = true + end end v = popfirst!(c.data) _increment_n_avail(c, -1) notify(c.cond_put, nothing, false, false) # notify only one, since only one slot has become available for a put!. return v finally - unlock(c) + locked && unlock(c) end end # 0-size channel -function take_unbuffered(c::Channel{T}) where T - lock(c) +function take_unbuffered(c::Channel{T}, cancel::CancelTokenArg=DEFAULT_CANCEL) where T + lock(c; cancel) + locked = true try check_channel_state(c) notify(c.cond_put, nothing, false, false) - return wait(c.cond_take)::T + locked = false + v = wait(c.cond_take, resolve_cancel_token(cancel))::T + locked = true + return v finally - unlock(c) + locked && unlock(c) end end """ isready(c::Channel) -Determines whether a [`Channel`](@ref) has a value stored in it. +Determine whether a [`Channel`](@ref) has a value stored in it. Returns immediately, does not block. For unbuffered channels, return `true` if there are tasks waiting on a [`put!`](@ref). # Examples -Buffered channel: +## Buffered channel ```jldoctest julia> c = Channel(1); @@ -575,7 +629,7 @@ julia> isready(c) true ``` -Unbuffered channel: +## Unbuffered channel ```jldoctest julia> c = Channel(); @@ -589,7 +643,6 @@ julia> schedule(task); # schedule a put! task julia> isready(c) true ``` - """ isready(c::Channel) = n_avail(c) > 0 isempty(c::Channel) = n_avail(c) == 0 @@ -601,7 +654,7 @@ end """ isfull(c::Channel) -Determines if a [`Channel`](@ref) is full, in the sense +Determine if a [`Channel`](@ref) is full, in the sense that calling `put!(c, some_value)` would have blocked. Returns immediately, does not block. @@ -616,7 +669,7 @@ tasks calling `put!` in parallel. # Examples -Buffered channel: +## Buffered channel ```jldoctest julia> c = Channel(1); # capacity = 1 @@ -629,7 +682,7 @@ julia> isfull(c) true ``` -Unbuffered channel: +## Unbuffered channel ```jldoctest julia> c = Channel(); # capacity = 0 @@ -639,7 +692,10 @@ true """ isfull(c::Channel) = n_avail(c) ≥ c.sz_max -lock(c::Channel) = lock(c.cond_take) +# the `cancel` forward makes the channel operations' preliminary lock +# acquisitions honor their resolved token (or explicit shield); see +# lock(::GenericCondition{ReentrantLock}) +lock(c::Channel; cancel::CancelTokenArg=DEFAULT_CANCEL) = lock(c.cond_take; cancel) lock(f, c::Channel) = lock(f, c.cond_take) unlock(c::Channel) = unlock(c.cond_take) trylock(c::Channel) = trylock(c.cond_take) @@ -647,7 +703,7 @@ trylock(c::Channel) = trylock(c.cond_take) """ wait(c::Channel) -Blocks until the `Channel` [`isready`](@ref). +Block until the `Channel` [`isready`](@ref). ```jldoctest julia> c = Channel(1); @@ -668,16 +724,23 @@ julia> istaskdone(task) # task is now unblocked true ``` """ -function wait(c::Channel) +function wait(c::Channel; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) isready(c) && return - lock(c) + lock(c; cancel) + locked = true try - while !isready(c) - check_channel_state(c) - wait(c.cond_wait) + if !isready(c) + tok = resolve_cancel_token(cancel) + while !isready(c) + check_channel_state(c) + locked = false + wait(c.cond_wait, tok) + locked = true + end end finally - unlock(c) + locked && unlock(c) end nothing end diff --git a/base/char.jl b/base/char.jl index 90636a6d9536e..34b55a7a706db 100644 --- a/base/char.jl +++ b/base/char.jl @@ -4,18 +4,22 @@ import Core: AbstractChar, Char """ The `AbstractChar` type is the supertype of all character implementations -in Julia. A character represents a Unicode code point, and can be converted -to an integer via the [`codepoint`](@ref) function in order to obtain the -numerical value of the code point, or constructed from the same integer. -These numerical values determine how characters are compared with `<` and `==`, -for example. New `T <: AbstractChar` types should define a `codepoint(::T)` +in Julia. A character normally represents a Unicode codepoint (and can +also encapsulate other information from an encoded byte sequence as described below), +and characters can be converted to integer codepoint values via the [`codepoint`](@ref) +function, or can be constructed from the same integer. At least for valid, +properly encoded Unicode characters, these numerical codepoint values +determine how characters are compared with `<` and `==`, for example. +New `T <: AbstractChar` types should define a `codepoint(::T)` method and a `T(::UInt32)` constructor, at minimum. A given `AbstractChar` subtype may be capable of representing only a subset of Unicode, in which case conversion from an unsupported `UInt32` value may throw an error. Conversely, the built-in [`Char`](@ref) type represents a *superset* of Unicode (in order to losslessly encode invalid byte streams), -in which case conversion of a non-Unicode value *to* `UInt32` throws an error. +in which case conversion of a non-Unicode value *to* `UInt32` throws an error +(see [`Base.ismalformed`](@ref)), and on the other hand a `Char` can also represent +a nonstandard "overlong" encoding ([`Base.isoverlong`](@ref)) of a codepoint. The [`isvalid`](@ref) function can be used to check which codepoints are representable in a given `AbstractChar` type. @@ -47,7 +51,7 @@ represents a valid Unicode character. """ Char -@constprop :aggressive (::Type{T})(x::Number) where {T<:AbstractChar} = T(UInt32(x)) +@constprop :aggressive (::Type{T})(x::Number) where {T<:AbstractChar} = T(UInt32(x)::UInt32) @constprop :aggressive AbstractChar(x::Number) = Char(x) @constprop :aggressive (::Type{T})(x::AbstractChar) where {T<:Union{Number,AbstractChar}} = T(codepoint(x)) @constprop :aggressive (::Type{T})(x::AbstractChar) where {T<:Union{Int32,Int64}} = codepoint(x) % T @@ -77,10 +81,19 @@ end codepoint(c::AbstractChar)::Integer Return the Unicode codepoint (an unsigned integer) corresponding -to the character `c` (or throw an exception if `c` does not represent -a valid character). For `Char`, this is a `UInt32` value, but +to the character `c` (or throw an exception if `c` represents +a malformed character). For `Char`, this is a `UInt32` value, but `AbstractChar` types that represent only a subset of Unicode may return a different-sized integer (e.g. `UInt8`). + +Should succeed for any non-malformed character, i.e. when +[`Base.ismalformed(c)`](@ref) returns `false`. This includes +invalid Unicode characters (such as unpaired surrogates) +and overlong encodings. + +!!! compat "Julia 1.12" + Prior to Julia 1.12, `codepoint(c)` fails for overlong encodings (when + [`Base.isoverlong(c)`](@ref) is `true`), and `Base.decode_overlong(c)` was needed. """ function codepoint end @@ -116,10 +129,19 @@ end """ ismalformed(c::AbstractChar)::Bool -Return `true` if `c` represents malformed (non-Unicode) data according to the +Return `true` if `c` represents malformed (non-codepoint / mis-encoded) data according to the encoding used by `c`. Defaults to `false` for non-`Char` types. -See also [`show_invalid`](@ref). +Any *non*-malformed `c` can be mapped to an integer codepoint +by [`codepoint(c)`](@ref); this includes codepoints that are +not valid Unicode characters ([`isvalid(c)`](@ref) is `false`). +For example, well-formed characters can include invalid Unicode +codepoints like `'\\U110000'`, unpaired surrogates such as `'\\ud800'`, +and can also include overlong encodings ([`Base.isoverlong`](@ref)). +Malformed data, in contrast, cannot be decoded to a codepoint +(`codepoint` will throw an exception). + +See also [`Base.show_invalid`](@ref). """ ismalformed(c::AbstractChar) = false @@ -129,7 +151,7 @@ ismalformed(c::AbstractChar) = false Return `true` if `c` represents an overlong UTF-8 sequence. Defaults to `false` for non-`Char` types. -See also [`decode_overlong`](@ref) and [`show_invalid`](@ref). +See also [`Base.show_invalid`](@ref). """ isoverlong(c::AbstractChar) = false @@ -140,7 +162,7 @@ isoverlong(c::AbstractChar) = false l1 = leading_ones(u) t0 = trailing_zeros(u) & 56 (l1 == 1) | (8l1 + t0 > 32) | - ((((u & 0x00c0c0c0) ⊻ 0x00808080) >> t0 != 0) | is_overlong_enc(u)) && + (((u & 0x00c0c0c0) ⊻ 0x00808080) >> t0 != 0) && throw_invalid_char(c) u &= 0xffffffff >> l1 u >>= t0 @@ -149,24 +171,43 @@ isoverlong(c::AbstractChar) = false end """ - decode_overlong(c::AbstractChar)::Integer + Base.unsafe_codepoint(c::AbstractChar)::UInt32 -When [`isoverlong(c)`](@ref) is `true`, `decode_overlong(c)` returns -the Unicode codepoint value of `c`. `AbstractChar` implementations -that support overlong encodings should implement `Base.decode_overlong`. -""" -function decode_overlong end +Like [`codepoint(c)`](@ref), but assumes `!`[`Base.ismalformed(c)`](@ref): for +`Char`, the result is unspecified when `c` is malformed. Intended as a low-level +helper for code that has already verified well-formedness, so that the compiler +can prove the result is `:nothrow`. Prefer `codepoint(c)` otherwise. -@constprop :aggressive function decode_overlong(c::Char) +For non-`Char` `AbstractChar` subtypes this falls back to `UInt32(c)`. +""" +unsafe_codepoint(c::AbstractChar) = UInt32(c)::UInt32 +@constprop :aggressive @assume_effects :nothrow :foldable function unsafe_codepoint(c::Char) u = bitcast(UInt32, c) + u < 0x80000000 && return u >> 24 l1 = leading_ones(u) t0 = trailing_zeros(u) & 56 u &= 0xffffffff >> l1 u >>= t0 - ((u & 0x0000007f) >> 0) | ((u & 0x00007f00) >> 2) | - ((u & 0x007f0000) >> 4) | ((u & 0x7f000000) >> 6) + return ((u & 0x0000007f) >> 0) | ((u & 0x00007f00) >> 2) | ((u & 0x007f0000) >> 4) | ((u & 0x7f000000) >> 6) end +""" + decode_overlong(c::AbstractChar)::Integer + +When [`isoverlong(c)`](@ref) is `true`, `decode_overlong(c)` returns +the Unicode codepoint value of `c`. Deprecated in favor of +`codepoint(c)`. + +!!! compat "Julia 1.12" + In Julia 1.12 or later, `decode_overlong(c)` simply calls + `codepoint(c)`, which should now work for overlong encodings. + `AbstractChar` implementations that support overlong encodings + should implement `Base.decode_overlong` on older releases. +""" +function decode_overlong end + +@constprop :aggressive decode_overlong(c::AbstractChar) = codepoint(c) + @constprop :aggressive function Char(u::UInt32) u < 0x80 && return bitcast(Char, u << 24) u < 0x00200000 || throw_code_point_err(u) @@ -222,16 +263,17 @@ in(x::AbstractChar, y::AbstractChar) = x == y ==(x::Char, y::Char) = bitcast(UInt32, x) == bitcast(UInt32, y) isless(x::Char, y::Char) = bitcast(UInt32, x) < bitcast(UInt32, y) hash(x::Char, h::UInt) = - hash_finalizer(((bitcast(UInt32, x) + UInt64(0xd4d64234)) << 32) ⊻ UInt64(h)) % UInt + hash_finalizer(((bitcast(UInt32, x) +% UInt64(0xd4d64234)) << 32) ⊻ UInt64(h)) % UInt # fallbacks: -isless(x::AbstractChar, y::AbstractChar) = isless(Char(x), Char(y)) -==(x::AbstractChar, y::AbstractChar) = Char(x) == Char(y) -hash(x::AbstractChar, h::UInt) = hash(Char(x), h) +isless(x::AbstractChar, y::AbstractChar) = isless(Char(x)::Char, Char(y)::Char) +==(x::AbstractChar, y::AbstractChar) = Char(x)::Char == Char(y)::Char +hash(x::AbstractChar, h::UInt) = hash(Char(x)::Char, h) widen(::Type{T}) where {T<:AbstractChar} = T +@inline -%(x::AbstractChar, y::AbstractChar) = Int(x) -% Int(y) @inline -(x::AbstractChar, y::AbstractChar) = Int(x) - Int(y) -@inline function -(x::T, y::Integer) where {T<:AbstractChar} +@inline function -%(x::T, y::Integer) where {T<:AbstractChar} if x isa Char u = Int32((bitcast(UInt32, x) >> 24) % Int8) if u >= 0 # inline the runtime fast path @@ -241,7 +283,7 @@ widen(::Type{T}) where {T<:AbstractChar} = T end return T(Int32(x) - Int32(y)) end -@inline function +(x::T, y::Integer) where {T<:AbstractChar} +@inline function +%(x::T, y::Integer) where {T<:AbstractChar} if x isa Char u = Int32((bitcast(UInt32, x) >> 24) % Int8) if u >= 0 # inline the runtime fast path @@ -251,13 +293,17 @@ end end return T(Int32(x) + Int32(y)) end -@inline +(x::Integer, y::AbstractChar) = y + x +@inline +%(x::Integer, y::AbstractChar) = y + x + +-(x::AbstractChar, y::Integer) = x -% y ++(x::AbstractChar, y::Integer) = x +% y ++(x::Integer, y::AbstractChar) = x +% y # `print` should output UTF-8 by default for all AbstractChar types. # (Packages may implement other IO subtypes to specify different encodings.) # In contrast, `write(io, c)` outputs a `c` in an encoding determined by typeof(c). print(io::IO, c::Char) = (write(io, c); nothing) -print(io::IO, c::AbstractChar) = print(io, Char(c)) # fallback: convert to output UTF-8 +print(io::IO, c::AbstractChar) = print(io, Char(c)::Char) # fallback: convert to output UTF-8 const hex_chars = UInt8['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', @@ -277,7 +323,7 @@ function show_invalid(io::IO, c::Char) end """ - show_invalid(io::IO, c::AbstractChar) + Base.show_invalid(io::IO, c::AbstractChar) Called by `show(io, c)` when [`isoverlong(c)`](@ref) or [`ismalformed(c)`](@ref) return `true`. Subclasses @@ -330,7 +376,7 @@ function show(io::IO, ::MIME"text/plain", c::T) where {T<:AbstractChar} print(io, ": ") if isoverlong(c) print(io, "[overlong] ") - u = decode_overlong(c) + u = decode_overlong(c) # backwards compat Julia < 1.12 c = T(u) else u = codepoint(c) diff --git a/base/checked.jl b/base/checked.jl index 39d487cba6e37..ca17dda193e42 100644 --- a/base/checked.jl +++ b/base/checked.jl @@ -18,10 +18,8 @@ export checked_neg, checked_abs, checked_add, checked_sub, checked_mul, import Core: Intrinsics import .Intrinsics: - checked_sadd_int, checked_ssub_int, checked_smul_int, checked_sdiv_int, - checked_srem_int, - checked_uadd_int, checked_usub_int, checked_umul_int, checked_udiv_int, - checked_urem_int + checked_sadd_int, checked_ssub_int, checked_smul_int, + checked_uadd_int, checked_usub_int, checked_umul_int import Base: no_op_err, @inline, @noinline, checked_length # define promotion behavior for checked operations @@ -99,7 +97,7 @@ throw_overflowerr_negation(x) = (@noinline; throw(OverflowError(Base.invokelatest(string, "checked arithmetic: cannot compute -x for x = ", x, "::", typeof(x))))) if BrokenSignedInt != Union{} function checked_neg(x::BrokenSignedInt) - r = -x + r = -%(x) (x<0) & (r<0) && throw_overflowerr_negation(x) r end @@ -131,8 +129,6 @@ end checked_abs(x::UnsignedInt) = x checked_abs(x::Bool) = x - - """ Base.add_with_overflow(x, y) -> (r, f) @@ -141,11 +137,11 @@ Calculates `r = x+y`, with the flag `f` indicating whether overflow has occurred function add_with_overflow end add_with_overflow(x::T, y::T) where {T<:SignedInt} = checked_sadd_int(x, y) add_with_overflow(x::T, y::T) where {T<:UnsignedInt} = checked_uadd_int(x, y) -add_with_overflow(x::Bool, y::Bool) = (x+y, false) +add_with_overflow(x::Bool, y::Bool) = (x +% y, false) if BrokenSignedInt != Union{} function add_with_overflow(x::T, y::T) where T<:BrokenSignedInt - r = x + y + r = x +% y # x and y have the same sign, and the result has a different sign f = (x<0) == (y<0) != (r<0) r, f @@ -155,11 +151,10 @@ if BrokenUnsignedInt != Union{} function add_with_overflow(x::T, y::T) where T<:BrokenUnsignedInt # x + y > typemax(T) # Note: ~y == -y-1 - x + y, x > ~y + x +% y, x > ~y end end - throw_overflowerr_binaryop(op, x, y) = (@noinline; throw(OverflowError(LazyString(x, " ", op, " ", y, " overflowed for type ", typeof(x))))) @@ -172,7 +167,11 @@ The overflow protection may impose a perceptible performance penalty. """ function checked_add(x::T, y::T) where T<:Integer @inline - z, b = add_with_overflow(x, y) + zb = add_with_overflow(x, y) + # Avoid use of tuple destructuring, which uses arithmetic internally, + # so that this can be used as a replacement for + + z = getfield(zb, 1) + b = getfield(zb, 2) b && throw_overflowerr_binaryop(:+, x, y) z end @@ -194,7 +193,6 @@ checked_add(x1::T, x2::T, x3::T, x4::T, x5::T, x6::T, x7::T) where {T} = checked_add(x1::T, x2::T, x3::T, x4::T, x5::T, x6::T, x7::T, x8::T) where {T} = checked_add(checked_add(x1, x2), x3, x4, x5, x6, x7, x8) - """ Base.sub_with_overflow(x, y) -> (r, f) @@ -207,7 +205,7 @@ sub_with_overflow(x::Bool, y::Bool) = (x-y, false) if BrokenSignedInt != Union{} function sub_with_overflow(x::T, y::T) where T<:BrokenSignedInt - r = x - y + r = x -% y # x and y have different signs, and the result has a different sign than x f = (x<0) != (y<0) == (r<0) r, f @@ -216,7 +214,7 @@ end if BrokenUnsignedInt != Union{} function sub_with_overflow(x::T, y::T) where T<:BrokenUnsignedInt # x - y < 0 - x - y, x < y + x -% y, x < y end end @@ -234,7 +232,6 @@ function checked_sub(x::T, y::T) where T<:Integer z end - """ Base.mul_with_overflow(x, y) -> (r, f) @@ -243,7 +240,7 @@ Calculates `r = x*y`, with the flag `f` indicating whether overflow has occurred function mul_with_overflow end mul_with_overflow(x::T, y::T) where {T<:SignedInt} = checked_smul_int(x, y) mul_with_overflow(x::T, y::T) where {T<:UnsignedInt} = checked_umul_int(x, y) -mul_with_overflow(x::Bool, y::Bool) = (x*y, false) +mul_with_overflow(x::Bool, y::Bool) = (x *% y, false) if BrokenSignedIntMul != Union{} && BrokenSignedIntMul != Int128 function mul_with_overflow(x::T, y::T) where T<:BrokenSignedIntMul @@ -274,14 +271,14 @@ if Int128 <: BrokenSignedIntMul else false end - x*y, f + x *% y, f end end if UInt128 <: BrokenUnsignedIntMul # Avoid BigInt function mul_with_overflow(x::T, y::T) where T<:UInt128 # x * y > typemax(T) - x * y, y > 0 && x > fld(typemax(T), y) + x *% y, y > 0 && x > fld(typemax(T), y) end end diff --git a/base/client.jl b/base/client.jl index 8ff925d0d9ba4..0f05964f62851 100644 --- a/base/client.jl +++ b/base/client.jl @@ -31,10 +31,18 @@ answer_color() = text_colors[repl_color("JULIA_ANSWER_COLOR", default_color_answ stackframe_lineinfo_color() = repl_color("JULIA_STACKFRAME_LINEINFO_COLOR", :bold) stackframe_function_color() = repl_color("JULIA_STACKFRAME_FUNCTION_COLOR", :bold) -function repl_cmd(cmd, out) - # Immediately expand all arguments, so that typing e.g. ~/bin/foo works. - cmd.exec .= expanduser.(cmd.exec) - +function repl_cmd(cmd::AbstractCmd, out) + if !(cmd isa Cmd) + # Pipelines and redirects: run directly without shell wrapping. + try + run(ignorestatus(cmd)) + catch + lasterr = current_exceptions() + lasterr = ExceptionStack(NamedTuple[(exception = e[1], backtrace = [] ) for e in lasterr]) + invokelatest(display_error, lasterr) + end + return nothing + end if isempty(cmd.exec) throw(ArgumentError("no cmd to execute")) elseif cmd.exec[1] == "cd" @@ -78,6 +86,9 @@ function repl_cmd(cmd, out) nothing end +repl_cmd(@nospecialize(cmd), out) = + throw(ArgumentError("repl_cmd: expected an `AbstractCmd`, got $(typeof(cmd))")) + # deprecated function--preserved for DocTests.jl function ip_matches_func(ip, func::Symbol) for fr in StackTraces.lookup(ip) @@ -89,12 +100,35 @@ function ip_matches_func(ip, func::Symbol) return false end +__script_entry_include(mod::Module, path::String) = _include(identity, mod, path) +__script_entry_include_string(mod::Module, code::String, filename::String) = + include_string(mod, code, filename) +__script_entry_eval(mod::Module, @nospecialize(ex)) = Core.eval(mod, ex) + +is_driver_entry(frame) = !frame.from_c && + (startswith(String(frame.func), "__repl_entry") || + startswith(String(frame.func), "__script_entry")) + +# `eval`/`include` machinery a driver entry runs user code through; directly +# above the cut these frames cannot belong to user code +function is_driver_machinery(frame) + frame.from_c && return false + mod = parentmodule(frame) + (mod === Base || mod === Core || mod === nothing) || return false + return frame.func in (:eval, :include_string, :_include, :include) +end + function scrub_repl_backtrace(bt) if bt !== nothing && !(bt isa Vector{Any}) # ignore our sentinel value types bt = bt isa Vector{StackFrame} ? copy(bt) : stacktrace(bt) - # remove REPL-related frames from interactive printing - eval_ind = findlast(frame -> !frame.from_c && startswith(String(frame.func), "__repl_entry"), bt) - eval_ind === nothing || deleteat!(bt, eval_ind:length(bt)) + # remove REPL/driver frames from interactive printing + eval_ind = findlast(is_driver_entry, bt) + if eval_ind !== nothing + deleteat!(bt, eval_ind:length(bt)) + while !isempty(bt) && is_driver_machinery(bt[end]) + pop!(bt) + end + end end return bt end @@ -120,6 +154,10 @@ function display_error(io::IO, er, bt) end display_error(er, bt=nothing) = display_error(stderr, er, bt) +# N.B.: Any functions starting with __repl_entry cut off backtraces when printing in the REPL. +__repl_entry_client_lower(mod::Module, @nospecialize(ast)) = Meta.lower(mod, ast) +__repl_entry_client_eval(mod::Module, @nospecialize(ast)) = Core.eval(mod, ast) + function eval_user_input(errio, @nospecialize(ast), show_value::Bool) errcount = 0 lasterr = nothing @@ -132,12 +170,18 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool) if lasterr !== nothing lasterr = scrub_repl_backtrace(lasterr) istrivialerror(lasterr) || setglobal!(Base.MainInclude, :err, lasterr) - invokelatest(display_error, errio, lasterr) - errcount = 0 - lasterr = nothing + # error display (user-extensible show methods) runs in a + # fresh ^C epoch: the failed evaluation's cancelled epoch + # must not poison it, and a stuck printout is cancellable + try + ScopedValues.@with(CANCEL_TOKEN => sigint_new_episode!(new_evaluation_cancel_source!()), + invokelatest(display_error, errio, lasterr)) + finally + sigint_close_episode!() + end else - ast = Meta.lower(Main, ast) - value = Core.eval(Main, ast) + ast = __repl_entry_client_lower(Main, ast) + value = __repl_entry_client_eval(Main, ast) setglobal!(Base.MainInclude, :ans, value) if !(value === nothing) && show_value if have_color @@ -169,8 +213,8 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool) nothing end -function _parse_input_line_core(s::String, filename::String) - ex = Meta.parseall(s, filename=filename) +function _parse_input_line_core(s::String, filename::String, mod::Union{Module, Nothing}) + ex = Meta.parseall(s; filename, _parse=invokelatest(Meta.parser_for_module, mod)) if ex isa Expr && ex.head === :toplevel if isempty(ex.args) return nothing @@ -185,18 +229,18 @@ function _parse_input_line_core(s::String, filename::String) return ex end -function parse_input_line(s::String; filename::String="none", depwarn=true) +function parse_input_line(s::String; filename::String="none", depwarn=true, mod::Union{Module, Nothing}=nothing) # For now, assume all parser warnings are depwarns ex = if depwarn - _parse_input_line_core(s, filename) + _parse_input_line_core(s, filename, mod) else with_logger(NullLogger()) do - _parse_input_line_core(s, filename) + _parse_input_line_core(s, filename, mod) end end return ex end -parse_input_line(s::AbstractString) = parse_input_line(String(s)) +parse_input_line(s::AbstractString; kwargs...) = parse_input_line(String(s); kwargs...) # detect the reason which caused an :incomplete expression # from the error message @@ -232,6 +276,20 @@ function exec_options(opts) global have_color = colored_text(opts) global is_interactive = (opts.isinteractive != 0) + # Enable verbose debugging options when requested by other frameworks + debug_env_vars = ( + "RUNNER_DEBUG", # github actions when UI "debug logging" is enabled + "CI_DEBUG_TRACE", # gitlab CI when UI "debug" toggle is enabled + "SYSTEM_DEBUG", # azure pipelines when UI "System diagnostics" is enabled + ) + for v in debug_env_vars + if get_bool_env(v, false) + Base.TRACE_EVAL = Base.TRACE_EVAL === :full ? :full : :loc # Enable --trace-eval (location only) + ENV["JULIA_TEST_VERBOSE"] = "true" # Set JULIA_TEST_VERBOSE for this session + break + end + end + # pre-process command line argument list arg_is_program = !isempty(ARGS) repl = !arg_is_program @@ -283,13 +341,13 @@ function exec_options(opts) # process cmds list for (cmd, arg) in cmds if cmd == 'e' - Core.eval(Main, parse_input_line(arg)) + __script_entry_eval(Main, parse_input_line(arg; mod=Main)) elseif cmd == 'E' - invokelatest(show, Core.eval(Main, parse_input_line(arg))) + invokelatest(show, __script_entry_eval(Main, parse_input_line(arg; mod=Main))) println() elseif cmd == 'm' entrypoint = push!(split(arg, "."), "main") - Base.eval(Main, Expr(:import, Expr(:., Symbol.(entrypoint)...))) + __script_entry_eval(Main, Expr(:import, Expr(:., Symbol.(entrypoint)...))) if !invokelatest(should_use_main_entrypoint) error("`main` in `$arg` not declared as entry point (use `@main` to do so)") end @@ -297,7 +355,7 @@ function exec_options(opts) elseif cmd == 'L' # load file immediately on all processors if !distributed_mode - include(Main, arg) + __script_entry_include(Main, arg) else # TODO: Move this logic to Distributed and use a callback @sync for p in invokelatest(Main.procs) @@ -315,9 +373,9 @@ function exec_options(opts) end try if PROGRAM_FILE == "-" - include_string(Main, read(stdin, String), "stdin") + __script_entry_include_string(Main, read(stdin, String), "stdin") else - include(Main, PROGRAM_FILE) + __script_entry_include(Main, PROGRAM_FILE) end catch invokelatest(display_error, scrub_repl_backtrace(current_exceptions())) @@ -356,9 +414,9 @@ end function load_julia_startup() global_file = _global_julia_startup_file() - (global_file !== nothing) && include(Main, global_file) + (global_file !== nothing) && __script_entry_include(Main, global_file) local_file = _local_julia_startup_file() - (local_file !== nothing) && include(Main, local_file) + (local_file !== nothing) && __script_entry_include(Main, local_file) return nothing end @@ -419,14 +477,13 @@ function run_fallback_repl(interactive::Bool) let input = stdin if isa(input, File) || isa(input, IOStream) # for files, we can slurp in the whole thing at once - ex = parse_input_line(read(input, String)) + ex = parse_input_line(read(input, String); mod=Main) if Meta.isexpr(ex, :toplevel) # if we get back a list of statements, eval them sequentially # as if we had parsed them sequentially for stmt in ex.args eval_user_input(stderr, stmt, true) end - body = ex.args else eval_user_input(stderr, ex, true) end @@ -442,12 +499,19 @@ function run_fallback_repl(interactive::Bool) ex = nothing while !eof(input) line *= readline(input, keep=true) - ex = parse_input_line(line) + ex = parse_input_line(line; mod=Main) if !(isa(ex, Expr) && ex.head === :incomplete) break end end - eval_user_input(stderr, ex, true) + # each interactive input is a fresh ^C epoch (an + # evaluation source, so leftovers stay session-sweepable) + try + ScopedValues.@with(CANCEL_TOKEN => sigint_new_episode!(new_evaluation_cancel_source!()), + eval_user_input(stderr, ex, true)) + finally + sigint_close_episode!() + end catch err isa(err, InterruptException) ? print("\n\n") : rethrow() end @@ -468,7 +532,7 @@ function run_std_repl(REPL::Module, quiet::Bool, banner::Symbol, history_file::B repl = REPL.LineEditREPL(term, get(stdout, :color, false), true) repl.history_file = history_file end - # Make sure any displays pushed in .julia/config/startup.jl ends up above the + # Make sure any displays pushed in .julia/config/startup.jl end up above the # REPLDisplay d = REPL.REPLDisplay(repl) last_active_repl = @isdefined(active_repl) ? active_repl : nothing @@ -482,7 +546,7 @@ function run_std_repl(REPL::Module, quiet::Bool, banner::Symbol, history_file::B finally popdisplay(d) active_repl = last_active_repl - active_repl_backend = last_active_repl_backend + global active_repl_backend = last_active_repl_backend end nothing end @@ -528,12 +592,17 @@ The thrown errors are collected in a stack of exceptions. """ global err = nothing +const main_parser = Base.ScopedValues.ScopedValue{Any}(Base.VersionedParse(VERSION)) +function var"#_internal_julia_parse"(args...) + main_parser[](args...) +end + # Used for memoizing require_stdlib of these modules global InteractiveUtils::Module global Distributed::Module # weakly exposes ans and err variables to Main -export ans, err +export ans, err, var"#_internal_julia_parse" end function should_use_main_entrypoint() @@ -543,13 +612,80 @@ function should_use_main_entrypoint() return true end + +## The interactive session's two-level cancellation-source tree +# +# An interactive driver (the REPL backend) runs every evaluation under its +# own cancellation source, each a child of one long-lived *session* source: +# cancelling an evaluation's source stops exactly that evaluation (and +# everything it spawned), while cancelling the session source sweeps every +# still-running piece of work any evaluation has started - the runaway +# `@async` from three prompts ago included. Cancellation is monotonic, so a +# swept session source is retired and the next evaluation starts a fresh +# session epoch: "everything so far" always means "since the last sweep". + +const _session_cancel_source = Ref{Union{Nothing, CancellationTokenSource}}(nothing) +const _session_cancel_lock = ReentrantLock() + +# The current session source, created on first use (and after each sweep). +function session_cancel_source!() + lock(_session_cancel_lock; cancel=nothing) + try + ses = _session_cancel_source[] + if ses === nothing + ses = CancellationTokenSource() + _session_cancel_source[] = ses + end + return ses + finally + unlock(_session_cancel_lock) + end +end + +# A fresh source governing one interactive evaluation, linked under the +# session source. +new_evaluation_cancel_source!() = + CancellationTokenSource(CancellationToken(session_cancel_source!())) + +""" + Base.cancel_session_work!() -> Bool + +Cancel every still-running piece of work started under the current +interactive session's evaluations (see the session-source tree above) and +start a fresh session epoch. Returns whether there was a session to sweep. +The REPL binds this to a repeated `^C` at an empty prompt. +""" +function cancel_session_work!() + lock(_session_cancel_lock; cancel=nothing) + ses = try + s = _session_cancel_source[] + _session_cancel_source[] = nothing + s + finally + unlock(_session_cancel_lock) + end + ses === nothing && return false + cancel!(ses) + return true +end + function _start() empty!(ARGS) append!(ARGS, Core.ARGS) # clear any postoutput hooks that were saved in the sysimage empty!(Base.postoutput_hooks) local ret = 0 - try + # `--project` has been processed at this point - latch the active project's syntax + # version and use it for `-L`, `argfile`, etc. If launched, the REPL will re-evaluate + # at each prompt. + # The whole foreground execution runs as a ^C episode: a SIGINT (with + # exit-on-sigint disabled) cancels the episode token's scope. An + # interactive session installs its own per-evaluation episodes later + # (see REPL.repl_backend_loop), superseding this one. Deliberately a + # standalone root, not a session child - see the `sigint_new_episode!` + # docstring. + sigint_tok = sigint_new_episode!() + @Base.ScopedValues.with MainInclude.main_parser=>parser_for_active_project() CANCEL_TOKEN=>sigint_tok try repl_was_requested = exec_options(JLOptions()) if invokelatest(should_use_main_entrypoint) && !is_interactive main = invokelatest(getglobal, Main, :main) @@ -564,10 +700,30 @@ function _start() ret = repl_main(ARGS) end ret === nothing && (ret = 0) - ret = Cint(ret) + ret = try + Cint(ret) + catch + @error "The return value of `main` should be `nothing` or convertible to `Cint`" + Cint(1) + end catch ret = Cint(1) - invokelatest(display_error, scrub_repl_backtrace(current_exceptions())) + # report the error in a fresh ^C epoch (the script's epoch may be + # the very cancellation being reported; level-triggered checks in + # the printing path would re-throw it mid-report) + local errs = scrub_repl_backtrace(current_exceptions()) + try + ScopedValues.@with(CANCEL_TOKEN => sigint_new_episode!(), + invokelatest(display_error, errs)) + catch + # The report itself failed - e.g. a further ^C cancelled the + # display epoch, or a user-defined `show` method errored. The + # exit code already reflects the original failure; leave a bare + # note rather than dying with an unhandled exception. + Core.print(Core.stderr, "\nSYSTEM: displaying the error report failed\n") + finally + sigint_close_episode!() + end end if is_interactive && get(stdout, :color, false) print(color_normal) @@ -598,8 +754,8 @@ entrypoint. The precise semantics of the entrypoint depend on the CLI driver. In the `julia` driver, if `Main.main` is marked as an entrypoint, it will be automatically called upon the completion of script execution. -The `@main` macro may be used standalone or as part of the function definition, though in the latter -case, parentheses are required. In particular, the following are equivalent: +The `@main` macro may be used standalone or as part of the function definition. +The following are equivalent: ``` function @main(args) diff --git a/base/cmd.jl b/base/cmd.jl index 12d576e0e4c4b..36bc6891c9c7d 100644 --- a/base/cmd.jl +++ b/base/cmd.jl @@ -6,6 +6,8 @@ abstract type AbstractCmd end const UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS = UInt32(1 << 2) const UV_PROCESS_DETACHED = UInt32(1 << 3) const UV_PROCESS_WINDOWS_HIDE = UInt32(1 << 4) +const UV_PROCESS_SETUID = UInt32(1 << 5) +const UV_PROCESS_SETGID = UInt32(1 << 6) const UV_PROCESS_WINDOWS_DISABLE_EXACT_NAME = UInt32(1 << 7) struct Cmd <: AbstractCmd @@ -15,13 +17,17 @@ struct Cmd <: AbstractCmd env::Union{Vector{String},Nothing} dir::String cpus::Union{Nothing,Vector{UInt16}} + uid::Union{Nothing,UInt32} + gid::Union{Nothing,UInt32} Cmd(exec::Vector{<:AbstractString}) = - new(exec, false, 0x00, nothing, "", nothing) - Cmd(cmd::Cmd, ignorestatus, flags, env, dir, cpus = nothing) = + new(exec, false, 0x00, nothing, "", nothing, nothing, nothing) + Cmd(cmd::Cmd, ignorestatus, flags, env, dir, cpus = nothing, uid = nothing, gid = nothing) = new(cmd.exec, ignorestatus, flags, env, - dir === cmd.dir ? dir : cstr(dir), cpus) + dir === cmd.dir ? dir : cstr(dir), cpus, uid, gid) function Cmd(cmd::Cmd; ignorestatus::Bool=cmd.ignorestatus, env=cmd.env, dir::AbstractString=cmd.dir, cpus::Union{Nothing,Vector{UInt16}} = cmd.cpus, + uid::Union{Nothing,UInt32} = cmd.uid, + gid::Union{Nothing,UInt32} = cmd.gid, detach::Bool = 0 != cmd.flags & UV_PROCESS_DETACHED, windows_verbatim::Bool = 0 != cmd.flags & UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS, windows_hide::Bool = 0 != cmd.flags & UV_PROCESS_WINDOWS_HIDE) @@ -29,7 +35,7 @@ struct Cmd <: AbstractCmd windows_verbatim * UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS | windows_hide * UV_PROCESS_WINDOWS_HIDE new(cmd.exec, ignorestatus, flags, byteenv(env), - dir === cmd.dir ? dir : cstr(dir), cpus) + dir === cmd.dir ? dir : cstr(dir), cpus, uid, gid) end end @@ -38,10 +44,12 @@ has_nondefault_cmd_flags(c::Cmd) = c.flags != 0x00 || c.env !== nothing || c.dir !== "" || - c.cpus !== nothing + c.cpus !== nothing || + c.uid !== nothing || + c.gid !== nothing """ - Cmd(cmd::Cmd; ignorestatus, detach, windows_verbatim, windows_hide, env, dir) + Cmd(cmd::Cmd; ignorestatus, detach, windows_verbatim, windows_hide, env, dir, uid, gid) Cmd(exec::Vector{String}) Construct a new `Cmd` object, representing an external program and arguments, from `cmd`, @@ -71,6 +79,8 @@ while changing the settings of the optional keyword arguments: elements, use [`addenv()`](@ref) which will return a `Cmd` object with the updated environment. * `dir::AbstractString`: Specify a working directory for the command (instead of the current directory). +* `uid::Union{Nothing,UInt32}`: Set the user ID for the process (Unix only). +* `gid::Union{Nothing,UInt32}`: Set the group ID for the process (Unix only). For any keywords that are not specified, the current settings from `cmd` are used. @@ -86,9 +96,9 @@ This can then be passed to the `Cmd` constructor to modify its settings, e.g. """ Cmd -hash(x::Cmd, h::UInt) = hash(x.exec, hash(x.env, hash(x.ignorestatus, hash(x.dir, hash(x.flags, h))))) +hash(x::Cmd, h::UInt) = hash(x.exec, hash(x.env, hash(x.ignorestatus, hash(x.dir, hash(x.flags, hash(x.uid, hash(x.gid, h))))))) ==(x::Cmd, y::Cmd) = x.exec == y.exec && x.env == y.env && x.ignorestatus == y.ignorestatus && - x.dir == y.dir && isequal(x.flags, y.flags) + x.dir == y.dir && isequal(x.flags, y.flags) && x.uid == y.uid && x.gid == y.gid struct OrCmds <: AbstractCmd a::AbstractCmd @@ -122,12 +132,69 @@ escape_microsoft_c_args(cmd::Cmd) = escape_microsoft_c_args(io::IO, cmd::Cmd) = escape_microsoft_c_args(io::IO, cmd.exec...) +# Patterns that indicate a sensitive environment variable name. +# Matched as whole components after splitting on non-alphanumeric characters, +# so e.g. "PAT" matches "GITHUB_PAT" but not "PATH". +const SENSITIVE_ENV_PATTERNS = ("KEY", "TOKEN", "SECRET", "JWT", "PAT", + "PASSWORD", "PASSWD", "PASS", "PWD", "PW", + "CREDENTIAL", "CRED", "AUTH", "PRIVATE", "PRIV") + +function is_sensitive_env_name(name::AbstractString) + uname = uppercase(name) + parts = eachsplit(uname, r"[^A-Za-z0-9]+") + return any(part -> part in SENSITIVE_ENV_PATTERNS, parts) +end + +function get_show_env_mode(io::IO) + val = get(io, :show_env, :_unset) + if val !== :_unset + return val::Symbol + end + envval = get(ENV, "JULIA_SHOW_ENV", nothing) + envval === nothing && return :redact + envval = lowercase(envval) + envval == "all" && return :all + envval == "none" && return :none + envval == "keys" && return :keys + return :redact +end + +function show_env(io::IO, env::Vector{String}) + mode = get_show_env_mode(io) + print(io, "[") + for (i, e) in enumerate(env) + i > 1 && print(io, ", ") + eqidx = findnext('=', e, 2) + if eqidx === nothing || mode === :all + show(io, e) + elseif mode === :keys + key = e[1:prevind(e, eqidx)] + show(io, key) + else # :redact + key = e[1:prevind(e, eqidx)] + if is_sensitive_env_name(key) + show(io, key) + else + show(io, e) + end + end + end + print(io, "]") +end + function show(io::IO, cmd::Cmd) - print_env = cmd.env !== nothing + env_mode = cmd.env !== nothing ? get_show_env_mode(io) : nothing + print_env = cmd.env !== nothing && env_mode !== :none print_dir = !isempty(cmd.dir) - (print_env || print_dir) && print(io, "setenv(") + print_uid = cmd.uid !== nothing + print_gid = cmd.gid !== nothing print_cpus = cmd.cpus !== nothing + + (print_env || print_dir) && print(io, "setenv(") print_cpus && print(io, "setcpuaffinity(") + print_gid && print(io, "setgid(") + print_uid && print(io, "setuid(") + print(io, '`') join(io, map(cmd.exec) do arg replace(sprint(context=io) do io @@ -137,14 +204,19 @@ function show(io::IO, cmd::Cmd) end, '`' => "\\`") end, ' ') print(io, '`') + + print_uid && (print(io, ", "); show(io, Int32(cmd.uid)); print(io, ")")) + print_gid && (print(io, ", "); show(io, Int32(cmd.gid)); print(io, ")")) if print_cpus print(io, ", ") show(io, collect(Int, something(cmd.cpus))) print(io, ")") end - print_env && (print(io, ","); show(io, cmd.env)) - print_dir && (print(io, "; dir="); show(io, cmd.dir)) - (print_dir || print_env) && print(io, ")") + if print_env || print_dir + print_env && (print(io, ","); show_env(io, cmd.env)) + print_dir && (print(io, "; dir="); show(io, cmd.dir)) + print(io, ")") + end nothing end @@ -188,7 +260,7 @@ if OS_HANDLE !== RawFD end setup_stdio(stdio::Union{DevNull,OS_HANDLE,RawFD}, ::Bool) = (stdio, false) -const Redirectable = Union{IO, FileRedirect, RawFD, OS_HANDLE} +const Redirectable = Union{IO, IOServer, FileRedirect, RawFD, OS_HANDLE} const StdIOSet = NTuple{3, Redirectable} struct CmdRedirect <: AbstractCmd @@ -225,6 +297,8 @@ Mark a command object so that running it will not throw an error if the result c ignorestatus(cmd::Cmd) = Cmd(cmd, ignorestatus=true) ignorestatus(cmd::Union{OrCmds,AndCmds}) = typeof(cmd)(ignorestatus(cmd.a), ignorestatus(cmd.b)) +ignorestatus(cmd::CmdRedirect) = + CmdRedirect(ignorestatus(cmd.cmd), cmd.handle, cmd.stream_no, cmd.readable) """ detach(command) @@ -248,7 +322,7 @@ byteenv(env::AbstractArray{<:AbstractString}) = byteenv(env::AbstractDict) = String[cstr(string(k)*"="*string(v)) for (k,v) in env] byteenv(env::Nothing) = nothing -byteenv(env::Union{AbstractVector{Pair{T,V}}, Tuple{Vararg{Pair{T,V}}}}) where {T<:AbstractString,V} = +byteenv(env::Union{AbstractVector{<:Pair{<:AbstractString}}, Tuple{Vararg{Pair{<:AbstractString}}}}) = String[cstr(k*"="*string(v)) for (k,v) in env] """ @@ -264,6 +338,17 @@ The `dir` keyword argument can be used to specify a working directory for the co `dir` defaults to the currently set `dir` for `command` (which is the current working directory if not specified already). +!!! warning + The display redaction behavior (including the `:show_env` IOContext key + and `JULIA_SHOW_ENV` environment variable) is experimental and may change + in a non-breaking release of Julia. + +When displaying a `Cmd`, environment variables whose names match sensitive patterns +(e.g. containing "KEY", "TOKEN", "SECRET", "PASSWORD", "AUTH", "JWT", "PAT") have +their values hidden. The display mode can be controlled via the `JULIA_SHOW_ENV` +environment variable or the `:show_env` IOContext key (which takes precedence), +with values `:none`, `:keys`, `:redact` (default), or `:all`. + See also [`Cmd`](@ref), [`addenv`](@ref), [`ENV`](@ref), [`pwd`](@ref). """ setenv(cmd::Cmd, env; dir=cmd.dir) = Cmd(cmd; env=byteenv(env), dir=dir) @@ -355,6 +440,54 @@ function setcpuaffinity end setcpuaffinity(cmd::Cmd, ::Nothing) = Cmd(cmd; cpus = nothing) setcpuaffinity(cmd::Cmd, cpus) = Cmd(cmd; cpus = collect(UInt16, cpus)) +""" + setuid(original_command::Cmd, uid) -> command::Cmd + +Set the user ID (UID) of the `command`. On Unix systems, this allows +the command to run as a different user. Passing `uid = nothing` removes +any previously set UID. + +This function is only supported on Unix-based systems (Linux, macOS, etc.). +Requires appropriate permissions to set UID. + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples + +```julia +julia> run(setuid(`id -u`, 1000)); +1000 +``` +""" +function setuid end +setuid(cmd::Cmd, ::Nothing) = Cmd(cmd; uid = nothing) +setuid(cmd::Cmd, uid::Integer) = Cmd(cmd; uid = UInt32(uid)) + +""" + setgid(original_command::Cmd, gid) -> command::Cmd + +Set the group ID (GID) of the `command`. On Unix systems, this allows +the command to run as a different group. Passing `gid = nothing` removes +any previously set GID. + +This function is only supported on Unix-based systems (Linux, macOS, etc.). +Requires appropriate permissions to set GID. + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples + +```julia +julia> run(setgid(`id -g`, 1000)); +1000 +``` +""" +function setgid end +setgid(cmd::Cmd, ::Nothing) = Cmd(cmd; gid = nothing) +setgid(cmd::Cmd, gid::Integer) = Cmd(cmd; gid = UInt32(gid)) + (&)(left::AbstractCmd, right::AbstractCmd) = AndCmds(left, right) redir_out(src::AbstractCmd, dest::AbstractCmd) = OrCmds(src, dest) redir_err(src::AbstractCmd, dest::AbstractCmd) = ErrOrCmds(src, dest) @@ -403,9 +536,21 @@ function pipeline(cmd::AbstractCmd; stdin=nothing, stdout=nothing, stderr=nothin return cmd end -pipeline(cmd::AbstractCmd, dest) = pipeline(cmd, stdout=dest) +pipeline(cmd::AbstractCmd, dest::Union{AbstractCmd, AbstractString, Redirectable}) = pipeline(cmd, stdout=dest) pipeline(src::Union{Redirectable,AbstractString}, cmd::AbstractCmd) = pipeline(cmd, stdin=src) +""" + pipeline(command, redir::Pair{<:Integer, <:Redirectable}) + +Redirect fd number `redir.first` of `command` to or from the given `redir.second`, which can be +an I/O stream, a filename, or a file descriptor. This method is primarily used to pass additional +fds beyond the standard ios that are not supported by the keyword argument interface. + +!!! compat "Julia 1.13" + This method requires Julia 1.13 or later. +""" +pipeline(cmd::AbstractCmd, redir::Pair{<:Integer, <:Redirectable}) = CmdRedirect(cmd, redir.second, Int(redir.first)) + """ pipeline(from, to, ...) @@ -468,12 +613,12 @@ function cmd_gen(parsed) args = String[] if length(parsed) >= 1 && isa(parsed[1], Tuple{Cmd}) cmd = (parsed[1]::Tuple{Cmd})[1] - (ignorestatus, flags, env, dir) = (cmd.ignorestatus, cmd.flags, cmd.env, cmd.dir) + (ignorestatus, flags, env, dir, cpus, uid, gid) = (cmd.ignorestatus, cmd.flags, cmd.env, cmd.dir, cmd.cpus, cmd.uid, cmd.gid) append!(args, cmd.exec) for arg in tail(parsed) append!(args, Base.invokelatest(arg_gen, arg...)::Vector{String}) end - return Cmd(Cmd(args), ignorestatus, flags, env, dir) + return Cmd(Cmd(args), ignorestatus, flags, env, dir, cpus, uid, gid) else for arg in parsed append!(args, arg_gen(arg...)::Vector{String}) @@ -506,5 +651,11 @@ Process(`echo 1`, ProcessExited(0)) """ macro cmd(str::String) cmd_ex = shell_parse(str, special=shell_special, filename=String(__source__.file))[1] - return :(cmd_gen($(esc(cmd_ex)))) + if Meta.isexpr(cmd_ex, :tuple) + return :(cmd_gen($(esc(cmd_ex)))) + else + # Pipeline/redirect expression: GlobalRefs inside are already resolved to Base. + # esc() ensures $interpolations inside are evaluated in the caller's scope. + return esc(cmd_ex) + end end diff --git a/base/cmem.jl b/base/cmem.jl index 531fac434d097..733d6d0eecac9 100644 --- a/base/cmem.jl +++ b/base/cmem.jl @@ -48,7 +48,7 @@ end Call `memcmp` from the C standard library. !!! compat "Julia 1.10" - Support for `memcmp` requires at least Julia 1.9. + Support for `memcmp` requires at least Julia 1.10. """ function memcmp(a::Ptr, b::Ptr, n::Integer) diff --git a/base/combinatorics.jl b/base/combinatorics.jl index 5180f830ce187..a7384b2b9c24c 100644 --- a/base/combinatorics.jl +++ b/base/combinatorics.jl @@ -44,7 +44,7 @@ end # Basic functions for working with permutations @inline function _foldoneto(op, acc, ::Val{N}) where N - @assert N::Integer > 0 + @assert N::Integer > 0 "N must be positive" if @generated quote acc_0 = acc @@ -122,7 +122,7 @@ function swaprows!(a::AbstractMatrix, i, j) end end -# like permute!! applied to each row of a, in-place in a (overwriting p). +# like permute!! applied to each column of a, in-place in a (overwriting p). function permutecols!!(a::AbstractMatrix, p::AbstractVector{<:Integer}) require_one_based_indexing(a, p) count = 0 diff --git a/base/complex.jl b/base/complex.jl index 6651581a96240..c1c2c75872d12 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -8,7 +8,7 @@ Complex number type with real and imaginary part of type `T`. `ComplexF16`, `ComplexF32` and `ComplexF64` are aliases for `Complex{Float16}`, `Complex{Float32}` and `Complex{Float64}` respectively. -See also: [`Real`](@ref), [`complex`](@ref), [`real`](@ref). +See also [`Real`](@ref), [`complex`](@ref), [`real`](@ref). """ struct Complex{T<:Real} <: Number re::T @@ -22,7 +22,7 @@ Complex(x::Real) = Complex(x, zero(x)) The imaginary unit. -See also: [`imag`](@ref), [`angle`](@ref), [`complex`](@ref). +See also [`imag`](@ref), [`angle`](@ref), [`complex`](@ref). # Examples ```jldoctest @@ -61,7 +61,7 @@ float(::Type{Complex{T}}) where {T} = Complex{float(T)} Return the real part of the complex number `z`. -See also: [`imag`](@ref), [`reim`](@ref), [`complex`](@ref), [`isreal`](@ref), [`Real`](@ref). +See also [`imag`](@ref), [`reim`](@ref), [`complex`](@ref), [`isreal`](@ref), [`Real`](@ref). # Examples ```jldoctest @@ -76,7 +76,7 @@ real(z::Complex) = z.re Return the imaginary part of the complex number `z`. -See also: [`conj`](@ref), [`reim`](@ref), [`adjoint`](@ref), [`angle`](@ref). +See also [`conj`](@ref), [`reim`](@ref), [`adjoint`](@ref), [`angle`](@ref). # Examples ```jldoctest @@ -105,7 +105,7 @@ reim(z) = (real(z), imag(z)) real(T::Type) Return the type that represents the real part of a value of type `T`. -e.g: for `T == Complex{R}`, returns `R`. +e.g., for `T == Complex{R}`, returns `R`. Equivalent to `typeof(real(zero(T)))`. # Examples @@ -120,7 +120,7 @@ Float64 real(T::Type) = typeof(real(zero(T))) real(::Type{T}) where {T<:Real} = T real(C::Type{<:Complex}) = fieldtype(C, 1) -real(::Type{Union{}}, slurp...) = Union{}(im) +real(::Type{Union{}}, slurp...) = Union{} """ isreal(x)::Bool @@ -188,6 +188,7 @@ Union{Missing, Complex{Int64}} """ complex(::Type{T}) where {T<:Real} = Complex{T} complex(::Type{Complex{T}}) where {T<:Real} = Complex{T} +complex(::Type{Union{}}, slurp...) = Union{} flipsign(x::Complex, y::Real) = ifelse(signbit(y), -x, x) @@ -195,16 +196,16 @@ function show(io::IO, z::Complex) r, i = reim(z) compact = get(io, :compact, false)::Bool show(io, r) - if signbit(i) && !isnan(i) + bufio = IOBuffer() + show(IOContext(bufio, io), i) + seekstart(bufio) + if peek(bufio) === UInt8('-') + seek(bufio, 1) print(io, compact ? "-" : " - ") - if isa(i,Signed) && !isa(i,BigInt) && i == typemin(typeof(i)) - show(io, -widen(i)) - else - show(io, -i) - end + write(io, bufio) else print(io, compact ? "+" : " + ") - show(io, i) + write(io, bufio) end if !(isa(i,Signed) || isa(i,AbstractFloat) && isfinite(i)) print(io, "*") @@ -248,11 +249,7 @@ isequal(z::Real, w::Complex) = isequal(z,real(w))::Bool & isequal(zero(z),imag(w in(x::Complex, r::AbstractRange{<:Real}) = isreal(x) && real(x) in r -if UInt === UInt64 - const h_imag = 0x32a7a07f3e7cd1f9 -else - const h_imag = 0x3e7cd1f9 -end +const h_imag = 0x32a7a07f3e7cd1f9 % UInt const hash_0_imag = hash(0, h_imag) function hash(z::Complex, h::UInt) @@ -268,7 +265,7 @@ end Compute the complex conjugate of a complex number `z`. -See also: [`angle`](@ref), [`adjoint`](@ref). +See also [`angle`](@ref), [`adjoint`](@ref). # Examples ```jldoctest @@ -293,9 +290,11 @@ inv(z::Complex{<:Integer}) = inv(float(z)) *(z::Complex, w::Complex) = Complex(real(z) * real(w) - imag(z) * imag(w), real(z) * imag(w) + imag(z) * real(w)) +_mulsub(a, b, c) = _mulsub(promote(a, b, c)...) +_mulsub(a::T, b::T, c::T) where {T<:Real} = muladd(a, b, -c) muladd(z::Complex, w::Complex, x::Complex) = - Complex(muladd(real(z), real(w), -muladd(imag(z), imag(w), -real(x))), - muladd(real(z), imag(w), muladd(imag(z), real(w), imag(x)))) + Complex(muladd(real(z), real(w), -_mulsub(imag(z), imag(w), real(x))), + muladd(real(z), imag(w), muladd(imag(z), real(w), imag(x)))) # handle Bool and Complex{Bool} # avoid type signature ambiguity warnings @@ -342,7 +341,7 @@ muladd(z::Complex, x::Real, w::Complex) = Complex(muladd(real(z),x,real(w)), muladd(imag(z),x,imag(w))) muladd(x::Real, y::Real, z::Complex) = Complex(muladd(x,y,real(z)), imag(z)) muladd(z::Complex, w::Complex, x::Real) = - Complex(muladd(real(z), real(w), -muladd(imag(z), imag(w), -x)), + Complex(muladd(real(z), real(w), -_mulsub(imag(z), imag(w), x)), muladd(real(z), imag(w), imag(z) * real(w))) /(a::R, z::S) where {R<:Real,S<:Complex} = (T = promote_type(R,S); a*inv(T(z))) @@ -514,7 +513,7 @@ function ssqs(x::T, y::T) where T<:Real ρ = convert(T, Inf) elseif isinf(ρ) || (ρ==0 && (x!=0 || y!=0)) || ρ cispi(10000) 1.0 + 0.0im @@ -622,7 +621,7 @@ Compute the phase angle in radians of a complex number `z`. Returns a number `-pi ≤ angle(z) ≤ pi`, and is thus discontinuous along the negative real axis. -See also: [`atan`](@ref), [`cis`](@ref), [`rad2deg`](@ref). +See also [`atan`](@ref), [`cis`](@ref), [`rad2deg`](@ref). # Examples ```jldoctest @@ -844,15 +843,23 @@ function _cpow(z::Union{T,Complex{T}}, p::Union{T,Complex{T}}) where T else r = -zᵣ θ = copysign(Tf(π),imag(z)) - rᵖ = r^pᵣ * exp(-pᵢ*θ) - ϕ = pᵣ*θ + pᵢ*log(r) + logr = log(r) + re_log, im_phase = pᵣ*logr, pᵢ*θ + lim = log(floatmax(Tf)) - one(Tf) + rᵖ = (abs(re_log) < lim && abs(im_phase) < lim) ? + r^pᵣ * exp(-im_phase) : exp(re_log - im_phase) + ϕ = pᵣ*θ + pᵢ*logr end else pᵣ, pᵢ = reim(p) r = abs(z) θ = angle(z) - rᵖ = r^pᵣ * exp(-pᵢ*θ) - ϕ = pᵣ*θ + pᵢ*log(r) + logr = log(r) + re_log, im_phase = pᵣ*logr, pᵢ*θ + lim = log(floatmax(Tf)) - one(Tf) + rᵖ = (abs(re_log) < lim && abs(im_phase) < lim) ? + r^pᵣ * exp(-im_phase) : exp(re_log - im_phase) + ϕ = pᵣ*θ + pᵢ*logr end if isfinite(ϕ) diff --git a/base/condition.jl b/base/condition.jl index fd771c9be346a..77aedd70e3342 100644 --- a/base/condition.jl +++ b/base/condition.jl @@ -21,7 +21,14 @@ function unlock end function trylock end function islocked end unlockall(l::AbstractLock) = unlock(l) # internal function for implementing `wait` -relockall(l::AbstractLock, token::Nothing) = lock(l) # internal function for implementing `wait` +relockall(l::AbstractLock, state::Nothing) = lock(l) # internal function for implementing `wait` +# Restore all but one level of a hold captured by `unlockall` - the +# enclosing frames' levels, consuming the waiting frame's own - for the +# exceptional unwind out of the internal wait layer. A no-op when that +# frame's level was the only one (the common case: a depth-1 cancellation +# unwind performs no lock operation at all). The state stays opaque; +# shielded like `relockall` - a restore has no correct cancellable use. +relockall_but_one(l::AbstractLock, state::Nothing) = nothing assert_havelock(l::AbstractLock, tid::Integer) = (islocked(l) && tid == Threads.threadid()) ? nothing : concurrency_violation() assert_havelock(l::AbstractLock, tid::Task) = @@ -54,52 +61,191 @@ islocked(::AlwaysLockedST) = true ## condition variables +# (The WaitEntry registration type and the wake-claim protocol live in +# cancellation.jl, which is included earlier in bootstrap: registrations +# carry the cancellation half of a parked wait.) + """ GenericCondition Abstract implementation of a condition object -for synchronizing tasks objects with a given lock. +for synchronizing task objects with a given lock. +""" +mutable struct GenericCondition{L<:AbstractLock} + # mutable for identity only + const waitq::IntrusiveLinkedList{WaitEntry} + const lock::L + + GenericCondition{L}() where {L<:AbstractLock} = new{L}(IntrusiveLinkedList{WaitEntry}(), L()) + GenericCondition{L}(l::L) where {L<:AbstractLock} = new{L}(IntrusiveLinkedList{WaitEntry}(), l) + GenericCondition(l::AbstractLock) = new{typeof(l)}(IntrusiveLinkedList{WaitEntry}(), l) +end + +waitqueue(c::GenericCondition) = ILLRef(c.waitq, c) + """ -struct GenericCondition{L<:AbstractLock} - waitq::IntrusiveLinkedList{Task} - lock::L + try_unlink_claimed!(w::WaitEntry) - GenericCondition{L}() where {L<:AbstractLock} = new{L}(IntrusiveLinkedList{Task}(), L()) - GenericCondition{L}(l::L) where {L<:AbstractLock} = new{L}(IntrusiveLinkedList{Task}(), l) - GenericCondition(l::AbstractLock) = new{typeof(l)}(IntrusiveLinkedList{Task}(), l) +Opportunistically attempt to unlink a wait entry from its queue. This is a memory pressure +optimization. If the queue is locked by another task, the entry will remain linked and will +be unlinked upon the next wakeup attempt. +""" +function try_unlink_claimed!(w::WaitEntry) + ok = true + for slot in slots(w) + q = slot.owner + q === nothing && continue + # Manual split for --trim (every waitq's identity is its + # condition - see waitqueue) + if q isa GenericCondition{Threads.SpinLock} + _try_unlink_from!(q, w) || (ok = false) + elseif q isa GenericCondition{ReentrantLock} + _try_unlink_from!(q, w) || (ok = false) + elseif q isa GenericCondition{AlwaysLockedST} + _try_unlink_from!(q, w) || (ok = false) + elseif q isa CancellationTokenSource + # sticky source registrations stay in place + else + ok = false + end + end + return ok +end + +function _try_unlink_from!(c::GenericCondition, w::WaitEntry) + trylock(c.lock) || return false + try + list_deletefirst!(waitqueue(c), w) + finally + unlock(c.lock) + end + return true end show(io::IO, c::GenericCondition) = print(io, GenericCondition, "(", c.lock, ")") assert_havelock(c::GenericCondition) = assert_havelock(c.lock) lock(c::GenericCondition) = lock(c.lock) +# (the `cancel`-forwarding lock method for ReentrantLock-backed conditions +# lives in lock.jl, after ReentrantLock is defined) unlock(c::GenericCondition) = unlock(c.lock) trylock(c::GenericCondition) = trylock(c.lock) islocked(c::GenericCondition) = islocked(c.lock) lock(f, c::GenericCondition) = lock(f, c.lock) -# have waiter wait for c -function _wait2(c::GenericCondition, waiter::Task, first::Bool=false) - ct = current_task() +## GenericCondition as a waitable (see base/park.jl): the lock-carried +## kind - its lock is caller-held from before phase 4 into the suspend +## bracket, which is what makes its recheck vacuous. +function wait_enqueue!(c::GenericCondition, w::WaitEntry, first::Bool) + if first + pushfirst!(waitqueue(c), w) + else + push!(waitqueue(c), w) + end + return true +end +function wait_dequeue!(c::GenericCondition, w::WaitEntry, why::UInt8) + # a no-op when a notify already popped the entry. WAKE_VALUE/WAKE_FIRED + # run under the caller's held lock (the settle after a wake; the fired + # branch); the cleanup/withdraw whys take it themselves - shielded, a + # cleanup may be unwinding the very request a cancellable acquire would + # rethrow + if why == WAKE_INTERRUPTED || why == WAKE_WITHDRAWN + _uncancellable_lock(c.lock) + try + list_deletefirst!(waitqueue(c), w) + finally + unlock(c.lock) + end + else + list_deletefirst!(waitqueue(c), w) + end + return nothing +end + +@noinline _fresh_waiter_error() = + throw(ConcurrencyViolationError("schedule_on_notify! requires a fresh task: never started, scheduled, or armed")) + +# Freshness is stricter than `!istaskstarted`: a task that +# `schedule`/`@async` has merely enqueued hasn't run yet, but arming it +# here would collide with the arm its own first park performs once it +# starts (that park's `_arm_wait` CAS fails and unwinds a wait's lock +# choreography from the outside). Reject anything started, queued, or +# already armed. +_assert_fresh_waiter(waiter::Task) = + if istaskstarted(waiter) || waiter.queue !== nothing || + (@atomic :monotonic waiter.waiting_on) !== nothing + _fresh_waiter_error() + end + +# Start `waiter` with the (level-triggered) cancellation of its birth +# source: a subscribed task whose governing source is cancelled dies +# instead of running its body - a never-started task raises the scheduled +# exception at start. +function _schedule_subscription_cancelled(waiter::Task, src::CancellationTokenSource) + st = @atomic :acquire src.state + schedule(waiter, CancellationRequest(st), error=true) + return nothing +end + +# Subscribe the not-yet-started task `waiter` to `c`'s next notify: arm +# its registration and enqueue it, so the notify's claim-and-schedule is +# the task's first schedule - a start trigger; the current task does not +# suspend. Waits of the *current* task go through `park!` (base/park.jl). +# +# Subscriptions are governed by the *waiter's* birth cancellation source +# (the CANCEL_TOKEN of the scope captured at its construction): if that +# source is - or becomes, while still subscribed - cancelled, the task +# dies with the CancellationRequest instead of starting. Cleanup-class +# subscribers (Timer/AsyncCondition callback tasks, channel close hooks, +# errormonitor, REPL teardown) are constructed under +# `CANCEL_TOKEN => nothing` and so are never killed this way. +# Returns the registration entry (or `nothing` when the subscription was +# refused and the waiter scheduled to die). +function schedule_on_notify!(c::GenericCondition, waiter::Task, first::Bool=false) assert_havelock(c) + _assert_fresh_waiter(waiter) + src = _birth_cancel_source(waiter) + if src !== nothing && iscancelled(src) + # born cancelled: never enqueue anything + _schedule_subscription_cancelled(waiter, src) + return nothing + end + w = src === nothing ? _cached_wait_entry(waiter) : + _cancel_wait_entry(waiter, src, 0x00) + _arm_wait(waiter, w) if first - pushfirst!(c.waitq, waiter) + pushfirst!(waitqueue(c), w) else - push!(c.waitq, waiter) + push!(waitqueue(c), w) + end + if src !== nothing + # the sticky source registration + the publish-then-recheck dance, + # exactly like a park's phases 4-5 - but claiming back the *waiter's* + # arm on refusal + sw = SourceWait(src, 0x00) + wait_enqueue!(sw, w, false) + if wait_recheck(sw, w) && disarm!(waiter, w) + list_deletefirst!(waitqueue(c), w) # under the held lock + _schedule_subscription_cancelled(waiter, src) + return nothing + end + # a lost disarm means a concurrent walk claimed the fresh arm: its + # delivery kills the waiter at start end - # since _wait2 is similar to schedule, we should observe the sticky bit now + # since this is similar to schedule, we should observe the sticky bit now if waiter.sticky && Threads.threadid(waiter) == 0 && !GC.in_finalizer() # Issue #41324 # t.sticky && tid == 0 is a task that needs to be co-scheduled with # the parent task. If the parent (current_task) is not sticky we must # set it to be sticky. # XXX: Ideally we would be able to unset this - ct.sticky = true + current_task().sticky = true tid = Threads.threadid() ccall(:jl_set_task_tid, Cint, (Any, Cint), waiter, tid-1) end - return + return w end """ @@ -126,25 +272,133 @@ proceeding. function wait end """ - wait(c::GenericCondition; first::Bool=false) + wait(c::GenericCondition; first::Bool=false, cancel=Base.DEFAULT_CANCEL) Wait for [`notify`](@ref) on `c` and return the `val` parameter passed to `notify`. If the keyword `first` is set to `true`, the waiter will be put _first_ in line to wake up on `notify`. Otherwise, `wait` has first-in-first-out (FIFO) behavior. + +The `cancel` keyword argument controls which cancellation token may interrupt +the wait (throwing the [`CancellationRequest`](@ref) into the waiter): by +default the scoped token (see `Base.CANCEL_TOKEN`); pass a +[`CancellationToken`](@ref) to override it, or `nothing` to make the wait +non-cancellable. """ -function wait(c::GenericCondition; first::Bool=false) +function wait(c::GenericCondition; first::Bool=false, + cancel::CancelTokenArg=DEFAULT_CANCEL) + # Check the caller contract here, not just in the internal layer: a + # violation (not locked, or locked by another task) must propagate + # with the lock state untouched - the restore in the catch below is + # only correct for throws that consumed this frame's lock level. + assert_havelock(c) + tok = check_cancel_arg(cancel) # an entry refusal throws lock-held + try + return wait(c, tok; first) + catch + # the internal layer throws having released this frame's lock + # level; the public contract is rethrow-with-lock-held (callers + # are written `lock(c); try ... finally unlock(c)`), so restore + # one shielded level + _uncancellable_lock(c.lock) + rethrow() + end +end + +# (The interrupted-wait cleanup lives in base/park.jl as +# interrupted_park_cleanup!, shared by every park site.) + +# `min_severity` is the lowest severity that may wake (cancel) this wait - +# the comparisons are inclusive on both the registration and walk sides. A +# teardown wait that re-parks after acknowledging a delivery at severity +# `s` must therefore pass `s + 0x01` (exclusive staging), so a re-cancel at +# the acknowledged severity leaves it parked and only an escalation wakes +# it; see e.g. _uv_write_cancelled_finish. +# +# `cancel_value` makes a cancellation of the governing token complete the +# wait as an ordinary (value-mode) wake returning the `CancellationRequest` +# instead of throwing it - callers that must keep working through a +# cancellation (structured teardown) check the return value rather than +# catching. Value-mode returns follow the normal wake contract (the +# caller's lock is held), and the registration rides the walk's watcher +# delivery, so no exception is constructed anywhere on the path. +function wait(c::GenericCondition, tok::MaybeToken; first::Bool=false, + min_severity::UInt8=0x00, cancel_value::Bool=false) ct = current_task() - _wait2(c, ct, first) - token = unlockall(c.lock) + assert_havelock(c) + src = cancel_source(tok) + # Entry check (skipped for throwing teardown re-parks; a value-mode + # wait checks against its floor). For the throwing flavor, like every + # throw out of this internal layer, the waiting frame's own lock level + # is released first - callers use the `locked && unlock` idiom; the + # public kwarg method restores the lock-held contract. + if src !== nothing && cancel_value + st = @atomic :acquire src.state + if st >= max(min_severity, 0x01) + return CancellationRequest(st) + end + elseif src !== nothing && min_severity == 0x00 && iscancelled(src) + unlock(c.lock) + checkcancel(src) + error("cancelled source did not throw") + end + if src === nothing + ws = (c,) + w = _cached_wait_entry(ct) + elseif cancel_value + # Watcher-mode registration: fresh single-use entry (the cache + # contract in base/park.jl - a cache slot may only carry ordinary + # cancellable-park registrations). + ws = (c, WatcherWait(src, max(min_severity, 0x01))) + w = WaitEntry2(ct) + else + ws = (c, SourceWait(src, min_severity)) + w = _cancel_wait_entry(ct, src, min_severity) + end + local parked::Bool try - return wait() + parked = park!(ws, w, first) catch - q = ct.queue; q === nothing || Base.list_deletefirst!(q::IntrusiveLinkedList{Task}, ct) + # an arm-phase throw (contract misuse: this task already carries + # a foreign registration) happens before any suspend. Nothing is + # enqueued to settle, but this frame's lock level must still be + # consumed like every other throw out of this layer - the + # caller's `locked && unlock` idiom would otherwise leak a held + # lock, and a leaked SpinLock wedges the process on the next + # lock of the same object + disarm!(ct, w) + withdraw!(ws, w, WAKE_FIRED) + unlock(c.lock) + rethrow() + end + if !parked + # the source refused at the registration recheck (the only + # fireable waitable here): withdraw under the still-held lock and + # deliver like the entry check (value mode returns lock-held) + withdraw!(ws, w, WAKE_FIRED) + if cancel_value + st = @atomic :acquire src.state + st >= max(min_severity, 0x01) || error("park fired without a cancelled source") + return CancellationRequest(st) + end + unlock(c.lock) + checkcancel(src) + error("park fired without a cancelled source") + end + lockstate = unlockall(c.lock) + r = try + wait_safe_interrupt(ws, w) + catch + # the cleanup already withdrew every registration; restore the + # enclosing frames' hold, consuming this waiting frame's level - + # a depth-1 unwind (the common case, e.g. the cancellation being + # delivered) touches no lock at all + relockall_but_one(c.lock, lockstate) rethrow() - finally - relockall(c.lock, token) end + relockall(c.lock, lockstate) + withdraw!(ws, w, WAKE_VALUE) # lazy settle under the reacquired lock + return r end """ @@ -161,7 +415,15 @@ function notify(c::GenericCondition, @nospecialize(arg), all, error) assert_havelock(c) cnt = 0 while !isempty(c.waitq) - t = popfirst!(c.waitq) + w = popfirst!(waitqueue(c)) + # An entry whose wake was already claimed by an interrupter does not + # count as woken: drop it and continue to the next waiter (the + # interrupted task resumes via whatever its claimer scheduled and + # will find its entry already unlinked). + t = @atomic :monotonic w.task + if !(t isa Task && claim_wait(t, w)) + continue + end schedule(t, arg, error=error) cnt += 1 all || break @@ -176,7 +438,7 @@ notify_error(c::GenericCondition, err) = notify(c, err, true, true) Return `true` if no tasks are waiting on the condition, `false` otherwise. """ -isempty(c::GenericCondition) = isempty(c.waitq) +isempty(c::GenericCondition) = _waitq_isempty(waitqueue(c)) # default (Julia v1.0) is currently single-threaded diff --git a/base/coreio.jl b/base/coreio.jl index b5c543a25d5ad..bddbb6214d128 100644 --- a/base/coreio.jl +++ b/base/coreio.jl @@ -1,5 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +function print end +typeof(print).name.max_methods = UInt8(1) + print(x) = print(stdout, x) print(x1, x2) = print(stdout, x1, x2) println(x) = print(stdout, x, "\n") @@ -12,7 +15,9 @@ println(io::IO) = print(io, "\n") function show end function repr end -struct DevNull <: IO end +struct DevNull <: IO + DevNull() = new() +end const devnull = DevNull() write(::DevNull, ::UInt8) = 1 unsafe_write(::DevNull, ::Ptr{UInt8}, n::UInt)::Int = n diff --git a/base/coreir.jl b/base/coreir.jl index 1cd226aae5f2d..6ad623cbccfd5 100644 --- a/base/coreir.jl +++ b/base/coreir.jl @@ -38,7 +38,7 @@ the following information about field defined-ness: - `undefs[i] === nothing` indicates the corresponding element in `fields` may be undefined - `undefs[i] === false` indicates the corresponding element in `fields` is guaranteed to be defined - `undefs[i] === true` indicates the corresponding element in `fields` is guaranteed to be undefined -If `field[i]` is of type `Union{}`, it means the `i`-th field is never be initialized and +If `field[i]` is of type `Union{}`, it means the `i`-th field will never be initialized and thus never be defined. In this case, `undefs[i]` should always be `true`. The same applies if `typ` is a `Tuple`, and because of how `Tuple` elements are initialized, @@ -112,5 +112,9 @@ while processing a call, then `Conditional` everywhere else. """ Core.InterConditional -Core.InterConditional(var::SlotNumber, @nospecialize(thentype), @nospecialize(elsetype)) = - InterConditional(slot_id(var), thentype, elsetype) +""" + alias::InterMustAlias + +This lattice element is used in a very similar way as `InterConditional`, but corresponds to `MustAlias`. +""" +Core.InterMustAlias diff --git a/base/cpuid.jl b/base/cpuid.jl index 0370bd33b83e5..1720ba798bdf5 100644 --- a/base/cpuid.jl +++ b/base/cpuid.jl @@ -10,8 +10,8 @@ export cpu_isa A structure which represents the Instruction Set Architecture (ISA) of a computer. It holds the `Set` of features of the CPU. -The numerical values of the features are automatically generated from the C -source code of Julia and stored in the `features_h.jl` Julia file. +Feature bit indices come from the cpufeatures library's generated tables +(extracted from LLVM's TableGen data at build time). """ struct ISA features::Set{UInt32} @@ -23,55 +23,156 @@ Base.isless(a::ISA, b::ISA) = a < b include(string(Base.BUILDROOT, "features_h.jl")) # include($BUILDROOT/base/features_h.jl) -# Keep in sync with `arch_march_isa_mapping`. +""" + _featurebytes_to_isa(buf::Vector{UInt8}) -> ISA + +Convert a raw feature byte buffer (from cpufeatures) into an ISA. +""" +function _featurebytes_to_isa(buf::Vector{UInt8}) + features = Set{UInt32}() + for byte_idx in 0:length(buf)-1 + b = buf[byte_idx + 1] + b == 0 && continue + for bit in 0:7 + if (b >> bit) & 1 != 0 + push!(features, UInt32(byte_idx * 8 + bit)) + end + end + end + return ISA(features) +end + +""" + _cross_lookup_cpu(arch::String, name::String) -> ISA + +Look up hardware features for a CPU on any architecture using the +cross-arch tables. Works regardless of host architecture. +Returns an empty ISA if the CPU or architecture is not found. +""" +function _cross_lookup_cpu(arch::String, name::String) + nbytes = ccall(:jl_cpufeatures_cross_nbytes, Csize_t, (Cstring,), arch) + nbytes == 0 && return ISA(Set{UInt32}()) + buf = Vector{UInt8}(undef, nbytes) + written = ccall(:jl_cpufeatures_cross_lookup, Csize_t, + (Cstring, Cstring, Ptr{UInt8}, Csize_t), + arch, name, buf, nbytes) + written == 0 && return ISA(Set{UInt32}()) + return _featurebytes_to_isa(buf) +end + +""" + _build_bit_to_name(arch::String) -> Dict{UInt32, String} + +Build a mapping from feature bit index to feature name for an architecture. +""" +function _build_bit_to_name(arch::String) + nfeats = ccall(:jl_cpufeatures_cross_num_features, UInt32, (Cstring,), arch) + result = Dict{UInt32, String}() + for i in 0:nfeats-1 + name_ptr = ccall(:jl_cpufeatures_cross_feature_name, Cstring, (Cstring, UInt32), arch, i) + name_ptr == C_NULL && continue + bit = ccall(:jl_cpufeatures_cross_feature_bit, Cint, (Cstring, UInt32), arch, i) + bit < 0 && continue + result[UInt32(bit)] = unsafe_string(name_ptr) + end + return result +end + +""" + feature_names(arch::String, cpu::String) -> Vector{String} + feature_names(arch::String, isa::ISA) -> Vector{String} + feature_names(isa::ISA) -> Vector{String} + feature_names() -> Vector{String} + +Return sorted hardware feature names. Can query by CPU name (on any +architecture) or by ISA. Defaults to the host architecture and CPU. + +# Examples +```julia +feature_names() # host CPU features +feature_names("x86_64", "haswell") # haswell's features +feature_names("aarch64", "cortex-x925") # cross-arch query +``` +""" +feature_names() = feature_names(string(Sys.ARCH), _host_isa()) +feature_names(isa::ISA) = feature_names(string(Sys.ARCH), isa) +function feature_names(arch::String, cpu::String) + isa = _cross_lookup_cpu(arch, cpu) + return feature_names(arch, isa) +end +function feature_names(arch::String, isa::ISA) + mapping = _build_bit_to_name(arch) + return sort([get(mapping, bit, "unknown_$bit") for bit in isa.features]) +end + +""" + _lookup_cpu(name::String) -> ISA + +Look up hardware features for the named CPU on the host architecture. +Returns an empty ISA if the CPU name is not found. +""" +function _lookup_cpu(name::String) + nbytes = ccall(:jl_cpufeatures_nbytes, Csize_t, ()) + buf = Vector{UInt8}(undef, nbytes) + ret = ccall(:jl_cpufeatures_lookup, Cint, (Cstring, Ptr{UInt8}, Csize_t), name, buf, nbytes) + ret != 0 && return ISA(Set{UInt32}()) + return _featurebytes_to_isa(buf) +end + +""" + _host_isa() -> ISA + +Get the hardware features of the host CPU from the cpufeatures library. +""" +function _host_isa() + nbytes = ccall(:jl_cpufeatures_nbytes, Csize_t, ()) + buf = Vector{UInt8}(undef, nbytes) + ccall(:jl_cpufeatures_host, Cvoid, (Ptr{UInt8}, Csize_t), buf, nbytes) + return _featurebytes_to_isa(buf) +end + +# Build an ISA list for a given architecture family. +# Uses cross-arch lookup so it works on any host. +# Entries with empty cpuname get an empty ISA (generic baseline). +function _make_isa_list(arch::String, entries::Vector{Pair{String,String}}) + result = Pair{String,ISA}[] + for (label, cpuname) in entries + if isempty(cpuname) + push!(result, label => ISA(Set{UInt32}())) + else + push!(result, label => _cross_lookup_cpu(arch, cpuname)) + end + end + return result +end + +# ISA definitions per architecture family. +# CPU names are LLVM names in the cpufeatures database. +# Keep in sync with `arch_march_isa_mapping` in binaryplatforms.jl. const ISAs_by_family = Dict( - "i686" => [ - # Source: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html. - # Implicit in all sets, because always required by Julia: mmx, sse, sse2 - "pentium4" => ISA(Set{UInt32}()), - "prescott" => ISA(Set((JL_X86_sse3,))), - ], - "x86_64" => [ - # Source: https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html. - # Implicit in all sets, because always required by x86-64 architecture: mmx, sse, sse2 - "x86_64" => ISA(Set{UInt32}()), - "core2" => ISA(Set((JL_X86_sse3, JL_X86_ssse3))), - "nehalem" => ISA(Set((JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt))), - "sandybridge" => ISA(Set((JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_aes, JL_X86_pclmul))), - "haswell" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c))), - "skylake" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c, JL_X86_rdseed, JL_X86_adx, JL_X86_prfchw, JL_X86_clflushopt, JL_X86_xsavec, JL_X86_xsaves))), - "skylake_avx512" => ISA(Set((JL_X86_movbe, JL_X86_sse3, JL_X86_ssse3, JL_X86_sse41, JL_X86_sse42, JL_X86_popcnt, JL_X86_pku, JL_X86_avx, JL_X86_avx2, JL_X86_aes, JL_X86_pclmul, JL_X86_fsgsbase, JL_X86_rdrnd, JL_X86_fma, JL_X86_bmi, JL_X86_bmi2, JL_X86_f16c, JL_X86_rdseed, JL_X86_adx, JL_X86_prfchw, JL_X86_clflushopt, JL_X86_xsavec, JL_X86_xsaves, JL_X86_avx512f, JL_X86_clwb, JL_X86_avx512vl, JL_X86_avx512bw, JL_X86_avx512dq, JL_X86_avx512cd))), - ], - "armv6l" => [ - # The only armv6l processor we know of that runs Julia on armv6l - # We don't have a good way to tell the different armv6l variants apart through features, - # and honestly we don't care much since it's basically this one chip that people want to use with Julia. - "arm1176jzfs" => ISA(Set{UInt32}()), - ], - "armv7l" => [ - "armv7l" => ISA(Set{UInt32}()), - "armv7l+neon" => ISA(Set((JL_AArch32_neon,))), - "armv7l+neon+vfpv4" => ISA(Set((JL_AArch32_neon, JL_AArch32_vfp4))), - ], - "aarch64" => [ - # Implicit in all sets, because always required: fp, asimd - "armv8.0-a" => ISA(Set{UInt32}()), - "armv8.1-a" => ISA(Set((JL_AArch64_v8_1a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm))), - "armv8.2-a+crypto" => ISA(Set((JL_AArch64_v8_2a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2))), - "a64fx" => ISA(Set((JL_AArch64_v8_2a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_sha2, JL_AArch64_ccpp, JL_AArch64_complxnum, JL_AArch64_fullfp16, JL_AArch64_sve))), - "apple_m1" => ISA(Set((JL_AArch64_v8_5a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2, JL_AArch64_sha3, JL_AArch64_ccpp, JL_AArch64_complxnum, JL_AArch64_fp16fml, JL_AArch64_fullfp16, JL_AArch64_dotprod, JL_AArch64_rcpc, JL_AArch64_altnzcv))), - ], - "riscv64" => [ - "riscv64" => ISA(Set{UInt32}()), - ], - "powerpc64le" => [ - # We have no way to test powerpc64le features yet, so we're only going to declare the lowest ISA: - "power8" => ISA(Set{UInt32}()), - ], - "riscv64" => [ - # We have no way to test riscv64 features yet, so we're only going to declare the lowest ISA: - "riscv64" => ISA(Set{UInt32}()), - ], + "i686" => _make_isa_list("x86_64", [ + "pentium4" => "", + "prescott" => "prescott", + ]), + "x86_64" => _make_isa_list("x86_64", [ + "x86_64" => "", + "core2" => "core2", + "nehalem" => "nehalem", + "sandybridge" => "sandybridge", + "haswell" => "haswell", + "skylake" => "skylake", + "skylake_avx512" => "skylake-avx512", + ]), + "aarch64" => _make_isa_list("aarch64", [ + "armv8.0-a" => "", + "armv8.1-a" => "cortex-a76", + "armv8.2-a+crypto" => "cortex-a78", + "a64fx" => "a64fx", + "apple_m1" => "apple-a14", + ]), + "riscv64" => _make_isa_list("riscv64", [ + "riscv64" => "", + ]), ) # Test a CPU feature exists on the currently-running host @@ -96,27 +197,13 @@ function normalize_arch(arch::String) return arch end -let - # Collect all relevant features for the current architecture, if any. - FEATURES = UInt32[] - arch = normalize_arch(String(Sys.ARCH)) - if arch in keys(ISAs_by_family) - for isa in ISAs_by_family[arch] - unique!(append!(FEATURES, last(isa).features)) - end - end - - # Use `@eval` to inline the list of features. - @eval function cpu_isa() - return ISA(Set{UInt32}(feat for feat in $(FEATURES) if test_cpu_feature(feat))) - end -end - """ cpu_isa() Return the [`ISA`](@ref) (instruction set architecture) of the current CPU. """ -cpu_isa +function cpu_isa() + return _host_isa() +end end # module CPUID diff --git a/base/deepcopy.jl b/base/deepcopy.jl index 58c753705a61f..3093d8cb2ba6f 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -34,7 +34,8 @@ function deepcopy(@nospecialize x) return deepcopy_internal(x, IdDict())::typeof(x) end -deepcopy_internal(x::Union{Symbol,Core.MethodInstance,Method,GlobalRef,DataType,Union,UnionAll,Task,Regex}, +deepcopy_internal(x::Union{Symbol,Core.MethodInstance,Method,GlobalRef,DataType,Union,UnionAll,Task,Regex, + Core.CancellationTokenSource}, stackdict::IdDict) = x deepcopy_internal(x::Tuple, stackdict::IdDict) = ntuple(i->deepcopy_internal(x[i], stackdict), length(x)) @@ -180,3 +181,21 @@ function deepcopy_internal(x::GenericCondition, stackdict::IdDict) stackdict[x] = y return y end + +# Core.WaitEntryN has a hidden variable-length slot tail that the generic +# path above - which allocates only the fixed datatype size - cannot +# reproduce (the GC would then scan a nonexistent tail). Allocate through +# the runtime allocator instead. The slots are transient registration +# state, meaningless outside the live wait registries (a copy is not +# enqueued anywhere), so the copy gets fresh, free slots +# (owner = nothing, next = nothing, aux = 0) rather than copies of the +# originals; the `task` reference is kept as-is (`deepcopy` of a Task is +# the identity, see above). +function deepcopy_internal(x::Core.WaitEntryN, stackdict::IdDict) + if haskey(stackdict, x) + return stackdict[x]::typeof(x) + end + y = WaitEntryN((@atomic :monotonic x.task), _nslots(x)) + stackdict[x] = y + return y +end diff --git a/base/deprecated.jl b/base/deprecated.jl index 0ee6b6b790837..4420eae524eda 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -2,7 +2,7 @@ # Internal changes mechanism. # Instructions for Julia Core Developers: -# 1. When making a breaking change that is known to be depnedet upon by an +# 1. When making a breaking change that is known to be depended upon by an # important and closely coupled package, decide on a unique `change_name` # for your PR and add it to the list below. In general, it is better to # err on the side of caution and assign a `change_name` even if it is not @@ -17,16 +17,19 @@ # 2. Upon tagging an -alpha version # a. On master, set __next_removal_version to v"1.(x+1)-alpha" # b. On the release branch, set __next_removal_version to v"1.x" (no -alpha) -# 3. Upong tagging a release candidate, clear the list of internal changes and +# 3. Upon tagging a release candidate, clear the list of internal changes and # set __next_removal_version to `nothing`. const __next_removal_version = v"1.12-alpha" const __internal_changes_list = ( :invertedlinetables, :codeinforefactor, :miuninferredrm, - :codeinfonargs, # #54341 + :codeinfonargs, #54341 :ocnopartial, :printcodeinfocalls, + :syntacticccall, #59165 + :svectvar, #61645 + :syntacticcglobal, #61709 # Add new change names above this line ) @@ -190,6 +193,7 @@ macro deprecate(old, new, export_old=true) newcall = sprint(show_unquoted, new) # if old.head is a :where, step down one level to the :call to avoid code duplication below callexpr = old.head === :call ? old : old.args[1] + maybe_export = nothing if callexpr.head === :call fnexpr = callexpr.args[1] if fnexpr isa Expr && fnexpr.head === :curly @@ -201,30 +205,28 @@ macro deprecate(old, new, export_old=true) else cannot_export_nonsymbol() end - else - maybe_export = nothing end else error("invalid usage of @deprecate") end - Expr(:toplevel, - maybe_export, - :($(esc(old)) = begin - $meta - depwarn($"`$oldcall` is deprecated, use `$newcall` instead.", Core.Typeof($(esc(fnexpr))).name.singletonname) - $(esc(new)) - end)) + ex = :($(esc(old)) = begin + $meta + depwarn($"`$oldcall` is deprecated, use `$newcall` instead.", Core.Typeof($(esc(fnexpr))).name.singletonname) + $(esc(new)) + end) + Expr(:toplevel, maybe_export, replace_linenums!(ex, __source__)) else if export_old && !(old isa Symbol) cannot_export_nonsymbol() end + ex = :(function $(esc(old))(args...; kwargs...) + $meta + depwarn($"`$old` is deprecated, use `$new` instead.", Core.Typeof($(esc(old))).name.singletonname) + $(esc(new))(args...; kwargs...) + end) Expr(:toplevel, export_old ? Expr(:export, esc(old)) : nothing, - :(function $(esc(old))(args...; kwargs...) - $meta - depwarn($"`$old` is deprecated, use `$new` instead.", Core.Typeof($(esc(old))).name.singletonname) - $(esc(new))(args...; kwargs...) - end)) + replace_linenums!(ex, __source__)) end end @@ -316,6 +318,31 @@ end deprecate(m::Module, s::Symbol, flag=1) = ccall(:jl_deprecate_binding, Cvoid, (Any, Any, Cint), m, s, flag) +""" + @deprecate_binding old new [export_old=true] [dep_message] [constant=true] + +Deprecate the binding `old`, making it an alias for `new` and printing a deprecation warning +on access to `old` (when `julia` is run with `--depwarn=yes`). This is for deprecating a +renamed or relocated global/constant, whereas [`@deprecate`](@ref) is for deprecating methods. + +`old` must be a symbol and `new` the replacement it forwards to. By default `old` is defined +as a `const`; pass `false` for `constant` to define it as a non-constant global instead. + +To prevent `old` from being exported, set `export_old` to `false`. A custom `dep_message` +string (printed after ".old is deprecated") may be given. + +See also [`@deprecate`](@ref) and [`Base.depwarn`](@ref). + +# Examples +```jldoctest +julia> const dep_new = 42; + +julia> Base.@deprecate_binding dep_old dep_new false; + +julia> dep_old +42 +``` +""" macro deprecate_binding(old, new, export_old=true, dep_message=:nothing, constant=true) dep_message === :nothing && (dep_message = ", use $new instead.") return Expr(:toplevel, @@ -343,7 +370,7 @@ macro deprecate_moved(old, new, export_old=true) "Run `Pkg.add(\"", new, "\")` to install it, restart Julia,\n", "and then run `using ", new, "` to load it.") return Expr(:toplevel, - :($eold(args...; kwargs...) = error($emsg)), + replace_linenums!(:($eold(args...; kwargs...) = error($emsg)), __source__), export_old ? Expr(:export, eold) : nothing, Expr(:call, :deprecate, __module__, Expr(:quote, old), 2)) end @@ -537,9 +564,9 @@ end """ isbindingresolved(m::Module, s::Symbol) -> Bool -Returns whether the binding of a symbol in a module is resolved. +Return whether the binding of a symbol in a module is resolved. -See also: [`isexported`](@ref), [`ispublic`](@ref), [`isdeprecated`](@ref) +See also [`isexported`](@ref), [`ispublic`](@ref), [`isdeprecated`](@ref). ```jldoctest julia> module Mod @@ -565,3 +592,155 @@ end to_power_type(x) = oftype(x*x, x) # END 1.12 deprecations + +# BEGIN 1.13 deprecations + +@deprecate merge(combine::Callable, d::AbstractDict, others::AbstractDict...) mergewith(combine, d, others...) + +# end 1.13 deprecations + +# BEGIN 1.14 deprecations + +# These operators are new in 1.14, but these fallback methods are added for +# compatibility while packages adjust to defining both operators, to allow +# Base and other packages to start using these. +*%(a::T, b::T) where {T} = *(a, b) ++%(a::T, b::T) where {T} = +(a, b) +-%(a::T, b::T) where {T} = -(a, b) + +# Revise calls this +function explicit_manifest_entry_path(args...) + spec = explicit_manifest_entry_load_spec(args...) + spec === nothing && return nothing + return spec.path +end + +# These functions get called from generated functions a lot where printing causes errors. We have the following options: +# 1. Use ordinary depwarn. This breaks generated functions that use these deprecations, defeating the point. +# 2. Always error in generated functions. This would probably be best (the depwarn would run at expansion time), +# but it completely breaks inference of these generated functions and some of them are important. +# 3. Never print a warning in generated functions (but do throw the error if --depwarn=error). +# +# We choose option 3. It's not ideal, because users that never use --depwarn=error will never see the warning, +# but it's the least bad tradeoff among the lot. +function depwarn_if_not_pure(args...) + opts = JLOptions() + opts.depwarn != 2 && ccall(:jl_is_in_pure_context, Bool, ()) && return + depwarn(args...) +end + +@noinline function getproperty(x::TypeEq, s::Symbol) + if s === :parameters + depwarn_if_not_pure("accessing `Type.parameters` is deprecated; use `Base.type_parameter(x)` instead", :getproperty) + return Core.svec(type_parameter(x)) + elseif s === :name + depwarn_if_not_pure("accessing `Type.name` is deprecated without replacement. If for detection, use `Base.isType(x)`.", :getproperty) + return Core.AnyType.name + elseif s === :hash + depwarn_if_not_pure("accessing `Type.hash` is deprecated; use `Base._jl_type_cache_hash(x)` instead", :getproperty) + return reinterpret(Int32, UInt32(_jl_type_cache_hash(x))) + end + return getfield(x, s) +end + +@noinline function typename(x::TypeEq) + depwarn_if_not_pure("calling `typename` on `Type` is deprecated. If for detection, use `Base.isType(x)`.", :typename) + return Core.AnyType.name +end + +@noinline function nameof(x::TypeEq) + depwarn_if_not_pure("calling `nameof` on `Type` is deprecated. If for detection, use `Base.isType(x)`.", :nameof) + return :Type +end + +@noinline function parentmodule(x::TypeEq) + depwarn_if_not_pure("calling `parentmodule` on `Type` is deprecated. If for detection, use `Base.isType(x)`.", :parentmodule) + return Core +end + +@noinline function isabstracttype(x::TypeEq) + depwarn_if_not_pure("calling `isabstracttype` on a `Type{...}` is deprecated; `Type{}` is now a kind. If for detection, use `Base.isType(x)`.", :isabstracttype) + return true +end + +@noinline function getproperty(x::Core.TypeEgal, s::Symbol) + if s === :parameters + depwarn_if_not_pure("accessing `Type.parameters` is deprecated; use `Base.type_parameter(x)` instead", :getproperty) + return Core.svec(type_parameter(x)) + elseif s === :name + depwarn_if_not_pure("accessing `Type.name` is deprecated without replacement. If for detection, use `Base.isType(x)`.", :getproperty) + return Core.AnyType.name + elseif s === :hash + depwarn_if_not_pure("accessing `Type.hash` is deprecated; use `Base._jl_type_cache_hash(x)` instead", :getproperty) + return reinterpret(Int32, UInt32(_jl_type_cache_hash(x))) + end + return getfield(x, s) +end + +@noinline function typename(x::Core.TypeEgal) + depwarn_if_not_pure("calling `typename` on `Type` is deprecated. If for detection, use `Base.isType(x)`.", :typename) + return Core.AnyType.name +end + +@noinline function nameof(x::Core.TypeEgal) + depwarn_if_not_pure("calling `nameof` on `Type` is deprecated. If for detection, use `Base.isType(x)`.", :nameof) + return :Type +end + +@noinline function parentmodule(x::Core.TypeEgal) + depwarn_if_not_pure("calling `parentmodule` on `Type` is deprecated. If for detection, use `Base.isType(x)`.", :parentmodule) + return Core +end + +@noinline function isabstracttype(x::Core.TypeEgal) + depwarn_if_not_pure("calling `isabstracttype` on a `Type{...}` is deprecated; `Type{}` is now a kind. If for detection, use `Base.isType(x)`.", :isabstracttype) + return true +end + +@deprecate SubString{T}(s::T, i::Int, j::Int, ::Val{:noshift}) where {T <: AbstractString} begin + @boundscheck if !(i == j == 0) + si, sj = i + 1, prevind(s, j + i + 1) + @inbounds isvalid(s, si) || string_index_err(s, si) + @inbounds isvalid(s, sj) || string_index_err(s, sj) + end + @inbounds raw_substring(s, i + 1, j) +end + +# This method is slightly different because it returns a SubString{SubString}, +# therefore it requires an explicit SubString{T}(ss) call at the end. +# We discourage creating substrings of substrings, but the deprecated method +# allowed it. +@deprecate SubString{T}(s::T, i::Int, j::Int, ::Val{:noshift}) where {T <: SubString} begin + @boundscheck if !(i == j == 0) + si, sj = i + 1, prevind(s, j + i + 1) + @inbounds isvalid(s, si) || string_index_err(s, si) + @inbounds isvalid(s, sj) || string_index_err(s, sj) + end + ss = @inbounds raw_substring(s, i + 1, j) + SubString{T}(ss) +end + +@deprecate _unsetindex!(A) Base.unsetindex!(A) false +@deprecate _unsetindex!(A, i) Base.unsetindex!(A, i) false + +# `a[] = v` on a `Threads.Atomic` (the `setindex!` form) is a footgun: read-modify-write +# expressions such as `a[] += 1` look atomic but expand to a separate non-atomic load and +# store. Steer users to the explicit `@atomic` form. This is written by hand rather than +# with `@deprecate` so the message can name the hazard and avoid the macro-expansion +# artifact `@deprecate` would embed in the suggested replacement. +# ## This is not yet enabled (deprecated) due to the need to transition a couple stdlibs to this form (or another equivalent form). +function setindex!(x::Threads.Atomic, v) + # depwarn(lazy"`a[] = v` on a `Threads.Atomic` is deprecated because read-modify-write uses like `a[] += 1` are not atomic; use `@atomic a[] = v` (or `@atomic a[] += 1`, `Threads.atomic_add!`, ...) instead.", :setindex!) + return @atomic x[] = v +end + +# The `*_domain_error` helpers were replaced by `Math.throw_finite_domainerror` in #62842, +# but some packages call them directly. Not deprecated, just kept for compatibility. +@eval Math begin + @noinline sin_domain_error(x) = throw_finite_domainerror(:sin, x) + @noinline cos_domain_error(x) = throw_finite_domainerror(:cos, x) + @noinline sincos_domain_error(x) = throw_finite_domainerror(:sincos, x) + @noinline tan_domain_error(x) = throw_finite_domainerror(:tan, x) +end + +# END 1.14 deprecations diff --git a/base/dict.jl b/base/dict.jl index 6cbf4429ca9e0..aae8db6c1c609 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -166,7 +166,7 @@ end @inbounds if (olds[i] & 0x80) != 0 k = oldk[i] v = oldv[i] - index, sh = hashindex(k, newsz) + index, _ = hashindex(k, newsz) index0 = index while slots[index] != 0 index = (index & (newsz-1)) + 1 @@ -223,8 +223,8 @@ function empty!(h::Dict{K,V}) where V where K fill!(h.slots, 0x0) sz = length(h.slots) for i in 1:sz - _unsetindex!(h.keys, i) - _unsetindex!(h.vals, i) + unsetindex!(h.keys, i) + unsetindex!(h.vals, i) end h.ndel = 0 h.count = 0 @@ -240,7 +240,7 @@ function ht_keyindex(h::Dict{K,V}, key) where V where K sz = length(h.keys) iter = 0 maxprobe = h.maxprobe - maxprobe < sz || throw(AssertionError()) # This error will never trigger, but is needed for terminates_locally to be valid + maxprobe < sz || throw(h) # This error will never trigger, but is needed for terminates_locally to be valid index, sh = hashindex(key, sz) keys = h.keys @@ -627,8 +627,8 @@ function _delete!(h::Dict{K,V}, index) where {K,V} @inbounds begin slots = h.slots sz = length(slots) - _unsetindex!(h.keys, index) - _unsetindex!(h.vals, index) + unsetindex!(h.keys, index) + unsetindex!(h.vals, index) # if the next slot is empty we don't need a tombstone # and can remove all tombstones that were required by the element we just deleted ndel = 1 @@ -698,11 +698,11 @@ function skip_deleted_floor!(h::Dict) idx end -@propagate_inbounds _iterate(t::Dict{K,V}, i) where {K,V} = i == 0 ? nothing : (Pair{K,V}(t.keys[i],t.vals[i]), i == typemax(Int) ? 0 : i+1) +@propagate_inbounds _iterate_dict(t::Dict{K,V}, i) where {K,V} = i == 0 ? nothing : (Pair{K,V}(t.keys[i],t.vals[i]), i == typemax(Int) ? 0 : i+1) @propagate_inbounds function iterate(t::Dict) - _iterate(t, skip_deleted(t, t.idxfloor)) + _iterate_dict(t, skip_deleted(t, t.idxfloor)) end -@propagate_inbounds iterate(t::Dict, i) = _iterate(t, skip_deleted(t, i)) +@propagate_inbounds iterate(t::Dict, i) = _iterate_dict(t, skip_deleted(t, i)) isempty(t::Dict) = (t.count == 0) length(t::Dict) = t.count @@ -782,16 +782,25 @@ end `ImmutableDict` is a dictionary implemented as an immutable linked list, which is optimal for small dictionaries that are constructed over many individual insertions. -Note that it is not possible to remove a value, although it can be partially overridden and hidden -by inserting a new value with the same key. +Note that it is not possible to remove a value, but a new value with the same +key may be added. Calling `getindex` will return the most recent value for a +particular key, but iterating will show all `key => value` pairs. - ImmutableDict(KV::Pair) +- Use `(key => value) in dict` to see if this particular combination is in the + properties set. +- Use `get(dict, key, default)` to retrieve the most recent value for a particular key. +- Iterate over `dict` to see all `key => value` pairs, including duplicate keys. + Iteration is in reverse order, from most recently added to least recently + added. -Create a new entry in the `ImmutableDict` for a `key => value` pair + ImmutableDict(KV::Pair...) - - use `(key => value) in dict` to see if this particular combination is in the properties set - - use `get(dict, key, default)` to retrieve the most recent value for a particular key +Create a new `ImmutableDict` containing the provided `key => value` pairs. + ImmutableDict(d::ImmutableDict, KV::Pair...) + +Return a new `ImmutableDict` containing all of the `key => value` pairs of `d` +as well as new entries for the provided `key => value` pairs. """ ImmutableDict ImmutableDict(KV::Pair{K,V}) where {K,V} = ImmutableDict{K,V}(KV[1], KV[2]) @@ -851,6 +860,28 @@ length(t::ImmutableDict) = count(Returns(true), t) isempty(t::ImmutableDict) = !isdefined(t, :parent) empty(::ImmutableDict, ::Type{K}, ::Type{V}) where {K, V} = ImmutableDict{K,V}() +""" + setindex(d::ImmutableDict, value, key) + +Creates a new `ImmutableDict` similar to `d` with the most recent value of the +key `key` set to `value`. + +Any existing values with the same key remain in `d`, but they are shadowed by +this new value when using [`getindex`](@ref). The existing values are still +present when iterating over `d`. + +# Examples +```jldoctest +julia> Base.setindex(Base.ImmutableDict(:a => 1), 2, :a) == Base.ImmutableDict(:a => 1, :a => 2) +true +julia> Base.setindex(Base.ImmutableDict(:a => 1), 2, :b) == Base.ImmutableDict(:a => 1, :b => 2) +true +``` +""" +function setindex(d::ImmutableDict, value, key) + ImmutableDict(d, key => value) +end + _similar_for(c::AbstractDict, ::Type{Pair{K,V}}, itr, isz, len) where {K, V} = empty(c, K, V) _similar_for(c::AbstractDict, ::Type{T}, itr, isz, len) where {T} = throw(ArgumentError("for AbstractDicts, similar requires an element type of Pair;\n if calling map, consider a comprehension instead")) @@ -874,10 +905,10 @@ struct PersistentDict{K,V} <: AbstractDict{K,V} dict::PersistentDict{K, V}, key, val) where {K, V} = @inline _keyvalueset(dict, key, val) @noinline Base.@assume_effects :nothrow :effect_free :terminates_globally KeyValue.set( dict::PersistentDict{K, V}, key::K, val::V) where {K, V} = @inline _keyvalueset(dict, key, val) - global function _keyvalueset(dict::PersistentDict{K, V}, key, val) where {K, V} + global function _keyvalueset(dict::PersistentDict{K, V}, key, @nospecialize(val)) where {K, V} trie = dict.trie h = HAMT.HashState(key) - found, present, trie, i, bi, top, hs = HAMT.path(trie, key, h, #=persistent=#true) + found, present, trie, i, bi, top, hs = HAMT.path(trie, h, #=persistent=#true) HAMT.insert!(found, present, trie, i, bi, hs, val) return new{K, V}(top) end @@ -888,7 +919,7 @@ struct PersistentDict{K,V} <: AbstractDict{K,V} global function _keyvalueset(dict::PersistentDict{K, V}, key) where {K, V} trie = dict.trie h = HAMT.HashState(key) - found, present, trie, i, bi, top, _ = HAMT.path(trie, key, h, #=persistent=#true) + found, present, trie, i, bi, top, _ = HAMT.path(trie, h, #=persistent=#true) if found && present deleteat!(trie.data, i) HAMT.unset!(trie, bi) @@ -900,7 +931,7 @@ end """ PersistentDict -`PersistentDict` is a dictionary implemented as an hash array mapped trie, +`PersistentDict` is a dictionary implemented as a hash array mapped trie, which is optimal for situations where you need persistence, each operation returns a new dictionary separate from the previous one, but the underlying implementation is space-efficient and may share storage across multiple @@ -975,7 +1006,7 @@ function in(key_val::Pair{K,V}, dict::PersistentDict{K,V}, valcmp=(==)) where {K key, val = key_val found = KeyValue.get(dict, key) found === nothing && return false - return valcmp(val, only(found)) + return valcmp(val, something(found)) end function haskey(dict::PersistentDict{K}, key::K) where K @@ -985,13 +1016,13 @@ end function getindex(dict::PersistentDict{K,V}, key::K) where {K,V} found = KeyValue.get(dict, key) found === nothing && throw(KeyError(key)) - return only(found) + return something(found) end function get(dict::PersistentDict{K,V}, key::K, default) where {K,V} found = KeyValue.get(dict, key) found === nothing && return default - return only(found) + return something(found) end @noinline function KeyValue.get(dict::PersistentDict{K, V}, key) where {K, V} @@ -1000,10 +1031,10 @@ end return nothing end h = HAMT.HashState(key) - found, present, trie, i, _, _, _ = HAMT.path(trie, key, h) + found, present, trie, i, _, _, _ = HAMT.path(trie, h) if found && present leaf = @inbounds trie.data[i]::HAMT.Leaf{K,V} - return (leaf.val,) + return Some{V}(leaf.val) end return nothing end @@ -1011,13 +1042,13 @@ end @noinline function KeyValue.get(default, dict::PersistentDict, key) found = KeyValue.get(dict, key) found === nothing && return default() - return only(found) + return something(found) end function get(default::Callable, dict::PersistentDict{K,V}, key::K) where {K,V} found = KeyValue.get(dict, key) found === nothing && return default() - return only(found) + return something(found) end function delete(dict::PersistentDict{K}, key::K) where K diff --git a/base/div.jl b/base/div.jl index 3fec8d2f5cdf3..4a355076d2afa 100644 --- a/base/div.jl +++ b/base/div.jl @@ -43,21 +43,34 @@ julia> div(4, 3, RoundFromZero) julia> div(-4, 3, RoundFromZero) -2 ``` -Because `div(x, y)` implements strictly correct truncated rounding based on the true -value of floating-point numbers, unintuitive situations can arise. For example: -```jldoctest -julia> div(6.0, 0.1) -59.0 -julia> 6.0 / 0.1 -60.0 -julia> 6.0 / big(0.1) -59.99999999999999666933092612453056361837965690217069245739573412231113406246995 -``` -What is happening here is that the true value of the floating-point number written -as `0.1` is slightly larger than the numerical value 1/10 while `6.0` represents -the number 6 precisely. Therefore the true value of `6.0 / 0.1` is slightly less -than 60. When doing division, this is rounded to precisely `60.0`, but -`div(6.0, 0.1, RoundToZero)` always truncates the true value, so the result is `59.0`. +!!! note "Floating-point numbers" + Accurate results for floating-point arguments are only guaranteed when the + mathematical value ``\\frac{x}{y}`` is within the range of exactly representable + integers for the given floating-point type, that is, when `eps(x/y) ≤ 1`, or + in other words, given `a = div(x, y)`, when `abs(a) < maxintfloat(a)`. + + Because `div(x, y)` implements strict truncated rounding based on the quotient + and remainder of the Euclidean division, and because the binary representation + of floating-point numbers (in most cases) only approximates the decimal + representation we use, unintuitive situations can arise. For example: + ```jldoctest + julia> div(6.0, 0.1) + 59.0 + julia> 6.0 / 0.1 + 60.0 + julia> 6.0 / big(0.1) + 59.99999999999999666933092612453056361837965690217069245739573412231113406246995 + ``` + What is happening here is that the binary representation of the `Float64` + number written as `0.1` is slightly larger than the numerical value ``0.1`` + (just like `0.3333333333333333` is less than ``1/3`` in decimal), while `6.0` + represents the number ``6`` precisely. Therefore the mathematical result of + `6.0` divided by (the `Float64` representation of) `0.1` is slightly less + than ``60``. The result of the floating-point division is rounded to precisely + `60.0`, but `div(6.0, 0.1, RoundToZero)` takes account of the quotient and + (here non-zero) remainder of the Euclidean division, so the result is `59.0`. + + See also [`rem`](@ref), [`divrem`](@ref). """ div(x, y, r::RoundingMode) @@ -79,16 +92,16 @@ without any intermediate rounding. - if `r == RoundToZero` (default), then the result is exact, and in the interval ``[0, |y|)`` if `x` is positive, or ``(-|y|, 0]`` otherwise. See also [`RoundToZero`](@ref). -- if `r == RoundDown`, then the result is in the interval ``[0, y)`` if `y` is positive, or - ``(y, 0]`` otherwise. The result may not be exact if `x` and `y` have different signs, and +- if `r == RoundDown`, then the result is in the interval ``[0, |y|)`` if `y` is positive, or + ``(-|y|, 0]`` otherwise. The result may not be exact if `x` and `y` have different signs, and `abs(x) < abs(y)`. See also [`RoundDown`](@ref). -- if `r == RoundUp`, then the result is in the interval ``(-y, 0]`` if `y` is positive, or - ``[0, -y)`` otherwise. The result may not be exact if `x` and `y` have the same sign, and +- if `r == RoundUp`, then the result is in the interval ``(-|y|, 0]`` if `y` is positive, or + ``[0, |y|)`` otherwise. The result may not be exact if `x` and `y` have the same sign, and `abs(x) < abs(y)`. See also [`RoundUp`](@ref). -- if `r == RoundFromZero`, then the result is in the interval ``(-y, 0]`` if `y` is positive, or - ``[0, -y)`` otherwise. The result may not be exact if `x` and `y` have the same sign, and +- if `r == RoundFromZero`, then the result is in the interval ``(-|y|, 0]`` if `x` is positive, or + ``[0, |y|)`` otherwise. The result may not be exact if `x` and `y` have the same sign, and `abs(x) < abs(y)`. See also [`RoundFromZero`](@ref). !!! compat "Julia 1.9" @@ -116,17 +129,31 @@ rem(x, y, ::RoundingMode{:Down}) = mod(x, y) rem(x, y, ::RoundingMode{:Up}) = mod(x, -y) rem(x, y, r::RoundingMode{:Nearest}) = x - y * div(x, y, r) rem(x::Integer, y::Integer, r::RoundingMode{:Nearest}) = divrem(x, y, r)[2] - +function rem(x::Integer, y::Integer, rnd::Union{typeof(RoundNearestTiesAway), + typeof(RoundNearestTiesUp)}) + divrem(x, y, rnd)[2] +end function rem(x, y, ::typeof(RoundFromZero)) signbit(x) == signbit(y) ? rem(x, y, RoundUp) : rem(x, y, RoundDown) end +function rem(x::AbstractFloat, y::AbstractFloat, rnd::Union{typeof(RoundNearestTiesAway), + typeof(RoundNearestTiesUp)}) + r = mod(x, y) + isnan(r) && return r + if !iszero(r) + m, n = abs(r) < floatmax(r)/2 ? abs.((2r, y)) : abs.((r, y/2)) + m < n && return r + m > n && return mod(x, -y) + end + rnd === RoundNearestTiesUp || signbit(x) == signbit(y) ? mod(x, -y) : r +end """ fld(x, y) Largest integer less than or equal to `x / y`. Equivalent to `div(x, y, RoundDown)`. -See also [`div`](@ref), [`cld`](@ref), [`fld1`](@ref). +See also [`div`](@ref), [`cld`](@ref), [`mod`](@ref), [`fldmod`](@ref). # Examples ```jldoctest @@ -137,21 +164,34 @@ julia> fld.(-5:5, 3)' 1×11 adjoint(::Vector{Int64}) with eltype Int64: -2 -2 -1 -1 -1 0 0 0 1 1 1 ``` -Because `fld(x, y)` implements strictly correct floored rounding based on the true -value of floating-point numbers, unintuitive situations can arise. For example: -```jldoctest -julia> fld(6.0, 0.1) -59.0 -julia> 6.0 / 0.1 -60.0 -julia> 6.0 / big(0.1) -59.99999999999999666933092612453056361837965690217069245739573412231113406246995 -``` -What is happening here is that the true value of the floating-point number written -as `0.1` is slightly larger than the numerical value 1/10 while `6.0` represents -the number 6 precisely. Therefore the true value of `6.0 / 0.1` is slightly less -than 60. When doing division, this is rounded to precisely `60.0`, but -`fld(6.0, 0.1)` always takes the floor of the true value, so the result is `59.0`. +!!! note "Floating-point numbers" + Accurate results for floating-point arguments are only guaranteed when the + mathematical value ``\\frac{x}{y}`` is within the range of exactly representable + integers for the given floating-point type, that is, when `eps(x/y) ≤ 1`, or + in other words, given `a = fld(x, y)`, when `abs(a) < maxintfloat(a)`. + + Because `fld(x, y)` implements strict floored rounding based on the quotient + and remainder of the Euclidean division, and because the binary representation + of floating-point numbers (in most cases) only approximates the decimal + representation we use, unintuitive situations can arise. For example: + ```jldoctest + julia> fld(6.0, 0.1) + 59.0 + julia> 6.0 / 0.1 + 60.0 + julia> 6.0 / big(0.1) + 59.99999999999999666933092612453056361837965690217069245739573412231113406246995 + ``` + What is happening here is that the binary representation of the `Float64` + number written as `0.1` is slightly larger than the numerical value ``0.1`` + (just like `0.3333333333333333` is less than ``1/3`` in decimal), while `6.0` + represents the number ``6`` precisely. Therefore the mathematical result of + `6.0` divided by (the `Float64` representation of) `0.1` is slightly less + than ``60``. The result of the floating-point division is rounded to precisely + `60.0`, but `fld(6.0, 0.1)` takes account of the quotient and (here non-zero) + remainder of the Euclidean division, so the result is `59.0`. + + See also [`rem`](@ref), [`divrem`](@ref). """ fld(a, b) = div(a, b, RoundDown) @@ -160,7 +200,7 @@ fld(a, b) = div(a, b, RoundDown) Smallest integer larger than or equal to `x / y`. Equivalent to `div(x, y, RoundUp)`. -See also [`div`](@ref), [`fld`](@ref). +See also [`div`](@ref), [`fld`](@ref), [`mod1`](@ref), [`cldmod1`](@ref). # Examples ```jldoctest @@ -171,9 +211,46 @@ julia> cld.(-5:5, 3)' 1×11 adjoint(::Vector{Int64}) with eltype Int64: -1 -1 -1 0 0 0 1 1 1 2 2 ``` +!!! note "Floating-point numbers" + Accurate results for floating-point arguments are only guaranteed when the + mathematical value ``\\frac{x}{y}`` is within the range of exactly representable + integers for the given floating-point type, that is, when `eps(x/y) ≤ 1`, or + in other words, given `a = cld(x, y)`, when `abs(a) < maxintfloat(a)`. + + Because `cld(x, y)` implements strict ceiled rounding based on the quotient + and remainder of the Euclidean division, and because the binary representation + of floating-point numbers (in most cases) only approximates the decimal + representation we use, unintuitive situations can arise. For example: + ```jldoctest + julia> cld(3.0, 0.3) + 11.0 + julia> 3.0 / 0.3 + 10.0 + julia> 3.0 / big(0.3) + 10.00000000000000037007434154171886050337904945061778828900298697586147515340753 + ``` + What is happening here is that the binary representation of the `Float64` + number written as `0.3` is slightly less than the numerical value ``0.3`` + (just like `0.3333333333333333` is less than ``1/3`` in decimal), while `3.0` + represents the number ``3`` precisely. Therefore the mathematical result of + `3.0` divided by (the `Float64` representation of) `0.3` is slightly larger + than ``10``. The result of the floating-point division is rounded to precisely + `10.0`, but `cld(3.0, 0.3)` takes account of the quotient and (here non-zero) + remainder of the Euclidean division, so the result is `11.0`. + + See also [`rem`](@ref), [`divrem`](@ref). """ cld(a, b) = div(a, b, RoundUp) +""" + fld1(a, b) + +Legacy spelling of `cld(a, b)` for integers. + +See also [`cld`](@ref). +""" +fld1(a, b) = cld(a, b) + # divrem """ divrem(x, y, r::RoundingMode=RoundToZero) @@ -182,7 +259,7 @@ The quotient and remainder from Euclidean division. Equivalent to `(div(x, y, r), rem(x, y, r))`. Equivalently, with the default value of `r`, this call is equivalent to `(x ÷ y, x % y)`. -See also: [`fldmod`](@ref), [`cld`](@ref). +See also [`fldmod`](@ref), [`cldmod1`](@ref), [`div`](@ref), [`rem`](@ref). # Examples ```jldoctest @@ -227,6 +304,25 @@ function divrem(a::Integer, b::Integer, r::Union{typeof(RoundUp), (d, a - d * b) end end +# For fixed-width integers the floored/ceiled quotient times `b` lies in +# `(a - b, a + b]` and so wraps on valid inputs (e.g. `fld(typemin(Int), 3)*3`); +# the remainder is correct only through wrap-cancellation. The truncated +# quotient satisfies `|d*b| <= |a|`, so the RoundToZero branch cannot overflow +# and stays on checkable operators. +function divrem(a::T, b::T, r::Union{typeof(RoundUp), + typeof(RoundDown), + typeof(RoundToZero)}) where T<:BitSigned + if r === RoundToZero + d = div(a, b) + (d, a - d * b) + elseif r === RoundDown + d = fld(a, b) + (d, a -% d *% b) + elseif r === RoundUp + d = div(a, b, r) + (d, a -% d *% b) + end +end function divrem(x::Integer, y::Integer, rnd::typeof(RoundNearest)) (q, r) = divrem(x, y) if x >= 0 @@ -286,11 +382,11 @@ end The floored quotient and modulus after division. A convenience wrapper for `divrem(x, y, RoundDown)`. Equivalent to `(fld(x, y), mod(x, y))`. -See also: [`fld`](@ref), [`cld`](@ref), [`fldmod1`](@ref). +See also [`fld`](@ref), [`mod`](@ref), [`divrem`](@ref), [`cldmod1`](@ref). """ fldmod(x, y) = divrem(x, y, RoundDown) -# We definite generic rounding methods for other rounding modes in terms of +# We define generic rounding methods for other rounding modes in terms of # RoundToZero. function div(x::Signed, y::Unsigned, ::typeof(RoundDown)) (q, r) = divrem(x, y) @@ -322,7 +418,7 @@ end # For bootstrapping purposes, we define div for integers directly. Provide the # generic signature also -div(a::T, b::T, ::typeof(RoundToZero)) where {T<:Union{BitSigned, BitUnsigned64}} = div(a, b) +div(a::T, b::T, ::typeof(RoundToZero)) where {T<:Union{BitSigned, BitUnsigned}} = div(a, b) div(a::Bool, b::Bool, r::RoundingMode) = div(a, b) # Prevent ambiguities for rm in (RoundUp, RoundDown, RoundToZero, RoundFromZero) @@ -335,10 +431,6 @@ function div(x::Bool, y::Bool, rnd::Union{typeof(RoundNearest), end fld(a::T, b::T) where {T<:Union{Integer,AbstractFloat}} = div(a, b, RoundDown) cld(a::T, b::T) where {T<:Union{Integer,AbstractFloat}} = div(a, b, RoundUp) -div(a::Int128, b::Int128, ::typeof(RoundToZero)) = div(a, b) -div(a::UInt128, b::UInt128, ::typeof(RoundToZero)) = div(a, b) -rem(a::Int128, b::Int128, ::typeof(RoundToZero)) = rem(a, b) -rem(a::UInt128, b::UInt128, ::typeof(RoundToZero)) = rem(a, b) # These are kept for compatibility with external packages overriding fld / cld. # In 2.0, packages should extend div(a, b, r) instead, in which case, these can @@ -381,7 +473,8 @@ function div(x::T, y::T, ::typeof(RoundUp)) where T<:Integer return d + (((x > 0) == (y > 0)) & (d * y != x)) end -# Real -# NOTE: C89 fmod() and x87 FPREM implicitly provide truncating float division, -# so it is used here as the basis of float div(). -div(x::T, y::T, r::RoundingMode) where {T<:AbstractFloat} = convert(T, round((x - rem(x, y, r)) / y)) +# Floats +# NB. If eps(x/y) > 1, x/y rounds to an unsafe integer which can't be floored +# or ceiled if it needs to since x/y ± 1 is not representable. +# @see https://github.com/JuliaLang/julia/issues/49450#issuecomment-3694946121 +div(x::T, y::T, r::RoundingMode) where {T<:AbstractFloat} = round(x / y - rem(x, y, r) / y) diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 13a5b35a115da..272c3856157aa 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -65,8 +65,7 @@ function. include("bindings.jl") -import .Base.Meta: quot, isexpr, unblock, unescape, uncurly -import .Base: Callable, with_output_color +import .Base.Meta: quot, isexpr, unblock, unescape using .Base: RefValue, mapany import ..CoreDocs: lazy_iterpolate @@ -159,7 +158,7 @@ Both the raw text, `.text`, and the parsed markdown, `.object`, are tracked by t Parsing of the raw text is done lazily when a request is made to render the docstring, which helps to reduce total precompiled image size. -The `.data` fields stores several values related to the docstring, such as: path, +The `.data` field stores several values related to the docstring, such as: path, linenumber, source code, and fielddocs. """ mutable struct DocStr @@ -295,16 +294,35 @@ catdoc(xs...) = vcat(xs...) const keywords = Dict{Symbol, DocStr}() -namify(@nospecialize x) = astname(x, isexpr(x, :macro))::Union{Symbol,Expr,GlobalRef} +namify(@nospecialize x) = astname(x, isexpr(x, :macro)) function astname(x::Expr, ismacro::Bool) head = x.head if head === :. ismacro ? macroname(x) : x - elseif head === :call && isexpr(x.args[1], :(::)) - return astname((x.args[1]::Expr).args[end], ismacro) +elseif head === :call && length(x.args) >= 1 && isexpr(x.args[1], :(::)) + # for documenting (x::y)(args...), extract the name from y + # otherwise, for documenting `x::y`, it will be extracted from x + astname((x.args[1]::Expr).args[end], ismacro) + elseif head === :overlay + # for documenting `Base.Experimental.@overlay mt f(args...)`, the callee is + # `Expr(:overlay, mt, f)`: extract the name from f + astname(x.args[end], ismacro) else - n = isexpr(x, (:module, :struct)) ? 2 : 1 + n = if isexpr(x, :module) + isa(x.args[1], Bool) ? 2 : 3 + elseif isexpr(x, :struct) + 2 + elseif isexpr(x, (:call, :macrocall, :function, :(=), :macro, :where, :curly, + :(::), :(<:), :(>:), :local, :global, :const, :atomic, + :copyast, :quote, :inert, :primitive, :abstract, + :escape, :var"hygienic-scope")) + # similar to is_function_def, but without -> and with various assignments, quoted statements, and miscellaneous that might be encountered in struct definitions also + 1 + else + return x # nothing to see here--bindingexpr will convert this to an error if defining a doc + end + length(x.args) < n && return x astname(x.args[n], ismacro) end end @@ -356,7 +374,7 @@ function metadata(__source__, __module__, expr, ismodule) if isa(eachex, Symbol) || isexpr(eachex, :(::)) # a field declaration if last_docstr !== nothing - push!(fields, P(namify(eachex::Union{Symbol,Expr}), last_docstr)) + push!(fields, P(namify(eachex)::Symbol, last_docstr)) last_docstr = nothing end elseif isexpr(eachex, :function) || isexpr(eachex, :(=)) @@ -383,8 +401,28 @@ function objectdoc(__source__, __module__, str, def, expr, sig = :(Union{})) @nospecialize str def expr sig binding = esc(bindingexpr(namify(expr))) docstr = esc(docexpr(__source__, __module__, lazy_iterpolate(str), metadata(__source__, __module__, expr, false))) - # Note: we want to avoid introducing line number nodes here (issue #24468) - return Expr(:block, esc(def), :($(doc!)($__module__, $binding, $docstr, $(esc(sig))))) + # Store the result of the definition and return it after documenting + docex = :($(doc!)($__module__, $binding, $docstr, $(esc(sig)))) + if def === nothing + return Expr(:block, docex) + else + exdef = esc(def) + if isexpr(def, :global, 1) && def.args[1] isa Union{Symbol,GlobalRef} + # Special case: `global x` should return nothing to avoid syntax errors with assigning to a value + val = nothing + else + if isexpr(def, :(=), 2) && isexpr(def.args[1], :curly) + # workaround for lowering bug #60001 + exdef = Expr(:block, exdef) + end + val = :val + # if-true hack: val should not be recognized as a struct field, + # including by @kwdef + exdef = Expr(:if, true, Expr(:(=), val, exdef)) + end + # Note: we want to avoid introducing line number nodes here (issue #24468) for def + return Expr(:block, exdef, docex, Expr(:if, true, val)) + end end function calldoc(__source__, __module__, str, def::Expr) @@ -408,9 +446,10 @@ function moduledoc(__source__, __module__, meta, def, def′::Expr) if def === nothing esc(:(Core.eval($name, $(quot(docex))))) else + has_version = !isa(def.args[1], Bool) def = unblock(def) - block = def.args[3].args - if !def.args[1] + block = def.args[3 + has_version].args + if !def.args[1 + has_version] pushfirst!(block, :(import Base: @doc)) end push!(block, docex) @@ -418,7 +457,9 @@ function moduledoc(__source__, __module__, meta, def, def′::Expr) end end -# Shares a single doc, `meta`, between several expressions from the tuple expression `ex`. +# Shares a single doc, `meta`, between several expressions from the tuple expression `ex` +# (but don't actually create the tuple for the result and just return the final one, +# as if this was a C++ comma operator or a block separated by `;` instead of `,`). function multidoc(__source__, __module__, meta, ex::Expr, define::Bool) @nospecialize meta out = Expr(:block) @@ -610,7 +651,13 @@ function simple_lookup_doc(ex) elseif !isa(ex, Expr) && !isa(ex, Symbol) return :($(_doc)($(typeof)($(esc(ex))))) end - binding = esc(bindingexpr(namify(ex))) + name = namify(ex) + # If namify couldn't extract a meaningful name and returned an Expr + # that can't be converted to a binding, treat it like a value + if isa(name, Expr) && !isexpr(name, :(.)) + return :($(_doc)($(typeof)($(esc(ex))))) + end + binding = esc(bindingexpr(name)) if isexpr(ex, :call) || isexpr(ex, :macrocall) || isexpr(ex, :where) sig = esc(signature(ex)) :($(_doc)($binding, $sig)) @@ -629,7 +676,6 @@ function docm(source::LineNumberNode, mod::Module, ex) else return simple_lookup_doc(ex) end - return nothing end # Drop incorrect line numbers produced by nested macro calls. docm(source::LineNumberNode, mod::Module, _, _, x...) = docm(source, mod, x...) @@ -638,7 +684,7 @@ docm(source::LineNumberNode, mod::Module, _, _, x...) = docm(source, mod, x...) # also part of a :where expression, so it unwraps the :where layers until it reaches the # "actual" expression iscallexpr(ex::Expr) = isexpr(ex, :where) ? iscallexpr(ex.args[1]) : isexpr(ex, :call) -iscallexpr(ex) = false +iscallexpr(@nospecialize ex) = false function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true) @nospecialize meta ex @@ -703,7 +749,7 @@ function _docm(source::LineNumberNode, mod::Module, meta, x, define::Bool = true # f(::T, ::U) where T where U # isexpr(x, FUNC_HEADS) && is_signature((x::Expr).args[1]) ? objectdoc(source, mod, meta, def, x::Expr, signature(x::Expr)) : - isexpr(x, [:function, :macro]) && !isexpr((x::Expr).args[1], :call) ? objectdoc(source, mod, meta, def, x::Expr) : + (isexpr(x, :function) || isexpr(x, :macro)) && !isexpr((x::Expr).args[1], :call) ? objectdoc(source, mod, meta, def, x::Expr) : iscallexpr(x) ? calldoc(source, mod, meta, x::Expr) : # Type definitions. @@ -723,7 +769,7 @@ function _docm(source::LineNumberNode, mod::Module, meta, x, define::Bool = true isexpr(x, BINDING_HEADS) && !isexpr((x::Expr).args[1], :call) ? objectdoc(source, mod, meta, def, x::Expr) : # Quoted macrocall syntax. `:@time` / `:(Base.@time)`. - isquotedmacrocall(x) ? objectdoc(source, mod, meta, def, x) : + isquotedmacrocall(x) ? objectdoc(source, mod, meta, nothing, x) : # Modules and baremodules. isexpr(x, :module) ? moduledoc(source, mod, meta, def, x::Expr) : # Document several expressions with the same docstring. `a, b, c`. @@ -837,7 +883,7 @@ Return a sorted vector of undocumented symbols in `module` (that is, lacking doc `export`, whereas `private=true` returns all symbols in the module (excluding compiler-generated hidden symbols starting with `#`). -See also: [`names`](@ref), [`Docs.hasdoc`](@ref), [`Base.ispublic`](@ref). +See also [`names`](@ref), [`Docs.hasdoc`](@ref), [`Base.ispublic`](@ref). """ function undocumented_names(mod::Module; private::Bool=false) filter!(names(mod; all=true)) do sym diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index a6abf3e384cfb..4ea39da912179 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -161,7 +161,7 @@ runtime initialization functions of external C libraries and initializing global that involve pointers returned by external libraries. See the [manual section about modules](@ref modules) for more details. -See also: [`OncePerProcess`](@ref). +See also [`OncePerProcess`](@ref). # Examples ```julia @@ -339,7 +339,6 @@ See the [manual section on variable scoping](@ref scope-of-variables) for more i See also [`for`](@ref). - # Examples ```jldoctest julia> function f() @@ -744,7 +743,7 @@ Quote an expression `expr`, returning the abstract syntax tree (AST) of `expr`. The AST may be of type `Expr`, `Symbol`, or a literal value. The syntax `:identifier` evaluates to a `Symbol`. -See also: [`Expr`](@ref), [`Symbol`](@ref), [`Meta.parse`](@ref) +See also [`Expr`](@ref), [`Symbol`](@ref), [`Meta.parse`](@ref). # Examples ```jldoctest @@ -1012,8 +1011,12 @@ kw"while" [`module`](@ref), [`struct`](@ref), [`mutable struct`](@ref), [`begin`](@ref), [`let`](@ref), [`for`](@ref) etc. -`end` may also be used when indexing to represent the last index of a -collection or the last index of a dimension of an array. +`end` may also be used when indexing with `[...]` to represent the last index of a +collection or the last index of a dimension of an array. For example, the expression +`A[end-1]` becomes `A[lastindex(A)-1]` and `A[:, end]` becomes `A[:, lastindex(A, 2)]`. +Every occurrence of `end` within the square bracket indexing syntax is lowered to a +call to [`lastindex`](@ref), using the one argument `lastindex(A)` there's only one index +argument and the two argument `lastindex(A, n)` for the n-th index argument. # Examples ```jldoctest @@ -1063,6 +1066,20 @@ end The syntax `catch e` (where `e` is any variable) assigns the thrown exception object to the given variable within the `catch` block. +```julia +try + a_dangerous_operation() +catch e + if isa(e, EOFError) + @warn "The operation failed - EOF." + elseif isa(e, OutOfMemoryError) + @warn "The operation failed - OOM." + else + rethrow() # ensure other exceptions can bubble up the call stack + end +end +``` + The power of the `try`/`catch` construct lies in the ability to unwind a deeply nested computation immediately to a much higher level in the stack of calling functions. @@ -1159,7 +1176,10 @@ kw"finally" """ break -Break out of a loop immediately. +Break out of the innermost loop or [`@label`](@ref) block immediately. + +`break` exits the innermost breakable scope, which may be a `for` or `while` loop, or +an `@label` block. # Examples ```jldoctest @@ -1177,6 +1197,23 @@ julia> while true 4 5 ``` + +Labeled break can be used to exit early from a labeled block created with [`@label`](@ref). + +```jldoctest +julia> result = @label myblock begin + for i in 1:10 + if i > 5 + break myblock i * 2 + end + end + 0 + end +12 +``` + +!!! compat "Julia 1.14" + Labeled `break` requires Julia 1.14. """ kw"break" @@ -1195,6 +1232,28 @@ julia> for i = 1:6 3 5 ``` + +Labeled continue can be used to skip to the next iteration of a labeled loop created with [`@label`](@ref). + +```jldoctest +julia> for i in 1:3 + @label inner for j in 1:3 + if j == 2 + continue inner + end + println((i, j)) + end + end +(1, 1) +(1, 3) +(2, 1) +(2, 3) +(3, 1) +(3, 3) +``` + +!!! compat "Julia 1.14" + Labeled `continue` requires Julia 1.14. """ kw"continue" @@ -1375,7 +1434,7 @@ Note that if `y` is an expression, it is only evaluated when `x` is `false`, whi Also, `y` does not need to have a boolean value. This means that `(condition) || (statement)` can be used as shorthand for `if !(condition); statement; end` for an arbitrary `statement`. -See also: [`|`](@ref), [`xor`](@ref), [`&&`](@ref). +See also [`|`](@ref), [`xor`](@ref), [`&&`](@ref). # Examples ```jldoctest @@ -1412,6 +1471,17 @@ In most cases, this simply results in a call to `convert(argtype, argvalue)`. """ kw"ccall" +""" + cglobal((symbol, library) [, type=Cvoid]) + +Obtain a pointer to a global variable in a C-exported shared library, specified +exactly as in [`ccall`](@ref). +Returns a `Ptr{Type}`, defaulting to `Ptr{Cvoid}` if no `Type` argument is supplied. +The values can be read or written by [`unsafe_load`](@ref) or [`unsafe_store!`](@ref), +respectively. +""" +Core.Intrinsics.cglobal + """ llvmcall(fun_ir::String, returntype, Tuple{argtype1, ...}, argvalue1, ...) llvmcall((mod_ir::String, entry_fn::String), returntype, Tuple{argtype1, ...}, argvalue1, ...) @@ -1431,8 +1501,6 @@ a tuple of types. All types, as well as the LLVM code, should be specified as li not as variables or expressions (it may be necessary to use `@eval` to generate these literals). -[Opaque pointers](https://llvm.org/docs/OpaquePointers.html) (written as `ptr`) are not allowed in the LLVM code. - See [`test/llvmcall.jl`](https://github.com/JuliaLang/julia/blob/v$VERSION/test/llvmcall.jl) for usage examples. @@ -1454,9 +1522,9 @@ end Usually `begin` will not be necessary, since keywords such as [`function`](@ref) and [`let`](@ref) implicitly begin blocks of code. See also [`;`](@ref). -`begin` may also be used when indexing to represent the first index of a -collection or the first index of a dimension of an array. For example, -`a[begin]` is the first element of an array `a`. +`begin` may also be used when indexing with `[...]` to represent the first index of a +collection or the first index of a dimension of an array, where it is lowered to +a call to [`firstindex`](@ref) along the relevant dimension (as determined by the context). For example, `a[begin]` is the first element of an array `a`. !!! compat "Julia 1.4" Use of `begin` as an index requires Julia 1.4 or later. @@ -1537,11 +1605,55 @@ See the manual section on [Composite Types](@ref) for more information. """ kw"mutable struct" +""" + typegroup + +`typegroup` introduces a block in which mutually recursive [`struct`](@ref) and +[`mutable struct`](@ref) definitions can refer to each other in their field types. +All types declared inside the block are atomically defined together at the end of +the block. + +```julia +typegroup + struct Node + edges::Vector{Edge} + end + struct Edge + from::Node + to::Node + end +end +``` + +Only `struct` or `mutable struct` definitions are allowed inside a `typegroup` block; +other declarations, including method definitions, are disallowed. Inner constructor +definitions are allowed inside the `struct` definitions and will semantically run +after all types have been atomically instantiated. + +!!! compat "Julia 1.14" + The `typegroup` keyword requires at least Julia 1.14. + +See the manual section on [Mutually Recursive Types](@ref) for more details. +""" +kw"typegroup" + """ new, or new{A,B,...} Special function available to inner constructors which creates a new object -of the type. The form new{A,B,...} explicitly specifies values of parameters for parametric types. +of the type. + +The form `new{A,B,...}` explicitly specifies values of parameters for parametric types. + +For constructors that have all of their type parameters after the function name, the +contents between the `{...}` are passed on to the short form `new()` automatically: + +```julia +struct NewExample{A,B} + NewExample{A,B}() where {A,B} = new() +end +``` + See the manual section on [Inner Constructor Methods](@ref man-inner-constructor-methods) for more information. """ @@ -1625,7 +1737,7 @@ devnull A type with no fields that is the type of [`nothing`](@ref). -See also: [`isnothing`](@ref), [`Some`](@ref), [`Missing`](@ref). +See also [`isnothing`](@ref), [`Some`](@ref), [`Missing`](@ref). """ Nothing @@ -1637,7 +1749,7 @@ The singleton instance of type [`Nothing`](@ref), used by convention when there A return value of `nothing` is not displayed by the REPL and similar interactive environments. -See also: [`isnothing`](@ref), [`something`](@ref), [`missing`](@ref). +See also [`isnothing`](@ref), [`something`](@ref), [`missing`](@ref). """ nothing @@ -2025,19 +2137,16 @@ TypeError """ InterruptException() -The process was stopped by a terminal interrupt (CTRL+C). - -Note that, in Julia script started without `-i` (interactive) option, -`InterruptException` is not thrown by default. Calling -[`Base.exit_on_sigint(false)`](@ref Base.exit_on_sigint) in the script -can recover the behavior of the REPL. Alternatively, a Julia script -can be started with +The exception historically thrown when the process was stopped by a terminal +interrupt (CTRL+C). A terminal interrupt is now delivered as a cancellation +of the current ^C episode's scope and observed at cancellation points as a +[`Base.CancellationRequest`](@ref); `InterruptException` remains for code +that throws it explicitly (e.g. to signal an interruption to another task). -```sh -julia -e "include(popfirst!(ARGS))" script.jl -``` - -to let `InterruptException` be thrown by CTRL+C during the execution. +Note that, in a Julia script started without the `-i` (interactive) option, +CTRL+C terminates the process by default. Calling +[`Base.exit_on_sigint(false)`](@ref Base.exit_on_sigint) in the script makes +CTRL+C observable again as a cancellation. """ InterruptException @@ -2078,7 +2187,7 @@ the runtime must do more work, `invoke` is generally also slower--sometimes sign so--than doing normal dispatch with a regular call. Be careful when using `invoke` for functions that you don't write. What definition is used -for given `argtypes` is an implementation detail unless the function is explicitly states +for given `argtypes` is an implementation detail unless the function explicitly states that calling with certain `argtypes` is a part of public API. For example, the change between `f1` and `f2` in the example below is usually considered compatible because the change is invisible by the caller with a normal (non-`invoke`) call. However, the change is @@ -2470,20 +2579,20 @@ julia> Tuple(Real[1, 2, pi]) # takes a collection tuple """ - getfield(value, name::Symbol, [boundscheck::Bool=true], [order::Symbol]) - getfield(value, i::Int, [boundscheck::Bool=true], [order::Symbol]) + getfield(value, name::Symbol, [order::Symbol], [boundscheck::Bool=true]) + getfield(value, i::Int, [order::Symbol], [boundscheck::Bool=true]) Extract a field from a composite `value` by name or position. -Optionally, an ordering can be defined for the operation. If the field was -declared `@atomic`, the specification is strongly recommended to be compatible -with the stores to that location. Otherwise, if not declared as `@atomic`, this -parameter must be `:not_atomic` if specified. +Optionally, an ordering can be defined for the operation. +If the field was declared `@atomic`, the specification is strongly recommended to be +compatible with the stores to that location. +Otherwise, if not declared as `@atomic`, this parameter must be `:not_atomic` if specified. The bounds check may be disabled, in which case the behavior of this function is undefined if `i` is out of bounds. -See also [`getproperty`](@ref Base.getproperty) and [`fieldnames`](@ref). +See also [`getproperty`](@ref Base.getproperty), [`fieldnames`](@ref). # Examples ```jldoctest @@ -2511,6 +2620,7 @@ mutable and `x` must be a subtype of `fieldtype(typeof(value), name)`. Additionally, an ordering can be specified for this operation. If the field was declared `@atomic`, this specification is mandatory. Otherwise, if not declared as `@atomic`, it must be `:not_atomic` if specified. + See also [`setproperty!`](@ref Base.setproperty!). # Examples @@ -2567,6 +2677,9 @@ optimized to the appropriate hardware instruction, otherwise it'll use a loop. !!! compat "Julia 1.7" This function requires Julia 1.7 or later. + +See also [`modifyproperty!`](@ref Base.modifyproperty!), +[`setfield!`](@ref Base.setfield!). """ modifyfield! @@ -2591,6 +2704,10 @@ instruction, otherwise it'll use a loop. !!! compat "Julia 1.7" This function requires Julia 1.7 or later. + +See also [`replaceproperty!`](@ref Base.replaceproperty!), +[`setfield!`](@ref Base.setfield!), +[`setfieldonce!`](@ref Base.setfieldonce!). """ replacefield! @@ -2609,6 +2726,10 @@ a given value, only if it was previously not set. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. + +See also [`setpropertyonce!`](@ref Base.setpropertyonce!), +[`setfield!`](@ref Base.setfield!), +[`replacefield!`](@ref Base.replacefield!). """ setfieldonce! @@ -2632,7 +2753,7 @@ Most users should not have to call this function directly -- The !!! compat "Julia 1.9" This function requires Julia 1.9 or later. -See also [`getproperty`](@ref Base.getproperty) and [`setglobal!`](@ref). +See also [`getproperty`](@ref Base.getproperty), [`setglobal!`](@ref). # Examples ```jldoctest @@ -2671,7 +2792,7 @@ cases. !!! compat "Julia 1.9" This function requires Julia 1.9 or later. -See also [`setproperty!`](@ref Base.setproperty!) and [`getglobal`](@ref) +See also [`setproperty!`](@ref Base.setproperty!), [`getglobal`](@ref) # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*\\n.*)*" @@ -2713,7 +2834,7 @@ Atomically perform the operations to simultaneously get and set a global. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`swapproperty!`](@ref Base.swapproperty!) and [`setglobal!`](@ref). +See also [`swapproperty!`](@ref Base.swapproperty!), [`setglobal!`](@ref). """ swapglobal! @@ -2726,7 +2847,7 @@ the function `op`. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`modifyproperty!`](@ref Base.modifyproperty!) and [`setglobal!`](@ref). +See also [`modifyproperty!`](@ref Base.modifyproperty!), [`setglobal!`](@ref). """ modifyglobal! @@ -2740,7 +2861,7 @@ a given value. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`replaceproperty!`](@ref Base.replaceproperty!) and [`setglobal!`](@ref). +See also [`replaceproperty!`](@ref Base.replaceproperty!), [`setglobal!`](@ref). """ replaceglobal! @@ -2754,10 +2875,74 @@ a given value, only if it was previously not set. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`setpropertyonce!`](@ref Base.setpropertyonce!) and [`setglobal!`](@ref). +See also [`setpropertyonce!`](@ref Base.setpropertyonce!), [`setglobal!`](@ref). """ setglobalonce! +""" + declare_global(module::Module, name::Symbol, strong::Bool=false, [ty::Type]) + +Declare the global `name` in module `module`. If `ty` is given, declares a +"strong" global, which cannot be replaced with a constant binding, otherwise +declares a weak global. + +See also [`global`](@ref), [`setglobal!`](@ref), [`get_binding_type`](@ref Core.get_binding_type). +""" +Core.declare_global + +""" + declare_const(module::Module, name::Symbol, [x]) + +Create or replace the constant `name` in `module` with the new value `x`. When +replacing, `x` does not need to have the same type as the original constant. + +When `x` is not given, `name` becomes an undefined constant; it cannot be read +or written to, but can be redefined. + +Unlike the syntax `const`, calling this function does not insert `Core.@latestworld` to update the world age of the current frame: +``` +julia> begin + const x = 1 + println(x) + const x = 2 + println(x) + Core.declare_const(Main, :x, 3) + println(x) + Core.@latestworld + println(x) + end +1 +2 +2 +3 +``` + +!!! compat "Julia 1.12" + This function requires Julia 1.12 or later. Redefining constants on earlier + versions of Julia is unpredictable. + +See also [`const`](@ref). +""" +Core.declare_const + +""" + define_method(module::Module, name::Symbol) + define_method(module::Module, fname_or_mt, argdata, code) + +Define a method. The 2-argument form declares a new generic function with the given `name`. +The 4-argument form defines a method for a function, where `fname_or_mt` is either the function +or its method table, `argdata` is the signature information, and `code` is the method body. + +This function is typically generated by lowering `function` definitions and should not +be called directly in most cases. + +!!! compat "Julia 1.14" + This function requires Julia 1.14 or later. + +See also [`function`](@ref). +""" +Core.define_method + """ _import(to::Module, from::Module, asname::Symbol, [sym::Symbol, imported::Bool]) @@ -3104,8 +3289,11 @@ Array{T,N}(::Missing, dims) UndefInitializer Singleton type used in array initialization, indicating the array-constructor-caller -would like an uninitialized array. See also [`undef`](@ref), -an alias for `UndefInitializer()`. +would like an uninitialized array. + +See the section in the manual on uninitialized memory. + +See also: [`undef`](@ref), an alias for `UndefInitializer()`. # Examples ```julia-repl @@ -3125,6 +3313,8 @@ Alias for `UndefInitializer()`, which constructs an instance of the singleton ty [`UndefInitializer`](@ref), used in array initialization to indicate the array-constructor-caller would like an uninitialized array. +See the section in the manual on uninitialized memory. + See also: [`missing`](@ref), [`similar`](@ref). # Examples @@ -3141,7 +3331,7 @@ undef """ Ptr{T}() -Creates a null pointer to type `T`. +Create a null pointer to type `T`. """ Ptr{T}() @@ -3175,12 +3365,36 @@ true """ (+)(x, y...) +""" + +%(x::Integer, y::Integer...) + +Addition operator with semantic wrapping. In the default Julia environment, this +is equivalent to the regular addition operator `+`. However, some users may choose to overwrite +`+` in their local environment to perform checked arithmetic instead (e.g. using +[`Experimental.@make_all_arithmetic_checked`](@ref)). The `+%` operator may be used to indicate +that wrapping behavior is semantically expected and correct and should thus be exempted from +any opt-in overflow checking. + +# Examples +```jldoctest +julia> 1 +% 20 +% 4 +25 + +julia> +%(1, 20, 4) +25 + +julia> typemax(Int) +% 1 +-9223372036854775808 +``` +""" +(+%)(x, y...) + """ -(x) Unary minus operator. -See also: [`abs`](@ref), [`flipsign`](@ref). +See also [`abs`](@ref), [`flipsign`](@ref). # Examples ```jldoctest @@ -3220,6 +3434,27 @@ julia> -(2, 4.5) """ -(x, y) +""" + -%(x::Integer, y::Integer...) + +Subtraction operator with semantic wrapping. In the default Julia environment, this +is equivalent to the regular subtraction operator `-`. However, some users may choose to overwrite +`-` in their local environment to perform checked arithmetic instead (e.g. using +[`Experimental.@make_all_arithmetic_checked`](@ref)). The `-%` operator may be used to indicate +that wrapping behavior is semantically expected and correct and should thus be exempted from +any opt-in overflow checking. + +# Examples +```jldoctest +julia> 2 -% 3 +-1 + +julia> -(typemin(Int)) +-9223372036854775808 +``` +""" +(-%)(x, y...) + """ *(x, y...) @@ -3258,6 +3493,30 @@ julia> x = [1, 2]; x'x # adjoint vector * vector """ (*)(x, y...) +""" + *%(x::Integer, y::Integer, z::Integer...) + +Multiplication operator with semantic wrapping. In the default Julia environment, this +is equivalent to the regular multiplication operator `*`. However, some users may choose to overwrite +`*` in their local environment to perform checked arithmetic instead (e.g. using +[`Experimental.@make_all_arithmetic_checked`](@ref)). The `*%` operator may be used to indicate +that wrapping behavior is semantically expected and correct and should thus be exempted from +any opt-in overflow checking. + +# Examples +```jldoctest +julia> 2 *% 7 *% 8 +112 + +julia> *(2, 7, 8) +112 + +julia> 0xff *% 0xff +0x01 +``` +""" +(*%)(x, y, z...) + """ /(x, y) @@ -3315,7 +3574,7 @@ AssertionError """ LoadError(file::AbstractString, line::Int, error) -An error occurred while [`include`](@ref Base.include)ing, [`require`](@ref Base.require)ing, or [`using`](@ref) a file. The error specifics +An error occurred while [`include`](@ref Base.include)ing, [`import`](@ref)ing, or [`using`](@ref) a file. The error specifics should be available in the `.error` field. !!! compat "Julia 1.7" @@ -3620,7 +3879,7 @@ Also note that using methods is often preferable. See also this style guide docu for more information: [Prefer exported methods over direct field access](@ref). See also [`getfield`](@ref Core.getfield), -[`propertynames`](@ref Base.propertynames) and +[`propertynames`](@ref Base.propertynames), [`setproperty!`](@ref Base.setproperty!). """ Base.getproperty @@ -3633,12 +3892,12 @@ The syntax `a.b = c` calls `setproperty!(a, :b, c)`. The syntax `@atomic order a.b = c` calls `setproperty!(a, :b, c, :order)` and the syntax `@atomic a.b = c` calls `setproperty!(a, :b, c, :sequentially_consistent)`. -!!! compat "Julia 1.8" - `setproperty!` on modules requires at least Julia 1.8. - See also [`setfield!`](@ref Core.setfield!), -[`propertynames`](@ref Base.propertynames) and +[`propertynames`](@ref Base.propertynames), [`getproperty`](@ref Base.getproperty). + +!!! compat "Julia 1.8" + `setproperty!` on modules requires at least Julia 1.8. """ Base.setproperty! @@ -3648,8 +3907,8 @@ Base.setproperty! The syntax `@atomic a.b, _ = c, a.b` returns `(c, swapproperty!(a, :b, c, :sequentially_consistent))`, where there must be one `getproperty` expression common to both sides. -See also [`swapfield!`](@ref Core.swapfield!) -and [`setproperty!`](@ref Base.setproperty!). +See also [`swapfield!`](@ref Core.swapfield!), +[`setproperty!`](@ref Base.setproperty!). """ Base.swapproperty! @@ -3665,8 +3924,8 @@ Invocation of `op(getproperty(x, f), v)` must return a value that can be stored [`setproperty!`](@ref Base.setproperty!), the `convert` function is not called automatically. -See also [`modifyfield!`](@ref Core.modifyfield!) -and [`setproperty!`](@ref Base.setproperty!). +See also [`modifyfield!`](@ref Core.modifyfield!), +[`setproperty!`](@ref Base.setproperty!). """ Base.modifyproperty! @@ -3677,7 +3936,7 @@ Perform a compare-and-swap operation on `x.f` from `expected` to `desired`, per egal. The syntax `@atomicreplace x.f expected => desired` can be used instead of the function call form. -See also [`replacefield!`](@ref Core.replacefield!) +See also [`replacefield!`](@ref Core.replacefield!), [`setproperty!`](@ref Base.setproperty!), [`setpropertyonce!`](@ref Base.setpropertyonce!). """ @@ -3689,12 +3948,12 @@ Base.replaceproperty! Perform a compare-and-swap operation on `x.f` to set it to `value` if previously unset. The syntax `@atomiconce x.f = value` can be used instead of the function call form. -See also [`setfieldonce!`](@ref Core.replacefield!), -[`setproperty!`](@ref Base.setproperty!), -[`replaceproperty!`](@ref Base.replaceproperty!). - !!! compat "Julia 1.11" This function requires Julia 1.11 or later. + +See also [`setfieldonce!`](@ref Core.setfieldonce!), +[`setproperty!`](@ref Base.setproperty!), +[`replaceproperty!`](@ref Base.replaceproperty!). """ Base.setpropertyonce! @@ -3832,7 +4091,7 @@ Unsafe pointer operations are compatible with loading and storing pointers decla `_Atomic` and `std::atomic` type in C11 and C++23 respectively. An error may be thrown if there is not support for atomically loading the Julia type `T`. -See also: [`unsafe_load`](@ref), [`unsafe_modify!`](@ref), [`unsafe_replace!`](@ref), [`unsafe_store!`](@ref), [`unsafe_swap!`](@ref) +See also [`unsafe_load`](@ref), [`unsafe_modify!`](@ref), [`unsafe_replace!`](@ref), [`unsafe_store!`](@ref), [`unsafe_swap!`](@ref) """ kw"atomic" @@ -3854,6 +4113,8 @@ This is intended for use in benchmarks that want to guarantee that `args` are actually computed. (Otherwise DCE may see that the result of the benchmark is unused and delete the entire benchmark code). +For a stronger compiler barrier, see [`Base.blackbox`](@ref). + !!! note `donotdelete` does not affect constant folding. For example, in `donotdelete(1+1)`, no add instruction needs to be executed at runtime and @@ -3885,6 +4146,39 @@ end """ Base.donotdelete +""" + Base.blackbox(x) -> x + +This function returns `x` unchanged, but treats the returned value as if it +came from an unknowable black-box source. The optimizer may not make any +assumptions about the output: it cannot be constant-folded, common-subexpression +eliminated (CSE'd), or treated as loop-invariant. +This is equivalent to `compilerbarrier(:blackbox, x)`. + +This is useful in benchmarking to prevent loop-invariant computations from being +hoisted out of benchmark loops. The output of `blackbox(x)` is opaque, so +any function call that depends on it must be re-executed each iteration. +For preventing deletion of results, see [`donotdelete`](@ref). + +!!! compat "Julia 1.14" + This method was added in Julia 1.14. + +# Examples + +```julia +function benchmark_loop(x, n) + for i in 1:n + # Without blackbox, the compiler may compute cbrt(x) once + # and reuse the result for all iterations. + y = blackbox(x) + z = cbrt(y) + donotdelete(z) + end +end +``` +""" +Base.blackbox + """ Base.compilerbarrier(setting::Symbol, val) @@ -3900,7 +4194,10 @@ Currently either of the following `setting`s is allowed: constant information on `val` * `:conditional`: the return type of this function call will be inferred with widening conditional information on `val` (see the example below) -- Any barriers on optimization aren't implemented yet +- Barriers on optimization: + * `:blackbox`: treat the returned value as if it came from an unknowable black-box + source, preventing common-subexpression elimination (CSE) and loop-invariant code motion on any computation that + depends on it. See [`blackbox`](@ref) for a convenience wrapper. !!! note This function is expected to be used with `setting` known precisely at compile-time. @@ -3960,6 +4257,30 @@ The current differences are: """ Core.finalizer +""" + Core._task(f, size) -> Task + Core._task(f, size, invoked) -> Task + +Create a new `Task` that will execute function `f` with the specified stack size. +The optional third argument `invoked` can be a `Method`, `CodeInstance`, or tuple +`Type` that will be used for optimized task invocation via `Core.invoke`. + +This builtin is an implementation detail used by the `Task` constructor and should +not be called directly by end-users. Use `Task(f)` instead. It is a low-level +interface that bypasses safety checks and initialization performed by the public +`Task` constructor. +""" +Core._task + +""" + Core.task_result_type(task) -> Type + +Return a conservative upper bound for the return type of the closure provided when +`task` was created. At runtime this builtin always returns the type `Any`; however, +inference may replace calls to it with a more precise result type. +""" +Core.task_result_type + """ ConcurrencyViolationError(msg) <: Exception diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index 34aa87bd13076..6b83a8b5ac74d 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -1,7 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -export @var - struct Binding mod::Module var::Symbol @@ -10,6 +8,19 @@ struct Binding # Normalise the binding module for module symbols so that: # Binding(Base, :Base) === Binding(Main, :Base) m = nameof(m) === v ? parentmodule(m) : m + # Walk every explicit by-name import to recover the canonical `(mod, name)` pair, + # threading `as`-renames through chained re-exports. + # `partition_restriction` only walks one hop; `binding_module` walks all hops + # but drops the name. + world = Base.get_world_counter() + while Base.invoke_in_world(world, isdefinedglobal, m, v) + bpart = Base.lookup_binding_partition(world, GlobalRef(m, v)) + Base.is_some_explicit_imported(Base.binding_kind(bpart)) || break + imported = Base.partition_restriction(bpart)::Core.Binding + next_m, next_v = imported.globalref.mod, imported.globalref.name + (next_m === m && next_v === v) && break + m, v = next_m, next_v + end new(Base.binding_module(m, v), v) end end @@ -20,28 +31,29 @@ defined(b::Binding) = invokelatest(isdefinedglobal, b.mod, b.var) resolve(b::Binding) = invokelatest(getglobal, b.mod, b.var) function splitexpr(x::Expr) - isexpr(x, :macrocall) ? splitexpr(x.args[1]) : - isexpr(x, :.) ? (x.args[1], x.args[2]) : - error("Invalid @var syntax `$x`.") + isexpr(x, :.) ? (x.args[1], x.args[2]) : error("Could not find something to document in `$x`.") end -splitexpr(s::Symbol) = Expr(:macrocall, getfield(Base, Symbol("@__MODULE__")), nothing), quot(s) +splitexpr(s::Symbol) = :($Base.@__MODULE__), quot(s) # this somewhat complex form allows deferring resolving the Module for module docstring until after the module is created splitexpr(r::GlobalRef) = r.mod, quot(r.name) -splitexpr(other) = error("Invalid @var syntax `$other`.") - -macro var(x) - esc(bindingexpr(x)) -end +splitexpr(other) = error("Could not find something to document in `$other`.") function Base.show(io::IO, b::Binding) if b.mod === Base.active_module() print(io, b.var) else - print(io, b.mod, '.', Base.isoperator(b.var) ? ":" : "", b.var) + print(io, b.mod, '.') + if Base.isoperator(b.var) + # ensures symbols are quoted right, so e.g. :(==), :(:), :+ or :- + show(io, b.var) + else + # print ordinary identifiers without any quoting + print(io, b.var) + end end end aliasof(b::Binding) = defined(b) ? (a = aliasof(resolve(b), b); defined(a) ? a : b) : b -aliasof(d::DataType, b) = Binding(d.name.module, d.name.name) -aliasof(λ::Function, b) = (m = typeof(λ).name; Binding(m.module, m.singletonname)) +aliasof(d::DataType, b) = Binding(parentmodule(d), nameof(d)) +aliasof(λ::Function, b) = Binding(parentmodule(λ), nameof(λ)) aliasof(m::Module, b) = Binding(m, nameof(m)) aliasof(other, b) = b diff --git a/base/docs/core.jl b/base/docs/core.jl index 93265416099f9..34f4fea06aede 100644 --- a/base/docs/core.jl +++ b/base/docs/core.jl @@ -2,7 +2,7 @@ module CoreDocs -import Core: @nospecialize, SimpleVector +import Core: SimpleVector struct DocLinkedList doc::SimpleVector diff --git a/base/docs/intrinsicsdocs.jl b/base/docs/intrinsicsdocs.jl index db54c1d0dc437..fe3246572bed3 100644 --- a/base/docs/intrinsicsdocs.jl +++ b/base/docs/intrinsicsdocs.jl @@ -10,7 +10,7 @@ Core.IR """ Core.IntrinsicFunction <: Core.Builtin <: Function -The `Core.IntrinsicFunction` function define some basic primitives for what defines the +The `Core.IntrinsicFunction` type defines some basic primitives for what defines the abilities and behaviors of a Julia program """ Core.IntrinsicFunction @@ -43,7 +43,7 @@ Return a `GenericMemoryRef` for a `GenericMemory`. See [`memoryref`](@ref). Core.memoryrefnew """ - Core..memoryrefoffset(::GenericMemoryRef) + Core.memoryrefoffset(::GenericMemoryRef) Return the offset index that was used to construct the `MemoryRef`. See [`memoryref`](@ref). @@ -94,7 +94,7 @@ Atomically perform the operations to simultaneously get and set a `MemoryRef` va !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`swapproperty!`](@ref Base.swapproperty!) and [`Core.memoryrefset!`](@ref). +See also [`swapproperty!`](@ref Base.swapproperty!), [`Core.memoryrefset!`](@ref). """ Core.memoryrefswap! @@ -107,7 +107,7 @@ the function `op`. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`modifyproperty!`](@ref Base.modifyproperty!) and [`Core.memoryrefset!`](@ref). +See also [`modifyproperty!`](@ref Base.modifyproperty!), [`Core.memoryrefset!`](@ref). """ Core.memoryrefmodify! @@ -120,7 +120,7 @@ Atomically perform the operations to get and conditionally set a `MemoryRef` val !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`replaceproperty!`](@ref Base.replaceproperty!) and [`Core.memoryrefset!`](@ref). +See also [`replaceproperty!`](@ref Base.replaceproperty!), [`Core.memoryrefset!`](@ref). """ Core.memoryrefreplace! @@ -134,7 +134,7 @@ a given value, only if it was previously not set. !!! compat "Julia 1.11" This function requires Julia 1.11 or later. -See also [`setpropertyonce!`](@ref Base.replaceproperty!) and [`Core.memoryrefset!`](@ref). +See also [`setpropertyonce!`](@ref Base.replaceproperty!), [`Core.memoryrefset!`](@ref). """ Core.memoryrefsetonce! @@ -216,3 +216,85 @@ Core.Intrinsics.atomic_pointermodify See [`unsafe_replace!`](@ref Base.unsafe_replace!). """ Core.Intrinsics.atomic_pointerreplace + +""" + Core.getglobal_partition(access::GlobalRef, partition::Core.BindingPartition, order::Symbol) + +Read the value from the global binding named by a binding partition, using the memory `order`. +`access` is the `GlobalRef` the read was written as, and names the binding if the read throws +an `UndefVarError` -- as [`getglobal`](@ref) reports the module and name it was given, rather +than the binding an import resolved to. + +If `partition` is an import (its restriction is another binding rather than a value or a +declared type), it is followed to the leaf partition of the access at the current world, as +[`getglobal`](@ref) would for the importing module and name. The partitions the compiler +freezes into code are already leaves, so no walk happens there. + +A deprecation the walk reaches warns as it would for [`getglobal`](@ref), but `partition` +itself does not: having named a partition, the caller owns its deprecation, and can ask for it +with [`Core.depwarn_partition`](@ref). +""" +Core.getglobal_partition + +""" + Core.setglobal_partition(partition::Core.BindingPartition, value, [order::Symbol]) + +Store to the global binding named by a resolved binding partition, as +[`setglobal!`](@ref) does for a module and name. + +Unlike the read builtins, the store targets the partition's own binding and follows no +import: assigning to a name imported from another module is an error, exactly as it is for +[`setglobal!`](@ref). +""" +Core.setglobal_partition + +""" + Core.swapglobal_partition(partition::Core.BindingPartition, value, [order::Symbol]) + +Store to the global binding named by a resolved binding partition and return its old value, +as [`swapglobal!`](@ref) does for a module and name. +""" +Core.swapglobal_partition + +""" + Core.modifyglobal_partition(partition::Core.BindingPartition, op, value, [order::Symbol]) + +Read the global binding named by a resolved binding partition, store `op(old, value)` back to +it, and return the pair `old => new`, as [`modifyglobal!`](@ref) does for a module and name. +""" +Core.modifyglobal_partition + +""" + Core.replaceglobal_partition(partition::Core.BindingPartition, expected, desired, [order::Symbol, [failorder::Symbol]]) + +Store to the global binding named by a resolved binding partition if it currently holds +`expected`, as [`replaceglobal!`](@ref) does for a module and name. +""" +Core.replaceglobal_partition + +""" + Core.setglobalonce_partition(partition::Core.BindingPartition, value, [order::Symbol, [failorder::Symbol]]) + +Store to the global binding named by a resolved binding partition if it is not already +defined, as [`setglobalonce!`](@ref) does for a module and name. +""" +Core.setglobalonce_partition + +""" + Core.isdefinedglobal_partition(partition::Core.BindingPartition, order::Symbol) + +Return whether the global binding named by a binding partition has a defined value, using the +memory `order`. An import partition is followed to its leaf, as for +[`Core.getglobal_partition`](@ref), making this the `allow_import=true` query. +""" +Core.isdefinedglobal_partition + +""" + Core.depwarn_partition(partition::Core.BindingPartition) + +Emit the deprecation warning `partition` calls for, if the command line argument `--depwarn` is +enabled (and throwing under `--depwarn=error`). The compiler emits this alongside a global read +or store it has resolved, in place of the warning [`getglobal`](@ref) or [`setglobal!`](@ref) +would have issued while resolving the name. +""" +Core.depwarn_partition diff --git a/base/env.jl b/base/env.jl index 5472456e22885..13d31195773e1 100644 --- a/base/env.jl +++ b/base/env.jl @@ -55,6 +55,21 @@ else # !windows _getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var) _hasenv(s::AbstractString) = _getenv(s) != C_NULL + if Sys.isapple() + _environ() = unsafe_load(ccall(:_NSGetEnviron, Ptr{Ptr{Cstring}}, ())) + else + const _environ_ptr = Base.OncePerProcess{Ptr{Ptr{Cstring}}}() do + # Look up `environ` in the main executable, since if Julia is loaded + # via RTLD_DEEPBIND this may resolve to NULL in our own namespace. + # (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111413) + executable_handle = ccall(:jl_dlopen, Ptr{Cvoid}, (Ptr{Cchar}, Cint), + C_NULL, Libc.Libdl.RTLD_LAZY) + return Ptr{Ptr{Cstring}}(Libc.Libdl.dlsym(executable_handle, :environ)) + end + + _environ() = unsafe_load(_environ_ptr()) + end + function access_env(onError::Function, var::AbstractString) val = _getenv(var) val == C_NULL ? onError(var) : unsafe_string(val) @@ -214,16 +229,19 @@ if Sys.iswindows() end else # !windows function iterate(::EnvDict, i=0) + envs = _environ() + envs == C_NULL && error("Failed to resolve `environ`.") while true - env = ccall(:jl_environ, Any, (Int32,), i) - env === nothing && return nothing - env = env::String + envp = unsafe_load(envs, i + 1) + envp == C_NULL && return nothing + env = unsafe_string(envp) + i += 1 m = findfirst('=', env) if m === nothing @warn "malformed environment entry" env continue end - return (Pair{String,String}(env[1:prevind(env, m)], env[nextind(env, m):end]), i+1) + return (Pair{String,String}(env[1:prevind(env, m)], env[nextind(env, m):end]), i) end end end # os-test @@ -231,7 +249,7 @@ end # os-test #TODO: Make these more efficient function length(::EnvDict) i = 0 - for (k,v) in ENV + for _ in ENV i += 1 end return i diff --git a/base/error.jl b/base/error.jl index e5eecf453ee75..c2ccd5af361c4 100644 --- a/base/error.jl +++ b/base/error.jl @@ -21,28 +21,23 @@ Throw an object as an exception. -See also: [`rethrow`](@ref), [`error`](@ref). +See also [`rethrow`](@ref), [`error`](@ref). """ throw ## native julia error handling ## -# This is `Experimental.@max_methods 2 function error end`, which is not available at this point in bootstrap. # NOTE It is important to always be able to infer the return type of `error` as `Union{}`, -# but there's a hitch when a package globally sets `@max_methods 1` and it causes inference -# for `error(::Any)` to fail (JuliaLang/julia#54029). -# This definition site `@max_methods 2` setting overrides any global `@max_methods 1` settings -# on package side, guaranteeing that return type inference on `error` is successful always. +# see issue (JuliaLang/julia#54029). Ensure that method counts are small enough with +# respect to the `max_methods` value of the function. function error end -typeof(error).name.max_methods = UInt8(2) """ error(message::AbstractString) Raise an `ErrorException` with the given message. """ -error(s::AbstractString) = throw(ErrorException(s)) -error() = throw(ErrorException("")) +error(::AbstractString) """ error(msg...) @@ -51,7 +46,14 @@ Raise an `ErrorException` with a message constructed by `string(msg...)`. """ function error(s::Vararg{Any,N}) where {N} @noinline - throw(ErrorException(Main.Base.string(s...))) + exc = if s === () + ErrorException("") + elseif s isa Tuple{AbstractString} + ErrorException(s...) + else + ErrorException(Main.Base.string(s...)) + end + throw(exc) end """ @@ -233,14 +235,12 @@ macro assert(ex, msgs...) msg = msg # pass-through elseif !isempty(msgs) && (isa(msg, Expr) || isa(msg, Symbol)) # message is an expression needing evaluating - # N.B. To reduce the risk of invalidation caused by the complex callstack involved - # with `string`, use `inferencebarrier` here to hide this `string` from the compiler. - msg = :(Main.Base.inferencebarrier(Main.Base.string)($(esc(msg)))) + msg = :($_assert_tostring($(esc(msg)))) elseif isdefined(Main, :Base) && isdefined(Main.Base, :string) && applicable(Main.Base.string, msg) msg = Main.Base.string(msg) else # string() might not be defined during bootstrap - msg = :(_assert_tostring($(Expr(:quote,msg)))) + msg = :($_assert_tostring($(Expr(:quote,msg)))) end return :($(esc(ex)) ? $(nothing) : throw(AssertionError($msg))) end diff --git a/base/errorshow.jl b/base/errorshow.jl index 1ae98378ff542..94397444ea3b5 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -91,6 +91,9 @@ function showerror(io::IO, ex::TypeError) end if ex.context == "" ctx = "in $(ex.func)" + elseif isa(ex.context, GlobalRef) + gr = ex.context + ctx = "in $(ex.func) of global binding `$(gr.mod).$(gr.name)`" elseif ex.func === :var"keyword argument" ctx = "in keyword argument $(ex.context)" else @@ -205,6 +208,8 @@ function showerror(io::IO, ex::CanonicalIndexError) print(io, "CanonicalIndexError: ", ex.func, " not defined for ", ex.type) end +# Must match `jl_inst_arg_tuple_type`: reflection through `typesof` should agree +# with actual dispatch, including egality keys for closed type-valued arguments. typesof(@nospecialize args...) = Tuple{Any[Core.Typeof(arg) for arg in args]...} function print_with_compare(io::IO, @nospecialize(a::DataType), @nospecialize(b::DataType), color::Symbol) @@ -259,7 +264,7 @@ function showerror(io::IO, ex::MethodError) is_arg_types = !isa(ex.args, Tuple) arg_types = is_arg_types ? ex.args : typesof(ex.args...) arg_types_param::SimpleVector = (unwrap_unionall(arg_types)::DataType).parameters - san_arg_types_param = Any[rewrap_unionall(a, arg_types) for a in arg_types_param] + san_arg_types_param = Any[rewrap_unionall(arg_types_param[i], arg_types) for i in 1:length(arg_types_param)] f = ex.f meth = methods_including_ambiguous(f, arg_types) if isa(meth, MethodList) && length(meth) > 1 @@ -391,6 +396,9 @@ function showerror(io::IO, ex::MethodError) "\nYou can convert to a column vector with the vec() function.") end end + if !is_arg_types && !(f isa Core.Builtin) + show_shadowed_type_hint(io, f, san_arg_types_param) + end Experimental.show_error_hints(io, ex, san_arg_types_param, kwargs) try show_method_candidates(io, ex, kwargs) @@ -423,10 +431,10 @@ function showerror_ambiguous(io::IO, meths, f, args::Type) sigfix = typeintersect(m.sig, sigfix) end if isa(unwrap_unionall(sigfix), DataType) && sigfix <: Tuple - let sigfix=sigfix - if all(m->morespecific(sigfix, m.sig), meths) + let sigfix=Core.Box(sigfix) + if all(m->morespecific(sigfix.contents, m.sig), meths) print(io, "\nPossible fix, define\n ") - show_tuple_as_call(io, :function, sigfix) + show_tuple_as_call(io, :function, sigfix.contents) else print(io, "To resolve the ambiguity, try making one of the methods more specific, or ") print(io, "adding a new method more specific than any of the existing applicable methods.") @@ -437,7 +445,7 @@ function showerror_ambiguous(io::IO, meths, f, args::Type) nothing end -#Show an error by directly calling jl_printf. +#Show an error by directly calling jl_printf and jl_static_show. #Useful in Base submodule __init__ functions where stderr isn't defined yet. function showerror_nostdio(@nospecialize(err), msg::AbstractString) stderr_stream = ccall(:jl_stderr_stream, Ptr{Cvoid}, ()) @@ -450,6 +458,198 @@ end stacktrace_expand_basepaths()::Bool = Base.get_bool_env("JULIA_STACKTRACE_EXPAND_BASEPATHS", false) === true stacktrace_contract_userdir()::Bool = Base.get_bool_env("JULIA_STACKTRACE_CONTRACT_HOMEDIR", true) === true stacktrace_linebreaks()::Bool = Base.get_bool_env("JULIA_STACKTRACE_LINEBREAKS", false) === true +stacktrace_full_loading()::Bool = Base.get_bool_env("JULIA_STACKTRACE_FULL_LOADING", false) === true + +# Print `::` with structural framing (type names, braces) in the default +# color, matching parameters and their separating commas in gray, and the +# topmost differing subtree(s) in `error_color`. +function show_type_diff(io::IO, @nospecialize(sig), @nospecialize(called), use_color::Bool, top_level::Bool=true) + show_namedtuple_diff(io, sig, called, use_color, top_level) && return nothing + params = descend_params(io, sig, called) + if params === nothing + return show_type_mismatch(io, sig, use_color, top_level) + end + top_level && print(io, "::") + sig_params, called_params, alias = params + if alias !== nothing + show_typealias_name(io, alias) + elseif sig isa TypeEq + print(io, "Type") + else + show_type_name(io, (sig::DataType).name) + end + print(io, "{") + for k in 1:length(sig_params) + k > 1 && show_separator(io, use_color) + sp = sig_params[k] + cp = called_params[k] + if sp === cp + show_type_match(io, sp, use_color) + else + show_type_diff(io, sp, cp, use_color, #=top_level=#false) + end + end + print(io, "}") +end + +function show_separator(io::IO, use_color::Bool) + if use_color + print(io, text_colors[:light_black], ", ", text_colors[:default]) + else + print(io, ", ") + end +end + +function show_type_match(io::IO, @nospecialize(ty), use_color::Bool) + if use_color + print(io, text_colors[:light_black]) + show(io, ty) + print(io, text_colors[:default]) + else + show(io, ty) + end +end + +function show_namedtuple_diff(io::IO, @nospecialize(sig), @nospecialize(called), + use_color::Bool, top_level::Bool) + sig isa DataType && called isa DataType || return false + sig.name === typename(NamedTuple) && called.name === typename(NamedTuple) || return false + length(sig.parameters) == 2 && length(called.parameters) == 2 || return false + s_syms, s_types = sig.parameters + c_syms, c_types = called.parameters + s_syms isa Tuple && c_syms isa Tuple && s_syms == c_syms || return false + s_types isa DataType && c_types isa DataType || return false + n = length(s_syms) + length(s_types.parameters) == n == length(c_types.parameters) || return false + (isvatuple(s_types) || isvatuple(c_types)) && return false + top_level && print(io, "::") + print(io, "@NamedTuple{") + for i in 1:n + i > 1 && show_separator(io, use_color) + show_sym(io, s_syms[i]) + sp = s_types.parameters[i] + cp = c_types.parameters[i] + if sp === cp + sp === Any && continue # match `show_at_namedtuple` and don't print `::Any` + print(io, "::") + show_type_match(io, sp, use_color) + else + print(io, "::") + show_type_diff(io, sp, cp, use_color, #=top_level=#false) + end + end + print(io, "}") + return true +end + +# Decide whether `sig` and `called` are pairwise-comparable at this level. +# Returns one of: +# `(sig.parameters, called.parameters, nothing)` — same name, no alias to print +# `(sa_env, ca_env, alias::GlobalRef)` — both resolve to the same alias +# `nothing` — bail; caller falls back to whole-subtree highlighting +function descend_params(io::IO, @nospecialize(sig), @nospecialize(called)) + if sig isa TypeEq && (called isa TypeEq || called isa Core.TypeEgal) + return Core.svec(type_parameter(sig)), Core.svec(type_parameter(called)), nothing + end + sig isa DataType && called isa DataType || return nothing + sig.name === called.name || return nothing + n = length(sig.parameters) + n > 0 && n == length(called.parameters) || return nothing + sig.name === typename(NamedTuple) && return nothing + (any(isvarargtype, sig.parameters) || any(isvarargtype, called.parameters)) && return nothing + sa = make_typealias(sig, io) + ca = make_typealias(called, io) + if sa === nothing && ca === nothing + return sig.parameters, called.parameters, nothing + elseif sa !== nothing && ca !== nothing && sa[1] === ca[1] + se = sa[2]::SimpleVector + ce = ca[2]::SimpleVector + length(se) == length(ce) > 0 || return nothing + return se, ce, sa[1] + else + return nothing + end +end + +function show_type_mismatch(io::IO, @nospecialize(ty), use_color::Bool, top_level::Bool) + if use_color + print(io, text_colors[error_color()]) + top_level && print(io, "::") + show(io, ty) + print(io, text_colors[:default]) + elseif top_level + print(io, "!Matched::") + show(io, ty) + else + print(io, "!Matched{") + show(io, ty) + print(io, "}") + end +end + +function _resolves_to_self(tn::Core.TypeName) + isdefined(tn, :module) || return true + m = tn.module + (isdefined(m, tn.name) && getglobal(m, tn.name) === tn.wrapper) || return false + while (p = parentmodule(m)) !== m + (isdefined(p, nameof(m)) && getglobal(p, nameof(m)) === m) || return false + m = p + end + return true +end + +function show_shadowed_type_hint(io::IO, @nospecialize(f), san_arg_types_param::Vector{Any}) + reported = IdSet{Core.TypeName}() + ft = Core.Typeof(f) + for method in methods(f) + msig = unwrap_unionall(method.sig)::DataType + mparams = msig.parameters + + # skip methods where the arity can't match the call + nargs = length(san_arg_types_param) + is_va = !isempty(mparams) && isa(mparams[end], Core.TypeofVararg) + is_va || nargs == length(mparams) - 1 || continue + + # build a list of potential shadows, max one candidate per argument + new_args = copy(san_arg_types_param) + shadows = Tuple{Core.TypeName,Core.TypeName}[] + for i in 1:nargs + # everything past nargs+1 hits vararg parameter + expected = mparams[min(i + 1, length(mparams))] + isa(expected, Core.TypeofVararg) && (expected = unwrapva(expected)) + + e_dt = unwrap_unionall(expected); isa(e_dt, DataType) || continue + a_dt = unwrap_unionall(san_arg_types_param[i]); isa(a_dt, DataType) || continue + e_tn, a_tn = e_dt.name, a_dt.name + + # actual shadowing heuristics + e_tn === a_tn && continue + e_tn.name === a_tn.name || continue + isdefined(e_tn, :module) && isdefined(a_tn, :module) || continue + new_args[i] = rewrap_unionall(expected, method.sig) + push!(shadows, (a_tn, e_tn)) + end + isempty(shadows) && continue + # make sure our suggestion hits an actual method + Tuple{ft, new_args...} <: method.sig || continue + for (a_tn, e_tn) in shadows + # don't print too many hints + a_tn in reported && continue + push!(reported, a_tn) + if !_resolves_to_self(e_tn) || !_resolves_to_self(a_tn) + print(io, "\nHint: `") + show_unquoted(io, a_tn.module); print(io, ".", a_tn.name) + print(io, "` appears to have been redefined, and methods refer to the older definition.") + else + print(io, "\nHint: You may have intended `") + show_unquoted(io, e_tn.module); print(io, ".", e_tn.name) + print(io, "` rather than `") + show_unquoted(io, a_tn.module); print(io, ".", a_tn.name) + print(io, "`.") + end + end + end +end function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) @nospecialize io @@ -460,7 +660,6 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) # Displays the closest candidates of the given function by looping over the # functions methods and counting the number of matching arguments. f = ex.f - ft = typeof(f) lines = String[] line_score = Int[] # These functions are special cased to only show if first argument is matched. @@ -473,11 +672,20 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) # pool MethodErrors for these two functions. if f === convert && !isempty(arg_types_param) at1 = arg_types_param[1] - if isType(at1) && !has_free_typevars(at1) && at1.parameters[1] isa Type - push!(funcs, (at1.parameters[1], arg_types_param[2:end])) + if isType(at1) && !has_free_typevars(at1) + at1p = type_parameter(at1) + if at1p isa Type + push!(funcs, (at1p, arg_types_param[2:end])) + end end end + # helpful when a parameterized struct has an unparameterized inner constructor + show_constructor_hint = isa(f, DataType) && (f !== f.name.wrapper) && isempty(methods(f)) + if show_constructor_hint + push!(funcs, (f.name.wrapper, arg_types_param)) + end + for (func, arg_types_param) in funcs for method in methods(func) buf = IOBuffer() @@ -506,15 +714,16 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) t_i = copy(arg_types_param) right_matches = 0 sig = sig0.parameters[2:end] + use_color = get(io, :color, false)::Bool for i = 1 : min(length(t_i), length(sig)) - i > 1 && print(iob, ", ") + i > 1 && show_separator(iob, use_color) # If isvarargtype then it checks whether the rest of the input arguments matches # the varargtype if Base.isvarargtype(sig[i]) - sigstr = (unwrapva(unwrap_unionall(sig[i])), "...") + sigstr = Core.svec(unwrapva(unwrap_unionall(sig[i])), "...") j = length(t_i) else - sigstr = (sig[i],) + sigstr = Core.svec(sig[i],) j = i end # Checks if the type of arg 1:i of the input intersects with the current method @@ -524,14 +733,18 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) # the type of the first argument is not matched. t_in === Union{} && special && i == 1 && break if t_in === Union{} - if get(io, :color, false)::Bool - let sigstr=sigstr - Base.with_output_color(Base.error_color(), iob) do iob - print(iob, "::", sigstr...) + if Base.isvarargtype(sig[i]) + if use_color + let sigstr=sigstr + Base.with_output_color(Base.error_color(), iob) do iob + print(iob, "::", sigstr...) + end end + else + print(iob, "!Matched::", sigstr...) end else - print(iob, "!Matched::", sigstr...) + show_type_diff(iob, sig[i], t_i[i], use_color) end # If there is no typeintersect then the type signature from the method is # inserted in t_i this ensures if the type at the next i matches the type @@ -539,7 +752,11 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) t_i[i] = sig[i] else right_matches += j==i ? 1 : 0 - print(iob, "::", sigstr...) + if use_color + print(iob, text_colors[:light_black], "::", sigstr..., text_colors[:default]) + else + print(iob, "::", sigstr...) + end end end special && right_matches == 0 && continue @@ -560,12 +777,12 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) for (k, sigtype) in enumerate(sig[length(t_i)+1:end]) sigtype = isvarargtype(sigtype) ? unwrap_unionall(sigtype) : sigtype if Base.isvarargtype(sigtype) - sigstr = (unwrapva(sigtype::Core.TypeofVararg), "...") + sigstr = Core.svec(unwrapva(sigtype::Core.TypeofVararg), "...") else - sigstr = (sigtype,) + sigstr = Core.svec(sigtype,) end if !((min(length(t_i), length(sig)) == 0) && k==1) - print(iob, ", ") + show_separator(iob, use_color) end if k == 1 && Base.isvarargtype(sigtype) # There wasn't actually a mismatch - the method match failed for @@ -598,7 +815,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) if !isempty(kwargs)::Bool unexpected = Symbol[] if isempty(kwords) || !(any(endswith(string(kword), "...") for kword in kwords)) - for (k, v) in kwargs + for (k, _) in kwargs if !(k::Symbol in kwords) push!(unexpected, k::Symbol) end @@ -626,7 +843,12 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) if !isempty(lines) # Display up to three closest candidates Base.with_output_color(:normal, io) do io - print(io, "\n\nClosest candidates are:") + if show_constructor_hint + print(io, "\n\nHint: constructors are defined for `", f.name.wrapper, + "`, but not for `", f, "`:") + else + print(io, "\n\nClosest candidates are:") + end permute!(lines, sortperm(line_score)) i = 0 for line in lines @@ -669,14 +891,18 @@ function _backtrace_find_and_remove_cycles(t) max_nested_cycles = 0 displayed_stackframes = [] repeated_cycles = Tuple{Int,Int,Int}[] - # First: index into `display_stackframes` to introuce the cycle bracket on + # First: index into `displayed_stackframes` to introduce the cycle bracket on # Second: length of the cycle as a count in the trace # Third: number of cycle repetitions + #= For each entry of the trace, where it ended up in `displayed_stackframes`, or 0 if it + was collapsed away, so that a cycle can be bracketed from where its turn began. =# + displayed_at = zeros(Int, length(t)) + t_curr = 1 - frame_counter = 1 while t_curr ≤ length(t) + t_this = t_curr (last_frame, n) = t[t_curr] current_hash = hash(t[t_curr]) positions = get(recorded_positions, current_hash, Int[]) @@ -702,7 +928,7 @@ function _backtrace_find_and_remove_cycles(t) if t_prev_end ≥ t_curr - 1 #= At least one cycle repeated =# ncycles = div(t_curr_end - t_prev + 1, t_cycle_length) - push!(repeated_cycles, (length(displayed_stackframes) - 1, t_cycle_length, ncycles)) + push!(repeated_cycles, (displayed_at[t_prev - 1], t_cycle_length, ncycles)) t_curr += t_cycle_length * (ncycles - 1) - 1 nnested_cycles += 1 end @@ -714,6 +940,7 @@ function _backtrace_find_and_remove_cycles(t) if ncycles == 0 push!(displayed_stackframes, (last_frame, n)) + displayed_at[t_this] = length(displayed_stackframes) end end return displayed_stackframes, repeated_cycles, max_nested_cycles @@ -784,7 +1011,7 @@ function show_processed_backtrace(io::IO, trace::Vector, num_frames::Int, repeat print_stackframe(io, frame_counter, frame, ndigits_max, max_nested_cycles, nactive_cycles, ncycle_starts, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) - frame_counter, nactive_cycles = _backtrace_print_repetition_closings!(io, i, current_cycles, frame_counter, max_nested_cycles, nactive_cycles, ndigits_max; prefix) + frame_counter, _nactive_cycles = _backtrace_print_repetition_closings!(io, i, current_cycles, frame_counter, max_nested_cycles, nactive_cycles, ndigits_max; prefix) frame_counter += 1 if i < length(trace) @@ -891,6 +1118,7 @@ Stacktrace processing pipeline: 4. `process_backtrace` filters a trace for internal implementation or redundant frames and summarizes repeated single frames: - `kwcall` frames removed - `include`-related stack frames removed + - code loading (`using`/`import`) stack frames collapsed to the frame that entered loading - Some frames that have the same location info are merged - Repeated frames are removed and summarized with a count - Output is an Any[] containing (StackFrame, count) tuple elements and this form is exposed to e.g. Revise @@ -967,7 +1195,7 @@ end function _backtrace_remove_kwcall_frames!(trace) todelete = findall(trace) do (frame, _) - code = frame.linfo + code = StackTraces.frame_mi(frame) if code isa MethodInstance def = code.def if def isa Method && def.name !== :kwcall && def.sig <: Tuple{typeof(Core.kwcall),NamedTuple,Any,Vararg} @@ -976,7 +1204,7 @@ function _backtrace_remove_kwcall_frames!(trace) # the argument list, since it has the right line number info) return true end - else + else # this branch may not be needed, from before current keyword argument handling frame.func === :kwcall && return true end return false @@ -1021,6 +1249,83 @@ function _backtrace_simplify_include_frames!(trace) keepat!(trace, kept_frames) end +# Functions making up the code loading machinery. Their frames are an implementation +# detail of `using`/`import` and are collapsed away by +# `_backtrace_simplify_loading_frames!` - see #52988. +const _LOADING_INTERNAL_FUNCS = ( + :__require, :_require_prelocked, :__require_prelocked, :_require_from_serialized, + :_tryrequire_from_serialized, :run_package_callbacks, :run_extension_callbacks, + :retry_load_extensions, :eval_import_path, :eval_import_path_all, :_eval_import, + :_eval_using) + +# The frames a collapsed run is allowed to be represented by. A run without one of +# these is left alone, so that frames which are only incidentally part of loading +# (e.g. `invoke_in_world`) are never hidden on their own. +const _LOADING_ANCHOR_FUNCS = (:require, :require_stdlib, :include_package_for_output) + +function _is_loading_frame(frame::StackFrame) + mod = parentmodule(frame) + # Hack: allow `mod === nothing` as a workaround for inlined functions, as in + # `_backtrace_simplify_include_frames!` + (mod === Base || mod === nothing) || return false + file = basename(string(frame.file)) + func = frame.func + if func === Symbol("macro expansion") + # the `@lock require_lock` and `@zone` blocks in `require`/`__require` + return file == "loading.jl" || file == "lock.jl" + elseif func === :invoke_in_world || func === :invokelatest + return file == "essentials.jl" + elseif func === :include || func === :_include + # `include`ing the package's own source, not a user-level `include` + return file == "Base.jl" || file == "loading.jl" + end + return (file == "loading.jl" || file == "module.jl") && + (func in _LOADING_INTERNAL_FUNCS || func in _LOADING_ANCHOR_FUNCS) +end + +_is_loading_anchor(frame::StackFrame) = frame.func in _LOADING_ANCHOR_FUNCS + +# For improved user experience, collapse runs of frames belonging to the code loading +# machinery down to the single frame that entered it - see #52988. Unlike hiding +# everything thrown through `require`, this keeps frames for user code that runs +# during loading (e.g. a package erroring while its source is being run). +function _backtrace_simplify_loading_frames!(trace) + stacktrace_full_loading() && return trace + kept_frames = trues(length(trace)) + i = firstindex(trace) + while i <= lastindex(trace) + if !_is_loading_frame(trace[i][1]::StackFrame) + i += 1 + continue + end + # find the extent of this run of loading frames + j = i + while j < lastindex(trace) && _is_loading_frame(trace[j+1][1]::StackFrame) + j += 1 + end + anchor = nothing + for k in i:j + frame = trace[k][1]::StackFrame + if frame.func === :require || frame.func === :require_stdlib + anchor = k + break + elseif frame.func === :include_package_for_output + # precompilation runs the package in a worker process, where the whole + # run is machinery. Represent it by its innermost frame, which at least + # reports the file being run, rather than by the long `input`/`depot_path` + # signature of `include_package_for_output` itself. + anchor = i + end + end + if anchor !== nothing + kept_frames[i:j] .= false + kept_frames[anchor] = true + end + i = j + 1 + end + keepat!(trace, kept_frames) +end + # Collapse frames that have the same location (in some cases) function _backtrace_collapse_repeated_locations!(trace) kept_frames = trues(length(trace)) @@ -1063,7 +1368,6 @@ function _backtrace_collapse_repeated_locations!(trace) params, last_params = Base.unwrap_unionall(m.sig).parameters::SimpleVector, Base.unwrap_unionall(last_m.sig).parameters::SimpleVector if last_m.nkw != 0 pos_sig_params = last_params[(last_m.nkw+2):end] - issame = true if pos_sig_params == params kept_frames[i] = false end @@ -1094,6 +1398,7 @@ end function process_backtrace(tracecount::Vector{Any}) _backtrace_remove_kwcall_frames!(tracecount) _backtrace_simplify_include_frames!(tracecount) + _backtrace_simplify_loading_frames!(tracecount) _backtrace_collapse_repeated_locations!(tracecount) return tracecount end @@ -1152,7 +1457,7 @@ function nonsetable_type_hint_handler(io, ex, arg_types, kwargs) printstyled(io, "a[1, 2]", color=:cyan) print(io, " rather than a[1][2]") elseif isType(T) - Tx = T.parameters[1] + Tx = type_parameter(T) print(io, "\nYou attempted to index the type $Tx, rather than an instance of the type. Make sure you create the type using its constructor: ") printstyled(io, "d = $Tx([...])", color=:cyan) print(io, " rather than d = $Tx") @@ -1174,6 +1479,18 @@ end Experimental.register_error_hint(string_concatenation_hint_handler, MethodError) +# Display a hint in case the user tries to use replace! on a string +# (replace! cannot modify a string in place; replace returns a new string) +function string_replace_hint_handler(@nospecialize(io::IO), ex::MethodError, arg_types::Vector{Any}, kwargs::Vector{Any}) + if ex.f === _replace! && any(@nospecialize(a) -> unwrapva(a) <: AbstractString, arg_types) + print(io, "\n`String`s cannot be modified with `replace!`. Use ") + printstyled(io, "replace", color=:cyan) + print(io, " instead, which returns a new string.") + end +end + +Experimental.register_error_hint(string_replace_hint_handler, MethodError) + # Display a hint in case the user tries to use the min or max function on an iterable # or tries to use something like `collect` on an iterator without defining either IteratorSize or length function methods_on_iterable(io, ex, arg_types, kwargs) @@ -1238,6 +1555,15 @@ end Experimental.register_error_hint(fielderror_listfields_hint_handler, FieldError) +function apply_type_unionall_hint_handler(io, ex) + @nospecialize + if ex.func === :apply_type && ex.expected === UnionAll + print(io, "\nHint: `", ex.got, "` takes no type parameters.") + end +end + +Experimental.register_error_hint(apply_type_unionall_hint_handler, TypeError) + function UndefVarError_hint(io::IO, ex::UndefVarError) var = ex.var if isdefined(ex, :scope) @@ -1291,9 +1617,11 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) warned = _UndefVarError_warnfor(io, [Base], var) if !warned - modules_to_check = (m for m in Base.loaded_modules_order - if m !== Core && m !== Base && m !== Main && m !== scope) - warned |= _UndefVarError_warnfor(io, modules_to_check, var) + let scope = scope + modules_to_check = (m for m in Base.loaded_modules_order + if m !== Core && m !== Base && m !== Main && m !== scope) + warned |= _UndefVarError_warnfor(io, modules_to_check, var) + end end warned || _UndefVarError_warnfor(io, [Core, Main], var) diff --git a/base/essentials.jl b/base/essentials.jl index f27346c4b43cb..bcab98c1a57c7 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -6,13 +6,17 @@ const Callable = Union{Function,Type} const Bottom = Union{} +blackbox(x) = compilerbarrier(:blackbox, x) + # Define minimal array interface here to help code used in macros: -length(a::Array{T, 0}) where {T} = 1 -length(a::Array{T, 1}) where {T} = getfield(a, :size)[1] -length(a::Array{T, 2}) where {T} = (sz = getfield(a, :size); sz[1] * sz[2]) -# other sizes are handled by generic prod definition for AbstractArray -length(a::GenericMemory) = getfield(a, :length) +size(a::Array) = getfield(a, :size) +length(t::AbstractArray) = (@inline; prod(size(t))) +size(a::GenericMemory) = (getfield(a, :length),) +throw_boundserror(A) = (@noinline; throw(BoundsError(A, ()))) throw_boundserror(A, I) = (@noinline; throw(BoundsError(A, I))) +throw_boundserror(A, i1, i2, I...) = (@noinline; throw(BoundsError(A, (i1, i2, I...)))) +_throw_boundserror_indices(A) = (@noinline; throw(BoundsError(A, ()))) +_throw_boundserror_indices(A, i1, I...) = (@noinline; throw(BoundsError(A, (i1, I...)))) # multidimensional getindex will be defined later on @@ -126,7 +130,7 @@ macro nospecialize(vars...) var.head = :kw end end - return Expr(:meta, :nospecialize, vars...) + return Expr(:escape, Expr(:meta, :nospecialize, vars...)) end """ @@ -143,7 +147,7 @@ macro specialize(vars...) var.head = :kw end end - return Expr(:meta, :specialize, vars...) + return Expr(:escape, Expr(:meta, :specialize, vars...)) end """ @@ -377,20 +381,21 @@ macro _nospecializeinfer_meta() return Expr(:meta, :nospecializeinfer) end -# These special checkbounds methods are defined early for bootstrapping +# These checkbounds methods are defined early for bootstrapping function checkbounds(::Type{Bool}, A::Union{Array, Memory}, i::Int) @inline ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) end -function checkbounds(A::Union{Array, GenericMemory}, i::Int) +function checkbounds(A::AbstractArray, I...) @inline - checkbounds(Bool, A, i) || throw_boundserror(A, (i,)) + checkbounds(Bool, A, I...) || _throw_boundserror_indices(A, I...) + nothing end -default_access_order(a::GenericMemory{:not_atomic}) = :not_atomic -default_access_order(a::GenericMemory{:atomic}) = :monotonic -default_access_order(a::GenericMemoryRef{:not_atomic}) = :not_atomic -default_access_order(a::GenericMemoryRef{:atomic}) = :monotonic +default_access_order(::GenericMemory{:not_atomic}) = :not_atomic +default_access_order(::GenericMemory{:atomic}) = :monotonic +default_access_order(::GenericMemoryRef{:not_atomic}) = :not_atomic +default_access_order(::GenericMemoryRef{:atomic}) = :monotonic function getindex(A::GenericMemory, i::Int) @_noub_if_noinbounds_meta @@ -461,12 +466,12 @@ julia> y === x true ``` -See also: [`round`](@ref), [`trunc`](@ref), [`oftype`](@ref), [`reinterpret`](@ref). +See also [`round`](@ref), [`trunc`](@ref), [`oftype`](@ref), [`reinterpret`](@ref). """ function convert end # ensure this is never ambiguous, and therefore fast for lookup -convert(T::Type{Union{}}, x...) = throw(ArgumentError("cannot convert a value to Union{} for assignment")) +convert(::Type{Union{}}, _...) = throw(ArgumentError("cannot convert a value to Union{} for assignment")) convert(::Type{Type}, x::Type) = x # the ssair optimizer is strongly dependent on this method existing to avoid over-specialization # in the absence of inlining-enabled @@ -503,26 +508,26 @@ end Pairs{K, V, I, A}(data, itr) where {K, V, I, A} = $(Expr(:new, :(Pairs{K, V, I, A}), :(data isa A ? data : convert(A, data)), :(itr isa I ? itr : convert(I, itr)))) Pairs{K, V}(data::A, itr::I) where {K, V, I, A} = $(Expr(:new, :(Pairs{K, V, I, A}), :data, :itr)) Pairs{K}(data::A, itr::I) where {K, I, A} = $(Expr(:new, :(Pairs{K, eltype(A), I, A}), :data, :itr)) - Pairs(data::A, itr::I) where {I, A} = $(Expr(:new, :(Pairs{eltype(I), eltype(A), I, A}), :data, :itr)) + Pairs(data::A, itr::I) where {I, A} = $(Expr(:new, :(Pairs{I !== Nothing ? eltype(I) : keytype(A), eltype(A), I, A}), :data, :itr)) end -pairs(::Type{NamedTuple}) = Pairs{Symbol, V, NTuple{N, Symbol}, NamedTuple{names, T}} where {V, N, names, T<:NTuple{N, Any}} +pairs(::Type{NamedTuple}) = Pairs{Symbol, V, Nothing, NT} where {V, NT <: NamedTuple} """ Base.Pairs(values, keys) <: AbstractDict{eltype(keys), eltype(values)} -Transforms an indexable container into a Dictionary-view of the same data. +Transform an indexable container into a Dictionary-view of the same data. Modifying the key-space of the underlying data may invalidate this object. """ Pairs -argtail(x, rest...) = rest +argtail(_, rest...) = rest """ tail(x::Tuple)::Tuple Return a `Tuple` consisting of all but the first component of `x`. -See also: [`front`](@ref Base.front), [`rest`](@ref Base.rest), [`first`](@ref), [`Iterators.peel`](@ref). +See also [`front`](@ref Base.front), [`rest`](@ref Base.rest), [`first`](@ref), [`Iterators.peel`](@ref). # Examples ```jldoctest @@ -578,7 +583,7 @@ end # remove concrete constraint on diagonal TypeVar if it comes from troot function widen_diagonal(@nospecialize(t), troot::UnionAll) - body = ccall(:jl_widen_diagonal, Any, (Any, Any), t, troot) + return ccall(:jl_widen_diagonal, Any, (Any, Any), t, troot) end function isvarargtype(@nospecialize(t)) @@ -618,7 +623,7 @@ function datatype_min_ninitialized(@nospecialize t0) if names isa Tuple return length(names) end - t = argument_datatype(types) + t = unwrap_unionall(types) t isa DataType || return 0 t.name === Tuple.name || return 0 end @@ -740,6 +745,11 @@ unsafe_convert(::Type{P}, x::Ptr) where {P<:Ptr} = convert(P, x) unsafe_convert(::Type{Ptr{UInt8}}, s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, (Any,), s) +# We don't add any _reinterpret methods until we include reinterpretarray.jl, +# but defining the function up front avoids a whole lot of invalidations when we +# do. +function _reinterpret end + """ reinterpret(::Type{Out}, x::In) @@ -829,6 +839,32 @@ Stacktrace: """ sizeof(x) = Core.sizeof(x) +""" + Core.bitsizeof(T::DataType) + Core.bitsizeof(obj) + +Logical size, in bits, of the canonical binary representation of the given `DataType` `T`, if any. +Or the logical size, in bits, of object `obj` if it is not a `DataType`. + +For primitive types, this may differ from `8*sizeof(T)` when the type uses byte-rounded storage +with unused bits in the last byte. + +# Examples +```jldoctest +julia> Core.bitsizeof(Float32) +32 + +julia> Core.bitsizeof(1.0) +64 + +julia> primitive type MyUInt63 <: Unsigned 63 end + +julia> Core.bitsizeof(MyUInt63) +63 +``` +""" +Core.bitsizeof + """ ifelse(condition::Bool, x, y) @@ -943,11 +979,72 @@ end Labels a statement with the symbolic label `name`. The label marks the end-point of an unconditional jump with [`@goto name`](@ref). + + @label expr + @label name expr + +Creates a labeled block that can be exited early with `break` / `break _ value` or +`break name value`. The block evaluates to `value` if a `break` statement is executed, +otherwise it evaluates to the result of `expr`. + +`@label expr` creates an anonymous block that participates in the default break scope: +a plain `break` (or `break _`) inside it will exit the block, just as `break` exits a loop. + +`@label name expr` creates a named block that can be exited with `break name` or +`break name value`. + +Using `_` as an explicit label name is not allowed — use `@label expr` instead. + +# Examples +```jldoctest +julia> @label begin + println("before") + break + println("after") + end +before + +julia> result = @label myblock begin + for i in 1:10 + if i > 5 + break myblock i * 2 # exits the @label block with value 12 + end + end + 0 # default value if no break + end +12 +``` """ macro label(name::Symbol) + name === :_ && error("use `@label expr` for anonymous blocks; `@label _` is not allowed") return esc(Expr(:symboliclabel, name)) end +macro label(name::Symbol, body) + name === :_ && error("use `@label expr` for anonymous blocks; `@label _ expr` is not allowed") + # If body is a syntactic loop, wrap its body in a continue block + # This allows `continue name` to work by breaking to `name#cont` + if body isa Expr && (body.head === :for || body.head === :while) + cont_name = Symbol(string(name, "#cont")) + if body.head === :for + loop_body = body.args[2] + wrapped_body = Expr(:symbolicblock, cont_name, loop_body) + body = Expr(:for, body.args[1], wrapped_body) + else # while + loop_body = body.args[2] + wrapped_body = Expr(:symbolicblock, cont_name, loop_body) + body = Expr(:while, body.args[1], wrapped_body) + end + end + return esc(Expr(:symbolicblock, name, body)) +end + +macro label(body) + # 1-arg form: create anonymous block that participates in the default break scope. + # Uses `loop-exit` as the internal label so that `break` and `break _` both target it. + return esc(Expr(:symbolicblock, Symbol("loop-exit"), body)) +end + """ @goto name @@ -973,7 +1070,12 @@ function setindex!(A::Array{Any}, @nospecialize(x), i::Int) memoryrefset!(memoryrefnew(getfield(A, :ref), i, false), x, :not_atomic, false) return A end -setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = (memoryrefset!(memoryrefnew(A, i, @_boundscheck), x, :not_atomic, @_boundscheck); A) +function setindex!(A::Memory{Any}, @nospecialize(x), i::Int) + @_noub_if_noinbounds_meta + (@_boundscheck) && checkbounds(A, i) + memoryrefset!(memoryrefnew(A, i, false), x, :not_atomic, false) + return A +end setindex!(A::MemoryRef{T}, x) where {T} = (memoryrefset!(A, convert(T, x), :not_atomic, @_boundscheck); A) setindex!(A::MemoryRef{Any}, @nospecialize(x)) = (memoryrefset!(A, x, :not_atomic, @_boundscheck); A) @@ -981,13 +1083,9 @@ setindex!(A::MemoryRef{Any}, @nospecialize(x)) = (memoryrefset!(A, x, :not_atomi getindex(v::SimpleVector, i::Int) = (@_foldable_meta; Core._svec_ref(v, i)) function length(v::SimpleVector) - @_total_meta - t = @_gc_preserve_begin v - len = unsafe_load(Ptr{Int}(pointer_from_objref(v))) - @_gc_preserve_end t - return len + Core._svec_len(v) end -firstindex(v::SimpleVector) = 1 +firstindex(::SimpleVector) = 1 lastindex(v::SimpleVector) = length(v) iterate(v::SimpleVector, i=1) = (length(v) < i ? nothing : (v[i], i + 1)) eltype(::Type{SimpleVector}) = Any @@ -1057,9 +1155,14 @@ The singleton instance of `Colon` is also a function used to construct ranges; see [`:`](@ref). """ struct Colon <: Function + Colon() = new() end const (:) = Colon() +function show(io::IO, ::Colon) + show_type_name(io, Colon.name) + print(io, "()") +end """ Val(c) @@ -1082,6 +1185,7 @@ julia> f(Val(true)) ``` """ struct Val{x} + Val{x}() where {x} = new() end Val(x) = Val{x}() @@ -1146,13 +1250,264 @@ julia> values([2]) """ values(itr) = itr +# Bootstrap operator definitions needed before _defaultctors +import Core: !== +(+)(x::Int, y::Int) = add_int(x, y) +(-)(x::Int, y::Int) = sub_int(x, y) + +""" + !(x) + +Boolean not. Implements [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic), +returning [`missing`](@ref) if `x` is `missing`. + +See also [`~`](@ref) for bitwise not. + +# Examples +```jldoctest +julia> !true +false + +julia> !false +true + +julia> !missing +missing + +julia> .![true false true] +1×3 BitMatrix: + 0 1 0 +``` +""" +!(x::Bool) = not_int(x) + +length(a::Array{T,1}) where {T} = getfield(getfield(a, :size), 1) +const C_NULL = bitcast(Ptr{Cvoid}, 0) +has_typevar(@nospecialize(t), v::TypeVar) = ccall(:jl_has_typevar, Int32, (Any, Any), t, v) !== Int32(0) + +# Check whether all type parameters are constrained by fields or other constrained tvars. +# `tvars` must be ordered from outermost to innermost `UnionAll`. +function _fieldtypes_constrain_typevars(tvars::Array{Any,1}, fts::Core.SimpleVector) + nparams = length(tvars) + n = length(fts) + i = nparams + while i !== 0 + @inbounds tv = tvars[i]::TypeVar + constrained = false + j = 1 + while j !== n + 1 + ft = fts[j] + if has_typevar(ft, tv) + constrained = true + break + end + j += 1 + end + if !constrained + j = i + 1 + remaining = nparams - i + while remaining !== 0 + @inbounds tv2 = tvars[j]::TypeVar + if has_typevar(tv2.ub, tv) + constrained = true + break + end + if tv2 === tv + constrained = false + break + end + j += 1 + remaining = remaining - 1 + end + end + constrained || return false + i -= 1 + end + return true +end + +# Return the DataType, outer-to-inner type variables, and field types for `ty`. +function _defaultctor_typeinfo(@nospecialize(ty::Type)) + nparams = 0 + ua = ty + while isa(ua, UnionAll) + nparams = nparams + 1 + ua = ua.body + end + dt = ua::DataType + tvars = Array{Any,1}(Core.undef, nparams) + ua = ty + i = 1 + while i !== nparams + 1 + @inbounds tvars[i] = (ua::UnionAll).var + ua = (ua::UnionAll).body + i = i + 1 + end + fts = ccall(:jl_get_fieldtypes, Any, (Any,), dt)::Core.SimpleVector + return dt, tvars, fts +end + +# Default constructor generation for structs without explicit inner constructors. +# Called by lowered code from struct definitions (both flisp and JuliaLowering). +# Uses jl_method_def directly with type objects, avoiding type-to-expression conversion. +function _defaultctors(@nospecialize(ty), functionloc) + typeinfo = _defaultctor_typeinfo(ty) + dt = getfield(typeinfo, 1) + tvars = getfield(typeinfo, 2) + fts = getfield(typeinfo, 3) + nparams = length(tvars) + + mod = dt.name.module + n = length(fts) + names = dt.name.names::Core.SimpleVector + src_file = ccall(:jl_symbol_name, Ptr{UInt8}, (Any,), functionloc.file) + src_line = UInt(functionloc.line) + + is_parametric = nparams !== 0 + + # Build argument names using actual field names for slot names (better debugging). + # The body references arguments via Core.Argument(N) to avoid issues with + # all-underscore field names being write-only in lowering. + self = Symbol("#ctor-self#") + argnames = Array{Any,1}(Core.undef, n + 1) + @inbounds argnames[1] = self + i = 1 + nany = 0 + while i !== n + 1 + @inbounds argnames[i + 1] = names[i]::Symbol + if fts[i] === Any + nany = nany + 1 + end + i = i + 1 + end + + if _fieldtypes_constrain_typevars(tvars, fts) + # Outer constructor: T(x::FT1, y::FT2, ...) = new{A,B,...}(x, y, ...) + # Build lambda body with direct `new`, no convert calls + if is_parametric + # new(apply_type(ty, static_parameter(1), ...), args...) + curly_args = Array{Any,1}(Core.undef, nparams + 1) + @inbounds curly_args[1] = ty + i = 1 + while i !== nparams + 1 + @inbounds curly_args[i + 1] = Expr(:static_parameter, i) + i = i + 1 + end + new_target = Expr(:curly, curly_args...) + else + new_target = Core.Argument(1) + end + new_args = Array{Any,1}(Core.undef, n + 1) + @inbounds new_args[1] = new_target + i = 1 + while i !== n + 1 + @inbounds new_args[i + 1] = Core.Argument(i + 1) + i = i + 1 + end + new_expr = Expr(:new, new_args...) + lambda = Expr(:lambda, argnames, + Expr(:block, functionloc, Expr(:return, new_expr))) + ci = ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, UInt, UInt, Int32), + lambda, mod, src_file, src_line, sub_int(UInt(0), UInt(1)), Int32(0))[1] + + # Build argdata: svec(svec(Type{ty}, ft1, ft2, ...), svec(tvars...), functionloc) + atypes_arr = Array{Any,1}(Core.undef, n + 1) + @inbounds atypes_arr[1] = Core.apply_type(Type, ty) + i = 1 + while i !== n + 1 + @inbounds atypes_arr[i + 1] = fts[i] + i = i + 1 + end + outer_atypes = Core.svec(atypes_arr...) + outer_tvars = Core.svec(tvars...) + argdata = Core.svec(outer_atypes, outer_tvars, functionloc) + ccall(:jl_method_def, Any, (Any, Ptr{Nothing}, Any, Any), + argdata, C_NULL, ci, mod) + + # For non-parametric types where all fields are Any, outer constructor suffices + if nparams === 0 + all_any = true + i = 1 + while i !== n + 1 + if fts[i] !== Any + all_any = false + break + end + i = i + 1 + end + if all_any + return + end + end + end + + # Inner constructor: (::Type{T{A,B,...}})(x, y, ...) with convert calls + # Build lambda body using Core.Argument references + nstmts = ((n - nany) + (n - nany)) + 1 + body_args = Array{Any,1}(Core.undef, nstmts) + new_args = Array{Any,1}(Core.undef, n) + i = 1 + bidx = 1 + while i !== n + 1 + ft = fts[i] + if ft === Any + @inbounds new_args[i] = Core.Argument(i + 1) + else + # Use an isa check to avoid depending on convert inlining. + # This matches the old convert-for-type-decl pattern: + # isa(arg, fieldtype(self, i)) ? arg : convert(fieldtype(self, i), arg) + # The isa check is important because user code may define ambiguous + # convert methods (e.g. convert(::Any, v::T) = v) that prevent the + # optimizer from inlining convert(fieldtype(self, i), arg) when the + # field type is Any after specialization. + ft_expr = Expr(:call, GlobalRef(Core, :fieldtype), Core.Argument(1), i) + ft_ssa = Expr(:ssavalue, bidx) + cnvt_ssa = Expr(:ssavalue, bidx + 1) + isa_check = Expr(:call, GlobalRef(Core, :isa), Core.Argument(i + 1), ft_ssa) + convert_expr = Expr(:call, GlobalRef(Base, :convert), ft_ssa, Core.Argument(i + 1)) + @inbounds body_args[bidx] = Expr(:(=), ft_ssa, ft_expr) + @inbounds body_args[bidx + 1] = Expr(:(=), cnvt_ssa, Expr(:if, isa_check, + Core.Argument(i + 1), convert_expr)) + @inbounds new_args[i] = cnvt_ssa + bidx = bidx + 2 + end + i = i + 1 + end + body_args[nstmts] = Expr(:return, Expr(:new, Core.Argument(1), new_args...)) + lambda = Expr(:lambda, argnames, + Expr(:block, functionloc, body_args...)) + ci = ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, UInt, UInt, Int32), + lambda, mod, src_file, src_line, sub_int(UInt(0), UInt(1)), Int32(0))[1] + + # Build argdata: svec(svec(UnionAll...Type{dt}..., Any, Any, ...), svec(), functionloc) + inner_atypes_arr = Array{Any,1}(Core.undef, n + 1) + typedt = Core.apply_type(Type, dt) + i = nparams + while i !== 0 + @inbounds typedt = UnionAll(tvars[i], typedt) + i = i - 1 + end + @inbounds inner_atypes_arr[1] = typedt + i = 1 + while i !== n + 1 + @inbounds inner_atypes_arr[i + 1] = Any + i = i + 1 + end + inner_atypes = Core.svec(inner_atypes_arr...) + inner_tvars = Core.svec() + argdata = Core.svec(inner_atypes, inner_tvars, functionloc) + ccall(:jl_method_def, Any, (Any, Ptr{Nothing}, Any, Any), + argdata, C_NULL, ci, mod) + return +end + """ Missing A type with no fields whose singleton instance [`missing`](@ref) is used to represent missing values. -See also: [`skipmissing`](@ref), [`nonmissingtype`](@ref), [`Nothing`](@ref). +See also [`skipmissing`](@ref), [`nonmissingtype`](@ref), [`Nothing`](@ref). """ struct Missing end @@ -1161,7 +1516,7 @@ struct Missing end The singleton instance of type [`Missing`](@ref) representing a missing value. -See also: [`NaN`](@ref), [`skipmissing`](@ref), [`nonmissingtype`](@ref). +See also [`NaN`](@ref), [`skipmissing`](@ref), [`nonmissingtype`](@ref). """ const missing = Missing() @@ -1170,7 +1525,7 @@ const missing = Missing() Indicate whether `x` is [`missing`](@ref). -See also: [`skipmissing`](@ref), [`isnothing`](@ref), [`isnan`](@ref). +See also [`skipmissing`](@ref), [`isnothing`](@ref), [`isnan`](@ref). """ ismissing(x) = x === missing @@ -1231,7 +1586,7 @@ to obtain a definitive answer. See also [`iterate`](@ref), [`isempty`](@ref) """ -isdone(itr, state...) = missing +isdone(_, _...) = missing """ iterate(iter [, state])::Union{Nothing, Tuple{Any, Any}} @@ -1266,7 +1621,10 @@ is newer than the world currently running. The `@world` macro is primarily used in the printing of bindings that are no longer available in the current world. -## Example +!!! compat "Julia 1.12" + This functionality requires at least Julia 1.12. + +# Examples ```julia-repl julia> struct Foo; a::Int; end Foo @@ -1282,9 +1640,6 @@ Foo julia> fold @world(Foo, 26866)(1) ``` - -!!! compat "Julia 1.12" - This functionality requires at least Julia 1.12. """ macro world(sym, world) if world == :∞ @@ -1305,7 +1660,7 @@ end _resolve_in_world(world::Integer, gr::GlobalRef) = invoke_in_world(UInt(world), Core.getglobal, gr.mod, gr.name) -# Special constprop heuristics for various binary opes +# Special constprop heuristics for various binary ops typename(typeof(function + end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function - end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function * end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC diff --git a/base/experimental.jl b/base/experimental.jl index 31cc12f1ab796..f09373c647e78 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -34,14 +34,14 @@ Base.axes(C::Const) = axes(C.a) """ @aliasscope expr -Allows the compiler to assume that all `Const`s are not being modified through stores +Allow the compiler to assume that all `Const`s are not being modified through stores within this scope, even if the compiler can't prove this to be the case. !!! warning Experimental API. Subject to change without deprecation. """ macro aliasscope(body) - sym = gensym() + sym = :aliasscope_result quote $(Expr(:aliasscope)) $sym = $(esc(body)) @@ -51,31 +51,47 @@ macro aliasscope(body) end -function sync_end(c::Channel{Any}) +function sync_end(c::Channel{Any}, src::Union{Nothing, Base.CancellationTokenSource}=nothing) if !isready(c) # there must be at least one item to begin with close(c) return end nremaining::Int = 0 - while true - event = take!(c) - if event === :__completion__ - nremaining -= 1 - if nremaining == 0 - break - end - else - nremaining += 1 - schedule(Task(()->begin - try - wait(event) - put!(c, :__completion__) - catch e - close(c, e) + try + while true + event = take!(c) + if event === :__completion__ + nremaining -= 1 + if nremaining == 0 + break end - end)) + else + nremaining += 1 + # The watcher must survive cancellation of the enclosing + # scope to deliver the completion (the `cancel` keyword is + # part of the waitable interface). + schedule(Task(()->begin + try + wait(event; cancel = nothing) + put!(c, :__completion__; cancel = nothing) + catch e + close(c, e) + end + end)) + end + end + catch e + # Per this macro's contract the exception (a child failure delivered + # via `close(c, e)`, or a cancellation of this block's scope) is + # rethrown immediately - we do not wait for the children to finish + # dying, but we cancel the block's own source so that all children + # observe the cancellation through the token tree. + if src !== nothing + e isa Base.CancellationRequest ? Base.cancel!(src, e) : Base.cancel!(src) end + close(c, e isa Exception ? e : ErrorException("sync_end interrupted")) + rethrow() end close(c) nothing @@ -87,8 +103,9 @@ end Wait until all lexically-enclosed uses of [`@async`](@ref), [`@spawn`](@ref Threads.@spawn), `Distributed.@spawnat` and `Distributed.@distributed` are complete, or at least one of them has errored. The first exception is immediately -rethrown. It is the responsibility of the user to cancel any still-running operations -during error handling. +rethrown; the block's cancellation scope is cancelled at the same time, so +still-running operations spawned within observe the failure through their +cancellation points rather than running unsupervised (they are not awaited). !!! Note This is different to [`@sync`](@ref) in that errors from wrapped tasks are thrown immediately, @@ -99,10 +116,18 @@ during error handling. """ macro sync(block) var = esc(sync_varname) + # like Base.@sync, the block runs in a new dynamic scope carrying the + # token of a fresh cancellation source; on the fail-fast path (and on + # cancellation from outside) the source is cancelled, reaching all + # children through the token tree without awaiting them + scoped_block = Expr(:tryfinally, esc(block), nothing, + :(Base.Scope(Core.current_scope()::Union{Nothing, Base.Scope}, + Base.CANCEL_TOKEN => Base.CancellationToken(var"#sync_src#")))) quote - let $var = Channel(Inf) - v = $(esc(block)) - sync_end($var) + let var"#sync_src#" = Base.CancellationTokenSource(Base.default_cancel_token()), + $var = Channel(Inf) + v = $scoped_block + sync_end($var, var"#sync_src#") v end end @@ -295,18 +320,18 @@ Closest candidates are: `if isdefined(Base.Experimental, :register_error_hint) ... end` block. """ function register_error_hint(@nospecialize(handler), @nospecialize(exct::Type)) - list = get!(Vector{Any}, _hint_handlers, exct) - push!(list, handler) + list = get!(Vector{Any}, _hint_handlers, Core.typename(exct)) + push!(list, (exct, handler)) return nothing end -const _hint_handlers = IdDict{Type,Vector{Any}}() +const _hint_handlers = IdDict{Core.TypeName,Vector{Any}}() """ Experimental.show_error_hints(io, ex, args...) Invoke all handlers from [`Experimental.register_error_hint`](@ref) for the particular -exception type `typeof(ex)`. `args` must contain any other arguments expected by +exception type `typeof(ex)` and all of its supertypes. `args` must contain any other arguments expected by the handler for that type. !!! compat "Julia 1.5" @@ -315,15 +340,21 @@ the handler for that type. This interface is experimental and subject to change or removal without notice. """ function show_error_hints(io, ex, args...) - hinters = get(_hint_handlers, typeof(ex), nothing) - isnothing(hinters) && return - for handler in hinters - try - @invokelatest handler(io, ex, args...) - catch - tn = typeof(handler).name - @error "Hint-handler $handler for $(typeof(ex)) in $(tn.module) caused an error" exception=current_exceptions() + @nospecialize + ex_supertype = typeof(ex) + while ex_supertype != Any + hinters = get(_hint_handlers, Core.typename(ex_supertype), Any[]) + for (exct, handler) in hinters + ex isa exct || continue + try + # TODO: deal with handlers accepting different signatures? + @invokelatest handler(io, ex, args...) + catch + tn = typeof(handler).name + @error "Hint-handler $handler for $(ex_supertype) in $(tn.module) caused an error" exception=current_exceptions() + end end + ex_supertype = supertype(ex_supertype) end end @@ -332,11 +363,13 @@ include("opaque_closure.jl") """ Base.Experimental.@overlay mt def + Base.Experimental.@overlay mt begin defs... end Define a method and add it to the method table `mt` instead of to the global method table. This can be used to implement a method override mechanism. Regular compilation will not consider these methods, and you should customize the compilation flow to look in these -method tables (e.g., using [`Core.Compiler.OverlayMethodTable`](@ref)). +method tables (e.g., using [`Core.Compiler.OverlayMethodTable`](@ref)). The block form +overlays every definition in the block; definitions may carry docstrings and other macros. !!! note Please be aware that when defining overlay methods using `@overlay`, it is not necessary @@ -361,8 +394,18 @@ method tables (e.g., using [`Core.Compiler.OverlayMethodTable`](@ref)). """ macro overlay(mt, def) inner = Base.unwrap_macrocalls(def) - is_function_def(inner) || error("@overlay requires a function definition") - overlay_def!(mt, inner) + if isexpr(inner, :block) + # `@overlay mt begin ... end`: overlay every definition in the block + for arg in inner.args + isa(arg, LineNumberNode) && continue + innerarg = Base.unwrap_macrocalls(arg) + is_function_def(innerarg) || error("@overlay requires a function definition") + overlay_def!(mt, innerarg) + end + else + is_function_def(inner) || error("@overlay requires a function definition") + overlay_def!(mt, inner) + end return esc(def) end @@ -387,7 +430,7 @@ For a detailed definition of `:consistent`-cy, consult the corresponding section !!! note Note that the requirements for `:consistent`-cy include not only that the return values are egal, but also that the manner of termination is the same. However, it's important - to aware that when they throw exceptions, the exceptions themselves don't necessarily + to be aware that when they throw exceptions, the exceptions themselves don't necessarily have to be egal. In other words, if ``fᵢ(x)`` throws an exception, ``fᵢ′(x)`` is required to also throw one, but the exact exceptions may differ. @@ -458,6 +501,29 @@ without adding them to the global method table. """ :@MethodTable +""" + Experimental.@make_all_arithmetic_checked() + +This macro defines methods that overwrite the base definition of basic arithmetic (+,-,*), +to use their checked variants instead. Explicitly overflowing arithmetic operators (+%,-%,*%) +are not affected. + +!!! warning + This macro is temporary and will likely be replaced by a more complete mechanism in the + future. It is subject to change or removal without notice. +""" +macro make_all_arithmetic_checked() + esc(quote + Base.:(-)(x::Base.BitInteger) = Base.Checked.checked_neg(x) + Base.:(-)(x::Base.Int, y::Base.Int) = Base.Checked.checked_sub(x, y) + Base.:(-)(x::T, y::T) where {T<:Base.BitInteger} = Base.Checked.checked_sub(x, y) + Base.:(+)(x::Base.Int, y::Base.Int) = Base.Checked.checked_add(x, y) + Base.:(+)(x::T, y::T) where {T<:Base.BitInteger} = Base.Checked.checked_add(x, y) + Base.:(*)(x::T, y::T) where {T<:Base.BitInteger} = Base.Checked.checked_mul(x, y) + Base.:(-)(x::Base.AbstractChar, y::Base.AbstractChar) = Base.Int(x) - Base.Int(y) + end) +end + """ Base.Experimental.make_io_thread() @@ -491,7 +557,10 @@ function entrypoint(@nospecialize(f), @nospecialize(argtypes::Tuple)) end function entrypoint(@nospecialize(argt::Type)) - ccall(:jl_add_entrypoint, Int32, (Any,), argt) + # Only add to entrypoint list if we're generating output and in trim mode + if ccall(:jl_generating_output, Cint, ()) != 0 + Base.Compiler.add_entrypoint(argt) + end nothing end @@ -534,7 +603,7 @@ This metric is only updated when `t` yields or completes unless `t` is the curre which it will be updated continuously. See also [`Base.Experimental.task_wall_time_ns`](@ref). -Returns `nothing` if task timings are not enabled. +Return `nothing` if task timings are not enabled. See [`Base.Experimental.task_metrics`](@ref). !!! note "This metric is from the Julia scheduler" @@ -549,7 +618,7 @@ function task_running_time_ns(t::Task=current_task()) if t == current_task() # These metrics fields can't update while we're running. # But since we're running we need to include the time since we last started running! - return t.running_time_ns + (time_ns() - t.last_started_running_at) + return t.running_time_ns +% (time_ns() -% t.last_started_running_at) else return t.running_time_ns end @@ -563,7 +632,7 @@ This is the time since the task first entered the run queue until the time at wh completed, or until the current time if the task has not yet completed. See also [`Base.Experimental.task_running_time_ns`](@ref). -Returns `nothing` if task timings are not enabled. +Return `nothing` if task timings are not enabled. See [`Base.Experimental.task_metrics`](@ref). !!! compat "Julia 1.12" @@ -574,8 +643,8 @@ function task_wall_time_ns(t::Task=current_task()) start_at = t.first_enqueued_at start_at == 0 && return UInt64(0) end_at = t.finished_at - end_at == 0 && return time_ns() - start_at - return end_at - start_at + end_at == 0 && return time_ns() -% start_at + return end_at -% start_at end # wait_with_timeout @@ -584,44 +653,13 @@ end # specification of a timeout. This is experimental as it will likely # be dropped when a cancellation framework is added. # -# The parallel behavior of wait_with_timeout is specified here. There -# are three concurrent entities that can interact: -# 1. Task W: the task that calls wait_with_timeout. -# 2. Task T: the task created to handle a timeout. -# 3. Task N: the task that notifies the Condition being waited on. -# -# Typical flow: -# - W enters the Condition's wait queue. -# - W creates T and stops running (calls wait()). -# - T, when scheduled, waits on a Timer. -# - Two common outcomes: -# - N notifies the Condition. -# - W starts running, closes the Timer, sets waiter_left and returns -# the notify'ed value. -# - The closed Timer throws an EOFError to T which simply ends. -# - The Timer expires. -# - T starts running and locks the Condition. -# - T confirms that waiter_left is unset and that W is still in the -# Condition's wait queue; it then removes W from the wait queue, -# sets dosched to true and unlocks the Condition. -# - If dosched is true, T schedules W with the special :timed_out -# value. -# - T ends. -# - W runs and returns :timed_out. -# -# Some possible interleavings: -# - N notifies the Condition but the Timer expires and T starts running -# before W: -# - W closing the expired Timer is benign. -# - T will find that W is no longer in the Condition's wait queue -# (which is protected by a lock) and will not schedule W. -# - N notifies the Condition; W runs and calls wait on the Condition -# again before the Timer expires: -# - W sets waiter_left before leaving. When T runs, it will find that -# waiter_left is set and will not schedule W. -# -# The lock on the Condition's wait queue and waiter_left together -# ensure proper synchronization and behavior of the tasks involved. +# Implemented as a `park!` over the condition, the governing cancellation +# source, and a `Base.TimeoutWait` deadline (see base/park.jl and +# base/asyncevent.jl): the deadline's claimer arbitrates against notifies +# and interrupters through the single wake-claim CAS on the waiting +# task's `waiting_on`, and the non-canonical waitable shape makes the +# entry cache hand out a fresh, single-use entry - which is exactly what +# makes the deadline's specific-wait claim sound. """ wait_with_timeout(c::GenericCondition; first::Bool=false, timeout::Real=0.0) @@ -635,54 +673,166 @@ If `timeout` is specified, cancel the `wait` when it expires and return `:timed_out`. The minimum value for `timeout` is 0.001 seconds, i.e. 1 millisecond. """ -function wait_with_timeout(c::GenericCondition; first::Bool=false, timeout::Real=0.0) +function wait_with_timeout(c::GenericCondition; first::Bool=false, timeout::Real=0.0, + cancel::Base.CancelTokenArg=Base.DEFAULT_CANCEL) + tok = Base.check_cancel_arg(cancel) + src = Base.cancel_source(tok) ct = current_task() - Base._wait2(c, ct, first) - token = Base.unlockall(c.lock) - - timer::Union{Timer, Nothing} = nothing - waiter_left::Union{Threads.Atomic{Bool}, Nothing} = nothing if timeout > 0.0 - timer = Timer(timeout) - waiter_left = Threads.Atomic{Bool}(false) - # start a task to wait on the timer - t = Task() do - try - wait(timer) - catch e - # if the timer was closed, the waiting task has been scheduled; do nothing - e isa EOFError && return - end - dosched = false - lock(c.lock) - # Confirm that the waiting task is still in the wait queue and remove it. If - # the task is not in the wait queue, it must have been notified already so we - # don't do anything here. - if !waiter_left[] && ct.queue === c.waitq - dosched = true - Base.list_deletefirst!(c.waitq, ct) - end - unlock(c.lock) - # send the waiting task a timeout - dosched && schedule(ct, :timed_out) - end - t.sticky = false - Threads._spawn_set_thrpool(t, :interactive) - schedule(t) + tw = Base.TimeoutWait(timeout) + ws = src === nothing ? (c, tw) : (c, Base.SourceWait(src, 0x00), tw) + else + ws = src === nothing ? (c,) : (c, Base.SourceWait(src, 0x00)) end - - try - res = wait() - if timer !== nothing - close(timer) - waiter_left[] = true - end - return res + # non-canonical shapes get a fresh entry - exactly what makes the + # deadline claimer's specific-wait CAS sound + w = Base.acquire_wait_entry!(ct, ws) + if !Base.park!(ws, w, first) + Base.withdraw!(ws, w, Base.WAKE_FIRED) + src === nothing || Base.checkcancel(src) + error("park fired without a cancelled source") + end + lockstate = Base.unlockall(c.lock) + r = try + Base.wait_safe_interrupt(ws, w) catch - q = ct.queue; q === nothing || Base.list_deletefirst!(q::IntrusiveLinkedList{Task}, ct) + Base.relockall(c.lock, lockstate) rethrow() - finally - Base.relockall(c.lock, token) + end + Base.relockall(c.lock, lockstate) + Base.withdraw!(ws, w, Base.WAKE_VALUE) # closes the timer, retires + return r +end + +""" + Base.Experimental.@reexport using Module + +Automatically re-export all exported names from a module when using it. + +# Examples + +```jldoctest +julia> module A + export foo + foo() = "foo from A" + end +A + +julia> module B + using Base.Experimental: @reexport + @reexport using ..A + # Now B exports foo, even though it's defined in A + end +B + +julia> using .B + +julia> foo() +"foo from A" +``` + +!!! warning + This interface is experimental and subject to change or removal without notice. +""" +macro reexport(ex) + if !Meta.isexpr(ex, :using) || isempty(ex.args) + error("@reexport must be used with a `using` statement, e.g., `@reexport using MyModule`") + end + + # Check for `using Foo: x, y` syntax (not supported) + if any(arg -> Meta.isexpr(arg, :(:)), ex.args) + error("@reexport does not support `using Module: names` syntax") + end + + # Generate _eval_using calls for each module in the using statement + calls = Expr(:block) + for mod_path in ex.args + push!(calls.args, :($(Core._eval_using)($(__module__), $(QuoteNode(mod_path)), $(Base.JL_MODULE_USING_REEXPORT)))) + end + push!(calls.args, Expr(:latestworld)) + push!(calls.args, :nothing) + + return esc(calls) +end + +struct VersionedLower + ver::VersionNumber +end + +function (vp::VersionedLower)(@nospecialize(code), mod::Module, + file="none", line=0, world=typemax(Csize_t), warn=false) + if !isdefined(Base, :JuliaLowering) + if vp.ver === VERSION + return Core._parse + end + error("JuliaLowering module is required for syntax version $(vp.ver), but it is not loaded.") + end + Base.JuliaLowering.core_lowering_hook(code, filename, lineno, offset, options; syntax_version=vp.ver) +end + +function Base.set_syntax_version(m::Module, ver::VersionNumber) + parser = Base.VersionedParse(ver) + Core.declare_const(m, Symbol("#_internal_julia_parse"), parser) + #lowerer = VersionedLower(ver) + #Core.declare_const(m, :_internal_julia_lower, lowerer) + nothing +end + +""" + Base.Experimental.@set_syntax_version ver + +Sets the syntax version of the current module to `ver`. This overrides settings of `syntax.julia_version` or +`compat.julia` from Project.toml. + +!!! compat "Julia 1.14" + This macro was added in Julia 1.14. + +!!! warning + The new syntax version will take effect only for code parsed after the *invocation* of the result of the macro + expansion. This may be unintuitive if the macro is used inside a module body, as the entire module will be parsed + before any statements therein are executed, e.g. consider. + + ``` + @set_syntax_version v"1.13" + module ChangeSyntax + @set_syntax_version v"1.14" + expr1 # Parsed with syntax version 1.13 + # The call itself is parsed with syntax version 1.13, but the included code is parsed with syntax version 1.14 + include_string(ChangeSyntax, "expr2") + expr3 # Parsed with syntax version 1.13 + end + ``` + + For this reason, the Project.toml mechanism is strongly preferred for packages. + However, this macro may be useful for scripts or the REPL. + +!!! warning + This interface is experimental and subject to change or removal without notice. +""" +macro set_syntax_version(ver) + Expr(:call, Base.set_syntax_version, __module__, esc(ver)) +end + +""" + Base.Experimental.@VERSION ver + +This macro provides access to parser (and possibly in the future other frontend component) language version +information. In particular, `(@VERSION).syntax` provides the syntax version used to parse the location where the macro is invoked. + +!!! compat "Julia 1.14" + This macro was added in Julia 1.14. + +!!! note + Calls to this macro have special handling in the parser and the name `@VERSION` is mandatory. At this time, other macros do not + have access to source syntax version information. +""" +function var"@VERSION"(__source__::Union{LineNumberNode, Core.MacroSource}, __module__::Module) + # This macro has special handling in the parser, which puts the current syntax + # version into __source__. + if isa(__source__, LineNumberNode) + return :((; syntax = v"1.13", runtime = VERSION)) + else + return :((; syntax = $(__source__.syntax_ver), runtime = VERSION)) end end diff --git a/base/exports.jl b/base/exports.jl index 2c30f095a3998..00b71dfa6abad 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -3,7 +3,7 @@ # Re-exports from `Core` export Core, # key types - Any, DataType, Vararg, NTuple, + Any, TypeEq, DataType, Vararg, NTuple, Tuple, Type, UnionAll, TypeVar, Union, Nothing, Cvoid, AbstractArray, DenseArray, NamedTuple, Pair, # special objects @@ -138,6 +138,7 @@ export StridedVector, SubArray, SubString, + StringView, SubstitutionString, Timer, UnitRange, @@ -227,8 +228,11 @@ export ÷, &, *, + *%, +, + +%, -, + -%, /, //, <, @@ -320,6 +324,7 @@ export fld1, fldmod, fldmod1, + cldmod1, flipsign, float, tryparse, @@ -837,6 +842,7 @@ export gensym, @kwdef, macroexpand, + macroexpand!, @macroexpand1, @macroexpand, parse, @@ -881,11 +887,14 @@ export exit, ntuple, splat, + tap, + unsplat, # I/O and events close, closewrite, countlines, + DirEntry, eachline, readeach, eof, @@ -936,6 +945,7 @@ export unsafe_read, unsafe_write, write, + writepartial, # multimedia I/O AbstractDisplay, @@ -1028,6 +1038,8 @@ export setenv, addenv, setcpuaffinity, + setuid, + setgid, success, withenv, @@ -1104,6 +1116,7 @@ export @gensym, @eval, @deprecate, + @deprecate_binding, # performance annotations @boundscheck, diff --git a/base/expr.jl b/base/expr.jl index b44c9336024e5..77c5720d18ece 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -9,7 +9,11 @@ const is_expr = isexpr """ gensym([tag]) -Generates a symbol which will not conflict with other variable names (in the same module). +Generate a symbol unique among all calls to this function within the same process. +If a string or symbol tag argument is specified, it is included in the generated name. + +Note that packages may be precompiled in separate processes, so names will not be unique +between definition time and run time. """ gensym() = ccall(:jl_gensym, Ref{Symbol}, ()) @@ -19,10 +23,10 @@ gensym(ss::String...) = map(gensym, ss) gensym(s::Symbol) = ccall(:jl_tagged_gensym, Ref{Symbol}, (Ptr{UInt8}, Csize_t), s, -1 % Csize_t) """ - @gensym + @gensym var1 var2 ... -Generates a gensym symbol for a variable. For example, `@gensym x y` is transformed into -`x = gensym("x"); y = gensym("y")`. +Generate symbols with [`gensym`](@ref) and assign them to the given variables. +For example, `@gensym x y` is transformed into `x = gensym("x"); y = gensym("y")`. """ macro gensym(names...) blk = Expr(:block) @@ -174,11 +178,12 @@ function ==(x::DebugInfo, y::DebugInfo) end """ - macroexpand(m::Module, x; recursive=true) + macroexpand(m::Module, x; recursive=true, legacyscope=true) Take the expression `x` and return an equivalent expression with all macros removed (expanded) for executing in module `m`. The `recursive` keyword controls whether deeper levels of nested macros are also expanded. +The `legacyscope` keyword controls whether legacy macroscope expansion is performed. This is demonstrated in the example below: ```jldoctest; filter = r"#= .*:6 =#" julia> module M @@ -197,13 +202,35 @@ julia> macroexpand(M, :(@m2()), recursive=true) julia> macroexpand(M, :(@m2()), recursive=false) :(#= REPL[1]:6 =# @m1) ``` + +!!! compat "Julia 1.13" + The `legacyscope` keyword argument requires at least Julia 1.13. """ -function macroexpand(m::Module, @nospecialize(x); recursive=true) - if recursive - ccall(:jl_macroexpand, Any, (Any, Any), x, m) - else - ccall(:jl_macroexpand1, Any, (Any, Any), x, m) - end +function macroexpand(m::Module, @nospecialize(x); recursive=true, legacyscope=true) + ccall(:jl_macroexpand, Any, (Any, Any, Cint, Cint, Cint), x, m, recursive, false, legacyscope) +end + +""" + macroexpand!(m::Module, x; recursive=true, legacyscope=false) + +Take the expression `x` and return an equivalent expression with all macros removed (expanded) +for executing in module `m`, modifying `x` in place without copying. +The `recursive` keyword controls whether deeper levels of nested macros are also expanded. +The `legacyscope` keyword controls whether legacy macroscope expansion is performed. + +This function performs macro expansion without the initial copy step, making it more efficient +when the original expression is no longer needed. By default, macroscope expansion is disabled +for in-place expansion as it can be called separately if needed. + +!!! warning + This function modifies the input expression `x` in place. Use `macroexpand` if you need + to preserve the original expression. + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. +""" +function macroexpand!(m::Module, @nospecialize(x); recursive=true, legacyscope=false) + ccall(:jl_macroexpand, Any, (Any, Any, Cint, Cint, Cint), x, m, recursive, true, legacyscope) end """ @@ -250,10 +277,10 @@ With `macroexpand` the expression expands in the module given as the first argum The two-argument form requires at least Julia 1.11. """ macro macroexpand(code) - return :(macroexpand($__module__, $(QuoteNode(code)), recursive=true)) + return :(macroexpand($__module__, $(QuoteNode(code)); recursive=true, legacyscope=true)) end macro macroexpand(mod, code) - return :(macroexpand($(esc(mod)), $(QuoteNode(code)), recursive=true)) + return :(macroexpand($(esc(mod)), $(QuoteNode(code)); recursive=true, legacyscope=true)) end """ @@ -262,10 +289,10 @@ end Non recursive version of [`@macroexpand`](@ref). """ macro macroexpand1(code) - return :(macroexpand($__module__, $(QuoteNode(code)), recursive=false)) + return :(macroexpand($__module__, $(QuoteNode(code)); recursive=false, legacyscope=true)) end macro macroexpand1(mod, code) - return :(macroexpand($(esc(mod)), $(QuoteNode(code)), recursive=false)) + return :(macroexpand($(esc(mod)), $(QuoteNode(code)); recursive=false, legacyscope=true)) end ## misc syntax ## @@ -352,7 +379,7 @@ Give a hint to the compiler that calls within `block` are worth inlining. ``` !!! warning - Although a callsite annotation will try to force inlining in regardless of the cost model, + Although a callsite annotation will try to force inlining regardless of the cost model, there are still chances it can't succeed in it. Especially, recursive calls can not be inlined even if they are annotated as `@inline`d. @@ -598,6 +625,7 @@ The following `setting`s are supported. - `:noub` - `:noub_if_noinbounds` - `:nortcall` +- `:reset_safe` - `:foldable` - `:removable` - `:total` @@ -676,7 +704,7 @@ were not executed. --- ## `:nothrow` -The `:nothrow` settings asserts that this method does not throw an exception +The `:nothrow` setting asserts that this method does not throw an exception (i.e. will either always return a value or never return). !!! note @@ -694,7 +722,7 @@ The `:nothrow` settings asserts that this method does not throw an exception --- ## `:terminates_globally` -The `:terminates_globally` settings asserts that this method will eventually terminate +The `:terminates_globally` setting asserts that this method will eventually terminate (either normally or abnormally), i.e. does not loop indefinitely. !!! note @@ -839,6 +867,25 @@ the following other `setting`s: number of effect overrides apply to a set of functions, a custom macro is recommended over the use of `:total`. +--- +## `:reset_safe` + +The `:reset_safe` asserts that it is safe to abandon execution of the annotated +function at any point. For functions so inferred, the compiler may extend the +reset region of a cancellation point through the `:reset_safe` regions. It thus +in particular implies `:effect_free`, but is a stronger assertion. For example, +an `:effect_free` function could in principle take a read-only lock (under appropriate +assumptions on how this is implemented and annotated), but a `:reset_safe` function +may not, because it could be abandoned inside the critical section. + +The same also applies to many implicitly inserted intrinsics and thus codegen for +a `:reset_safe` function requires cooperation by the code generator to uphold the +invariant throughout the entire body of the generated code. + +As such, annotating a function as `:reset_safe` is currently ignored as an effect +override and will only apply to [`@ccall`](@ref) sites. See the ccall documentation +for further details on this interaction. + --- ## Negated effects @@ -848,14 +895,32 @@ the call is generally total, it may however throw. """ macro assume_effects(args...) lastex = args[end] - override = compute_assumed_settings(args[begin:end-1]) - if is_function_def(unwrap_macrocalls(lastex)) - return esc(pushmeta!(lastex::Expr, form_purity_expr(override))) - elseif isexpr(lastex, :macrocall) && lastex.args[1] === Symbol("@ccall") + settings = args[begin:end-1] + if isexpr(lastex, :macrocall) && lastex.args[1] === Symbol("@ccall") + # `:reset_safe` is a foreigncall-only setting (it marks the call for + # the cancellation lowering, not the enclosing method): peel it off + # and carry it in a dedicated bit above the standard effects + # overrides in the `@ccall_effects` word. + reset_safe = false + rest = () + for st in settings + if st === QuoteNode(:reset_safe) + reset_safe = true + else + rest = (rest..., st) + end + end + override = compute_assumed_settings(rest) + word = encode_effects_override(override) + reset_safe && (word |= CCALL_EFFECT_RESET_SAFE) lastex.args[1] = GlobalRef(Base, Symbol("@ccall_effects")) - insert!(lastex.args, 3, encode_effects_override(override)) + insert!(lastex.args, 3, word) return esc(lastex) end + override = compute_assumed_settings(settings) + if is_function_def(unwrap_macrocalls(lastex)) + return esc(pushmeta!(lastex::Expr, form_purity_expr(override))) + end override′ = compute_assumed_setting(override, lastex) if override′ !== nothing # anonymous function case @@ -924,6 +989,9 @@ end const NUM_EFFECTS_OVERRIDES = 11 # sync with julia.h +# `:reset_safe` is ccall_only at the moment. +const CCALL_EFFECT_RESET_SAFE = 0x0800 + function compute_assumed_setting(override::EffectsOverride, @nospecialize(setting), val::Bool=true) if isexpr(setting, :call) && setting.args[1] === :(!) return compute_assumed_setting(override, setting.args[2], !val) @@ -1431,8 +1499,14 @@ function make_atomic(order, ex) if length(ex.args) == 2 if ex.head === :(+=) op = :+ + elseif ex.head === Symbol("+%=") + op = Symbol("+%") elseif ex.head === :(-=) op = :- + elseif ex.head === Symbol("-%=") + op = Symbol("-%") + elseif ex.head === Symbol("*%=") + op = Symbol("*%") elseif ex.head === :(|=) op = :| elseif ex.head === :(&=) @@ -1759,6 +1833,7 @@ function isa_ast_node(@nospecialize x) x isa Argument || x isa QuoteNode || x isa GlobalRef || + x isa Core.BindingPartition || x isa Symbol || x isa PiNode || x isa PhiNode || @@ -1780,9 +1855,9 @@ types of AST object inside, and even may sometimes evaluate and interpolate any quoted(@nospecialize(x)) = isa_ast_node(x) ? QuoteNode(x) : x # Implementation of generated functions -function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) +function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool, loc::LineNumberNode) ci = ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), - ex, defmod, "none", 0, typemax(Csize_t), 0)[1] + ex, defmod, loc.file, loc.line, typemax(Csize_t), 0)[1] if !isa(ci, CodeInfo) if isa(ci, Expr) && ci.head === :error msg = ci.args[1] @@ -1811,15 +1886,18 @@ function (g::Core.GeneratedFunctionStub)(world::UInt, source::Method, @nospecial body = g.gen(args...) file = source.file file isa Symbol || (file = :none) + loc = LineNumberNode(Int(source.line), source.file) lam = Expr(:lambda, Expr(:argnames, g.argnames...).args, Expr(:var"scope-block", Expr(:block, - LineNumberNode(Int(source.line), source.file), + loc, Expr(:meta, :push_loc, file, :var"@generated body"), - Expr(:return, body), + Expr(:return, Expr(:toplevel_pure, body)), Expr(:meta, :pop_loc)))) spnames = g.spnames - return generated_body_to_codeinfo(spnames === Core.svec() ? lam : Expr(Symbol("with-static-parameters"), lam, spnames...), + return generated_body_to_codeinfo( + spnames === Core.svec() ? lam : Expr(Symbol("with-static-parameters"), lam, spnames...), source.module, - source.isva) + source.isva, + loc) end diff --git a/base/fastmath.jl b/base/fastmath.jl index f2f60519b99ac..8bfbbf4e2e23b 100644 --- a/base/fastmath.jl +++ b/base/fastmath.jl @@ -297,12 +297,13 @@ exp10_fast(x::Union{Float32,Float64}) = Base.Math.exp10_fast(x) # builtins -function pow_fast(x::Float64, y::Integer) +@inline function pow_fast(x::T, y::Integer) where T <: Base.IEEEFloat z = y % Int32 z == y ? pow_fast(x, z) : x^y end -pow_fast(x::Float32, y::Integer) = x^y -pow_fast(x::Float64, y::Int32) = ccall("llvm.powi.f64.i32", llvmcall, Float64, (Float64, Int32), x, y) +pow_fast(x::Float16, y::Int32) = ccall("llvm.powi", llvmcall, Float16, (Float16, Int32), x, y) +pow_fast(x::Float32, y::Int32) = ccall("llvm.powi", llvmcall, Float32, (Float32, Int32), x, y) +pow_fast(x::Float64, y::Int32) = ccall("llvm.powi", llvmcall, Float64, (Float64, Int32), x, y) pow_fast(x::FloatTypes, ::Val{p}) where {p} = pow_fast(x, p) # inlines already via llvm.powi @inline pow_fast(x, v::Val) = Base.literal_pow(^, x, v) diff --git a/base/file.jl b/base/file.jl index 24a21396721d3..197a027b90ee2 100644 --- a/base/file.jl +++ b/base/file.jl @@ -8,6 +8,7 @@ export chown, cp, cptree, + DirEntry, diskstat, hardlink, mkdir, @@ -253,9 +254,9 @@ function mkpath(path::AbstractString; mode::Integer = 0o777) return path end -# Files that were requested to be deleted but can't be by the current process -# i.e. loaded DLLs on Windows -delayed_delete_dir() = joinpath(tempdir(), "julia_delayed_deletes") +# Files that were requested to be deleted but can't be by the current process, +# i.e. loaded DLLs on Windows, are listed in the directory below +delayed_delete_ref() = joinpath(tempdir(), "julia_delayed_deletes_ref") """ rm(path::AbstractString; force::Bool=false, recursive::Bool=false) @@ -288,13 +289,7 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allo force && err.code==Base.UV_ENOENT && return @static if Sys.iswindows() if allow_delayed_delete && err.code==Base.UV_EACCES && endswith(path, ".dll") - # Loaded DLLs cannot be deleted on Windows, even with posix delete mode - # but they can be moved. So move out to allow the dir to be deleted. - # Pkg.gc() cleans up this dir when possible - dir = mkpath(delayed_delete_dir()) - temp_path = tempname(dir, cleanup = false, suffix = string("_", basename(path))) - @debug "Could not delete DLL most likely because it is loaded, moving to tempdir" path temp_path - mv(path, temp_path) + delayed_delete_dll(path) return end end @@ -330,6 +325,22 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allo end +# Loaded DLLs cannot be deleted on Windows, even with posix delete mode but they can be renamed. +# delayed_delete_dll(path) does so temporarily, until later cleanup by Pkg.gc(). +function delayed_delete_dll(path) + # in-use DLL must be kept on the same drive + temp_path = _tempname(abspath(dirname(path)), string("_", basename(path))) + @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path + mkpath(delayed_delete_ref()) + io = Base.open(_win_mkstemp(delayed_delete_ref()), "r+") + try + print(io, temp_path) # record the temporary path for Pkg.gc() + finally + close(io) + end + rename(path, temp_path) # do not call mv which could recursively call rm(path) +end + # The following use Unix command line facilities function checkfor_mv_cp_cptree(src::AbstractString, dst::AbstractString, txt::AbstractString; force::Bool=false) @@ -384,12 +395,19 @@ symbolic link. If `follow_symlinks=true` and `src` is a symbolic link, `dst` wil of the file or directory `src` refers to. Return `dst`. +The timestamps, permissions, and ownership (if possible) of the destination file(s) are copied +from those of the source file(s), similar to the Unix `cp -p` command. + !!! note The `cp` function is different from the `cp` Unix command. The `cp` function always operates on the assumption that `dst` is a file, while the command does different things depending on whether `dst` is a directory or a file. Using `force=true` when `dst` is a directory will result in loss of all the contents present in the `dst` directory, and `dst` will become a file that has the contents of `src` instead. + +!!! compat "Julia 1.13" + Prior to Julia 1.13, the file permissions and other metadata were not necessarily + preserved (e.g. the permissions were modified by the current `umask` on Unix systems). """ function cp(src::AbstractString, dst::AbstractString; force::Bool=false, follow_symlinks::Bool=false) @@ -399,7 +417,7 @@ function cp(src::AbstractString, dst::AbstractString; force::Bool=false, elseif isdir(src) cptree(src, dst; force=force, follow_symlinks=follow_symlinks) else - sendfile(src, dst) + sendfile(src, dst; force) end dst end @@ -544,7 +562,7 @@ end """ tempdir() -Gets the path of the temporary directory. On Windows, `tempdir()` uses the first environment +Get the path of the temporary directory. On Windows, `tempdir()` uses the first environment variable found in the ordered list `TMP`, `TEMP`, `USERPROFILE`. On all other operating systems, `tempdir()` uses the first environment variable found in the ordered list `TMPDIR`, `TMP`, `TEMP`, and `TEMPDIR`. If none of these are found, the path `"/tmp"` is used. @@ -598,7 +616,7 @@ function prepare_for_deletion(path::AbstractString) catch ex ex isa IOError || ex isa SystemError || rethrow() end - for (root, dirs, files) in walkdir(path; onerror=x->()) + for (root, dirs, _) in walkdir(path; onerror=x->()) for dir in dirs dpath = joinpath(root, dir) try @@ -656,18 +674,18 @@ function temp_cleanup_purge_prelocked(force::Bool) end function temp_cleanup_purge_all() - may_need_gc = false - @lock TEMP_CLEANUP_LOCK filter!(TEMP_CLEANUP) do (path, asap) + may_need_gc = Ref(false) + @lock TEMP_CLEANUP_LOCK filter!(TEMP_CLEANUP) do (path, _) try ispath(path) || return false - may_need_gc = true + may_need_gc[] = true return true catch ex ex isa InterruptException && rethrow() return true end end - if may_need_gc + if may_need_gc[] # this is only usually required on Sys.iswindows(), but may as well do it everywhere GC.gc(true) end @@ -678,8 +696,35 @@ end # deprecated internal function used by some packages temp_cleanup_purge(; force=false) = force ? temp_cleanup_purge_all() : @lock TEMP_CLEANUP_LOCK temp_cleanup_purge_prelocked(false) +function temp_cleanup_postprocess(cleanup_dirs) + if !isempty(cleanup_dirs) + rmcmd = """ + cleanuplist = readlines(stdin) # This loop won't start running until stdin is closed, which is supposed to be sequenced after the process exits + sleep(1) # Wait for the operating system to hopefully be ready, since the OS implementation is probably incorrect, given the history of buggy work-arounds like this that have existed for ages in dotNet and libuv + for path in cleanuplist + try + rm(path, force=true, recursive=true) + catch ex + @warn "Failed to clean up temporary path \$(repr(path))\n\$ex" _group=:file + end + end + """ + cmd = Cmd(Base.cmd_gen(((Base.julia_cmd(),), ("--startup-file=no",), ("-e",), (rmcmd,))); ignorestatus = true, detach = true) + pw = Base.PipeEndpoint() + run(cmd, pw, devnull, stderr; wait=false) + join(pw, cleanup_dirs, "\n") + Base.dup(Base._fd(pw)) # intentionally leak a reference, until the process exits + close(pw) + end +end + +function temp_cleanup_atexit() + temp_cleanup_purge_all() + @lock TEMP_CLEANUP_LOCK temp_cleanup_postprocess(keys(TEMP_CLEANUP)) +end + function __postinit__() - Base.atexit(temp_cleanup_purge_all) + Base.atexit(temp_cleanup_atexit) end const temp_prefix = "jl_" @@ -697,11 +742,17 @@ end # Obtain a temporary filename. function tempname(parent::AbstractString=tempdir(); max_tries::Int = 100, cleanup::Bool=true, suffix::AbstractString="") + filename = _tempname(parent, suffix, max_tries) + cleanup && temp_cleanup_later(filename) + return filename +end + +function _tempname(parent::AbstractString, suffix::AbstractString, max_tries::Int = 100) isdir(parent) || throw(ArgumentError("$(repr(parent)) is not a directory")) prefix = joinpath(parent, temp_prefix) filename = nothing - for i in 1:max_tries + for _ in 1:max_tries filename = string(prefix, _rand_filename(), suffix) if ispath(filename) filename = nothing @@ -714,7 +765,6 @@ function tempname(parent::AbstractString=tempdir(); max_tries::Int = 100, cleanu error("tempname: max_tries exhausted") end - cleanup && temp_cleanup_later(filename) return filename end @@ -732,7 +782,7 @@ function _win_mkstemp(temppath::AbstractString) tempp, temppfx, UInt32(0), tname) windowserror("GetTempFileName", uunique == 0) lentname = something(findfirst(iszero, tname)) - @assert lentname > 0 + @assert lentname > 0 "unexpected index" resize!(tname, lentname - 1) return transcode(String, tname) end @@ -830,7 +880,7 @@ See also: [`mktemp`](@ref), [`mkdir`](@ref). """ function mktempdir(parent::AbstractString=tempdir(); prefix::AbstractString=temp_prefix, cleanup::Bool=true) - if isempty(parent) || occursin(path_separator_re, parent[end:end]) + if isempty(parent) || isseparator(last(parent)) # append a path_separator only if parent didn't already have one tpath = "$(parent)$(prefix)XXXXXX" else @@ -970,7 +1020,7 @@ julia> readdir("base") ⋮ "version_git.sh" "views.jl" - "weakkeydict.jl" + "weakdict.jl" julia> readdir("base", join=true) 145-element Vector{String}: @@ -980,7 +1030,7 @@ julia> readdir("base", join=true) ⋮ "base/version_git.sh" "base/views.jl" - "base/weakkeydict.jl" + "base/weakdict.jl" julia> readdir(abspath("base"), join=true) 145-element Vector{String}: @@ -990,7 +1040,7 @@ julia> readdir(abspath("base"), join=true) ⋮ "/home/JuliaUser/dev/julia/base/version_git.sh" "/home/JuliaUser/dev/julia/base/views.jl" - "/home/JuliaUser/dev/julia/base/weakkeydict.jl" + "/home/JuliaUser/dev/julia/base/weakdict.jl" ``` """ readdir(; join::Bool=false, kwargs...) = readdir(join ? pwd() : "."; join, kwargs...)::Vector{String} @@ -1011,65 +1061,87 @@ const UV_DIRENT_BLOCK = Cint(7) DirEntry A type representing a filesystem entry that contains the name of the entry, the directory, and -the raw type of the entry. The full path of the entry can be obtained lazily by accessing the -`path` field. The type of the entry can be checked for by calling [`isfile`](@ref), [`isdir`](@ref), +the raw type of the entry. The full path of the entry can be obtained lazily via [`joinpath(entry)`](@ref). + +The directory and name components are accessed via [`dirname`](@ref) and [`basename`](@ref), +respectively, mirroring the behavior of the corresponding string-returning functions. + +The type of the entry can be checked for by calling [`isfile`](@ref), [`isdir`](@ref), [`islink`](@ref), [`isfifo`](@ref), [`issocket`](@ref), [`ischardev`](@ref), and [`isblockdev`](@ref) +on the entry object. These predicates use the raw type cached at scan time when available; on +filesystems that report `UV_DIRENT_UNKNOWN` (some network/FUSE mounts) and for symlinks they fall +through to a `stat` syscall on each call. Callers in tight loops that need multiple predicates for +the same entry should call [`stat`](@ref) once and reuse the result. + +!!! warning "Staleness" + A `DirEntry` is a snapshot from when the directory was scanned. The underlying filesystem may + have changed in the meantime: the entry may no longer exist, may have been replaced by a + different type, or the cached `rawtype` may be wrong. Treat `DirEntry` values as advisory + and re-`stat` if up-to-date information is required. """ struct DirEntry dir::String name::String rawtype::Cint end -function Base.getproperty(obj::DirEntry, p::Symbol) - if p === :path - return joinpath(obj.dir, obj.name) - else - return getfield(obj, p) - end -end -Base.propertynames(::DirEntry) = (:dir, :name, :path, :rawtype) -Base.isless(a::DirEntry, b::DirEntry) = a.dir == b.dir ? isless(a.name, b.name) : isless(a.dir, b.dir) -Base.hash(o::DirEntry, h::UInt) = hash(o.dir, hash(o.name, hash(o.rawtype, h))) -Base.:(==)(a::DirEntry, b::DirEntry) = a.name == b.name && a.dir == b.dir && a.rawtype == b.rawtype -joinpath(obj::DirEntry, args...) = joinpath(obj.path, args...) +basename(obj::DirEntry) = getfield(obj, :name) +dirname(obj::DirEntry) = getfield(obj, :dir) +joinpath(obj::DirEntry, args...) = joinpath(dirname(obj), basename(obj), args...) +Base.isless(a::DirEntry, b::DirEntry) = dirname(a) == dirname(b) ? isless(basename(a), basename(b)) : isless(dirname(a), dirname(b)) +Base.hash(o::DirEntry, h::UInt) = hash(dirname(o), hash(basename(o), hash(o.rawtype, h))) +Base.:(==)(a::DirEntry, b::DirEntry) = basename(a) == basename(b) && dirname(a) == dirname(b) && a.rawtype == b.rawtype isunknown(obj::DirEntry) = obj.rawtype == UV_DIRENT_UNKNOWN -islink(obj::DirEntry) = isunknown(obj) ? islink(obj.path) : obj.rawtype == UV_DIRENT_LINK -isfile(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfile(obj.path) : obj.rawtype == UV_DIRENT_FILE -isdir(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isdir(obj.path) : obj.rawtype == UV_DIRENT_DIR -isfifo(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfifo(obj.path) : obj.rawtype == UV_DIRENT_FIFO -issocket(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? issocket(obj.path) : obj.rawtype == UV_DIRENT_SOCKET -ischardev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? ischardev(obj.path) : obj.rawtype == UV_DIRENT_CHAR -isblockdev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isblockdev(obj.path) : obj.rawtype == UV_DIRENT_BLOCK -realpath(obj::DirEntry) = realpath(obj.path) +islink(obj::DirEntry) = isunknown(obj) ? islink(joinpath(obj)) : obj.rawtype == UV_DIRENT_LINK +isfile(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfile(joinpath(obj)) : obj.rawtype == UV_DIRENT_FILE +isdir(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isdir(joinpath(obj)) : obj.rawtype == UV_DIRENT_DIR +isfifo(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfifo(joinpath(obj)) : obj.rawtype == UV_DIRENT_FIFO +issocket(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? issocket(joinpath(obj)) : obj.rawtype == UV_DIRENT_SOCKET +ischardev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? ischardev(joinpath(obj)) : obj.rawtype == UV_DIRENT_CHAR +isblockdev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isblockdev(joinpath(obj)) : obj.rawtype == UV_DIRENT_BLOCK +realpath(obj::DirEntry) = realpath(joinpath(obj)) """ - _readdirx(dir::AbstractString=pwd(); sort::Bool = true)::Vector{DirEntry} + readdir(dir::AbstractString, ::Type{DirEntry}; sort::Bool=true)::Vector{DirEntry} + readdir(::Type{DirEntry}; sort::Bool=true)::Vector{DirEntry} + readdir(entry::DirEntry, ::Type{DirEntry}; sort::Bool=true)::Vector{DirEntry} + readdir(entry::DirEntry; join::Bool=false, sort::Bool=true)::Vector{String} Return a vector of [`DirEntry`](@ref) objects representing the contents of the directory `dir`, or the current working directory if not given. If `sort` is true, the returned vector is sorted by name. -Unlike [`readdir`](@ref), `_readdirx` returns [`DirEntry`](@ref) objects, which contain the name of the -file, the directory it is in, and the type of the file which is determined during the -directory scan. This means that calls to [`isfile`](@ref), [`isdir`](@ref), [`islink`](@ref), [`isfifo`](@ref), -[`issocket`](@ref), [`ischardev`](@ref), and [`isblockdev`](@ref) can be made on the -returned objects without further stat calls. However, for some filesystems, the type of the file -cannot be determined without a stat call. In these cases the `rawtype` field of the [`DirEntry`](@ref)) -object will be 0 (`UV_DIRENT_UNKNOWN`) and [`isfile`](@ref) etc. will fall back to a `stat` call. +The element type is selected by the trailing `DirEntry` type argument (mirroring +[`read(io, String)`](@ref)): include it to get [`DirEntry`](@ref) objects (which carry the +type of each entry as determined during the directory scan, so [`isfile`](@ref), +[`isdir`](@ref), etc. can be called without further `stat` calls), or omit it to get a +`Vector{String}` of names — independent of whether the directory was specified as an +`AbstractString` or a `DirEntry`. For some filesystems the type of the entry cannot be +determined without a `stat` call; in those cases the `rawtype` field of the +[`DirEntry`](@ref) is 0 (`UV_DIRENT_UNKNOWN`) and the predicates fall back to a `stat` call. +# Examples ```julia -for obj in _readdirx() - isfile(obj) && println("\$(obj.name) is a file with path \$(obj.path)") +for entry in readdir(".", DirEntry) + if isfile(entry) + println("\$(basename(entry)) is a file with path \$(joinpath(entry))") + continue + end + isdir(entry) || continue + for entry2 in readdir(entry, DirEntry) + ... + end end ``` """ -_readdirx(dir::AbstractString=pwd(); sort::Bool=true) = _readdir(dir; return_objects=true, sort)::Vector{DirEntry} +readdir(dir::AbstractString, ::Type{DirEntry}; sort::Bool=true) = _readdir(dir; return_objects=true, sort)::Vector{DirEntry} +readdir(::Type{DirEntry}; sort::Bool=true) = readdir(pwd(), DirEntry; sort)::Vector{DirEntry} +readdir(entry::DirEntry, ::Type{DirEntry}; sort::Bool=true) = readdir(joinpath(entry), DirEntry; sort)::Vector{DirEntry} +readdir(entry::DirEntry; kwargs...) = readdir(joinpath(entry); kwargs...)::Vector{String} function _readdir(dir::AbstractString; return_objects::Bool=false, join::Bool=false, sort::Bool=true) # Allocate space for uv_fs_t struct req = Libc.malloc(_sizeof_uv_fs) try - # defined in sys.c, to call uv_fs_readdir, which sets errno on error. err = ccall(:uv_fs_scandir, Int32, (Ptr{Cvoid}, Ptr{Cvoid}, Cstring, Cint, Ptr{Cvoid}), C_NULL, req, dir, 0, C_NULL) err < 0 && uv_error("readdir($(repr(dir)))", err) @@ -1150,43 +1222,45 @@ julia> (path, dirs, files) = first(itr) ``` """ function walkdir(path = pwd(); topdown=true, follow_symlinks=false, onerror=throw) - function _walkdir(chnl, path) - tryf(f, p) = try - f(p) - catch err - isa(err, IOError) || rethrow() - try - onerror(err) - catch err2 - close(chnl, err2) - end - return - end - entries = tryf(_readdirx, path) - entries === nothing && return - dirs = Vector{String}() - files = Vector{String}() - for entry in entries - # If we're not following symlinks, then treat all symlinks as files - if (!follow_symlinks && something(tryf(islink, entry), true)) || !something(tryf(isdir, entry), false) - push!(files, entry.name) - else - push!(dirs, entry.name) - end - end + return Channel{Tuple{String,Vector{String},Vector{String}}}(chnl -> + _walkdir(chnl, path, topdown, follow_symlinks, onerror)) +end - if topdown - push!(chnl, (path, dirs, files)) - end - for dir in dirs - _walkdir(chnl, joinpath(path, dir)) +function _walkdir(chnl, path, topdown, follow_symlinks, onerror) + tryf(f, p) = try + f(p) + catch err + isa(err, IOError) || rethrow() + try + onerror(err) + catch err2 + close(chnl, err2) + end + return end - if !topdown - push!(chnl, (path, dirs, files)) + entries = tryf(p -> readdir(p, DirEntry), path) + entries === nothing && return + dirs = Vector{String}() + files = Vector{String}() + for entry in entries + # If we're not following symlinks, then treat all symlinks as files + if (!follow_symlinks && something(tryf(islink, entry), true)) || !something(tryf(isdir, entry), false) + push!(files, basename(entry)) + else + push!(dirs, basename(entry)) end - nothing end - return Channel{Tuple{String,Vector{String},Vector{String}}}(chnl -> _walkdir(chnl, path)) + + if topdown + push!(chnl, (path, dirs, files)) + end + for dir in dirs + _walkdir(chnl, joinpath(path, dir), topdown, follow_symlinks, onerror) + end + if !topdown + push!(chnl, (path, dirs, files)) + end + nothing end function unlink(p::AbstractString) @@ -1229,26 +1303,15 @@ function rename(oldpath::AbstractString, newpath::AbstractString) newpath end -function sendfile(src::AbstractString, dst::AbstractString) - src_open = false - dst_open = false - local src_file, dst_file - try - src_file = open(src, JL_O_RDONLY) - src_open = true - dst_file = open(dst, JL_O_CREAT | JL_O_TRUNC | JL_O_WRONLY, filemode(src_file)) - dst_open = true +const UV_FS_COPYFILE_EXCL = 0x0001 +const UV_FS_COPYFILE_FICLONE = 0x0002 - bytes = filesize(stat(src_file)) - sendfile(dst_file, src_file, Int64(0), Int(bytes)) - finally - if src_open && isopen(src_file) - close(src_file) - end - if dst_open && isopen(dst_file) - close(dst_file) - end - end +function sendfile(src::AbstractString, dst::AbstractString; force::Bool=true) + flags = force ? UV_FS_COPYFILE_FICLONE : UV_FS_COPYFILE_FICLONE | UV_FS_COPYFILE_EXCL + result = ccall(:jl_fs_copyfile, Cint, (Cstring, Cstring, Cint), + src, dst, flags % Cint) + uv_error("copyfile", result) + return nothing end if Sys.iswindows() @@ -1260,7 +1323,7 @@ end """ hardlink(src::AbstractString, dst::AbstractString) -Creates a hard link to an existing source file `src` with the name `dst`. The +Create a hard link to an existing source file `src` with the name `dst`. The destination, `dst`, must not exist. See also: [`symlink`](@ref). @@ -1280,7 +1343,7 @@ end """ symlink(target::AbstractString, link::AbstractString; dir_target = false) -Creates a symbolic link to `target` with the name `link`. +Create a symbolic link to `target` with the name `link`. On Windows, symlinks must be explicitly declared as referring to a directory or not. If `target` already exists, by default the type of `link` will be auto- @@ -1367,7 +1430,7 @@ function readlink(path::AbstractString) if ret < 0 uv_fs_req_cleanup(req) uv_error("readlink($(repr(path)))", ret) - @assert false + @assert false "unexpected uv readlink error" end tgt = unsafe_string(ccall(:jl_uv_fs_t_ptr, Cstring, (Ptr{Cvoid},), req)) uv_fs_req_cleanup(req) @@ -1453,7 +1516,7 @@ Base.show(io::IO, x::DiskStat) = """ diskstat(path=pwd()) -Returns statistics in bytes about the disk that contains the file or directory pointed at by +Return statistics in bytes about the disk that contains the file or directory pointed at by `path`. If no argument is passed, statistics about the disk that contains the current working directory are returned. diff --git a/base/filesystem.jl b/base/filesystem.jl index a5f1327b5cff1..4cc762e34da06 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -1,8 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## File Operations (Libuv-based) ## +# NB: This file is `Core.eval`-uated into the (pre-existing) module Filesystem -module Filesystem +## File Operations (Libuv-based) ## """ JL_O_APPEND @@ -149,14 +149,15 @@ if Sys.iswindows() import .Base: cwstring end -# Average buffer size including null terminator for several filesystem operations. -# On Windows we use the MAX_PATH = 260 value on Win32. -const AVG_PATH = Sys.iswindows() ? 260 : 512 - # helper function to clean up libuv request uv_fs_req_cleanup(req) = ccall(:uv_fs_req_cleanup, Cvoid, (Ptr{Cvoid},), req) -include("path.jl") +if Sys.iswindows() + const path_separator_re = r"[/\\]+"sa # May be used by some external packages +elseif Sys.isunix() + const path_separator_re = r"/+"sa # May be used by some external packages +end + include("stat.jl") include("file.jl") include(string(Base.BUILDROOT, "file_constants.jl")) # include($BUILDROOT/base/file_constants.jl) @@ -219,10 +220,10 @@ function sendfile(dst::File, src::File, src_offset::Int64, bytes::Int) check_open(dst) check_open(src) while true + nsent = min(typemax(Cssize_t), bytes) # biggest allowed chunk result = ccall(:jl_fs_sendfile, Int32, (OS_HANDLE, OS_HANDLE, Int64, Csize_t), - src.handle, dst.handle, src_offset, bytes) + src.handle, dst.handle, src_offset, nsent) uv_error("sendfile", result) - nsent = result bytes -= nsent src_offset += nsent bytes <= 0 && break @@ -268,16 +269,17 @@ function read(f::File, ::Type{UInt8}) ret = ccall(:jl_fs_read, Int32, (OS_HANDLE, Ptr{Cvoid}, Csize_t), f.handle, p, 1) uv_error("read", ret) - @assert ret <= sizeof(p) == 1 + @assert ret <= sizeof(p) == 1 "unexpected read size" ret < 1 && throw(EOFError()) return p[] % UInt8 end function read(f::File, ::Type{Char}) b0 = read(f, UInt8) - l = 0x08 * (0x04 - UInt8(leading_ones(b0))) + lo = UInt8(leading_ones(b0)) c = UInt32(b0) << 24 - if l ≤ 0x10 + if 0x02 ≤ lo ≤ 0x04 + l = 0x08 * (0x04 - lo) s = 16 while s ≥ l && !eof(f) # this works around lack of peek(::File) @@ -390,7 +392,7 @@ function isexecutable(path::String) X_OK = 0x01 return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, X_OK) == 0 end -isexecutable(path::AbstractString) = isexecutable(String(path)) +isexecutable(path::AbstractString) = isexecutable(String(path)::String) """ isreadable(path::String) @@ -417,7 +419,7 @@ function isreadable(path::String) R_OK = 0x04 return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, R_OK) == 0 end -isreadable(path::AbstractString) = isreadable(String(path)) +isreadable(path::AbstractString) = isreadable(String(path)::String) """ iswritable(path::String) @@ -440,11 +442,8 @@ See also [`ispath`](@ref), [`isexecutable`](@ref), [`isreadable`](@ref). """ function iswritable(path::String) # We use `access()` and `W_OK` to determine if a given path is - # writeable by the current user. `W_OK` comes from `unistd.h`. + # writable by the current user. `W_OK` comes from `unistd.h`. W_OK = 0x02 return ccall(:jl_fs_access, Cint, (Cstring, Cint), path, W_OK) == 0 end -iswritable(path::AbstractString) = iswritable(String(path)) - - -end +iswritable(path::AbstractString) = iswritable(String(path)::String) diff --git a/base/flfrontend.jl b/base/flfrontend.jl index 86b291cf7328b..334620adb78ac 100644 --- a/base/flfrontend.jl +++ b/base/flfrontend.jl @@ -1,11 +1,14 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license # Call Julia's builtin flisp-based parser. `offset` is 0-based offset into the -# byte buffer or string. +# byte buffer or string. `version` may be nothing during bootstrap. function fl_parse(text::Union{Core.SimpleVector,String}, - filename::String, lineno, offset, options) + filename::String, lineno, offset, options, version) + if version !== nothing && Base.thisminor(version) != Base.thisminor(VERSION) + error("Syntax version $version not supported by flisp parser ($VERSION)") + end if text isa Core.SimpleVector - # Will be generated by C entry points jl_parse_string etc + # Will be generated by C entry points jl_parse etc text, text_len = text else text_len = sizeof(text) @@ -14,13 +17,17 @@ function fl_parse(text::Union{Core.SimpleVector,String}, text, text_len, filename, lineno, offset, options) end -function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options) - fl_parse(String(text), String(filename), lineno, offset, options) +function fl_parse(text::AbstractString, filename::AbstractString, lineno, + offset, options, version) + fl_parse(String(text), String(filename), lineno, offset, options, version) +end + +function fl_parse_bootstrap(text, filename, lineno, offset, options) + fl_parse(text, filename, lineno, offset, options, nothing) end -function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", - lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) - warn = warn ? 1 : 0 - ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), +function fl_lower(ex, mod::Module, filename::String="none", + lineno::Int=0, world::UInt=typemax(Csize_t), warn::Bool=false) + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint), ex, mod, filename, lineno, world, warn) end diff --git a/base/float.jl b/base/float.jl index d5bfd044cf3b0..c040d17d38f74 100644 --- a/base/float.jl +++ b/base/float.jl @@ -166,9 +166,7 @@ IEEE 754 definition of the minimum exponent. """ ieee754_exponent_min(::Type{T}) where {T<:IEEEFloat} = Int(1 - exponent_max(T))::Int -exponent_min(::Type{Float16}) = ieee754_exponent_min(Float16) -exponent_min(::Type{Float32}) = ieee754_exponent_min(Float32) -exponent_min(::Type{Float64}) = ieee754_exponent_min(Float64) +exponent_min(T::Union{Type{Float16},Type{Float32},Type{Float64}}) = ieee754_exponent_min(T) function ieee754_representation( ::Type{F}, sign_bit::Bool, exponent_field::Integer, significand_field::Integer @@ -179,6 +177,7 @@ function ieee754_representation( ret |= exponent_field ret <<= significand_bits(F) ret |= significand_field + return ret end # ±floatmax(T) @@ -224,7 +223,7 @@ function ieee754_representation( end """ - uabs(x::Integer) + Base.uabs(x::Integer) Return the absolute value of `x`, possibly returning a different type should the operation be susceptible to overflow. This typically arises when `x` is a two's complement @@ -345,17 +344,8 @@ Float32(x::Float16) = fpext(Float32, x) Float64(x::Float32) = fpext(Float64, x) Float64(x::Float16) = fpext(Float64, x) -AbstractFloat(x::Bool) = Float64(x) -AbstractFloat(x::Int8) = Float64(x) -AbstractFloat(x::Int16) = Float64(x) -AbstractFloat(x::Int32) = Float64(x) -AbstractFloat(x::Int64) = Float64(x) # LOSSY -AbstractFloat(x::Int128) = Float64(x) # LOSSY -AbstractFloat(x::UInt8) = Float64(x) -AbstractFloat(x::UInt16) = Float64(x) -AbstractFloat(x::UInt32) = Float64(x) -AbstractFloat(x::UInt64) = Float64(x) # LOSSY -AbstractFloat(x::UInt128) = Float64(x) # LOSSY +# lossy for the 64- and 128-bit integer types +AbstractFloat(x::Union{Bool, BitInteger}) = Float64(x) Bool(x::Float16) = x==0 ? false : x==1 ? true : throw(InexactError(:Bool, Bool, x)) @@ -391,7 +381,7 @@ Float64 """ float(::Type{T}) where {T<:Number} = typeof(float(zero(T))) float(::Type{T}) where {T<:AbstractFloat} = T -float(::Type{Union{}}, slurp...) = Union{}(0.0) +float(::Type{Union{}}, slurp...) = Union{} """ unsafe_trunc(T, x) @@ -425,7 +415,9 @@ end function unsafe_trunc(::Type{UInt128}, x::Float64) xu = reinterpret(UInt64,x) - k = Int(xu >> 52) & 0x07ff - 1075 + # use `% Int` instead of `Int(...)` to preserve `:nothrow` (the shifted value + # fits in 11 bits, but `Int(::UInt64)` would otherwise add a bounds check) + k = ((xu >> 52) % Int) & 0x07ff - 1075 xu = (xu & 0x000f_ffff_ffff_ffff) | 0x0010_0000_0000_0000 if k <= 0 UInt128(xu >> -k) @@ -517,7 +509,6 @@ function _to_float(number::U, ep) where {U<:Unsigned} lz::signed(U) = unsafe_trunc(S, Core.Intrinsics.ctlz_int(number) - U(exponent_bits(F))) number <<= lz epint -= lz - bits = U(0) if epint >= 0 bits = number & significand_mask(F) bits |= ((epint + S(1)) << significand_bits(F)) & exponent_mask(F) @@ -619,7 +610,6 @@ end ## floating point comparisons ## ==(x::T, y::T) where {T<:IEEEFloat} = eq_float(x, y) -!=(x::T, y::T) where {T<:IEEEFloat} = ne_float(x, y) <( x::T, y::T) where {T<:IEEEFloat} = lt_float(x, y) <=(x::T, y::T) where {T<:IEEEFloat} = le_float(x, y) @@ -709,7 +699,7 @@ See also: [`iszero`](@ref), [`isone`](@ref), [`isinf`](@ref), [`ismissing`](@ref isnan(x::AbstractFloat) = (x != x)::Bool isnan(x::Number) = false -isfinite(x::AbstractFloat) = !isnan(x - x) +isfinite(x::AbstractFloat) = !(isnan(x - x)::Bool) isfinite(x::Real) = decompose(x)[3] != 0 isfinite(x::Integer) = true @@ -754,7 +744,7 @@ function decompose(x::Float16)::NTuple{3,Int} e = ((n & 0x7c00) >> 10) % Int s |= Int16(e != 0) << 10 d = ifelse(signbit(x), -1, 1) - s, e - 25 + (e == 0), d + s%Int, e - 25 + (e == 0), d end function decompose(x::Float32)::NTuple{3,Int} @@ -765,7 +755,7 @@ function decompose(x::Float32)::NTuple{3,Int} e = ((n & 0x7f800000) >> 23) % Int s |= Int32(e != 0) << 23 d = ifelse(signbit(x), -1, 1) - s, e - 150 + (e == 0), d + s%Int, e - 150 + (e == 0), d end function decompose(x::Float64)::Tuple{Int64, Int, Int} @@ -808,13 +798,8 @@ precision(::Type{T}; base::Integer=2) where {T<:AbstractFloat} = _precision(T, b precision(::T; base::Integer=2) where {T<:AbstractFloat} = precision(T; base) -""" - nextfloat(x::AbstractFloat, n::Integer) - -The result of `n` iterative applications of `nextfloat` to `x` if `n >= 0`, or `-n` -applications of [`prevfloat`](@ref) if `n < 0`. -""" -function nextfloat(f::IEEEFloat, d::Integer) +function _nextfloat(f::IEEEFloat, dneg::Bool, da::Integer) + # da must be > 0 F = typeof(f) fumax = reinterpret(Unsigned, F(Inf)) U = typeof(fumax) @@ -824,8 +809,6 @@ function nextfloat(f::IEEEFloat, d::Integer) fneg = fi < 0 fu = unsigned(fi & typemax(fi)) - dneg = d < 0 - da = uabs(d) if da > typemax(U) fneg = dneg fu = fumax @@ -852,6 +835,14 @@ function nextfloat(f::IEEEFloat, d::Integer) reinterpret(F, fu) end +""" + nextfloat(x::AbstractFloat, n::Integer) + +The result of `n` iterative applications of `nextfloat` to `x` if `n >= 0`, or `-n` +applications of [`prevfloat`](@ref) if `n < 0`. +""" +nextfloat(f::AbstractFloat, d::Integer) = _nextfloat(f, isnegative(d), uabs(d)) + """ nextfloat(x::AbstractFloat) @@ -860,7 +851,7 @@ If no such `y` exists (e.g. if `x` is `Inf` or `NaN`), then return `x`. See also: [`prevfloat`](@ref), [`eps`](@ref), [`issubnormal`](@ref). """ -nextfloat(x::AbstractFloat) = nextfloat(x,1) +nextfloat(x::AbstractFloat) = nextfloat(x, 1) """ prevfloat(x::AbstractFloat, n::Integer) @@ -868,7 +859,7 @@ nextfloat(x::AbstractFloat) = nextfloat(x,1) The result of `n` iterative applications of `prevfloat` to `x` if `n >= 0`, or `-n` applications of [`nextfloat`](@ref) if `n < 0`. """ -prevfloat(x::AbstractFloat, d::Integer) = nextfloat(x, -d) +prevfloat(x::AbstractFloat, d::Integer) = _nextfloat(x, ispositive(d), uabs(d)) """ prevfloat(x::AbstractFloat) @@ -876,7 +867,7 @@ prevfloat(x::AbstractFloat, d::Integer) = nextfloat(x, -d) Return the largest floating point number `y` of the same type as `x` such that `y < x`. If no such `y` exists (e.g. if `x` is `-Inf` or `NaN`), then return `x`. """ -prevfloat(x::AbstractFloat) = nextfloat(x,-1) +prevfloat(x::AbstractFloat) = nextfloat(x, -1) for Ti in (Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UInt128) for Tf in (Float16, Float32, Float64) @@ -899,7 +890,7 @@ for Ti in (Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UIn if ($(Tf(typemin(Ti))) <= x < $(Tf(typemax(Ti))+one(Tf))) && isinteger(x) return unsafe_trunc($Ti,x) else - throw(InexactError($(Expr(:quote,Ti.name.name)), $Ti, x)) + throw(InexactError($(Expr(:quote,nameof(Ti))), $Ti, x)) end end end @@ -920,7 +911,7 @@ for Ti in (Int8, Int16, Int32, Int64, Int128, UInt8, UInt16, UInt32, UInt64, UIn if ($(Tf(typemin(Ti))) <= x < $(Tf(typemax(Ti)))) && isinteger(x) return unsafe_trunc($Ti,x) else - throw(InexactError($(Expr(:quote,Ti.name.name)), $Ti, x)) + throw(InexactError($(Expr(:quote,nameof(Ti))), $Ti, x)) end end end @@ -1066,7 +1057,7 @@ julia> 1.0 + eps()/2 ``` More generally, for any floating-point numeric type, `eps` corresponds to an -upper bound on the distance to the nearest floating-point complex value: if ``\text{fl}(x)`` is the closest +upper bound on the distance to the nearest floating-point value: if ``\text{fl}(x)`` is the closest floating-point value to a number ``x`` (e.g. an arbitrary real number), then ``\text{fl}(x)`` satisfies ``|x - \text{fl}(x)| ≤ \text{eps}(x)/2``, not including overflow cases. This allows the definition of `eps` to be extended to complex numbers, diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index 437f8ae0157ec..b9f99c2bd9def 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -27,7 +27,7 @@ That is, `maxintfloat` returns the smallest positive integer-valued floating-poi When an `Integer`-type value is needed, use `Integer(maxintfloat(T))`. -See also: [`typemax`](@ref), [`floatmax`](@ref). +See also [`typemax`](@ref), [`floatmax`](@ref). """ maxintfloat(::Type{Float64}) = 9007199254740992. maxintfloat(::Type{Float32}) = Float32(16777216.) @@ -228,16 +228,70 @@ function isapprox(x::Number, y::Number; (nans && isnan(x) && isnan(y)) end +""" + _uabsdiff(x::Integer, y::Integer) + +Compute the exact absolute difference, widening the result type when necessary. +""" +_uabsdiff(x::Integer, y::Integer) = x < y ? y - x : x - y + +_uabsdiff(x::Bool, y::BitInteger) = _uabsdiff(oftype(y, x), y) +_uabsdiff(x::BitInteger, y::Bool) = _uabsdiff(x, oftype(x, y)) + +function _uabsdiff(x::BitUnsigned, y::BitUnsigned) + lo, hi = minmax(x, y) + return hi - lo +end +function _uabsdiff(x::BitSigned, y::BitSigned) + lo, hi = minmax(x, y) + # Unsigned subtraction gives the exact distance even across zero. + return unsigned(hi) - unsigned(lo) +end + +# Return (m, u, d, carried), where d is the wrapped absolute difference. +# On carry, m = |x| and the exact difference is m + u. +function _mixed_uabsdiff(x::BitSigned, y::BitUnsigned) + U = promote_type(unsigned(typeof(x)), typeof(y)) + v, u = x % U, y % U + d = ifelse(x < y, u - v, v - u) + # For x < 0, the distance is |x| + y, which carries exactly when d < u. + return -v, u, d, (x < 0) & (d < u) +end + +function _uabsdiff(x::BitSigned, y::BitUnsigned) + m, u, d, carried = _mixed_uabsdiff(x, y) + return carried ? widen(m) + widen(u) : d +end +_uabsdiff(x::BitUnsigned, y::BitSigned) = _uabsdiff(y, x) + +_uabsdiff_le(x::Integer, y::Integer, b::Real) = _uabsdiff(x, y) <= b +# Keep widening out of the caller to limit code size. +@noinline _widesum_le(m::T, u::T, b::Real) where {T<:BitUnsigned} = widen(m) + widen(u) <= b +function _uabsdiff_le(x::BitSigned, y::BitUnsigned, b::Real) + m, u, d, carried = _mixed_uabsdiff(x, y) + # Widen only if both the distance and the bound exceed typemax(d). + carried & (b > typemax(d)) && return _widesum_le(m, u, b) + return (d <= b) & !carried +end +_uabsdiff_le(x::BitUnsigned, y::BitSigned, b::Real) = _uabsdiff_le(y, x, b) + +_scaled_rtol(rtol::Real, scale::Integer) = (rtol * scale, false) +_scaled_rtol(rtol::BitInteger, scale::BitInteger) = mul_with_overflow(promote(rtol, scale)...) + function isapprox(x::Integer, y::Integer; atol::Real=0, rtol::Real=rtoldefault(x,y,atol), nans::Bool=false, norm::Function=abs) - if norm === abs && atol < 1 && rtol == 0 - return x == y - else - # We need to take the difference `max` - `min` when comparing unsigned integers. - _x, _y = x < y ? (x, y) : (y, x) - return norm(_y - _x) <= max(atol, rtol*max(norm(_x), norm(_y))) + if norm === abs + atol < 1 && rtol == 0 && return x == y + # Check equality before forming the bound, since Inf * 0 is NaN. + x == y && return true + # uabs handles typemin and avoids signed/unsigned promotion. + b, overflowed = _scaled_rtol(rtol, max(uabs(x), uabs(y))) + # Overflow implies rtol >= 2, hence rtol * max(|x|, |y|) >= |x - y|. + return overflowed || _uabsdiff_le(x, y, max(atol, b)) end + return x == y || + norm(_uabsdiff(x, y)) <= max(atol, rtol*max(norm(uabs(x)), norm(uabs(y)))) end """ @@ -273,7 +327,7 @@ end """ fma(x, y, z) -Computes `x*y+z` without rounding the intermediate result `x*y`. On some systems this is +Compute `x*y+z` without rounding the intermediate result `x*y`. On some systems this is significantly more expensive than `x*y+z`. `fma` is used to improve accuracy in certain algorithms. See [`muladd`](@ref). """ @@ -293,21 +347,43 @@ end """ Splits a Float64 into a hi bit and a low bit where the high bit has 27 trailing 0s and the low bit has 26 trailing 0s""" @inline function splitbits(x::Float64) - hi = reinterpret(Float64, reinterpret(UInt64, x) & 0xffff_ffff_f800_0000) + hi = truncbits(x, 27) return hi, x-hi end -function twomul(a::Float64, b::Float64) - ahi, alo = splitbits(a) - bhi, blo = splitbits(b) - abhi = a*b - blohi, blolo = splitbits(blo) - ablo = alo*blohi - (((abhi - ahi*bhi) - alo*bhi) - ahi*blo) + blolo*alo - return abhi, ablo +# two-product: returns (hi, lo) with hi + lo == x*y exactly. Uses a hardware +# fma when available, otherwise a split-based error-free transformation. +function two_mul(x::T, y::T) where {T<:Number} + xy = x*y + xy, fma(x, y, -xy) +end + +@assume_effects :consistent @inline function two_mul(x::Float64, y::Float64) + if Core.Intrinsics.have_fma(Float64) + xy = x*y + return xy, fma_float(x, y, -xy) + end + # fma-free fallback; `fma_emulated` relies on this branch never calling `fma` + xhi, xlo = splitbits(x) + yhi, ylo = splitbits(y) + xy = x*y + ylohi, ylolo = splitbits(ylo) + xylo = xlo*ylohi - (((xy - xhi*yhi) - xlo*yhi) - xhi*ylo) + ylolo*xlo + return xy, xylo +end + +@assume_effects :consistent @inline function two_mul(x::T, y::T) where T<:Union{Float16, Float32} + if Core.Intrinsics.have_fma(T) + xy = x*y + return xy, fma(x, y, -xy) + end + xy = widen(x)*y + Txy = T(xy) + return Txy, T(xy-Txy) end function fma_emulated(a::Float64, b::Float64,c::Float64) - abhi, ablo = @inline twomul(a,b) + abhi, ablo = @inline two_mul(a, b) if !isfinite(abhi+c) || isless(abs(abhi), nextfloat(0x1p-969)) || issubnormal(a) || issubnormal(b) aandbfinite = isfinite(a) && isfinite(b) if !(isfinite(c) && aandbfinite) @@ -324,7 +400,7 @@ function fma_emulated(a::Float64, b::Float64,c::Float64) a = reinterpret(Float64, (reinterpret(UInt64, a) & ~Base.exponent_mask(Float64)) | Base.exponent_one(Float64)) b = reinterpret(Float64, (reinterpret(UInt64, b) & ~Base.exponent_mask(Float64)) | Base.exponent_one(Float64)) c = c_denorm - abhi, ablo = twomul(a,b) + abhi, ablo = two_mul(a, b) # abhi <= 4 -> isfinite(r) (α) r = abhi+c # s ≈ 0 (β) @@ -341,7 +417,7 @@ function fma_emulated(a::Float64, b::Float64,c::Float64) bits_lost = -bias-Math._exponent_finite_nonzero(sumhi)-1022 sumhiInt = reinterpret(UInt64, sumhi) if (bits_lost != 1) ⊻ (sumhiInt&1 == 1) - sumhi = nextfloat(sumhi, cmp(sumlo,0)) + sumhi = nextfloat(sumhi, cmp(sumlo, 0)) end end return ldexp(sumhi, bias) diff --git a/base/gcutils.jl b/base/gcutils.jl index d5e6f4597739f..f5521ea6aba4b 100644 --- a/base/gcutils.jl +++ b/base/gcutils.jl @@ -157,7 +157,7 @@ enable_finalizers(on::Bool) = on ? enable_finalizers() : disable_finalizers() function enable_finalizers() @inline ccall(:jl_gc_enable_finalizers_internal, Cvoid, ()) - if Core.Intrinsics.atomic_pointerref(cglobal(:jl_gc_have_pending_finalizers, Cint), :monotonic) != 0 + if unsafe_load(cglobal(:jl_gc_have_pending_finalizers, Cint), :monotonic) != 0 ccall(:jl_gc_run_pending_finalizers, Cvoid, (Ptr{Cvoid},), C_NULL) end end @@ -169,7 +169,7 @@ end """ GC.in_finalizer()::Bool -Returns `true` if the current task is running a finalizer, returns `false` +Return `true` if the current task is running a finalizer, return `false` otherwise. Will also return `false` within a finalizer which was inlined by the compiler's eager finalization optimization, or if `finalize` is called on the finalizer directly. diff --git a/base/generator.jl b/base/generator.jl index 26bb7c7d91b5d..9c45f386dac70 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -34,11 +34,12 @@ struct Generator{I,F} iter::I end -Generator(f, I1, I2, Is...) = Generator(a->f(a...), zip(I1, I2, Is...)) +Generator(f, I1, I2, Is...) = Generator(splat(f), zip(I1, I2, Is...)) -Generator(::Type{T}, iter::I) where {T,I} = Generator{I,Type{T}}(T, iter) +Generator(::Type{T}, iter::I) where {T,I} = + Generator{I,has_free_typevars(T) ? typeof(T) : Core.TypeEgal{T}}(T, iter) -Generator(::Type{T}, I1, I2, Is...) where {T} = Generator(a->T(a...), zip(I1, I2, Is...)) +Generator(::Type{T}, I1, I2, Is...) where {T} = Generator(splat(T), zip(I1, I2, Is...)) function iterate(g::Generator, s...) @inline diff --git a/base/genericmemory.jl b/base/genericmemory.jl index b180462115f41..a7c082566f33a 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -61,45 +61,127 @@ AtomicMemory using Core: memoryrefoffset, memoryref_isassigned # import more functions which were not essential -size(a::GenericMemory, d::Int) = - d < 1 ? error("dimension out of range") : - d == 1 ? length(a) : - 1 -size(a::GenericMemory, d::Integer) = size(a, convert(Int, d)) -size(a::GenericMemory) = (length(a),) - IndexStyle(::Type{<:GenericMemory}) = IndexLinear() parent(ref::GenericMemoryRef) = ref.mem +""" + memoryindex(ref::GenericMemoryRef)::Int + +Get the 1-based index of `ref` in its `GenericMemory`. + +# Examples +```jldoctest +julia> mem = Memory{String}(undef, 10); + +julia> ref = Base.memoryindex(memoryref(mem, 3)) +3 + +julia> Base.memoryindex(memoryref(Memory{Nothing}(undef, 10), 8)) +8 +``` + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. +""" +memoryindex(ref::GenericMemoryRef) = memoryrefoffset(ref) + pointer(mem::GenericMemoryRef) = unsafe_convert(Ptr{Cvoid}, mem) # no bounds check, even for empty array -_unsetindex!(A::Memory, i::Int) = (@_propagate_inbounds_meta; _unsetindex!(memoryref(A, i)); A) -function _unsetindex!(A::MemoryRef{T}) where T - @_terminates_locally_meta +""" + Base.unsetindex!(ref::MemoryRef) -> ref + +Unset the reference from `ref` to its underlying value, leaving it +as if uninitialized, and return `ref`. +This is equivalent to `Base.unsetindex!(parent(ref), Base.memoryindex(ref))`. + +See the section of uninitialized memory in the manual for more details. + +# Examples +```jldoctest +julia> ref = memoryref(fill!(Memory{Int}(undef, 3), 4)); ref[] +4 + +julia> Base.unsetindex!(ref); ref[] isa Int # specific value not guaranteed +true + +julia> ref = memoryref(fill!(Memory{String}(undef, 3), "abc")); ref[] +"abc" + +julia> Base.unsetindex!(ref); ref[] +ERROR: UndefRefError: access to undefined reference +[...] +``` + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. +""" +unsetindex!(A::MemoryRef) = (@_propagate_inbounds_meta; Core.memoryrefunset!(A, :not_atomic, @_boundscheck); A) + +""" + unsetindex!(A::Union{Memory, Array}, i::Integer) -> A + +Unset the reference from `A` at index `i` to its underlying value and return `A`. +This leaves the slot as it was uninitialized. + +See the section of uninitialized memory in the manual for more details. + +# Examples +```jldoctest +julia> A = [6, 7, 8]; A[2] +7 + +julia> Base.unsetindex!(A, 2); A[2] isa Int # specific value not guaranteed +true + +julia> A = ["abc", "def", "ghi"]; A[2] +"def" + +julia> Base.unsetindex!(A, 2); A[2] +ERROR: UndefRefError: access to undefined reference +[...] +``` + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. +""" +unsetindex!(A::Memory, i::Int) = (@_propagate_inbounds_meta; unsetindex!(memoryref(A, i)); A) +unsetindex!(A::Union{Array, Memory}, i::Integer) = unsetindex!(A, to_index(i)) + +""" + unsetindex_atomic!(ref::AtomicMemoryRef, ordering::Symbol) -> ref + +Same as `unsetindex!(::MemoryRef)`, but the index is atomically unset with the +atomic memory ordering set by `ordering`. + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. +""" +function unsetindex_atomic!(A::AtomicMemoryRef, order::Symbol) @_propagate_inbounds_meta - @inline - @boundscheck memoryref(A, 1) - mem = A.mem - MemT = typeof(mem) - arrayelem = datatype_arrayelem(MemT) - elsz = datatype_layoutsize(MemT) - isbits = 0; isboxed = 1; isunion = 2 - arrayelem == isbits && datatype_pointerfree(T::DataType) && return A - t = @_gc_preserve_begin mem - p = Ptr{Ptr{Cvoid}}(@inbounds pointer(A)) - if arrayelem == isboxed - Intrinsics.atomic_pointerset(p, C_NULL, :monotonic) - elseif arrayelem != isunion - for j = 1:Core.sizeof(Ptr{Cvoid}):elsz - # XXX: this violates memory ordering, since it writes more than one C_NULL to each - Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic) - end - end - @_gc_preserve_end t + Core.memoryrefunset!(A, order, @_boundscheck) + return A +end + +""" + unsetindex_atomic!(A::AtomicMemory, order::Symbol, i::Integer) -> A + +Same as `unsetindex!(::Memory, ::Integer)`, but the index is atomically unset with the +atomic memory ordering set by `ordering`. + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. +""" +function unsetindex_atomic!(A::AtomicMemory, order::Symbol, i::Int) + @_propagate_inbounds_meta + unsetindex_atomic!(memoryref(A, i), order) return A end +function unsetindex_atomic!(A::AtomicMemory, order::Symbol, i::Integer) + return unsetindex_atomic!(A, order, to_index(i)) +end + elsize(@nospecialize _::Type{A}) where {T,A<:GenericMemory{<:Any,T}} = aligned_sizeof(T) # XXX: probably supposed to be the stride? sizeof(a::GenericMemory) = Core.sizeof(a) @@ -157,7 +239,7 @@ function unsafe_copyto!(dest::Memory, doffs, src::Memory, soffs, n) if isassigned(src, soffs + i - 1) dest[doffs + i - 1] = src[soffs + i - 1] else - _unsetindex!(dest, doffs + i - 1) + unsetindex!(dest, doffs + i - 1) end end else @@ -165,7 +247,7 @@ function unsafe_copyto!(dest::Memory, doffs, src::Memory, soffs, n) if isassigned(src, soffs + i - 1) dest[doffs + i - 1] = src[soffs + i - 1] else - _unsetindex!(dest, doffs + i - 1) + unsetindex!(dest, doffs + i - 1) end end end @@ -245,8 +327,9 @@ getindex(A::Memory, c::Colon) = copy(A) ## Indexing: setindex! ## function _setindex!(A::Memory{T}, x::T, i1::Int) where {T} - ref = memoryrefnew(memoryref(A), i1, @_boundscheck) - memoryrefset!(ref, x, :not_atomic, @_boundscheck) + @_noub_if_noinbounds_meta + (@_boundscheck) && checkbounds(A, i1) + memoryrefset!(memoryrefnew(memoryref(A), i1, false), x, :not_atomic, false) return A end @@ -258,7 +341,7 @@ end function setindex!(A::Memory{T}, x, i1::Int, i2::Int, I::Int...) where {T} @inline - @boundscheck (i2 == 1 && all(==(1), I)) || throw_boundserror(A, (i1, i2, I...)) + @boundscheck (i2 == 1 && all(==(1), I)) || throw_boundserror(A, i1, i2, I...) setindex!(A, x, i1) end diff --git a/base/gmp.jl b/base/gmp.jl index e4d9294766aaa..7a7e314a8976a 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -4,14 +4,14 @@ module GMP export BigInt -import .Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor, nand, nor, +import .Base: *, *%, +, +%, -, -%, /, <, <<, >>, >>>, <=, ==, >, >=, ^, ~, &, |, xor, binomial, cmp, convert, div, divrem, factorial, cld, fld, gcd, gcdx, lcm, mod, ndigits, promote_rule, rem, show, isqrt, string, powermod, sum, prod, trailing_zeros, trailing_ones, count_ones, count_zeros, tryparse_internal, - bin, oct, dec, hex, isequal, invmod, _prevpow2, _nextpow2, ndigits0zpb, - widen, signed, unsafe_trunc, trunc, iszero, isone, big, flipsign, signbit, - sign, isodd, iseven, digits!, hash, hash_integer, top_set_bit, - ispositive, isnegative, clamp, unsafe_takestring + invmod, _prevpow2, _nextpow2, ndigits0zpb, + widen, signed, unsafe_trunc, iszero, isone, big, flipsign, signbit, + sign, isodd, iseven, digits!, hash, hash_integer, top_set_bit, exponent, + ispositive, isnegative, clamp import Core: Signed, Float16, Float32, Float64 @@ -114,11 +114,14 @@ function __init__() bits_per_limb() != BITS_PER_LIMB ? @error(msg) : @warn(msg) end + # GMP calls may run under a reset region and re-enter the runtime + # through these allocation hooks. The hooks unpublish the region + # around the allocator. ccall((:__gmp_set_memory_functions, libgmp), Cvoid, (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), - cglobal(:jl_gc_counted_malloc), - cglobal(:jl_gc_counted_realloc_with_old_size), - cglobal(:jl_gc_counted_free_with_size)) + cglobal(:jl_gmp_counted_malloc), + cglobal(:jl_gmp_counted_realloc_with_old_size), + cglobal(:jl_gmp_counted_free_with_size)) ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) catch ex @@ -151,7 +154,7 @@ using ..GMP: BigInt, Limb, BITS_PER_LIMB, libgmp const mpz_t = Ref{BigInt} const bitcnt_t = Culong -gmpz(op::Symbol) = (Symbol(:__gmpz_, op), libgmp) +gmpz(op::Symbol) = Expr(:tuple, QuoteNode(Symbol(:__gmpz_, op)), GlobalRef(MPZ, :libgmp)) init!(x::BigInt) = (ccall((:__gmpz_init, libgmp), Cvoid, (mpz_t,), x); x) init2!(x::BigInt, a) = (ccall((:__gmpz_init2, libgmp), Cvoid, (mpz_t, bitcnt_t), x, a); x) @@ -167,28 +170,32 @@ for (op, nbits) in (:add => :(BITS_PER_LIMB*(1 + max(abs(a.size), abs(b.size)))) :fdiv_r => 0, :tdiv_r => 0, :cdiv_r => 0, :gcd => 0, :lcm => 0, :and => 0, :ior => 0, :xor => 0) op! = Symbol(op, :!) + fname = Symbol(:__gmpz_, op) @eval begin - $op!(x::BigInt, a::BigInt, b::BigInt) = (ccall($(gmpz(op)), Cvoid, (mpz_t, mpz_t, mpz_t), x, a, b); x) + $op!(x::BigInt, a::BigInt, b::BigInt) = + (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.$fname(x::mpz_t, a::mpz_t, b::mpz_t)::Cvoid); x) $op(a::BigInt, b::BigInt) = $op!(BigInt(nbits=$nbits), a, b) $op!(x::BigInt, b::BigInt) = $op!(x, x, b) end end invert!(x::BigInt, a::BigInt, b::BigInt) = - ccall((:__gmpz_invert, libgmp), Cint, (mpz_t, mpz_t, mpz_t), x, a, b) + (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall libgmp.__gmpz_invert(x::mpz_t, a::mpz_t, b::mpz_t)::Cint) invert!(x::BigInt, b::BigInt) = invert!(x, x, b) invert(a::BigInt, b::BigInt) = (ret=BigInt(); invert!(ret, a, b); ret) for op in (:add_ui, :sub_ui, :mul_ui, :mul_2exp, :fdiv_q_2exp, :pow_ui, :bin_ui) op! = Symbol(op, :!) + fname = Symbol(:__gmpz_, op) @eval begin - $op!(x::BigInt, a::BigInt, b) = (ccall($(gmpz(op)), Cvoid, (mpz_t, mpz_t, Culong), x, a, b); x) + $op!(x::BigInt, a::BigInt, b) = + (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.$fname(x::mpz_t, a::mpz_t, b::Culong)::Cvoid); x) $op(a::BigInt, b) = $op!(BigInt(), a, b) $op!(x::BigInt, b) = $op!(x, x, b) end end -ui_sub!(x::BigInt, a, b::BigInt) = (ccall((:__gmpz_ui_sub, libgmp), Cvoid, (mpz_t, Culong, mpz_t), x, a, b); x) +ui_sub!(x::BigInt, a, b::BigInt) = (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.__gmpz_ui_sub(x::mpz_t, a::Culong, b::mpz_t)::Cvoid); x) ui_sub(a, b::BigInt) = ui_sub!(BigInt(), a, b) for op in (:scan1, :scan0) @@ -197,21 +204,26 @@ for op in (:scan1, :scan0) @eval $op(a::BigInt, b) = Int(signed(ccall($(gmpz(op)), Culong, (mpz_t, Culong), a, b))) end -mul_si!(x::BigInt, a::BigInt, b) = (ccall((:__gmpz_mul_si, libgmp), Cvoid, (mpz_t, mpz_t, Clong), x, a, b); x) +mul_si!(x::BigInt, a::BigInt, b) = (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.__gmpz_mul_si(x::mpz_t, a::mpz_t, b::Clong)::Cvoid); x) mul_si(a::BigInt, b) = mul_si!(BigInt(), a, b) mul_si!(x::BigInt, b) = mul_si!(x, x, b) for op in (:neg, :com, :sqrt, :set) op! = Symbol(op, :!) + fname = Symbol(:__gmpz_, op) @eval begin - $op!(x::BigInt, a::BigInt) = (ccall($(gmpz(op)), Cvoid, (mpz_t, mpz_t), x, a); x) + $op!(x::BigInt, a::BigInt) = + (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.$fname(x::mpz_t, a::mpz_t)::Cvoid); x) $op(a::BigInt) = $op!(BigInt(), a) end op === :set && continue # MPZ.set!(x) would make no sense @eval $op!(x::BigInt) = $op!(x, x) end -for (op, T) in ((:fac_ui, Culong), (:set_ui, Culong), (:set_si, Clong), (:set_d, Cdouble)) +fac_ui!(x::BigInt, a) = (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.__gmpz_fac_ui(x::mpz_t, a::Culong)::Cvoid); x) +fac_ui(a) = fac_ui!(BigInt(), a) + +for (op, T) in ((:set_ui, Culong), (:set_si, Clong), (:set_d, Cdouble)) op! = Symbol(op, :!) @eval begin $op!(x::BigInt, a) = (ccall($(gmpz(op)), Cvoid, (mpz_t, $T), x, a); x) @@ -225,18 +237,20 @@ mpn_popcount(d::Ptr{Limb}, s::Integer) = Int(ccall((:__gmpn_popcount, libgmp), C mpn_popcount(a::BigInt) = mpn_popcount(a.d, abs(a.size)) function tdiv_qr!(x::BigInt, y::BigInt, a::BigInt, b::BigInt) - ccall((:__gmpz_tdiv_qr, libgmp), Cvoid, (mpz_t, mpz_t, mpz_t, mpz_t), x, y, a, b) + Base.@cancel_check + Base.@assume_effects :reset_safe @ccall libgmp.__gmpz_tdiv_qr(x::mpz_t, y::mpz_t, a::mpz_t, b::mpz_t)::Cvoid x, y end tdiv_qr(a::BigInt, b::BigInt) = tdiv_qr!(BigInt(), BigInt(), a, b) powm!(x::BigInt, a::BigInt, b::BigInt, c::BigInt) = - (ccall((:__gmpz_powm, libgmp), Cvoid, (mpz_t, mpz_t, mpz_t, mpz_t), x, a, b, c); x) + (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.__gmpz_powm(x::mpz_t, a::mpz_t, b::mpz_t, c::mpz_t)::Cvoid); x) powm(a::BigInt, b::BigInt, c::BigInt) = powm!(BigInt(), a, b, c) powm!(x::BigInt, b::BigInt, c::BigInt) = powm!(x, x, b, c) function gcdext!(x::BigInt, y::BigInt, z::BigInt, a::BigInt, b::BigInt) - ccall((:__gmpz_gcdext, libgmp), Cvoid, (mpz_t, mpz_t, mpz_t, mpz_t, mpz_t), x, y, z, a, b) + Base.@cancel_check + Base.@assume_effects :reset_safe @ccall libgmp.__gmpz_gcdext(x::mpz_t, y::mpz_t, z::mpz_t, a::mpz_t, b::mpz_t)::Cvoid x, y, z end gcdext(a::BigInt, b::BigInt) = gcdext!(BigInt(), BigInt(), BigInt(), a, b) @@ -249,8 +263,8 @@ cmp_d(a::BigInt, b) = Int(ccall((:__gmpz_cmp_d, libgmp), Cint, (mpz_t, Cdouble), mpn_cmp(a::Ptr{Limb}, b::Ptr{Limb}, c) = ccall((:__gmpn_cmp, libgmp), Cint, (Ptr{Limb}, Ptr{Limb}, Clong), a, b, c) mpn_cmp(a::BigInt, b::BigInt, c) = mpn_cmp(a.d, b.d, c) -get_str!(x, a, b::BigInt) = (ccall((:__gmpz_get_str,libgmp), Ptr{Cchar}, (Ptr{Cchar}, Cint, mpz_t), x, a, b); x) -set_str!(x::BigInt, a, b) = Int(ccall((:__gmpz_set_str, libgmp), Cint, (mpz_t, Ptr{UInt8}, Cint), x, a, b)) +get_str!(x, a, b::BigInt) = (Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.__gmpz_get_str(x::Ptr{Cchar}, a::Cint, b::mpz_t)::Ptr{Cchar}); x) +set_str!(x::BigInt, a, b) = Int(begin Base.@cancel_check; Base.@assume_effects :reset_safe @ccall(libgmp.__gmpz_set_str(x::mpz_t, a::Ptr{UInt8}, b::Cint)::Cint) end) get_d(a::BigInt) = ccall((:__gmpz_get_d, libgmp), Cdouble, (mpz_t,), a) function export!(a::AbstractVector{T}, n::BigInt; order::Integer=-1, nails::Integer=0, endian::Integer=0) where {T<:Base.BitInteger} @@ -261,7 +275,7 @@ function export!(a::AbstractVector{T}, n::BigInt; order::Integer=-1, nails::Inte count = Ref{Csize_t}() ccall((:__gmpz_export, libgmp), Ptr{T}, (Ptr{T}, Ref{Csize_t}, Cint, Csize_t, Cint, Csize_t, mpz_t), a, count, order, sizeof(T), endian, nails, n) - @assert count[] ≤ length(a) + @assert count[] ≤ length(a) "count[] > length(a)" return a, Int(count[]) end @@ -332,7 +346,7 @@ function BigInt(x::Integer) isbits(x) && typemin(Clong) <= x <= typemax(Clong) && return BigInt((x % Clong)::Clong) nd = ndigits(x, base=2) z = MPZ.realloc2(nd) - ux = unsigned(x < 0 ? -x : x) + ux = unsigned(x < 0 ? -%(x) : x) size = 0 limbnbits = sizeof(Limb) << 3 while nd > 0 @@ -484,6 +498,12 @@ promote_rule(::Type{BigInt}, ::Type{<:Integer}) = BigInt Convert a number to a maximum precision representation (typically [`BigInt`](@ref) or `BigFloat`). See [`BigFloat`](@ref BigFloat(::Any, rounding::RoundingMode)) for information about some pitfalls with floating-point numbers. + +!!! note "big(x::BigFloat)" + Unlike `BigFloat(x)`, `big(x)` is a no-op when `x` is already a `BigFloat`, + ie. when doing `x = big(x)`, the precision of `x` remains unchanged even if the + current `BigFloat` precision is different. + ``` """ function big end @@ -494,6 +514,7 @@ big(n::Integer) = convert(BigInt, n) # Binary ops for (fJ, fC) in ((:+, :add), (:-,:sub), (:*, :mul), + (:+%, :add), (:-%,:sub), (:*%, :mul), (:mod, :fdiv_r), (:rem, :tdiv_r), (:gcd, :gcd), (:lcm, :lcm), (:&, :and), (:|, :ior), (:xor, :xor)) @@ -530,7 +551,7 @@ function invmod(x::BigInt, y::BigInt) if y < 0 MPZ.add!(z, y) end - # The postcondition is: mod(z * x, y) == mod(big(1), m) && div(z, y) == 0 + # The postcondition is: mod(z * x, y) == mod(big(1), y) && div(z, y) == 0 return z end @@ -552,10 +573,10 @@ end -(x::BigInt, c::CulongMax) = MPZ.sub_ui(x, c) -(c::CulongMax, x::BigInt) = MPZ.ui_sub(c, x) -+(x::BigInt, c::ClongMax) = c < 0 ? -(x, -(c % Culong)) : x + convert(Culong, c) -+(c::ClongMax, x::BigInt) = c < 0 ? -(x, -(c % Culong)) : x + convert(Culong, c) --(x::BigInt, c::ClongMax) = c < 0 ? +(x, -(c % Culong)) : -(x, convert(Culong, c)) --(c::ClongMax, x::BigInt) = c < 0 ? -(x + -(c % Culong)) : -(convert(Culong, c), x) ++(x::BigInt, c::ClongMax) = c < 0 ? -(x, -%(c % Culong)) : x + convert(Culong, c) ++(c::ClongMax, x::BigInt) = c < 0 ? -(x, -%(c % Culong)) : x + convert(Culong, c) +-(x::BigInt, c::ClongMax) = c < 0 ? +(x, -%(c % Culong)) : -(x, convert(Culong, c)) +-(c::ClongMax, x::BigInt) = c < 0 ? -(x + -%(c % Culong)) : -(convert(Culong, c), x) *(x::BigInt, c::CulongMax) = MPZ.mul_ui(x, c) *(c::CulongMax, x::BigInt) = x * c @@ -605,10 +626,21 @@ Number of ones in the binary representation of abs(x). """ count_ones_abs(x::BigInt) = iszero(x) ? 0 : MPZ.mpn_popcount(x) +# all uses of _bit_magnitude MUST ensure at callsite that `x` is nonzero, otherwise it is UB +function _bit_magnitude(x::BigInt) + n = abs(x.size) + return n * sizeof(Limb) << 3 - leading_zeros(GC.@preserve x unsafe_load(x.d, n)) +end + +function exponent(x::BigInt) + iszero(x) && throw(DomainError(x, "cannot be zero")) + return _bit_magnitude(x) - 1 +end + function top_set_bit(x::BigInt) isnegative(x) && throw(DomainError(x, "top_set_bit only supports negative arguments when they have type BitSigned.")) iszero(x) && return 0 - x.size * sizeof(Limb) << 3 - leading_zeros(GC.@preserve x unsafe_load(x.d, x.size)) + return _bit_magnitude(x) end divrem(x::BigInt, y::BigInt, ::typeof(RoundToZero) = RoundToZero) = MPZ.tdiv_qr(x, y) @@ -680,24 +712,27 @@ sum(arr::Union{AbstractArray{BigInt}, Tuple{BigInt, Vararg{BigInt}}}) = foldl(MPZ.add!, arr; init=BigInt(0)) function prod(arr::AbstractArray{BigInt}) - # compute first the needed number of bits for the result, - # to avoid re-allocations; - # GMP will always request n+m limbs for the result in MPZ.mul!, - # if the arguments have n and m limbs; so we add all the bits - # taken by the array elements, and add BITS_PER_LIMB to that, - # to account for the rounding to limbs in MPZ.mul! - # (BITS_PER_LIMB-1 would typically be enough, to which we add - # 1 for the initial multiplication by init=1 in foldl) - nbits = BITS_PER_LIMB - for x in arr - iszero(x) && return zero(BigInt) - xsize = abs(x.size) - lz = GC.@preserve x leading_zeros(unsafe_load(x.d, xsize)) - nbits += xsize * BITS_PER_LIMB - lz + any(iszero, arr) && return zero(BigInt) + _prod(arr, firstindex(arr), lastindex(arr)) +end +function _prod(arr::AbstractArray{BigInt}, lo, hi) + if hi - lo + 1 <= 16 + # compute first the needed number of bits for the result, + # to avoid re-allocations + nlimbs = 0 + for i in lo:hi + nlimbs += abs(arr[i].size) + end + init = BigInt(; nbits=nlimbs*BITS_PER_LIMB) + MPZ.set_si!(init, 1) + for i in lo:hi + MPZ.mul!(init, arr[i]) + end + init + else + mid = (lo + hi) ÷ 2 + MPZ.mul!(_prod(arr, lo, mid), _prod(arr, mid+1, hi)) end - init = BigInt(; nbits) - MPZ.set_si!(init, 1) - foldl(MPZ.mul!, arr; init) end factorial(n::BigInt) = !isnegative(n) ? MPZ.fac_ui(n) : throw(DomainError(n, "`n` must not be negative.")) @@ -754,13 +789,17 @@ function string(n::BigInt; base::Integer = 10, pad::Integer = 1) iszero(n) && pad < 1 && return "" nd1 = ndigits(n, base=base) nd = max(nd1, pad) - sv = Base.StringMemory(nd + isnegative(n)) - GC.@preserve sv MPZ.get_str!(pointer(sv) + nd - nd1, base, n) - @inbounds for i = (1:nd-nd1) .+ isnegative(n) - sv[i] = '0' % UInt8 + str = Base._string_n(nd + isnegative(n)) + GC.@preserve str begin + p = pointer(str) + MPZ.get_str!(p + nd - nd1, base, n) + pad_len = nd - nd1 + if pad_len > 0 + Base.memset(p + isnegative(n), UInt8('0'), pad_len) + end + isnegative(n) && unsafe_store!(p, UInt8('-')) end - isnegative(n) && (sv[1] = '-' % UInt8) - unsafe_takestring(sv) + return str end function digits!(a::AbstractVector{T}, n::BigInt; base::Integer = 10) where {T<:Integer} @@ -850,23 +889,50 @@ if Limb === UInt64 === UInt # an optimized version for BigInt of hash_integer (used e.g. for Rational{BigInt}), # and of hash - using .Base: HASH_SECRET, hash_bytes, hash_finalizer + using .Base: HASH_SECRET, hash_bytes + + # UnsafeLimbView provides a safe iterator interface to BigInt limb data + struct UnsafeLimbView <: AbstractVector{UInt8} + bigint::BigInt + start_byte::Int + num_bytes::Int + end + + function Base.size(view::UnsafeLimbView) + return (view.num_bytes,) + end + + function Base.getindex(view::UnsafeLimbView, i::Int) + @boundscheck checkbounds(view, i) + GC.@preserve view begin + limb_index = div(view.start_byte + i - 2, 8) + 1 + byte_in_limb = (view.start_byte + i - 2) % 8 + limb = unsafe_load(view.bigint.d, limb_index) + return UInt8((limb >> (8 * byte_in_limb)) & 0xff) + end + end + + function Base.iterate(view::UnsafeLimbView, state::Int = 1) + state > view.num_bytes && return nothing + return @inbounds(view[state]), state + 1 + end + + function Base.length(view::UnsafeLimbView) + return view.num_bytes + end function hash_integer(n::BigInt, h::UInt) iszero(n) && return hash_integer(0, h) - GC.@preserve n begin - s = n.size - h ⊻= (s < 0) - - us = abs(s) - leading_zero_bytes = div(leading_zeros(unsafe_load(n.d, us)), 8) - hash_bytes( - Ptr{UInt8}(n.d), - 8 * us - leading_zero_bytes, - h, - HASH_SECRET - ) - end + s = n.size + h ⊻= (s < 0) + + us = abs(s) + leading_zero_bytes = div(leading_zeros(unsafe_load(n.d, us)), 8) + num_bytes = 8 * us - leading_zero_bytes + + # Use UnsafeLimbView for safe iterator-based access + limb_view = UnsafeLimbView(n, 1, num_bytes) + return hash_bytes(limb_view, h, HASH_SECRET) end function hash(x::BigInt, h::UInt) @@ -878,7 +944,7 @@ if Limb === UInt64 === UInt return hash(unsafe_load(ptr), h) elseif sz == -1 limb = unsafe_load(ptr) - limb <= typemin(Int) % UInt && return hash(-(limb % Int), h) + limb <= typemin(Int) % UInt && return hash(-%(limb % Int), h) end pow = trailing_zeros(x) nd = Base.ndigits0z(x, 2) @@ -901,12 +967,11 @@ if Limb === UInt64 === UInt h ⊻= (sz < 0) leading_zero_bytes = div(leading_zeros(unsafe_load(x.d, asz)), 8) trailing_zero_bytes = div(pow, 8) - return hash_bytes( - Ptr{UInt8}(x.d) + trailing_zero_bytes, - 8 * asz - (leading_zero_bytes + trailing_zero_bytes), - h, - HASH_SECRET - ) + num_bytes = 8 * asz - (leading_zero_bytes + trailing_zero_bytes) + + # Use UnsafeLimbView for safe iterator-based access + limb_view = UnsafeLimbView(x, trailing_zero_bytes + 1, num_bytes) + return hash_bytes(limb_view, h, HASH_SECRET) end end end @@ -917,7 +982,7 @@ module MPQ import .Base: unsafe_rational, __throw_rational_argerror_zero import ..GMP: BigInt, MPZ, Limb, libgmp -gmpq(op::Symbol) = (Symbol(:__gmpq_, op), libgmp) +gmpq(op::Symbol) = Expr(:tuple, QuoteNode(Symbol(:__gmpq_, op)), GlobalRef(MPZ, :libgmp)) mutable struct _MPQ num_alloc::Cint diff --git a/base/hamt.jl b/base/hamt.jl index c77c592b17e58..1f81cbe12edd3 100644 --- a/base/hamt.jl +++ b/base/hamt.jl @@ -34,7 +34,7 @@ export HAMT # into the storage array. # # HAMT can be both persistent and non-persistent. -# The `path` function searches for a matching entries, and for persistency +# The `path` function searches for a matching entry, and for persistency # optionally copies the path so that it can be safely mutated. # TODO: @@ -91,16 +91,16 @@ struct HashState{K} shift::Int end HashState(key) = HashState(key, objectid(key), 0, 0) -# Reconstruct -Base.@assume_effects :terminates_locally function HashState(other::HashState, key) - h = HashState(key) +Base.@assume_effects :terminates_locally function HashState{K}(@nospecialize(other::HashState), @nospecialize(key)) where {K} + h = HashState{K}(key, objectid(key), 0, 0) while h.depth !== other.depth h = next(h) end return h end +Base.@assume_effects :terminates_locally HashState(other::HashState, key) = HashState{typeof(key)}(other, key) -function next(h::HashState) +function next(h::HashState{K}) where {K} depth = h.depth + 1 shift = h.shift + BITS_PER_LEVEL # Assert disabled for effect precision @@ -113,7 +113,7 @@ function next(h::HashState) else h_hash = h.hash end - return HashState(h.key, h_hash, depth, shift) + return HashState{K}(h.key, h_hash, depth, shift) end struct BitmapIndex @@ -155,7 +155,7 @@ as the current `level`. If a copy function is provided `copyf` use the return `top` for the new persistent tree. """ -@inline @Base.assume_effects :noub :terminates_locally function path(trie::HAMT{K,V}, key, h::HashState, copy=false) where {K, V} +@inline Base.@assume_effects :noub :terminates_locally function path(trie::HAMT{K,V}, h::HashState, copy=false) where {K, V} if copy trie = top = HAMT{K,V}(Base.copy(trie.data), trie.bitmap) else @@ -199,10 +199,10 @@ or grows the HAMT by inserting a new trie instead. end set!(trie, bi) else - @assert present + @assert present "!found && !present" # collision -> grow leaf = @inbounds trie.data[i]::Leaf{K,V} - leaf_h = HashState(h, leaf.key) + leaf_h = HashState{K}(h, leaf.key) if leaf_h.hash == h.hash error("Perfect hash collision") end diff --git a/base/hashing.jl b/base/hashing.jl index 897a0d73cb874..2fad477a29abc 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -1,16 +1,17 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license const HASH_SEED = UInt == UInt64 ? 0xbdd89aa982704029 : 0xeabe9406 -const HASH_SECRET = tuple( +const HASH_SECRET = ( 0x2d358dccaa6c78a5, 0x8bb84b93962eacc9, 0x4b33a62ed433d4a3, + 0xaaaaaaaaaaaaaaaa, ) """ hash(x[, h::UInt])::UInt -Compute an integer hash code such that `isequal(x,y)` implies `hash(x)==hash(y)`. The +Compute an integer hash code such that `isequal(x,y)` implies `isequal(hash(x), hash(y))`. The optional second argument `h` is another hash code to be mixed with the result. New types should implement the 2-argument form, typically by calling the 2-argument `hash` @@ -20,6 +21,12 @@ Typically, any type that implements `hash` should also implement its own [`==`]( The hash value may change when a new Julia process is started. +!!! warning + When implementing the 2-argument form, the second argument `h` should _not_ be given a + default value such `h = UInt(0)` as this will implicitly create a 1-argument method that + is more specific than the fallback (see [Note on Optional and keyword Arguments](@ref)), + but potentially with the wrong seed, causing hash inconsistencies. + ```jldoctest; filter = r"0x[0-9a-f]{16}" julia> a = hash(10) 0x759d18cc5346a65f @@ -28,13 +35,14 @@ julia> hash(10, a) # only use the output of another hash function as the second 0x03158cd61b1b0bd1 ``` -See also: [`objectid`](@ref), [`Dict`](@ref), [`Set`](@ref). +See also [`objectid`](@ref), [`Dict`](@ref), [`Set`](@ref). """ hash(data::Any) = hash(data, HASH_SEED) hash(w::WeakRef, h::UInt) = hash(w.value, h) # Types can't be deleted, so marking as total allows the compiler to look up the hash @noinline _jl_type_hash(T::Type) = @assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T) +@noinline _jl_type_cache_hash(T::Type) = @assume_effects :total ccall(:jl_type_cache_hash, UInt, (Any,), T) hash(T::Type, h::UInt) = hash(_jl_type_hash(T), h) hash(@nospecialize(data), h::UInt) = hash(objectid(data), h) @@ -45,10 +53,10 @@ end hash_mix(a::UInt64, b::UInt64) = ⊻(mul_parts(a, b)...) # faster-but-weaker than hash_mix intended for small keys -hash_mix_linear(x::Union{UInt64, UInt32}, h::UInt) = 3h - x +hash_mix_linear(x::Union{UInt64, UInt32}, h::UInt) = 3 *% h -% x function hash_finalizer(x::UInt64) x ⊻= (x >> 32) - x *= 0x63652a4cd374b267 + x *%= 0x63652a4cd374b267 x ⊻= (x >> 33) return x end @@ -69,78 +77,86 @@ hash(x::UInt64, h::UInt) = hash_uint64(hash_mix_linear(x, h)) hash(x::Int64, h::UInt) = hash(bitcast(UInt64, x), h) hash(x::Union{Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32}, h::UInt) = hash(Int64(x), h) -hash_integer(x::Integer, h::UInt) = _hash_integer(x, UInt64(h)) % UInt -function _hash_integer( - x::Integer, - seed::UInt64 = HASH_SEED, - secret::NTuple{3, UInt64} = HASH_SECRET - ) - seed ⊻= (x < 0) - u = abs(x) +# IntegerCodeUnits provides a little-endian byte representation of integers +struct IntegerCodeUnits{T<:Integer} <: AbstractVector{UInt8} + uvalue::T + num_bytes::Int - # always left-pad to full byte - buflen = UInt(max(cld(top_set_bit(u), 8), 1)) - seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) + function IntegerCodeUnits(x::T) where {T<:Integer} + # Calculate number of bytes needed (always pad to full byte) + u = abs(x) + num_bytes = max(cld(top_set_bit(u), 8), 1) + return new{T}(u, num_bytes) + end +end +size(units::IntegerCodeUnits) = (units.num_bytes,) +length(units::IntegerCodeUnits) = units.num_bytes +@inline getindex(units::IntegerCodeUnits, i::Int) = (units.uvalue >>> (8 * (i - 1))) % UInt8 +@inline load_le_array(::Type{UInt64}, units::IntegerCodeUnits, idx) = (units.uvalue >>> (8 * (idx - 1))) % UInt64 +@inline load_le_array(::Type{UInt32}, units::IntegerCodeUnits, idx) = (units.uvalue >>> (8 * (idx - 1))) % UInt32 - a = zero(UInt64) - b = zero(UInt64) - if buflen ≤ 16 - if buflen ≥ 4 - a = (UInt64(u % UInt32) << 32) | - UInt64((u >>> ((buflen - 4) * 8)) % UInt32) - - delta = (buflen & 24) >>> (buflen >>> 3) - - b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) | - UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32) - else # buflen > 0 - b0 = u % UInt8 - b1 = (u >>> (8 * div(buflen, 2))) % UInt8 - b2 = (u >>> (8 * (buflen - 1))) % UInt8 - a = (UInt64(b0) << 56) | - (UInt64(b1) << 32) | - UInt64(b2) - end - else - a = (u >>> 8(buflen - 16)) % UInt - b = (u >>> 8(buflen - 8)) % UInt +# Main interface function to get little-endian byte representation of integers +codeunits(x::Integer) = IntegerCodeUnits(x) - i = buflen - if i > 48 - see1 = seed - see2 = seed - while i ≥ 48 - l0 = u % UInt; u >>>= 64 - l1 = u % UInt; u >>>= 64 - l2 = u % UInt; u >>>= 64 - l3 = u % UInt; u >>>= 64 - l4 = u % UInt; u >>>= 64 - l5 = u % UInt; u >>>= 64 - - seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed) - see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1) - see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2) - i -= 48 - end - seed = seed ⊻ see1 ⊻ see2 - end - if i > 16 - l0 = u % UInt; u >>>= 64 - l1 = u % UInt; u >>>= 64 - seed = hash_mix(l0 ⊻ secret[3], l1 ⊻ seed ⊻ secret[2]) - if i > 32 - l2 = u % UInt; u >>>= 64 - l3 = u % UInt; u >>>= 64 - seed = hash_mix(l2 ⊻ secret[3], l3 ⊻ seed) - end - end +# UTF8Units provides UTF-8 byte iteration for any AbstractString +struct UTF8Units{T<:AbstractString} + string::T +end + +utf8units(s::AbstractString) = codeunit(s) <: UInt8 ? codeunits(s) : UTF8Units(s) + +# Iterator state: (char_iter_state, remaining_utf8_bytes) +function iterate(units::UTF8Units) + char_result = iterate(units.string) + char_result === nothing && return nothing + char, char_state = char_result + + # Decode char to UTF-8 bytes (similar to the write function) + u = bswap(reinterpret(UInt32, char)) + + # Return first byte and set up state for remaining bytes + first_byte = u % UInt8 + remaining_bytes = u >> 8 + return first_byte, (char_state, remaining_bytes) +end + +function iterate(units::UTF8Units, state) + char_state, remaining_bytes = state + # If we have more bytes from current char, return next byte + if remaining_bytes != 0 + byte = remaining_bytes % UInt8 + new_remaining = remaining_bytes >> 8 + return byte, (char_state, new_remaining) end - a = a ⊻ secret[2] - b = b ⊻ seed - b, a = mul_parts(a, b) - return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2]) + # Move to next char + char_result = iterate(units.string, char_state) + char_result === nothing && return nothing + char, new_char_state = char_result + + # Decode new char to UTF-8 bytes + u = bswap(reinterpret(UInt32, char)) + + # Return first byte and set up state for remaining bytes + first_byte = u % UInt8 + remaining_bytes = u >> 8 + + return first_byte, (new_char_state, remaining_bytes) +end + +hash_integer(x::Integer, h::UInt) = _hash_integer(x, UInt64(h)) % UInt +function _hash_integer( + x::Integer, + seed::UInt64, + secret::NTuple{4, UInt64} = HASH_SECRET + ) + # Handle sign by XOR-ing with seed + seed ⊻= (x < 0) + # Get little-endian byte representation of absolute value + # and hash using the new safe hash_bytes function + u = abs(x) # n.b.: this hashes typemin(IntN) correctly even if abs fails + return hash_bytes(codeunits(u), seed, secret) end @@ -214,7 +230,7 @@ function hash(x::Real, h::UInt) left <= 64 && !signbit(num) && return hash(UInt64(num) << Int(pow), h) end # typemin(Int64) handled by Float64 case # 2^1024 is the maximum Float64 so if the power is greater, not a Float64 - # Float64s only have 53 mantisa bits (including implicit bit) + # Float64s have 53 mantissa bits (including implicit bit) left <= 1024 && left - pow <= 53 && return hash(ldexp(Float64(num), pow), h) end else @@ -233,17 +249,21 @@ end ## symbol & expression hashing ## if UInt === UInt64 # conservatively hash using == equality of all of the data, even though == often uses === internally - hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x83c7900696d26dc6)) hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x2c97bf8b3de87020) hash(x::PhiNode, h::UInt) = hash(x.edges, hash(x.values, h ⊻ 0x2c97bf8b3de87020)) hash(x::PhiCNode, h::UInt) = hash(x.values, h ⊻ 0x2c97bf8b3de87020) else - hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x469d72af)) hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x469d72af) hash(x::PhiNode, h::UInt) = hash(x.edges, hash(x.values, h ⊻ 0x469d72af)) hash(x::PhiCNode, h::UInt) = hash(x.values, h ⊻ 0x469d72af) end +function hash(x::Expr, h::UInt) + h = hash(x.head, h ⊻ (UInt === UInt64 ? 0x83c7900696d26dc6 : 0x469d72af)) + # Hint that `x.args::Vector{Any}` is mostly Expr, Symbol, and LineNumberNode. + hash_shaped(x.args, h ⊻ hash_abstractarray_seed, (Val{Expr}(), Val{Symbol}(), Val{LineNumberNode}())) +end + function hash(x::CodeInfo, h::UInt) h ⊻= UInt === UInt64 ? 0x2c97bf8b3de87020 : 0x469d72af for i in 1:nfields(x) @@ -262,47 +282,55 @@ end hash(x::Symbol) = objectid(x) +const hashgr_seed = 0xe19bede84d316c06 % UInt +const hashps_seed = 0xee5c84c1439961a8 % UInt +hash(x::GlobalRef, h::UInt) = hash(x.name, hash(x.mod, h ⊻ hashgr_seed)) +function hash(x::PartialStruct, h::UInt) + h ⊻= hashps_seed + h = hash(x.typ, h) + h = hash(x.undefs, h) + h = hash(x.fields, h) + return h +end + load_le(::Type{T}, ptr::Ptr{UInt8}, i) where {T <: Union{UInt32, UInt64}} = unsafe_load(convert(Ptr{T}, ptr + i - 1)) -function read_small(ptr::Ptr{UInt8}, n::Int) - return (UInt64(unsafe_load(ptr)) << 56) | - (UInt64(unsafe_load(ptr, div(n, 2) + 1)) << 32) | - UInt64(unsafe_load(ptr, n)) -end - @assume_effects :terminates_globally function hash_bytes( ptr::Ptr{UInt8}, n::Int, seed::UInt64, - secret::NTuple{3, UInt64} + secret::NTuple{4, UInt64} ) # Adapted with gratitude from [rapidhash](https://github.com/Nicoshev/rapidhash) buflen = UInt64(n) - seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) + seed = seed ⊻ hash_mix(seed ⊻ secret[3], secret[2]) a = zero(UInt64) b = zero(UInt64) + i = buflen if buflen ≤ 16 if buflen ≥ 4 - a = (UInt64(load_le(UInt32, ptr, 1)) << 32) | - UInt64(load_le(UInt32, ptr, n - 3)) - - delta = (buflen & 24) >>> (buflen >>> 3) - b = (UInt64(load_le(UInt32, ptr, delta + 1)) << 32) | - UInt64(load_le(UInt32, ptr, n - 3 - delta)) + seed ⊻= buflen + if buflen ≥ 8 + a = load_le(UInt64, ptr, 1) + b = load_le(UInt64, ptr, n - 7) + else + a = UInt64(load_le(UInt32, ptr, 1)) + b = UInt64(load_le(UInt32, ptr, n - 3)) + end elseif buflen > 0 - a = read_small(ptr, n) + a = (UInt64(unsafe_load(ptr)) << 45) | UInt64(unsafe_load(ptr, n)) + b = UInt64(unsafe_load(ptr, div(n, 2) + 1)) end else pos = 1 - i = buflen if i > 48 see1 = seed see2 = seed - while i ≥ 48 + while i > 48 seed = hash_mix( load_le(UInt64, ptr, pos) ⊻ secret[1], load_le(UInt64, ptr, pos + 8) ⊻ seed @@ -318,12 +346,13 @@ end pos += 48 i -= 48 end - seed = seed ⊻ see1 ⊻ see2 + seed ⊻= see1 + seed ⊻= see2 end if i > 16 seed = hash_mix( load_le(UInt64, ptr, pos) ⊻ secret[3], - load_le(UInt64, ptr, pos + 8) ⊻ seed ⊻ secret[2] + load_le(UInt64, ptr, pos + 8) ⊻ seed ) if i > 32 seed = hash_mix( @@ -333,19 +362,288 @@ end end end - a = load_le(UInt64, ptr, n - 15) + a = load_le(UInt64, ptr, n - 15) ⊻ i b = load_le(UInt64, ptr, n - 7) end a = a ⊻ secret[2] b = b ⊻ seed b, a = mul_parts(a, b) - return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2]) + return hash_mix(a ⊻ secret[4], b ⊻ secret[2] ⊻ i) +end + +@inline function load_le_array(::Type{UInt64}, arr::AbstractArray{UInt8}, idx) + # n.b. for whatever reason, writing this as a loop ensures LLVM + # optimizations (particular SROA) don't make a disaster of this code + # early on so it can actually emit the optimal result + result = zero(UInt64) + for i in 0:7 + byte = @inbounds arr[idx + i] + result |= UInt64(byte) << (8 * i) + end + return result +end + +@inline function load_le_array(::Type{UInt32}, arr::AbstractArray{UInt8}, idx) + result = zero(UInt32) + for i in 0:3 + byte = @inbounds arr[idx + i] + result |= UInt32(byte) << (8 * i) + end + return result +end + +@assume_effects :terminates_globally function hash_bytes( + arr::AbstractArray{UInt8}, + seed::UInt64, + secret::NTuple{4, UInt64} + ) + # Adapted with gratitude from [rapidhash](https://github.com/Nicoshev/rapidhash) + n = length(arr) + buflen = UInt64(n) + seed = seed ⊻ hash_mix(seed ⊻ secret[3], secret[2]) + firstidx = firstindex(arr) + + a = zero(UInt64) + b = zero(UInt64) + i = buflen + + if buflen ≤ 16 + if buflen ≥ 4 + seed ⊻= buflen + if buflen ≥ 8 + a = load_le_array(UInt64, arr, firstidx) + b = load_le_array(UInt64, arr, firstidx + n - 8) + else + a = UInt64(load_le_array(UInt32, arr, firstidx)) + b = UInt64(load_le_array(UInt32, arr, firstidx + n - 4)) + end + elseif buflen > 0 + a = (UInt64(@inbounds arr[firstidx]) << 45) | UInt64(@inbounds arr[firstidx + n - 1]) + b = UInt64(@inbounds arr[firstidx + div(n, 2)]) + end + else + pos = 0 + if i > 48 + see1 = seed + see2 = seed + while i > 48 + seed = hash_mix( + load_le_array(UInt64, arr, firstidx + pos) ⊻ secret[1], + load_le_array(UInt64, arr, firstidx + pos + 8) ⊻ seed + ) + see1 = hash_mix( + load_le_array(UInt64, arr, firstidx + pos + 16) ⊻ secret[2], + load_le_array(UInt64, arr, firstidx + pos + 24) ⊻ see1 + ) + see2 = hash_mix( + load_le_array(UInt64, arr, firstidx + pos + 32) ⊻ secret[3], + load_le_array(UInt64, arr, firstidx + pos + 40) ⊻ see2 + ) + pos += 48 + i -= 48 + end + seed ⊻= see1 + seed ⊻= see2 + end + if i > 16 + seed = hash_mix( + load_le_array(UInt64, arr, firstidx + pos) ⊻ secret[3], + load_le_array(UInt64, arr, firstidx + pos + 8) ⊻ seed + ) + if i > 32 + seed = hash_mix( + load_le_array(UInt64, arr, firstidx + pos + 16) ⊻ secret[3], + load_le_array(UInt64, arr, firstidx + pos + 24) ⊻ seed + ) + end + end + + a = load_le_array(UInt64, arr, firstidx + n - 16) ⊻ i + b = load_le_array(UInt64, arr, firstidx + n - 8) + end + + a = a ⊻ secret[2] + b = b ⊻ seed + b, a = mul_parts(a, b) + return hash_mix(a ⊻ secret[4], b ⊻ secret[2] ⊻ i) end + +# Helper function to concatenate two UInt64 values with a byte shift +# Returns the result of shifting 'low' right by 'shift_bytes' bytes and +# filling the high bits with the low bits of 'high' +@inline function concat_shift(low::UInt64, high::UInt64, shift_bytes::UInt8) + shift_bits = (shift_bytes * 0x8) & 0x3f + return (low >> shift_bits) | (high << (0x40 - shift_bits)) +end + +@inline function read_uint64_from_uint8_iter(iter, state) + value = zero(UInt64) + @nexprs 8 i -> begin + next_result = iterate(iter, state) + next_result === nothing && return value, state, UInt8(i - 1) + byte, state = next_result + value |= UInt64(byte) << ((i - 1) * 8) + end + return value, state, 0x8 +end + +@inline function read_uint64_from_uint8_iter(iter) + next_result = iterate(iter) + next_result === nothing && return nothing + byte, state = next_result + value = UInt64(byte) + @nexprs 7 i -> begin + next_result = iterate(iter, state) + next_result === nothing && return value, state, UInt8(i) + byte, state = next_result + value |= UInt64(byte::UInt8) << (i * 8) + end + return value, state, 0x8 +end + +@assume_effects :terminates_globally function hash_bytes( + iter, + seed::UInt64, + secret::NTuple{4, UInt64} + ) + seed = seed ⊻ hash_mix(seed ⊻ secret[3], secret[2]) + + a = zero(UInt64) + b = zero(UInt64) + buflen = zero(UInt64) + + see1 = seed + see2 = seed + l0 = zero(UInt64) + l1 = zero(UInt64) + l2 = zero(UInt64) + l3 = zero(UInt64) + l4 = zero(UInt64) + l5 = zero(UInt64) + b0 = 0x0 + b1 = 0x0 + b2 = 0x0 + b3 = 0x0 + b4 = 0x0 + b5 = 0x0 + t0 = zero(UInt64) + t1 = zero(UInt64) + + # Handle first iteration separately + read = read_uint64_from_uint8_iter(iter) + if read !== nothing + l0, state, b0 = read + # Repeat hashing chunks until a short read + while true + l1, state, b1 = read_uint64_from_uint8_iter(iter, state) + if b1 == 0x8 + l2, state, b2 = read_uint64_from_uint8_iter(iter, state) + if b2 == 0x8 + l3, state, b3 = read_uint64_from_uint8_iter(iter, state) + if b3 == 0x8 + l4, state, b4 = read_uint64_from_uint8_iter(iter, state) + if b4 == 0x8 + l5, state, b5 = read_uint64_from_uint8_iter(iter, state) + if b5 == 0x8 + # Read start of next chunk + read = read_uint64_from_uint8_iter(iter, state) + if read[3] == 0x0 + # Read exactly 48 bytes + t0 = l4 + t1 = l5 + break + else + # Read more than 48 bytes - process and continue to next chunk + seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed) + see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1) + see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2) + buflen += 48 + l0, state, b0 = read + b1 = 0 + b2 = 0 + b3 = 0 + b4 = 0 + b5 = 0 + if b0 < 8 + t0 = concat_shift(l4, l5, b0) + t1 = concat_shift(l5, l0, b0) + break + end + end + else + # Extract final 16 bytes at the first short read + t0 = concat_shift(l3, l4, b5) + t1 = concat_shift(l4, l5, b5) + break + end + else + t0 = concat_shift(l2, l3, b4) + t1 = concat_shift(l3, l4, b4) + break + end + else + t0 = concat_shift(l1, l2, b3) + t1 = concat_shift(l2, l3, b3) + break + end + else + t0 = concat_shift(l0, l1, b2) + t1 = concat_shift(l1, l2, b2) + break + end + else + t0 = concat_shift(l5, l0, b1) + t1 = concat_shift(l0, l1, b1) + break + end + end + end + + # Partial chunk, handle based on size + bytes_chunk = b0 + b1 + b2 + b3 + b4 + b5 + if buflen > 0 + # Finalize last full chunk + seed ⊻= see1 + seed ⊻= see2 + end + buflen += bytes_chunk + if buflen ≤ 16 + if bytes_chunk ≥ 0x4 + seed ⊻= bytes_chunk + if bytes_chunk ≥ 0x8 + a = l0 + b = t1 + else + a = UInt64(l0 % UInt32) + b = UInt64((l0 >>> ((0x8 * (bytes_chunk - 0x4)) % 0x3f)) % UInt32) + end + elseif bytes_chunk > 0x0 + b0 = l0 % UInt8 + b1 = (l0 >>> ((0x8 * div(bytes_chunk, 0x2)) % 0x3f)) % UInt8 + b2 = (l0 >>> ((0x8 * (bytes_chunk - 0x1)) % 0x3f)) % UInt8 + a = (UInt64(b0) << 45) | UInt64(b2) + b = UInt64(b1) + end + else + if bytes_chunk > 0x10 + seed = hash_mix(l0 ⊻ secret[3], l1 ⊻ seed) + if bytes_chunk > 0x20 + seed = hash_mix(l2 ⊻ secret[3], l3 ⊻ seed) + end + end + a = t0 ⊻ bytes_chunk + b = t1 + end + + a = a ⊻ secret[2] + b = b ⊻ seed + b, a = mul_parts(a, b) + return hash_mix(a ⊻ secret[4], b ⊻ secret[2] ⊻ bytes_chunk) +end + +hash(data::AbstractString, h::UInt) = + hash_bytes(utf8units(data), UInt64(h), HASH_SECRET) % UInt @assume_effects :total hash(data::String, h::UInt) = GC.@preserve data hash_bytes(pointer(data), sizeof(data), UInt64(h), HASH_SECRET) % UInt - -# no longer used in Base, but a lot of packages access these internals -const memhash = UInt === UInt64 ? :memhash_seed : :memhash32_seed -const memhash_seed = UInt === UInt64 ? 0x71e729fd56419c81 : 0x56419c81 diff --git a/base/iddict.jl b/base/iddict.jl index ec5392cf7b5b8..cd0caa8921b75 100644 --- a/base/iddict.jl +++ b/base/iddict.jl @@ -32,6 +32,9 @@ mutable struct IdDict{K,V} <: AbstractDict{K,V} function IdDict{K,V}(itr) where {K, V} d = IdDict{K,V}() + if IteratorSize(itr) isa Union{HasLength, HasShape} + sizehint!(d, length(itr)) + end for (k,v) in itr; d[k] = v; end d end diff --git a/base/idset.jl b/base/idset.jl index 4ad9426172909..46e9f1f5195ba 100644 --- a/base/idset.jl +++ b/base/idset.jl @@ -10,6 +10,9 @@ In the example below, the values are all `isequal` so they get overwritten in the ordinary `Set`. The `IdSet` compares by `===` and so preserves the 3 different values. +!!! compat "Julia 1.11" + Exported in Julia 1.11 and later. + # Examples ```jldoctest; filter = r"\\n\\s*(1|1\\.0|true)" julia> Set(Any[true, 1, 1.0]) @@ -49,7 +52,7 @@ function push!(s::IdSet, @nospecialize(x)) else if s.max < length(s.list) idx = s.max - @assert !isassigned(s.list, idx + 1) + @assert !isassigned(s.list, idx + 1) "bucket is already occupied" s.list[idx + 1] = x s.max = idx + 1 else @@ -58,7 +61,7 @@ function push!(s::IdSet, @nospecialize(x)) idx = newidx[] s.max = idx < 0 ? -idx : idx + 1 end - @assert s.list[s.max] === x + @assert s.list[s.max] === x "unexpected object in bucket" setfield!(s, :idxs, ccall(:jl_idset_put_idx, Any, (Any, Any, Int), s.list, s.idxs, idx)) s.count += 1 end @@ -105,7 +108,7 @@ function empty!(s::IdSet) _zero!(s.idxs) list = s.list for i = 1:s.max - _unsetindex!(list, i) + unsetindex!(list, i) end s.count = 0 s.max = 0 diff --git a/base/indices.jl b/base/indices.jl index 88e48a2b331ee..7965cb6e42152 100644 --- a/base/indices.jl +++ b/base/indices.jl @@ -215,6 +215,7 @@ end # those are the permutations that preserve the order of the non-singleton # dimensions. function setindex_shape_check(X::AbstractArray, I::Integer...) + @inline li = ndims(X) lj = length(I) i = j = 1 @@ -385,15 +386,13 @@ end Slice(S::Slice) = S Slice{T}(S::Slice) where {T<:AbstractUnitRange} = Slice{T}(T(S.indices)) -axes(S::Slice) = (IdentityUnitRange(S.indices),) +axes(S::Slice) = (axes1(S),) axes1(S::Slice) = IdentityUnitRange(S.indices) -axes(S::Slice{<:OneTo}) = (S.indices,) -axes1(S::Slice{<:OneTo}) = S.indices +axes1(S::Slice{<:AbstractOneTo{<:Integer}}) = S.indices first(S::Slice) = first(S.indices) last(S::Slice) = last(S.indices) size(S::Slice) = (length(S.indices),) -length(S::Slice) = length(S.indices) getindex(S::Slice, i::Int) = (@inline; @boundscheck checkbounds(S, i); i) getindex(S::Slice, i::AbstractUnitRange{<:Integer}) = (@inline; @boundscheck checkbounds(S, i); i) getindex(S::Slice, i::StepRange{<:Integer}) = (@inline; @boundscheck checkbounds(S, i); i) @@ -414,15 +413,13 @@ IdentityUnitRange(S::IdentityUnitRange) = S IdentityUnitRange{T}(S::IdentityUnitRange) where {T<:AbstractUnitRange} = IdentityUnitRange{T}(T(S.indices)) # IdentityUnitRanges are offset and thus have offset axes, so they are their own axes -axes(S::IdentityUnitRange) = (S,) +axes(S::IdentityUnitRange) = (axes1(S),) axes1(S::IdentityUnitRange) = S -axes(S::IdentityUnitRange{<:OneTo}) = (S.indices,) -axes1(S::IdentityUnitRange{<:OneTo}) = S.indices +axes1(S::IdentityUnitRange{<:AbstractOneTo{<:Integer}}) = S.indices first(S::IdentityUnitRange) = first(S.indices) last(S::IdentityUnitRange) = last(S.indices) size(S::IdentityUnitRange) = (length(S.indices),) -length(S::IdentityUnitRange) = length(S.indices) unsafe_length(S::IdentityUnitRange) = unsafe_length(S.indices) getindex(S::IdentityUnitRange, i::Integer) = (@inline; @boundscheck checkbounds(S, i); convert(eltype(S), i)) getindex(S::IdentityUnitRange, i::Bool) = throw(ArgumentError("invalid index: $i of type Bool")) @@ -465,11 +462,11 @@ end show(io::IO, r::IdentityUnitRange) = print(io, "Base.IdentityUnitRange(", r.indices, ")") iterate(S::IdentityUnitRange, s...) = iterate(S.indices, s...) -# For OneTo, the values and indices of the values are identical, so this may be defined in Base. +# For AbstractOneTo, the values and indices of the values are identical, so this may be defined in Base. # In general such an indexing operation would produce offset ranges # This should also ideally return an AbstractUnitRange{eltype(S)}, but currently # we're restricted to eltype(::IdentityUnitRange) == Int by definition -function getindex(S::OneTo, I::IdentityUnitRange{<:AbstractUnitRange{<:Integer}}) +function getindex(S::AbstractOneTo{<:Integer}, I::IdentityUnitRange{<:AbstractUnitRange{<:Integer}}) @inline @boundscheck checkbounds(S, I) return I diff --git a/base/initdefs.jl b/base/initdefs.jl index cc829f1823dd4..c89126a4ba515 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -30,6 +30,41 @@ exit() = exit(0) const roottask = current_task() +const _foreground_task = Lockable(Ref{Union{Task, Nothing}}(nothing)) + +""" + Base.foreground_task() -> Union{Task, Nothing} + +Return the task currently designated as the "foreground" task — typically the task +that owns interactive stdin (e.g. a REPL command-execution task). Returns `nothing` +if no task has been registered as foreground. +""" +foreground_task() = lock(getindex, _foreground_task) + +""" + Base.@as_foreground_task expr + +Evaluate `expr` with [`current_task()`](@ref) registered as the foreground task +(see [`foreground_task`](@ref)), restoring the previous foreground task on exit. +Used to mark the section of code where a particular task "owns" interactive stdin +so that other components (e.g. the precompile keyboard menu) can defer to it. +""" +macro as_foreground_task(expr) + quote + local ref = $(GlobalRef(@__MODULE__, :_foreground_task)) + local prev = lock(ref) do r + local old = r[] + r[] = current_task() + old + end + try + $(esc(expr)) + finally + lock(r -> r[] = prev, ref) + end + end +end + is_interactive::Bool = false """ @@ -97,7 +132,7 @@ const DEPOT_PATH = String[] function append_bundled_depot_path!(DEPOT_PATH) path = abspath(Sys.BINDIR, "..", "local", "share", "julia") path in DEPOT_PATH || push!(DEPOT_PATH, path) - path = abspath(Sys.BINDIR, "..", "share", "julia") + path = abspath(Sys.BINDIR, DATAROOTDIR, "julia") path in DEPOT_PATH || push!(DEPOT_PATH, path) return DEPOT_PATH end @@ -165,8 +200,7 @@ the [`JULIA_LOAD_PATH`](@ref JULIA_LOAD_PATH) environment variable if set; otherwise it defaults to `["@", "@v#.#", "@stdlib"]`. Entries starting with `@` have special meanings: -- `@` refers to the "current active environment", the initial value of which is - initially determined by the [`JULIA_PROJECT`](@ref JULIA_PROJECT) environment +- `@` refers to the "current active environment", whose value is determined by the [`JULIA_PROJECT`](@ref JULIA_PROJECT) environment variable or the `--project` command-line option. - `@stdlib` expands to the absolute path of the current Julia installation's @@ -262,7 +296,7 @@ end function init_active_project() project = (JLOptions().project != C_NULL ? - unsafe_string(Base.JLOptions().project) : + unsafe_string(JLOptions().project) : get(ENV, "JULIA_PROJECT", nothing)) set_active_project( project === nothing ? nothing : @@ -271,6 +305,22 @@ function init_active_project() ) end +function init_named_env!(path) + try + mkpath(dirname(path)) + io = open(path, "w") + try + print(io, "syntax.julia_version = \"",VERSION,"\"") + finally + close(io) + end + return path + catch e + @warn "Failed to initialize named environment at $path: $e" + return nothing + end +end + ## load path expansion: turn LOAD_PATH entries into concrete paths ## cmd_suppresses_program(cmd) = cmd in ('e', 'E') @@ -307,7 +357,8 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing} end end isempty(DEPOT_PATH) && return nothing - return abspath(DEPOT_PATH[1], "environments", name, project_names[end]) + new_named_env_path = abspath(DEPOT_PATH[1], "environments", name, project_names[end]) + return init_named_env!(new_named_env_path) end # otherwise, it's a path path = abspath(env) @@ -364,13 +415,37 @@ function set_active_project(projfile::Union{AbstractString,Nothing}) ACTIVE_PROJECT[] = projfile for f in active_project_callbacks try - Base.invokelatest(f) + invokelatest(f) catch @error "active project callback $f failed" maxlog=1 end end end +""" + active_manifest() + active_manifest(project_file::AbstractString) + +Return the path of the active manifest file, or the manifest file that would be used for a given `project_file`. + +In a stacked environment (where multiple environments exist in the load path), this returns the manifest +file for the primary (active) environment only, not the manifests from other environments in the stack. +See the manual section on [Environment stacks](@ref) for more details on how stacked environments work. + +See [`Project environments`](@ref project-environments) for details on the difference between a project and a manifest, and the naming +options and their priority in package loading. + +See also [`Base.active_project`](@ref), [`Base.set_active_project`](@ref). + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. +""" +function active_manifest(project_file::Union{AbstractString,Nothing}=nothing; search_load_path::Bool=true) + # If `project_file` was specified, use that, otherwise get the active project: + project_file = !isnothing(project_file) ? project_file : active_project(search_load_path) + project_file === nothing && return nothing + return project_file_manifest_path(project_file) +end """ load_path() @@ -425,7 +500,9 @@ This situation may occur if you are registering exit hooks from background Tasks may still be executing concurrently during shutdown. """ function atexit(f::Function) - Base.@lock _atexit_hooks_lock begin + # HACK: if generating output, hint that we might want to compile `f`, so that it is available for the no-codegen test + generating_output() && (precompile(f, (Cint,)) || precompile(f, ())) + @lock _atexit_hooks_lock begin _atexit_hooks_finished && error("cannot register new atexit hook; already exiting.") pushfirst!(atexit_hooks, f) return nothing @@ -437,7 +514,25 @@ function _atexit(exitcode::Cint) # this exit came from a signal for example), then try to clear that state # to minimize scheduler issues later ct = current_task() - q = ct.queue; q === nothing || list_deletefirst!(q::IntrusiveLinkedList{Task}, ct) + q = ct.queue + if q !== nothing + try + list_deletefirst!(q::StickyWorkqueue, ct) + catch + # best-effort cleanup on the way out + end + end + # If this task's wait registration is still armed (e.g. this exit came + # from a signal while parked), disarm it: the waitee-queue entry is + # stale and gets dropped by whoever pops it, and the sticky source + # registration is collected by a walk once this task is done. + @atomicswap ct.waiting_on = nothing + # We are exiting: any pending cancellation of this task's scope is moot + # and would only disrupt the atexit hooks - run them in a shielded scope. + return ScopedValues.@with(CANCEL_TOKEN => nothing, _run_atexit_hooks(exitcode)) +end + +function _run_atexit_hooks(exitcode::Cint) # Don't hold the lock around the iteration, just in case any other thread executing in # parallel tries to register a new atexit hook while this is running. We don't want to # block that thread from proceeding, and we can allow it to register its hook which we @@ -494,10 +589,13 @@ end ## hook for disabling threaded libraries ## library_threading_enabled::Bool = true -const disable_library_threading_hooks = [] + +# Base.OncePerProcess ensures that any registered hooks do not outlive the session. +# (even if they are registered during the sysimage build process by top-level code) +const disable_library_threading_hooks = OncePerProcess(Vector{Any}) function at_disable_library_threading(f) - push!(disable_library_threading_hooks, f) + push!(disable_library_threading_hooks(), f) if !library_threading_enabled disable_library_threading() end @@ -506,8 +604,8 @@ end function disable_library_threading() global library_threading_enabled = false - while !isempty(disable_library_threading_hooks) - f = pop!(disable_library_threading_hooks) + while !isempty(disable_library_threading_hooks()) + f = pop!(disable_library_threading_hooks()) try f() catch err diff --git a/base/int.jl b/base/int.jl index 57ebeda4812ad..4432758690727 100644 --- a/base/int.jl +++ b/base/int.jl @@ -87,6 +87,11 @@ signed(::Type{T}) where {T<:Signed} = T (+)(x::T, y::T) where {T<:BitInteger} = add_int(x, y) (*)(x::T, y::T) where {T<:BitInteger} = mul_int(x, y) +(-%)(x::BitInteger) = neg_int(x) +(-%)(x::T, y::T) where {T<:BitInteger} = sub_int(x, y) +(+%)(x::T, y::T) where {T<:BitInteger} = add_int(x, y) +(*%)(x::T, y::T) where {T<:BitInteger} = mul_int(x, y) + negate(x) = -x negate(x::Unsigned) = -convert(Signed, x) #widenegate(x) = -convert(widen(signed(typeof(x))), x) @@ -96,6 +101,44 @@ inv(x::Integer) = float(one(x)) / float(x) # skip promotion for system integer types (/)(x::BitInteger, y::BitInteger) = float(x) / float(y) + +""" + mul_hi(a::T, b::T) where {T<:Base.Integer} + +Return the higher half of the product of `a` and `b` where `T` is a fixed size integer. + +# Examples +```jldoctest +julia> Base.mul_hi(12345678987654321, 123456789) +82624 + +julia> (widen(12345678987654321) * 123456789) >> 64 +82624 + +julia> Base.mul_hi(0xff, 0xff) +0xfe +``` +""" +function mul_hi(a::T, b::T) where {T<:Integer} + ((widen(a)*b) >>> Base.top_set_bit(-1 % T)) % T +end + +function mul_hi(a::UInt128, b::UInt128) + shift = sizeof(a)*4 + mask = typemax(UInt128) >> shift + a1, a2 = a >>> shift, a & mask + b1, b2 = b >>> shift, b & mask + a1b1, a1b2, a2b1, a2b2 = a1*b1, a1*b2, a2*b1, a2*b2 + carry = ((a1b2 & mask) + (a2b1 & mask) + (a2b2 >>> shift)) >>> shift + a1b1 + (a1b2 >>> shift) + (a2b1 >>> shift) + carry +end + +function mul_hi(a::Int128, b::Int128) + shift = sizeof(a)*8 - 1 + t1, t2 = (a >> shift) & b % UInt128, (b >> shift) & a % UInt128 + (mul_hi(a % UInt128, b % UInt128) -% t1 -% t2) % Int128 +end + """ isodd(x::Number)::Bool @@ -261,7 +304,7 @@ exceptions, see note below). type, and so rounding error may occur. In particular, if the exact result is very close to `y`, then it may be rounded to `y`. -See also: [`rem`](@ref), [`div`](@ref), [`fld`](@ref), [`mod1`](@ref), [`invmod`](@ref). +See also: [`rem`](@ref), [`fld`](@ref), [`mod1`](@ref), [`fldmod`](@ref), [`invmod`](@ref). ```jldoctest julia> mod(8, 3) @@ -288,9 +331,17 @@ function mod(x::T, y::T) where T<:Integer y == -1 && return T(0) # avoid potential overflow in fld return x - fld(x, y) * y end +# For fixed-width integers the subtraction and product above wrap on valid +# inputs (`fld(x, y)*y` lies in `(x - y, x + y]`, e.g. `mod(typemin(Int), 3)`) +# and the result is correct only through the double wrap-cancellation. +function mod(x::T, y::T) where T<:BitSigned + y == -1 && return T(0) # avoid potential overflow in fld + return x -% fld(x, y) *% y +end function mod(x::BitSigned, y::Unsigned) remval = rem(x, y) # correct iff remval>=0 - return unsigned(remval + (remval0 so correct iff y>0 or remval==0 @@ -522,12 +573,19 @@ top_set_bit(x::BitInteger) = 8sizeof(x) - leading_zeros(x) (<=)(x::T, y::T) where {T<:BitSigned} = sle_int(x, y) (<=)(x::T, y::T) where {T<:BitUnsigned} = ule_int(x, y) -==(x::BitSigned, y::BitUnsigned) = (x >= 0) & (unsigned(x) == y) -==(x::BitUnsigned, y::BitSigned ) = (y >= 0) & (x == unsigned(y)) -<( x::BitSigned, y::BitUnsigned) = (x < 0) | (unsigned(x) < y) -<( x::BitUnsigned, y::BitSigned ) = (y >= 0) & (x < unsigned(y)) -<=(x::BitSigned, y::BitUnsigned) = (x < 0) | (unsigned(x) <= y) -<=(x::BitUnsigned, y::BitSigned ) = (y >= 0) & (x <= unsigned(y)) +==(x::BitSigned, y::BitUnsigned) = y == x +<(x::BitSigned, y::BitUnsigned) = !(y <= x) +<=(x::BitSigned, y::BitUnsigned) = !(y < x) + +for op in (:(==), :<, :<=) + @eval function $op(x::BitUnsigned, y::BitSigned) + if Core.sizeof(x) < Core.sizeof(y) + $op(typeof(y)(x), y) + else + (y >= 0) & $op(x, unsigned(y) % typeof(x)) + end + end +end ## integer shifts ## @@ -850,17 +908,169 @@ widemul(x::Bool,y::Number) = x * y widemul(x::Number,y::Bool) = x * y -# Int128 multiply and divide -*(x::T, y::T) where {T<:Union{Int128,UInt128}} = mul_int(x, y) +## wide multiplication, Int128 multiply and divide ## -div(x::Int128, y::Int128) = checked_sdiv_int(x, y) -div(x::UInt128, y::UInt128) = checked_udiv_int(x, y) +if Core.sizeof(Int) == 4 + function widemul(u::Int64, v::Int64) + local u0::UInt64, v0::UInt64, w0::UInt64 + local u1::Int64, v1::Int64, w1::UInt64, w2::Int64, t::UInt64 + + u0 = u & 0xffffffff; u1 = u >> 32 + v0 = v & 0xffffffff; v1 = v >> 32 + w0 = u0 *% v0 + t = reinterpret(UInt64, u1) *% v0 +% (w0 >>> 32) + w2 = reinterpret(Int64, t) >> 32 + w1 = u0 *% reinterpret(UInt64, v1) +% (t & 0xffffffff) + hi = u1 *% v1 +% w2 +% (reinterpret(Int64, w1) >> 32) + lo = (w0 & 0xffffffff) +% (w1 << 32) + return (Int128(hi) << 64) +% Int128(lo) + end -rem(x::Int128, y::Int128) = checked_srem_int(x, y) -rem(x::UInt128, y::UInt128) = checked_urem_int(x, y) + function widemul(u::UInt64, v::UInt64) + local u0::UInt64, v0::UInt64, w0::UInt64 + local u1::UInt64, v1::UInt64, w1::UInt64, w2::UInt64, t::UInt64 + + u0 = u & 0xffffffff; u1 = u >>> 32 + v0 = v & 0xffffffff; v1 = v >>> 32 + w0 = u0 *% v0 + t = u1 *% v0 +% (w0 >>> 32) + w2 = t >>> 32 + w1 = u0 *% v1 +% (t & 0xffffffff) + hi = u1 *% v1 +% w2 +% (w1 >>> 32) + lo = (w0 & 0xffffffff) +% (w1 << 32) + return (UInt128(hi) << 64) +% UInt128(lo) + end + + function *(u::Int128, v::Int128) + u0 = u % UInt64; u1 = Int64(u >> 64) + v0 = v % UInt64; v1 = Int64(v >> 64) + lolo = widemul(u0, v0) + lohi = widemul(reinterpret(Int64, u0), v1) + hilo = widemul(u1, reinterpret(Int64, v0)) + t = reinterpret(UInt128, hilo) +% (lolo >>> 64) + w1 = reinterpret(UInt128, lohi) +% (t & 0xffffffffffffffff) + return Int128(lolo & 0xffffffffffffffff) +% (reinterpret(Int128, w1) << 64) + end + + function *(u::UInt128, v::UInt128) + u0 = u % UInt64; u1 = UInt64(u>>>64) + v0 = v % UInt64; v1 = UInt64(v>>>64) + lolo = widemul(u0, v0) + lohi = widemul(u0, v1) + hilo = widemul(u1, v0) + t = hilo +% (lolo >>> 64) + w1 = lohi +% (t & 0xffffffffffffffff) + return (lolo & 0xffffffffffffffff) +% (UInt128(w1) << 64) + end + + function _setbit(x::UInt128, i) + # faster version of `return x | (UInt128(1) << i)` + j = i >> 5 + y = UInt128(one(UInt32) << (i & 0x1f)) + if j == 0 + return x | y + elseif j == 1 + return x | (y << 32) + elseif j == 2 + return x | (y << 64) + elseif j == 3 + return x | (y << 96) + end + return x + end + + function divrem(x::UInt128, y::UInt128) + iszero(y) && throw(DivideError()) + if (x >> 64) % UInt64 == 0 + if (y >> 64) % UInt64 == 0 + # fast path: upper 64 bits are zero, so we can fallback to UInt64 division + q64, x64 = divrem(x % UInt64, y % UInt64) + return UInt128(q64), UInt128(x64) + else + # this implies y>x, so + return zero(UInt128), x + end + end + n = leading_zeros(y) - leading_zeros(x) + q = zero(UInt128) + ys = y << n + while n >= 0 + # ys == y * 2^n + if ys <= x + x -= ys + q = _setbit(q, n) + if (x >> 64) % UInt64 == 0 + # exit early, similar to above fast path + if (y >> 64) % UInt64 == 0 + q64, x64 = divrem(x % UInt64, y % UInt64) + q |= q64 + x = UInt128(x64) + end + return q, x + end + end + ys >>>= 1 + n -= 1 + end + return q, x + end + + function div(x::Int128, y::Int128) + (x == typemin(Int128)) & (y == -1) && throw(DivideError()) + return Int128(div(BigInt(x), BigInt(y)))::Int128 + end + div(x::UInt128, y::UInt128) = divrem(x, y)[1] + + function rem(x::Int128, y::Int128) + return Int128(rem(BigInt(x), BigInt(y)))::Int128 + end + + function rem(x::UInt128, y::UInt128) + iszero(y) && throw(DivideError()) + if (x >> 64) % UInt64 == 0 + if (y >> 64) % UInt64 == 0 + # fast path: upper 64 bits are zero, so we can fallback to UInt64 division + return UInt128(rem(x % UInt64, y % UInt64)) + else + # this implies y>x, so + return x + end + end + n = leading_zeros(y) - leading_zeros(x) + ys = y << n + while n >= 0 + # ys == y * 2^n + if ys <= x + x -= ys + if (x >> 64) % UInt64 == 0 + # exit early, similar to above fast path + if (y >> 64) % UInt64 == 0 + x = UInt128(rem(x % UInt64, y % UInt64)) + end + return x + end + end + ys >>>= 1 + n -= 1 + end + return x + end + + function mod(x::Int128, y::Int128) + return Int128(mod(BigInt(x), BigInt(y)))::Int128 + end +else + *(x::T, y::T) where {T<:Union{Int128,UInt128}} = mul_int(x, y) + + div(x::Int128, y::Int128) = checked_sdiv_int(x, y) + div(x::UInt128, y::UInt128) = checked_udiv_int(x, y) + + rem(x::Int128, y::Int128) = checked_srem_int(x, y) + rem(x::UInt128, y::UInt128) = checked_urem_int(x, y) +end # issue #15489: since integer ops are unchecked, they shouldn't check promotion -for op in (:+, :-, :*, :&, :|, :xor) +for op in (:+, :-, :*, :&, :|, :xor, Symbol("+%"), Symbol("-%"), Symbol("*%")) @eval function $op(a::Integer, b::Integer) T = promote_typeof(a, b) aT, bT = a % T, b % T diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 925dafdd8f378..c40acb900bdac 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -143,11 +143,19 @@ function lcm(a::T, b::T) where T<:Integer end end +function _promote_mixed_signs(a::Signed, b::Unsigned) + # handle the case a == typemin(typeof(a)) if R != typeof(a) + R = promote_typeof(a, b) + promote(abs(a % signed(R)), b) +end + gcd(a::Integer) = checked_abs(a) gcd(a::Rational) = checked_abs(a.num) // a.den lcm(a::Union{Integer,Rational}) = gcd(a) -gcd(a::Unsigned, b::Signed) = gcd(promote(a, abs(b))...) -gcd(a::Signed, b::Unsigned) = gcd(promote(abs(a), b)...) +gcd(a::Unsigned, b::Signed) = gcd(b, a) +gcd(a::Signed, b::Unsigned) = gcd(_promote_mixed_signs(a, b)...) +lcm(a::Unsigned, b::Signed) = lcm(promote(a, abs(b))...) +lcm(a::Signed, b::Unsigned) = lcm(_promote_mixed_signs(a, b)...) gcd(a::Real, b::Real) = gcd(promote(a,b)...) lcm(a::Real, b::Real) = lcm(promote(a,b)...) gcd(a::Real, b::Real, c::Real...) = gcd(a, gcd(b, c...)) @@ -178,11 +186,14 @@ function gcd(abc::AbstractArray{<:Integer}) return a end +_gcdx_update(s0::T, q::T, s1::T) where {T<:Unsigned} = s0 -% q *% s1 +_gcdx_update(s0, q, s1) = s0 - q*s1 + # return (gcd(a, b), x, y) such that ax+by == gcd(a, b) """ gcdx(a, b...) -Computes the greatest common (positive) divisor of `a` and `b` and their Bézout +Compute the greatest common (positive) divisor of `a` and `b` and their Bézout coefficients, i.e. the integer coefficients `u` and `v` that satisfy ``u*a + v*b = d = gcd(a, b)``. ``gcdx(a, b)`` returns ``(d, u, v)``. @@ -221,22 +232,31 @@ julia> gcdx(15, 12, 20) their `typemax`, and the identity then holds only via the unsigned integers' modulo arithmetic. """ -Base.@assume_effects :terminates_locally function gcdx(a::Integer, b::Integer) - T = promote_type(typeof(a), typeof(b)) - a == b == 0 && return (zero(T), zero(T), zero(T)) +Base.@assume_effects :terminates_locally function gcdx(a::T, b::T) where {T<:Integer} + if iszero(a) && iszero(b) + return (zero(T), zero(T), zero(T)) + elseif isone(abs(b)) + # handles (typemin(::Signed), -1) + return (one(T), zero(T), b) + elseif isone(abs(a)) + return (one(T), a, zero(T)) + end # a0, b0 = a, b s0, s1 = oneunit(T), zero(T) t0, t1 = s1, s0 # The loop invariant is: s0*a0 + t0*b0 == a && s1*a0 + t1*b0 == b - x = a % T - y = b % T - while y != 0 - q, r = divrem(x, y) - x, y = y, r - s0, s1 = s1, s0 - q*s1 - t0, t1 = t1, t0 - q*t1 + while !iszero(b) + q, r = divrem(a, b) + a, b = b, r + s0, s1 = s1, _gcdx_update(s0, q, s1) + t0, t1 = t1, _gcdx_update(t0, q, t1) + end + # for cases like abs(Int8(-128)) + if isnegative(a) && isnegative(abs(a)) + throw(DomainError((a, b), LazyString("gcd not representable in ", T))) + else + return isnegative(a) ? (abs(a), -s0, -t0) : (a, s0, t0) end - x < 0 ? (-x, -s0, -t0) : (x, s0, t0) end gcdx(a::Real, b::Real) = gcdx(promote(a,b)...) gcdx(a::T, b::T) where T<:Real = throw(MethodError(gcdx, (a,b))) @@ -253,6 +273,17 @@ function gcdx(a::Real, b::Real, cs::Real...) return d′, i*x, j*x, ys... end +function gcdx(a::Signed, b::Unsigned) + R = promote_typeof(a, b) + d, u, v = gcdx(promote(abs(a % signed(R)), b)...) + flip_typemin = isnegative(a) & (R <: Signed) + d, flipsign(u, a -% oftype(a, flip_typemin)), v +end +function gcdx(a::Unsigned, b::Signed) + d, v, u = gcdx(b, a) + d, u, v +end + # multiplicative inverse of n mod m, error if none """ @@ -275,24 +306,38 @@ julia> invmod(5, 6) ``` """ function invmod(n::Integer, m::Integer) + # The postcondition is: mod(widemul(result, n), m) == mod(one(T), m) && iszero(div(result, m)) iszero(m) && throw(DomainError(m, "`m` must not be 0.")) - if n isa Signed && hastypemax(typeof(n)) - # work around inconsistencies in gcdx - # https://github.com/JuliaLang/julia/issues/33781 - T = promote_type(typeof(n), typeof(m)) - n == typemin(typeof(n)) && m == typeof(n)(-1) && return T(0) - n == typeof(n)(-1) && m == typemin(typeof(n)) && return T(-1) + R = promote_typeof(n, m) + if R <: Signed + x = _bezout_coef(n, m) + return mod(x, m) + else + S = signed(R) + if !hastypemax(S) || (n <= typemax(S)) && (m <= typemax(S)) + x = _bezout_coef(n % S, m % S) + + # this branch is only hit if R <: Unsigned, so we don't have + # to worry about abs(typemin(::Signed)) overflow. If `m` is + # signed then `x` must be unsigned, and thus never negative + isnegative(x) && (x += abs(m)) + return mod(x % R, m) + else + # since gcdx only promises bezout w.r.t overflow for unsigned ints, + # we have to widen to a signed type + W = widen(S) + x = _bezout_coef(n % W, m % W) + t = mod(x, m % W) + isnegative(m) && (t -= m) + return mod(t % R, m) + end end - g, x, y = gcdx(n, m) +end + +function _bezout_coef(n, m) + g, x, _ = gcdx(n, m) g != 1 && throw(DomainError((n, m), LazyString("Greatest common divisor is ", g, "."))) - # Note that m might be negative here. - if x isa Unsigned && hastypemax(typeof(x)) && x > typemax(x)>>1 - # x might have wrapped if it would have been negative - # adding back m forces a correction - x += m - end - # The postcondition is: mod(result * n, m) == mod(T(1), m) && div(result, m) == 0 - return mod(x, m) + return x end """ @@ -316,7 +361,8 @@ inconvenient since the modulus is by definition too big to be represented by the type. The modular inverse is computed much more efficiently than the general case -using the algorithm described in https://arxiv.org/pdf/2204.04342.pdf. +using the algorithm described in [*An Improved Integer Modular Multiplicative +Inverse (modulo ``2^w``)* by Jeffrey Hurchalla](https://arxiv.org/abs/2204.04342). !!! compat "Julia 1.11" The `invmod(n)` and `invmod(n, T)` methods require Julia 1.11 or later. @@ -325,11 +371,11 @@ invmod(n::Integer, ::Type{T}) where {T<:BitInteger} = invmod(n % T) function invmod(n::T) where {T<:BitInteger} isodd(n) || throw(DomainError(n, "Argument must be odd.")) - x = (3*n ⊻ 2) % T - y = (1 - n*x) % T + x = (3 *% n ⊻ 2) % T + y = (one(T) -% n *% x) % T for _ = 1:trailing_zeros(2*sizeof(T)) - x *= y + true - y *= y + x = x *% (y +% one(T)) + y = y *% y end return x end @@ -477,34 +523,63 @@ julia> powermod(5, 3, 19) function powermod(x::Integer, p::Integer, m::T) where T<:Integer p == 0 && return mod(one(m),m) # When the concrete type of p is signed and has the lowest value, - # `p != 0 && p == -p` is equivalent to `p == typemin(typeof(p))` for 2's complement representation. + # `p < 0 && p == -p` is equivalent to `p == typemin(typeof(p))` for 2's complement representation. # but will work for integer types like `BigInt` that don't have `typemin` defined # It needs special handling otherwise will cause overflow problem. - if p == -p - imod = invmod(x, m) - rhalf = powermod(imod, -(p÷2), m) - r::T = mod(widemul(rhalf, rhalf), m) - isodd(p) && (r = mod(widemul(r, imod), m)) - #else odd - return r - elseif p < 0 - return powermod(invmod(x, m), -p, m) + if p < 0 + if p == -p + imod = invmod(x, m) + rhalf = powermod(imod, -(p÷2), m) + r::T = mod(widemul(rhalf, rhalf), m) + isodd(p) && (r = mod(widemul(r, imod), m)) + #else odd + return r + else + return powermod(invmod(x, m), -p, m) + end end (m == 1 || m == -1) && return zero(m) - b = oftype(m,mod(x,m)) # this also checks for divide by zero - - t = prevpow(2, p) - r = 1 - while true - if p >= t - r = mod(widemul(r,b),m) - p -= t + + mm = uabs(m) + rr = one(mm) + bb = oftype(mm, mod(x, mm)) + + # legal && profitable + if _powermod_mi_legal(mm) && (p > 2sizeof(mm)) + if bb == 0 + rr = zero(mm) + else + mis = MultiplicativeInverses.multiplicativeinverse(mm) + Base.@assume_effects :terminates_locally while true + if (p & 1) != 0 + rr = mod(rr * bb, mis) + end + p >>= 1 + p == 0 && break + bb = mod(bb * bb, mis) + end + end + else + if bb == 0 + rr = zero(mm) + else + Base.@assume_effects :terminates_locally while true + if (p & 1) != 0 + rr = oftype(mm, mod(widemul(rr, bb), mm)) + end + p >>= 1 + p == 0 && break + bb = oftype(mm, mod(widemul(bb, bb), mm)) + end end - t >>>= 1 - t <= 0 && break - r = mod(widemul(r,r),m) end - return r + r = oftype(m, rr) + return (iszero(r) || (m > 0)) ? r : r + m +end + +_powermod_mi_legal(::Integer) = false +function _powermod_mi_legal(mm::T) where {T<:Unsigned} + return Base.hastypemax(T) && (mm <= (typemax(T) >> (sizeof(T) << 2))) end # optimization: promote the modulus m to BigInt only once (cf. widemul in generic powermod above) @@ -617,7 +692,7 @@ julia> prevpow(4, 16) """ function prevpow(a::T, x::Real) where T <: Real x < 1 && throw(DomainError(x, "`x` must be ≥ 1.")) - # See comment in nextpos() for a == special case. + # See comment in nextpow() for a == special case. a == 2 && isa(x, Integer) && return _prevpow2(x) a <= 1 && throw(DomainError(a, "`a` must be greater than 1.")) n = floor(Integer,log(a, x)) @@ -649,7 +724,7 @@ const powers_of_ten = [ function bit_ndigits0z(x::Base.BitUnsigned64) lz = top_set_bit(x) nd = (1233*lz)>>12+1 - nd -= x < powers_of_ten[nd] + return nd - (x < powers_of_ten[nd]) end function bit_ndigits0z(x::UInt128) n = 0 @@ -686,6 +761,7 @@ ndigits0znb(x::Bool, b::Integer) = x % Int # The suffix "pb" stands for "positive base" function ndigits0zpb(x::Integer, b::Integer) + @constprop :aggressive # precondition: b > 1 x == 0 && return 0 b = Int(b) @@ -750,6 +826,7 @@ julia> Base.ndigits0z(10) See also [`ndigits`](@ref). """ function ndigits0z(x::Integer, b::Integer) + @constprop :aggressive if b < -1 ndigits0znb(x, b) elseif b > 1 @@ -760,7 +837,8 @@ function ndigits0z(x::Integer, b::Integer) end # Extends the definition in base/int.jl -top_set_bit(x::Integer) = ceil(Integer, log2(x + oneunit(x))) +# assume x >= 0. result is implementation-defined for negative values +top_set_bit(x::Integer) = iszero(x) ? 0 : exponent(x) + 1 """ ndigits(n::Integer; base::Integer=10, pad::Integer=1) @@ -799,42 +877,45 @@ ndigits(x::Integer; base::Integer=10, pad::Integer=1) = max(pad, ndigits0z(x, ba function bin(x::Unsigned, pad::Int, neg::Bool) m = top_set_bit(x) n = neg + max(pad, m) - a = StringMemory(n) - # for i in 0x0:UInt(n-1) # automatic vectorization produces redundant codes - # @inbounds a[n - i] = 0x30 + (((x >> i) % UInt8)::UInt8 & 0x1) - # end - i = n - @inbounds while i >= 4 - b = UInt32((x % UInt8)::UInt8) - d = 0x30303030 + ((b * 0x08040201) >> 0x3) & 0x01010101 - a[i-3] = (d >> 0x00) % UInt8 - a[i-2] = (d >> 0x08) % UInt8 - a[i-1] = (d >> 0x10) % UInt8 - a[i] = (d >> 0x18) % UInt8 - x >>= 0x4 - i -= 4 - end - while i > neg - @inbounds a[i] = 0x30 + ((x % UInt8)::UInt8 & 0x1) - x >>= 0x1 - i -= 1 + str = _string_n(n) + GC.@preserve str begin + p = pointer(str) + i = n + while i >= 4 + b = UInt32((x % UInt8)::UInt8) + d = 0x30303030 +% ((b *% 0x08040201) >> 0x3) & 0x01010101 + unsafe_store!(p, (d >> 0x00) % UInt8, i-3) + unsafe_store!(p, (d >> 0x08) % UInt8, i-2) + unsafe_store!(p, (d >> 0x10) % UInt8, i-1) + unsafe_store!(p, (d >> 0x18) % UInt8, i) + x >>= 0x4 + i -= 4 + end + while i > neg + unsafe_store!(p, 0x30 + ((x % UInt8)::UInt8 & 0x1), i) + x >>= 0x1 + i -= 1 + end + neg && unsafe_store!(p, 0x2d, 1) # UInt8('-') end - neg && (@inbounds a[1] = 0x2d) # UInt8('-') - unsafe_takestring(a) + return str end function oct(x::Unsigned, pad::Int, neg::Bool) m = div(top_set_bit(x) + 2, 3) n = neg + max(pad, m) - a = StringMemory(n) - i = n - while i > neg - @inbounds a[i] = 0x30 + ((x % UInt8)::UInt8 & 0x7) - x >>= 0x3 - i -= 1 + str = _string_n(n) + GC.@preserve str begin + p = pointer(str) + i = n + while i > neg + unsafe_store!(p, 0x30 + ((x % UInt8)::UInt8 & 0x7), i) + x >>= 0x3 + i -= 1 + end + neg && unsafe_store!(p, 0x2d, 1) # UInt8('-') end - neg && (@inbounds a[1] = 0x2d) # UInt8('-') - unsafe_takestring(a) + return str end # 2-digit decimal characters ("00":"99") @@ -865,7 +946,6 @@ function append_c_digits(olength::Int, digits::Unsigned, buf, pos::Int) end if i == 1 @inbounds buf[pos] = UInt8('0') + rem(digits, 0xa) % UInt8 - i -= 1 end return pos + olength end @@ -901,31 +981,63 @@ end function dec(x::Unsigned, pad::Int, neg::Bool) n = neg + ndigits(x, pad=pad) - a = StringMemory(n) - append_c_digits_fast(n, x, a, 1) - neg && (@inbounds a[1] = 0x2d) # UInt8('-') - unsafe_takestring(a) + str = _string_n(n) + GC.@preserve str begin + p = pointer(str) + i = n + while i > 9 && x > typemax(UInt) + d, r = divrem(x, 0x3b9aca00) # 10^9 + x = oftype(x, d) + r32 = r % UInt32 + for j in 0:3 + q, s = divrem(r32, 0x64) + r32 = q + v = @inbounds _dec_d100[1 + (s % Int)] + unsafe_store!(p, (v >> 8) % UInt8, i - 2*j) + unsafe_store!(p, v % UInt8, i - 2*j - 1) + end + unsafe_store!(p, 0x30 + (r32 % UInt8), i - 8) + i -= 9 + end + y = x % UInt + while i >= 2 + d, r = divrem(y, 0x64) + y = d + v = @inbounds _dec_d100[1 + (r % Int)] + unsafe_store!(p, (v >> 8) % UInt8, i) + unsafe_store!(p, v % UInt8, i - 1) + i -= 2 + end + if i > neg + unsafe_store!(p, 0x30 + (rem(y, 0xa) % UInt8), i) + end + neg && unsafe_store!(p, 0x2d, 1) # '-' + end + return str end function hex(x::Unsigned, pad::Int, neg::Bool) m = 2 * sizeof(x) - (leading_zeros(x) >> 2) n = neg + max(pad, m) - a = StringMemory(n) - i = n - while i >= 2 - b = (x % UInt8)::UInt8 - d1, d2 = b >> 0x4, b & 0xf - @inbounds a[i-1] = d1 + ifelse(d1 > 0x9, 0x57, 0x30) - @inbounds a[i] = d2 + ifelse(d2 > 0x9, 0x57, 0x30) - x >>= 0x8 - i -= 2 - end - if i > neg - d = (x % UInt8)::UInt8 & 0xf - @inbounds a[i] = d + ifelse(d > 0x9, 0x57, 0x30) + str = _string_n(n) + GC.@preserve str begin + p = pointer(str) + i = n + while i >= 2 + b = (x % UInt8)::UInt8 + d1, d2 = b >> 0x4, b & 0xf + unsafe_store!(p, d1 + ifelse(d1 > 0x9, 0x57, 0x30), i-1) + unsafe_store!(p, d2 + ifelse(d2 > 0x9, 0x57, 0x30), i) + x >>= 0x8 + i -= 2 + end + if i > neg + d = (x % UInt8)::UInt8 & 0xf + unsafe_store!(p, d + ifelse(d > 0x9, 0x57, 0x30), i) + end + neg && unsafe_store!(p, 0x2d, 1) # UInt8('-') end - neg && (@inbounds a[1] = 0x2d) # UInt8('-') - unsafe_takestring(a) + return str end const base36digits = UInt8['0':'9';'a':'z'] @@ -937,24 +1049,27 @@ function _base(base::Integer, x::Integer, pad::Int, neg::Bool) b = (base % Int)::Int digits = abs(b) <= 36 ? base36digits : base62digits n = neg + ndigits(x, base=b, pad=pad) - a = StringMemory(n) - i = n - @inbounds while i > neg - if b > 0 - a[i] = digits[1 + (rem(x, b) % Int)::Int] - x = div(x,b) - else - a[i] = digits[1 + (mod(x, -b) % Int)::Int] - x = cld(x,b) + str = _string_n(n) + GC.@preserve str begin + p = pointer(str) + i = n + while i > neg + if b > 0 + unsafe_store!(p, @inbounds(digits[1 + (rem(x, b) % Int)::Int]), i) + x = div(x,b) + else + unsafe_store!(p, @inbounds(digits[1 + (mod(x, -b) % Int)::Int]), i) + x = cld(x,b) + end + i -= 1 end - i -= 1 + neg && unsafe_store!(p, 0x2d, 1) # UInt8('-') end - neg && (@inbounds a[1] = 0x2d) # UInt8('-') - unsafe_takestring(a) + return str end -split_sign(n::Integer) = unsigned(abs(n)), n < 0 -split_sign(n::Unsigned) = n, false +split_sign(n::Integer) = (unsigned(abs(n)), n < 0) +split_sign(n::Unsigned) = (n, false) """ string(n::Integer; base::Integer = 10, pad::Integer = 1) @@ -962,7 +1077,8 @@ split_sign(n::Unsigned) = n, false Convert an integer `n` to a string in the given `base`, optionally specifying a number of digits to pad to. -See also [`digits`](@ref), [`bitstring`](@ref), [`count_zeros`](@ref). +See also [`digits`](@ref), [`bitstring`](@ref), [`count_zeros`](@ref), +and the Printf standard library. # Examples ```jldoctest @@ -971,9 +1087,18 @@ julia> string(5, base = 13, pad = 4) julia> string(-13, base = 5, pad = 4) "-0023" + +julia> using Printf + +julia> @sprintf("%04i", 5) +"0005" + +julia> @sprintf("%4i", 5) +" 5" ``` """ function string(n::Integer; base::Integer = 10, pad::Integer = 1) + @constprop :aggressive pad = (min(max(pad, typemin(Int)), typemax(Int)) % Int)::Int if base == 2 (n_positive, neg) = split_sign(n) @@ -1013,20 +1138,31 @@ julia> bitstring(2.2) """ function bitstring(x::T) where {T} isprimitivetype(T) || throw(ArgumentError(LazyString(T, " not a primitive type"))) - sz = sizeof(T) * 8 - str = StringMemory(sz) - i = sz - @inbounds while i >= 4 - b = UInt32(sizeof(T) == 1 ? bitcast(UInt8, x) : trunc_int(UInt8, x)) - d = 0x30303030 + ((b * 0x08040201) >> 0x3) & 0x01010101 - str[i-3] = (d >> 0x00) % UInt8 - str[i-2] = (d >> 0x08) % UInt8 - str[i-1] = (d >> 0x10) % UInt8 - str[i] = (d >> 0x18) % UInt8 - x = lshr_int(x, 4) - i -= 4 + sz = Core.bitsizeof(T) + onebyte = sz == 8 + subbyte = sz < 8 + str = _string_n(sz) + GC.@preserve str begin + p = pointer(str) + i = sz + while i >= 4 + b = UInt32(onebyte ? bitcast(UInt8, x) : subbyte ? zext_int(UInt8, x) : trunc_int(UInt8, x)) + d = 0x30303030 +% ((b *% 0x08040201) >> 0x3) & 0x01010101 + unsafe_store!(p, (d >> 0x00) % UInt8, i-3) + unsafe_store!(p, (d >> 0x08) % UInt8, i-2) + unsafe_store!(p, (d >> 0x10) % UInt8, i-1) + unsafe_store!(p, (d >> 0x18) % UInt8, i) + x = lshr_int(x, 4) + i -= 4 + end + while i > 0 + b = UInt8(onebyte ? bitcast(UInt8, x) : subbyte ? zext_int(UInt8, x) : trunc_int(UInt8, x)) + unsafe_store!(p, 0x30 + (b & 0x01), i) + x = lshr_int(x, 1) + i -= 1 + end end - return unsafe_takestring(str) + return str end """ diff --git a/base/io.jl b/base/io.jl index 71e4c404dfeac..5b917cb1381bf 100644 --- a/base/io.jl +++ b/base/io.jl @@ -2,6 +2,13 @@ # Generic IO stubs -- all subtypes should implement these (if meaningful) +""" + IO + +Abstract supertype for input/output types. +""" +IO + """ EOFError() @@ -24,7 +31,9 @@ struct SystemError <: Exception end lock(::IO) = nothing +typeof(lock).name.max_methods = UInt8(1) unlock(::IO) = nothing +typeof(unlock).name.max_methods = UInt8(1) """ reseteof(io) @@ -59,6 +68,7 @@ false ``` """ function isopen end +typeof(isopen).name.max_methods = UInt8(1) """ close(io::IO) @@ -76,6 +86,7 @@ closed IO does not cause undefined behaviour. See also: [`isopen`](@ref) """ function close end +typeof(close).name.max_methods = UInt8(1) """ closewrite(stream) @@ -103,6 +114,7 @@ julia> read(io, String) ``` """ function closewrite end +typeof(closewrite).name.max_methods = UInt8(1) """ flush(io::IO) @@ -112,6 +124,7 @@ This has a default implementation `flush(::IO) = nothing`, so may be called in generic IO code. """ function flush end +typeof(flush).name.max_methods = UInt8(1) """ bytesavailable(io) @@ -127,6 +140,7 @@ julia> bytesavailable(io) ``` """ function bytesavailable end +typeof(bytesavailable).name.max_methods = UInt8(1) """ readavailable(stream) @@ -140,8 +154,10 @@ data has already been buffered. The result is a `Vector{UInt8}`. should generally be used instead. """ function readavailable end +typeof(readavailable).name.max_methods = UInt8(1) function isexecutable end +typeof(isexecutable).name.max_methods = UInt8(1) """ isreadable(io)::Bool @@ -212,6 +228,7 @@ true ``` """ function eof end +typeof(eof).name.max_methods = UInt8(1) function copy end function wait_readnb end @@ -312,6 +329,20 @@ Base.RefValue{MyStruct}(MyStruct(42.0)) """ function write end +""" + writepartial(io::IO, x) -> Int + +Write `x` to `io` with partial-write cancellation semantics: where +[`write`](@ref) throws the `CancellationRequest` when its governing +cancellation token is cancelled mid-write, `writepartial` returns the +number of bytes the stream had already accepted, and the (level-triggered) +cancellation is delivered at the next cancellation point instead. Callers +using it must be prepared for short counts. For IO types whose writes +cannot block on cancellable resources it is equivalent to `write`. +""" +writepartial(io::IO, x) = write(io, x) +typeof(write).name.max_methods = UInt8(1) + read(s::IO, ::Type{UInt8}) = error(typeof(s)," does not support byte I/O") write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") @@ -414,7 +445,7 @@ julia> open(io->read(io, String), "myfile.txt") julia> rm("myfile.txt") ``` """ -function open(f::Function, args...; kwargs...) +@inline function open(f::Function, args...; kwargs...) io = open(args...; kwargs...) try f(io) @@ -454,32 +485,56 @@ end function pipe_reader end function pipe_writer end -for f in (:flush, :closewrite, :iswritable) - @eval $(f)(io::AbstractPipe) = $(f)(pipe_writer(io)::IO) +iswritable(io::AbstractPipe) = iswritable(pipe_writer(io)::IO) +# flush/closewrite/unsafe_write accept `cancel` like their LibuvStream +# counterparts, so a keyword call on a compound pipe does not miss these +# forwarders (only an explicit token is forwarded: the sentinel keeps the +# plain call, which any user-defined method of the inner IO supports; see +# readbytes! below) +for f in (:flush, :closewrite) + @eval $(f)(io::AbstractPipe; cancel::CancelTokenArg=DEFAULT_CANCEL) = + cancel === DEFAULT_CANCEL ? $(f)(pipe_writer(io)::IO) : + $(f)(pipe_writer(io)::IO; cancel) end write(io::AbstractPipe, byte::UInt8) = write(pipe_writer(io)::IO, byte) +writepartial(io::AbstractPipe, x) = writepartial(pipe_writer(io)::IO, x) write(to::IO, from::AbstractPipe) = write(to, pipe_reader(from)) -unsafe_write(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_write(pipe_writer(io)::IO, p, nb)::Union{Int,UInt} +unsafe_write(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) = + (cancel === DEFAULT_CANCEL ? unsafe_write(pipe_writer(io)::IO, p, nb) : + unsafe_write(pipe_writer(io)::IO, p, nb; cancel))::Union{Int,UInt} buffer_writes(io::AbstractPipe, args...) = buffer_writes(pipe_writer(io)::IO, args...) for f in ( # peek/mark interface :mark, :unmark, :reset, :ismarked, # Simple reader functions - :read, :readavailable, :bytesavailable, :reseteof, :isreadable) + :read, :bytesavailable, :reseteof, :isreadable) @eval $(f)(io::AbstractPipe) = $(f)(pipe_reader(io)::IO) end -read(io::AbstractPipe, byte::Type{UInt8}) = read(pipe_reader(io)::IO, byte)::UInt8 +# explicit-token-forwarding reader forwarders (same pattern as readbytes! +# below: the sentinel keeps the plain call) +readavailable(io::AbstractPipe; cancel::CancelTokenArg=DEFAULT_CANCEL) = + cancel === DEFAULT_CANCEL ? readavailable(pipe_reader(io)::IO) : + readavailable(pipe_reader(io)::IO; cancel) +read(io::AbstractPipe, byte::Type{UInt8}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + (cancel === DEFAULT_CANCEL ? read(pipe_reader(io)::IO, byte) : + read(pipe_reader(io)::IO, byte; cancel))::UInt8 unsafe_read(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_read(pipe_reader(io)::IO, p, nb) copyuntil(out::IO, io::AbstractPipe, arg::UInt8; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) copyuntil(out::IO, io::AbstractPipe, arg::AbstractChar; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) copyuntil(out::IO, io::AbstractPipe, arg::AbstractString; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) copyuntil(out::IO, io::AbstractPipe, arg::AbstractVector; kw...) = copyuntil(out, pipe_reader(io)::IO, arg; kw...) readuntil_vector!(io::AbstractPipe, target::AbstractVector, keep::Bool, out) = readuntil_vector!(pipe_reader(io)::IO, target, keep, out) -readbytes!(io::AbstractPipe, target::AbstractVector{UInt8}, n=length(target)) = readbytes!(pipe_reader(io)::IO, target, n) +# (only an explicit token is forwarded: the sentinel keeps the plain call, +# which any user-defined readbytes! method supports) +readbytes!(io::AbstractPipe, target::AbstractVector{UInt8}, n=length(target); cancel::CancelTokenArg=DEFAULT_CANCEL) = + cancel === DEFAULT_CANCEL ? readbytes!(pipe_reader(io)::IO, target, n) : + readbytes!(pipe_reader(io)::IO, target, n; cancel) peek(io::AbstractPipe, ::Type{T}) where {T} = peek(pipe_reader(io)::IO, T)::T wait_readnb(io::AbstractPipe, nb::Int) = wait_readnb(pipe_reader(io)::IO, nb) -eof(io::AbstractPipe) = eof(pipe_reader(io)::IO)::Bool +eof(io::AbstractPipe; cancel::CancelTokenArg=DEFAULT_CANCEL) = + (cancel === DEFAULT_CANCEL ? eof(pipe_reader(io)::IO) : + eof(pipe_reader(io)::IO; cancel))::Bool isopen(io::AbstractPipe) = isopen(pipe_writer(io)::IO) || isopen(pipe_reader(io)::IO) close(io::AbstractPipe) = (close(pipe_writer(io)::IO); close(pipe_reader(io)::IO)) @@ -523,9 +578,17 @@ read(filename::AbstractString, ::Type{T}) where {T} = open(io->read(io, T), conv Read binary data from an I/O stream or file, filling in `array`. """ function read! end +typeof(read!).name.max_methods = UInt8(1) read!(filename::AbstractString, a) = open(io->read!(io, a), convert(String, filename)::String) +# The generic-IO `cancel` convention (referenced as such below): in methods +# over abstract `IO`, the inner reads/writes go through arbitrary, possibly +# user-extended methods that need not accept a `cancel` keyword. The +# resolved token therefore gates *between* those calls, via explicit +# cancellation points, while any parks inside them run under the ambient +# scope. + """ readuntil(stream::IO, delim; keep::Bool = false) readuntil(filename::AbstractString, delim; keep::Bool = false) @@ -553,9 +616,12 @@ julia> rm("my_file.txt") ``` """ readuntil(filename::AbstractString, delim; kw...) = open(io->readuntil(io, delim; kw...), convert(String, filename)::String) -readuntil(stream::IO, delim::UInt8; kw...) = _unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)) -readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = takestring!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)) -readuntil(stream::IO, delim::T; keep::Bool=false) where T = _copyuntil(Vector{T}(), stream, delim, keep) +readuntil(stream::IO, delim::UInt8; cancel::CancelTokenArg=DEFAULT_CANCEL, kw...) = + _unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; cancel, kw...)) +readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; cancel::CancelTokenArg=DEFAULT_CANCEL, kw...) = + takestring!(copyuntil(IOBuffer(sizehint=16), stream, delim; cancel, kw...)) +readuntil(stream::IO, delim::T; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) where T = + _copyuntil(Vector{T}(), stream, delim, keep, resolve_cancel_token(cancel)) """ @@ -584,6 +650,9 @@ julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true)) julia> rm("my_file.txt") ``` + +!!! compat "Julia 1.11" + `copyuntil` was introduced in Julia 1.11. """ copyuntil(out::IO, filename::AbstractString, delim; kw...) = open(io->copyuntil(out, io, delim; kw...), convert(String, filename)::String) @@ -624,9 +693,12 @@ Logan "Logan" ``` """ -readline(filename::AbstractString; keep::Bool=false) = - open(io -> readline(io; keep), filename) -readline(s::IO=stdin; keep::Bool=false) = takestring!(copyline(IOBuffer(sizehint=16), s; keep)) +function readline(filename::AbstractString; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + return open(io -> readline(io; keep, cancel=tok), filename) +end +readline(s::IO=stdin; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) = + takestring!(copyline(IOBuffer(sizehint=16), s; keep, cancel)) """ copyline(out::IO, io::IO=stdin; keep::Bool=false) @@ -659,17 +731,27 @@ julia> takestring!(copyline(IOBuffer(), "my_file.txt", keep=true)) julia> rm("my_file.txt") ``` + +!!! compat "Julia 1.11" + `copyline` was introduced in Julia 1.11. """ -copyline(out::IO, filename::AbstractString; keep::Bool=false) = - open(io -> copyline(out, io; keep), filename) +function copyline(out::IO, filename::AbstractString; keep::Bool=false, + cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + return open(io -> copyline(out, io; keep, cancel=tok), filename) +end # fallback to optimized methods for IOBuffer in iobuffer.jl -function copyline(out::IO, s::IO; keep::Bool=false) +function copyline(out::IO, s::IO; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) if keep - return copyuntil(out, s, 0x0a, keep=true) + return copyuntil(out, s, 0x0a; keep=true, cancel) else + tok = resolve_cancel_token(cancel) + @cancel_check tok # more complicated to deal with CRLF logic while !eof(s) + # token-gate between the reads (generic-IO cancel convention) + @cancel_check tok b = read(s, UInt8) b == 0x0a && break if b == 0x0d && !eof(s) @@ -714,7 +796,8 @@ function readlines(filename::AbstractString; kw...) readlines(f; kw...) end end -readlines(s=stdin; kw...) = collect(eachline(s; kw...)) +readlines(s=stdin; cancel::CancelTokenArg=DEFAULT_CANCEL, kw...) = + collect(eachline(s; cancel, kw...)) ## byte-order mark, ntoh & hton ## @@ -799,9 +882,13 @@ isreadonly(s) = isreadable(s) && !iswritable(s) ## binary I/O ## write(io::IO, x) = throw(MethodError(write, (io, x))) -function write(io::IO, x1, xs...) +function write(io::IO, x1, xs...; cancel::CancelTokenArg=DEFAULT_CANCEL) + # check for cancellations between each write + tok = resolve_cancel_token(cancel) + @cancel_check tok written::Int = write(io, x1) for x in xs + @cancel_check tok written += write(io, x) end return written @@ -826,7 +913,9 @@ end write(s::IO, x::Bool) = write(s, UInt8(x)) write(to::IO, p::Ptr) = write(to, convert(UInt, p)) -function write(s::IO, A::AbstractArray) +function write(s::IO, A::AbstractArray; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok if !isbitstype(eltype(A)) error("`write` is not supported on non-isbits arrays") end @@ -839,7 +928,9 @@ function write(s::IO, A::AbstractArray) return nb end -function write(s::IO, A::StridedArray) +function write(s::IO, A::StridedArray; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok if !isbitstype(eltype(A)) error("`write` is not supported on non-isbits arrays") end @@ -917,7 +1008,9 @@ end read(s::IO, ::Type{Bool}) = (read(s, UInt8) != 0) read(s::IO, ::Type{Ptr{T}}) where {T} = convert(Ptr{T}, read(s, UInt)) -function read!(s::IO, A::AbstractArray{T}) where {T} +function read!(s::IO, A::AbstractArray{T}; cancel::CancelTokenArg=DEFAULT_CANCEL) where {T} + tok = resolve_cancel_token(cancel) + @cancel_check tok if isbitstype(T) && _checkcontiguous(Bool, A) GC.@preserve A unsafe_read(s, pointer(A), elsize(A) * length(A)) else @@ -936,7 +1029,29 @@ function read!(s::IO, A::AbstractArray{T}) where {T} return A end -function read!(s::IO, A::StridedArray{T}) where {T} +# bitarray.jl loads before cancellation machinery in bootstrap order, so +# write the specialization here. +# TODO: because the generic `write` design relies on a bug (#9498), we have to +# explicitly add a `cancel` kwarg to avoid hitting the ::AbstractArray method +function write(s::IO, B::BitArray; cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) + return write(s, B.chunks) +end +function read!(s::IO, B::BitArray; cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) + n = length(B) + Bc = B.chunks + read!(s, Bc) + if length(Bc) > 0 && Bc[end] & _msk_end(n) ≠ Bc[end] + Bc[end] &= _msk_end(n) # ensure that the BitArray is not broken + throw(DimensionMismatch("read mismatch, found non-zero bits after BitArray length")) + end + return B +end + +function read!(s::IO, A::StridedArray{T}; cancel::CancelTokenArg=DEFAULT_CANCEL) where {T} + tok = resolve_cancel_token(cancel) + @cancel_check tok if !isbitstype(T) || _checkcontiguous(Bool, A) return invoke(read!, Tuple{IO, AbstractArray}, s, A) end @@ -969,9 +1084,10 @@ end function read(io::IO, ::Type{Char}) b0 = read(io, UInt8)::UInt8 - l = 0x08 * (0x04 - UInt8(leading_ones(b0))) + lo = UInt8(leading_ones(b0)) c = UInt32(b0) << 24 - if l ≤ 0x10 + if 0x02 ≤ lo ≤ 0x04 + l = 0x08 * (0x04 - lo) s = 16 while s ≥ l && !eof(io)::Bool peek(io) & 0xc0 == 0x80 || break @@ -985,11 +1101,15 @@ end # read(io, T) is not defined for other AbstractChar: implementations # must provide their own encoding-specific method. -function copyuntil(out::IO, s::IO, delim::AbstractChar; keep::Bool=false) +function copyuntil(out::IO, s::IO, delim::AbstractChar; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) if delim ≤ '\x7f' - return copyuntil(out, s, delim % UInt8; keep) + return copyuntil(out, s, delim % UInt8; keep, cancel) end + tok = resolve_cancel_token(cancel) + @cancel_check tok for c in readeach(s, Char) + # token-gate between the reads (generic-IO cancel convention) + @cancel_check tok if c == delim keep && write(out, c) break @@ -1001,12 +1121,15 @@ end # note: optimized methods of copyuntil for IOStreams and delim::UInt8 in iostream.jl # and for IOBuffer with delim::UInt8 in iobuffer.jl -copyuntil(out::IO, s::IO, delim; keep::Bool=false) = _copyuntil(out, s, delim, keep) +copyuntil(out::IO, s::IO, delim; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) = + _copyuntil(out, s, delim, keep, resolve_cancel_token(cancel)) # supports out::Union{IO, AbstractVector} for use with both copyuntil & readuntil -function _copyuntil(out, s::IO, delim::T, keep::Bool) where T +function _copyuntil(out, s::IO, delim::T, keep::Bool, tok::MaybeToken=nothing) where T output! = isa(out, IO) ? write : push! for c in readeach(s, T) + # token-gate between the reads (generic-IO cancel convention) + @cancel_check tok if c == delim keep && output!(out, c) break @@ -1102,29 +1225,40 @@ function readuntil_vector!(io::IO, target::AbstractVector{T}, keep::Bool, out) w return false end -function copyuntil(out::IO, io::IO, target::AbstractString; keep::Bool=false) +function copyuntil(out::IO, io::IO, target::AbstractString; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) # small-string target optimizations x = Iterators.peel(target) isnothing(x) && return out c, rest = x if isempty(rest) && c <= '\x7f' - return copyuntil(out, io, c % UInt8; keep) + return copyuntil(out, io, c % UInt8; keep, cancel) end # convert String to a utf8-byte-iterator if !(target isa String) && !(target isa SubString{String}) target = String(target) end target = codeunits(target)::AbstractVector - return copyuntil(out, io, target, keep=keep) + return copyuntil(out, io, target; keep, cancel) end -function readuntil(io::IO, target::AbstractVector{T}; keep::Bool=false) where T +# like the vector copyuntil below: without the `cancel` keyword here, a +# keyword call would bypass this method for the scalar-delimiter catch-all +# above (keyword dispatch only sees keyword-accepting methods) +function readuntil(io::IO, target::AbstractVector{T}; keep::Bool=false, + cancel::CancelTokenArg=DEFAULT_CANCEL) where T + # entry gate only (generic-IO cancel convention) + @cancel_check resolve_cancel_token(cancel) out = (T === UInt8 ? resize!(StringVector(16), 0) : Vector{T}()) readuntil_vector!(io, target, keep, out) return out end -copyuntil(out::IO, io::IO, target::AbstractVector; keep::Bool=false) = - (readuntil_vector!(io, target, keep, out); out) +function copyuntil(out::IO, io::IO, target::AbstractVector; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + # entry gate only, ahead of readuntil_vector!'s generic reads + # (generic-IO cancel convention) + @cancel_check resolve_cancel_token(cancel) + readuntil_vector!(io, target, keep, out) + return out +end """ readchomp(x) @@ -1142,7 +1276,12 @@ julia> readchomp("my_file.txt") julia> rm("my_file.txt"); ``` """ -readchomp(x) = chomp(read(x, String)) +function readchomp(x; cancel::CancelTokenArg=DEFAULT_CANCEL) + # `x` may be anything readable (a stream, file name, command): entry + # gate only (generic-IO cancel convention) + @cancel_check resolve_cancel_token(cancel) + return chomp(read(x, String)) +end # read up to nb bytes into nb, returning # bytes read @@ -1153,11 +1292,15 @@ Read at most `nb` bytes from `stream` into `b`, returning the number of bytes re The size of `b` will be increased if needed (i.e. if `nb` is greater than `length(b)` and enough bytes could be read), but it will never be decreased. """ -function readbytes!(s::IO, b::AbstractArray{UInt8}, nb=length(b)) +function readbytes!(s::IO, b::AbstractArray{UInt8}, nb=length(b); cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok require_one_based_indexing(b) olb = lb = length(b) nr = 0 while nr < nb && !eof(s) + # token-gate between the reads (generic-IO cancel convention) + @cancel_check tok a = read(s, UInt8) nr += 1 if nr > lb @@ -1177,15 +1320,25 @@ end Read at most `nb` bytes from `s`, returning a `Vector{UInt8}` of the bytes read. """ -function read(s::IO, nb::Integer = typemax(Int)) +function read(s::IO, nb::Integer = typemax(Int); cancel::CancelTokenArg=DEFAULT_CANCEL) # Let readbytes! grow the array progressively by default - # instead of taking of risk of over-allocating + # instead of taking the risk of over-allocating b = Vector{UInt8}(undef, nb == typemax(Int) ? 1024 : nb) - nr = readbytes!(s, b, nb) + # an explicit token is forwarded to readbytes! (whose Base methods all + # accept it); the default sentinel keeps the plain call, which any + # user-defined readbytes! method supports + nr = cancel === DEFAULT_CANCEL ? readbytes!(s, b, nb) : readbytes!(s, b, nb; cancel) return resize!(b, nr) end -read(s::IO, ::Type{String}) = String(read(s)::Vector{UInt8}) +function read(s::IO, ::Type{String}; cancel::CancelTokenArg=DEFAULT_CANCEL) + # thread the token (or an explicit `nothing` shield) into the inner + # read, which does the actual blocking; the sentinel keeps the plain + # call, which any user-defined `read` method supports + cancel === DEFAULT_CANCEL && return String(read(s)::Vector{UInt8}) + cancel = check_cancel_arg(cancel) + return String(read(s; cancel)::Vector{UInt8}) +end read(s::IO, T::Type) = error("The IO stream does not support reading objects of type $T.") ## high-level iterator interfaces ## @@ -1194,8 +1347,10 @@ struct EachLine{IOT <: IO} stream::IOT ondone::Function keep::Bool - EachLine(stream::IO=stdin; ondone::Function=()->nothing, keep::Bool=false) = - new{typeof(stream)}(stream, ondone, keep) + cancel::MaybeToken + EachLine(stream::IO=stdin; ondone::Function=()->nothing, keep::Bool=false, + cancel::MaybeToken=nothing) = + new{typeof(stream)}(stream, ondone, keep, cancel) end """ @@ -1231,18 +1386,20 @@ julia> rm("my_file.txt"); !!! compat "Julia 1.8" Julia 1.8 is required to use `Iterators.reverse` or `last` with `eachline` iterators. """ -function eachline(stream::IO=stdin; keep::Bool=false) - EachLine(stream, keep=keep)::EachLine +function eachline(stream::IO=stdin; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + EachLine(stream; keep, cancel=resolve_cancel_token(cancel))::EachLine end -function eachline(filename::AbstractString; keep::Bool=false) +function eachline(filename::AbstractString; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) s = open(filename) - EachLine(s, ondone=()->close(s), keep=keep)::EachLine + EachLine(s; ondone=()->close(s), keep, cancel=resolve_cancel_token(cancel))::EachLine end function iterate(itr::EachLine, state=nothing) + # token-gate between lines (generic-IO cancel convention) + @cancel_check itr.cancel eof(itr.stream) && return (itr.ondone(); nothing) - (readline(itr.stream, keep=itr.keep), nothing) + (readline(itr.stream; keep=itr.keep, cancel=itr.cancel), nothing) end eltype(::Type{<:EachLine}) = String @@ -1390,7 +1547,8 @@ readeach(stream::IOT, T::Type) where IOT<:IO = ReadEachIterator{T,IOT}(stream) iterate(itr::ReadEachIterator{T}, state=nothing) where T = eof(itr.stream) ? nothing : (read(itr.stream, T), nothing) -eltype(::Type{ReadEachIterator{T}}) where T = T +eltype(::Type{<:ReadEachIterator{T}}) where {T} = T +eltype(::Type{ReadEachIterator}) = Any IteratorSize(::Type{<:ReadEachIterator}) = SizeUnknown() diff --git a/base/iobuffer.jl b/base/iobuffer.jl index dd3b757c0e3f0..d9d5ad2434cb9 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -59,7 +59,7 @@ mutable struct GenericIOBuffer{T<:AbstractVector{UInt8}} <: IO # When the buffer is resized, or a new buffer allocated, this is the maximum size of the buffer. # A new GenericIOBuffer may be constructed with an existing data larger than `maxsize`. - # When that happensm we must make sure to not have more than `maxsize` bytes in the buffer, + # When that happens, we must make sure to not have more than `maxsize` bytes in the buffer, # else reallocating will lose data. So, never write to indices > `maxsize + get_offset(io)` # This value is always in 0:typemax(Int). maxsize::Int @@ -191,6 +191,7 @@ When `data` is not given, the buffer will be both readable and writable by defau offset leaving behind arbitrary values at other offsets. If `maxsize > length(data)`, the IOBuffer might re-allocate the data entirely, which may or may not be visible in any outstanding bindings to `array`. + # Examples ```jldoctest julia> io = IOBuffer(); @@ -515,7 +516,7 @@ function seek(io::GenericIOBuffer, n::Int) end # TODO: REPL.jl relies on the fact that this does not throw (by seeking past the beginning or end - # of an GenericIOBuffer), so that would need to be fixed in order to throw an error here + # of a GenericIOBuffer), so that would need to be fixed in order to throw an error here max_ptr = io.size + 1 min_ptr = get_offset(io) + 1 io.ptr = clamp(translate_seek_position(io, n), min_ptr, max_ptr) @@ -604,7 +605,8 @@ end end # The fast path here usually checks there is already room, then does nothing. # When append is true, new data is added after io.size, not io.ptr - existing_space = min(lastindex(io.data), io.maxsize + get_offset(io)) - (io.append ? io.size : io.ptr - 1) + start_offset = io.append ? io.size : io.ptr - 1 + existing_space = min(lastindex(io.data) - start_offset, io.maxsize - (start_offset - get_offset(io))) if existing_space < nshort % Int # Outline this function to make it more likely that ensureroom inlines itself return ensureroom_slowpath(io, nshort, existing_space) @@ -641,7 +643,7 @@ end # If we can't fit all the requested data in the new buffer, we need to # fit as much as possible, so we must compact if !iszero(reclaimable_bytes) - desired_size -= compact!(io) + compact!(io) end # Max out the buffer size if we want more than the buffer size if length(io.data) < io.maxsize @@ -729,7 +731,7 @@ julia> String(take!(io)) function take!(io::GenericIOBuffer) io.mark = -1 if io.seekable - # If the buffer is seekable, then the previously consumed bytes from ptr+1:size + # If the buffer is seekable, then the previously consumed bytes from 1:ptr-1 # must still be output, as they are not truly gone. # Hence, we output all bytes from 1:io.size offset = get_offset(io) @@ -783,11 +785,13 @@ function take!(io::IOBuffer) return data end -"Internal method. This method can be faster than takestring!, because it does not +""" +Internal method. This method can be faster than takestring!, because it does not reset the buffer to a usable state, and it does not check for io.reinit. Using the buffer after calling unsafe_takestring! may cause undefined behaviour. This function is meant to be used when the buffer is only used as a temporary -string builder, which is discarded after the string is built." +string builder, which is discarded after the string is built. +""" function unsafe_takestring!(io::IOBuffer) used_span = get_used_span(io) nbytes = length(used_span) @@ -805,7 +809,7 @@ function unsafe_takestring!(io::IOBuffer) end """ - takestring!(io::IOBuffer) -> String + takestring!(io::IOBuffer)::String Return the content of `io` as a `String`, resetting the buffer to its initial state. @@ -892,18 +896,23 @@ function write(to::IO, from::GenericIOBuffer) return written end -function unsafe_write(to::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt) +# writing to an in-memory buffer never blocks; accept (and entry-gate) +# `cancel` so that explicit-token writes forwarded here (e.g. +# `write(io, ::String; cancel=...)`) keep working +function unsafe_write(to::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) + precheck_cancel_arg(cancel) ensureroom(to, nb) size = to.size append = to.append ptr = append ? size+1 : to.ptr data = to.data - to_write = min(nb, (min(Int(length(data))::Int, to.maxsize + get_offset(to)) - ptr + 1) % UInt) % Int + start_offset = ptr - 1 + to_write = max(0, min(nb, (min(Int(length(data))::Int - start_offset, to.maxsize - (start_offset - get_offset(to)))) % UInt) % Int) # Dispatch based on the type of data, to possibly allow using memcpy _unsafe_write(data, p, ptr, to_write % UInt) # Update to.size only if the ptr has advanced to higher than # the previous size. Otherwise, we just overwrote existing data - to.size = max(size, ptr + to_write - 1) + to.size = max(size, start_offset + to_write) # If to.append, we only update size, not ptr. if !append to.ptr = ptr + to_write @@ -941,7 +950,7 @@ end ptr = (to.append ? to.size+1 : to.ptr) # We have just ensured there is room for 1 byte, EXCEPT if we were to exceed # maxsize. So, we just need to check that here. - if ptr > to.maxsize + get_offset(to) + if ptr - get_offset(to) > to.maxsize return 0 else to.data[ptr] = a @@ -953,7 +962,7 @@ end return sizeof(UInt8) end -readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb)) +readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb)::Int) function readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb::Int) io.readable || _throw_not_readable() @@ -976,7 +985,12 @@ function occursin(delim::UInt8, buf::GenericIOBuffer) return in(delim, view(buf.data, buf.ptr:buf.size)) end -function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) +# Reading from an in-memory buffer never blocks, but the write to `out` +# may: an explicit token (or `nothing` shield) is forwarded to it, so +# shielded chains (e.g. `readline(stream; cancel=nothing)`, which lands +# here through the stream's copyuntil) compose; the sentinel keeps the +# plain call, which any IO's write supports. +function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) data = view(io.data, io.ptr:io.size) # note: findfirst + copyto! is much faster than a single loop # except for nout ≲ 20. A single loop is 2x faster for nout=5. @@ -984,12 +998,20 @@ function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) if !keep && nout > 0 && data[nout] == delim nout -= 1 end - write(out, view(io.data, io.ptr:io.ptr+nout-1)) + if cancel === DEFAULT_CANCEL + write(out, view(io.data, io.ptr:io.ptr+nout-1)) + else + write(out, view(io.data, io.ptr:io.ptr+nout-1); cancel) + end io.ptr += nread return out end -function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) +function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok + # the resolved token (or explicit shield) governs the inner copyuntil, + # which does the actual blocking reads # If the data is copied into the middle of the buffer of `out` instead of appended to the end, # and !keep, and the line copied ends with \r\n, then the copyuntil (even if keep=false) # will overwrite one too many bytes with the new \r byte. @@ -997,7 +1019,7 @@ function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) # Could perhaps be done better if !out.append && out.ptr < out.size + 1 newbuf = IOBuffer() - copyuntil(newbuf, s, 0x0a, keep=true) + copyuntil(newbuf, s, 0x0a; keep=true, cancel=tok) v = take!(newbuf) # Remove \r\n or \n if present if !keep @@ -1008,12 +1030,14 @@ function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) pop!(v) end end - write(out, v) + # `tok` is resolved above: the write to `out` runs under it (or its + # explicit shield), not the ambient scope + write(out, v; cancel=tok) return out else # Else, we can just copy the data directly into the buffer, and then # subtract the last one or two bytes depending on `keep`. - copyuntil(out, s, 0x0a, keep=true) + copyuntil(out, s, 0x0a; keep=true, cancel=tok) line = out.data i = out.size if keep || i == out.offset_or_compacted || line[i] != 0x0a @@ -1031,7 +1055,7 @@ function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) end end -function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) +function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) data = view(io.data, io.ptr:io.size) # note: findfirst + copyto! is much faster than a single loop # except for nout ≲ 20. A single loop is 2x faster for nout=5. @@ -1041,13 +1065,20 @@ function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) nout -= 1 nout > 0 && data[nout] == 0x0d && (nout -= 1) end - write(out, view(io.data, io.ptr:io.ptr+nout-1)) + if cancel === DEFAULT_CANCEL + write(out, view(io.data, io.ptr:io.ptr+nout-1)) + else + write(out, view(io.data, io.ptr:io.ptr+nout-1); cancel) + end io.ptr += nread return out end -copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) -copyline(out::GenericIOBuffer, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) +# reading from an in-memory buffer never blocks, but the write to `out` +# may; an explicit token (or shield) is forwarded to it (see copyuntil +# above) +copyline(out::IO, io::GenericIOBuffer; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) = _copyline(out, io; keep, cancel) +copyline(out::GenericIOBuffer, io::GenericIOBuffer; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) = _copyline(out, io; keep, cancel) # copy-free crc32c of IOBuffer: diff --git a/base/iostream.jl b/base/iostream.jl index c58818968c0b1..97c780eecd581 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -12,13 +12,13 @@ Mostly used to represent files returned by [`open`](@ref). """ mutable struct IOStream <: IO handle::Ptr{Cvoid} - ios::Array{UInt8,1} + ios::Vector{UInt8} name::String mark::Int64 lock::ReentrantLock _dolock::Bool - IOStream(name::AbstractString, buf::Array{UInt8,1}) = new(pointer(buf), buf, name, -1, ReentrantLock(), true) + IOStream(name::AbstractString, buf::Vector{UInt8}) = new(pointer(buf), buf, name, -1, ReentrantLock(), true) end function IOStream(name::AbstractString, finalize::Bool) @@ -260,7 +260,12 @@ function filesize(s::IOStream) end _eof_nolock(s::IOStream) = ccall(:ios_eof_blocking, Cint, (Ptr{Cvoid},), s.ios) != 0 -eof(s::IOStream) = @_lock_ios s _eof_nolock(s) +function eof(s::IOStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + # entry check only: IOStream reads block in C and are not + # interruptible (likewise for every bare `@cancel_check` below) + @cancel_check resolve_cancel_token(cancel) + @_lock_ios s _eof_nolock(s) +end ## constructing and opening streams ## @@ -403,7 +408,14 @@ function write(s::IOStream, b::UInt8) Int(@_lock_ios s ccall(:ios_putc, Cint, (Cint, Ptr{Cvoid}), b, s.ios)) end -function unsafe_write(s::IOStream, p::Ptr{UInt8}, nb::UInt) +function unsafe_write(s::IOStream, p::Ptr{UInt8}, nb::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) + # Explicit tokens gate at entry (which keeps explicit-token writes - + # e.g. `write(io, ::String; cancel=...)` - working on files); the + # sentinel deliberately passes without resolving the ambient scope: a + # plain print to a file is not a cancellation point (unlike the + # C-side-blocking reads above), so e.g. logging during cancellation + # cleanup keeps working unshielded. + precheck_cancel_arg(cancel) iswritable(s) || throw(ArgumentError("write failed, IOStream is not writeable")) return Int(@_lock_ios s ccall(:ios_write, Csize_t, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t), s.ios, p, nb)) end @@ -411,7 +423,8 @@ end # num bytes available without blocking bytesavailable(s::IOStream) = @_lock_ios s ccall(:jl_nb_available, Int32, (Ptr{Cvoid},), s.ios) -function readavailable(s::IOStream) +function readavailable(s::IOStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) lock(s.lock) nb = ccall(:jl_nb_available, Int32, (Ptr{Cvoid},), s.ios) if nb == 0 @@ -469,23 +482,29 @@ end take!(s::IOStream) = @_lock_ios s ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Cvoid},), s.ios) -function readuntil(s::IOStream, delim::UInt8; keep::Bool=false) - @_lock_ios s ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 0, !keep) +function readuntil(s::IOStream, delim::UInt8; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) + @_lock_ios s ccall(:jl_readuntil, Vector{UInt8}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 0, !keep) end # like readuntil, above, but returns a String without requiring a copy function readuntil_string(s::IOStream, delim::UInt8, keep::Bool) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 1, !keep) end -readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false) = - isascii(delim) ? readuntil_string(s, delim % UInt8, keep) : - takestring!(copyuntil(IOBuffer(sizehint=70), s, delim; keep)) +function readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok + return isascii(delim) ? readuntil_string(s, delim % UInt8, keep) : + takestring!(copyuntil(IOBuffer(sizehint=70), s, delim; keep, cancel=tok)) +end -function readline(s::IOStream; keep::Bool=false) +function readline(s::IOStream; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2) end -function copyuntil(out::IOBuffer, s::IOStream, delim::UInt8; keep::Bool=false) +function copyuntil(out::IOBuffer, s::IOStream, delim::UInt8; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) ensureroom(out, 1) # make sure we can read at least 1 byte, for iszero(n) check below while true d = out.data @@ -504,12 +523,13 @@ function copyuntil(out::IOBuffer, s::IOStream, delim::UInt8; keep::Bool=false) (eof(s) || len == out.maxsize) && break len = min(2len + 64, out.maxsize) ensureroom(out, len) - @assert length(out.data) >= len + @assert length(out.data) >= len "length(out.data) < len" end return out end -function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) +function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) @_lock_ios out @_lock_ios s ccall(:ios_copyuntil, Csize_t, (Ptr{Cvoid}, Ptr{Cvoid}, UInt8, Cint), out.ios, s.ios, delim, keep) return out @@ -572,7 +592,9 @@ requested bytes, until an error or end-of-file occurs. If `all` is `false`, at m `read` call is performed, and the amount of data returned is device-dependent. Note that not all stream types support the `all` option. """ -function readbytes!(s::IOStream, b::MutableDenseArrayType{UInt8}, nb=length(b); all::Bool=true) +function readbytes!(s::IOStream, b::MutableDenseArrayType{UInt8}, nb=length(b); all::Bool=true, + cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) return all ? readbytes_all!(s, b, nb) : readbytes_some!(s, b, nb) end @@ -617,7 +639,8 @@ requested bytes, until an error or end-of-file occurs. If `all` is `false`, at m `read` call is performed, and the amount of data returned is device-dependent. Note that not all stream types support the `all` option. """ -function read(s::IOStream, nb::Integer; all::Bool=true) +function read(s::IOStream, nb::Integer; all::Bool=true, cancel::CancelTokenArg=DEFAULT_CANCEL) + @cancel_check resolve_cancel_token(cancel) # When all=false we have to allocate a buffer of the requested size upfront # since a single call will be made b = Vector{UInt8}(undef, all && nb == typemax(Int) ? 1024 : nb) diff --git a/base/irrationals.jl b/base/irrationals.jl index e0e7fc3bc2e1d..e551a72d6abe1 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -171,18 +171,36 @@ function <(x::Rational{T}, y::AbstractIrrational) where T return x < ry end end -<(x::AbstractIrrational, y::Rational{BigInt}) = big(x) < y -<(x::Rational{BigInt}, y::AbstractIrrational) = x < big(y) +function <(x::AbstractIrrational, y::Rational{BigInt}) + Float64(x) != Float64(y) && return Float64(x) < Float64(y) + p = precision(BigFloat) + 32 + @assume_effects :terminates_locally while true + xf, yf = BigFloat(x; precision=p), BigFloat(y; precision=p) + xf != yf && return xf < yf + p *= 2 + end +end +<(x::Rational{BigInt}, y::AbstractIrrational) = !(y < x) <=(x::AbstractIrrational, y::Rational) = x < y <=(x::Rational, y::AbstractIrrational) = x < y +cmp(::Irrational{s}, ::Irrational{s}) where {s} = 0 +cmp(x::AbstractIrrational, y::AbstractIrrational) = x < y ? -1 : 1 +cmp(x::AbstractIrrational, y::Real) = x < y ? -1 : 1 +cmp(x::Real, y::AbstractIrrational) = x < y ? -1 : 1 + +cmp(<, ::Irrational{s}, ::Irrational{s}) where {s} = 0 +cmp(<, x::AbstractIrrational, y::AbstractIrrational) = x < y ? -1 : 1 +cmp(<, x::AbstractIrrational, y::Real) = x < y ? -1 : 1 +cmp(<, x::Real, y::AbstractIrrational) = x < y ? -1 : 1 + isfinite(::AbstractIrrational) = true isinteger(::AbstractIrrational) = false iszero(::AbstractIrrational) = false isone(::AbstractIrrational) = false -hash(x::Irrational, h::UInt) = 3h - objectid(x) +hash(x::Irrational, h::UInt) = 3 *% h -% objectid(x) widen(::Type{T}) where {T<:Irrational} = T @@ -200,6 +218,8 @@ for op in Symbol[:+, :-, :*, :/, :^] end *(x::Bool, y::AbstractIrrational) = ifelse(x, Float64(y), 0.0) +^(x::AbstractIrrational, y::Integer) = float(x)^y + round(x::Irrational, r::RoundingMode) = round(float(x), r) """ @@ -253,7 +273,7 @@ function irrational(sym, val, def) bigconvert = isa(def,Symbol) ? quote function Base.BigFloat(::Irrational{$qsym}, r::MPFR.MPFRRoundingMode=Rounding.rounding_raw(BigFloat); precision=precision(BigFloat)) c = BigFloat(;precision=precision) - ccall(($(string("mpfr_const_", def)), :libmpfr), + ccall(($(string("mpfr_const_", def)), Base.MPFR.libmpfr), Cint, (Ref{BigFloat}, MPFR.MPFRRoundingMode), c, r) return c end diff --git a/base/iterators.jl b/base/iterators.jl index 9812feb2d62cd..5802792d2fded 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -9,31 +9,31 @@ baremodule Iterators import Base: @__MODULE__, parentmodule const Base = parentmodule(@__MODULE__) using .Base: - @inline, Pair, Pairs, AbstractDict, IndexLinear, IndexStyle, AbstractVector, Vector, + @inline, Pair, Pairs, IndexLinear, AbstractVector, Vector, SizeUnknown, HasLength, HasShape, IsInfinite, EltypeUnknown, HasEltype, OneTo, - @propagate_inbounds, @isdefined, @boundscheck, @inbounds, Generator, IdDict, + @propagate_inbounds, @boundscheck, @inbounds, Generator, IdDict, AbstractRange, AbstractUnitRange, UnitRange, LinearIndices, TupleOrBottom, - (:), |, +, -, *, !==, !, ==, !=, <=, <, >, >=, =>, missing, - any, _counttuple, eachindex, ntuple, zero, prod, reduce, in, firstindex, lastindex, - tail, fieldtypes, min, max, minimum, zero, oneunit, promote, promote_shape, LazyString, - afoldl, mod1 + :, |, +, -, *, !==, !, ==, !=, <=, <, >, >=, =>, missing, + any, eachindex, ntuple, zero, identity, reduce, in, firstindex, lastindex, + tail, fieldtypes, min, max, zero, oneunit, promote, promote_shape, LazyString, + afoldl, mod1, @default_eltype using .Core using Core: @doc using Base: - cld, fld, resize!, IndexCartesian, Checked + cld, resize!, IndexCartesian, Checked using .Checked: checked_mul import Base: first, last, - isempty, length, size, axes, ndims, + length, size, axes, ndims, eltype, IteratorSize, IteratorEltype, promote_typejoin, haskey, keys, values, pairs, getindex, setindex!, get, iterate, popfirst!, isdone, peek, intersect -export enumerate, zip, rest, countfrom, take, drop, takewhile, dropwhile, cycle, repeated, product, flatten, flatmap, partition, nth -public accumulate, filter, map, peel, reverse, Stateful +export enumerate, zip, rest, countfrom, take, drop, takewhile, dropwhile, cycle, repeated, product, flatten, flatmap, partition, nth, findeach +public accumulate, filter, map, peel, reverse, Reverse, Stateful """ Iterators.map(f, iterators...) @@ -71,6 +71,8 @@ end and_iteratorsize(isz::T, ::T) where {T} = isz and_iteratorsize(::HasLength, ::HasShape) = HasLength() and_iteratorsize(::HasShape, ::HasLength) = HasLength() +and_iteratorsize(::HasShape{N}, ::HasShape{N}) where {N} = HasShape{N}() +and_iteratorsize(::HasShape, ::HasShape) = HasLength() and_iteratorsize(a, b) = SizeUnknown() and_iteratoreltype(iel::T, ::T) where {T} = iel @@ -110,6 +112,16 @@ julia> foreach(println, Iterators.reverse(1:5)) """ reverse(itr) = Reverse(itr) +""" + Iterators.Reverse{T} + +A type representing a reverse-order iterator for an iterator of type `T`, which +is stored in the `itr` field. Typically returned by [`Iterators.reverse(itr::T)`](@ref), +which constructs an `Iterators.Reverse{T}` wrapper around `itr` by default. + +To support lazy reverse-order iteration, a type `T` should either implement an [`iterate`](@ref) +method for `Iterators.Reverse{T}` or overload `Iterators.reverse` to return a different type. +""" struct Reverse{T} itr::T end @@ -137,6 +149,10 @@ function iterate(A::Reverse, state=(reverse(eachindex(A.itr)),)) (A.itr[idx], (state[1], itrs)) end +# Guard against invalidations due to spurious `Reverse{Union{}}` intersections +iterate(r::Reverse{Union{}}) = throw(ArgumentError("cannot iterate a reversed iterator of type Union{}")) +iterate(r::Reverse{Union{}}, state) = throw(ArgumentError("cannot iterate a reversed iterator of type Union{}")) + reverse(R::AbstractRange) = Base.reverse(R) # copying ranges is cheap reverse(G::Generator) = Generator(G.f, reverse(G.iter)) reverse(r::Reverse) = r.itr @@ -198,7 +214,7 @@ size(e::Enumerate) = size(e.itr) n === nothing && return n (i, n[1]), (i+1, n[2]) end -last(e::Enumerate) = (length(e.itr), e.itr[end]) +last(e::Enumerate) = (length(e.itr), last(e.itr)) eltype(::Type{Enumerate{I}}) where {I} = TupleOrBottom(Int, eltype(I)) @@ -260,14 +276,14 @@ CartesianIndex(1, 2) d CartesianIndex(2, 2) e ``` -See also [`IndexStyle`](@ref), [`axes`](@ref). +See also [`Base.IndexStyle`](@ref), [`axes`](@ref). """ pairs(::IndexLinear, A::AbstractArray) = Pairs(A, LinearIndices(A)) # preserve indexing capabilities for known indexable types # faster than zip(keys(a), values(a)) for arrays pairs(tuple::Tuple) = Pairs{Int}(tuple, keys(tuple)) -pairs(nt::NamedTuple) = Pairs{Symbol}(nt, keys(nt)) +pairs(nt::NamedTuple) = Pairs{Symbol}(nt, nothing) pairs(v::Core.SimpleVector) = Pairs(v, LinearIndices(v)) pairs(A::AbstractVector) = pairs(IndexLinear(), A) # pairs(v::Pairs) = v # listed for reference, but already defined from being an AbstractDict @@ -275,45 +291,45 @@ pairs(A::AbstractVector) = pairs(IndexLinear(), A) pairs(::IndexCartesian, A::AbstractArray) = Pairs(A, Base.CartesianIndices(axes(A))) pairs(A::AbstractArray) = pairs(IndexCartesian(), A) -length(v::Pairs) = length(getfield(v, :itr)) -axes(v::Pairs) = axes(getfield(v, :itr)) -size(v::Pairs) = size(getfield(v, :itr)) +length(v::Pairs) = length(keys(v)) +axes(v::Pairs) = axes(keys(v)) +size(v::Pairs) = size(keys(v)) Base.@eval @propagate_inbounds function _pairs_elt(p::Pairs{K, V}, idx) where {K, V} return $(Expr(:new, :(Pair{K, V}), :idx, :(getfield(p, :data)[idx]))) end @propagate_inbounds function iterate(p::Pairs{K, V}, state...) where {K, V} - x = iterate(getfield(p, :itr), state...) + x = iterate(keys(p), state...) x === nothing && return x idx, next = x return (_pairs_elt(p, idx), next) end -@propagate_inbounds function iterate(r::Reverse{<:Pairs}, state=(reverse(getfield(r.itr, :itr)),)) +@propagate_inbounds function iterate(r::Reverse{<:Pairs}, state=(reverse(keys(r.itr)),)) x = iterate(state...) x === nothing && return x idx, next = x return (_pairs_elt(r.itr, idx), (state[1], next)) end -@inline isdone(v::Pairs, state...) = isdone(getfield(v, :itr), state...) +@inline isdone(v::Pairs, state...) = isdone(keys(v), state...) IteratorSize(::Type{<:Pairs{<:Any, <:Any, I}}) where {I} = IteratorSize(I) IteratorSize(::Type{<:Pairs{<:Any, <:Any, <:AbstractUnitRange, <:Tuple}}) = HasLength() function last(v::Pairs{K, V}) where {K, V} - idx = last(getfield(v, :itr)) + idx = last(keys(v)) return Pair{K, V}(idx, v[idx]) end -haskey(v::Pairs, key) = (key in getfield(v, :itr)) -keys(v::Pairs) = getfield(v, :itr) +haskey(v::Pairs, key) = key in keys(v) +keys(v::Pairs) = getfield(v, :itr) === nothing ? keys(getfield(v, :data)) : getfield(v, :itr) values(v::Pairs) = getfield(v, :data) # TODO: this should be a view of data subset by itr -getindex(v::Pairs, key) = getfield(v, :data)[key] -setindex!(v::Pairs, value, key) = (getfield(v, :data)[key] = value; v) -get(v::Pairs, key, default) = get(getfield(v, :data), key, default) -get(f::Base.Callable, v::Pairs, key) = get(f, getfield(v, :data), key) +getindex(v::Pairs, key) = values(v)[key] +setindex!(v::Pairs, value, key) = (values(v)[key] = value; v) +get(v::Pairs, key, default) = get(values(v), key, default) +get(f::Base.Callable, v::Pairs, key) = get(f, values(v), key) # zip @@ -399,13 +415,21 @@ _promote_tuple_shape((m,)::Tuple{Integer}, (n,)::Tuple{Integer}) = (min(m, n),) _promote_tuple_shape(a, b) = promote_shape(a, b) _promote_tuple_shape(a, b...) = _promote_tuple_shape(a, _promote_tuple_shape(b...)) _promote_tuple_shape(a) = a -eltype(::Type{Zip{Is}}) where {Is<:Tuple} = TupleOrBottom(map(eltype, fieldtypes(Is))...) +# bind the tuple length `N` as a static parameter: `fieldtypes` needs an +# egality-pinned `Is` to fold, while the sparam binds through `==` (#61323). +# NOTE: an indefinite-length `Is` (e.g. `Tuple{Vararg{Vector{Int}}}`) matches +# no `N`, so type-level queries on such (necessarily abstract) `Zip`/`Product` +# types skip these methods and fall back to the generic trait defaults (they +# used to raise a TypeError from `_counttuple`) +eltype(::Type{Zip{Is}}) where {N, Is<:Tuple{Vararg{Any, N}}} = TupleOrBottom(ntuple(n -> eltype(fieldtype(Is, n)), N)...) #eltype(::Type{Zip{Tuple{}}}) = Tuple{} #eltype(::Type{Zip{Tuple{A}}}) where {A} = Tuple{eltype(A)} #eltype(::Type{Zip{Tuple{A, B}}}) where {A, B} = Tuple{eltype(A), eltype(B)} + @inline isdone(z::Zip) = _zip_any_isdone(z.is, Base.map(_ -> (), z.is)) @inline isdone(z::Zip, ss) = _zip_any_isdone(z.is, Base.map(tuple, ss)) -@inline function _zip_any_isdone(is, ss) + +@inline function _zip_any_isdone(is::Tuple, ss::Tuple) d1 = isdone(is[1], ss[1]...) d1 === true && return true return d1 | _zip_any_isdone(tail(is), tail(ss)) @@ -429,36 +453,36 @@ end return _zip_iterate_interleave(xs1, xs2, ds) end -@propagate_inbounds function _zip_iterate_some(is, ss, ds::Tuple{T,Vararg{Any}}, f::T) where T +@propagate_inbounds function _zip_iterate_some(is::Tuple, ss::Tuple, ds::Tuple{T,Vararg{Any}}, f::T) where T x = iterate(is[1], ss[1]...) x === nothing && return nothing y = _zip_iterate_some(tail(is), tail(ss), tail(ds), f) y === nothing && return nothing return (x, y...) end -@propagate_inbounds _zip_iterate_some(is, ss, ds::Tuple{Any,Vararg{Any}}, f) = +@propagate_inbounds _zip_iterate_some(is::Tuple, ss::Tuple, ds::Tuple{Any,Vararg{Any}}, f) = _zip_iterate_some(tail(is), tail(ss), tail(ds), f) _zip_iterate_some(::Tuple{}, ::Tuple{}, ::Tuple{}, ::Any) = () -function _zip_iterate_interleave(xs1, xs2, ds) +function _zip_iterate_interleave(xs1::Tuple, xs2::Tuple, ds::Tuple) t = _zip_iterate_interleave(tail(xs1), xs2, tail(ds)) ((xs1[1][1], t[1]...), (xs1[1][2], t[2]...)) end -function _zip_iterate_interleave(xs1, xs2, ds::Tuple{Bool,Vararg{Any}}) +function _zip_iterate_interleave(xs1::Tuple, xs2::Tuple, ds::Tuple{Bool,Vararg{Any}}) t = _zip_iterate_interleave(xs1, tail(xs2), tail(ds)) ((xs2[1][1], t[1]...), (xs2[1][2], t[2]...)) end _zip_iterate_interleave(::Tuple{}, ::Tuple{}, ::Tuple{}) = ((), ()) -function _zip_isdone(is, ss) +function _zip_isdone(is::Tuple, ss::Tuple) d = isdone(is[1], ss[1]...) d´, ds = _zip_isdone(tail(is), tail(ss)) return (d === true || d´, (d, ds...)) end _zip_isdone(::Tuple{}, ::Tuple{}) = (false, ()) -IteratorSize(::Type{Zip{Is}}) where {Is<:Tuple} = zip_iteratorsize(ntuple(n -> IteratorSize(fieldtype(Is, n)), _counttuple(Is)::Int)...) -IteratorEltype(::Type{Zip{Is}}) where {Is<:Tuple} = zip_iteratoreltype(ntuple(n -> IteratorEltype(fieldtype(Is, n)), _counttuple(Is)::Int)...) +IteratorSize(::Type{Zip{Is}}) where {N, Is<:Tuple{Vararg{Any, N}}} = zip_iteratorsize(ntuple(n -> IteratorSize(fieldtype(Is, n)), N)...) +IteratorEltype(::Type{Zip{Is}}) where {N, Is<:Tuple{Vararg{Any, N}}} = zip_iteratoreltype(ntuple(n -> IteratorEltype(fieldtype(Is, n)), N)...) zip_iteratorsize() = IsInfinite() zip_iteratorsize(I) = I @@ -473,7 +497,8 @@ zip_iteratoreltype() = HasEltype() zip_iteratoreltype(a) = a zip_iteratoreltype(a, tail...) = and_iteratoreltype(a, zip_iteratoreltype(tail...)) -last(z::Zip) = getindex.(z.is, minimum(Base.map(lastindex, z.is))) +last(z::Zip) = nth(z, length(z)) + function reverse(z::Zip) if !first(_zip_lengths_finite_equal(z.is)) throw(ArgumentError("Cannot reverse zipped iterators of unknown, infinite, or unequal lengths")) @@ -838,7 +863,7 @@ length(d::Drop) = _diff_length(d.xs, 1:d.n, IteratorSize(d.xs), HasLength()) function iterate(it::Drop) y = iterate(it.xs) - for i in 1:it.n + for _ in 1:it.n y === nothing && return y y = iterate(it.xs, y[2]) end @@ -858,7 +883,7 @@ end """ takewhile(pred, iter) -An iterator that generates element from `iter` as long as predicate `pred` is true, +An iterator that generates elements from `iter` as long as predicate `pred` is true, afterwards, drops every element. !!! compat "Julia 1.4" @@ -905,7 +930,7 @@ end """ dropwhile(pred, iter) -An iterator that drops element from `iter` as long as predicate `pred` is true, +An iterator that drops elements from `iter` as long as predicate `pred` is true, afterwards, returns every element. !!! compat "Julia 1.4" @@ -945,6 +970,34 @@ IteratorSize(::Type{<:DropWhile}) = SizeUnknown() eltype(::Type{DropWhile{I,P}}) where {I,P} = eltype(I) IteratorEltype(::Type{DropWhile{I,P}}) where {I,P} = IteratorEltype(I) +""" + findeach(f, it) + findeach(it) + +An iterator that generates every key from the key/value pairs of `pairs(it)`, +where `f(value)` returns `true`. + +If `f` is not specified, default to `identity`. + +`Iterators.findeach` is the lazy equivalent of `findall`. + +!!! compat "Julia 1.13" + `findeach` requires at least Julia 1.13. + +# Examples +```jldoctest +julia> collect(Iterators.findeach(isodd, Dict(2 => 3, 3 => 2))) +1-element Vector{Int64}: + 2 + +julia> only(Iterators.findeach(==(1), [3,6,2,1])) +4 +``` +""" +findeach(f, it) = (k for (k, v) in pairs(it) if f(v)) + +findeach(it) = findeach(identity, it) + # Cycle an iterator forever @@ -1076,8 +1129,8 @@ true product(iters...) = ProductIterator(iters) IteratorSize(::Type{ProductIterator{Tuple{}}}) = HasShape{0}() -IteratorSize(::Type{ProductIterator{T}}) where {T<:Tuple} = - prod_iteratorsize(ntuple(n -> IteratorSize(fieldtype(T, n)), _counttuple(T)::Int)..., HasShape{0}()) +IteratorSize(::Type{ProductIterator{T}}) where {N, T<:Tuple{Vararg{Any, N}}} = + prod_iteratorsize(ntuple(n -> IteratorSize(fieldtype(T, n)), N)..., HasShape{0}()) prod_iteratorsize() = HasShape{0}() prod_iteratorsize(I) = I @@ -1116,15 +1169,15 @@ length(P::ProductIterator) = reduce(checked_mul, size(P); init=1) IteratorEltype(::Type{ProductIterator{Tuple{}}}) = HasEltype() IteratorEltype(::Type{ProductIterator{Tuple{I}}}) where {I} = IteratorEltype(I) -function IteratorEltype(::Type{ProductIterator{T}}) where {T<:Tuple} - E = ntuple(n -> IteratorEltype(fieldtype(T, n)), _counttuple(T)::Int) +function IteratorEltype(::Type{ProductIterator{T}}) where {N, T<:Tuple{Vararg{Any, N}}} + E = ntuple(n -> IteratorEltype(fieldtype(T, n)), N) any(I -> I == EltypeUnknown(), E) && return EltypeUnknown() return E[end] end eltype(::Type{ProductIterator{I}}) where {I} = _prod_eltype(I) _prod_eltype(::Type{Tuple{}}) = Tuple{} -_prod_eltype(::Type{I}) where {I<:Tuple} = TupleOrBottom(ntuple(n -> eltype(fieldtype(I, n)), _counttuple(I)::Int)...) +_prod_eltype(::Type{I}) where {N, I<:Tuple{Vararg{Any, N}}} = TupleOrBottom(ntuple(n -> eltype(fieldtype(I, n)), N)...) iterate(::ProductIterator{Tuple{}}) = (), true iterate(::ProductIterator{Tuple{}}, state) = nothing @@ -1147,6 +1200,8 @@ end next === nothing && return nothing restnext = _piterate(rest...) restnext === nothing && return nothing + VS = @default_eltype(iter1) + next = Pair{VS, typeof(next[2])}(next[1], next[2]) return (next, restnext...) end @inline function iterate(P::ProductIterator) @@ -1159,8 +1214,8 @@ end @inline _piterate1(::Tuple{}, ::Tuple{}) = nothing @inline function _piterate1(iters, states) iter1 = first(iters) - next = iterate(iter1, first(states)[2]) - restnext = tail(states) + state1, restnext... = states + next = iterate(iter1, state1[2]) if next === nothing isdone(iter1) === true && return nothing restnext = _piterate1(tail(iters), restnext) @@ -1168,6 +1223,7 @@ end next = iterate(iter1) next === nothing && return nothing end + next = Pair{fieldtype(typeof(state1), 1), typeof(next[2])}(next[1], next[2]) return (next, restnext...) end @inline function iterate(P::ProductIterator, states) @@ -1218,8 +1274,12 @@ flatten(itr) = Flatten(itr) eltype(::Type{Flatten{I}}) where {I} = eltype(eltype(I)) # For tuples, we statically know the element type of each index, so we can compute -# this at compile time. -function eltype(::Type{Flatten{I}}) where {I<:Union{Tuple,NamedTuple}} +# this at compile time. Like `Zip`'s `eltype`, bind the length `N` as a static +# parameter: `fieldtypes` needs an egality-pinned `I` to fold (#61323). +function eltype(::Type{Flatten{I}}) where {N, I<:Tuple{Vararg{Any, N}}} + afoldl((T, i) -> promote_typejoin(T, eltype(i)), Union{}, ntuple(n -> fieldtype(I, n), N)...) +end +function eltype(::Type{Flatten{I}}) where {I<:NamedTuple} afoldl((T, i) -> promote_typejoin(T, eltype(i)), Union{}, fieldtypes(I)...) end @@ -1242,7 +1302,7 @@ IteratorSize(::Type{Flatten{I}}) where {I} = _flatten_iteratorsize(IteratorSize( flatten_length(f, T::Type{Union{}}, slurp...) = 0 function flatten_length(f, T::Type{<:NTuple{N,Any}}) where {N} - return N * length(f.it) + return checked_mul(N, length(f.it)) end flatten_length(f, ::Type{<:Number}) = length(f.it) flatten_length(f, T) = throw(ArgumentError( @@ -1509,7 +1569,7 @@ approx_iter_type(itrT::Type) = _approx_iter_type(itrT, Base._return_type(iterate # having to typesplit on Nothing function doiterate(itr, valstate::Union{Nothing, Tuple{Any, Any}}) valstate === nothing && return nothing - val, st = valstate + _, st = valstate return iterate(itr, st) end function _approx_iter_type(itrT::Type, vstate::Type) @@ -1530,7 +1590,7 @@ convert(::Type{Stateful}, itr) = Stateful(itr) throw(Base.EOFError()) else val, state = vs - Core.setfield!(s, :nextvalstate, iterate(s.itr, state)) + setfield!(s, :nextvalstate, iterate(s.itr, state)) return val end end @@ -1645,8 +1705,8 @@ pairs(s::AbstractString) = IterableStatePairs(s) """ nth(itr, n::Integer) -Get the `n`th element of an iterable collection. Throw a `BoundsError`[@ref] if not existing. -Will advance any `Stateful`[@ref] iterator. +Get the `n`th element of an iterable collection. Throw a [`BoundsError`](@ref) if not existing. +Will advance any [`Stateful`](@ref) iterator. See also: [`first`](@ref), [`last`](@ref) @@ -1664,6 +1724,9 @@ julia> stateful = Iterators.Stateful(1:10); Iterators.nth(stateful, 7) julia> first(stateful) 8 ``` + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. """ nth(itr, n::Integer) = _nth(IteratorSize(itr), itr, n) nth(itr::Cycle{I}, n::Integer) where I = _nth(IteratorSize(I), itr, n) @@ -1700,6 +1763,9 @@ function _nth(::IteratorSize, itr, n::Integer) y === nothing && throw(BoundsError(itr, n)) y[1] end + +_nth(::IteratorSize, z::Zip, n::Integer) = Base.map(nth(n), z.is) + """ nth(n::Integer) diff --git a/base/libc.jl b/base/libc.jl index f575690c370d6..2fd9fc5f7b6dd 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -7,7 +7,7 @@ Interface to libc, the C standard library. import Base: transcode, windowserror, show # these need to be defined separately for bootstrapping but belong to Libc -import Base: memcpy, memmove, memset, memcmp +import Base: memcmp, memcpy, memmove, memset import Core.Intrinsics: bitcast export FILE, TmStruct, strftime, strptime, getpid, gethostname, free, malloc, memcpy, @@ -62,6 +62,7 @@ if Sys.iswindows() WindowsRawSocket(handle::WindowsRawSocket) = handle Base.cconvert(::Type{Ptr{Cvoid}}, fd::WindowsRawSocket) = bitcast(Ptr{Cvoid}, fd) + show(io::IO, fd::WindowsRawSocket) = (print(io, "WindowsRawSocket("); show(io, bitcast(UInt, fd)); print(io, ')')) # avoids invalidation via show_default _get_osfhandle(fd::RawFD) = ccall(:_get_osfhandle, WindowsRawSocket, (RawFD,), fd) _get_osfhandle(fd::WindowsRawSocket) = fd function dup(src::WindowsRawSocket) @@ -266,9 +267,39 @@ function strptime(fmt::AbstractString, timestr::AbstractString) throw(ArgumentError("invalid arguments")) end @static if Sys.isapple() + function shouldcallmktime(s::AbstractString) + # Equivalent to !occursin(r"([^%]|^)%(a|A|j|w|Ow)"a, s), but + # without regex since this is not available yet + + c = Base.utf8units(s) + N = length(c) + i = findfirst(==(UInt8('%')), c) + + while true + isnothing(i) && break + i >= N && break + + # Ignore % following after another % + if i == firstindex(c) || c[i-1] != UInt8('%') + # Detect %a, %A, %j, %w, %Ow + if c[i+1] in b"aAjw" + return false + end + if (checkbounds(Bool, c, i+2) && + c[i+1] == UInt8('O') && + c[i+2] == UInt8('w') + ) + return false + end + end + + i = findnext(==(UInt8('%')), c, i+1) + end + return true + end # if we didn't explicitly parse the weekday or year day, use mktime # to fill them in automatically. - if !occursin(r"([^%]|^)%(a|A|j|w|Ow)"a, fmt) + if shouldcallmktime(fmt) ccall(:mktime, Int, (Ref{TmStruct},), tm) end end @@ -280,7 +311,7 @@ end """ time(t::TmStruct)::Float64 -Converts a `TmStruct` struct to a number of seconds since the epoch. +Convert a `TmStruct` struct to a number of seconds since the epoch. """ time(tm::TmStruct) = Float64(ccall(:mktime, Int, (Ref{TmStruct},), tm)) @@ -574,6 +605,7 @@ end getuid() = ccall(:jl_getuid, Culong, ()) geteuid() = ccall(:jl_geteuid, Culong, ()) +getegid() = Sys.iswindows() ? Culong(-1) : ccall(:getegid, Culong, ()) # Include dlopen()/dlpath() code include("libdl.jl") diff --git a/base/libdl.jl b/base/libdl.jl index 73fd9d9c26871..2781cf0ad4144 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -60,8 +60,8 @@ function dlsym(hnd::Ptr, s::Union{Symbol,AbstractString}; throw_error::Bool = tr hnd == C_NULL && throw(ArgumentError("NULL library handle")) val = Ref(Ptr{Cvoid}(0)) symbol_found = ccall(:jl_dlsym, Cint, - (Ptr{Cvoid}, Cstring, Ref{Ptr{Cvoid}}, Cint), - hnd, s, val, Int64(throw_error) + (Ptr{Cvoid}, Cstring, Ref{Ptr{Cvoid}}, Cint, Cint), + hnd, s, val, Int64(throw_error), Int64(1) ) if symbol_found == 0 return nothing @@ -171,7 +171,7 @@ end """ dlclose(::Nothing) -For the very common pattern usage pattern of +For the very common usage pattern of try hdl = dlopen(library_name) @@ -223,6 +223,7 @@ find_library(libname::Union{Symbol,AbstractString}, extrapaths=String[]) = Given a library `handle` from `dlopen`, return the full path. """ function dlpath(handle::Ptr{Cvoid}) + handle == C_NULL && throw(ArgumentError("NULL library handle")) p = ccall(:jl_pathname_for_handle, Cstring, (Ptr{Cvoid},), handle) s = unsafe_string(p) Sys.iswindows() && Libc.free(p) @@ -262,30 +263,6 @@ File extension for dynamic libraries (e.g. dll, dylib, so) on the current platfo """ dlext -if (Sys.islinux() || Sys.isbsd()) && !Sys.isapple() - struct dl_phdr_info - # Base address of object - addr::Cuint - - # Null-terminated name of object - name::Ptr{UInt8} - - # Pointer to array of ELF program headers for this object - phdr::Ptr{Cvoid} - - # Number of program headers for this object - phnum::Cshort - end - - # This callback function called by dl_iterate_phdr() on Linux and BSD's - # DL_ITERATE_PHDR(3) on freebsd - function dl_phdr_info_callback(di::dl_phdr_info, size::Csize_t, dynamic_libraries::Array{String,1}) - name = unsafe_string(di.name) - push!(dynamic_libraries, name) - return Cint(0) - end -end - """ dllist() @@ -302,13 +279,10 @@ function dllist() name = unsafe_string(ccall(:_dyld_get_image_name, Cstring, (UInt32,), i)) push!(dynamic_libraries, name) end - elseif Sys.islinux() || Sys.isbsd() - callback = @cfunction(dl_phdr_info_callback, Cint, - (Ref{dl_phdr_info}, Csize_t, Ref{Vector{String}})) - ccall(:dl_iterate_phdr, Cint, (Ptr{Cvoid}, Ref{Vector{String}}), callback, dynamic_libraries) - popfirst!(dynamic_libraries) - filter!(!isempty, dynamic_libraries) - elseif Sys.iswindows() + elseif Sys.iswindows() || Sys.islinux() || Sys.isbsd() + # `dl_iterate_phdr` must be handled by C, since otherwise arbitrary Julia + # code (in finalizers / ccall symbol resolution) may compete for the dynamic + # linker lock held during its callback ccall(:jl_dllist, Cint, (Any,), dynamic_libraries) else # unimplemented @@ -319,14 +293,21 @@ end """ - LazyLibraryPath + LazyLibraryPath(path_pieces...) -Helper type for lazily constructed library paths for use with `LazyLibrary`. -Arguments are passed to `joinpath()`. Arguments must be able to have -`string()` called on them. +Helper type for lazily constructed library paths for use with [`LazyLibrary`](@ref). +Path pieces are stored unevaluated and joined with `joinpath()` when the library is first +accessed. Arguments must be able to have `string()` called on them. -``` -libfoo = LazyLibrary(LazyLibraryPath(prefix, "lib/libfoo.so.1.2.3")) +!!! compat "Julia 1.11" + `LazyLibraryPath` was added in Julia 1.11. + +See also [`LazyLibrary`](@ref), [`BundledLazyLibraryPath`](@ref). + +# Examples + +```julia +const mylib = LazyLibrary(LazyLibraryPath(artifact_dir, "lib", "libmylib.so.1.2.3")) ``` """ struct LazyLibraryPath @@ -347,29 +328,75 @@ end Base.string(::PrivateShlibdirGetter) = private_shlibdir() """ - BundledLazyLibraryPath + BundledLazyLibraryPath(subpath) -Helper type for lazily constructed library paths that are stored within the -bundled Julia distribution, primarily for use by Base modules. +Helper type for lazily constructed library paths within the Julia distribution. +Constructs paths relative to Julia's private shared library directory. +Primarily used by Julia's standard library. For example: +```julia +const libgmp = LazyLibrary(BundledLazyLibraryPath("libgmp.so.10")) ``` -libfoo = LazyLibrary(BundledLazyLibraryPath("libfoo.so.1.2.3")) -``` + +!!! compat "Julia 1.11" + `BundledLazyLibraryPath` was added in Julia 1.11. + +See also [`LazyLibrary`](@ref), [`LazyLibraryPath`](@ref). """ BundledLazyLibraryPath(subpath) = LazyLibraryPath(PrivateShlibdirGetter(), subpath) +# Small helper struct to initialize a LazyLibrary with its initial set of dependencies +struct InitialDependencies{T} + dependencies::Vector{T} +end +(init::InitialDependencies)() = copy(init.dependencies) """ - LazyLibrary(name, flags = , + LazyLibrary(name; flags = , dependencies = LazyLibrary[], on_load_callback = nothing) -Represents a lazily-loaded library that opens itself and its dependencies on first usage -in a `dlopen()`, `dlsym()`, or `ccall()` usage. While this structure contains the -ability to run arbitrary code on first load via `on_load_callback`, we caution that this -should be used sparingly, as it is not expected that `ccall()` should result in large -amounts of Julia code being run. You may call `ccall()` from within the -`on_load_callback` but only for the current library and its dependencies, and user should -not call `wait()` on any tasks within the on load callback. +Represents a lazily-loaded shared library that delays loading itself and its dependencies +until first use in a `ccall()`, `@ccall`, `dlopen()`, `dlsym()`, `dlpath()`, or `cglobal()`. +This is a thread-safe mechanism for on-demand library initialization. + +# Arguments + +- `name`: Library name (or lazy path computation) as a `String`, + [`LazyLibraryPath`](@ref), or [`BundledLazyLibraryPath`](@ref). +- `flags`: Optional `dlopen` flags (default: `RTLD_LAZY | RTLD_DEEPBIND`). See [`dlopen`](@ref). +- `dependencies`: Vector of `LazyLibrary` object references to load before this one. +- `on_load_callback`: Optional function to run arbitrary code on first load (use sparingly, + as it is not expected that `ccall()` should result in large amounts of Julia code being run. + You may call `ccall()` from within the `on_load_callback` but only for the current library + and its dependencies, and user should not call `wait()` on any tasks within the on load + callback as they may deadlock). + +The dlopen operation is thread-safe: only one thread loads the library, acquired after the +release store of the reference to each dependency from loading of each dependency. Other +tasks block until loading completes. The handle is then cached and reused for all subsequent +calls (there is no dlclose for lazy library and dlclose should not be called on the returned handle). + +!!! compat "Julia 1.11" + `LazyLibrary` was added in Julia 1.11. + +See also [`LazyLibraryPath`](@ref), [`BundledLazyLibraryPath`](@ref), [`dlopen`](@ref), +[`dlsym`](@ref), [`add_dependency!`](@ref). + +# Examples + +```julia +# Basic usage +const mylib = LazyLibrary("libmylib") +@ccall mylib.myfunc(42::Cint)::Cint + +# With dependencies +const libfoo = LazyLibrary("libfoo") +const libbar = LazyLibrary("libbar"; dependencies=[libfoo]) +``` + +For more examples including platform-specific libraries, lazy path construction, and +migration from `__init__()` patterns, see the manual section on +[Using LazyLibrary for Lazy Loading](@ref man-lazylibrary). """ mutable struct LazyLibrary # Name and flags to open with @@ -377,7 +404,11 @@ mutable struct LazyLibrary const flags::UInt32 # Dependencies that must be loaded before we can load - dependencies::Vector{LazyLibrary} + # + # The OncePerProcess is introduced here so that any registered dependencies are + # always ephemeral to a given process (instead of, e.g., persisting depending + # on whether they were added in the process where this LazyLibrary was created) + dependencies::Base.OncePerProcess{Vector{LazyLibrary}, InitialDependencies{LazyLibrary}} # Function that get called once upon initial load on_load_callback @@ -390,7 +421,9 @@ mutable struct LazyLibrary return new( path, UInt32(flags), - collect(dependencies), + Base.OncePerProcess{Vector{LazyLibrary}}( + InitialDependencies{LazyLibrary}(dependencies) + ), on_load_callback, Base.ReentrantLock(), C_NULL, @@ -400,15 +433,31 @@ end # We support adding dependencies only because of very special situations # such as LBT needing to have OpenBLAS_jll added as a dependency dynamically. +""" + add_dependency!(library::LazyLibrary, dependency::LazyLibrary) + +Dynamically add a dependency that must be loaded before `library`. Only needed when +dependencies cannot be determined at construction time. + +!!! warning + Dependencies added with this function are **ephemeral** and only persist within the + current process. They will not persist across precompilation boundaries. + +Prefer specifying dependencies in the `LazyLibrary` constructor when possible. + +!!! compat "Julia 1.11" + `add_dependency!` was added in Julia 1.11. + +See also [`LazyLibrary`](@ref). +""" function add_dependency!(ll::LazyLibrary, dep::LazyLibrary) @lock ll.lock begin - push!(ll.dependencies, dep) + push!(ll.dependencies(), dep) end end # Register `jl_libdl_dlopen_func` so that `ccall()` lowering knows -# how to call `dlopen()`, during bootstrap. -# See `post_image_load_hooks` for non-bootstrapping. +# how to call `dlopen()`. Base.unsafe_store!(cglobal(:jl_libdl_dlopen_func, Any), dlopen) function dlopen(ll::LazyLibrary, flags::Integer = ll.flags; kwargs...) @@ -418,7 +467,7 @@ function dlopen(ll::LazyLibrary, flags::Integer = ll.flags; kwargs...) # Check to see if another thread has already run this if ll.handle == C_NULL # Ensure that all dependencies are loaded - for dep in ll.dependencies + for dep in ll.dependencies() dlopen(dep; kwargs...) end @@ -430,6 +479,9 @@ function dlopen(ll::LazyLibrary, flags::Integer = ll.flags; kwargs...) if ll.on_load_callback !== nothing ll.on_load_callback() end + else + # Another thread loaded the library while we were waiting + handle = @atomic :acquire ll.handle end end else diff --git a/base/linked_list.jl b/base/linked_list.jl index c477dc56bdb2b..fd56d092321e7 100644 --- a/base/linked_list.jl +++ b/base/linked_list.jl @@ -1,12 +1,21 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license mutable struct IntrusiveLinkedList{T} - # Invasive list requires that T have a field `.next >: U{T, Nothing}` and `.queue >: U{ILL{T}, Nothing}` + # Invasive list requires that T have a field `.next >: U{T, Nothing}` and `.queue::Any` head::Union{T, Nothing} tail::Union{T, Nothing} IntrusiveLinkedList{T}() where {T} = new{T}(nothing, nothing) end +struct ILLRef{T} + list::IntrusiveLinkedList{T} + waitee::Any # Invariant: waitqueue(waitee).list === list +end + +# waitqueue(x) returns the ILLRef for the queue of waiters registered on `x`. +# Methods are added for each waitee type (conditions, tasks, workqueues, ...). +function waitqueue end + #const list_append!! = append! #const list_deletefirst! = delete! @@ -49,9 +58,13 @@ function list_append!!(q::IntrusiveLinkedList{T}, q2::IntrusiveLinkedList{T}) wh return q end -function push!(q::IntrusiveLinkedList{T}, val::T) where T +isempty(qr::ILLRef) = isempty(qr.list) +length(qr::ILLRef) = length(qr.list) + +function push!(qr::ILLRef{T}, val::T) where T val.queue === nothing || error("val already in a list") - val.queue = q + val.queue = qr.waitee + q = qr.list tail = q.tail if tail === nothing q.head = q.tail = val @@ -62,9 +75,10 @@ function push!(q::IntrusiveLinkedList{T}, val::T) where T return q end -function pushfirst!(q::IntrusiveLinkedList{T}, val::T) where T +function pushfirst!(qr::ILLRef{T}, val::T) where T val.queue === nothing || error("val already in a list") - val.queue = q + val.queue = qr.waitee + q = qr.list head = q.head if head === nothing q.head = q.tail = val @@ -75,21 +89,34 @@ function pushfirst!(q::IntrusiveLinkedList{T}, val::T) where T return q end -function pop!(q::IntrusiveLinkedList{T}) where {T} - val = q.tail::T - list_deletefirst!(q, val) # expensive! +function pop!(qr::ILLRef{T}) where {T} + val = qr.list.tail::T + _list_deletefirst!(qr.list, val) # expensive! return val end -function popfirst!(q::IntrusiveLinkedList{T}) where {T} - val = q.head::T - list_deletefirst!(q, val) # cheap +function popfirst!(qr::ILLRef{T}) where {T} + val = qr.list.head::T + _list_deletefirst!(qr.list, val) # cheap return val end +# Delete `val` from the list, but only if it is actually in it, as witnessed by +# `val.queue` holding the ILLRef's waitee. This makes deletion a no-op if `val` +# was concurrently popped, which various cleanup paths rely upon. +function list_deletefirst!(qr::ILLRef{T}, val::T) where T + val.queue === qr.waitee || return qr.list + return _list_deletefirst!(qr.list, val) +end + +push!(q::IntrusiveLinkedList{T}, val::T) where T = push!(ILLRef(q, q), val) +pushfirst!(q::IntrusiveLinkedList{T}, val::T) where T = pushfirst!(ILLRef(q, q), val) +pop!(q::IntrusiveLinkedList{T}) where T = pop!(ILLRef(q, q)) +popfirst!(q::IntrusiveLinkedList{T}) where T = popfirst!(ILLRef(q, q)) +list_deletefirst!(q::IntrusiveLinkedList{T}, val::T) where T = list_deletefirst!(ILLRef(q, q), val) + # this function assumes `val` is found in `q` -function list_deletefirst!(q::IntrusiveLinkedList{T}, val::T) where T - val.queue === q || return +function _list_deletefirst!(q::IntrusiveLinkedList{T}, val::T) where T head = q.head::T if head === val if q.tail::T === val @@ -150,3 +177,117 @@ function list_deletefirst!(q::LinkedList{T}, val::T) where T end return q end + +## Wait-entry lists +# +# Wait entries carry uniform {owner, next, aux} slots (see cancellation.jl) +# and can be registered on several waitables at once (wait-any), so the +# list operations locate each entry's slot for *this* list through the +# ILLRef's waitee identity - every list is populated under exactly one +# identity (the ILLRef invariant above). The generic `.next`/`.queue` +# methods above keep serving Task scheduler lists. + +function push!(qr::ILLRef{WaitEntry}, val::WaitEntry) + _find_slot(val, qr.waitee) == 0 || error("val already in this list") + _acquire_slot!(val, qr.waitee) + q = qr.list + tail = q.tail + if tail === nothing + q.head = q.tail = val + else + tail = tail::WaitEntry + _set_slot_next!(tail, _find_slot(tail, qr.waitee), val) + q.tail = val + end + return q +end + +function pushfirst!(qr::ILLRef{WaitEntry}, val::WaitEntry) + _find_slot(val, qr.waitee) == 0 || error("val already in this list") + i = _acquire_slot!(val, qr.waitee) + q = qr.list + head = q.head + if head === nothing + q.head = q.tail = val + else + _set_slot_next!(val, i, head) + q.head = val + end + return q +end + +function popfirst!(qr::ILLRef{WaitEntry}) + q = qr.list + val = q.head::WaitEntry + i = _find_slot(val, qr.waitee) + vnext = _slot_next(val, i) + q.head = vnext + vnext === nothing && (q.tail = nothing) + _release_slot!(val, i) + return val +end + +function pop!(qr::ILLRef{WaitEntry}) + val = qr.list.tail::WaitEntry + list_deletefirst!(qr, val) # expensive! + return val +end + +# Delete `val` from the list, but only if it is actually in it, as witnessed +# by its slot for this list's identity. This makes deletion a no-op if `val` +# was concurrently popped, which various cleanup paths rely upon. +function list_deletefirst!(qr::ILLRef{WaitEntry}, val::WaitEntry) + vi = _find_slot(val, qr.waitee) + vi == 0 && return qr.list + q = qr.list + o = qr.waitee + head = q.head + if head === val + vnext = _slot_next(val, vi) + q.head = vnext + vnext === nothing && (q.tail = nothing) + else + head === nothing && return q + prev = head::WaitEntry + while true + previ = _find_slot(prev, o) + previ == 0 && return q + prevslot = slots(prev)[previ] + cur = prevslot.next + cur === nothing && return q + cur = cur::WaitEntry + if cur === val + vnext = _slot_next(val, vi) + prevslot.next = vnext + vnext === nothing && (q.tail = prev) + break + end + prev = cur + end + end + _release_slot!(val, vi) + return q +end + +function length(qr::ILLRef{WaitEntry}) + n = 0 + w = qr.list.head + while w !== nothing + n += 1 + w = _next_on(w::WaitEntry, qr.waitee) + end + return n +end + +# Whether any entry on the list is still armed (the emptiness that matters +# to waiters-pending queries; claimed corpses linger until popped). +function _waitq_isempty(qr::ILLRef{WaitEntry}) + w = qr.list.head + while w !== nothing + w = w::WaitEntry + t = @atomic :monotonic w.task + t isa Task && (@atomic t.waiting_on) === w && return false + w = _next_on(w, qr.waitee) + end + return true +end diff --git a/base/linking.jl b/base/linking.jl index edb70e8061119..dda8442515f28 100644 --- a/base/linking.jl +++ b/base/linking.jl @@ -91,16 +91,25 @@ end function ld() default_args = `` @static if Sys.iswindows() - # LLD supports mingw style linking + # From`x86_64-w64-mingw32-gcc -shared -Wl,--verbose` flavor = "gnu" m = Sys.ARCH == :x86_64 ? "i386pep" : "i386pe" - default_args = `-m $m -Bdynamic --enable-auto-image-base --allow-multiple-definition --disable-auto-import --disable-runtime-pseudo-reloc` + entry = Sys.ARCH == :x86_64 ? "DllMainCRTStartup" : "_DllMainCRTStartup" + default_args = `-m $m -Bdynamic -e $entry --enable-auto-image-base --allow-multiple-definition --disable-auto-import --disable-runtime-pseudo-reloc` elseif Sys.isapple() flavor = "darwin" arch = Sys.ARCH == :aarch64 ? :arm64 : Sys.ARCH default_args = `-arch $arch -undefined dynamic_lookup -platform_version macos $(Base.MACOS_PRODUCT_VERSION) $(Base.MACOS_PLATFORM_VERSION)` + # due to an lld bug: https://github.com/llvm/llvm-project/issues/193646 + # we must make sure the syslibroot does not point to the system or else + # it will not respect the provided `libSystem.tbd` file + default_args = `$default_args -syslibroot $(private_libdir())` else flavor = "gnu" + # From `gcc -shared -Wl,--verbose`; `-z defs` added to enforce that all symbols + # the pkgimage references are resolvable at link time (catches regressions early + # instead of deferring to first-call crashes at runtime). + default_args = `--build-id --eh-frame-hdr --hash-style=gnu --as-needed -z relro -z defs` end `$(lld()) -flavor $flavor $default_args` @@ -118,6 +127,17 @@ else "--no-whole-archive" end +# Prefer whole_archive to WHOLE_ARCHIVE +whole_archive(paths::String; is_cc=false) = whole_archive([paths]; is_cc) +function whole_archive(paths::Vector{String}; is_cc=false) + cc_arg(a) = is_cc ? "-Wl,$a" : a + if Sys.isapple() + Cmd(collect(Iterators.flatmap(p -> (cc_arg("-force_load"), p), paths))) + else + `$(cc_arg("--whole-archive")) $paths $(cc_arg("--no-whole-archive"))` + end +end + const SHARED = if Sys.isapple() "-dylib" else @@ -134,25 +154,64 @@ end verbose_linking() = something(Base.get_bool_env("JULIA_VERBOSE_LINKING", false), false) +function _find_static(lib) + if isfile(joinpath(private_libdir(), lib)) + return joinpath(private_libdir(), lib) + else + return joinpath(libdir(), lib) + end +end + +function _find_loaded(re::Regex) + for p in Libdl.dllist() + occursin(re, p) && return p + end + error("no loaded shared object matching $re") +end + function link_image_cmd(path, out) PRIVATE_LIBDIR = "-L$(private_libdir())" + LIBDIR = "-L$(libdir())" SHLIBDIR = "-L$(shlibdir())" - LIBS = isdebugbuild() ? ("-ljulia-debug", "-ljulia-internal-debug") : - ("-ljulia", "-ljulia-internal") + LIBS = String[] + if isdebugbuild() + push!(LIBS, "-ljulia-debug") + push!(LIBS, "-ljulia-internal-debug") + else + push!(LIBS, "-ljulia") + push!(LIBS, "-ljulia-internal") + end + crtbegin = String[] + crtend = String[] @static if Sys.iswindows() - LIBS = (LIBS..., "-lopenlibm", "-lgcc_s", "-lgcc", "-lmsvcrt") - if isdebugbuild() - LIBS = (LIBS..., "-lssp") - if isfile(joinpath(private_libdir(), "libmingwex.a")) - # In MinGW 11, the ssp implementation was moved from libssp to - # libmingwex with ssp only being a stub. See #59020. - LIBS = (LIBS..., "-lmingwex", "-lkernel32") - end - end + # From `x86_64-w64-mingw32-gcc -shared -Wl,--verbose`. + append!(LIBS, String["-lopenlibm"]) + # libmsvcrt-os.a contains MinGW CRT objects that can refer back to + # libmingw32.a/libmingwex.a; keep the selected CRT last. + append!(LIBS, String["-lmingw32", "-lgcc_s", "-lgcc", "-lmoldname", "-lmingwex", "-lmsvcrt-os", "-lmingw32", "-lmingwex", "-lmsvcrt-os", "-lkernel32"]) + append!(LIBS, String["-lpthread", "-ladvapi32", "-lshell32", "-luser32"]) + append!(crtbegin, String[_find_static("dllcrt2.o"), _find_static("crtbegin.o")]) + append!(crtend, String[_find_static("crtend.o")]) + isdebugbuild() && push!(LIBS, "-lssp") + append!(LIBS, String["-lmingwex", "-lmsvcrt-os", "-lmingw32", "-lmingwex", "-lmsvcrt-os"]) + elseif Sys.isapple() + # From `clang -dynamiclib -Wl,-v` + append!(LIBS, String[_find_static("libclang_rt.osx.a"), _find_static("libSystem.tbd")]) + else + # From `gcc -shared -Wl,--verbose` + # but without repeated libraries (lld auto-resolves circular library references) + libc = _find_loaded(r"/libc\.so\.\d+$") # system libc + ld_linux = _find_loaded(r"/ld-(?:linux|musl|elf)[^/]*\.so\.\d+$") # system ld + libc_nonshared = _find_static("libc_nonshared.a") + append!(LIBS, String["-lgcc", "--as-needed", "-lgcc_s", "-latomic", "-lopenlibm", "--no-as-needed", libc]) + isfile(libc_nonshared) && push!(LIBS, libc_nonshared) + append!(LIBS, String["--as-needed", ld_linux, "--no-as-needed"]) + append!(crtbegin, String[_find_static("crti.o"), _find_static("crtbeginS.o")]) + append!(crtend, String[_find_static("crtendS.o"), _find_static("crtn.o")]) end V = verbose_linking() ? "--verbose" : "" - `$(ld()) $V $SHARED -o $out $WHOLE_ARCHIVE $path $NO_WHOLE_ARCHIVE $PRIVATE_LIBDIR $SHLIBDIR $LIBS` + `$(ld()) $V $SHARED -o $out $crtbegin $(whole_archive(path)) $PRIVATE_LIBDIR $LIBDIR $SHLIBDIR $LIBS $crtend` end function link_image(path, out, internal_stderr::IO=stderr, internal_stdout::IO=stdout) diff --git a/base/loading.jl b/base/loading.jl index 33799805e0601..5faf8f6ff8c31 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -110,36 +110,39 @@ function isaccessiblepath(path) end end -## SHA1 ## - -struct SHA1 - bytes::NTuple{20, UInt8} -end -function SHA1(bytes::Vector{UInt8}) - length(bytes) == 20 || - throw(ArgumentError("wrong number of bytes for SHA1 hash: $(length(bytes))")) - return SHA1(ntuple(i->bytes[i], Val(20))) -end -SHA1(s::AbstractString) = SHA1(hex2bytes(s)) -parse(::Type{SHA1}, s::AbstractString) = SHA1(s) -function tryparse(::Type{SHA1}, s::AbstractString) - try - return parse(SHA1, s) - catch e - if isa(e, ArgumentError) - return nothing +## SHA1 and SHA256 ## +for (name, namestr, numbytes) in [(:SHA1, "SHA1", 20), (:SHA256, "SHA256", 32)] + @eval begin + struct $name + bytes::NTuple{$numbytes, UInt8} + end + function $name(bytes::Vector{UInt8}) + length(bytes) == $numbytes || + throw(ArgumentError("wrong number of bytes for " * string($namestr) * ": Expected " * string($numbytes) * " bytes, got $(length(bytes))")) + return $name(ntuple(i->bytes[i], Val($numbytes))) + end + $name(s::AbstractString) = $name(hex2bytes(s)) + parse(::Type{$name}, s::AbstractString) = $name(s) + function tryparse(::Type{$name}, s::AbstractString) + try + return parse($name, s) + catch e + if isa(e, ArgumentError) + return nothing + end + rethrow(e) + end end - rethrow(e) - end -end -string(hash::SHA1) = bytes2hex(hash.bytes) -print(io::IO, hash::SHA1) = bytes2hex(io, hash.bytes) -show(io::IO, hash::SHA1) = print(io, "SHA1(\"", hash, "\")") + string(hash::$name) = bytes2hex(hash.bytes) + print(io::IO, hash::$name) = bytes2hex(io, hash.bytes) + show(io::IO, hash::$name) = print(io, $namestr * "(\"", hash, "\")") -isless(a::SHA1, b::SHA1) = isless(a.bytes, b.bytes) -hash(a::SHA1, h::UInt) = hash((SHA1, a.bytes), h) -==(a::SHA1, b::SHA1) = a.bytes == b.bytes + isless(a::$name, b::$name) = isless(a.bytes, b.bytes) + hash(a::$name, h::UInt) = hash(($name, a.bytes), h) + ==(a::$name, b::$name) = a.bytes == b.bytes + end +end # fake uuid5 function (for self-assigned UUIDs) # TODO: delete and use real uuid5 once it's in stdlib @@ -179,14 +182,14 @@ function dummy_uuid(project_file::String) end end -## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ## +## package path slugs: turning UUID + SHA1 into a pair of 5-byte "slugs" ## const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9']) function slug(x::UInt32, p::Int) sprint(sizehint=p) do io y = x - n = length(slug_chars) + n = UInt32(length(slug_chars)) for i = 1:p y, d = divrem(y, n) write(io, slug_chars[1+d]) @@ -250,6 +253,17 @@ function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict) return f.d end +""" + struct PkgLoadSpec + +A PkgLoadSpec is the result of a `locate_package` operation and specifies how +and wherefrom to load a julia package. +""" +struct PkgLoadSpec + path::String + julia_syntax_version::VersionNumber +end + struct LoadingCache load_path::Vector{String} dummy_uuid::Dict{String, UUID} @@ -258,10 +272,19 @@ struct LoadingCache require_parsed::Set{String} identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, String}}} identified::Dict{String, Union{Nothing, Tuple{PkgId, String}}} - located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{String, String}, Nothing}} -end -const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) -LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set(), Dict(), Dict(), Dict()) + located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{PkgLoadSpec, String}, Nothing}} +end +const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) # n.b.: all access to and through this are protected by require_lock +LoadingCache() = LoadingCache( + load_path(), + Dict{String, UUID}(), + Dict{String, Union{Bool, String}}(), + Dict{String, Union{Nothing, String}}(), + Set{String}(), + Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, String}}}(), + Dict{String, Union{Nothing, Tuple{PkgId, String}}}(), + Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{PkgLoadSpec, String}, Nothing}}() +) struct TOMLCache{Dates} @@ -302,10 +325,12 @@ end # Used by Pkg but not used in loading itself function find_package(arg) # ::Union{Nothing,String} + @lock require_lock begin pkgenv = identify_package_env(arg) pkgenv === nothing && return nothing pkg, env = pkgenv return locate_package(pkg, env) + end end # is there a better/faster ground truth? @@ -332,56 +357,78 @@ is also returned, except when the identity is not identified. """ identify_package_env(where::Module, name::String) = identify_package_env(PkgId(where), name) function identify_package_env(where::PkgId, name::String) + # Special cases + if where.name === name + # Project tries to load itself + return (where, nothing) + elseif where.uuid === nothing + # Project without Project.toml - treat as toplevel load + return identify_package_env(nothing, name) + end + + # Check if we have a cached answer for this + assert_havelock(require_lock) cache = LOADING_CACHE[] + cache_key = (where, name) if cache !== nothing - pkg_env = get(cache.identified_where, (where, name), missing) + pkg_env = get(cache.identified_where, cache_key, missing) pkg_env === missing || return pkg_env end + + # Main part: Search through all environments in the load path to see if we have + # a matching entry. pkg_env = nothing - if where.name === name - return (where, nothing) - elseif where.uuid === nothing - pkg_env = identify_package_env(name) # ignore `where` - else - for env in load_path() - pkgid = manifest_deps_get(env, where, name) - pkgid === nothing && continue # not found--keep looking - if pkgid.uuid !== nothing - pkg_env = pkgid, env # found in explicit environment--use it - end - break # found in implicit environment--return "not found" - end - if pkg_env === nothing && is_stdlib(where) - # if not found it could be that manifests are from a different julia version/commit - # where stdlib dependencies have changed, so look up deps based on the stdlib Project.toml - # as a fallback - pkg_env = identify_stdlib_project_dep(where, name) - end + for env in load_path() + pkgid = environment_deps_get(env, where, name) + # If we didn't find `where` at all, keep looking through the environment stack + pkgid === nothing && continue + if pkgid.uuid !== nothing + pkg_env = (pkgid, env) + end + # If we don't have pkgid.uuid, still break here - this is a sentinel that indicates + # that we've found `where` but it did not have the required dependency. We terminate the search. + break end + if pkg_env === nothing && is_stdlib(where) + # if not found it could be that manifests are from a different julia version/commit + # where stdlib dependencies have changed, so look up deps based on the stdlib Project.toml + # as a fallback + pkg_env = identify_stdlib_project_dep(where, name) + end + + # Cache the result if cache !== nothing - cache.identified_where[(where, name)] = pkg_env + cache.identified_where[cache_key] = pkg_env end return pkg_env end -function identify_package_env(name::String) +function identify_package_env(where::Nothing, name::String) + # Check if we have a cached answer for this + assert_havelock(require_lock) cache = LOADING_CACHE[] if cache !== nothing pkg_env = get(cache.identified, name, missing) pkg_env === missing || return pkg_env end + + # Main part: Search through all environments in the load path to see if we have + # a matching entry. pkg_env = nothing for env in load_path() - pkg = project_deps_get(env, name) - if pkg !== nothing - pkg_env = pkg, env # found--return it - break - end + pkgid = environment_deps_get(env, nothing, name) + # If we didn't find `where` at all, keep looking through the environment stack + pkgid === nothing && continue + pkg_env = (pkgid, env) + break end + + # Cache the result if cache !== nothing cache.identified[name] = pkg_env end return pkg_env end +identify_package_env(name::String) = identify_package_env(nothing, name) function identify_stdlib_project_dep(stdlib::PkgId, depname::String) @debug """ @@ -426,71 +473,67 @@ julia> using LinearAlgebra julia> Base.identify_package(LinearAlgebra, "Pkg") # Pkg is not a dependency of LinearAlgebra ``` """ -identify_package(where::Module, name::String) = _nothing_or_first(identify_package_env(where, name)) -identify_package(where::PkgId, name::String) = _nothing_or_first(identify_package_env(where, name)) -identify_package(name::String) = _nothing_or_first(identify_package_env(name)) +identify_package(where::Module, name::String) = @lock require_lock _nothing_or_first(identify_package_env(where, name)) +identify_package(where::PkgId, name::String) = @lock require_lock _nothing_or_first(identify_package_env(where, name)) +identify_package(name::String) = @lock require_lock _nothing_or_first(identify_package_env(name)) -function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,Tuple{String,String}} +function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,Tuple{PkgLoadSpec, String}} + assert_havelock(require_lock) cache = LOADING_CACHE[] if cache !== nothing - pathenv = get(cache.located, (pkg, stopenv), missing) - pathenv === missing || return pathenv - end - path = nothing - env′ = nothing - if pkg.uuid === nothing - for env in load_path() - # look for the toplevel pkg `pkg.name` in this entry - found = project_deps_get(env, pkg.name) - if found !== nothing - @assert found.name == pkg.name - if found.uuid === nothing - # pkg.name is present in this directory or project file, - # return the path the entry point for the code, if it could be found - # otherwise, signal failure - path = implicit_manifest_uuid_path(env, pkg) - env′ = env - @goto done + specenv = get(cache.located, (pkg, stopenv), missing) + specenv === missing || return specenv + end + (env′, spec) = @label found begin + if pkg.uuid === nothing + # The project we're looking for does not have a Project.toml (n.b. - present + # `Project.toml` without UUID gets a path-based dummy UUID). It must have + # come from an implicit manifest environment, so go through those only. + # N.B.: Implicitly loaded packages do not participate in syntax versioning. + for env in load_path() + project_file = env_project_file(env) + (project_file isa Bool && project_file) || continue + found = implicit_manifest_pkgid(env, pkg.name) + if found !== nothing && found.uuid === nothing + @assert found.name == pkg.name + break found (env, implicit_manifest_uuid_load_spec(env, pkg)) + end + if !(loading_extension || precompiling_extension) + stopenv == env && break found (nothing, nothing) end end - if !(loading_extension || precompiling_extension) - stopenv == env && @goto done - end - end - else - for env in load_path() - path = manifest_uuid_path(env, pkg) - # missing is used as a sentinel to stop looking further down in envs - if path === missing - path = nothing - @goto done - end - if path !== nothing - env′ = env - @goto done + else + for env in load_path() + spec = manifest_uuid_load_spec(env, pkg) + # missing is used as a sentinel to stop looking further down in envs + if spec === missing + is_stdlib(pkg) && break + break found (nothing, nothing) + end + if spec !== nothing + break found (env, spec) + end + if !(loading_extension || precompiling_extension) + stopenv == env && break + end end - if !(loading_extension || precompiling_extension) - stopenv == env && break + # Allow loading of stdlibs if the name/uuid are given + # e.g. if they have been explicitly added to the project/manifest + mbyspec = manifest_uuid_load_spec(Sys.STDLIB, pkg) + if mbyspec isa PkgLoadSpec + break found (Sys.STDLIB, mbyspec) end end - # Allow loading of stdlibs if the name/uuid are given - # e.g. if they have been explicitly added to the project/manifest - mbypath = manifest_uuid_path(Sys.STDLIB, pkg) - if mbypath isa String - path = mbypath - env′ = Sys.STDLIB - @goto done - end + (nothing, nothing) end - @label done - if path !== nothing && !isfile_casesensitive(path) - path = nothing + if spec !== nothing && !isfile_casesensitive(spec.path) + spec = nothing end if cache !== nothing - cache.located[(pkg, stopenv)] = path === nothing ? nothing : (path, something(env′)) + cache.located[(pkg, stopenv)] = spec === nothing ? nothing : (spec, something(env′)) end - path === nothing && return nothing - return path, something(env′) + spec === nothing && return nothing + return spec, something(env′) end """ @@ -508,7 +551,19 @@ julia> Base.locate_package(pkg) ``` """ function locate_package(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,String} - _nothing_or_first(locate_package_env(pkg, stopenv)) + @lock require_lock begin + specenv = locate_package_env(pkg, stopenv) + specenv === nothing && return nothing + specenv[1].path + end +end + +function locate_package_load_spec(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,PkgLoadSpec} + @lock require_lock begin + specenv = locate_package_env(pkg, stopenv) + specenv === nothing && return nothing + specenv[1] + end end """ @@ -593,9 +648,8 @@ end """ pkgversion(m::Module) -Return the version of the package that imported module `m`, -or `nothing` if `m` was not imported from a package, or imported -from a package without a version field set. +If the module `m` belongs to a versioned package, return the +version number of that package. Otherwise return `nothing`. The version is read from the package's Project.toml during package load. @@ -607,13 +661,15 @@ the form `pkgversion(@__MODULE__)` can be used. This function was introduced in Julia 1.9. """ function pkgversion(m::Module) - path = pkgdir(m) - path === nothing && return nothing @lock require_lock begin - v = get_pkgversion_from_path(path) pkgorigin = get(pkgorigins, PkgId(moduleroot(m)), nothing) - # Cache the version - if pkgorigin !== nothing && pkgorigin.version === nothing + if pkgorigin !== nothing && pkgorigin.version !== nothing + return pkgorigin.version + end + path = pkgdir(m) + path === nothing && return nothing + v = get_pkgversion_from_path(path) + if pkgorigin !== nothing pkgorigin.version = v end return v @@ -667,57 +723,74 @@ function env_project_file(env::String)::Union{Bool,String} end function base_project(project_file) - base_dir = abspath(joinpath(dirname(project_file), "..")) - base_project_file = env_project_file(base_dir) - base_project_file isa String || return nothing - d = parsed_toml(base_project_file) - workspace = get(d, "workspace", nothing)::Union{Dict{String, Any}, Nothing} - if workspace === nothing - return nothing - end - projects = get(workspace, "projects", nothing)::Union{Vector{String}, Nothing, String} - projects === nothing && return nothing - if projects isa Vector && basename(dirname(project_file)) in projects - return base_project_file + home_dir = abspath(homedir()) + project_dir = abspath(dirname(project_file)) + current_dir = project_dir + # Only stop at home boundary if we started under home + started_in_home = startswith(project_dir, home_dir) + + while true + parent_dir = dirname(current_dir) + # Stop if we've reached root + if parent_dir == current_dir + return nothing + end + # Stop if we started in home and have now left it + if started_in_home && !startswith(parent_dir, home_dir) + return nothing + end + + base_project_file = env_project_file(parent_dir) + if base_project_file isa String + d = parsed_toml(base_project_file) + workspace = get(d, "workspace", nothing)::Union{Dict{String, Any}, Nothing} + if workspace !== nothing + projects = get(workspace, "projects", nothing)::Union{Vector{String}, Nothing, String} + if projects isa Vector + # Check if any project in the workspace matches the original project + workspace_root = dirname(base_project_file) + for project in projects + project_path = joinpath(workspace_root, project) + if isdir(project_path) + if samefile(project_path, project_dir) + return base_project_file + end + end + end + end + end + end + current_dir = parent_dir end - return nothing end -function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} - project_file = env_project_file(env) - if project_file isa String - pkg_uuid = explicit_project_deps_get(project_file, name) - pkg_uuid === nothing || return PkgId(pkg_uuid, name) - elseif project_file - return implicit_project_deps_get(env, name) - end - return nothing +function package_get_here(project_file, name::String) + # if `where` matches the project, use [deps] section as manifest, and stop searching + pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid === nothing && return PkgId(name) + return PkgId(pkg_uuid, name) end -function package_get(project_file, where::PkgId, name::String) - proj = project_file_name_uuid(project_file, where.name) - if proj == where - # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name) - return PkgId(pkg_uuid, name) +function package_get(project_file, where::Union{Nothing, PkgId}, name::String) + if where !== nothing + proj = project_file_name_uuid(project_file, where.name) + proj != where && return nothing end - return nothing + return package_get_here(project_file, name) end -function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} - uuid = where.uuid - @assert uuid !== nothing - project_file = env_project_file(env) - if project_file isa String - pkg = package_get(project_file, where, name) - pkg === nothing || return pkg - d = parsed_toml(project_file) - exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} - if exts !== nothing - proj = project_file_name_uuid(project_file, where.name) - # Check if `where` is an extension of the project - if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name) - # Extensions can load weak deps... +ext_may_load_weakdep(exts::String, name::String) = exts == name +ext_may_load_weakdep(exts::Vector{String}, name::String) = name in exts + +function package_extension_get(project_file, where::PkgId, name::String) + d = parsed_toml(project_file) + exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} + if exts !== nothing + proj = project_file_name_uuid(project_file, where.name) + # Check if `where` is an extension of the project + if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name) + # Extensions can load weak deps if they are an extension trigger + if ext_may_load_weakdep(exts[where.name]::Union{String, Vector{String}}, name) weakdeps = get(d, "weakdeps", nothing)::Union{Dict{String, Any}, Nothing} if weakdeps !== nothing wuuid = get(weakdeps, name, nothing)::Union{String, Nothing} @@ -725,37 +798,94 @@ function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothi return PkgId(UUID(wuuid), name) end end - # ... and they can load same deps as the project itself - mby_uuid = explicit_project_deps_get(project_file, name) - mby_uuid === nothing || return PkgId(mby_uuid, name) end + # ... and they can load same deps as the project itself + return package_get_here(project_file, name) end - # look for manifest file and `where` stanza - return explicit_manifest_deps_get(project_file, where, name) - elseif project_file - # if env names a directory, search it - return implicit_manifest_deps_get(env, where, name) end return nothing end -function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missing} +function environment_deps_get(env::String, where::Union{Nothing,PkgId}, name::String)::Union{Nothing,PkgId} + @assert where === nothing || where.uuid !== nothing + project_file = env_project_file(env) + implicit_manifest = !(project_file isa String) + if implicit_manifest + project_file || return nothing + if where === nothing + # Toplevel load with a directory (implicit manifest) - all we look for is the + # existence of the package name in the directory. + pkg = implicit_manifest_pkgid(env, name) + return pkg + end + project_file = implicit_manifest_project(env, where) + project_file === nothing && return nothing + end + + # Are we + # a) loading into a top-level project itself + # b) loading into a non-top-level project that was part of an implicit + # manifest environment (and for which we found the project file above) + # c) performing a top-level load (where === nothing) - i.e. we're looking + # at an environment's project file. + # + # If so, we may load either: + # I: the project itself (if name matches where) + # II: a dependency from [deps] section of the project file + # + # N.B.: Here "top-level" includes package loaded from an implicit manifest, which + # uses the same code path. Otherwise this is the active project. + pkg = package_get(project_file, where, name) + if pkg !== nothing + if where === nothing && pkg.uuid === nothing + # This is a top-level load - even though we didn't find the dependency + # here, we still want to keep looking through the top-level environment stack. + return nothing + end + return pkg + end + + @assert where !== nothing + + # Are we an extension of a project from cases a), b) above + # If so, in addition to I, II above, we get: + # III: A dependency from [weakdeps] section of the project file as long + # as it is an extension trigger for `where` in the `extensions` section. + pkg = package_extension_get(project_file, where, name) + pkg === nothing || return pkg + + if implicit_manifest + # With an implicit manifest, getting here means that our (implicit) environment + # *has* the package `where`. If we don't find it, it just means that `where` doesn't + # have `name` as a dependency - c.f. the analogous case in `explicit_manifest_deps_get`. + return PkgId(name) + end + + # All other cases, dependencies come from the (top-level) manifest + return explicit_manifest_deps_get(project_file, where, name) +end + +function manifest_uuid_load_spec(env::String, pkg::PkgId)::Union{Nothing,PkgLoadSpec,Missing} project_file = env_project_file(env) if project_file isa String proj = project_file_name_uuid(project_file, pkg.name) if proj == pkg # if `pkg` matches the project, return the project itself - return project_file_path(project_file, pkg.name) + return project_file_load_spec(project_file, pkg.name) end - mby_ext = project_file_ext_path(project_file, pkg) + mby_ext = project_file_ext_load_spec(project_file, pkg) mby_ext === nothing || return mby_ext # look for manifest file and `where` stanza - return explicit_manifest_uuid_path(project_file, pkg) + return explicit_manifest_uuid_load_spec(project_file, pkg) elseif project_file # if env names a directory, search it - proj = implicit_manifest_uuid_path(env, pkg) + # Implicit environments do not participate in syntax versioning + proj = implicit_manifest_uuid_load_spec(env, pkg) proj === nothing || return proj - # if not found + # if not found, this might be an extension - first we fast path needing + # to scan the whole directory for a matching extension by peeking at + # EXT_PRIMED. However, this only works if the parent package was loaded. + # This is usually the case, but not always, e.g. in precompilation. triggers = get(EXT_PRIMED, pkg, nothing) if triggers !== nothing parentid = triggers[1] @@ -763,10 +893,14 @@ function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missi if parent_project_file !== nothing parentproj = project_file_name_uuid(parent_project_file, parentid.name) if parentproj == parentid - mby_ext = project_file_ext_path(parent_project_file, pkg) + mby_ext = project_file_ext_load_spec(parent_project_file, pkg) mby_ext === nothing || return mby_ext end end + else + # We still need to scan the whole directory for extensions. + ext_ls, ext_proj = implicit_env_project_file_extension(env, pkg) + ext_ls === nothing || return ext_ls end end return nothing @@ -779,13 +913,14 @@ function find_ext_path(project_path::String, extname::String) return joinpath(project_path, "ext", extname * ".jl") end -function project_file_ext_path(project_file::String, ext::PkgId) +function project_file_ext_load_spec(project_file::String, ext::PkgId) d = parsed_toml(project_file) p = dirname(project_file) exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} if exts !== nothing if ext.name in keys(exts) && ext.uuid == uuid5(UUID(d["uuid"]::String), ext.name) - return find_ext_path(p, ext.name) + # Syntax version of the main package applies to its extensions + return PkgLoadSpec(find_ext_path(p, ext.name), project_get_syntax_version(d)) end end return nothing @@ -800,14 +935,47 @@ function project_file_name_uuid(project_file::String, name::String)::PkgId return PkgId(uuid, name) end -function project_file_path(project_file::String, name::String) +const NON_VERSIONED_SYNTAX = v"1.13" + +function project_get_syntax_version(d::Dict) + # Syntax Evolution. First check syntax.julia_version entry + sv = nothing + ds = get(d, "syntax", nothing) + if ds !== nothing + sv = VersionNumber(get(ds, "julia_version", nothing)) + end + # If not found, default to minimum(compat["julia"]) + if sv === nothing + cs = get(d, "compat", nothing) + if cs !== nothing + jv = get(cs, "julia", nothing) + if jv !== nothing + sv = VersionNumber(minimum(semver_spec(jv)).t...) + end + end + end + # Finally, if neither of those are set, default to the current Julia version. + # N.B.: This choice is less "compatible" than defaulting to a fixed older version. + # However, it avoids surprises from moving over scripts and REPL code to packages + if sv === nothing + sv = VERSION + elseif sv <= NON_VERSIONED_SYNTAX + # Syntax versioning was first introduced in Julia 1.14 - we do not support + # going back to versions before syntax version 1.13. + sv = NON_VERSIONED_SYNTAX + end + return sv +end + +function project_file_load_spec(project_file::String, name::String) d = parsed_toml(project_file) entryfile = get(d, "path", nothing)::Union{String, Nothing} # "path" entry in project file is soft deprecated if entryfile === nothing entryfile = get(d, "entryfile", nothing)::Union{String, Nothing} end - return entry_path(dirname(project_file), name, entryfile) + sv = project_get_syntax_version(d) + return PkgLoadSpec(entry_path(dirname(project_file), name, entryfile), sv) end function workspace_manifest(project_file) @@ -818,6 +986,33 @@ function workspace_manifest(project_file) return nothing end +struct VersionedParse + ver::VersionNumber +end + +function (vp::VersionedParse)(code, filename::String, lineno::Int, offset::Int, options::Symbol) + pm = parentmodule(Core._parse) + # hack to support old copies of JuliaSyntax + if !isdefined(pm, :_has_v1_14_version_hooks) && isdefined(pm, :_has_v1_10_hooks) + invokelatest(Core._parse, code, filename, lineno, offset, options) + else + invokelatest(Core._parse, code, filename, lineno, offset, options, vp.ver) + end +end + +function parser_for_active_project() + project = active_project() + sv = VERSION + if project !== nothing && isfile(project) + try + sv = project_get_syntax_version(parsed_toml(project)) + catch e + @warn "Failed to read project $project - defaulting to latest syntax. err=$e" + end + end + VersionedParse(sv) +end + # find project file's corresponding manifest file function project_file_manifest_path(project_file::String)::Union{Nothing,String} @lock require_lock begin @@ -827,6 +1022,7 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String} manifest_path === missing || return manifest_path end dir = abspath(dirname(project_file)) + isfile_casesensitive(project_file) || return nothing d = parsed_toml(project_file) base_manifest = workspace_manifest(project_file) if base_manifest !== nothing @@ -889,9 +1085,9 @@ function implicit_env_project_file_extension(dir::String, ext::PkgId) for pkg in readdir(dir; join=true) project_file = env_project_file(pkg) project_file isa String || continue - path = project_file_ext_path(project_file, ext) - if path !== nothing - return path, project_file + ls = project_file_ext_load_spec(project_file, ext) + if ls !== nothing + return ls, project_file end end return nothing, nothing @@ -909,7 +1105,7 @@ end # find project file root or deps `name => uuid` mapping # `ext` is the name of the extension if `name` is loaded from one # return `nothing` if `name` is not found -function explicit_project_deps_get(project_file::String, name::String, ext::Union{String,Nothing}=nothing)::Union{Nothing,UUID} +function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} d = parsed_toml(project_file) if get(d, "name", nothing)::Union{String, Nothing} === name root_uuid = dummy_uuid(project_file) @@ -921,19 +1117,6 @@ function explicit_project_deps_get(project_file::String, name::String, ext::Unio uuid = get(deps, name, nothing)::Union{String, Nothing} uuid === nothing || return UUID(uuid) end - if ext !== nothing - extensions = get(d, "extensions", nothing) - extensions === nothing && return nothing - ext_data = get(extensions, ext, nothing) - ext_data === nothing && return nothing - if (ext_data isa String && name == ext_data) || (ext_data isa Vector{String} && name in ext_data) - weakdeps = get(d, "weakdeps", nothing)::Union{Dict{String, Any}, Nothing} - weakdeps === nothing && return nothing - wuuid = get(weakdeps, name, nothing)::Union{String, Nothing} - wuuid === nothing && return nothing - return UUID(wuuid) - end - end return nothing end @@ -960,14 +1143,27 @@ function get_deps(raw_manifest::Dict) end end -# find `where` stanza and return the PkgId for `name` -# return `nothing` if it did not find `where` (indicating caller should continue searching) +function dep_stanza_get(stanza::Dict{String, Any}, name::String)::Union{Nothing, PkgId} + for (dep, uuid) in stanza + uuid::String + if dep === name + return PkgId(UUID(uuid), name) + end + end + return nothing +end + +function dep_stanza_get(stanza::Vector{String}, name::String)::Union{Nothing, PkgId} + name in stanza && return PkgId(name) + return nothing +end + +dep_stanza_get(stanza::Nothing, name::String) = nothing + function explicit_manifest_deps_get(project_file::String, where::PkgId, name::String)::Union{Nothing,PkgId} manifest_file = project_file_manifest_path(project_file) manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH d = get_deps(parsed_toml(manifest_file)) - found_where = false - found_name = false for (dep_name, entries) in d entries::Vector{Any} for entry in entries @@ -977,71 +1173,67 @@ function explicit_manifest_deps_get(project_file::String, where::PkgId, name::St # deps is either a list of names (deps = ["DepA", "DepB"]) or # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."} deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing} - if UUID(uuid) === where.uuid - found_where = true - if deps isa Vector{String} - found_name = name in deps - found_name && @goto done - elseif deps isa Dict{String, Any} - deps = deps::Dict{String, Any} - for (dep, uuid) in deps - uuid::String - if dep === name - return PkgId(UUID(uuid), name) - end - end - end - else # Check for extensions - extensions = get(entry, "extensions", nothing) - if extensions !== nothing - if haskey(extensions, where.name) && where.uuid == uuid5(UUID(uuid), where.name) - found_where = true - if name == dep_name - return PkgId(UUID(uuid), name) - end - exts = extensions[where.name]::Union{String, Vector{String}} - weakdeps = get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing} - if (exts isa String && name == exts) || (exts isa Vector{String} && name in exts) - for deps′ in [weakdeps, deps] - if deps′ !== nothing - if deps′ isa Vector{String} - found_name = name in deps′ - found_name && @goto done - elseif deps′ isa Dict{String, Any} - deps′ = deps′::Dict{String, Any} - for (dep, uuid) in deps′ - uuid::String - if dep === name - return PkgId(UUID(uuid), name) - end - end - end - end - end + local dep::Union{Nothing, PkgId} + @label resolved begin + if UUID(uuid) === where.uuid + dep = dep_stanza_get(deps, name) + + # We found `where` in this environment, but it did not have a deps entry for + # `name`. This is likely because the dependency was modified without a corresponding + # change to dependency's Project or our Manifest. Return a sentinel here indicating + # that we know the package, but do not know its UUID. The caller will terminate the + # search and provide an appropriate error to the user. + dep === nothing && return PkgId(name) + else + # Check if we're trying to load into an extension of this package + extensions = get(entry, "extensions", nothing) + if extensions !== nothing + if haskey(extensions, where.name) && where.uuid == uuid5(UUID(uuid), where.name) + if name == dep_name + # Extension loads its base package + return PkgId(UUID(uuid), name) + end + exts = extensions[where.name]::Union{String, Vector{String}} + # Extensions are allowed to load: + # 1. Any ordinary dep of the parent package + # 2. Any weakdep of the parent package declared as an extension trigger + for deps′ in (ext_may_load_weakdep(exts, name) ? + (get(entry, "weakdeps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing}, deps) : + (deps,)) + dep = dep_stanza_get(deps′, name) + dep === nothing && continue + break resolved end - # `name` is not an ext, do standard lookup as if this was the parent - return identify_package(PkgId(UUID(uuid), dep_name), name) + return PkgId(name) + end end + continue end end + + dep.uuid !== nothing && return dep + + # We have the dep, but it did not specify a UUID. In this case, + # it must be that the name is unique in the manifest - so lookup + # the UUID at the top level by name + name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}} + if name_deps === nothing || length(name_deps) != 1 + error("expected a single entry for $(repr(name)) in $(repr(project_file))") + end + entry = first(name_deps::Vector{Any})::Dict{String, Any} + uuid = get(entry, "uuid", nothing)::Union{String, Nothing} + uuid === nothing && return PkgId(name) + return PkgId(UUID(uuid), name) end end - @label done - found_where || return nothing - found_name || return PkgId(name) - # Only reach here if deps was not a dict which mean we have a unique name for the dep - name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}} - if name_deps === nothing || length(name_deps) != 1 - error("expected a single entry for $(repr(name)) in $(repr(project_file))") - end - entry = first(name_deps::Vector{Any})::Dict{String, Any} - uuid = get(entry, "uuid", nothing)::Union{String, Nothing} - uuid === nothing && return nothing - return PkgId(UUID(uuid), name) + + # We did not find `where` in this environment, either as a package or as an extension. + # The caller should continue searching the environment stack. + return nothing end # find `uuid` stanza, return the corresponding path -function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String,Missing} +function explicit_manifest_uuid_load_spec(project_file::String, pkg::PkgId)::Union{Nothing,PkgLoadSpec,Missing} manifest_file = project_file_manifest_path(project_file) manifest_file === nothing && return nothing # no manifest, skip env @@ -1053,7 +1245,7 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No uuid = get(entry, "uuid", nothing)::Union{Nothing, String} uuid === nothing && continue if UUID(uuid) === pkg.uuid - return explicit_manifest_entry_path(manifest_file, pkg, entry) + return explicit_manifest_entry_load_spec(manifest_file, pkg, entry) end end end @@ -1065,30 +1257,50 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No uuid = get(entry, "uuid", nothing)::Union{Nothing, String} extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid - parent_path = locate_package(PkgId(UUID(uuid), name)) - if parent_path === nothing + parent_load_spec = explicit_manifest_entry_load_spec(manifest_file, PkgId(UUID(uuid), name), entry) + if parent_load_spec === nothing || parent_load_spec === missing error("failed to find source of parent package: \"$name\"") end + parent_path = parent_load_spec.path p = normpath(dirname(parent_path), "..") - return find_ext_path(p, pkg.name) + return PkgLoadSpec(find_ext_path(p, pkg.name), parent_load_spec.julia_syntax_version) end end end return nothing end -function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry::Dict{String,Any}) +function explicit_manifest_entry_load_spec(manifest_file::String, pkg::PkgId, entry::Dict{String,Any})::Union{Nothing, Missing, PkgLoadSpec} + # Resolve syntax version. N.B.: Unlike in project files, an absent syntax.julia_version + # entry in manifest files means defaulting to 1.13. This is because we assume the + # manifest was created by an older version of julia that did not support syntax versioning. + # Newer versions of Pkg will provide syntax version information in the manifest, + # even if absent from the project file. + syntax_version = NON_VERSIONED_SYNTAX + syntax_table = get(entry, "syntax", nothing) + if syntax_table !== nothing + syntax_version = VersionNumber(get(syntax_table, "julia_version", nothing)) + # Clamp to minimum supported syntax version + if syntax_version <= NON_VERSIONED_SYNTAX + syntax_version = NON_VERSIONED_SYNTAX + end + end + + # Resolve path path = get(entry, "path", nothing)::Union{Nothing, String} entryfile = get(entry, "entryfile", nothing)::Union{Nothing, String} if path !== nothing path = entry_path(normpath(abspath(dirname(manifest_file), path)), pkg.name, entryfile) - return path + return PkgLoadSpec(path, syntax_version) end hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String} if hash === nothing - mbypath = manifest_uuid_path(Sys.STDLIB, pkg) - if mbypath isa String && isfile(mbypath) - return mbypath + # stdlibs do not have a git-hash so cannot be loaded from depots. As + # a special case, we allow loading these directly from the stdlib location + # (treated as an implicit environment). + mbyspec = manifest_uuid_load_spec(Sys.STDLIB, pkg) + if mbyspec isa PkgLoadSpec && isfile(mbyspec.path) + return mbyspec end return nothing end @@ -1098,7 +1310,7 @@ function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry:: for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4)) for depot in DEPOT_PATH path = joinpath(depot, "packages", pkg.name, slug) - ispath(path) && return entry_path(abspath(path), pkg.name, entryfile) + ispath(path) && return PkgLoadSpec(entry_path(abspath(path), pkg.name, entryfile), syntax_version) end end # no depot contains the package, return missing to stop looking @@ -1106,10 +1318,7 @@ function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry:: end ## implicit project & manifest API ## - -# look for an entry point for `name` from a top-level package (no environment) -# otherwise return `nothing` to indicate the caller should keep searching -function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId} +function implicit_manifest_pkgid(dir::String, name::String)::Union{Nothing,PkgId} path, project_file = entry_point_and_project_file(dir, name) if project_file === nothing path === nothing && return nothing @@ -1120,47 +1329,29 @@ function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,Pkg return proj end -# look for an entry-point for `name`, check that UUID matches -# if there's a project file, look up `name` in its deps and return that -# otherwise return `nothing` to indicate the caller should keep searching -function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} - @assert where.uuid !== nothing - project_file = entry_point_and_project_file(dir, where.name)[2] +function implicit_manifest_project(dir::String, pkg::PkgId)::Union{Nothing, String} + @assert pkg.uuid !== nothing + project_file = entry_point_and_project_file(dir, pkg.name)[2] if project_file === nothing # `where` could be an extension - project_file = implicit_env_project_file_extension(dir, where)[2] - project_file === nothing && return nothing + return implicit_env_project_file_extension(dir, pkg)[2] end - proj = project_file_name_uuid(project_file, where.name) - ext = nothing - if proj !== where - # `where` could be an extension in `proj` - d = parsed_toml(project_file) - exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} - if exts !== nothing && where.name in keys(exts) - if where.uuid !== uuid5(proj.uuid, where.name) - return nothing - end - ext = where.name - else - return nothing - end - end - # this is the correct project, so stop searching here - pkg_uuid = explicit_project_deps_get(project_file, name, ext) - return PkgId(pkg_uuid, name) + proj = project_file_name_uuid(project_file, pkg.name) + proj == pkg || return nothing + return project_file end # look for an entry-point for `pkg` and return its path if UUID matches -function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} +function implicit_manifest_uuid_load_spec(dir::String, pkg::PkgId)::Union{Nothing, PkgLoadSpec} path, project_file = entry_point_and_project_file(dir, pkg.name) if project_file === nothing pkg.uuid === nothing || return nothing - return path + # Without a project file, treat as empty - which defaults to VERSION + return PkgLoadSpec(path, VERSION) end proj = project_file_name_uuid(project_file, pkg.name) proj == pkg || return nothing - return path + return PkgLoadSpec(path, project_get_syntax_version(parsed_toml(project_file))) end ## other code loading functionality ## @@ -1180,56 +1371,74 @@ function cache_file_entry(pkg::PkgId) uuid === nothing ? pkg.name : package_slug(uuid) end -function find_all_in_cache_path(pkg::PkgId, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) - paths = String[] +# Return the cache file candidates in `depot` and whether each has a pkgimage +# next to it, from a single directory listing without opening or stat'ing any +# entry; invalid entries are rejected during validation. +function cachefile_candidates_in_depot(pkg::PkgId, depot::String) entrypath, entryfile = cache_file_entry(pkg) - for path in DEPOT_PATH - path = joinpath(path, entrypath) - isdir(path) || continue - for file in readdir(path, sort = false) # no sort given we sort later - if !((pkg.uuid === nothing && file == entryfile * ".ji") || - (pkg.uuid !== nothing && startswith(file, entryfile * "_") && - endswith(file, ".ji"))) - continue - end - filepath = joinpath(path, file) - isfile_casesensitive(filepath) && push!(paths, filepath) - end - end - if length(paths) > 1 - function sort_by(path) - # when using pkgimages, consider those cache files first - pkgimage = if JLOptions().use_pkgimages != 0 - io = open(path, "r") - try - if iszero(isvalid_cache_header(io)) - false - else - _, _, _, _, _, _, _, flags = parse_cache_header(io, path) - CacheFlags(flags).use_pkgimages - end - finally - close(io) - end - else - false - end - (; pkgimage, mtime=mtime(path)) + dir = joinpath(depot, entrypath) + paths = String[] + pkgimages = Bool[] + isdir(dir) || return paths, pkgimages + entries = readdir(dir, sort = false) + use_pkgimages = JLOptions().use_pkgimages != 0 + for file in entries + if !((pkg.uuid === nothing && file == entryfile * ".ji") || + (pkg.uuid !== nothing && startswith(file, entryfile * "_") && + endswith(file, ".ji"))) + continue end - function sort_lt(a, b) - if a.pkgimage != b.pkgimage - return a.pkgimage < b.pkgimage - end - return a.mtime < b.mtime + push!(paths, joinpath(dir, file)) + push!(pkgimages, use_pkgimages && ocachefile_from_cachefile(file) in entries) + end + return paths, pkgimages +end + +# Sort candidates best-first: those with a pkgimage before those without, and +# most recently modified first within each group, so that `touch`ing a cache +# file makes it the first one tried on the next load. +function sort_cachefile_candidates!(paths::Vector{String}, pkgimages::Vector{Bool}) + length(paths) > 1 || return paths + sort_by(i) = (; pkgimage=pkgimages[i], mtime=mtime(paths[i])) + function sort_lt(a, b) + if a.pkgimage != b.pkgimage + return a.pkgimage < b.pkgimage end + return a.mtime < b.mtime + end + # allocating the sort vector is less expensive than using sort!(.. by=sort_by), + # which would call the relatively slow mtime multiple times per path + p = sortperm(sort_by.(eachindex(paths)), lt=sort_lt, rev=true) + permute!(paths, p) + return paths +end - # allocating the sort vector is less expensive than using sort!(.. by=sort_by), - # which would call the relatively slow mtime multiple times per path - p = sortperm(sort_by.(paths), lt=sort_lt, rev=true) - return paths[p] - else - return paths +function sorted_cachefile_candidates_in_depot(pkg::PkgId, depot::String) + return sort_cachefile_candidates!(cachefile_candidates_in_depot(pkg, depot)...) +end + +# Search the depot containing Sys.STDLIB first for bundled stdlibs. +function cache_search_depots(sourcepath::String, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) + stdlib_dir = joinpath(Sys.STDLIB, "") # trailing separator for path-boundary matching + startswith(sourcepath, stdlib_dir) || return DEPOT_PATH + return sort(DEPOT_PATH, by = depot -> !startswith(stdlib_dir, joinpath(depot, ""))) +end + +# Search one depot at a time and stop enumerating once a candidate is accepted. +function lazy_cachefile_candidates(pkg::PkgId, sourcepath::String, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) + return Iterators.flatten(sorted_cachefile_candidates_in_depot(pkg, depot) + for depot in cache_search_depots(sourcepath, DEPOT_PATH)) +end + +function find_all_in_cache_path(pkg::PkgId, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) + paths = String[] + pkgimages = Bool[] + for depot in DEPOT_PATH + depot_paths, depot_pkgimages = cachefile_candidates_in_depot(pkg, depot) + append!(paths, depot_paths) + append!(pkgimages, depot_pkgimages) end + return sort_cachefile_candidates!(paths, pkgimages) end ocachefile_from_cachefile(cachefile) = string(chopsuffix(cachefile, ".ji"), ".", Libc.Libdl.dlext) @@ -1257,7 +1466,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No for i in eachindex(depmods) dep = depmods[i] dep isa Module && continue - _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128} + _, depkey, depbuild_id = dep::Tuple{PkgLoadSpec, PkgId, UInt128} dep = something(maybe_loaded_precompile(depkey, depbuild_id)) @assert PkgId(dep) == depkey && module_build_id(dep) === depbuild_id depmods[i] = dep @@ -1283,12 +1492,9 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No end sv = sv::SimpleVector - edges = sv[3]::Vector{Any} - ext_edges = sv[4]::Union{Nothing,Vector{Any}} - extext_methods = sv[5]::Vector{Any} - internal_methods = sv[6]::Vector{Any} + internal_methods = sv[3]::Vector{Any} Compiler.@zone "CC: INSERT_BACKEDGES" begin - StaticData.insert_backedges(edges, ext_edges, extext_methods, internal_methods) + ReinferUtils.insert_backedges_typeinf(internal_methods) end restored = register_restored_modules(sv, pkg, path) @@ -1297,8 +1503,8 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No if is_root_module(M) && PkgId(M) == pkg register && register_root_module(M) if timing_imports - elapsed_time = time_ns() - t_before - comp_time, recomp_time = cumulative_compile_time_ns() .- t_comp_before + elapsed_time = time_ns() -% t_before + comp_time, recomp_time = map(-%, cumulative_compile_time_ns(), t_comp_before) print_time_imports_report(M, elapsed_time, comp_time, recomp_time) end return M @@ -1422,9 +1628,9 @@ function run_module_init(mod::Module, i::Int=1) ccall(:jl_init_restored_module, Cvoid, (Any,), mod) - elapsed_time = time_ns() - elapsed_time + elapsed_time = time_ns() -% elapsed_time cumulative_compile_timing(false); - comp_time, recomp_time = cumulative_compile_time_ns() .- compile_elapsedtimes + comp_time, recomp_time = map(-%, cumulative_compile_time_ns(), compile_elapsedtimes) print_time_imports_report_init(mod, i, elapsed_time, comp_time, recomp_time) end @@ -1448,6 +1654,55 @@ function run_package_callbacks(modkey::PkgId) nothing end +""" + Base.CACHE_FETCH_HOOK + +An optional callable that is consulted when `require` or precompilation has +determined that no valid compile cache exists for a package and it is about +to compile one. The hook is called as + + hook(pkg::PkgId, sourcepath::String)::Bool + +A return value of `true` indicates that the hook may have placed a cachefile +in one of the `DEPOT_PATH` compile cache directories (e.g. by fetching it +from a cache server); the caller then rescans the cache candidates, +revalidates them through the normal staleness machinery, and falls back to +compiling if nothing valid appeared. Any other return value — or a thrown +error, which is caught and logged at debug level — proceeds directly to +compilation. + +The hook is advisory and its output is untrusted: fetched files undergo the +same validation as any other cache candidate. Implementations must not load +non-sysimage code, and must be safe to call from concurrent tasks. The hook +is never invoked from output-generating (precompile worker) processes, nor +reentrantly. +""" +const CACHE_FETCH_HOOK = Ref{Any}(nothing) + +""" + Base.maybe_fetch_cache(pkg::PkgId, sourcepath::String) -> Bool + +Invoke `CACHE_FETCH_HOOK` under its safety guards (never while generating +output, never reentrantly, errors demoted to `false`) and return whether +the caller should rescan the compile cache candidates. +""" +function maybe_fetch_cache(pkg::PkgId, sourcepath::String) + h = CACHE_FETCH_HOOK[] + h === nothing && return false + generating_output() && return false + tls = task_local_storage() + haskey(tls, :in_cache_fetch_hook) && return false + tls[:in_cache_fetch_hook] = true + try + return @invokelatest(h(pkg, sourcepath)) === true + catch err + @debug "CACHE_FETCH_HOOK failed" pkg exception=(err, catch_backtrace()) + return false + finally + delete!(tls, :in_cache_fetch_hook) + end +end + ############## # Extensions # @@ -1468,7 +1723,7 @@ function insert_extension_triggers(pkg::PkgId) pkg.uuid === nothing && return path_env_loc = locate_package_env(pkg) path_env_loc === nothing && return - path, env_loc = path_env_loc + _, env_loc = path_env_loc insert_extension_triggers(env_loc, pkg) end @@ -1507,6 +1762,12 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi uuid = get(entry, "uuid", nothing)::Union{String, Nothing} uuid === nothing && continue if UUID(uuid) == pkg.uuid + if get(entry, "path", nothing) === nothing && get(entry, "git-tree-sha1", nothing) === nothing + # a stdlib entry is loaded from Sys.STDLIB (see `explicit_manifest_uuid_path`), and + # the manifest may have been resolved by a Julia version whose copy of the stdlib + # had different extensions, so take them from the stdlib's own Project.toml + return insert_extension_triggers(Sys.STDLIB, pkg) + end extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return weakdeps = get(Dict{String, Any}, entry, "weakdeps")::Union{Vector{String}, Dict{String,Any}} @@ -1520,8 +1781,8 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi if length(entries) != 1 error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") end - entry = first(entries)::Dict{String, Any} - uuid = entry["uuid"]::String + local entry = first(entries)::Dict{String, Any} + local uuid = entry["uuid"]::String deps′_expanded[dep_name] = uuid end return deps′_expanded @@ -1666,33 +1927,38 @@ struct CacheFlags check_bounds::Int inline::Bool opt_level::Int + # coverage instrumentation of the image (jl_image_coverage_config): + # 0 none, 1 hit counters, 2 execution counters + coverage::Int end -function CacheFlags(f::UInt8) +function CacheFlags(f::UInt8, coverage::Integer=0) use_pkgimages = Bool(f & 1) debug_level = Int((f >> 1) & 3) check_bounds = Int((f >> 3) & 3) inline = Bool((f >> 5) & 1) - opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils - CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level) + opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in staticdata_utils + CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level, Int(coverage)) end CacheFlags(f::Int) = CacheFlags(UInt8(f)) -function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flags, UInt8, ())); +function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flags, UInt8, ()), ccall(:jl_image_coverage_config, UInt8, ())); use_pkgimages::Union{Nothing,Bool}=nothing, debug_level::Union{Nothing,Int}=nothing, check_bounds::Union{Nothing,Int}=nothing, inline::Union{Nothing,Bool}=nothing, - opt_level::Union{Nothing,Int}=nothing + opt_level::Union{Nothing,Int}=nothing, + coverage::Union{Nothing,Int}=nothing ) return CacheFlags( use_pkgimages === nothing ? cf.use_pkgimages : use_pkgimages, debug_level === nothing ? cf.debug_level : debug_level, check_bounds === nothing ? cf.check_bounds : check_bounds, inline === nothing ? cf.inline : inline, - opt_level === nothing ? cf.opt_level : opt_level + opt_level === nothing ? cf.opt_level : opt_level, + coverage === nothing ? cf.coverage : coverage ) end # reflecting jloptions.c defaults -const DefaultCacheFlags = CacheFlags(use_pkgimages=true, debug_level=isdebugbuild() ? 2 : 1, check_bounds=0, inline=true, opt_level=2) +const DefaultCacheFlags = CacheFlags(use_pkgimages=true, debug_level=isdebugbuild() ? 2 : 1, check_bounds=0, inline=true, opt_level=2, coverage=0) function _cacheflag_to_uint8(cf::CacheFlags)::UInt8 f = UInt8(0) @@ -1711,9 +1977,22 @@ function translate_cache_flags(cacheflags::CacheFlags, defaultflags::CacheFlags) cacheflags.check_bounds != defaultflags.check_bounds && push!(opts, ("--check-bounds=auto", "--check-bounds=yes", "--check-bounds=no")[cacheflags.check_bounds + 1]) cacheflags.inline != defaultflags.inline && push!(opts, cacheflags.inline ? "--inline=yes" : "--inline=no") cacheflags.opt_level != defaultflags.opt_level && push!(opts, "-O$(cacheflags.opt_level)") + cacheflags.coverage != defaultflags.coverage && append!(opts, coverage_cache_options(cacheflags)) return opts end +# Image instrumentation is independent of the collecting process's scope. +function coverage_cache_options(cf::CacheFlags) + cf.coverage == 0 && return ["--code-coverage=none"] + mode = cf.coverage == 2 ? "count" : "hit" + return ["--code-coverage=user", "--code-coverage-mode=" * mode] +end + +# Whether a cache with instrumentation `actual` serves `requested`. +function match_cache_coverage(requested::CacheFlags, actual::CacheFlags) + return @ccall(jl_match_cache_coverage(UInt8(requested.coverage)::UInt8, UInt8(actual.coverage)::UInt8)::Cint) != 0 +end + function show(io::IO, cf::CacheFlags) print(io, "CacheFlags(") print(io, "; use_pkgimages=") @@ -1726,27 +2005,50 @@ function show(io::IO, cf::CacheFlags) print(io, cf.inline) print(io, ", opt_level=") print(io, cf.opt_level) + print(io, ", coverage=") + print(io, cf.coverage) print(io, ")") end +function Base.parse(::Type{CacheFlags}, s::AbstractString) + e = Meta.parse(s) + if !(e isa Expr && e.head === :call && length(e.args) == 2 && + e.args[1] === :CacheFlags && + e.args[2] isa Expr && e.args[2].head == :parameters) + throw(ArgumentError("Malformed CacheFlags string")) + end + params = Dict{Symbol, Any}(p.args[1] => p.args[2] for p in e.args[2].args) + use_pkgimages = get(params, :use_pkgimages, nothing) + debug_level = get(params, :debug_level, nothing) + check_bounds = get(params, :check_bounds, nothing) + inline = get(params, :inline, nothing) + opt_level = get(params, :opt_level, nothing) + coverage = get(params, :coverage, nothing) + return CacheFlags(; use_pkgimages, debug_level, check_bounds, inline, opt_level, coverage) +end + struct ImageTarget name::String flags::Int32 + base::Int32 ext_features::String - features_en::Vector{UInt8} - features_dis::Vector{UInt8} + features_en::String + features_dis::String end function parse_image_target(io::IO) flags = read(io, Int32) - nfeature = read(io, Int32) - feature_en = read(io, 4*nfeature) - feature_dis = read(io, 4*nfeature) + base = read(io, Int32) + nwords = read(io, Int32) # number of uint64_t feature words + feature_en_raw = read(io, 8*nwords) + feature_dis_raw = read(io, 8*nwords) name_len = read(io, Int32) name = String(read(io, name_len)) ext_features_len = read(io, Int32) ext_features = String(read(io, ext_features_len)) - ImageTarget(name, flags, ext_features, feature_en, feature_dis) + features_en = @ccall jl_feature_bits_to_string(feature_en_raw::Ptr{UInt8}, nwords::Int32)::Ref{String} + features_dis = @ccall jl_feature_bits_to_string(feature_dis_raw::Ptr{UInt8}, nwords::Int32)::Ref{String} + ImageTarget(name, flags, base, ext_features, features_en, features_dis) end function parse_image_targets(targets::Vector{UInt8}) @@ -1764,105 +2066,99 @@ function current_image_targets() return parse_image_targets(targets) end -struct FeatureName - name::Cstring - bit::UInt32 # bit index into a `uint32_t` array; - llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support -end - -function feature_names() - fnames = Ref{Ptr{FeatureName}}() - nf = Ref{Csize_t}() - @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid - if fnames[] == C_NULL - @assert nf[] == 0 - return Vector{FeatureName}(undef, 0) - end - Base.unsafe_wrap(Array, fnames[], nf[], own=false) -end - -function test_feature(features::Vector{UInt8}, feat::FeatureName) - bitidx = feat.bit - u8idx = div(bitidx, 8) + 1 - bit = bitidx % 8 - return (features[u8idx] & (1 << bit)) != 0 -end - function show(io::IO, it::ImageTarget) print(io, it.name) if !isempty(it.ext_features) print(io, ",", it.ext_features) end + if it.base >= 0 + print(io, "; base=", it.base) + end print(io, "; flags=", it.flags) - print(io, "; features_en=(") - first = true - for feat in feature_names() - if test_feature(it.features_en, feat) - name = Base.unsafe_string(feat.name) - if first - first = false - print(io, name) - else - print(io, ", ", name) - end - end + if !isempty(it.features_en) + print(io, "; features_en=(", it.features_en, ")") + end + if !isempty(it.features_dis) + print(io, "; features_dis=(", it.features_dis, ")") end - print(io, ")") - # Is feature_dis useful? end # should sync with the types of arguments of `stale_cachefile` -const StaleCacheKey = Tuple{PkgId, UInt128, String, String} +const StaleCacheKey = Tuple{PkgId, UInt128, PkgLoadSpec, String, Bool, CacheFlags} -function compilecache_path(pkg::PkgId; +function compilecache_freshest_path(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(), - cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg), - sourcepath::Union{String,Nothing}=Base.locate_package(pkg), - flags::CacheFlags=CacheFlags()) - path = nothing - isnothing(sourcepath) && error("Cannot locate source for $(repr("text/plain", pkg))") - for path_to_try in cachepaths - staledeps = stale_cachefile(sourcepath, path_to_try; ignore_loaded, requested_flags=flags) - if staledeps === true - continue - end - staledeps, _, _ = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128} - # finish checking staledeps module graph - for dep in staledeps - dep isa Module && continue - modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} - modpaths = get!(() -> find_all_in_cache_path(modkey), cachepath_cache, modkey) - for modpath_to_try in modpaths::Vector{String} - stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey - if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded, requested_flags=flags) === true, - stale_cache, stale_cache_key) - continue + cachepaths::Vector{String}=get(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg), + sourcespec::Union{PkgLoadSpec,Nothing}=Base.locate_package_load_spec(pkg), + flags::CacheFlags=CacheFlags(), + # skipping the full-file corruption checks is only safe for advisory + # queries (e.g. `isprecompiled`): a caller whose verdict decides what + # gets loaded without further validation (like the precompilation + # driver) must keep them on, so that a corrupted cache is recompiled + # rather than reported as loadable + verify_checksums::Bool=true, + reasons::Union{Dict{Symbol,Int},Nothing}=nothing) + isnothing(sourcespec) && error("Cannot locate source for $(repr("text/plain", pkg))") + @lock require_lock begin + set_cache = LOADING_CACHE[] === nothing + set_cache && (LOADING_CACHE[] = LoadingCache()) + try + try_build_ids = UInt128[UInt128(0)] + if !ignore_loaded + let loaded = get(loaded_precompiles, pkg, nothing) + if loaded !== nothing + for mod in loaded # try these in reverse original load order to see if one is already valid + pushfirst!(try_build_ids, module_build_id(mod)) end - @goto check_next_dep end - @goto check_next_path - @label check_next_dep end - try - # update timestamp of precompilation file so that it is the first to be tried by code loading - touch(path_to_try) - catch ex - # file might be read-only and then we fail to update timestamp, which is fine - ex isa IOError || rethrow() + end + for build_id in try_build_ids + @label next_path for path_to_try in cachepaths + staledeps = stale_cachefile(pkg, build_id, sourcespec, path_to_try; ignore_loaded, requested_flags=flags, verify_checksums, reasons) + if staledeps === true + continue + end + staledeps, _, id_build = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128} + # Record the result so dependents don't check this file again. + stale_cache[(pkg, id_build, sourcespec, path_to_try, ignore_loaded, flags)::StaleCacheKey] = false + # finish checking staledeps module graph + @label next_dep for dep in staledeps + dep isa Module && continue + modspec, modkey, modbuild_id = dep::Tuple{PkgLoadSpec, PkgId, UInt128} + modpaths = get(() -> find_all_in_cache_path(modkey), cachepath_cache, modkey) + for modpath_to_try in modpaths::Vector{String} + stale_cache_key = (modkey, modbuild_id, modspec, modpath_to_try, ignore_loaded, flags)::StaleCacheKey + if get!(() -> stale_cachefile(modkey, modbuild_id, modspec, modpath_to_try; ignore_loaded, requested_flags=flags, verify_checksums) === true, + stale_cache, stale_cache_key) + continue + end + continue next_dep + end + continue next_path + end + try + # update timestamp of precompilation file so that it is the first to be tried by code loading + touch(path_to_try) + catch + # file might be read-only and then we fail to update timestamp, which is fine + end + return path_to_try end - path = path_to_try - break - @label check_next_path end - return path + return nothing + finally + set_cache && (LOADING_CACHE[] = nothing) + end + end end """ Base.isprecompiled(pkg::PkgId; ignore_loaded::Bool=false) -Returns whether a given PkgId within the active project is precompiled. +Return whether a given PkgId within the active project is precompiled. By default this check observes the same approach that code loading takes with respect to when different versions of dependencies are currently loaded @@ -1872,32 +2168,26 @@ fresh julia session specify `ignore_loaded=true`. !!! compat "Julia 1.10" This function requires at least Julia 1.10. """ -function isprecompiled(pkg::PkgId; - ignore_loaded::Bool=false, - stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), - cachepath_cache::Dict{PkgId, Vector{String}}=Dict{PkgId, Vector{String}}(), - cachepaths::Vector{String}=get!(() -> find_all_in_cache_path(pkg), cachepath_cache, pkg), - sourcepath::Union{String,Nothing}=Base.locate_package(pkg), - flags::CacheFlags=CacheFlags()) - path = compilecache_path(pkg; ignore_loaded, stale_cache, cachepath_cache, cachepaths, sourcepath, flags) +function isprecompiled(pkg::PkgId; ignore_loaded::Bool=false) + path = compilecache_freshest_path(pkg; ignore_loaded, verify_checksums=false) return !isnothing(path) end """ Base.isrelocatable(pkg::PkgId) -Returns whether a given PkgId within the active project is precompiled and the +Return whether a given PkgId within the active project is precompiled and the associated cache is relocatable. !!! compat "Julia 1.11" This function requires at least Julia 1.11. """ function isrelocatable(pkg::PkgId) - path = compilecache_path(pkg) + path = compilecache_freshest_path(pkg; verify_checksums=false) isnothing(path) && return false io = open(path, "r") try - iszero(isvalid_cache_header(io)) && throw(ArgumentError("Incompatible header in cache file $cachefile.")) + isvalid_cache_header(io) === nothing && throw(ArgumentError("Incompatible header in cache file $path.")) _, (includes, includes_srcfiles, _), _... = _parse_cache_header(io, path) for inc in includes !startswith(inc.filename, "@depot") && return false @@ -1913,17 +2203,35 @@ function isrelocatable(pkg::PkgId) return true end +function parse_cache_buildid(cachepath::String) + f = open(cachepath, "r") + try + checksum = isvalid_cache_header(f) + checksum === nothing && throw(ArgumentError("Incompatible header in cache file $cachepath.")) + read(f, UInt8) # flags + read(f, UInt8) # coverage + read(f, UInt8) # syntax_version + n = read(f, Int32) + n == 0 && error("no module defined in $cachepath") + skip(f, n) # module name + uuid = UUID((read(f, UInt64), read(f, UInt64))) # pkg UUID + build_id = (UInt128(checksum) << 64) | read(f, UInt64) + return build_id, uuid + finally + close(f) + end +end + # search for a precompile cache file to load, after some various checks function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) assert_havelock(require_lock) loaded = start_loading(modkey, build_id, false) if loaded === nothing try - modpath = locate_package(modkey) - isnothing(modpath) && error("Cannot locate source for $(repr("text/plain", modkey))") - modpath = String(modpath)::String - set_pkgorigin_version_path(modkey, modpath) - loaded = _require_search_from_serialized(modkey, modpath, build_id, true) + modspec = locate_package_load_spec(modkey) + isnothing(modspec) && error("Cannot locate source for $(repr("text/plain", modkey))") + set_pkgorigin_version_path(modkey, modspec.path) + loaded = _require_search_from_serialized(modkey, modspec, build_id, true) finally end_loading(modkey, loaded) end @@ -1945,8 +2253,8 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union local depmodnames io = open(path, "r") try - iszero(isvalid_cache_header(io)) && return ArgumentError("Incompatible header in cache file $path.") - _, (includes, _, _), depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io, path) + isvalid_cache_header(io) === nothing && return ArgumentError("Incompatible header in cache file $path.") + _, (includes, _, _), depmodnames, _, _, clone_targets, _ = parse_cache_header(io, path) pkgimage = !isempty(clone_targets) @@ -1980,9 +2288,8 @@ end # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it or it was stale # returns the set of modules restored if the cache load succeeded -@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128, stalecheck::Bool; reasons=nothing, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) +@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcespec::PkgLoadSpec, build_id::UInt128, stalecheck::Bool; reasons=nothing, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) assert_havelock(require_lock) - paths = find_all_in_cache_path(pkg, DEPOT_PATH) newdeps = PkgId[] try_build_ids = UInt128[build_id] if build_id == UInt128(0) @@ -1994,9 +2301,18 @@ end end end end + # iterating the lazy candidates more than once would repeat directory listings + paths = length(try_build_ids) == 1 ? + lazy_cachefile_candidates(pkg, sourcespec.path, DEPOT_PATH) : + find_all_in_cache_path(pkg, DEPOT_PATH) + # Try the driver's validated cache first; fall back to the normal search. + pre = get(preresolved_cachefiles, pkg, nothing) + pre !== nothing && (paths = Iterators.flatten(((pre,), paths))) for build_id in try_build_ids - for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck) + @label next_path for path_to_try in paths + trusted = path_to_try === pre + staledeps = stale_cachefile(pkg, build_id, sourcespec, path_to_try; reasons, + stalecheck = stalecheck && !trusted, verify_checksums = !trusted) if staledeps === true continue end @@ -2014,7 +2330,7 @@ end i += 1 dep = staledeps[i] dep isa Module && continue - _, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} + _, modkey, modbuild_id = dep::Tuple{PkgLoadSpec, PkgId, UInt128} dep = canstart_loading(modkey, modbuild_id, stalecheck) if dep isa Module if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id @@ -2022,7 +2338,7 @@ end continue else @debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected." - @goto check_next_path + continue next_path end continue elseif dep === nothing @@ -2032,27 +2348,30 @@ end i = 0 end end - for i in reverse(eachindex(staledeps)) + @label next_dep for i in reverse(eachindex(staledeps)) dep = staledeps[i] dep isa Module && continue - modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} + modspec, modkey, modbuild_id = dep::Tuple{PkgLoadSpec, PkgId, UInt128} # inline a call to start_loading here @assert canstart_loading(modkey, modbuild_id, stalecheck) === nothing package_locks[modkey] = (current_task(), Threads.Condition(require_lock), modbuild_id) startedloading = i - modpaths = find_all_in_cache_path(modkey, DEPOT_PATH) + mpre = get(preresolved_cachefiles, modkey, nothing) + modpaths = lazy_cachefile_candidates(modkey, modspec.path, DEPOT_PATH) + mpre !== nothing && (modpaths = Iterators.flatten(((mpre,), modpaths))) for modpath_to_try in modpaths - modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck) + modtrusted = modpath_to_try === mpre + modstaledeps = stale_cachefile(modkey, modbuild_id, modspec, modpath_to_try; + stalecheck = stalecheck && !modtrusted, verify_checksums = !modtrusted) if modstaledeps === true continue end modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128} - staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath) - @goto check_next_dep + staledeps[i] = (modspec, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath) + continue next_dep end @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache." - @goto check_next_path - @label check_next_dep + continue next_path end M = maybe_loaded_precompile(pkg, newbuild_id) if isa(M, Module) @@ -2062,8 +2381,8 @@ end if stalecheck try touch(path_to_try) # update timestamp of precompilation file - catch ex # file might be read-only and then we fail to update timestamp, which is fine - ex isa IOError || rethrow() + catch + # file might be read-only and then we fail to update timestamp, which is fine end end # finish loading module graph into staledeps @@ -2071,12 +2390,12 @@ end for i in eachindex(staledeps) dep = staledeps[i] dep isa Module && continue - modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}} - set_pkgorigin_version_path(modkey, modpath) + modspec, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{PkgLoadSpec, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}} + set_pkgorigin_version_path(modkey, modspec.path) dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck) if !isa(dep, Module) @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep - @goto check_next_path + continue next_path else startedloading = i + 1 end_loading(modkey, dep) @@ -2090,16 +2409,15 @@ end end isa(restored, Module) && return restored @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored - @label check_next_path finally # cancel all start_loading locks that were taken but not fulfilled before failing for i in startedloading:length(staledeps) dep = staledeps[i] dep isa Module && continue - if dep isa Tuple{String, PkgId, UInt128} + if dep isa Tuple{PkgLoadSpec, PkgId, UInt128} _, modkey, _ = dep else - _, modkey, _ = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}} + _, modkey, _ = dep::Tuple{PkgLoadSpec, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}} end end_loading(modkey, nothing) end @@ -2147,8 +2465,12 @@ function canstart_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool) for each in package_locks cond2 = each[2][2] assert_havelock(cond2.lock) - for waiting in cond2.waitq - push!(waiters, waiting => (each[2][1] => each[1])) + w = cond2.waitq.head + while w !== nothing + w = w::WaitEntry + waiting = @atomic :monotonic w.task + waiting isa Task && push!(waiters, waiting => (each[2][1] => each[1])) + w = _next_on(w, cond2) end end while true @@ -2219,6 +2541,9 @@ const include_callbacks = Any[] # used to optionally track dependencies when requiring a module: const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", because they are explicitly loaded, and the process should try to avoid invalidating them + +# Cache files supplied by the parent precompile driver. +const preresolved_cachefiles = Dict{PkgId,String}() # protected by require_lock const _require_dependencies = Any[] # a list of (mod::Module, abspath::String, fsize::UInt64, hash::UInt32, mtime::Float64) tuples that are the file dependencies of the module currently being precompiled const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies @@ -2256,6 +2581,60 @@ function _include_dependency!(dep_list::Vector{Any}, track_dependencies::Bool, return path, prev end +# Hidden binding, written into a package's own root module, holding the non-identity `mapexpr` +# functions passed to `include(mapexpr, mod, path)` while the package loaded. Stored inside the +# module so it is serialized into the package image during precompilation and is therefore +# retrievable after load (e.g. by revision tools) without re-parsing or re-executing any source. +const _include_mapexprs_name = Symbol("#include_mapexprs#") + +# Keyed by `(including_module, absolute_path)`. A plain `Dict` (not `IdDict`): the path component +# is a `String`, so keys must compare by value rather than by `===`. +const IncludeMapexprs = Dict{Tuple{Module,String},Any} + +function _record_include_mapexpr!(mod::Module, path::AbstractString, @nospecialize(mapexpr)) + # `identity` includes are reconstructible from the source snapshot alone, so the overwhelmingly + # common case stays zero-overhead and never allocates the table. + mapexpr === identity && return nothing + root = moduleroot(mod) + @lock require_lock begin + if isdefined(root, _include_mapexprs_name) + # The binding was created in an earlier world, so reading it now is safe. + table = getglobal(root, _include_mapexprs_name)::IncludeMapexprs + else + # First non-identity include for this root: create the table. `Core.eval` defines the + # binding because `setglobal!` cannot create one that does not yet exist (#56933). The + # rest of this call must use the local `table`, not look the binding back up: within the + # call that defines it, the binding lives in a world the call cannot yet observe. + table = IncludeMapexprs() + Core.eval(root, Expr(:const, Expr(:(=), _include_mapexprs_name, table))) + end + table[(mod, String(path))] = mapexpr + end + return nothing +end + +""" + Base.include_mapexprs(mod::Module) -> Union{Nothing,Dict{Tuple{Module,String},Any}} + +Return the `mapexpr` functions used by `include(mapexpr, …)` calls (with `mapexpr !== identity`) +while loading the package rooted at `mod`, keyed by `(including_module, absolute_path)`. Return +`nothing` when no such includes occurred — the common case, so that querying every loaded package +allocates nothing. + +This lets revision tools (e.g. Revise) re-apply the original transform when an edited file is +re-evaluated: the table records the exact function object used at load/precompile time, which a +`mapexpr` that captures runtime state could not be reconstructed by re-parsing. + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. +""" +function include_mapexprs(mod::Module) + root = moduleroot(mod) + isdefined(root, _include_mapexprs_name) || return nothing + return getglobal(root, _include_mapexprs_name)::IncludeMapexprs +end +public include_mapexprs + """ include_dependency(path::AbstractString; track_content::Bool=true) @@ -2286,7 +2665,7 @@ precompilableerror(ex::PrecompilableError) = true precompilableerror(ex::WrappedException) = precompilableerror(ex.error) precompilableerror(@nospecialize ex) = false -# Call __precompile__(false) at the top of a tile prevent it from being precompiled (false) +# Call __precompile__(false) at the top of a file to prevent it from being precompiled (false) """ __precompile__(isprecompilable::Bool) @@ -2308,26 +2687,36 @@ const toplevel_load = Ref(true) const _require_world_age = Ref{UInt}(typemax(UInt)) """ - require(into::Module, module::Symbol) + Base.TRACE_EVAL + +Global control for expression tracing during top-level evaluation. This setting takes priority +over the `--trace-eval` command-line option. -This function is part of the implementation of [`using`](@ref) / [`import`](@ref), if a module is not -already defined in `Main`. It can also be called directly to force reloading a module, -regardless of whether it has been loaded before (for example, when interactively developing -libraries). +Set to: +- `nothing` - use the command-line `--trace-eval` setting (default) +- `:no` - disable expression tracing +- `:loc` - show only location information during evaluation +- `:full` - show full expressions being evaluated -Loads a source file, in the context of the `Main` module, on every active node, searching -standard locations for files. `require` is considered a top-level operation, so it sets the -current `include` path but does not use it to search for files (see help for [`include`](@ref)). -This function is typically used to load library code, and is implicitly called by `using` to -load packages. +# Examples +```julia +# Enable full expression tracing +Base.TRACE_EVAL = :full -When searching for files, `require` first looks for package code in the global array -[`LOAD_PATH`](@ref). `require` is case-sensitive on all platforms, including those with -case-insensitive filesystems like macOS and Windows. +# Show only locations +Base.TRACE_EVAL = :loc -For more details regarding code loading, see the manual sections on [modules](@ref modules) and -[parallel computing](@ref code-availability). +# Disable tracing (overrides command-line setting) +Base.TRACE_EVAL = :no + +# Reset to use command-line setting +Base.TRACE_EVAL = nothing +``` + +See also: [Command-line Interface](@ref cli) for the `--trace-eval` option. """ +TRACE_EVAL::Union{Symbol,Nothing} = nothing + function require(into::Module, mod::Symbol) world = _require_world_age[] if world == typemax(UInt) @@ -2410,19 +2799,20 @@ function find_unsuitable_manifests_versions() manifest_file isa String || continue # no manifest file m = parsed_toml(manifest_file) man_julia_version = get(m, "julia_version", nothing) - man_julia_version isa String || @goto mark - man_julia_version = VersionNumber(man_julia_version) - thispatch(man_julia_version) != thispatch(VERSION) && @goto mark - isempty(man_julia_version.prerelease) != isempty(VERSION.prerelease) && @goto mark - isempty(man_julia_version.prerelease) && continue - man_julia_version.prerelease[1] != VERSION.prerelease[1] && @goto mark - if VERSION.prerelease[1] == "DEV" - # manifests don't store the 2nd part of prerelease, so cannot check further - # so treat them specially in the warning - push!(dev_manifests, manifest_file) - end - continue - @label mark + @label check begin + man_julia_version isa String || break check + man_julia_version = VersionNumber(man_julia_version) + thispatch(man_julia_version) != thispatch(VERSION) && break check + isempty(man_julia_version.prerelease) != isempty(VERSION.prerelease) && break check + isempty(man_julia_version.prerelease) && continue + man_julia_version.prerelease[1] != VERSION.prerelease[1] && break check + if VERSION.prerelease[1] == "DEV" + # manifests don't store the 2nd part of prerelease, so cannot check further + # so treat them specially in the warning + push!(dev_manifests, manifest_file) + end + continue + end push!(unsuitable_manifests, string(manifest_file, " (v", man_julia_version, ")")) end return unsuitable_manifests, dev_manifests @@ -2458,24 +2848,43 @@ function require(uuidkey::PkgId) return invoke_in_world(world, __require, uuidkey) end __require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey) +# Enabled by `include_package_for_output` so the precompile worker can attribute +# wall-clock time spent loading dependencies from disk. Only outermost (depth==0) +# calls accumulate to avoid double-counting transitive `require`s. +const _precompile_track_dep_load = Ref{Bool}(false) +const _precompile_dep_load_ns = Ref{UInt64}(0) +const _precompile_dep_load_depth = Ref{Int}(0) function _require_prelocked(uuidkey::PkgId, env=nothing) assert_havelock(require_lock) - m = start_loading(uuidkey, UInt128(0), true) - if m === nothing - last = toplevel_load[] - try - toplevel_load[] = false - m = __require_prelocked(uuidkey, env) - m isa Module || check_package_module_loaded_error(uuidkey) - finally - toplevel_load[] = last - end_loading(uuidkey, m) + track = _precompile_track_dep_load[] + t0 = UInt64(0) + if track + _precompile_dep_load_depth[] == 0 && (t0 = time_ns()) + _precompile_dep_load_depth[] += 1 + end + try + m = start_loading(uuidkey, UInt128(0), true) + if m === nothing + last = toplevel_load[] + try + toplevel_load[] = false + m = __require_prelocked(uuidkey, env) + m isa Module || check_package_module_loaded_error(uuidkey) + finally + toplevel_load[] = last + end_loading(uuidkey, m) + end + insert_extension_triggers(uuidkey) + # After successfully loading, notify downstream consumers + run_package_callbacks(uuidkey) + end + return m + finally + if track + _precompile_dep_load_depth[] -= 1 + _precompile_dep_load_depth[] == 0 && (_precompile_dep_load_ns[] = _precompile_dep_load_ns[] +% (time_ns() -% t0)) end - insert_extension_triggers(uuidkey) - # After successfully loading, notify downstream consumers - run_package_callbacks(uuidkey) end - return m end mutable struct PkgOrigin @@ -2537,6 +2946,11 @@ register_root_module(Main) # to the loaded_modules table instead of getting bindings. baremodule __toplevel__ using Base +global var"#_internal_julia_parse" = Base.VersionedParse(VERSION) +global _internal_julia_lower = Core._lower + +# Used for version checking of precompiled cache files only +global _internal_syntax_version::UInt8 = 0 end # get a top-level Module from the given key @@ -2576,8 +2990,6 @@ function set_pkgorigin_version_path(pkg::PkgId, path::String) nothing end -# Unused -const PKG_PRECOMPILE_HOOK = Ref{Function}() disable_parallel_precompile::Bool = false # Returns `nothing` or the new(ish) module @@ -2585,32 +2997,35 @@ function __require_prelocked(pkg::PkgId, env) assert_havelock(require_lock) # perform the search operation to select the module file require intends to load - path = locate_package(pkg, env) - if path === nothing + specenv = locate_package_env(pkg, env) + if specenv === nothing throw(ArgumentError(""" Package $(repr("text/plain", pkg)) is required but does not seem to be installed: - Run `Pkg.instantiate()` to install all recorded dependencies. """)) end + spec = specenv[1] + path = spec.path set_pkgorigin_version_path(pkg, path) - parallel_precompile_attempted = false # being safe to avoid getting stuck in a precompilepkgs loop - reasons = Dict{String,Int}() + parallel_precompile_attempted = Ref(false) # being safe to avoid getting stuck in a precompilepkgs loop + cache_fetch_attempted = Ref(false) # the cache-fetch hook gets one shot, then we compile + reasons = Dict{Symbol,Int}() # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 @label load_from_cache - loaded = _require_search_from_serialized(pkg, path, UInt128(0), true; reasons) + loaded = _require_search_from_serialized(pkg, spec, UInt128(0), true; reasons) if loaded isa Module return loaded end end - if JLOptions().use_compiled_modules == 3 - error("Precompiled image $pkg not available with flags $(CacheFlags())") - end - # if the module being required was supposed to have a particular version - # but it was not handled by the precompile loader, complain + # but it was not handled by the precompile loader, complain. This runs before + # the strict-mode check: the pinned build id is the one the parent session has + # loaded, and once its cache file is gone nothing a worker can do will produce + # it again, so the dependent has to be loaded from source in that session + # rather than reported as a precompilation failure. for (concrete_pkg, concrete_build_id) in _concrete_dependencies if pkg == concrete_pkg @warn """Module $(pkg.name) with build ID $((UUID(concrete_build_id))) is missing from the cache. @@ -2622,43 +3037,78 @@ function __require_prelocked(pkg::PkgId, env) end end + if JLOptions().use_compiled_modules == 3 + error("Precompiled image $pkg not available with flags $(CacheFlags())$(list_reasons(reasons; full=true))") + end + if JLOptions().use_compiled_modules == 1 if !generating_output(#=incremental=#false) - project = active_project() - if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) - parallel_precompile_attempted = true - unlock(require_lock) - try - Precompilation.precompilepkgs([pkg.name]; _from_loading=true, ignore_loaded=false) - finally - lock(require_lock) - end + # If a background precompile task is working on this package, + # temporarily monitor it until done, then retry from cache. + if Precompilation.wait_for_pending_package(pkg) @goto load_from_cache end # spawn off a new incremental pre-compile task for recursive `require` calls - loaded = maybe_cachefile_lock(pkg, path) do - # double-check the search now that we have lock - m = _require_search_from_serialized(pkg, path, UInt128(0), true) - m isa Module && return m - triggers = get(EXT_PRIMED, pkg, nothing) - loadable_exts = nothing - if triggers !== nothing # extension - loadable_exts = PkgId[] - for (ext′, triggers′) in EXT_PRIMED - if triggers′ ⊊ triggers - push!(loadable_exts, ext′) + loaded = let spec = spec, reasons = reasons, parallel_precompile_attempted = parallel_precompile_attempted, + cache_fetch_attempted = cache_fetch_attempted + maybe_cachefile_lock(pkg, spec.path) do + # double-check the search now that we have lock + m = _require_search_from_serialized(pkg, spec, UInt128(0), true) + m isa Module && return m + + local verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug + @logmsg verbosity "Precompiling $(pkg_log_name(pkg))$(list_reasons(reasons))" + + unlock(require_lock) + try + # a cache-fetch hook gets one chance to materialize a + # cachefile before we spend time compiling; returning + # `nothing` retries the cache search from the top + if !cache_fetch_attempted[] && CACHE_FETCH_HOOK[] !== nothing + cache_fetch_attempted[] = true + maybe_fetch_cache(pkg, spec.path) && return nothing + end + if !generating_output() && !parallel_precompile_attempted[] && !disable_parallel_precompile && @isdefined(Precompilation) + parallel_precompile_attempted[] = true + # Note that we use @invokelatest here to avoid world + # age issues when printing, see: + # https://github.com/JuliaLang/julia/issues/60223 + precompiled = @invokelatest Precompilation.precompilepkgs([pkg]; _from_loading=true, ignore_loaded=false) + # precompiled returns either nothing, indicating it needs serial precompile, + # or the entry(ies) that it found would be best to load (possibly because it just created it) + # or an empty set of entries (indicating the precompile should be skipped) + if precompiled !== nothing + isempty(precompiled) && return PrecompilableError() # oops, Precompilation forgot to report what this might actually be + local cachefile = precompiled[1] + local ocachefile = nothing + if JLOptions().use_pkgimages == 1 + ocachefile = ocachefile_from_cachefile(cachefile) + end + return cachefile, ocachefile + end end + triggers = get(EXT_PRIMED, pkg, nothing) + loadable_exts = nothing + if triggers !== nothing # extension + loadable_exts = PkgId[] + for (ext′, triggers′) in EXT_PRIMED + if triggers′ ⊊ triggers + push!(loadable_exts, ext′) + end + end + end + return compilecache(pkg, spec; loadable_exts) + finally + lock(require_lock) end end - return compilecache(pkg, path; reasons, loadable_exts) end loaded isa Module && return loaded if isnothing(loaded) # maybe_cachefile_lock returns nothing if it had to wait for another process @goto load_from_cache # the new cachefile will have the newest mtime so will come first in the search elseif isa(loaded, Exception) if precompilableerror(loaded) - verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug - @logmsg verbosity "Skipping precompilation due to precompilable error. Importing $(repr("text/plain", pkg))." exception=loaded + # Intentionally not logging - __precompile__(false) is not an error else @warn "The call to compilecache failed to create a usable precompiled cache file for $(repr("text/plain", pkg))" exception=loaded end @@ -2687,11 +3137,13 @@ function __require_prelocked(pkg::PkgId, env) if uuid !== old_uuid ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, uuid) end + __toplevel__.var"#_internal_julia_parse" = VersionedParse(spec.julia_syntax_version) unlock(require_lock) try include(__toplevel__, path) loaded = maybe_root_module(pkg) finally + __toplevel__.var"#_internal_julia_parse" = VersionedParse(VERSION) lock(require_lock) if uuid !== old_uuid ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid) @@ -2718,6 +3170,15 @@ end """ require_stdlib(package_uuidkey::PkgId, [ext::String, from::Module]) +Load a standard library package from the bundled Julia depot, loading precompiled cache +files without requiring source files to be present. This function is designed to load +stdlib packages even when `JULIA_DEPOT_PATH` doesn't include the bundled depot directory, +enabling stdlib usage in isolated or restricted environments. + +Unlike `require`, this function loads `.ji` cache files directly from the bundled depot +without source staleness checks (since stdlibs are immutable for a given Julia version). +If the bundled depot cache is unavailable, it falls back to normal package loading. + !!! warning "May load duplicate copies of stdlib packages." This requires that all stdlib packages loaded are compatible with having concurrent @@ -2789,7 +3250,7 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}, fro sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext) end set_pkgorigin_version_path(this_uuidkey, sourcepath) - newm = _require_search_from_serialized(this_uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + newm = _require_search_from_serialized(this_uuidkey, PkgLoadSpec(sourcepath, VERSION), UInt128(0), false; DEPOT_PATH=depot_path) end finally end_loading(this_uuidkey, newm) @@ -2800,7 +3261,7 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}, fro run_package_callbacks(this_uuidkey) else # if the user deleted their bundled depot, next try to load it completely normally - # if it is an extension, we first need to indicate where to find its parant via EXT_PRIMED + # if it is an extension, we first need to indicate where to find its parent via EXT_PRIMED ext isa String && (EXT_PRIMED[this_uuidkey] = PkgId[package_uuidkey]) newm = _require_prelocked(this_uuidkey) end @@ -2826,12 +3287,8 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString, filename::AbstractString="string") loc = LineNumberNode(1, Symbol(filename)) try - ast = Meta.parseall(code, filename=filename) - if !Meta.isexpr(ast, :toplevel) - @assert Core._lower != fl_lower - # Only reached when JuliaLowering and alternate parse functions are activated - return Core.eval(mod, ast) - end + _parse = invokelatest(Meta.parser_for_module, mod) + ast = Meta.parseall(code; filename, _parse) result = nothing line_and_ex = Expr(:toplevel, loc, nothing) for ex in ast.args @@ -2844,6 +3301,28 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString, # Wrap things to be eval'd in a :toplevel expr to carry line # information as part of the expr. line_and_ex.args[2] = ex + # Check global TRACE_EVAL first, fall back to command line option + trace_eval_setting = TRACE_EVAL + trace_eval = if trace_eval_setting !== nothing + # Convert symbol to integer value + setting = trace_eval_setting + if setting === :no + 0 + elseif setting === :loc + 1 + elseif setting === :full + 2 + else + error("Invalid TRACE_EVAL value: $(setting). Must be :no, :loc, or :full") + end + else + JLOptions().trace_eval + end + if trace_eval == 2 # show everything + println(stderr, "eval: ", line_and_ex) + elseif trace_eval == 1 # show top location only + println(stderr, "eval: ", line_and_ex.args[1]) + end result = Core.eval(mod, line_and_ex) end return result @@ -2897,6 +3376,8 @@ Base.include # defined in Base.jl function _include(mapexpr::Function, mod::Module, _path::AbstractString) @noinline # Workaround for module availability in _simplify_include_frames path, prev = _include_dependency(mod, _path) + # Record a non-identity transform so it survives precompilation and can be re-applied on revision. + _record_include_mapexpr!(mod, path, mapexpr) for callback in include_callbacks # to preserve order, must come before eval in include_string invokelatest(callback, mod, path) end @@ -2970,11 +3451,12 @@ function load_path_setup_code(load_path::Bool=true) end # Const global for GC root -const newly_inferred = CodeInstance[] +const newly_inferred = [] # this is called in the external process that generates precompiled package files -function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, - concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}) +function include_package_for_output(pkg::PkgId, input::String, syntax_version::VersionNumber, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, + concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}, + preresolved::Vector{Pair{PkgId,String}}=Pair{PkgId,String}[]) @lock require_lock begin m = start_loading(pkg, UInt128(0), false) @@ -2987,6 +3469,9 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto Base._track_dependencies[] = true get!(Base.PkgOrigin, Base.pkgorigins, pkg).path = input append!(empty!(Base._concrete_dependencies), concrete_deps) + for (k, v) in preresolved + preresolved_cachefiles[k] = v + end end uuid_tuple = pkg.uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, pkg.uuid) @@ -2997,14 +3482,43 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto end ccall(:jl_set_newly_inferred, Cvoid, (Any,), newly_inferred) + # When this worker is producing a native object pkgimage, retain raw + # inferred IR on `CodeInstance.inferred` for the non-inlineable methods + # that would otherwise be discarded, so the irgen pass can short-circuit + # `typeinf_ext` instead of re-inferring. `jl_finalize_precompile_inferred` + # clears them (and the flag) before staticdata serialization. + keep_ir = JLOptions().outputo != C_NULL + keep_ir && ccall(:jl_set_precompile_keep_ir, Cvoid, (Int8,), 1) + # This one changes the parser behavior + __toplevel__.var"#_internal_julia_parse" = VersionedParse(syntax_version) + # This one is the compatibility marker for cache loading + __toplevel__._internal_syntax_version = cache_syntax_version(syntax_version) + cumulative_compile_timing(true) + _precompile_dep_load_ns[] = 0 + _precompile_dep_load_depth[] = 0 + _precompile_track_dep_load[] = true + t_include_start = time_ns() + t_comp_before, _ = cumulative_compile_time_ns() try - Base.include(Base.__toplevel__, input) + Compiler.@zone "PRECOMPILE_INCLUDE" Base.include(Base.__toplevel__, input) catch ex precompilableerror(ex) || rethrow() @debug "Aborting `create_expr_cache'" exception=(ErrorException("Declaration of __precompile__(false) not allowed"), catch_backtrace()) exit(125) # we define status = 125 means PrecompileableError finally + t_comp_after, _ = cumulative_compile_time_ns() + t_include_end = time_ns() + cumulative_compile_timing(false) + _precompile_track_dep_load[] = false + if Base.get_bool_env("JULIA_PRECOMP_REPORT_TIMING", false) + println(stderr, PRECOMPILE_VERBOSE_TIMING_MARKER, + " include_ns=", t_include_end -% t_include_start, + " deps_ns=", _precompile_dep_load_ns[], + " compilation_ns=", t_comp_after -% t_comp_before, + " methods=", length(newly_inferred)) + end ccall(:jl_set_newly_inferred, Cvoid, (Any,), nothing) + keep_ir && ccall(:jl_set_precompile_keep_ir, Cvoid, (Int8,), 0) end # check that the package defined the expected module so we can give a nice error message if not m = maybe_root_module(pkg) @@ -3032,12 +3546,15 @@ _pkg_str(_pkg::Pair{PkgId}) = _pkg_str(_pkg.first) * " => " * repr(_pkg.second) _pkg_str(_pkg::Nothing) = "nothing" const PRECOMPILE_TRACE_COMPILE = Ref{String}() -function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, +# Marker prefix used by the precompile subprocess to report per-package timing +# buckets back to the parent process; the parent surfaces them in verbose mode. +const PRECOMPILE_VERBOSE_TIMING_MARKER = "__JL_PRECOMP_VERBOSE_TIMING__" +function create_expr_cache(pkg::PkgId, input::PkgLoadSpec, output::String, output_o::Union{Nothing, String}, concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), - internal_stderr::IO = stderr, internal_stdout::IO = stdout, loadable_exts::Union{Vector{PkgId},Nothing}=nothing) + internal_stderr::IO = stderr, internal_stdout::IO = stdout, loadable_exts::Union{Vector{PkgId},Nothing}=nothing; + report_timing::Bool=false, + preresolved::Vector{Pair{PkgId,String}} = @lock(require_lock, collect(preresolved_cachefiles))) @nospecialize internal_stderr internal_stdout - rm(output, force=true) # Remove file if it exists - output_o === nothing || rm(output_o, force=true) depot_path = String[abspath(x) for x in DEPOT_PATH] dl_load_path = String[abspath(x) for x in DL_LOAD_PATH] load_path = String[abspath(x) for x in Base.load_path()] @@ -3067,6 +3584,9 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: cacheflags = CacheFlags(cacheflags, opt_level=0) end opts = translate_cache_flags(cacheflags, CacheFlags()) # julia_cmd is generated for the running system, and must be fixed if running for precompile instead + # julia_cmd forwards --code-coverage only for pid-dependent output paths, + # so request the image's instrumentation explicitly (later options win) + append!(opts, coverage_cache_options(cacheflags)) if output_o !== nothing @debug "Generating object cache file for $(repr("text/plain", pkg))" cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) @@ -3081,16 +3601,18 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: push!(opts, "--trace-compile-timing") end - io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) - $(flags) - $(opts) - --output-incremental=yes - --startup-file=no --history-file=no --warn-overwrite=yes - $(have_color === nothing ? "--color=auto" : have_color ? "--color=yes" : "--color=no") - -`, - "OPENBLAS_NUM_THREADS" => 1, - "JULIA_NUM_THREADS" => 1), - stderr = internal_stderr, stdout = internal_stdout), + cmd = `$(julia_cmd(;cpu_target)::Cmd) + $(flags) + $(opts) + --output-incremental=yes + --startup-file=no --history-file=no --warn-overwrite=yes + $(have_color === nothing ? "--color=auto" : have_color ? "--color=yes" : "--color=no") + -` + cmd = addenv(cmd, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) + # Only request per-package timing reports when explicitly asked for (e.g. by + # precompilepkgs), so that the marker lines don't leak into normal load logs. + report_timing && (cmd = addenv(cmd, "JULIA_PRECOMP_REPORT_TIMING" => 1)) + io = open(pipeline(cmd, stderr = internal_stderr, stdout = internal_stdout), "w", stdout) # write data over stdin to avoid the (unlikely) case of exceeding max command line size write(io.in, """ @@ -3098,8 +3620,8 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: Base.track_nested_precomp($(_pkg_str(vcat(Base.precompilation_stack, pkg)))) Base.loadable_extensions = $(_pkg_str(loadable_exts)) Base.precompiling_extension = $(loading_extension) - Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), - $(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing)))) + Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input.path))), $(repr(input.julia_syntax_version)), $(repr(depot_path)), $(repr(dl_load_path)), + $(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing))), $(_pkg_str(preresolved))) """) close(io.in) return io @@ -3122,7 +3644,7 @@ function compilecache_dir(pkg::PkgId) return joinpath(DEPOT_PATH[1], entrypath) end -function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=CacheFlags(), project::String=something(Base.active_project(), ""))::String +function compilecache_path(pkg::PkgId, prefs_blob::String; flags::CacheFlags=CacheFlags(), project::String=something(Base.active_project(), ""))::String entrypath, entryfile = cache_file_entry(pkg) cachepath = joinpath(DEPOT_PATH[1], entrypath) isdir(cachepath) || mkpath(cachepath) @@ -3133,14 +3655,14 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=Cac crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) crc = _crc32c(_cacheflag_to_uint8(flags), crc) + crc = _crc32c(UInt8(flags.coverage), crc) cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) if cpu_target === nothing cpu_target = unsafe_string(JLOptions().cpu_target) end crc = _crc32c(cpu_target, crc) - - crc = _crc32c(prefs_hash, crc) + crc = _crc32c(prefs_blob, crc) project_precompile_slug = slug(crc, 5) abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) end @@ -3149,23 +3671,25 @@ end """ Base.compilecache(module::PkgId) -Creates a precompiled cache file for a module and all of its dependencies. +Create a precompiled cache file for a module and all of its dependencies. This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), loadable_exts::Union{Vector{PkgId},Nothing}=nothing) +function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), loadable_exts::Union{Vector{PkgId},Nothing}=nothing, signal_channel::Union{Channel{Int32},Nothing}=nothing, report_timing::Bool=false) @nospecialize internal_stderr internal_stdout - path = locate_package(pkg) - path === nothing && throw(ArgumentError("$(repr("text/plain", pkg)) not found during precompilation")) - return compilecache(pkg, path, internal_stderr, internal_stdout; flags, cacheflags, reasons, loadable_exts) + spec = locate_package_load_spec(pkg) + spec === nothing && throw(ArgumentError("$(repr("text/plain", pkg)) not found during precompilation")) + return compilecache(pkg, spec, internal_stderr, internal_stdout; flags, cacheflags, loadable_exts, signal_channel, report_timing) end const MAX_NUM_PRECOMPILE_FILES = Ref(10) -function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, +function compilecache(pkg::PkgId, spec::PkgLoadSpec, internal_stderr::IO = stderr, internal_stdout::IO = stdout, keep_loaded_modules::Bool = true; flags::Cmd=``, cacheflags::CacheFlags=CacheFlags(), - reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}(), loadable_exts::Union{Vector{PkgId},Nothing}=nothing) + loadable_exts::Union{Vector{PkgId},Nothing}=nothing, signal_channel::Union{Channel{Int32},Nothing}=nothing, + pid_channel::Union{Channel{Int32},Nothing}=nothing, report_timing::Bool=false, + preresolved::Vector{Pair{PkgId,String}} = @lock(require_lock, collect(preresolved_cachefiles))) @nospecialize internal_stderr internal_stdout # decide where to put the resulting cache file @@ -3183,8 +3707,6 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in concrete_deps = empty(_concrete_dependencies) end # run the expression and cache the result - verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug - @logmsg verbosity "Precompiling $(repr("text/plain", pkg)) $(list_reasons(reasons))" # create a temporary file in `cachepath` directory, write the cache in it, # write the checksum, _and then_ atomically move the file to `cachefile`. @@ -3198,36 +3720,68 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in else tmppath_o = nothing end - local p + local p::Base.Process try close(tmpio) if cache_objects close(tmpio_o) close(tmpio_so) end - p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, cacheflags, internal_stderr, internal_stdout, loadable_exts) + p = create_expr_cache(pkg, spec, tmppath, tmppath_o, concrete_deps, flags, cacheflags, internal_stderr, internal_stdout, loadable_exts; report_timing, preresolved) + + # Report the PID of the compilation subprocess + if pid_channel !== nothing + try; put!(pid_channel, Int32(getpid(p))); catch; end + end - if success(p) + # Forward signals from the channel to the subprocess + if signal_channel !== nothing + let p = p + Base.errormonitor(Threads.@spawn :samepool begin + for sig in signal_channel + process_running(p) || break + try + kill(p, sig) + catch e + e isa IOError && break + rethrow() + end + end + end) + end + end + + local result + try + result = success(p) + finally + if signal_channel !== nothing + close(signal_channel) + end + end + + if result if cache_objects # Run linker over tmppath_o - Linking.link_image(tmppath_o, tmppath_so) + Compiler.@zone "PRECOMPILE_LINK" Linking.link_image(tmppath_o, tmppath_so) end - # Read preferences hash back from .ji file (we can't precompute because - # we don't actually know what the list of compile-time preferences are without compiling) - prefs_hash = preferences_hash(tmppath) - cachefile = compilecache_path(pkg, prefs_hash; flags=cacheflags) + # Read preferences blob back from .ji file (we can't precompute because we don't + # actually know what the list of compile-time preferences are without compiling) + prefs_blob = preferences_blob(tmppath) + cachefile = compilecache_path(pkg, prefs_blob; flags=cacheflags) ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing # append checksum for so to the end of the .ji file: - crc_so = UInt32(0) - if cache_objects - crc_so = open(_crc32c, tmppath_so, "r") + crc_so = if cache_objects + open(_crc32c, tmppath_so, "r") + else + UInt32(0) end # append extra crc to the end of the .ji file: open(tmppath, "r+") do f - if iszero(isvalid_cache_header(f)) + if isvalid_cache_header(f) === nothing error("Incompatible header for $(repr("text/plain", pkg)) in new cache file $(repr(tmppath)).") end seekend(f) @@ -3237,7 +3791,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in end # inherit permission from the source file (and make them writable) - chmod(tmppath, filemode(path) & 0o777 | 0o200) + chmod(tmppath, filemode(spec.path) & 0o777 | 0o200) # prune the directory with cache files if pkg.uuid !== nothing @@ -3276,6 +3830,9 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in return cachefile, ocachefile end finally + if signal_channel !== nothing + close(signal_channel) + end rm(tmppath, force=true) if cache_objects rm(tmppath_o::String, force=true) @@ -3301,7 +3858,7 @@ function rename_unique_ocachefile(tmppath_so::String, ocachefile_orig::String, o end # Windows prevents renaming a file that is in use so if there is a Julia session started # with a package image loaded, we cannot rename that file. - # The code belows append a `_i` to the name of the cache file where `i` is the smallest number such that + # The code below appends a `_i` to the name of the cache file where `i` is the smallest number such that # that cache file does not exist. ocachename, ocacheext = splitext(ocachefile_orig) ocachefile_unique = ocachename * "_$num" * ocacheext @@ -3318,15 +3875,19 @@ function object_build_id(obj) return module_build_id(mod::Module) end +const JI_FLAG_PKGIMAGE::UInt32 = 1 << 0 +const JI_FLAG_SPLIT::UInt32 = 1 << 1 + function isvalid_cache_header(f::IOStream) - pkgimage = Ref{UInt8}() - checksum = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid}, Ptr{UInt8}, Ptr{Int64}, Ptr{Int64}), f.ios, pkgimage, Ref{Int64}(), Ref{Int64}()) # returns checksum id or zero + flags = Ref{UInt32}() + checksum = Ref{UInt32}() + err = ccall(:jl_read_verify_header, Cint, (Ptr{Cvoid}, Ptr{UInt32}, Ptr{UInt32}, Ptr{Int64}, Ptr{Int64}), f.ios, flags, checksum, Ref{Int64}(), Ref{Int64}()) - if !iszero(checksum) && pkgimage[] != 0 - @debug "Cache header was for pkgimage" - return UInt64(0) # We somehow read the header for a pkgimage and not a ji + if err == 0 && (flags[] & JI_FLAG_PKGIMAGE == 0) + @debug "Cache header was for a system image" + return nothing # We somehow read the header for a system image end - return checksum + return err == 0 ? checksum[] : nothing end isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32)) @@ -3378,8 +3939,13 @@ function restore_depot_path(path::AbstractString, depot::AbstractString) replace(path, r"^@depot" => depot; count=1) end -function resolve_depot(inc::AbstractString) +function resolve_depot(inc::AbstractString, hint::Union{String, Nothing}=nothing) startswith(inc, string("@depot", Filesystem.pathsep())) || return :not_relocatable + # include_dependency() files usually live in the same depot as the source + # files of their cache, so try that depot first. + if hint !== nothing && ispath(restore_depot_path(inc, hint)) + return hint + end for depot in DEPOT_PATH ispath(restore_depot_path(inc, depot)) && return depot end @@ -3401,7 +3967,8 @@ function read_module_list(f::IO, has_buildid_hi::Bool) end function _parse_cache_header(f::IO, cachefile::AbstractString) - flags = read(f, UInt8) + flags = CacheFlags(read(f, UInt8), read(f, UInt8)) + syntax_version = read(f, UInt8) modules = read_module_list(f, false) totbytes = Int64(read(f, UInt64)) # total bytes for file dependencies + preferences # read the list of requirements @@ -3445,18 +4012,10 @@ function _parse_cache_header(f::IO, cachefile::AbstractString) push!(includes, CacheHeaderIncludes(modkey, depname, fsize, hash, mtime, modpath)) end end - prefs = String[] - while true - n2 = read(f, Int32) - totbytes -= 4 - if n2 == 0 - break - end - push!(prefs, String(read(f, n2))) - totbytes -= n2 - end - prefs_hash = read(f, UInt64) - totbytes -= 8 + n2 = read(f, Int32) + totbytes -= 4 + prefs_blob = String(read(f, n2)) + totbytes -= n2 srctextpos = read(f, Int64) totbytes -= 8 @assert totbytes == 0 "header of cache file appears to be corrupt (totbytes == $(totbytes))" @@ -3467,12 +4026,12 @@ function _parse_cache_header(f::IO, cachefile::AbstractString) srcfiles = srctext_files(f, srctextpos, includes) - return modules, (includes, srcfiles, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags + return modules, (includes, srcfiles, requires), required_modules, srctextpos, prefs_blob, clone_targets, flags, syntax_version end function parse_cache_header(f::IO, cachefile::AbstractString) modules, (includes, srcfiles, requires), required_modules, - srctextpos, prefs, prefs_hash, clone_targets, flags = _parse_cache_header(f, cachefile) + srctextpos, prefs_blob, clone_targets, flags, syntax_version = _parse_cache_header(f, cachefile) includes_srcfiles = CacheHeaderIncludes[] includes_depfiles = CacheHeaderIncludes[] @@ -3489,34 +4048,49 @@ function parse_cache_header(f::IO, cachefile::AbstractString) # 1. If the cache is not relocatable because of an absolute path, # we ignore that path for the depot search. # Recompilation will be triggered by stale_cachefile() if that absolute path does not exist. - # 2. If we can't find a depot for a relocatable path, - # we still replace it with the depot we found from other files. - # Recompilation will be triggered by stale_cachefile() because the resolved path does not exist. - # 3. We require that relocatable paths all resolve to the same depot. - # 4. We explicitly check that all relocatable paths resolve to the same depot. This has two reasons: - # - We want to scan all source files in order to provide logs for 1. and 2. above. - # - It is possible that a depot might be missing source files. - # Assume that we have two depots on DEPOT_PATH, depot_complete and depot_incomplete. - # If DEPOT_PATH=["depot_complete","depot_incomplete"] then no recompilation shall happen, - # because depot_complete will be picked. - # If DEPOT_PATH=["depot_incomplete","depot_complete"] we trigger recompilation and - # hopefully a meaningful error about missing files is thrown. - # If we were to just select the first depot we find, then whether recompilation happens would - # depend on whether the first relocatable file resolves to depot_complete or depot_incomplete. - srcdepot = nothing + # 2. We require that a single depot contains all relocatable paths, and pick the first such + # depot in DEPOT_PATH order. This choice does not depend on the iteration order of + # `srcfiles`: with a depot missing some of the files ("depot_incomplete") and a depot + # containing all of them ("depot_complete") on DEPOT_PATH, depot_complete is picked + # regardless of their relative order. Probing a depot stops at its first missing file, + # so files of a cache belonging to a later depot (e.g. bundled stdlib caches) do not + # pay one miss per file on each depot preceding theirs. + # 3. If no depot contains all relocatable paths, we scan the files individually, only to + # provide logs distinguishing files missing from every depot from files spread over + # several. The paths keep their @depot tag and stale_cachefile() will trigger + # recompilation. any_not_relocatable = false any_no_depot_found = false multiple_depots_found = false + depot_tag = string("@depot", Filesystem.pathsep()) + relocatable_srcfiles = String[] for src in srcfiles - depot = resolve_depot(src) - if depot === :not_relocatable + if startswith(src, depot_tag) + push!(relocatable_srcfiles, src) + else any_not_relocatable = true - elseif depot === :no_depot_found - any_no_depot_found = true - elseif isnothing(srcdepot) - srcdepot = depot - elseif depot != srcdepot - multiple_depots_found = true + end + end + srcdepot = nothing + if !isempty(relocatable_srcfiles) + for depot in DEPOT_PATH + if all(src -> ispath(restore_depot_path(src, depot)), relocatable_srcfiles) + srcdepot = depot + break + end + end + if srcdepot === nothing + founddepot = nothing + for src in relocatable_srcfiles + depot = resolve_depot(src) + if depot === :no_depot_found + any_no_depot_found = true + elseif founddepot === nothing + founddepot = depot + elseif depot != founddepot + multiple_depots_found = true + end + end end end if any_no_depot_found @@ -3536,7 +4110,7 @@ function parse_cache_header(f::IO, cachefile::AbstractString) # unlike include() files, we allow each relocatable include_dependency() file to resolve # to a separate depot, #52161 for inc in includes_depfiles - depot = resolve_depot(inc.filename) + depot = resolve_depot(inc.filename, srcdepot) if depot === :no_depot_found @debug("Unable to resolve @depot tag for include_dependency() file $(inc.filename) from cache file $cachefile", _group=:relocatable) elseif depot === :not_relocatable @@ -3546,13 +4120,13 @@ function parse_cache_header(f::IO, cachefile::AbstractString) end end - return modules, (includes, includes_srcfiles, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags + return modules, (includes, includes_srcfiles, requires), required_modules, srctextpos, prefs_blob, clone_targets, flags, syntax_version end function parse_cache_header(cachefile::String) io = open(cachefile, "r") try - iszero(isvalid_cache_header(io)) && throw(ArgumentError("Incompatible header in cache file $cachefile.")) + isvalid_cache_header(io) === nothing && throw(ArgumentError("Incompatible header in cache file $cachefile.")) ret = parse_cache_header(io, cachefile) return ret finally @@ -3560,14 +4134,14 @@ function parse_cache_header(cachefile::String) end end -preferences_hash(f::IO, cachefile::AbstractString) = parse_cache_header(f, cachefile)[6] -function preferences_hash(cachefile::String) +preferences_blob(f::IO, cachefile::AbstractString) = parse_cache_header(f, cachefile)[5] +function preferences_blob(cachefile::String) io = open(cachefile, "r") try - if iszero(isvalid_cache_header(io)) + if isvalid_cache_header(io) === nothing throw(ArgumentError("Incompatible header in cache file $cachefile.")) end - return preferences_hash(io, cachefile) + return preferences_blob(io, cachefile) finally close(io) end @@ -3581,7 +4155,7 @@ end function cache_dependencies(cachefile::String) io = open(cachefile, "r") try - iszero(isvalid_cache_header(io)) && throw(ArgumentError("Incompatible header in cache file $cachefile.")) + isvalid_cache_header(io) === nothing && throw(ArgumentError("Incompatible header in cache file $cachefile.")) return cache_dependencies(io, cachefile) finally close(io) @@ -3589,7 +4163,7 @@ function cache_dependencies(cachefile::String) end function read_dependency_src(io::IO, cachefile::AbstractString, filename::AbstractString) - _, (includes, _, _), _, srctextpos, _, _, _, _ = parse_cache_header(io, cachefile) + _, (includes, _, _), _, srctextpos, _, _, _ = parse_cache_header(io, cachefile) srctextpos == 0 && error("no source-text stored in cache file") seek(io, srctextpos) return _read_dependency_src(io, filename, includes) @@ -3621,7 +4195,7 @@ end function read_dependency_src(cachefile::String, filename::AbstractString) io = open(cachefile, "r") try - iszero(isvalid_cache_header(io)) && throw(ArgumentError("Incompatible header in cache file $cachefile.")) + isvalid_cache_header(io) === nothing && throw(ArgumentError("Incompatible header in cache file $cachefile.")) return read_dependency_src(io, cachefile, filename) finally close(io) @@ -3726,34 +4300,34 @@ function collect_preferences(project_toml::String, uuid::Union{UUID,Nothing}) end """ - recursive_prefs_merge(base::Dict, overrides::Dict...) + recursive_prefs_merge(base::Dict{String, Any}, overrides::Vector{Dict{String, Any}}) Helper function to merge preference dicts recursively, honoring overrides in nested dictionaries properly. """ -function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String, Any}...) - new_base = Base._typeddict(base, overrides...) +function recursive_prefs_merge(base::Dict{String, Any}, overrides::Vector{Dict{String, Any}}) + merged = copy(base) for override in overrides # Clear entries are keys that should be deleted from any previous setting. override_clear = get(override, "__clear__", nothing) if override_clear isa Vector{String} for k in override_clear - delete!(new_base, k) + delete!(merged, k) end end for (k, override_k) in override # Note that if `base` has a mapping that is _not_ a `Dict`, and `override` - new_base_k = get(new_base, k, nothing) - if new_base_k isa Dict{String, Any} && override_k isa Dict{String, Any} - new_base[k] = recursive_prefs_merge(new_base_k, override_k) + merged_k = get(merged, k, nothing) + if merged_k isa Dict{String, Any} && override_k isa Dict{String, Any} + merged[k] = recursive_prefs_merge(merged_k, Dict{String,Any}[override_k]) else - new_base[k] = override_k + merged[k] = override_k end end end - return new_base + return merged end function get_projects_workspace_to_root(project_file) @@ -3784,47 +4358,53 @@ function get_preferences(uuid::Union{UUID,Nothing} = nothing) # Collect all dictionaries from the current point in the load path, then merge them in dicts = collect_preferences(project_toml, uuid) - merged_prefs = recursive_prefs_merge(merged_prefs, dicts...) + merged_prefs = recursive_prefs_merge(merged_prefs, dicts) end return merged_prefs end -function get_preferences_hash(uuid::Union{UUID, Nothing}, prefs_list::Vector{String}) - # Start from a predictable hash point to ensure that the same preferences always - # hash to the same value, modulo changes in how Dictionaries are hashed. - h = UInt(0) - uuid === nothing && return UInt64(h) - - # Load the preferences - prefs = get_preferences(uuid) - - # Walk through each name that's called out as a compile-time preference - for name in prefs_list - prefs_value = get(prefs, name, nothing) - if prefs_value !== nothing - h = hash(prefs_value, h)::UInt - end - end - # We always return a `UInt64` so that our serialization format is stable - return UInt64(h) -end - -get_preferences_hash(m::Module, prefs_list::Vector{String}) = get_preferences_hash(PkgId(m).uuid, prefs_list) - # This is how we keep track of who is using what preferences at compile-time const COMPILETIME_PREFERENCES = Dict{UUID,Set{String}}() -# In `Preferences.jl`, if someone calls `load_preference(@__MODULE__, key)` while we're precompiling, -# we mark that usage as a usage at compile-time and call this method, so that at the end of `.ji` generation, -# we can record the list of compile-time preferences and embed that into the `.ji` header +# This is used by `Preferences.load_preference` to record any (pre)compile-time preference +# queries, which the resulting pkgimage will be keyed on to detect preference changes. function record_compiletime_preference(uuid::UUID, key::String) pref = get!(Set{String}, COMPILETIME_PREFERENCES, uuid) push!(pref, key) return nothing end -get_compiletime_preferences(uuid::UUID) = collect(get(Vector{String}, COMPILETIME_PREFERENCES, uuid)) -get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid) -get_compiletime_preferences(::Nothing) = String[] + +# Return a serialized "blob" of all observed (at compile-time) preferences. +# This is called at pkgimage write time (.ji generation). +function get_preferences_blob() + isempty(COMPILETIME_PREFERENCES) && return "" + observed = Dict{String, Any}() + unset = Dict{String, Any}() + for (uuid, pref_keys) in COMPILETIME_PREFERENCES + uuid_prefs = get_preferences(uuid) + uuid_str = string(uuid) + for key in pref_keys + if haskey(uuid_prefs, key) + uuid_observed = get!(Dict{String, Any}, observed, uuid_str) + uuid_observed[key] = uuid_prefs[key] + else + # preferences that were not set are tracked in a + # separate table to prevent name / value collisions + uuid_unset = get!(Set{Any}, unset, uuid_str) + push!(uuid_unset, key) + end + end + end + for (uuid, uuid_unset) in pairs(unset) + unset[uuid] = sort!(collect(uuid_unset)) + end + if !isempty(unset) + observed["unset"] = unset + end + buf = IOBuffer() + TOML.Printer.print(buf, observed; sorted=true) + return String(take!(buf)) +end function check_clone_targets(clone_targets) rejection_reason = ccall(:jl_check_pkgimage_clones, Any, (Ptr{Cchar},), clone_targets) @@ -3838,10 +4418,10 @@ global mkpidlock_hook::Any global trymkpidlock_hook::Any global parse_pidfile_hook::Any -# The preferences hash is only known after precompilation so just assume no preferences. +# The preferences blob is only known after precompilation so just assume no preferences. # Also ignore the active project, which means that if all other conditions are equal, # the same package cannot be precompiled from different projects and/or different preferences at the same time. -compilecache_pidfile_path(pkg::PkgId; flags::CacheFlags=CacheFlags()) = compilecache_path(pkg, UInt64(0); project="", flags) * ".pidfile" +compilecache_pidfile_path(pkg::PkgId; flags::CacheFlags=CacheFlags()) = compilecache_path(pkg, ""; project="", flags) * ".pidfile" const compilecache_pidlock_stale_age = 10 @@ -3876,36 +4456,139 @@ function maybe_cachefile_lock(f, pkg::PkgId, srcpath::String; stale_age=compilec end end -function record_reason(reasons::Dict{String,Int}, reason::String) +function record_reason(reasons::Dict{Symbol,Int}, reason::Symbol) reasons[reason] = get(reasons, reason, 0) + 1 end -record_reason(::Nothing, ::String) = nothing -function list_reasons(reasons::Dict{String,Int}) +record_reason(::Nothing, ::Symbol) = nothing + +# Reasons why a candidate cache file may be rejected, mapped to a category and a +# human-readable description for the loading log message. +# :actionable — an otherwise-usable cache was rejected, so the details are useful +# :wrong_julia — the cache was built by a different version or build of Julia, which +# is expected when switching versions, so it is collapsed into a single +# generic message and only reported if nothing actionable was seen +# :internal — the candidate simply wasn't the cache being searched for; not reported +const CACHE_REJECT_REASONS = Dict{Symbol,Pair{Symbol,String}}( + :unresolved_depot => :actionable => "file location uses unresolved depot path", + :source_missing => :actionable => "source file not found", + :mtime_changed => :actionable => "file modification time changed", + :fsize_changed => :actionable => "file size changed", + :content_changed => :actionable => "file content changed", + :flags_mismatch => :actionable => "different compilation options", + :pkgimages_disabled => :actionable => "native code caching disabled", + :cpu_target => :actionable => "different system or CPU target", + :ocachefile_missing => :actionable => "native code cache file not found", + :dep_loaded_incompatible => :actionable => "a dependency is already loaded at a different version", + :dep_missing => :actionable => "dependency source file not found", + :source_path_changed => :actionable => "different source file path", + :dep_identity_changed => :actionable => "dependency identifier changed", + :checksum_invalid => :actionable => "cache file checksum is invalid", + :ocache_checksum_invalid => :actionable => "native code cache checksum is invalid", + :preferences_changed => :actionable => "package preferences changed", + :incompatible_header => :wrong_julia => "incompatible cache header", + :julia_version => :wrong_julia => "different Julia version", + :syntax_version => :wrong_julia => "different Julia syntax version", + :pkgid_mismatch => :internal => "different package identifier", + :buildid_mismatch => :internal => "different build identifier", + :dep_buildid_mismatch => :internal => "different dependency build identifier", +) + +# `:dep_loaded_incompatible` is recorded with the dependency's name appended, so the +# message can say which package is loaded at a different version than the cache expects. +const DEP_LOADED_INCOMPATIBLE_PREFIX = "dep_loaded_incompatible:" + +function reject_reason(key::Symbol) + reason = get(CACHE_REJECT_REASONS, key, nothing) + reason === nothing || return reason + keystr = String(key) + if startswith(keystr, DEP_LOADED_INCOMPATIBLE_PREFIX) + name = keystr[length(DEP_LOADED_INCOMPATIBLE_PREFIX)+1:end] + return :actionable => "$name is already loaded at a different version" + end + return :actionable => keystr +end + +function list_reasons(reasons::Dict{Symbol,Int}; full::Bool=false) isempty(reasons) && return "" - return "(cache misses: $(join(("$k ($v)" for (k,v) in reasons), ", ")))" + actionable = String[] + wrong_julia = false + verbose = String[] + for (key, count) in reasons + category, desc = reject_reason(key) + push!(verbose, "$count for $desc") + if category === :actionable + push!(actionable, desc) + elseif category === :wrong_julia + wrong_julia = true + end + end + @debug "Caches not reused: $(join(verbose, ", "))" + full && return " (cache not reused: $(join(sort!(verbose), ", ")))" + if !isempty(actionable) + return " (cache not reused: $(join(sort!(actionable), ", ")))" + elseif wrong_julia + return " (no compatible cache for this version of Julia)" + else + return "" + end +end +list_reasons(::Nothing; full::Bool=false) = "" + +# How a package is named in loading log messages: the bare name when the load path's +# manifests map it to no other uuid, otherwise name and uuid. An extension is named +# by its parent, as the precompile driver does. +function pkg_log_name(pkg::PkgId) + triggers = get(EXT_PRIMED, pkg, nothing) + triggers === nothing || return pkg_log_name(pkg, triggers[1]) + uuid = pkg.uuid + uuid === nothing && return pkg.name + @lock require_lock begin + for env in load_path() + project_file = env_project_file(env) + project_file isa String || continue + manifest_file = project_file_manifest_path(project_file) + manifest_file === nothing && continue + for entry in get(Vector{Any}, get_deps(parsed_toml(manifest_file)), pkg.name) + entry_uuid = get(entry::Dict{String, Any}, "uuid", nothing)::Union{String, Nothing} + entry_uuid === nothing || UUID(entry_uuid) == uuid || return repr("text/plain", pkg) + end + end + end + return pkg.name end -list_reasons(::Nothing) = "" +pkg_log_name(ext::PkgId, parent::PkgId) = "$(pkg_log_name(parent)) → $(ext.name)" -function any_includes_stale(includes::Vector{CacheHeaderIncludes}, cachefile::String, reasons::Union{Dict{String,Int},Nothing}=nothing) +function in_package_store(path::String) + for depot in DEPOT_PATH + prefix = joinpath(depot, "packages", "") + startswith(path, prefix) && return true + end + return false +end + +function any_includes_stale(includes::Vector{CacheHeaderIncludes}, cachefile::String, reasons::Union{Dict{Symbol,Int},Nothing}=nothing) for chi in includes f, fsize_req, hash_req, ftime_req = chi.filename, chi.fsize, chi.hash, chi.mtime if startswith(f, string("@depot", Filesystem.pathsep())) @debug("Rejecting stale cache file $cachefile because its depot could not be resolved") - record_reason(reasons, "nonresolveable depot") + record_reason(reasons, :unresolved_depot) return true end - if !ispath(f) + # A single stat provides existence, mtime and size; permission errors + # propagate like they did from ispath()/mtime()/stat() before. + fstat = stat(f) + if !ispath(fstat) _f = fixup_stdlib_path(f) if _f != f && isfile(_f) && startswith(_f, Sys.STDLIB) continue end @debug "Rejecting stale cache file $cachefile because file $f does not exist" - record_reason(reasons, "missing sourcefile") + record_reason(reasons, :source_missing) return true end if ftime_req >= 0.0 # this is an include_dependency for which we only recorded the mtime - ftime = mtime(f) + ftime = mtime(fstat) is_stale = ( ftime != ftime_req ) && ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes ( ftime != ceil(ftime_req) ) && # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching @@ -3914,21 +4597,25 @@ function any_includes_stale(includes::Vector{CacheHeaderIncludes}, cachefile::St !( 0 < (ftime_req - ftime) < 1e-6 ) # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond if is_stale @debug "Rejecting stale cache file $cachefile because mtime of include_dependency $f has changed (mtime $ftime, before $ftime_req)" - record_reason(reasons, "include_dependency mtime change") + record_reason(reasons, :mtime_changed) return true end else - fstat = stat(f) fsize = filesize(fstat) if fsize != fsize_req @debug "Rejecting stale cache file $cachefile because file size of $f has changed (file size $fsize, before $fsize_req)" - record_reason(reasons, "include_dependency fsize change") + record_reason(reasons, :fsize_changed) return true end + # a read-only file in the content-addressed package store cannot have + # changed since precompilation, so trust the recorded hash + if !isdir(fstat) && filemode(fstat) & 0o222 == 0 && in_package_store(f) + continue + end hash = isdir(fstat) ? _crc32c(join(readdir(f))) : open(_crc32c, f, "r") if hash != hash_req @debug "Rejecting stale cache file $cachefile because hash of $f has changed (hash $hash, before $hash_req)" - record_reason(reasons, "include_dependency fhash change") + record_reason(reasons, :content_changed) return true end end @@ -3936,14 +4623,70 @@ function any_includes_stale(includes::Vector{CacheHeaderIncludes}, cachefile::St return false end +function cache_syntax_version(ver::VersionNumber) + UInt8(clamp(ver.minor - 13, 0, 255)) +end + +# This custom equality predicate is analogous to `===`, except that it also +# compares mutable containers structurally. This allows us to differentiate +# `1.0` / `1` / `true` in Preferences, despite these being normally `isequal` +toml_egal(@nospecialize(A), @nospecialize(B)) = A === B + +function toml_egal(A::AbstractArray, B::AbstractArray) + axes(A) != axes(B) && return false + for (a, b) in zip(A, B) + !toml_egal(a, b) && return false + end + return true +end + +function toml_egal(A::AbstractDict, B::AbstractDict) + isa(A,IdDict) != isa(B,IdDict) && return false + length(A) != length(B) && return false + for pair in A + !in(pair, B, toml_egal) && return false + end + return true +end + +function stale_prefs(prefs_blob::String) + # ensure any preferences observed match their precompile-time values + prefs_blob == "" && return false # no observed preferences (fast-path) + prefs_data = TOML.parse(TOML.Parser{nothing}(prefs_blob)) + + for (uuid, observed) in prefs_data + uuid == "unset" && continue + curr = get_preferences(UUID(uuid)) + for (key, val) in observed::Dict{String,Any} + # any set preferences should have the same value + !haskey(curr, key) && return true + !toml_egal(curr[key], val) && return true + end + end + if haskey(prefs_data, "unset") + for (uuid, observed) in prefs_data["unset"]::Dict{String,Any} + curr = get_preferences(UUID(uuid)) + for key in observed::Vector{String} + # any unset preferences should still be unset + haskey(curr, key) && return true + end + end + end + return false +end + # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check -@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons=nothing) - return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, requested_flags, reasons) +@constprop :none function stale_cachefile(modpath::String, cachefile::String; kwargs...) + return stale_cachefile(PkgLoadSpec(modpath, VERSION), cachefile; kwargs...) +end +@constprop :none function stale_cachefile(modspec::PkgLoadSpec, cachefile::String; ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons=nothing, verify_checksums::Bool=true) + return stale_cachefile(PkgId(""), UInt128(0), modspec, cachefile; ignore_loaded, requested_flags, reasons, verify_checksums) end -@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; +@constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modspec::PkgLoadSpec, cachefile::String; ignore_loaded::Bool=false, requested_flags::CacheFlags=CacheFlags(), - reasons::Union{Dict{String,Int},Nothing}=nothing, stalecheck::Bool=true) + reasons::Union{Dict{Symbol,Int},Nothing}=nothing, stalecheck::Bool=true, + verify_checksums::Bool=true) # n.b.: this function does nearly all of the file validation, not just those checks related to stale, so the name is potentially unclear io = try open(cachefile, "r") @@ -3954,22 +4697,28 @@ end end try checksum = isvalid_cache_header(io) - if iszero(checksum) + if checksum === nothing @debug "Rejecting cache file $cachefile due to it containing an incompatible cache header" - record_reason(reasons, "incompatible header") + record_reason(reasons, :incompatible_header) return true # incompatible cache file end - modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, actual_flags = parse_cache_header(io, cachefile) + modules, (includes, _, requires), required_modules, srctextpos, prefs_blob, clone_targets, actual_flags, syntax_version = parse_cache_header(io, cachefile) if isempty(modules) return true # ignore empty file end - if @ccall(jl_match_cache_flags(_cacheflag_to_uint8(requested_flags)::UInt8, actual_flags::UInt8)::UInt8) == 0 + if @ccall(jl_match_cache_flags(_cacheflag_to_uint8(requested_flags)::UInt8, _cacheflag_to_uint8(actual_flags)::UInt8)::UInt8) == 0 || + !match_cache_coverage(requested_flags, actual_flags) @debug """ Rejecting cache file $cachefile for $modkey since the flags are mismatched requested flags: $(requested_flags) [$(_cacheflag_to_uint8(requested_flags))] - cache file: $(CacheFlags(actual_flags)) [$actual_flags] + cache file: $(actual_flags) [$(_cacheflag_to_uint8(actual_flags))] """ - record_reason(reasons, "mismatched flags") + record_reason(reasons, :flags_mismatch) + return true + end + if stalecheck && syntax_version != cache_syntax_version(modspec.julia_syntax_version) + @debug "Rejecting cache file $cachefile for $modkey since it was parsed for a different Julia syntax version" + record_reason(reasons, :syntax_version) return true end pkgimage = !isempty(clone_targets) @@ -3978,7 +4727,7 @@ end if JLOptions().use_pkgimages == 0 # presence of clone_targets means native code cache @debug "Rejecting cache file $cachefile for $modkey since it would require usage of pkgimage" - record_reason(reasons, "requires pkgimages") + record_reason(reasons, :pkgimages_disabled) return true end rejection_reasons = check_clone_targets(clone_targets) @@ -3987,12 +4736,12 @@ end Reasons=rejection_reasons, var"Image Targets"=parse_image_targets(clone_targets), var"Current Targets"=current_image_targets()) - record_reason(reasons, "target mismatch") + record_reason(reasons, :cpu_target) return true end if !isfile(ocachefile) @debug "Rejecting cache file $cachefile for $modkey since pkgimage $ocachefile was not found" - record_reason(reasons, "missing ocachefile") + record_reason(reasons, :ocachefile_missing) return true end else @@ -4001,7 +4750,7 @@ end id = first(modules) if id.first != modkey && modkey != PkgId("") @debug "Rejecting cache file $cachefile for $modkey since it is for $id instead" - record_reason(reasons, "for different pkgid") + record_reason(reasons, :pkgid_mismatch) return true end id_build = id.second @@ -4009,7 +4758,7 @@ end if build_id != UInt128(0) if id_build != build_id @debug "Ignoring cache file $cachefile for $modkey ($(UUID(id_build))) since it does not provide desired build_id ($((UUID(build_id))))" - record_reason(reasons, "for different buildid") + record_reason(reasons, :buildid_mismatch) return true end end @@ -4030,60 +4779,64 @@ end end M = maybe_root_module(req_key) if M isa Module - if PkgId(M) == req_key && module_build_id(M) === req_build_id + # With `ignore_loaded` the verdict has to reflect the environment rather than the + # session: a dependency loaded at the version this cache was built against says + # nothing about the version the manifest resolves now, so only sysimage modules, + # which cannot differ, are accepted on that basis; everything else is checked below + # against its located source and on-disk cache. + if PkgId(M) == req_key && module_build_id(M) === req_build_id && (!ignore_loaded || in_sysimage(req_key)) depmods[i] = M continue elseif M == Core @debug "Rejecting cache file $cachefile because it was made with a different julia version" - record_reason(reasons, "wrong julia version") + record_reason(reasons, :julia_version) return true # Won't be able to fulfill dependency elseif ignore_loaded || !stalecheck # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages else @debug "Rejecting cache file $cachefile because module $req_key is already loaded and incompatible." - record_reason(reasons, "wrong dep version loaded") + record_reason(reasons, Symbol(DEP_LOADED_INCOMPATIBLE_PREFIX, req_key.name)) return true # Won't be able to fulfill dependency end end - path = locate_package(req_key) # TODO: add env and/or skip this when stalecheck is false - if path === nothing + spec = locate_package_load_spec(req_key) # TODO: add env and/or skip this when stalecheck is false + if spec === nothing @debug "Rejecting cache file $cachefile because dependency $req_key not found." - record_reason(reasons, "dep missing source") + record_reason(reasons, :dep_missing) return true # Won't be able to fulfill dependency end - depmods[i] = (path, req_key, req_build_id) + depmods[i] = (spec, req_key, req_build_id) end # check if this file is going to provide one of our concrete dependencies # or if it provides a version that conflicts with our concrete dependencies - # or neither - if stalecheck - for (req_key, req_build_id) in _concrete_dependencies - build_id = get(modules, req_key, UInt64(0)) - if build_id !== UInt64(0) - build_id |= UInt128(checksum) << 64 - if build_id === req_build_id - stalecheck = false - break - end - @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))" - record_reason(reasons, "wrong dep buildid") - return true # cachefile doesn't provide the required version of the dependency + # or neither. This is not skipped for a trusted (driver-validated) file: + # the driver only checks that the file is fresh, not that it carries the + # build id the parent session pinned. + for (req_key, req_build_id) in _concrete_dependencies + build_id = get(modules, req_key, UInt64(0)) + if build_id !== UInt64(0) + build_id |= UInt128(checksum) << 64 + if build_id === req_build_id + stalecheck = false + break end + @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))" + record_reason(reasons, :dep_buildid_mismatch) + return true # cachefile doesn't provide the required version of the dependency end end # now check if this file's content hash has changed relative to its source files if stalecheck - if !samefile(includes[1].filename, modpath) + if !samefile(includes[1].filename, modspec.path) # In certain cases the path rewritten by `fixup_stdlib_path` may # point to an unreadable directory, make sure we can `stat` the - # file before comparing it with `modpath`. + # file before comparing it with `modspec.path`. stdlib_path = fixup_stdlib_path(includes[1].filename) - if !(isreadable(stdlib_path) && samefile(stdlib_path, modpath)) - !samefile(fixup_stdlib_path(includes[1].filename), modpath) - @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" - record_reason(reasons, "wrong source") + if !(isreadable(stdlib_path) && samefile(stdlib_path, modspec.path)) + @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $(modspec.path)" + record_reason(reasons, :source_path_changed) return true # cache file was compiled from a different path end end @@ -4092,7 +4845,7 @@ end pkg = identify_package(modkey, req_modkey.name) if pkg != req_modkey @debug "Rejecting cache file $cachefile because uuid mapping for $modkey => $req_modkey has changed, expected $modkey => $(repr("text/plain", pkg))" - record_reason(reasons, "dep uuid changed") + record_reason(reasons, :dep_identity_changed) return true end end @@ -4101,24 +4854,25 @@ end end end - if !isvalid_file_crc(io) - @debug "Rejecting cache file $cachefile because it has an invalid checksum" - record_reason(reasons, "invalid checksum") - return true - end - - if pkgimage - if !isvalid_pkgimage_crc(io, ocachefile::String) - @debug "Rejecting cache file $cachefile because $ocachefile has an invalid checksum" - record_reason(reasons, "ocachefile invalid checksum") + if verify_checksums + if !isvalid_file_crc(io) + @debug "Rejecting cache file $cachefile because it has an invalid checksum" + record_reason(reasons, :checksum_invalid) return true end + + if pkgimage + if !isvalid_pkgimage_crc(io, ocachefile::String) + @debug "Rejecting cache file $cachefile because $ocachefile has an invalid checksum" + record_reason(reasons, :ocache_checksum_invalid) + return true + end + end end - curr_prefs_hash = get_preferences_hash(id.uuid, prefs) - if prefs_hash != curr_prefs_hash - @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" - record_reason(reasons, "preferences hash mismatch") + if stale_prefs(prefs_blob) + @debug "Rejecting cache file $cachefile because preferences have changed" + record_reason(reasons, :preferences_changed) return true end @@ -4236,12 +4990,7 @@ function precompile(@nospecialize(f), @nospecialize(argtypes::Tuple), m::Method) end function precompile(@nospecialize(argt::Type), m::Method) - atype, sparams = ccall(:jl_type_intersection_with_env, Any, (Any, Any), argt, m.sig)::SimpleVector + atype, sparams = typeintersect_env(argt, m.sig) mi = Base.Compiler.specialize_method(m, atype, sparams) return precompile(mi) end - -precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) || @assert false -precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) || @assert false -precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), Cmd, CacheFlags, IO, IO)) || @assert false -precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), Cmd, CacheFlags, IO, IO)) || @assert false diff --git a/base/lock.jl b/base/lock.jl index 461cb2b12a807..d99f3807f1726 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -21,7 +21,7 @@ const MAX_SPIN_ITERS = 40 """ ReentrantLock() -Creates a re-entrant lock for synchronizing [`Task`](@ref)s. The same task can +Create a re-entrant lock for synchronizing [`Task`](@ref)s. The same task can acquire the lock as many times as required (this is what the "Reentrant" part of the name means). Each [`lock`](@ref) must be matched with an [`unlock`](@ref). @@ -31,7 +31,7 @@ should naturally be supported, but beware of inverting the try/lock order or missing the try block entirely (e.g. attempting to return with the lock still held): -This provides a acquire/release memory ordering on lock/unlock calls. +This provides an acquire/release memory ordering on lock/unlock calls. ``` lock(l) @@ -74,11 +74,11 @@ mutable struct ReentrantLock <: AbstractLock # | | potentially never getting woken up). @atomic havelock::UInt8 # offset32 = 28, offset64 = 32 - cond_wait::ThreadSynchronizer # 2 words - # offset32 = 36, offset64 = 48 - # sizeof32 = 20, sizeof64 = 32 + cond_wait::ThreadSynchronizer # 1 word (mutable, held by reference) + # offset32 = 32, offset64 = 40 + # sizeof32 = 16, sizeof64 = 24 # now add padding to make this a full cache line to minimize false sharing between objects - _::NTuple{Int === Int32 ? 2 : 3, Int} + _::NTuple{Int === Int32 ? 3 : 4, Int} # offset32 = 44, offset64 = 72 == sizeof+offset # sizeof32 = 28, sizeof64 = 56 @@ -191,9 +191,13 @@ wait for it to become available. Each `lock` must be matched by an [`unlock`](@ref). """ -@inline function lock(rl::ReentrantLock) - trylock(rl) || (@noinline function slowlock(rl::ReentrantLock) +@inline function lock(rl::ReentrantLock; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + trylock(rl) || (@noinline function slowlock(rl::ReentrantLock, cancel::CancelTokenArg) Threads.lock_profiling() && Threads.inc_lock_conflict_count() + # resolve the scoped default once; a cancelled acquisition throws the + # CancellationRequest without the lock held + tok = resolve_cancel_token(cancel) c = rl.cond_wait ct = current_task() iteration = 1 @@ -206,6 +210,13 @@ Each `lock` must be matched by an [`unlock`](@ref). if result.success rl.reentrancy_cnt = 0x0000_0001 @atomic :release rl.locked_by = ct + # Mirror the park path's refusal (see wait_no_relock): a + # token that got cancelled while we were spinning must + # not hand out the lock - release it and deliver. + if tok !== nothing && iscancelled(tok.source) + unlock(rl) + checkcancel(tok.source) + end return end GC.enable_finalizers() @@ -236,28 +247,63 @@ Each `lock` must be matched by an [`unlock`](@ref). end # It was locked, so now wait for the unlock to notify us - wait_no_relock(c) + wait_no_relock(c, tok) # Loop back and try locking again iteration = 1 end - end)(rl) + end)(rl, cancel) return end -function wait_no_relock(c::GenericCondition) +# ReentrantLock-backed conditions (Threads.Condition) forward `cancel` into +# the lock acquisition, so operations that resolved a token (or an explicit +# `nothing` shield) can make their preliminary lock honor it instead of the +# ambient scope - see e.g. the Channel operations. +lock(c::GenericCondition{ReentrantLock}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + lock(c.lock; cancel) + +function wait_no_relock(c::GenericCondition, tok::MaybeToken) + # relock=false: a normal wake returns without retaking the lock (the + # notifier's handoff popped our entry), the refusal releases before + # throwing (e.g. a token cancelled while we were still spinning and + # thus not interruptibly waiting), and the interrupted cleanup retakes + # the lock only for its unlink ct = current_task() - _wait2(c, ct) - token = unlockall(c.lock) - try - return wait() - catch - ct.queue === nothing || list_deletefirst!(ct.queue::IntrusiveLinkedList{Task}, ct) - rethrow() + src = cancel_source(tok) + if src === nothing + ws = (c,) + w = _cached_wait_entry(ct) + else + ws = (c, SourceWait(src, 0x00)) + w = _cancel_wait_entry(ct, src, 0x00) + end + if !park!(ws, w, false) + # refused at the registration recheck (e.g. a token cancelled + # while we were still spinning and thus not interruptibly + # waiting): withdraw under the held lock, release, deliver + withdraw!(ws, w, WAKE_FIRED) + unlock(c.lock) + checkcancel(src) + error("park fired without a cancelled source") end + unlockall(c.lock) + # no relock on any path: a normal wake was handed the lock's baton by + # the notifying unlock, and an exceptional unwind - possibly the + # delivered cancellation itself - must not sleep on locks it does not + # need (the cleanup's unlink takes the lock transiently itself) + return wait_safe_interrupt(ws, w) end +_uncancellable_lock(l::ReentrantLock) = lock(l; cancel=nothing) + +function relockall_but_one(rl::ReentrantLock, state::UInt32) + state == 0x0000_0001 && return nothing + relockall(rl, state - 0x0000_0001) + return nothing +end + """ unlock(lock) @@ -308,7 +354,11 @@ function unlockall(rl::ReentrantLock) end function relockall(rl::ReentrantLock, n::UInt32) - lock(rl) + # The reacquire is the cleanup half of a wait whose outcome (including a + # cancellation) has already been delivered: it must not itself be + # cancellable, or a cancelled scope would throw out of here without the + # lock and the caller's queue cleanup would run unlocked. + lock(rl; cancel=nothing) old = @atomicswap :not_atomic rl.reentrancy_cnt = n old == 0x0000_0001 || concurrency_violation() return @@ -324,10 +374,10 @@ available. When this function returns, the `lock` has been released, so the caller should not attempt to `unlock` it. -See also: [`@lock`](@ref). - !!! compat "Julia 1.7" Using a [`Channel`](@ref) as the second argument requires Julia 1.7 or later. + +See also [`@lock`](@ref). """ function lock(f, l::AbstractLock) lock(l) @@ -338,6 +388,15 @@ function lock(f, l::AbstractLock) end end +function lock(f, l::ReentrantLock; cancel::CancelTokenArg=DEFAULT_CANCEL) + lock(l; cancel) + try + return f() + finally + unlock(l) + end +end + function trylock(f, l::AbstractLock) if trylock(l) try @@ -366,7 +425,7 @@ This is similar to using [`lock`](@ref) with a `do` block, but avoids creating a and thus can improve the performance. !!! compat - `@lock` was added in Julia 1.3, and exported in Julia 1.10. + `@lock` was added in Julia 1.3, and exported in Julia 1.7. """ macro lock(l, expr) quote @@ -400,7 +459,7 @@ end """ Lockable(value, lock = ReentrantLock()) -Creates a `Lockable` object that wraps `value` and +Create a `Lockable` object that wraps `value` and associates it with the provided `lock`. This object supports [`@lock`](@ref), [`lock`](@ref), [`trylock`](@ref), [`unlock`](@ref). To access the value, index the lockable object while @@ -409,7 +468,7 @@ holding the lock. !!! compat "Julia 1.11" Requires at least Julia 1.11. -## Example +# Examples ```jldoctest julia> locked_list = Base.Lockable(Int[]); @@ -500,7 +559,7 @@ Create a counting semaphore that allows at most `sem_size` acquires to be in use at any time. Each acquire must be matched with a release. -This provides a acquire & release memory ordering on acquire/release calls. +This provides an acquire & release memory ordering on acquire/release calls. """ mutable struct Semaphore sem_size::Int @@ -515,15 +574,23 @@ end Wait for one of the `sem_size` permits to be available, blocking until one can be acquired. """ -function acquire(s::Semaphore) - lock(s.cond_wait) +function acquire(s::Semaphore; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + lock(s.cond_wait; cancel) + locked = true try - while s.curr_cnt >= s.sem_size - wait(s.cond_wait) + if s.curr_cnt >= s.sem_size + tok = resolve_cancel_token(cancel) + while s.curr_cnt >= s.sem_size + # a cancelled wait throws before the permit is taken + locked = false + wait(s.cond_wait, tok) + locked = true + end end s.curr_cnt = s.curr_cnt + 1 finally - unlock(s.cond_wait) + locked && unlock(s.cond_wait) end return end @@ -552,8 +619,8 @@ end This method requires at least Julia 1.8. """ -function acquire(f, s::Semaphore) - acquire(s) +function acquire(f, s::Semaphore; cancel::CancelTokenArg=DEFAULT_CANCEL) + acquire(s; cancel) try return f() finally @@ -637,22 +704,26 @@ mutable struct Event Event(autoreset::Bool=false) = new(Threads.Condition(), autoreset, false) end -function wait(e::Event) +function wait(e::Event; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) if e.autoreset (@atomicswap :acquire_release e.set = false) && return else (@atomic e.set) && return # full barrier also end - lock(e.notify) # acquire barrier + lock(e.notify; cancel) # acquire barrier + locked = true try if e.autoreset (@atomicswap :acquire_release e.set = false) && return else e.set && return end - wait(e.notify) + locked = false + wait(e.notify, resolve_cancel_token(cancel)) + locked = true finally - unlock(e.notify) # release barrier + locked && unlock(e.notify) # release barrier end nothing end @@ -707,7 +778,7 @@ which won't get serialized. !!! compat "Julia 1.12" This type requires Julia 1.12 or later. -## Example +# Examples ```jldoctest julia> const global_state = Base.OncePerProcess{Vector{UInt32}}() do @@ -797,7 +868,7 @@ end # share a lock/condition, since we just need it briefly, so some contention is okay -const PerThreadLock = ThreadSynchronizer() +const PerThreadLock = Threads.SpinLock() """ OncePerThread{T}(init::Function)() -> T @@ -815,12 +886,12 @@ if that behavior is correct within your library's threading-safety design. may get deprecated in the future. If initializer yields, the thread running the current task after the call might not be the same as the one at the start of the call. -See also: [`OncePerTask`](@ref). - !!! compat "Julia 1.12" This type requires Julia 1.12 or later. -## Example +See also [`OncePerTask`](@ref). + +# Examples ```jldoctest julia> const thread_state = Base.OncePerThread{Vector{UInt32}}() do @@ -901,7 +972,15 @@ OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof( state = @atomic :monotonic ss[tid] while state == PerStateConcurrent # lost race, wait for notification this is done running elsewhere - wait(PerThreadLock) # wait for initializer to finish without releasing this thread + # without releasing this thread + unlock(PerThreadLock) + while state == PerStateConcurrent + # spin loop until ready + ss = @atomic :acquire once.ss + state = @atomic :monotonic ss[tid] + GC.safepoint() + end + lock(PerThreadLock) ss = @atomic :monotonic once.ss state = @atomic :monotonic ss[tid] end @@ -915,7 +994,6 @@ OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof( lock(PerThreadLock) ss = @atomic :monotonic once.ss @atomic :release ss[tid] = PerStateErrored - notify(PerThreadLock) rethrow() end # store result and notify waiters @@ -924,7 +1002,6 @@ OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof( @atomic :release xs[tid] = result ss = @atomic :monotonic once.ss @atomic :release ss[tid] = PerStateHasrun - notify(PerThreadLock) elseif state == PerStateErrored error("OncePerThread initializer failed previously") elseif state != PerStateHasrun @@ -946,12 +1023,12 @@ end Calling a `OncePerTask` object returns a value of type `T` by running the function `initializer` exactly once per Task. All future calls in the same Task will return exactly the same value. -See also: [`task_local_storage`](@ref). - !!! compat "Julia 1.12" This type requires Julia 1.12 or later. -## Example +See also [`task_local_storage`](@ref). + +# Examples ```jldoctest julia> const task_state = Base.OncePerTask{Vector{UInt32}}() do diff --git a/base/locks-mt.jl b/base/locks-mt.jl index 237e0d9856996..a43d11413f3b4 100644 --- a/base/locks-mt.jl +++ b/base/locks-mt.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import .Base: unsafe_convert, lock, trylock, unlock, islocked, wait, notify, AbstractLock +import .Base: lock, trylock, unlock, islocked, AbstractLock export SpinLock public PaddedSpinLock @@ -27,6 +27,9 @@ succeeds unless there are other tasks attempting to hold the lock "at the same t Test-and-test-and-set spin locks are quickest up to about 30ish contending threads. If you have more contention than that, different synchronization approaches should be considered. + +!!! compat "Julia 1.13" + This type requires at least Julia 1.13. """ abstract type AbstractSpinLock <: AbstractLock end @@ -39,7 +42,6 @@ See also [`PaddedSpinLock`](@ref). See the documentation for [`AbstractSpinLock`](@ref) regarding correct usage. """ mutable struct SpinLock <: AbstractSpinLock - # we make this much larger than necessary to minimize false-sharing @atomic owned::Int SpinLock() = new(0) end @@ -56,6 +58,9 @@ false sharing. See also [`SpinLock`](@ref). See the documentation for [`AbstractSpinLock`](@ref) regarding correct usage. + +!!! compat "Julia 1.13" + This type requires at least Julia 1.13. """ mutable struct PaddedSpinLock <: AbstractSpinLock # we make this much larger than necessary to minimize false-sharing @@ -85,9 +90,9 @@ function lock(l::AbstractSpinLock) end function trylock(l::AbstractSpinLock) - if l.owned == 0 + if l.owned::Int == 0 GC.disable_finalizers() - p = @atomicswap :acquire l.owned = 1 + p = (@atomicswap :acquire l.owned = 1)::Int if p == 0 return true end @@ -97,7 +102,7 @@ function trylock(l::AbstractSpinLock) end function unlock(l::AbstractSpinLock) - if (@atomicswap :release l.owned = 0) == 0 + if (@atomicswap :release l.owned = 0)::Int == 0 error("unlock count must match lock count") end GC.enable_finalizers() @@ -106,5 +111,5 @@ function unlock(l::AbstractSpinLock) end function islocked(l::AbstractSpinLock) - return (@atomic :monotonic l.owned) != 0 + return (@atomic :monotonic l.owned)::Int != 0 end diff --git a/base/logging/ConsoleLogger.jl b/base/logging/ConsoleLogger.jl index 6521bf49b3e66..cc4afa6b7b7bc 100644 --- a/base/logging/ConsoleLogger.jl +++ b/base/logging/ConsoleLogger.jl @@ -57,6 +57,7 @@ min_enabled_level(logger::ConsoleLogger) = logger.min_level showvalue(io, msg) = show(io, "text/plain", msg) function showvalue(io, e::Tuple{Exception,Any}) ex,bt = e + bt = Base.scrub_repl_backtrace(bt) showerror(io, ex, bt; backtrace = bt!==nothing) end showvalue(io, ex::Exception) = showerror(io, ex) @@ -124,8 +125,8 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module # split into lines. This is specialised to improve type inference, # and reduce the risk of resulting method invalidations. message = string(message) - msglines = if Base._isannotated(message) && !isempty(Base.annotations(message)) - message = Base.AnnotatedString(String(message), Base.annotations(message)) + msglines = if Base._isannotated(message) && !isempty(Base.annotations(message)::Vector{Base.RegionAnnotation}) + message = Base.AnnotatedString(String(message)::String, Base.annotations(message)::Vector{Base.RegionAnnotation}) @NamedTuple{indent::Int, msg::Union{SubString{Base.AnnotatedString{String}}, SubString{String}}}[ (indent=0, msg=l) for l in split(chomp(message), '\n')] else @@ -142,11 +143,13 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module valbuf = IOBuffer() rows_per_value = max(1, dsize[1] ÷ (nkwargs + 1 - hasmaxlog)) valio = IOContext(IOContext(valbuf, stream), - :displaysize => (rows_per_value, dsize[2] - 5), :limit => logger.show_limited) for (key, val) in kwargs key === :maxlog && continue - showvalue(valio, val) + keyio = IOContext(valio, + :displaysize => (rows_per_value, + dsize[2] - 7 - textwidth(string(key)))) + showvalue(keyio, val) vallines = split(takestring!(valbuf), '\n') if length(vallines) == 1 push!(msglines, (indent=2, msg=SubString("$key = $(vallines[1])"))) diff --git a/base/logging/logging.jl b/base/logging/logging.jl index 25f4dbe4902be..9b0fb2bb9faa2 100644 --- a/base/logging/logging.jl +++ b/base/logging/logging.jl @@ -3,7 +3,7 @@ module CoreLogging import Base: isless, +, -, convert, show -import Base.ScopedValues: ScopedValue, with, @with +import Base.ScopedValues: ScopedValue, @with export AbstractLogger, @@ -60,7 +60,7 @@ function min_enabled_level end catch_exceptions(logger) Return `true` if the logger should catch exceptions which happen during log -record construction. By default, messages are caught. +record construction. By default, exceptions are caught. By default all exceptions are caught to prevent log message generation from crashing the program. This lets users confidently toggle little-used @@ -129,6 +129,8 @@ end LogLevel(level::LogLevel) = level isless(a::LogLevel, b::LogLevel) = isless(a.level, b.level) +isless(a::LogLevel, b::Integer) = isless(a.level, b) +isless(a::Integer, b::LogLevel) = isless(a, b.level) +(level::LogLevel, inc::Integer) = LogLevel(level.level+inc) -(level::LogLevel, inc::Integer) = LogLevel(level.level-inc) convert(::Type{LogLevel}, level::Integer) = LogLevel(level) @@ -305,7 +307,7 @@ function log_record_id(_module, level, message, log_kws) push!(_log_record_ids, id) return id end - h += 1 + h +%= 1 end end @@ -504,7 +506,12 @@ end catch ex LazyString("Exception handling log message: ", ex) end - bt = real ? catch_backtrace() : backtrace() + bt = Base.scrub_repl_backtrace(real ? catch_backtrace() : stacktrace()) + if !real + # drop the frames of the logging machinery itself + i = findfirst(fr -> fr.func === :logging_error, bt) + i === nothing || deleteat!(bt, 1:i) + end handle_message( logger, Error, msg, _module, :logevent_error, id, filepath, line; exception=(err,bt)) @@ -556,7 +563,7 @@ Logging.disable_logging(Logging.Info) # Disable debug and info ``` """ function disable_logging(level::LogLevel) - _min_enabled_level[] = level + 1 + @atomic _min_enabled_level[] = level + 1 end let _debug_groups_include::Vector{Symbol} = Symbol[], diff --git a/base/math.jl b/base/math.jl index d6efadf56eccd..2f4d27f7871d8 100644 --- a/base/math.jl +++ b/base/math.jl @@ -23,7 +23,7 @@ import .Base: log, exp, sin, cos, tan, sinh, cosh, tanh, asin, using .Base: sign_mask, exponent_mask, exponent_one, exponent_half, uinttype, significand_mask, significand_bits, exponent_bits, exponent_bias, - exponent_max, exponent_raw_max, clamp, clamp! + exponent_raw_max, clamp, clamp!, two_mul using Core.Intrinsics: sqrt_llvm, min_float, max_float @@ -37,6 +37,9 @@ end throw(DomainError(x, LazyString(f," was called with a real argument < -1 but will only return a complex result if called with a complex argument. Try ", f,"(Complex(x))."))) end +@noinline function throw_finite_domainerror(f::Symbol, x) + throw(DomainError(x, LazyString("`", f, "(x)` is only defined for finite `x`."))) +end @noinline function throw_exp_domainerror(x) throw(DomainError(x, LazyString( "Exponentiation yielding a complex result requires a ", @@ -46,30 +49,6 @@ end # non-type specific math functions -function two_mul(x::T, y::T) where {T<:Number} - xy = x*y - xy, fma(x, y, -xy) -end - -@assume_effects :consistent @inline function two_mul(x::Float64, y::Float64) - if Core.Intrinsics.have_fma(Float64) - xy = x*y - return xy, fma(x, y, -xy) - end - return Base.twomul(x,y) -end - -@assume_effects :consistent @inline function two_mul(x::T, y::T) where T<: Union{Float16, Float32} - if Core.Intrinsics.have_fma(T) - xy = x*y - return xy, fma(x, y, -xy) - end - xy = widen(x)*y - Txy = T(xy) - return Txy, T(xy-Txy) -end - - """ evalpoly(x, p) @@ -85,6 +64,8 @@ a Goertzel-like [^DK62] algorithm if `x` is complex. !!! compat "Julia 1.4" This function requires Julia 1.4 or later. +See also [`@evalpoly`](@ref). + # Examples ```jldoctest julia> evalpoly(2, (1, 2, 3)) @@ -130,13 +111,13 @@ function evalpoly(z::Complex, p::Tuple) end ai = :a0 push!(as, :($ai = $a)) - C = Expr(:block, - :(x = real(z)), - :(y = imag(z)), - :(r = x + x), - :(s = muladd(x, x, y*y)), - as..., - :(muladd($ai, z, $b))) + Expr(:block, + :(x = real(z)), + :(y = imag(z)), + :(r = x + x), + :(s = muladd(x, x, y*y)), + as..., + :(muladd($ai, z, $b))) else _evalpoly(z, p) end @@ -237,24 +218,16 @@ function _pi_over_180(z::AbstractFloat) end # rounded to closest representable number where necessary -function _180_over_pi(z::Union{Float16, Float32}) - if z isa Float16 - r = Float16(57.28) - elseif z isa Float32 - r = 57.29578f0 - end - r -end -function _pi_over_180(::Float16) - Float16(0.01746) -end +_180_over_pi(::Float16) = Float16(57.28) +_180_over_pi(::Float32) = 57.29578f0 +_pi_over_180(::Float16) = Float16(0.01746) """ rad2deg(x) Convert `x` from radians to degrees. -See also [`deg2rad`](@ref). +See also [`deg2rad`](@ref), [`pi`](@ref). # Examples ```jldoctest @@ -269,7 +242,7 @@ rad2deg(z::AbstractFloat) = z * _180_over_pi(z) Convert `x` from degrees to radians. -See also [`rad2deg`](@ref), [`sind`](@ref), [`pi`](@ref). +See also [`rad2deg`](@ref), [`pi`](@ref). # Examples ```jldoctest @@ -291,6 +264,8 @@ log(b::T, x::T) where {T<:Number} = log(x)/log(b) Compute the base `b` logarithm of `x`. Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. +See also [`log2`](@ref), [`log10`](@ref). + # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*" julia> log(4,8) @@ -328,17 +303,14 @@ Stacktrace: """ log(b::Number, x::Number) = log(promote(b,x)...) -# type specific math functions - const libm = Base.libm_name - # functions with no domain error """ sinh(x) Compute hyperbolic sine of `x`. -See also [`sin`](@ref). +See also [`sin`](@ref), [`asinh`](@ref). """ sinh(x::Number) @@ -347,7 +319,7 @@ sinh(x::Number) Compute hyperbolic cosine of `x`. -See also [`cos`](@ref). +See also [`cos`](@ref), [`acosh`](@ref). """ cosh(x::Number) @@ -415,6 +387,8 @@ atan(x::Number) asinh(x) Compute the inverse hyperbolic sine of `x`. + +See also [`sinh`](@ref), [`asin`](@ref). """ asinh(x::Number) @@ -444,7 +418,7 @@ julia> sind(45) 0.7071067811865476 julia> sinpi(1/4) -0.7071067811865475 +0.7071067811865476 julia> round.(sincos(pi/6), digits=3) (0.5, 0.866) @@ -465,7 +439,7 @@ Compute cosine of `x`, where `x` is in radians. Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. -See also [`cosd`](@ref), [`cospi`](@ref), [`sincos`](@ref), [`cis`](@ref). +See also [`cosd`](@ref), [`cospi`](@ref), [`sincos`](@ref), [`cis`](@ref), [`acos`](@ref). """ cos(x::Number) @@ -506,6 +480,8 @@ asin(x::Number) Compute the inverse cosine of `x`, where the output is in radians. Return a `T(NaN)` if `isnan(x)`. + +See also [`acosd`](@ref) for output in degrees. """ acos(x::Number) @@ -513,6 +489,8 @@ acos(x::Number) acosh(x) Compute the inverse hyperbolic cosine of `x`. + +See also [`cosh`](@ref), [`acos`](@ref). """ acosh(x::Number) @@ -520,6 +498,8 @@ acosh(x::Number) atanh(x) Compute the inverse hyperbolic tangent of `x`. + +See also [`tanh`](@ref), [`atanh`](@ref). """ atanh(x::Number) @@ -535,7 +515,7 @@ Use [`Complex`](@ref) arguments to obtain [`Complex`](@ref) results. `log` has a branch cut along the negative real axis; `-0.0im` is taken to be below the axis. -See also [`ℯ`](@ref), [`log1p`](@ref), [`log2`](@ref), [`log10`](@ref). +See also [`ℯ`](@ref), [`exp`](@ref), [`log1p`](@ref), [`log2`](@ref), [`log10`](@ref). # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*" @@ -570,7 +550,7 @@ log(x::Number) Compute the logarithm of `x` to base 2. Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. -See also: [`exp2`](@ref), [`ldexp`](@ref), [`ispow2`](@ref). +See also: [`exp2`](@ref), [`log`](@ref), [`ldexp`](@ref), [`ispow2`](@ref). # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*" @@ -602,6 +582,8 @@ log2(x) Compute the logarithm of `x` to base 10. Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. +See also: [`exp10`](@ref), [`log`](@ref). + # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*" julia> log10(100) @@ -644,7 +626,7 @@ Stacktrace: """ log1p(x) -@inline function sqrt(x::Union{Float32,Float64}) +@inline function sqrt(x::IEEEFloat) x < zero(x) && throw_complex_domainerror(:sqrt, x) sqrt_llvm(x) end @@ -663,7 +645,7 @@ The prefix operator `√` is equivalent to `sqrt`. `sqrt` has a branch cut along the negative real axis; `-0.0im` is taken to be below the axis. -See also: [`hypot`](@ref). +See also [`cbrt`](@ref), [`fourthroot`](@ref), [`hypot`](@ref). # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*" @@ -696,7 +678,9 @@ sqrt(x) """ fourthroot(x) -Return the fourth root of `x` by applying `sqrt` twice successively. +Return the fourth root of `x`. + +See also [`cbrt`](@ref), [`sqrt`](@ref). """ fourthroot(x::Number) = sqrt(sqrt(x)) @@ -706,10 +690,7 @@ fourthroot(x::Number) = sqrt(sqrt(x)) Compute the hypotenuse ``\\sqrt{|x|^2+|y|^2}`` avoiding overflow and underflow. This code is an implementation of the algorithm described in: -An Improved Algorithm for `hypot(a,b)` -by Carlos F. Borges -The article is available online at arXiv at the link - https://arxiv.org/abs/1904.09481 +[*An Improved Algorithm for `hypot(a,b)`* by Carlos F. Borges](https://arxiv.org/abs/1904.09481) hypot(x...) @@ -764,7 +745,7 @@ function _hypot(x, y) # Order the operands if ay > ax - axu, ayu = ayu, axu + axu = ayu ax, ay = ay, ax end @@ -803,14 +784,16 @@ function _hypot(x, y) end return h*scale*oneunit(axu) end -@inline function _hypot(x::Float32, y::Float32) +# @assume_effects :nothrow: isinf guards handle Inf inputs; muladd(x,x,y*y) is always ≥ 0 +# so the sqrt call never throws. +@assume_effects :nothrow @inline function _hypot(x::Float32, y::Float32) if isinf(x) || isinf(y) return Inf32 end _x, _y = Float64(x), Float64(y) return Float32(sqrt(muladd(_x, _x, _y*_y))) end -@inline function _hypot(x::Float16, y::Float16) +@assume_effects :nothrow @inline function _hypot(x::Float16, y::Float16) if isinf(x) || isinf(y) return Inf16 end @@ -852,13 +835,8 @@ min(x::T, y::T) where {T<:AbstractFloat} = isnan(x) || ~isnan(y) && _isless(x, y max(x::T, y::T) where {T<:AbstractFloat} = isnan(x) || ~isnan(y) && _isless(y, x) ? x : y minmax(x::T, y::T) where {T<:AbstractFloat} = min(x, y), max(x, y) -function min(x::T, y::T) where {T<:IEEEFloat} - return min_float(x, y) -end - -function max(x::T, y::T) where {T<:IEEEFloat} - return max_float(x, y) -end +min(x::T, y::T) where {T<:IEEEFloat} = min_float(x, y) +max(x::T, y::T) where {T<:IEEEFloat} = max_float(x, y) """ ldexp(x, n) @@ -895,7 +873,7 @@ function ldexp(x::T, e::Integer) where T<:IEEEFloat return flipsign(T(0.0), x) end n = e % Int - k += n + k = k +% n # overflow, if k is larger than maximum possible exponent if k >= exponent_raw_max(T) return flipsign(T(Inf), x) @@ -927,6 +905,9 @@ For a normalized floating-point number `x`, this corresponds to the exponent of Throws a `DomainError` when `x` is zero, infinite, or [`NaN`](@ref). For any other non-subnormal floating-point number `x`, this corresponds to the exponent bits of `x`. +!!! compat "Julia 1.14" + Calling `exponent` on a `Bool` requires Julia 1.14 or later. + See also [`signbit`](@ref), [`significand`](@ref), [`frexp`](@ref), [`issubnormal`](@ref), [`log2`](@ref), [`ldexp`](@ref). # Examples ```jldoctest @@ -956,7 +937,9 @@ function exponent(x::T) where T<:IEEEFloat @noinline throw2(x) = throw(DomainError(x, "Cannot be ±0.0.")) xs = reinterpret(Unsigned, x) & ~sign_mask(T) xs >= exponent_mask(T) && throw1(x) - k = Int(xs >> significand_bits(T)) + # use `% Int` instead of `Int(...)` to preserve `:nothrow` (the shifted value + # always fits in `exponent_bits(T)` bits, well below `typemax(Int)`) + k = (xs >> significand_bits(T)) % Int if k == 0 # x is subnormal xs == 0 && throw2(x) m = leading_zeros(xs) - exponent_bits(T) @@ -980,6 +963,40 @@ function _exponent_finite_nonzero(x::T) where T<:IEEEFloat return k - exponent_bias(T) end +function _ilog2_step(y::T, d::T, s) where {T<:Integer} + if (y >> s) >= d + y, n = _ilog2_step(y, d*d, s+s) + else + n = 0 + end + if y >= d + y >>= s + n = Base.checked_add(n, s) + end + return y, n +end + +function exponent(x::Integer) + iszero(x) && throw(DomainError(x, "cannot be zero")) + ux = Base.uabs(x) + _, n = _ilog2_step(ux, one(ux) + one(ux), 1) + return n +end + +function exponent(x::Base.BitInteger) + iszero(x) && throw(DomainError(x, "cannot be zero")) + ux = Base.uabs(x) + return 8sizeof(ux) - leading_zeros(ux) - 1 +end + +function exponent(x::Bool) + if x + 0 + else + throw(DomainError(x, "cannot be zero")) + end +end + """ significand(x) @@ -1043,7 +1060,9 @@ function frexp(x::T) where T<:IEEEFloat xu = reinterpret(Unsigned, x) xs = xu & ~sign_mask(T) xs >= exponent_mask(T) && return x, 0 # NaN or Inf - k = Int(xs >> significand_bits(T)) + # use `% Int` instead of `Int(...)` to preserve `:nothrow` (after masking the sign + # bit, xs >> significand_bits(T) is at most 2^exponent_bits(T)-1, which always fits in Int) + k = (xs >> significand_bits(T)) % Int if k == 0 # x is subnormal xs == 0 && return x, 0 # +-0 m = leading_zeros(xs) - exponent_bits(T) @@ -1149,154 +1168,13 @@ function modf(x::T) where T<:IEEEFloat return (rx, ix) end -@inline function use_power_by_squaring(n::Integer) - -2^12 <= n <= 3 * 2^13 -end - -# @constprop aggressive to help the compiler see the switch between the integer and float -# variants for callers with constant `y` -@constprop :aggressive function ^(x::Float64, y::Float64) - xu = reinterpret(UInt64, x) - xu == reinterpret(UInt64, 1.0) && return 1.0 - # Exponents greater than this will always overflow or underflow. - # Note that NaN can pass through this, but that will end up fine. - if !(abs(y)<0x1.8p62) - isnan(y) && return y - y = sign(y)*0x1.8p62 - end - yint = unsafe_trunc(Int64, y) # This is actually safe since julia freezes the result - yisint = y == yint - if yisint - yint == 0 && return 1.0 - use_power_by_squaring(yint) && return @noinline pow_body(x, yint) - end - 2*xu==0 && return abs(y)*Inf*(!(y>0)) # if x === +0.0 or -0.0 (Inf * false === 0.0) - s = 1 - if x < 0 - !yisint && throw_exp_domainerror(x) # y isn't an integer - s = ifelse(isodd(yint), -1, 1) - end - !isfinite(x) && return copysign(x,s)*(y>0 || isnan(x)) # x is inf or NaN - return copysign(pow_body(abs(x), y), s) -end - -@assume_effects :foldable @noinline function pow_body(x::Float64, y::Float64) - xu = reinterpret(UInt64, x) - if xu < (UInt64(1)<<52) # x is subnormal - xu = reinterpret(UInt64, x * 0x1p52) # normalize x - xu &= ~sign_mask(Float64) - xu -= UInt64(52) << 52 # mess with the exponent - end - logxhi,logxlo = _log_ext(xu) - xyhi, xylo = two_mul(logxhi,y) - xylo = muladd(logxlo, y, xylo) - hi = xyhi+xylo - return @inline Base.Math.exp_impl(hi, xylo-(hi-xyhi), Val(:ℯ)) -end - -# @constprop aggressive to help the compiler see the switch between the integer and float -# variants for callers with constant `y` -@constprop :aggressive function ^(x::T, y::T) where T <: Union{Float16, Float32} - x == 1 && return one(T) - # Exponents greater than this will always overflow or underflow. - # Note that NaN can pass through this, but that will end up fine. - max_exp = T == Float16 ? T(3<<14) : T(0x1.Ap30) - if !(abs(y)0 || isnan(x)) # x is inf or NaN - return copysign(pow_body(abs(x), y), s) -end - -@inline function pow_body(x::T, y) where T <: Union{Float16, Float32} - return T(exp2(log2(abs(widen(x))) * y)) -end - -@inline function pow_body(x::Union{Float16, Float32}, n::Int32) - n == -2 && return (i=inv(x); i*i) - n == 3 && return x*x*x #keep compatibility with literal_pow - n < 0 && return oftype(x, Base.power_by_squaring(inv(widen(x)), -n)) - return oftype(x, Base.power_by_squaring(widen(x), n)) -end -@constprop :aggressive @inline function ^(x::Float64, n::Integer) - n = clamp(n, Int64) - n == 0 && return one(x) - if use_power_by_squaring(n) - return pow_body(x, n) - else - s = ifelse(x < 0 && isodd(n), -1.0, 1.0) - x = abs(x) - y = float(n) - if y == n - return copysign(pow_body(x, y), s) - else - n2 = n % 1024 - y = float(n - n2) - return pow_body(x, y) * copysign(pow_body(x, n2), s) - end - end -end - -# compensated power by squaring -# this method is only reliable for -2^20 < n < 2^20 (cf. #53881 #53886) -@assume_effects :terminates_locally @noinline function pow_body(x::Float64, n::Integer) - y = 1.0 - xnlo = -0.0 - ynlo = 0.0 - n == 3 && return x*x*x # keep compatibility with literal_pow - if n < 0 - rx = inv(x) - n==-2 && return rx*rx #keep compatibility with literal_pow - isfinite(x) && (xnlo = -fma(x, rx, -1.) * rx) - x = rx - n = -n - end - while n > 1 - if n&1 > 0 - err = muladd(y, xnlo, x*ynlo) - y, ynlo = two_mul(x,y) - ynlo += err - end - err = x*2*xnlo - x, xnlo = two_mul(x, x) - xnlo += err - n >>>= 1 - end - err = muladd(y, xnlo, x*ynlo) - return ifelse(isfinite(x) & isfinite(err), muladd(x, y, err), x*y) -end - -# @constprop aggressive to help the compiler see the switch between the integer and float -# variants for callers with constant `y` -@constprop :aggressive @inline function ^(x::T, n::Integer) where T <: Union{Float16, Float32} - n = clamp(n, Int32) - # Exponents greater than this will always overflow or underflow. - # Note that NaN can pass through this, but that will end up fine. - n == 0 && return one(x) - use_power_by_squaring(n) && return pow_body(x, n) - s = ifelse(x < 0 && isodd(n), -one(T), one(T)) - x = abs(x) - return pow_body(x, widen(T)(n)) -end ## rem2pi-related calculations ## function add22condh(xh::Float64, xl::Float64, yh::Float64, yl::Float64) # This algorithm, due to Dekker, computes the sum of two - # double-double numbers and return the high double. References: + # double-double numbers and returns the high double. References: # [1] http://www.digizeitschriften.de/en/dms/img/?PID=GDZPPN001170007 # [2] https://doi.org/10.1007/BF01397083 r = xh+yh @@ -1305,19 +1183,6 @@ function add22condh(xh::Float64, xl::Float64, yh::Float64, yl::Float64) return zh end -# multiples of pi/2, as double-double (ie with "tail") -const pi1o2_h = 1.5707963267948966 # convert(Float64, pi * BigFloat(1/2)) -const pi1o2_l = 6.123233995736766e-17 # convert(Float64, pi * BigFloat(1/2) - pi1o2_h) - -const pi2o2_h = 3.141592653589793 # convert(Float64, pi * BigFloat(1)) -const pi2o2_l = 1.2246467991473532e-16 # convert(Float64, pi * BigFloat(1) - pi2o2_h) - -const pi3o2_h = 4.71238898038469 # convert(Float64, pi * BigFloat(3/2)) -const pi3o2_l = 1.8369701987210297e-16 # convert(Float64, pi * BigFloat(3/2) - pi3o2_h) - -const pi4o2_h = 6.283185307179586 # convert(Float64, pi * BigFloat(2)) -const pi4o2_l = 2.4492935982947064e-16 # convert(Float64, pi * BigFloat(2) - pi4o2_h) - """ rem2pi(x, r::RoundingMode) @@ -1350,135 +1215,6 @@ julia> rem2pi(7pi/4, RoundDown) ``` """ function rem2pi end -function rem2pi(x::Float64, ::RoundingMode{:Nearest}) - isnan(x) && return x - isinf(x) && return NaN - - abs(x) < pi && return x - - n,y = rem_pio2_kernel(x) - - if iseven(n) - if n & 2 == 2 # n % 4 == 2: add/subtract pi - if y.hi <= 0 - return add22condh(y.hi,y.lo,pi2o2_h,pi2o2_l) - else - return add22condh(y.hi,y.lo,-pi2o2_h,-pi2o2_l) - end - else # n % 4 == 0: add 0 - return y.hi+y.lo - end - else - if n & 2 == 2 # n % 4 == 3: subtract pi/2 - return add22condh(y.hi,y.lo,-pi1o2_h,-pi1o2_l) - else # n % 4 == 1: add pi/2 - return add22condh(y.hi,y.lo,pi1o2_h,pi1o2_l) - end - end -end -function rem2pi(x::Float64, ::RoundingMode{:ToZero}) - isnan(x) && return x - isinf(x) && return NaN - - ax = abs(x) - ax <= 2*Float64(pi,RoundDown) && return x - - n,y = rem_pio2_kernel(ax) - - if iseven(n) - if n & 2 == 2 # n % 4 == 2: add pi - z = add22condh(y.hi,y.lo,pi2o2_h,pi2o2_l) - else # n % 4 == 0: add 0 or 2pi - if y.hi > 0 - z = y.hi+y.lo - else # negative: add 2pi - z = add22condh(y.hi,y.lo,pi4o2_h,pi4o2_l) - end - end - else - if n & 2 == 2 # n % 4 == 3: add 3pi/2 - z = add22condh(y.hi,y.lo,pi3o2_h,pi3o2_l) - else # n % 4 == 1: add pi/2 - z = add22condh(y.hi,y.lo,pi1o2_h,pi1o2_l) - end - end - copysign(z,x) -end -function rem2pi(x::Float64, ::RoundingMode{:Down}) - isnan(x) && return x - isinf(x) && return NaN - - if x < pi4o2_h - if x >= 0 - return x - elseif x > -pi4o2_h - return add22condh(x,0.0,pi4o2_h,pi4o2_l) - end - end - - n,y = rem_pio2_kernel(x) - - if iseven(n) - if n & 2 == 2 # n % 4 == 2: add pi - return add22condh(y.hi,y.lo,pi2o2_h,pi2o2_l) - else # n % 4 == 0: add 0 or 2pi - if y.hi > 0 - return y.hi+y.lo - else # negative: add 2pi - return add22condh(y.hi,y.lo,pi4o2_h,pi4o2_l) - end - end - else - if n & 2 == 2 # n % 4 == 3: add 3pi/2 - return add22condh(y.hi,y.lo,pi3o2_h,pi3o2_l) - else # n % 4 == 1: add pi/2 - return add22condh(y.hi,y.lo,pi1o2_h,pi1o2_l) - end - end -end -function rem2pi(x::Float64, ::RoundingMode{:Up}) - isnan(x) && return x - isinf(x) && return NaN - - if x > -pi4o2_h - if x <= 0 - return x - elseif x < pi4o2_h - return add22condh(x,0.0,-pi4o2_h,-pi4o2_l) - end - end - - n,y = rem_pio2_kernel(x) - - if iseven(n) - if n & 2 == 2 # n % 4 == 2: sub pi - return add22condh(y.hi,y.lo,-pi2o2_h,-pi2o2_l) - else # n % 4 == 0: sub 0 or 2pi - if y.hi < 0 - return y.hi+y.lo - else # positive: sub 2pi - return add22condh(y.hi,y.lo,-pi4o2_h,-pi4o2_l) - end - end - else - if n & 2 == 2 # n % 4 == 3: sub pi/2 - return add22condh(y.hi,y.lo,-pi1o2_h,-pi1o2_l) - else # n % 4 == 1: sub 3pi/2 - return add22condh(y.hi,y.lo,-pi3o2_h,-pi3o2_l) - end - end -end - -rem2pi(x::Float32, r::RoundingMode) = Float32(rem2pi(Float64(x), r)) -rem2pi(x::Float16, r::RoundingMode) = Float16(rem2pi(Float64(x), r)) -rem2pi(x::Int32, r::RoundingMode) = rem2pi(Float64(x), r) - -# general fallback -function rem2pi(x::Integer, r::RoundingMode) - fx = float(x) - fx == x || throw(ArgumentError(LazyString(typeof(x), " argument to rem2pi is too large: ", x))) - rem2pi(fx, r) -end """ mod2pi(x) @@ -1558,13 +1294,16 @@ include("special/exp.jl") include("special/hyperbolic.jl") include("special/trig.jl") include("special/rem_pio2.jl") +include("special/rem2pi.jl") include("special/log.jl") +include("special/pow.jl") + # Float16 definitions for func in (:sin,:cos,:tan,:asin,:acos,:atan,:cosh,:tanh,:asinh,:acosh, - :atanh,:log,:log2,:log10,:sqrt,:fourthroot,:log1p) + :atanh,:log,:log2,:log10,:log1p) @eval begin $func(a::Float16) = Float16($func(Float32(a))) $func(a::ComplexF16) = ComplexF16($func(ComplexF32(a))) diff --git a/base/meta.jl b/base/meta.jl index 5d880a7442b3e..7389d15437bb1 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -18,9 +18,121 @@ export quot, public parse -using Base: isidentifier, isoperator, isunaryoperator, isbinaryoperator, ispostfixoperator import Base: isexpr +## AST decoding helpers ## + +is_id_start_char(c::AbstractChar) = ccall(:jl_id_start_char, Cint, (UInt32,), c) != 0 +is_id_char(c::AbstractChar) = ccall(:jl_id_char, Cint, (UInt32,), c) != 0 + +""" + isidentifier(s) -> Bool + +Return whether the symbol or string `s` contains characters that are parsed as +a valid ordinary identifier (not a binary/unary operator) in Julia code; +see also [`Base.isoperator`](@ref). + +Internally Julia allows any sequence of characters in a `Symbol` (except `\\0`s), +and macros automatically use variable names containing `#` in order to avoid +naming collision with the surrounding code. In order for the parser to +recognize a variable, it uses a limited set of characters (greatly extended by +Unicode). `isidentifier()` makes it possible to query the parser directly +whether a symbol contains valid characters. + +# Examples +```jldoctest +julia> Meta.isidentifier(:x), Meta.isidentifier("1x") +(true, false) +``` +""" +function isidentifier(s::AbstractString) + x = Iterators.peel(s) + isnothing(x) && return false + (s == "true" || s == "false") && return false + c, rest = x + is_id_start_char(c) || return false + return all(is_id_char, rest) +end +isidentifier(s::Symbol) = isidentifier(string(s)) + +is_op_suffix_char(c::AbstractChar) = ccall(:jl_op_suffix_char, Cint, (UInt32,), c) != 0 + +_isoperator(s) = ccall(:jl_is_operator, Cint, (Cstring,), s) != 0 + +""" + isoperator(s::Symbol) + +Return `true` if the symbol can be used as an operator, `false` otherwise. + +# Examples +```jldoctest +julia> Meta.isoperator(:+), Meta.isoperator(:f) +(true, false) +``` +""" +isoperator(s::Union{Symbol,AbstractString}) = _isoperator(s) || ispostfixoperator(s) + +""" + isunaryoperator(s::Symbol) + +Return `true` if the symbol can be used as a unary (prefix) operator, `false` otherwise. + +# Examples +```jldoctest +julia> Meta.isunaryoperator(:-), Meta.isunaryoperator(:√), Meta.isunaryoperator(:f) +(true, true, false) +``` +""" +isunaryoperator(s::Symbol) = ccall(:jl_is_unary_operator, Cint, (Cstring,), s) != 0 +is_unary_and_binary_operator(s::Symbol) = ccall(:jl_is_unary_and_binary_operator, Cint, (Cstring,), s) != 0 +is_syntactic_operator(s::Symbol) = ccall(:jl_is_syntactic_operator, Cint, (Cstring,), s) != 0 + +""" + isbinaryoperator(s::Symbol) + +Return `true` if the symbol can be used as a binary (infix) operator, `false` otherwise. + +# Examples +```jldoctest +julia> Meta.isbinaryoperator(:-), Meta.isbinaryoperator(:√), Meta.isbinaryoperator(:f) +(true, false, false) +``` +""" +function isbinaryoperator(s::Symbol) + return _isoperator(s) && (!isunaryoperator(s) || is_unary_and_binary_operator(s)) && + s !== Symbol("'") +end + +""" + ispostfixoperator(s::Union{Symbol,AbstractString}) + +Return `true` if the symbol can be used as a postfix operator, `false` otherwise. + +# Examples +```jldoctest +julia> Meta.ispostfixoperator(Symbol("'")), Meta.ispostfixoperator(Symbol("'ᵀ")), Meta.ispostfixoperator(:-) +(true, true, false) +``` +""" +function ispostfixoperator(s::Union{Symbol,AbstractString}) + s = String(s)::String + return startswith(s, '\'') && all(is_op_suffix_char, SubString(s, 2)) +end + +const keyword_syms = IdSet{Symbol}([ + :baremodule, :begin, :break, :catch, :const, :continue, :do, :else, :elseif, + :end, :export, :var"false", :finally, :for, :function, :global, :if, :import, + :let, :local, :macro, :module, :public, :quote, :return, :struct, :var"true", + :try, :using, :while ]) + +function is_valid_identifier(sym) + return (isidentifier(sym) && !(sym in keyword_syms)) || + (_isoperator(sym) && + !(sym in (Symbol("'"), :(::), :?)) && + !is_syntactic_operator(sym) + ) +end + """ Meta.quot(ex)::Expr @@ -192,12 +304,23 @@ end ParseError(msg::AbstractString) = ParseError(msg, nothing) +function parser_for_module(mod::Union{Module, Nothing}) + if mod isa Module && invokelatest(isdefined, mod, Symbol("#_internal_julia_parse")) + invokelatest(getglobal, mod, Symbol("#_internal_julia_parse")) + elseif isdefined(Base, :VersionedParse) + Base.VersionedParse(VERSION) + else + Base.fl_parse_bootstrap + end +end + function _parse_string(text::AbstractString, filename::AbstractString, - lineno::Integer, index::Integer, options) + lineno::Integer, index::Integer, options, + _parse=parser_for_module(nothing)) if index < 1 || index > ncodeunits(text) + 1 throw(BoundsError(text, index)) end - ex, offset::Int = Core._parse(text, filename, lineno, index-1, options) + ex, offset::Int = _parse(text, filename, lineno, index-1, options) ex, offset+1 end @@ -234,8 +357,8 @@ julia> Meta.parse("(α, β) = 3, 5", 11, greedy=false) ``` """ function parse(str::AbstractString, pos::Integer; - filename="none", greedy::Bool=true, raise::Bool=true, depwarn::Bool=true) - ex, pos = _parse_string(str, String(filename), 1, pos, greedy ? :statement : :atom) + filename="none", greedy::Bool=true, raise::Bool=true, depwarn::Bool=true, mod::Union{Nothing, Module}=nothing, _parse = parser_for_module(mod)) + ex, pos = _parse_string(str, String(filename), 1, pos, greedy ? :statement : :atom, _parse) if raise && isexpr(ex, :error) err = ex.args[1] if err isa String @@ -274,8 +397,8 @@ julia> Meta.parse("x = ") ``` """ function parse(str::AbstractString; - filename="none", raise::Bool=true, depwarn::Bool=true) - ex, pos = parse(str, 1; filename, greedy=true, raise, depwarn) + filename="none", raise::Bool=true, depwarn::Bool=true, mod::Union{Nothing, Module}=nothing, _parse = parser_for_module(mod)) + ex, pos = parse(str, 1; filename, greedy=true, raise, depwarn, _parse) if isexpr(ex, :error) return ex end @@ -286,12 +409,16 @@ function parse(str::AbstractString; return ex end -function parseatom(text::AbstractString, pos::Integer; filename="none", lineno=1) - return _parse_string(text, String(filename), lineno, pos, :atom) +function parseatom(text::AbstractString, pos::Integer; filename="none", + lineno=1, mod::Union{Nothing, Module}=nothing, + _parse = parser_for_module(mod)) + return _parse_string(text, String(filename), lineno, pos, :atom, _parse) end -function parseall(text::AbstractString; filename="none", lineno=1) - ex,_ = _parse_string(text, String(filename), lineno, 1, :all) +function parseall(text::AbstractString; filename="none", lineno=1, + mod::Union{Nothing, Module}=nothing, + _parse = parser_for_module(mod)) + ex,_ = _parse_string(text, String(filename), lineno, 1, :all, _parse) return ex end @@ -409,7 +536,7 @@ function _partially_inline!(@nospecialize(x), slot_replacements::Vector{Any}, end return x elseif head === :cfunction - @assert !isa(type_signature, UnionAll) || !isempty(spvals) + @assert !isa(type_signature, UnionAll) || !isempty(static_param_values) if !isa(x.args[2], QuoteNode) # very common no-op x.args[2] = _partially_inline!(x.args[2], slot_replacements, type_signature, static_param_values, slot_offset, @@ -427,7 +554,9 @@ function _partially_inline!(@nospecialize(x), slot_replacements::Vector{Any}, elseif i == 4 @assert isa(x.args[4], Int) elseif i == 5 - @assert isa((x.args[5]::QuoteNode).value, Union{Symbol, Tuple{Symbol, UInt16, Bool}}) + @assert isa((x.args[5]::QuoteNode).value, + Union{Symbol, Tuple{Symbol, UInt16, Bool}, Tuple{Symbol, UInt16, Bool, Bool}, + Tuple{Symbol, UInt16, Bool, Bool, Bool}}) else x.args[i] = _partially_inline!(x.args[i], slot_replacements, type_signature, static_param_values, @@ -516,6 +645,24 @@ function unescape(@nospecialize ex) return ex end +""" + Meta.reescape(unescaped_expr, original_expr) + +Re-wrap `unescaped_expr` with the same level of escaping as `original_expr` had. +This is the inverse operation of [`unescape`](@ref) - if the original expression +was escaped, the unescaped expression is wrapped in `:escape` again. +""" +function reescape(@nospecialize(unescaped_expr), @nospecialize(original_expr)) + if isexpr(original_expr, :escape) + return reescape(Expr(:escape, unescaped_expr), original_expr.args[1]) + elseif isexpr(original_expr, :var"hygienic-scope") + next, ctx... = original_expr.args + return reescape(Expr(:var"hygienic-scope", unescaped_expr, ctx...), next) + else + return unescaped_expr + end +end + """ Meta.uncurly(expr) diff --git a/base/methodshow.jl b/base/methodshow.jl index 1470303a01bbc..ecc5ae294897e 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -71,17 +71,39 @@ function arg_decl_parts(m::Method, html=false) decls[1] = ("", sprint(show_signature_function, unwrapva(sig.parameters[1]), false, decls[1][1], html, context = show_env)) else - decls = Tuple{String,String}[("", "") for i = 1:length(sig.parameters::SimpleVector)] + decls = Tuple{String,String}[("", "") for _ = 1:length(sig.parameters::SimpleVector)] end return tv, decls, file, line end # NOTE: second argument is deprecated and is no longer used -function kwarg_decl(m::Method, kwtype = nothing) +function kwarg_decl(m::Method, kwtype = nothing; world::UInt=get_world_counter()) if !(m.sig === Tuple || m.sig <: Tuple{Core.Builtin, Vararg}) # OpaqueClosure or Builtin kwtype = typeof(Core.kwcall) - sig = rewrap_unionall(Tuple{kwtype, NamedTuple, (unwrap_unionall(m.sig)::DataType).parameters...}, m.sig) - kwli = ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, get_world_counter()) + sig_params = (unwrap_unionall(m.sig)::DataType).parameters + sig = rewrap_unionall(Tuple{kwtype, NamedTuple, sig_params...}, m.sig) + kwli = ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, world) + if kwli === nothing + # a compiled keyword sorter is specialized on the dispatch (egality) + # spelling of closed type-valued slots, so retry the lookup with + # `Type{X}` slots as `Core.TypeEgal{X}` + new_params = Any[kwtype, NamedTuple] + changed = false + for p in sig_params + if p isa TypeEq + tp = type_parameter(p) + if tp isa Type && !has_free_typevars(tp) + p = Core.TypeEgal{tp} + changed = true + end + end + push!(new_params, p) + end + if changed + sig = rewrap_unionall(Tuple{new_params...}, m.sig) + kwli = ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, world) + end + end if kwli !== nothing kwli = kwli::Method slotnames = ccall(:jl_uncompress_argnames, Vector{Symbol}, (Any,), kwli.slot_syms) @@ -140,7 +162,7 @@ function fixup_stdlib_path(path::String) if isdefined(@__MODULE__, :Core) && isdefined(Core, :Compiler) compiler_folder = dirname(String(Base.moduleloc(Core.Compiler).file)) if dirname(path) == compiler_folder - return abspath(Sys.STDLIB, "..", "..", "Compiler", "src", basename(path)) + return abspath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "Compiler", "src", basename(path)) end end end @@ -286,7 +308,7 @@ function show_method_list_header(io::IO, ms::MethodList, namefmt::Function) printstyled(io, tn.module, color=col) elseif '#' in sname print(io, " for anonymous function ", namedisplay) - elseif tn === _TYPE_NAME || iskindtype(tn.wrapper) + elseif iskindtype(tn.wrapper) print(io, " for type constructor") else print(io, " for callable object") @@ -306,13 +328,8 @@ function _modulecolor(method::Method) # method table is shared, we now need to distinguish "primary" methods by trying to # check if there is a primary `DataType` to identify it with. c.f. how `jl_method_def` # would derive this same information (for the name). - ft = argument_datatype((unwrap_unionall(method.sig)::DataType).parameters[1]) - # `ft` should be the type associated with the first argument in the method signature. - # If it's `Type`, try to unwrap it again. - if isType(ft) - ft = argument_datatype(ft.parameters[1]) - end - if ft === nothing || parentmodule(method) === parentmodule(ft) !== Core + ft = argument_datatypename((unwrap_unionall(method.sig)::DataType).parameters[1]) + if ft === nothing || parentmodule(method) === ft.module !== Core return nothing end m = parentmodule_before_main(method) @@ -328,7 +345,7 @@ function show_method_table(io::IO, ms::MethodList, max::Int=-1, header::Bool=tru show_method_list_header(io, ms, str -> "\""*str*"\"") end n = rest = 0 - local last + last = nothing last_shown_line_infos = get(io, :last_shown_line_infos, nothing) last_shown_line_infos === nothing || empty!(last_shown_line_infos) @@ -355,7 +372,7 @@ function show_method_table(io::IO, ms::MethodList, max::Int=-1, header::Bool=tru end if rest > 0 println(io) - if rest == 1 + if rest == 1 && last isa Method show_method(io, last) else print(io, "... $rest methods not shown") @@ -401,19 +418,21 @@ function url(m::Method) if LibGit2 isa Module try d = dirname(file) - return LibGit2.with(LibGit2.GitRepoExt(d)) do repo - LibGit2.with(LibGit2.GitConfig(repo)) do cfg - u = LibGit2.get(cfg, "remote.origin.url", "") - u = (match(LibGit2.GITHUB_REGEX,u)::AbstractMatch).captures[1] - commit = string(LibGit2.head_oid(repo)) - root = LibGit2.path(repo) - if startswith(file, root) || startswith(realpath(file), root) - "https://github.com/$u/tree/$commit/"*file[length(root)+1:end]*"#L$line" - else - fileurl(file) + return let file = file + LibGit2.with(LibGit2.GitRepoExt(d)) do repo + LibGit2.with(LibGit2.GitConfig(repo)) do cfg + u = LibGit2.get(cfg, "remote.origin.url", "") + u = (match(LibGit2.GITHUB_REGEX,u)::AbstractMatch).captures[1] + commit = string(LibGit2.head_oid(repo)) + root = LibGit2.path(repo) + if startswith(file, root) || startswith(realpath(file), root) + "https://github.com/$u/tree/$commit/"*file[length(root)+1:end]*"#L$line" + else + fileurl(file) + end end end - end + end::String catch # oops, this was a bad idea end @@ -422,7 +441,7 @@ function url(m::Method) end function show(io::IO, ::MIME"text/html", m::Method) - tv, decls, file, line = arg_decl_parts(m, true) + tv, decls, _file, line = arg_decl_parts(m, true) sig = unwrap_unionall(m.sig) if sig <: Tuple{Core.Builtin, Vararg} print(io, m.name, "(...) in ", parentmodule(m)) diff --git a/base/missing.jl b/base/missing.jl index 5fc0e6ec9e620..a103f4c5dfbfd 100644 --- a/base/missing.jl +++ b/base/missing.jl @@ -102,19 +102,14 @@ for f in (:(!), :(~), :(+), :(-), :(*), :(&), :(|), :(xor), :(real), :(imag), :(sign), :(inv)) @eval ($f)(::Missing) = missing end -for f in (:(Base.zero), :(Base.one), :(Base.oneunit)) +for f in (:zero, :one, :oneunit) + @eval ($f)(::Type{Any}) = throw(MethodError($f, (Any,))) # To prevent StackOverflowError @eval ($f)(::Type{Missing}) = missing - @eval function $(f)(::Type{Union{T, Missing}}) where T - T === Any && throw(MethodError($f, (Any,))) # To prevent StackOverflowError - $f(T) - end + @eval ($f)(::Type{T}) where {T>:Missing} = $f(nonmissingtype_checked(T)) end -for f in (:(Base.float), :(Base.complex)) - @eval $f(::Type{Missing}) = Missing - @eval function $f(::Type{Union{T, Missing}}) where T - T === Any && throw(MethodError($f, (Any,))) # To prevent StackOverflowError - Union{$f(T), Missing} - end +for f in (:float, :real, :complex) + @eval ($f)(::Type{Any}) = throw(MethodError($f, (Any,))) # To prevent StackOverflowError + @eval ($f)(::Type{T}) where {T>:Missing} = Union{$f(nonmissingtype(T)), Missing} end # Binary operators/functions @@ -131,12 +126,14 @@ div(::Missing, ::Missing, r::RoundingMode) = missing div(::Missing, ::Number, r::RoundingMode) = missing div(::Number, ::Missing, r::RoundingMode) = missing -min(::Missing, ::Missing) = missing -min(::Missing, ::Any) = missing -min(::Any, ::Missing) = missing -max(::Missing, ::Missing) = missing -max(::Missing, ::Any) = missing -max(::Any, ::Missing) = missing +for (f, result) in ((:min, missing), (:max, missing), (:minmax, (missing, missing))) + @eval begin + $f(::Missing, ::Missing) = $result + $f(::Missing, ::Any) = $result + $f(::Any, ::Missing) = $result + $f(::Missing) = $result + end +end clamp(::Missing, lo, hi) = missing missing_conversion_msg(@nospecialize T) = diff --git a/base/module.jl b/base/module.jl index c21414179446e..b1fcec9746c4f 100644 --- a/base/module.jl +++ b/base/module.jl @@ -12,17 +12,16 @@ end function eval_import_path(at::Module, from::Union{Module, Nothing}, path::Expr, keyword::String) isempty(path.args) && error("malformed import statement") - i::Int = 1 + i = RefValue(1) function next!() local v - i <= length(path.args) || error("invalid module path") - v = path.args[i] - i += 1 + i[] <= length(path.args) || error("invalid module path") + v = path.args[i[]] + i[] += 1 v isa Symbol || throw(TypeError(Symbol(keyword), "", Symbol, v)) v end v = next!() - m = nothing if from !== nothing m = from @@ -36,7 +35,7 @@ function eval_import_path(at::Module, from::Union{Module, Nothing}, path::Expr, m = require(at, v) m isa Module || error("failed to load module $v") end - i > lastindex(path.args) && return m, nothing + i[] > lastindex(path.args) && return m, nothing v = next!() else # `.A.B.C`: strip off leading dots by following parent links @@ -48,7 +47,7 @@ function eval_import_path(at::Module, from::Union{Module, Nothing}, path::Expr, while true v === :. && error("invalid $keyword path: \".\" in identifier path") - i > lastindex(path.args) && break + i[] > lastindex(path.args) && break m = getglobal(m, v) m isa Module || error("invalid $keyword path: \"$v\" does not name a module") v = next!() @@ -128,14 +127,14 @@ making its exports available to the `to` module; this is the lowered form of `using A`. ``` -using A.B => _module_using(Main, Expr(:., :A, :B)) +using A.B => _eval_using(Main, Expr(:., :A, :B)) ``` See also [`_using`](@ref Core._using). """ -function _eval_using(to::Module, path::Expr) +function _eval_using(to::Module, path::Expr, flags::UInt8=UInt8(0)) from = eval_import_path_all(to, path, "using") - Core._using(to, from) + Core._using(to, from, flags) is_package = length(path.args) == 1 && path.args[1] !== :. if to == Main && is_package Core._import(to, from, nameof(from)) diff --git a/base/mpfr.jl b/base/mpfr.jl index cb51d21aa5bc5..836a33a2d78b3 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -9,14 +9,14 @@ export import .Base: *, +, -, /, <, <=, ==, >, >=, ^, ceil, cmp, convert, copysign, div, inv, exp, exp2, exponent, factorial, floor, fma, muladd, hypot, isinteger, - isfinite, isinf, isnan, ldexp, log, log2, log10, max, min, mod, modf, + isfinite, isinf, isnan, issubnormal, ldexp, log, log2, log10, max, min, modf, nextfloat, prevfloat, promote_rule, rem, rem2pi, round, show, float, sum, sqrt, string, print, trunc, precision, _precision, exp10, expm1, log1p, - eps, signbit, sign, sin, cos, sincos, tan, sec, csc, cot, acos, asin, atan, + eps, signbit, sign, sin, cos, sincos, tan, sec, csc, acos, asin, atan, cosh, sinh, tanh, sech, csch, coth, acosh, asinh, atanh, lerpi, cbrt, typemax, typemin, unsafe_trunc, floatmin, floatmax, rounding, setrounding, maxintfloat, widen, significand, frexp, tryparse, iszero, - isone, big, _string_n, decompose, minmax, _precision_with_base_2, + isone, big, decompose, minmax, _precision_with_base_2, sinpi, cospi, sincospi, tanpi, sind, cosd, tand, asind, acosd, atand, uinttype, exponent_max, exponent_min, ieee754_representation, significand_mask, ispositive, isnegative @@ -197,7 +197,7 @@ end end # While BigFloat (like all Numbers) is considered immutable, for practical reasons -# of writing the algorithms on it we allow mutating sign, exp, and the contents of d +# of writing the algorithms on it we allow mutating sign and exp @inline function Base.setproperty!(x::BigFloat, s::Symbol, v) d = getfield(x, :d) p = Base.unsafe_convert(Ptr{Limb}, d) @@ -205,9 +205,8 @@ end return GC.@preserve d unsafe_store!(Ptr{Cint}(p) + offset_sign, v) elseif s === :exp return GC.@preserve d unsafe_store!(Ptr{Clong}(p) + offset_exp, v) - #elseif s === :d || s === :prec # not mutable else - return throw(FieldError(x, s)) + throw(FieldError(BigFloat, s)) end end @@ -747,7 +746,7 @@ end # More efficient commutative operations -for (fJ, fC, fI) in ((:+, :add, 0), (:*, :mul, 1)) +for (fJ, fC) in ((:+, :add), (:*, :mul)) @eval begin function ($fJ)(a::BigFloat, b::BigFloat, c::BigFloat) z = BigFloat() @@ -990,9 +989,7 @@ end # Utility functions ==(x::BigFloat, y::BigFloat) = ccall((:mpfr_equal_p, libmpfr), Int32, (Ref{BigFloat}, Ref{BigFloat}), x, y) != 0 <=(x::BigFloat, y::BigFloat) = ccall((:mpfr_lessequal_p, libmpfr), Int32, (Ref{BigFloat}, Ref{BigFloat}), x, y) != 0 ->=(x::BigFloat, y::BigFloat) = ccall((:mpfr_greaterequal_p, libmpfr), Int32, (Ref{BigFloat}, Ref{BigFloat}), x, y) != 0 <(x::BigFloat, y::BigFloat) = ccall((:mpfr_less_p, libmpfr), Int32, (Ref{BigFloat}, Ref{BigFloat}), x, y) != 0 ->(x::BigFloat, y::BigFloat) = ccall((:mpfr_greater_p, libmpfr), Int32, (Ref{BigFloat}, Ref{BigFloat}), x, y) != 0 function cmp(x::BigFloat, y::BigInt) isnan(x) && return 1 @@ -1137,6 +1134,8 @@ function isnan(x::BigFloat) return x.exp == mpfr_special_exponent_nan end +issubnormal(x::BigFloat) = false + isfinite(x::BigFloat) = !isinf(x) && !isnan(x) iszero(x::BigFloat) = x.exp == mpfr_special_exponent_zero @@ -1151,7 +1150,7 @@ isnegative(x::BigFloat) = signbit(x) && !iszero(x) && !isnan(x) function nextfloat!(x::BigFloat, n::Integer=1) signbit(n) && return prevfloat!(x, abs(n)) - for i = 1:n + for _ = 1:n ccall((:mpfr_nextabove, libmpfr), Int32, (Ref{BigFloat},), x) end return x @@ -1159,7 +1158,7 @@ end function prevfloat!(x::BigFloat, n::Integer=1) signbit(n) && return nextfloat!(x, abs(n)) - for i = 1:n + for _ = 1:n ccall((:mpfr_nextbelow, libmpfr), Int32, (Ref{BigFloat},), x) end return x @@ -1199,10 +1198,10 @@ Note: `nextfloat()`, `prevfloat()` do not use the precision mentioned by The `base` keyword requires at least Julia 1.8. """ function setprecision(f::Function, ::Type{T}, prec::Integer; kws...) where T - depwarn(""" - The fallback `setprecision(::Function, ...)` method is deprecated. Packages overloading this method should - implement their own specialization using `ScopedValue` instead. - """, :setprecision) + Base.depwarn(""" + The fallback `setprecision(::Function, ...)` method is deprecated. Packages overloading this method should + implement their own specialization using `ScopedValue` instead. + """, :setprecision) old_prec = precision(T) setprecision(T, prec; kws...) try @@ -1256,7 +1255,7 @@ function _prettify_bigfloat(s::String)::String else neg = startswith(int, '-') neg == true && (int = lstrip(int, '-')) - @assert length(int) == 1 + @assert length(int) == 1 "length(int) != 1" string(neg ? '-' : "", '0', '.', '0'^(-expo-1), int, frac == "0" ? "" : frac) end else diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 83a03fc1b45bf..afb87b0d20105 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -345,6 +345,8 @@ module IteratorsMD CartesianIndices((2:1:3, 1:2:3)) ``` """ + (:)(::CartesianIndex, ::CartesianIndex, ::CartesianIndex) + (:)(I::CartesianIndex{N}, J::CartesianIndex{N}) where N = CartesianIndices(map((i,j) -> i:j, Tuple(I), Tuple(J))) (:)(I::CartesianIndex{N}, S::CartesianIndex{N}, J::CartesianIndex{N}) where N = @@ -447,7 +449,7 @@ module IteratorsMD end iterfirst, iterfirst end - @inline function iterate(iter::CartesianIndices, state) + @inline function iterate(iter::CartesianIndices, state::CartesianIndex) valid, I = __inc(state.I, iter.indices) valid || return nothing return CartesianIndex(I...), CartesianIndex(I...) @@ -466,7 +468,7 @@ module IteratorsMD __inc(::Tuple{}, ::Tuple{}) = false, () @inline function __inc(state::Tuple{Int}, indices::Tuple{OrdinalRangeInt}) rng = indices[1] - I = state[1] + step(rng) + I = state[1] +% step(rng) valid = state[1] != last(rng) return valid, (I,) end @@ -481,12 +483,10 @@ module IteratorsMD end # 0-d cartesian ranges are special-cased to iterate once and only once - iterate(iter::CartesianIndices{0}, done=false) = done ? nothing : (CartesianIndex(), true) + iterate(iter::CartesianIndices{0}, done::Bool=false) = done ? nothing : (CartesianIndex(), true) size(iter::CartesianIndices) = map(length, iter.indices) - length(iter::CartesianIndices) = prod(size(iter)) - # make CartesianIndices a multidimensional range Base.step(iter::CartesianIndices) = CartesianIndex(map(step, iter.indices)) @@ -592,13 +592,13 @@ module IteratorsMD @inline __dec(::Tuple{}, ::Tuple{}) = false, () @inline function __dec(state::Tuple{Int}, indices::Tuple{OrdinalRangeInt}) rng = indices[1] - I = state[1] - step(rng) + I = state[1] -% step(rng) valid = state[1] != first(rng) return valid, (I,) end @inline function __dec(state::Tuple{Int,Int,Vararg{Int}}, indices::Tuple{OrdinalRangeInt,OrdinalRangeInt,Vararg{OrdinalRangeInt}}) rng = indices[1] - I = state[1] - step(rng) + I = state[1] -% step(rng) if state[1] != first(rng) return true, (I, tail(state)...) end @@ -673,21 +673,24 @@ module IteratorsMD # CartesianPartition. mi = iter.parent.mi ci = iter.parent.parent - ax, ax1 = axes(ci), Base.axes1(ci) - subs = Base.ind2sub_rs(ax, mi, first(iter.indices[1])) - vl, fl = Base._sub2ind(tail(ax), tail(subs)...), subs[1] - vr, fr = divrem(last(iter.indices[1]) - 1, mi[end]) .+ (1, first(ax1)) + ax1 = Base.axes1(ci) + function splitdim1(i, mi) + d, r = divrem(i - 1, mi) + d + 1, r + first(ax1) + end + vl, fl = splitdim1(first(iter.indices[1]), mi[1]) + vr, fr = splitdim1(last(iter.indices[1]), mi[1]) + # form the iterator for outer dimensions, equivalent to vec(oci), but mi is reused oci = CartesianIndices(tail(ci.indices)) - # A fake CartesianPartition to reuse the outer iterate fallback - outer = @inbounds view(ReshapedArray(oci, (length(oci),), mi), vl:vr) - init = @inbounds dec(oci[tail(subs)...].I, oci.indices) # real init state + roci = ReshapedArray(oci, (length(oci),), tail(mi)) + outer = @inbounds view(roci, vl:vr) # Use Generator to make inner loop branchless @inline function skip_len_I(i::Int, I::CartesianIndex) l = i == 1 ? fl : first(ax1) r = i == length(outer) ? fr : last(ax1) l - first(ax1), r - l + 1, I end - (skip_len_I(i, I) for (i, I) in Iterators.enumerate(Iterators.rest(outer, (init, 0)))) + (skip_len_I(i, I) for (i, I) in Iterators.enumerate(outer)) end @inline function simd_outer_range(iter::CartesianPartition{CartesianIndex{2}}) # But for two-dimensional Partitions the above is just a simple one-dimensional range @@ -756,12 +759,12 @@ end # Here we try to consume N of the indices (if there are that many available) @inline function checkbounds_indices(::Type{Bool}, inds::Tuple, I::Tuple{CartesianIndex,Vararg}) inds1, rest = IteratorsMD.split(inds, Val(length(I[1]))) - checkindex(Bool, inds1, I[1]) & checkbounds_indices(Bool, rest, tail(I)) + checkindex(Bool, inds1, I[1]) && checkbounds_indices(Bool, rest, tail(I)) end @inline checkindex(::Type{Bool}, inds::Tuple, I::CartesianIndex) = checkbounds_indices(Bool, inds, I.I) @inline checkindex(::Type{Bool}, inds::Tuple, i::AbstractRange{<:CartesianIndex}) = - isempty(i) | (checkindex(Bool, inds, first(i)) & checkindex(Bool, inds, last(i))) + isempty(i) | (checkindex(Bool, inds, first(i)) && checkindex(Bool, inds, last(i))) # Indexing into Array with mixtures of Integers and CartesianIndices is # extremely performance-sensitive. While the abstract fallbacks support this, @@ -778,7 +781,7 @@ end # Here we try to consume N of the indices (if there are that many available) @inline function checkbounds_indices(::Type{Bool}, inds::Tuple, I::Tuple{AbstractArray{CartesianIndex{N}},Vararg}) where N inds1, rest = IteratorsMD.split(inds, Val(N)) - checkindex(Bool, inds1, I[1]) & checkbounds_indices(Bool, rest, tail(I)) + checkindex(Bool, inds1, I[1]) && checkbounds_indices(Bool, rest, tail(I)) end @inline checkindex(::Type{Bool}, inds::Tuple, I::CartesianIndices) = checkbounds_indices(Bool, inds, I.indices) @@ -833,7 +836,6 @@ LogicalIndex(mask::AbstractVector{Bool}) = LogicalIndex{Int, typeof(mask)}(mask) LogicalIndex(mask::AbstractArray{Bool, N}) where {N} = LogicalIndex{CartesianIndex{N}, typeof(mask)}(mask) LogicalIndex{Int}(mask::AbstractArray) = LogicalIndex{Int, typeof(mask)}(mask) size(L::LogicalIndex) = (L.sum,) -length(L::LogicalIndex) = L.sum collect(L::LogicalIndex) = [i for i in L] show(io::IO, r::LogicalIndex) = print(io,collect(r)) print_array(io::IO, X::LogicalIndex) = print_array(io, collect(X)) @@ -905,12 +907,12 @@ checkbounds(::Type{Bool}, A::AbstractArray, i::AbstractVector{Bool}) = checkindex(Bool, eachindex(IndexLinear(), A), i) @inline function checkbounds_indices(::Type{Bool}, inds::Tuple, I::Tuple{AbstractArray{Bool},Vararg}) inds1, rest = IteratorsMD.split(inds, Val(ndims(I[1]))) - checkindex(Bool, inds1, I[1]) & checkbounds_indices(Bool, rest, tail(I)) + checkindex(Bool, inds1, I[1]) && checkbounds_indices(Bool, rest, tail(I)) end checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractVector{Bool}) = axes1(I) == inds checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractRange{Bool}) = axes1(I) == inds checkindex(::Type{Bool}, inds::Tuple, I::AbstractArray{Bool}) = _check_boolean_axes(inds, axes(I)) -_check_boolean_axes(inds::Tuple, axes::Tuple) = (inds[1] == axes[1]) & _check_boolean_axes(tail(inds), tail(axes)) +_check_boolean_axes(inds::Tuple, axes::Tuple) = (inds[1] == axes[1]) && _check_boolean_axes(tail(inds), tail(axes)) _check_boolean_axes(::Tuple{}, axes::Tuple) = all(==(OneTo(1)), axes) ensure_indexable(I::Tuple{}) = () @@ -960,7 +962,7 @@ Fallback to `iterate` by default, but optimized for indices type in `Base`. _prechecked_iterate(iter::AbstractUnitRange, i = first(iter)) = i, convert(eltype(iter), i + step(iter)) _prechecked_iterate(iter::LinearIndices, i = first(iter)) = i, i + 1 _prechecked_iterate(iter::CartesianIndices) = first(iter), first(iter) -function _prechecked_iterate(iter::CartesianIndices, i) +function _prechecked_iterate(iter::CartesianIndices, i::CartesianIndex) i′ = IteratorsMD.inc(i.I, iter.indices) return i′, i′ end @@ -1116,7 +1118,7 @@ end function mightalias(A::SubArray, B::SubArray) # There are three ways that SubArrays might _problematically_ alias one another: # 1. The parents are the same we can conservatively check if the indices might overlap OR - # 2. The parents alias eachother in a more complicated manner (and we can't trace indices) OR + # 2. The parents alias each other in a more complicated manner (and we can't trace indices) OR # 3. One's parent is used in the other's indices # Note that it's ok for just the indices to alias each other as those should not be mutated, # so we can always do better than the default !_isdisjoint(dataids(A), dataids(B)) @@ -1173,46 +1175,6 @@ end # And in general, checking the intersection is too much work _indicesmightoverlap(A::Tuple{Any, Vararg{Any}}, B::Tuple{Any, Vararg{Any}}) = true -""" - fill!(A, x) - -Fill array `A` with the value `x`. If `x` is an object reference, all elements will refer to -the same object. `fill!(A, Foo())` will return `A` filled with the result of evaluating -`Foo()` once. - -# Examples -```jldoctest -julia> A = zeros(2,3) -2×3 Matrix{Float64}: - 0.0 0.0 0.0 - 0.0 0.0 0.0 - -julia> fill!(A, 2.) -2×3 Matrix{Float64}: - 2.0 2.0 2.0 - 2.0 2.0 2.0 - -julia> a = [1, 1, 1]; A = fill!(Vector{Vector{Int}}(undef, 3), a); a[1] = 2; A -3-element Vector{Vector{Int64}}: - [2, 1, 1] - [2, 1, 1] - [2, 1, 1] - -julia> x = 0; f() = (global x += 1; x); fill!(Vector{Int}(undef, 3), f()) -3-element Vector{Int64}: - 1 - 1 - 1 -``` -""" -function fill!(A::AbstractArray{T}, x) where T - xT = convert(T, x) - for I in eachindex(A) - @inbounds A[I] = xT - end - A -end - function copyto!(dest::AbstractArray{T1,N}, Rdest::CartesianIndices{N}, src::AbstractArray{T2,N}, Rsrc::CartesianIndices{N}) where {T1,T2,N} isempty(Rdest) && return dest @@ -1439,7 +1401,7 @@ end end end -# in the general multidimensional non-scalar case, can we do about 10% better +# in the general multidimensional non-scalar case, we can do about 10% better # in most cases by manually hoisting the bitarray chunks access out of the loop # (This should really be handled by the compiler or with an immutable BitArray) @generated function _unsafe_getindex!(X::BitArray, B::BitArray, I::Union{Int,AbstractArray{Int}}...) @@ -1526,7 +1488,7 @@ function copy_to_bitarray_chunks!(Bc::Vector{UInt64}, pos_d::Int, C::StridedArra end @inbounds if bind ≤ kd1 - @assert bind == kd1 + @assert bind == kd1 "bind != kd1" c = UInt64(0) for j = 0:ld1 c |= (UInt64(unchecked_bool_convert(C[ind])) << j) @@ -1539,7 +1501,7 @@ end # contiguous multidimensional indexing: if the first dimension is a range, # we can get some performance from using copy_chunks! -@inline function setindex!(B::BitArray, X::Union{StridedArray,BitArray}, J0::Union{Colon,AbstractUnitRange{Int}}) +@inline function setindex!(B::BitArray, X::Union{StridedArray,BitArray}, J0::D) where {D<:Union{Colon,AbstractUnitRange{Int}}} I0 = to_indices(B, (J0,))[1] @boundscheck checkbounds(B, I0) l0 = length(I0) @@ -1551,7 +1513,7 @@ end end @inline function setindex!(B::BitArray, X::Union{StridedArray,BitArray}, - I0::Union{Colon,AbstractUnitRange{Int}}, I::Union{Int,AbstractUnitRange{Int},Colon}...) + I0::DI0, I::Union{Int,AbstractUnitRange{Int},Colon}...) where {DI0<:Union{Colon,AbstractUnitRange{Int}}, } J = to_indices(B, (I0, I...)) @boundscheck checkbounds(B, J...) _unsafe_setindex!(B, X, J...) @@ -1592,7 +1554,7 @@ end end @propagate_inbounds function setindex!(B::BitArray, X::AbstractArray, - I0::Union{Colon,AbstractUnitRange{Int}}, I::Union{Int,AbstractUnitRange{Int},Colon}...) + I0::DI0, I::Union{Int,AbstractUnitRange{Int},Colon}...) where {DI0<:Union{Colon,AbstractUnitRange{Int}}} _setindex!(IndexStyle(B), B, X, to_indices(B, (I0, I...))...) end @@ -1787,7 +1749,7 @@ julia> unique(A, dims=3) 0 0 ``` """ -unique(A::AbstractArray; dims::Union{Colon,Integer} = :) = _unique_dims(A, dims) +unique(A::AbstractArray; dims::D = :) where {D<:Union{Colon,Integer}} = _unique_dims(A, dims) _unique_dims(A::AbstractArray, dims::Colon) = invoke(unique, Tuple{Any}, A) @@ -2083,7 +2045,34 @@ function _hash_fib(A, h::UInt) return hash_uint(h) end -function hash_shaped(A, h::UInt) +""" + union_split(f, x, ts::Tuple{Vararg{Val}}, args...) + +Call `f(x, args...)`, union-splitting on all the types specified by `ts` + +`union_split(f, x, (Val{T1}(), Val{T2}()), y, z)` is equivalent to + +``` +if x isa T1 + f(x, y, z) +elseif x isa T2 + f(x, y, z) +else + f(x, y, z) +end +``` +""" +@inline function union_split(f, @nospecialize(x), ts::Tuple{Val{T}, Vararg{Val,N}}, args...) where {T, N} + if x isa T + f(x, args...) + else + union_split(f, x, Base.tail(ts), args...) + end +end +@inline union_split(f, x, ::Tuple{}, args::Vararg{Any, N}) where {N} = f(x, args...) + +function hash_shaped(A, h0::UInt, eltype_hint=()) + h::UInt = h0 # Axes are themselves AbstractArrays, so hashing them directly would stack overflow # Instead hash the tuple of firsts and lasts along each dimension h = hash(map(first, axes(A)), h) @@ -2093,20 +2082,20 @@ function hash_shaped(A, h::UInt) if len < 8 # for the shortest arrays we chain directly for elt in A - h = hash(elt, h) + h = union_split(hash, elt, eltype_hint, h) end return h elseif len < 32768 # separate accumulator streams, unrolled - @nexprs 8 i -> p_i = h + @nexprs 8 i -> p_i::UInt = h n = 1 limit = len - 7 while n <= limit - @nexprs 8 i -> p_i = hash(A[n + i - 1], p_i) + @nexprs 8 i -> p_i = union_split(hash, A[n + i - 1], eltype_hint, p_i) n += 8 end while n <= len - p_1 = hash(A[n], p_1) + p_1 = union_split(hash, A[n], eltype_hint, p_1) n += 1 end # fold all streams back together diff --git a/base/multimedia.jl b/base/multimedia.jl index 883e4831a39fb..0d068edb043b1 100644 --- a/base/multimedia.jl +++ b/base/multimedia.jl @@ -2,7 +2,7 @@ module Multimedia -import .Base: show, print, convert, repr +import .Base: show, print, repr export AbstractDisplay, display, pushdisplay, popdisplay, displayable, redisplay, MIME, @MIME_str, istextmime, @@ -16,18 +16,49 @@ export AbstractDisplay, display, pushdisplay, popdisplay, displayable, redisplay """ MIME -A type representing a standard internet data format. "MIME" stands for -"Multipurpose Internet Mail Extensions", since the standard was originally -used to describe multimedia attachments to email messages. +A parametric type representing a content type or format. +It is most commonly used when defining [`show`](@ref) methods to specify how to output objects in different formats. +It also allows [`display`](@ref) to automatically detect which formats are available for a given object in a particular display environment. -A `MIME` object can be passed as the second argument to [`show`](@ref) to -request output in that format. +"MIME" stands for "Multipurpose Internet Mail Extensions", since the standard was originally +used to describe multimedia attachments to email messages. +The [current standard](https://www.iana.org/assignments/media-types/media-types.xhtml), +maintained by the Internet Assigned Numbers Authority, +now refers to "MIME types" as "media types". +Each media type is defined as a string in the form `"/"`. +There are over one thousand official media types, along with innumerable unofficial application-specific +types (though in practice only a few are usually supported in any given `display` environment). +Examples of common media types include `"text/plain"`, `"text/html"`, `"image/jpeg"`, `"video/mpeg"`. + +A specific singleton MIME **type** is constructed by passing the MIME string as a symbol, +e.g. `MIME{Symbol("text/plain")}`. +The string macro [`@MIME_str`](@ref) allows `MIME{Symbol("...")}` types to be specified more succinctly +as `MIME"..."`, for example `MIME"text/plain"`. +Singleton MIME types can be used to add new methods to the [`show`](@ref) function. + +A `MIME` **instance** is created by calling the MIME constructor, either directly, +e.g. `MIME("text/plain")`, or after the string macro, e.g. `MIME"text/plain"()`. +A `mime::MIME` instance can be passed as the second argument to [`show(io, mime, x)`](@ref) +to request output in that format (if it is implemented for `x`), as well as to various other functions like +[`showable`](@ref), [`repr`](@ref), and [`display`](@ref); most such functions also allow +passing a string (e.g. `"text/plain"`) that will be converted to a `MIME` instance automatically. # Examples ```jldoctest julia> show(stdout, MIME("text/plain"), "hi") "hi" ``` + +```jldoctest +julia> struct MyType + val + end + +julia> Base.show(io::IO, ::MIME"text/plain", x::MyType) = print(io, "My value is ", x.val); + +julia> show(stdout, MIME"text/plain"(), MyType(5)) +My value is 5 +``` """ struct MIME{mime} end @@ -372,7 +403,7 @@ function displayable(m::MIME) end ########################################################################### -# The redisplay method can be overridden by a AbstractDisplay in order to +# The redisplay method can be overridden by an AbstractDisplay in order to # update an existing display (instead of, for example, opening a new # window), and is used by the IJulia interface to defer display # until the next interactive prompt. This is especially useful diff --git a/base/multinverses.jl b/base/multinverses.jl index 70033de12fcd8..1dc98210d882a 100644 --- a/base/multinverses.jl +++ b/base/multinverses.jl @@ -2,8 +2,7 @@ module MultiplicativeInverses -import Base: div, divrem, rem, unsigned -using Base: IndexLinear, IndexCartesian, tail +import Base: div, divrem, mul_hi, rem, unsigned, mod export multiplicativeinverse unsigned(::Type{Bool}) = UInt @@ -68,21 +67,21 @@ struct SignedMultiplicativeInverse{T<:Signed} <: MultiplicativeInverse{T} q1 = q1<<1 r1 = r1<<1 if r1 >= anc # must be unsigned comparison - q1 += one(UT) + q1 = q1 +% one(UT) r1 -= anc end # update q2, r2 = divrem(2^p, abs(d)) q2 = q2<<1 r2 = r2<<1 if r2 >= ad - q2 += one(UT) + q2 = q2 +% one(UT) r2 -= ad end delta = ad - r2 (q1 < delta || (q1 == delta && r1 == 0)) || break end - m = flipsign((q2 + one(UT)) % T, d) # resulting magic number + m = flipsign((q2 +% one(UT)) % T, d) # resulting magic number s = p - sizeof(d)*8 # resulting shift new(d, m, d > 0 && m < 0 ? Int8(1) : d < 0 && m > 0 ? Int8(-1) : Int8(0), UInt8(s)) end @@ -100,7 +99,7 @@ struct UnsignedMultiplicativeInverse{T<:Unsigned} <: MultiplicativeInverse{T} add = false signedmin = one(d) << (sizeof(d)*8-1) signedmax = signedmin - one(T) - allones = (zero(d) - 1) % T + allones = (zero(d) -% one(T)) % T nc = allones - rem(convert(T, allones - d), d) p = 8*sizeof(d) - 1 @@ -109,59 +108,39 @@ struct UnsignedMultiplicativeInverse{T<:Unsigned} <: MultiplicativeInverse{T} while true p += 1 if r1 >= convert(T, nc - r1) - q1 = q1 + q1 + one(T) - r1 = r1 + r1 - nc + q1 = q1 +% q1 +% one(T) + r1 = r1 +% r1 -% nc else - q1 = q1 + q1 - r1 = r1 + r1 + q1 = q1 +% q1 + r1 = r1 +% r1 end - if convert(T, r2 + one(T)) >= convert(T, d - r2) + if convert(T, r2 +% one(T)) >= convert(T, d -% r2) add |= q2 >= signedmax - q2 = q2 + q2 + one(T) - r2 = r2 + r2 + one(T) - d + q2 = q2 +% q2 +% one(T) + r2 = r2 +% r2 +% one(T) -% d else add |= q2 >= signedmin - q2 = q2 + q2 - r2 = r2 + r2 + one(T) + q2 = q2 +% q2 + r2 = r2 +% r2 +% one(T) end - delta = d - one(T) - r2 + delta = d -% one(T) -% r2 (p < sizeof(d)*16 && (q1 < delta || (q1 == delta && r1 == 0))) || break end - m = q2 + one(T) # resulting magic number + m = q2 +% one(T) # resulting magic number s = p - sizeof(d)*8 - add # resulting shift new(d, m, add, s % UInt8) end end UnsignedMultiplicativeInverse(x::Unsigned) = UnsignedMultiplicativeInverse{typeof(x)}(x) -# Returns the higher half of the product a*b -function _mul_high(a::T, b::T) where {T<:Union{Signed, Unsigned}} - ((widen(a)*b) >>> (sizeof(a)*8)) % T -end - -function _mul_high(a::UInt128, b::UInt128) - shift = sizeof(a)*4 - mask = typemax(UInt128) >> shift - a1, a2 = a >>> shift, a & mask - b1, b2 = b >>> shift, b & mask - a1b1, a1b2, a2b1, a2b2 = a1*b1, a1*b2, a2*b1, a2*b2 - carry = ((a1b2 & mask) + (a2b1 & mask) + (a2b2 >>> shift)) >>> shift - a1b1 + (a1b2 >>> shift) + (a2b1 >>> shift) + carry -end -function _mul_high(a::Int128, b::Int128) - shift = sizeof(a)*8 - 1 - t1, t2 = (a >> shift) & b % UInt128, (b >> shift) & a % UInt128 - (_mul_high(a % UInt128, b % UInt128) - t1 - t2) % Int128 -end - function div(a::T, b::SignedMultiplicativeInverse{T}) where T - x = _mul_high(a, b.multiplier) - x += (a*b.addmul) % T + x = mul_hi(a, b.multiplier) + x = x +% ((a *% b.addmul) % T) ifelse(abs(b.divisor) == 1, a*b.divisor, (signbit(x) + (x >> b.shift)) % T) end function div(a::T, b::UnsignedMultiplicativeInverse{T}) where T - x = _mul_high(a, b.multiplier) - x = ifelse(b.add, convert(T, convert(T, (convert(T, a - x) >>> 1)) + x), x) + x = mul_hi(a, b.multiplier) + x = ifelse(b.add, convert(T, convert(T, (convert(T, a -% x) >>> 1)) +% x), x) ifelse(b.divisor == 1, a, x >>> b.shift) end @@ -173,6 +152,13 @@ function divrem(a::T, b::MultiplicativeInverse{T}) where T (d, a - d*b.divisor) end +mod(a::T, b::UnsignedMultiplicativeInverse{T}) where {T} = rem(a, b) + +function mod(a::T, b::SignedMultiplicativeInverse{T}) where {T} + r = rem(a, b) + return (iszero(r) || signbit(r) == signbit(b.divisor)) ? r : r + b.divisor +end + multiplicativeinverse(x::Signed) = SignedMultiplicativeInverse(x) multiplicativeinverse(x::Unsigned) = UnsignedMultiplicativeInverse(x) diff --git a/base/namedtuple.jl b/base/namedtuple.jl index f71b13852b953..77c6415ee46c7 100644 --- a/base/namedtuple.jl +++ b/base/namedtuple.jl @@ -159,7 +159,7 @@ end length(t::NamedTuple) = nfields(t) iterate(t::NamedTuple, iter=1) = iter > nfields(t) ? nothing : (getfield(t, iter), iter + 1) rest(t::NamedTuple) = t -@inline rest(t::NamedTuple{names}, i::Int) where {names} = NamedTuple{rest(names,i)}(t) +@inline rest(t::NamedTuple{names}, i) where {names} = NamedTuple{rest(names,i::Int)}(t) firstindex(t::NamedTuple) = 1 lastindex(t::NamedTuple) = nfields(t) getindex(t::NamedTuple, i::Int) = getfield(t, i) @@ -185,7 +185,7 @@ function convert(::Type{NamedTuple{names,T}}, nt::NamedTuple{names}) where {name end function convert(::Type{NT}, nt::NamedTuple{names}) where {names, NT<:NamedTuple{names}} - # converting abstract NT to an abstract Tuple type, to a concrete NT1, is not straightforward, so this could just be an error, but we define it anyways + # converting abstract NT to an abstract Tuple type, to a concrete NT1, is not straightforward, so this could just be an error, but we define it anyway # _tuple_error(NT, nt) T1 = Tuple{ntuple(i -> fieldtype(NT, i), Val(length(names)))...} NT1 = NamedTuple{names, T1} @@ -256,7 +256,7 @@ same_names(::NamedTuple{names}...) where {names} = true same_names(::NamedTuple...) = false # NOTE: this method signature makes sure we don't define map(f) -function map(f, nt::NamedTuple{names}, nts::NamedTuple...) where names +function map(f::F, nt::NamedTuple{names}, nts::NamedTuple...) where {names, F} if !same_names(nt, nts...) throw(ArgumentError("Named tuple names do not match.")) end @@ -343,7 +343,7 @@ merge(a::NamedTuple, b::NamedTuple{()}) = a merge(a::NamedTuple{()}, b::NamedTuple{()}) = a merge(a::NamedTuple{()}, b::NamedTuple) = b -merge(a::NamedTuple, b::Iterators.Pairs{<:Any,<:Any,<:Any,<:NamedTuple}) = merge(a, getfield(b, :data)) +merge(a::NamedTuple, b::Iterators.Pairs{<:Any,<:Any,Nothing,<:NamedTuple}) = merge(a, getfield(b, :data)) merge(a::NamedTuple, b::Iterators.Zip{<:Tuple{Any,Any}}) = merge(a, NamedTuple{Tuple(b.is[1])}(b.is[2])) @@ -535,7 +535,7 @@ when it is printed in the stack trace view. ```julia julia> @Kwargs{init::Int} # the internal representation of keyword arguments -Base.Pairs{Symbol, Int64, Tuple{Symbol}, @NamedTuple{init::Int64}} +Base.Pairs{Symbol, Int64, Nothing, @NamedTuple{init::Int64}} julia> sum("julia"; init=1) ERROR: MethodError: no method matching +(::Char, ::Char) @@ -578,7 +578,7 @@ Stacktrace: macro Kwargs(ex) return :(let NT = @NamedTuple $ex - Base.Pairs{keytype(NT),eltype(NT),typeof(NT.parameters[1]),NT} + Base.Pairs{keytype(NT),eltype(NT),Nothing,NT} end) end diff --git a/base/ntuple.jl b/base/ntuple.jl index 185c42601280f..fa0a3821d7047 100644 --- a/base/ntuple.jl +++ b/base/ntuple.jl @@ -45,12 +45,6 @@ function ntupleany(f, n) (Any[f(i) for i = 1:n]...,) end -# inferable ntuple (enough for bootstrapping) -ntuple(f, ::Val{0}) = () -ntuple(f, ::Val{1}) = (@inline; (f(1),)) -ntuple(f, ::Val{2}) = (@inline; (f(1), f(2))) -ntuple(f, ::Val{3}) = (@inline; (f(1), f(2), f(3))) - """ ntuple(f, ::Val{N}) @@ -77,7 +71,6 @@ julia> ntuple(i -> 2*i, Val(4)) Tuple(f(i) for i = 1:(N::Int)) end end -typeof(function ntuple end).name.max_methods = UInt8(5) @inline function fill_to_length(t::Tuple, val, ::Val{_N}) where {_N} M = length(t) @@ -97,5 +90,5 @@ end function reverse(t::NTuple{N}) where N ntuple(Val{N}()) do i t[end+1-i] - end + end::typeof(t) end diff --git a/base/number.jl b/base/number.jl index 8314c546147c7..2d69d33efc50d 100644 --- a/base/number.jl +++ b/base/number.jl @@ -11,6 +11,8 @@ convert(::Type{T}, x::Number) where {T<:Number} = T(x)::T Test whether `x` is numerically equal to some integer. +See also [`iszero`](@ref), [`isone`](@ref), [`isfinite`](@ref), [`isnan`](@ref). + # Examples ```jldoctest julia> isinteger(4.0) @@ -25,7 +27,7 @@ isinteger(x::Integer) = true Return `true` if `x == zero(x)`; if `x` is an array, this checks whether all of the elements of `x` are zero. -See also: [`isone`](@ref), [`isinteger`](@ref), [`isfinite`](@ref), [`isnan`](@ref). +See also [`isone`](@ref), [`isinteger`](@ref), [`isfinite`](@ref), [`isnan`](@ref). # Examples ```jldoctest @@ -47,6 +49,8 @@ iszero(x) = x == zero(x) # fallback method Return `true` if `x == one(x)`; if `x` is an array, this checks whether `x` is an identity matrix. +See also [`iszero`](@ref), [`isinteger`](@ref), [`isfinite`](@ref), [`isnan`](@ref). + # Examples ```jldoctest julia> isone(1.0) @@ -66,6 +70,8 @@ isone(x) = x == one(x) # fallback method Test whether a number is finite. +See also [`iszero`](@ref), [`isone`](@ref), [`isinteger`](@ref), [`isnan`](@ref). + # Examples ```jldoctest julia> isfinite(5) @@ -117,7 +123,7 @@ copy(x::Number) = x # some code treats numbers as collection-like Return `true` if the value of the sign of `x` is negative, otherwise `false`. -See also [`sign`](@ref) and [`copysign`](@ref). +See also [`sign`](@ref), [`copysign`](@ref). # Examples ```jldoctest diff --git a/base/opaque_closure.jl b/base/opaque_closure.jl index 5e38c8523f4a8..9e283227c963b 100644 --- a/base/opaque_closure.jl +++ b/base/opaque_closure.jl @@ -24,7 +24,7 @@ end macro opaque(ty, ex) if Base.isexpr(ty, :->) (AT, body) = ty.args - filter!((n)->!isa(n, Core.LineNumberNode), body.args) + filter!((n)->!isa(n, LineNumberNode), body.args) if !Base.isexpr(body, :block) || length(body.args) != 1 error("Opaque closure type must be specified in the form Tuple{T,U...}->RT") end diff --git a/base/operators.jl b/base/operators.jl index 51729b852070d..d78882ea69588 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -83,8 +83,23 @@ julia> supertype(Vector) DenseVector (alias for DenseArray{T, 1} where T) ``` """ -supertype(T::DataType) = (@_total_meta; T.super) -supertype(T::UnionAll) = (@_total_meta; UnionAll(T.var, supertype(T.body))) +function supertype(T::DataType) + @_foldable_meta + # force the computation of a deferred supertype (an instantiation of a + # self-referential definition materializes its supertype graph lazily, + # one level per demand, see issue #61347); idempotent, so still foldable. + # The subsequent plain field read keeps `getfield`'s inference precision. + ccall(:jl_datatype_compute_super, Ptr{Cvoid}, (Any,), T) + return getfield(T, :super) +end +function getproperty(T::DataType, s::Symbol) + @inline + # fill the deferred supertype cache on access, so raw `.super` reads keep + # working for instantiations of self-referential definitions (#61347) + s === :super && return supertype(T) + return getfield(T, s) +end +supertype(T::UnionAll) = (@_foldable_meta; UnionAll(T.var, supertype(T.body))) ## generic comparison ## @@ -288,12 +303,11 @@ orderings such as [`isless`](@ref). !!! compat "Julia 1.7" This function requires Julia 1.7 or later. """ -isunordered(x) = false +isunordered(_) = false isunordered(x::AbstractFloat) = isnan(x) -isunordered(x::Missing) = true +isunordered(::Missing) = true ==(T::Type, S::Type) = (@_total_meta; ccall(:jl_types_equal, Cint, (Any, Any), T, S) != 0) -!=(T::Type, S::Type) = (@_total_meta; !(T == S)) ==(T::TypeVar, S::Type) = false ==(T::Type, S::TypeVar) = false @@ -319,6 +333,7 @@ false ``` """ !=(x, y) = !(x == y) +typeof(!=).name.max_methods = UInt8(1) const ≠ = != """ @@ -423,6 +438,7 @@ true ``` """ >(x, y) = y < x +typeof(>).name.max_methods = UInt8(1) """ <=(x, y) @@ -454,6 +470,14 @@ const ≤ = <= Greater-than-or-equals comparison operator. Falls back to `y <= x`. +# Implementation + +New types should prefer to implement [`<=`](@ref) instead of this function, +and rely on the fallback definition `>=(x, y) = y <= x`. + +Furthermore, in many cases it is enough to implement just [`<`](@ref) and +[`==`](@ref), relying on the fallback definitions of both `<=` and `>=`. + # Examples ```jldoctest julia> 'a' >= 'b' @@ -470,6 +494,7 @@ true ``` """ >=(x, y) = (y <= x) +typeof(>=).name.max_methods = UInt8(1) const ≥ = >= # this definition allows Number types to implement < instead of isless, @@ -566,7 +591,7 @@ minmax(x,y) = isless(y, x) ? (y, x) : (x, y) The identity function. Returns its argument. -See also: [`one`](@ref), [`oneunit`](@ref), and [`LinearAlgebra`](@ref man-linalg)'s `I`. +See also [`one`](@ref), [`oneunit`](@ref), [`LinearAlgebra.I`](@ref). # Examples ```jldoctest @@ -635,7 +660,7 @@ function afoldl(op, a, bs...) end setfield!(typeof(afoldl).name, :max_args, Int32(34), :monotonic) -for op in (:+, :*, :&, :|, :xor, :min, :max, :kron) +for op in (:+, :(+%), :*, :(*%), :&, :|, :xor, :min, :max, :kron) @eval begin # note: these definitions must not cause a dispatch loop when +(a,b) is # not defined, and must only try to call 2-argument definitions, so @@ -809,7 +834,7 @@ end Remainder from Euclidean division, returning a value of the same sign as `x`, and smaller in magnitude than `y`. This value is always exact. -See also: [`div`](@ref), [`mod`](@ref), [`mod1`](@ref), [`divrem`](@ref). +See also [`div`](@ref), [`mod`](@ref), [`mod1`](@ref), [`divrem`](@ref). # Examples ```jldoctest @@ -836,7 +861,7 @@ const % = rem The quotient from Euclidean (integer) division. Generally equivalent to a mathematical operation x/y without a fractional part. -See also: [`cld`](@ref), [`fld`](@ref), [`rem`](@ref), [`divrem`](@ref). +See also [`cld`](@ref), [`fld`](@ref), [`rem`](@ref), [`divrem`](@ref). # Examples ```jldoctest @@ -860,68 +885,68 @@ const ÷ = div """ mod1(x, y) -Modulus after flooring division, returning a value `r` such that `mod(r, y) == mod(x, y)` -in the range ``(0, y]`` for positive `y` and in the range ``[y,0)`` for negative `y`. +Equivalent to `rem(x, y, RoundUp) + x*sign(y)`. Returns a value in the range +``(0, y]`` for positive `y` and ``[-|y|,0)`` for negative `y`. -With integer arguments and positive `y`, this is equal to `mod(x, 1:y)`, and hence natural -for 1-based indexing. By comparison, `mod(x, y) == mod(x, 0:y-1)` is natural for computations with -offsets or strides. +With integer arguments and positive `y`, this is equal to `mod(x, 1:y)`, and hence natural for +1-based indexing. By comparison, `mod(x, y) == mod(x, 0:y-1)` is natural for 0-based indexing. -See also [`mod`](@ref), [`fld1`](@ref), [`fldmod1`](@ref). +See also [`rem`](@ref), [`mod`](@ref), [`cld`](@ref), [`cldmod1`](@ref). # Examples ```jldoctest julia> mod1(4, 2) 2 -julia> mod1.(-5:5, 3)' -1×11 adjoint(::Vector{Int64}) with eltype Int64: - 1 2 3 1 2 3 1 2 3 1 2 +julia> [-7:7 mod1.(-7:7, 3)]' +2×15 adjoint(::Matrix{Int64}) with eltype Int64: + -7 -6 -5 -4 -3 -2 -1 0 1 2 3 4 5 6 7 + 2 3 1 2 3 1 2 3 1 2 3 1 2 3 1 -julia> mod1.([-0.1, 0, 0.1, 1, 2, 2.9, 3, 3.1]', 3) +julia> mod1.([-0.1 0 0.1 1 2 2.9 3 3.1], 3) 1×8 Matrix{Float64}: 2.9 3.0 0.1 1.0 2.0 2.9 3.0 0.1 ``` """ -mod1(x::T, y::T) where {T<:Real} = (m = mod(x, y); ifelse(m == 0, y, m)) +mod1(x::T, y::T) where {T<:Real} = (m = mod(x, y); iszero(m) ? y : m) """ - fld1(x, y) + cldmod1(x, y) -Flooring division, returning a value consistent with `mod1(x,y)` +Return `(cld(x,y), mod1(x,y))`. For positive integer inputs, this is the (col, row) index +of the xᵗʰ element in a column major matrix with y rows. -See also [`mod1`](@ref), [`fldmod1`](@ref). +See also [`cld`](@ref), [`mod1`](@ref), [`divrem`](@ref), [`fldmod`](@ref). # Examples ```jldoctest -julia> x = 15; y = 4; +julia> col, row = cldmod1(20, 6) +(4, 2) -julia> fld1(x, y) -4 - -julia> x == fld(x, y) * y + mod(x, y) +julia> 20 == (col - 1) * 6 + row true -julia> x == (fld1(x, y) - 1) * y + mod1(x, y) -true +julia> reshape(1:36, 6, 6) +6×6 reshape(::UnitRange{Int64}, 6, 6) with eltype Int64: + 1 7 13 19 25 31 + 2 8 14 20 26 32 + 3 9 15 21 27 33 + 4 10 16 22 28 34 + 5 11 17 23 29 35 + 6 12 18 24 30 36 ``` """ -fld1(x::T, y::T) where {T<:Real} = (m = mod1(x, y); fld((x - m) + y, y)) -function fld1(x::T, y::T) where T<:Integer - d = div(x, y) - return d + (!signbit(x ⊻ y) & (d * y != x)) -end +cldmod1(x, y) = (cld(x, y), mod1(x, y)) """ fldmod1(x, y) -Return `(fld1(x,y), mod1(x,y))`. +Legacy spelling of `cldmod1(x, y)` for integers. -See also [`fld1`](@ref), [`mod1`](@ref). +See also [`cldmod1`](@ref). """ -fldmod1(x, y) = (fld1(x, y), mod1(x, y)) - +fldmod1(x, y) = cldmod1(x, y) """ widen(x) @@ -973,7 +998,7 @@ julia> [0 1; 2 3] .|> (x -> x^2) |> sum |>(x, f) = f(x) _stable_typeof(x) = typeof(x) -_stable_typeof(::Type{T}) where {T} = @isdefined(T) ? Type{T} : DataType +_stable_typeof(::Type{T}) where {T} = @isdefined(T) && !Core.has_free_typevars(T) ? Type{T} : DataType """ f = Returns(value) @@ -1017,7 +1042,7 @@ entered in the Julia REPL (and most editors, appropriately configured) by typing Function composition also works in prefix form: `∘(f, g)` is the same as `f ∘ g`. The prefix form supports composition of multiple functions: `∘(f, g, h) = f ∘ g ∘ h` and splatting `∘(fs...)` for composing an iterable collection of functions. -The last argument to `∘` execute first. +The last argument to `∘` executes first. !!! compat "Julia 1.4" Multiple function composition requires at least Julia 1.4. @@ -1187,13 +1212,27 @@ end function (f::Fix{N})(args::Vararg{Any,M}; kws...) where {N,M} M < N-1 && throw(ArgumentError(LazyString("expected at least ", N-1, " arguments to `Fix{", N, "}`, but got ", M))) - return f.f(args[begin:begin+(N-2)]..., f.x, args[begin+(N-1):end]...; kws...) + (left, right) = _split_tuple(args, N-1) + return f.f(left..., f.x, right...; kws...) end # Special cases for improved constant propagation (f::Fix{1})(arg; kws...) = f.f(f.x, arg; kws...) (f::Fix{2})(arg; kws...) = f.f(arg, f.x; kws...) +function Base.show(io::IO, fix::Fix{N}) where {N} + constr = Fix{N} + callable = fix.f + fixed_argument = fix.x + show(io, constr) + print(io, '(') + show(io, callable) + print(io, ',') + print(io, ' ') + show(io, fixed_argument) + print(io, ')') +end + """ Alias for `Fix{1}`. See [`Fix`](@ref Base.Fix). """ @@ -1342,6 +1381,57 @@ end (s::Splat)(args) = s.f(args...) show(io::IO, s::Splat) = (print(io, "splat("); show(io, s.f); print(io, ")")) +""" + unsplat(f) + +Given a function `f` that takes a single tuple argument, return a new function +that takes any number of arguments and bundles them into a tuple to pass to the +original function. + +That is, the return value is *effectively* equivalent to `(args...) -> f(args)`, except that +it may use a more specialized function type (such as `f ∘ tuple` via [`∘`](@ref) and [`tuple`](@ref)) +for improved clarity or efficiency, rather than creating a new anonymous function. + +`unsplat` is the inverse of [`splat`](@ref): `unsplat(splat(f)) === f` and `splat(unsplat(f)) === f`. +This operation is also sometimes referred to as "slurp". + +# Examples +```jldoctest +julia> unsplat(sum)(1, 2, 3) +6 + +julia> unsplat(splat(+)) === + +true +``` + +!!! compat "Julia 1.14" + `unsplat` requires at least Julia 1.14. +""" +unsplat(f) = f ∘ tuple +unsplat(s::Splat) = s.f +splat(f::ComposedFunction{<:Any,typeof(tuple)}) = f.outer + +""" + tap(f) + +Create a function that calls `f(x)` and returns `x`. + +# Examples +```jldoctest +julia> 2 |> sqrt |> tap(println) |> inv +1.4142135623730951 +0.7071067811865475 + +julia> "hello" |> uppercase |> tap(Base.Fix1(println, stderr)) |> length +HELLO +5 +``` + +!!! compat "Julia 1.14" + `tap` requires at least Julia 1.14. +""" +tap(f) = x -> (f(x); x) + ## in and related operators """ @@ -1357,38 +1447,22 @@ used to implement specialized methods. """ in(x) = Fix2(in, x) -function in(x, itr::Any) - anymissing = false - for y in itr - v = (y == x) - if ismissing(v) - anymissing = true - elseif v - return true - end - end - return anymissing ? missing : false -end +in(x, itr::Any) = any(==(x), itr) # Specialized variant of in for Tuple, which can generate typed comparisons for each element # of the tuple, skipping values that are statically known to be != at compile time. -in(x, itr::Tuple) = _in_tuple(x, itr, false) +in(x, itr::Tuple) = _in_tuple(x, itr) + # This recursive function will be unrolled at compiletime, and will not generate separate # llvm-compiled specializations for each step of the recursion. -function _in_tuple(x, @nospecialize(itr::Tuple), anymissing::Bool) +function _in_tuple(x, @nospecialize(itr::Tuple), result = false) @inline - # Base case - if isempty(itr) - return anymissing ? missing : false - end - # Recursive case + isempty(itr) && return result v = (itr[1] == x) - if ismissing(v) - anymissing = true - elseif v + if v === true return true end - return _in_tuple(x, tail(itr), anymissing) + return _in_tuple(x, tail(itr), result | v) end # fallback to the loop implementation after some number of arguments to avoid inference blowup @@ -1436,13 +1510,14 @@ contains `missing` but not `item`, in which case `missing` is returned ([three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic), matching the behavior of [`any`](@ref) and [`==`](@ref)). Some collections follow a slightly different definition. For example, -[`Set`](@ref)s check whether the item [`isequal`](@ref) to one of the elements; -[`Dict`](@ref)s look for `key=>value` pairs, and the `key` is compared using -[`isequal`](@ref). +[`Set`](@ref)s check whether the item [`isequal`](@ref) to one of the elements. +For [`Dict`](@ref), [`ImmutableDict`](@ref), and [`WeakKeyDict`](@ref), +`key=>value` membership compares keys using [`isequal`](@ref) and values using +[`==`](@ref); [`IdDict`](@ref) instead compares keys using [`===`](@ref). To test for the presence of a key in a dictionary, use [`haskey`](@ref) -or `k in keys(dict)`. For the collections mentioned above, -the result is always a `Bool`. +or `k in keys(dict)`. For the dictionaries mentioned above, +the result of `haskey(dict, k)` or `k in keys(dict)` is always a `Bool`. When broadcasting with `in.(items, collection)` or `items .∈ collection`, both `items` and `collection` are broadcasted over, which is often not what is intended. @@ -1452,7 +1527,7 @@ corresponding position in `collection`. To get a vector indicating whether each in `items` is in `collection`, wrap `collection` in a tuple or a `Ref` like this: `in.(items, Ref(collection))` or `items .∈ Ref(collection)`. -See also: [`∉`](@ref), [`insorted`](@ref), [`contains`](@ref), [`occursin`](@ref), [`issubset`](@ref). +See also [`∉`](@ref), [`insorted`](@ref), [`contains`](@ref), [`occursin`](@ref), [`issubset`](@ref). # Examples ```jldoctest diff --git a/base/optimized_generics.jl b/base/optimized_generics.jl index 6b1d146b6172b..3e61a795ca97e 100644 --- a/base/optimized_generics.jl +++ b/base/optimized_generics.jl @@ -23,7 +23,7 @@ Implements a key-value like interface where the compiler has liberty to perform the following transformations. The core optimization semantically allowed for the compiler is: - get(set(x, key, val), key) -> (val,) + get(set(x, key, val), key) -> Some(val) where the compiler will recursively look through `x`. Keys are compared by egality. @@ -47,8 +47,8 @@ module KeyValue """ get(collection, key) - Retrieve the value corresponding to `key` in `collection` as a single - element tuple or `nothing` if no value corresponding to the key was found. + Retrieve the value corresponding to `key` in `collection` wrapped in + `Some` or return `nothing` if no value corresponding to the key was found. `key`s are compared by egal. """ function get end @@ -70,8 +70,7 @@ module CompilerPlugins """ typeinf(owner, mi, source_mode)::CodeInstance - Return a `CodeInstance` for the given `mi` whose valid results include at - the least current tls world and satisfies the requirements of `source_mode`. + Return a `CodeInstance` for the given `mi` whose valid results include at least the current tls world and satisfies the requirements of `source_mode`. """ function typeinf end diff --git a/base/options.jl b/base/options.jl index 203a91ca8f641..480b5bd1617af 100644 --- a/base/options.jl +++ b/base/options.jl @@ -24,6 +24,7 @@ struct JLOptions startupfile::Int8 compile_enabled::Int8 code_coverage::Int8 + code_coverage_mode::Int8 malloc_log::Int8 tracked_path::Ptr{UInt8} opt_level::Int8 @@ -64,12 +65,18 @@ struct JLOptions heap_target_increment::UInt64 trace_compile_timing::Int8 trim::Int8 + trace_eval::Int8 task_metrics::Int8 timeout_for_safepoint_straggler_s::Int16 gc_sweep_always_full::Int8 + compress_sysimage::Int8 + alert_on_critical_error::Int8 + target_sanitize_memory::Int8 + target_sanitize_thread::Int8 + target_sanitize_address::Int8 end -# This runs early in the sysimage != is not defined yet +# This runs early in the sysimage when `!=` is not defined yet if sizeof(JLOptions) === ccall(:jl_sizeof_jl_options, Int, ()) else ccall(:jl_throw, Cvoid, (Any,), "Option structure mismatch") @@ -77,6 +84,12 @@ end JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions)) +# NOTE: Keep in sync with the JL_OPTIONS_COMPILE_* defines in src/julia.h +const JL_OPTIONS_COMPILE_OFF = 0 +const JL_OPTIONS_COMPILE_ON = 1 +const JL_OPTIONS_COMPILE_ALL = 2 +const JL_OPTIONS_COMPILE_MIN = 3 + function colored_text(opts::JLOptions) return if opts.color != 0 opts.color == 1 diff --git a/base/ordering.jl b/base/ordering.jl index f2ddd20ab09f0..485e3857f5b23 100644 --- a/base/ordering.jl +++ b/base/ordering.jl @@ -6,8 +6,8 @@ module Order import Base: @__MODULE__, parentmodule const Base = parentmodule(@__MODULE__) import .Base: - AbstractVector, @propagate_inbounds, isless, identity, getindex, reverse, - +, -, !, &, <, | + AbstractVector, @propagate_inbounds, isless, identity, reverse, + !, &, <, | ## notions of element ordering ## @@ -50,7 +50,7 @@ ReverseOrdering() = ReverseOrdering(ForwardOrdering()) """ reverse(o::Base.Ordering) -reverses ordering specified by `o`. +Reverses ordering specified by `o`. """ reverse(o::Ordering) = ReverseOrdering(o) diff --git a/base/osinfo.jl b/base/osinfo.jl new file mode 100644 index 0000000000000..f04aecb15ef72 --- /dev/null +++ b/base/osinfo.jl @@ -0,0 +1,172 @@ + +# NB: This file is `Core.eval`-uated into the (pre-existing) module Sys + +export KERNEL, + isapple, + isbsd, + isdragonfly, + isfreebsd, + islinux, + isnetbsd, + isopenbsd, + isunix, + iswindows, + isjsvm, + detectwsl + +""" + Sys.KERNEL::Symbol + +A symbol representing the name of the operating system, as returned by `uname` of the build configuration. +""" +const KERNEL = ccall(:jl_get_UNAME, Any, ())::Symbol +""" + Sys.isunix([os]) + +Predicate for testing if the OS provides a Unix-like interface. +See documentation in [Handling Operating System Variation](@ref). +""" +function isunix(os::Symbol) + if iswindows(os) + return false + elseif islinux(os) || isbsd(os) + return true + elseif os === :Emscripten + # Emscripten implements the POSIX ABI and provides traditional + # Unix-style operating system functions such as file system support. + # Therefore, we consider it a unix, even though this need not be + # generally true for a jsvm embedding. + return true + else + throw(ArgumentError("unknown operating system \"$os\"")) + end +end + +""" + Sys.islinux([os]) + +Predicate for testing if the OS is a derivative of Linux. +See documentation in [Handling Operating System Variation](@ref). +""" +islinux(os::Symbol) = (os === :Linux) + +""" + Sys.isbsd([os]) + +Predicate for testing if the OS is a derivative of BSD. +See documentation in [Handling Operating System Variation](@ref). + +!!! note + The Darwin kernel descends from BSD, which means that `Sys.isbsd()` is + `true` on macOS systems. To exclude macOS from a predicate, use + `Sys.isbsd() && !Sys.isapple()`. +""" +isbsd(os::Symbol) = (isfreebsd(os) || isopenbsd(os) || isnetbsd(os) || isdragonfly(os) || isapple(os)) + +""" + Sys.isfreebsd([os]) + +Predicate for testing if the OS is a derivative of FreeBSD. +See documentation in [Handling Operating System Variation](@ref). + +!!! note + Not to be confused with `Sys.isbsd()`, which is `true` on FreeBSD but also on + other BSD-based systems. `Sys.isfreebsd()` refers only to FreeBSD. +!!! compat "Julia 1.1" + This function requires at least Julia 1.1. +""" +isfreebsd(os::Symbol) = (os === :FreeBSD) + +""" + Sys.isopenbsd([os]) + +Predicate for testing if the OS is a derivative of OpenBSD. +See documentation in [Handling Operating System Variation](@ref). + +!!! note + Not to be confused with `Sys.isbsd()`, which is `true` on OpenBSD but also on + other BSD-based systems. `Sys.isopenbsd()` refers only to OpenBSD. +!!! compat "Julia 1.1" + This function requires at least Julia 1.1. +""" +isopenbsd(os::Symbol) = (os === :OpenBSD) + +""" + Sys.isnetbsd([os]) + +Predicate for testing if the OS is a derivative of NetBSD. +See documentation in [Handling Operating System Variation](@ref). + +!!! note + Not to be confused with `Sys.isbsd()`, which is `true` on NetBSD but also on + other BSD-based systems. `Sys.isnetbsd()` refers only to NetBSD. +!!! compat "Julia 1.1" + This function requires at least Julia 1.1. +""" +isnetbsd(os::Symbol) = (os === :NetBSD) + +""" + Sys.isdragonfly([os]) + +Predicate for testing if the OS is a derivative of DragonFly BSD. +See documentation in [Handling Operating System Variation](@ref). + +!!! note + Not to be confused with `Sys.isbsd()`, which is `true` on DragonFly but also on + other BSD-based systems. `Sys.isdragonfly()` refers only to DragonFly. +!!! compat "Julia 1.1" + This function requires at least Julia 1.1. +""" +isdragonfly(os::Symbol) = (os === :DragonFly) + +""" + Sys.iswindows([os]) + +Predicate for testing if the OS is a derivative of Microsoft Windows NT. +See documentation in [Handling Operating System Variation](@ref). +""" +iswindows(os::Symbol) = (os === :Windows || os === :NT) + +""" + Sys.isapple([os]) + +Predicate for testing if the OS is a derivative of Apple Macintosh OS X or Darwin. +See documentation in [Handling Operating System Variation](@ref). +""" +isapple(os::Symbol) = (os === :Apple || os === :Darwin) + +""" + Sys.isjsvm([os]) + +Predicate for testing if Julia is running in a JavaScript VM (JSVM), +including e.g. a WebAssembly JavaScript embedding in a web browser. + +!!! compat "Julia 1.2" + This function requires at least Julia 1.2. +""" +isjsvm(os::Symbol) = (os === :Emscripten) + +""" + Sys.detectwsl() + +Runtime predicate for testing if Julia is running inside +Windows Subsystem for Linux (WSL). + +!!! note + Unlike `Sys.iswindows`, `Sys.islinux` etc., this is a runtime test, and thus + cannot meaningfully be used in `@static if` constructs. + +!!! compat "Julia 1.12" + This function requires at least Julia 1.12. +""" +function detectwsl() + # We use the same approach as canonical/snapd do to detect WSL + islinux() && ( + isfile("/proc/sys/fs/binfmt_misc/WSLInterop") + || isdir("/run/WSL") + ) +end + +for f in (:isunix, :islinux, :isbsd, :isapple, :iswindows, :isfreebsd, :isopenbsd, :isnetbsd, :isdragonfly, :isjsvm) + @eval $f() = $(getfield(@__MODULE__, f)(KERNEL)) +end diff --git a/base/pair.jl b/base/pair.jl index 1953dc2886053..f41b93f4281b3 100644 --- a/base/pair.jl +++ b/base/pair.jl @@ -41,7 +41,10 @@ eltype(p::Type{Pair{A, B}}) where {A, B} = Union{A, B} iterate(p::Pair, i=1) = i > 2 ? nothing : (getfield(p, i), i + 1) indexed_iterate(p::Pair, i::Int, state=1) = (getfield(p, i), i + 1) -hash(p::Pair, h::UInt) = hash(p.second, hash(p.first, h)) +let seed = (UInt === UInt64) ? 0x94cb2bb20a28ce96 : 0x1f60a087 + global hash + hash(p::Pair, h::UInt) = hash(p.second, hash(p.first, xor(seed, h))) +end ==(p::Pair, q::Pair) = (p.first==q.first) & (p.second==q.second) isequal(p::Pair, q::Pair) = isequal(p.first,q.first)::Bool & isequal(p.second,q.second)::Bool diff --git a/base/park.jl b/base/park.jl new file mode 100644 index 0000000000000..8e523a36e9212 --- /dev/null +++ b/base/park.jl @@ -0,0 +1,429 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## The parked-wait driver +# +# Every parked wait proceeds in six phases, uniformly over the set of +# things it waits for: +# +# 1. CHECK the caller's fast-path satisfaction tests - no allocation, +# no publication (stays at the call sites). +# 2. ENTRY allocate the wait entry or reuse the task's cached one; +# stage per-slot aux (severity floors) that the claim side +# may read through an already-linked entry. +# 3. ARM publish the entry as the task's sole claimable +# registration: CAS(t.waiting_on, nothing => w). +# 4. ENQUEUE register the entry with every waitable, each under that +# waitable's own protection discipline. +# 5. RECHECK re-test satisfaction where the enqueue could have raced +# the firing; on a hit, self-claim: won => withdraw and +# return (or deliver the refusal); lost => a waker owns our +# wake - suspend and consume it. +# 6. SUSPEND release the locks that must not be held while parked, +# `wait()`, then reacquire per policy and settle. +# +# A *waitable* is the identity a slot's `owner` records. The waitable +# protocol is ordinary generic functions - the set is open; each call +# site's concrete container type carries the dispatch: +# +# wait_enqueue!(x, w, first) -> Bool phase 4, under x's discipline; +# false = x already fired and +# declined (one-shot waitables) +# wait_recheck(x, w) -> Bool phase 5, after ALL enqueues; +# vacuous default +# wait_dequeue!(x, w, why) -> Nothing withdraw x's slot. Lock +# discipline is per `why`: on +# WAKE_VALUE/WAKE_FIRED the caller +# holds the kind's protection; on +# WAKE_INTERRUPTED/WAKE_WITHDRAWN +# the method takes it itself +# +# The driver takes and releases no locks: lock choreography is plain +# caller code (`unlockall`/`relockall` around `wait_safe_interrupt`, +# `iolock_end`/`iolock_begin` at the uv sites, ...), so there is no +# release/reacquire protocol and no relock policy. +# +# Lock rule: from phase 3 on the entry is the task's armed registration, +# so a lock acquisition *inside* phase 4 must never park (a nested park +# would try to arm a second registration). Parking locks must be +# caller-held before `park!` begins (the `wait(c)` contract); locks a +# `wait_enqueue!` method takes itself must be spin locks. The cleanup's +# reacquire is exempt: it runs disarmed and may itself park with a fresh +# entry (the cache-blank dance below exists for exactly that). + +# why-codes for wait_dequeue! +const WAKE_VALUE = 0x00 # normal wake consumed (lazy settle) +const WAKE_FIRED = 0x01 # fired; self-claim won; no suspend +const WAKE_INTERRUPTED = 0x03 # exceptional wake (cleanup path) +const WAKE_WITHDRAWN = 0x04 # withdraw! - the caller is done waiting + +function wait_enqueue! end +function wait_dequeue! end +wait_recheck(@nospecialize(x), w::WaitEntry) = false + +# Lock acquisition for self-protecting `wait_dequeue!` methods (the +# cleanup/withdraw paths): must not observe cancellation - the cleanup +# may be unwinding the very CancellationRequest a cancellable acquire +# would rethrow. The `ReentrantLock` method (lock.jl) shields itself. +_uncancellable_lock(l) = lock(l) + +## The cancellation source as a waitable +# +# `SourceWait(src, floor)` in a park's waitables makes the wait +# cancellable under `src` at severities >= `floor`: its enqueue is the +# sticky lock-free registration, its recheck is the seq_cst state read +# closing the arm-vs-cancel race (a Dekker - both sides need their +# seq_cst upgrade), and its fired outcome throws the refusal. +# Registrations are sticky: no dequeue on any path - collection is the +# walk's job (prune/dead accounting). +struct SourceWait + src::CancellationTokenSource + floor::UInt8 +end + +## Entry acquisition - the cache contract +# +# The two canonical tuple shapes are served from the task's caches; any +# other iterable gets a fresh, single-use entry. Freshness is what makes +# specific-wait wakers (a timeout's expected-entry CAS) safe: entry +# identity scopes a claim to this wait - the same principle that gives +# shielded parks a distinct entry from cancellable ones. +acquire_wait_entry!(ct::Task, ws::Tuple{Any}) = _cached_wait_entry(ct) +acquire_wait_entry!(ct::Task, ws::Tuple{Any, SourceWait}) = + _cancel_wait_entry(ct, ws[2].src, ws[2].floor) +acquire_wait_entry!(ct::Task, ws) = WaitEntryN(ct, length(ws)) + +# Retire a fresh (uncached) entry that is done waiting: its sticky source +# registrations become prunable corpses. Cached entries stay live. +function release_wait_entry!(ct::Task, w::WaitEntry) + (w === ct.cached_wait_entry || w === ct.cached_cancel_entry) && return nothing + # idempotent: several exits may withdraw the same fresh entry + (@atomic :monotonic w.task) === nothing && return nothing + retire_cancellation_entry!(w) + return nothing +end + +# Self-claim: take back the wake of our own armed registration. +disarm!(ct::Task, w::WaitEntry) = (@atomicreplace ct.waiting_on w => nothing).success + +## The verbs + +""" + park!(ws, w::WaitEntry, first::Bool) -> Bool + +Phases 3-5 of the protocol over the flat waitable iterable `ws`: arm +`w`, enqueue it with every waitable, and run the rechecks. Returns +`true` when the task is parked - a wake is (or will be) in flight, and +the caller must suspend through [`wait_safe_interrupt`](@ref) to consume +it. Returns `false` when a waitable fired and the self-claim won: +nothing is in flight, and the caller owns the outcome (for a fired +source, re-checking cancellation - `checkcancel` - throws the refusal +exactly like the entry check) as well as every remaining registration +(usually `withdraw!(ws, w, WAKE_FIRED)` under its still-held locks). A +fired recheck whose self-claim *loses* returns `true`: the concurrent +claimer's wake delivers the outcome. + +The driver takes and releases no locks: protection the caller holds is +held throughout - which is what makes the enqueue/recheck window sound - +and lock choreography around the suspend is plain caller code. The one +dequeue the driver performs itself is the *fired slot* on the `false` +path (self-protecting; a done-but-still-linked predicate would re-fire +on every `repark!` until its notify drains). +""" +function park!(ws, w::WaitEntry, first::Bool) + ct = current_task() + _arm_wait(ct, w) # 3 + fx = _enqueue_until_fired(ws, w, first) # 4 + fx === nothing && (fx = _recheck_until_fired(ws, w)) # 5 + if fx !== nothing && disarm!(ct, w) + wait_dequeue!(fx, w, WAKE_FIRED) + return false + end + return true +end + +# Per-waitable phases. Unrolled by tuple recursion: a generic `for x in ws` +# loop would re-box every element through its dynamic tuple index. +@inline _enqueue_until_fired(ws::Tuple{}, w::WaitEntry, first::Bool) = nothing +@inline function _enqueue_until_fired(ws::Tuple, w::WaitEntry, first::Bool) + x = ws[1] + wait_enqueue!(x, w, first) || return x + return _enqueue_until_fired(tail(ws), w, first) +end +@inline function _enqueue_until_fired(ws, w::WaitEntry, first::Bool) + for x in ws + wait_enqueue!(x, w, first) || return x + end + return nothing +end +@inline _recheck_until_fired(ws::Tuple{}, w::WaitEntry) = nothing +@inline function _recheck_until_fired(ws::Tuple, w::WaitEntry) + x = ws[1] + wait_recheck(x, w) && return x + return _recheck_until_fired(tail(ws), w) +end +@inline function _recheck_until_fired(ws, w::WaitEntry) + for x in ws + wait_recheck(x, w) && return x + end + return nothing +end +@inline _dequeue_each!(ws::Tuple{}, w::WaitEntry, why::UInt8) = nothing +@inline function _dequeue_each!(ws::Tuple, w::WaitEntry, why::UInt8) + wait_dequeue!(ws[1], w, why) + return _dequeue_each!(tail(ws), w, why) +end +@inline function _dequeue_each!(ws, w::WaitEntry, why::UInt8) + for x in ws + wait_dequeue!(x, w, why) + end + return nothing +end + +""" + repark!(ws, w::WaitEntry) -> Bool + +Re-arm and recheck `w`, with the same `Bool` contract as [`park!`](@ref). +The caller must ensure every pending waitable is still registered and its +recheck observes notifications delivered while `w` was unarmed. `DoneWait` +does not satisfy this contract: notification removes the slot required by +its recheck. Multi-task waits must enqueue pending tasks with [`park!`](@ref). +""" +function repark!(ws, w::WaitEntry) + ct = current_task() + _arm_wait(ct, w) + fx = _recheck_until_fired(ws, w) + if fx !== nothing && disarm!(ct, w) + wait_dequeue!(fx, w, WAKE_FIRED) + return false + end + return true +end + +""" + wait_safe_interrupt(ws, w::WaitEntry) + +Phase 6: suspend and consume exactly one wake of the park `park!` +armed, returning its payload. This is the only legal way to suspend on +an armed park - a raw `wait()` would miss the interrupted-wait cleanup. +The caller must have released any parking locks it holds (plain caller +code, e.g. `unlockall`); on a normal wake it returns with no locks +touched, and the caller reacquires per its own contract. + +On an exceptional resume (an interrupter's claim, a delivered +cancellation, a raw `throwto`) the cleanup runs here, then the exception +propagates: disarm; blank the entry's cache slot; withdraw every +registration through its kind's *self-protecting* dequeue - each takes +its own lock, and those round-trips serialize any claimer's in-flight +schedule - and only then drop a claimed-and-enqueued wake this unwind +will never consume (dropping earlier would race the in-flight claimer +and leak the wake into the task's next park); finally restore or retire +the entry. Because the registrations are already withdrawn when the +exception reaches the caller, its catch owes nothing to the protocol - +it only restores whatever lock contract its own callers require +(reacquiring shielded, and only what that contract demands: an unwind +that is itself a cancellation should not sleep on locks it does not +need). +""" +function wait_safe_interrupt(ws, w::WaitEntry) + ct = current_task() + local r + try + r = wait() + catch + interrupted_park_cleanup!(ct, ws, w) + rethrow() + end + return r +end + +""" + withdraw!(ws, w::WaitEntry, why::UInt8=WAKE_WITHDRAWN) + +Withdraw every registration of `w` per its kind's policy and release the +entry (retiring it when fresh; idempotent). The lock discipline follows +`why`: `WAKE_VALUE`/`WAKE_FIRED` run under the caller's still-held +protection (the lazy settle after a wake; the fired branch), while +`WAKE_WITHDRAWN` dequeues are self-protecting (leaving a multi-wait). +""" +function withdraw!(ws, w::WaitEntry, why::UInt8=WAKE_WITHDRAWN) + _dequeue_each!(ws, w, why) + release_wait_entry!(current_task(), w) + return nothing +end + +# The interrupted-wait cleanup (see wait_safe_interrupt). In order: +# 1. Disarm the registration - before any reacquire below can register a +# new wait. When the disarm loses, a claimer got the wake: its +# schedule is either already enqueued or still in flight under the +# waitee's protection. +# 2. Blank the entry's cache slot: a notifier may have popped the stale +# entry without scheduling us and may still retain its identity for +# the wake-claim CAS, so `w` must not be reused (e.g. by a park +# inside a self-protecting dequeue's lock acquire) before the +# unlinks below. +# 3. Withdraw every registration through its kind's self-protecting +# dequeue; the per-kind lock round-trips serialize any claimer's +# in-flight schedule ... +# 4. ... which is what makes the pending-wake drop here deterministic: +# a wake claimed-and-enqueued by such a claimer must not leak into +# this task's next wait. (A claim-less raw wake delivered outside any +# lock - the documented-unsafe `schedule(t, exc, error=true)` of a +# running task - can still land after this drop; that hazard is the +# primitive's, not this path's.) +# 5. Restore `w` to its cache slot unless a nested park cached a +# replacement - then `w` is unreachable garbage (unarmed, off every +# waitq), so retire it; fresh entries are always retired. +function interrupted_park_cleanup!(ct::Task, ws, w::WaitEntry) + @atomicreplace ct.waiting_on w => nothing + was_plain = ct.cached_wait_entry === w + was_cancel = !was_plain && ct.cached_cancel_entry === w + was_plain && (ct.cached_wait_entry = nothing) + was_cancel && (ct.cached_cancel_entry = nothing) + _dequeue_each!(ws, w, WAKE_INTERRUPTED) + q = ct.queue + q === nothing || list_deletefirst!(q::StickyWorkqueue, ct) + if was_plain + if ct.cached_wait_entry === nothing + ct.cached_wait_entry = w + else + retire_cancellation_entry!(w) + end + elseif was_cancel + if ct.cached_cancel_entry === nothing + ct.cached_cancel_entry = w + else + retire_cancellation_entry!(w) + end + else + release_wait_entry!(ct, w) + end + return nothing +end + +## SourceWait methods (the struct and its doc live above, before the +## entry-acquisition contract that dispatches on it) + +function _source_wait_enqueue!(src::CancellationTokenSource, w::WaitEntry, aux::UInt64) + i = _find_slot(w, src) + if i == 0 + # First registration under `src`: claim a slot (the slot's `owner` + # is the push ticket) and stage the aux (the floor, plus the + # watcher bit for a `WatcherWait`) - pre-publication, so any walk + # that can see the slot sees its aux - then publish with a + # lock-free push. seq_cst, pairing with the cancellation walk's + # state-write-then-head-read: if the walk's head read misses this + # push, this push is later in the total order, so the recheck + # below observes the raised state. + i = _acquire_slot!(w, src) + _set_slot_aux!(w, i, aux) + slot = slots(w)[i] + while true + h = _waiters_head(src) + slot.next = h + if (@atomicreplace :sequentially_consistent :monotonic src.waiters_head h => w).success + break + end + end + # approximate list length, feeding the scaled prune threshold + # (resynced by every walk; see _note_dead_registration!) + @atomic :monotonic src.reg_count += UInt32(1) + else + # Sticky re-arm (already registered): upgrade this thread's + # arm-then-recheck to the store-load ordering the race argument + # needs (the arm CAS itself is only `release`). + Core.Intrinsics.atomic_fence(:sequentially_consistent, :system) + end + return true +end + +wait_enqueue!(x::SourceWait, w::WaitEntry, first::Bool) = + _source_wait_enqueue!(x.src, w, UInt64(x.floor)) + +function wait_recheck(x::SourceWait, w::WaitEntry) + # Post-publication recheck: either the concurrent cancellation walk + # observes our push/arm, or we observe its state write here. + st = @atomic :sequentially_consistent x.src.state + return st != 0x00 && st >= x.floor +end + +wait_dequeue!(x::SourceWait, w::WaitEntry, why::UInt8) = nothing + +## Watching a cancellation source (`wait(::CancellationToken)`) + +# `WatcherWait(src)` in a park's waitables makes the wait *complete* when +# `src` is cancelled: its slot stages the watcher aux bit, which the +# cancellation walk delivers value-mode - the request is the payload, at +# any severity (see `_cancel_walk_node!`). +# The registration and recheck follow `SourceWait`'s Dekker exactly; only +# the staged aux and the meaning of firing (the awaited event, not a +# refusal) differ. Watcher parks always use a fresh single-use entry: +# neither cache slot may ever carry a registration on a source the arm is +# not an ordinary cancellable park under (the shield/claim soundness +# argument in the cache contract above). +struct WatcherWait + src::CancellationTokenSource + # The lowest severity that fires this watcher (inclusive; normalized to + # at least SAFE). Non-throwing teardown waits re-park with the + # acknowledged severity's successor, so only an escalation wakes them. + floor::UInt8 +end +WatcherWait(src::CancellationTokenSource) = WatcherWait(src, CANCEL_REQUEST_SAFE.request) + +acquire_wait_entry!(ct::Task, ws::Tuple{WatcherWait}) = WaitEntry1(ct) +acquire_wait_entry!(ct::Task, ws::Tuple{WatcherWait, SourceWait}) = WaitEntry2(ct) + +wait_enqueue!(x::WatcherWait, w::WaitEntry, first::Bool) = + _source_wait_enqueue!(x.src, w, + WAIT_AUX_WATCHER_BIT | UInt64(max(x.floor, CANCEL_REQUEST_SAFE.request))) + +wait_recheck(x::WatcherWait, w::WaitEntry) = + (@atomic :sequentially_consistent x.src.state) >= max(x.floor, CANCEL_REQUEST_SAFE.request) + +wait_dequeue!(x::WatcherWait, w::WaitEntry, why::UInt8) = nothing + +""" + wait(tok::CancellationToken; cancel=...) + +Block until `tok`'s source is cancelled, and return the corresponding +[`CancellationRequest`](@ref) as an ordinary value; return immediately if it +already is. This inverts the usual delivery - cancellation of `tok` is the +event this operation waits *for*, not an interruption of it - and is the +building block of the watcher-task ("cancellation callback") pattern. + +The wait itself accepts the standard `cancel` keyword argument (defaulting +to the scoped token) and is interrupted by that token like any other +blocking operation. Waiting on the token that also governs the wait is +refused with an `ArgumentError`, since completing and interrupting the wait +would be the same event; pass `cancel = nothing` to wait for `tok` +unconditionally. +""" +function wait(tok::CancellationToken; cancel::CancelTokenArg=DEFAULT_CANCEL) + src = tok.source + gov = resolve_cancel_token(cancel) + govsrc = gov === nothing ? nothing : gov.source + if govsrc === src + throw(ArgumentError( + "cannot wait for a token's cancellation under the same governing token; " * + "pass `cancel = nothing` to wait for it unconditionally")) + end + govsrc === nothing || checkcancel(govsrc) + st = @atomic :acquire src.state + st != 0x00 && return CancellationRequest(st) + ct = current_task() + ws = govsrc === nothing ? (WatcherWait(src),) : + (WatcherWait(src), SourceWait(govsrc, CANCEL_REQUEST_SAFE.request)) + w = acquire_wait_entry!(ct, ws) + if !park!(ws, w, true) + # Fired, and the self-claim won: either the watched source's + # cancellation (the awaited event, delivered as the return value) + # or the governing token's (the refusal) - re-inspect to tell. + withdraw!(ws, w, WAKE_FIRED) + st = @atomic :acquire src.state + st != 0x00 && return CancellationRequest(st) + checkcancel(govsrc::CancellationTokenSource) # throws the refusal + error("cancellation wait fired with no cancellation") + end + r = wait_safe_interrupt(ws, w) + withdraw!(ws, w, WAKE_VALUE) + return r::CancellationRequest +end diff --git a/base/parse.jl b/base/parse.jl index 7e6f35cf6b35a..545e04896545a 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -81,7 +81,7 @@ function parseint_preamble(signed::Bool, base::Int, s::AbstractString, startpos: c, i = iterate(s,i)::Tuple{Char, Int} base = c=='b' ? 2 : c=='o' ? 8 : c=='x' ? 16 : 10 if base != 10 - c, i, j = parseint_iterate(s,i,endpos) + _c, _i, j = parseint_iterate(s,i,endpos) end else base = 10 @@ -106,6 +106,7 @@ end _A <= _c <= _Z ? _c-_A+ UInt32(10) : _a <= _c <= _z ? _c-_a+a : base + return d end @@ -259,47 +260,28 @@ tryparse(::Type{Union{}}, slurp...; kwargs...) = error("cannot parse a value as ## string to float functions ## -function tryparse(::Type{Float64}, s::String) +function tryparse(::Type{Float64}, s::DenseUTF8String) hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s) % UInt) hasvalue ? val : nothing end -function tryparse(::Type{Float64}, s::SubString{String}) - hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits % UInt) - hasvalue ? val : nothing -end -function tryparse_internal(::Type{Float64}, s::String, startpos::Int, endpos::Int) +function tryparse_internal(::Type{Float64}, s::DenseUTF8String, startpos::Int, endpos::Int) hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, (Ptr{UInt8},Csize_t,Csize_t), s, startpos-1, endpos-startpos+1) hasvalue ? val : nothing end -function tryparse_internal(::Type{Float64}, s::SubString{String}, startpos::Int, endpos::Int) - hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset+startpos-1, endpos-startpos+1) - hasvalue ? val : nothing -end -function tryparse(::Type{Float32}, s::String) +function tryparse(::Type{Float32}, s::DenseUTF8String) hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s) % UInt) hasvalue ? val : nothing end -function tryparse(::Type{Float32}, s::SubString{String}) - hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits % UInt) - hasvalue ? val : nothing -end -function tryparse_internal(::Type{Float32}, s::String, startpos::Int, endpos::Int) +function tryparse_internal(::Type{Float32}, s::DenseUTF8String, startpos::Int, endpos::Int) hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, (Ptr{UInt8},Csize_t,Csize_t), s, startpos-1, endpos-startpos+1) hasvalue ? val : nothing end -function tryparse_internal(::Type{Float32}, s::SubString{String}, startpos::Int, endpos::Int) - hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset+startpos-1, endpos-startpos+1) - hasvalue ? val : nothing -end -tryparse(::Type{T}, s::AbstractString) where {T<:Union{Float32,Float64}} = tryparse(T, String(s)) + +tryparse(::Type{T}, s::AbstractString) where {T<:Union{Float32,Float64}} = tryparse(T, String(s)::String) tryparse(::Type{Float16}, s::AbstractString) = convert(Union{Float16, Nothing}, tryparse(Float32, s)) tryparse_internal(::Type{Float16}, s::AbstractString, startpos::Int, endpos::Int) = @@ -307,7 +289,7 @@ tryparse_internal(::Type{Float16}, s::AbstractString, startpos::Int, endpos::Int ## string to complex functions ## -function tryparse_internal(::Type{Complex{T}}, s::Union{String,SubString{String}}, i::Int, e::Int, raise::Bool) where {T<:Real} +function tryparse_internal(::Type{Complex{T}}, s::DenseUTF8String, i::Int, e::Int, raise::Bool) where {T<:Real} # skip initial whitespace while i ≤ e && isspace(s[i]) i = nextind(s, i) diff --git a/base/partr.jl b/base/partr.jl index d488330f0c87e..6ecdf702ccd8e 100644 --- a/base/partr.jl +++ b/base/partr.jl @@ -2,7 +2,7 @@ module Partr -using ..Threads: SpinLock, maxthreadid, threadid +using ..Threads: SpinLock # a task minheap mutable struct taskheap @@ -44,7 +44,7 @@ Base.@assume_effects :removable :inaccessiblememonly :notaskstate function rand_ return val % UInt32 end -# This implementation is based on OpenSSLs implementation of rand_uniform +# This implementation is based on OpenSSL's implementation of rand_uniform # https://github.com/openssl/openssl/blob/1d2cbd9b5a126189d5e9bc78a3bdb9709427d02b/crypto/rand/rand_uniform.c#L13-L99 # Comments are vendored from their implementation as well. # For the original developer check the PR to swift https://github.com/apple/swift/pull/39143. @@ -68,7 +68,7 @@ Base.@assume_effects :total function rand_uniform_max_int32(max::UInt32, seed::U # Multiplying this by the range gives us a number on [0, upper). # The high word of the multiplication result represents the integral part # This is not completely unbiased as it's missing the fractional part of the original implementation but it's good enough for our purposes - seed = UInt64(69069) * seed + UInt64(362437) + seed = UInt64(69069) *% seed +% UInt64(362437) prod = (UInt64(max)) * (seed % UInt32) # 64 bit product i = prod >> 32 % UInt32 # integral part return i % UInt32, seed @@ -109,6 +109,12 @@ end function multiq_size(tpid::Int8) nt = UInt32(Threads._nthreads_in_pool(tpid)) + if nt == 0 + # A pool with no threads can still receive tasks (e.g. during + # sysimage bootstrap); size the heaps so insertion can park them + # instead of indexing an empty heap vector. + nt = UInt32(1) + end tp = tpid + 1 tpheaps = heaps[tp] heap_c = UInt32(2) @@ -119,8 +125,14 @@ function multiq_size(tpid::Int8) end @lock heaps_lock[tp] begin + # Re-read under the lock; growing from a stale copy would replace the + # current heaps and orphan tasks concurrently inserted into them (#62144). + tpheaps = heaps[tp] heap_p = UInt32(length(tpheaps)) nt = UInt32(Threads._nthreads_in_pool(tpid)) + if nt == 0 + nt = UInt32(1) + end if heap_c * nt <= heap_p return heap_p end @@ -139,7 +151,7 @@ end function multiq_insert(task::Task, priority::UInt16) tpid = ccall(:jl_get_task_threadpoolid, Int8, (Any,), task) - @assert tpid > -1 + @assert tpid > -1 "invalid tpid" heap_p = multiq_size(tpid) tp = tpid + 1 @@ -171,6 +183,7 @@ end function multiq_deletemin() local rn1::UInt32 + local heap, task tid = Threads.threadid() tp = ccall(:jl_threadpoolid, Int8, (Int16,), tid-1) + 1 @@ -179,43 +192,52 @@ function multiq_deletemin() end tpheaps = heaps[tp] - @label retry - GC.safepoint() - heap_p = UInt32(length(tpheaps)) - for i = UInt32(0):heap_p - if i == heap_p - return nothing - end - rn1 = cong(heap_p) - rn2 = cong(heap_p) - prio1 = tpheaps[rn1].priority - prio2 = tpheaps[rn2].priority - if prio1 > prio2 - prio1 = prio2 - rn1 = rn2 - elseif prio1 == prio2 && prio1 == typemax(UInt16) - continue - end - if trylock(tpheaps[rn1].lock) - if prio1 == tpheaps[rn1].priority - break + while true + GC.safepoint() + heap_p = UInt32(length(tpheaps)) + for i = UInt32(0):heap_p + if i == heap_p + return nothing + end + rn1 = cong(heap_p) + rn2 = cong(heap_p) + prio1 = tpheaps[rn1].priority + prio2 = tpheaps[rn2].priority + if prio1 > prio2 + prio1 = prio2 + rn1 = rn2 + elseif prio1 == prio2 && prio1 == typemax(UInt16) + continue + end + if trylock(tpheaps[rn1].lock) + if prio1 == tpheaps[rn1].priority + break + end + unlock(tpheaps[rn1].lock) end - unlock(tpheaps[rn1].lock) end - end - - @assert @isdefined(rn1) "Assertion to tell the compiler about the definedness of this variable" - heap = tpheaps[rn1] - task = heap.tasks[1] - if ccall(:jl_set_task_tid, Cint, (Any, Cint), task, tid-1) == 0 - unlock(heap.lock) - @goto retry + @assert @isdefined(rn1) "Assertion to tell the compiler about the definedness of this variable" + + heap = tpheaps[rn1] + task = heap.tasks[1] + if ccall(:jl_set_task_tid, Cint, (Any, Cint), task, tid-1) == 0 + # This task is sticky to a different thread, so we can't run it. + # Wake that thread so it can come pick up its own work, then keep + # looking for something we are allowed to run. + task_tid = ccall(:jl_get_task_tid, Int16, (Any,), task) + unlock(heap.lock) + if task_tid != Int16(-1) + ccall(:jl_wakeup_thread, Cint, (Int16,), task_tid) + end + continue + end + break end ntasks = heap.ntasks @atomic :monotonic heap.ntasks = ntasks - Int32(1) heap.tasks[1] = heap.tasks[ntasks] - Base._unsetindex!(heap.tasks, Int(ntasks)) + Base.unsetindex!(heap.tasks, Int(ntasks)) prio1 = typemax(UInt16) if ntasks > 1 multiq_sift_down(heap, Int32(1)) diff --git a/base/path.jl b/base/path.jl index a1221e0bdc844..386bb42f11013 100644 --- a/base/path.jl +++ b/base/path.jl @@ -1,5 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +# NB: This file is `Core.eval`-uated into the (pre-existing) module Filesystem + +import Base: StringVector, utf8units + export abspath, basename, @@ -20,40 +24,126 @@ export if Sys.isunix() const path_separator = "/" - const path_separator_re = r"/+"sa - const path_directory_re = r"(?:^|/)\.{0,2}$"sa - const path_dir_splitter = r"^(.*?)(/+)([^/]*)$"sa - const path_ext_splitter = r"^((?:.*/)?(?:\.|[^/\.])[^/]*?)(\.[^/\.]*|)$"sa + @inline isseparator(c::Char) = c === '/' + @inline isseparator(c::UInt8) = c === UInt8('/') splitdrive(path::String) = ("",path) elseif Sys.iswindows() const path_separator = "\\" - const path_separator_re = r"[/\\]+"sa - const path_absolute_re = r"^(?:[A-Za-z]+:)?[/\\]"sa - const path_directory_re = r"(?:^|[/\\])\.{0,2}$"sa - const path_dir_splitter = r"^(.*?)([/\\]+)([^/\\]*)$"sa - const path_ext_splitter = r"^((?:.*[/\\])?(?:\.|[^/\\\.])[^/\\]*?)(\.[^/\\\.]*|)$"sa - - const splitdrive_re = let - # Slash in either direction. - S = raw"[\\/]" - # Not a slash in either direction. - N = raw"[^\\/]" - # Drive letter, e.g. `C:` - drive = "$(N)+:" - # UNC path, e.g. `\\server\share` - unc = "$(S)$(S)$(N)+$(S)$(N)+" - # Long drive letter, e.g. `\\?\C:` - long_drive = "$(S)$(S)\\?$(S)$(drive)" + @inline isseparator(c::Char) = c === '/' || c === '\\' + @inline isseparator(c::UInt8) = c === UInt8('/') || c === UInt8('\\') + + @inline isdriveletter(c::Char) = isascii(c) && isdriveletter(UInt8(c)) + @inline function isdriveletter(c::UInt8) + UInt8('A') <= c <= UInt8('Z') || + UInt8('a') <= c <= UInt8('z') + end + + function _split_longunc(s::String)::Tuple{String, String} # Long UNC path, e.g. `\\?\UNC\server\share` - long_unc = "$(S)$(S)\\?$(S)UNC$(S)$(N)+$(S)$(N)+" - # Need to match the long patterns first so they get priority. - Regex("^($long_unc|$long_drive|$unc|$drive|)(.*)\$", "sa") + # Based on previous implementation matching with regex + # S = raw"[\\/]"; N = raw"[^\\/]"; + # r"^$(S)$(S)\?$(S)UNC$(S)$(N)+$(S)$(N)+"sa + + if (ncodeunits(s) >= 11 && + isseparator(codeunit(s, 1)) && + isseparator(codeunit(s, 2)) && + codeunit(s, 3) === UInt8('?') && + isseparator(codeunit(s, 4)) && + codeunit(s, 5) === UInt8('U') && + codeunit(s, 6) === UInt8('N') && + codeunit(s, 7) === UInt8('C') && + isseparator(codeunit(s, 8)) + ) + # Ensure we have [sequence of non-separator] - single separator - [sequence of non-separator]. + # Since the prefix raw"\\?\UNC\" is always 8 codeunits, we start at index 9. + i = findnext(isseparator, s, 9) + if (!isnothing(i) && + i >= 10 && # implies !isseparator(s[9]) + ncodeunits(s) > i && # Need something after the separator + !isseparator(codeunit(s, i+1)) # Consecutive separators does not count + ) + # Stop just before next separator if it exists, + # otherwise the whole string is a drive + j = something(findnext(isseparator, s, i+1), lastindex(s)+1) + return s[1:prevind(s, j)], s[j:end] + end + end + return "", s + end + + function _split_longdriveletter(s::String)::Tuple{String, String} + # Long drive letter, e.g. `\\?\C:` + # Based on implementation matching with regex + # S = raw"[\\/]"; N = raw"[^\\/]"; drive = "$(N):"; + # r"$(S)$(S)\?$(S)$(drive)"sa + if (ncodeunits(s) >= 6 && + isseparator(codeunit(s, 1)) && + isseparator(codeunit(s, 2)) && + codeunit(s, 3) === UInt8('?') && + isseparator(codeunit(s, 4)) && + !isseparator(codeunit(s, 5)) && # Any ascii char except separators passes as the drive letter + codeunit(s, 6) == UInt8(':') # This effectively limits codeunit(s, 5) to ascii + ) + return s[1:6], s[nextind(s, 6):end] + end + return "", s end - function splitdrive(path::String) - m = match(splitdrive_re, path)::AbstractMatch - String(something(m.captures[1])), String(something(m.captures[2])) + function _split_uncpath(s::String)::Tuple{String, String} + # UNC path, e.g. `\\server\share` + # Based on previous implementation matching with regex + # S = raw"[\\/]"; N = raw"[^\\/]"; + # r"$(S)$(S)$(N)+$(S)$(N)+"sa + if (ncodeunits(s) >= 5 && # Not shorter than `\\a\b` + isseparator(codeunit(s, 1)) && + isseparator(codeunit(s, 2)) + ) + # Ensure we have [sequence of non-separator] - single separator - [sequence of non-separator]. + # Since the prefix raw"\\" is always 2 codeunits, we start at index 3. + i = findnext(isseparator, s, 3) + if (!isnothing(i) && + i >= 4 && # implies !isseparator(s[3]) + ncodeunits(s) > i && # Need something after the separator + !isseparator(codeunit(s, i+1)) # Consecutive separators does not count + ) + # Stop just before next separator if it exists, + # otherwise the whole string is a drive + j = something(findnext(isseparator, s, i+1), lastindex(s)+1) + return s[1:prevind(s, j)], s[j:end] + end + end + return "", s + end + + function splitdrive(path::String)::Tuple{String, String} + if !isempty(path) + # Fast return if path does not contain a drive + if !isseparator(codeunit(path, 1)) && (codeunit(path, 1) < 0x80) + # Drive letter, e.g. `C:` + # Any ascii char except separators passes as the drive letter + colonind = nextind(path, 1) + if checkbounds(Bool, path, colonind) && path[colonind] === ':' + return path[1:colonind], path[colonind+1:end] + end + elseif ncodeunits(path) >= 2 && isseparator(codeunit(path, 2)) + # All other drive types must start with two separators + + # Long UNC path, e.g. `\\?\UNC\server\share` + drive, rest = _split_longunc(path) + !isempty(drive) && return drive, rest + + # Long drive letter, e.g. `\\?\C:` + drive, rest = _split_longdriveletter(path) + !isempty(drive) && return drive, rest + + # UNC path, e.g. `\\server\share` + drive, rest = _split_uncpath(path) + !isempty(drive) && return drive, rest + end + end + + return "", path end else error("path primitives for this OS need to be defined") @@ -68,6 +158,10 @@ first component is always the empty string. """ splitdrive(path::AbstractString) +# Average buffer size including null terminator for several filesystem operations. +# On Windows we use the MAX_PATH = 260 value on Win32. +const AVG_PATH = Sys.iswindows() ? 260 : 512 + """ homedir()::String @@ -78,11 +172,18 @@ Return the current user's home directory. (for example on how to specify the home directory via environment variables), see the [`uv_os_homedir` documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_homedir). + homedir(username::AbstractString)::Union{String,Nothing} + +Return the home directory for the given `username`, or `nothing` if the user does not exist. +On Unix, this performs a lookup via `getpwnam_r`. On Windows, the user's SID is resolved and +the profile path is read from the registry; if that fails, the profile directory is inferred +from the current user's home directory. + See also [`Sys.username`](@ref). """ function homedir() buf = Base.StringVector(AVG_PATH - 1) # space for null-terminator implied by StringVector - sz = RefValue{Csize_t}(length(buf) + 1) # total buffer size including null + sz = Base.RefValue{Csize_t}(length(buf) + 1) # total buffer size including null while true rc = ccall(:uv_os_homedir, Cint, (Ptr{UInt8}, Ptr{Csize_t}), buf, sz) if rc == 0 @@ -96,11 +197,29 @@ function homedir() end end - -if Sys.iswindows() - isabspath(path::AbstractString) = occursin(path_absolute_re, path) -else - isabspath(path::AbstractString) = startswith(path, '/') +function isabspath(path::String) + isempty(path) && return false + # Paths starting with "/" are considered absolute also on windows + # This captures e.g. UNC paths, but does not guarantee a valid path. + # Also note that isabspath(x) does not imply !isempty(splitdrive(x)[1]) + isseparator(codeunit(path, 1) ) && return true + + @static if Sys.iswindows() + # the letter before : in e.g. "C:\" must be a valid drive letter. + # This differs from `splitdrive`, where any non-separator single codeunit char is + # accepted. + firstsep = findfirst(isseparator, codeunits(path)) + if (!isnothing(firstsep) && + firstsep >= 3 && + codeunit(path, firstsep-1) == UInt(':') + ) + for b in codeunits(path)[1:firstsep-2] + !isdriveletter(b) && return false + end + return true + end + end + return false end """ @@ -133,7 +252,12 @@ julia> isdirpath("/home/") true ``` """ -isdirpath(path::String) = occursin(path_directory_re, splitdrive(path)[2]) +function isdirpath(path::String)::Bool + # Reimplements occursin(r"(?:^|/)\.{0,2}$"sa, splitdrive(path)[2]) + + _, after_last_separator = _splitdir_nodrive("", splitdrive(path)[2]) + return after_last_separator in ("", ".", "..") +end """ splitdir(path::AbstractString) -> (dir::AbstractString, file::AbstractString) @@ -153,14 +277,15 @@ end # Common splitdir functionality without splitdrive, needed for splitpath. _splitdir_nodrive(path::String) = _splitdir_nodrive("", path) -function _splitdir_nodrive(a::String, b::String) - m = match(path_dir_splitter,b) - m === nothing && return (a,b) - cs = m.captures - getcapture(cs, i) = cs[i]::AbstractString - c1, c2, c3 = getcapture(cs, 1), getcapture(cs, 2), getcapture(cs, 3) - a = string(a, isempty(c1) ? c2[1] : c1) - a, String(c3) +function _splitdir_nodrive(drive::String, path::String)::Tuple{String, String} + lastsepind = findlast(isseparator, path) + + isnothing(lastsepind) && return drive, path + + dir = path[1:something(findprev(!isseparator, path, lastsepind), 1)] + tail = path[nextind(path, lastsepind):end] + + return drive * dir, tail end """ @@ -223,11 +348,23 @@ julia> splitext("/home/my.user/example") ("/home/my.user/example", "") ``` """ -function splitext(path::String) - a, b = splitdrive(path) - m = match(path_ext_splitter, b) - m === nothing && return (path,"") - (a*something(m.captures[1])), String(something(m.captures[2])) +function splitext(path::String)::Tuple{String, String} + drive, p = splitdrive(path) + lastdot = findlast('.', p) + if !isnothing(lastdot) + # No separator after the last dot + if isnothing(findnext(isseparator, p, lastdot)) + # No separator just before the last dot + prev = prevind(p, lastdot) + if checkbounds(Bool, p, prev) + if !isseparator(p[prev]) + return drive * p[1:prev], p[lastdot:end] + end + end + end + end + + return (path, "") end # NOTE: deprecated in 1.4 @@ -253,7 +390,7 @@ julia> splitpath("/home/myuser/example.jl") "example.jl" ``` """ -splitpath(p::AbstractString) = splitpath(String(p)) +splitpath(p::AbstractString) = splitpath(String(p)::String) function splitpath(p::String) drive, p = splitdrive(p) @@ -287,7 +424,7 @@ function joinpath(paths::Union{Tuple, AbstractVector})::String assertstring(paths[i]) p_drive, p_path = splitdrive(paths[i]) - if startswith(p_path, ('\\', '/')) + if !isempty(p_path) && isseparator(first(p_path)) # second path is absolute if !isempty(p_drive) || !isempty(result_drive) result_drive = p_drive @@ -304,7 +441,7 @@ function joinpath(paths::Union{Tuple, AbstractVector})::String end # second path is relative to the first - if !isempty(result_path) && result_path[end] ∉ ('\\', '/') + if !isempty(result_path) && !isseparator(result_path[end]) result_path *= "\\" end @@ -312,7 +449,11 @@ function joinpath(paths::Union{Tuple, AbstractVector})::String end # add separator between UNC and non-absolute path - if !isempty(p_path) && result_path[1] ∉ ('\\', '/') && !isempty(result_drive) && result_drive[end] != ':' + if (!isempty(p_path) && + !isseparator(result_path[1]) && + !isempty(result_drive) && + result_drive[end] != ':' + ) return result_drive * "\\" * result_path end @@ -357,7 +498,7 @@ the join of the preceding paths, then prior components are dropped. Note on Windows since there is a current directory for each drive, `joinpath("c:", "foo")` represents a path relative to the current directory on drive "c:" so this is equal to "c:foo", not "c:\\foo". Furthermore, `joinpath` treats this as a non-absolute path and ignores the drive -letter casing, hence `joinpath("C:\\A","c:b") = "C:\\A\\b"`. +letter casing, hence `joinpath("C:\\\\A","c:b") = "C:\\\\A\\\\b"`. # Examples ```jldoctest @@ -372,6 +513,30 @@ julia> joinpath(["/home/myuser", "example.jl"]) """ joinpath +function _split_at_separators(path::AbstractString; keepempty = true) + # Equivalent to Base.split(path, r"/+"sa; keepempty) (r"[\\/]+"sa on windows) + # Since there is no split between consecutive separators, keepempty + # only has an effect on strings starting or ending with separators. + out = String[] + start = 1 + + while true + nextsep = findnext(isseparator, path, start) + + stop = isnothing(nextsep) ? lastindex(path) : prevind(path, nextsep) + + substr = String(view(path, start:stop)) + if keepempty || !isempty(substr) + push!(out, substr) + end + + isnothing(nextsep) && break + + start = something(findnext(!isseparator, path, nextsep+1), nextind(path, lastindex(path))) + end + return out +end + """ normpath(path::AbstractString)::String @@ -391,7 +556,7 @@ function normpath(path::String) isabs = isabspath(path) isdir = isdirpath(path) drive, path = splitdrive(path) - parts = split(path, path_separator_re; keepempty=false) + parts = _split_at_separators(path, keepempty = false) filter!(!=("."), parts) while true clean = true @@ -445,7 +610,7 @@ Which gives a path like `"/home/JuliaUser/data/"`. See also [`joinpath`](@ref), [`pwd`](@ref), [`expanduser`](@ref). """ -function abspath(a::String)::String +@noinline function abspath(a::String)::String if !isabspath(a) cwd = pwd() a_drive, a_nodrive = splitdrive(a) @@ -514,50 +679,233 @@ function realpath(path::AbstractString) end if Sys.iswindows() -# on windows, ~ means "temporary file" -expanduser(path::AbstractString) = path -contractuser(path::AbstractString) = path -else -function expanduser(path::AbstractString) + +function homedir(username::AbstractString) + # For the current user, just return homedir(). + current_user = try + Sys.username() + catch err + err isa IOError || rethrow() + nothing + end + if username == current_user + return homedir() + end + # Look up the user's SID, then query the registry for their profile path. + # This is the same approach Go uses (os/user.Lookup on Windows). + home = _win_profile_from_registry(username) + home !== nothing && return home + # Fallback: assume profiles are siblings in the same parent directory, + # but only if the current user's home follows the / + # convention. If not, we can't guess reliably. + userhome = homedir() + if current_user !== nothing && basename(userhome) == current_user + home = joinpath(dirname(userhome), username) + isdir(home) && return home + end + return nothing +end +function _win_profile_from_registry(username::AbstractString) + # Step 1: Resolve username to a SID via LookupAccountNameW. + # First call with zero-length buffers to get required sizes. + wuser = cwstring(username) + sid_size = Ref{UInt32}(0) + domain_size = Ref{UInt32}(0) + use = Ref{Int32}(0) + ccall((:LookupAccountNameW, "advapi32"), stdcall, Cint, + (Ptr{UInt16}, Ptr{UInt16}, Ptr{Cvoid}, Ptr{UInt32}, + Ptr{UInt16}, Ptr{UInt32}, Ptr{Int32}), + C_NULL, wuser, C_NULL, sid_size, C_NULL, domain_size, use) + sid_size[] == 0 && return nothing + sid_buf = Vector{UInt8}(undef, sid_size[]) + domain_buf = Vector{UInt16}(undef, domain_size[]) + ret = ccall((:LookupAccountNameW, "advapi32"), stdcall, Cint, + (Ptr{UInt16}, Ptr{UInt16}, Ptr{UInt8}, Ptr{UInt32}, + Ptr{UInt16}, Ptr{UInt32}, Ptr{Int32}), + C_NULL, wuser, sid_buf, sid_size, domain_buf, domain_size, use) + ret == 0 && return nothing + # Step 2: Convert SID to string form (e.g. "S-1-5-21-..."). + str_sid_ptr = Ref{Ptr{UInt16}}(C_NULL) + ret = ccall((:ConvertSidToStringSidW, "advapi32"), stdcall, Cint, + (Ptr{UInt8}, Ref{Ptr{UInt16}}), sid_buf, str_sid_ptr) + ret == 0 && return nothing + len = ccall(:wcslen, Csize_t, (Ptr{UInt16},), str_sid_ptr[]) + sid_str = transcode(String, unsafe_wrap(Array, str_sid_ptr[], len)) + ccall((:LocalFree, "kernel32"), stdcall, Ptr{Cvoid}, (Ptr{Cvoid},), str_sid_ptr[]) + # Step 3: Query the registry for the user's ProfileImagePath. + subkey = cwstring("SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\ProfileList\\$sid_str") + value = cwstring("ProfileImagePath") + buf_size = Ref{UInt32}(0) + # RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ = 0x00000006 + HKEY_LOCAL_MACHINE = 0x80000002 % UInt + ccall((:RegGetValueW, "advapi32"), stdcall, Clong, + (UInt, Ptr{UInt16}, Ptr{UInt16}, UInt32, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}), + HKEY_LOCAL_MACHINE, subkey, value, 0x00000006, C_NULL, C_NULL, buf_size) + buf_size[] == 0 && return nothing + buf = Vector{UInt16}(undef, buf_size[] ÷ 2) + ret = ccall((:RegGetValueW, "advapi32"), stdcall, Clong, + (UInt, Ptr{UInt16}, Ptr{UInt16}, UInt32, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}), + HKEY_LOCAL_MACHINE, subkey, value, 0x00000006, C_NULL, buf, buf_size) + ret != 0 && return nothing + # Remove trailing null and convert to String. + n = buf_size[] ÷ 2 + n > 0 && buf[n] == 0 && (n -= 1) + home = transcode(String, buf[1:n]) + return isdir(home) ? home : nothing +end +function contractuser(path::Union{String, SubString{String}})::String + # Walk prefixes of path, checking if any matches homedir() via inode. + # Only checks the current user's home (no ~username on Windows). + # Preserves the original path string after the matched prefix verbatim. + home_st = stat(homedir()) + ispath(home_st) || return path + # check the full path (home directory itself, no trailing separator) + samefile(stat(path), home_st) && return "~" + # scan for separators; start after the first one to skip the root + m = findnext(path_separator_re, path, firstindex(path)) + m === nothing && return path + while true + m = findnext(path_separator_re, path, nextind(path, last(m))) + m === nothing && return path + prefix = SubString(path, 1, prevind(path, first(m))) + st = stat(prefix) + ispath(st) || return path + if samefile(st, home_st) + return "~" * SubString(path, first(m)) + end + end +end + +else # !Sys.iswindows() + +function homedir(username::AbstractString) + # Thread-safe user lookup via getpwnam_r. + # pwd_storage holds the struct passwd; 256 bytes is a generous upper bound + # for all supported platforms (Linux x86-64: ~56 bytes, macOS arm64: ~80 bytes). + pwd_storage = zeros(UInt8, 256) + # The string buffer holds the pointed-to strings (pw_name, pw_dir, etc.). + # Start at 1024 and double on ERANGE if any string is unusually long. + buflen = 1024 + while buflen <= 65536 + str_buf = Vector{UInt8}(undef, buflen) + result = Ref{Ptr{Cvoid}}(C_NULL) + ret = ccall(:getpwnam_r, Cint, + (Cstring, Ptr{Cvoid}, Ptr{UInt8}, Csize_t, Ptr{Ptr{Cvoid}}), + username, pwd_storage, str_buf, Csize_t(buflen), result) + if ret == 34 # ERANGE: string buffer too small, retry with more space + buflen *= 2 + elseif ret == 0 && result[] != C_NULL + # pw_uid sits at offset 2*sizeof(Ptr) in struct passwd on all supported + # platforms (after pw_name and pw_passwd, which are both pointer-sized) + uid = unsafe_load(Ptr{Cuint}(pointer(pwd_storage) + 2 * sizeof(Ptr{Cvoid}))) + pd = Libc.getpwuid(uid, false) + return pd !== nothing ? pd.homedir : nothing + else + return nothing # user not found or error + end + end + return nothing +end +function contractuser(path::Union{String, SubString{String}})::String + # Walk path prefixes from shortest to longest. At each existing prefix, + # check against the current user's home first, then the directory owner's + # home via inode comparison. This handles symlinks transparently. + # Preserves the original path string after the matched prefix verbatim. + home_st = stat(homedir()) + ispath(home_st) || return path + cache_uid = ccall(:getuid, Cuint, ()) + cache_uname = nothing + cache_home_st = nothing + # Check the full path (home directory itself, no trailing separator) + samefile(stat(path), home_st) && return "~" + # Scan for separators; start after the first one to skip the root + m = findnext(path_separator_re, path, firstindex(path)) + m === nothing && return path + while true + m = findnext(path_separator_re, path, nextind(path, last(m))) + m === nothing && return path + prefix = SubString(path, 1, prevind(path, first(m))) + st = stat(prefix) + ispath(st) || return path + rest = SubString(path, first(m)) + if samefile(st, home_st) + return "~" * rest + end + uid = st.uid + if uid != cache_uid + cache_uid = uid + pd = Libc.getpwuid(uid, false) + cache_uname = pd !== nothing && !isempty(pd.username) ? pd.username : nothing + if cache_uname !== nothing + pw_home = homedir(cache_uname) + cache_home_st = pw_home !== nothing ? stat(pw_home) : nothing + else + cache_home_st = nothing + end + end + if cache_home_st !== nothing && ispath(cache_home_st) && samefile(st, cache_home_st) + return "~$(cache_uname)" * rest + end + end +end + +end # if Sys.iswindows() + +function expanduser(path::Union{String, SubString{String}})::String y = iterate(path) y === nothing && return path c, i = y::Tuple{eltype(path),Int} c != '~' && return path - y = iterate(path, i) - y === nothing && return homedir() - y[1]::eltype(path) == '/' && return homedir() * path[i:end] - throw(ArgumentError("~user tilde expansion not yet implemented")) -end -function contractuser(path::AbstractString) - home = homedir() - if path == home - return "~" - elseif startswith(path, home) - return joinpath("~", relpath(path, home)) - else + # collect username: everything after ~ up to separator or end + m = findnext(path_separator_re, path, i) + j = prevind(path, m === nothing ? + nextind(path, lastindex(path)) : first(m)) + username = SubString(path, i, j) + # can't use a regex because of bootstrap order + if isempty(username) + home = homedir() + elseif Sys.iswindows() || # ~username not supported on Windows + !all(c -> isletter(c) || isdigit(c) || c in "._-", username) # invalid return path + else + home = homedir(username) + home === nothing && return path end -end + # use first separator in the rest of path in home + if m !== nothing + if Sys.iswindows() + sep = path[first(m)] + home = replace(home, path_separator_re => sep) + end + return home * SubString(path, first(m)) + end + return home end """ expanduser(path::AbstractString)::AbstractString -On Unix systems, replace a tilde character at the start of a path with the current user's home directory. +Replace a tilde character at the start of a path with the current user's home directory. +On Unix, `~username` at the start of a path is replaced with that user's home directory; +if the user does not exist the path is returned unchanged. On Windows, only `~` expansion +is supported (not `~username`). See also: [`contractuser`](@ref). """ -expanduser(path::AbstractString) +expanduser(path::AbstractString) = expanduser(String(path)) """ contractuser(path::AbstractString)::AbstractString -On Unix systems, if the path starts with `homedir()`, replace it with a tilde character. +Replace a home directory prefix in `path` with a tilde. If the path starts with the +current user's home directory it is replaced with `~`. On Unix, if it starts with +another user's home directory it is replaced with `~username`. The path is returned +unchanged if no home directory prefix is found. See also: [`expanduser`](@ref). """ -contractuser(path::AbstractString) +contractuser(path::AbstractString) = contractuser(String(path)) """ @@ -581,11 +929,11 @@ function relpath(path::String, startpath::String = ".") startpath_drive, startpath_without_drive = splitdrive(startpath) isempty(startpath_drive) && (startpath_drive = path_drive) # by default assume same as path drive uppercase(path_drive) == uppercase(startpath_drive) || return abspath(path) # if drives differ return first path - path_arr = split(abspath(path_drive * path_without_drive), path_separator_re) - start_arr = split(abspath(path_drive * startpath_without_drive), path_separator_re) + path_arr = _split_at_separators(abspath(path_drive * path_without_drive)) + start_arr = _split_at_separators(abspath(path_drive * startpath_without_drive)) else - path_arr = split(abspath(path), path_separator_re) - start_arr = split(abspath(startpath), path_separator_re) + path_arr = _split_at_separators(abspath(path)) + start_arr = _split_at_separators(abspath(startpath)) end i = 0 while i < min(length(path_arr), length(start_arr)) @@ -608,16 +956,30 @@ function relpath(path::String, startpath::String = ".") return isempty(relpath_) ? curdir : relpath_ end relpath(path::AbstractString, startpath::AbstractString) = - relpath(String(path), String(startpath)) + relpath(String(path)::String, String(startpath)::String) -for f in (:isdirpath, :splitdir, :splitdrive, :splitext, :normpath, :abspath) - @eval $f(path::AbstractString) = $f(String(path)) +for f in (:isdirpath, :splitdir, :splitdrive, :splitext, :normpath, :abspath, :isabspath) + @eval $f(path::AbstractString) = $f(String(path)::String) end -# RFC3986 Section 2.1 -percent_escape(s) = '%' * join(map(b -> uppercase(string(b, base=16)), codeunits(s)), '%') -# RFC3986 Section 2.3 -encode_uri_component(s) = replace(s, r"[^A-Za-z0-9\-_.~/]+" => percent_escape) +function encode_uri_component(s::AbstractString) + out = empty!(StringVector(sizeof(s))) + for cu in utf8units(s) + # RFC3986 Section 2.3 + if (UInt8('A') <= cu <= UInt8('Z') || + UInt8('a') <= cu <= UInt8('z') || + UInt8('0') <= cu <= UInt8('9') || + cu in map(UInt8, ('-', '_', '.', '~', '/')) + ) + push!(out, cu) + else + # RFC3986 Section 2.1 + push!(out, UInt8('%')) + append!(out, codeunits(uppercase(string(cu, base = 16)))) + end + end + String(out) +end """ uripath(path::AbstractString) @@ -643,18 +1005,18 @@ function uripath end function uripath(path::String) path = abspath(path) if startswith(path, "\\\\") # UNC path, RFC8089 Appendix E.3 - unixpath = join(eachsplit(path, path_separator_re, keepempty=false), '/') + unixpath = join(_split_at_separators(path, keepempty=false), '/') string("file://", encode_uri_component(unixpath)) # RFC8089 Section 2 else drive, localpath = splitdrive(path) # Assuming that non-UNC absolute paths on Windows always have a drive component - unixpath = join(eachsplit(localpath, path_separator_re, keepempty=false), '/') + unixpath = join(_split_at_separators(localpath, keepempty=false), '/') encdrive = replace(encode_uri_component(drive), "%3A" => ':', "%7C" => '|') # RFC8089 Appendices D.2, E.2.1, and E.2.2 string("file:///", encdrive, '/', encode_uri_component(unixpath)) # RFC8089 Section 2 end end else function uripath(path::String) - localpath = join(eachsplit(abspath(path), path_separator_re, keepempty=false), '/') + localpath = join(_split_at_separators(abspath(path), keepempty=false), '/') host = if ispath("/proc/sys/fs/binfmt_misc/WSLInterop") # WSL sigil distro = get(ENV, "WSL_DISTRO_NAME", "") # See "wsl\$/$distro" # See and @@ -665,4 +1027,4 @@ else end end -uripath(path::AbstractString) = uripath(String(path)) +uripath(path::AbstractString) = uripath(String(path)::String) diff --git a/base/pcre.jl b/base/pcre.jl index 213fc1890f51d..852f0b384d465 100644 --- a/base/pcre.jl +++ b/base/pcre.jl @@ -4,7 +4,7 @@ module PCRE -import ..RefValue +import ..RefValue, ..DenseUTF8String # include($BUILDROOT/base/pcre_h.jl) include(string(Base.BUILDROOT, "pcre_h.jl")) @@ -196,7 +196,7 @@ function err_message(errno::Integer) return GC.@preserve buffer unsafe_string(pointer(buffer)) end -exec(re, subject::Union{String,SubString{String}}, offset, options, match_data) = +exec(re, subject::DenseUTF8String, offset, options, match_data) = _exec(re, subject, offset, options, match_data) exec(re, subject, offset, options, match_data) = _exec(re, String(subject)::String, offset, options, match_data) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index cf9748168aac2..1d63839af8387 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -50,6 +50,8 @@ Base.size(A::PermutedDimsArray{T,N,perm}) where {T,N,perm} = genperm(size(parent Base.axes(A::PermutedDimsArray{T,N,perm}) where {T,N,perm} = genperm(axes(parent(A)), perm) Base.has_offset_axes(A::PermutedDimsArray) = Base.has_offset_axes(A.parent) Base.similar(A::PermutedDimsArray, T::Type, dims::Base.Dims) = similar(parent(A), T, dims) +Base.dataids(A::PermutedDimsArray) = Base.dataids(parent(A)) +Base.unaliascopy(A::PermutedDimsArray) = typeof(A)(Base.unaliascopy(parent(A))) Base.cconvert(::Type{Ptr{T}}, A::PermutedDimsArray{T}) where {T} = Base.cconvert(Ptr{T}, parent(A)) # It's OK to return a pointer to the first element, and indeed quite @@ -288,11 +290,12 @@ function permutedims!(dest, src::AbstractArray, perm) return dest end -function Base.copyto!(dest::PermutedDimsArray{T,N}, src::AbstractArray{T,N}) where {T,N} +function Base.copyto!(dest::PermutedDimsArray{<:Any,N}, src::AbstractArray{<:Any,N}) where {N} + isempty(src) && return dest checkbounds(dest, axes(src)...) - _copy!(dest, src) + src′ = Base.unalias(dest, src) + _copy!(dest, src′) end -Base.copyto!(dest::PermutedDimsArray, src::AbstractArray) = _copy!(dest, src) function _copy!(P::PermutedDimsArray{T,N,perm}, src) where {T,N,perm} # If dest/src are "close to dense," then it pays to be cache-friendly. diff --git a/base/pkgid.jl b/base/pkgid.jl index 7ef7c58eee4cc..580f52ffbe17b 100644 --- a/base/pkgid.jl +++ b/base/pkgid.jl @@ -38,7 +38,7 @@ end function binunpack(s::String) io = IOBuffer(s) z = read(io, UInt8) - @assert z === 0x00 + @assert z === 0x00 "unexpected data" uuid = read(io, UInt128) name = read(io, String) return PkgId(UUID(uuid), name) diff --git a/base/pointer.jl b/base/pointer.jl index 72c567eaf2a85..bd6d65bd66c8e 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -78,7 +78,7 @@ function unsafe_convert(::Type{Ptr{Cvoid}}, a::GenericMemoryRef{<:Any,T,Core.CPU MemT = typeof(mem) arrayelem = datatype_arrayelem(MemT) elsz = datatype_layoutsize(MemT) - isboxed = 1; isunion = 2 + isunion = 2 if arrayelem == isunion || elsz == 0 offset = UInt(offset) * elsz offset += unsafe_convert(Ptr{Cvoid}, mem) @@ -309,7 +309,8 @@ isless(x::Ptr{T}, y::Ptr{T}) where {T} = x < y ==(x::Ptr, y::Ptr) = UInt(x) == UInt(y) <(x::Ptr, y::Ptr) = UInt(x) < UInt(y) --(x::Ptr, y::Ptr) = UInt(x) - UInt(y) +-(x::Ptr, y::Ptr) = UInt(x) -% UInt(y) +-%(x::Ptr, y::Ptr) = UInt(x) -% UInt(y) +(x::Ptr, y::Integer) = add_ptr(x, (y % UInt) % UInt) -(x::Ptr, y::Integer) = sub_ptr(x, (y % UInt) % UInt) diff --git a/base/precompilation.jl b/base/precompilation.jl index 1b609f3e2015a..79fe6807d63c2 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -1,19 +1,546 @@ module Precompilation -using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names, - project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive, +using Base: CoreLogging, PkgId, UUID, SHA1, StaleCacheKey, parsed_toml, + project_file_manifest_path, get_deps, preferences_names, base_project, isdefined +const Config = Pair{Cmd, Base.CacheFlags} +const PkgConfig = Tuple{PkgId,Config} + +# --- Precompile jobserver (JuliaLang/julia#58591) ---------------------------- +# A token pool of size `ntokens` shared with worker subprocesses that caps total +# CPU threads across all parallel workers: each holds one baseline token while +# CPU-active and its AOT imaging phase draws extra codegen threads from the same +# pool. Only one pool exists per process; a concurrent session (e.g. from package +# loading) whose create fails joins the existing pool instead, holding baselines +# against it without tearing it down. Returns: +# :created -- created and owns the pool (must tear it down) +# :joined -- sharing a pool another session in this process owns +# :none -- no pool available; run uncoordinated +function setup_precompile_jobserver!(ntokens::Int) + namep = ccall(:jl_precompile_jobserver_create, Ptr{UInt8}, (Cint,), ntokens) + if namep != C_NULL + ENV["JULIA_PRECOMPILE_JOBSERVER"] = unsafe_string(namep) + return :created + end + # create failed: join an existing pool, else run uncoordinated (OS refused it) + ccall(:jl_precompile_jobserver_active, Cint, ()) != 0 ? :joined : :none +end + +function teardown_precompile_jobserver!() + ccall(:jl_precompile_jobserver_destroy, Cvoid, ()) + delete!(ENV, "JULIA_PRECOMPILE_JOBSERVER") + return nothing +end + +# Bounds concurrently CPU-active workers. The `Base.Semaphore` enforces the +# process-count cap; when the jobserver is active each worker also holds one +# baseline token from the shared pool while CPU-active, balancing worker +# baselines and imaging codegen threads against one budget. A worker's main +# thread sleeps during imaging, so its baseline token doubles as that phase's +# first codegen thread. +# +# Baseline acquisition is best-effort: it proceeds tokenless when the pool is +# torn down, cancelled, or starved past a timeout (tokens leak if a worker is +# killed mid-imaging), degrading to bounded oversubscription instead of hanging. +# `ntokens` tracks tokens actually held, so releases only post back while it is +# positive and tokenless acquires can never over-post. +# +# Slots are handed out by priority rather than in request order: a package that a +# long chain of other packages is waiting on starts before one nothing depends on, +# so the environment's critical path is not delayed behind leaves that happened to +# become ready first. +mutable struct WorkerLimiter + const cond::Threads.Condition # guards `active`, `seq` and `waiting` + const max::Int + const jobserver::Bool + active::Int + seq::Int + const waiting::Vector{Tuple{Float64,Int}} # (priority, -request order) of tasks waiting for a slot + @atomic ntokens::Int +end +WorkerLimiter(max::Int, jobserver::Bool) = + WorkerLimiter(Threads.Condition(), max, jobserver, 0, 0, Tuple{Float64,Int}[], 0) + +# How long an acquire keeps polling for a baseline token before proceeding +# without one. Generous enough that it is only ever hit when the pool has been +# starved abnormally (leaked tokens), not during routine imaging contention. +const JOBSERVER_BASELINE_TIMEOUT_S = 240.0 + +# Try to take one baseline token from the shared pool, with a yielding backoff +# so the task scheduler keeps running while the pool is drained by imaging +# workers. Returns whether a token was acquired; bails out tokenless when the +# jobserver reports inactive (torn down), `cancel` returns true, or the timeout +# expires. +function _jobserver_acquire_baseline(w::WorkerLimiter; cancel=Returns(false)) + delay = 0.005 + # time_ns is monotonic, unlike time(); UInt64 subtraction is wrap-safe + start_ns = time_ns() + timeout_ns = round(UInt64, JOBSERVER_BASELINE_TIMEOUT_S * 1e9) + while true + r = ccall(:jl_precompile_jobserver_acquire, Cint, ()) + if r == 1 + @atomic w.ntokens += 1 + return true + end + if r == -1 || cancel() + return false + end + if (time_ns() -% start_ns) >= timeout_ns + @debug "Precompilation jobserver baseline token unavailable for $(JOBSERVER_BASELINE_TIMEOUT_S)s; proceeding without one" + return false + end + Base.sleep(delay) + delay = min(2 * delay, 0.1) + end +end + +function _jobserver_release_baseline(w::WorkerLimiter) + while true + old = @atomic w.ntokens + old == 0 && return false + (; success) = @atomicreplace w.ntokens old => old - 1 + if success + ccall(:jl_precompile_jobserver_release, Cvoid, ()) + return true + end + end +end + +function _acquire_slot(w::WorkerLimiter, priority::Float64) + @lock w.cond begin + key = (priority, -(w.seq += 1)) + push!(w.waiting, key) + # admitted when a slot is free and no waiter outranks us + while w.active >= w.max || maximum(w.waiting) != key + wait(w.cond) + end + deleteat!(w.waiting, findfirst(==(key), w.waiting)::Int) + w.active += 1 + end + return nothing +end + +function _release_slot(w::WorkerLimiter) + @lock w.cond begin + w.active -= 1 + notify(w.cond) + end + return nothing +end + +function Base.acquire(w::WorkerLimiter; priority::Float64=0.0, cancel=Returns(false)) + _acquire_slot(w, priority) + if w.jobserver + try + _jobserver_acquire_baseline(w; cancel) + catch + _release_slot(w) + rethrow() + end + end + return nothing +end + +function Base.release(w::WorkerLimiter) + w.jobserver && _jobserver_release_baseline(w) + _release_slot(w) + return nothing +end + +function Base.acquire(f, w::WorkerLimiter; priority::Float64=0.0, cancel=Returns(false)) + Base.acquire(w; priority, cancel) + try + return f() + finally + Base.release(w) + end +end + +## PrecompileJob + +# Per-package compilation status and associated state. +@enum JobStatus::UInt8 begin + JOB_PENDING # not yet started + JOB_STARTED # compilation in progress + JOB_RECOMPILED # successfully compiled + JOB_SOFT_ERROR # compilecache returned an Exception (may work after restart) + JOB_FAILED # hard compile error + JOB_DEP_FAILED # not attempted because a dependency failed to precompile +end + +mutable struct PrecompileJob + status::JobStatus + started_at::Float64 + error_msg::String + output::IOBuffer + pid::Int32 + had_pid::Bool # sticky: true once a subprocess was actually spawned for this job + lock_holder::String + waiting_for_bg::Bool + verbose_timing::String # raw payload from the worker subprocess; surfaced in verbose mode + peak_rss_bytes::UInt64 # max RSS observed by `poll_process_stats!`; 0 on unsupported platforms + PrecompileJob() = new(JOB_PENDING, 0.0, "", IOBuffer(), Int32(0), false, "", false, "", UInt64(0)) +end + +is_pending(j::PrecompileJob) = j.status == JOB_PENDING +is_started(j::PrecompileJob) = j.status == JOB_STARTED +is_recompiled(j::PrecompileJob) = j.status == JOB_RECOMPILED +is_soft_error(j::PrecompileJob) = j.status == JOB_SOFT_ERROR +is_failed(j::PrecompileJob) = j.status == JOB_FAILED +is_dep_failed(j::PrecompileJob) = j.status == JOB_DEP_FAILED +has_pid(j::PrecompileJob) = j.pid > 0 +had_pid(j::PrecompileJob) = j.had_pid +is_locked(j::PrecompileJob) = !isempty(j.lock_holder) +is_waiting(j::PrecompileJob) = j.waiting_for_bg + +mark_started!(j::PrecompileJob, t::Float64=time()) = (j.status = JOB_STARTED; j.started_at = t) +mark_recompiled!(j::PrecompileJob) = (j.status = JOB_RECOMPILED) +mark_soft_error!(j::PrecompileJob) = (j.status = JOB_SOFT_ERROR) +mark_failed!(j::PrecompileJob, msg::String) = (j.status = JOB_FAILED; j.error_msg = msg) +# `root` names the package whose failure caused the skip +mark_dep_failed!(j::PrecompileJob, root::String) = (j.status = JOB_DEP_FAILED; j.error_msg = root) +set_pid!(j::PrecompileJob, pid::Int32) = (j.pid = pid; j.had_pid = true) +clear_pid!(j::PrecompileJob) = (j.pid = Int32(0)) + +function clear_failure!(j::PrecompileJob) + j.status = JOB_PENDING + j.error_msg = "" + truncate(j.output, 0) +end + +# A request to do precompilation work, either in a new session or merged into a running one. +struct PrecompileRequest + pkgs::Union{Vector{String}, Vector{PkgId}} + internal_call::Bool + strict::Bool + warn_loaded::Bool + timing::Bool + _from_loading::Bool + configs::Vector{Config} + io::IOContext + fancyprint::Bool + manifest::Bool + ignore_loaded::Bool + detachable::Bool + skip_dependents::Bool + force::Bool + force_stdlibs::Bool + result::Channel{Any} +end + +# Shared mutable state for a precompilation session. +Base.@kwdef mutable struct PrecompileSession + # Configuration (set once, read-only after construction) + configs::Vector{Config} + io::IOContext + logio::IOContext + logcalls::Union{Nothing, CoreLogging.LogLevel} + fancyprint::Bool + hascolor::Bool + warn_loaded::Bool + ignore_loaded::Bool + internal_call::Bool + strict::Bool + _from_loading::Bool + skip_dependents::Bool + force::Bool + force_stdlibs::Bool + time_start::UInt64 + print_lock::ReentrantLock + parallel_limiter::WorkerLimiter + num_tasks::Int + start_loaded_modules::Set{PkgId} + requested_pkgids::Vector{PkgId} + requested_all::Bool = false # requested_pkgids was defaulted to the project deps + + # Dependency graph (built by setup, extended by drainer) + direct_deps::Dict{PkgId, Vector{PkgId}} + ext_to_parent::Dict{PkgId, PkgId} + parent_to_exts::Dict{PkgId, Vector{PkgId}} + triggers::Dict{PkgId, Vector{PkgId}} + project_deps::Vector{PkgId} + serial_deps::Vector{PkgId} + circular_deps::Vector{PkgId} + + # Progress counters and flags + n_done::Int = 0 + n_already_precomp::Int = 0 + n_loaded::Int = 0 + loaded_pkgs::Vector{PkgId} = PkgId[] + n_total::Int + n_batches::Int = 1 + interrupted::Bool = false + canceled::Bool = false + interrupted_or_done::Bool = false + printloop_should_exit::Bool + target::String + pkg_liveprinted::Union{Nothing, PkgId} = nothing + + # Per-package tracking + jobs::Dict{PkgConfig, PrecompileJob} = Dict{PkgConfig,PrecompileJob}() + was_processed::Dict{PkgConfig, Base.Event} + stale_cache::Dict{StaleCacheKey, Bool} = Dict{StaleCacheKey,Bool}() + cachepath_cache::Dict{PkgId, Vector{String}} = Dict{PkgId,Vector{String}}() + pkg_queue::Vector{PkgConfig} = PkgConfig[] + prev_cpu_times::Dict{Int32, UInt64} = Dict{Int32,UInt64}() + + # Synchronization + first_started::Base.Event = Base.Event() + cache_lock::ReentrantLock = ReentrantLock() + + # Task tracking + tasks::Vector{Task} = Task[] + injected_tasks::Vector{Task} = Task[] + result_waiters::Vector{Task} = Task[] +end + +# Background precompilation state +# +# Lock ordering (outermost first): BG.lock → BG.pkg_done → BG.task_done +# All three have independent ReentrantLocks. Always acquire in this order to avoid deadlocks. +mutable struct BackgroundPrecompileState + task::Union{Nothing, Task} + interrupt_requested::Bool + cancel_requested::Bool + monitoring::Bool + completed_at::Union{Nothing, Float64} + result::Union{Nothing, String} + return_value::Any + exception::Any + lock::ReentrantLock + task_done::Threads.Condition + signal_channels::Vector{Channel{Int32}} # channels for broadcasting signals to all subprocesses + pending_pkgids::Dict{PkgId, Int} # packages queued and currently being precompiled (refcount for multi-config) + completed_pkgids::Set{PkgId} # packages that have finished precompiling (for fast-finish detection) + pkg_done::Threads.Condition # notified when a package finishes precompiling + work_channel::Channel{PrecompileRequest} # channel for injecting requests into running task + verbose::Bool # show PIDs and CPU% for each worker + detachable::Bool # whether the monitor can be detached + confirming::Symbol # :none, :cancel, or :info — action awaiting Enter to confirm + confirm_deadline::Float64 # time() deadline for confirmation + info_requested::Bool # whether SIGINFO/SIGUSR1 has been broadcast at least once + key_listening::Bool # whether a key listener task is currently consuming stdin +end +Base.lock(f, bg::BackgroundPrecompileState) = lock(f, bg.lock) +Base.lock(bg::BackgroundPrecompileState) = lock(bg.lock) +Base.unlock(bg::BackgroundPrecompileState) = unlock(bg.lock) + +const BG = BackgroundPrecompileState(nothing, false, false, false, nothing, nothing, nothing, nothing, ReentrantLock(), Threads.Condition(), Channel{Int32}[], Dict{PkgId, Int}(), Set{PkgId}(), Threads.Condition(), Channel{PrecompileRequest}(Inf), false, false, :none, 0.0, false, false) + +# Serializes the inject-vs-launch decision in `_precompilepkgs` with the launch +# itself. Lock ordering: acquired before (outside) BG.lock, never while holding it. +const launch_lock = ReentrantLock() + +## Constants and formatting utilities + +const ansi_movecol1 = "\e[1G" +const ansi_cleartoend = "\e[0J" +const ansi_cleartoendofline = "\e[0K" +const ansi_enablecursor = "\e[?25h" +const ansi_disablecursor = "\e[?25l" +const ansi_clearline = "\e[2K" +ansi_moveup(n::Int) = string("\e[", n, "A") + +struct PkgPrecompileError <: Exception + msg::String +end +Base.showerror(io::IO, err::PkgPrecompileError) = print(io, err.msg) +Base.showerror(io::IO, err::PkgPrecompileError, bt; kw...) = Base.showerror(io, err) # hide stacktrace + +# This needs a show method to make `julia> err` show nicely +Base.show(io::IO, err::PkgPrecompileError) = print(io, "PkgPrecompileError: ", err.msg) + +can_fancyprint(io::IO) = @something(get(io, :force_fancyprint, nothing), (Base.unwrapcontext(io)[1] isa Base.TTY && (get(ENV, "CI", nothing) != "true"))) + +# The driver prints through one concrete io type whatever stream it was given, so it and +# the closures it spawns are compiled once (into the sysimage) rather than once per +# stream type. A pipe, for example, is otherwise re-inferred by every process that +# captures `Pkg.precompile` output. +unstable_iocontext(io::IOContext{IO}) = io +unstable_iocontext(io::IOContext) = IOContext{IO}(io.io, io.dict) +unstable_iocontext(io::IO) = IOContext{IO}(io) + +function printpkgstyle(io, header, msg; color=:green) + return @lock io begin + printstyled(io, header; color, bold=true) + println(io, " ", msg) + end +end + +timing_string(t) = string(lpad(round(t, digits = 1), 6), " s") + +# Parse the marker payload emitted by `Base.include_package_for_output` and +# format an inline breakdown (include / compilation / image-gen seconds, +# cache file size and cached method count) appended to the per-package timing +# line in verbose mode. Column labels are emitted once via +# `format_verbose_timing_header`. Returns an empty string if the payload is +# missing or unparsable. +function format_verbose_timing(payload::AbstractString, total_seconds::Float64, cache_bytes::Int, peak_rss_bytes::UInt64, hascolor::Bool) + isempty(payload) && return "" + include_ns = compilation_ns = deps_ns = UInt64(0) + methods = 0 + seen = false + for tok in split(payload) + m = match(r"^(include|compilation|deps)_ns=(\d+)$", tok) + if m !== nothing + v = parse(UInt64, m.captures[2]) + tag = m.captures[1] + if tag == "include" + include_ns = v; seen = true + elseif tag == "compilation" + compilation_ns = v + else + deps_ns = v + end + continue + end + m = match(r"^methods=(\d+)$", tok) + m === nothing || (methods = parse(Int, m.captures[1])) + end + seen || return "" + inc_s = include_ns / 1e9 + comp_s = compilation_ns / 1e9 + deps_s = deps_ns / 1e9 + img_s = max(total_seconds - inc_s, 0.0) + dim(s, isz) = isz ? color_string(s, :light_black, hascolor) : s + function col(x) + s = string(round(x, digits = 2)) + dot = findfirst('.', s) + if dot === nothing + s *= ".00" + elseif length(s) - dot == 1 + s *= "0" + end + dim(string(lpad(s, 6), "s"), x < 0.005) + end + cache_mb = cache_bytes / 1024^2 + cache_str = cache_bytes <= 0 ? dim(lpad("-", 8), true) : + string(lpad(round(cache_mb, digits = 1), 6), "MB") + peak_mb = peak_rss_bytes / 1024^2 + peak_str = peak_rss_bytes == 0 ? dim(lpad("-", 7), true) : + string(lpad(round(Int, peak_mb), 5), "MB") + methods_str = dim(lpad(methods, 7), methods == 0) + bar = " │ " + return string(bar, col(inc_s), " ", col(deps_s), " ", col(comp_s), + bar, methods_str, " ", col(img_s), " ", cache_str, " ", peak_str) +end + +# Header that labels the columns produced by `format_verbose_timing`. The leading +# 8 spaces account for `timing_string`'s width so columns line up. +format_verbose_timing_header() = " total │ include (deps) (comp) │ methods img-gen cache ~pk-rss" + +# Sum the on-disk size of the `.ji` cache file and (optionally) its companion +# pkgimage (`.so`/`.dylib`/`.dll`). Returns 0 if files are missing. +function _precompile_cache_bytes(cf_jl::AbstractString, cf_so::Union{Nothing,AbstractString}) + total = 0 + try + isfile(cf_jl) && (total += filesize(cf_jl)) + cf_so === nothing || (isfile(cf_so) && (total += filesize(cf_so))) + catch + end + return total +end + + + +function color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) + if hascolor + enable_ansi = get(Base.text_colors, col, Base.text_colors[:default]) + disable_ansi = get(Base.disable_text_style, col, Base.text_colors[:default]) + return string(enable_ansi, cstr, disable_ansi) + else + return cstr + end +end + +# using Printf +Base.@kwdef mutable struct MiniProgressBar + max::Int = 1 + header::String = "" + color::Symbol = :nothing + width::Int = 40 + current::Int = 0 + prev::Int = 0 + has_shown::Bool = false + time_shown::Float64 = 0.0 + percentage::Bool = true + always_reprint::Bool = false + indent::Int = 4 +end + +const PROGRESS_BAR_TIME_GRANULARITY = Ref(1 / 30.0) # 30 fps +const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1) + +start_progress(io::IO, _::MiniProgressBar) = print(io, ansi_disablecursor) + +function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagereturn=true) + if p.max == 0 + perc = 0.0 + prev_perc = 0.0 + else + perc = p.current / p.max * 100 + prev_perc = p.prev / p.max * 100 + end + # Bail early if we are not updating the progress bar, + # Saves printing to the terminal + if !p.always_reprint && p.has_shown && !((perc - prev_perc) > PROGRESS_BAR_PERCENTAGE_GRANULARITY[]) + return + end + t = time() + if !p.always_reprint && p.has_shown && (t - p.time_shown) < PROGRESS_BAR_TIME_GRANULARITY[] + return + end + p.time_shown = t + p.prev = p.current + p.has_shown = true + + progress_text = string(p.current, "/", p.max) + termwidth = @something termwidth (displaysize(io)::Tuple{Int,Int})[2] + max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) + filled = max_progress_width * clamp(perc / 100, 0.0, 1.0) + (partial_filled, n_filled::Int64) = modf(filled) # get fractional / integer part + n_left = max_progress_width - n_filled + headers = split(p.header, ' ') + to_print = sprint(; context=io) do io + print(io, " "^p.indent) + printstyled(io, headers[1], " "; color=:green, bold=true) + printstyled(io, join(headers[2:end], ' ')) + print(io, " ") + printstyled(io, "━"^n_filled; color=p.color) + if n_left > 0 + if partial_filled > 0.5 + printstyled(io, "╸"; color=p.color) # More filled, use ╸ + else + printstyled(io, "╺"; color=:light_black) # Less filled, use ╺ + end + printstyled(io, "━"^(n_left-1); color=:light_black) + end + printstyled(io, " "; color=:light_black) + print(io, progress_text) + carriagereturn && print(io, "\r") + end + # Print everything in one call + print(io, to_print) +end + +end_progress(io, p::MiniProgressBar) = print(io, ansi_enablecursor, ansi_clearline) + +print_progress_bottom(io::IO) = print(io, "\e[S", ansi_moveup(1), ansi_clearline, ansi_movecol1) + +## ExplicitEnv + # This is currently only used for pkgprecompile but the plan is to use this in code loading in the future # see the `kc/codeloading2.0` branch struct ExplicitEnv path::String - project_deps::Dict{String, UUID} # [deps] in Project.toml - project_weakdeps::Dict{String, UUID} # [weakdeps] in Project.toml - project_extras::Dict{String, UUID} # [extras] in Project.toml - project_extensions::Dict{String, Vector{UUID}} # [exts] in Project.toml - deps::Dict{UUID, Vector{UUID}} # all dependencies in Manifest.toml - weakdeps::Dict{UUID, Vector{UUID}} # all weak dependencies in Manifest.toml + project_deps::Dict{String, UUID} # [deps] in the active project's Project.toml + project_weakdeps::Dict{String, UUID} # [weakdeps] in the active project's Project.toml + project_extras::Dict{String, UUID} # [extras] in the active project's Project.toml + project_extensions::Dict{String, Vector{UUID}} # [extensions] in the active project's Project.toml + workspace_deps::Dict{UUID, String} # packages and [deps] from all workspace member Project.tomls + deps::Dict{UUID, Vector{UUID}} # full dependency graph from Manifest.toml + weakdeps::Dict{UUID, Vector{UUID}} # full weak dependency graph from Manifest.toml extensions::Dict{UUID, Dict{String, Vector{UUID}}} # Lookup name for a UUID names::Dict{UUID, String} @@ -33,6 +560,7 @@ function ExplicitEnv(::Nothing, envpath::String="") Dict{String, UUID}(), # project_weakdeps Dict{String, UUID}(), # project_extras Dict{String, Vector{UUID}}(), # project_extensions + Dict{UUID, String}(), # workspace_deps Dict{UUID, Vector{UUID}}(), # deps Dict{UUID, Vector{UUID}}(), # weakdeps Dict{UUID, Dict{String, Vector{UUID}}}(), # extensions @@ -41,7 +569,7 @@ function ExplicitEnv(::Nothing, envpath::String="") end function ExplicitEnv(envpath::String) # Handle missing project file by creating an empty environment - if !isfile(envpath) + if !isfile(envpath) || project_file_manifest_path(envpath) === nothing envpath = abspath(envpath) return ExplicitEnv(nothing, envpath) end @@ -50,7 +578,7 @@ function ExplicitEnv(envpath::String) # TODO: Perhaps verify that two packages with the same UUID do not have different names? names = Dict{UUID, String}() - project_uuid_to_name = Dict{String, UUID}() + project_name_to_uuid = Dict{String, UUID}() project_deps = Dict{String, UUID}() project_weakdeps = Dict{String, UUID}() @@ -65,8 +593,8 @@ function ExplicitEnv(envpath::String) error() uuid = UUID(_uuid::String) v[name] = uuid - names[UUID(uuid)] = name - project_uuid_to_name[name] = UUID(uuid) + names[uuid] = name + project_name_to_uuid[name] = uuid end end @@ -84,8 +612,8 @@ function ExplicitEnv(envpath::String) project_is_package = proj_name !== nothing && proj_uuid !== nothing if project_is_package # TODO: Error on missing uuid? - project_deps[proj_name] = UUID(proj_uuid) - names[UUID(proj_uuid)] = proj_name + project_deps[proj_name] = proj_uuid + names[proj_uuid] = proj_name end project_extensions = Dict{String, Vector{UUID}}() @@ -98,7 +626,7 @@ function ExplicitEnv(envpath::String) end uuids = UUID[] for trigger in triggers - uuid = get(project_uuid_to_name, trigger, nothing) + uuid = get(project_name_to_uuid, trigger, nothing) if uuid === nothing error("Trigger $trigger for extension $name not found in project") end @@ -239,6 +767,8 @@ function ExplicitEnv(envpath::String) extensions_expanded[pkg] = exts_expanded end + fixup_stdlib_deps!(deps_expanded, weakdeps_expanded, extensions_expanded, names, lookup_strategy) + # Everything that does not yet have a lookup_strategy is missing from the manifest for (_, uuid) in project_deps get!(lookup_strategy, uuid, missing) @@ -260,125 +790,116 @@ function ExplicitEnv(envpath::String) end =# + workspace_deps = collect_workspace_deps(envpath) + return ExplicitEnv(envpath, project_deps, project_weakdeps, project_extras, - project_extensions, deps_expanded, weakdeps_expanded, extensions_expanded, + project_extensions, workspace_deps, + deps_expanded, weakdeps_expanded, extensions_expanded, names, lookup_strategy, #=prefs, local_prefs=#) end -## PROGRESS BAR +function collect_workspace_deps(project_file::String) + while true + base = base_project(project_file) + base === nothing && break + project_file = base + end -# using Printf -Base.@kwdef mutable struct MiniProgressBar - max::Int = 1.0 - header::String = "" - color::Symbol = :nothing - width::Int = 40 - current::Int = 0.0 - prev::Int = 0.0 - has_shown::Bool = false - time_shown::Float64 = 0.0 - percentage::Bool = true - always_reprint::Bool = false - indent::Int = 4 + workspace_deps = Dict{UUID, String}() + collect_workspace_deps!(workspace_deps, Set{String}(), project_file) + return workspace_deps end -const PROGRESS_BAR_TIME_GRANULARITY = Ref(1 / 30.0) # 30 fps -const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1) - -function start_progress(io::IO, _::MiniProgressBar) - ansi_disablecursor = "\e[?25l" - print(io, ansi_disablecursor) -end +function collect_workspace_deps!(workspace_deps::Dict{UUID, String}, seen::Set{String}, project_file::String) + project_file = abspath(project_file) + project_file in seen && return + push!(seen, project_file) -function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagereturn=true) - if p.max == 0 - perc = 0.0 - prev_perc = 0.0 - else - perc = p.current / p.max * 100 - prev_perc = p.prev / p.max * 100 - end - # Bail early if we are not updating the progress bar, - # Saves printing to the terminal - if !p.always_reprint && p.has_shown && !((perc - prev_perc) > PROGRESS_BAR_PERCENTAGE_GRANULARITY[]) - return + project = parsed_toml(project_file) + for (name, _uuid) in get(Dict{String, Any}, project, "deps")::Dict{String, Any} + workspace_deps[UUID(_uuid::String)] = name end - t = time() - if !p.always_reprint && p.has_shown && (t - p.time_shown) < PROGRESS_BAR_TIME_GRANULARITY[] - return + + name = get(project, "name", nothing)::Union{String, Nothing} + _uuid = get(project, "uuid", nothing)::Union{String, Nothing} + if name !== nothing && _uuid !== nothing + workspace_deps[UUID(_uuid)] = name end - p.time_shown = t - p.prev = p.current - p.has_shown = true - progress_text = if false # p.percentage - # @sprintf "%2.1f %%" perc - else - string(p.current, "/", p.max) + workspace = get(project, "workspace", nothing)::Union{Dict{String, Any}, Nothing} + workspace === nothing && return + projects = get(workspace, "projects", nothing)::Union{Vector{String}, Nothing, String} + projects isa Vector || return + for member in projects + member_file = Base.env_project_file(joinpath(dirname(project_file), member)) + member_file isa String || continue + collect_workspace_deps!(workspace_deps, seen, member_file) end - termwidth = @something termwidth (displaysize(io)::Tuple{Int,Int})[2] - max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) - n_filled = floor(Int, max_progress_width * perc / 100) - partial_filled = (max_progress_width * perc / 100) - n_filled - n_left = max_progress_width - n_filled - headers = split(p.header, ' ') - to_print = sprint(; context=io) do io - print(io, " "^p.indent) - printstyled(io, headers[1], " "; color=:green, bold=true) - printstyled(io, join(headers[2:end], ' ')) - print(io, " ") - printstyled(io, "━"^n_filled; color=p.color) - if n_left > 0 - if partial_filled > 0.5 - printstyled(io, "╸"; color=p.color) # More filled, use ╸ - else - printstyled(io, "╺"; color=:light_black) # Less filled, use ╺ + return +end + +# A manifest resolved by a different Julia version can record stale information for +# stdlibs: a dependency or extension the stdlib gained later, a dependency that is not in +# the manifest at all, or a git-tree-sha1 from when the package was not a stdlib yet. +# Code loading tolerates this by falling back to the stdlib's own Project.toml (see +# `Base.identify_stdlib_project_dep` and `Base.insert_extension_triggers`), so the +# dependency graph must include those edges too, otherwise a missing dependency is never +# precompiled before the stdlib that needs it and the strict precompile worker fails. +function fixup_stdlib_deps!(deps::Dict{UUID, Vector{UUID}}, weakdeps::Dict{UUID, Vector{UUID}}, + extensions::Dict{UUID, Dict{String, Vector{UUID}}}, names::Dict{UUID, String}, + lookup_strategy::Dict{UUID, Union{SHA1, String, Nothing, Missing}}) + stdlib_names = Set(readdir(Sys.STDLIB)) + stack = collect(keys(lookup_strategy)) + while !isempty(stack) + uuid = pop!(stack) + name = names[uuid] + # same check as `Base.is_stdlib`, but keeping the parsed Project.toml + name in stdlib_names || continue + project_file = Base.locate_project_file(joinpath(Sys.STDLIB, name)) + project_file isa String || continue + project_d = parsed_toml(project_file) + project_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} + (project_uuid !== nothing && UUID(project_uuid) == uuid) || continue + project_deps = get(Dict{String, Any}, project_d, "deps")::Dict{String, Any} + project_weakdeps = get(Dict{String, Any}, project_d, "weakdeps")::Dict{String, Any} + project_extensions = get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} + pkg_deps = get!(Vector{UUID}, deps, uuid) + for (dep_name, _dep_uuid) in project_deps + dep_uuid = UUID(_dep_uuid::String) + dep_uuid in pkg_deps && continue + push!(pkg_deps, dep_uuid) + if !haskey(lookup_strategy, dep_uuid) + # not in the manifest at all, so it is loaded as a stdlib + names[dep_uuid] = dep_name + lookup_strategy[dep_uuid] = nothing + push!(stack, dep_uuid) end - printstyled(io, "━"^(n_left-1); color=:light_black) end - printstyled(io, " "; color=:light_black) - print(io, progress_text) - carriagereturn && print(io, "\r") + pkg_weakdeps = get!(Vector{UUID}, weakdeps, uuid) + for (dep_name, _dep_uuid) in project_weakdeps + dep_uuid = UUID(_dep_uuid::String) + dep_uuid in pkg_weakdeps && continue + push!(pkg_weakdeps, dep_uuid) + get!(names, dep_uuid, dep_name) + end + pkg_extensions = get!(Dict{String, Vector{UUID}}, extensions, uuid) + for (ext, triggers) in project_extensions + haskey(pkg_extensions, ext) && continue + triggers = triggers isa String ? [triggers] : triggers::Vector{String} + trigger_uuids = UUID[] + for trigger in triggers + _trigger_uuid = get(project_weakdeps, trigger, get(project_deps, trigger, nothing))::Union{String, Nothing} + _trigger_uuid === nothing && break + push!(trigger_uuids, UUID(_trigger_uuid)) + end + length(trigger_uuids) == length(triggers) || continue + pkg_extensions[ext] = trigger_uuids + end end - # Print everything in one call - print(io, to_print) -end - -function end_progress(io, p::MiniProgressBar) - ansi_enablecursor = "\e[?25h" - ansi_clearline = "\e[2K" - print(io, ansi_enablecursor * ansi_clearline) -end - -function print_progress_bottom(io::IO) - ansi_clearline = "\e[2K" - ansi_movecol1 = "\e[1G" - ansi_moveup(n::Int) = string("\e[", n, "A") - print(io, "\e[S" * ansi_moveup(1) * ansi_clearline * ansi_movecol1) -end - - -############ -struct PkgPrecompileError <: Exception - msg::String -end -Base.showerror(io::IO, err::PkgPrecompileError) = print(io, err.msg) -Base.showerror(io::IO, err::PkgPrecompileError, bt; kw...) = Base.showerror(io, err) # hide stacktrace - -# This needs a show method to make `julia> err` show nicely -Base.show(io::IO, err::PkgPrecompileError) = print(io, "PkgPrecompileError: ", err.msg) - -import Base: StaleCacheKey - -can_fancyprint(io::IO) = @something(get(io, :force_fancyprint, nothing), (io isa Base.TTY && (get(ENV, "CI", nothing) != "true"))) - -function printpkgstyle(io, header, msg; color=:green) - printstyled(io, header; color, bold=true) - println(io, " ", msg) + return deps end -const Config = Pair{Cmd, Base.CacheFlags} -const PkgConfig = Tuple{PkgId,Config} +## Dependency graph # name or parent → ext function full_name(ext_to_parent::Dict{PkgId, PkgId}, pkg::PkgId) @@ -389,9 +910,10 @@ function full_name(ext_to_parent::Dict{PkgId, PkgId}, pkg::PkgId) end end -function excluded_circular_deps_explanation(io::IOContext{IO}, ext_to_parent::Dict{PkgId, PkgId}, circular_deps, cycles) +function excluded_circular_deps_explanation(io::IOContext, ext_to_parent::Dict{PkgId, PkgId}, circular_deps, cycles) outer_deps = copy(circular_deps) cycles_names = "" + hascolor = get(io, :color, false)::Bool for cycle in cycles filter!(!in(cycle), outer_deps) cycle_str = "" @@ -406,8 +928,7 @@ function excluded_circular_deps_explanation(io::IOContext{IO}, ext_to_parent::Di else line = " └" * "─" ^j * " " end - hascolor = get(io, :color, false)::Bool - line = _color_string(line, :light_black, hascolor) * full_name(ext_to_parent, pkg) * "\n" + line = color_string(line, :light_black, hascolor) * full_name(ext_to_parent, pkg) * "\n" cycle_str *= line end cycles_names *= cycle_str @@ -470,102 +991,53 @@ function collect_all_deps(direct_deps, dep, alldeps=Set{Base.PkgId}()) return alldeps end +function visit_indirect_deps!(direct_deps::Dict{PkgId, Vector{PkgId}}, visited::Set{PkgId}, + node::PkgId, all_deps::Set{PkgId}) + if node in visited + return + end + push!(visited, node) + for dep in get(Set{PkgId}, direct_deps, node) + if !(dep in all_deps) + push!(all_deps, dep) + visit_indirect_deps!(direct_deps, visited, dep, all_deps) + end + end + return +end -function precompilepkgs(pkgs::Vector{String}=String[]; - internal_call::Bool=false, - strict::Bool = false, - warn_loaded::Bool = true, - timing::Bool = false, - _from_loading::Bool=false, - configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), - io::IO=stderr, - # asking for timing disables fancy mode, as timing is shown in non-fancy mode - fancyprint::Bool = can_fancyprint(io) && !timing, - manifest::Bool=false, - ignore_loaded::Bool=true) - @debug "precompilepkgs called with" pkgs internal_call strict warn_loaded timing _from_loading configs fancyprint manifest ignore_loaded - # monomorphize this to avoid latency problems - _precompilepkgs(pkgs, internal_call, strict, warn_loaded, timing, _from_loading, - configs isa Vector{Config} ? configs : [configs], - IOContext{IO}(io), fancyprint, manifest, ignore_loaded) -end - -function _precompilepkgs(pkgs::Vector{String}, - internal_call::Bool, - strict::Bool, - warn_loaded::Bool, - timing::Bool, - _from_loading::Bool, - configs::Vector{Config}, - _io::IOContext{IO}, - fancyprint::Bool, - manifest::Bool, - ignore_loaded::Bool) - requested_pkgs = copy(pkgs) # for understanding user intent - - time_start = time_ns() - - env = ExplicitEnv() - - # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Issue #2323 - # TODO: Investigate why this happens in windows and restore the full task limit - default_num_tasks = Sys.iswindows() ? div(Sys.CPU_THREADS::Int, 2) + 1 : Sys.CPU_THREADS::Int + 1 - default_num_tasks = min(default_num_tasks, 16) # limit for better stability on shared resource systems - - num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) - parallel_limiter = Base.Semaphore(num_tasks) - - # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests - io = (_from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false)) ? IOContext{IO}(devnull) : _io - - - nconfigs = length(configs) - hascolor = get(io, :color, false)::Bool - color_string(cstr::String, col::Union{Int64, Symbol}) = _color_string(cstr, col, hascolor) - - stale_cache = Dict{StaleCacheKey, Bool}() - cachepath_cache = Dict{PkgId, Vector{String}}() - - # a map from packages/extensions to their direct deps - direct_deps = Dict{Base.PkgId, Vector{Base.PkgId}}() - # a map from parent → extension, including all extensions that are loadable - # in the current environment (i.e. their triggers are present) - parent_to_exts = Dict{Base.PkgId, Vector{Base.PkgId}}() - # inverse map of `parent_to_ext` above (ext → parent) - ext_to_parent = Dict{Base.PkgId, Base.PkgId}() - - function describe_pkg(pkg::PkgId, is_project_dep::Bool, is_serial_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) - name = full_name(ext_to_parent, pkg) - name = is_project_dep ? name : color_string(name, :light_black) - if is_serial_dep - name *= color_string(" (serial)", :light_black) - end - if nconfigs > 1 && !isempty(flags) - config_str = join(flags, " ") - name *= color_string(" `$config_str`", :light_black) - end - if nconfigs > 1 - config_str = join(Base.translate_cache_flags(cacheflags, Base.DefaultCacheFlags), " ") - name *= color_string(" $config_str", :light_black) - end - return name +# Build dependency graph from an ExplicitEnv. +# Returns a NamedTuple of (direct_deps, ext_to_parent, parent_to_exts, triggers, project_deps, serial_deps). +function build_dep_graph(env::ExplicitEnv, manifest::Bool, _from_loading::Bool, requested_pkgids::Vector{PkgId}) + direct_deps = Dict{PkgId, Vector{PkgId}}() + parent_to_exts = Dict{PkgId, Vector{PkgId}}() + ext_to_parent = Dict{PkgId, PkgId}() + triggers = Dict{PkgId, Vector{PkgId}}() + + # Determine which packages to consider for precompilation by walking + # transitive dependencies from the appropriate roots. + # `manifest` controls the scope: workspace_deps (all members) vs project_deps (current project). + root_uuids = manifest ? keys(env.workspace_deps) : values(env.project_deps) + pkg_uuids = Set{UUID}() + for uuid in root_uuids + _collect_reachable!(pkg_uuids, env.deps, uuid) end - triggers = Dict{Base.PkgId,Vector{Base.PkgId}}() - for (dep, deps) in env.deps - pkg = Base.PkgId(dep, env.names[dep]) + for dep in pkg_uuids + haskey(env.deps, dep) || continue + pkg = PkgId(dep, env.names[dep]) Base.in_sysimage(pkg) && continue - deps = [Base.PkgId(x, env.names[x]) for x in deps] + deps = [PkgId(x, env.names[x]) for x in env.deps[dep]] direct_deps[pkg] = filter!(!Base.in_sysimage, deps) - for (ext_name, trigger_uuids) in env.extensions[dep] + for (ext_name, trigger_uuids) in get(Dict{String, Vector{UUID}}, env.extensions, dep) ext_uuid = Base.uuid5(pkg.uuid, ext_name) - ext = Base.PkgId(ext_uuid, ext_name) - triggers[ext] = Base.PkgId[pkg] # depends on parent package + ext = PkgId(ext_uuid, ext_name) + triggers[ext] = PkgId[pkg] all_triggers_available = true for trigger_uuid in trigger_uuids - trigger_name = env.names[trigger_uuid] - if trigger_uuid in keys(env.deps) - push!(triggers[ext], Base.PkgId(trigger_uuid, trigger_name)) + trigger_name = PkgId(trigger_uuid, env.names[trigger_uuid]) + if trigger_uuid in pkg_uuids || Base.in_sysimage(trigger_name) + push!(triggers[ext], trigger_name) else all_triggers_available = false break @@ -574,647 +1046,2195 @@ function _precompilepkgs(pkgs::Vector{String}, all_triggers_available || continue ext_to_parent[ext] = pkg direct_deps[ext] = filter(!Base.in_sysimage, triggers[ext]) - if !haskey(parent_to_exts, pkg) - parent_to_exts[pkg] = Base.PkgId[ext] + parent_to_exts[pkg] = PkgId[ext] else push!(parent_to_exts[pkg], ext) end end end - project_deps = [ - Base.PkgId(uuid, name) - for (name, uuid) in env.project_deps if !Base.in_sysimage(Base.PkgId(uuid, name)) - ] - + project_deps = PkgId[] + if manifest + for (uuid, name) in env.workspace_deps + push!(project_deps, PkgId(uuid, name)) + end + else + for (name, uuid) in env.project_deps + push!(project_deps, PkgId(uuid, name)) + end + end + filter!(!Base.in_sysimage, project_deps) # consider exts of project deps to be project deps so that errors are reported - append!(project_deps, keys(filter(d->last(d).name in keys(env.project_deps), ext_to_parent))) - - @debug "precompile: deps collected" + append!(project_deps, keys(filter(d -> last(d) in project_deps, ext_to_parent))) # An extension effectively depends on another extension if it has a strict superset of its triggers for ext_a in keys(ext_to_parent) for ext_b in keys(ext_to_parent) if triggers[ext_a] ⊋ triggers[ext_b] + push!(triggers[ext_a], ext_b) push!(direct_deps[ext_a], ext_b) end end end # A package depends on an extension if it (indirectly) depends on all extension triggers - function expand_indirect_dependencies(direct_deps) - function visit!(visited, node, all_deps) - if node in visited - return - end - push!(visited, node) - for dep in get(Set{Base.PkgId}, direct_deps, node) - if !(dep in all_deps) - push!(all_deps, dep) - visit!(visited, dep, all_deps) - end - end - end - - local indirect_deps = Dict{Base.PkgId, Set{Base.PkgId}}() + # Iterate to a fixed point because adding an extension edge (e.g. ExtA → TopPkg) + # may cause another extension (e.g. ExtAB, which depends on ExtA) to become + # loadable in TopPkg on the next iteration. + changed = true + while changed + changed = false + indirect_deps = Dict{PkgId, Set{PkgId}}() for package in keys(direct_deps) - # Initialize a set to keep track of all dependencies for 'package' - all_deps = Set{Base.PkgId}() - visited = Set{Base.PkgId}() - visit!(visited, package, all_deps) - # Update direct_deps with the complete set of dependencies for 'package' + all_deps = Set{PkgId}() + visited = Set{PkgId}() + visit_indirect_deps!(direct_deps, visited, package, all_deps) indirect_deps[package] = all_deps end - return indirect_deps - end - - # this loop must be run after the full direct_deps map has been populated - indirect_deps = expand_indirect_dependencies(direct_deps) - for ext in keys(ext_to_parent) - ext_loadable_in_pkg = Dict{Base.PkgId,Bool}() - for pkg in keys(direct_deps) - is_trigger = in(pkg, direct_deps[ext]) - is_extension = in(pkg, keys(ext_to_parent)) - has_triggers = issubset(direct_deps[ext], indirect_deps[pkg]) - ext_loadable_in_pkg[pkg] = !is_extension && has_triggers && !is_trigger - end - for (pkg, ext_loadable) in ext_loadable_in_pkg - if ext_loadable && !any((dep)->ext_loadable_in_pkg[dep], direct_deps[pkg]) - # add an edge if the extension is loadable by pkg, and was not loadable in any - # of the pkg's dependencies - push!(direct_deps[pkg], ext) + for ext in keys(ext_to_parent) + ext_loadable_in_pkg = Dict{PkgId,Bool}() + for pkg in keys(direct_deps) + is_trigger = in(pkg, direct_deps[ext]) + is_extension = in(pkg, keys(ext_to_parent)) + has_triggers = issubset(direct_deps[ext], indirect_deps[pkg]) + ext_loadable_in_pkg[pkg] = !is_extension && has_triggers && !is_trigger + end + for (pkg, ext_loadable) in ext_loadable_in_pkg + if ext_loadable && !any((dep)->ext_loadable_in_pkg[dep], direct_deps[pkg]) + if ext ∉ direct_deps[pkg] + # add an edge if the extension is loadable by pkg, and was not loadable in any + # of the pkg's dependencies + push!(direct_deps[pkg], ext) + changed = true + end + end end end end - @debug "precompile: extensions collected" - - serial_deps = Base.PkgId[] # packages that are being precompiled in serial + serial_deps = PkgId[] if _from_loading - # if called from loading precompilation it may be a package from another environment stack - # where we don't have access to the dep graph, so just add as a single package and do serial - # precompilation of its deps within the job. - for pkg in requested_pkgs # In case loading asks for multiple packages - pkgid = Base.identify_package(pkg) + for pkgid in requested_pkgids pkgid === nothing && continue if !haskey(direct_deps, pkgid) @debug "precompile: package `$(pkgid)` is outside of the environment, so adding as single package serial job" - direct_deps[pkgid] = Base.PkgId[] # no deps, do them in serial in the job - push!(project_deps, pkgid) # add to project_deps so it doesn't show up in gray + direct_deps[pkgid] = PkgId[] + push!(project_deps, pkgid) push!(serial_deps, pkgid) end end end - # return early if no deps - if isempty(direct_deps) - if isempty(pkgs) - return - else - error("No direct dependencies outside of the sysimage found matching $(pkgs)") - end - end - - # initialize signalling - started = Dict{PkgConfig,Bool}() - was_processed = Dict{PkgConfig,Base.Event}() - was_recompiled = Dict{PkgConfig,Bool}() - for config in configs - for pkgid in keys(direct_deps) - pkg_config = (pkgid, config) - started[pkg_config] = false - was_processed[pkg_config] = Base.Event() - was_recompiled[pkg_config] = false - end - end - @debug "precompile: signalling initialized" - - # find and guard against circular deps - cycles = Vector{Base.PkgId}[] - # For every scanned package, true if pkg found to be in a cycle - # or depends on packages in a cycle and false otherwise. - could_be_cycle = Dict{Base.PkgId, Bool}() - # temporary stack for the SCC-like algorithm below - stack = Base.PkgId[] + return (; direct_deps, ext_to_parent, parent_to_exts, triggers, project_deps, serial_deps) +end - # set of packages that depend on a cycle (either because they are - # a part of a cycle themselves or because they transitively depend - # on a package in some cycle) - circular_deps = Base.PkgId[] +# Detect circular dependencies and notify their Events so waiting tasks can skip them. +# Returns the list of packages involved in or dependent on cycles. +function detect_circular_deps!(direct_deps, serial_deps, was_processed, io, ext_to_parent) + cycles = Vector{PkgId}[] + could_be_cycle = Dict{PkgId, Bool}() + stack = PkgId[] + circular_deps = PkgId[] for pkg in keys(direct_deps) @assert isempty(stack) + pkg in serial_deps && continue if scan_pkg!(stack, could_be_cycle, cycles, pkg, direct_deps) push!(circular_deps, pkg) - for pkg_config in keys(was_processed) - # notify all to allow skipping - pkg_config[1] == pkg && notify(was_processed[pkg_config]) + for (pkg_config, evt) in was_processed + pkg_config[1] == pkg && notify(evt) end end end if !isempty(circular_deps) - @warn excluded_circular_deps_explanation(io, ext_to_parent, circular_deps, cycles) + printpkgstyle(io, :Warning, excluded_circular_deps_explanation(io, ext_to_parent, circular_deps, cycles), color=Base.warn_color()) end - @debug "precompile: circular dep check done" + return circular_deps +end - if !manifest - if isempty(pkgs) - pkgs = [pkg.name for pkg in project_deps] +# Filter the dependency graph to only include requested packages and their transitive deps. +# Returns true if the graph became empty (caller should return early). +function filter_dep_graph!(direct_deps, pkg_names, ext_to_parent, requested_pkgids) + isempty(pkg_names) && return false + keep = Set{PkgId}() + for dep_pkgid in keys(direct_deps) + if dep_pkgid.name in pkg_names + push!(keep, dep_pkgid) + collect_all_deps(direct_deps, dep_pkgid, keep) end - keep = Set{Base.PkgId}() - for dep in direct_deps - dep_pkgid = first(dep) - if dep_pkgid.name in pkgs - push!(keep, dep_pkgid) - collect_all_deps(direct_deps, dep_pkgid, keep) - end + end + for requested_pkgid in requested_pkgids + if haskey(direct_deps, requested_pkgid) + push!(keep, requested_pkgid) + collect_all_deps(direct_deps, requested_pkgid, keep) end - for ext in keys(ext_to_parent) - if issubset(collect_all_deps(direct_deps, ext), keep) # if all extension deps are kept - push!(keep, ext) - end + end + for ext in keys(ext_to_parent) + if issubset(collect_all_deps(direct_deps, ext), keep) + push!(keep, ext) + end + end + filter!(d->in(first(d), keep), direct_deps) + return isempty(direct_deps) +end + +## Public API + +""" + precompilepkgs(pkgs; kwargs...) + +Precompile packages and their dependencies, with support for parallel compilation, +progress tracking, and various compilation configurations. + +`pkgs::Union{Vector{String}, Vector{PkgId}}`: Packages to precompile. When +empty (default), precompiles all project dependencies. When specified, +precompiles only the given packages and their dependencies (unless +`manifest=true`). + +!!! note + Errors will only throw when precompiling the top-level dependencies, given that + not all manifest dependencies may be loaded by the top-level dependencies on the given system. + This can be overridden to make errors in all dependencies throw by setting the kwarg `strict` to `true` + +# Keyword Arguments +- `internal_call::Bool`: Indicates this is an automatic precompilation call + from somewhere external (e.g. Pkg). Do not use this parameter. + +- `strict::Bool`: Controls error reporting scope. When `false` (default), only reports + errors for direct project dependencies. Only relevant when `manifest=true`. + +- `warn_loaded::Bool`: When `true` (default), checks for and warns about packages that are + precompiled but already loaded with a different version. Displays a warning that Julia + needs to be restarted to use the newly precompiled versions. + +- `timing::Bool`: When `true` (not default), displays timing information for + each package compilation, but only if compilation might have succeeded. + Disables fancy progress bar output (timing is shown in simple text mode). + +- `verbose::Bool`: When `true` (not default), enables verbose timing mode: implies + `timing=true` and appends a per-package breakdown to each completion line with + these columns: + * `total` — total wall-clock time for the worker subprocess. + * `include` — time spent in `Base.include` of the package source. + * `(deps)` — subset of `include`: time spent loading already-precompiled + dependencies from disk (via `require`). + * `(comp)` — subset of `include`: cumulative compile time (type inference + and code generation) reported by `cumulative_compile_time_ns`. + * `methods` — number of newly-inferred methods cached for this package. + * `img-gen` — `total - include`: post-include work (native-code generation, + serialization, writing the `.ji` and pkgimage to disk). + * `cache` — combined on-disk size of the `.ji` cache and any pkgimage. + * `~pk-rss` — approximate peak resident set size of the worker subprocess. + The leading `~` indicates the value is sampled (every ~0.5 s) + rather than observed continuously, so transient peaks between + samples may be missed. Linux/macOS only, `-` elsewhere. + Values under 5 ms and zero counts are dimmed for readability. + +- `_from_loading::Bool`: Internal flag indicating the call originated from the + package loading system. When `true` (not default): returns early instead of + throwing when packages are not found; suppresses progress messages when not + in an interactive session; allows packages outside the current environment to + be added as serial precompilation jobs; skips LOADING_CACHE initialization; + and changes cachefile locking behavior. + +- `configs::Union{Config,Vector{Config}}`: Compilation configurations to use. Each Config + is a `Pair{Cmd, Base.CacheFlags}` specifying command flags and cache flags. When + multiple configs are provided, each package is precompiled for each configuration. + +- `io::IO`: Output stream for progress messages, warnings, and errors. Can be + redirected (e.g., to `devnull` when called from loading in non-interactive mode). + +- `fancyprint::Bool`: Controls output format. When `true`, displays an animated progress + bar with spinners. When `false`, instead enables `timing` mode. Automatically + disabled when `timing=true` or when called from loading in non-interactive mode. + +- `manifest::Bool`: Controls the scope of packages to precompile. When `false` (default), + precompiles only packages specified in `pkgs` and their dependencies. When `true`, + precompiles all packages in the manifest (workspace mode), typically used by Pkg for + workspace precompile requests. + +- `ignore_loaded::Bool`: Controls whether already-loaded packages affect cache + freshness checks. When `false` (not default), loaded package versions are considered when + determining if cache files are fresh. + +- `detachable::Bool`: When `true` (not default), allows detaching from the + precompilation monitor with the `d` key, letting precompilation continue in + the background. The monitor can be reattached later via + [`Base.Precompilation.monitor_background_precompile`](@ref). Pkg.jl passes + `detachable=true` in interactive sessions. + +- `skip_dependents::Bool`: When `true` (default), packages that depend on a package + which failed to precompile are not attempted, since loading the failed dependency + would fail again. Set to `false` to attempt them anyway, for example when a package + only loads that dependency on some platforms. Extensions always load their parent + and triggers, so they are never attempted when one of those failed, regardless of + this setting. + +- `force::Bool`: When `true` (not default), recompiles packages whose cache files are + already fresh. Standard libraries are left alone. + +# Keyboard Controls + +When running interactively in a TTY, the following keys are available during +precompilation: + + - **`c`** — Cancel precompilation. Prompts for Enter to confirm; ignored after + 5 seconds or if any other key is pressed. + - **`d`/`q`/`]`** — Detach (only when `detachable=true`). Returns to the REPL while + precompilation continues in the background. + - **`i`** — Info. Sends a profiling signal (SIGINFO on macOS/BSD, SIGUSR1 on + Linux) to subprocesses, triggering a profile peek without interrupting + compilation. Prompts for Enter to confirm; ignored after 5 seconds or if any + other key is pressed. + - **`v`** — Toggle verbose mode. Shows elapsed time and worker PID for each actively + compiling package, plus CPU% and memory (RSS) on Linux and macOS. + - **`?`/`h`** — Show keyboard shortcut help. + - **Ctrl-C** — Interrupt. Sends SIGINT to subprocesses and displays their output. + +# Return +- `Vector{String}`: Paths to cache files for the requested packages. +- `Nothing`: precompilation should be skipped + +# Notes +- Packages in circular dependency cycles are skipped with a warning. +- Packages with `__precompile__(false)` are skipped if they are from loading to + avoid repeated work on every session. +- Parallel compilation is controlled by `JULIA_NUM_PRECOMPILE_TASKS` environment variable + (defaults to CPU_THREADS + 1, capped at 16, halved on Windows). The total CPU-thread + budget shared across those workers (worker baselines plus native-image codegen threads) + is controlled by `JULIA_PRECOMPILE_THREADS` (defaults to CPU_THREADS + 1). +- Extensions are precompiled when all their triggers are available in the environment. +""" +# Include only cache files that are ready for workers. +function preresolved_snapshot(s::PrecompileSession) + @lock s.cache_lock Pair{Base.PkgId,String}[k => first(v) for (k, v) in s.cachepath_cache if !isempty(v)] +end + +function precompilepkgs(pkgs::Union{Vector{String}, Vector{PkgId}}=String[]; + internal_call::Bool=false, + strict::Bool = false, + warn_loaded::Bool = true, + timing::Bool = false, + verbose::Bool = false, + _from_loading::Bool=false, + configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), + io::IO=stderr, + # asking for timing disables fancy mode, as timing is shown in non-fancy mode; + # verbose implies timing (see below), so also disables fancy mode + fancyprint::Bool = can_fancyprint(io) && !timing && !verbose, + manifest::Bool=false, + ignore_loaded::Bool=true, + detachable::Bool=false, + skip_dependents::Bool=true, + force::Bool=false, + # undocumented: `force` for stdlibs too. Warning: their new cache files land in + # the user depot rather than the one julia ships them in, and shadow those from then on + force_stdlibs::Bool=false) + # verbose timing mode requires timing to be enabled (per-package breakdown + # is only shown alongside timing lines in non-fancy mode) + verbose && (timing = true) + force_stdlibs && (force = true) + @debug "precompilepkgs called with" pkgs internal_call strict warn_loaded timing verbose _from_loading configs fancyprint manifest ignore_loaded detachable skip_dependents force force_stdlibs + # monomorphize this to avoid latency problems + _precompilepkgs(pkgs, internal_call, strict, warn_loaded, timing, verbose, _from_loading, + configs isa Vector{Config} ? configs : [configs], + unstable_iocontext(io), fancyprint, manifest, ignore_loaded, detachable, + skip_dependents, force, force_stdlibs) +end + +""" + precompile_for_loading(into::Module, names::Vector{Symbol}) + +Look-ahead for a `using`/`import` statement that is about to load the packages +`names` into `into`. Precompiles in one parallel session every package of the +statement that is not loaded yet and has no usable cache, together with the +extensions that become loadable once they all are (including extensions whose +other triggers are already loaded). Each subsequent `require` then finds a fresh +cache instead of starting its own session per package, and extensions do not +trail as extra sessions after their triggers load. + +Any failure is left for `require` to report: this only does work `require` +would otherwise do itself, one package at a time. +""" +function precompile_for_loading(into::Module, names::Vector{Symbol}) + Base.JLOptions().use_compiled_modules == 1 || return + Base.generating_output() && return + Base.disable_parallel_precompile && return + try + _precompile_for_loading(into, names) + catch err + err isa Union{InterruptException, Base.CancellationRequest} && rethrow() + @debug "Look-ahead precompilation failed, leaving it to `require`" exception=(err, catch_backtrace()) + end + return +end + +function _precompile_for_loading(into::Module, names::Vector{Symbol}) + pkgs = PkgId[] + for name in names + pkg = Base.identify_package(into, String(name)) + pkg === nothing && continue + Base.root_module_exists(pkg) && continue + pkg in pkgs || push!(pkgs, pkg) + end + isempty(pkgs) && return + + # Extensions the statement makes loadable: the parent is reachable from what is + # being loaded or already loaded, at least one trigger is being loaded, and every + # trigger is reachable, already loaded, or in the sysimage. + env = ExplicitEnv() + reachable = Set{UUID}() + for pkg in pkgs + pkg.uuid === nothing && continue + _collect_reachable!(reachable, env.deps, pkg.uuid) + end + loaded = @lock Base.require_lock Set{UUID}(pkg.uuid::UUID for pkg in keys(Base.loaded_modules) if pkg.uuid !== nothing) + available(uuid::UUID) = uuid in reachable || uuid in loaded || + (haskey(env.names, uuid) && Base.in_sysimage(PkgId(uuid, env.names[uuid]))) + ext_parents = Dict{PkgId, PkgId}() + for uuid in Iterators.flatten((reachable, loaded)) + for (ext_name, trigger_uuids) in get(Dict{String, Vector{UUID}}, env.extensions, uuid) + all(available, trigger_uuids) || continue + (uuid in reachable || any(in(reachable), trigger_uuids)) || continue + ext = PkgId(Base.uuid5(uuid, ext_name), ext_name) + (Base.root_module_exists(ext) || haskey(ext_parents, ext)) && continue + push!(pkgs, ext) + ext_parents[ext] = PkgId(uuid, env.names[uuid]) + end + end + # a single package gets no benefit over the session `require` starts itself + length(pkgs) > 1 || return + + stale = PkgId[] + reasons = Dict{Symbol,Int}() + stale_cache = Dict{Base.StaleCacheKey,Bool}() + cachepath_cache = Dict{PkgId, Vector{String}}() + for pkg in pkgs + fresh = try + Base.compilecache_freshest_path(pkg; ignore_loaded=false, stale_cache, cachepath_cache, + verify_checksums=false, reasons) !== nothing + catch + true # e.g. no source located: leave it to `require` to report + end + fresh || push!(stale, pkg) + end + isempty(stale) && return + + verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug + label(pkg) = haskey(ext_parents, pkg) ? Base.pkg_log_name(pkg, ext_parents[pkg]) : Base.pkg_log_name(pkg) + Base.@logmsg verbosity "Precompiling $(join((label(pkg) for pkg in stale), ", "))$(Base.list_reasons(reasons))" + # `@invokelatest` for the same reason as in `Base.__require_prelocked`, see #60223 + @invokelatest precompilepkgs(pkgs; _from_loading=true, ignore_loaded=false) + return +end + +## Background lifecycle + +function is_precompiling_in_background() + @lock BG (BG.task !== nothing && !istaskdone(BG.task)) +end + +# Check if `pkg` is currently being precompiled by a background task. +# This should be called while holding require_lock to avoid races. +# Returns true if the package is pending. +function is_package_pending(pkg::PkgId) + @lock BG begin + BG.task === nothing && return false + istaskdone(BG.task) && return false + return get(BG.pending_pkgids, pkg, 0) > 0 + end +end + +# Monitor the precompile progress for `pkg` until that package finishes. +# This should be called after unlocking require_lock. +function wait_for_pending_package(pkg::PkgId) + is_package_pending(pkg) || return false + printpkgstyle(stderr, :Info, "$(pkg.name) is currently being precompiled in the background. Waiting for it to finish...", color = Base.info_color()) + monitor_background_precompile(stderr, false, pkg) + return true +end + +# Broadcast a signal to all active precompilation subprocesses. +function broadcast_signal(sig::Int32) + @lock BG begin + for ch in BG.signal_channels + try; isopen(ch) && put!(ch, sig); catch e; e isa InvalidStateException || rethrow(); end end - filter!(d->in(first(d), keep), direct_deps) - if isempty(direct_deps) - if _from_loading - # if called from loading precompilation it may be a package from another environment stack so - # don't error and allow serial precompilation to try - # TODO: actually handle packages from other envs in the stack - return + end +end +broadcast_signal(sig::Integer) = broadcast_signal(Int32(sig)) + +# Record that `pkg` will not be precompiled by this session (e.g. missing source, +# `__precompile__(false)`), so watchers waiting for it are released promptly. +function mark_completed_pkgid!(pkg::PkgId) + @lock BG @lock BG.pkg_done begin + push!(BG.completed_pkgids, pkg) + notify(BG.pkg_done) + end + return nothing +end + +function stop_background_precompile(; graceful::Bool = true) + @lock BG begin + if BG.task !== nothing && !istaskdone(BG.task) + if graceful + BG.interrupt_requested = true else - return + BG.cancel_requested = true + broadcast_signal(Base.SIGKILL) # re-entrant: broadcast_signal also @lock's BG end + try; close(BG.work_channel); catch; end + return true end + return false end +end - target = Ref{Union{Nothing, String}}(nothing) - if nconfigs == 1 - if !isempty(only(configs)[1]) - target[] = "for configuration $(join(only(configs)[1], " "))" - end +const register_atexit_hook = Base.OncePerProcess{Nothing}() do + Base.atexit() do + task = @lock BG BG.task + task === nothing && return + istaskdone(task) && return + stop_background_precompile(; graceful=false) + wait(task) + end + nothing +end + +const _confirm_messages = Dict{Symbol, String}( + :cancel => "cancel precompilation", + :info => "send profiling signal", +) + +function keyboard_tip(s::BackgroundPrecompileState) + s.monitoring || return "", :default + s.key_listening || return "", :default + if s.confirming !== :none + remaining = max(0, ceil(Int, s.confirm_deadline - time())) + msg = get(_confirm_messages, s.confirming, string(s.confirming)) + return "Press Enter to $(msg) (ignoring in $(remaining)s)", Base.info_color() + end + if s.detachable + return "Press `?` for help, `c` to cancel, `d` to detach.", :default else - target[] = "for $nconfigs compilation configurations..." + return "Press `?` for help, `c` to cancel.", :default end - @debug "precompile: packages filtered" +end - pkg_queue = PkgConfig[] - failed_deps = Dict{PkgConfig, String}() - precomperr_deps = PkgConfig[] # packages that may succeed after a restart (i.e. loaded packages with no cache file) +function monitor_background_precompile(io::IO = stderr, detachable::Bool = true, wait_for_pkg::Union{Nothing, PkgId} = nothing; + key_controls::Union{Bool, Nothing} = nothing) + _monitor_background_precompile(unstable_iocontext(io), detachable, wait_for_pkg; key_controls) +end - print_lock = io.io isa Base.LibuvStream ? io.io.lock::ReentrantLock : ReentrantLock() - first_started = Base.Event() - printloop_should_exit = Ref{Bool}(!fancyprint) # exit print loop immediately if not fancy printing - interrupted_or_done = Base.Event() - - ansi_moveup(n::Int) = string("\e[", n, "A") - ansi_movecol1 = "\e[1G" - ansi_cleartoend = "\e[0J" - ansi_cleartoendofline = "\e[0K" - ansi_enablecursor = "\e[?25h" - ansi_disablecursor = "\e[?25l" - n_done = Ref(0) - n_already_precomp = Ref(0) - n_loaded = Ref(0) - interrupted = Ref(false) - - function handle_interrupt(err, in_printloop::Bool) - notify(interrupted_or_done) - in_printloop || wait(t_print) # wait to let the print loop cease first - if err isa InterruptException - @lock print_lock begin - println(io, " Interrupted: Exiting precompilation...", ansi_cleartoendofline) +function _monitor_background_precompile(io::IOContext{IO}, detachable::Bool, wait_for_pkg::Union{Nothing, PkgId}; + key_controls::Union{Bool, Nothing}) + # By default only enable key controls when this task is the foreground task (see #61563, #61698). + # Falls back to roottask when no foreground task is registered (e.g. non-REPL interactive scripts). + key_controls = @something key_controls current_task() === something(Base.foreground_task(), Base.roottask) + local completed_at::Union{Nothing, Float64} + local task + + @lock BG begin + completed_at = BG.completed_at + task = BG.task + end + + if task === nothing || istaskdone(task) + if completed_at !== nothing + elapsed = time() - completed_at + time_str = if elapsed < 60 + "$(round(Int, elapsed)) seconds ago" + elseif elapsed < 3600 + "$(round(Int, elapsed / 60)) minutes ago" + elseif elapsed < 86400 + "$(round(Int, elapsed / 3600)) hours ago" + else + "$(round(Int, elapsed / 86400)) days ago" + end + printpkgstyle(io, :Info, "Background precompilation completed $time_str", color = Base.info_color()) + result = @lock BG BG.result + if result !== nothing && !isempty(result) + println(io, " ", result) end - interrupted[] = true - return true else - return false + printpkgstyle(io, :Info, "No background precompilation is running or has been run in this session", color = Base.info_color()) end + return end - std_outputs = Dict{PkgConfig,IOBuffer}() - taskwaiting = Set{PkgConfig}() - pkgspidlocked = Dict{PkgConfig,String}() - pkg_liveprinted = Ref{Union{Nothing, PkgId}}(nothing) - function monitor_std(pkg_config, pipe; single_requested_pkg=false) - pkg, config = pkg_config - try - liveprinting = false - while !eof(pipe) - local str = readline(pipe, keep=true) - if single_requested_pkg && (liveprinting || !isempty(str)) - @lock print_lock begin - if !liveprinting - printpkgstyle(io, :Info, "Given $(pkg.name) was explicitly requested, output will be shown live $ansi_cleartoendofline", - color = Base.info_color()) - liveprinting = true - pkg_liveprinted[] = pkg + # Enable output from do_precompile + @lock BG BG.monitoring = true + + exit_requested = Ref(false) + cancel_requested = Ref(false) + interrupt_requested = Ref(false) + + # Start a task to listen for keypresses. Skipped if another reader already holds + # raw mode on stdin (e.g. runtests.jl's stdin_monitor). + key_task = if key_controls && stdin isa Base.TTY + Threads.@spawn :samepool try + trylock(stdin.raw_lock) || return + @lock BG begin + BG.detachable = detachable + BG.confirming = :none + BG.key_listening = true + end + buffered_input = UInt8[] + try + term = Base.Terminals.TTYTerminal(get(ENV, "TERM", "dumb"), stdin, stdout, stderr) + Base.Terminals.raw!(term, true) + # Drain any pre-existing input (e.g. pasted or pre-typed text) + # before entering the key listener. This avoids accidentally + # triggering menu actions from stale input (see #61520). + # The drained bytes are replayed into stdin on exit. + # We must start_reading + yield first so libuv delivers any + # bytes sitting in the kernel tty buffer into stdin.buffer. + Base.start_reading(stdin) + yield() + while bytesavailable(stdin) > 0 + push!(buffered_input, read(stdin, UInt8)) + end + try + while true + completed = @lock BG (BG.completed_at !== nothing) + if completed || exit_requested[] || cancel_requested[] || interrupt_requested[] + break + end + Base.wait_readnb(stdin, 1) + completed = @lock BG (BG.completed_at !== nothing) + if completed || exit_requested[] || cancel_requested[] || interrupt_requested[] + break + end + bytesavailable(stdin) > 0 || continue + c = read(stdin, Char) + # If waiting for confirmation, Enter confirms, anything else aborts + confirmed_action = @lock BG begin + prev = BG.confirming + BG.confirming = :none + (prev !== :none && c in ('\r', '\n')) ? prev : :none + end + if confirmed_action == :cancel + cancel_requested[] = true + println(io) + @lock BG BG.cancel_requested = true + broadcast_signal(Base.SIGKILL) + break + elseif confirmed_action == :info + @lock BG BG.info_requested = true + broadcast_signal(Sys.isapple() ? Base.SIGINFO : Base.SIGUSR1) + continue + end + if c in ('c', 'C') + @lock BG begin + BG.confirming = :cancel + BG.confirm_deadline = time() + 5.0 + end + elseif detachable && c in ('d', 'D', 'q', 'Q', ']') + exit_requested[] = true + println(io) # newline after keypress + break + elseif c == '\x03' # Ctrl-C + interrupt_requested[] = true + println(io) # newline after keypress + @lock BG BG.interrupt_requested = true + broadcast_signal(Base.SIGINT) + break + elseif c in ('i', 'I') + @lock BG begin + BG.confirming = :info + BG.confirm_deadline = time() + 5.0 + end + elseif c in ('v', 'V') + @lock BG BG.verbose = !BG.verbose + elseif c in ('?', 'h', 'H') + @lock io begin + println(io, " Keyboard shortcuts:", ansi_cleartoendofline) + println(io, " c Cancel precompilation via killing subprocesses (press Enter to confirm)", ansi_cleartoendofline) + if detachable + println(io, " d/q/] Detach (precompilation continues in background)", ansi_cleartoendofline) + end + println(io, " i Send profiling signal to subprocesses (press Enter to confirm)", ansi_cleartoendofline) + fields = Sys.iswindows() ? "elapsed time and PID" : "elapsed time, PID, CPU% and memory" + println(io, " v Toggle verbose mode (show $(fields) for each worker)", ansi_cleartoendofline) + println(io, " Ctrl-C Interrupt (sends SIGINT, shows output)", ansi_cleartoendofline) + println(io, " ?/h Show this help", ansi_cleartoendofline) + end end - print(io, ansi_cleartoendofline, str) end + finally + Base.Terminals.raw!(term, false) + end + catch err + err isa EOFError && return + exit_requested[] = true + rethrow() + finally + # Replay any buffered input back into stdin so the REPL + # (or whatever reads stdin next) sees it as typed text. + if !isempty(buffered_input) + lock(stdin.cond) + try + write(stdin.buffer, buffered_input) + notify(stdin.cond) + finally + unlock(stdin.cond) + end + end + Base.reseteof(stdin) + @lock BG begin + BG.confirming = :none + BG.key_listening = false + end + unlock(stdin.raw_lock) + end + finally + @lock BG.task_done notify(BG.task_done) + end + else + nothing + end + + # Wake up key_task by signaling EOF on stdin so wait_readnb returns + wake_key_task = () -> begin + if key_task !== nothing && !istaskdone(key_task) + lock(stdin.cond) + try + stdin.status = Base.StatusEOF + notify(stdin.cond) + finally + unlock(stdin.cond) + end + end + end + + # If waiting for a specific package, spawn a watcher that exits when it's done + if wait_for_pkg !== nothing + Threads.@spawn :samepool begin + @lock BG.pkg_done begin + # Wait for the package to appear in pending_pkgids (it may not be + # registered yet if the request was just injected via work_channel + # and drain_work_channel! hasn't processed it). + # Also check completed_pkgids in case the package was added and + # removed before we started watching. + while get(BG.pending_pkgids, wait_for_pkg, 0) == 0 && wait_for_pkg ∉ BG.completed_pkgids && BG.completed_at === nothing + wait(BG.pkg_done) + end + # Now wait for it to finish + while get(BG.pending_pkgids, wait_for_pkg, 0) > 0 + wait(BG.pkg_done) + end + end + exit_requested[] = true + @lock BG.task_done notify(BG.task_done) + end + else + nothing + end + + return try + # Wait for task completion or user action + @lock BG.task_done begin + while !exit_requested[] && !cancel_requested[] && !interrupt_requested[] + BG.completed_at !== nothing && break + wait(BG.task_done) + end + end + + # If user requested cancel, stop the background task + if cancel_requested[] + key_task !== nothing && wait(key_task) + print(io, ansi_enablecursor, ansi_cleartoend) + printpkgstyle(io, :Info, "Canceling precompilation...$(ansi_cleartoend)", color = Base.info_color()) + # Wait for the task to emit its final report before clearing + # `BG.monitoring`, which gates that report's output. + wait(task; throw=false) + @lock BG BG.monitoring = false + return + end + + # If user requested interrupt, wait for background task to finish cleanly + if interrupt_requested[] + key_task !== nothing && wait(key_task) + # Escalate to SIGKILL if background task doesn't finish promptly + escalation = Timer(5) do _ + broadcast_signal(Base.SIGKILL) + end + wait(task; throw=false) + close(escalation) + return + end + + # If we were waiting for a specific package and it finished, clean up silently + if exit_requested[] && wait_for_pkg !== nothing + @lock BG BG.monitoring = false + if key_task !== nothing + wake_key_task() + wait(key_task) + end + print(io, ansi_enablecursor, ansi_cleartoend) + return + end + + # If user requested exit, clean up and return + if exit_requested[] + @lock BG BG.monitoring = false + key_task !== nothing && wait(key_task) + print(io, ansi_enablecursor, ansi_cleartoend) + n_pending = @lock BG length(BG.pending_pkgids) + progress = n_pending > 0 ? " ($n_pending packages remaining)." : "." + printpkgstyle(io, :Precompiling, "detached$(progress) Precompilation will continue in the background. Monitor with `precompile --monitor`.$(ansi_cleartoend)", color = Base.info_color()) + return + end + + # Normal completion - signal key_task to exit and wait + if key_task !== nothing + wake_key_task() + wait(key_task) + end + + wait(task; throw=false) + catch + # Clean up on error + @lock BG BG.monitoring = false + if key_task !== nothing + exit_requested[] = true + wake_key_task() + try; wait(key_task); catch; end + end + rethrow() + end +end + +function launch_background_precompile(pkgs::Union{Vector{String}, Vector{PkgId}}, + internal_call::Bool, + strict::Bool, + warn_loaded::Bool, + timing::Bool, + verbose::Bool, + _from_loading::Bool, + configs::Vector{Config}, + io::IOContext, + fancyprint::Bool, + manifest::Bool, + ignore_loaded::Bool, + detachable::Bool, + skip_dependents::Bool, + force::Bool, + force_stdlibs::Bool) + # Stop any existing background precompilation + @lock BG begin + if BG.task !== nothing && !istaskdone(BG.task) + BG.interrupt_requested = true + @lock BG.task_done notify(BG.task_done) + end + end + + # Wait for previous task to complete + old_task = @lock BG BG.task + if old_task !== nothing + wait(old_task) + end + + @lock BG begin + BG.interrupt_requested = false + BG.cancel_requested = false + BG.info_requested = false + BG.verbose = verbose + empty!(BG.signal_channels) + BG.monitoring = true + BG.completed_at = nothing + BG.result = nothing + BG.return_value = nothing + BG.exception = nothing + empty!(BG.pending_pkgids) + empty!(BG.completed_pkgids) + BG.work_channel = Channel{PrecompileRequest}(Inf) + end + + # Register an atexit hook (once) to cleanly shut down background precompilation + # before the event loop is torn down. + register_atexit_hook() + + # Launch new background precompilation + @lock BG begin + wc = BG.work_channel + BG.task = Threads.@spawn :samepool begin + try + # pass the ids through: an extension has no name resolvable from `Main` + ret = do_precompile(pkgs, internal_call, strict, warn_loaded, timing, _from_loading, + configs, io, fancyprint, manifest, ignore_loaded, detachable, + skip_dependents, force, force_stdlibs, wc) + + @lock BG begin + BG.return_value = ret end - write(get!(IOBuffer, std_outputs, pkg_config), str) - if !in(pkg_config, taskwaiting) && occursin("waiting for IO to finish", str) - !fancyprint && @lock print_lock begin - println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) + catch e + @lock BG begin + if BG.interrupt_requested || BG.cancel_requested + BG.result = "Background precompilation was interrupted" + else + BG.exception = e + BG.result = "Background precompilation failed: $(sprint(showerror, e))" end - push!(taskwaiting, pkg_config) end - if !fancyprint && in(pkg_config, taskwaiting) - @lock print_lock begin - print(io, str) + finally + close(wc) + # Drain pending requests with error + while isready(wc) + req = try; take!(wc); catch; break; end + try; put!(req.result, InterruptException()); catch; end + end + @lock BG begin + BG.task = nothing + BG.interrupt_requested = false + BG.cancel_requested = false + BG.info_requested = false + foreach(close, BG.signal_channels) + empty!(BG.signal_channels) + @lock BG.pkg_done begin + empty!(BG.pending_pkgids) + notify(BG.pkg_done) end + BG.monitoring = false + BG.completed_at = time() + @lock BG.task_done notify(BG.task_done) + end + end + end + end + + return nothing +end + +function _collect_reachable!(pkg_uuids::Set{UUID}, deps::Dict{UUID, Vector{UUID}}, uuid::UUID) + uuid in pkg_uuids && return + push!(pkg_uuids, uuid) + for dep_uuid in get(Vector{UUID}, deps, uuid) + _collect_reachable!(pkg_uuids, deps, dep_uuid) + end +end + +function _precompilepkgs(pkgs::Union{Vector{String}, Vector{PkgId}}, + internal_call::Bool, + strict::Bool, + warn_loaded::Bool, + timing::Bool, + verbose::Bool, + _from_loading::Bool, + configs::Vector{Config}, + io::IOContext, + fancyprint′::Bool, + manifest::Bool, + ignore_loaded::Bool, + detachable::Bool, + skip_dependents::Bool, + force::Bool, + force_stdlibs::Bool) + # Try to inject into a running background task, else launch a new one, under + # launch_lock so concurrent callers cannot spawn competing background tasks. + local req = nothing + injected = @lock launch_lock begin + did_inject = @lock BG begin + if BG.task !== nothing && !istaskdone(BG.task) && + isopen(BG.work_channel) + req = PrecompileRequest(copy(pkgs), internal_call, strict, warn_loaded, timing, _from_loading, + configs, io, fancyprint′, manifest, ignore_loaded, detachable, + skip_dependents, force, force_stdlibs, Channel{Any}(1)) + try + # Enable verbose before enqueueing, and only ever turn it on so a + # non-verbose merge doesn't disable an already-verbose run. + verbose && (BG.verbose = true) + put!(BG.work_channel, req) + true + catch + req = nothing + false end + else + false + end + end + if !did_inject + launch_background_precompile(pkgs, internal_call, strict, warn_loaded, timing, verbose, _from_loading, + configs, io, fancyprint′, manifest, ignore_loaded, detachable, + skip_dependents, force, force_stdlibs) + end + did_inject + end + if injected + printpkgstyle(io, :Precompiling, "Merging precompilation request into existing run...", color = Base.info_color()) + end + + if req !== nothing + # Injected into existing task — monitor until our package finishes, then read result + wait_for_pkg = if _from_loading && length(pkgs) == 1 + pkgs[1] isa PkgId ? pkgs[1] : Base.identify_package(pkgs[1]) + else + nothing + end + monitor_background_precompile(io, detachable, wait_for_pkg) + if _from_loading + # _from_loading: package just left pending_pkgids, waiter will put result shortly + result = take!(req.result) + result isa Exception && throw(result) + return result + end + # Interactive: if result is ready (task completed), use it; if detached, return nothing + if isready(req.result) + result = take!(req.result) + result isa Exception && throw(result) + return result + end + return nothing + end + + # Launched new task — wait for full completion + monitor_background_precompile(io, detachable) + + local ret_val, ret_ex + @lock BG begin + ret_val = BG.return_value + ret_ex = BG.exception + end + ret_ex !== nothing && throw(ret_ex) + return ret_val +end + +## Session implementation + +# Mach timebase info for converting Mach absolute time → nanoseconds on macOS. +# Queried once per process at runtime (not at sysimage build time). +@static if Sys.isapple() + const _mach_timebase = Base.OncePerProcess{Tuple{UInt64,UInt64}}() do + buf = zeros(UInt32, 2) + ccall(:mach_timebase_info, Cvoid, (Ptr{UInt32},), buf) + (UInt64(buf[1]), UInt64(buf[2])) + end +end + +# Read cumulative CPU time (user+system) in nanoseconds and RSS in bytes for a process. +# Returns (cpu_ns=0, rss_bytes=0) if the process no longer exists or on unsupported platforms. +function process_stats(pid::Int32) + @static if Sys.islinux() + try + stat = read("/proc/$(pid)/stat", String) + # Field 2 (comm) is in parens and may contain spaces; skip past it + i = findlast(')', stat) + i === nothing && return (cpu_ns=UInt64(0), rss_bytes=UInt64(0)) + fields = split(@view(stat[nextind(stat, i):end])) + # fields[1] = state (field 3), so utime=field 14 is at index 12, + # stime=field 15 at index 13, rss=field 24 at index 22 + length(fields) >= 22 || return (cpu_ns=UInt64(0), rss_bytes=UInt64(0)) + utime = parse(UInt64, fields[12]) + stime = parse(UInt64, fields[13]) + rss_pages = parse(UInt64, fields[22]) + # CLK_TCK is almost always 100 on Linux; 1 tick = 10ms = 10_000_000 ns + cpu_ns = (utime + stime) * UInt64(10_000_000) + rss_bytes = rss_pages * UInt64(ccall(:getpagesize, Cint, ())) + return (; cpu_ns, rss_bytes) + catch + return (cpu_ns=UInt64(0), rss_bytes=UInt64(0)) + end + elseif Sys.isapple() + try + buf = Vector{UInt8}(undef, 96) # sizeof(struct proc_taskinfo) + ret = ccall((:proc_pidinfo, "libproc"), Cint, + (Cint, Cint, UInt64, Ptr{UInt8}, Cint), + pid, Cint(4), UInt64(0), buf, Cint(96)) + ret <= 0 && return (cpu_ns=UInt64(0), rss_bytes=UInt64(0)) + # pti_resident_size at offset 8 (bytes) + rss_bytes = GC.@preserve buf unsafe_load(Ptr{UInt64}(pointer(buf) + 8)) + # pti_total_user at offset 16, pti_total_system at offset 24 (Mach absolute time) + user = GC.@preserve buf unsafe_load(Ptr{UInt64}(pointer(buf) + 16)) + sys = GC.@preserve buf unsafe_load(Ptr{UInt64}(pointer(buf) + 24)) + # Convert Mach absolute time to nanoseconds via mach_timebase_info + numer, denom = _mach_timebase() + cpu_ns = div((user + sys) * numer, denom) + return (; cpu_ns, rss_bytes) + catch + return (cpu_ns=UInt64(0), rss_bytes=UInt64(0)) + end + else + return (cpu_ns=UInt64(0), rss_bytes=UInt64(0)) + end +end + +# Compute CPU% and RSS for all active PIDs since last poll. +# Mutates cpu_pcts and rss in place. +function poll_process_stats!(cpu_pcts::Dict{Int32, Float64}, rss::Dict{Int32, UInt64}, + prev_cpu_times::Dict{Int32, UInt64}, jobs::Dict{PkgConfig, PrecompileJob}, dt::Float64) + empty!(cpu_pcts) + empty!(rss) + @static if !(Sys.islinux() || Sys.isapple()) + return + end + for (_, job) in jobs + has_pid(job) || continue + pid = job.pid + haskey(cpu_pcts, pid) && continue + stats = process_stats(pid) + stats.cpu_ns == 0 && continue + prev = get(prev_cpu_times, pid, stats.cpu_ns) + delta = stats.cpu_ns >= prev ? stats.cpu_ns - prev : UInt64(0) + prev_cpu_times[pid] = stats.cpu_ns + pct = dt > 0 ? (delta / 1.0e9) / dt * 100.0 : 0.0 + cpu_pcts[pid] = min(pct, 999.9) + if stats.rss_bytes > 0 + rss[pid] = stats.rss_bytes + stats.rss_bytes > job.peak_rss_bytes && (job.peak_rss_bytes = stats.rss_bytes) + end + end + pids_set = Set(job.pid for (_, job) in jobs if has_pid(job)) + for pid in keys(prev_cpu_times) + pid in pids_set || delete!(prev_cpu_times, pid) + end + return +end + +# Lightweight peak-RSS-only sampler used in non-fancy verbose timing mode, +# where the live progress UI (which would otherwise drive `poll_process_stats!`) +# is not running but the per-package verbose timing column still wants peak RSS. +function sample_peak_rss!(jobs::Dict{PkgConfig, PrecompileJob}) + @static if !(Sys.islinux() || Sys.isapple()) + return + end + seen = Set{Int32}() + for (_, job) in jobs + has_pid(job) || continue + pid = job.pid + pid in seen && continue + push!(seen, pid) + stats = process_stats(pid) + if stats.rss_bytes > job.peak_rss_bytes + job.peak_rss_bytes = stats.rss_bytes + end + end + return +end + +function should_stop(s::PrecompileSession) + ir, cr = @lock BG begin + BG.interrupt_requested, BG.cancel_requested + end + if ir || cr + @lock s.print_lock begin + s.interrupted = s.interrupted || ir + s.canceled = s.canceled || cr + if !s.interrupted_or_done + s.interrupted_or_done = true + foreach(notify, values(s.was_processed)) + notify(s.first_started) end - catch err - err isa InterruptException || rethrow() end + return true + end + return s.interrupted_or_done +end + +function make_signal_channel() + ch = Channel{Int32}(32) + @lock BG push!(BG.signal_channels, ch) + return ch +end + +function describe_pkg(s::PrecompileSession, pkg::PkgId, is_project_dep::Bool, is_serial_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) + name = full_name(s.ext_to_parent, pkg) + name = is_project_dep ? name : color_string(name, :light_black, s.hascolor) + if is_serial_dep + name *= color_string(" (serial)", :light_black, s.hascolor) + end + if length(s.configs) > 1 && !isempty(flags) + config_str = join(flags, " ") + name *= color_string(" `$config_str`", :light_black, s.hascolor) + end + if length(s.configs) > 1 + config_str = join(Base.translate_cache_flags(cacheflags, Base.DefaultCacheFlags), " ") + name *= color_string(" $config_str", :light_black, s.hascolor) end + return name +end - ## fancy print loop - t_print = @async begin +function spawn_print_loop!(s::PrecompileSession) + Threads.@spawn :samepool begin + cursor_disabled = false + t = nothing try - wait(first_started) - (isempty(pkg_queue) || interrupted_or_done.set) && return - @lock print_lock begin - if target[] !== nothing - printpkgstyle(io, :Precompiling, target[]) - end - if fancyprint - print(io, ansi_disablecursor) + wait(s.first_started) + (isempty(s.pkg_queue) || s.interrupted_or_done) && return + @lock s.print_lock begin + if BG.monitoring + printpkgstyle(s.logio, :Precompiling, s.target) + BG.verbose && println(s.logio, format_verbose_timing_header()) end end t = Timer(0; interval=1/10) anim_chars = ["◐","◓","◑","◒"] i = 1 - last_length = 0 bar = MiniProgressBar(; indent=0, header = "Precompiling packages ", color = :green, percentage=false, always_reprint=true) - n_total = length(direct_deps) * length(configs) - bar.max = n_total - n_already_precomp[] + bar.max = s.n_total - s.n_already_precomp final_loop = false - n_print_rows = 0 - while !printloop_should_exit[] - @lock print_lock begin - term_size = displaysize(io)::Tuple{Int, Int} + last_poll_time = time() + cpu_pcts = Dict{Int32, Float64}() + rss_bytes = Dict{Int32, UInt64}() + while !s.printloop_should_exit + # Propagate cancel/interrupt requested via BG into the session so the loop exits. + should_stop(s) + @lock s.print_lock begin + verbose = BG.verbose + now_time = time() + dt = now_time - last_poll_time + if verbose && dt >= 0.5 + poll_process_stats!(cpu_pcts, rss_bytes, s.prev_cpu_times, s.jobs, dt) + last_poll_time = now_time + end + term_size = displaysize(s.logio)::Tuple{Int, Int} num_deps_show = max(term_size[1] - 3, 2) # show at least 2 deps - pkg_queue_show = if !interrupted_or_done.set && length(pkg_queue) > num_deps_show - last(pkg_queue, num_deps_show) + pkg_queue_show = if !s.interrupted_or_done && length(s.pkg_queue) > num_deps_show + last(s.pkg_queue, num_deps_show) else - pkg_queue + s.pkg_queue + end + i_local = i + final_loop_local = final_loop + tip, tip_color = @lock BG begin + if BG.confirming !== :none && time() >= BG.confirm_deadline + BG.confirming = :none + end + keyboard_tip(BG) end - str_ = sprint() do iostr - if i > 1 + str_ = sprint(; context=s.logio) do iostr + if i_local > 1 print(iostr, ansi_cleartoend) end - bar.current = n_done[] - n_already_precomp[] - bar.max = n_total - n_already_precomp[] - # when sizing to the terminal width subtract a little to give some tolerance to resizing the - # window between print cycles - termwidth = (displaysize(io)::Tuple{Int,Int})[2] - 4 - if !final_loop - s = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) - print(iostr, Base._truncate_at_width_or_chars(true, s, termwidth), "\n") + bar.header = verbose ? "Precompiling ($(s.num_tasks) tasks) " : "Precompiling packages " + # max(0,...) guards against a race where the print loop runs after + # n_already_precomp is incremented but before n_done is incremented, + # which would otherwise produce a negative value and crash repeat(). + bar.current = max(0, s.n_done - s.n_already_precomp) + bar.max = max(0, s.n_total - s.n_already_precomp) + termwidth = (displaysize(s.logio)::Tuple{Int,Int})[2] + if !final_loop_local + tip_width = isempty(tip) ? 0 : textwidth(tip) + 1 + bar_termwidth = termwidth - tip_width + s_bar = sprint(io -> show_progress(io, bar; termwidth=bar_termwidth, carriagereturn=false); context=s.logio) + if !isempty(tip) + s_bar = string(s_bar, " ", color_string(tip, tip_color, s.hascolor)) + end + print(iostr, Base._truncate_at_width_or_chars(true, s_bar, termwidth), "\n") end for pkg_config in pkg_queue_show dep, config = pkg_config - loaded = warn_loaded && haskey(Base.loaded_modules, dep) + loaded = s.warn_loaded && (dep in s.start_loaded_modules) flags, cacheflags = config - name = describe_pkg(dep, dep in project_deps, dep in serial_deps, flags, cacheflags) - line = if pkg_config in precomperr_deps - string(color_string(" ? ", Base.warn_color()), name) - elseif haskey(failed_deps, pkg_config) - string(color_string(" ✗ ", Base.error_color()), name) - elseif was_recompiled[pkg_config] - !loaded && interrupted_or_done.set && continue - loaded || @async begin # keep successful deps visible for short period + name = describe_pkg(s, dep, dep in s.project_deps, dep in s.serial_deps, flags, cacheflags) + job = s.jobs[pkg_config] + line = if is_soft_error(job) + string(color_string(" ? ", Base.warn_color(), s.hascolor), name) + elseif is_failed(job) + string(color_string(" ✗ ", Base.error_color(), s.hascolor), name) + elseif is_dep_failed(job) + string(color_string(" ✗ ", Base.error_color(), s.hascolor), name, + color_string(" (skipped, $(job.error_msg) failed to precompile)", :light_black, s.hascolor)) + elseif is_recompiled(job) + !loaded && s.interrupted_or_done && continue + loaded || Base.errormonitor(Threads.@spawn :samepool begin sleep(1); - filter!(!isequal(pkg_config), pkg_queue) + @lock s.print_lock filter!(!isequal(pkg_config), s.pkg_queue) + end) + string(color_string(" ✓ ", loaded ? Base.warn_color() : :green, s.hascolor), name) + elseif is_started(job) && s.interrupted_or_done + # Cancel/interrupt: show a static marker for jobs that actually had + # a subprocess running, so the user sees what was in flight without + # it looking still-running. Skip jobs that hadn't reached subprocess + # spawn yet (e.g. still acquiring the parallel_limiter). + had_pid(job) || continue + string(color_string(" - ", :light_black, s.hascolor), name) + elseif is_started(job) + anim_char = anim_chars[(i_local + Int(dep.name[1])) % length(anim_chars) + 1] + anim_char_colored = dep in s.project_deps ? anim_char : color_string(anim_char, :light_black, s.hascolor) + waiting = if is_locked(job) + color_string(" Being precompiled by $(job.lock_holder)", Base.info_color(), s.hascolor) + elseif is_waiting(job) + color_string(" Waiting for background task / IO / timer. Interrupt to inspect", Base.warn_color(), s.hascolor) + else + "" end - string(color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) - elseif started[pkg_config] - # Offset each spinner animation using the first character in the package name as the seed. - # If not offset, on larger terminal fonts it looks odd that they all sync-up - anim_char = anim_chars[(i + Int(dep.name[1])) % length(anim_chars) + 1] - anim_char_colored = dep in project_deps ? anim_char : color_string(anim_char, :light_black) - waiting = if haskey(pkgspidlocked, pkg_config) - who_has_lock = pkgspidlocked[pkg_config] - color_string(" Being precompiled by $(who_has_lock)", Base.info_color()) - elseif pkg_config in taskwaiting - color_string(" Waiting for background task / IO / timer. Interrupt to inspect", Base.warn_color()) + + pid_info = if verbose && has_pid(job) + elapsed_str = string(round(Int, now_time - job.started_at), "s") + pid = job.pid + pct = get(cpu_pcts, pid, -1.0) + pct_str = pct >= 0 ? string(" | cpu ", round(Int, pct), "%") : "" + mem = get(rss_bytes, pid, UInt64(0)) + mem_str = mem > 0 ? string(" | ", Base.format_bytes(mem)) : "" + color_string(" $(elapsed_str) | pid $(pid)$(pct_str)$(mem_str)", Base.info_color(), s.hascolor) else "" end - string(" ", anim_char_colored, " ", name, waiting) + string(" ", anim_char_colored, " ", name, pid_info, waiting) else string(" ", name) end println(iostr, Base._truncate_at_width_or_chars(true, line, termwidth)) end end - last_length = length(pkg_queue_show) n_print_rows = count("\n", str_) - print(io, str_) - printloop_should_exit[] = interrupted_or_done.set && final_loop - final_loop = interrupted_or_done.set # ensures one more loop to tidy last task after finish + s.printloop_should_exit = s.interrupted_or_done && final_loop + final_loop = s.interrupted_or_done i += 1 - printloop_should_exit[] || print(io, ansi_moveup(n_print_rows), ansi_movecol1) + if BG.monitoring + if s.fancyprint && !cursor_disabled + print(s.logio, ansi_disablecursor) + cursor_disabled = true + end + if s.printloop_should_exit + print(s.logio, str_) + else + print(s.logio, str_, ansi_moveup(n_print_rows), ansi_movecol1) + end + elseif cursor_disabled + print(s.logio, ansi_enablecursor) + cursor_disabled = false + end end wait(t) end - catch err - handle_interrupt(err, true) || rethrow() finally - fancyprint && print(io, ansi_enablecursor) + t === nothing || close(t) + cursor_disabled && print(s.logio, ansi_enablecursor) + end + end +end + +function precompilepkgs_monitor_std(s::PrecompileSession, pkg_config, job::PrecompileJob, pipe, single_requested_pkg::Bool) + pkg, _ = pkg_config + liveprinting = false + thistaskwaiting = false + while !eof(pipe) + str = readline(pipe, keep=true) + if startswith(str, Base.PRECOMPILE_VERBOSE_TIMING_MARKER) + job.verbose_timing = strip(str) + continue + end + if single_requested_pkg && (liveprinting || !isempty(str)) + BG.monitoring && @lock s.print_lock begin + if !liveprinting + liveprinting = true + s.pkg_liveprinted = pkg + end + # in fancy mode clear the progress bar residue from the line first + print(s.io, s.fancyprint ? ansi_cleartoendofline : "", str) + end + end + write(job.output, str) + if !thistaskwaiting && occursin("Waiting for background task / IO / timer", str) + thistaskwaiting = true + !liveprinting && !s.fancyprint && BG.monitoring && @lock s.print_lock begin + println(s.io, full_name(s.ext_to_parent, pkg), color_string(str, Base.warn_color(), s.hascolor)) + end + job.waiting_for_bg = true + elseif !thistaskwaiting + # XXX: don't just re-enable IO for random packages without printing the context for them first + !liveprinting && !s.fancyprint && BG.monitoring && @lock s.print_lock begin + print(s.io, str) + end + end + end +end + +# Can be merged with `maybe_cachefile_lock` in loading? +# Wraps the precompilation function `f` with cachefile lock handling. +# Returns the result from `f()`, which can be: +# - `nothing`: cache already existed +# - `Tuple{String, Union{Nothing, String}}`: this process just compiled +# - `Exception`: compilation failed +function precompile_pkgs_maybe_cachefile_lock(f, s::PrecompileSession, pkg_config, job::PrecompileJob, fullname) + if !(isdefined(Base, :mkpidlock_hook) && isdefined(Base, :trymkpidlock_hook) && Base.isdefined(Base, :parse_pidfile_hook)) + return f() + end + pkg, config = pkg_config + _, cacheflags = config + stale_age = Base.compilecache_pidlock_stale_age + pidfile = Base.compilecache_pidfile_path(pkg, flags=cacheflags) + cachefile = @invokelatest Base.trymkpidlock_hook(f, pidfile; stale_age) + if cachefile === false + pid, hostname, _ = @invokelatest Base.parse_pidfile_hook(pidfile) + job.lock_holder = if isempty(hostname) || hostname == gethostname() + if pid == getpid() + "an async task in this process (pidfile: $pidfile)" + else + "another process (pid: $pid, pidfile: $pidfile)" + end + else + "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" + end + !s.fancyprint && BG.monitoring && @lock s.print_lock begin + println(s.io, " ", fullname, color_string(" Being precompiled by $(job.lock_holder)", Base.info_color(), s.hascolor)) + end + Base.release(s.parallel_limiter) # release so other work can be done while waiting + try + # wait until the lock is available + cachefile = @invokelatest Base.mkpidlock_hook(() -> begin + job.lock_holder = "" + # this worker already had a slot before it waited for the lock, so take the next one + Base.acquire(f, s.parallel_limiter; priority=Inf, cancel=() -> should_stop(s)) + end, + pidfile; stale_age) + finally + Base.acquire(s.parallel_limiter; priority=Inf, cancel=() -> should_stop(s)) # re-acquire so the outer release is balanced + end + end + return cachefile +end + +# Rough proxy for how long a package takes to precompile, used only to order +# scheduling: the bytes of Julia source next to its entry file. +function precompile_cost_estimate(spec::Union{Nothing,Base.PkgLoadSpec}) + spec === nothing && return 1.0 + total = 0 + try + for (root, _, files) in walkdir(dirname(spec.path)) + for f in files + endswith(f, ".jl") && (total += filesize(joinpath(root, f))) + end end + catch + end + return max(1.0, Float64(total)) +end + +# Scheduling priority of each package: its own estimated cost plus that of the +# longest chain of packages that cannot start until it is done. Handing worker +# slots out in this order starts the environment's critical path as early as +# possible; in a cold precompile of a large environment the last package on that +# path, not the total amount of work, sets the wall-clock time. +# Computed on demand, so a run with little to compile doesn't read every package's source. +struct SchedulePriorities + lock::ReentrantLock + dependents::Dict{PkgId,Vector{PkgId}} + sourcespecs::Dict{PkgId,Union{Nothing,Base.PkgLoadSpec}} + cost::Dict{PkgId,Float64} + height::Dict{PkgId,Float64} + visiting::Set{PkgId} +end + +function SchedulePriorities(direct_deps::Dict{PkgId,Vector{PkgId}}, + sourcespecs::Dict{PkgId,Union{Nothing,Base.PkgLoadSpec}}) + dependents = Dict{PkgId,Vector{PkgId}}() + for (pkg, deps) in direct_deps, dep in deps + push!(get!(Vector{PkgId}, dependents, dep), pkg) + end + return SchedulePriorities(ReentrantLock(), dependents, sourcespecs, + Dict{PkgId,Float64}(), Dict{PkgId,Float64}(), Set{PkgId}()) +end + +schedule_priority(p::SchedulePriorities, pkg::PkgId) = @lock p.lock schedule_height!(p, pkg) + +# A top-level function rather than a local one so the recursion does not box it. +function schedule_height!(p::SchedulePriorities, pkg::PkgId) + haskey(p.height, pkg) && return p.height[pkg] + pkg in p.visiting && return 0.0 # circular dependency, reported elsewhere + push!(p.visiting, pkg) + best = 0.0 + for d in get(p.dependents, pkg, PkgId[]) + best = max(best, schedule_height!(p, d)) end - tasks = Task[] - if !_from_loading - Base.LOADING_CACHE[] = Base.LoadingCache() + delete!(p.visiting, pkg) + cost = get!(() -> precompile_cost_estimate(get(p.sourcespecs, pkg, nothing)), p.cost, pkg) + return p.height[pkg] = cost + best +end + +# Standard libraries ship precompiled with julia, so `force` leaves them alone +# unless `force_stdlibs` is set. +is_stdlib_source(spec::Base.PkgLoadSpec) = startswith(spec.path, Sys.STDLIB) + +# Name of the package whose failure means `deps` cannot be loaded: the first +# dependency whose worker ran and failed, or the root cause recorded on one that +# was itself skipped. A job that failed without a worker (no source file, e.g. a +# stdlib an old manifest does not list) may still load through the stdlib +# fallback, so its dependents are left to try. +function failed_dependency(s::PrecompileSession, deps::Vector{PkgId}, config::Config) + for dep in deps + job = @lock s.print_lock get(s.jobs, (dep, config), nothing) + job === nothing && continue + is_failed(job) && had_pid(job) && return full_name(s.ext_to_parent, dep) + is_dep_failed(job) && return job.error_msg end - @debug "precompile: starting precompilation loop" direct_deps project_deps - ## precompilation loop + return nothing +end +function spawn_precompile_tasks!(s::PrecompileSession; + direct_deps, was_processed, configs, circular_deps, + requested_pkgids, pkg_names, requested_pkgs, from_loading) + batch_tasks = Task[] + sourcespecs = Dict{PkgId,Union{Nothing,Base.PkgLoadSpec}}(pkg => Base.locate_package_load_spec(pkg) for pkg in keys(direct_deps)) + priorities = SchedulePriorities(direct_deps, sourcespecs) for (pkg, deps) in direct_deps - cachepaths = get!(() -> Base.find_all_in_cache_path(pkg), cachepath_cache, pkg) - sourcepath = Base.locate_package(pkg) - single_requested_pkg = length(requested_pkgs) == 1 && only(requested_pkgs) == pkg.name + cachepaths = Base.find_all_in_cache_path(pkg) + freshpaths = String[] + @lock s.cache_lock s.cachepath_cache[pkg] = freshpaths + sourcespec = sourcespecs[pkg] + single_requested_pkg = length(requested_pkgs) == 1 && + (pkg in requested_pkgids || pkg.name in pkg_names) for config in configs pkg_config = (pkg, config) - if sourcepath === nothing - failed_deps[pkg_config] = "Error: Missing source file for $(pkg)" + # look up under print_lock (the drainer may insert into s.jobs concurrently) + job = @lock s.print_lock s.jobs[pkg_config] + if sourcespec === nothing + mark_failed!(job, "Error: Missing source file for $(pkg)") + mark_completed_pkgid!(pkg) + @lock s.print_lock (s.n_done += 1) notify(was_processed[pkg_config]) continue end - # Heuristic for when precompilation is disabled - if occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcepath, String)) + if from_loading && single_requested_pkg && occursin(r"\b__precompile__\(\s*false\s*\)", read(sourcespec.path, String)) + @lock s.print_lock begin + Base.@logmsg s.logcalls "Disabled precompiling $(repr("text/plain", pkg)) since the text `__precompile__(false)` was found in file." + end + mark_completed_pkgid!(pkg) + @lock s.print_lock (s.n_done += 1) notify(was_processed[pkg_config]) continue end + @lock BG @lock BG.pkg_done begin + BG.pending_pkgids[pkg] = get(BG.pending_pkgids, pkg, 0) + 1 + notify(BG.pkg_done) + end flags, cacheflags = config - task = @async begin - try - loaded = warn_loaded && haskey(Base.loaded_modules, pkg) - for dep in deps # wait for deps to finish - wait(was_processed[(dep,config)]) + task = Threads.@spawn :samepool try + loaded = s.warn_loaded && (pkg in s.start_loaded_modules) + for dep in deps + wait(was_processed[(dep,config)]) + if should_stop(s) + return + end + end + circular = pkg in circular_deps + forced = s.force && !circular && (s.force_stdlibs || !is_stdlib_source(sourcespec)) + freshpath = @lock s.cache_lock Base.compilecache_freshest_path(pkg; ignore_loaded=s.ignore_loaded, + stale_cache=s.stale_cache, cachepath_cache=s.cachepath_cache, cachepaths, sourcespec, flags=cacheflags) + is_stale = forced || freshpath === nothing + if is_stale && !forced && !circular && Base.CACHE_FETCH_HOOK[] !== nothing + # a cache-fetch hook gets one chance to materialize a + # cachefile before we schedule a local compile; this runs + # after the dep waits above, so dependency cachefiles are + # in place for the hook to key against + if Base.maybe_fetch_cache(pkg, sourcespec.path) + local fetched_cachepaths = Base.find_all_in_cache_path(pkg) + freshpath = @lock s.cache_lock Base.compilecache_freshest_path(pkg; ignore_loaded=s.ignore_loaded, + stale_cache=s.stale_cache, cachepath_cache=s.cachepath_cache, + cachepaths=fetched_cachepaths, sourcespec, flags=cacheflags) + is_stale = freshpath === nothing + end + end + if !is_stale + @lock s.cache_lock push!(freshpaths, freshpath) + end + if !circular && is_stale + is_serial_dep = pkg in s.serial_deps + is_project_dep = pkg in s.project_deps + # an extension always loads its parent and triggers, so it can never + # be attempted when one of them failed; that is not worth reporting + is_ext = haskey(s.ext_to_parent, pkg) + root = (s.skip_dependents || is_ext) ? failed_dependency(s, deps, config) : nothing + if root !== nothing + # loading `root` would fail again, so do not spend a worker on it + @lock s.print_lock mark_dep_failed!(job, root) + # only project deps get a line; a failure low in the graph would otherwise + # list most of the environment, and the count in the summary covers the rest + (is_ext || !is_project_dep) && return + name = describe_pkg(s, pkg, is_project_dep, is_serial_dep, flags, cacheflags) + @lock s.print_lock begin + if !s.fancyprint && isempty(s.pkg_queue) && BG.monitoring + printpkgstyle(s.logio, :Precompiling, s.target) + end + push!(s.pkg_queue, pkg_config) + !s.fancyprint && BG.monitoring && println(s.logio, " "^12, + color_string(" ✗ ", Base.error_color(), s.hascolor), name, + color_string(" (skipped, $root failed to precompile)", :light_black, s.hascolor)) + end + return end - circular = pkg in circular_deps - is_stale = !Base.isprecompiled(pkg; ignore_loaded, stale_cache, cachepath_cache, cachepaths, sourcepath, flags=cacheflags) - if !circular && is_stale - Base.acquire(parallel_limiter) - is_project_dep = pkg in project_deps - is_serial_dep = pkg in serial_deps - - # std monitoring - std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) - t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) - - name = describe_pkg(pkg, is_project_dep, is_serial_dep, flags, cacheflags) - @lock print_lock begin - if !fancyprint && isempty(pkg_queue) - printpkgstyle(io, :Precompiling, something(target[], "packages...")) + Base.acquire(s.parallel_limiter; priority=schedule_priority(priorities, pkg), cancel=() -> should_stop(s)) + + std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) + t_monitor = Threads.@spawn :samepool precompilepkgs_monitor_std(s, pkg_config, job, std_pipe, + single_requested_pkg) + + name = describe_pkg(s, pkg, is_project_dep, is_serial_dep, flags, cacheflags) + try + @lock s.print_lock begin + if !s.fancyprint && isempty(s.pkg_queue) && BG.monitoring + printpkgstyle(s.logio, :Precompiling, s.target) + BG.verbose && println(s.logio, format_verbose_timing_header()) end + push!(s.pkg_queue, pkg_config) end - push!(pkg_queue, pkg_config) - started[pkg_config] = true - fancyprint && notify(first_started) - if interrupted_or_done.set - notify(was_processed[pkg_config]) - Base.release(parallel_limiter) + mark_started!(job) + s.fancyprint && notify(s.first_started) + if should_stop(s) return end - try - # allows processes to wait if another process is precompiling a given package to - # a functionally identical package cache (except for preferences, which may differ) - t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(io, print_lock, fancyprint, pkg_config, pkgspidlocked, hascolor, parallel_limiter, ignore_loaded) do - Base.with_logger(Base.NullLogger()) do - # whether to respect already loaded dependency versions - keep_loaded_modules = !ignore_loaded - # for extensions, any extension in our direct dependencies is one we have a right to load - # for packages, we may load any extension (all possible triggers are accounted for above) - loadable_exts = haskey(ext_to_parent, pkg) ? filter((dep)->haskey(ext_to_parent, dep), direct_deps[pkg]) : nothing - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, keep_loaded_modules; - flags, cacheflags, loadable_exts) + loadable_exts = haskey(s.ext_to_parent, pkg) ? filter((dep)->haskey(s.ext_to_parent, dep), s.triggers[pkg]) : nothing + + flags_ = if !isempty(deps) + `$flags --compiled-modules=strict` + else + flags + end + + pid_ch = Channel{Int32}(1) + if from_loading && pkg in requested_pkgids + Base.errormonitor(Threads.@spawn :samepool begin + pid = try; take!(pid_ch); catch; Int32(0); end + pid > 0 && @lock s.print_lock set_pid!(job, pid) + end) + t = @elapsed ret = begin + Base.compilecache(pkg, sourcespec, std_pipe, std_pipe, !s.ignore_loaded; + flags=flags_, cacheflags, loadable_exts, signal_channel=make_signal_channel(), + pid_channel=pid_ch, report_timing=true, + preresolved=preresolved_snapshot(s)) + end + else + fullname = full_name(s.ext_to_parent, pkg) + Base.errormonitor(Threads.@spawn :samepool begin + pid = try; take!(pid_ch); catch; Int32(0); end + pid > 0 && @lock s.print_lock set_pid!(job, pid) + end) + t = @elapsed ret = precompile_pkgs_maybe_cachefile_lock(s, pkg_config, job, fullname) do + if should_stop(s) + return ErrorException("canceled") + end + local cachepaths = Base.find_all_in_cache_path(pkg) + local freshpath = @lock s.cache_lock Base.compilecache_freshest_path(pkg; ignore_loaded=s.ignore_loaded, + stale_cache=s.stale_cache, cachepath_cache=s.cachepath_cache, cachepaths, sourcespec, flags=cacheflags) + local is_stale = forced || freshpath === nothing + if !is_stale + @lock s.cache_lock push!(freshpaths, freshpath) + return nothing + end + s.logcalls === CoreLogging.Debug && @lock s.print_lock begin + @debug "Precompiling $(repr("text/plain", pkg))" end + Base.compilecache(pkg, sourcespec, std_pipe, std_pipe, !s.ignore_loaded; + flags=flags_, cacheflags, loadable_exts, signal_channel=make_signal_channel(), + pid_channel=pid_ch, report_timing=true, + preresolved=preresolved_snapshot(s)) end - if ret isa Base.PrecompilableError - push!(precomperr_deps, pkg_config) - !fancyprint && @lock print_lock begin - println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) + end + if ret isa Exception + mark_soft_error!(job) + !s.fancyprint && BG.monitoring && @lock s.print_lock begin + println(s.logio, timing_string(t), color_string(" ? ", Base.warn_color(), s.hascolor), name) + end + else + cache_bytes = 0 + if ret !== nothing + cf_jl, cf_so = ret::Tuple{String, Union{Nothing, String}} + cache_bytes = _precompile_cache_bytes(cf_jl, cf_so) + end + !s.fancyprint && BG.monitoring && @lock s.print_lock begin + verbose_prefix = BG.verbose ? format_verbose_timing(job.verbose_timing, t, cache_bytes, job.peak_rss_bytes, s.hascolor) : "" + println(s.logio, timing_string(t), verbose_prefix, color_string(" ✓ ", loaded ? Base.warn_color() : :green, s.hascolor), name) + end + if ret !== nothing + mark_recompiled!(job) + cachefile, _ = ret::Tuple{String, Union{Nothing, String}} + build_id, _ = Base.parse_cache_buildid(cachefile) + stale_cache_key = (pkg, build_id, sourcespec, cachefile, s.ignore_loaded, cacheflags)::StaleCacheKey + @lock s.cache_lock begin + push!(freshpaths, cachefile) + s.stale_cache[stale_cache_key] = false end - else - !fancyprint && @lock print_lock begin - println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) + if loaded && Base.module_build_id(Base.loaded_modules[pkg]) != build_id + @lock s.print_lock begin + s.n_loaded += 1 + push!(s.loaded_pkgs, pkg) + end end - was_recompiled[pkg_config] = true + elseif loaded + # another process compiled this package; conservatively warn + @lock s.print_lock begin + s.n_loaded += 1 + push!(s.loaded_pkgs, pkg) + end + end + end + catch err + close(std_pipe.in) + wait(t_monitor) + err isa InterruptException && rethrow() + # If cancel was requested, this failure is almost certainly the + # subprocess being SIGKILL'd by the cancel; don't report it as + # a precompile failure. + if !(@lock BG BG.cancel_requested) + mark_failed!(job, sprint(showerror, err)) + !s.fancyprint && BG.monitoring && @lock s.print_lock begin + println(s.logio, " "^12, color_string(" ✗ ", Base.error_color(), s.hascolor), name) end - loaded && (n_loaded[] += 1) - catch err - # @show err - close(std_pipe.in) # close pipe to end the std output monitor - wait(t_monitor) - if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file")) - errmsg = String(take!(get(IOBuffer, std_outputs, pkg_config))) - delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report - failed_deps[pkg_config] = (strict || is_project_dep) ? string(sprint(showerror, err), "\n", strip(errmsg)) : "" - !fancyprint && @lock print_lock begin - println(io, " "^12, color_string(" ✗ ", Base.error_color()), name) + end + finally + isopen(std_pipe.in) && close(std_pipe.in) + wait(t_monitor) + try; close(pid_ch); catch; end + @lock s.print_lock clear_pid!(job) + Base.release(s.parallel_limiter) + end + else + if !is_stale + @lock s.print_lock begin + s.n_already_precomp += 1 + if loaded + fresh_build_id, _ = Base.parse_cache_buildid(freshpath) + if Base.module_build_id(Base.loaded_modules[pkg]) != fresh_build_id + s.n_loaded += 1 + push!(s.loaded_pkgs, pkg) end - else - rethrow() end - finally - isopen(std_pipe.in) && close(std_pipe.in) # close pipe to end the std output monitor - wait(t_monitor) - Base.release(parallel_limiter) end + end + end + finally + @lock BG @lock BG.pkg_done begin + n = get(BG.pending_pkgids, pkg, 0) - 1 + if n <= 0 + delete!(BG.pending_pkgids, pkg) + push!(BG.completed_pkgids, pkg) else - is_stale || (n_already_precomp[] += 1) + BG.pending_pkgids[pkg] = n end - n_done[] += 1 - notify(was_processed[pkg_config]) - catch err_outer - # For debugging: - # println("Task failed $err_outer") - # Base.display_error(ErrorException(""), Base.catch_backtrace())# logging doesn't show here - handle_interrupt(err_outer, false) || rethrow() - notify(was_processed[pkg_config]) - finally - filter!(!istaskdone, tasks) - length(tasks) == 1 && notify(interrupted_or_done) + notify(BG.pkg_done) end + @lock s.print_lock (s.n_done += 1) + notify(was_processed[pkg_config]) end - Base.errormonitor(task) # interrupts are handled separately so ok to watch for other errors like this - push!(tasks, task) + push!(batch_tasks, task) end end - isempty(tasks) && notify(interrupted_or_done) - try - wait(interrupted_or_done) - catch err - handle_interrupt(err, false) || rethrow() - finally - Base.LOADING_CACHE[] = nothing - end - notify(first_started) # in cases of no-op or !fancyprint - fancyprint && wait(t_print) - quick_exit = !all(istaskdone, tasks) || interrupted[] # if some not finished internal error is likely - seconds_elapsed = round(Int, (time_ns() - time_start) / 1e9) - ndeps = count(values(was_recompiled)) - if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) - str = sprint(context=io) do iostr - if !quick_exit - if fancyprint # replace the progress bar - what = isempty(requested_pkgs) ? "packages finished." : "$(join(requested_pkgs, ", ", " and ")) finished." + return batch_tasks +end + +function drain_work_channel!(s::PrecompileSession, work_channel::Channel{PrecompileRequest}) + drainer = Threads.@spawn :samepool begin + while true + isopen(work_channel) || break + should_stop(s) && break + request = try; take!(work_channel); catch; break; end + waiter_spawned = false + try + new_env = ExplicitEnv() + req_pkgids = PkgId[] + if request.pkgs isa Vector{PkgId} + append!(req_pkgids, request.pkgs) + else + for name in request.pkgs + pkgid = Base.identify_package(name) + pkgid !== nothing && push!(req_pkgids, pkgid) + end + end + new_graph = build_dep_graph(new_env, request.manifest, request._from_loading, req_pkgids) + # When no specific packages were requested, treat project deps as the requested set + effective_pkgids = isempty(req_pkgids) ? new_graph.project_deps : req_pkgids + @lock s.print_lock begin + merge!(s.direct_deps, new_graph.direct_deps) + merge!(s.ext_to_parent, new_graph.ext_to_parent) + merge!(s.parent_to_exts, new_graph.parent_to_exts) + merge!(s.triggers, new_graph.triggers) + union!(s.project_deps, new_graph.project_deps) + union!(s.serial_deps, new_graph.serial_deps) + union!(s.requested_pkgids, effective_pkgids) + end + new_pkg_names = String[pkg isa PkgId ? pkg.name : pkg for pkg in request.pkgs] + new_dd = new_graph.direct_deps + filter_dep_graph!(new_dd, new_pkg_names, new_graph.ext_to_parent, req_pkgids) + skip_pkgs = Set{PkgId}() + @lock BG begin + for pkgid in keys(new_dd) + if haskey(BG.pending_pkgids, pkgid) + push!(skip_pkgs, pkgid) + end + end + end + # Replace skip_pkgs with leaf nodes so circular dep detection + # can traverse through them, but no tasks will be spawned. + for pkgid in skip_pkgs + new_dd[pkgid] = PkgId[] + end + new_wp = Dict{PkgConfig, Base.Event}() + for config in request.configs + for pkgid in keys(new_dd) + pkg_config = (pkgid, config) + if pkgid in skip_pkgs + # Wire existing event so new tasks wait for the already-compiling dep + if haskey(s.was_processed, pkg_config) + new_wp[pkg_config] = s.was_processed[pkg_config] + else + evt = Base.Event() + notify(evt) + new_wp[pkg_config] = evt + end + else + evt = Base.Event() + # workers read s.jobs and should_stop iterates s.was_processed under this lock + @lock s.print_lock begin + get!(PrecompileJob, s.jobs, pkg_config) + s.was_processed[pkg_config] = evt + end + new_wp[pkg_config] = evt + end + end + end + new_circular = detect_circular_deps!(new_dd, new_graph.serial_deps, new_wp, s.io, s.ext_to_parent) + for pkgid in skip_pkgs + delete!(new_dd, pkgid) + end + if isempty(new_dd) + try; put!(request.result, String[]); catch; end + continue + end + @lock s.print_lock begin + s.n_total += length(new_dd) * length(request.configs) + s.n_batches += 1 + end + new_tasks = spawn_precompile_tasks!(s; + direct_deps=new_dd, was_processed=new_wp, configs=request.configs, + circular_deps=new_circular, requested_pkgids=effective_pkgids, + pkg_names=new_pkg_names, requested_pkgs=request.pkgs, + from_loading=request._from_loading) + append!(s.injected_tasks, new_tasks) + waiter = Threads.@spawn :samepool begin + try + waitall(new_tasks; failfast=false, throw=false) + # also wait for skipped packages being compiled by another request + foreach(wait, values(new_wp)) + paths = @lock s.cache_lock collect(String, Iterators.flatten((v for (pkgid, v) in s.cachepath_cache if pkgid in effective_pkgids))) + try; put!(request.result, paths); catch; end + finally + isready(request.result) || try; put!(request.result, String[]); catch; end + end + end + push!(s.result_waiters, waiter) + waiter_spawned = true + catch e + try; put!(request.result, e); catch; end + finally + # Only write the fallback here if no waiter was spawned — the waiter + # has its own finally that guarantees a result is written after tasks finish. + if !waiter_spawned && !isready(request.result) + try; put!(request.result, String[]); catch; end + end + end + end + end + return drainer +end + +function report_precompile_results!(s::PrecompileSession) + if !s._from_loading + @lock Base.require_lock begin + Base.LOADING_CACHE[] = nothing + end + end + notify(s.first_started) # in cases of no-op or !fancyprint + + quick_exit = any(t -> !istaskdone(t) || istaskfailed(t), s.tasks) || s.interrupted || s.canceled + seconds_elapsed = round(Int, (s.time_start > 0 ? (time_ns() -% s.time_start) : 0) / 1e9) + ndeps = count(j -> is_recompiled(j), values(s.jobs)) + + requested_errs = false + for ((dep, _), job) in s.jobs + if (is_failed(job) || is_dep_failed(job)) && dep in s.requested_pkgids + requested_errs = true + break + end + end + if !s.strict && !requested_errs && !s.interrupted && !s.canceled + for (_, job) in s.jobs + (is_failed(job) || is_dep_failed(job)) && clear_failure!(job) + end + end + if s.canceled && !(@lock BG BG.info_requested) + # Drop captured stdout/stderr from jobs that didn't fail before cancel, + # since their output is just truncated cancel-induced noise. If the user + # asked for info (SIGINFO/SIGUSR1) at any point, keep the output so the + # profiling info gets surfaced in the cancel report. Soft errors + # (e.g. `?` packages) are completed jobs whose captured output is the + # actual precompile error message, so preserve it as well. + for (_, job) in s.jobs + (is_failed(job) || is_soft_error(job)) && continue + job.output.size > 0 && truncate(job.output, 0) + end + end + n_failed = count(j -> is_failed(j), values(s.jobs)) + # skipped extensions are implied by their parent or trigger failing, so only count packages + n_dep_failed = count(((k, j),) -> is_dep_failed(j) && !haskey(s.ext_to_parent, first(k)), s.jobs) + if ndeps > 0 || n_failed > 0 + if !quick_exit + logstr = sprint(context=s.logio) do iostr + if s.fancyprint + what = if s.n_batches > 1 + "done." + elseif isempty(s.requested_pkgids) || s.requested_all + "packages finished." + else + "$(join((full_name(s.ext_to_parent, p) for p in s.requested_pkgids), ", ", " and ")) finished." + end printpkgstyle(iostr, :Precompiling, what) end - plural = length(configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" + plural = length(s.configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" print(iostr, " $(ndeps) $(plural) successfully precompiled in $(seconds_elapsed) seconds") - if n_already_precomp[] > 0 || !isempty(circular_deps) - n_already_precomp[] > 0 && (print(iostr, ". $(n_already_precomp[]) already precompiled")) - !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) + if s.n_already_precomp > 0 || !isempty(s.circular_deps) || n_dep_failed > 0 + s.n_already_precomp > 0 && (print(iostr, ". $(s.n_already_precomp) already precompiled")) + !isempty(s.circular_deps) && (print(iostr, ". $(length(s.circular_deps)) skipped due to circular dependency")) + n_dep_failed > 0 && (print(iostr, ". $(n_dep_failed) skipped because a dependency failed to precompile")) print(iostr, ".") end - if n_loaded[] > 0 - local plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded[] == 1 ? "dependency" : "dependencies" - local plural2 = n_loaded[] == 1 ? "a different version is" : "different versions are" - local plural3 = n_loaded[] == 1 ? "" : "s" - local plural4 = n_loaded[] == 1 ? "this package" : "these packages" + if s.n_loaded > 0 + plural1 = length(s.configs) > 1 ? "dependency configurations" : s.n_loaded == 1 ? "dependency" : "dependencies" + plural2 = s.n_loaded == 1 ? "a different version is" : "different versions are" + plural3 = s.n_loaded == 1 ? "" : "s" + loaded_names_vec = sort!([full_name(s.ext_to_parent, p) for p in s.loaded_pkgs]) + max_loaded_names = 5 + if length(loaded_names_vec) > max_loaded_names + loaded_names = string( + join(first(loaded_names_vec, max_loaded_names), ", ", " and "), + ", and ", + length(loaded_names_vec) - max_loaded_names, + " more" + ) + else + loaded_names = join(loaded_names_vec, ", ", " and ") + end + # compute how many precompiled packages transitively depend on the loaded packages + loaded_set = Set{PkgId}(s.loaded_pkgs) + n_affected = let reverse_deps = Dict{PkgId, Vector{PkgId}}() + for (p, deps) in s.direct_deps + for d in deps + push!(get!(Vector{PkgId}, reverse_deps, d), p) + end + end + affected = Set{PkgId}() + frontier = PkgId[p for p in loaded_set] + while !isempty(frontier) + p = pop!(frontier) + for rdep in get(reverse_deps, p, PkgId[]) + if rdep ∉ affected && rdep ∉ loaded_set + push!(affected, rdep) + push!(frontier, rdep) + end + end + end + length(affected) + end print(iostr, "\n ", - color_string(string(n_loaded[]), Base.warn_color()), + color_string(string(s.n_loaded), Base.warn_color(), s.hascolor), " $(plural1) precompiled but ", - color_string("$(plural2) currently loaded", Base.warn_color()), - ". Restart julia to access the new version$(plural3). \ - Otherwise, loading dependents of $(plural4) may trigger further precompilation to work with the unexpected version$(plural3)." + color_string("$(plural2) currently loaded", Base.warn_color(), s.hascolor), + " (", loaded_names, ")", + ". Restart julia to access the new version$(plural3)." + ) + if n_affected > 0 + affected_plural = length(s.configs) > 1 ? "dependency configurations" : n_affected == 1 ? "dependent" : "dependents" + print(iostr, + " Otherwise, $(n_affected) $(affected_plural) of ", + s.n_loaded == 1 ? "this package" : "these packages", + " may trigger further precompilation to work with the unexpected version$(plural3)." + ) + end + end + n_soft_errors = count(j -> is_soft_error(j), values(s.jobs)) + if n_soft_errors > 0 + pluralpc = length(s.configs) > 1 ? "dependency configurations" : n_soft_errors == 1 ? "dependency" : "dependencies" + print(iostr, "\n ", + color_string(string(n_soft_errors), Base.warn_color(), s.hascolor), + " $(pluralpc) failed but may be precompilable after restarting julia" ) end - if !isempty(precomperr_deps) - pluralpc = length(configs) > 1 ? "dependency configurations" : precomperr_deps == 1 ? "dependency" : "dependencies" + end + @lock BG BG.result = logstr + BG.monitoring && @lock s.print_lock begin + println(s.logio, logstr) + end + elseif s.interrupted || s.canceled + istr = sprint(context=s.logio) do iostr + if s.fancyprint + printpkgstyle(iostr, :Precompiling, s.canceled && !s.interrupted ? "canceled." : "interrupted.") + end + # On cancel we don't mark in-flight jobs as failed (their subprocesses + # were killed by the cancel itself), so report them as the "canceled" count. + # Use the sticky `had_pid` flag so we count jobs that actually spawned a + # subprocess, not ones that were just past mark_started!. Soft errors are + # completed jobs that errored before cancel; report them separately. + n_soft_errors = count(j -> is_soft_error(j), values(s.jobs)) + n_canceled_i = s.canceled && !s.interrupted ? + count(j -> had_pid(j) && !is_recompiled(j) && !is_soft_error(j), values(s.jobs)) : + n_failed + verb = s.canceled && !s.interrupted ? "canceled" : "interrupted" + print(iostr, " $(ndeps) dependenc$(ndeps == 1 ? "y" : "ies") precompiled, ", + color_string("$(n_canceled_i)", Base.error_color(), s.hascolor), + " $verb after $(seconds_elapsed) seconds") + if n_soft_errors > 0 + pluralpc = length(s.configs) > 1 ? "dependency configurations" : n_soft_errors == 1 ? "dependency" : "dependencies" print(iostr, "\n ", - color_string(string(length(precomperr_deps)), Base.warn_color()), + color_string(string(n_soft_errors), Base.warn_color(), s.hascolor), " $(pluralpc) failed but may be precompilable after restarting julia" ) end end - # show any stderr output, even if Pkg.precompile has been interrupted (quick_exit=true), given user may be - # interrupting a hanging precompile job with stderr output. julia#48371 - let std_outputs = Tuple{PkgConfig,SubString{String}}[(pkg_config, strip(String(take!(io)))) for (pkg_config,io) in std_outputs] - filter!(kv -> !isempty(last(kv)), std_outputs) + @lock BG BG.result = istr + BG.monitoring && @lock s.print_lock begin + println(s.logio, istr) + end + end + end + if any(j -> j.output.size > 0, values(s.jobs)) + str = sprint(context=s.io) do iostr + let std_outputs = Tuple{PkgConfig,SubString{String}}[(pc, strip(String(take!(job.output)))) for (pc, job) in s.jobs] + filter!(!isempty∘last, std_outputs) if !isempty(std_outputs) - local plural1 = length(std_outputs) == 1 ? "y" : "ies" - local plural2 = length(std_outputs) == 1 ? "" : "s" - print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") + plural1 = length(std_outputs) == 1 ? "y" : "ies" + print(iostr, " ", color_string("$(length(std_outputs))", Base.warn_color(), s.hascolor), " dependenc$(plural1) had output during precompilation:") for (pkg_config, err) in std_outputs - pkg, config = pkg_config - err = if pkg == pkg_liveprinted[] + pkg, _ = pkg_config + err = if pkg == s.pkg_liveprinted "[Output was shown above]" else - join(split(err, "\n"), color_string("\n│ ", Base.warn_color())) + join(split(err, "\n"), color_string("\n│ ", Base.warn_color(), s.hascolor)) end - name = full_name(ext_to_parent, pkg) - print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color())) + name = full_name(s.ext_to_parent, pkg) + print(iostr, color_string("\n┌ ", Base.warn_color(), s.hascolor), name, color_string("\n│ ", Base.warn_color(), s.hascolor), err, color_string("\n└ ", Base.warn_color(), s.hascolor)) end end end end - let str=str - @lock print_lock begin - println(io, str) - end + !isempty(str) && BG.monitoring && @lock s.print_lock begin + println(s.io, str) end - quick_exit && return + end + if n_failed > 0 err_str = IOBuffer() - n_direct_errs = 0 - for (pkg_config, err) in failed_deps - dep, config = pkg_config - if strict || (dep in project_deps) - print(err_str, "\n", dep.name, " ") - for cfg in config[1] - print(err_str, cfg, " ") - end - print(err_str, "\n\n", err) - n_direct_errs > 0 && write(err_str, "\n") - n_direct_errs += 1 - end + for ((dep, config), job) in s.jobs + is_failed(job) || continue + write(err_str, "\n") + print(err_str, "\n", full_name(s.ext_to_parent, dep), " ") + join(err_str, config[1], " ") + print(err_str, "\n", job.error_msg) end - if position(err_str) > 0 - skip(err_str, -1) - truncate(err_str, position(err_str)) - pluralde = n_direct_errs == 1 ? "y" : "ies" - direct = strict ? "" : "direct " - err_msg = "The following $n_direct_errs $(direct)dependenc$(pluralde) failed to precompile:\n$(String(take!(err_str)))" - if internal_call # aka. auto-precompilation - if isinteractive() - plural1 = length(failed_deps) == 1 ? "y" : "ies" - println(io, " ", color_string("$(length(failed_deps))", Base.error_color()), " dependenc$(plural1) errored.") - println(io, " For a report of the errors see `julia> err`. To retry use `pkg> precompile`") - setglobal!(Base.MainInclude, :err, PkgPrecompileError(err_msg)) - else - # auto-precompilation shouldn't throw but if the user can't easily access the - # error messages, just show them - print(io, "\n", err_msg) - end - else - println(io) - throw(PkgPrecompileError(err_msg)) + if n_dep_failed > 0 + # name only the skipped project deps; the rest are implied by them + skipped = sort!(String[dep.name for ((dep, _), job) in s.jobs + if is_dep_failed(job) && dep in s.project_deps && !haskey(s.ext_to_parent, dep)]) + print(err_str, "\n\n", n_dep_failed, n_dep_failed == 1 ? " package was" : " packages were", + " skipped because a dependency failed to precompile") + if length(skipped) == n_dep_failed + print(err_str, ": ", join(skipped, ", ")) + elseif !isempty(skipped) + print(err_str, ", including ", join(skipped, ", ")) end + print(err_str, "\nTo attempt them anyway, pass `skip_dependents=false` (`pkg> precompile --noskip`).") + end + pluraled = n_failed == 1 ? "" : "s" + err_msg = "The following $n_failed package$(pluraled) failed to precompile:$(String(take!(err_str)))\n" + if s.internal_call + print(s.io, err_msg) + else + throw(PkgPrecompileError(err_msg)) end end - nothing + if s.interrupted + throw(InterruptException()) + end + return @lock s.cache_lock collect(String, Iterators.flatten((v for (pkgid, v) in s.cachepath_cache if pkgid in s.requested_pkgids))) end -_timing_string(t) = string(lpad(round(t * 1e3, digits = 1), 9), " ms") - -function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) - if hascolor - enable_ansi = get(Base.text_colors, col, Base.text_colors[:default]) - disable_ansi = get(Base.disable_text_style, col, Base.text_colors[:default]) - return string(enable_ansi, cstr, disable_ansi) +# The actual precompilation implementation (mode-agnostic) +function do_precompile(pkgs::Union{Vector{String}, Vector{PkgId}}, + internal_call::Bool, + strict::Bool, + warn_loaded::Bool, + timing::Bool, + _from_loading::Bool, + configs::Vector{Config}, + io::IOContext, + fancyprint′::Bool, + manifest::Bool, + ignore_loaded::Bool, + detachable::Bool, + skip_dependents::Bool, + force::Bool, + force_stdlibs::Bool, + work_channel::Channel{PrecompileRequest}) + requested_pkgs = copy(pkgs) + pkg_names = pkgs isa Vector{String} ? copy(pkgs) : String[pkg.name for pkg in pkgs] + if pkgs isa Vector{PkgId} + requested_pkgids = copy(pkgs) else - return cstr + requested_pkgids = PkgId[] + for name in pkgs + pkgid = Base.identify_package(name) + if pkgid === nothing + _from_loading && return + throw(PkgPrecompileError("Unknown package: $name")) + end + push!(requested_pkgids, pkgid) + end end -end -# Can be merged with `maybe_cachefile_lock` in loading? -function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg_config, pkgspidlocked, hascolor, parallel_limiter::Base.Semaphore, ignore_loaded::Bool) - if !(isdefined(Base, :mkpidlock_hook) && isdefined(Base, :trymkpidlock_hook) && Base.isdefined(Base, :parse_pidfile_hook)) - return f() - end - pkg, config = pkg_config - flags, cacheflags = config - stale_age = Base.compilecache_pidlock_stale_age - pidfile = Base.compilecache_pidfile_path(pkg, flags=cacheflags) - cachefile = @invokelatest Base.trymkpidlock_hook(f, pidfile; stale_age) - if cachefile === false - pid, hostname, age = @invokelatest Base.parse_pidfile_hook(pidfile) - pkgspidlocked[pkg_config] = if isempty(hostname) || hostname == gethostname() - if pid == getpid() - "an async task in this process (pidfile: $pidfile)" - else - "another process (pid: $pid, pidfile: $pidfile)" - end + time_start = time_ns() + env = ExplicitEnv() + + # Windows sometimes hits a ReadOnlyMemoryError, so we halve the default number of tasks. Pkg.jl#2323 + # TODO: Investigate why this happens in windows and restore the full task limit + default_num_tasks = Sys.iswindows() ? div(Sys.EFFECTIVE_CPU_THREADS::Int, 2) + 1 : Sys.EFFECTIVE_CPU_THREADS::Int + 1 + default_num_tasks = min(default_num_tasks, 16) # limit for better stability on shared resource systems + num_tasks = max(1, something(tryparse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))), 1)) + + # Suppress precompilation progress messages when precompiling for loading packages, except during + # interactive sessions, since the complicated IO can have disastrous consequences in the background (#59599) + logio = io + logcalls = nothing + if _from_loading + if isinteractive() + logcalls = CoreLogging.Info else - "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" + logio = IOContext{IO}(devnull) + fancyprint′ = false + logcalls = CoreLogging.Debug + end + end + fancyprint = fancyprint′ + hascolor = get(logio, :color, false)::Bool + + # Build dependency graph + graph = build_dep_graph(env, manifest, _from_loading, requested_pkgids) + + # Return early if no deps + if isempty(graph.direct_deps) + isempty(pkgs) && return + # a request for packages that are all in the sysimage has nothing to do + all(Base.in_sysimage, requested_pkgids) && return + error("No direct dependencies outside of the sysimage found matching $(pkgs)") + end + + # Initialize signalling + was_processed = Dict{PkgConfig,Base.Event}() + jobs = Dict{PkgConfig,PrecompileJob}() + for config in configs + for pkgid in keys(graph.direct_deps) + pkg_config = (pkgid, config) + jobs[pkg_config] = PrecompileJob() + was_processed[pkg_config] = Base.Event() end - !fancyprint && @lock print_lock begin - println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg_config])", Base.info_color(), hascolor)) + end + + circular_deps = detect_circular_deps!(graph.direct_deps, graph.serial_deps, was_processed, io, graph.ext_to_parent) + + if filter_dep_graph!(graph.direct_deps, pkg_names, graph.ext_to_parent, requested_pkgids) + @lock BG BG.result = "" + return + end + + # When no specific packages were requested, treat project deps as the requested + # set so direct-dep failures are reported and the returned paths cover the project. + requested_all = isempty(requested_pkgids) + requested_all && append!(requested_pkgids, graph.project_deps) + + nconfigs = length(configs) + target = if nconfigs == 1 + flags = only(configs)[1] + isempty(flags) ? (requested_all ? "project..." : "packages...") : "for configuration $(join(flags, " "))" + else + "for $nconfigs compilation configurations" + end + + print_lock = io.io isa Base.LibuvStream ? io.io.lock::ReentrantLock : ReentrantLock() + + # Size the shared CPU-thread budget for the parallel workers. Defaults to + # `EFFECTIVE_CPU_THREADS + 1` (one baseline per worker plus a spare so a lone + # worker can fill every core during imaging); `JULIA_PRECOMPILE_THREADS` + # overrides it. Skipped for a single task or when JULIA_IMAGE_THREADS pins a + # per-worker count (a hard override that bypasses the shared budget), unless + # `JULIA_PRECOMPILE_THREADS` is also set, in which case it takes precedence. + # Examples: + # neither set -> jobserver budget = EFFECTIVE_CPU_THREADS + 1 + # JULIA_PRECOMPILE_THREADS=8 -> jobserver budget = 8 + # JULIA_IMAGE_THREADS=4 -> no jobserver; each worker pinned to 4 threads + # JULIA_IMAGE_THREADS=4 JULIA_PRECOMPILE_THREADS=8 -> jobserver budget = 8 (JULIA_IMAGE_THREADS ignored here) + precompile_jobserver = if num_tasks > 1 && (haskey(ENV, "JULIA_PRECOMPILE_THREADS") || !haskey(ENV, "JULIA_IMAGE_THREADS")) + default_budget = Sys.EFFECTIVE_CPU_THREADS + 1 + budget = max(1, something(tryparse(Int, get(ENV, "JULIA_PRECOMPILE_THREADS", string(default_budget))), default_budget)) + setup_precompile_jobserver!(budget) + else + :none + end + + s = PrecompileSession(; + configs, io, logio, logcalls, fancyprint, hascolor, + warn_loaded, ignore_loaded, internal_call, strict, _from_loading, + skip_dependents, force, force_stdlibs, + time_start, print_lock, + parallel_limiter=WorkerLimiter(num_tasks, precompile_jobserver !== :none), num_tasks, + start_loaded_modules=Set{PkgId}(keys(Base.loaded_modules)), requested_pkgids, requested_all, + direct_deps=graph.direct_deps, + ext_to_parent=graph.ext_to_parent, parent_to_exts=graph.parent_to_exts, + triggers=graph.triggers, project_deps=graph.project_deps, + serial_deps=graph.serial_deps, circular_deps, + n_total=length(graph.direct_deps) * nconfigs, + printloop_should_exit=!fancyprint, target, + jobs, was_processed, + ) + + # Start print loop + t_print = spawn_print_loop!(s) + # In non-fancy mode the print loop exits immediately and so `poll_process_stats!` + # never runs; sample peak RSS on a timer instead for the `~pk-rss` column. + # Gated per tick since verbose can be enabled mid-run (injected request, `v` key). + # Locks taken sequentially: the print loop nests BG inside print_lock. + peak_rss_timer = if !fancyprint + Timer(0.1; interval=0.5, spawn=true) do _ + verbose_now = @lock BG BG.verbose + verbose_now && @lock s.print_lock sample_peak_rss!(s.jobs) end - Base.release(parallel_limiter) # release so other work can be done while waiting - try - # wait until the lock is available - @invokelatest Base.mkpidlock_hook(() -> begin - # double-check in case the other process crashed or the lock expired - if Base.isprecompiled(pkg; ignore_loaded, flags=cacheflags) # don't use caches for this as the env state will have changed - return nothing # returning nothing indicates a process waited for another - else - delete!(pkgspidlocked, pkg_config) - Base.acquire(f, parallel_limiter) # precompile - end - end, - pidfile; stale_age) - finally - Base.acquire(parallel_limiter) # re-acquire so the outer release is balanced + else + nothing + end + + try + if !_from_loading + @lock Base.require_lock begin + Base.LOADING_CACHE[] = Base.LoadingCache() + end end + @debug "precompile: starting precompilation loop" graph.direct_deps graph.project_deps + + # Initial pass + initial_tasks = spawn_precompile_tasks!(s; + direct_deps=graph.direct_deps, was_processed, configs, circular_deps, + requested_pkgids, pkg_names, requested_pkgs, from_loading=_from_loading) + append!(s.tasks, initial_tasks) + + # Concurrent drainer for injected requests + drainer = drain_work_channel!(s, work_channel) + + waitall(initial_tasks; failfast=false, throw=false) + @lock BG close(work_channel) + wait(drainer) + append!(s.tasks, s.injected_tasks) + waitall(s.injected_tasks; failfast=false, throw=false) + waitall(s.result_waiters; failfast=false, throw=false) + + # Final output + s.interrupted_or_done = true + notify(s.first_started) # unblock print loop if nothing ever started (no-op case) + fancyprint && wait(t_print) + return report_precompile_results!(s) + finally + # Ensure print loop exits even on exception + s.interrupted_or_done = true + notify(s.first_started) + peak_rss_timer === nothing || close(peak_rss_timer) + precompile_jobserver === :created && teardown_precompile_jobserver!() end - return cachefile end end diff --git a/base/process.jl b/base/process.jl index d1ed1acb9fcf4..81cb29c206a48 100644 --- a/base/process.jl +++ b/base/process.jl @@ -36,9 +36,9 @@ end pipe_reader(p::ProcessChain) = p.out pipe_writer(p::ProcessChain) = p.in -# a lightweight pair of a child OS_HANDLE and associated Task that will +# A lightweight pair of a child OS_HANDLE and associated Task that will # complete only after all content has been read from it for synchronizing -# state without the kernel to aide +# state without the kernel to aid. struct SyncCloseFD fd t::Task @@ -84,7 +84,7 @@ function _uv_hook_close(proc::Process) nothing end -const SpawnIO = Union{IO, RawFD, OS_HANDLE, SyncCloseFD} # internal copy of Redirectable, removing FileRedirect and adding SyncCloseFD +const SpawnIO = Union{IO, IOServer, RawFD, OS_HANDLE, SyncCloseFD} # internal copy of Redirectable, removing FileRedirect and adding SyncCloseFD const SpawnIOs = Memory{SpawnIO} # convenience name for readability (used for dispatch also to clearly distinguish from Vector{Redirectable}) function as_cpumask(cpus::Vector{UInt16}) @@ -97,7 +97,15 @@ function as_cpumask(cpus::Vector{UInt16}) end # handle marshalling of `Cmd` arguments from Julia to C -@noinline function _spawn_primitive(file, cmd::Cmd, stdio::SpawnIOs) +@noinline function _spawn_primitive(file, cmd::Cmd, stdio::SpawnIOs, + tok::MaybeToken=default_cancel_token()) + # Entry cancellation check before the child exists: a spawn under an + # already-cancelled token must not run the command (its side effects + # cannot be taken back). `tok` is the *resolved* token of the public + # operation (run/open/success/read), threaded down here so an explicit + # token - or an explicit `cancel = nothing` shield - governs the spawn + # itself, not whatever the ambient scope happens to be. + @cancel_check tok loop = eventloop() cpumask = cmd.cpus cpumask === nothing || (cpumask = as_cpumask(cpumask)) @@ -113,13 +121,20 @@ end syncd = Task[io.t for io in stdio if io isa SyncCloseFD] handle = Libc.malloc(_sizeof_uv_process) disassociate_julia_struct(handle) - (; exec, flags, env, dir) = cmd + (; exec, flags, env, dir, uid, gid) = cmd flags ⊻= UV_PROCESS_WINDOWS_DISABLE_EXACT_NAME # libuv inverts the default for this, so flip this bit now + if uid !== nothing + flags |= UV_PROCESS_SETUID + end + if gid !== nothing + flags |= UV_PROCESS_SETGID + end iolock_begin() err = ccall(:jl_spawn, Int32, (Cstring, Ptr{Cstring}, Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Tuple{Cint, UInt}}, Int, - UInt32, Ptr{Cstring}, Cstring, Ptr{Bool}, Csize_t, Ptr{Cvoid}), + UInt32, Ptr{Cstring}, Cstring, Ptr{Bool}, Csize_t, + UInt32, UInt32, Ptr{Cvoid}), file, exec, loop, handle, iohandles, length(iohandles), flags, @@ -127,39 +142,44 @@ end isempty(dir) ? C_NULL : dir, cpumask === nothing ? C_NULL : cpumask, cpumask === nothing ? 0 : length(cpumask), + uid === nothing ? typemax(UInt32) : uid, + gid === nothing ? typemax(UInt32) : gid, @cfunction(uv_return_spawn, Cvoid, (Ptr{Cvoid}, Int64, Int32))) if err == 0 pp = Process(cmd, handle, syncd) associate_julia_struct(handle, pp) + iolock_end() + return pp else ccall(:jl_forceclose_uv, Cvoid, (Ptr{Cvoid},), handle) # will call free on handle eventually + iolock_end() + throw(_UVError("could not spawn " * repr(cmd), err)) end - iolock_end() end - if err != 0 - throw(_UVError("could not spawn " * repr(cmd), err)) - end - return pp end -_spawn(cmds::AbstractCmd) = _spawn(cmds, SpawnIOs()) +# The `tok::MaybeToken` threaded through the _spawn chain is the resolved +# token of the public operation; see _spawn_primitive. +_spawn(cmds::AbstractCmd, tok::MaybeToken=default_cancel_token()) = + _spawn(cmds, SpawnIOs(), tok) -function _spawn(cmd::AbstractCmd, stdios::Vector{Redirectable}) +function _spawn(cmd::AbstractCmd, stdios::Vector{Redirectable}, + tok::MaybeToken=default_cancel_token()) pp = setup_stdios(stdios) do stdios - return _spawn(cmd, stdios) + return _spawn(cmd, stdios, tok) end return pp end # optimization: we can spawn `Cmd` directly without allocating the ProcessChain -function _spawn(cmd::Cmd, stdios::SpawnIOs) +function _spawn(cmd::Cmd, stdios::SpawnIOs, tok::MaybeToken=default_cancel_token()) isempty(cmd.exec) && throw(ArgumentError("cannot spawn empty command")) - return _spawn_primitive(cmd.exec[1], cmd, stdios) + return _spawn_primitive(cmd.exec[1], cmd, stdios, tok) end # assume that having a ProcessChain means that the stdio are setup -function _spawn(cmds::AbstractCmd, stdios::SpawnIOs) - return _spawn(cmds, stdios, ProcessChain()) +function _spawn(cmds::AbstractCmd, stdios::SpawnIOs, tok::MaybeToken=default_cancel_token()) + return _spawn(cmds, stdios, ProcessChain(), tok) end # helper function for making a copy of a SpawnIOs, with replacement @@ -171,24 +191,35 @@ function _stdio_copy(stdios::SpawnIOs, fd::Int, @nospecialize replace) return new end -function _spawn(redirect::CmdRedirect, stdios::SpawnIOs, args...) +function _spawn(redirect::CmdRedirect, stdios::SpawnIOs, tok::MaybeToken=default_cancel_token()) fdnum = redirect.stream_no + 1 io, close_io = setup_stdio(redirect.handle, redirect.readable) try stdios = _stdio_copy(stdios, fdnum, io) - return _spawn(redirect.cmd, stdios, args...) + return _spawn(redirect.cmd, stdios, tok) finally close_io && close_stdio(io) end end -function _spawn(cmds::OrCmds, stdios::SpawnIOs, chain::ProcessChain) +function _spawn(redirect::CmdRedirect, stdios::SpawnIOs, chain::ProcessChain, tok::MaybeToken) + fdnum = redirect.stream_no + 1 + io, close_io = setup_stdio(redirect.handle, redirect.readable) + try + stdios = _stdio_copy(stdios, fdnum, io) + return _spawn(redirect.cmd, stdios, chain, tok) + finally + close_io && close_stdio(io) + end +end + +function _spawn(cmds::OrCmds, stdios::SpawnIOs, chain::ProcessChain, tok::MaybeToken) in_pipe, out_pipe = link_pipe(false, false) try stdios_left = _stdio_copy(stdios, 2, out_pipe) - _spawn(cmds.a, stdios_left, chain) + _spawn(cmds.a, stdios_left, chain, tok) stdios_right = _stdio_copy(stdios, 1, in_pipe) - _spawn(cmds.b, stdios_right, chain) + _spawn(cmds.b, stdios_right, chain, tok) finally close_pipe_sync(out_pipe) close_pipe_sync(in_pipe) @@ -196,13 +227,13 @@ function _spawn(cmds::OrCmds, stdios::SpawnIOs, chain::ProcessChain) return chain end -function _spawn(cmds::ErrOrCmds, stdios::SpawnIOs, chain::ProcessChain) +function _spawn(cmds::ErrOrCmds, stdios::SpawnIOs, chain::ProcessChain, tok::MaybeToken) in_pipe, out_pipe = link_pipe(false, false) try stdios_left = _stdio_copy(stdios, 3, out_pipe) - _spawn(cmds.a, stdios_left, chain) + _spawn(cmds.a, stdios_left, chain, tok) stdios_right = _stdio_copy(stdios, 1, in_pipe) - _spawn(cmds.b, stdios_right, chain) + _spawn(cmds.b, stdios_right, chain, tok) finally close_pipe_sync(out_pipe) close_pipe_sync(in_pipe) @@ -210,15 +241,15 @@ function _spawn(cmds::ErrOrCmds, stdios::SpawnIOs, chain::ProcessChain) return chain end -function _spawn(cmds::AndCmds, stdios::SpawnIOs, chain::ProcessChain) - _spawn(cmds.a, stdios, chain) - _spawn(cmds.b, stdios, chain) +function _spawn(cmds::AndCmds, stdios::SpawnIOs, chain::ProcessChain, tok::MaybeToken) + _spawn(cmds.a, stdios, chain, tok) + _spawn(cmds.b, stdios, chain, tok) return chain end -function _spawn(cmd::Cmd, stdios::SpawnIOs, chain::ProcessChain) +function _spawn(cmd::Cmd, stdios::SpawnIOs, chain::ProcessChain, tok::MaybeToken) isempty(cmd.exec) && throw(ArgumentError("cannot spawn empty command")) - pp = _spawn_primitive(cmd.exec[1], cmd, stdios) + pp = _spawn_primitive(cmd.exec[1], cmd, stdios, tok) push!(chain.processes, pp) return chain end @@ -341,20 +372,22 @@ close_stdio(stdio::SyncCloseFD) = close_stdio(stdio.fd) spawn_opts_swallow(stdios::StdIOSet) = Redirectable[stdios...] spawn_opts_inherit(stdios::StdIOSet) = Redirectable[stdios...] -spawn_opts_swallow(in::Redirectable=devnull, out::Redirectable=devnull, err::Redirectable=devnull) = - Redirectable[in, out, err] +spawn_opts_swallow(in::Redirectable=devnull, out::Redirectable=devnull, err::Redirectable=devnull, extra::Redirectable...) = + Redirectable[in, out, err, extra...] # pass original descriptors to child processes by default, because we might # have already exhausted and closed the libuv object for our standard streams. # ref issue #8529 -spawn_opts_inherit(in::Redirectable=RawFD(0), out::Redirectable=RawFD(1), err::Redirectable=RawFD(2)) = - Redirectable[in, out, err] +spawn_opts_inherit(in::Redirectable=RawFD(0), out::Redirectable=RawFD(1), err::Redirectable=RawFD(2), extra::Redirectable...) = + Redirectable[in, out, err, extra...] -function eachline(cmd::AbstractCmd; keep::Bool=false) +function eachline(cmd::AbstractCmd; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok out = PipeEndpoint() - processes = _spawn(cmd, Redirectable[devnull, out, stderr]) + processes = _spawn(cmd, Redirectable[devnull, out, stderr], tok) # if the user consumes all the data, also check process exit status for success - ondone = () -> (success(processes) || pipeline_error(processes); nothing) - return EachLine(out, keep=keep, ondone=ondone)::EachLine + ondone = () -> (success(processes; cancel=tok) || pipeline_error(processes); nothing) + return EachLine(out; keep, ondone, cancel=tok)::EachLine end """ @@ -371,13 +404,14 @@ Possible mode strings are: | `r+` | read, write | `read = true, write = true` | | `w+` | read, write | `read = true, write = true` | """ -function open(cmds::AbstractCmd, mode::AbstractString, stdio::Redirectable=devnull) +function open(cmds::AbstractCmd, mode::AbstractString, stdio::Redirectable=devnull; + cancel::CancelTokenArg=DEFAULT_CANCEL) if mode == "r+" || mode == "w+" - return open(cmds, stdio, read = true, write = true) + return open(cmds, stdio, read = true, write = true; cancel) elseif mode == "r" - return open(cmds, stdio) + return open(cmds, stdio; cancel) elseif mode == "w" - return open(cmds, stdio, write = true) + return open(cmds, stdio, write = true; cancel) else throw(ArgumentError("mode must be \"r\", \"w\", \"r+\", or \"w+\", not $(repr(mode))")) end @@ -394,25 +428,27 @@ the process's standard input and `stdio` optionally specifies the process's stan stream. The process's standard error stream is connected to the current global `stderr`. """ -function open(cmds::AbstractCmd, stdio::Redirectable=devnull; write::Bool=false, read::Bool=!write) +function open(cmds::AbstractCmd, stdio::Redirectable=devnull; write::Bool=false, read::Bool=!write, + cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) if read && write stdio === devnull || throw(ArgumentError("no stream can be specified for `stdio` in read-write mode")) in = PipeEndpoint() out = PipeEndpoint() - processes = _spawn(cmds, Redirectable[in, out, stderr]) + processes = _spawn(cmds, Redirectable[in, out, stderr], tok) processes.in = in processes.out = out elseif read out = PipeEndpoint() - processes = _spawn(cmds, Redirectable[stdio, out, stderr]) + processes = _spawn(cmds, Redirectable[stdio, out, stderr], tok) processes.out = out elseif write in = PipeEndpoint() - processes = _spawn(cmds, Redirectable[in, stdio, stderr]) + processes = _spawn(cmds, Redirectable[in, stdio, stderr], tok) processes.in = in else stdio === devnull || throw(ArgumentError("no stream can be specified for `stdio` in no-access mode")) - processes = _spawn(cmds, Redirectable[devnull, devnull, stderr]) + processes = _spawn(cmds, Redirectable[devnull, devnull, stderr], tok) end return processes end @@ -476,10 +512,12 @@ end Run `command` and return the resulting output as an array of bytes. """ -function read(cmd::AbstractCmd) - procs = open(cmd, "r", devnull) - bytes = read(procs.out) - success(procs) || pipeline_error(procs) +function read(cmd::AbstractCmd; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok + procs = open(cmd, "r", devnull; cancel=tok) + bytes = read(procs.out; cancel=tok) + success(procs; cancel=tok) || pipeline_error(procs) return bytes::Vector{UInt8} end @@ -488,7 +526,8 @@ end Run `command` and return the resulting output as a `String`. """ -read(cmd::AbstractCmd, ::Type{String}) = String(read(cmd))::String +read(cmd::AbstractCmd, ::Type{String}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + String(read(cmd; cancel))::String """ run(command, args...; wait::Bool = true) @@ -511,13 +550,15 @@ Use [`pipeline`](@ref) to control I/O redirection. See also: [`Cmd`](@ref). """ -function run(cmds::AbstractCmd, args...; wait::Bool = true) +function run(cmds::AbstractCmd, args...; wait::Bool = true, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok if wait - ps = _spawn(cmds, spawn_opts_inherit(args...)) - success(ps) || pipeline_error(ps) + ps = _spawn(cmds, spawn_opts_inherit(args...), tok) + success(ps; cancel=tok) || pipeline_error(ps) else stdios = spawn_opts_swallow(args...) - ps = _spawn(cmds, stdios) + ps = _spawn(cmds, stdios, tok) # for each stdio input argument, guess whether the user # passed a `stdio` placeholder object as input, and thus # might be able to use the return AbstractProcess as an IO object @@ -544,11 +585,13 @@ const SIGHUP = 1 const SIGINT = 2 const SIGQUIT = 3 # !windows const SIGKILL = 9 +const SIGUSR1 = Sys.isapple() ? 30 : 10 # !windows const SIGPIPE = 13 # !windows const SIGTERM = 15 +const SIGINFO = 29 # apple/BSD only; use SIGUSR1 on linux function test_success(proc::Process) - @assert process_exited(proc) + @assert process_exited(proc) "process did not exit successfully" if proc.exitcode < 0 #TODO: this codepath is not currently tested throw(_UVError("could not start process " * repr(proc.cmd), proc.exitcode)) @@ -556,12 +599,14 @@ function test_success(proc::Process) return proc.exitcode == 0 && proc.termsignal == 0 end -function success(x::Process) - wait(x) +function success(x::Process; cancel::CancelTokenArg=DEFAULT_CANCEL) + wait(x; cancel) return test_success(x) end -success(procs::Vector{Process}) = mapreduce(success, &, procs) -success(procs::ProcessChain) = success(procs.processes) +success(procs::Vector{Process}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + mapreduce(p -> success(p; cancel), &, procs) +success(procs::ProcessChain; cancel::CancelTokenArg=DEFAULT_CANCEL) = + success(procs.processes; cancel) """ success(command) @@ -570,7 +615,11 @@ Run a command object, constructed with backticks (see the [Running External Prog section in the manual), and tell whether it was successful (exited with a code of 0). An exception is raised if the process cannot be started. """ -success(cmd::AbstractCmd) = success(_spawn(cmd)) +function success(cmd::AbstractCmd; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(cancel) + @cancel_check tok + return success(_spawn(cmd, tok); cancel=tok) +end """ @@ -626,7 +675,7 @@ permissions). function kill(p::Process, signum::Integer=SIGTERM) iolock_begin() if process_running(p) - @assert p.handle != C_NULL + @assert p.handle != C_NULL "invalid handle" err = ccall(:uv_process_kill, Int32, (Ptr{Cvoid}, Int32), p.handle, signum) if err != 0 && err != UV_ESRCH throw(_UVError("kill", err)) @@ -687,17 +736,21 @@ function process_status(s::Process) error("process status error") end -function wait(x::Process, syncd::Bool=true) +function wait(x::Process, syncd::Bool=true; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = check_cancel_arg(cancel) if !process_exited(x) iolock_begin() if !process_exited(x) preserve_handle(x) lock(x.exitnotify) iolock_end() + locked = true try - wait(x.exitnotify) + locked = false + wait(x.exitnotify, tok) + locked = true finally - unlock(x.exitnotify) + locked && unlock(x.exitnotify) unpreserve_handle(x) end else @@ -706,12 +759,13 @@ function wait(x::Process, syncd::Bool=true) end # and make sure all sync'd Tasks are complete too syncd && for t in x.syncd - wait(t) + wait(t, tok) end nothing end -wait(x::ProcessChain, syncd::Bool=true) = foreach(p -> wait(p, syncd), x.processes) +wait(x::ProcessChain, syncd::Bool=true; cancel::CancelTokenArg=DEFAULT_CANCEL) = + foreach(p -> wait(p, syncd; cancel), x.processes) show(io::IO, p::Process) = print(io, "Process(", p.cmd, ", ", process_status(p), ")") diff --git a/base/promotion.jl b/base/promotion.jl index f935c546915be..8ae39eaaeb0ec 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -21,6 +21,19 @@ typejoin() = Bottom typejoin(@nospecialize(t)) = (@_nospecializeinfer_meta; t) typejoin(@nospecialize(t), @nospecialize(s), @nospecialize(u)) = (@_foldable_meta; @_nospecializeinfer_meta; typejoin(typejoin(t, s), u)) typejoin(@nospecialize(t), @nospecialize(s), @nospecialize(u), ts...) = (@_foldable_meta; @_nospecializeinfer_meta; afoldl(typejoin, typejoin(t, s, u), ts...)) + +function _has_ancestor_typename(@nospecialize(a), name::Core.TypeName) + @_foldable_meta + @_nothrow_meta + @_nospecializeinfer_meta + a = a::DataType + while true + a.name === name && return true + a === Any && return false + a = supertype(a)::DataType + end +end + function typejoin(@nospecialize(a), @nospecialize(b)) @_foldable_meta @_nothrow_meta @@ -45,12 +58,27 @@ function typejoin(@nospecialize(a), @nospecialize(b)) return typejoin(typejoin(a.a, a.b), b) elseif isa(b, Union) return typejoin(a, typejoin(b.a, b.b)) + elseif isTypeEgal(a) || isTypeEgal(b) + a = isTypeEgal(a) ? typeof(type_parameter(a)) : a + b = isTypeEgal(b) ? typeof(type_parameter(b)) : b + return typejoin(a, b) + elseif isTypeEq(a) || isTypeEq(b) + # At least one operand is a `Type{X}` kind. We have already ruled out + # `a <: b`, `b <: a`, and any `UnionAll`/`Union`/`TypeVar`. The least supertype + # of a `Type{X}` kind is the abstract `Type`, so widen each kind to `Type` and + # join the two by subtyping. We compare directly instead of recursing through + # `typejoin`, because `Type === (Type{T} where T)` would re-enter this branch and + # not terminate. + a = isTypeEq(a) ? Type : a + b = isTypeEq(b) ? Type : b + return a <: b ? b : + b <: a ? a : Any end # a and b are DataTypes # We have to hide Constant info from inference, see #44390 a, b = inferencebarrier(a)::DataType, inferencebarrier(b)::DataType - if a <: Tuple - if !(b <: Tuple) + if a.name === Tuple.name + if !(b.name === Tuple.name) return Any end ap, bp = a.parameters, b.parameters @@ -94,23 +122,14 @@ function typejoin(@nospecialize(a), @nospecialize(b)) c[i] = i == length(c) && (isvarargtype(ai) || isvarargtype(bi)) ? Vararg{ci} : ci end return Tuple{c...} - elseif b <: Tuple + elseif b.name === Tuple.name return Any end while !(b === Any) - if a <: b.name.wrapper + if _has_ancestor_typename(a, b.name) while !(a.name === b.name) a = supertype(a)::DataType end - if a.name === Type.body.name - ap = a.parameters[1] - bp = b.parameters[1] - if ((isa(ap,TypeVar) && ap.lb === Bottom && ap.ub === Any) || - (isa(bp,TypeVar) && bp.lb === Bottom && bp.ub === Any)) - # handle special Type{T} supertype - return Type - end - end aprimary = a.name.wrapper # join on parameters n = length(a.parameters) @@ -178,7 +197,7 @@ Float64 """ function promote_typejoin(@nospecialize(a), @nospecialize(b)) c = typejoin(_promote_typesubtract(a), _promote_typesubtract(b)) - return Union{a, b, c}::Type + return Union{a, b, c} end _promote_typesubtract(@nospecialize(a)) = a === Any ? a : @@ -197,6 +216,8 @@ function promote_typejoin_union(::Type{T}) where T elseif T isa DataType T <: Tuple && return typejoin_union_tuple(T) return T + elseif isType(T) + return T else error("unreachable") # not a type?? end @@ -435,6 +456,10 @@ end -(x::Number, y::Number) = -(promote(x,y)...) /(x::Number, y::Number) = /(promote(x,y)...) ++%(x::Number, y::Number) = +%(promote(x,y)...) +*%(x::Number, y::Number) = *%(promote(x,y)...) +-%(x::Number, y::Number) = -%(promote(x,y)...) + """ ^(x, y) @@ -482,7 +507,11 @@ true ^(x::Number, y::Number) = ^(promote(x,y)...) fma(x::Number, y::Number, z::Number) = fma(promote(x,y,z)...) -muladd(x::Number, y::Number, z::Number) = muladd(promote(x,y,z)...) +function muladd(a::Number, b::Number, c::Number) + _a, _b, _c = promote(a, b, c) + ((a === false) || (b === false)) && return _c + return muladd(_a, _b, _c) +end ==(x::Number, y::Number) = (==)(promote(x,y)...) <( x::Real, y::Real) = (< )(promote(x,y)...) @@ -490,9 +519,7 @@ muladd(x::Number, y::Number, z::Number) = muladd(promote(x,y,z)...) rem(x::Real, y::Real) = rem(promote(x,y)...) mod(x::Real, y::Real) = mod(promote(x,y)...) - mod1(x::Real, y::Real) = mod1(promote(x,y)...) -fld1(x::Real, y::Real) = fld1(promote(x,y)...) max(x::Real, y::Real) = max(promote(x,y)...) min(x::Real, y::Real) = min(promote(x,y)...) diff --git a/base/public.jl b/base/public.jl index bc3d76e86eadc..7bcd53a6c9c3d 100644 --- a/base/public.jl +++ b/base/public.jl @@ -34,6 +34,9 @@ public # arrays has_offset_axes, require_one_based_indexing, + memoryindex, + unsetindex!, + unsetindex_atomic!, # collections IteratorEltype, @@ -52,11 +55,13 @@ public DL_LOAD_PATH, load_path, active_project, + active_manifest, # Reflection and introspection get_extension, isambiguous, isexpr, + isfieldatomic, isidentifier, issingletontype, identify_package, @@ -67,6 +72,7 @@ public isexported, ispublic, remove_linenums!, + generating_output, # AST handling IR, @@ -80,6 +86,10 @@ public isoperator, isunaryoperator, +# Integer math + uabs, + mul_hi, + # C interface cconvert, unsafe_convert, @@ -93,6 +103,8 @@ public @constprop, @locals, @propagate_inbounds, + @__doc__, + @nospecializeinfer, # External processes shell_escape, @@ -104,6 +116,13 @@ public # Strings escape_raw_string, + raw_substring, + unannotate, + +# Chars + ismalformed, + isoverlong, + show_invalid, # IO # types @@ -121,9 +140,13 @@ public # filesystem operations rename, +# promotion + promote_typejoin, + # misc notnothing, runtests, text_colors, depwarn, + blackbox, donotdelete diff --git a/base/range.jl b/base/range.jl index e9d28daf3ba3b..423b5b65d4b91 100644 --- a/base/range.jl +++ b/base/range.jl @@ -28,6 +28,16 @@ _colon(::Ordered, ::ArithmeticRounds, start::T, step, stop::T) where {T} = _colon(::Any, ::Any, start::T, step, stop::T) where {T} = StepRangeLen(start, step, convert(Integer, fld(stop - start, step)) + 1) +const _RangeWrappingInteger = Union{ + Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128} + +_range_wrapping_add(x::_RangeWrappingInteger, y::_RangeWrappingInteger) = x +% y +_range_wrapping_add(x, y) = x + y +_range_wrapping_sub(x::_RangeWrappingInteger, y::_RangeWrappingInteger) = x -% y +_range_wrapping_sub(x, y) = x - y +_range_wrapping_mul(x::_RangeWrappingInteger, y::_RangeWrappingInteger) = x *% y +_range_wrapping_mul(x, y) = x * y + """ (:)(start, [step], stop) @@ -49,6 +59,8 @@ The operator `:` is also used in indexing to select whole dimensions, e.g. in `A `:` is also used to [`quote`](@ref) code, e.g. `:(x + y) isa Expr` and `:x isa Symbol`. Since `:2 isa Int`, it does *not* create a range in indexing: `v[:2] == v[2] != v[begin:2]`. """ +(:)(::Any, ::Any, ::Any) + (:)(start::T, step, stop::T) where {T} = _colon(start, step, stop) (:)(start::T, step, stop::T) where {T<:Real} = _colon(start, step, stop) # without the second method above, the first method above is ambiguous with @@ -185,7 +197,7 @@ range_stop(stop) = range_start_stop(oftype(stop, 1), stop) range_stop(stop::Integer) = range_length(stop) function range_step_stop_length(step, a, len::Integer) - start = a - step * (len - oneunit(len)) + start = _range_wrapping_sub(a, _range_wrapping_mul(step, len - oneunit(len))) if start isa Signed # overflow in recomputing length from stop is okay return StepRange{typeof(start),typeof(step)}(start, step, convert(typeof(start), a)) @@ -196,7 +208,7 @@ end # Stop and length as the only argument function range_stop_length(a, len::Integer) step = oftype(a - a, 1) # assert that step is representable - start = a - (len - oneunit(len)) + start = _range_wrapping_sub(a, len - oneunit(len)) if start isa Signed # overflow in recomputing length from stop is okay return UnitRange(start, oftype(start, a)) @@ -207,7 +219,7 @@ end # Start and length as the only argument function range_start_length(a, len::Integer) step = oftype(a - a, 1) # assert that step is representable - stop = a + (len - oneunit(len)) + stop = _range_wrapping_add(a, len - oneunit(len)) if stop isa Signed # overflow in recomputing length from stop is okay return UnitRange(oftype(stop, a), stop) @@ -218,7 +230,7 @@ end range_start_stop(start, stop) = start:stop function range_start_step_length(a, step, len::Integer) - stop = a + step * (len - oneunit(len)) + stop = _range_wrapping_add(a, _range_wrapping_mul(step, len - oneunit(len))) if stop isa Signed # overflow in recomputing length from stop is okay return StepRange{typeof(stop),typeof(step)}(convert(typeof(stop), a), step, stop) @@ -356,7 +368,8 @@ function steprange_last(start, step, stop)::typeof(stop) else # Compute absolute value of difference between `start` and `stop` # (to simplify handling both signed and unsigned T and checking for signed overflow): - absdiff, absstep = stop > start ? (stop - start, step) : (start - stop, -step) + absdiff, absstep = stop > start ? (_range_wrapping_sub(stop, start), step) : + (_range_wrapping_sub(start, stop), -step) # Compute remainder as a non-negative number: if absdiff isa Signed && absdiff < zero(absdiff) @@ -377,9 +390,9 @@ function steprange_last_empty(start::Integer, step, stop)::typeof(stop) # empty range has a special representation where stop = start-1, # which simplifies arithmetic for Signed numbers if step > zero(step) - last = start - oneunit(step) + last = _range_wrapping_sub(start, oneunit(step)) else - last = start + oneunit(step) + last = _range_wrapping_add(start, oneunit(step)) end return last end @@ -424,7 +437,8 @@ end # if stop and start are integral, we know that their difference is a multiple of 1 unitrange_last(start::Integer, stop::Integer) = - stop >= start ? stop : convert(typeof(stop), start - oneunit(start - stop)) + stop >= start ? stop : convert(typeof(stop), + _range_wrapping_sub(start, oneunit(_range_wrapping_sub(start, stop)))) # otherwise, use `floor` as a more efficient way to compute modulus with step=1 unitrange_last(start, stop) = stop >= start ? convert(typeof(stop), start + floor(stop - start)) : @@ -455,6 +469,9 @@ end Base.AbstractOneTo Abstract type for ranges that start at 1 and have a step size of 1. + +!!! compat "Julia 1.13" + This type requires at least Julia 1.13. """ abstract type AbstractOneTo{T} <: AbstractUnitRange{T} end @@ -651,9 +668,8 @@ function print_range(io::IO, r::AbstractArray, if !haskey(io, :compact) io = IOContext(io, :compact => true) end - screenheight, screenwidth = sz[1] - 4, sz[2] + _, screenwidth = sz[1] - 4, sz[2] screenwidth -= length(pre) + length(post) - postsp = "" sepsize = length(sep) m = 1 # treat the range as a one-row matrix n = length(r) @@ -803,7 +819,7 @@ let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128}, # (near typemax) for types with known `unsigned` functions function length(r::OrdinalRange{T}) where T<:bigints s = step(r) - diff = last(r) - first(r) + diff = last(r) -% first(r) isempty(r) && return zero(diff) # Compute `(diff ÷ s) + 1` in a manner robust to signed overflow # by using the absolute values as unsigneds for non-empty ranges. @@ -819,10 +835,10 @@ let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128}, isempty(r) && return zero(ET) # n.b. !(s isa T) if s > 1 - diff = stop - start + diff = stop -% start a = convert(ET, div(unsigned(diff), s)) elseif s < -1 - diff = start - stop + diff = start -% stop a = convert(ET, div(unsigned(diff), -s)) elseif s > 0 a = convert(ET, div(checked_sub(stop, start), s)) @@ -969,7 +985,7 @@ unsafe_getindex(v::OneTo{T}, i::Integer) where T = convert(T, i) unsafe_getindex(v::AbstractRange{T}, i::Integer) where T = convert(T, first(v) + (i - oneunit(i))*step_hp(v)) function unsafe_getindex(r::StepRangeLen{T}, i::Integer) where T u = oftype(r.offset, i) - r.offset - T(r.ref + u*r.step) + convert(T, (r.ref + u*r.step)) end unsafe_getindex(r::LinRange, i::Integer) = lerpi(i-oneunit(i), r.lendiv, r.start, r.stop) @@ -993,7 +1009,7 @@ function getindex(r::AbstractUnitRange, s::AbstractUnitRange{T}) where {T<:Integ f = first(r) start = oftype(f, f + first(s) - firstindex(r)) len = length(s) - stop = oftype(f, start + (len - oneunit(len))) + stop = oftype(f, _range_wrapping_add(start, len - oneunit(len))) return range(start, stop) end end @@ -1166,6 +1182,17 @@ function ==(r::AbstractRange, s::AbstractRange) return true end +function cmp(r1::T, r2::T) where {T <: AbstractRange} + firstindex(r1) == firstindex(r2) || return cmp(firstindex(r1), firstindex(r2)) + (isempty(r1) || isempty(r2)) && return cmp(isempty(r2), isempty(r1)) + first(r1) != first(r2) && return cmp(first(r1), first(r2)) + # Assume that ranges are monotonic and use the last shared element as a high precision proxy for step. + n = min(lastindex(r1), lastindex(r2)) + x1, x2 = r1[n], r2[n] + x1 != x2 && return cmp(x1, x2) + cmp(length(r1), length(r2)) +end + intersect(r::OneTo, s::OneTo) = OneTo(min(r.stop,s.stop)) union(r::OneTo, s::OneTo) = OneTo(max(r.stop,s.stop)) @@ -1216,7 +1243,7 @@ function intersect(r::StepRange, s::StepRange) start2, step2, stop2 = first_step_last_ascending(s) a = lcm(step1, step2) - g, x, y = gcdx(step1, step2) + g, x, _ = gcdx(step1, step2) if !iszero(rem(start1 - start2, g)) # Unaligned, no overlap possible. @@ -1378,7 +1405,7 @@ promote_rule(::Type{LinRange{A,L}}, b::Type{StepRangeLen{T2,R2,S2,L2}}) where {A function vcat(rs::AbstractRange{T}...) where T n::Int = 0 for ra in rs - n += length(ra) + n = checked_add(n, length(ra)) end a = Vector{T}(undef, n) i = 1 diff --git a/base/rational.jl b/base/rational.jl index e04f14760d8f4..ac3571547c64a 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -52,7 +52,7 @@ Rational(n::Integer) = unsafe_rational(n, one(n)) """ divgcd(x::Integer, y::Integer) -Returns `(x÷gcd(x,y), y÷gcd(x,y))`. +Return `(x÷gcd(x,y), y÷gcd(x,y))`. See also [`div`](@ref), [`gcd`](@ref). """ @@ -105,8 +105,42 @@ function //(x::Rational, y::Rational) end //(x::Complex, y::Real) = complex(real(x)//y, imag(x)//y) -//(x::Number, y::Complex) = x*conj(y)//abs2(y) +# Return a complex numerator and real denominator +# of the exact inverse of a Complex number. +function _complex_exact_inv(y::Complex) + c, d = reim(y) + num = if (isinf(c) | isinf(d)) + conj(zero(y)) + else + conj(y) + end + num, abs2(y) +end +function _complex_exact_inv(y::Complex{<:Integer}) + c, d = reim(y) + c_r, d_r = divgcd(c, d) + abs2y_r = checked_add(checked_mul(c, c_r), checked_mul(d, d_r)) + num = complex(c_r, checked_neg(d_r)) + num, abs2y_r +end + +function //(x::Number, y::Complex) + num, den = _complex_exact_inv(y) + (x * num) // den +end +function //(x::Integer, y::Complex{<:Integer}) + complex(x) // y +end +function //(x::Complex{<:Integer}, y::Complex{<:Integer}) + a, b, c, d = promote(reim(x)..., reim(y)...) + c_r, d_r = divgcd(c, d) + abs2y_r = checked_add(checked_mul(c, c_r), checked_mul(d, d_r)) + complex( + checked_add(checked_mul(a, c_r), checked_mul(b, d_r)), + checked_add(checked_mul(b, c_r), checked_neg(checked_mul(a, d_r))) + )//abs2y_r +end //(X::AbstractArray, y::Number) = X .// y @@ -210,11 +244,8 @@ julia> typeof(numerator(a)) BigInt ``` """ -function rationalize(::Type{T}, x::Union{AbstractFloat, Rational}, tol::Real) where T<:Integer - if tol < 0 - throw(ArgumentError("negative tolerance $tol")) - end - +function rationalize(::Type{T}, x::AbstractFloat, tol::Real) where T<:Integer + tol < 0 && throw(ArgumentError("Tolerance can not be negative. tol=$tol")) T<:Unsigned && x < 0 && __throw_negate_unsigned() isnan(x) && return T(x)//one(T) isinf(x) && return unsafe_rational(x < 0 ? -one(T) : one(T), zero(T)) @@ -228,21 +259,24 @@ function rationalize(::Type{T}, x::Union{AbstractFloat, Rational}, tol::Real) wh y = one(x) tolx = oftype(x, tol) nt, t, tt = tolx, zero(tolx), tolx - ia = np = nq = zero(T) # compute the successive convergents of the continued fraction # np // nq = (p*a + pp) // (q*a + qq) while r > nt try ia = convert(T,a) - np = checked_add(checked_mul(ia,p),pp) nq = checked_add(checked_mul(ia,q),qq) p, pp = np, p q, qq = nq, q catch e isa(e,InexactError) || isa(e,OverflowError) || rethrow() - return p // q + (a ≤ 2 || ((-p == p) && (p < 0))) && return p // q + # find best semiconvergent that fits in T + ia_p = iszero(p) ? typemax(T) : fld(typemax(T) - abs(pp), abs(p)) + ia_q = iszero(q) ? typemax(T) : fld(typemax(T) - qq, q) + ia = min(ia_p, ia_q) + return ia > a/2 ? (ia*p + pp) // (ia*q + qq) : p // q end # naive approach of using @@ -259,25 +293,38 @@ function rationalize(::Type{T}, x::Union{AbstractFloat, Rational}, tol::Real) wh # find optimal semiconvergent # smallest a such that x-a*y < a*t+tt - a = cld(x-tt,y+t) + a_min = cld(x-tt,y+t) try - ia = convert(T,a) + ia = convert(T,a_min) np = checked_add(checked_mul(ia,p),pp) nq = checked_add(checked_mul(ia,q),qq) return np // nq catch e isa(e,InexactError) || isa(e,OverflowError) || rethrow() - return p // q + (a ≤ 2 || ((-p == p) && (p < 0))) && return p // q + ia_p = iszero(p) ? typemax(T) : fld(typemax(T) - abs(pp), abs(p)) + ia_q = iszero(q) ? typemax(T) : fld(typemax(T) - qq, q) + ia = min(ia_p, ia_q) + return ia > a/2 ? (ia*p + pp) // (ia*q + qq) : p // q end end rationalize(::Type{T}, x::AbstractFloat; tol::Real = eps(x)) where {T<:Integer} = rationalize(T, x, tol) rationalize(x::Real; kvs...) = rationalize(Int, x; kvs...) rationalize(::Type{T}, x::Complex; kvs...) where {T<:Integer} = Complex(rationalize(T, x.re; kvs...), rationalize(T, x.im; kvs...)) rationalize(x::Complex; kvs...) = Complex(rationalize(Int, x.re; kvs...), rationalize(Int, x.im; kvs...)) -rationalize(::Type{T}, x::Rational; tol::Real = 0) where {T<:Integer} = rationalize(T, x, tol) -rationalize(x::Rational; kvs...) = x +rationalize(::Type{T}, x::Rational; tol::Real = eps(float(x))) where {T<:Integer} = rationalize(T, x, tol) +rationalize(x::Rational{T}; kvs...) where {T<:Integer} = rationalize(T, x; kvs...) +function rationalize(::Type{T}, x::Rational, tol::Real) where {T<:Integer} + T<:Unsigned && x < 0 && __throw_negate_unsigned() + if !hastypemax(T) || (typemin(T) ≤ x.num ≤ typemax(T) && x.den ≤ typemax(T)) + return Rational{T}(x) + end + isfinite(float(x)) && tol ≥ eps(float(x))/2 || throw(InexactError(:rationalize, Rational{T}, x)) + return rationalize(T, float(x), tol) +end rationalize(x::Integer; kvs...) = Rational(x) function rationalize(::Type{T}, x::Integer; kvs...) where {T<:Integer} + T<:Unsigned && x < 0 && __throw_negate_unsigned() if Base.hastypemax(T) # BigInt doesn't x < typemin(T) && return unsafe_rational(-one(T), zero(T)) x > typemax(T) && return unsafe_rational(one(T), zero(T)) @@ -354,21 +401,39 @@ function -(x::Rational{T}) where T<:Unsigned end function +(x::Rational, y::Rational) + xp, _ = promote(x, y)::NTuple{2,Rational} + if isinf(x) && x == y + return xp + end + xd, yd = divgcd(promote(x.den, y.den)...) + Rational(checked_add(checked_mul(x.num, yd), checked_mul(y.num, xd)), checked_mul(x.den, yd)) +end + +function +%(x::Rational, y::Rational) xp, yp = promote(x, y)::NTuple{2,Rational} if isinf(x) && x == y return xp end xd, yd = divgcd(promote(x.den, y.den)...) - Rational(checked_add(checked_mul(x.num,yd), checked_mul(y.num,xd)), checked_mul(x.den,yd)) + Rational(+%(*%(x.num,yd), *%(y.num,xd)), *%(x.den,yd)) end function -(x::Rational, y::Rational) + xp, _ = promote(x, y)::NTuple{2,Rational} + if isinf(x) && x == -y + return xp + end + xd, yd = divgcd(promote(x.den, y.den)...) + Rational(checked_sub(checked_mul(x.num, yd), checked_mul(y.num, xd)), checked_mul(x.den, yd)) +end + +function -%(x::Rational, y::Rational) xp, yp = promote(x, y)::NTuple{2,Rational} if isinf(x) && x == -y return xp end xd, yd = divgcd(promote(x.den, y.den)...) - Rational(checked_sub(checked_mul(x.num,yd), checked_mul(y.num,xd)), checked_mul(x.den,yd)) + Rational(-%(*%(x.num, yd), *%(y.num, xd)), *%(x.den, yd)) end for (op,chop) in ((:rem,:rem), (:mod,:mod)) @@ -421,8 +486,8 @@ function *(x::Bool, y::T)::promote_type(Bool,T) where T<:Rational return ifelse(x, y, copysign(zero(y), y)) end *(y::Rational, x::Bool) = x * y -/(x::Rational, y::Union{Rational, Integer, Complex{<:Union{Integer,Rational}}}) = x//y -/(x::Union{Integer, Complex{<:Union{Integer,Rational}}}, y::Rational) = x//y +/(x::Rational, y::Union{Rational, Integer}) = x//y +/(x::Integer, y::Rational) = x//y inv(x::Rational{T}) where {T} = checked_den(x.den, x.num) fma(x::Rational, y::Rational, z::Rational) = x*y+z @@ -443,7 +508,7 @@ fma(x::Rational, y::Rational, z::Rational) = x*y+z function ==(x::AbstractFloat, q::Rational) if isfinite(x) - (count_ones(q.den) == 1) & (x*q.den == q.num) + (count_ones(q.den) == 1) && (ldexp(x, top_set_bit(q.den-1)) == q.num) else x == q.num/q.den end @@ -620,7 +685,7 @@ function hash(x::Rational{<:BitInteger64}, h::UInt) end end h = hash_integer(pow, h) - h = hash_integer((pow > 0) ? (num << (pow % 64)) : num, h) + h = hash_integer((pow > 0) ? (num << (pow % 8)) : num, h) return h end diff --git a/base/rawbigfloats.jl b/base/rawbigfloats.jl index 4377edfc463d8..a8b46b313bdb1 100644 --- a/base/rawbigfloats.jl +++ b/base/rawbigfloats.jl @@ -30,7 +30,7 @@ word_is_nonzero(x::BigFloatData, v::Val) = let x = x end """ -Returns a `Bool` indicating whether the `len` least significant words +Return a `Bool` indicating whether the `len` least significant words of `x` are nonzero. """ function tail_is_nonzero(x::BigFloatData, len::Int, ::Val{:words}) @@ -38,7 +38,7 @@ function tail_is_nonzero(x::BigFloatData, len::Int, ::Val{:words}) end """ -Returns a `Bool` indicating whether the `len` least significant bits of +Return a `Bool` indicating whether the `len` least significant bits of the `i`-th (zero-based index) word of `x` are nonzero. """ function tail_is_nonzero(x::BigFloatData, len::Int, i::Int, ::Val{:word}) @@ -47,7 +47,7 @@ function tail_is_nonzero(x::BigFloatData, len::Int, i::Int, ::Val{:word}) end """ -Returns a `Bool` indicating whether the `len` least significant bits of +Return a `Bool` indicating whether the `len` least significant bits of `x` are nonzero. """ function tail_is_nonzero(x::BigFloatData, len::Int, ::Val{:bits}) @@ -61,14 +61,14 @@ function tail_is_nonzero(x::BigFloatData, len::Int, ::Val{:bits}) end """ -Returns a `Bool` that is the `i`-th (zero-based index) bit of `x`. +Return a `Bool` that is the `i`-th (zero-based index) bit of `x`. """ function get_elem(x::Unsigned, i::Int, ::Val{:bits}, ::Val{:ascending}) (x >>> i) % Bool end """ -Returns a `Bool` that is the `i`-th (zero-based index) bit of `x`. +Return a `Bool` that is the `i`-th (zero-based index) bit of `x`. """ function get_elem(x::BigFloatData, i::Int, ::Val{:bits}, v::Val{:ascending}) vb = Val(:bits) @@ -82,7 +82,7 @@ function get_elem(x::BigFloatData, i::Int, ::Val{:bits}, v::Val{:ascending}) end """ -Returns an integer of type `R`, consisting of the `len` most +Return an integer of type `R`, consisting of the `len` most significant bits of `x`. If there are less than `len` bits in `x`, the least significant bits are zeroed. """ diff --git a/base/reduce.jl b/base/reduce.jl index 743713360ed35..4a87e7583432b 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -198,10 +198,11 @@ foldl(op, itr; kw...) = mapfoldl(identity, op, itr; kw...) ## foldr & mapfoldr function mapfoldr_impl(f, op, nt, itr) - op′, itr′ = _xfadjoint(BottomRF(FlipArgs(op)), Generator(f, itr)) - return foldl_impl(op′, nt, _reverse_iter(itr′)) + op′, itr′ = _xfadjoint(BottomRF(FlipArgs(op)), Generator(f, _reverse_iter(itr))) + return foldl_impl(op′, nt, itr′) end + _reverse_iter(itr) = Iterators.reverse(itr) _reverse_iter(itr::Union{Tuple,NamedTuple}) = length(itr) <= 32 ? reverse(itr) : Iterators.reverse(itr) #33235 @@ -218,7 +219,7 @@ Like [`mapreduce`](@ref), but with guaranteed right associativity, as in [`foldr provided, the keyword argument `init` will be used exactly once. In general, it will be necessary to provide `init` to work with empty collections. """ -mapfoldr(f, op, itr; init=_InitialValue()) = mapfoldr_impl(f, op, init, itr) +mapfoldr(f::F, op::F2, itr; init=_InitialValue()) where {F,F2} = mapfoldr_impl(f, op, init, itr) """ @@ -237,7 +238,7 @@ julia> foldr(=>, 1:4; init=0) 1 => (2 => (3 => (4 => 0))) ``` """ -foldr(op, itr; kw...) = mapfoldr(identity, op, itr; kw...) +foldr(op::F, itr; kw...) where {F} = mapfoldr(identity, op, itr; kw...) ## reduce & mapreduce @@ -374,11 +375,19 @@ mapreduce_empty(f::typeof(abs), ::typeof(max), T) = abs(zero(T)) mapreduce_empty(f::typeof(abs2), ::typeof(max), T) = abs2(zero(T)) # For backward compatibility: -mapreduce_empty_iter(f, op, itr, ItrEltype) = +mapreduce_empty_iter(f::F, op::F2, itr, ItrEltype) where {F,F2} = reduce_empty_iter(MappingRF(f, op), itr, ItrEltype) @inline reduce_empty_iter(op, itr) = reduce_empty_iter(op, itr, IteratorEltype(itr)) @inline reduce_empty_iter(op, itr, ::HasEltype) = reduce_empty(op, eltype(itr)) +# a homogeneous tuple binds the element type as a static parameter, which stays +# precise for abstract `Tuple{Vararg{T}}` queries where `eltype` of the +# `@nospecialize`d tuple type does not (#61323). `T` is undefined only for the +# empty tuple (vacuous `Vararg` match), whose eltype is `Union{}`; the +# `@isdefined` guard keeps this method's intentional unbound sparam (see the +# `detect_unbound_args` allow-list in test/ambiguous.jl), like `_eltype_ntuple`. +@inline reduce_empty_iter(op, itr::Tuple{Vararg{T}}, ::HasEltype) where {T} = + reduce_empty(op, @isdefined(T) ? T : Union{}) reduce_empty_iter(op, itr, ::EltypeUnknown) = throw(ArgumentError(""" reducing over an empty collection of unknown element type is not allowed. You may be able to prevent this error by supplying an `init` value to the reducer.""")) diff --git a/base/reducedim.jl b/base/reducedim.jl index ba3434494bc0b..12b07c65bf578 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -49,7 +49,6 @@ for (Op, initval) in ((:(typeof(and_all)), true), (:(typeof(or_any)), false)) @eval initarray!(a::AbstractArray, ::Any, ::$(Op), init::Bool, src::AbstractArray) = (init && fill!(a, $initval); a) end -# reducedim_initarray is called by reducedim_initarray(A::AbstractArrayOrBroadcasted, region, init, ::Type{R}) where {R} = fill!(similar(A,R,reduced_indices(A,region)), init) reducedim_initarray(A::AbstractArrayOrBroadcasted, region, init::T) where {T} = reducedim_initarray(A, region, init, T) @@ -240,7 +239,7 @@ Extract first entry of slices of array A into existing array R. copyfirst!(R::AbstractArray, A::AbstractArray) = mapfirst!(identity, R, A) function mapfirst!(f::F, R::AbstractArray, A::AbstractArray{<:Any,N}) where {N, F} - lsiz = check_reducedims(R, A) + check_reducedims(R, A) t = _firstreducedslice(axes(R), axes(A)) map!(f, R, view(A, t...)) end @@ -260,7 +259,6 @@ function _mapreducedim!(f, op, R::AbstractArray, A::AbstractArrayOrBroadcasted) if has_fast_linear_indexing(A) && lsiz > 16 # use mapreduce_impl, which is probably better tuned to achieve higher performance - nslices = div(length(A), lsiz) ibase = first(LinearIndices(A))-1 for i in eachindex(R) r = op(@inbounds(R[i]), mapreduce_impl(f, op, A, ibase+1, ibase+lsiz)) @@ -271,7 +269,7 @@ function _mapreducedim!(f, op, R::AbstractArray, A::AbstractArrayOrBroadcasted) end indsAt, indsRt = safe_tail(axes(A)), safe_tail(axes(R)) # handle d=1 manually keep, Idefault = Broadcast.shapeindexer(indsRt) - if reducedim1(R, A) + if reducedim1(R) # keep the accumulator as a local variable when reducing along the first dimension i1 = first(axes1(R)) for IA in CartesianIndices(indsAt) @@ -327,23 +325,38 @@ julia> mapreduce(isodd, |, a, dims=1) 1 1 1 1 ``` """ -mapreduce(f, op, A::AbstractArrayOrBroadcasted; dims=:, init=_InitialValue()) = +mapreduce(f, op, A::AbstractArrayOrBroadcasted; dims::D=:, init=_InitialValue()) where {D} = _mapreduce_dim(f, op, init, A, dims) mapreduce(f, op, A::AbstractArrayOrBroadcasted, B::AbstractArrayOrBroadcasted...; kw...) = reduce(op, map(f, A, B...); kw...) -_mapreduce_dim(f, op, nt, A::AbstractArrayOrBroadcasted, ::Colon) = - mapfoldl_impl(f, op, nt, A) +function _mapreduce_dim(f, op, nt, A::AbstractArrayOrBroadcasted, ::Colon) + # equivalent to `mapfoldl_impl(f, op, nt, A)` this logic is expanded here + # to work around limitations in inference's recursion heuristic + y = iterate(A) + y === nothing && return nt + v = op(nt, f(y[1])) + for x in Iterators.rest(A, y[2]) + v = op(v, f(x)) + end + return v +end _mapreduce_dim(f, op, ::_InitialValue, A::AbstractArrayOrBroadcasted, ::Colon) = _mapreduce(f, op, IndexStyle(A), A) -_mapreduce_dim(f, op, nt, A::AbstractArrayOrBroadcasted, dims) = +_mapreduce_dim(f, op, nt, A::AbstractArrayOrBroadcasted, dims::D) where {D} = mapreducedim!(f, op, reducedim_initarray(A, dims, nt), A) -_mapreduce_dim(f, op, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims) = +_mapreduce_dim(f, op, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims::D) where {D} = mapreducedim!(f, op, reducedim_init(f, op, A, dims), A) +_mapreduce_dim(f, op, nt, R::ReshapedArray, ::Colon) = + _mapreduce_dim(f, op, nt, parent(R), :) + +_mapreduce_dim(f, op, i::_InitialValue, R::ReshapedArray, ::Colon) = + _mapreduce_dim(f, op, i, parent(R), :) + """ reduce(f, A::AbstractArray; dims=:, [init]) @@ -409,8 +422,8 @@ julia> count(<=(2), A, dims=2) 0 ``` """ -count(A::AbstractArrayOrBroadcasted; dims=:, init=0) = count(identity, A; dims, init) -count(f, A::AbstractArrayOrBroadcasted; dims=:, init=0) = _count(f, A, dims, init) +count(A::AbstractArrayOrBroadcasted; dims::D=:, init=0) where {D} = count(identity, A; dims, init) +count(f, A::AbstractArrayOrBroadcasted; dims::D=:, init=0) where {D} = _count(f, A, dims, init) _count(f, A::AbstractArrayOrBroadcasted, dims::Colon, init) = _simple_count(f, A, init) _count(f, A::AbstractArrayOrBroadcasted, dims, init) = mapreduce(_bool(f), add_sum, A; dims, init) @@ -871,7 +884,7 @@ julia> all(i -> i > 0, A, dims=2) 1 ``` """ -all(::Function, ::AbstractArray; dims) +all(f, ::AbstractArray; dims) """ all!(r, A) @@ -945,7 +958,7 @@ julia> any(i -> i > 0, A, dims=2) 1 ``` """ -any(::Function, ::AbstractArray; dims) +any(f, ::AbstractArray; dims) """ any!(r, A) @@ -980,21 +993,21 @@ for (fname, _fname, op) in [(:sum, :_sum, :add_sum), (:prod, :_prod, mapf = fname === :extrema ? :(ExtremaMap(f)) : :f @eval begin # User-facing methods with keyword arguments - @inline ($fname)(a::AbstractArray; dims=:, kw...) = ($_fname)(a, dims; kw...) - @inline ($fname)(f, a::AbstractArray; dims=:, kw...) = ($_fname)(f, a, dims; kw...) + @inline ($fname)(a::AbstractArray; dims::D=:, kw...) where {D} = ($_fname)(a, dims; kw...) + @inline ($fname)(f, a::AbstractArray; dims::D=:, kw...) where {D} = ($_fname)(f, a, dims; kw...) # Underlying implementations using dispatch ($_fname)(a, ::Colon; kw...) = ($_fname)(identity, a, :; kw...) ($_fname)(f, a, ::Colon; kw...) = mapreduce($mapf, $op, a; kw...) + + ($_fname)(R::ReshapedArray, ::Colon) = ($fname)(parent(R)) end end -any(a::AbstractArray; dims=:) = _any(a, dims) -any(f::Function, a::AbstractArray; dims=:) = _any(f, a, dims) -_any(a, ::Colon) = _any(identity, a, :) -all(a::AbstractArray; dims=:) = _all(a, dims) -all(f::Function, a::AbstractArray; dims=:) = _all(f, a, dims) -_all(a, ::Colon) = _all(identity, a, :) +any(a::AbstractArray; kw...) = any(identity, a; kw...) +any(f, a::AbstractArray; dims::D=:) where D = _any(f, a, dims) +all(a::AbstractArray; kw...) = all(identity, a; kw...) +all(f, a::AbstractArray; dims::D=:) where D = _all(f, a, dims) for (fname, op) in [(:sum, :add_sum), (:prod, :mul_prod), (:maximum, :max), (:minimum, :min), @@ -1004,12 +1017,12 @@ for (fname, op) in [(:sum, :add_sum), (:prod, :mul_prod), _fname = Symbol('_', fname) mapf = fname === :extrema ? :(ExtremaMap(f)) : :f @eval begin - $(fname!)(f::Function, r::AbstractArray, A::AbstractArray; init::Bool=true) = + $(fname!)(f, r::AbstractArray, A::AbstractArray; init::Bool=true) = mapreducedim!($mapf, $(op), initarray!(r, $mapf, $(op), init, A), A) $(fname!)(r::AbstractArray, A::AbstractArray; init::Bool=true) = $(fname!)(identity, r, A; init=init) - $(_fname)(A, dims; kw...) = $(_fname)(identity, A, dims; kw...) - $(_fname)(f, A, dims; kw...) = mapreduce($mapf, $(op), A; dims=dims, kw...) + $(_fname)(A, dims::D; kw...) where {D} = $(_fname)(identity, A, dims; kw...) + $(_fname)(f, A, dims::D; kw...) where {D} = mapreduce($mapf, $(op), A; dims=dims, kw...) end end @@ -1018,7 +1031,7 @@ end # function findminmax!(f, op, Rval, Rind, A::AbstractArray{T,N}) where {T,N} (isempty(Rval) || isempty(A)) && return Rval, Rind - lsiz = check_reducedims(Rval, A) + check_reducedims(Rval, A) for i = 1:N axes(Rval, i) == axes(Rind, i) || throw(DimensionMismatch("Find-reduction: outputs must have the same indices")) end @@ -1029,7 +1042,7 @@ function findminmax!(f, op, Rval, Rind, A::AbstractArray{T,N}) where {T,N} ks = keys(A) y = iterate(ks) zi = zero(eltype(ks)) - if reducedim1(Rval, A) + if reducedim1(Rval) i1 = first(axes1(Rval)) for IA in CartesianIndices(indsAt) IR = Broadcast.newindex(IA, keep, Idefault) @@ -1100,8 +1113,8 @@ julia> findmin(A, dims=2) ([1.0; 3.0;;], CartesianIndex{2}[CartesianIndex(1, 1); CartesianIndex(2, 1);;]) ``` """ -findmin(A::AbstractArray; dims=:) = _findmin(A, dims) -_findmin(A, dims) = _findmin(identity, A, dims) +findmin(A::AbstractArray; dims::D=:) where {D} = _findmin(A, dims) +_findmin(A, dims::D) where {D} = _findmin(identity, A, dims) """ findmin(f, A; dims) -> (f(x), index) @@ -1123,9 +1136,9 @@ julia> findmin(abs2, A, dims=2) ([1.0; 0.25;;], CartesianIndex{2}[CartesianIndex(1, 1); CartesianIndex(2, 1);;]) ``` """ -findmin(f, A::AbstractArray; dims=:) = _findmin(f, A, dims) +findmin(f, A::AbstractArray; dims::D=:) where {D} = _findmin(f, A, dims) -function _findmin(f, A, region) +function _findmin(f, A, region::D) where {D} ri = reduced_indices0(A, region) if isempty(A) if prod(map(length, reduced_indices(A, region))) != 0 @@ -1173,8 +1186,8 @@ julia> findmax(A, dims=2) ([2.0; 4.0;;], CartesianIndex{2}[CartesianIndex(1, 2); CartesianIndex(2, 2);;]) ``` """ -findmax(A::AbstractArray; dims=:) = _findmax(A, dims) -_findmax(A, dims) = _findmax(identity, A, dims) +findmax(A::AbstractArray; dims::D=:) where {D} = _findmax(A, dims) +_findmax(A, dims::D) where {D} = _findmax(identity, A, dims) """ findmax(f, A; dims) -> (f(x), index) @@ -1196,9 +1209,9 @@ julia> findmax(abs2, A, dims=2) ([1.0; 4.0;;], CartesianIndex{2}[CartesianIndex(1, 1); CartesianIndex(2, 2);;]) ``` """ -findmax(f, A::AbstractArray; dims=:) = _findmax(f, A, dims) +findmax(f, A::AbstractArray; dims::D=:) where {D} = _findmax(f, A, dims) -function _findmax(f, A, region) +function _findmax(f, A, region::D) where {D} ri = reduced_indices0(A, region) if isempty(A) if prod(map(length, reduced_indices(A, region))) != 0 @@ -1219,10 +1232,10 @@ function _findminmax_inittype(f, A::AbstractArray) # Second conditional: handle missing specifically, as most often, f(missing) = missing; # certainly, some predicate functions return Bool, but not all. # Else, return the type of the transformation. - Tr = v0 isa T ? T : Missing <: eltype(A) ? Union{Missing, typeof(v0)} : typeof(v0) + return v0 isa T ? T : Missing <: eltype(A) ? Union{Missing, typeof(v0)} : typeof(v0) end -reducedim1(R, A) = length(axes1(R)) == 1 +reducedim1(R) = length(axes1(R)) == 1 """ argmin(A; dims) -> indices @@ -1247,7 +1260,7 @@ julia> argmin(A, dims=2) CartesianIndex(2, 1) ``` """ -argmin(A::AbstractArray; dims=:) = findmin(A; dims=dims)[2] +argmin(A::AbstractArray; dims::D=:) where {D} = findmin(A; dims=dims)[2] """ argmax(A; dims) -> indices @@ -1272,4 +1285,4 @@ julia> argmax(A, dims=2) CartesianIndex(2, 2) ``` """ -argmax(A::AbstractArray; dims=:) = findmax(A; dims=dims)[2] +argmax(A::AbstractArray; dims::D=:) where {D} = findmax(A; dims=dims)[2] diff --git a/base/reflection.jl b/base/reflection.jl index 5ad05b615ddfd..44eac1f74e6e1 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -34,7 +34,7 @@ function code_lowered(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); gener code = ccall(:jl_code_for_staged, Ref{CodeInfo}, (Any, UInt, Ptr{Cvoid}), m, world, C_NULL) else error("Could not expand generator for `@generated` method ", m, ". ", - "This can happen if the provided argument types (", t, ") are ", + "This can happen if the provided argument types (", argtypes, ") are ", "not concrete types, but the `generated` argument is `true`.") end else @@ -103,7 +103,7 @@ struct CodegenParams If enabled, generate the necessary code to support the --code-coverage command line flag to julia itself. Note that the option itself does not enable code coverage. Rather, it merely generates the support code necessary - to code coverage if requested by the command line option. + to perform code coverage if requested by the command line option. """ code_coverage::Cint @@ -160,27 +160,51 @@ struct CodegenParams targets. The option may be disabled for use in environments where the julia runtime is unavailable, but is otherwise recommended to be enabled, even if lazy resolution is not required, as the Julia PLT mechanism may have superior - performance compared to the native platform mechanism. The options is enabled by default. + performance compared to the native platform mechanism. The option is enabled by default. """ use_jlplt::Cint """ - If enabled emit LLVM IR for all functions even if wouldn't be compiled - for some reason (i.e functions that return a constant value). + If enabled emit LLVM IR for all functions even if they wouldn't be compiled + for some reason (i.e. functions that return a constant value). """ force_emit_all::Cint + """ + When enabled, run the MemorySanitizer pass. + """ + sanitize_memory::Cint + """ + When enabled, run the ThreadSanitizer pass. + """ + sanitize_thread::Cint + """ + When enabled, run the AddressSanitizer pass. + """ + sanitize_address::Cint + + """ + When enabled, generate names that are globally unique in this Julia session, + across all code generated with this flag set. Intended for llvmpasses + tests. + """ + unique_names::Cint + function CodegenParams(; track_allocations::Bool=true, code_coverage::Bool=true, prefer_specsig::Bool=false, gnu_pubnames::Bool=true, debug_info_kind::Cint = default_debug_info_kind(), debug_info_level::Cint = Cint(JLOptions().debug_level), safepoint_on_entry::Bool=true, - gcstack_arg::Bool=true, use_jlplt::Bool=true, force_emit_all::Bool=false) + gcstack_arg::Bool=true, use_jlplt::Bool=true, force_emit_all::Bool=false, + sanitize_memory::Bool=false, sanitize_thread::Bool=false, sanitize_address::Bool=false, + unique_names::Bool=false) return new( Cint(track_allocations), Cint(code_coverage), Cint(prefer_specsig), Cint(gnu_pubnames), debug_info_kind, debug_info_level, Cint(safepoint_on_entry), - Cint(gcstack_arg), Cint(use_jlplt), Cint(force_emit_all)) + Cint(gcstack_arg), Cint(use_jlplt), Cint(force_emit_all), + Cint(sanitize_memory), Cint(sanitize_thread), Cint(sanitize_address), + Cint(unique_names)) end end @@ -196,7 +220,7 @@ end """ code_typed(f, types; kw...) -Returns an array of type-inferred lowered form (IR) for the methods matching the given +Return an array of type-inferred lowered form (IR) for the methods matching the given generic function and type signature. # Keyword Arguments @@ -260,7 +284,9 @@ function code_typed(@nospecialize(f), @nospecialize(types=default_tt(f)); kwargs return code_typed_by_type(tt; kwargs...) end -# support 'functor'-like queries, such as `(::Foo)(::Int, ::Int)` via `code_typed((Foo, Int, Int))` +# support queries with signatures rather than objects to better support +# non-singleton function objects such as `(::Foo)(::Int, ::Int)` +# via `code_typed((Foo, Int, Int))` or `code_typed(Tuple{Foo, Int, Int})`. function code_typed(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) tt = to_tuple_type(argtypes) return code_typed_by_type(tt; kwargs...) @@ -306,7 +332,7 @@ function invoke_interp_compiler(interp, fname::Symbol, args...) T = typeof(interp) while true Tname = typename(T).name - Tname === :Any && error("Expected Interpreter") + Tname === :Any && error("Expected AbstractInterpreter") Tname === :AbstractInterpreter && break T = supertype(T) end @@ -314,6 +340,21 @@ function invoke_interp_compiler(interp, fname::Symbol, args...) end end +function invoke_mt_compiler(mt, fname::Symbol, args...) + if mt === nothing + return invoke_default_compiler(fname, args...) + else + T = typeof(mt) + while true + Tname = typename(T).name + Tname === :Any && error("Expected MethodTableView") + Tname === :MethodTableView && break + T = supertype(T) + end + return getglobal(typename(T).module, fname)(args...) + end +end + """ code_typed_by_type(types::Type{<:Tuple}; ...) @@ -564,7 +605,7 @@ function return_types(@nospecialize(f), @nospecialize(types=default_tt(f)); interp = passed_interp === nothing ? invoke_default_compiler(:_default_interp, world) : interp check_generated_context(world) if isa(f, Core.OpaqueClosure) - _, rt = only(code_typed_opaque_closure(f, types; Compiler)) + _, rt = only(code_typed_opaque_closure(f, types; interp=passed_interp)) return Any[rt] elseif isa(f, Core.Builtin) return Any[_builtin_return_type(passed_interp, interp, f, types)] @@ -586,7 +627,7 @@ end world::UInt=get_world_counter(), interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world)) -> rt::Type -Returns an inferred return type of the function call specified by `f` and `types`. +Return an inferred return type of the function call specified by `f` and `types`. # Arguments - `f`: The function to analyze. @@ -620,7 +661,7 @@ julia> Base.return_types(checksym, (Union{Symbol,String},)) ``` It's important to note the difference here: `Base.return_types` gives back inferred results -for each method that matches the given signature `checksum(::Union{Symbol,String})`. +for each method that matches the given signature `checksym(::Union{Symbol,String})`. On the other hand `Base.infer_return_type` returns one collective result that sums up all those possibilities. !!! warning @@ -732,7 +773,7 @@ end world::UInt=get_world_counter(), interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world)) -> exct::Type -Returns the type of exception potentially thrown by the function call specified by `f` and `types`. +Return the type of exception potentially thrown by the function call specified by `f` and `types`. # Arguments - `f`: The function to analyze. @@ -801,7 +842,7 @@ end world::UInt=get_world_counter(), interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world)) -> effects::Effects -Returns the possible computation effects of the function call specified by `f` and `types`. +Return the possible computation effects of the function call specified by `f` and `types`. # Arguments - `f`: The function to analyze. @@ -827,7 +868,7 @@ Returns the possible computation effects of the function call specified by `f` a julia> f1(x) = x * 2; julia> Base.infer_effects(f1, (Int,)) -(+c,+e,+n,+t,+s,+m,+i) +(+c,+e,+re,+n,+t,+s,+m,+u,+o,+r) ``` This function will return an `Effects` object with information about the computational @@ -837,7 +878,7 @@ effects of the function `f1` when called with an `Int` argument. julia> f2(x::Int) = x * 2; julia> Base.infer_effects(f2, (Integer,)) -(+c,+e,!n,+t,+s,+m,+i) +(+c,+e,+re,!n,+t,+s,+m,+u,+o,+r) ``` This case is pretty much the same as with `f1`, but there's a key difference to note. For @@ -920,7 +961,7 @@ function _which(@nospecialize(tt::Type); world::UInt=get_world_counter(), raise::Bool=true) world == typemax(UInt) && error("code reflection cannot be used from generated functions") - match, = invoke_default_compiler(:findsup_mt, tt, world, method_table) + match, = invoke_mt_compiler(method_table, :findsup_mt, tt, world, method_table) if match === nothing raise && error("no unique matching method found for the specified argument types") return nothing @@ -931,11 +972,11 @@ end """ which(f, types) -Returns the method of `f` (a `Method` object) that would be called for arguments of the given `types`. +Return the method of `f` (a `Method` object) that would be called for arguments of the given `types`. If `types` is an abstract type, then the method that would be called by `invoke` is returned. -See also: [`parentmodule`](@ref), [`@which`](@ref Main.InteractiveUtils.@which), and [`@edit`](@ref Main.InteractiveUtils.@edit). +See also [`parentmodule`](@ref), [`@which`](@ref Main.InteractiveUtils.@which), [`@edit`](@ref Main.InteractiveUtils.@edit). """ function which(@nospecialize(f), @nospecialize(t)) tt = signature_type(f, t) @@ -955,7 +996,7 @@ end """ which(types::Type{<:Tuple}) -Returns the method that would be called by the given type signature (as a tuple type). +Return the method that would be called by the given type signature (as a tuple type). """ function which(@nospecialize(tt#=::Type=#)) return _which(tt).method @@ -1019,8 +1060,8 @@ end Return the module in which the given method `m` is defined. -!!! compat "Julia 1.9" - Passing a `Method` as an argument requires Julia 1.9 or later. +!!! compat "Julia 1.10" + Passing a `Method` as an argument requires Julia 1.10 or later. """ parentmodule(m::Method) = m.module @@ -1089,13 +1130,20 @@ function hasmethod(f, t, kwnames::Tuple{Vararg{Symbol}}; world::UInt=get_world_c end """ - fbody = bodyfunction(basemethod::Method) + fbody = bodyfunction(basemethod::Method; world::UInt=Base.get_world_counter()) Find the keyword "body function" (the function that contains the body of the method as written, called after all missing keyword-arguments have been assigned default values). `basemethod` is the method you obtain via [`which`](@ref) or [`methods`](@ref). + +The binding of the body function is looked up in the world age given by `world`, which +defaults to the current world counter. + +!!! compat "Julia 1.14" + The `world` keyword argument requires Julia 1.14 or later. Before Julia 1.14, + the binding was looked up in the world age of the calling task. """ -function bodyfunction(basemethod::Method) +function bodyfunction(basemethod::Method; world::UInt=get_world_counter()) fmod = parentmodule(basemethod) # The lowered code for `basemethod` should look like # %1 = mkw(kwvalues..., #self#, args...) @@ -1108,7 +1156,7 @@ function bodyfunction(basemethod::Method) fsym = callexpr.args[1] while true if isa(fsym, Symbol) - return getfield(fmod, fsym) + return invoke_in_world(world, getglobal, fmod, fsym) elseif isa(fsym, GlobalRef) if fsym.mod === Core && fsym.name === :_apply fsym = callexpr.args[2] @@ -1116,9 +1164,9 @@ function bodyfunction(basemethod::Method) fsym = callexpr.args[3] end if isa(fsym, Symbol) - return getfield(fmod, fsym)::Function + return invoke_in_world(world, getglobal, fmod, fsym)::Function elseif isa(fsym, GlobalRef) - return getfield(fsym.mod, fsym.name)::Function + return invoke_in_world(world, getglobal, fsym.mod, fsym.name)::Function elseif isa(fsym, Core.SSAValue) fsym = ast.code[fsym.id] else @@ -1273,22 +1321,22 @@ It also supports the following syntax: ```jldoctest julia> @macroexpand @invoke f(x::T, y) -:(Core.invoke(f, Tuple{T, Core.Typeof(y)}, x, y)) +:(Core.invoke(f, Base.Tuple{T, Core.Typeof(y)}, x, y)) julia> @invoke 420::Integer % Unsigned 0x00000000000001a4 julia> @macroexpand @invoke (x::X).f -:(Core.invoke(Base.getproperty, Tuple{X, Core.Typeof(:f)}, x, :f)) +:(Core.invoke(Base.getproperty, Base.Tuple{X, Core.Typeof(:f)}, x, :f)) julia> @macroexpand @invoke (x::X).f = v::V -:(Core.invoke(Base.setproperty!, Tuple{X, Core.Typeof(:f), V}, x, :f, v)) +:(Core.invoke(Base.setproperty!, Base.Tuple{X, Core.Typeof(:f), V}, x, :f, v)) julia> @macroexpand @invoke (xs::Xs)[i::I] -:(Core.invoke(Base.getindex, Tuple{Xs, I}, xs, i)) +:(Core.invoke(Base.getindex, Base.Tuple{Xs, I}, xs, i)) julia> @macroexpand @invoke (xs::Xs)[i::I] = v::V -:(Core.invoke(Base.setindex!, Tuple{Xs, V, I}, xs, v, i)) +:(Core.invoke(Base.setindex!, Base.Tuple{Xs, V, I}, xs, v, i)) ``` !!! compat "Julia 1.7" @@ -1305,19 +1353,19 @@ macro invoke(ex) f, args, kwargs = destructure_callex(topmod, ex) types = Expr(:curly, :Tuple) out = Expr(:call, GlobalRef(Core, :invoke)) - isempty(kwargs) || push!(out.args, Expr(:parameters, kwargs...)) - push!(out.args, f) + isempty(kwargs) || push!(out.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...)) + push!(out.args, esc(f)) push!(out.args, types) for arg in args if isexpr(arg, :(::)) - push!(out.args, arg.args[1]) - push!(types.args, arg.args[2]) + push!(out.args, esc(arg.args[1])) + push!(types.args, esc(arg.args[2])) else - push!(out.args, arg) - push!(types.args, Expr(:call, GlobalRef(Core, :Typeof), arg)) + push!(out.args, esc(arg)) + push!(types.args, Expr(:call, GlobalRef(Core, :Typeof), esc(arg))) end end - return esc(out) + return out end getglobalref(gr::GlobalRef, world::UInt) = ccall(:jl_eval_globalref, Any, (Any, UInt), gr, world) @@ -1367,42 +1415,42 @@ macro invokelatest(ex) if !isa(f, GlobalRef) out_f = Expr(:call, GlobalRef(Base, :invokelatest)) - isempty(kwargs) || push!(out_f.args, Expr(:parameters, kwargs...)) + isempty(kwargs) || push!(out_f.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...)) if isexpr(f, :(.)) - s = gensym() + s = :s check = quote - $s = $(f.args[1]) + $s = $(esc(f.args[1])) isa($s, Module) end - push!(out_f.args, Expr(:(.), s, f.args[2])) + push!(out_f.args, Expr(:(.), s, esc(f.args[2]))) else - push!(out_f.args, f) + push!(out_f.args, esc(f)) end - append!(out_f.args, args) + append!(out_f.args, Any[esc(arg) for arg in args]) if @isdefined(s) - f = :(GlobalRef($s, $(f.args[2]))) - elseif !isa(f, Symbol) - return esc(out_f) + f = :(GlobalRef($s, $(esc(f.args[2])))) + elseif isa(f, Symbol) + check = esc(:($(Expr(:isglobal, f)))) else - check = :($(Expr(:isglobal, f))) + return out_f end end out_gr = Expr(:call, GlobalRef(Base, :invokelatest_gr)) - isempty(kwargs) || push!(out_gr.args, Expr(:parameters, kwargs...)) + isempty(kwargs) || push!(out_gr.args, Expr(:parameters, Any[esc(kw) for kw in kwargs]...)) push!(out_gr.args, isa(f, GlobalRef) ? QuoteNode(f) : isa(f, Symbol) ? QuoteNode(GlobalRef(__module__, f)) : f) - append!(out_gr.args, args) + append!(out_gr.args, Any[esc(arg) for arg in args]) if isa(f, GlobalRef) - return esc(out_gr) + return out_gr end # f::Symbol - return esc(:($check ? $out_gr : $out_f)) + return :($check ? $out_gr : $out_f) end function destructure_callex(topmod::Module, @nospecialize(ex)) @@ -1453,3 +1501,20 @@ function destructure_callex(topmod::Module, @nospecialize(ex)) end return f, args, kwargs end + +""" + Base.drop_all_caches() + +Internal function to drop all native code caches and increment world age. +This invalidates all compiled code as if a method was added that intersects +with all existing methods. +""" +function drop_all_caches() + ccall(:jl_drop_all_caches, Cvoid, ()) + + # Reset loading.jl world age so that loading code is regenerated + _require_world_age[] = typemax(UInt) + + # Call Base.Compiler.activate!() after dropping caching to activate coverage of the Compiler code itself + Base.Compiler.activate!() +end diff --git a/base/refvalue.jl b/base/refvalue.jl index 188ab67f26a78..18a114a4bc54b 100644 --- a/base/refvalue.jl +++ b/base/refvalue.jl @@ -42,7 +42,7 @@ function unsafe_convert(P::Union{Type{Ptr{T}},Type{Ptr{Cvoid}}}, b::RefValue{T}) p = pointer_from_objref(b.x) else # If the slot is not leaf type, it could be either immutable or not. - # If it is actually an immutable, then we can't take it's pointer directly + # If it is actually an immutable, then we can't take its pointer directly # Instead, explicitly load the pointer from the `RefValue`, # which also ensures this returns same pointer as the one rooted in the `RefValue` object. p = atomic_pointerref(Ptr{Ptr{Cvoid}}(pointer_from_objref(b)), :monotonic) diff --git a/base/regex.jl b/base/regex.jl index 691dbc94c5563..a8bc21f9fc79f 100644 --- a/base/regex.jl +++ b/base/regex.jl @@ -240,7 +240,7 @@ RegexMatch(match::SubString{S}, captures::Vector{Union{Nothing,SubString{S}}}, Return a vector of keys for all capture groups of the underlying regex. A key is included even if the capture group fails to match. -That is, `idx` will be in the return value even if `m[idx] == nothing`. +That is, `idx` will be in the return value even if `m[idx] === nothing`. Unnamed capture groups will have integer keys corresponding to their index. Named capture groups will have string keys. @@ -260,7 +260,7 @@ julia> keys(match(r"(?\\d+):(?\\d+)(am|pm)?", "11:30")) function keys(m::RegexMatch) idx_to_capture_name = PCRE.capture_names(m.regex.regex) return map(eachindex(m.captures)) do i - # If the capture group is named, return it's name, else return it's index + # If the capture group is named, return its name, else return its index get(idx_to_capture_name, i, i) end end @@ -308,7 +308,7 @@ function occursin(r::Regex, s::AbstractString; offset::Integer=0) return PCRE.exec_r(r.regex, String(s), offset, r.match_options) end -function occursin(r::Regex, s::SubString{String}; offset::Integer=0) +function occursin(r::Regex, s::DenseUTF8String; offset::Integer=0) compile(r) return PCRE.exec_r(r.regex, s, offset, r.match_options) end @@ -324,7 +324,7 @@ Return `true` if `s` starts with the regex pattern, `prefix`. `match_option` to PCRE. If compile time is amortized, `occursin(r"^...", s)` is faster than `startswith(s, r"...")`. -See also [`occursin`](@ref) and [`endswith`](@ref). +See also [`occursin`](@ref), [`endswith`](@ref), [`match`](@ref) !!! compat "Julia 1.2" This method requires at least Julia 1.2. @@ -340,7 +340,7 @@ function startswith(s::AbstractString, r::Regex) return PCRE.exec_r(r.regex, String(s), 0, r.match_options | PCRE.ANCHORED) end -function startswith(s::SubString{String}, r::Regex) +function startswith(s::DenseUTF8String, r::Regex) compile(r) return PCRE.exec_r(r.regex, s, 0, r.match_options | PCRE.ANCHORED) end @@ -356,7 +356,7 @@ Return `true` if `s` ends with the regex pattern, `suffix`. `match_option` to PCRE. If compile time is amortized, `occursin(r"...\$", s)` is faster than `endswith(s, r"...")`. -See also [`occursin`](@ref) and [`startswith`](@ref). +See also [`occursin`](@ref), [`startswith`](@ref), [`match`](@ref) !!! compat "Julia 1.2" This method requires at least Julia 1.2. @@ -372,7 +372,7 @@ function endswith(s::AbstractString, r::Regex) return PCRE.exec_r(r.regex, String(s), 0, r.match_options | PCRE.ENDANCHORED) end -function endswith(s::SubString{String}, r::Regex) +function endswith(s::DenseUTF8String, r::Regex) compile(r) return PCRE.exec_r(r.regex, s, 0, r.match_options | PCRE.ENDANCHORED) end @@ -421,10 +421,13 @@ julia> m.match julia> match(rx, "cabac", 3) === nothing true ``` +# See also +[`eachmatch`](@ref), [`occursin`](@ref), [`findfirst`](@ref) + """ function match end -function match(re::Regex, str::Union{SubString{String}, String}, idx::Integer, +function match(re::Regex, str::DenseUTF8String, idx::Integer, add_opts::UInt32=UInt32(0)) compile(re) opts = re.match_options | add_opts @@ -436,10 +439,15 @@ function match(re::Regex, str::Union{SubString{String}, String}, idx::Integer, n = div(PCRE.ovec_length(data), 2) - 1 p = PCRE.ovec_ptr(data) mat = SubString(str, unsafe_load(p, 1)+1, prevind(str, unsafe_load(p, 2)+1)) - cap = Union{Nothing,SubString{String}}[unsafe_load(p,2i+1) == PCRE.UNSET ? nothing : + T = if str isa SubString + typeof(str) + else + SubString{typeof(str)} + end + cap = Union{Nothing,T}[unsafe_load(p,2i+1) == PCRE.UNSET ? nothing : SubString(str, unsafe_load(p,2i+1)+1, prevind(str, unsafe_load(p,2i+2)+1)) for i=1:n] - off = Int[ unsafe_load(p,2i+1)+1 for i=1:n ] + off = Int[ unsafe_load(p,2i+1) == PCRE.UNSET ? 0 : unsafe_load(p,2i+1)+1 for i=1:n ] result = RegexMatch(mat, cap, unsafe_load(p,1)+1, off, re) PCRE.free_match_data(data) return result @@ -447,12 +455,10 @@ end function _annotatedmatch(m::RegexMatch{S}, str::AnnotatedString{S}) where {S<:AbstractString} RegexMatch{AnnotatedString{S}}( - (@inbounds SubString{AnnotatedString{S}}( - str, m.match.offset, m.match.ncodeunits, Val(:noshift))), + (@inbounds raw_substring(str, m.match.offset + 1, m.match.ncodeunits)), Union{Nothing,SubString{AnnotatedString{S}}}[ if !isnothing(cap) - (@inbounds SubString{AnnotatedString{S}}( - str, cap.offset, cap.ncodeunits, Val(:noshift))) + (@inbounds raw_substring(str, cap.offset + 1, cap.ncodeunits)) end for cap in m.captures], m.offset, m.offsets, m.regex) end @@ -473,10 +479,12 @@ end match(r::Regex, s::AbstractString) = match(r, s, firstindex(s)) match(r::Regex, s::AbstractString, i::Integer) = throw(ArgumentError( - "regex matching is only available for the String and AnnotatedString types; use String(s) to convert" + "regex matching is only available for some dense (memory backed strings); use String(s) to convert s to String" )) -findnext(re::Regex, str::Union{String,SubString}, idx::Integer) = _findnext_re(re, str, idx, C_NULL) +function findnext(re::Regex, str::DenseUTF8String, idx::Integer) + _findnext_re(re, str, idx, C_NULL) +end # TODO: return only start index and update deprecation # duck-type str so that external UTF-8 string packages like StringViews can hook in @@ -558,6 +566,9 @@ julia> count(r"a(.)a", "cabacabac", overlap=true) julia> count(r"a(.)a", "cabacabac") 2 ``` +# See also +[`eachmatch`](@ref), [`occursin`](@ref), [`findall`](@ref) + """ function count(t::Union{AbstractChar,AbstractString,AbstractPattern}, s::AbstractString; overlap::Bool=false) n = 0 @@ -643,17 +654,17 @@ replace_err(repl) = error("Bad replacement string: $repl") function _write_capture(io::IO, group::Int, str, r, re::RegexAndMatchData) len = PCRE.substring_length_bynumber(re.match_data, group) # in the case of an optional group that doesn't match, len == 0 - len == 0 && return + len == 0 && return len ensureroom(io, len+1) PCRE.substring_copy_bynumber(re.match_data, group, pointer(io.data, io.ptr), len+1) io.ptr += len io.size = max(io.size, io.ptr - 1) - nothing + return len end function _write_capture(io::IO, group::Int, str, r, re) group == 0 || replace_err("pattern is not a Regex") - return print(io, SubString(str, r)) + return write(io, SubString(str, r)) end @@ -667,12 +678,13 @@ function _replace(io, repl_s::SubstitutionString, str, r, re) repl = unescape_string(repl_s.string, KEEP_ESC) i = firstindex(repl) e = lastindex(repl) + nb = 0 while i <= e if repl[i] == SUB_CHAR next_i = nextind(repl, i) next_i > e && replace_err(repl) if repl[next_i] == SUB_CHAR - write(io, SUB_CHAR) + nb += write(io, SUB_CHAR) i = nextind(repl, next_i) elseif isdigit(repl[next_i]) group = parse(Int, repl[next_i]) @@ -685,7 +697,7 @@ function _replace(io, repl_s::SubstitutionString, str, r, re) break end end - _write_capture(io, group, str, r, re) + nb += _write_capture(io, group, str, r, re) elseif repl[next_i] == GROUP_CHAR i = nextind(repl, next_i) if i > e || repl[i] != LBRACKET @@ -707,16 +719,17 @@ function _replace(io, repl_s::SubstitutionString, str, r, re) else group = -1 end - _write_capture(io, group, str, r, re) + nb += _write_capture(io, group, str, r, re) i = nextind(repl, i) else replace_err(repl) end else - write(io, repl[i]) + nb += write(io, repl[i]) i = nextind(repl, i) end end + nb end struct RegexMatchIterator{S <: AbstractString} @@ -789,6 +802,9 @@ julia> collect(eachmatch(rx, "a1a2a3a", overlap = true)) RegexMatch("a2a") RegexMatch("a3a") ``` +# See also +[`match`](@ref), [`findall`](@ref), [`count`](@ref) + """ eachmatch(re::Regex, str::AbstractString; overlap = false) = RegexMatchIterator(re, str, overlap) @@ -830,7 +846,7 @@ RegexMatch("Hello world") julia> r = r"a|b" * "c|d" r"(?:a|b)\\Qc|d\\E" -julia> match(r, "ac") == nothing +julia> match(r, "ac") === nothing true julia> match(r, "ac|d") diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index dc993515ed0cd..02b354b99cc9b 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -24,6 +24,11 @@ struct ReinterpretArray{T,N,S,A<:AbstractArray{S},IsReshaped} <: AbstractArray{T @noinline throw(ArgumentError(LazyString("cannot reinterpret a `", S, "` array to `", T, "` which is a singleton type"))) end + function throwbitpadding(S::Type, T::Type, U::Type) + @noinline + throw(ArgumentError(LazyString("cannot reinterpret a `", S, "` array to `", T, + "` because type `", U, "` contains non-byte-aligned primitive fields"))) + end global reinterpret @@ -79,14 +84,16 @@ struct ReinterpretArray{T,N,S,A<:AbstractArray{S},IsReshaped} <: AbstractArray{T end isbitstype(T) || throwbits(S, T, T) isbitstype(S) || throwbits(S, T, S) - (N != 0 || sizeof(T) == sizeof(S)) || throwsize0(S, T, "different") - if N != 0 && sizeof(S) != sizeof(T) + has_bit_padding(T) && throwbitpadding(S, T, T) + has_bit_padding(S) && throwbitpadding(S, T, S) + (N != 0 || aligned_sizeof(T) == aligned_sizeof(S)) || throwsize0(S, T, "different") + if N != 0 && aligned_sizeof(S) != aligned_sizeof(T) ax1 = axes(a)[1] dim = length(ax1) if issingletontype(T) issingletontype(S) || throwsingleton(S, T) else - rem(dim*sizeof(S),sizeof(T)) == 0 || thrownonint(S, T, dim) + rem(dim*aligned_sizeof(S),aligned_sizeof(T)) == 0 || thrownonint(S, T, dim) end first(ax1) == 1 || throwaxes1(S, T, ax1) end @@ -100,14 +107,14 @@ struct ReinterpretArray{T,N,S,A<:AbstractArray{S},IsReshaped} <: AbstractArray{T function reinterpret(::typeof(reshape), ::Type{T}, a::A) where {T,S,A<:AbstractArray{S}} function throwintmult(S::Type, T::Type) @noinline - throw(ArgumentError(LazyString("`reinterpret(reshape, T, a)` requires that one of `sizeof(T)` (got ", - sizeof(T), ") and `sizeof(eltype(a))` (got ", sizeof(S), ") be an integer multiple of the other"))) + throw(ArgumentError(LazyString("`reinterpret(reshape, T, a)` requires that one of `Base.elsize(Array{T})` (got ", + aligned_sizeof(T), ") and `Base.elsize(Array{eltype(a)})` (got ", aligned_sizeof(S), ") be an integer multiple of the other"))) end function throwsize1(a::AbstractArray, T::Type) @noinline throw(ArgumentError(LazyString("`reinterpret(reshape, ", T, ", a)` where `eltype(a)` is ", eltype(a), " requires that `axes(a, 1)` (got ", axes(a, 1), ") be equal to 1:", - sizeof(T) ÷ sizeof(eltype(a)), " (from the ratio of element sizes)"))) + aligned_sizeof(T) ÷ aligned_sizeof(eltype(a)), " (from the ratio of element sizes)"))) end function throwfromsingleton(S, T) @noinline @@ -116,18 +123,20 @@ struct ReinterpretArray{T,N,S,A<:AbstractArray{S},IsReshaped} <: AbstractArray{T end isbitstype(T) || throwbits(S, T, T) isbitstype(S) || throwbits(S, T, S) - if sizeof(S) == sizeof(T) + has_bit_padding(T) && throwbitpadding(S, T, T) + has_bit_padding(S) && throwbitpadding(S, T, S) + if aligned_sizeof(S) == aligned_sizeof(T) N = ndims(a) - elseif sizeof(S) > sizeof(T) + elseif aligned_sizeof(S) > aligned_sizeof(T) issingletontype(T) && throwsingleton(S, T) - rem(sizeof(S), sizeof(T)) == 0 || throwintmult(S, T) + rem(aligned_sizeof(S), aligned_sizeof(T)) == 0 || throwintmult(S, T) N = ndims(a) + 1 else issingletontype(S) && throwfromsingleton(S, T) - rem(sizeof(T), sizeof(S)) == 0 || throwintmult(S, T) + rem(aligned_sizeof(T), aligned_sizeof(S)) == 0 || throwintmult(S, T) N = ndims(a) - 1 N > -1 || throwsize0(S, T, "larger") - axes(a, 1) == OneTo(sizeof(T) ÷ sizeof(S)) || throwsize1(a, T) + axes(a, 1) == OneTo(aligned_sizeof(T) ÷ aligned_sizeof(S)) || throwsize1(a, T) end readable = array_subpadding(T, S) writable = array_subpadding(S, T) @@ -144,9 +153,9 @@ NonReshapedReinterpretArray{T,N,S,A<:AbstractArray{S, N}} = ReinterpretArray{T,N Change the type-interpretation of `A` while consuming or adding a "channel dimension." -If `sizeof(T) = n*sizeof(S)` for `n>1`, `A`'s first dimension must be -of size `n` and `B` lacks `A`'s first dimension. Conversely, if `sizeof(S) = n*sizeof(T)` for `n>1`, -`B` gets a new first dimension of size `n`. The dimensionality is unchanged if `sizeof(T) == sizeof(S)`. +If `Base.elsize(Array{T}) = n*Base.elsize(Array{S})` for `n>1`, `A`'s first dimension must be +of size `n` and `B` lacks `A`'s first dimension. Conversely, if `Base.elsize(Array{S}) = n*Base.elsize(Array{T})` for `n>1`, +`B` gets a new first dimension of size `n`. The dimensionality is unchanged if `Base.elsize(Array{T}) == Base.elsize(Array{S})`. !!! compat "Julia 1.6" This method requires at least Julia 1.6. @@ -199,20 +208,19 @@ stride(A::Union{DenseArray,StridedReshapedArray,StridedReinterpretArray}, k::Int function strides(a::ReinterpretArray{T,<:Any,S,<:AbstractArray{S},IsReshaped}) where {T,S,IsReshaped} _checkcontiguous(Bool, a) && return size_to_strides(1, size(a)...) stp = strides(parent(a)) - els, elp = sizeof(T), sizeof(S) + els, elp = aligned_sizeof(T), aligned_sizeof(S) els == elp && return stp # 0dim parent is also handled here. - IsReshaped && els < elp && return (1, _checked_strides(stp, els, elp)...) - stp[1] == 1 || throw(ArgumentError("Parent must be contiguous in the 1st dimension!")) - st′ = _checked_strides(tail(stp), els, elp) + # Note: ep typically equals elp, but a custom strided parent may define elsize differently + ep = elsize(parent(a)) + byte_strides = map(i -> i * ep, stp) + IsReshaped && els < elp && return (1, _checked_strides(byte_strides, els)...) + first(byte_strides) == elp || throw(ArgumentError("Parent must be contiguous in the 1st dimension!")) + st′ = _checked_strides(tail(byte_strides), els) return IsReshaped ? st′ : (1, st′...) end -@inline function _checked_strides(stp::Tuple, els::Integer, elp::Integer) - if elp > els && rem(elp, els) == 0 - N = div(elp, els) - return map(i -> N * i, stp) - end - drs = map(i -> divrem(elp * i, els), stp) +@inline function _checked_strides(byte_strides::Tuple, els::Integer) + drs = map(i -> divrem(i, els), byte_strides) all(i->iszero(i[2]), drs) || throw(ArgumentError("Parent's strides could not be exactly divided!")) map(first, drs) @@ -221,6 +229,7 @@ end _checkcontiguous(::Type{Bool}, A::ReinterpretArray) = _checkcontiguous(Bool, parent(A)) similar(a::ReinterpretArray, T::Type, d::Dims) = similar(a.parent, T, d) +similar(::Type{TA}, dims::Dims) where {T,N,O,P,TA<:ReinterpretArray{T,N,O,P}} = similar(P, dims) function check_readable(a::ReinterpretArray{T, N, S} where N) where {T,S} # See comment in check_writable @@ -245,12 +254,12 @@ end # For `reinterpret(reshape, T, a)` where we're adding a channel dimension and with # `IndexStyle(a) == IndexLinear()`, it's advantageous to retain pseudo-linear indexing. -struct IndexSCartesian2{K} <: IndexStyle end # K = sizeof(S) ÷ sizeof(T), a static-sized 2d cartesian iterator +struct IndexSCartesian2{K} <: IndexStyle end # K = aligned_sizeof(S) ÷ aligned_sizeof(T), a static-sized 2d cartesian iterator IndexStyle(::Type{ReinterpretArray{T,N,S,A,false}}) where {T,N,S,A<:AbstractArray{S,N}} = IndexStyle(A) function IndexStyle(::Type{ReinterpretArray{T,N,S,A,true}}) where {T,N,S,A<:AbstractArray{S}} - if sizeof(T) < sizeof(S) - IndexStyle(A) === IndexLinear() && return IndexSCartesian2{sizeof(S) ÷ sizeof(T)}() + if aligned_sizeof(T) < aligned_sizeof(S) + IndexStyle(A) === IndexLinear() && return IndexSCartesian2{aligned_sizeof(S) ÷ aligned_sizeof(T)}() return IndexCartesian() end return IndexStyle(A) @@ -266,7 +275,7 @@ to_index(i::SCartesianIndex2) = i struct SCartesianIndices2{K,R<:AbstractUnitRange{Int}} <: AbstractMatrix{SCartesianIndex2{K}} indices2::R end -SCartesianIndices2{K}(indices2::AbstractUnitRange{Int}) where {K} = (@assert K::Int > 1; SCartesianIndices2{K,typeof(indices2)}(indices2)) +SCartesianIndices2{K}(indices2::AbstractUnitRange{Int}) where {K} = (@assert K::Int > 1 "invalid index"; SCartesianIndices2{K,typeof(indices2)}(indices2)) eachindex(::IndexSCartesian2{K}, A::ReshapedReinterpretArray) where {K} = SCartesianIndices2{K}(eachindex(IndexLinear(), parent(A))) @inline function eachindex(style::IndexSCartesian2{K}, A::AbstractArray, B::AbstractArray...) where {K} @@ -360,13 +369,13 @@ unaliascopy(a::ReshapedReinterpretArray{T}) where {T} = reinterpret(reshape, T, function size(a::NonReshapedReinterpretArray{T,N,S} where {N}) where {T,S} psize = size(a.parent) - size1 = issingletontype(T) ? psize[1] : div(psize[1]*sizeof(S), sizeof(T)) + size1 = issingletontype(T) ? psize[1] : div(psize[1]*aligned_sizeof(S), aligned_sizeof(T)) tuple(size1, tail(psize)...) end function size(a::ReshapedReinterpretArray{T,N,S} where {N}) where {T,S} psize = size(a.parent) - sizeof(S) > sizeof(T) && return (div(sizeof(S), sizeof(T)), psize...) - sizeof(S) < sizeof(T) && return tail(psize) + aligned_sizeof(S) > aligned_sizeof(T) && return (div(aligned_sizeof(S), aligned_sizeof(T)), psize...) + aligned_sizeof(S) < aligned_sizeof(T) && return tail(psize) return psize end size(a::NonReshapedReinterpretArray{T,0}) where {T} = () @@ -374,37 +383,45 @@ size(a::NonReshapedReinterpretArray{T,0}) where {T} = () function axes(a::NonReshapedReinterpretArray{T,N,S} where {N}) where {T,S} paxs = axes(a.parent) f, l = first(paxs[1]), length(paxs[1]) - size1 = issingletontype(T) ? l : div(l*sizeof(S), sizeof(T)) + size1 = issingletontype(T) ? l : div(l*aligned_sizeof(S), aligned_sizeof(T)) tuple(oftype(paxs[1], f:f+size1-1), tail(paxs)...) end function axes(a::ReshapedReinterpretArray{T,N,S} where {N}) where {T,S} paxs = axes(a.parent) - sizeof(S) > sizeof(T) && return (OneTo(div(sizeof(S), sizeof(T))), paxs...) - sizeof(S) < sizeof(T) && return tail(paxs) + aligned_sizeof(S) > aligned_sizeof(T) && return (OneTo(div(aligned_sizeof(S), aligned_sizeof(T))), paxs...) + aligned_sizeof(S) < aligned_sizeof(T) && return tail(paxs) return paxs end axes(a::NonReshapedReinterpretArray{T,0}) where {T} = () has_offset_axes(a::ReinterpretArray) = has_offset_axes(a.parent) -elsize(::Type{<:ReinterpretArray{T}}) where {T} = sizeof(T) +function elsize(::Type{<:ReinterpretArray{T,<:Any,S,A}}) where {T,S,A} + if aligned_sizeof(T) == aligned_sizeof(S) + elsize(A) + else + aligned_sizeof(T) + end +end cconvert(::Type{Ptr{T}}, a::ReinterpretArray{T,N,S} where N) where {T,S} = cconvert(Ptr{S}, a.parent) unsafe_convert(::Type{Ptr{T}}, a::ReinterpretArray{T,N,S} where N) where {T,S} = Ptr{T}(unsafe_convert(Ptr{S},a.parent)) @propagate_inbounds function getindex(a::NonReshapedReinterpretArray{T,0,S}) where {T,S} - if isprimitivetype(T) && isprimitivetype(S) + if _is_scalar_reinterpret_applicable(T, S) reinterpret(T, a.parent[]) else a[firstindex(a)] end end -check_ptr_indexable(a::ReinterpretArray, sz = elsize(a)) = check_ptr_indexable(parent(a), sz) -check_ptr_indexable(a::ReshapedArray, sz) = check_ptr_indexable(parent(a), sz) -check_ptr_indexable(a::FastContiguousSubArray, sz) = check_ptr_indexable(parent(a), sz) -check_ptr_indexable(a::Array, sz) = sizeof(eltype(a)) !== sz -check_ptr_indexable(a::Memory, sz) = true -check_ptr_indexable(a::AbstractArray, sz) = false +function check_ptr_indexable(a::ReinterpretArray{T,<:Any,S}) where {T,S} + aligned_sizeof(T) === aligned_sizeof(S) && return false + return check_ptr_indexable(parent(a)) +end +check_ptr_indexable(a::ReshapedArray) = check_ptr_indexable(parent(a)) +check_ptr_indexable(a::FastContiguousSubArray) = check_ptr_indexable(parent(a)) +check_ptr_indexable(a::Union{Array, Memory}) = true +check_ptr_indexable(a::AbstractArray) = false @propagate_inbounds getindex(a::ReshapedReinterpretArray{T,0}) where {T} = a[firstindex(a)] @@ -441,25 +458,27 @@ end @boundscheck checkbounds(a, inds...) li = _to_linear_index(a, inds...) ap = cconvert(Ptr{T}, a) - p = unsafe_convert(Ptr{T}, ap) + sizeof(T) * (li - 1) + p = unsafe_convert(Ptr{T}, ap) + elsize(a) * (li - 1) GC.@preserve ap return unsafe_load(p) end +function _is_scalar_reinterpret_applicable(T, S) + isprimitivetype(S) && isprimitivetype(T) && Core.bitsizeof(T) === Core.bitsizeof(S) +end + @propagate_inbounds function _getindex_ra(a::NonReshapedReinterpretArray{T,N,S}, i1::Int, tailinds::TT) where {T,N,S,TT} - # Make sure to match the scalar reinterpret if that is applicable - if sizeof(T) == sizeof(S) && (fieldcount(T) + fieldcount(S)) == 0 - if issingletontype(T) # singleton types - @boundscheck checkbounds(a, i1, tailinds...) - return T.instance - end + if issingletontype(T) # singleton types + @boundscheck checkbounds(a, i1, tailinds...) + return T.instance + elseif _is_scalar_reinterpret_applicable(T, S) return reinterpret(T, a.parent[i1, tailinds...]) else @boundscheck checkbounds(a, i1, tailinds...) - ind_start, sidx = divrem((i1-1)*sizeof(T), sizeof(S)) + ind_start, sidx = divrem((i1-1)*aligned_sizeof(T), aligned_sizeof(S)) # Optimizations that avoid branches - if sizeof(T) % sizeof(S) == 0 + if aligned_sizeof(T) % aligned_sizeof(S) == 0 # T is bigger than S and contains an integer number of them - n = sizeof(T) ÷ sizeof(S) + n = aligned_sizeof(T) ÷ aligned_sizeof(S) t = Ref{T}() GC.@preserve t begin sptr = Ptr{S}(unsafe_convert(Ref{T}, t)) @@ -469,7 +488,7 @@ end end end return t[] - elseif sizeof(S) % sizeof(T) == 0 + elseif aligned_sizeof(S) % aligned_sizeof(T) == 0 # S is bigger than T and contains an integer number of them s = Ref{S}(a.parent[ind_start + 1, tailinds...]) GC.@preserve s begin @@ -487,9 +506,9 @@ end GC.@preserve s t begin sptr = Ptr{S}(unsafe_convert(Ref{S}, s)) tptr = Ptr{T}(unsafe_convert(Ref{T}, t)) - while nbytes_copied < sizeof(T) + while nbytes_copied < aligned_sizeof(T) s[] = a.parent[ind_start + i, tailinds...] - nb = min(sizeof(S) - sidx, sizeof(T)-nbytes_copied) + nb = min(aligned_sizeof(S) - sidx, aligned_sizeof(T)-nbytes_copied) memcpy(tptr + nbytes_copied, sptr + sidx, nb) nbytes_copied += nb sidx = 0 @@ -502,22 +521,20 @@ end end @propagate_inbounds function _getindex_ra(a::ReshapedReinterpretArray{T,N,S}, i1::Int, tailinds::TT) where {T,N,S,TT} - # Make sure to match the scalar reinterpret if that is applicable - if sizeof(T) == sizeof(S) && (fieldcount(T) + fieldcount(S)) == 0 - if issingletontype(T) # singleton types - @boundscheck checkbounds(a, i1, tailinds...) - return T.instance - end + if issingletontype(T) # singleton types + @boundscheck checkbounds(a, i1, tailinds...) + return T.instance + elseif _is_scalar_reinterpret_applicable(T, S) return reinterpret(T, a.parent[i1, tailinds...]) end @boundscheck checkbounds(a, i1, tailinds...) - if sizeof(T) >= sizeof(S) + if aligned_sizeof(T) >= aligned_sizeof(S) t = Ref{T}() GC.@preserve t begin sptr = Ptr{S}(unsafe_convert(Ref{T}, t)) - if sizeof(T) > sizeof(S) + if aligned_sizeof(T) > aligned_sizeof(S) # Extra dimension in the parent array - n = sizeof(T) ÷ sizeof(S) + n = aligned_sizeof(T) ÷ aligned_sizeof(S) if isempty(tailinds) && IndexStyle(a.parent) === IndexLinear() offset = n * (i1 - firstindex(a)) for i = 1:n @@ -539,7 +556,7 @@ end return t[] end # S is bigger than T and contains an integer number of them - # n = sizeof(S) ÷ sizeof(T) + # n = aligned_sizeof(S) ÷ aligned_sizeof(T) s = Ref{S}() GC.@preserve s begin tptr = Ptr{T}(unsafe_convert(Ref{S}, s)) @@ -549,7 +566,7 @@ end end @propagate_inbounds function setindex!(a::NonReshapedReinterpretArray{T,0,S}, v) where {T,S} - if isprimitivetype(S) && isprimitivetype(T) + if _is_scalar_reinterpret_applicable(T, S) a.parent[] = reinterpret(S, convert(T, v)::T) return a end @@ -590,37 +607,34 @@ end @boundscheck checkbounds(a, inds...) li = _to_linear_index(a, inds...) ap = cconvert(Ptr{T}, a) - p = unsafe_convert(Ptr{T}, ap) + sizeof(T) * (li - 1) + p = unsafe_convert(Ptr{T}, ap) + elsize(a) * (li - 1) GC.@preserve ap unsafe_store!(p, v) return a end @propagate_inbounds function _setindex_ra!(a::NonReshapedReinterpretArray{T,N,S}, v, i1::Int, tailinds::TT) where {T,N,S,TT} v = convert(T, v)::T - # Make sure to match the scalar reinterpret if that is applicable - if sizeof(T) == sizeof(S) && (fieldcount(T) + fieldcount(S)) == 0 - if issingletontype(T) # singleton types - @boundscheck checkbounds(a, i1, tailinds...) - # setindex! is a noop except for the index check - else - setindex!(a.parent, reinterpret(S, v), i1, tailinds...) - end + if issingletontype(T) # singleton types + @boundscheck checkbounds(a, i1, tailinds...) + # setindex! is a noop except for the index check + elseif _is_scalar_reinterpret_applicable(T, S) + setindex!(a.parent, reinterpret(S, v), i1, tailinds...) else @boundscheck checkbounds(a, i1, tailinds...) - ind_start, sidx = divrem((i1-1)*sizeof(T), sizeof(S)) + ind_start, sidx = divrem((i1-1)*aligned_sizeof(T), aligned_sizeof(S)) # Optimizations that avoid branches - if sizeof(T) % sizeof(S) == 0 + if aligned_sizeof(T) % aligned_sizeof(S) == 0 # T is bigger than S and contains an integer number of them t = Ref{T}(v) GC.@preserve t begin sptr = Ptr{S}(unsafe_convert(Ref{T}, t)) - n = sizeof(T) ÷ sizeof(S) + n = aligned_sizeof(T) ÷ aligned_sizeof(S) for i = 1:n s = unsafe_load(sptr, i) a.parent[ind_start + i, tailinds...] = s end end - elseif sizeof(S) % sizeof(T) == 0 + elseif aligned_sizeof(S) % aligned_sizeof(T) == 0 # S is bigger than T and contains an integer number of them s = Ref{S}(a.parent[ind_start + 1, tailinds...]) GC.@preserve s begin @@ -640,25 +654,24 @@ end # element from the original array and overwrite the relevant parts if sidx != 0 s[] = a.parent[ind_start + i, tailinds...] - nb = min((sizeof(S) - sidx) % UInt, sizeof(T) % UInt) + nb = min((aligned_sizeof(S) - sidx) % UInt, aligned_sizeof(T) % UInt) memcpy(sptr + sidx, tptr, nb) nbytes_copied += nb a.parent[ind_start + i, tailinds...] = s[] i += 1 - sidx = 0 end # Deal with the main body of elements - while nbytes_copied < sizeof(T) && (sizeof(T) - nbytes_copied) > sizeof(S) - nb = min(sizeof(S), sizeof(T) - nbytes_copied) + while nbytes_copied < aligned_sizeof(T) && (aligned_sizeof(T) - nbytes_copied) > aligned_sizeof(S) + nb = min(aligned_sizeof(S), aligned_sizeof(T) - nbytes_copied) memcpy(sptr, tptr + nbytes_copied, nb) nbytes_copied += nb a.parent[ind_start + i, tailinds...] = s[] i += 1 end # Deal with trailing partial elements - if nbytes_copied < sizeof(T) + if nbytes_copied < aligned_sizeof(T) s[] = a.parent[ind_start + i, tailinds...] - nb = min(sizeof(S), sizeof(T) - nbytes_copied) + nb = min(aligned_sizeof(S), aligned_sizeof(T) - nbytes_copied) memcpy(sptr, tptr + nbytes_copied, nb) a.parent[ind_start + i, tailinds...] = s[] end @@ -670,65 +683,52 @@ end @propagate_inbounds function _setindex_ra!(a::ReshapedReinterpretArray{T,N,S}, v, i1::Int, tailinds::TT) where {T,N,S,TT} v = convert(T, v)::T - # Make sure to match the scalar reinterpret if that is applicable - if sizeof(T) == sizeof(S) && (fieldcount(T) + fieldcount(S)) == 0 - if issingletontype(T) # singleton types - @boundscheck checkbounds(a, i1, tailinds...) - # setindex! is a noop except for the index check - else - setindex!(a.parent, reinterpret(S, v), i1, tailinds...) - end - end - @boundscheck checkbounds(a, i1, tailinds...) - if sizeof(T) >= sizeof(S) - t = Ref{T}(v) - GC.@preserve t begin - sptr = Ptr{S}(unsafe_convert(Ref{T}, t)) - if sizeof(T) > sizeof(S) - # Extra dimension in the parent array - n = sizeof(T) ÷ sizeof(S) - if isempty(tailinds) && IndexStyle(a.parent) === IndexLinear() - offset = n * (i1 - firstindex(a)) - for i = 1:n - s = unsafe_load(sptr, i) - a.parent[i + offset] = s - end - else - for i = 1:n - s = unsafe_load(sptr, i) - a.parent[i, i1, tailinds...] = s + if issingletontype(T) # singleton types + @boundscheck checkbounds(a, i1, tailinds...) + # setindex! is a noop except for the index check + elseif _is_scalar_reinterpret_applicable(T, S) + setindex!(a.parent, reinterpret(S, v), i1, tailinds...) + else + @boundscheck checkbounds(a, i1, tailinds...) + if aligned_sizeof(T) >= aligned_sizeof(S) + t = Ref{T}(v) + GC.@preserve t begin + sptr = Ptr{S}(unsafe_convert(Ref{T}, t)) + if aligned_sizeof(T) > aligned_sizeof(S) + # Extra dimension in the parent array + n = aligned_sizeof(T) ÷ aligned_sizeof(S) + if isempty(tailinds) && IndexStyle(a.parent) === IndexLinear() + offset = n * (i1 - firstindex(a)) + for i = 1:n + s = unsafe_load(sptr, i) + a.parent[i + offset] = s + end + else + for i = 1:n + s = unsafe_load(sptr, i) + a.parent[i, i1, tailinds...] = s + end end + else # aligned_sizeof(T) == aligned_sizeof(S) + # No extra dimension + s = unsafe_load(sptr) + a.parent[i1, tailinds...] = s end - else # sizeof(T) == sizeof(S) - # No extra dimension - s = unsafe_load(sptr) - a.parent[i1, tailinds...] = s end - end - else - # S is bigger than T and contains an integer number of them - s = Ref{S}() - GC.@preserve s begin - tptr = Ptr{T}(unsafe_convert(Ref{S}, s)) - s[] = a.parent[tailinds...] - unsafe_store!(tptr, v, i1) - a.parent[tailinds...] = s[] + else + # S is bigger than T and contains an integer number of them + s = Ref{S}() + GC.@preserve s begin + tptr = Ptr{T}(unsafe_convert(Ref{S}, s)) + s[] = a.parent[tailinds...] + unsafe_store!(tptr, v, i1) + a.parent[tailinds...] = s[] + end end end return a end -# Padding -struct Padding - offset::Int # 0-indexed offset of the next valid byte; sizeof(T) indicates trailing padding - size::Int # bytes of padding before a valid byte -end -function intersect(p1::Padding, p2::Padding) - start = max(p1.offset, p2.offset) - stop = min(p1.offset + p1.size, p2.offset + p2.size) - Padding(start, max(0, stop-start)) -end - struct PaddingError <: Exception S::Type T::Type @@ -738,102 +738,103 @@ function showerror(io::IO, p::PaddingError) print(io, "Padding of type $(p.S) is not compatible with type $(p.T).") end -""" - CyclePadding(padding, total_size) - -Cycles an iterator of `Padding` structs, restarting the padding at `total_size`. -E.g. if `padding` is all the padding in a struct and `total_size` is the total -aligned size of that array, `CyclePadding` will correspond to the padding in an -infinite vector of such structs. -""" -struct CyclePadding{P} - padding::P - total_size::Int -end -eltype(::Type{<:CyclePadding}) = Padding -IteratorSize(::Type{<:CyclePadding}) = IsInfinite() -isempty(cp::CyclePadding) = isempty(cp.padding) -function iterate(cp::CyclePadding) - y = iterate(cp.padding) - y === nothing && return nothing - y[1], (0, y[2]) -end -function iterate(cp::CyclePadding, state::Tuple) - y = iterate(cp.padding, tail(state)...) - y === nothing && return iterate(cp, (state[1]+cp.total_size,)) - Padding(y[1].offset+state[1], y[1].size), (state[1], tail(y)...) +# Return a byte index to Bool map for each byte of an aligned `T` +# if false, that byte is undefined padding that should not be observed. +# Preconditions, already checked by the `reinterpret` constructors and `_reinterpret`: +# `isbitstype(T)` and `!has_bit_padding(T)` +function non_padding_bytes(T::DataType)::Memory{Bool} + # @assert isbitstype(T) + # @assert !has_bit_padding(T) + used = Memory{Bool}(undef, aligned_sizeof(T)) + fill!(used, false) + fill_nonpadding_bytes!(T, 0, used) + used +end +function fill_nonpadding_bytes!(T::DataType, offset::Int, used::Memory{Bool}) + if isprimitivetype(T) + for i in 1:sizeof(T) + used[i + offset] = true + end + else + for i in 1:fieldcount(T) + fill_nonpadding_bytes!(fieldtype(T, i), offset + Int(fieldoffset(T, i)), used) + end + end end -""" - Compute the location of padding in an isbits datatype. Recursive over the fields of that type. -""" -@assume_effects :foldable function padding(T::DataType, baseoffset::Int = 0) - pads = Padding[] - last_end::Int = baseoffset - for i = 1:fieldcount(T) - offset = baseoffset + Int(fieldoffset(T, i)) - fT = fieldtype(T, i) - append!(pads, padding(fT, offset)) - if offset != last_end - push!(pads, Padding(offset, offset-last_end)) +@assume_effects :foldable function isarraypacked(T) + !datatype_haspadding(T) && sizeof(T) == aligned_sizeof(T) +end + +# Preconditions, already checked by the `reinterpret` constructors: +# `isbitstype(T/S)` and `!has_bit_padding(T/S)` +@assume_effects :foldable function array_subpadding(S, T) + # Fast path: if every byte of `T` is a non-padding byte, any byte can be + # read. This also covers zero-size `T`. + if isarraypacked(T) + return true + end + # `T` has at least one padding byte here. If `S` has none, the byte cycle + # below visits every byte of `T`, so some readable byte of `S` must land on + # padding in `T`. This also covers zero-size `S`. + if isarraypacked(S) + return false + end + s_used, t_used = non_padding_bytes(S), non_padding_bytes(T) + # zero-size T and S are unreachable here (handled by the fast paths above); + # these guards only keep the loop below well-defined + isempty(t_used) && return true + isempty(s_used) && return false + s_n, t_n = length(s_used), length(t_used) + s_i, t_i = 1, 1 + while true + # If a byte can be read in S, but is padding in T, return false + if s_used[s_i] && !t_used[t_i] + return false + end + # Advance the indexes with wrapping around + s_i += 1 + if s_i > s_n + s_i = 1 + end + t_i += 1 + if t_i > t_n + t_i = 1 + end + # We made it back to the start + if isone(s_i) && isone(t_i) + return true end - last_end = offset + sizeof(fT) - end - if 0 < last_end - baseoffset < sizeof(T) - push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end + baseoffset)) end - return Core.svec(pads...) end -function CyclePadding(T::DataType) - a, s = datatype_alignment(T), sizeof(T) - as = s + (a - (s % a)) % a - pad = padding(T) - if s != as - pad = Core.svec(pad..., Padding(s, as - s)) - end - CyclePadding(pad, as) +@assume_effects :foldable function struct_subpadding(Out, In) + non_padding_bytes(Out) == non_padding_bytes(In) end -@assume_effects :total function array_subpadding(S, T) - lcm_size = lcm(sizeof(S), sizeof(T)) - s, t = CyclePadding(S), CyclePadding(T) - checked_size = 0 - # use of Stateful harms inference and makes this vulnerable to invalidation - (pad, tstate) = let - it = iterate(t) - it === nothing && return true - it - end - (ps, sstate) = let - it = iterate(s) - it === nothing && return false - it - end - while checked_size < lcm_size - while true - # See if there's corresponding padding in S - ps.offset > pad.offset && return false - intersect(ps, pad) == pad && break - ps, sstate = iterate(s, sstate) - end - checked_size = pad.offset + pad.size - pad, tstate = iterate(t, tstate) +@assume_effects :foldable function packedsize(T) + if !datatype_haspadding(T) + sizeof(T) + else + count(non_padding_bytes(T)) end - return true end -@assume_effects :foldable function struct_subpadding(::Type{Out}, ::Type{In}) where {Out, In} - padding(Out) == padding(In) +@assume_effects :foldable function ispacked(T) + !datatype_haspadding(T) end -@assume_effects :foldable function packedsize(::Type{T}) where T - pads = padding(T) - return sizeof(T) - sum((p.size for p ∈ pads), init = 0) +@assume_effects :foldable function has_bit_padding(::Type{T}) where T + if isprimitivetype(T) + return Core.bitsizeof(T) % 8 != 0 + end + T isa Union && return any(has_bit_padding, uniontypes(T)) + for i in 1:fieldcount(T) + has_bit_padding(fieldtype(T, i)) && return true + end + return false end -@assume_effects :foldable ispacked(::Type{T}) where T = isempty(padding(T)) - function _copytopacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} writeoffset = 0 for i ∈ 1:fieldcount(In) @@ -870,6 +871,10 @@ end # handle non-primitive types isbitstype(Out) || throw(ArgumentError("Target type for `reinterpret` must be isbits")) isbitstype(In) || throw(ArgumentError("Source type for `reinterpret` must be isbits")) + has_bit_padding(Out) && throw(ArgumentError(LazyString( + "cannot `reinterpret` type ", Out, " containing non-byte-aligned primitive fields"))) + has_bit_padding(In) && throw(ArgumentError(LazyString( + "cannot `reinterpret` type ", In, " containing non-byte-aligned primitive fields"))) inpackedsize = packedsize(In) outpackedsize = packedsize(Out) inpackedsize == outpackedsize || @@ -877,7 +882,7 @@ end " do not match; got ", outpackedsize, " and ", inpackedsize, ", respectively."))) in = Ref{In}(x) out = Ref{Out}() - if struct_subpadding(Out, In) + if (ispacked(In) && ispacked(Out)) || struct_subpadding(Out, In) # if packed the same, just copy GC.@preserve in out begin ptr_in = unsafe_convert(Ptr{In}, in) @@ -917,6 +922,12 @@ function _reinterpret_padding(::Type{Out}, x::In) where {Out, In} return out[] end +function String(v::ReinterpretArray{UInt8,1,S,<:Union{Vector{S},Memory{S}},IsReshaped}) where {S,IsReshaped} + len = length(v) + len == 0 && return "" + check_readable(v) # stringifying empty arrays is always allowed + return ccall(:jl_pchar_to_string, Ref{String}, (Ptr{UInt8}, Int), v, len) +end # Reductions with IndexSCartesian2 @@ -960,3 +971,17 @@ end mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, ifirst::SCartesianIndex2, ilast::SCartesianIndex2) where {F,OP} = mapreduce_impl(f, op, A, ifirst, ilast, pairwise_blocksize(f, op)) + +# Fast path for `NTuple{N,E}(::Union{Array,Memory})` with `N >= 32`: when the +# eltype matches and is isbits, the storage is layout-identical to the tuple, +# so a single `reinterpret` load suffices (O(1) in N, unlike the All32 fallback). +function _totuple(T::Type{All32{E,N}}, itr::Union{Array,Memory}) where {E,N} + len = N + 32 + length(itr) >= len || _totuple_err(T) + if isbitstype(E) && eltype(itr) === E + v = length(itr) == len ? itr : view(itr, 1:len) + return @inbounds reinterpret(T, v)[1] + end + elts = collect(E, Iterators.take(itr, len)) + return (elts...,) +end diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 5bfa64d756a97..1f656b333cc8e 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -50,7 +50,7 @@ eltype(::Type{<:ReshapedArrayIterator{I}}) where {I} = @isdefined(I) ? ReshapedI return $(Expr(:new, :(Array{T,N}), :ref, :dims)) end -## reshape!(::Array, ::Dims) returns the original array, but must have the same dimensions and length as the original +## reshape!(::Array, ::Dims) returns the original array, but the new dimensions must have the same total length as the original # see also resize! for a similar operation that can change the length function reshape!(a::Array{T,N}, dims::NTuple{N,Int}) where {T,N} len = Core.checked_dims(dims...) # make sure prod(dims) doesn't overflow (and because of the comparison to length(a)) @@ -220,7 +220,7 @@ end # General reshape function _reshape(parent::AbstractArray, dims::Dims) n = length(parent) - prod(dims) == n || _throw_dmrs(n, "size", dims) + Core.checked_dims(dims...) == n || _throw_dmrs(n, "size", dims) __reshape((parent, IndexStyle(parent)), dims) end @@ -234,10 +234,10 @@ _reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims) function __reshape(p::Tuple{AbstractArray,IndexStyle}, dims::Dims) parent = p[1] - strds = front(size_to_strides(map(length, axes(parent))..., 1)) - strds1 = map(s->max(1,Int(s)), strds) # for resizing empty arrays - mi = map(SignedMultiplicativeInverse, strds1) - ReshapedArray(parent, dims, reverse(mi)) + szs = front(size(parent)) + szs1 = map(s -> max(1, Int(s)), szs) # for resizing empty arrays + mi = map(SignedMultiplicativeInverse, szs1) + ReshapedArray(parent, dims, mi) end function __reshape(p::Tuple{AbstractArray{<:Any,0},IndexCartesian}, dims::Dims) @@ -253,10 +253,10 @@ end size(A::ReshapedArray) = A.dims length(A::ReshapedArray) = length(parent(A)) similar(A::ReshapedArray, eltype::Type, dims::Dims) = similar(parent(A), eltype, dims) +similar(::Type{TA}, dims::Dims) where {T,N,P,TA<:ReshapedArray{T,N,P}} = similar(P, dims) IndexStyle(::Type{<:ReshapedArrayLF}) = IndexLinear() parent(A::ReshapedArray) = A.parent parentindices(A::ReshapedArray) = map(oneto, size(parent(A))) -reinterpret(::Type{T}, A::ReshapedArray, dims::Dims) where {T} = reinterpret(T, parent(A), dims) elsize(::Type{<:ReshapedArray{<:Any,<:Any,P}}) where {P} = elsize(P) unaliascopy(A::ReshapedArray) = typeof(A)(unaliascopy(A.parent), A.dims, A.mi) @@ -268,11 +268,11 @@ mightalias(A::ReshapedArray, B::SubArray) = mightalias(parent(A), B) mightalias(A::SubArray, B::ReshapedArray) = mightalias(A, parent(B)) @inline ind2sub_rs(ax, ::Tuple{}, i::Int) = (i,) -@inline ind2sub_rs(ax, strds, i) = _ind2sub_rs(ax, strds, i - 1) +@inline ind2sub_rs(ax, szs, i) = _ind2sub_rs(ax, szs, i - 1) @inline _ind2sub_rs(ax, ::Tuple{}, ind) = (ind + first(ax[end]),) -@inline function _ind2sub_rs(ax, strds, ind) - d, r = divrem(ind, strds[1]) - (_ind2sub_rs(front(ax), tail(strds), r)..., d + first(ax[end])) +@inline function _ind2sub_rs(ax, szs, ind) + d, r = divrem(ind, szs[1]) + (r + first(ax[1]), _ind2sub_rs(tail(ax), tail(szs), d)...) end offset_if_vec(i::Integer, axs::Tuple{<:AbstractUnitRange}) = i + first(axs[1]) - 1 offset_if_vec(i::Integer, axs::Tuple) = i @@ -354,22 +354,64 @@ unsafe_convert(::Type{Ptr{T}}, a::ReshapedArray{T}) where {T} = unsafe_convert(P const ReshapedUnitRange{T,N,A<:AbstractUnitRange} = ReshapedArray{T,N,A,Tuple{}} viewindexing(I::Tuple{Slice, ReshapedUnitRange, Vararg{ScalarIndex}}) = IndexLinear() viewindexing(I::Tuple{ReshapedRange, Vararg{ScalarIndex}}) = IndexLinear() -compute_stride1(s, inds, I::Tuple{ReshapedRange, Vararg{Any}}) = s*step(I[1].parent) +compute_stride1(s, inds, I::Tuple{ReshapedRange, Vararg{Any}}) = s * Int(step(I[1].parent)) compute_offset1(parent::AbstractVector, stride1::Integer, I::Tuple{ReshapedRange}) = - (@inline; first(I[1]) - first(axes1(I[1]))*stride1) -substrides(strds::NTuple{N,Int}, I::Tuple{ReshapedUnitRange, Vararg{Any}}) where N = - (size_to_strides(strds[1], size(I[1])...)..., substrides(tail(strds), tail(I))...) + (@inline; Int(first(I[1])) - Int(first(axes1(I[1])))*stride1) +substrides(strds::NTuple{N,Int}, I::Tuple{ReshapedRange{<:Integer}, Vararg{Any}}) where N = + (size_to_strides(strds[1]*Int(step(I[1].parent)), size(I[1])...)..., substrides(tail(strds), tail(I))...) -# cconvert(::Type{<:Ptr}, V::SubArray{T,N,P,<:Tuple{Vararg{Union{RangeIndex,ReshapedUnitRange}}}}) where {T,N,P} = V -function unsafe_convert(::Type{Ptr{S}}, V::SubArray{T,N,P,<:Tuple{Vararg{Union{RangeIndex,ReshapedUnitRange}}}}) where {S,T,N,P} +# The indices of a strided SubArray: scalar positions, ranges with a constant +# step, and reshaped ranges with a constant step. +const StridedSubArrayIndex = Union{Integer, AbstractRange{<:Integer}, ReshapedRange{<:Integer}} + +# This exists for backwards compatibility, normally the cconvert method below will be used +function unsafe_convert(::Type{Ptr{S}}, V::SubArray{T,N,P,<:Tuple{Vararg{StridedSubArrayIndex}}}) where {S,T,N,P} + parent = V.parent + Δmem = if _checkcontiguous(Bool, parent) + (first_index(V) - firstindex(parent)) * elsize(parent) + else + _memory_offset(parent, map(first, V.indices)...) + end + return Ptr{S}(unsafe_convert(Ptr{T}, parent) + Int(Δmem)) +end + +struct OffsetCConvert{T, C} + byte_offset::Int + cconv_parent::C +end + +# Avoid unneeded nesting +function _offset_cconvert(::Type{Ptr{T}}, byte_offset::Int, cconv_parent::OffsetCConvert{T}) where {T} + _offset_cconvert( + Ptr{T}, + cconv_parent.byte_offset + byte_offset, + cconv_parent.cconv_parent, + ) +end +function _offset_cconvert(::Type{Ptr{T}}, byte_offset::Int, cconv_parent::C) where {T,C} + OffsetCConvert{T,C}( + byte_offset, + cconv_parent, + ) +end + +function unsafe_convert(::Type{Ptr{S}}, c::OffsetCConvert{T}) where {S, T} + Ptr{S}(unsafe_convert(Ptr{T}, c.cconv_parent) + c.byte_offset) +end + +function cconvert(::Type{Ptr{S}}, V::SubArray{T,N,P,<:Tuple{Vararg{StridedSubArrayIndex}}}) where {S,T,N,P} parent = V.parent - p = cconvert(Ptr{T}, parent) # XXX: this should occur in cconvert, the result is not GC-rooted + p = cconvert(Ptr{T}, parent) Δmem = if _checkcontiguous(Bool, parent) (first_index(V) - firstindex(parent)) * elsize(parent) else _memory_offset(parent, map(first, V.indices)...) end - return Ptr{S}(unsafe_convert(Ptr{T}, p) + Δmem) + _offset_cconvert( + Ptr{T}, + Int(Δmem), + p, + ) end _checkcontiguous(::Type{Bool}, A::AbstractArray) = false @@ -420,3 +462,13 @@ function merge_adjacent_dim(apsz::Dims{N}, apst::Dims{N}, n::Int = 1) where {N} end return sz, st, n end + +map(f, R::ReshapedArray) = reshape(map(f, parent(R)), size(R)) + +iterate(R::ReshapedArray) = iterate(parent(R)) +iterate(R::ReshapedArray, state) = iterate(parent(R), state) + +mapfoldl_impl(f, op, nt, R::ReshapedArray) = mapfoldl_impl(f, op, nt, parent(R)) +mapfoldr_impl(f, op, nt, R::ReshapedArray) = mapfoldr_impl(f, op, nt, parent(R)) + +in(x, R::ReshapedArray) = in(x, parent(R)) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index ba224acf897d4..96050b850516a 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -7,7 +7,7 @@ Get a module's enclosing `Module`. `Main` is its own parent. -See also: [`names`](@ref), [`nameof`](@ref), [`fullname`](@ref), [`@__MODULE__`](@ref). +See also [`names`](@ref), [`nameof`](@ref), [`fullname`](@ref), [`@__MODULE__`](@ref). # Examples ```jldoctest @@ -88,19 +88,23 @@ function moduleloc(m::Module) end """ - names(x::Module; all::Bool=false, imported::Bool=false, usings::Bool=false)::Vector{Symbol} + names(x::Module; all::Bool=false, imported::Bool=false, usings::Bool=false, world::UInt=Base.tls_world_age())::Vector{Symbol} Get a vector of the public names of a `Module`, excluding deprecated names. If `all` is true, then the list also includes non-public names defined in the module, deprecated names, and compiler-generated names. If `imported` is true, then names explicitly imported from other modules are also included. -If `usings` is true, then names explicitly imported via `using` are also included. +If `usings` is true, then names explicitly or implicitly imported via `using` are also included. Names are returned in sorted order. As a special case, all names defined in `Main` are considered \"public\", since it is not idiomatic to explicitly mark names from `Main` as public. +The `world` argument controls the world age used to look up binding partitions, defaulting +to the current task's world age. Pass `world=Base.get_world_counter()` to include names +from the latest world. + !!! note `sym ∈ names(SomeModule)` does *not* imply `isdefined(SomeModule, sym)`. `names` may return symbols marked with `public` or `export`, even if @@ -110,18 +114,22 @@ since it is not idiomatic to explicitly mark names from `Main` as public. `names` may return duplicate names. The duplication happens, e.g. if an `import`ed name conflicts with an already existing identifier. -See also: [`Base.isexported`](@ref), [`Base.ispublic`](@ref), [`Base.@locals`](@ref), [`@__MODULE__`](@ref). +!!! compat "Julia 1.12" + The `usings` argument requires Julia 1.12 or later. + +See also [`Base.isexported`](@ref), [`Base.ispublic`](@ref), [`Base.@locals`](@ref), [`@__MODULE__`](@ref). """ names(m::Module; kwargs...) = sort!(unsorted_names(m; kwargs...)) -unsorted_names(m::Module; all::Bool=false, imported::Bool=false, usings::Bool=false) = - ccall(:jl_module_names, Array{Symbol,1}, (Any, Cint, Cint, Cint), m, all, imported, usings) +unsorted_names(m::Module; all::Bool=false, imported::Bool=false, usings::Bool=false, + world::UInt=tls_world_age()) = + ccall(:jl_module_names, Array{Symbol,1}, (Any, Cint, Cint, Cint, UInt), m, all, imported, usings, world) """ isexported(m::Module, s::Symbol)::Bool -Returns whether a symbol is exported from a module. +Return whether a symbol is exported from a module. -See also: [`ispublic`](@ref), [`names`](@ref) +See also [`ispublic`](@ref), [`names`](@ref). ```jldoctest julia> module Mod @@ -145,14 +153,14 @@ isexported(m::Module, s::Symbol) = ccall(:jl_module_exports_p, Cint, (Any, Any), """ ispublic(m::Module, s::Symbol)::Bool -Returns whether a symbol is marked as public in a module. +Return whether a symbol is marked as public in a module. Exported symbols are considered public. !!! compat "Julia 1.11" This function and the notion of publicity were added in Julia 1.11. -See also: [`isexported`](@ref), [`names`](@ref) +See also [`isexported`](@ref), [`names`](@ref). ```jldoctest julia> module Mod @@ -205,13 +213,17 @@ julia> bar() = nameof(@__FUNCTION__); julia> bar() :bar ``` + +!!! compat "Julia 1.13" + This macro requires at least Julia 1.13. """ macro __FUNCTION__() Expr(:thisfunction) end -# TODO: this is vaguely broken because it only works for explicit calls to -# `Base.deprecate`, not the @deprecated macro: +# Reflects binding-partition deprecation (as set by `Base.deprecate` / `Base.@deprecate_binding`), +# including a binding reached through an implicit `using`, transparently through reexports. +# `@deprecate` deprecates a method rather than the binding, so it is intentionally not reported here. isdeprecated(m::Module, s::Symbol) = ccall(:jl_is_binding_deprecated, Cint, (Any, Any), m, s) != 0 function binding_module(m::Module, s::Symbol) @@ -250,12 +262,16 @@ const PARTITION_KIND_BACKDATED_CONST = 0xb const PARTITION_FLAG_EXPORTED = 0x10 const PARTITION_FLAG_DEPRECATED = 0x20 const PARTITION_FLAG_DEPWARN = 0x40 +const PARTITION_FLAG_IMPLICITLY_EXPORTED = 0x80 +const PARTITION_FLAG_IMPLICITLY_DEPRECATED = 0x100 const PARTITION_MASK_KIND = 0x0f -const PARTITION_MASK_FLAG = 0xf0 +const PARTITION_MASK_FLAG = 0x1f0 const BINDING_FLAG_ANY_IMPLICIT_EDGES = 0x8 +const JL_MODULE_USING_REEXPORT = 0x1 + is_defined_const_binding(kind::UInt8) = (kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_BACKDATED_CONST) is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == PARTITION_KIND_UNDEF_CONST) is_some_imported(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT_GLOBAL || kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) @@ -263,6 +279,8 @@ is_some_implicit(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT_GLOBAL || kind is_some_explicit_imported(kind::UInt8) = (kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) is_some_binding_imported(kind::UInt8) = is_some_explicit_imported(kind) || kind == PARTITION_KIND_IMPLICIT_GLOBAL is_some_guard(kind::UInt8) = (kind == PARTITION_KIND_GUARD || kind == PARTITION_KIND_FAILED || kind == PARTITION_KIND_UNDEF_CONST) +# A global variable: accessed through the binding's own value slot (typed, or `Any` if weakly declared). +is_some_global(kind::UInt8) = (kind == PARTITION_KIND_GLOBAL || kind == PARTITION_KIND_DECLARED) function lookup_binding_partition(world::UInt, b::Core.Binding) ccall(:jl_get_binding_partition, Ref{Core.BindingPartition}, (Any, UInt), b, world) @@ -287,7 +305,18 @@ end partition_restriction(bpart::Core.BindingPartition) = ccall(:jl_bpart_get_restriction_value, Any, (Any,), bpart) -binding_kind(bpart::Core.BindingPartition) = ccall(:jl_bpart_get_kind, UInt8, (Any,), bpart) +# Recover the `Core.Binding` that owns `bpart` by walking the chain to its end: the +# last (oldest) partition's `next` is a backreference to the owning binding, so the +# list can be traversed circularly. +function partition_owner(bpart::Core.BindingPartition) + next = bpart.next + while next isa Core.BindingPartition + next = next.next + end + return next::Core.Binding +end + +binding_kind(bpart::Core.BindingPartition) = UInt8(bpart.kind & PARTITION_MASK_KIND) binding_kind(m::Module, s::Symbol) = binding_kind(lookup_binding_partition(tls_world_age(), GlobalRef(m, s))) """ @@ -306,6 +335,40 @@ function delete_binding(mod::Module, sym::Symbol) ccall(:jl_disable_binding, Cvoid, (Any,), GlobalRef(mod, sym)) end +""" + set_binding_visibility!(mod::Module, sym::Symbol, vis::Symbol) + +Select the declared visibility of `mod.sym`, one of `:export`, `:public`, or +`:none`. This is the programmatic counterpart to the `export` and `public` +keywords; unlike them it can also *retract* a declaration, since `:none` +removes a name's `export` or `public` status. + +Retracting an `export` causes modules that did `using \$mod` to stop resolving +`sym` implicitly, once their world age advances past this call (see +[`invokelatest`](@ref) and [`get_world_counter`](@ref)). Because that +invalidates dependent compiled code, it can be expensive. + +Unlike the exported flag, the public flag is not world-versioned: setting or +clearing it takes effect in every world age at once. Retracting to `:none` +therefore drops public status in all world ages, including ones in which the name +is declared public. Combined with the world-versioned export flag, this means an +older world age can still report a name as [`isexported`](@ref) while no longer +reporting it as [`ispublic`](@ref). + +See also [`isexported`](@ref), [`ispublic`](@ref), [`delete_binding`](@ref). + +!!! compat "Julia 1.14" + This function was added in Julia 1.14. +""" +function set_binding_visibility!(mod::Module, sym::Symbol, vis::Symbol) + state = vis === :none ? Cint(0) : + vis === :public ? Cint(1) : + vis === :export ? Cint(2) : + throw(ArgumentError(LazyString("visibility must be :none, :public, or :export, got ", repr(vis)))) + ccall(:jl_module_set_visibility, Cvoid, (Any, Any, Cint), mod, sym, state) + return nothing +end + """ fieldname(x::DataType, i::Integer) @@ -568,7 +631,14 @@ struct DataTypeLayout # arrayelem_isatomic : 1; # arrayelem_islocked : 1; # isbitsegal : 1; - # padding : 8; + # unused_bits : 3; + # padding : 5; +end + +function DataTypeLayout(dt::DataType) + layout = dt.layout::Ptr{Cvoid} + layout == C_NULL && throw(UndefRefError()) + return unsafe_load(convert(Ptr{DataTypeLayout}, layout)) end """ @@ -578,12 +648,8 @@ Memory allocation minimum alignment for instances of this type. Can be called on any `isconcretetype`, although for Memory it will give the alignment of the elements, not the whole object. """ -function datatype_alignment(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - alignment = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).alignment - return Int(alignment) -end +datatype_alignment(dt::DataType) = (@_foldable_meta; datatype_alignment(DataTypeLayout(dt))) +datatype_alignment(dtl::DataTypeLayout) = Int(dtl.alignment) function uniontype_layout(@nospecialize T::Type) sz = RefValue{Csize_t}(0) @@ -605,6 +671,12 @@ function aligned_sizeof(@nospecialize T::Type) return LLT_ALIGN(sz, al) end elseif allocatedinline(T) + if T === Type{Union{}} + # allocated with the layout of the `typeof(Union{})` singleton + # (cf. `normalize_typeofbottom_layout_alias`), which is what the + # `DataType`-only layout queries below expect + T = Core.TypeofBottom + end al = datatype_alignment(T) return LLT_ALIGN(Core.sizeof(T), al) end @@ -622,27 +694,19 @@ with no intervening padding bits (defined as bits whose value does not impact the semantic value of the instance itself). Can be called on any `isconcretetype`. """ -function datatype_haspadding(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags - return flags & 1 == 1 -end +datatype_haspadding(dt::DataType) = (@_foldable_meta; datatype_haspadding(DataTypeLayout(dt))) +datatype_haspadding(dtl::DataTypeLayout) = dtl.flags & 1 == 1 """ Base.datatype_isbitsegal(dt::DataType)::Bool Return whether egality of the (non-padding bits of the) in-memory representation -of an instance of this type implies semantic egality of the instance itself. -This may not be the case if the type contains to other values whose egality is +of an instance of this type is equivalent to semantic egality of the instance itself. +This may not be the case if the type contains pointers to other values whose egality is independent of their identity (e.g. immutable structs, some types, etc.). """ -function datatype_isbitsegal(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags - return (flags & (1<<7)) != 0 -end +datatype_isbitsegal(dt::DataType) = (@_foldable_meta; datatype_isbitsegal(DataTypeLayout(dt))) +datatype_isbitsegal(dtl::DataTypeLayout) = (dtl.flags & (1<<7)) != 0 """ Base.datatype_nfields(dt::DataType)::UInt32 @@ -651,22 +715,16 @@ Return the number of fields known to this datatype's layout. This may be different from the number of actual fields of the type for opaque types. Can be called on any `isconcretetype`. """ -function datatype_nfields(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - return unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).nfields -end +datatype_nfields(dt::DataType) = (@_foldable_meta; datatype_nfields(DataTypeLayout(dt))) +datatype_nfields(dtl::DataTypeLayout) = dtl.nfields """ Base.datatype_npointers(dt::DataType)::Int Return the number of pointers in the layout of a datatype. """ -function datatype_npointers(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - return unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).npointers -end +datatype_npointers(dt::DataType) = (@_foldable_meta; datatype_npointers(DataTypeLayout(dt))) +datatype_npointers(dtl::DataTypeLayout) = dtl.npointers """ Base.datatype_pointerfree(dt::DataType)::Bool @@ -688,12 +746,8 @@ Can be called on any `isconcretetype`. See also [`fieldoffset`](@ref). """ -function datatype_fielddesc_type(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags - return (flags >> 1) & 3 -end +datatype_fielddesc_type(dt::DataType) = (@_foldable_meta; datatype_fielddesc_type(DataTypeLayout(dt))) +datatype_fielddesc_type(dtl::DataTypeLayout) = (dtl.flags >> 1) & 3 """ Base.datatype_arrayelem(dt::DataType)::Int @@ -705,19 +759,11 @@ Can be called on any `isconcretetype`, but only meaningful on `Memory`. 1 = isboxed 2 = isbitsunion """ -function datatype_arrayelem(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags - return (flags >> 3) & 3 -end +datatype_arrayelem(dt::DataType) = (@_foldable_meta; datatype_arrayelem(DataTypeLayout(dt))) +datatype_arrayelem(dtl::DataTypeLayout) = (dtl.flags >> 3) & 3 -function datatype_layoutsize(dt::DataType) - @_foldable_meta - dt.layout == C_NULL && throw(UndefRefError()) - size = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).size - return size % Int -end +datatype_layoutsize(dt::DataType) = (@_foldable_meta; datatype_layoutsize(DataTypeLayout(dt))) +datatype_layoutsize(dtl::DataTypeLayout) = dtl.size % Int # For type stability, we only expose a single struct that describes everything @@ -751,14 +797,13 @@ function getindex(dtfd::DataTypeFieldDesc, i::Int) fielddesc_type = (layout.flags >> 1) & 3 nfields = layout.nfields @boundscheck ((1 <= i <= nfields) || throw(BoundsError(dtfd, i))) - if fielddesc_type == 0 + if fielddesc_type == 0 # JL_FIELDDESC_8 return FieldDesc(unsafe_load(Ptr{FieldDescStorage{UInt8}}(fd_ptr), i)) - elseif fielddesc_type == 1 + elseif fielddesc_type == 1 # JL_FIELDDESC_16 return FieldDesc(unsafe_load(Ptr{FieldDescStorage{UInt16}}(fd_ptr), i)) - elseif fielddesc_type == 2 + elseif fielddesc_type == 2 # JL_FIELDDESC_32 return FieldDesc(unsafe_load(Ptr{FieldDescStorage{UInt32}}(fd_ptr), i)) - else - # fielddesc_type == 3 + else # fielddesc_type == 3 # JL_FIELDDESC_FOREIGN return FieldDesc(true, true, 0, 0) end end @@ -896,7 +941,10 @@ Determine whether type `T` is a [`Tuple`](@ref) that could appear as a type signature in dispatch. For this to be true, every element of the tuple type must be either: - [concrete](@ref isconcretetype) but not a [kind type](@ref Base.iskindtype) -- a [`Type{U}`](@ref Type) with no free type variables in `U` +- the egality kind `Core.TypeEgal{U}` with no free type variables in `U` (a + `Type{U}` slot is not enough, since it also admits `==`-equal but non-`===` + argument values; `Type{Union{}}` is the exception, the bottom object being + unique) !!! note A dispatch tuple is relevant for method dispatch because it has no inhabited @@ -926,6 +974,9 @@ julia> isdispatchtuple(Tuple{DataType}) false julia> isdispatchtuple(Tuple{Type{Int}}) +false + +julia> isdispatchtuple(Tuple{Core.TypeEgal{Int}}) true julia> isdispatchtuple(Tuple{Type}) @@ -949,6 +1000,9 @@ function ismutationfree(@nospecialize(t)) t = unwrap_unionall(t) if isa(t, DataType) return datatype_ismutationfree(t) + elseif isType(t) + T = type_parameter(t) + return isa(T, Type) && ismutationfree(typeof(T)) elseif isa(t, Union) return ismutationfree(t.a) && ismutationfree(t.b) end @@ -969,6 +1023,9 @@ function isidentityfree(@nospecialize(t)) t = unwrap_unionall(t) if isa(t, DataType) return datatype_isidentityfree(t) + elseif isType(t) + T = type_parameter(t) + return isa(T, Type) && isidentityfree(typeof(T)) elseif isa(t, Union) return isidentityfree(t.a) && isidentityfree(t.b) end @@ -985,15 +1042,15 @@ or [`Core.TypeofBottom`](@ref). All kinds are [concrete](@ref isconcretetype) because types are Julia values. """ -iskindtype(@nospecialize t) = (t === DataType || t === UnionAll || t === Union || t === typeof(Bottom)) +iskindtype(@nospecialize t) = (t === Core.AnyType || t === DataType || t === UnionAll || t === Union || t === TypeEq || t === Core.TypeEgal || t === typeof(Bottom)) """ Base.isconcretedispatch(T) -Returns true if `T` is a [concrete type](@ref isconcretetype) that could appear +Return true if `T` is a [concrete type](@ref isconcretetype) that could appear as an element of a [dispatch tuple](@ref isdispatchtuple). -See also: [`isdispatchtuple`](@ref). +See also [`isdispatchtuple`](@ref). # Examples ```jldoctest @@ -1018,11 +1075,37 @@ using Core: has_free_typevars # and is thus perhaps most similar to the old (pre-1.0) `isconcretetype` query function isdispatchelem(@nospecialize v) return (v === Bottom) || (v === typeof(Bottom)) || isconcretedispatch(v) || - (isType(v) && !has_free_typevars(v)) + isTypeEgal(v) || (isTypeEq(v) && type_parameter(v) === Union{}) end -const _TYPE_NAME = Type.body.name -isType(@nospecialize t) = isa(t, DataType) && t.name === _TYPE_NAME +""" + Base.isType(t) + +Determine whether `t` is a kind whose values are Julia type objects. This is +true for both equality-keyed `Type{T}`/`TypeEq{T}` kinds and egality-keyed +`Core.TypeEgal{T}` kinds. + +Use [`Base.isTypeEq`](@ref) or [`Base.isTypeEgal`](@ref) when the distinction +between equality and egality matters. +""" +isType(@nospecialize t) = isTypeEq(t) || isTypeEgal(t) + +""" + Base.isTypeEq(t) + +Determine whether `t` is an equality-keyed `Type{T}`/`TypeEq{T}` kind. +""" +isTypeEq(@nospecialize t) = isa(t, TypeEq) + +""" + Base.isTypeEgal(t) + +Determine whether `t` is an egality-keyed `Core.TypeEgal{T}` kind. +""" +isTypeEgal(@nospecialize t) = isa(t, Core.TypeEgal) + +type_parameter(t::TypeEq) = getfield(t, :T) +type_parameter(t::Core.TypeEgal) = getfield(t, :T) """ isconcretetype(T) @@ -1042,7 +1125,7 @@ If `T` is not a type, then return `false`. possible for a type `U` to exist such that `T == U`, `isconcretetype(T)`, but `!isconcretetype(U)`. -See also: [`isbits`](@ref), [`isabstracttype`](@ref), [`issingletontype`](@ref). +See also [`isbits`](@ref), [`isabstracttype`](@ref), [`issingletontype`](@ref). # Examples ```jldoctest @@ -1085,7 +1168,7 @@ If `T` is not a type, then return `false`. vice versa, types can be neither concrete nor abstract (for example, `Vector` (a [`UnionAll`](@ref))). -See also: [`isconcretetype`](@ref). +See also [`isconcretetype`](@ref). # Examples ```jldoctest @@ -1099,6 +1182,7 @@ false function isabstracttype(@nospecialize(t)) @_total_meta t = unwrap_unionall(t) + isType(t) && return true # TODO: what to do for `Union`? return isa(t, DataType) && (t.name.flags & 0x1) == 0x1 end @@ -1162,7 +1246,7 @@ We can use it to summarize information about a struct: julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:fieldcount(T)]; julia> structinfo(Base.Filesystem.StatStruct) -14-element Vector{Tuple{UInt64, Symbol, Type}}: +14-element Vector{Tuple{UInt64, Symbol, Core.AnyType}}: (0x0000000000000000, :desc, Union{RawFD, String}) (0x0000000000000008, :device, UInt64) (0x0000000000000010, :inode, UInt64) @@ -1251,12 +1335,25 @@ function _fieldindex_nothrow(T::DataType, name::Symbol) end function fieldindex(t::UnionAll, name::Symbol, err::Bool=true) - t = argument_datatype(t) - if t === nothing + return _fieldindex(t, name, err) +end + +function fieldindex(t::Union, name::Symbol, err::Bool=true) + return _fieldindex(t, name, err) +end + +function _fieldindex(@nospecialize(t), name::Symbol, err::Bool) + idx = _fieldindex_noerror(t, name) + if idx === nothing err && throw(ArgumentError("type does not have definite fields")) return 0 end - return fieldindex(t, name, err) + if idx == 0 && err + t = _fieldindex_error_type(t) + t === nothing && throw(ArgumentError("type does not have definite fields")) + return fieldindex(t, name, true) + end + return idx end function argument_datatype(@nospecialize t) @@ -1265,6 +1362,12 @@ function argument_datatype(@nospecialize t) return ccall(:jl_argument_datatype, Any, (Any,), t)::Union{Nothing,DataType} end +function argument_datatypename(@nospecialize t) + @_total_meta + @noinline + return ccall(:jl_argument_datatypename, Any, (Any,), t)::Union{Nothing,Core.TypeName} +end + function datatype_fieldcount(t::DataType) if t.name === _NAMEDTUPLE_NAME names, types = t.parameters[1], t.parameters[2] @@ -1287,6 +1390,64 @@ function datatype_fieldcount(t::DataType) return length(t.name.names) end +function _typename_noerror(@nospecialize(t)) + t = unwrap_unionall(t) + if t isa DataType + return t.name + elseif t isa Union + aname = _typename_noerror(t.a) + aname === nothing && return nothing + bname = _typename_noerror(t.b) + return aname === bname ? aname : nothing + end + return nothing +end + +function _fieldindex_error_type(@nospecialize(t)) + t = unwrap_unionall(t) + if t isa DataType + fieldcount_noerror(t) === nothing && return nothing + return t + elseif t isa Union + tn = _typename_noerror(t) + tn === nothing && return nothing + t = unwrap_unionall(tn.wrapper) + t isa DataType || return nothing + return t + end + return nothing +end + +function _fieldindex_noerror(@nospecialize(t), name::Symbol) + t = unwrap_unionall(t) + if t isa Union + _typename_noerror(t) === nothing && return nothing + aidx = _fieldindex_noerror(t.a, name) + aidx === nothing && return nothing + bidx = _fieldindex_noerror(t.b, name) + return aidx === bidx ? aidx : nothing + elseif t isa DataType + return fieldindex(t, name, false) + end + return nothing +end + +function _fieldcount_noerror(@nospecialize(t)) + t === Union{} && return 0 + t = unwrap_unionall(t) + if t isa Union + _typename_noerror(t) === nothing && return nothing + acount = _fieldcount_noerror(t.a) + acount === nothing && return nothing + bcount = _fieldcount_noerror(t.b) + return acount === bcount ? acount : nothing + elseif t === Union{} + return 0 + end + t isa DataType || return nothing + return datatype_fieldcount(t) +end + """ fieldcount(t::Type) @@ -1295,14 +1456,15 @@ An error is thrown if the type is too abstract to determine this. """ function fieldcount(@nospecialize t) @_foldable_meta - if t isa UnionAll || t isa Union - t = argument_datatype(t) - if t === nothing - throw(ArgumentError("type does not have a definite number of fields")) - end - elseif t === Union{} + if t === Union{} throw(ArgumentError("The empty type does not have a well-defined number of fields since it does not have instances.")) end + t = unwrap_unionall(t) + if t isa Union + fcount = _fieldcount_noerror(t) + fcount === nothing && throw(ArgumentError("type does not have a definite number of fields")) + return fcount + end if !(t isa DataType) throw(TypeError(:fieldcount, DataType, t)) end @@ -1314,28 +1476,7 @@ function fieldcount(@nospecialize t) end function fieldcount_noerror(@nospecialize t) - if t isa UnionAll || t isa Union - t = argument_datatype(t) - if t === nothing - return nothing - end - elseif t === Union{} - return 0 - end - t isa DataType || return nothing - if t.name === _NAMEDTUPLE_NAME - names, types = t.parameters - if names isa Tuple - return length(names) - end - if types isa DataType && types <: Tuple - return fieldcount_noerror(types) - end - return nothing - elseif isabstracttype(t) || (t.name === Tuple.name && isvatuple(t)) - return nothing - end - return isdefined(t, :types) ? length(t.types) : length(t.name.names) + return _fieldcount_noerror(t) end @@ -1358,7 +1499,7 @@ julia> fieldtypes(Foo) (Int64, String) ``` """ -fieldtypes(T::Type) = (@_foldable_meta; ntupleany(i -> fieldtype(T, i), fieldcount(T))) +fieldtypes(@nospecialize T::Type) = (@_foldable_meta; ntupleany(i -> fieldtype(T, i), fieldcount(T))) # return all instances, for types that can be enumerated @@ -1387,7 +1528,7 @@ function to_tuple_type(@nospecialize(t)) if isa(p, Core.TypeofVararg) p = unwrapva(p) end - if !(isa(p, Type) || isa(p, TypeVar)) + if !(isa(p, Core.AnyType) || isa(p, TypeVar)) error("argument tuple type must contain only types") end end @@ -1399,6 +1540,8 @@ end function signature_type(@nospecialize(f), @nospecialize(argtypes)) argtypes = to_tuple_type(argtypes) + # `Core.Typeof` matches the per-argument key of the dispatch tuple + # constructed by `jl_inst_arg_tuple_type`. ft = Core.Typeof(f) u = unwrap_unionall(argtypes)::DataType return rewrap_unionall(Tuple{ft, u.parameters...}, argtypes) @@ -1417,18 +1560,66 @@ end Determine whether `t` is a Type for which one or more of its parameters is `Union{}`. """ -function has_bottom_parameter(t::DataType) - for p in t.parameters - has_bottom_parameter(p) && return true +function has_bottom_parameter(@nospecialize(t::Core.AnyType)) + t === Bottom && return true + ty = typeof(t) + if ty === DataType + for p in getfield(t, :parameters) + has_bottom_parameter(p) && return true + end + elseif ty === TypeEq || ty === Core.TypeEgal + return has_bottom_parameter(type_parameter(t)) + elseif ty === UnionAll + return has_bottom_parameter(unwrap_unionall(t)) + elseif ty === Union + return has_bottom_parameter(getfield(t, :a)) & has_bottom_parameter(getfield(t, :b)) end return false end -has_bottom_parameter(t::typeof(Bottom)) = true -has_bottom_parameter(t::UnionAll) = has_bottom_parameter(unwrap_unionall(t)) -has_bottom_parameter(t::Union) = has_bottom_parameter(t.a) & has_bottom_parameter(t.b) has_bottom_parameter(t::TypeVar) = has_bottom_parameter(t.ub) has_bottom_parameter(::Any) = false +function find_free_typevars(@nospecialize(t)) + return ccall(:jl_find_free_typevars, Array{Any, 1}, (Any,), t) +end + +""" + rewrap_free_typevars(@nospecialize(t), pre=Core.svec()) + +Wrap `t` in `UnionAll` for each `TypeVar` that is free in `t` but not referenced +in `pre` (the typevars that were free in the consumer's input before an env was +applied). + +This is the consumer-side complement to `jl_type_intersection_env` / +`jl_subtype_env`: the env svec may contain entries whose typevar identity is +shared across slots. Rather than wrapping each slot independently (which breaks +that identity), callers that build a new type from the env should apply this +helper once to the final reconstructed type. +""" +function rewrap_free_typevars(@nospecialize(t), pre=Core.svec()) + has_free_typevars(t) || return t + # UnionAll cannot directly wrap a Vararg; the caller must wrap it in Tuple first + isvarargtype(t) && return t + fv = find_free_typevars(t) + for i in length(fv):-1:1 + v = fv[i]::TypeVar + wrap = true + for p in pre + if p === v + wrap = false + break + end + end + wrap && (t = UnionAll(v, t)) + end + return t +end + +function typeintersect_env(@nospecialize(a), @nospecialize(b)) + (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), a, b)::SimpleVector + Pair{Any, SimpleVector}(ti, env) +end + min_world(m::Core.CodeInstance) = m.min_world max_world(m::Core.CodeInstance) = m.max_world min_world(m::Core.CodeInfo) = m.min_world @@ -1437,7 +1628,7 @@ max_world(m::Core.CodeInfo) = m.max_world """ get_world_counter() -Returns the current maximum world-age counter. This counter is monotonically +Return the current maximum world-age counter. This counter is monotonically increasing. !!! warning @@ -1451,7 +1642,7 @@ get_world_counter() = ccall(:jl_get_world_counter, UInt, ()) """ tls_world_age() -Returns the world the [current_task()](@ref) is executing within. +Return the world the [`current_task`](@ref) is executing within. """ tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) @@ -1470,7 +1661,7 @@ of the documented interface of `x`. If you want it to also return "private" property names intended for internal use, pass `true` for the optional second argument. REPL tab completion on `x.` shows only the `private=false` properties. -See also: [`hasproperty`](@ref), [`hasfield`](@ref). +See also [`hasproperty`](@ref), [`hasfield`](@ref). """ propertynames(x) = fieldnames(typeof(x)) propertynames(m::Module) = names(m) @@ -1485,7 +1676,7 @@ Return a boolean indicating whether the object `x` has `s` as one of its own pro !!! compat "Julia 1.2" This function requires at least Julia 1.2. -See also: [`propertynames`](@ref), [`hasfield`](@ref). +See also [`propertynames`](@ref), [`hasfield`](@ref). """ hasproperty(x, s::Symbol) = s in propertynames(x) @@ -1535,10 +1726,14 @@ If `types` is specified, return an array of methods whose types match. If `module` is specified, return an array of methods defined in that module. A list of modules can also be specified as an array or set. +The methods are ordered from most to least specific. The relative order of +methods without a specificity relationship (i.e. ambiguous or incomparable) +is unspecified. + !!! compat "Julia 1.4" At least Julia 1.4 is required for specifying a module. -See also: [`which`](@ref), [`@which`](@ref Main.InteractiveUtils.@which) and [`methodswith`](@ref Main.InteractiveUtils.methodswith). +See also [`which`](@ref), [`@which`](@ref Main.InteractiveUtils.@which), [`methodswith`](@ref Main.InteractiveUtils.methodswith). """ function methods(@nospecialize(f), @nospecialize(t), mod::Union{Tuple{Module},AbstractArray{Module},AbstractSet{Module},Nothing}=nothing) @@ -1644,7 +1839,7 @@ ast_slotflag(@nospecialize(code), i) = ccall(:jl_ir_slotflag, UInt8, (Any, Csize """ may_invoke_generator(method, atype, sparams)::Bool -Computes whether or not we may invoke the generator for the given `method` on +Compute whether or not we may invoke the generator for the given `method` on the given `atype` and `sparams`. For correctness, all generated function are required to return monotonic answers. However, since we don't expect users to be able to successfully implement this criterion, we only call generated @@ -1687,7 +1882,7 @@ function may_invoke_generator(method::Method, @nospecialize(atype), sparams::Sim firstarg = 1 for i = 1:nsparams - if isa(sparams[i], TypeVar) + if isa(sparams[i], SimpleVector) if (ast_slotflag(code, firstarg + i) & SLOT_USED) != 0 return false end @@ -1704,7 +1899,7 @@ function may_invoke_generator(method::Method, @nospecialize(atype), sparams::Sim end if method.isva # If the va argument is used, we need to ensure that all arguments that - # contribute to the va tuple are dispatchelemes + # contribute to the va tuple are dispatch elements if (ast_slotflag(code, firstarg + nargs + nsparams) & SLOT_USED) != 0 for i = (non_va_args+1):length(at.parameters) if !isdispatchelem(at.parameters[i]) @@ -1746,8 +1941,7 @@ function normalize_typevars(method::Method, @nospecialize(atype), sparams::Simpl at2 = subst_trivial_bounds(atype) if at2 !== atype && at2 == atype atype = at2 - sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), at2, method.sig)::SimpleVector - sparams = sp_[2]::SimpleVector + (_, sparams) = typeintersect_env(at2, method.sig) end return Pair{Any,SimpleVector}(atype, sparams) end @@ -1765,6 +1959,9 @@ is_nospecializeinfer(method::Method) = method.nospecializeinfer && is_nospeciali Return MethodInstance corresponding to `atype` and `sparams`. No widening / narrowing / compileable-normalization of `atype` is performed. +A slot for an argument known by egality must already carry the egality kind +(`TypeEgal{X}`, as `Compiler.widenconst` produces); a closed `Type{X}` slot +means the argument is only known up to type equality (#61323). """ function specialize_method(method::Method, @nospecialize(atype), sparams::SimpleVector; preexisting::Bool=false) @inline @@ -1792,9 +1989,6 @@ hasintersect(@nospecialize(a), @nospecialize(b)) = typeintersect(a, b) !== Botto # scoping # ########### -_topmod(m::Module) = ccall(:jl_base_relative_to, Any, (Any,), m)::Module - - # high-level, more convenient method lookup functions function visit(f, mt::Core.MethodTable) @@ -1803,11 +1997,13 @@ function visit(f, mt::Core.MethodTable) end function visit(f, mc::Core.TypeMapLevel) function avisit(f, e::Memory{Any}) - for i in 2:2:length(e) + # slot 1 holds the smallintset index; key/value pairs follow, so values + # live on the odd slots starting at 3 (see mtcache layout in src/typemap.c) + for i in 3:2:length(e) isassigned(e, i) || continue ei = e[i] if ei isa Memory{Any} - for j in 2:2:length(ei) + for j in 3:2:length(ei) isassigned(ei, j) || continue visit(f, ei[j]) end @@ -1869,11 +2065,11 @@ end length(specs::MethodSpecializations) = count(Returns(true), specs) function length(mt::Core.MethodTable) - n = 0 + n = Ref(0) visit(mt) do m - n += 1 + n[] += 1 end - return n::Int + return n[] end isempty(mt::Core.MethodTable) = (mt.defs === nothing) diff --git a/base/ryu/Ryu.jl b/base/ryu/Ryu.jl index 86b9a64c002d5..63bc2d0c12262 100644 --- a/base/ryu/Ryu.jl +++ b/base/ryu/Ryu.jl @@ -1,7 +1,7 @@ module Ryu using .Base.Libc -import .Base: significand_bits, significand_mask, exponent_bits, exponent_mask, exponent_bias, exponent_max, uinttype +using .Base: significand_bits, significand_mask, exponent_mask, exponent_bias, exponent_max, uinttype include("utils.jl") include("shortest.jl") @@ -17,6 +17,15 @@ neededdigits(::Type{Float64}) = 309 + 17 neededdigits(::Type{Float32}) = 39 + 9 + 2 neededdigits(::Type{Float16}) = 9 + 5 + 9 +""" + Ryu.neededfloatdigits(T) + +Number of digits necessary to represent type `T` in shortest precision. +""" +neededfloatdigits(::Type{Float64}) = 24 +neededfloatdigits(::Type{Float32}) = 17 +neededfloatdigits(::Type{Float16}) = 20 + """ Ryu.writeshortest(x, plus=false, space=false, hash=true, precision=-1, expchar=UInt8('e'), padexp=false, decchar=UInt8('.'), typed=false, compact=false) Ryu.writeshortest(buf::AbstractVector{UInt8}, pos::Int, x, args...) @@ -46,14 +55,17 @@ function writeshortest(x::T, decchar::UInt8=UInt8('.'), typed::Bool=false, compact::Bool=false) where {T <: Base.IEEEFloat} - buf = Base.StringVector(neededdigits(T)) + precision = Int(precision) + precision >= -1 || throw(ArgumentError("precision must be at least -1")) + bufsize = precision < 0 ? neededdigits(T) : Base.checked_add(precision, neededdigits(T)) + buf = Base.StringVector(bufsize) pos = writeshortest(buf, 1, x, plus, space, hash, precision, expchar, padexp, decchar, typed, compact) return String(resize!(buf, pos - 1)) end """ Ryu.writefixed(x, precision, plus=false, space=false, hash=false, decchar=UInt8('.'), trimtrailingzeros=false) - Ryu.writefixed(buf::AbstractVector{UInt8}, pos::Int, x, args...) + Ryu.writefixed(buf::AbstractVector{UInt8}, pos::Int, x, precision, args...) Convert a float value `x` into a "fixed" size decimal string of the provided precision. This function allows achieving the `%f` printf format. @@ -63,7 +75,7 @@ Various options for the output format include: * `plus`: for positive `x`, prefix decimal string with a `'+'` character * `space`: for positive `x`, prefix decimal string with a `' '` character; overridden if `plus=true` * `hash`: whether the decimal point should be written, even if no additional digits are needed for precision - * `precision`: minimum number of significant digits to be included in the decimal string; extra `'0'` characters will be added for padding if necessary + * `precision`: exact number of digits after the decimal point to be included in the decimal string; extra `'0'` characters will be added for padding if necessary * `decchar`: decimal point character to be used * `trimtrailingzeros`: whether trailing zeros of fractional part should be removed """ @@ -74,14 +86,16 @@ function writefixed(x::T, hash::Bool=false, decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} - buf = Base.StringVector(precision + neededdigits(T)) + precision = Int(precision) + precision >= 0 || throw(ArgumentError("precision must be non-negative")) + buf = Base.StringVector(Base.checked_add(precision, neededdigits(T))) pos = writefixed(buf, 1, x, precision, plus, space, hash, decchar, trimtrailingzeros) return String(resize!(buf, pos - 1)) end """ Ryu.writeexp(x, precision, plus=false, space=false, hash=false, expchar=UInt8('e'), decchar=UInt8('.'), trimtrailingzeros=false) - Ryu.writeexp(buf::AbstractVector{UInt8}, pos::Int, x, args...) + Ryu.writeexp(buf::AbstractVector{UInt8}, pos::Int, x, precision, args...) Convert a float value `x` into a scientific notation decimal string. This function allows achieving the `%e` printf format. @@ -91,7 +105,7 @@ Various options for the output format include: * `plus`: for positive `x`, prefix decimal string with a `'+'` character * `space`: for positive `x`, prefix decimal string with a `' '` character; overridden if `plus=true` * `hash`: whether the decimal point should be written, even if no additional digits are needed for precision - * `precision`: minimum number of significant digits to be included in the decimal string; extra `'0'` characters will be added for padding if necessary + * `precision`: exact number of digits after the decimal point to be included in the decimal string; extra `'0'` characters will be added for padding if necessary * `expchar`: character to use exponent component in scientific notation * `decchar`: decimal point character to be used * `trimtrailingzeros`: whether trailing zeros should be removed @@ -104,23 +118,25 @@ function writeexp(x::T, expchar::UInt8=UInt8('e'), decchar::UInt8=UInt8('.'), trimtrailingzeros::Bool=false) where {T <: Base.IEEEFloat} - buf = Base.StringVector(precision + neededdigits(T)) + precision = Int(precision) + precision >= 0 || throw(ArgumentError("precision must be non-negative")) + buf = Base.StringVector(Base.checked_add(precision, neededdigits(T))) pos = writeexp(buf, 1, x, precision, plus, space, hash, expchar, decchar, trimtrailingzeros) return String(resize!(buf, pos - 1)) end function Base.show(io::IO, x::T, forceuntyped::Bool=false, fromprint::Bool=false) where {T <: Base.IEEEFloat} compact = get(io, :compact, false)::Bool - buf = Base.StringVector(neededdigits(T)) + buf = Memory{UInt8}(undef, neededfloatdigits(T)) typed = !forceuntyped && !compact && Base.nonnothing_nonmissing_typeinfo(io) !== typeof(x) pos = writeshortest(buf, 1, x, false, false, true, -1, (x isa Float32 && !fromprint) ? UInt8('f') : UInt8('e'), false, UInt8('.'), typed, compact) - write(io, resize!(buf, pos - 1)) + write(io, view(buf, 1:pos - 1)) return end function Base.string(x::T) where {T <: Base.IEEEFloat} - buf = Base.StringVector(neededdigits(T)) + buf = Base.StringVector(neededfloatdigits(T)) pos = writeshortest(buf, 1, x, false, false, true, -1, UInt8('e'), false, UInt8('.'), false, false) return String(resize!(buf, pos - 1)) diff --git a/base/ryu/exp.jl b/base/ryu/exp.jl index 4f749668867e2..f3097159e7168 100644 --- a/base/ryu/exp.jl +++ b/base/ryu/exp.jl @@ -1,7 +1,10 @@ function writeexp(buf, pos, v::T, - precision=-1, plus=false, space=false, hash=false, + precision, plus=false, space=false, hash=false, expchar=UInt8('e'), decchar=UInt8('.'), trimtrailingzeros=false) where {T <: Base.IEEEFloat} - @assert 0 < pos <= length(buf) + pos = Int(pos) + precision = Int(precision) + precision >= 0 || throw(ArgumentError("precision must be non-negative")) + @assert 0 < pos <= length(buf) "invalid pos" startpos = pos x = Float64(v) pos = append_sign(x, plus, space, buf, pos) @@ -38,7 +41,7 @@ function writeexp(buf, pos, v::T, return pos + 3 end - bits = Core.bitcast(UInt64, x) + bits = reinterpret(UInt64, x) mant = bits & MANTISSA_MASK exp = Int((bits >> 52) & EXP_MASK) @@ -49,7 +52,6 @@ function writeexp(buf, pos, v::T, e2 = exp - 1023 - 52 m2 = (Int64(1) << 52) | mant end - nonzero = false precision += 1 digits = zero(UInt32) printedDigits = 0 @@ -140,12 +142,11 @@ function writeexp(buf, pos, v::T, end lastDigit = zero(UInt32) if availableDigits > maximum - for k = 0:(availableDigits - maximum - 1) + for _ = 0:(availableDigits - maximum - 1) lastDigit = digits % UInt32(10) digits = div(digits, UInt32(10)) end end - roundUp = 0 if lastDigit != 5 roundUp = lastDigit > 5 ? 1 : 0 else diff --git a/base/ryu/fixed.jl b/base/ryu/fixed.jl index 96777059bc284..334e6d2dc3cd2 100644 --- a/base/ryu/fixed.jl +++ b/base/ryu/fixed.jl @@ -1,7 +1,10 @@ function writefixed(buf, pos, v::T, - precision=-1, plus=false, space=false, hash=false, + precision, plus=false, space=false, hash=false, decchar=UInt8('.'), trimtrailingzeros=false) where {T <: Base.IEEEFloat} - @assert 0 < pos <= length(buf) + pos = Int(pos) + precision = Int(precision) + precision >= 0 || throw(ArgumentError("precision must be non-negative")) + @assert 0 < pos <= length(buf) "invalid pos" startpos = pos x = Float64(v) pos = append_sign(x, plus, space, buf, pos) @@ -34,7 +37,7 @@ function writefixed(buf, pos, v::T, return pos + 3 end - bits = Core.bitcast(UInt64, x) + bits = reinterpret(UInt64, x) mant = bits & MANTISSA_MASK exp = Int((bits >> 52) & EXP_MASK) diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index b4ba8255e0b8a..9b10f0e1305b2 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -14,7 +14,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. # mf * 2^ef == f mf = (one(U) << significand_bits(T)) | m ef = e - exponent_bias(T) - significand_bits(T) - f_isinteger = mf & ((one(U) << -ef) - one(U)) == 0 + f_isinteger = mf & ((one(U) << -ef) -% one(U)) == 0 if ef > 0 || ef < -Base.significand_bits(T) || !f_isinteger # fixup subnormals @@ -56,7 +56,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. end end if q <= qinvbound(T) - if ((v % UInt32) - 5 * div(v, 5)) == 0 + if ((v % UInt32) -% 5 *% div(v, 5)) == 0 b_allzero = pow5(v, q) elseif mf_iseven a_allzero = pow5(u, q) @@ -99,9 +99,9 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. if c_div10 <= a_div10 break end - a_mod10 = (a % UInt32) - UInt32(10) * (a_div10 % UInt32) + a_mod10 = (a % UInt32) -% UInt32(10) *% (a_div10 % UInt32) b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) a_allzero &= a_mod10 == 0 b_allzero &= b_lastdigit == 0 b_lastdigit = b_mod10 % UInt8 @@ -113,13 +113,13 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. if a_allzero while true a_div10 = div(a, 10) - a_mod10 = (a % UInt32) - UInt32(10) * (a_div10 % UInt32) + a_mod10 = (a % UInt32) -% UInt32(10) *% (a_div10 % UInt32) if a_mod10 != 0 && (maxsignif === nothing || b < maxsignif) break end c_div10 = div(c, 10) b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) b_allzero &= b_lastdigit == 0 b_lastdigit = b_mod10 % UInt8 b = b_div10 @@ -139,7 +139,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. a_div100 = div(a, 100) if c_div100 > a_div100 b_div100 = div(b, 100) - b_mod100 = (b % UInt32) - UInt32(100) * (b_div100 % UInt32) + b_mod100 = (b % UInt32) -% UInt32(100) *% (b_div100 % UInt32) roundup = b_mod100 >= 50 b = b_div100 c = c_div100 @@ -153,7 +153,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. break end b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) roundup = b_mod10 >= 5 b = b_div10 c = c_div10 @@ -166,7 +166,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. # reduce to max significant digits while true b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) if b <= maxsignif break end @@ -180,7 +180,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. # remove trailing zeros while true b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) if b_mod10 != 0 break end @@ -201,7 +201,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. # reduce to max significant digits while true b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) if b <= maxsignif break end @@ -214,7 +214,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. end while true b_div10 = div(b, 10) - b_mod10 = (b % UInt32) - UInt32(10) * (b_div10 % UInt32) + b_mod10 = (b % UInt32) -% UInt32(10) *% (b_div10 % UInt32) if b_mod10 != 0 break end @@ -229,7 +229,10 @@ function writeshortest(buf::AbstractVector{UInt8}, pos, x::T, plus=false, space=false, hash=true, precision=-1, expchar=UInt8('e'), padexp=false, decchar=UInt8('.'), typed=false, compact=false) where {T} - @assert 0 < pos <= length(buf) + pos = Int(pos) + precision = Int(precision) + precision >= -1 || throw(ArgumentError("precision must be at least -1")) + @assert 0 < pos <= length(buf) "invalid pos" # special cases if x == 0 if typed && x isa Float16 @@ -378,8 +381,11 @@ function writeshortest(buf::AbstractVector{UInt8}, pos, x::T, else pointoff = olength - abs(nexp) # shift bytes after pointoff to make room for decchar - ptr = pointer(buf) - memmove(ptr + pos + pointoff, ptr + pos + pointoff - 1, olength - pointoff + 1) + buf_cconv = Base.cconvert(Ptr{UInt8}, buf) + GC.@preserve buf_cconv begin + ptr = Base.unsafe_convert(Ptr{UInt8}, buf_cconv) + memmove(ptr + pos + pointoff, ptr + pos + pointoff - 1, (olength - pointoff)%Csize_t) + end @inbounds buf[pos + pointoff] = decchar pos += olength + 1 precision -= olength diff --git a/base/ryu/utils.jl b/base/ryu/utils.jl index 2064dfbefcecd..c24dce7b274ad 100644 --- a/base/ryu/utils.jl +++ b/base/ryu/utils.jl @@ -22,7 +22,7 @@ qbound(::Type{Float64}) = 63 """ Ryu.log10pow2(e::Integer) -Computes `floor(log10(2^e))`. This is valid for all `e < 1651`. +Compute `floor(log10(2^e))`. This is valid for all `e < 1651`. """ log10pow2(e) = (e * 78913) >> 18 @@ -30,19 +30,19 @@ log10pow2(e) = (e * 78913) >> 18 """ Ryu.log10pow5(e::Integer) -Computes `floor(log10(5^e))`. This is valid for all `e < 2621`. +Compute `floor(log10(5^e))`. This is valid for all `e < 2621`. """ log10pow5(e) = (e * 732923) >> 20 """ Ryu.pow5bits(e) -Computes `e == 0 ? 1 : ceil(log2(5^e))`. This is valid for `e < 3529` (if performend in `Int32` arithmetic). +Compute `e == 0 ? 1 : ceil(log2(5^e))`. This is valid for `e < 3529` (if performed in `Int32` arithmetic). """ pow5bits(e) = ((e * 1217359) >> 19) + 1 """" - Ryu.mulshift(m::U, mula, j) where {U<:Unsigned} + Ryu.mulshift(m::U, mul, j) where {U<:Unsigned} Compute `(m * mul) >> j`, where `j >= 8*sizeof(U)`. The type of the results is the larger of `U` or `UInt32`. """ @@ -61,7 +61,23 @@ lengthforindex(idx) = div(((Int64(16 * idx) * 1292913986) >> 32) + 1 + 16 + 8, 9 Return `true` if `5^p` is a divisor of `x`. """ -pow5(x, p) = x % (UInt64(5)^p) == 0 +function pow5(x, p) + # Assumptions: + # 0 < x < (1 << 53) + # 0 <= p <= 308 + wrapped_pow = Base.power_by_squaring(UInt64(5), p; mul=*%) + + # We are using wrapping arithmetic here, which can overflow, but we get the right answer anyway + # by the following argument (#47511): + # 1. False Negative + # No overflow occurs unless big(5)^p > typemax(UInt64), but typemax(UInt64) > (1 << 53), so + # z is never divisible by big(5)^p. QED. + # 2. False Positive + # We ask ∃ m2, p s.t. big(5)^p && m2 % wrapped_pow == 0. + # By exhaustive checking `all(p->UInt64(5)^p > 1<<53, 28:308)`, so there does not exist any such + # `m2` that satiesfies our assumptions above. QED. + x % wrapped_pow == 0 +end """ Ryu.pow2(x, p) @@ -146,15 +162,15 @@ function umul256(a::UInt128, bHi::UInt64, bLo::UInt64) b00Lo = b00 % UInt64 b00Hi = (b00 >> 64) % UInt64 - mid1 = b10 + b00Hi + mid1 = b10 +% b00Hi mid1Lo = mid1 % UInt64 mid1Hi = (mid1 >> 64) % UInt64 - mid2 = b01 + mid1Lo + mid2 = b01 +% mid1Lo mid2Lo = mid2 % UInt64 mid2Hi = (mid2 >> 64) % UInt64 - pHi = b11 + mid1Hi + mid2Hi + pHi = b11 +% mid1Hi +% mid2Hi pLo = (UInt128(mid2Lo) << 64) | b00Lo return pLo, pHi end @@ -166,6 +182,14 @@ Compute `pHi = (a*b)>>128` where `b = bLo + bHi<<64`. """ umul256_hi(a::UInt128, bHi::UInt64, bLo::UInt64) = umul256(a, bHi, bLo)[2] +function mod1e9(v::UInt128) + # TODO: Fix LLVM to perform this optimization itself on 128 bit integers + # return (v % 10^9) % UInt32 + multiplied = umul256_hi(v, 0x89705F4136B4A597, 0x31680A88F8953031) + shifted = (multiplied >> 29) % UInt32 + return (v % UInt32) -% UInt32(1000000000) *% shifted +end + """ Ryu.mulshiftmod1e9(m, mula, mulb, mulc, j)::UInt32 @@ -175,12 +199,10 @@ function mulshiftmod1e9(m, mula, mulb, mulc, j) b0 = UInt128(m) * mula b1 = UInt128(m) * mulb b2 = UInt128(m) * mulc - mid = b1 + ((b0 >> 64) % UInt64) - s1 = b2 + ((mid >> 64) % UInt64) + mid = b1 +% ((b0 >> 64) % UInt64) + s1 = b2 +% ((mid >> 64) % UInt64) v = s1 >> (j - 128) - multiplied = umul256_hi(v, 0x89705F4136B4A597, 0x31680A88F8953031) - shifted = (multiplied >> 29) % UInt32 - return (v % UInt32) - UInt32(1000000000) * shifted + return mod1e9(v) end function append_sign(x, plus::Bool, space::Bool, buf, pos::Int) diff --git a/base/scope.jl b/base/scope.jl new file mode 100644 index 0000000000000..3e3448f0687ff --- /dev/null +++ b/base/scope.jl @@ -0,0 +1,47 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Early definitions of the dynamic-scope types used by `Core.current_scope()`. +# The user-facing API lives in the `ScopedValues` module (scopedvalues.jl, included +# much later in bootstrap); the types live here so that early Base code — in +# particular the cancellation machinery (`cancellation.jl`) — can resolve +# scoped state during bootstrap. + +""" + AbstractScopedValue{T} + +Abstract base type for scoped values that propagate values across +dynamic scopes. All scoped value types must extend this abstract type. + +See also: [`ScopedValues.ScopedValue`](@ref), [`ScopedValues.LazyScopedValue`](@ref) + +!!! compat "Julia 1.13" + AbstractScopedValue requires Julia 1.13+. +""" +abstract type AbstractScopedValue{T} end + +const ScopeStorage = PersistentDict{AbstractScopedValue, Any} + +struct Scope + values::ScopeStorage +end + +Scope(scope::Scope) = scope + +function Scope(parent::Union{Nothing, Scope}, key::AbstractScopedValue{T}, value) where T + val = convert(T, value) + if parent === nothing + return Scope(ScopeStorage(key=>val)) + end + return Scope(ScopeStorage(parent.values, key=>val)) +end + +function Scope(scope, pair::Pair{<:AbstractScopedValue}) + return Scope(scope, pair...) +end + +function Scope(scope, pair1::Pair{<:AbstractScopedValue}, pair2::Pair{<:AbstractScopedValue}, pairs::Pair{<:AbstractScopedValue}...) + # Unroll this loop through recursion to make sure that + # our compiler optimization support works + return Scope(Scope(scope, pair1...), pair2, pairs...) +end +Scope(::Nothing) = nothing diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 39e3c2c076718..fbd852eea2416 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -2,8 +2,44 @@ module ScopedValues -export ScopedValue, with, @with +export ScopedValue, LazyScopedValue, with, @with, ScopedThunk public get +using Base: AbstractScopedValue, Scope, ScopeStorage + + +""" + LazyScopedValue{T}(f::OncePerProcess{T}) + +A scoped value that uses an `OncePerProcess{T}` to lazily compute its default value +when none has been set in the current scope. Unlike `ScopedValue`, the default is +not evaluated at construction time but only when first accessed. + +# Examples + +```julia-repl +julia> using Base.ScopedValues; + +julia> const editor = LazyScopedValue(OncePerProcess(() -> ENV["JULIA_EDITOR"])); + +julia> editor[] +"vim" + +julia> with(editor => "emacs") do + editor[] + end +"emacs" + +julia> editor[] +"vim" +``` + +!!! compat "Julia 1.13" + LazyScopedValue requires Julia 1.13+. +""" +mutable struct LazyScopedValue{T} <: AbstractScopedValue{T} + const getdefault::OncePerProcess{T} +end + """ ScopedValue(x) @@ -40,17 +76,25 @@ julia> sval[] Scoped values were introduced in Julia 1.11. In Julia 1.8+ a compatible implementation is available from the package ScopedValues.jl. """ -mutable struct ScopedValue{T} +mutable struct ScopedValue{T} <: AbstractScopedValue{T} # NOTE this struct must be defined as mutable one since it's used as a key of # `ScopeStorage` dictionary and thus needs object identity - const has_default::Bool # this field is necessary since isbitstype `default` field may be initialized with undefined value + const hasdefault::Bool # this field is necessary since isbitstype `default` field may be initialized with undefined value const default::T - ScopedValue{T}() where T = new(false) + ScopedValue{T}() where T = new{T}(false) ScopedValue{T}(val) where T = new{T}(true, val) ScopedValue(val::T) where T = new{T}(true, val) end -Base.eltype(::ScopedValue{T}) where {T} = T +Base.eltype(::AbstractScopedValue{T}) where {T} = T + +hasdefault(val::ScopedValue) = val.hasdefault +hasdefault(val::LazyScopedValue) = true +hasdefault(val::Base.CancelTokenKey) = true + +getdefault(val::ScopedValue) = val.hasdefault ? val.default : throw(KeyError(val)) +getdefault(val::LazyScopedValue) = val.getdefault() +getdefault(val::Base.CancelTokenKey) = nothing """ isassigned(val::ScopedValue) @@ -72,40 +116,13 @@ julia> isassigned(b) false ``` """ -function Base.isassigned(val::ScopedValue) - val.has_default && return true +function Base.isassigned(val::AbstractScopedValue) + hasdefault(val) && return true scope = Core.current_scope()::Union{Scope, Nothing} scope === nothing && return false return haskey((scope::Scope).values, val) end -const ScopeStorage = Base.PersistentDict{ScopedValue, Any} - -struct Scope - values::ScopeStorage -end - -Scope(scope::Scope) = scope - -function Scope(parent::Union{Nothing, Scope}, key::ScopedValue{T}, value) where T - val = convert(T, value) - if parent === nothing - return Scope(ScopeStorage(key=>val)) - end - return Scope(ScopeStorage(parent.values, key=>val)) -end - -function Scope(scope, pair::Pair{<:ScopedValue}) - return Scope(scope, pair...) -end - -function Scope(scope, pair1::Pair{<:ScopedValue}, pair2::Pair{<:ScopedValue}, pairs::Pair{<:ScopedValue}...) - # Unroll this loop through recursion to make sure that - # our compiler optimization support works - return Scope(Scope(scope, pair1...), pair2, pairs...) -end -Scope(::Nothing) = nothing - function Base.show(io::IO, scope::Scope) print(io, Scope, "(") first = true @@ -115,7 +132,7 @@ function Base.show(io::IO, scope::Scope) else print(io, ", ") end - print(io, typeof(key), "@") + print(io, isa(key, ScopedValue) ? ScopedValue{eltype(key)} : typeof(key), "@") show(io, Base.objectid(key)) print(io, " => ") show(IOContext(io, :typeinfo => eltype(key)), value) @@ -123,11 +140,9 @@ function Base.show(io::IO, scope::Scope) print(io, ")") end -struct NoValue end -const novalue = NoValue() - """ get(val::ScopedValue{T})::Union{Nothing, Some{T}} + get(val::LazyScopedValue{T})::Union{Nothing, Some{T}} If the scoped value isn't set and doesn't have a default value, return `nothing`. Otherwise returns `Some{T}` with the current @@ -148,36 +163,38 @@ julia> isnothing(ScopedValues.get(b)) true ``` """ -function get(val::ScopedValue{T}) where {T} +function get(val::AbstractScopedValue{T}) where {T} scope = Core.current_scope()::Union{Scope, Nothing} if scope === nothing - val.has_default && return Some{T}(val.default) - return nothing + !hasdefault(val) && return nothing + return Some{T}(getdefault(val)) end scope = scope::Scope - if val.has_default - return Some{T}(Base.get(scope.values, val, val.default)::T) + if hasdefault(val) + return Some{T}(Base.get(Base.Fix1(getdefault, val), scope.values, val)::T) else - v = Base.get(scope.values, val, novalue) - v === novalue || return Some{T}(v::T) + v = Base.KeyValue.get(scope.values, val) + v === nothing && return nothing + return Some{T}(something(v)::T) end - return nothing end -function Base.getindex(val::ScopedValue{T})::T where T +function Base.getindex(val::AbstractScopedValue{T})::T where T maybe = get(val) maybe === nothing && throw(KeyError(val)) return something(maybe)::T end -function Base.show(io::IO, val::ScopedValue) - print(io, ScopedValue) - print(io, '{', eltype(val), '}') +function Base.show(io::IO, val::AbstractScopedValue) + if isa(val, ScopedValue) + print(io, ScopedValue) + print(io, '{', eltype(val), '}') + else + print(io, typeof(val)) + end print(io, '(') v = get(val) - if v === nothing - print(io, "undefined") - else + if v !== nothing show(IOContext(io, :typeinfo => eltype(val)), something(v)) end print(io, ')') @@ -265,9 +282,29 @@ julia> with(() -> a[] * b[], a=>3, b=>4) 12 ``` """ -function with(f, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) +@inline function with(f, pair::Pair{<:AbstractScopedValue}, rest::Pair{<:AbstractScopedValue}...) @with(pair, rest..., f()) end with(@nospecialize(f)) = f() +macro enter_scope(scope, expr) + Expr(:tryfinally, esc(expr), nothing, :(Scope($(esc(scope))::Union{Nothing, Scope}))) +end + +""" + ScopedThunk(f) + +Create a callable that records the current dynamic scope, i.e. all current +`ScopedValue`s, along with `f`. When the callable is invoked, it runs `f` +in the recorded dynamic scope. +""" +struct ScopedThunk{F} + f::F + scope::Union{Nothing, Scope} + + ScopedThunk{F}(f) where {F} = new{F}(f, Core.current_scope()) +end +ScopedThunk(f) = ScopedThunk{typeof(f)}(f) +(sf::ScopedThunk)() = @enter_scope sf.scope sf.f() + end # module ScopedValues diff --git a/base/set.jl b/base/set.jl index 8b8f3d44603c8..7763e82e01359 100644 --- a/base/set.jl +++ b/base/set.jl @@ -9,8 +9,8 @@ Elements in a `Set` are unique, as determined by the elements' definition of `isequal`. The order of elements in a `Set` is an implementation detail and cannot be relied on. -See also: [`AbstractSet`](@ref), [`BitSet`](@ref), [`Dict`](@ref), -[`push!`](@ref), [`empty!`](@ref), [`union!`](@ref), [`in`](@ref), [`isequal`](@ref) +See also [`AbstractSet`](@ref), [`BitSet`](@ref), [`Dict`](@ref), +[`push!`](@ref), [`empty!`](@ref), [`union!`](@ref), [`in`](@ref), [`isequal`](@ref). # Examples ```jldoctest; filter = r"^ '.'"ma @@ -64,7 +64,7 @@ function _Set(itr, ::EltypeUnknown) return Set{T}(itr) end -empty(s::AbstractSet{T}, ::Type{U}=T) where {T,U} = Set{U}() +empty(s::AbstractSet{T}, ::Type{U}=T) where {T,U} = emptymutable(s, U) # return an empty set with eltype T, which is mutable (can be grown) # by default, a Set is returned @@ -98,11 +98,11 @@ If `x` is in `s`, return `true`. If not, push `x` into `s` and return `false`. This is equivalent to `in(x, s) ? true : (push!(s, x); false)`, but may have a more efficient implementation. -See also: [`in`](@ref), [`push!`](@ref), [`Set`](@ref) - !!! compat "Julia 1.11" This function requires at least 1.11. +See also [`in`](@ref), [`push!`](@ref), [`Set`](@ref). + # Examples ```jldoctest; filter = r"^\\s+\\d\$"m julia> s = Set{Any}([1, 2, 3]); in!(4, s) @@ -177,7 +177,7 @@ iterate(s::Set, i...) = iterate(KeySet(s.dict), i...) @propagate_inbounds Iterators.only(s::Set) = Iterators._only(s, first) -# In case the size(s) is smaller than size(t) its more efficient to iterate through +# In case the size(s) is smaller than size(t) it's more efficient to iterate through # elements of s instead and only delete the ones also contained in t. # The threshold for this decision boils down to a tradeoff between # size(s) * cost(in() + delete!()) ≶ size(t) * cost(delete!()) @@ -200,12 +200,12 @@ end """ unique(itr) -Return an array containing only the unique elements of collection `itr`, +Return an `AbstractArray` containing only the unique elements of collection `itr`, as determined by [`isequal`](@ref) and [`hash`](@ref), in the order that the first of each set of equivalent elements originally appears. The element type of the input is preserved. -See also: [`unique!`](@ref), [`allunique`](@ref), [`allequal`](@ref). +See also [`unique!`](@ref), [`allunique`](@ref), [`allequal`](@ref). # Examples ```jldoctest @@ -264,7 +264,7 @@ unique(r::AbstractRange) = allunique(r) ? r : oftype(r, r[begin]:r[begin]) """ unique(f, itr) -Return an array containing one value from `itr` for each unique value produced by `f` +Return an `AbstractArray` containing one value from `itr` for each unique value produced by `f` applied to elements of `itr`. # Examples @@ -489,11 +489,11 @@ The precise number of calls is regarded as an implementation detail. `allunique` may use a specialized implementation when the input is sorted. -See also: [`unique`](@ref), [`issorted`](@ref), [`allequal`](@ref). - !!! compat "Julia 1.11" The method `allunique(f, itr)` requires at least Julia 1.11. +See also [`unique`](@ref), [`issorted`](@ref), [`allequal`](@ref). + # Examples ```jldoctest julia> allunique([1, 2, 3]) @@ -526,7 +526,7 @@ function _hashed_allunique(C) seen = Set{@default_eltype(C)}() x = iterate(C) if haslength(C) && length(C) > 1000 - for i in OneTo(1000) + for _ in OneTo(1000) v, s = something(x) in!(v, seen) && return false x = iterate(C, s) @@ -545,6 +545,11 @@ allunique(::Union{AbstractSet,AbstractDict}) = true allunique(r::AbstractRange) = !iszero(step(r)) || length(r) <= 1 +# ncodeunits is O(1) and bounds character count; match StridedArray's short cutoff. +function allunique(s::AbstractString) + ncodeunits(s) < 32 ? _indexed_allunique(s) : _hashed_allunique(s) +end + function allunique(A::StridedArray) if length(A) < 32 _indexed_allunique(A) @@ -611,14 +616,14 @@ Or if all of `[f(x) for x in itr]` are equal, for the second method. Note that `allequal(f, itr)` may call `f` fewer than `length(itr)` times. The precise number of calls is regarded as an implementation detail. -See also: [`unique`](@ref), [`allunique`](@ref). - !!! compat "Julia 1.8" The `allequal` function requires at least Julia 1.8. !!! compat "Julia 1.11" The method `allequal(f, itr)` requires at least Julia 1.11. +See also [`unique`](@ref), [`allunique`](@ref). + # Examples ```jldoctest julia> allequal([]) @@ -656,15 +661,37 @@ allequal(r::AbstractRange) = iszero(step(r)) || length(r) <= 1 allequal(f, xs) = allequal(Generator(f, xs)) -function allequal(f, xs::Tuple) - length(xs) <= 1 && return true - f1 = f(xs[1]) - for x in tail(xs) - isequal(f1, f(x)) || return false +function _allequal_loop(f, xs::Tuple) + val = f(xs[1]) + for i in 2:length(xs) + isequal(val, f(xs[i])) || return false end return true end +function allequal(f, xs::Tuple) + if @generated + n = fieldcount(xs) + if n <= 32 + n <= 1 && return true + checks = Expr[:(isequal(val, f(getfield(xs, $i)))) for i in 2:n] + expr = foldr((a, b) -> :($a && $b), checks) + return quote + val = f(getfield(xs, 1)) + $expr + end + else + return :(return _allequal_loop(f, xs)) + end + else + length(xs) <= 1 && return true + return _allequal_loop(f, xs) + end +end + +allequal(f, ::Tuple{}) = true +allequal(xs::Tuple) = allequal(identity, xs) + filter!(f, s::Set) = unsafe_filter!(f, s) const hashs_seed = UInt === UInt64 ? 0x852ada37cfe8e0ce : 0xcfe8e0ce @@ -881,8 +908,8 @@ askey(k, ::AbstractSet) = k function _replace!(new::Callable, res::Union{AbstractDict,AbstractSet}, A::Union{AbstractDict,AbstractSet}, count::Int) - @assert res isa AbstractDict && A isa AbstractDict || - res isa AbstractSet && A isa AbstractSet + @assert (res isa AbstractDict && A isa AbstractDict || + res isa AbstractSet && A isa AbstractSet) "type mismatch" count == 0 && return res c = 0 if res === A # cannot replace elements while iterating over A diff --git a/base/shell.jl b/base/shell.jl index 68925cbd5d5af..a6c7cb258cab9 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -2,7 +2,7 @@ ## shell-like command parsing ## -const shell_special = "#{}()[]<>|&*?~;" +const shell_special = "#{}()[]<>|&*?;" (@doc raw""" rstrip_shell(s::AbstractString) @@ -28,8 +28,12 @@ function rstrip_shell(s::AbstractString) SubString(s, 1, 0) end) -function shell_parse(str::AbstractString, interpolate::Bool=true; - special::AbstractString="", filename="none") +shell_parse(str::AbstractString, interpolate::Bool=true; + special::AbstractString="", filename="none") = + __repl_entry_shell_parse(str, interpolate, special, filename) + +# N.B.: Any functions starting with __repl_entry cut off backtraces when printing in the REPL. +function __repl_entry_shell_parse(str::AbstractString, interpolate::Bool, special::AbstractString, filename) last_arg = firstindex(str) # N.B.: This is used by REPLCompletions s = SubString(str, last_arg) s = rstrip_shell(lstrip(s)) @@ -44,6 +48,17 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; i = firstindex(s) st = Iterators.Stateful(pairs(s)) update_last_arg = false # true after spaces or interpolate + word_has_special = false # true if current word contains any quoted/escaped content + + # Pipeline/redirection state (used only when interpolate=true). + # pipeline_parts accumulates completed segments as (seg_tuple, stdin, stdout, stdout_append, stderr, stderr_append). + pipeline_parts = [] + seg_stdin = nothing # stdin redirect expression for current segment + seg_stdout = nothing # stdout redirect expression for current segment + seg_stdout_append = false # true if stdout redirect is >> (append mode) + seg_stderr = nothing # stderr redirect expression for current segment + seg_stderr_append = false # true if stderr redirect is >> (append mode) + redirect_mode = :none # :none | :stdin | :stdout | :stderr function push_nonempty!(list, x) if !isa(x,AbstractString) || !isempty(x) @@ -63,12 +78,57 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; C = eltype(str) P = Pair{Int,C} + + # Parse the expression following a `$` interpolation marker. + # Pops the first char of the expression from `st`, parses the atom, advances + # `s` past it, and resets `st`. Returns `(expr, new_s, interp_pos)` where + # `interp_pos` is the absolute position in `str` (for REPLCompletions). + function parse_dollar_interp(st, s) + isempty(st) && error("\$ right before end of command") + stpos, c = popfirst!(st)::P + isspace(c) && error("space not allowed right after \$") + if startswith(SubString(s, stpos), "var\"") + # Disallow var"#" syntax in cmd interpolations. + # TODO: Allow only identifiers after the $ for consistency with + # string interpolation syntax (see #3150) + atom, j = :var, stpos+3 + else + # use parseatom instead of parse to respect filename (#28188) + atom, j = Meta.parseatom(s, stpos, filename=filename) + end + interp_pos = stpos + s.offset + s = SubString(s, j) + Iterators.reset!(st, pairs(s)) + return atom, s, interp_pos + end + + # Convert a word (list of string/expr parts) to a redirect filename expression. + function redirect_word_expr(word) + if length(word) == 1 && isa(word[1], AbstractString) + return String(word[1]) + else + return Expr(:call, GlobalRef(Base, :cmd_interpolate), word...) + end + end + for (j, c) in st j, c = j::Int, c::C if !in_single_quotes && !in_double_quotes && isspace(c) update_last_arg = true i = consume_upto!(arg, s, i, j) - append_2to1!(args, arg) + if redirect_mode != :none && (!isempty(arg) || word_has_special) + # This word is the redirect filename. + re = redirect_word_expr(arg) + if redirect_mode === :stdin; seg_stdin = re + elseif redirect_mode === :stdout; seg_stdout = re + else; seg_stderr = re; end + empty!(arg) + redirect_mode = :none + elseif redirect_mode == :none && (!isempty(arg) || word_has_special) + append_2to1!(args, arg) + end + word_has_special = false + # else: in redirect_mode but arg empty — still waiting for the filename. while !isempty(st) # We've made sure above that we don't end in whitespace, # so updating `i` here is ok @@ -77,25 +137,67 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; popfirst!(st) end elseif interpolate && !in_single_quotes && c == '$' + consume_upto!(arg, s, i, j) + result = parse_dollar_interp(st, s) + s = result[2] + last_arg = result[3] + update_last_arg = true + push!(arg, result[1]) + i = firstindex(s) + elseif interpolate && !in_single_quotes && !in_double_quotes && c == '|' + # Pipeline operator: finalize current word/redirect, then save segment. i = consume_upto!(arg, s, i, j) - isempty(st) && error("\$ right before end of command") - stpos, c = popfirst!(st)::P - isspace(c) && error("space not allowed right after \$") - if startswith(SubString(s, stpos), "var\"") - # Disallow var"#" syntax in cmd interpolations. - # TODO: Allow only identifiers after the $ for consistency with - # string interpolation syntax (see #3150) - ex, j = :var, stpos+3 - else - # use parseatom instead of parse to respect filename (#28188) - ex, j = Meta.parseatom(s, stpos, filename=filename) + if redirect_mode != :none && (!isempty(arg) || word_has_special) + re = redirect_word_expr(arg) + if redirect_mode === :stdin; seg_stdin = re + elseif redirect_mode === :stdout; seg_stdout = re + else; seg_stderr = re; end + empty!(arg) + elseif redirect_mode == :none && (!isempty(arg) || word_has_special) + append_2to1!(args, arg) + end + seg_tuple = Expr(:tuple) + for a in args; push!(seg_tuple.args, Expr(:tuple, a...)); end + push!(pipeline_parts, (seg_tuple, seg_stdin, seg_stdout, seg_stdout_append, + seg_stderr, seg_stderr_append)) + empty!(args) + seg_stdin = nothing; seg_stdout = nothing; seg_stdout_append = false + seg_stderr = nothing; seg_stderr_append = false; redirect_mode = :none + word_has_special = false + update_last_arg = true + elseif interpolate && !in_single_quotes && !in_double_quotes && (c == '<' || c == '>') + # Redirection operator: finalize any pending word, then set redirect mode. + # A pure-integer word immediately before the operator is an fd number (e.g. 2>). + i = consume_upto!(arg, s, i, j) + fd = nothing + if !word_has_special && length(arg) == 1 && isa(arg[1], AbstractString) && + !isempty(arg[1]::AbstractString) && all(isdigit, arg[1]::AbstractString) + fd = parse(Int, arg[1]::AbstractString) + empty!(arg) + word_has_special = false + elseif !isempty(arg) + append_2to1!(args, arg) + end + if c == '>' + append = false + if !isempty(st) && (peek(st)::P).second == '>' + popfirst!(st) # consume second > + append = true + i = something(peek(st), lastindex(s)::Int+1 => '\0').first::Int + end + if fd === nothing || fd == 1 + redirect_mode = :stdout; seg_stdout_append = append + elseif fd == 2 + redirect_mode = :stderr; seg_stderr_append = append + else + error("parsing command `$str`: unsupported fd $fd in redirection") + end + else # '<' + (fd === nothing || fd == 0) || + error("parsing command `$str`: unsupported fd $fd in redirection") + redirect_mode = :stdin end - last_arg = stpos + s.offset update_last_arg = true - push!(arg, ex) - s = SubString(s, j) - Iterators.reset!(st, pairs(s)) - i = firstindex(s) else if update_last_arg last_arg = i + s.offset @@ -103,11 +205,14 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; end if !in_double_quotes && c == '\'' in_single_quotes = !in_single_quotes + word_has_special = true i = consume_upto!(arg, s, i, j) elseif !in_single_quotes && c == '"' in_double_quotes = !in_double_quotes + word_has_special = true i = consume_upto!(arg, s, i, j) elseif !in_single_quotes && c == '\\' + word_has_special = true if !isempty(st) && (peek(st)::P)[2] in ('\n', '\r') i = consume_upto!(arg, s, i, j) + 1 if popfirst!(st)[2] == '\r' && (peek(st)::P)[2] == '\n' @@ -120,7 +225,7 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; end elseif in_double_quotes isempty(st) && error("unterminated double quote") - k, c′ = peek(st)::P + _, c′ = peek(st)::P if c′ == '"' || c′ == '$' || c′ == '\\' i = consume_upto!(arg, s, i, j) _ = popfirst!(st) @@ -130,6 +235,41 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; i = consume_upto!(arg, s, i, j) _ = popfirst!(st) end + elseif interpolate && !in_single_quotes && !in_double_quotes && + c == '~' && i == j && isempty(arg) + # Tilde expansion: `~` or `~username` at start of a word. + # Collect the username, which may include $interpolations (e.g. `~$user`). + # Stop at /, whitespace, uninterpolable chars (quotes, backslash), or operators. + i = consume_upto!(arg, s, i, j) # i now points past the ~ + user_parts = [] + user_lit_start = i + while !isempty(st) + nxt = peek(st)::P + nc = nxt[2] + (nc == '/' || isspace(nc) || nc == '\'' || nc == '"' || nc == '\\' || + nc == '|' || nc == '<' || nc == '>') && break + if nc == '$' + push_nonempty!(user_parts, s[user_lit_start:prevind(s, nxt[1])]) + popfirst!(st) # consume $ + result = parse_dollar_interp(st, s) + push!(user_parts, result[1]) + s = result[2] + i = firstindex(s) + user_lit_start = i + else + popfirst!(st) + end + end + user_end = something(peek(st), (lastindex(s) + 1) => '\0').first + push_nonempty!(user_parts, s[user_lit_start:prevind(s, user_end)]) + if isempty(user_parts) + push!(arg, :(expanduser("~"))) + elseif length(user_parts) == 1 && isa(user_parts[1], AbstractString) + push!(arg, :(expanduser($("~" * user_parts[1])))) + else + push!(arg, :(let _u = string($(user_parts...)); isempty(_u) ? "~" : expanduser(string('~', _u)) end)) + end + i = user_end elseif !in_single_quotes && !in_double_quotes && c in special error("parsing command `$str`: special characters \"$special\" must be quoted in commands") end @@ -140,16 +280,72 @@ function shell_parse(str::AbstractString, interpolate::Bool=true; if in_double_quotes; error("unterminated double quote"); end push_nonempty!(arg, s[i:end]) - append_2to1!(args, arg) + if interpolate && redirect_mode != :none + isempty(arg) && error("parsing command `$str`: redirect operator without filename") + re = redirect_word_expr(arg) + if redirect_mode === :stdin; seg_stdin = re + elseif redirect_mode === :stdout; seg_stdout = re + else; seg_stderr = re; end + empty!(arg) + elseif !isempty(arg) || word_has_special + append_2to1!(args, arg) + end interpolate || return args, last_arg - # construct an expression - ex = Expr(:tuple) - for arg in args - push!(ex.args, Expr(:tuple, arg...)) + # If no pipeline operators and no redirects: return existing tuple format (backward compat). + if isempty(pipeline_parts) && seg_stdin === nothing && seg_stdout === nothing && + seg_stderr === nothing + ex = Expr(:tuple) + for arg in args + push!(ex.args, Expr(:tuple, arg...)) + end + return ex, last_arg end - return ex, last_arg + + # Build a cmd_gen(seg_tuple) expression, wrapped in pipeline() calls for each redirect. + # stdin and stdout (with its append flag) are combined into one call; stderr is separate + # so that different append modes can be handled independently. + function build_seg_expr(seg_tuple, s_in, s_out, s_out_app, s_err, s_err_app) + cmd_ex = Expr(:call, GlobalRef(Base, :cmd_gen), seg_tuple) + if s_in !== nothing || s_out !== nothing + kwargs = Any[] + s_in !== nothing && push!(kwargs, Expr(:kw, :stdin, s_in)) + s_out !== nothing && push!(kwargs, Expr(:kw, :stdout, s_out)) + s_out_app && push!(kwargs, Expr(:kw, :append, true)) + cmd_ex = Expr(:call, GlobalRef(Base, :pipeline), Expr(:parameters, kwargs...), cmd_ex) + end + if s_err !== nothing + kwargs = Any[Expr(:kw, :stderr, s_err)] + s_err_app && push!(kwargs, Expr(:kw, :append, true)) + cmd_ex = Expr(:call, GlobalRef(Base, :pipeline), Expr(:parameters, kwargs...), cmd_ex) + end + return cmd_ex + end + + # Build the final segment's tuple. + final_seg_tuple = Expr(:tuple) + for a in args; push!(final_seg_tuple.args, Expr(:tuple, a...)); end + + if isempty(pipeline_parts) + # Only redirects, no pipes. + return build_seg_expr(final_seg_tuple, seg_stdin, seg_stdout, seg_stdout_append, + seg_stderr, seg_stderr_append), last_arg + end + + # Chain pipeline segments left-to-right: pipeline(seg1, pipeline(seg2, seg3, ...)). + (s0_args, s0_in, s0_out, s0_out_app, s0_err, s0_err_app) = pipeline_parts[1] + result = build_seg_expr(s0_args, s0_in, s0_out, s0_out_app, s0_err, s0_err_app) + for k in 2:length(pipeline_parts) + (sk_args, sk_in, sk_out, sk_out_app, sk_err, sk_err_app) = pipeline_parts[k] + result = Expr(:call, GlobalRef(Base, :pipeline), result, + build_seg_expr(sk_args, sk_in, sk_out, sk_out_app, sk_err, sk_err_app)) + end + result = Expr(:call, GlobalRef(Base, :pipeline), result, + build_seg_expr(final_seg_tuple, seg_stdin, seg_stdout, seg_stdout_append, + seg_stderr, seg_stderr_append)) + + return result, last_arg end """ @@ -171,7 +367,7 @@ function shell_split(s::AbstractString) parsed = shell_parse(s, false)[1] args = String[] for arg in parsed - push!(args, string(arg...)) + push!(args, string(arg...)::String) end args end @@ -196,7 +392,7 @@ function print_shell_word(io::IO, word::AbstractString, special::AbstractString else print(io, '"') for c in word - if c == '"' || c == '$' + if c == '"' || c == '$' || c == '\\' print(io, '\\') end print(io, c) @@ -244,34 +440,40 @@ function print_shell_escaped_posixly(io::IO, args::AbstractString...) first || print(io, ' ') # avoid printing quotes around simple enough strings # that any (reasonable) shell will definitely never consider them to be special - have_single::Bool = false - have_double::Bool = false - function isword(c::AbstractChar) - if '0' <= c <= '9' || 'a' <= c <= 'z' || 'A' <= c <= 'Z' - # word characters - elseif c == '_' || c == '/' || c == '+' || c == '-' || c == '.' - # other common characters - elseif c == '\'' - have_single = true - elseif c == '"' - have_double && return false # switch to single quoting - have_double = true - elseif !first && c == '=' - # equals is special if it is first (e.g. `env=val ./cmd`) - else - # anything else - return false - end - return true - end if isempty(arg) print(io, "''") - elseif all(isword, arg) - have_single && (arg = replace(arg, '\'' => "\\'")) - have_double && (arg = replace(arg, '"' => "\\\"")) - print(io, arg) else - print(io, '\'', replace(arg, '\'' => "'\\''"), '\'') + have_single = false + have_double = false + isword = true + for c in arg + if '0' <= c <= '9' || 'a' <= c <= 'z' || 'A' <= c <= 'Z' + # word characters + elseif c == '_' || c == '/' || c == '+' || c == '-' || c == '.' + # other common characters + elseif c == '\'' + have_single = true + elseif c == '"' + if have_double + isword = false + break # switch to single quoting + end + have_double = true + elseif !first && c == '=' + # equals is special if it is first (e.g. `env=val ./cmd`) + else + # anything else + isword = false + break + end + end + if isword + have_single && (arg = replace(arg, '\'' => "\\'")) + have_double && (arg = replace(arg, '"' => "\\\"")) + print(io, arg) + else + print(io, '\'', replace(arg, '\'' => "'\\''"), '\'') + end end first = false end diff --git a/base/show.jl b/base/show.jl index 870b2f47fe4a6..c82877a63ab6c 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using .Compiler: has_typevar +using .Meta: _isoperator, is_id_start_char, is_unary_and_binary_operator, + is_valid_identifier function show(io::IO, ::MIME"text/plain", u::UndefInitializer) show(io, u) @@ -173,6 +174,7 @@ function show(io::IO, ::MIME"text/plain", t::AbstractDict{K,V}) where {K,V} isempty(t) && return print(io, ":") show_circular(io, t) && return + keywidth = 0 if limit sz = displaysize(io) rows, cols = sz[1] - 3, sz[2] @@ -185,7 +187,6 @@ function show(io::IO, ::MIME"text/plain", t::AbstractDict{K,V}) where {K,V} hascolor = get(recur_io, :color, false) ks = Vector{String}(undef, min(rows, length(t))) vs = Vector{String}(undef, min(rows, length(t))) - keywidth = 0 valwidth = 0 for (i, (k, v)) in enumerate(t) i > rows && break @@ -292,6 +293,25 @@ function show(io::IO, ::MIME"text/plain", t::Task) end end +# Compact summary: the default field-recursive show would descend the +# intrusive child list (unbounded, possibly very deep) via `child_head`. +function show(io::IO, src::Core.CancellationTokenSource) + print(io, "CancellationTokenSource(") + sev = cancel_severity(src) + if sev === nothing + print(io, "active") + else + r = sev.request + print(io, r == 0x1 ? "cancelled" : + r == 0x3 ? "cancelled, abandon external" : + r == 0x4 ? "cancelled, abandon all" : + "cancelled, severity $(repr(r))") + end + np = Int(src.nparents) + np > 0 && print(io, ", ", np, np == 1 ? " parent" : " parents") + print(io, ")") +end + print(io::IO, s::Symbol) = (write(io,s); nothing) @@ -382,6 +402,11 @@ The following properties are in common use: - `:color`: Boolean specifying whether ANSI color/escape codes are supported/expected. By default, this is determined by whether `io` is a compatible terminal and by any `--color` command-line flag when `julia` was launched. + - `:hexunsigned`: Boolean specifying whether to print unsigned integers in + hexadecimal. Defaults to `true`, otherwise they will be printed in decimal. + +!!! compat "Julia 1.14" + The `:hexunsigned` option requires Julia 1.14 or later. # Examples @@ -518,10 +543,18 @@ function _show_default(io::IO, @nospecialize(x)) else print(io, "0x") r = Ref{Any}(x) + nbits = Core.bitsizeof(t) + nbytes = cld(nbits, 8) GC.@preserve r begin p = unsafe_convert(Ptr{Cvoid}, r) - for i in (nb - 1):-1:0 - print(io, string(unsafe_load(convert(Ptr{UInt8}, p + i)), base = 16, pad = 2)) + for i in (nbytes - 1):-1:0 + byte = unsafe_load(convert(Ptr{UInt8}, p + i)) + if i == nbytes - 1 && nbits % 8 != 0 + byte &= (UInt8(1) << (nbits % 8)) - UInt8(1) + print(io, string(byte, base = 16, pad = cld(nbits % 8, 4))) + else + print(io, string(byte, base = 16, pad = 2)) + end end end end @@ -573,7 +606,6 @@ end print(io::IO, f::Core.IntrinsicFunction) = print(io, nameof(f)) -show(io::IO, ::Core.TypeofBottom) = print(io, "Union{}") show(io::IO, ::MIME"text/plain", ::Core.TypeofBottom) = print(io, "Union{}") function print_without_params(@nospecialize(x)) @@ -592,10 +624,16 @@ end io_has_tvar_name(io::IO, name::Symbol, @nospecialize(x)) = false modulesof!(s::Set{Module}, x::TypeVar) = modulesof!(s, x.ub) +modulesof!(s::Set{Module}, x::TypeEq) = modulesof!(s, type_parameter(x)) +modulesof!(s::Set{Module}, x::Core.TypeEgal) = modulesof!(s, type_parameter(x)) function modulesof!(s::Set{Module}, x::Type) x = unwrap_unionall(x) if x isa DataType push!(s, parentmodule(x)) + elseif x isa TypeEq + modulesof!(s, x) + elseif x isa Core.TypeEgal + modulesof!(s, x) elseif x isa Union modulesof!(s, x.a) modulesof!(s, x.b) @@ -603,11 +641,45 @@ function modulesof!(s::Set{Module}, x::Type) s end -# given an IO context for printing a type, reconstruct the proper type that -# we're attempting to represent. -# Union{T} where T is a degenerate case and is equal to T.ub, but we don't want -# to print them that way, so filter those out from our aliases completely. -function makeproper(io::IO, @nospecialize(x::Type)) +function has_other_free_typevars(@nospecialize(x), free_before) + has_free_typevars(x) || return false + for v in find_free_typevars(x) + seen = false + for p in free_before + if p === v + seen = true + break + end + end + seen || return true + end + return false +end + +# Return a copy of the type alias `alias` with every bounded binder replaced by +# an unbounded one, so that `typeintersect_env` can match an open `x` (whose free +# typevars are not yet known to satisfy the alias' bounds) against the alias. +# The binders are rewritten from the innermost outward, so that a bound that +# references an outer binder is rewritten consistently with that binder. +function unbounded_typealias(@nospecialize(alias)) + alias isa UnionAll || return alias + body = unbounded_typealias(alias.body) + var = alias.var + if var.lb === Union{} && var.ub === Any + body === alias.body && return alias + return UnionAll(var, body) + end + newvar = TypeVar(var.name) + return UnionAll(newvar, UnionAll(var, body){newvar}) +end + +# Reconstruct the closed type that the (possibly open) `x` is a piece of, by +# re-wrapping it in the typevars bound by the surrounding printing context (the +# `:unionall_env` entries of `io`). Subtype tests use this so that the +# context-bound typevars are quantified rather than treated as rigid free +# variables, while the rest of the alias machinery keeps operating on the open +# `x` whose free typevars must be matched against the alias' parameters. +function reapply_unionall_env(io::Union{IO,Nothing}, @nospecialize(x)) if io isa IOContext for (key, val) in io.dict if key === :unionall_env && val isa TypeVar @@ -615,30 +687,28 @@ function makeproper(io::IO, @nospecialize(x::Type)) end end end - has_free_typevars(x) && return Any return x end -function make_typealias(@nospecialize(x::Type)) +function make_typealias(@nospecialize(x::Type), io::Union{IO,Nothing}=nothing) Any === x && return nothing x <: Tuple && return nothing mods = modulesof!(Set{Module}(), x) replace!(mods, Core=>Base) + properx = reapply_unionall_env(io, x) aliases = Tuple{GlobalRef,SimpleVector}[] - xenv = UnionAll[] - for p in uniontypes(unwrap_unionall(x)) - p isa UnionAll && push!(xenv, p) - end - x isa UnionAll && push!(xenv, x) for mod in mods for name in unsorted_names(mod) if isdefinedglobal(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) alias = getglobal(mod, name) - if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && x <: alias + if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && properx <: alias if alias isa UnionAll - (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), x, alias)::SimpleVector + free_before = find_free_typevars(x) + (_ti, env) = typeintersect_env(x, unbounded_typealias(alias)) # ti === Union{} && continue # impossible, since we already checked that x <: alias env = env::SimpleVector + # unwrap `svec(tvar, constrained)` env markers down to the TypeVar + env = Core.svec(Any[e isa SimpleVector ? e[1] : e for e in env]...) # TODO: In some cases (such as the following), the `env` is over-approximated. # We'd like to disable `fix_inferred_var_bound` since we'll already do that fix-up here. # (or detect and reverse the computation of it here). @@ -656,11 +726,9 @@ function make_typealias(@nospecialize(x::Type)) ex isa TypeError || rethrow() continue end - for p in xenv - applied = rewrap_unionall(applied, p) - end - has_free_typevars(applied) && continue - applied === x || continue # it couldn't figure out the parameter matching + applied = rewrap_free_typevars(applied, free_before) + has_other_free_typevars(applied, free_before) && continue + applied == x || continue # it couldn't figure out the parameter matching elseif alias === x env = Core.svec() else @@ -698,8 +766,8 @@ function show_typeparams(io::IO, env::SimpleVector, orig::SimpleVector, wheres:: elide = length(wheres) function egal_var(p::TypeVar, @nospecialize o) return o isa TypeVar && - ccall(:jl_types_egal, Cint, (Any, Any), p.ub, o.ub) != 0 && - ccall(:jl_types_egal, Cint, (Any, Any), p.lb, o.lb) != 0 + ccall(:jl_types_struct_equiv, Cint, (Any, Any), p.ub, o.ub) != 0 && + ccall(:jl_types_struct_equiv, Cint, (Any, Any), p.lb, o.lb) != 0 end for i = n:-1:1 p = env[i] @@ -739,7 +807,7 @@ function show_typeparams(io::IO, env::SimpleVector, orig::SimpleVector, wheres:: nothing end -function show_typealias(io::IO, name::GlobalRef, x::Type, env::SimpleVector, wheres::Vector) +function show_typealias_name(io::IO, name::GlobalRef) if !(get(io, :compact, false)::Bool) # Print module prefix unless alias is visible from module passed to # IOContext. If :module is not set, default to Main. @@ -751,6 +819,11 @@ function show_typealias(io::IO, name::GlobalRef, x::Type, env::SimpleVector, whe end end print(io, name.name) + return nothing +end + +function show_typealias(io::IO, name::GlobalRef, env::SimpleVector, wheres::Vector) + show_typealias_name(io, name) isempty(env) && return io = IOContext(io) for p in wheres @@ -811,11 +884,10 @@ function show_wheres(io::IO, wheres::Vector{TypeVar}) end function show_typealias(io::IO, @nospecialize(x::Type)) - properx = makeproper(io, x) - alias = make_typealias(properx) + alias = make_typealias(x, io) alias === nothing && return false wheres = make_wheres(io, alias[2], x) - show_typealias(io, alias[1], x, alias[2], wheres) + show_typealias(io, alias[1], alias[2], wheres) show_wheres(io, wheres) return true end @@ -826,7 +898,7 @@ function make_typealiases(@nospecialize(x::Type)) x <: Tuple && return aliases, Union{} mods = modulesof!(Set{Module}(), x) replace!(mods, Core=>Base) - vars = Dict{Symbol,TypeVar}() + free_before = find_free_typevars(x) xenv = UnionAll[] each = Any[] for p in uniontypes(unwrap_unionall(x)) @@ -839,12 +911,14 @@ function make_typealiases(@nospecialize(x::Type)) if isdefinedglobal(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) alias = getglobal(mod, name) if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && !(alias <: Tuple) - (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), x, alias)::SimpleVector + (ti, env) = typeintersect_env(x, unbounded_typealias(alias)) ti === Union{} && continue # make sure this alias wasn't from an unrelated part of the Union mod2 = modulesof!(Set{Module}(), alias) mod in mod2 || (mod === Base && Core in mod2) || continue env = env::SimpleVector + # unwrap `svec(tvar, constrained)` env markers down to the TypeVar + env = Core.svec(Any[e isa SimpleVector ? e[1] : e for e in env]...) applied = alias if !isempty(env) applied = try @@ -863,10 +937,12 @@ function make_typealiases(@nospecialize(x::Type)) for p in xenv applied = rewrap_unionall(applied, p) end - has_free_typevars(applied) && continue + applied = rewrap_free_typevars(applied, free_before) + has_other_free_typevars(applied, free_before) && continue applied <: x || continue # parameter matching didn't make a subtype print_without_params(x) && (env = Core.svec()) for typ in each # check that the alias also fully subsumes at least component of the input + typ isa TypeVar && continue if typ <: applied push!(aliases, Core.svec(GlobalRef(mod, name), env, applied, (ul, -length(env)))) break @@ -902,8 +978,7 @@ function make_typealiases(@nospecialize(x::Type)) end function show_unionaliases(io::IO, x::Union) - properx = makeproper(io, x) - aliases, applied = make_typealiases(properx) + aliases, applied = make_typealiases(x) isempty(aliases) && return false first = true tvar = false @@ -911,7 +986,7 @@ function show_unionaliases(io::IO, x::Union) if isa(typ, TypeVar) tvar = true # sort bare TypeVars to the end continue - elseif rewrap_unionall(typ, properx) <: applied + elseif typ <: applied continue end print(io, first ? "Union{" : ", ") @@ -922,7 +997,7 @@ function show_unionaliases(io::IO, x::Union) alias = aliases[1] env = alias[2]::SimpleVector wheres = make_wheres(io, env, x) - show_typealias(io, alias[1], x, env, wheres) + show_typealias(io, alias[1], env, wheres) show_wheres(io, wheres) else for alias in aliases @@ -930,7 +1005,7 @@ function show_unionaliases(io::IO, x::Union) first = false env = alias[2]::SimpleVector wheres = make_wheres(io, env, x) - show_typealias(io, alias[1], x, env, wheres) + show_typealias(io, alias[1], env, wheres) show_wheres(io, wheres) end if tvar @@ -948,8 +1023,7 @@ end function show(io::IO, ::MIME"text/plain", @nospecialize(x::Type)) if !print_without_params(x) - properx = makeproper(io, x) - if make_typealias(properx) !== nothing || (unwrap_unionall(x) isa Union && x <: make_typealiases(properx)[2]) + if make_typealias(x, io) !== nothing || (unwrap_unionall(x) isa Union && x <: make_typealiases(x)[2]) show(IOContext(io, :compact => true), x) if !(get(io, :compact, false)::Bool) printstyled(io, " (alias for "; color = :light_black) @@ -972,13 +1046,43 @@ function show(io::IO, ::MIME"text/plain", @nospecialize(x::Type)) end end -show(io::IO, @nospecialize(x::Type)) = _show_type(io, inferencebarrier(x)) +function show_typeegal(io::IO, @nospecialize(x::Core.TypeEgal)) + print(io, "Core.TypeEgal{") + show(io, type_parameter(x)) + print(io, "}") +end +function show(io::IO, @nospecialize(x::Core.AnyType)) + if x isa Core.TypeofBottom + print(io, "Union{}") + elseif x isa Core.TypeEgal + show_typeegal(io, x) + elseif x isa TypeEq + show_typeeq(io, x) + else + _show_type(io, inferencebarrier(x)) + end +end +# `Type{T}` is the familiar user-facing spelling and is used for all normal +# (compact) printing. In non-compact contexts (e.g. the REPL's `text/plain` +# display) the canonical kind name `TypeEq{T}` is shown instead, so that a +# concrete `Type{T}` renders as `Type{T} (alias for TypeEq{T})`. +function show_typeeq(io::IO, @nospecialize(x::TypeEq)) + print(io, get(io, :compact, true)::Bool ? "Type{" : "TypeEq{") + show(io, type_parameter(x)) + print(io, "}") +end function _show_type(io::IO, @nospecialize(x::Type)) - if print_without_params(x) + if x isa Core.TypeEgal + show_typeegal(io, x) + return + elseif print_without_params(x) show_type_name(io, (unwrap_unionall(x)::DataType).name) return elseif get(io, :compact, true)::Bool && show_typealias(io, x) return + elseif x isa TypeEq + show_typeeq(io, x) + return elseif x isa DataType show_datatype(io, x) return @@ -989,6 +1093,9 @@ function _show_type(io::IO, @nospecialize(x::Type)) print(io, "Union") show_delim_array(io, uniontypes(x), '{', ',', '}', false) return + elseif x === Union{} + print(io, "Union{}") + return end x = x::UnionAll @@ -1025,7 +1132,9 @@ end # Check whether 'sym' (defined in module 'parent') is visible from module 'from' # If an object with this name exists in 'from', we need to check that it's the same binding # and that it's not deprecated. -function isvisible(sym::Symbol, parent::Module, from::Module) +# `@constprop :none` so concrete-eval doesn't bake binding edges (often on `Main.sym`) +# into the show machinery, where any rebinding of the name would invalidate it (#61667) +@constprop :none function isvisible(sym::Symbol, parent::Module, from::Module) isdeprecated(parent, sym) && return false isdefinedglobal(from, sym) || return false isdefinedglobal(parent, sym) || return false @@ -1070,8 +1179,10 @@ function check_world_bounded(tn::Core.TypeName) return Int(partition.min_world):Int(max_world) end end - isdefined(partition, :next) || return nothing - partition = @atomic partition.next + next = @atomic partition.next + # The last partition's `next` is a backreference to the owning Binding. + next isa Core.BindingPartition || return nothing + partition = next end end @@ -1113,6 +1224,8 @@ function show_type_name(io::IO, tn::Core.TypeName) end function maybe_kws_nt(x::DataType) + # manually-written version of + # x <: (Pairs{Symbol, eltype(NT), Nothing, NT} where NT <: NamedTuple) x.name === typename(Pairs) || return nothing length(x.parameters) == 4 || return nothing x.parameters[1] === Symbol || return nothing @@ -1122,7 +1235,7 @@ function maybe_kws_nt(x::DataType) types isa DataType || return nothing x.parameters[2] === eltype(p4) || return nothing isa(syms, Tuple) || return nothing - x.parameters[3] === typeof(syms) || return nothing + x.parameters[3] === Nothing || return nothing return p4 end return nothing @@ -1205,7 +1318,7 @@ function show_datatype(io::IO, x::DataType, wheres::Vector{TypeVar}=TypeVar[]) return elseif isnamedtuple syms, types = parameters - if syms isa Tuple && types isa DataType + if syms isa Tuple && types isa DataType && length(types.parameters) == length(syms) && !isvatuple(types) print(io, "@NamedTuple{") show_at_namedtuple(io, syms, types) print(io, "}") @@ -1255,7 +1368,7 @@ show_supertypes(typ::DataType) = show_supertypes(stdout, typ) Prints one or more expressions, and their results, to `stdout`, and returns the last result. -See also: [`show`](@ref), [`@info`](@ref man-logging), [`println`](@ref). +See also [`show`](@ref), [`@info`](@ref man-logging), [`println`](@ref). # Examples ```jldoctest @@ -1288,7 +1401,17 @@ nonnothing_nonmissing_typeinfo(io::IO) = nonmissingtype(nonnothingtype(get(io, : show(io::IO, b::Bool) = print(io, nonnothing_nonmissing_typeinfo(io) === Bool ? (b ? "1" : "0") : (b ? "true" : "false")) show(io::IO, ::Nothing) = print(io, "nothing") show(io::IO, n::Signed) = (write(io, string(n)); nothing) -show(io::IO, n::Unsigned) = print(io, "0x", string(n, pad = sizeof(n)<<1, base = 16)) +function show(io::IO, n::Unsigned) + if get(io, :hexunsigned, true)::Bool + print(io, "0x", string(n, pad = cld(Core.bitsizeof(n), 4), base = 16)) + else + if get(io, :typeinfo, Nothing)::Type == typeof(n) + print(io, n) + else + print(io, typeof(n), "($(n))") + end + end +end print(io::IO, n::Unsigned) = print(io, string(n)) has_tight_type(p::Pair) = @@ -1380,6 +1503,9 @@ function show(io::IO, codeinst::Core.CodeInstance) else show_mi(io, def::MethodInstance) end + if codeinst.owner !== nothing + print(io, " (foreign)") + end end function show_mi(io::IO, mi::Core.MethodInstance, from_stackframe::Bool=false) @@ -1477,7 +1603,7 @@ end show(io::IO, t::Tuple) = show_delim_array(io, t, '(', ',', ')', true) show(io::IO, v::SimpleVector) = show_delim_array(io, v, "svec(", ',', ')', false) -show(io::IO, s::Symbol) = show_unquoted_quote_expr(io, s, 0, 0, 0) +show(io::IO, s::Symbol) = show_unquoted_quote_expr(io, s, 0, 0) ## Abstract Syntax Tree (AST) printing ## @@ -1516,7 +1642,7 @@ const ExprNode = Union{Expr, QuoteNode, SlotNumber, LineNumberNode, SSAValue, # head is :$ and which is not inside a quote to fallback to the "unhandled" case: # this is behavior is triggered by IOContext(io, :unquote_fallback => true) print( io::IO, ex::ExprNode) = (show_unquoted(IOContext(io, :unquote_fallback => false), ex, 0, -1); nothing) -show( io::IO, ex::ExprNode) = show_unquoted_quote_expr(IOContext(io, :unquote_fallback => true), ex, 0, -1, 0) +show( io::IO, ex::ExprNode) = show_unquoted_quote_expr(IOContext(io, :unquote_fallback => true), ex, 0, -1) show_unquoted(io::IO, ex) = show_unquoted(io, ex, 0, 0) show_unquoted(io::IO, ex, indent::Int) = show_unquoted(io, ex, indent, 0) show_unquoted(io::IO, ex, ::Int,::Int) = show(io, ex) @@ -1542,111 +1668,16 @@ const expr_parens = Dict(:tuple=>('(',')'), :vcat=>('[',']'), :ncat =>('[',']'), :nrow =>('[',']'), :braces=>('{','}'), :bracescat=>('{','}')) -## AST decoding helpers ## - -is_id_start_char(c::AbstractChar) = ccall(:jl_id_start_char, Cint, (UInt32,), c) != 0 -is_id_char(c::AbstractChar) = ccall(:jl_id_char, Cint, (UInt32,), c) != 0 - -""" - isidentifier(s) -> Bool - -Return whether the symbol or string `s` contains characters that are parsed as -a valid ordinary identifier (not a binary/unary operator) in Julia code; -see also [`Base.isoperator`](@ref). - -Internally Julia allows any sequence of characters in a `Symbol` (except `\\0`s), -and macros automatically use variable names containing `#` in order to avoid -naming collision with the surrounding code. In order for the parser to -recognize a variable, it uses a limited set of characters (greatly extended by -Unicode). `isidentifier()` makes it possible to query the parser directly -whether a symbol contains valid characters. - -# Examples -```jldoctest -julia> Meta.isidentifier(:x), Meta.isidentifier("1x") -(true, false) -``` -""" -function isidentifier(s::AbstractString) - x = Iterators.peel(s) - isnothing(x) && return false - (s == "true" || s == "false") && return false - c, rest = x - is_id_start_char(c) || return false - return all(is_id_char, rest) -end -isidentifier(s::Symbol) = isidentifier(string(s)) - -is_op_suffix_char(c::AbstractChar) = ccall(:jl_op_suffix_char, Cint, (UInt32,), c) != 0 - -_isoperator(s) = ccall(:jl_is_operator, Cint, (Cstring,), s) != 0 - -""" - isoperator(s::Symbol) - -Return `true` if the symbol can be used as an operator, `false` otherwise. - -# Examples -```jldoctest -julia> Meta.isoperator(:+), Meta.isoperator(:f) -(true, false) -``` -""" -isoperator(s::Union{Symbol,AbstractString}) = _isoperator(s) || ispostfixoperator(s) - -""" - isunaryoperator(s::Symbol) - -Return `true` if the symbol can be used as a unary (prefix) operator, `false` otherwise. - -# Examples -```jldoctest -julia> Meta.isunaryoperator(:-), Meta.isunaryoperator(:√), Meta.isunaryoperator(:f) -(true, true, false) -``` -""" -isunaryoperator(s::Symbol) = ccall(:jl_is_unary_operator, Cint, (Cstring,), s) != 0 -is_unary_and_binary_operator(s::Symbol) = ccall(:jl_is_unary_and_binary_operator, Cint, (Cstring,), s) != 0 -is_syntactic_operator(s::Symbol) = ccall(:jl_is_syntactic_operator, Cint, (Cstring,), s) != 0 - -""" - isbinaryoperator(s::Symbol) - -Return `true` if the symbol can be used as a binary (infix) operator, `false` otherwise. - -# Examples -```jldoctest -julia> Meta.isbinaryoperator(:-), Meta.isbinaryoperator(:√), Meta.isbinaryoperator(:f) -(true, false, false) -``` -""" -function isbinaryoperator(s::Symbol) - return _isoperator(s) && (!isunaryoperator(s) || is_unary_and_binary_operator(s)) && - s !== Symbol("'") -end - -""" - ispostfixoperator(s::Union{Symbol,AbstractString}) - -Return `true` if the symbol can be used as a postfix operator, `false` otherwise. - -# Examples -```jldoctest -julia> Meta.ispostfixoperator(Symbol("'")), Meta.ispostfixoperator(Symbol("'ᵀ")), Meta.ispostfixoperator(:-) -(true, true, false) -``` -""" -function ispostfixoperator(s::Union{Symbol,AbstractString}) - s = String(s)::String - return startswith(s, '\'') && all(is_op_suffix_char, SubString(s, 2)) -end """ operator_precedence(s::Symbol) -Return an integer representing the precedence of operator `s`, relative to +Return an integer representing the precedence of a binary operator `s`, relative to other operators. Higher-numbered operators take precedence over lower-numbered -operators. Return `0` if `s` is not a valid operator. +operators. Return `0` if `s` is not a valid binary operator. + +(The precedence of *unary* operators is handled differently, including cases like `+` +where an operator can be either unary or binary.) # Examples ```jldoctest @@ -1763,7 +1794,7 @@ function show_list(io::IO, items, sep, indent::Int, prec::Int=0, quote_level::In show_unquoted(io, Expr(:(=), item.args[1], item.args[2]), indent, parens ? 0 : prec, quote_level) elseif kw && is_expr(item, :(=), 2) item = item::Expr - show_unquoted_expr_fallback(io, item, indent, quote_level) + show_unquoted_expr_fallback(io, item) else show_unquoted(io, item, indent, parens ? 0 : prec, quote_level) end @@ -1778,19 +1809,6 @@ function show_enclosed_list(io::IO, op, items, sep, cl, indent, prec=0, quote_le print(io, cl) end -const keyword_syms = Set([ - :baremodule, :begin, :break, :catch, :const, :continue, :do, :else, :elseif, - :end, :export, :var"false", :finally, :for, :function, :global, :if, :import, - :let, :local, :macro, :module, :public, :quote, :return, :struct, :var"true", - :try, :using, :while ]) - -function is_valid_identifier(sym) - return (isidentifier(sym) && !(sym in keyword_syms)) || - (_isoperator(sym) && - !(sym in (Symbol("'"), :(::), :?)) && - !is_syntactic_operator(sym) - ) -end # show a normal (non-operator) function call, e.g. f(x, y) or A[z] # kw: `=` expressions are parsed with head `kw` in this context @@ -1799,7 +1817,8 @@ function show_call(io::IO, head, func, func_args, indent, quote_level, kw::Bool) if (isa(func, Symbol) && func !== :(:) && !(head === :. && isoperator(func))) || (isa(func, Symbol) && !is_valid_identifier(func)) || (isa(func, Expr) && (func.head === :. || func.head === :curly || func.head === :macroname)) || - isa(func, GlobalRef) + isa(func, GlobalRef) || + isa(func, Core.BindingPartition) show_unquoted(io, func, indent, 0, quote_level) else print(io, '(') @@ -1841,13 +1860,22 @@ function show_unquoted(io::IO, val::SSAValue, ::Int, ::Int) # invalid SSAValue, print this in red for better recognition printstyled(io, "%", val.id; color=:red) else - print(io, "%", val.id) + cls = Base.Compiler.IRShow.ssa_warn_type_class(io, val.id) + if cls === Base.Compiler.IRShow.SSA_WARN_TYPE_STRONG + printstyled(io, "%", val.id; color=:light_red, bold=true) + elseif cls === Base.Compiler.IRShow.SSA_WARN_TYPE_MILD + printstyled(io, "%", val.id; color=warn_color(), bold=true) + else + print(io, "%", val.id) + end end end show_unquoted(io::IO, sym::Symbol, ::Int, ::Int) = show_sym(io, sym, allow_macroname=false) show_unquoted(io::IO, ex::LineNumberNode, ::Int, ::Int) = show_linenumber(io, ex.line, ex.file) show_unquoted(io::IO, ex::GotoNode, ::Int, ::Int) = print(io, "goto %", ex.label) show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int) = show_globalref(io, ex) +show_unquoted(io::IO, bpart::Core.BindingPartition, ::Int, ::Int) = + show_globalref(io, partition_owner(bpart).globalref) function show_globalref(io::IO, ex::GlobalRef; allow_macroname=false) print(io, ex.mod) @@ -1871,9 +1899,9 @@ function show_unquoted(io::IO, ex::SlotNumber, ::Int, ::Int) end end -function show_unquoted(io::IO, ex::QuoteNode, indent::Int, prec::Int) +function show_unquoted(io::IO, ex::QuoteNode, indent::Int, _prec::Int) if isa(ex.value, Symbol) - show_unquoted_quote_expr(io, ex.value, indent, prec, 0) + show_unquoted_quote_expr(io, ex.value, indent, 0) else print(io, "\$(QuoteNode(") # QuoteNode does not allows for interpolation, so if ex.value is an @@ -1884,7 +1912,7 @@ function show_unquoted(io::IO, ex::QuoteNode, indent::Int, prec::Int) end end -function show_unquoted_quote_expr(io::IO, @nospecialize(value), indent::Int, prec::Int, quote_level::Int) +function show_unquoted_quote_expr(io::IO, @nospecialize(value), indent::Int, quote_level::Int) if isa(value, Symbol) sym = value::Symbol if value in quoted_syms @@ -1990,7 +2018,7 @@ is_core_macro(@nospecialize(arg), macro_name::Symbol) = false # as an ordinary symbol, which is true in indexing expressions. const beginsym = gensym(:beginsym) -function show_unquoted_expr_fallback(io::IO, ex::Expr, indent::Int, quote_level::Int) +function show_unquoted_expr_fallback(io::IO, ex::Expr) print(io, "\$(Expr(") show(io, ex.head) for arg in ex.args @@ -2038,6 +2066,9 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int, quote_level::In elseif (head in expr_infix_any && nargs==2) func_prec = operator_precedence(head) head_ = head in expr_infix_wide ? " $head " : head + if head == :-> && is_expr(args[1], :...) + args = Any[Expr(:tuple, args[1]), args[2]] + end if func_prec <= prec show_enclosed_list(io, '(', args, head_, ')', indent, func_prec, quote_level, true) else @@ -2269,7 +2300,12 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int, quote_level::In print(io, "end") end + elseif head === :module && nargs==4 && isa(args[1],VersionNumber) && isa(args[2],Bool) + # New 4-argument form: (version, baremodule_flag, name, body) + show_block(IOContext(io, beginsym=>false), args[2] ? :module : :baremodule, args[3], args[4], indent, quote_level) + print(io, "end") elseif head === :module && nargs==3 && isa(args[1],Bool) + # Old 3-argument form: (baremodule_flag, name, body) show_block(IOContext(io, beginsym=>false), args[1] ? :module : :baremodule, args[2], args[3], indent, quote_level) print(io, "end") @@ -2433,7 +2469,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int, quote_level::In end elseif head === :quote && nargs == 1 && isa(args[1], Symbol) - show_unquoted_quote_expr(IOContext(io, beginsym=>false), args[1]::Symbol, indent, 0, quote_level+1) + show_unquoted_quote_expr(IOContext(io, beginsym=>false), args[1]::Symbol, indent, quote_level+1) elseif head === :quote && !(get(io, :unquote_fallback, true)::Bool) if nargs == 1 && is_expr(args[1], :block) show_block(IOContext(io, beginsym=>false), "quote", Expr(:quote, (args[1]::Expr).args...), indent, @@ -2539,12 +2575,12 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int, quote_level::In # Reset SOURCE_SLOTNAMES. Raw SlotNumbers are not valid in Expr(:toplevel), but # we want to show bad ASTs reasonably to make errors understandable. lambda_io = IOContext(io, :SOURCE_SLOTNAMES => false) - show_unquoted_expr_fallback(lambda_io, ex, indent, quote_level) + show_unquoted_expr_fallback(lambda_io, ex) else unhandled = true end if unhandled - show_unquoted_expr_fallback(io, ex, indent, quote_level) + show_unquoted_expr_fallback(io, ex) end nothing end @@ -2570,7 +2606,7 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg end s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.singletonname), context=io) print_within_stacktrace(io, s, bold=true) - elseif isType(ft) && (f = ft.parameters[1]; !isa(f, TypeVar)) + elseif isType(ft) && (f = type_parameter(ft); !isa(f, TypeVar)) uwf = unwrap_unionall(f) parens = isa(f, UnionAll) && !(isa(uwf, DataType) && f === uwf.name.wrapper) parens && print(io, "(") @@ -2674,11 +2710,11 @@ function type_depth_limit(str::String, n::Int; maxdepth = nothing) levelcount = Int[] # number of nodes at each level strwid = 0 st_0, st_backslash, st_squote, st_dquote = 0,1,2,4 - state::Int = st_0 - stateis(s) = (state & s) != 0 + state = Ref(st_0) + stateis(s) = (state[] & s) != 0 quoted() = stateis(st_squote) || stateis(st_dquote) - enter(s) = (state |= s) - leave(s) = (state &= ~s) + enter(s) = (state[] |= s) + leave(s) = (state[] &= ~s) for (i, c) in ANSIIterator(str) if c isa ANSIDelimiter depths[i] = depth @@ -2814,11 +2850,12 @@ function show(io::IO, tv::TypeVar) # Otherwise, the lower bound should be printed if it is not `Bottom` # and the upper bound should be printed if it is not `Any`. in_env = (:unionall_env => tv) in io - function show_bound(io::IO, @nospecialize(b)) + function show_bound(io::IO, @nospecialize(b)) # b::Union{Core.AnyType,TypeVar} parens = isa(b,UnionAll) && !print_without_params(b) parens && print(io, "(") - show(io, b) + b isa TypeVar ? show(io, b) : show(io, b::Core.AnyType) parens && print(io, ")") + nothing end lb, ub = tv.lb, tv.ub if !in_env && lb !== Bottom @@ -3112,6 +3149,8 @@ end nocolor(io::IO) = IOContext(io, :color => false) alignment_from_show(io::IO, x::Any) = textwidth(sprint(show, x, context=nocolor(io), sizehint=0)) +alignment_from_show(io::IO, x::AbstractString) = + textwidth(sprint(show, MIME"text/plain"(), x, context=nocolor(io), sizehint=0)) """ `alignment(io, X)` returns a tuple (left,right) showing how many characters are @@ -3216,21 +3255,35 @@ function array_summary(io::IO, a, inds) print(io, " with indices ", inds2string(inds)) end +## `summary` for GenericMemoryRef +function summary(io::IO, mref::GenericMemoryRef) + offset = Core.memoryrefoffset(mref) + len_after_offset = length(mref.mem) - offset + 1 + print(io, len_after_offset, "-element ") + showarg(io, mref, true) +end + ## `summary` for Function summary(io::IO, f::Function) = show(io, MIME"text/plain"(), f) """ - showarg(io::IO, x, toplevel) + Base.showarg(io::IO, x, toplevel) + +Show the quasi-type of `x` where quasi-type is the type of `x` or an expression (possibly +containing quasi-types) that would generate an object of the same type as `x`. The shorter +of these two options is typically used. -Show `x` as if it were an argument to a function. This function is -used by [`summary`](@ref) to display type information in terms of sequences of -function calls on objects. `toplevel` is `true` if this is -the direct call from `summary` and `false` for nested (recursive) calls. +This function is used by `summary` to display type information in terms of sequences of +function calls on objects. -The fallback definition is to print `x` as "::\\\$(typeof(x))", -representing argument `x` in terms of its type. (The double-colon is -omitted if `toplevel=true`.) However, you can -specialize this function for specific types to customize printing. +Show a leading `::` if `toplevel` is `false` and showing a type. `toplevel` is `true` if +this is the direct call from `summary` and `false` for nested (recursive) calls. + +The fallback definition is to print `x` as "::\\\$(typeof(x))" or "\\\$(typeof(x))", +representing argument `x` in terms of its type. However, you can specialize +this function for specific types to customize printing. This customization is useful for +types that have simple, public constructors and verbose and/or internal types and type +parameters such as `reinterpret`ed arrays or `SubArray`s. # Examples @@ -3259,14 +3312,14 @@ type, indicating that any recursed calls are not at the top level. Printing the parent as `::Array{Float64,3}` is the fallback (non-toplevel) behavior, because no specialized method for `Array` has been defined. """ -function showarg(io::IO, T::Type, toplevel) - toplevel || print(io, "::") - print(io, "Type{", T, "}") -end function showarg(io::IO, @nospecialize(x), toplevel) toplevel || print(io, "::") print(io, typeof(x)) end +function showarg(io::IO, T::Type, toplevel) + toplevel || print(io, "::") + print(io, "Type{", T, "}") +end # This method resolves an ambiguity for packages that specialize on eltype function showarg(io::IO, a::Array{Union{}}, toplevel) toplevel || print(io, "::") @@ -3327,7 +3380,7 @@ function Base.showarg(io::IO, r::Iterators.Pairs{<:Integer, <:Any, <:Any, T}, to print(io, "pairs(IndexLinear(), ::", T, ")") end -function Base.showarg(io::IO, r::Iterators.Pairs{Symbol, <:Any, <:Any, T}, toplevel) where {T <: NamedTuple} +function Base.showarg(io::IO, r::Iterators.Pairs{Symbol, <:Any, Nothing, T}, toplevel) where {T <: NamedTuple} print(io, "pairs(::NamedTuple)") end @@ -3389,32 +3442,22 @@ function print_partition(io::IO, partition::Core.BindingPartition) print(io, max_world) end if (partition.kind & PARTITION_MASK_FLAG) != 0 - first = false - print(io, " [") - if (partition.kind & PARTITION_FLAG_EXPORTED) != 0 - print(io, "exported") - end - if (partition.kind & PARTITION_FLAG_DEPRECATED) != 0 - first ? (first = false) : print(io, ",") - print(io, "deprecated") - end - if (partition.kind & PARTITION_FLAG_DEPWARN) != 0 - first ? (first = false) : print(io, ",") - print(io, "depwarn") - end - print(io, "]") + flags = String[] + (partition.kind & PARTITION_FLAG_EXPORTED) != 0 && push!(flags, "exported") + (partition.kind & PARTITION_FLAG_IMPLICITLY_EXPORTED) != 0 && push!(flags, "re-exported") + (partition.kind & PARTITION_FLAG_DEPRECATED) != 0 && push!(flags, "deprecated") + (partition.kind & PARTITION_FLAG_DEPWARN) != 0 && push!(flags, "depwarn") + (partition.kind & PARTITION_FLAG_IMPLICITLY_DEPRECATED) != 0 && push!(flags, "implicitly-deprecated") + print(io, " [", join(flags, ","), "]") end print(io, " - ") kind = binding_kind(partition) if kind == PARTITION_KIND_BACKDATED_CONST - print(io, "backdated constant binding to ") - print(io, partition_restriction(partition)) + print(io, "backdated constant binding") elseif kind == PARTITION_KIND_CONST - print(io, "constant binding to ") - print(io, partition_restriction(partition)) + print(io, "constant binding") elseif kind == PARTITION_KIND_CONST_IMPORT - print(io, "constant binding (declared with `import`) to ") - print(io, partition_restriction(partition)) + print(io, "constant binding (declared with `import`)") elseif kind == PARTITION_KIND_UNDEF_CONST print(io, "undefined const binding") elseif kind == PARTITION_KIND_GUARD @@ -3427,8 +3470,7 @@ function print_partition(io::IO, partition::Core.BindingPartition) print(io, "implicit `using` resolved to global ") print(io, partition_restriction(partition).globalref) elseif kind == PARTITION_KIND_IMPLICIT_CONST - print(io, "implicit `using` resolved to constant ") - print(io, partition_restriction(partition)) + print(io, "implicit `using` resolved to constant") elseif kind == PARTITION_KIND_EXPLICIT print(io, "explicit `using` from ") print(io, partition_restriction(partition).globalref) @@ -3436,14 +3478,21 @@ function print_partition(io::IO, partition::Core.BindingPartition) print(io, "explicit `import` from ") print(io, partition_restriction(partition).globalref) else - @assert kind == PARTITION_KIND_GLOBAL + @assert kind == PARTITION_KIND_GLOBAL "unexpected partition kind" print(io, "global variable with type ") print(io, partition_restriction(partition)) end end +function show(io::IO, partition::Core.BindingPartition) + print(io, "BindingPartition(for ", partition_owner(partition).globalref, ": ") + print_partition(io, partition) + print(io, ")") +end + function show(io::IO, ::MIME"text/plain", partition::Core.BindingPartition) print(io, "BindingPartition ") + print(io, "for ", partition_owner(partition).globalref, "\n ") print_partition(io, partition) end @@ -3458,8 +3507,10 @@ function show(io::IO, ::MIME"text/plain", bnd::Core.Binding) println(io) print(io, " ") print_partition(io, partition) - isdefined(partition, :next) || break - partition = @atomic partition.next + next = @atomic partition.next + # The last partition's `next` is a backreference to the owning Binding. + next isa Core.BindingPartition || break + partition = next end end end diff --git a/base/simdloop.jl b/base/simdloop.jl index 797b77ed75a99..40bd52f0fce37 100644 --- a/base/simdloop.jl +++ b/base/simdloop.jl @@ -60,22 +60,22 @@ function compile(x, ivdep) check_body!(x) var,range = parse_iteration_space(x.args[1]) - r = gensym("r") # Range value - j = gensym("i") # Iteration variable for outer loop - n = gensym("n") # Trip count for inner loop - i = gensym("i") # Trip index for inner loop - quote + # r: Range value + # j: Iteration variable for outer loop + # n: Trip count for inner loop + # i: Trip index for inner loop + return quote # Evaluate range value once, to enhance type and data flow analysis by optimizers. - let $r = $range - for $j in Base.simd_outer_range($r) - let $n = Base.simd_inner_length($r,$j) - if zero($n) < $n + let r = $(esc(range)) + for j in Base.simd_outer_range(r) + let n = Base.simd_inner_length(r,j) + if zero(n) < n # Lower loop in way that seems to work best for LLVM 3.3 vectorizer. - let $i = zero($n) - while $i < $n - local $var = Base.simd_index($r,$j,$i) - $(x.args[2]) # Body of loop - $i += 1 + let i = zero(n) + while i < n + local $(esc(var)) = Base.simd_index(r,j,i) + $(esc(x.args[2])) # Body of loop + i += 1 $(Expr(:loopinfo, Symbol("julia.simdloop"), ivdep)) # Mark loop as SIMD loop end end @@ -125,12 +125,12 @@ either case, your inner loop should have the following properties to allow vecto * No iteration ever waits on a previous iteration to make forward progress. """ macro simd(forloop) - esc(compile(forloop, nothing)) + compile(forloop, nothing) end macro simd(ivdep, forloop) if ivdep === :ivdep - esc(compile(forloop, Symbol("julia.ivdep"))) + compile(forloop, Symbol("julia.ivdep")) else throw(SimdError("Only ivdep is valid as the first argument to @simd")) end diff --git a/base/slicearray.jl b/base/slicearray.jl index 1928020a1155a..a5222e7e70d91 100644 --- a/base/slicearray.jl +++ b/base/slicearray.jl @@ -48,15 +48,15 @@ function Slices(A::P, slicemap::SM, ax::AX) where {P,SM,AX} Slices{P,SM,AX,S,N}(A, slicemap, ax) end -_slice_check_dims(N) = nothing -function _slice_check_dims(N, dim, dims...) +_slice_check_dims() = nothing +function _slice_check_dims(dim, dims...) 1 <= dim || throw(DimensionMismatch("Invalid dimension $dim")) dim in dims && throw(DimensionMismatch("Dimensions $dims are not unique")) - _slice_check_dims(N,dims...) + _slice_check_dims(dims...) end @constprop :aggressive function _eachslice(A::AbstractArray{T,N}, dims::NTuple{M,Integer}, drop::Bool) where {T,N,M} - _slice_check_dims(N,dims...) + _slice_check_dims(dims...) N_ = foldl(max, dims; init=N) if drop @@ -93,14 +93,14 @@ dimensions having size 1. See [`stack`](@ref)`(slices; dims)` for the inverse of `eachslice(A; dims::Integer)`. -See also [`eachrow`](@ref), [`eachcol`](@ref), [`mapslices`](@ref) and [`selectdim`](@ref). - !!! compat "Julia 1.1" This function requires at least Julia 1.1. !!! compat "Julia 1.9" Prior to Julia 1.9, this returned an iterator, and only a single dimension `dims` was supported. +See also [`eachrow`](@ref), [`eachcol`](@ref), [`mapslices`](@ref), [`selectdim`](@ref). + # Examples ```jldoctest @@ -141,14 +141,14 @@ Row slices are returned as `AbstractVector` views of `A`. For the inverse, see [`stack`](@ref)`(rows; dims=1)`. -See also [`eachcol`](@ref), [`eachslice`](@ref) and [`mapslices`](@ref). - !!! compat "Julia 1.1" This function requires at least Julia 1.1. !!! compat "Julia 1.9" Prior to Julia 1.9, this returned an iterator. +See also [`eachcol`](@ref), [`eachslice`](@ref), [`mapslices`](@ref). + # Examples ```jldoctest @@ -179,14 +179,14 @@ Column slices are returned as `AbstractVector` views of `A`. For the inverse, see [`stack`](@ref)`(cols)` or `reduce(`[`hcat`](@ref)`, cols)`. -See also [`eachrow`](@ref), [`eachslice`](@ref) and [`mapslices`](@ref). - !!! compat "Julia 1.1" This function requires at least Julia 1.1. !!! compat "Julia 1.9" Prior to Julia 1.9, this returned an iterator. +See also [`eachrow`](@ref), [`eachslice`](@ref), [`mapslices`](@ref). + # Examples ```jldoctest diff --git a/base/some.jl b/base/some.jl index 4269b2d78aedd..dbc021c0e63f6 100644 --- a/base/some.jl +++ b/base/some.jl @@ -14,8 +14,6 @@ end Some(::Type{T}) where {T} = Some{Type{T}}(T) -promote_rule(::Type{Some{T}}, ::Type{Some{S}}) where {T, S<:T} = Some{T} - nonnothingtype(@nospecialize(T::Type)) = typesplit(T, Nothing) promote_rule(T::Type{Nothing}, S::Type) = Union{S, Nothing} function promote_rule(T::Type{>:Nothing}, S::Type) @@ -152,8 +150,9 @@ macro something(args...) which is why we need the last argument first when building the final expression. =# - for arg in reverse(args) - val = gensym() + for i in reverse(eachindex(args)) + arg = args[i] + val = Cartesian.inlineanonymous(:val, i) expr = quote $val = $(esc(arg)) if !isnothing($val) @@ -170,4 +169,4 @@ end ==(a::Some, b::Some) = a.value == b.value isequal(a::Some, b::Some)::Bool = isequal(a.value, b.value) const hash_some_seed = UInt == UInt64 ? 0xde5c997007a4ca3a : 0x78c29c09 -hash(s::Some, h::UInt) = hash(s.value, hash_some_seed + h) +hash(s::Some, h::UInt) = hash(s.value, hash_some_seed +% h) diff --git a/base/sort.jl b/base/sort.jl index bfa65d2459680..e8f3a903ebb61 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -5,14 +5,13 @@ module Sort using Base.Order using Base: copymutable, midpoint, require_one_based_indexing, uinttype, tail, - sub_with_overflow, add_with_overflow, OneTo, BitSigned, BitIntegerType, top_set_bit + sub_with_overflow, add_with_overflow, BitSigned, BitIntegerType, top_set_bit import Base: sort, sort!, issorted, - sortperm, - to_indices + sortperm export # also exported by Base # order-only: @@ -40,8 +39,7 @@ export # not exported by Base Algorithm, DEFAULT_UNSTABLE, DEFAULT_STABLE, - SMALL_ALGORITHM, - SMALL_THRESHOLD + SMALL_ALGORITHM abstract type Algorithm end @@ -62,7 +60,7 @@ function issorted(itr, order::Ordering) end """ - issorted(v, lt=isless, by=identity, rev::Bool=false, order::Base.Order.Ordering=Base.Order.Forward) + issorted(itr, lt=isless, by=identity, rev::Bool=false, order::Base.Order.Ordering=Base.Order.Forward) Test whether a collection is in sorted order. The keywords modify what order is considered sorted, as described in the [`sort!`](@ref) documentation. @@ -113,7 +111,7 @@ maybeview(v, k) = view(v, k) maybeview(v, k::Integer) = v[k] """ - partialsort!(v, k; by=identity, lt=isless, rev=false) + partialsort!(v, k; by=identity, lt=isless, rev=false, order::Base.Order.Ordering=Base.Order.Forward) Mutate the vector `v` so that the value at index `k` (or range of adjacent values if `k` is a range) occurs @@ -170,7 +168,7 @@ partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}; partialsort!(v, k, ord(lt,by,rev,order); kws...) """ - partialsort(v, k, by=identity, lt=isless, rev=false) + partialsort(v, k, by=identity, lt=isless, rev=false, order::Base.Order.Ordering=Base.Order.Forward) Variant of [`partialsort!`](@ref) that copies `v` before partially sorting it, thereby returning the same thing as `partialsort!` but leaving `v` unmodified. @@ -325,7 +323,7 @@ searched value `x` as well as the values in `v`. The range is generally found using binary search, but there are optimized implementations for some inputs. -See also: [`searchsortedfirst`](@ref), [`sort!`](@ref), [`insorted`](@ref), [`findall`](@ref). +See also [`searchsortedfirst`](@ref), [`sort!`](@ref), [`insorted`](@ref), [`findall`](@ref). # Examples ```jldoctest @@ -364,7 +362,7 @@ values in `v`. The index is generally found using binary search, but there are optimized implementations for some inputs. -See also: [`searchsortedlast`](@ref), [`searchsorted`](@ref), [`findfirst`](@ref). +See also [`searchsortedlast`](@ref), [`searchsorted`](@ref), [`findfirst`](@ref). # Examples ```jldoctest @@ -496,7 +494,7 @@ end """ make_scratch(scratch::Union{Nothing, Vector}, T::Type, len::Integer) -Returns `(s, t)` where `t` is an `AbstractVector` of type `T` with length at least `len` +Return `(s, t)` where `t` is an `AbstractVector` of type `T` with length at least `len` that is backed by the `Vector` `s`. If `scratch !== nothing`, then `s === scratch`. This function will allocate a new vector if `scratch === nothing`, `resize!` `scratch` if it @@ -563,12 +561,15 @@ function _sort!(v::UnwrappableSubArray, a::SubArrayOptimization, o::Ordering, kw @getkw lo hi # @assert v.stride1 == 1 parent = v.parent - if parent isa Array && !(parent isa Vector) && hi - lo < 100 + if parent isa Array && !(parent isa Vector) && hi - lo < 100 || !iszero(v.offset1) # vec(::Array{T, ≠1}) allocates and is therefore somewhat expensive. # We don't want that for small inputs. + + # Additionally, if offset1 is non-zero, then this optimization is incompatible with + # algorithms that track absolute first and last indices (e.g. ScratchQuickSort) _sort!(v, a.next, o, kw) else - _sort!(vec(parent), a.next, o, (;kw..., lo = lo + v.offset1, hi = hi + v.offset1)) + _sort!(vec(parent), a.next, o, kw) end end @@ -596,7 +597,7 @@ struct WithoutMissingVector{T, U} <: AbstractVector{T} end Base.@propagate_inbounds function Base.getindex(v::WithoutMissingVector, i::Integer) out = v.data[i] - @assert !(out isa Missing) + @assert !(out isa Missing) "encountered `missing` in WithoutMissingVector" out::eltype(v) end Base.@propagate_inbounds function Base.setindex!(v::WithoutMissingVector, x, i::Integer) @@ -691,7 +692,7 @@ end Move NaN values to the end, partition by sign, and reinterpret the rest as unsigned integers. IEEE floating point numbers (`Float64`, `Float32`, and `Float16`) compare the same as -unsigned integers with the bits with a few exceptions. This pass +unsigned integers with the bits with a few exceptions. This pass is triggered for both `sort([1.0, NaN, 3.0])` and `sortperm([1.0, NaN, 3.0])`. """ @@ -760,7 +761,7 @@ end """ IsUIntMappable(yes, no) isa Base.Sort.Algorithm -Determines if the elements of a vector can be mapped to unsigned integers while preserving +Determine if the elements of a vector can be mapped to unsigned integers while preserving their order under the specified ordering. If they can be, dispatch to the `yes` algorithm and record the unsigned integer type that @@ -921,7 +922,7 @@ end ConsiderCountingSort(next) = ConsiderCountingSort(CountingSort(), next) function _sort!(v::AbstractVector{<:Integer}, a::ConsiderCountingSort, o::DirectOrdering, kw) @getkw lo hi mn mx - range = maybe_unsigned(o === Reverse ? mn-mx : mx-mn) + range = maybe_unsigned(o === Reverse ? mn -% mx : mx -% mn) if range < (sizeof(eltype(v)) > 8 ? 5(hi-lo)-100 : div(hi-lo, 2)) _sort!(v, a.counting, o, kw) @@ -946,18 +947,18 @@ maybe_reverse(o::ForwardOrdering, x) = x maybe_reverse(o::ReverseOrdering, x) = reverse(x) function _sort!(v::AbstractVector{<:Integer}, ::CountingSort, o::DirectOrdering, kw) @getkw lo hi mn mx scratch - range = maybe_unsigned(o === Reverse ? mn-mx : mx-mn) - offs = 1 - (o === Reverse ? mx : mn) + range = maybe_unsigned(o === Reverse ? mn -% mx : mx -% mn) + offs = 1 -% (o === Reverse ? mx : mn) counts = fill(0, range+1) # TODO use scratch (but be aware of type stability) @inbounds for i = lo:hi - counts[v[i] + offs] += 1 + counts[v[i] +% offs] += 1 end idx = lo @inbounds for i = maybe_reverse(o, 1:range+1) lastidx = idx + counts[i] - 1 - val = i-offs + val = i -% offs for j = idx:lastidx v[j] = val isa Unsigned && eltype(v) <: Signed ? signed(val) : val end @@ -1248,13 +1249,13 @@ function move!(v, target, source) # This function never dominates runtime—only add `@inbounds` if you can demonstrate a # performance improvement. And if you do, also double check behavior when `target` # is out of bounds. - @assert length(target) == length(source) + @assert length(target) == length(source) "length mismatch" if length(target) == 1 || isdisjoint(target, source) for (i, j) in zip(target, source) v[i], v[j] = v[j], v[i] end else - @assert minimum(source) <= minimum(target) + @assert minimum(source) <= minimum(target) "range mismatch" reverse!(v, minimum(source), maximum(target)) reverse!(v, minimum(target), maximum(target)) end @@ -1325,7 +1326,7 @@ function _sort!(v::AbstractVector, a::BracketedSort, o::Ordering, kw) # Specifically, this means that expected_middle_ln == ln, so # ln <= ... + 2.0expected_middle_ln && return ... # will trigger. - @assert false + @assert false "this should never happen" # But if it does happen, the kernel reduces to 0, hi elseif lo_signpost_i <= lo @@ -1745,7 +1746,7 @@ end Variant of [`sort!`](@ref) that returns a sorted copy of `v` leaving `v` itself unmodified. -When calling `sort` on the [`keys`](@ref) or [`values](@ref) of a dictionary, `v` is +When calling `sort` on the [`keys`](@ref) or [`values`](@ref) of a dictionary, `v` is collected and then sorted. !!! compat "Julia 1.12" @@ -1891,7 +1892,7 @@ julia> partialsortperm!(ix, v, 2:3) 4 3 ``` - """ +""" function partialsortperm!(ix::AbstractVector{<:Integer}, v::AbstractVector, k::Union{Integer, OrdinalRange}; lt::Function=isless, @@ -1932,7 +1933,7 @@ To sort slices of an array, refer to [`sortslices`](@ref). # Examples ```jldoctest -julia> v = [3, 1, 2]; +julia> v = [13, 11, 12]; julia> p = sortperm(v) 3-element Vector{Int64}: @@ -1942,9 +1943,9 @@ julia> p = sortperm(v) julia> v[p] 3-element Vector{Int64}: - 1 - 2 - 3 + 11 + 12 + 13 julia> A = [8 7; 5 6] 2×2 Matrix{Int64}: @@ -2055,13 +2056,13 @@ end # sortperm for vectors of few unique integers function sortperm_int_range(x::Vector{<:Integer}, rangelen, minval) - offs = 1 - minval + offs = 1 -% minval n = length(x) counts = fill(0, rangelen+1) counts[1] = 1 @inbounds for i = 1:n - counts[x[i] + offs + 1] += 1 + counts[x[i] +% offs +% 1] += 1 end #cumsum!(counts, counts) @@ -2071,7 +2072,7 @@ function sortperm_int_range(x::Vector{<:Integer}, rangelen, minval) P = Vector{Int}(undef, n) @inbounds for i = 1:n - label = x[i] + offs + label = x[i] +% offs P[counts[label]] = i counts[label] += 1 end @@ -2238,14 +2239,14 @@ UIntMappable(T::Type, order::Ordering) = nothing """ uint_map(x, order::Base.Order.Ordering)::Unsigned -Map `x` to an un unsigned integer, maintaining sort order. +Map `x` to an unsigned integer, maintaining sort order. The map should be reversible with [`uint_unmap`](@ref), so `isless(order, a, b)` must be a linear ordering for `a, b <: typeof(x)`. Satisfies `isless(order, a, b) === (uint_map(a, order) < uint_map(b, order))` and `x === uint_unmap(typeof(x), uint_map(x, order), order)` -See also: [`UIntMappable`](@ref) [`uint_unmap`](@ref) +See also [`UIntMappable`](@ref), [`uint_unmap`](@ref). """ function uint_map end @@ -2255,7 +2256,7 @@ function uint_map end Reconstruct the unique value `x::T` that uint_maps to `u`. Satisfies `x === uint_unmap(T, uint_map(x::T, order), order)` for all `x <: T`. -See also: [`uint_map`](@ref) [`UIntMappable`](@ref) +See also [`uint_map`](@ref), [`UIntMappable`](@ref). """ function uint_unmap end @@ -2329,7 +2330,7 @@ Characteristics: compare equal (e.g. "a" and "A" in a sort of letters that ignores case). * *in-place* in memory. - * *divide-and-conquer*: sort strategy similar to [`MergeSort`](@ref). + * *divide-and-conquer*: sort strategy similar to [`QuickSort`](@ref). Note that `PartialQuickSort(k)` does not necessarily sort the whole array. For example, @@ -2451,17 +2452,25 @@ function partition!(v::AbstractVector, lo::Integer, hi::Integer, o::Ordering) end function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::QuickSortAlg, o::Ordering) + checkbounds(v, lo:hi) + _quicksort!(v, lo, hi, a, o) +end + +function _quicksort!(v::AbstractVector, lo::Integer, hi::Integer, a::QuickSortAlg, o::Ordering) @inbounds while lo < hi - hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o) + if hi-lo <= SMALL_THRESHOLD + _sort!(v, SMALL_ALGORITHM, o, (; lo, hi, scratch=nothing)) + return v + end j = partition!(v, lo, hi, o) if j-lo < hi-j # recurse on the smaller chunk # this is necessary to preserve O(log(n)) # stack space in the worst case (rather than O(n)) - lo < (j-1) && sort!(v, lo, j-1, a, o) + lo < (j-1) && _quicksort!(v, lo, j-1, a, o) lo = j+1 else - j+1 < hi && sort!(v, j+1, hi, a, o) + j+1 < hi && _quicksort!(v, j+1, hi, a, o) hi = j-1 end end @@ -2472,8 +2481,17 @@ sort!(v::AbstractVector{T}, lo::Integer, hi::Integer, a::MergeSortAlg, o::Orderi invoke(sort!, Tuple{typeof.((v, lo, hi, a, o))..., AbstractVector{T}}, v, lo, hi, a, o, t0) # For disambiguation function sort!(v::AbstractVector{T}, lo::Integer, hi::Integer, a::MergeSortAlg, o::Ordering, t0::Union{AbstractVector{T}, Nothing}=nothing) where T + checkbounds(v, lo:hi) + _mergesort!(v, lo, hi, a, o, t0) +end + +function _mergesort!(v::AbstractVector{T}, lo::Integer, hi::Integer, a::MergeSortAlg, o::Ordering, + t0::Union{AbstractVector{T}, Nothing}) where T @inbounds if lo < hi - hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o) + if hi-lo <= SMALL_THRESHOLD + _sort!(v, SMALL_ALGORITHM, o, (; lo, hi, scratch=nothing)) + return v + end m = midpoint(lo, hi) @@ -2481,8 +2499,8 @@ function sort!(v::AbstractVector{T}, lo::Integer, hi::Integer, a::MergeSortAlg, length(t) < m-lo+1 && resize!(t, m-lo+1) Base.require_one_based_indexing(t) - sort!(v, lo, m, a, o, t) - sort!(v, m+1, hi, a, o, t) + _mergesort!(v, lo, m, a, o, t) + _mergesort!(v, m+1, hi, a, o, t) i, j = 1, lo while j <= m @@ -2514,8 +2532,17 @@ end function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::PartialQuickSort, o::Ordering) + checkbounds(v, lo:hi) + _partialquicksort!(v, lo, hi, a, o) +end + +function _partialquicksort!(v::AbstractVector, lo::Integer, hi::Integer, a::PartialQuickSort, + o::Ordering) @inbounds while lo < hi - hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o) + if hi-lo <= SMALL_THRESHOLD + _sort!(v, SMALL_ALGORITHM, o, (; lo, hi, scratch=nothing)) + return v + end j = partition!(v, lo, hi, o) if j <= first(a.k) @@ -2527,10 +2554,10 @@ function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::PartialQuickSort, # this is necessary to preserve O(log(n)) # stack space in the worst case (rather than O(n)) if j-lo < hi-j - lo < (j-1) && sort!(v, lo, j-1, a, o) + lo < (j-1) && _partialquicksort!(v, lo, j-1, a, o) lo = j+1 else - hi > (j+1) && sort!(v, j+1, hi, a, o) + hi > (j+1) && _partialquicksort!(v, j+1, hi, a, o) hi = j-1 end end @@ -2543,11 +2570,13 @@ end # Support 3-, 5-, and 6-argument versions of sort! for calling into the internals in the old way sort!(v::AbstractVector, a::Algorithm, o::Ordering) = sort!(v, firstindex(v), lastindex(v), a, o) function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::Algorithm, o::Ordering) + checkbounds(v, lo:hi) _sort!(v, a, o, (; lo, hi, legacy_dispatch_entry=a)) v end sort!(v::AbstractVector, lo::Integer, hi::Integer, a::Algorithm, o::Ordering, _) = sort!(v, lo, hi, a, o) function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::Algorithm, o::Ordering, scratch::Vector) + checkbounds(v, lo:hi) _sort!(v, a, o, (; lo, hi, scratch, legacy_dispatch_entry=a)) v end diff --git a/base/special/cbrt.jl b/base/special/cbrt.jl index ce3a3d67e3ba4..23cab5b630574 100644 --- a/base/special/cbrt.jl +++ b/base/special/cbrt.jl @@ -22,6 +22,8 @@ Return the cube root of `x`, i.e. ``x^{1/3}``. Negative values are accepted The prefix operator `∛` is equivalent to `cbrt`. +See also [`sqrt`](@ref), [`fourthroot`](@ref). + # Examples ```jldoctest julia> cbrt(big(27)) @@ -77,7 +79,7 @@ These implementations assume that NaNs, infinities and zeros have already been f end @inline function _improve_cbrt(x::Float32, t::Float32) - # Newton iterations solving + # Halley iterations solving # t^2 - x/t == 0 # with update # t <- t*(t^3 + 2*x)/(2*t^3 + x) @@ -105,7 +107,7 @@ end # # where P(r) is a polynomial of degree 4 that approximates 1/cbrt(r) # to within 2^-23.5 when |r - 1| < 1/10. The rough approximation - # has produced t such than |t/cbrt(x) - 1| ~< 1/32, and cubing this + # has produced t such that |t/cbrt(x) - 1| ~< 1/32, and cubing this # gives us bounds for r = t^3/x. r = (t*t)*(t/x) @@ -122,7 +124,7 @@ end # before the final error is larger than 0.667 ulps. u = reinterpret(UInt64, t) - u = (u + 0x8000_0000) & UInt64(0xffff_ffff_c000_0000) + u = (u +% 0x8000_0000) & UInt64(0xffff_ffff_c000_0000) t = reinterpret(Float64, u) # one step Newton iteration solving diff --git a/base/special/exp.jl b/base/special/exp.jl index f423551b02f29..62da8c374c2d8 100644 --- a/base/special/exp.jl +++ b/base/special/exp.jl @@ -221,8 +221,8 @@ end x <= MIN_EXP(base, T) && return 0.0 if k <= -53 # The UInt64 forces promotion. (Only matters for 32 bit systems.) - twopk = (k + UInt64(53)) << 52 - return reinterpret(T, twopk + reinterpret(UInt64, small_part))*0x1p-53 + twopk = ((k % UInt64) +% UInt64(53)) << 52 + return reinterpret(T, twopk +% reinterpret(UInt64, small_part))*0x1p-53 end #k == 1024 && return (small_part * 2.0) * 0x1p1023 end @@ -249,8 +249,8 @@ end x <= MIN_EXP(base, T) && return 0.0 if k <= -53 # The UInt64 forces promotion. (Only matters for 32 bit systems.) - twopk = (k + UInt64(53)) << 52 - return reinterpret(T, twopk + reinterpret(UInt64, small_part))*0x1p-53 + twopk = ((k % UInt64) +% UInt64(53)) << 52 + return reinterpret(T, twopk +% reinterpret(UInt64, small_part))*0x1p-53 end k == 1024 && return (small_part * 2.0) * 0x1p1023 end @@ -267,7 +267,7 @@ end r = muladd(N_float, LogBo256U(base, T), x) r = muladd(N_float, LogBo256L(base, T), r) k = N >> 8 - jU = reinterpret(Float64, JU_CONST | (@inbounds J_TABLE[N&255 + 1] & JU_MASK)) + jU, _ = table_unpack(N) small_part = muladd(jU, expm1b_kernel(base, r), jU) twopk = Int64(k) << 52 return reinterpret(T, twopk + reinterpret(Int64, small_part)) @@ -336,7 +336,7 @@ end Compute the natural base exponential of `x`, in other words ``ℯ^x``. -See also [`exp2`](@ref), [`exp10`](@ref) and [`cis`](@ref). +See also [`exp2`](@ref), [`exp10`](@ref), [`expm1`](@ref), [`cis`](@ref), [`log`](@ref). # Examples ```jldoctest @@ -353,7 +353,7 @@ true Compute the base 2 exponential of `x`, in other words ``2^x``. -See also [`ldexp`](@ref), [`<<`](@ref). +See also [`exp`](@ref), [`log2`](@ref), [`ldexp`](@ref), [`<<`](@ref). # Examples ```jldoctest @@ -374,6 +374,8 @@ exp2(x) Compute the base 10 exponential of `x`, in other words ``10^x``. +See also, [`exp`](@ref), [`log10`](@ref). + # Examples ```jldoctest julia> exp10(2) @@ -395,7 +397,7 @@ exp10(x) reinterpret(Float64, 0x0000_0000_0000_0001 << ((x + 1074) % UInt)) else # We will cast everything to Int64 to avoid errors in case of Int128 - # If x is a Int128, and is outside the range of Int64, then it is not -1023 expm1(1e-16) diff --git a/base/special/hyperbolic.jl b/base/special/hyperbolic.jl index 333951b6f6024..8f917243be6c9 100644 --- a/base/special/hyperbolic.jl +++ b/base/special/hyperbolic.jl @@ -106,9 +106,9 @@ function cosh(x::T) where T<:Union{Float32,Float64} # 1. Replace x by |x| (cosh(x) = cosh(-x)). # 2. Find the branch and the expression to calculate and return it # a) x <= COSH_SMALL_X - # approximate sinh(x) with a minimax polynomial + # approximate cosh(x) with a minimax polynomial # b) COSH_SMALL_X <= x < H_LARGE_X - # return cosh(x) = = (exp(x) + exp(-x))/2 + # return cosh(x) = (exp(x) + exp(-x))/2 # e) H_LARGE_X <= x # return cosh(x) = exp(x/2)/2 * exp(x/2) # Note that this branch automatically deals with Infs and NaNs @@ -162,7 +162,7 @@ end AH_LN2(::Type{Float64}) = 6.93147180559945286227e-01 AH_LN2(::Type{Float32}) = 6.9314718246f-01 # asinh methods -function asinh(x::T) where T <: Union{Float32, Float64} +@assume_effects :nothrow function asinh(x::T) where T <: Union{Float32, Float64} # Method # mathematically asinh(x) = sign(x)*log(|x| + sqrt(x*x + 1)) # is the principle value of the inverse hyperbolic sine @@ -175,6 +175,8 @@ function asinh(x::T) where T <: Union{Float32, Float64} # return sign(x)*log(2|x|+1/(|x|+sqrt(x*x+1))) # d) |x| >= 2^28 # return sign(x)*(log(x)+ln2)) + # Non-finite inputs are handled below; all log/log1p calls receive positive + # arguments, so this function never throws for Float32/Float64. if !isfinite(x) return x end @@ -240,7 +242,7 @@ end @noinline atanh_domain_error(x) = throw(DomainError(x, "atanh(x) is only defined for |x| ≤ 1.")) function atanh(x::T) where T <: Union{Float32, Float64} # Method - # 1.Reduced x to positive by atanh(-x) = -atanh(x) + # 1. Reduce x to positive by atanh(-x) = -atanh(x) # 2. Find the branch and the expression to calculate and return it # a) 0 <= x < 0.5 # return 0.5*log1p(2x/(1-x)) diff --git a/base/special/log.jl b/base/special/log.jl index 029394b7a63f1..43aadbc1fedf7 100644 --- a/base/special/log.jl +++ b/base/special/log.jl @@ -138,12 +138,6 @@ const t_log_Float32 = (0.0,0.007782140442054949,0.015504186535965254,0.023167059 0.6773988235918061,0.6813592248079031,0.6853040030989194,0.689233281238809, 0.6931471805599453) -# truncate lower order bits (up to 26) -# ideally, this should be able to use ANDPD instructions, see #9868. -@inline function truncbits(x::Float64) - reinterpret(Float64, reinterpret(UInt64,x) & 0xffff_ffff_f800_0000) -end - logb(::Type{Float32},::Val{2}) = 1.4426950408889634 logb(::Type{Float32},::Val{:ℯ}) = 1.0 logb(::Type{Float32},::Val{10}) = 0.4342944819032518 @@ -273,11 +267,12 @@ end # Step 3 xu = reinterpret(UInt64,x) - m = Int(xu >> 52) & 0x07ff + # `% Int` rather than `Int(...)` to preserve `:nothrow` (the shifted value fits in 11 bits) + m = ((xu >> 52) % Int) & 0x07ff if m == 0 # x is subnormal x *= 1.8014398509481984e16 # 0x1p54, normalise significand xu = reinterpret(UInt64,x) - m = Int(xu >> 52) & 0x07ff - 54 + m = ((xu >> 52) % Int) & 0x07ff - 54 end m -= 1023 y = reinterpret(Float64,(xu & 0x000f_ffff_ffff_ffff) | 0x3ff0_0000_0000_0000) @@ -347,7 +342,8 @@ function log1p(x::Float64) z = 1.0 + x zu = reinterpret(UInt64,z) s = reinterpret(Float64,0x7fe0_0000_0000_0000 - (zu & 0xfff0_0000_0000_0000)) # 2^-m - m = Int(zu >> 52) & 0x07ff - 1023 # z cannot be subnormal + # `% Int` rather than `Int(...)` to preserve `:nothrow` (the shifted value fits in 11 bits) + m = ((zu >> 52) % Int) & 0x07ff - 1023 # z cannot be subnormal c = m > 0 ? 1.0-(z-x) : x-(z-1.0) # 1+x = z+c exactly y = reinterpret(Float64,(zu & 0x000f_ffff_ffff_ffff) | 0x3ff0_0000_0000_0000) @@ -560,8 +556,8 @@ function _log_ext(xu::UInt64) # x = 2^k z; where z is in range [0x1.69555p-1,0x1.69555p-0) and exact. # The range is split into N subintervals. # The ith subinterval contains z and c is near the center of the interval. - tmp = reinterpret(Int64, xu - 0x3fe6955500000000) #0x1.69555p-1 - z = reinterpret(Float64, xu - (tmp & 0xfff0000000000000)) + tmp = reinterpret(Int64, xu -% 0x3fe6955500000000) #0x1.69555p-1 + z = reinterpret(Float64, xu -% (tmp & 0xfff0000000000000)) k = Float64(tmp >> 52) # log(x) = k*Ln2 + log(c) + log1p(z/c-1). # N.B. :nothrow and :noub since `idx` is known to be `1 ≤ idx ≤ length(t_log_table_compact)` diff --git a/base/special/pow.jl b/base/special/pow.jl new file mode 100644 index 0000000000000..e7da9b45b266a --- /dev/null +++ b/base/special/pow.jl @@ -0,0 +1,142 @@ +@inline function use_power_by_squaring(n::Integer) + -2^12 <= n <= 3 * 2^13 +end + +# @constprop aggressive to help the compiler see the switch between the integer and float +# variants for callers with constant `y` +@constprop :aggressive function ^(x::Float64, y::Float64) + xu = reinterpret(UInt64, x) + xu == reinterpret(UInt64, 1.0) && return 1.0 + # Exponents greater than this will always overflow or underflow. + # Note that NaN can pass through this, but that will end up fine. + if !(abs(y)<0x1.8p62) + isnan(y) && return y + y = sign(y)*0x1.8p62 + end + yint = unsafe_trunc(Int64, y) # This is actually safe since julia freezes the result + yisint = y == yint + if yisint + yint == 0 && return 1.0 + use_power_by_squaring(yint) && return @noinline pow_body(x, yint) + end + 2*xu==0 && return abs(y)*Inf*(!(y>0)) # if x === +0.0 or -0.0 (Inf * false === 0.0) + s = 1 + if x < 0 + !yisint && throw_exp_domainerror(x) # y isn't an integer + s = ifelse(isodd(yint), -1, 1) + end + !isfinite(x) && return copysign(x,s)*(y>0 || isnan(x)) # x is inf or NaN + return copysign(pow_body(abs(x), y), s) +end + +# @constprop aggressive to help the compiler see the switch between the integer and float +# variants for callers with constant `y` +@constprop :aggressive function ^(x::T, y::T) where T <: Union{Float16, Float32} + x == 1 && return one(T) + # Exponents greater than this will always overflow or underflow. + # Note that NaN can pass through this, but that will end up fine. + max_exp = T == Float16 ? T(3<<14) : T(0x1.Ap30) + if !(abs(y)0 || isnan(x)) # x is inf or NaN + return copysign(pow_body(abs(x), y), s) +end + +@constprop :aggressive @inline function ^(x::Float64, n::Integer) + n64 = clamp(n, Int64) + n64 == 0 && return one(x) + if use_power_by_squaring(n64) + return pow_body(x, n64) + else + s = ifelse(signbit(x) && isodd(n), -1.0, 1.0) + x = abs(x) + y = float(n64) + if y == n64 + return copysign(pow_body(x, y), s) + else + n2 = n64 % 1024 + y = float(n64 - n2) + return pow_body(x, y) * copysign(pow_body(x, n2), s) + end + end +end + +# @constprop aggressive to help the compiler see the switch between the integer and float +# variants for callers with constant `y` +@constprop :aggressive @inline function ^(x::T, n::Integer) where T <: Union{Float16, Float32} + n32 = clamp(n, Int32) + # Exponents greater than this will always overflow or underflow. + # Note that NaN can pass through this, but that will end up fine. + n32 == 0 && return one(x) + use_power_by_squaring(n32) && return pow_body(x, n32) + s = ifelse(signbit(x) && isodd(n), -one(T), one(T)) + x = abs(x) + return copysign(pow_body(x, widen(T)(n32)), s) +end + +@assume_effects :foldable @noinline function pow_body(x::Float64, y::Float64) + xu = reinterpret(UInt64, x) + if xu < (UInt64(1)<<52) # x is subnormal + xu = reinterpret(UInt64, x * 0x1p52) # normalize x + xu &= ~sign_mask(Float64) + xu -%= (UInt64(52) << 52) # mess with the exponent + end + logxhi,logxlo = _log_ext(xu) + xyhi, xylo = two_mul(logxhi,y) + xylo = muladd(logxlo, y, xylo) + hi = xyhi+xylo + return @inline Base.Math.exp_impl(hi, xylo-(hi-xyhi), Val(:ℯ)) +end + +@inline function pow_body(x::T, y) where T <: Union{Float16, Float32} + return T(exp2(log2(abs(widen(x))) * y)) +end + +@inline function pow_body(x::Union{Float16, Float32}, n::Int32) + n == -2 && return (i=inv(x); i*i) + n == 3 && return x*x*x #keep compatibility with literal_pow + n < 0 && return oftype(x, Base.power_by_squaring(inv(widen(x)), -n)) + return oftype(x, Base.power_by_squaring(widen(x), n)) +end + +# compensated power by squaring +# this method is only reliable for -2^20 < n < 2^20 (cf. #53881 #53886) +@assume_effects :terminates_locally @noinline function pow_body(x::Float64, n::Integer) + y = 1.0 + xnlo = -0.0 + ynlo = 0.0 + n == 3 && return x*x*x # keep compatibility with literal_pow + if n < 0 + rx = inv(x) + n==-2 && return rx*rx #keep compatibility with literal_pow + isfinite(x) && (xnlo = -fma(x, rx, -1.) * rx) + x = rx + n = -n + end + while n > 1 + if n&1 > 0 + err = muladd(y, xnlo, x*ynlo) + y, ynlo = two_mul(x,y) + ynlo += err + end + err = x*2*xnlo + x, xnlo = two_mul(x, x) + xnlo += err + n >>>= 1 + end + err = muladd(y, xnlo, x*ynlo) + return ifelse(isfinite(x) & isfinite(err), muladd(x, y, err), x*y) +end diff --git a/base/special/rem2pi.jl b/base/special/rem2pi.jl new file mode 100644 index 0000000000000..2b8417416a5bd --- /dev/null +++ b/base/special/rem2pi.jl @@ -0,0 +1,142 @@ +# multiples of pi/2, as double-double (ie with "tail") +const pi1o2_h = 1.5707963267948966 # convert(Float64, pi * BigFloat(1/2)) +const pi1o2_l = 6.123233995736766e-17 # convert(Float64, pi * BigFloat(1/2) - pi1o2_h) + +const pi2o2_h = 3.141592653589793 # convert(Float64, pi * BigFloat(1)) +const pi2o2_l = 1.2246467991473532e-16 # convert(Float64, pi * BigFloat(1) - pi2o2_h) + +const pi3o2_h = 4.71238898038469 # convert(Float64, pi * BigFloat(3/2)) +const pi3o2_l = 1.8369701987210297e-16 # convert(Float64, pi * BigFloat(3/2) - pi3o2_h) + +const pi4o2_h = 6.283185307179586 # convert(Float64, pi * BigFloat(2)) +const pi4o2_l = 2.4492935982947064e-16 # convert(Float64, pi * BigFloat(2) - pi4o2_h) + +function rem2pi(x::Float64, ::RoundingMode{:Nearest}) + isnan(x) && return x + isinf(x) && return NaN + + abs(x) < pi && return x + + n,y = rem_pio2_kernel(x) + + if iseven(n) + if n & 2 == 2 # n % 4 == 2: add/subtract pi + if y.hi <= 0 + return add22condh(y.hi,y.lo,pi2o2_h,pi2o2_l) + else + return add22condh(y.hi,y.lo,-pi2o2_h,-pi2o2_l) + end + else # n % 4 == 0: add 0 + return y.hi+y.lo + end + else + if n & 2 == 2 # n % 4 == 3: subtract pi/2 + return add22condh(y.hi,y.lo,-pi1o2_h,-pi1o2_l) + else # n % 4 == 1: add pi/2 + return add22condh(y.hi,y.lo,pi1o2_h,pi1o2_l) + end + end +end +function rem2pi(x::Float64, ::RoundingMode{:ToZero}) + isnan(x) && return x + isinf(x) && return NaN + + ax = abs(x) + ax <= 2*Float64(pi,RoundDown) && return x + + n,y = rem_pio2_kernel(ax) + + if iseven(n) + if n & 2 == 2 # n % 4 == 2: add pi + z = add22condh(y.hi,y.lo,pi2o2_h,pi2o2_l) + else # n % 4 == 0: add 0 or 2pi + if y.hi > 0 + z = y.hi+y.lo + else # negative: add 2pi + z = add22condh(y.hi,y.lo,pi4o2_h,pi4o2_l) + end + end + else + if n & 2 == 2 # n % 4 == 3: add 3pi/2 + z = add22condh(y.hi,y.lo,pi3o2_h,pi3o2_l) + else # n % 4 == 1: add pi/2 + z = add22condh(y.hi,y.lo,pi1o2_h,pi1o2_l) + end + end + copysign(z,x) +end +function rem2pi(x::Float64, ::RoundingMode{:Down}) + isnan(x) && return x + isinf(x) && return NaN + + if x < pi4o2_h + if x >= 0 + return x + elseif x > -pi4o2_h + return add22condh(x,0.0,pi4o2_h,pi4o2_l) + end + end + + n,y = rem_pio2_kernel(x) + + if iseven(n) + if n & 2 == 2 # n % 4 == 2: add pi + return add22condh(y.hi,y.lo,pi2o2_h,pi2o2_l) + else # n % 4 == 0: add 0 or 2pi + if y.hi > 0 + return y.hi+y.lo + else # negative: add 2pi + return add22condh(y.hi,y.lo,pi4o2_h,pi4o2_l) + end + end + else + if n & 2 == 2 # n % 4 == 3: add 3pi/2 + return add22condh(y.hi,y.lo,pi3o2_h,pi3o2_l) + else # n % 4 == 1: add pi/2 + return add22condh(y.hi,y.lo,pi1o2_h,pi1o2_l) + end + end +end +function rem2pi(x::Float64, ::RoundingMode{:Up}) + isnan(x) && return x + isinf(x) && return NaN + + if x > -pi4o2_h + if x <= 0 + return x + elseif x < pi4o2_h + return add22condh(x,0.0,-pi4o2_h,-pi4o2_l) + end + end + + n,y = rem_pio2_kernel(x) + + if iseven(n) + if n & 2 == 2 # n % 4 == 2: sub pi + return add22condh(y.hi,y.lo,-pi2o2_h,-pi2o2_l) + else # n % 4 == 0: sub 0 or 2pi + if y.hi < 0 + return y.hi+y.lo + else # positive: sub 2pi + return add22condh(y.hi,y.lo,-pi4o2_h,-pi4o2_l) + end + end + else + if n & 2 == 2 # n % 4 == 3: sub pi/2 + return add22condh(y.hi,y.lo,-pi1o2_h,-pi1o2_l) + else # n % 4 == 1: sub 3pi/2 + return add22condh(y.hi,y.lo,-pi3o2_h,-pi3o2_l) + end + end +end + +rem2pi(x::Float32, r::RoundingMode) = Float32(rem2pi(Float64(x), r)) +rem2pi(x::Float16, r::RoundingMode) = Float16(rem2pi(Float64(x), r)) +rem2pi(x::Int32, r::RoundingMode) = rem2pi(Float64(x), r) + +# general fallback +function rem2pi(x::Integer, r::RoundingMode) + fx = float(x) + fx == x || throw(ArgumentError(LazyString(typeof(x), " argument to rem2pi is too large: ", x))) + rem2pi(fx, r) +end diff --git a/base/special/rem_pio2.jl b/base/special/rem_pio2.jl index b0a17fdc25087..24ec742c2296d 100644 --- a/base/special/rem_pio2.jl +++ b/base/special/rem_pio2.jl @@ -82,7 +82,7 @@ end y1 = r-w high = highword(y1) i = j-((high>>20)&0x7ff) - if i>49 # 3rd iteration need, 151 bits acc + if i>49 # 3rd iteration needed, 151 bits acc t = r # will cover all possible cases w = fn*pio2_3 r = t-w @@ -97,6 +97,7 @@ end """ fromfraction(f::Int128) + Compute a tuple of values `(z1,z2)` such that ``z1 + z2 == f / 2^128`` and the significand of `z1` has 27 trailing zeros. @@ -185,10 +186,10 @@ function paynehanek(x::Float64) # # (i.e. ignoring integer and lowest bit parts of result) - w1 = UInt128(X*a1) << 64 # overflow becomes integer - w2 = widemul(X,a2) - w3 = widemul(X,a3) >> 64 - w = w1 + w2 + w3 # quotient fraction after division by 2π + w1 = UInt128(X *% a1) << 64 # overflow becomes integer + w2 = widemul(X, a2) + w3 = widemul(X, a3) >> 64 + w = w1 +% w2 +% w3 # quotient fraction after division by 2π # adjust for sign of x w = flipsign(w,x) @@ -211,6 +212,7 @@ end """ rem_pio2_kernel(x::Union{Float32, Float64}) + Calculate `x` divided by `π/2` accurately for arbitrarily large `x`. Returns a pair `(k, r)`, where `k` is the quadrant of the result (multiple of π/2) and `r` is the remainder, such that ``k * π/2 = x - r``. diff --git a/base/special/trig.jl b/base/special/trig.jl index 96a2da5f5b968..b196a022a81e4 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -25,7 +25,6 @@ end # Trigonometric functions # sin methods -@noinline sin_domain_error(x) = throw(DomainError(x, "sin(x) is only defined for finite x.")) function sin(x::T) where T<:Union{Float32, Float64} absx = abs(x) if absx < T(pi)/4 #|x| ~<= pi/4, no need for reduction @@ -36,7 +35,7 @@ function sin(x::T) where T<:Union{Float32, Float64} elseif isnan(x) return x elseif isinf(x) - sin_domain_error(x) + return throw_finite_domainerror(:sin, x) end n, y = rem_pio2_kernel(x) n = n&3 @@ -64,7 +63,7 @@ const DS6 = 1.58969099521155010221e-10 """ sin_kernel(yhi, ylo) -Computes the sine on the interval [-π/4; π/4]. +Compute the sine on the interval [-π/4; π/4]. """ @inline function sin_kernel(y::DoubleFloat64) y² = y.hi*y.hi @@ -94,7 +93,6 @@ end end # cos methods -@noinline cos_domain_error(x) = throw(DomainError(x, "cos(x) is only defined for finite x.")) function cos(x::T) where T<:Union{Float32, Float64} absx = abs(x) if absx < T(pi)/4 @@ -105,7 +103,7 @@ function cos(x::T) where T<:Union{Float32, Float64} elseif isnan(x) return x elseif isinf(x) - cos_domain_error(x) + return throw_finite_domainerror(:cos, x) else n, y = rem_pio2_kernel(x) n = n&3 @@ -162,7 +160,6 @@ cos_kernel(x::Float32) = cos_kernel(DoubleFloat32(x)) end ### sincos methods -@noinline sincos_domain_error(x) = throw(DomainError(x, "sincos(x) is only defined for finite x.")) """ sincos(x::T) where T -> Tuple{float(T),float(T)} @@ -183,14 +180,14 @@ function sincos(x::T) where T<:Union{Float32, Float64} elseif isnan(x) return x, x elseif isinf(x) - sincos_domain_error(x) + return throw_finite_domainerror(:sincos, x) end n, y = rem_pio2_kernel(x) n = n&3 # calculate both kernels at the reduced y... si, co = sincos_kernel(y) # ... and use the same selection scheme as above: (sin, cos, -sin, -cos) for - # for sin and (cos, -sin, -cos, sin) for cos + # sin and (cos, -sin, -cos, sin) for cos if n == 0 return si, co elseif n == 1 @@ -214,7 +211,6 @@ sincos(x) = _sincos(float(x)) @inline sincos_kernel(y::Union{Float32, Float64, DoubleFloat32, DoubleFloat64}) = (sin_kernel(y), cos_kernel(y)) # tangent methods -@noinline tan_domain_error(x) = throw(DomainError(x, "tan(x) is only defined for finite x.")) function tan(x::T) where T<:Union{Float32, Float64} absx = abs(x) if absx < T(pi)/4 @@ -225,7 +221,7 @@ function tan(x::T) where T<:Union{Float32, Float64} elseif isnan(x) return x elseif isinf(x) - tan_domain_error(x) + return throw_finite_domainerror(:tan, x) end n, y = rem_pio2_kernel(x) if iseven(n) @@ -636,7 +632,7 @@ function atan(y::T, x::T) where T<:Union{Float32, Float64} ypw = poshighword(y) xpw = poshighword(x) # compute y/x for Float32 - k = reinterpret(Int32, ypw-xpw)>>ATAN2_RATIO_BIT_SHIFT(T) + k = reinterpret(Int32, ypw -% xpw)>>ATAN2_RATIO_BIT_SHIFT(T) if k > ATAN2_RATIO_THRESHOLD(T) # |y/x| > threshold z=T(pi)/2+T(0.5)*ATAN2_PI_LO(T) @@ -727,18 +723,13 @@ end # Uses minimax polynomial of sin(π * x) for π * x in [0, .25] @inline function sinpi_kernel(x::Float64) - sinpi_kernel_wide(x) + _sinpi_kernel_f64(x) end @inline function sinpi_kernel_wide(x::Float64) - x² = x*x - x⁴ = x²*x² - r = evalpoly(x², (2.5501640398773415, -0.5992645293202981, 0.08214588658006512, - -7.370429884921779e-3, 4.662827319453555e-4, -2.1717412523382308e-5)) - return muladd(3.141592653589793, x, x*muladd(-5.16771278004997, - x², muladd(x⁴, r, 1.2245907532225998e-16))) + _sinpi_kernel_f64(x) end @inline function sinpi_kernel(x::Float32) - Float32(sinpi_kernel_wide(x)) + _sinpi_kernel_f32(x) end @inline function sinpi_kernel_wide(x::Float32) x = Float64(x) @@ -756,20 +747,13 @@ end # Uses minimax polynomial of cos(π * x) for π * x in [0, .25] @inline function cospi_kernel(x::Float64) - cospi_kernel_wide(x) + _cospi_kernel_f64(x) end @inline function cospi_kernel_wide(x::Float64) - x² = x*x - r = x²*evalpoly(x², (4.058712126416765, -1.3352627688537357, 0.23533063027900392, - -0.025806887811869204, 1.9294917136379183e-3, -1.0368935675474665e-4)) - a_x² = 4.934802200544679 * x² - a_x²lo = muladd(3.109686485461973e-16, x², muladd(4.934802200544679, x², -a_x²)) - - w = 1.0-a_x² - return w + muladd(x², r, ((1.0-w)-a_x²) - a_x²lo) + _cospi_kernel_f64(x) end @inline function cospi_kernel(x::Float32) - Float32(cospi_kernel_wide(x)) + _cospi_kernel_f32(x) end @inline function cospi_kernel_wide(x::Float32) x = Float64(x) @@ -784,6 +768,160 @@ end return evalpoly(x*x, (1.0f0, -4.934802f0, 4.058712f0, -1.3352624f0, 0.23531426f0, -0.0255071f0)) end +function _cospi_kernel(x::AbstractFloat, sch) + x² = x * x + c₀ = sch.c₀ + (c₂, c₂_lo) = sch.c₂ + rest = sch.rest + c₂ = -c₂ + c₂_lo = -c₂_lo + r = evalpoly(x², rest) * x² + a_x² = c₂ * x² + a_x²_lo = muladd(c₂_lo, x², muladd(c₂, x², -a_x²)) + w = c₀ - a_x² + w + muladd(x², r, ((c₀ - w) - a_x²) - a_x²_lo) +end + +function _sinpi_kernel(x::AbstractFloat, sch) + x² = x * x + (c₁, c₁_lo) = sch.c₁ + (c₃, rest...) = sch.rest + if rest === () + r = typeof(x)(0) + else + r = evalpoly(x², rest) + end + muladd(c₁, x, x * muladd(c₃, x², muladd(x² * x², r, c₁_lo))) +end + +#= +# Polynomial approximation for `cospi` and `sinpi` kernels + +## `cospi` + +Constrain the zeroth coefficient to `1` to achieve exact behavior for zero input. + +* `Float32`: + + ```sollya + handTuned = 3; + prec = 500!; + accurate = cos(pi * x); + kernelDomain = [-2^-3, 2^-2]; + constrainedPart = 1; + machinePrecision = 24; + doubleWordPrecision = 2 * machinePrecision + handTuned; + freeMonomials = [|2, 4, 6, 8|]; + freeMonomialPrecisions = [|doubleWordPrecision, machinePrecision, machinePrecision, machinePrecision|]; + polynomial = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, kernelDomain, constrainedPart); + supnormPrecision = 2^-10; + sup(supnorm(polynomial, accurate, kernelDomain, relative, supnormPrecision)); + polynomial; + ``` + +* `Float64`: + + ```sollya + handTuned = 1; + prec = 500!; + accurate = cos(pi * x); + kernelDomain = [-2^-3, 2^-2]; + constrainedPart = 1; + machinePrecision = 53; + doubleWordPrecision = 2 * machinePrecision + handTuned; + freeMonomials = [|2, 4, 6, 8, 10, 12, 14|]; + freeMonomialPrecisions = [|doubleWordPrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision|]; + polynomial = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, kernelDomain, constrainedPart); + supnormPrecision = 2^-10; + sup(supnorm(polynomial, accurate, kernelDomain, relative, supnormPrecision)); + polynomial; + ``` + +## `sinpi` + +* `Float32`: + + ```sollya + handTuned = 5; + prec = 500!; + accurate = sin(pi * x); + kernelDomain = [-2^-3, 2^-2]; + machinePrecision = 24; + doubleWordPrecision = 2 * machinePrecision + handTuned; + freeMonomials = [|1, 3, 5, 7, 9|]; + freeMonomialPrecisions = [|doubleWordPrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision|]; + polynomial = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, kernelDomain); + supnormPrecision = 2^-10; + sup(supnorm(polynomial, accurate, kernelDomain, relative, supnormPrecision)); + polynomial; + ``` + +* `Float64`: + + ```sollya + handTuned = 3; + prec = 500!; + accurate = sin(pi * x); + kernelDomain = [-2^-3, 2^-2]; + machinePrecision = 53; + doubleWordPrecision = 2 * machinePrecision + handTuned; + freeMonomials = [|1, 3, 5, 7, 9, 11, 13, 15|]; + freeMonomialPrecisions = [|doubleWordPrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision|]; + polynomial = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, kernelDomain); + supnormPrecision = 2^-10; + sup(supnorm(polynomial, accurate, kernelDomain, relative, supnormPrecision)); + polynomial; + ``` +=# + +const _cospi_kernel_polynomial_f32 = (; + c₀ = Float32(1), + c₂ = (-4.934802f0, -1.1607644f-7), + rest = ( + 4.0587077f0, + -1.3350532f0, + 0.23138562f0, + ), +) +const _sinpi_kernel_polynomial_f32 = (; + c₁ = (3.1415927f0, -8.764345f-8), + rest = ( + -5.1677127f0, + 2.5501568f0, + -0.5990627f0, + 0.079937235f0, + ), +) +const _cospi_kernel_polynomial_f64 = (; + c₀ = Float64(1), + c₂ = (-4.934802200544679, -2.6451348079795815e-16), + rest = ( + 4.058712126416749, + -1.3352627688519947, + 0.2353306301924776, + -0.025806885661227713, + 0.0019294656071154924, + -0.00010356606727649327, + ), +) +const _sinpi_kernel_polynomial_f64 = (; + c₁ = (3.141592653589793, 1.2267151843884804e-16), + rest = ( + -5.16771278004997, + 2.550164039877393, + -0.5992645293247603, + 0.082145886770189, + -0.007370434116378644, + 0.000466329949762989, + -2.1925990105975317e-5, + ), +) + +const _cospi_kernel_f32 = Base.Fix2(_cospi_kernel, _cospi_kernel_polynomial_f32) +const _cospi_kernel_f64 = Base.Fix2(_cospi_kernel, _cospi_kernel_polynomial_f64) +const _sinpi_kernel_f32 = Base.Fix2(_sinpi_kernel, _sinpi_kernel_polynomial_f32) +const _sinpi_kernel_f64 = Base.Fix2(_sinpi_kernel, _sinpi_kernel_polynomial_f64) + """ sinpi(x::T) where T -> float(T) @@ -797,7 +935,7 @@ function sinpi(_x::T) where T<:IEEEFloat x = abs(_x) if !isfinite(x) isnan(x) && return x - throw(DomainError(x, "`sinpi(x)` is only defined for finite `x`.")) + return throw_finite_domainerror(:sinpi, x) end # For large x, answers are all 1 or zero. x >= maxintfloat(T) && return copysign(zero(T), _x) @@ -817,6 +955,7 @@ function sinpi(_x::T) where T<:IEEEFloat end return ifelse(signbit(_x), -res, res) end + """ cospi(x::T) where T -> float(T) @@ -824,13 +963,13 @@ Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. -See also: [`cispi`](@ref), [`sincosd`](@ref), [`sinpi`](@ref). +See also [`cispi`](@ref), [`sincosd`](@ref), [`sinpi`](@ref). """ function cospi(x::T) where T<:IEEEFloat x = abs(x) if !isfinite(x) isnan(x) && return x - throw(DomainError(x, "`cospi(x)` is only defined for finite `x`.")) + return throw_finite_domainerror(:cospi, x) end # For large x, answers are all 1 or zero. x >= maxintfloat(T) && return one(T) @@ -849,6 +988,7 @@ function cospi(x::T) where T<:IEEEFloat return sinpi_kernel(rx) end end + """ sincospi(x::T) where T -> Tuple{float(T),float(T)} @@ -860,13 +1000,13 @@ Throw a [`DomainError`](@ref) if `isinf(x)`, return a `(T(NaN), T(NaN))` tuple i !!! compat "Julia 1.6" This function requires Julia 1.6 or later. -See also: [`cispi`](@ref), [`sincosd`](@ref), [`sinpi`](@ref). +See also [`cispi`](@ref), [`sincosd`](@ref), [`sinpi`](@ref). """ function sincospi(_x::T) where T<:IEEEFloat x = abs(_x) if !isfinite(x) isnan(x) && return x, x - throw(DomainError(x, "`sincospi(x)` is only defined for finite `x`.")) + return throw_finite_domainerror(:sincospi, x) end # For large x, answers are all 1 or zero. x >= maxintfloat(T) && return (copysign(zero(T), _x), one(T)) @@ -907,7 +1047,7 @@ function tanpi(_x::T) where T<:IEEEFloat x = abs(_x) if !isfinite(x) isnan(x) && return x - throw(DomainError(x, "`tanpi(x)` is only defined for finite `x`.")) + return throw_finite_domainerror(:tanpi, x) end # For large x, answers are all zero. # All integer values for floats larger than maxintfloat are even. @@ -940,7 +1080,15 @@ sinpi(x::AbstractFloat) = sin(pi*x) cospi(x::AbstractFloat) = cos(pi*x) sincospi(x::AbstractFloat) = sincos(pi*x) tanpi(x::AbstractFloat) = tan(pi*x) -tanpi(x::Complex) = sinpi(x) / cospi(x) # Is there a better way to do this? + +function tanpi(z::Complex) + zr, zi = reim(z) + iszero(zi) && return Complex(tanpi(zr)) + sr, cr = sincospi(zr) + ti = tanh(zi * pi) + cz = Complex(cr, ti * sr) + Complex(sr, ti * cr) * cz / abs2(cz) +end function sinpi(z::Complex{T}) where T F = float(T) @@ -1104,6 +1252,10 @@ Return a `T(NaN)` if `isnan(x)`. See also [`sinc`](@ref). """ cosc(x::Number) = _cosc(float(x)) +function _cosc_generic(x) + pi_x = pi * x + (pi_x*cospi(x)-sinpi(x))/(pi_x*x) +end function _cosc(x::Number) # naive cosc formula is susceptible to catastrophic # cancellation error near x=0, so we use the Taylor series @@ -1112,6 +1264,7 @@ function _cosc(x::Number) # generic Taylor series: π ∑ (-1)^n (πx)^{2n-1}/a(n) where # a(n) = (1+2n)*(2n-1)! (= OEIS A174549) s = (term = -(π*x))/3 + iszero(s) && return s # preserve floating-point signed zero π²x² = term^2 ε = eps(fastabs(term)) # error threshold to stop sum n = 1 @@ -1123,17 +1276,128 @@ function _cosc(x::Number) end return π*s else - return isinf_real(x) ? zero(x) : ((pi*x)*cospi(x)-sinpi(x))/((pi*x)*x) + return isinf_real(x) ? zero(x) : _cosc_generic(x) end end + +#= + +## `cosc(x)` for `x` around the first zero, at `x = 0` + +`Float32`: + +```sollya +prec = 500!; +accurate = ((pi * x) * cos(pi * x) - sin(pi * x)) / (pi * x * x); +b1 = 0.27001953125; +b2 = 0.449951171875; +domain_0 = [-b1/2, b1]; +domain_1 = [b1, b2]; +machinePrecision = 24; +freeMonomials = [|1, 3, 5, 7|]; +freeMonomialPrecisions = [|machinePrecision, machinePrecision, machinePrecision, machinePrecision|]; +polynomial_0 = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, domain_0); +polynomial_1 = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, domain_1); +polynomial_0; +polynomial_1; +``` + +`Float64`: + +```sollya +prec = 500!; +accurate = ((pi * x) * cos(pi * x) - sin(pi * x)) / (pi * x * x); +b1 = 0.1700439453125; +b2 = 0.27001953125; +b3 = 0.340087890625; +b4 = 0.39990234375; +domain_0 = [-b1/2, b1]; +domain_1 = [b1, b2]; +domain_2 = [b2, b3]; +domain_3 = [b3, b4]; +machinePrecision = 53; +freeMonomials = [|1, 3, 5, 7, 9, 11|]; +freeMonomialPrecisions = [|machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision, machinePrecision|]; +polynomial_0 = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, domain_0); +polynomial_1 = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, domain_1); +polynomial_2 = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, domain_2); +polynomial_3 = fpminimax(accurate, freeMonomials, freeMonomialPrecisions, domain_3); +polynomial_0; +polynomial_1; +polynomial_2; +polynomial_3; +``` + +=# + +function _cos_cardinal_eval(x::AbstractFloat, polynomials_close_to_origin::NTuple) + function choose_poly(a::AbstractFloat, polynomials_close_to_origin::NTuple{2}) + ((b1, p0), (_, p1)) = polynomials_close_to_origin + if a ≤ b1 + p0 + else + p1 + end + end + function choose_poly(a::AbstractFloat, polynomials_close_to_origin::NTuple{4}) + ((b1, p0), (b2, p1), (b3, p2), (_, p3)) = polynomials_close_to_origin + if a ≤ b2 # hardcoded binary search + if a ≤ b1 + p0 + else + p1 + end + else + if a ≤ b3 + p2 + else + p3 + end + end + end + a = abs(x) + if (polynomials_close_to_origin !== ()) && (a ≤ polynomials_close_to_origin[end][1]) + x * evalpoly(x * x, choose_poly(a, polynomials_close_to_origin)) + elseif isinf(x) + typeof(x)(0) + else + _cosc_generic(x) + end +end + +const _cosc_f32 = let b = Float32 ∘ Float16 + ( + (b(0.27), (-3.289868f0, 3.246966f0, -1.1443111f0, 0.20542027f0)), + (b(0.45), (-3.2898617f0, 3.2467577f0, -1.1420113f0, 0.1965574f0)), + ) +end + +const _cosc_f64 = let b = Float64 ∘ Float16 + ( + (b(0.17), (-3.289868133696453, 3.2469697011333203, -1.1445109446992934, 0.20918277797812262, -0.023460519561502552, 0.001772485141534688)), + (b(0.27), (-3.289868133695205, 3.246969700970421, -1.1445109360543062, 0.20918254132488637, -0.023457115021035743, 0.0017515112964895303)), + (b(0.34), (-3.289868133634355, 3.246969697075094, -1.1445108347839286, 0.209181201609773, -0.023448079433318045, 0.001726628430505518)), + (b(0.4), (-3.289868133074254, 3.2469696736659346, -1.1445104406286049, 0.20917785794416457, -0.02343378376047161, 0.0017019796223768677)), + ) +end + +function _cosc(x::Union{Float32, Float64}) + if x isa Float32 + pols = _cosc_f32 + else + pols = _cosc_f64 + end + _cos_cardinal_eval(x, pols) +end + # hard-code Float64/Float32 Taylor series, with coefficients # Float64.([(-1)^n*big(pi)^(2n)/((2n+1)*factorial(2n-1)) for n = 1:6]) -_cosc(x::Union{Float64,ComplexF64}) = +_cosc(x::ComplexF64) = fastabs(x) < 0.14 ? x*evalpoly(x^2, (-3.289868133696453, 3.2469697011334144, -1.1445109447325053, 0.2091827825412384, -0.023460810354558236, 0.001781145516372852)) : - isinf_real(x) ? zero(x) : ((pi*x)*cospi(x)-sinpi(x))/((pi*x)*x) -_cosc(x::Union{Float32,ComplexF32}) = + isinf_real(x) ? zero(x) : _cosc_generic(x) +_cosc(x::ComplexF32) = fastabs(x) < 0.26f0 ? x*evalpoly(x^2, (-3.289868f0, 3.2469697f0, -1.144511f0, 0.20918278f0)) : - isinf_real(x) ? zero(x) : ((pi*x)*cospi(x)-sinpi(x))/((pi*x)*x) + isinf_real(x) ? zero(x) : _cosc_generic(x) _cosc(x::Float16) = Float16(_cosc(Float32(x))) _cosc(x::ComplexF16) = ComplexF16(_cosc(ComplexF32(x))) @@ -1208,7 +1472,7 @@ deg2rad_ext(x::Real) = deg2rad(x) # Fallback function sind(x::Real) if isinf(x) - return throw(DomainError(x, "`sind(x)` is only defined for finite `x`.")) + return throw_finite_domainerror(:sind, x) elseif isnan(x) return x end @@ -1239,7 +1503,7 @@ end function cosd(x::Real) if isinf(x) - return throw(DomainError(x, "`cosd(x)` is only defined for finite `x`.")) + return throw_finite_domainerror(:cosd, x) elseif isnan(x) return x end diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 8f2866af170f7..908e950ec5bee 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -47,6 +47,14 @@ Stack information representing execution context, with the following fields: Representation of the pointer to the execution context as returned by `backtrace`. +- `pc::Int` + + If `from_c`, this is a column number. Otherwise, it is a 1-based statement + index within the frame's `linfo.debuginfo`, recovered from the DWARF column + emitted by codegen, or `0` when debuginfo is unavailable. Used internally by + `StackTraces.lookup` to resolve the `MethodInstance` for inlined frames; also + surfaced in low-level safe backtrace output (e.g. on segfault) but not in + Julia's default `show` for `StackFrame`. """ struct StackFrame # this type should be kept platform-agnostic so that profiles can be dumped on one machine and read on another "the name of the function containing the execution context" @@ -64,10 +72,14 @@ struct StackFrame # this type should be kept platform-agnostic so that profiles inlined::Bool "representation of the pointer to the execution context as returned by `backtrace`" pointer::UInt64 # Large enough to be read losslessly on 32- and 64-bit machines. + "if !from_c, 1-based statement index (PC) within the frame's CodeInfo, or 0 if unavailable" + pc::Int end +StackFrame(func, file, line, linfo, from_c, inlined, pointer) = + StackFrame(func, file, line, linfo, from_c, inlined, pointer, 0) StackFrame(func, file, line) = StackFrame(Symbol(func), Symbol(file), line, - nothing, false, false, 0) + nothing, false, false, 0, 0) """ StackTrace @@ -99,6 +111,36 @@ function hash(frame::StackFrame, h::UInt) return h end +function _add_linetable_frames!(frames, pointer, di::Core.DebugInfo, pc::Int) + lt, ltpc = Base.Compiler.prev_debuginfo(di, pc) + if lt isa Core.DebugInfo + _add_linetable_frames!(frames, pointer, lt, ltpc) + edi, epc = Base.Compiler.edge_debuginfo(lt, ltpc) + edi !== nothing && _add_di_frames!(frames, pointer, edi, epc) + end + nothing +end + +# 1. Push our own frame +# 2. If there is a linetable, recurse on edges there (and not the linetable) +# 3. Recurse on any of our own edges +function _add_di_frames!(frames, pointer, di::Core.DebugInfo, pc::Int) + @assert pc > 0 "invalid pc" + push!(frames, StackFrame( + di.def isa Symbol ? Symbol("macro expansion") : IRShow.method_name(di.def), + IRShow.debuginfo_file1(di), + @ccall(jl_cdi_firstxy(di::Any, pc::Int32)::NTuple{2, Int32})[1], + di.def isa Core.MethodInstance ? di.def : nothing, + false, # we can assume C frames aren't inlined into julia + !isempty(frames), + pointer, + pc)) + _add_linetable_frames!(frames, pointer, di, pc) + edi, epc = Base.Compiler.edge_debuginfo(di, pc) + edi !== nothing && _add_di_frames!(frames, pointer, edi, epc) + nothing +end + """ lookup(pointer::Ptr{Cvoid})::Vector{StackFrame} @@ -107,20 +149,44 @@ up stack frame context information. Returns an array of frame information for al inlined at that point, innermost function first. """ Base.@constprop :none function lookup(pointer::Ptr{Cvoid}) - infos = ccall(:jl_lookup_code_address, Any, (Ptr{Cvoid}, Cint), pointer, false)::Core.SimpleVector + frames = @ccall jl_lookup_code_address(pointer::Ptr{Cvoid}, false::Cint)::Core.SimpleVector pointer = convert(UInt64, pointer) - isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, nothing, true, false, pointer)] # this is equal to UNKNOWN - res = Vector{StackFrame}(undef, length(infos)) - for i in 1:length(infos) - info = infos[i]::Core.SimpleVector - @assert length(info) == 6 "corrupt return from jl_lookup_code_address" - func = info[1]::Symbol - file = info[2]::Symbol - linenum = info[3]::Int - linfo = info[4] - res[i] = StackFrame(func, file, linenum, linfo, info[5]::Bool, info[6]::Bool, pointer) + + # this is equal to UNKNOWN + isempty(frames) && return [StackFrame( + empty_sym, empty_sym, -1, nothing, true, false, pointer)] + + # If we aren't given a PC and CodeInstance for the last (non-inlined) frame, + # we can't recover any more information than `frames`, so use those. + # Otherwise, DebugInfo lets us recover `linfo` for inlined frames too, so + # ignore `frames` and construct them by traversing the DebugInfo tree + # instead. This is a separate code path since attempting to enhance + # existing `frames` would require matching them to our tree traversal. + pc = frames[end][5]::Bool ? 0 : frames[end][7]::Int + di = let x = frames[end][4] + x isa Core.CodeInstance ? x.debuginfo : nothing end - return res + if pc <= 0 || !(di isa Core.DebugInfo) + out = Vector{StackFrame}(undef, length(frames)) + for i in 1:length(frames) + f = frames[i] + @assert length(f) == 7 "corrupt return from jl_lookup_code_address" + func = f[1]::Symbol + file = f[2]::Symbol + linenum = f[3]::Int + linfo = f[4] + from_c = f[5]::Bool + inlined = f[6]::Bool + sv_pc = from_c ? f[7]::Int : 0 + out[i] = StackFrame( + func, file, linenum, linfo, from_c, inlined, pointer, sv_pc) + end + else + out = Vector{StackFrame}() + _add_di_frames!(out, pointer, di, pc) + reverse!(out) + end + return out end const top_level_scope_sym = Symbol("top-level scope") @@ -147,11 +213,14 @@ function lookup(ip::Base.InterpreterIP) if isa(def, Core.ABIOverride) def = def.def end - if isa(def, MethodInstance) && isa(def.def, Method) - meth = def.def - func = meth.name - file = meth.file - line = meth.line + if isa(def, MethodInstance) + let meth = def.def + if isa(meth, Method) + func = meth.name + file = meth.file + line = meth.line + end + end end else codeinfo = code::CodeInfo @@ -164,19 +233,21 @@ function lookup(ip::Base.InterpreterIP) if isempty(scopes) return [StackFrame(func, file, line, code, false, false, 0)] end + res = Vector{StackFrame}(undef, length(scopes)) inlined = false - scopes = map(scopes) do lno + for i in eachindex(scopes) + lno = scopes[i] if inlined - def = lno.method - def isa Union{Method,Core.CodeInstance,MethodInstance} || (def = nothing) + def_local = lno.method + def_local isa Union{Method,Core.CodeInstance,MethodInstance} || (def_local = nothing) else - def = codeinfo + def_local = codeinfo end - sf = StackFrame(IRShow.normalize_method_name(lno.method), lno.file, lno.line, def, false, inlined, 0) + res[i] = StackFrame(IRShow.normalize_method_name(lno.method), lno.file, lno.line, + def_local, false, inlined, 0) inlined = true - return sf end - return scopes + return res end """ @@ -279,7 +350,8 @@ function show_spec_linfo(io::IO, frame::StackFrame) else # Equivalent to the default implementation of `show_custom_spec_sig` # for `linfo isa CodeInstance`, but saves an extra dynamic dispatch. - show_spec_sig(io, def, frame_mi(frame).specTypes) + mi = frame_mi(frame)::MethodInstance + show_spec_sig(io, def::Method, mi.specTypes) end else m = linfo::Method @@ -291,7 +363,8 @@ end # Can be extended by compiler packages to customize backtrace display of custom code instance frames function show_custom_spec_sig(io::IO, @nospecialize(owner), linfo::CodeInstance, frame::StackFrame) mi = Base.get_ci_mi(linfo) - return show_spec_sig(io, mi.def, mi.specTypes) + m = mi.def::Method # the case ::Module is handled in show_spec_linfo + return show_spec_sig(io, m, mi.specTypes) end function show_spec_sig(io::IO, m::Method, @nospecialize(sig::Type)) @@ -375,4 +448,4 @@ function from(frame::StackFrame, m::Module) return parentmodule(frame) === m end -end +end # module StackTraces diff --git a/base/stat.jl b/base/stat.jl index fbab5126d39bc..ee157031512e2 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -54,7 +54,7 @@ The following fields of this struct are considered public API: | mtime | `Float64` | Unix timestamp of when the file was last modified | | ctime | `Float64` | Unix timestamp of when the file's metadata was changed | -See also: [`stat`](@ref) +See also [`stat`](@ref). """ struct StatStruct desc :: Union{String, OS_HANDLE} # for show method, not included in equality or hash @@ -380,7 +380,7 @@ function ispath(path::String) end return r == 0 end -ispath(path::AbstractString) = ispath(String(path)) +ispath(path::AbstractString) = ispath(String(path)::String) """ isfifo(path)::Bool @@ -406,6 +406,8 @@ ischardev(st::StatStruct) = filemode(st) & 0xf000 == 0x2000 Return `true` if `path` points to a directory, `false` otherwise. +See also [`isfile`](@ref), [`ispath`](@ref). + # Examples ```jldoctest julia> isdir(homedir()) @@ -414,8 +416,6 @@ true julia> isdir("not/a/directory") false ``` - -See also [`isfile`](@ref) and [`ispath`](@ref). """ isdir(st::StatStruct) = filemode(st) & 0xf000 == 0x4000 @@ -434,6 +434,8 @@ isblockdev(st::StatStruct) = filemode(st) & 0xf000 == 0x6000 Return `true` if `path` points to a regular file, `false` otherwise. +See also [`isdir`](@ref), [`ispath`](@ref). + # Examples ```jldoctest julia> isfile(homedir()) @@ -451,8 +453,6 @@ julia> rm(filename); julia> isfile(filename) false ``` - -See also [`isdir`](@ref) and [`ispath`](@ref). """ isfile(st::StatStruct) = filemode(st) & 0xf000 == 0x8000 @@ -519,7 +519,7 @@ is read+write, the bitfield is "110", which maps to the decimal value of 0+2+4=6. This is reflected in the printing of the returned `UInt8` value. -See also [`gperm`](@ref) and [`operm`](@ref). +See also [`gperm`](@ref), [`operm`](@ref). ```jldoctest julia> touch("dummy_file"); # Create test-file without contents diff --git a/base/stream.jl b/base/stream.jl index d88f164f3ccca..5554a5a497b2b 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -101,9 +101,10 @@ end bytesavailable(s::LibuvStream) = bytesavailable(s.buffer) -function eof(s::LibuvStream) +function eof(s::LibuvStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) bytesavailable(s) > 0 && return false - wait_readnb(s, 1) + wait_readnb(s, 1, resolve_cancel_token(cancel)) # This function is race-y if used from multiple threads, but we guarantee # it to never return true until the stream is definitively exhausted # and that we won't return true if there's a readerror pending (it'll instead get thrown). @@ -202,12 +203,7 @@ end function PipeEndpoint(fd::OS_HANDLE) pipe = PipeEndpoint() - iolock_begin() - err = ccall(:uv_pipe_open, Int32, (Ptr{Cvoid}, OS_HANDLE), pipe.handle, fd) - uv_error("pipe_open", err) - pipe.status = StatusOpen - iolock_end() - return pipe + return open_pipe!(pipe, fd) end if OS_HANDLE != RawFD PipeEndpoint(fd::RawFD) = PipeEndpoint(Libc._get_osfhandle(fd)) @@ -223,6 +219,7 @@ mutable struct TTY <: LibuvStream sendbuf::Union{IOBuffer, Nothing} lock::ReentrantLock # advisory lock throttle::Int + raw_lock::ReentrantLock # exclusive access to raw mode @static if Sys.iswindows(); ispty::Bool; end function TTY(handle::Ptr{Cvoid}, status) tty = new( @@ -233,7 +230,8 @@ mutable struct TTY <: LibuvStream nothing, nothing, ReentrantLock(), - DEFAULT_READ_BUFFER_SZ) + DEFAULT_READ_BUFFER_SZ, + ReentrantLock()) associate_julia_struct(handle, tty) finalizer(uvfinalize, tty) @static if Sys.iswindows() @@ -283,8 +281,8 @@ end lock(s::LibuvStream) = lock(s.lock) unlock(s::LibuvStream) = unlock(s.lock) -setup_stdio(stream::LibuvStream, ::Bool) = (stream, false) -rawhandle(stream::LibuvStream) = stream.handle +setup_stdio(stream::Union{LibuvStream, LibuvServer}, ::Bool) = (stream, false) +rawhandle(stream::Union{LibuvStream, LibuvServer}) = stream.handle unsafe_convert(::Type{Ptr{Cvoid}}, s::Union{LibuvStream, LibuvServer}) = s.handle function init_stdio(handle::Ptr{Cvoid}) @@ -378,7 +376,7 @@ end function isopen(x::Union{LibuvStream, LibuvServer}) if x.status == StatusUninit || x.status == StatusInit || x.handle === C_NULL - throw(ArgumentError("$x is not initialized")) + throw(ArgumentError("stream not initialized")) end return x.status != StatusClosed end @@ -389,7 +387,7 @@ function check_open(x::Union{LibuvStream, LibuvServer}) end end -function wait_readnb(x::LibuvStream, nb::Int) +function wait_readnb(x::LibuvStream, nb::Int, tok::MaybeToken=default_cancel_token()) # fast path before iolock acquire bytesavailable(x.buffer) >= nb && return open = isopen(x) && x.status != StatusEOF # must precede readerror check @@ -405,6 +403,7 @@ function wait_readnb(x::LibuvStream, nb::Int) oldthrottle = x.throttle preserve_handle(x) lock(x.cond) + locked = true try while bytesavailable(x.buffer) < nb x.readerror === nothing || throw(x.readerror) @@ -413,12 +412,20 @@ function wait_readnb(x::LibuvStream, nb::Int) x.throttle = max(nb, x.throttle) start_reading(x) # ensure we are reading iolock_end() - wait(x.cond) + locked = false + wait(x.cond, tok) + locked = true unlock(x.cond) + locked = false iolock_begin() lock(x.cond) + locked = true end finally + # the teardown reads the waiter queue and the throttle under the + # cond lock: reacquire when a wait-throw released this frame's + # level (a spin lock; stop_reading takes the iolock itself) + locked || lock(x.cond) if isempty(x.cond) stop_reading(x) # stop reading iff there are currently no other read clients of the stream end @@ -434,12 +441,290 @@ function wait_readnb(x::LibuvStream, nb::Int) nothing end -function closewrite(s::LibuvStream) +## Waits on libuv requests +# +# A libuv request (write, shutdown, UDP send, DNS lookup) is issued under the +# iolock with its data field C_NULL and completes by a callback (also run +# under the iolock) that wakes the parked issuer through the standard +# wake-claim protocol. The request's data field tracks who owns freeing it: +# pointer to the wait entry - a waiter is parked on the request; +# C_NULL - the completion callback has run; the waiter +# owns the request; +# UV_REQ_DETACHED - the waiter departed (interrupted/refused +# wait, or fire-and-forget): the callback owns +# freeing it, plus any buffer root recorded in +# `_detached_uvreq_roots`. + +# Sentinel for a uv request's data field: see above. +const UV_REQ_DETACHED = Ptr{Cvoid}(UInt(0x1)) + +# Aux word on a uv write wait's *witness* slot (the slot whose owner is the +# stream; its aux is otherwise unused). Every reader and writer runs under +# the iolock, so the packing needs no atomics: +# +# bit 0 - CANCEL_REQUESTED: set immediately before Julia itself +# issues `uv_cancel` on write request(s) it keeps awaiting, +# so the completion callback can tell a Julia-requested +# cancellation (UV_ECANCELED is then the expected outcome, +# delivered as a partial-count wake) from a close-induced +# one (a real write error - libuv also fails queued writes +# with UV_ECANCELED when the stream closes). +# bits 8..23 - the negated status of the first real error a chunk of a +# split write completed with (0 = none): a real error must +# win the final wake even when later chunks fail with the +# close-induced UV_ECANCELED cascade. +# bits 32..63 - the number of pending completion callbacks of a split +# write. 0 means the wait is on a single request (the +# common case); a split write counts down here and only +# the callback that reaches 0 claims and wakes the waiter. +# +# Cleared whenever the slot is released for reuse (_release_slot!/ +# _clear_uv_witness!). The detach paths do not set the flag: there the +# requests are detached in the same iolock critical section, so no callback +# can observe a waiter - the flag only matters to waited requests. +const _UVREQ_AUX_CANCEL_REQUESTED = UInt64(0x1) +const _UVREQ_AUX_STATUS_SHIFT = 8 +const _UVREQ_AUX_STATUS_MASK = UInt64(0xffff) << _UVREQ_AUX_STATUS_SHIFT +const _UVREQ_AUX_PENDING_SHIFT = 32 + +# The witness slot of a uv-request wait entry: the one whose owner is +# neither empty nor the cancellation source. Callbacks locate it without +# knowing the witness object. +function _uvreq_witness_slot(w::WaitEntry) + for (i, slot) in enumerate(slots(w)) + o = slot.owner + (o === nothing || o isa CancellationTokenSource) && continue + return i + end + return 0 +end + +function _mark_uvreq_cancel_requested!(w::WaitEntry, @nospecialize(witness)) + i = _find_slot(w, witness) + i == 0 || _set_slot_aux!(w, i, _slot_aux(w, i) | _UVREQ_AUX_CANCEL_REQUESTED) + return nothing +end + +# Whether the waiter flagged this request's cancellation as its own. Read +# by the completion callback (under the iolock) *before* the claim releases +# the witness slot. +function _uvreq_cancel_requested(req::Ptr{Cvoid}) + d = uv_req_data(req) + (d == C_NULL || d == UV_REQ_DETACHED) && return false + w = unsafe_pointer_to_objref(d)::WaitEntry + i = _uvreq_witness_slot(w) + return i != 0 && _slot_aux(w, i) & _UVREQ_AUX_CANCEL_REQUESTED != 0 +end + +# Buffers of detached write requests. A detached request keeps referencing +# the caller's memory until its completion callback runs - long after the +# issuing frame, whose GC.@preserve was the buffer's only root, has unwound. +# When the write's Julia owner is known it is rooted here, keyed by the +# request, and released by the completion callback. Guarded by the iolock +# (callbacks run under it). Raw-pointer writes (the generic unsafe_write +# interface) have no discoverable owner - their contract is pointer validity +# for the duration of the call - and detaching them retains the pre-existing +# hazard; owner-carrying entry points avoid it. +const _detached_uvreq_roots = IdDict{Ptr{Cvoid}, Any}() + +function _root_detached_uvreq!(req::Ptr{Cvoid}, @nospecialize(owner)) + owner === nothing || (_detached_uvreq_roots[req] = owner) + return nothing +end +_unroot_detached_uvreq!(req::Ptr{Cvoid}) = + (isempty(_detached_uvreq_roots) || delete!(_detached_uvreq_roots, req); nothing) + +# The entry check of a cancellable uv operation, made under the iolock +# before the request is issued: when `src` is already cancelled, release the +# iolock and throw the request - before the operation has any side effects. +function _iolocked_checkcancel(src::Union{Nothing, CancellationTokenSource}) + if src !== nothing && iscancelled(src) + iolock_end() + checkcancel(src) + end + return nothing +end + +# Arm the wait for the issued uv request `req`: pick the wait entry (with a +# cancellation-source slot when governed by `src`), arm it, record `witness` +# (the waitee identity - the handle, or the request itself when there is +# none) as the entry's reuse gate, and point the request at the entry. +# Leaves the caller's iolock held, inside a sigatomic section. +function _begin_uvreq_wait!(src::Union{Nothing, CancellationTokenSource}, + @nospecialize(witness), req::Ptr{Cvoid}) + ct = current_task() + w = src === nothing ? _cached_wait_entry(ct) : _cancel_wait_entry(ct, src, 0x00) + _arm_wait(ct, w) + _set_wait_witness!(w, witness) + preserve_handle(ct) + sigatomic_begin() + uv_req_set_data(req, pointer_from_objref(w)) + return w +end + +# Wind down a uv request wait (caller holds the iolock, at the sigatomic +# level of the wait): resolve the request's ownership, release the wait +# witness, disarm the registration (a no-op when a claimer already did), +# and unwind the iolock, the sigatomic section, and the handle preservation. +# When the completion callback has already run (data == C_NULL) the request +# is ours to free; otherwise - the wait was interrupted or refused, or the +# task was resumed by an unexpected `schedule` - the request is detached for +# the callback to free, `uv_cancel`ed first when `trycancel` is set (so e.g. +# an abandoned write stops spamming the stream), with `owner` kept rooted +# until then. The sticky source registration needs no cleanup at all. +function _end_uvreq_wait!(w::WaitEntry, @nospecialize(witness), req::Ptr{Cvoid}, + trycancel::Bool, @nospecialize(owner)) + ct = current_task() + if uv_req_data(req) == C_NULL + Libc.free(req) + else + trycancel && ccall(:uv_cancel, Cint, (Ptr{Cvoid},), req) # ignore any errors + uv_req_set_data(req, UV_REQ_DETACHED) + _root_detached_uvreq!(req, owner) + end + _clear_wait_witness!(w, witness) + @atomicreplace ct.waiting_on w => nothing + # Drop a claimed-and-enqueued wake an interrupted teardown will never + # consume. The completion callback claims and schedules under the + # iolock this function holds, so the drop is deterministic here (cf. + # the interrupted cleanup in base/park.jl). + q = ct.queue + q === nothing || list_deletefirst!(q::StickyWorkqueue, ct) + iolock_end() + sigatomic_end() + unpreserve_handle(ct) + return nothing +end + +## An in-flight libuv request as a waitable (see base/park.jl): the +## held-resource kind. The caller issues the request under the iolock and +## keeps it held into `park!` - the completion callback runs under that +## same lock, so the recheck is vacuous - and the request's data pointer +## doubles as the completion/detach handshake: the dequeue resolves its +## ownership on every exit path (callback already ran => we free it; +## otherwise it is detached to the callback, `uv_cancel`ed when +## requested, with the write's buffer kept rooted until then). The +## dequeue also consumes the reacquired iolock/sigatomic bracket. +struct UvReqWait + req::Ptr{Cvoid} + witness::Any # the waitee identity (the handle, or the request) + owner::Any # buffer root for a detached request + trycancel::Bool +end + +function wait_enqueue!(x::UvReqWait, w::WaitEntry, first::Bool) + _set_wait_witness!(w, x.witness) + uv_req_set_data(x.req, pointer_from_objref(w)) + return true +end + +# Resolve the request's ownership and release the witness. On +# WAKE_VALUE/WAKE_FIRED the caller holds the iolock (the settle and the +# refusal both run under it); the cleanup why takes it itself. The +# iolock/sigatomic/handle-preservation choreography around the park is +# the site's own code (_wait_uvreq, _uv_write_wait). +function wait_dequeue!(x::UvReqWait, w::WaitEntry, why::UInt8) + selflock = why == WAKE_INTERRUPTED || why == WAKE_WITHDRAWN + selflock && iolock_begin() + req = x.req + if uv_req_data(req) == C_NULL + Libc.free(req) + else + x.trycancel && ccall(:uv_cancel, Cint, (Ptr{Cvoid},), req) # ignore any errors + uv_req_set_data(req, UV_REQ_DETACHED) + _root_detached_uvreq!(req, x.owner) + end + _clear_wait_witness!(w, x.witness) + selflock && iolock_end() + return nothing +end + +# Park on the completion of the issued uv request `req` and return the value +# its completion callback delivers. A cancellation of `src` interrupts the +# wait and throws the CancellationRequest - without parking at all when +# `src` got cancelled between the caller's entry check and the registration +# here; either way the request is left to its callback (see the dequeue +# above). The caller must have issued `req` under the still-held iolock - +# so the callback cannot have run yet - and gets it back with the iolock +# released. +function _wait_uvreq(src::Union{Nothing, CancellationTokenSource}, @nospecialize(witness), + req::Ptr{Cvoid}, trycancel::Bool, @nospecialize(owner)) + ct = current_task() + uvw = UvReqWait(req, witness, owner, trycancel) + if src === nothing + ws = (uvw,) + w = _cached_wait_entry(ct) + else + ws = (uvw, SourceWait(src, 0x00)) + w = _cancel_wait_entry(ct, src, 0x00) + end + preserve_handle(ct) + sigatomic_begin() + if !park!(ws, w, false) + # refused at the registration recheck: resolve the request under + # the still-held iolock, unwind the brackets, deliver + withdraw!(ws, w, WAKE_FIRED) + iolock_end() + sigatomic_end() + unpreserve_handle(ct) + checkcancel(src) + error("park fired without a cancelled source") + end + iolock_end() + sigatomic_end() + local r + try + r = wait_safe_interrupt(ws, w) + catch + # (the catch restored the sigatomic level from the try entry; the + # cleanup's dequeue resolved the request under its own iolock) + unpreserve_handle(ct) + rethrow() + end + sigatomic_begin() + iolock_begin() + withdraw!(ws, w, WAKE_VALUE) # resolve the request under the iolock + iolock_end() + sigatomic_end() + unpreserve_handle(ct) + return r +end + +# Completion-callback side of the wait (runs under the iolock): resolve +# request ownership and claim the parked waiter's wake. Returns the task to +# schedule, or `nothing` when there is nobody to wake - either no waiter +# remains (fire-and-forget, or the waiter departed and detached the request, +# which is freed here together with any recorded buffer root), or a +# canceller claimed the wake first (the waiter's teardown then observes +# data == C_NULL and takes over the request). +function _claim_uvreq_waiter(req::Ptr{Cvoid}) + d = uv_req_data(req) + if d == C_NULL || d == UV_REQ_DETACHED + _unroot_detached_uvreq!(req) + Libc.free(req) + return nothing + end + # mark the callback as done; the waiter (which inspects the request + # under the iolock) owns freeing it + uv_req_set_data(req, C_NULL) + w = unsafe_pointer_to_objref(d)::WaitEntry + t = @atomic :monotonic w.task + if t isa Task && claim_wait(t, w) + _clear_uv_witness!(w) + return t + end + return nothing +end + +function closewrite(s::LibuvStream; cancel::CancelTokenArg=DEFAULT_CANCEL) iolock_begin() if !iswritable(s) iolock_end() return end + src = cancel_source(resolve_cancel_token(cancel)) + # entry check: throw before issuing the shutdown request + _iolocked_checkcancel(src) req = Libc.malloc(_sizeof_uv_shutdown) uv_req_set_data(req, C_NULL) # in case we get interrupted before arriving at the wait call err = ccall(:uv_shutdown, Int32, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}), @@ -448,32 +733,9 @@ function closewrite(s::LibuvStream) Libc.free(req) uv_error("shutdown", err) end - ct = current_task() - preserve_handle(ct) - sigatomic_begin() - uv_req_set_data(req, ct) - iolock_end() - local status - try - sigatomic_end() - status = wait()::Cint - sigatomic_begin() - finally - # try-finally unwinds the sigatomic level, so need to repeat sigatomic_end - sigatomic_end() - iolock_begin() - q = ct.queue; q === nothing || Base.list_deletefirst!(q::IntrusiveLinkedList{Task}, ct) - if uv_req_data(req) != C_NULL - # req is still alive, - # so make sure we won't get spurious notifications later - uv_req_set_data(req, C_NULL) - else - # done with req - Libc.free(req) - end - iolock_end() - unpreserve_handle(ct) - end + # A shutdown request itself cannot be cancelled, so an interrupted wait + # detaches it (its completion callback frees it). + status = _wait_uvreq(src, s, req, false, nothing)::Cint if isopen(s) if status < 0 || ccall(:uv_is_readable, Cint, (Ptr{Cvoid},), s.handle) == 0 close(s) @@ -488,12 +750,18 @@ end function wait_close(x::Union{LibuvStream, LibuvServer}) preserve_handle(x) lock(x.cond) + locked = true try while isopen(x) - wait(x.cond) + # close is the cleanup primitive: its completion wait is shielded + # from cancellation (completion depends only on the event loop, + # not on any peer, so this wait is bounded) + locked = false + wait(x.cond, nothing) + locked = true end finally - unlock(x.cond) + locked && unlock(x.cond) unpreserve_handle(x) end nothing @@ -569,7 +837,7 @@ displaysize(io::IO) = displaysize() displaysize() = (parse(Int, get(ENV, "LINES", "24")), parse(Int, get(ENV, "COLUMNS", "80")))::Tuple{Int, Int} -# This is a fancy way to make de-specialize a call to `displaysize(io::IO)` +# This is a fancy way to de-specialize a call to `displaysize(io::IO)` # which is unfortunately invalidated by REPL # (https://github.com/JuliaLang/julia/issues/56080) # @@ -617,7 +885,8 @@ end function alloc_request(buffer::IOBuffer, recommended_size::UInt) ensureroom(buffer, recommended_size) ptr = buffer.append ? buffer.size + 1 : buffer.ptr - nb = min(length(buffer.data), buffer.maxsize + get_offset(buffer)) - ptr + 1 + start_offset = ptr - 1 + nb = max(0, min(length(buffer.data) - start_offset, buffer.maxsize - (start_offset - get_offset(buffer)))) return (Ptr{Cvoid}(pointer(buffer.data, ptr)), nb) end @@ -892,7 +1161,7 @@ end if Sys.iswindows() # the low performance version of stop_reading is required - # on Windows due to a NT kernel bug that we can't use a blocking + # on Windows due to an NT kernel bug that we can't use a blocking # stream for non-blocking (overlapped) calls, # and a ReadFile call blocking on one thread # causes all other operations on that stream to lockup @@ -919,25 +1188,26 @@ end # bulk read / write readbytes!(s::LibuvStream, a::Vector{UInt8}, nb = length(a)) = readbytes!(s, a, Int(nb)) -function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) +function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) iolock_begin() sbuf = s.buffer - @assert sbuf.seekable == false - @assert sbuf.maxsize >= nb + @assert sbuf.seekable == false "buffer should not be seekable" + @assert sbuf.maxsize >= nb "insufficient buffer size" - function wait_locked(s, buf, nb) + function wait_locked(s, buf, nb, tok) while bytesavailable(buf) < nb s.readerror === nothing || throw(s.readerror) isopen(s) || break s.status != StatusEOF || break iolock_end() - wait_readnb(s, nb) + wait_readnb(s, nb, tok) iolock_begin() end end if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer - wait_locked(s, sbuf, nb) + wait_locked(s, sbuf, nb, tok) end if bytesavailable(sbuf) >= nb nread = readbytes!(sbuf, a, nb) @@ -947,7 +1217,7 @@ function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) nread = try s.buffer = newbuf write(newbuf, sbuf) - wait_locked(s, newbuf, nb) + wait_locked(s, newbuf, nb, tok) bytesavailable(newbuf) finally s.buffer = sbuf @@ -959,33 +1229,34 @@ function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) return nread end -function read(stream::LibuvStream) - wait_readnb(stream, typemax(Int)) +function read(stream::LibuvStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + wait_readnb(stream, typemax(Int), resolve_cancel_token(precheck_cancel_arg(cancel))) iolock_begin() bytes = take!(stream.buffer) iolock_end() return bytes end -function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) +function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) iolock_begin() sbuf = s.buffer - @assert sbuf.seekable == false - @assert sbuf.maxsize >= nb + @assert sbuf.seekable == false "buffer should not be seekable" + @assert sbuf.maxsize >= nb "insufficient buffer size" - function wait_locked(s, buf, nb) + function wait_locked(s, buf, nb, tok) while bytesavailable(buf) < nb s.readerror === nothing || throw(s.readerror) isopen(s) || throw(EOFError()) s.status != StatusEOF || throw(EOFError()) iolock_end() - wait_readnb(s, nb) + wait_readnb(s, nb, tok) iolock_begin() end end if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer - wait_locked(s, sbuf, Int(nb)) + wait_locked(s, sbuf, Int(nb), tok) end if bytesavailable(sbuf) >= nb unsafe_read(sbuf, p, nb) @@ -994,7 +1265,7 @@ function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) try s.buffer = newbuf write(newbuf, sbuf) - wait_locked(s, newbuf, Int(nb)) + wait_locked(s, newbuf, Int(nb), tok) finally s.buffer = sbuf end @@ -1003,13 +1274,14 @@ function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) nothing end -function read(this::LibuvStream, ::Type{UInt8}) +function read(this::LibuvStream, ::Type{UInt8}; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) iolock_begin() sbuf = this.buffer - @assert sbuf.seekable == false + @assert sbuf.seekable == false "buffer should not be seekable" while bytesavailable(sbuf) < 1 iolock_end() - eof(this) && throw(EOFError()) + eof(this; cancel) && throw(EOFError()) iolock_begin() end c = read(sbuf, UInt8) @@ -1017,25 +1289,27 @@ function read(this::LibuvStream, ::Type{UInt8}) return c end -function readavailable(this::LibuvStream) - wait_readnb(this, 1) # unlike the other `read` family of functions, this one doesn't guarantee error reporting +function readavailable(this::LibuvStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + wait_readnb(this, 1, resolve_cancel_token(precheck_cancel_arg(cancel))) # unlike the other `read` family of functions, this one doesn't guarantee error reporting iolock_begin() buf = this.buffer - @assert buf.seekable == false + @assert buf.seekable == false "buffer should not be seekable" bytes = take!(buf) iolock_end() return bytes end -function copyuntil(out::IO, x::LibuvStream, c::UInt8; keep::Bool=false) +function copyuntil(out::IO, x::LibuvStream, c::UInt8; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) iolock_begin() buf = x.buffer - @assert buf.seekable == false + @assert buf.seekable == false "buffer should not be seekable" if !occursin(c, buf) # fast path checks first x.readerror === nothing || throw(x.readerror) if isopen(x) && x.status != StatusEOF preserve_handle(x) lock(x.cond) + locked = true try while !occursin(c, x.buffer) x.readerror === nothing || throw(x.readerror) @@ -1043,12 +1317,18 @@ function copyuntil(out::IO, x::LibuvStream, c::UInt8; keep::Bool=false) x.status != StatusEOF || break start_reading(x) # ensure we are reading iolock_end() - wait(x.cond) + locked = false + wait(x.cond, tok) + locked = true unlock(x.cond) + locked = false iolock_begin() lock(x.cond) + locked = true end finally + # see wait_readnb's teardown note + locked || lock(x.cond) if isempty(x.cond) stop_reading(x) # stop reading iff there are currently no other read clients of the stream end @@ -1057,60 +1337,311 @@ function copyuntil(out::IO, x::LibuvStream, c::UInt8; keep::Bool=false) end end end - copyuntil(out, buf, c; keep) + # thread the resolved token (or explicit shield): the in-memory copy's + # write to `out` is itself token-gated + copyuntil(out, buf, c; keep, cancel=tok) iolock_end() return out end -uv_write(s::LibuvStream, p::Vector{UInt8}) = GC.@preserve p uv_write(s, pointer(p), UInt(sizeof(p))) - -# caller must have acquired the iolock -function uv_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt) - uvw = uv_write_async(s, p, n) - ct = current_task() - preserve_handle(ct) - sigatomic_begin() - uv_req_set_data(uvw, ct) +uv_write(s::LibuvStream, p::Vector{UInt8}, cancel::CancelTokenArg=DEFAULT_CANCEL, + partial::Bool=false) = + GC.@preserve p _uv_write_owned(s, pointer(p), UInt(sizeof(p)), cancel, p, partial) + +# Issue the write and wait for its completion, delivering a cancellation of +# `tok` by interrupting the wait. A cancelled wait first resolves the +# in-flight request per the severity (SAFE awaits the completion callback, +# so the buffer is provably out of OS hands); then, with `partial` unset, +# the CancellationRequest is thrown - bytes already accepted stay written - +# and with `partial` set the count of accepted bytes is returned and the +# (level-triggered) cancellation is left to the caller's next cancellation +# point. The caller must have acquired the iolock, which is released before +# returning. +function _uv_write_wait(s::LibuvStream, p::Ptr{UInt8}, n::UInt, + tok::MaybeToken, @nospecialize(owner), partial::Bool, + chunk::UInt=MAX_OS_WRITE) + src = cancel_source(tok) + # entry check: throw before handing anything to libuv + _iolocked_checkcancel(src) + local uvw, lastn, others + try + uvw, lastn, others = uv_write_async(s, p, n, chunk) + catch + # E.g. the stream was (or gets) closed: release the iolock so that + # errors propagate without it. + iolock_end() + rethrow() + end + w = _begin_uvreq_wait!(src, s, uvw) + if others !== nothing + # A split write: point every chunk request at the wait entry and + # store the pending-callback count on the witness slot's aux (all + # under the iolock hold that covers submission, so no callback can + # have run yet). Completions count down; only the last claims and + # wakes. The waiter owns - and, once all callbacks have run, sums + # and frees - every chunk request, which is what makes the total + # accepted-byte count exact even when a cancellation sweep + # completes the requests out of order. + wp = pointer_from_objref(w) + for r in others + uv_req_set_data(r, wp) + end + i = _find_slot(w, s) + _set_slot_aux!(w, i, _slot_aux(w, i) | + (UInt64(length(others) + 1) << _UVREQ_AUX_PENDING_SHIFT)) + end + refused = false + if src !== nothing + sw = SourceWait(src, 0x00) + wait_enqueue!(sw, w, false) + # a lost self-claim means a concurrent walk claimed the freshly + # armed entry: its wake delivers the request into the park below + refused = wait_recheck(sw, w) && disarm!(current_task(), w) + end + if refused + # The token was cancelled since the entry check and the recheck's + # self-claim won the wake back: don't park. Resolve the in-flight + # write and report the bytes that reached the OS; the caller + # observes the cancellation at its next cancellation point. + iolock_end() + nwritten = _uv_write_cancelled_finish(s, w, uvw, + severity(cancel_severity(src)::CancellationRequest), src, owner, others) + # level-triggered: the throw below (and any later cancellation + # point) reads the current severity + partial || checkcancel(src) + return others === nothing ? Int(n - lastn + nwritten) : Int(nwritten) + end iolock_end() - local status + local nwritten::Csize_t + creq = nothing try sigatomic_end() - # wait for the last chunk to complete (or error) - # assume that any errors would be sticky, - # (so we don't need to monitor the error status of the intermediate writes) - status = wait()::Cint + # wait for the write to complete (or error): the last request's + # callback for a single-request write, the pending count reaching + # zero for a split one (whose wake also carries the first real + # error of any chunk) + nwritten = wait()::Csize_t sigatomic_begin() - finally - # try-finally unwinds the sigatomic level, so need to repeat sigatomic_end - sigatomic_end() + catch err + # (catch restored the sigatomic level from the try entry) + if err isa CancellationRequest + # Cancellation is an expected outcome of a write, not an error: + # resolve the request's ownership below and return the partial + # count - the caller's next cancellation point (re)delivers the + # request. (The delivery is level-triggered, so nothing is + # lost by not propagating this throw.) + creq = err + else + # interrupted by something other than a cancellation (an + # interrupter or raw throwto): cancel whatever is still + # cancellable and detach the rest to the callbacks + iolock_begin() + if others === nothing + _end_uvreq_wait!(w, s, uvw, true, owner) + else + _end_split_uvreq_wait!(w, s, uvw, others, true, owner) + end + rethrow() + end + end + if creq !== nothing + nwritten = _uv_write_cancelled_finish(s, w, uvw, severity(creq), src, owner, others) + # the in-flight request is resolved (SAFE: the buffer is provably + # out of OS hands); deliver the cancellation now unless the caller + # asked for the partial count + partial || checkcancel(src::CancellationTokenSource) + else + # normal completion (or the wake lost the race to a completion that + # ran anyway): the callback(s) handed us the request(s) to free iolock_begin() - q = ct.queue; q === nothing || Base.list_deletefirst!(q::IntrusiveLinkedList{Task}, ct) - if uv_req_data(uvw) != C_NULL - # uvw is still alive, - # so make sure we won't get spurious notifications later - uv_req_set_data(uvw, C_NULL) + if others === nothing + _end_uvreq_wait!(w, s, uvw, true, owner) else - # done with uvw - Libc.free(uvw) + nwritten = _end_split_uvreq_wait!(w, s, uvw, others, false, owner) end - iolock_end() - unpreserve_handle(ct) end - if status < 0 - throw(_UVError("write", status)) + return others === nothing ? Int(n - lastn + nwritten) : Int(nwritten) +end + +# Resolve the ownership of a cancelled write's uv request(s) and return the +# partial write count: the last chunk's for a single-request write, the +# exact total of accepted bytes for a split one (0 for whatever had to be +# detached without awaiting completion). Every in-flight request is +# `uv_cancel`ed, tail-first for a split write: our chunks sit contiguously +# in the stream's queue, so cancelling from the tail dequeues behind the +# still-active head and the wire always keeps a clean prefix. For SAFE +# cancellations this awaits the completion callback(s), so the caller's +# buffer is provably no longer in use by the OS when this returns; only a +# severity escalation interrupts that bounded wait. Enters with the iolock +# released at the sigatomic level of the interrupted wait; unwinds both and +# the handle preservation. +function _uv_write_cancelled_finish(s::LibuvStream, w::WaitEntry, uvw::Ptr{Cvoid}, + sev::UInt8, src::CancellationTokenSource, + @nospecialize(owner), + others::Union{Nothing, Vector{Ptr{Cvoid}}}) + ct = current_task() + nwritten::Csize_t = 0 + awaited = false + iolock_begin() + if others === nothing ? uv_req_data(uvw) == C_NULL : _split_complete(uvw, others) + # the completion callback(s) already ran (the wake lost its claim + # race): the requests are ours and know their counts + awaited = true + if others === nothing + nwritten = ccall(:uv_write_nwritten, Csize_t, (Ptr{Cvoid},), uvw) + end + else + # The write is still in flight: flag it as cancelled-by-us first + # (on the witness slot's aux, under the iolock), so the completion + # callback(s) can tell our UV_ECANCELED from a close-induced one - + # see uv_writecb_task - then sweep, tail-first. + _mark_uvreq_cancel_requested!(w, s) + uv_req_data(uvw) == C_NULL || + ccall(:uv_cancel, Cint, (Ptr{Cvoid},), uvw) # ignore any errors + if others !== nothing + for j in lastindex(others):-1:firstindex(others) + r = others[j] + uv_req_data(r) == C_NULL || + ccall(:uv_cancel, Cint, (Ptr{Cvoid},), r) # ignore any errors + end + end + if sev < severity(CANCEL_REQUEST_ABANDON_EXTERNAL) + # SAFE cancellation: await the completion callback, so that the + # caller's buffer is provably no longer in use when this + # returns. Only a severity escalation may interrupt this bounded + # teardown wait: stage the raised eligibility floor, then re-arm + # the (still registered) entry - the re-park needs no new + # registration. + slots(w)[_find_slot(w, src)].aux = UInt64(sev + 0x01) + sw = SourceWait(src, UInt8(sev + 0x01)) + _arm_wait(ct, w) + wait_enqueue!(sw, w, false) # sticky: the seq_cst re-arm fence + if !(wait_recheck(sw, w) && disarm!(ct, w)) + iolock_end() + try + nwritten = wait()::Csize_t + awaited = true + catch + # escalated (or interrupted) during the teardown wait: + # fall through to detach + end + iolock_begin() + if !awaited && + (others === nothing ? uv_req_data(uvw) == C_NULL : + _split_complete(uvw, others)) + # the callback(s) ran anyway (our wake lost a race) + awaited = true + if others === nothing + nwritten = ccall(:uv_write_nwritten, Csize_t, (Ptr{Cvoid},), uvw) + end + end + end + # (a refused registration means the state already escalated) + end end - return Int(n) + # Resolve ownership. Awaited or not, completed requests are ours (a + # split write sums their exact counts); anything still in flight - an + # abandoning severity, or an escalation during the teardown wait - is + # detached for its callback to free, with the written buffer kept + # rooted until every detached callback has run; those requests' counts + # are unknown and contribute 0. (No re-cancel: the sweep above already + # cancelled everything that was in flight.) + if others === nothing + _end_uvreq_wait!(w, s, uvw, false, owner) + else + nwritten = _end_split_uvreq_wait!(w, s, uvw, others, false, owner) + end + return nwritten +end + +# Whether every request of a split write has completed (data == C_NULL). +function _split_complete(uvw::Ptr{Cvoid}, others::Vector{Ptr{Cvoid}}) + uv_req_data(uvw) == C_NULL || return false + for r in others + uv_req_data(r) == C_NULL || return false + end + return true end -# helper function for uv_write that returns the uv_write_t struct for the write -# rather than waiting on it, caller must hold the iolock -function uv_write_async(s::LibuvStream, p::Ptr{UInt8}, n::UInt) +# Split-write counterpart of _end_uvreq_wait! (caller holds the iolock, at +# the sigatomic level of the wait): resolve the ownership of every chunk +# request and return the total of the completed ones' accepted-byte counts. +# Completed requests (data == C_NULL) are summed and freed; in-flight ones +# are detached for their callbacks to free - `uv_cancel`ed first, tail-first +# (last request first, then earlier chunks in reverse submission order, so +# the queue keeps a clean prefix), when `trycancel` is set - with `owner` +# kept rooted under each until its callback runs. Then unwinds the witness, +# the registration, the iolock, the sigatomic section, and the handle +# preservation, exactly like _end_uvreq_wait!. +function _end_split_uvreq_wait!(w::WaitEntry, @nospecialize(witness), uvw::Ptr{Cvoid}, + others::Vector{Ptr{Cvoid}}, trycancel::Bool, + @nospecialize(owner)) + ct = current_task() + nwritten::Csize_t = 0 + for j in (lastindex(others) + 1):-1:firstindex(others) + req = j > lastindex(others) ? uvw : others[j] + if uv_req_data(req) == C_NULL + nwritten += ccall(:uv_write_nwritten, Csize_t, (Ptr{Cvoid},), req) + Libc.free(req) + else + trycancel && ccall(:uv_cancel, Cint, (Ptr{Cvoid},), req) # ignore any errors + uv_req_set_data(req, UV_REQ_DETACHED) + _root_detached_uvreq!(req, owner) + end + end + _clear_wait_witness!(w, witness) + @atomicreplace ct.waiting_on w => nothing + # Drop a claimed-and-enqueued wake an interrupted teardown will never + # consume (cf. _end_uvreq_wait!). + q = ct.queue + q === nothing || list_deletefirst!(q::StickyWorkqueue, ct) + iolock_end() + sigatomic_end() + unpreserve_handle(ct) + return nwritten +end + +function uv_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL, + owner=nothing) + return _uv_write_owned(s, p, n, cancel, owner, false) +end + +# Positional core of uv_write: an `owner::Any` keyword call is not statically +# resolvable (the NamedTuple type is abstract), which breaks trimmed builds. +function _uv_write_owned(s::LibuvStream, p::Ptr{UInt8}, n::UInt, cancel::CancelTokenArg, + @nospecialize(owner), partial::Bool) + tok = resolve_cancel_token(cancel) + # branch on the token explicitly: a Union-typed `tok` alongside the + # deliberately unspecialized `owner` leaves the callee unresolvable + # for trimmed builds + nb = tok === nothing ? _uv_write_wait(s, p, n, nothing, owner, partial) : + _uv_write_wait(s, p, n, tok, owner, partial) + # With `partial` set, nb < n means the wait was cancelled: the write + # reports the count of accepted bytes (like a short write) and the + # cancellation itself is delivered at the caller's next cancellation + # point - level-triggered, nothing is lost by returning normally here. + # Without it, a cancelled wait has already thrown. + return nb +end + +# helper function for uv_write that submits the write as one request per +# OS-sized chunk rather than waiting, caller must hold the iolock. Returns +# the last chunk's uv_write_t, that chunk's size, and - for split writes - +# the vector of the earlier chunks' requests (nothing for the common +# single-request case). All requests go out with data C_NULL +# (fire-and-forget); a waiter that wants completion re-points them at its +# wait entry under the same iolock hold (see _uv_write_wait). +function uv_write_async(s::LibuvStream, p::Ptr{UInt8}, n::UInt, chunk::UInt=MAX_OS_WRITE) check_open(s) + others = nothing while true uvw = Libc.malloc(_sizeof_uv_write) uv_req_set_data(uvw, C_NULL) # in case we get interrupted before arriving at the wait call - nwrite = min(n, MAX_OS_WRITE) # split up the write into chunks the OS can handle. - # TODO: use writev instead of a loop + nwrite = min(n, chunk) # split up the write into chunks the OS can handle. + # TODO: use writev instead of a loop (requires libuv to bound the + # per-syscall submission: as of now it recombines all bufs of a + # request into one writev/WSASend, blowing through the OS limits + # this chunking exists to respect) err = ccall(:jl_uv_write, Int32, (Ptr{Cvoid}, Ptr{Cvoid}, UInt, Ptr{Cvoid}, Ptr{Cvoid}), @@ -1123,8 +1654,10 @@ function uv_write_async(s::LibuvStream, p::Ptr{UInt8}, n::UInt) n -= nwrite p += nwrite if n == 0 - return uvw + return uvw, nwrite, others end + others === nothing && (others = Ptr{Cvoid}[]) + push!(others::Vector{Ptr{Cvoid}}, uvw) end end @@ -1133,7 +1666,80 @@ end # - smaller writes are buffered, final uv write on flush or when buffer full # - large isbits arrays are unbuffered and written directly -function unsafe_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt) +function write(s::LibuvStream, a::Vector{UInt8}; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + # Like the generic unsafe_write fallback below, but carrying the buffer's + # owner so a detached (abandoned) write keeps it rooted; see + # _detached_uvreq_roots. + GC.@preserve a begin + return Int(_unsafe_write_owned(s, pointer(a), UInt(sizeof(a)), cancel, a, false)) + end +end + +function writepartial(s::LibuvStream, a::Vector{UInt8}; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + GC.@preserve a begin + return Int(_unsafe_write_owned(s, pointer(a), UInt(sizeof(a)), cancel, a, true)) + end +end + +# The public String path must be owner-carrying for the same reason: the +# generic method (strings/io.jl) preserves the string only until +# unsafe_write returns, but an abandoning cancellation can detach the +# in-flight request, which then references the string until its completion +# callback runs. +function write(s::LibuvStream, str::Union{String, SubString{String}}; + cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + GC.@preserve str begin + return Int(_unsafe_write_owned(s, pointer(str), UInt(sizeof(str)), cancel, str, false)) + end +end + +function writepartial(s::LibuvStream, str::Union{String, SubString{String}}; + cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + GC.@preserve str begin + return Int(_unsafe_write_owned(s, pointer(str), UInt(sizeof(str)), cancel, str, true)) + end +end + +# Owner-carrying counterpart of the generic `write(::IO, ::Array)` methods, +# again so a detached request keeps the array rooted (bits element types +# only; others error in the generic method anyway). +function write(s::LibuvStream, a::Array; cancel::CancelTokenArg=DEFAULT_CANCEL) + if isbitstype(eltype(a)) + cancel = precheck_cancel_arg(cancel) + GC.@preserve a begin + return Int(_unsafe_write_owned(s, Ptr{UInt8}(pointer(a)), UInt(sizeof(a)), cancel, a, false)) + end + end + return invoke(write, Tuple{IO, AbstractArray}, s, a; cancel) +end + +function writepartial(s::LibuvStream, a::Array; cancel::CancelTokenArg=DEFAULT_CANCEL) + if isbitstype(eltype(a)) + cancel = precheck_cancel_arg(cancel) + GC.@preserve a begin + return Int(_unsafe_write_owned(s, Ptr{UInt8}(pointer(a)), UInt(sizeof(a)), cancel, a, true)) + end + end + return invoke(write, Tuple{IO, AbstractArray}, s, a; cancel) +end + +# Raw-pointer writes have no discoverable owner: the caller's documented +# contract is pointer validity for the duration of the call - which an +# abandoning cancellation extends past the return, until the detached +# request's completion callback has run (see _detached_uvreq_roots). The +# owner-carrying entry points above (Vector{UInt8}, Array, String) do not +# have this hazard and are what the public `write` paths use. +function unsafe_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) + return _unsafe_write_owned(s, p, n, cancel, nothing, false) +end + +function _unsafe_write_owned(s::LibuvStream, p::Ptr{UInt8}, n::UInt, cancel::CancelTokenArg, + @nospecialize(owner), partial::Bool) while true # try to add to the send buffer iolock_begin() @@ -1148,23 +1754,72 @@ function unsafe_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt) bytesavailable(buf) == 0 && break # perform flush(s) arr = take!(buf) - uv_write(s, arr) + nb = uv_write(s, arr, cancel, true) + if nb < length(arr) + # a cancelled flush-write returns short: splice the unwritten + # tail back (ahead of concurrent appends) so no buffered byte + # is lost, then deliver - or, for the partial form, let the + # next iteration's entry check deliver (level-triggered) + _requeue_unwritten!(s, arr, nb) + if !partial + tok = resolve_cancel_token(cancel) + tok === nothing || checkcancel(tok.source) + end + end end # perform the output to the kernel - return uv_write(s, p, n) + return _uv_write_owned(s, p, n, cancel, owner, partial) end -function flush(s::LibuvStream) +# Splice the unwritten tail of a cancelled buffered-flush write back to the +# *front* of `s.sendbuf` - ahead of anything appended while the write was in +# flight, preserving stream order. `arr` was taken off the send buffer and +# only its first `nwritten` bytes reached the stream. +function _requeue_unwritten!(s::LibuvStream, arr::Vector{UInt8}, nwritten::Int) + nwritten < length(arr) || return nothing + iolock_begin() + buf = s.sendbuf + if buf !== nothing + appended = bytesavailable(buf) > 0 ? take!(buf) : nothing + write(buf, @view arr[nwritten+1:end]) + appended === nothing || write(buf, appended) + end + iolock_end() + return nothing +end + +# Cancellation contract of flush: a cancellation interrupts the wait and +# throws, but never silently discards data - buffered bytes that did not +# reach the stream are put back in the send buffer (a later flush retries +# them) and the partial state is left consistent. +function flush(s::LibuvStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + cancel = precheck_cancel_arg(cancel) iolock_begin() buf = s.sendbuf if buf !== nothing if bytesavailable(buf) > 0 arr = take!(buf) - uv_write(s, arr) + nb = uv_write(s, arr, cancel, true) + if nb < length(arr) + # cancelled short: requeue the unwritten tail rather than + # drop it - it stays buffered for a later flush - then + # deliver the cancellation + _requeue_unwritten!(s, arr, nb) + tok = resolve_cancel_token(cancel) + tok === nothing || checkcancel(tok.source) + end return end end - uv_write(s, Ptr{UInt8}(Base.eventloop()), UInt(0)) # zero write from a random pointer to flush current queue + # zero write from a random pointer to flush current queue, ignoring any + # errors from it: previously queued writes have already reported their + # errors to their writers, and the peer closing the stream after a + # completed exchange must not make flush throw + try + _uv_write_owned(s, Ptr{UInt8}(Base.eventloop()), UInt(0), cancel, nothing, false) + catch ex + ex isa IOError || rethrow() + end return end @@ -1189,32 +1844,71 @@ function write(s::LibuvStream, b::UInt8) end iolock_end() end - return write(s, Ref{UInt8}(b)) + # carry the Ref as the owner (rather than the generic ownerless + # `write(s, ::Ref)` path): an abandoning cancellation may detach the + # in-flight request, which must keep the byte's storage rooted + r = Ref{UInt8}(b) + GC.@preserve r begin + return Int(_unsafe_write_owned(s, unsafe_convert(Ptr{UInt8}, r), UInt(1), DEFAULT_CANCEL, r, false)) + end end function uv_writecb_task(req::Ptr{Cvoid}, status::Cint) + # A chunk of a waited *split* write counts down on the witness slot's + # aux instead of waking: the waiter owns (and later frees) every chunk + # request, so the callback only records completion (data = C_NULL) and + # any first real error; the callback that brings the count to 0 falls + # through to the ordinary claim-and-wake below. d = uv_req_data(req) - if d != C_NULL - uv_req_set_data(req, C_NULL) # let the Task know we got the writecb - t = unsafe_pointer_to_objref(d)::Task - schedule(t, status) - else - # no owner for this req, safe to just free it - Libc.free(req) + if d != C_NULL && d != UV_REQ_DETACHED + w = unsafe_pointer_to_objref(d)::WaitEntry + i = _uvreq_witness_slot(w) + if i != 0 + aux = _slot_aux(w, i) + pending = aux >> _UVREQ_AUX_PENDING_SHIFT + if pending > 1 + uv_req_set_data(req, C_NULL) + if status != 0 && + !(status == UV_ECANCELED && aux & _UVREQ_AUX_CANCEL_REQUESTED != 0) && + aux & _UVREQ_AUX_STATUS_MASK == 0 + aux |= (UInt64(-status) << _UVREQ_AUX_STATUS_SHIFT) & _UVREQ_AUX_STATUS_MASK + end + _set_slot_aux!(w, i, (aux & ~(typemax(UInt64) << _UVREQ_AUX_PENDING_SHIFT)) | + ((pending - 1) << _UVREQ_AUX_PENDING_SHIFT)) + return nothing + end + # pending == 1: final chunk of a split write - a real error + # recorded by an earlier chunk must win the wake even if this + # chunk itself completed cleanly (or with the expected + # UV_ECANCELED of a cancellation sweep) + if pending == 1 && aux & _UVREQ_AUX_STATUS_MASK != 0 + status = Cint(-((aux & _UVREQ_AUX_STATUS_MASK) >> _UVREQ_AUX_STATUS_SHIFT)) + end + end + end + # An expected-by-the-waiter UV_ECANCELED must be read off the entry + # before the claim below releases the witness slot. + cancel_requested = status == UV_ECANCELED && _uvreq_cancel_requested(req) + t = _claim_uvreq_waiter(req) + if t !== nothing + if status == 0 || cancel_requested + # For writes cancelled by the waiter, this is the partial write + # count. (For a split write the value is ignored: the waiter + # recomputes the total from the chunk requests it owns.) + schedule(t, ccall(:uv_write_nwritten, Csize_t, (Ptr{Cvoid},), req)) + else + # A real error - including a close-induced UV_ECANCELED (libuv + # fails still-queued writes with it when the stream closes), + # which must not masquerade as a benign short write. + schedule(t, _UVError("write", status); error=true) + end end nothing end function uv_shutdowncb_task(req::Ptr{Cvoid}, status::Cint) - d = uv_req_data(req) - if d != C_NULL - uv_req_set_data(req, C_NULL) # let the Task know we got the shutdowncb - t = unsafe_pointer_to_objref(d)::Task - schedule(t, status) - else - # no owner for this req, safe to just free it - Libc.free(req) - end + t = _claim_uvreq_waiter(req) + t === nothing || schedule(t, status) nothing end @@ -1225,7 +1919,7 @@ _fd(x::Union{OS_HANDLE, RawFD}) = x function _fd(x::Union{LibuvStream, LibuvServer}) fd = Ref{OS_HANDLE}(INVALID_OS_HANDLE) if x.status != StatusUninit && x.status != StatusClosed && x.handle != C_NULL - err = ccall(:uv_fileno, Int32, (Ptr{Cvoid}, Ptr{OS_HANDLE}), x.handle, fd) + ccall(:uv_fileno, Int32, (Ptr{Cvoid}, Ptr{OS_HANDLE}), x.handle, fd) # handle errors by returning INVALID_OS_HANDLE end return fd[] @@ -1249,7 +1943,15 @@ function _redirect_io_libc(stream, unix_fd::Int) -10 - unix_fd, Libc._get_osfhandle(posix_fd)) end end - dup(posix_fd, RawFD(unix_fd)) + GC.@preserve stream dup(posix_fd, RawFD(unix_fd)) + nothing +end +function _redirect_io_cglobal(handle::Union{LibuvStream, IOStream, Nothing}, unix_fd::Int) + c_sym = unix_fd == 0 ? cglobal(:jl_uv_stdin, Ptr{Cvoid}) : + unix_fd == 1 ? cglobal(:jl_uv_stdout, Ptr{Cvoid}) : + unix_fd == 2 ? cglobal(:jl_uv_stderr, Ptr{Cvoid}) : + C_NULL + c_sym == C_NULL || unsafe_store!(c_sym, handle === nothing ? Ptr{Cvoid}(unix_fd) : handle.handle) nothing end function _redirect_io_global(io, unix_fd::Int) @@ -1260,11 +1962,7 @@ function _redirect_io_global(io, unix_fd::Int) end function (f::RedirectStdStream)(handle::Union{LibuvStream, IOStream}) _redirect_io_libc(handle, f.unix_fd) - c_sym = f.unix_fd == 0 ? cglobal(:jl_uv_stdin, Ptr{Cvoid}) : - f.unix_fd == 1 ? cglobal(:jl_uv_stdout, Ptr{Cvoid}) : - f.unix_fd == 2 ? cglobal(:jl_uv_stderr, Ptr{Cvoid}) : - C_NULL - c_sym == C_NULL || unsafe_store!(c_sym, handle.handle) + _redirect_io_cglobal(handle, f.unix_fd) _redirect_io_global(handle, f.unix_fd) return handle end @@ -1273,6 +1971,7 @@ function (f::RedirectStdStream)(::DevNull) handle = open(nulldev, write=f.writable) _redirect_io_libc(handle, f.unix_fd) close(handle) # handle has been dup'ed in _redirect_io_libc + _redirect_io_cglobal(nothing, f.unix_fd) _redirect_io_global(devnull, f.unix_fd) return devnull end @@ -1519,7 +2218,15 @@ end isopen(s::BufferStream) = s.status != StatusClosed -closewrite(s::BufferStream) = close(s) +# BufferStream <: LibuvStream, so every keyword-accepting LibuvStream +# method a `cancel=` call would select must be re-specialized here: keyword +# calls dispatch only among keyword-accepting methods, and the LibuvStream +# ones touch fields (handle, sendbuf) a BufferStream does not have. +function closewrite(s::BufferStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + # closing is in-memory and immediate: entry gate only + precheck_cancel_arg(cancel) + close(s) +end function close(s::BufferStream) lock(s.cond) do @@ -1531,16 +2238,18 @@ end uvfinalize(s::BufferStream) = nothing setup_stdio(stream::BufferStream, child_readable::Bool) = invoke(setup_stdio, Tuple{IO, Bool}, stream, child_readable) -function read(s::BufferStream, ::Type{UInt8}) +function read(s::BufferStream, ::Type{UInt8}; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) nread = lock(s.cond) do - wait_readnb(s, 1) + wait_readnb(s, 1, tok) read(s.buffer, UInt8) end return nread end -function unsafe_read(s::BufferStream, a::Ptr{UInt8}, nb::UInt) +function unsafe_read(s::BufferStream, a::Ptr{UInt8}, nb::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) lock(s.cond) do - wait_readnb(s, Int(nb)) + wait_readnb(s, Int(nb), tok) unsafe_read(s.buffer, a, nb) nothing end @@ -1550,49 +2259,59 @@ bytesavailable(s::BufferStream) = bytesavailable(s.buffer) isreadable(s::BufferStream) = (isopen(s) || bytesavailable(s) > 0) && s.buffer.readable iswritable(s::BufferStream) = isopen(s) && s.buffer.writable -function wait_readnb(s::BufferStream, nb::Int) - lock(s.cond) do +function wait_readnb(s::BufferStream, nb::Int, tok::MaybeToken=default_cancel_token()) + lock(s.cond) + locked = true + try while isopen(s) && bytesavailable(s.buffer) < nb - wait(s.cond) + locked = false + wait(s.cond, tok) + locked = true end + finally + locked && unlock(s.cond) end + nothing end -function readavailable(this::BufferStream) +function readavailable(this::BufferStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) bytes = lock(this.cond) do - wait_readnb(this, 1) + wait_readnb(this, 1, tok) buf = this.buffer - @assert buf.seekable == false + @assert buf.seekable == false "buffer should not be seekable" take!(buf) end return bytes end -function read(stream::BufferStream) +function read(stream::BufferStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) bytes = lock(stream.cond) do - wait_close(stream) + wait_close(stream, tok) take!(stream.buffer) end return bytes end -function readbytes!(s::BufferStream, a::Vector{UInt8}, nb::Int) +function readbytes!(s::BufferStream, a::Vector{UInt8}, nb::Int; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) sbuf = s.buffer - @assert sbuf.seekable == false - @assert sbuf.maxsize >= nb + @assert sbuf.seekable == false "buffer should not be seekable" + @assert sbuf.maxsize >= nb "insufficient buffer size" - function wait_locked(s, buf, nb) + function wait_locked(s, buf, nb, tok) while bytesavailable(buf) < nb s.readerror === nothing || throw(s.readerror) isopen(s) || break s.status != StatusEOF || break - wait_readnb(s, nb) + wait_readnb(s, nb, tok) end end bytes = lock(s.cond) do if nb <= SZ_UNBUFFERED_IO # Under this limit we are OK with copying the array from the stream's buffer - wait_locked(s, sbuf, nb) + wait_locked(s, sbuf, nb, tok) end if bytesavailable(sbuf) >= nb nread = readbytes!(sbuf, a, nb) @@ -1602,7 +2321,7 @@ function readbytes!(s::BufferStream, a::Vector{UInt8}, nb::Int) nread = try s.buffer = newbuf write(newbuf, sbuf) - wait_locked(s, newbuf, nb) + wait_locked(s, newbuf, nb, tok) bytesavailable(newbuf) finally s.buffer = sbuf @@ -1617,29 +2336,74 @@ end show(io::IO, s::BufferStream) = print(io, "BufferStream(bytes waiting=", bytesavailable(s.buffer), ", isopen=", isopen(s), ")") -function readuntil(s::BufferStream, c::UInt8; keep::Bool=false) - bytes = lock(s.cond) do +function readuntil(s::BufferStream, c::UInt8; keep::Bool=false, cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) + lock(s.cond) + locked = true + bytes = try while isopen(s) && !occursin(c, s.buffer) - wait(s.cond) + # a cancelled wait unwinds with the buffer intact (nothing has + # been consumed until the delimiter is present) + locked = false + wait(s.cond, tok) + locked = true end readuntil(s.buffer, c, keep=keep) + finally + locked && unlock(s.cond) end return bytes end -function wait_close(s::BufferStream) - lock(s.cond) do +function wait_close(s::BufferStream, tok::MaybeToken=default_cancel_token()) + lock(s.cond) + locked = true + try while isopen(s) - wait(s.cond) + locked = false + wait(s.cond, tok) + locked = true end + finally + locked && unlock(s.cond) end + nothing end start_reading(s::BufferStream) = Int32(0) stop_reading(s::BufferStream) = nothing write(s::BufferStream, b::UInt8) = write(s, Ref{UInt8}(b)) -function unsafe_write(s::BufferStream, p::Ptr{UInt8}, nb::UInt) +# BufferStream counterparts of the owner-carrying LibuvStream write +# methods, which must not apply here (they track uv requests and the send +# buffer; a BufferStream write is in-memory, so no request can outlive the +# caller). These writes cannot block for long (the advisory lock's park +# runs under the ambient scope): an explicit token gates at entry only. +function write(s::BufferStream, a::Vector{UInt8}; cancel::CancelTokenArg=DEFAULT_CANCEL) + precheck_cancel_arg(cancel) + GC.@preserve a begin + return Int(unsafe_write(s, pointer(a), UInt(sizeof(a)))) + end +end +function write(s::BufferStream, str::Union{String, SubString{String}}; + cancel::CancelTokenArg=DEFAULT_CANCEL) + precheck_cancel_arg(cancel) + GC.@preserve str begin + return Int(unsafe_write(s, pointer(str), UInt(sizeof(str)))) + end +end +function write(s::BufferStream, a::Array; cancel::CancelTokenArg=DEFAULT_CANCEL) + precheck_cancel_arg(cancel) + if isbitstype(eltype(a)) + GC.@preserve a begin + return Int(unsafe_write(s, Ptr{UInt8}(pointer(a)), UInt(sizeof(a)))) + end + end + return invoke(write, Tuple{IO, AbstractArray}, s, a) +end +function unsafe_write(s::BufferStream, p::Ptr{UInt8}, nb::UInt; cancel::CancelTokenArg=DEFAULT_CANCEL) + # entry gate only (see the BufferStream `write` methods above) + precheck_cancel_arg(cancel) nwrite = lock(s.cond) do check_open(s) rv = unsafe_write(s.buffer, p, nb) @@ -1649,10 +2413,11 @@ function unsafe_write(s::BufferStream, p::Ptr{UInt8}, nb::UInt) return nwrite end -function eof(s::BufferStream) +function eof(s::BufferStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + tok = resolve_cancel_token(precheck_cancel_arg(cancel)) bytesavailable(s) > 0 && return false iseof = lock(s.cond) do - wait_readnb(s, 1) + wait_readnb(s, 1, tok) return !isopen(s) && bytesavailable(s) <= 0 end return iseof @@ -1660,7 +2425,9 @@ end # If buffer_writes is called, it will delay notifying waiters till a flush is called. buffer_writes(s::BufferStream, bufsize=0) = (s.buffer_writes = true; s) -function flush(s::BufferStream) +function flush(s::BufferStream; cancel::CancelTokenArg=DEFAULT_CANCEL) + # in-memory flush (a notify), nothing to wait for: entry gate only + precheck_cancel_arg(cancel) lock(s.cond) do check_open(s) notify(s.cond) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index cb2d09f5b7054..a9653dc3b3356 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -50,7 +50,7 @@ AnnotatedString(s::S<:AbstractString) -> AnnotatedString{S} AnnotatedString(s::S<:AbstractString, annotations::Vector{$RegionAnnotation}) ``` -A AnnotatedString can also be created with [`annotatedstring`](@ref), which acts much +An AnnotatedString can also be created with [`annotatedstring`](@ref), which acts much like [`string`](@ref) but preserves any annotations present in the arguments. # Examples @@ -75,8 +75,8 @@ More specifically, this is a simple wrapper around any other [`AbstractChar`](@ref), which holds a list of arbitrary labelled annotations (`$Annotation`) with the wrapped character. -See also: [`AnnotatedString`](@ref), [`annotatedstring`](@ref), `annotations`, -and `annotate!`. +See also [`AnnotatedString`](@ref), [`annotatedstring`](@ref), `annotations`, +`annotate!`. # Constructors @@ -158,6 +158,37 @@ eltype(::Type{<:AnnotatedString{S}}) where {S} = AnnotatedChar{eltype(S)} firstindex(s::AnnotatedString) = firstindex(s.string) lastindex(s::AnnotatedString) = lastindex(s.string) +""" + unannotate(s::AnnotatedString{S})::S + unannotate(s::SubString{AnnotatedString{S}})::SubString{S} + unannotate(s::SubString{AnnotatedString{SubString{S}}})::SubString{S} + +Get the underlying string of `s`, without copying. + +# Examples +```jldoctest; setup=:(using Base: AnnotatedString) +julia> s = AnnotatedString("abcde", [(1:3, :A, 4)]) +"abcde" + +julia> u = Base.unannotate(s) +"abcde" + +julia> typeof(u) +String +``` + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. + +""" +unannotate(s::AnnotatedString) = s.string + +function unannotate(s::SubString{<:AnnotatedString}) + start_index = first(parentindices(s)[1]) + @inbounds raw_substring(parent(s).string, start_index, ncodeunits(s)) +end + + function getindex(s::AnnotatedString, i::Integer) @boundscheck checkbounds(s, i) @inbounds if isvalid(s, i) @@ -204,16 +235,14 @@ cmp(a::AnnotatedString, b::AnnotatedString) = cmp(a.string, b.string) # To prevent substring equality from hitting the generic fallback function ==(a::SubString{<:AnnotatedString}, b::SubString{<:AnnotatedString}) - SubString(a.string.string, a.offset, a.ncodeunits, Val(:noshift)) == - SubString(b.string.string, b.offset, b.ncodeunits, Val(:noshift)) && - annotations(a) == annotations(b) + unannotate(a) == unannotate(b) && annotations(a) == annotations(b) end ==(a::SubString{<:AnnotatedString}, b::AnnotatedString) = - annotations(a) == annotations(b) && SubString(a.string.string, a.offset, a.ncodeunits, Val(:noshift)) == b.string + annotations(a) == annotations(b) && unannotate(a) == b.string ==(a::SubString{<:AnnotatedString}, b::AbstractString) = - isempty(annotations(a)) && SubString(a.string.string, a.offset, a.ncodeunits, Val(:noshift)) == b + isempty(annotations(a)) && unannotate(a) == b ==(a::AbstractString, b::SubString{<:AnnotatedString}) = b == a @@ -228,13 +257,13 @@ Create a `AnnotatedString` from any number of `values` using their This acts like [`string`](@ref), but takes care to preserve any annotations present (in the form of [`AnnotatedString`](@ref) or [`AnnotatedChar`](@ref) values). -See also [`AnnotatedString`](@ref) and [`AnnotatedChar`](@ref). +See also [`AnnotatedString`](@ref), [`AnnotatedChar`](@ref). ## Examples ```jldoctest; setup=:(using Base: AnnotatedString, annotatedstring) -julia> annotatedstring("now a AnnotatedString") -"now a AnnotatedString" +julia> annotatedstring("now an AnnotatedString") +"now an AnnotatedString" julia> annotatedstring(AnnotatedString("annotated", [(1:9, :label, 1)]), ", and unannotated") "annotated, and unannotated" @@ -250,7 +279,7 @@ function annotatedstring(xs...) size = filesize(s.io) if x isa AnnotatedString for annot in x.annotations - push!(annotations, setindex(annot, annot.region .+ size, :region)) + push!(annotations, @inline(setindex(annot, annot.region .+ size, :region))) end print(s, x.string) elseif x isa SubString{<:AnnotatedString} @@ -259,10 +288,10 @@ function annotatedstring(xs...) if start <= x.offset + x.ncodeunits && stop > x.offset rstart = size + max(0, start - x.offset - 1) + 1 rstop = size + min(stop, x.offset + x.ncodeunits) - x.offset - push!(annotations, setindex(annot, rstart:rstop, :region)) + push!(annotations, @inline(setindex(annot, rstart:rstop, :region))) end end - print(s, SubString(x.string.string, x.offset, x.ncodeunits, Val(:noshift))) + print(s, unannotate(x)) elseif x isa AnnotatedChar for annot in x.annotations push!(annotations, (region=1+size:1+size, annot...)) @@ -293,12 +322,12 @@ function repeat(str::AnnotatedString, r::Integer) elseif allequal(a -> a.region, str.annotations) && first(str.annotations).region == fullregion newfullregion = firstindex(unannot):lastindex(unannot) for annot in str.annotations - push!(annotations, setindex(annot, newfullregion, :region)) + push!(annotations, @inline(setindex(annot, newfullregion, :region))) end else for offset in 0:len:(r-1)*len for annot in str.annotations - push!(annotations, setindex(annot, annot.region .+ offset, :region)) + push!(annotations, @inline(setindex(annot, annot.region .+ offset, :region))) end end end @@ -318,10 +347,10 @@ function reverse(s::AnnotatedString) lastind = lastindex(s) AnnotatedString( reverse(s.string), - [setindex(annot, + [@inline(setindex(annot, UnitRange(1 + lastind - last(annot.region), 1 + lastind - first(annot.region)), - :region) + :region)) for annot in s.annotations]) end @@ -367,8 +396,8 @@ annotate!(s::SubString{<:AnnotatedString}, label::Symbol, @nospecialize(val::Any Annotate `char` with the labeled value `(label, value)`. """ -annotate!(c::AnnotatedChar, label::Symbol, @nospecialize(val::Any)) = - (push!(c.annotations, Annotation((; label, val))); c) +annotate!(c::AnnotatedChar, label::Symbol, @nospecialize(value::Any)) = + (push!(c.annotations, Annotation((; label, value))); c) """ annotations(str::Union{AnnotatedString, SubString{AnnotatedString}}, @@ -384,21 +413,33 @@ a vector of region–annotation tuples. In accordance with the semantics documented in [`AnnotatedString`](@ref), the order of annotations returned matches the order in which they were applied. -See also: [`annotate!`](@ref). +See also [`annotate!`](@ref). """ annotations(s::AnnotatedString) = s.annotations function annotations(s::SubString{<:AnnotatedString}) - RegionAnnotation[ - setindex(ann, first(ann.region)-s.offset:last(ann.region)-s.offset, :region) - for ann in annotations(s.string, s.offset+1:s.offset+s.ncodeunits)] + substr_range = s.offset+1:s.offset+s.ncodeunits + result = RegionAnnotation[] + for ann in annotations(s.string, substr_range) + # Shift the region to be relative to the substring start + shifted_region = first(ann.region)-s.offset:last(ann.region)-s.offset + # @inline setindex makes :region const knowable (#60365) + push!(result, @inline(setindex(ann, shifted_region, :region))) + end + return result end function annotations(s::AnnotatedString, pos::UnitRange{<:Integer}) # TODO optimise - RegionAnnotation[ - setindex(ann, max(first(pos), first(ann.region)):min(last(pos), last(ann.region)), :region) - for ann in s.annotations if !isempty(intersect(pos, ann.region))] + result = RegionAnnotation[] + for ann in s.annotations + if !isempty(intersect(pos, ann.region)) + clamped_region = max(first(pos), first(ann.region)):min(last(pos), last(ann.region)) + # @inline setindex makes :region const knowable (#60365) + push!(result, @inline(setindex(ann, clamped_region, :region))) + end + end + return result end annotations(s::AnnotatedString, pos::Integer) = annotations(s, pos:pos) @@ -430,7 +471,7 @@ This works by comparing the number of code units of each character before and after transforming with `f`, recording and aggregating any differences, then applying them to the annotation regions. -Returns an `AnnotatedString{String}` (regardless of the original underling +Returns an `AnnotatedString{String}` (regardless of the original underlying string type of `str`). """ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothing) @@ -455,7 +496,7 @@ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothin start, stop = first(annot.region), last(annot.region) start_offset = last(offsets[findlast(<=(start) ∘ first, offsets)::Int]) stop_offset = last(offsets[findlast(<=(stop) ∘ first, offsets)::Int]) - push!(annots, setindex(annot, (start + start_offset):(stop + stop_offset), :region)) + push!(annots, @inline(setindex(annot, (start + start_offset):(stop + stop_offset), :region))) end AnnotatedString(takestring!(outstr), annots) end @@ -509,7 +550,7 @@ function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s): pos = first(events).pos if pos > first(subregion) push!(regions, thisind(s, first(subregion)):prevind(s, pos)) - push!(annots, []) + push!(annots, Annotation[]) end activelist = Int[] for event in events @@ -526,7 +567,7 @@ function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s): end if last(events).pos < nextind(s, last(subregion)) push!(regions, last(events).pos:thisind(s, last(subregion))) - push!(annots, []) + push!(annots, Annotation[]) end RegionIterator(s.string, regions, annots) end diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 9698fd5909b68..7ad2d9a279a11 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -42,8 +42,9 @@ end write(io::AnnotatedIOBuffer, c::AnnotatedChar) = write(io, AnnotatedString(string(c), [(region=1:ncodeunits(c), a...) for a in c.annotations])) -write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x) +write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x)::Int write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, s) +write(io::AnnotatedIOBuffer, s::StringViewAndSub) = write(io.io, s)::Int write(io::AnnotatedIOBuffer, b::UInt8) = write(io.io, b) function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) @@ -52,7 +53,7 @@ function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) srcpos = position(src) nb = write(dest.io, src.io) isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) - srcannots = [setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region) + srcannots = [@inline(setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region)) for annot in src.annotations if first(annot.region) >= srcpos] _insert_annotations!(dest, srcannots, destpos - srcpos) nb @@ -78,10 +79,11 @@ function write(io::AbstractPipe, c::AnnotatedChar) end function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} - if (start = position(io)) == 0 + start = position(io) + if start == 0 AnnotatedString(read(io.io, T), copy(io.annotations)) else - annots = [setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region) + annots = [@inline(setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region)) for annot in io.annotations if last(annot.region) > start] AnnotatedString(read(io.io, T), annots) end @@ -101,7 +103,7 @@ read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char function truncate(io::AnnotatedIOBuffer, size::Integer) truncate(io.io, size) filter!(ann -> first(ann.region) <= size, io.annotations) - map!(ann -> setindex(ann, first(ann.region):min(size, last(ann.region)), :region), + map!(ann -> @inline(setindex(ann, first(ann.region):min(size, last(ann.region)), :region)), io.annotations, io.annotations) io end @@ -125,17 +127,17 @@ function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, sp # Test for partial overlap if first(region) <= first(span) <= last(region) || first(region) <= last(span) <= last(region) annotations[i] = - setindex(annot, + @inline(setindex(annot, if first(region) < first(span) first(region):first(span)-1 else last(span)+1:last(region) end, - :region) + :region)) # If `span` fits exactly within `region`, then we've only copied over # the beginning overhang, but also need to conserve the end overhang. if first(region) < first(span) && last(span) < last(region) - push!(extras, (i, setindex(annot, last(span)+1:last(region), :region))) + push!(extras, (i, @inline(setindex(annot, last(span)+1:last(region), :region)))) end end end @@ -151,7 +153,7 @@ end Register new `annotations` in `io`, applying an `offset` to their regions. -The largely consists of simply shifting the regions of `annotations` by `offset` +This largely consists of simply shifting the regions of `annotations` by `offset` and pushing them onto `io`'s annotations. However, when it is possible to merge the new annotations with recent annotations in accordance with the semantics outlined in [`AnnotatedString`](@ref), we do so. More specifically, when there @@ -163,43 +165,182 @@ This is implemented so that one can say write an `AnnotatedString` to an `AnnotatedIOBuffer` one character at a time without needlessly producing a new annotation for each character. """ -function _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{RegionAnnotation}, offset::Int = position(io)) - run = 0 - if !isempty(io.annotations) && last(last(io.annotations).region) == offset - for i in reverse(axes(annotations, 1)) - annot = annotations[i] - first(annot.region) == 1 || continue - i <= length(io.annotations) || continue - if annot.label == last(io.annotations).label && annot.value == last(io.annotations).value - valid_run = true - for runlen in 1:i - new = annotations[begin+runlen-1] - old = io.annotations[end-i+runlen] - if last(old.region) != offset || first(new.region) != 1 || old.label != new.label || old.value != new.value - valid_run = false - break - end - end - if valid_run - run = i - break - end +function _insert_annotations!(annots::Vector{RegionAnnotation}, newannots::Vector{RegionAnnotation}, offset::Int = 0) + run = @label search begin + if !isempty(annots) && last(last(annots).region) == offset + for i in reverse(axes(newannots, 1)) + annot = newannots[i] + first(annot.region) == 1 || continue + i <= length(annots) || continue + annot.label == last(annots).label || continue + annot.value == last(annots).value || continue + all(1:i) do runlen + new = newannots[begin+runlen-1] + old = annots[end-i+runlen] + !(last(old.region) != offset || + first(new.region) != 1 || + old.label != new.label || + old.value != new.value) + end || continue + break search i end end + 0 end for runindex in 0:run-1 - old_index = lastindex(io.annotations) - run + 1 + runindex - old = io.annotations[old_index] - new = annotations[begin+runindex] - io.annotations[old_index] = setindex(old, first(old.region):last(new.region)+offset, :region) + old_index = lastindex(annots) - run + 1 + runindex + old = annots[old_index] + new = newannots[begin+runindex] + extannot = (region = first(old.region):last(new.region)+offset, + label = old.label, + value = old.value) + annots[old_index] = extannot end - for index in run+1:lastindex(annotations) - annot = annotations[index] + for index in run+1:lastindex(newannots) + annot = newannots[index] start, stop = first(annot.region), last(annot.region) - push!(io.annotations, setindex(annotations[index], start+offset:stop+offset, :region)) + # REVIEW: For some reason, construction of `newannot` + # can be a significant contributor to the overall runtime + # of this function. For instance, executing: + # + # replace(AnnotatedIOBuffer(), S"apple", + # 'e' => S"{red:x}", 'p' => S"{green:y}") + # + # results in 3 calls to `_insert_annotations!`. It takes + # ~570ns in total, compared to ~200ns if we push `annot` + # instead of `newannot`. Commenting out the `_insert_annotations!` + # line reduces the runtime to ~170ns, from which we can infer + # that constructing `newannot` is somehow responsible for + # a ~30ns -> ~400ns (~13x) increase in runtime!! + # This also comes with a marginal increase in allocations + # (compared to the commented out version) of 2 -> 14 (250b -> 720b). + # + # This seems quite strange, but I haven't dug into the generated + # LLVM or ASM code. If anybody reading this is interested in checking + # this out, that would be brilliant 🙏. + # + # What I have done is found that "direct tuple reconstruction" + # (as below) is several times faster than using `setindex`. + newannot = (region = start+offset:stop+offset, + label = annot.label, + value = annot.value) + push!(annots, newannot) + end +end + +_insert_annotations!(io::AnnotatedIOBuffer, newannots::Vector{RegionAnnotation}, offset::Int = position(io)) = + _insert_annotations!(io.annotations, newannots, offset) + +# String replacement + +# REVIEW: For some reason the `Core.kwcall` indirection seems to cause a +# substantial slowdown here. If we remove `; count` from the signature +# and run the sample code above in `_insert_annotations!`, the runtime +# drops from ~4400ns to ~580ns (~7x faster). I cannot guess why this is. +function replace(out::AnnotatedIOBuffer, str::AnnotatedString, pat_f::Pair...; count = typemax(Int)) + if count == 0 || isempty(pat_f) + write(out, str) + return out + end + e1, patterns, replacers, repspans, notfound = _replace_init(str.string, pat_f, count) + if notfound + foreach(_free_pat_replacer, patterns) + write(out, str) + return out + end + # Modelled after `Base.annotated_chartransform`, but needing + # to handle a bit more complexity. + isappending = eof(out) + newannots = empty(out.annotations) + bytepos = bytestart = firstindex(str.string) + replacements = [(region = (bytestart - 1):(bytestart - 1), offset = position(out))] + nrep = 1 + while nrep <= count + repspans, ridx, xspan, newbytes, bytepos = @inline _replace_once( + out.io, str.string, bytestart, e1, patterns, replacers, repspans, count, nrep, bytepos) + first(xspan) >= e1 && break + nrep += 1 + # NOTE: When the replaced pattern ends with a multi-codeunit character, + # `xspan` only covers up to the start of that character. However, + # for us to correctly account for the changes to the string we need + # the /entire/ span of codeunits that were replaced. + if !isempty(xspan) && codeunit(str.string, last(xspan)) > 0x80 + xspan = first(xspan):nextind(str.string, last(xspan))-1 + end + drift = last(replacements).offset + thisrep = (region = xspan, offset = drift + newbytes - length(xspan)) + destoff = first(xspan) - 1 + drift + push!(replacements, thisrep) + replacement = replacers[ridx] + _isannotated(replacement) || continue + annots = annotations(replacement) + annots′ = if eltype(annots) == Annotation # When it's a char not a string + region = 1:newbytes + [@NamedTuple{region::UnitRange{Int}, label::Symbol, value}((region, label, value)) + for (; label, value) in annots] + else + annots + end::Vector{RegionAnnotation} + _insert_annotations!(newannots, annots′, destoff) + end + push!(replacements, (region = e1:(e1-1), offset = last(replacements).offset)) + foreach(_free_pat_replacer, patterns) + write(out.io, SubString(str.string, bytepos)) + # NOTE: To enable more efficient annotation clearing, + # we make use of the fact that `_replace_once` picks + # replacements ordered by their match start position. + # This means that the start of `.region`s in + # `replacements` is monotonically increasing. + isappending || _clear_annotations_in_region!(out.annotations, first(replacements).offset:position(out)) + for (; region, label, value) in str.annotations + start, stop = first(region), last(region) + prioridx = searchsortedlast( + replacements, (region = start:start, offset = 0), + by = r -> first(r.region)) + postidx = searchsortedfirst( + replacements, (region = stop:stop, offset = 0), + by = r -> first(r.region)) + priorrep, postrep = replacements[prioridx], replacements[postidx] + if prioridx == postidx && start >= first(priorrep.region) && stop <= last(priorrep.region) + # Region contained within a replacement + continue + elseif postidx - prioridx <= 1 && start > last(priorrep.region) && stop < first(postrep.region) + # Lies between replacements + shiftregion = (start + priorrep.offset):(stop + priorrep.offset) + shiftann = (region = shiftregion, label, value) + push!(out.annotations, shiftann) + else + # Split between replacements + prevrep = replacements[max(begin, prioridx - 1)] + for rep in @view replacements[max(begin, prioridx - 1):min(end, postidx + 1)] + gap = max(start, last(prevrep.region)+1):min(stop, first(rep.region)-1) + if !isempty(gap) + shiftregion = (first(gap) + prevrep.offset):(last(gap) + prevrep.offset) + shiftann = (; region = shiftregion, label, value) + push!(out.annotations, shiftann) + end + prevrep = rep + end + end end + append!(out.annotations, newannots) + out +end + +replace(out::IO, str::AnnotatedString, pat_f::Pair...; count=typemax(Int)) = + replace(out, str.string, pat_f...; count) + +function replace(str::AnnotatedString, pat_f::Pair...; count=typemax(Int)) + isempty(pat_f) || iszero(count) && return str + out = AnnotatedIOBuffer() + replace(out, str, pat_f...; count) + read(seekstart(out), AnnotatedString) end +# Printing + +function printstyled end + # NOTE: This is an interim solution to the invalidations caused # by the split styled display implementation. This should be # replaced by a more robust solution (such as a consolidation of @@ -227,7 +368,7 @@ function Base.write(io::IO, aio::AnnotatedIOBuffer) # This does introduce an overhead that technically # could be avoided, but I'm not sure that it's currently # worth the effort to implement an efficient version of - # writing from a AnnotatedIOBuffer with style. + # writing from an AnnotatedIOBuffer with style. # In the meantime, by converting to an `AnnotatedString` we can just # reuse all the work done to make that work. ansi_write_(write, io, read(aio, AnnotatedString))::Int @@ -250,6 +391,14 @@ Base.print(io::AnnotatedIOBuffer, s::Union{<:AnnotatedString, SubString{<:Annota Base.print(io::AnnotatedIOBuffer, c::AnnotatedChar) = (write(io, c); nothing) +styled_print(io::AnnotatedIOBuffer, msg::Any, kwargs::Any) = print(io, msg...) + +styled_print_(io::AnnotatedIOBuffer, @nospecialize(msg), @nospecialize(kwargs)) = + invoke_in_world(tls_world_age(), styled_print, io, msg, kwargs)::Nothing + +Base.printstyled(io::AnnotatedIOBuffer, msg...; kwargs...) = + styled_print_(io, msg, kwargs) + # Escape Base.escape_string(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}, diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 352d42eb3a40f..5d86046395ab6 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -105,7 +105,7 @@ UInt8 See also [`ncodeunits`](@ref), [`checkbounds`](@ref). """ @propagate_inbounds codeunit(s::AbstractString, i::Integer) = i isa Int ? - throw(MethodError(codeunit, (s, i))) : codeunit(s, Int(i)) + throw(MethodError(codeunit, (s, i))) : codeunit(s, Int(i)::Int) """ isvalid(s::AbstractString, i::Integer)::Bool @@ -141,7 +141,7 @@ Stacktrace: ``` """ @propagate_inbounds isvalid(s::AbstractString, i::Integer) = i isa Int ? - throw(MethodError(isvalid, (s, i))) : isvalid(s, Int(i)) + throw(MethodError(isvalid, (s, i))) : isvalid(s, Int(i)::Int) """ iterate(s::AbstractString, i::Integer)::Union{Tuple{<:AbstractChar, Int}, Nothing} @@ -154,7 +154,7 @@ of the iteration protocol may assume that `i` is the start of a character in `s` See also [`getindex`](@ref), [`checkbounds`](@ref). """ @propagate_inbounds iterate(s::AbstractString, i::Integer) = i isa Int ? - throw(MethodError(iterate, (s, i))) : iterate(s, Int(i)) + throw(MethodError(iterate, (s, i))) : iterate(s, Int(i)::Int) ## basic generic definitions ## @@ -221,7 +221,7 @@ Vector{UInt8}(s::AbstractString) = unsafe_wrap(Vector{UInt8}, String(s)) Array{UInt8}(s::AbstractString) = unsafe_wrap(Vector{UInt8}, String(s)) Vector{T}(s::AbstractString) where {T<:AbstractChar} = collect(T, s) -Symbol(s::AbstractString) = Symbol(String(s)) +Symbol(s::AbstractString) = Symbol(String(s)::String) Symbol(x...) = Symbol(string(x...)) convert(::Type{T}, s::T) where {T<:AbstractString} = s @@ -320,7 +320,7 @@ end ==(a::AbstractString, b::AbstractString)::Bool Test whether two strings are equal character by character (technically, Unicode -code point by code point). Should either string be a [`AnnotatedString`](@ref) the +code point by code point). Should either string be an [`AnnotatedString`](@ref) the string properties must match too. # Examples @@ -362,10 +362,6 @@ end isless(a::Symbol, b::Symbol) = cmp(a, b) < 0 -# hashing - -hash(s::AbstractString, h::UInt) = hash(String(s), h) - ## character index arithmetic ## """ @@ -412,7 +408,7 @@ function length(s::AbstractString, i::Int, j::Int) end @propagate_inbounds length(s::AbstractString, i::Integer, j::Integer) = - length(s, Int(i), Int(j)) + length(s, Int(i)::Int, Int(j)::Int) """ thisind(s::AbstractString, i::Integer)::Int @@ -446,7 +442,7 @@ ERROR: BoundsError: attempt to access 2-codeunit String at index [-1] [...] ``` """ -thisind(s::AbstractString, i::Integer) = thisind(s, Int(i)) +thisind(s::AbstractString, i::Integer) = thisind(s, Int(i)::Int) function thisind(s::AbstractString, i::Int) z = ncodeunits(s)::Int + 1 @@ -502,8 +498,8 @@ julia> prevind("α", 2, 3) -1 ``` """ -prevind(s::AbstractString, i::Integer, n::Integer) = prevind(s, Int(i), Int(n)) -prevind(s::AbstractString, i::Integer) = prevind(s, Int(i)) +prevind(s::AbstractString, i::Integer, n::Integer) = prevind(s, Int(i)::Int, Int(n)::Int) +prevind(s::AbstractString, i::Integer) = prevind(s, Int(i)::Int) prevind(s::AbstractString, i::Int) = prevind(s, i, 1) function prevind(s::AbstractString, i::Int, n::Int) @@ -561,8 +557,8 @@ julia> nextind("α", 1, 2) 4 ``` """ -nextind(s::AbstractString, i::Integer, n::Integer) = nextind(s, Int(i), Int(n)) -nextind(s::AbstractString, i::Integer) = nextind(s, Int(i)) +nextind(s::AbstractString, i::Integer, n::Integer) = nextind(s, Int(i)::Int, Int(n)::Int) +nextind(s::AbstractString, i::Integer) = nextind(s, Int(i)::Int, 1) nextind(s::AbstractString, i::Int) = nextind(s, i, 1) function nextind(s::AbstractString, i::Int, n::Int) @@ -631,7 +627,7 @@ isascii(c::AbstractChar) = UInt32(c) < 0x80 return 0 ≤ r < 0x80 end -#The chunking algorithm makes the last two chunks overlap inorder to keep the size fixed +#The chunking algorithm makes the last two chunks overlap in order to keep the size fixed @inline function _isascii_chunks(chunk_size,cu::AbstractVector{CU}, first,last) where {CU} n=first while n <= last - chunk_size @@ -647,6 +643,7 @@ Test whether all values in the vector belong to the ASCII character set (0x00 to This function is intended to be used by other string implementations that need a fast ASCII check. """ function isascii(cu::AbstractVector{CU}) where {CU <: Integer} + # Note that String and SubString{String} assume this is :nothrow :foldable chunk_size = 1024 chunk_threshold = chunk_size + (chunk_size ÷ 2) first = firstindex(cu); last = lastindex(cu) @@ -757,7 +754,7 @@ julia> repeat("ha", 3) "hahaha" ``` """ -repeat(s::AbstractString, r::Integer) = repeat(String(s), r) +repeat(s::AbstractString, r::Integer) = repeat(String(s)::String, r) """ ^(s::Union{AbstractString,AbstractChar}, n::Integer)::AbstractString @@ -791,14 +788,15 @@ struct CodeUnits{T,S<:AbstractString} <: DenseVector{T} CodeUnits(s::S) where {S<:AbstractString} = new{codeunit(s),S}(s) end -length(s::CodeUnits) = ncodeunits(s.s) sizeof(s::CodeUnits{T}) where {T} = ncodeunits(s.s) * sizeof(T) -size(s::CodeUnits) = (length(s),) +size(s::CodeUnits) = (ncodeunits(s.s),) elsize(s::Type{<:CodeUnits{T}}) where {T} = sizeof(T) @propagate_inbounds getindex(s::CodeUnits, i::Int) = codeunit(s.s, i) IndexStyle(::Type{<:CodeUnits}) = IndexLinear() checkbounds(::Type{Bool}, s::CodeUnits, i::Integer) = checkbounds(Bool, s.s, i) +dataids(s::CodeUnits) = dataids(s.s) + write(io::IO, s::CodeUnits) = write(io, s.s) diff --git a/base/strings/cstring.jl b/base/strings/cstring.jl index 3a377ab0e7b1e..f8c449cd77ac1 100644 --- a/base/strings/cstring.jl +++ b/base/strings/cstring.jl @@ -109,7 +109,7 @@ if ccall(:jl_get_UNAME, Any, ()) === :NT """ Base.cwstring(s) -Converts a string `s` to a NUL-terminated `Vector{Cwchar_t}`, suitable for passing to C +Convert a string `s` to a NUL-terminated `Vector{Cwchar_t}`, suitable for passing to C functions expecting a `Ptr{Cwchar_t}`. The main advantage of using this over the implicit conversion provided by [`Cwstring`](@ref) is if the function is called multiple times with the same argument. @@ -176,6 +176,10 @@ function transcode(::Type{UInt8}, src::Vector{<:Union{Int32,UInt32}}) end transcode(::Type{String}, src::String) = src transcode(T, src::String) = transcode(T, codeunits(src)) +# `String(::Vector{UInt8})` consumes (empties) its argument, so wrap a caller's +# own vector in a view: `String` then copies the bytes out without touching the +# source, and only that one copy is made. (#28612) +transcode(::Type{String}, src::Vector{UInt8}) = String(view(src, :)) transcode(::Type{String}, src) = String(transcode(UInt8, src)) function transcode(::Type{UInt16}, src::AbstractVector{UInt8}) @@ -209,7 +213,7 @@ function transcode(::Type{UInt16}, src::AbstractVector{UInt8}) elseif a == 0xf0 && b < 0x90 # overlong encoding push!(dst, xor(0x2080, UInt16(b) << 12, UInt16(c) << 6, d)) else # 4-byte UTF-8 - push!(dst, 0xe5b8 + (UInt16(a) << 8) + (UInt16(b) << 2) + (c >> 4), + push!(dst, 0xe5b8 +% (UInt16(a) << 8) +% (UInt16(b) << 2) +% UInt16(c >> 4), xor(0xdc80, UInt16(c & 0xf) << 6, d)) end else # too short @@ -277,7 +281,7 @@ function transcode(::Type{UInt8}, src::AbstractVector{UInt16}) b = src[i += 1] if (b & 0xfc00) == 0xdc00 # 2-unit UTF-16 sequence => 4-byte UTF-8 - a += 0x2840 + a +%= 0x2840 dst[j += 1] = 0xf0 | ((a >> 8) % UInt8) dst[j += 1] = 0x80 | ((a % UInt8) >> 2) dst[j += 1] = xor(0xf0, ((a % UInt8) << 4) & 0x3f, (b >> 6) % UInt8) @@ -301,14 +305,27 @@ function transcode(::Type{UInt8}, src::AbstractVector{UInt16}) return dst end +""" + unsafe_string(p::Ptr{T}, [length::Integer]) where {T<:Union{UInt16,UInt32,Cwchar_t}} + unsafe_string(p::Cwstring) + +Transcode a string from the address of a C-style (NUL-terminated) string encoded as UTF-16 +(`T=UInt16`), UTF-32 (`T=UInt32`), or the system-dependent `wchar_t` (`T=Cwchar_t` or `Cwstring`), +returning a `String` (UTF-8 encoding), similar to [`transcode`](@ref) but reading directly +from a pointer. (The pointer can be safely freed afterwards.) If `length` is specified +(the length of the data in encoding units), the string does not have to be NUL-terminated. + +This function is labeled "unsafe" because it will crash if `p` is not +a valid memory address to data of the requested length (or NUL-terminated data). +""" function unsafe_string(p::Ptr{T}, length::Integer) where {T<:Union{UInt16,UInt32,Cwchar_t}} transcode(String, unsafe_wrap(Array, p, length; own=false)) end -function unsafe_string(cw::Cwstring) - p = convert(Ptr{Cwchar_t}, cw) +function unsafe_string(p::Ptr{T}) where {T<:Union{UInt16,UInt32,Cwchar_t}} n = 1 while unsafe_load(p, n) != 0 n += 1 end return unsafe_string(p, n - 1) end +unsafe_string(cw::Cwstring) = unsafe_string(convert(Ptr{Cwchar_t}, cw)) diff --git a/base/strings/io.jl b/base/strings/io.jl index f3a0783e98a9b..bd6d9de9fc1af 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -42,8 +42,10 @@ end function print(io::IO, xs...) lock(io) try - for x in xs - print(io, x) + # xs[i] might be a known Union, and under --trim that gets split regardless of length. + # In contrast, `for x in xs` will fall back to Any for unions longer than 3. + for i in 1:nfields(xs) + print(io, xs[i]) end finally unlock(io) @@ -51,8 +53,6 @@ function print(io::IO, xs...) return nothing end -setfield!(typeof(print).name, :max_args, Int32(10), :monotonic) - """ println([io::IO], xs...) @@ -75,8 +75,6 @@ julia> takestring!(io) ``` """ println(io::IO, xs...) = print(io, xs..., "\n") - -setfield!(typeof(println).name, :max_args, Int32(10), :monotonic) ## conversion of general objects to strings ## """ @@ -143,12 +141,11 @@ function print_to_string(xs...) end # specialized for performance reasons s = IOBuffer(sizehint=siz) - for x in xs - print(s, x) + for i in 1:nfields(xs) + print(s, xs[i]) end takestring!(s) end -setfield!(typeof(print_to_string).name, :max_args, Int32(10), :monotonic) function string_with_env(env, xs...) if isempty(xs) @@ -161,8 +158,8 @@ function string_with_env(env, xs...) # specialized for performance reasons s = IOBuffer(sizehint=siz) env_io = IOContext(s, env) - for x in xs - print(env_io, x) + for i in 1:nfields(xs) + print(env_io, xs[i]) end takestring!(s) end @@ -178,7 +175,7 @@ highly efficient, then it may make sense to add a method to `string` and define `print(io::IO, x::MyType) = print(io, string(x))` to ensure the functions are consistent. -See also: [`String`](@ref), [`repr`](@ref), [`sprint`](@ref), [`show`](@ref @show). +See also [`String`](@ref), [`repr`](@ref), [`sprint`](@ref), [`show`](@ref @show). # Examples ```jldoctest @@ -239,8 +236,13 @@ function show( end # optimized methods to avoid iterating over chars -write(io::IO, s::Union{String,SubString{String}}) = - GC.@preserve s (unsafe_write(io, pointer(s), reinterpret(UInt, sizeof(s))) % Int)::Int +# (an explicit token is forwarded to unsafe_write, whose cancellable methods +# accept it; the default sentinel keeps the plain call, which any IO's +# unsafe_write method supports) +write(io::IO, s::Union{String,SubString{String}}; cancel::CancelTokenArg=DEFAULT_CANCEL) = + cancel === DEFAULT_CANCEL ? + GC.@preserve(s, (unsafe_write(io, pointer(s), reinterpret(UInt, sizeof(s))) % Int)::Int) : + GC.@preserve(s, (unsafe_write(io, pointer(s), reinterpret(UInt, sizeof(s)); cancel) % Int)::Int) print(io::IO, s::Union{String,SubString{String}}) = (write(io, s); nothing) """ @@ -477,8 +479,8 @@ end unescape_string(io, s::AbstractString, keep = ())::Nothing General unescaping of traditional C and Unicode escape sequences. The first form returns -the escaped string, the second prints the result to `io`. -The argument `keep` specifies a collection of characters which (along with backlashes) are +the unescaped string, the second prints the result to `io`. +The argument `keep` specifies a collection of characters which (along with backslashes) are to be kept as they are. The following escape sequences are recognised: @@ -596,9 +598,9 @@ end Create a raw string without interpolation and unescaping. The exception is that quotation marks still must be escaped. Backslashes escape both quotation marks and other backslashes, but only when a sequence -of backslashes precedes a quote character. Thus, 2n backslashes followed by -a quote encodes n backslashes and the end of the literal while 2n+1 backslashes -followed by a quote encodes n backslashes followed by a quote character. +of backslashes precedes a quote character. Thus, ``2n`` backslashes followed by +a quote encode ``n`` backslashes and the end of the literal while ``2n+1`` backslashes +followed by a quote encode ``n`` backslashes followed by a quote character. # Examples ```jldoctest @@ -735,7 +737,7 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) col = div(col + tabwidth, tabwidth) * tabwidth elseif ch == '\n' # Now we need to output enough indentation - for i = 1:col-indent + for _ = 1:col-indent print(buf, ' ') end col = 0 @@ -744,7 +746,7 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) cutting = false # Now we need to output enough indentation to get to # correct place - for i = 1:col-indent + for _ = 1:col-indent print(buf, ' ') end col += 1 @@ -754,7 +756,7 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) upd = div(col + tabwidth, tabwidth) * tabwidth # output the number of spaces that would have been seen # with original indentation - for i = 1:(upd-col) + for _ = 1:(upd-col) print(buf, ' ') end col = upd @@ -770,7 +772,7 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) # If we were still "cutting" when we hit the end of the string, # we need to output the right number of spaces for the indentation if cutting - for i = 1:col-indent + for _ = 1:col-indent print(buf, ' ') end end diff --git a/base/strings/lazy.jl b/base/strings/lazy.jl index eaaa6397d37f2..5512f0f3214e5 100644 --- a/base/strings/lazy.jl +++ b/base/strings/lazy.jl @@ -35,18 +35,22 @@ times, though only exactly one result will be returned. """ mutable struct LazyString <: AbstractString const parts::Tuple + const compact::Bool # compact & limited output # Created on first access @atomic str::Union{String,Nothing} - global _LazyString(parts, str) = new(parts, str) - LazyString(args...) = new(args, nothing) + global _LazyString(parts, str) = new(parts, false, str) + LazyString(args...; compact::Bool=false) = new(args, compact, nothing) end """ lazy"str" + lazy"str"c Create a [`LazyString`](@ref) using regular string interpolation syntax. Note that interpolations are *evaluated* at LazyString construction time, -but *printing* is delayed until the first access to the string. +but *printing* is delayed until the first access to the string. The `lazy"str"c` +variant uses an [`IOContext`](@ref) with `:compact=>true, :limit=>true` when +printing, in order to keep the output string compact. See [`LazyString`](@ref) documentation for the safety properties for concurrent programs. @@ -62,8 +66,13 @@ LazyString !!! compat "Julia 1.8" `lazy"str"` requires Julia 1.8 or later. + +!!! compat "Julia 1.14" + `lazy"str"c` requires Julia 1.14 or later. """ -macro lazy_str(text) +macro lazy_str(text, flags...) + compact = flags == ("c",) + isempty(flags) || compact || throw(ArgumentError("unknown lazy-string flag: $flags")) parts = Any[] lastidx = idx = 1 while (idx = findnext('$', text, idx)) !== nothing @@ -74,15 +83,17 @@ macro lazy_str(text) lastidx = idx end lastidx <= lastindex(text) && push!(parts, text[lastidx:end]) - :(LazyString($(parts...))) + :(LazyString($(parts...); compact=$compact)) end function String(l::LazyString) old = @atomic :acquire l.str old === nothing || return old str = sprint() do io - for p in l.parts - print(io, p) + if l.compact + foreach(Fix1(print, IOContext(io, :compact=>true, :limit=>true)), l.parts) + else + foreach(Fix1(print, io), l.parts) end end old, ok = @atomicreplace :acquire_release :acquire l.str nothing => str @@ -96,6 +107,7 @@ iterate(s::LazyString, i::Integer) = iterate(String(s), i) isequal(a::LazyString, b::LazyString) = isequal(String(a), String(b)) ==(a::LazyString, b::LazyString) = (String(a) == String(b)) ncodeunits(s::LazyString) = ncodeunits(String(s)) -codeunit(s::LazyString) = codeunit(String(s)) +codeunit(s::LazyString) = codeunit("") # returns UInt8 codeunit(s::LazyString, i::Integer) = codeunit(String(s), i) +codeunits(s::LazyString) = codeunits(String(s)) isvalid(s::LazyString, i::Integer) = isvalid(String(s), i) diff --git a/base/strings/search.jl b/base/strings/search.jl index a68b5f52b9f0a..55a5850103396 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -41,12 +41,9 @@ end # This holds even in the presence of invalid UTF8 is_standalone_byte(x::UInt8) = (x < 0x80) | (x > 0xf7) -last_byteindex(x::Union{String, SubString{String}}) = ncodeunits(x) -last_byteindex(x::DenseUInt8OrInt8) = lastindex(x) - # Internal type - lazy iterator over positions of char in string struct FwCharPosIter{S} - string::S # S is assumed to be either String or SubString{String} + string::S # S is assumed to be either String, StringView or SubString of those char::Char # Char searchers search for the last UTF8 byte, because this byte tends to # have the most variety in real texts, so any individual value is rarer. @@ -54,7 +51,7 @@ struct FwCharPosIter{S} last_char_byte::UInt8 end -function FwCharPosIter(s::Union{String, SubString{String}}, c::AbstractChar) +function FwCharPosIter(s::UTF8String, c::AbstractChar) char = Char(c)::Char byte = last_utf8_byte(char) FwCharPosIter{typeof(s)}(s, char, byte) @@ -70,13 +67,13 @@ function Base.iterate(s::FwCharPosIter, i::Int=1) # Hence, we can simply search for the occurrence of the byte itself. if is_standalone_byte(s.last_char_byte) i > scu && return nothing - i = _search(s.string, s.last_char_byte, i) + i = findnext(==(s.last_char_byte), codeunits(s.string), i) i === nothing ? nothing : (i, i + 1) else ncu = ncodeunits(s.char) while true i > scu && return nothing - i = _search(s.string, s.last_char_byte, i) + i = findnext(==(s.last_char_byte), codeunits(s.string), i) i === nothing && return nothing # Increment i before the continue to avoid infinite loop. # Since we search for the last byte in the char, the index has an offset. @@ -97,7 +94,7 @@ end # Internal type - lazy iterator over positions of char in string, in reverse order struct RvCharPosIter{S} - string::S # S is assumed to be either String or SubString{String} + string::S # S is assumed to be String, StringView or SubString of those char::Char last_char_byte::UInt8 end @@ -105,7 +102,7 @@ end IteratorSize(s::Type{<:Union{FwCharPosIter, RvCharPosIter}}) = SizeUnknown() eltype(::Type{<:Union{FwCharPosIter, RvCharPosIter}}) = Int -function RvCharPosIter(s::Union{String, SubString{String}}, c::AbstractChar) +function RvCharPosIter(s::UTF8String, c::AbstractChar) char = Char(c)::Char byte = last_utf8_byte(char) RvCharPosIter{typeof(s)}(s, char, byte) @@ -121,13 +118,13 @@ function Base.iterate(s::RvCharPosIter, i::Int=ncodeunits(s.string)) ncu = ncodeunits(s.char) if is_standalone_byte(s.last_char_byte) i < ncu && return nothing - i = _rsearch(s.string, s.last_char_byte, i) + i = findprev(==(s.last_char_byte), codeunits(s.string), i) i === nothing ? nothing : (i, i - 1) else i = min(ncodeunits(s.string), i + ncu - 1) while true i < ncu && return nothing - i = _rsearch(s.string, s.last_char_byte, i) + i = findprev(==(s.last_char_byte), codeunits(s.string), i) i === nothing && return nothing index = i - ncu + 1 i -= 1 @@ -145,7 +142,7 @@ end function findnext( pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:AbstractChar}, - s::Union{String, SubString{String}}, + s::UTF8String, i::Integer, ) # TODO: Redesign these strange rules for errors, see #54584 @@ -162,45 +159,56 @@ function findnext( i = Int(i)::Int isvalid(s, i) || string_index_err(s, i) return if is_standalone_byte(u) - _search(s, u, i) + findnext(==(u), codeunits(s), i) else try_next(FwCharPosIter(s, c, last_utf8_byte(c)), i) end end -function findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},UInt8}, a::DenseUInt8, i::Integer) - @boundscheck i < firstindex(a) && throw(BoundsError(a, i)) - i > lastindex(a) && return nothing - _search(a, pred.x, i) -end - function findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},Int8}, a::DenseInt8, i::Integer) - @boundscheck i < firstindex(a) && throw(BoundsError(a, i)) - i > lastindex(a) && return nothing - _search(a, pred.x, i) + i = Int(i)::Int + @boundscheck i < Int(firstindex(a))::Int && throw(BoundsError(a, i)) + i > Int(lastindex(a))::Int && return nothing + return memchr(a, pred.x, i) end -# iszero is special, in that the bitpattern for zero for Int8 and UInt8 is the same, -# so we can use memchr even if we search for an Int8 in an UInt8 array or vice versa -function findnext(::typeof(iszero), a::DenseUInt8OrInt8, i::Integer) - @boundscheck i < firstindex(a) && throw(BoundsError(a, i)) - i > lastindex(a) && return nothing - _search(a, zero(UInt8), i) +function findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},UInt8}, a::DenseUInt8, i::Integer) + i = Int(i)::Int + @boundscheck i < Int(firstindex(a))::Int && throw(BoundsError(a, i)) + i > Int(lastindex(a))::Int && return nothing + return memchr(a, pred.x, i) end -# This is essentially just a wrapper around memchr. i must be inbounds. -function _search(a::Union{String,SubString{String},DenseUInt8OrInt8}, b::Union{Int8,UInt8}, i::Integer = firstindex(a)) - fst = firstindex(a) +function memchr(a::DenseUInt8, b::UInt8, i::Int) + fst = Int(firstindex(a))::Int GC.@preserve a begin p = pointer(a) - q = ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p+i-fst, b, last_byteindex(a) - i + 1) + q = ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p+i-fst, b, lastindex(a) - i + 1) end return q == C_NULL ? nothing : (q-p+fst) % Int end +function memchr(a::DenseInt8, b::Int8, i::Int) + fst = Int(firstindex(a))::Int + GC.@preserve a begin + p = Ptr{UInt8}(pointer(a)) + q = ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p+i-fst, b, lastindex(a) - i + 1) + end + return q == C_NULL ? nothing : (q-p+fst) % Int +end + +# iszero is special, in that the bitpattern for zero for Int8 and UInt8 is the same, +# so we can use memchr even if we search for an Int8 in an UInt8 array or vice versa +function findnext(::typeof(iszero), a::DenseUInt8OrInt8, i::Integer) + i = Int(i)::Int + @boundscheck i < Int(firstindex(a))::Int && throw(BoundsError(a, i)) + i > Int(lastindex(a))::Int && return nothing + return memchr(a, zero(eltype(a)), i) +end + function findprev( pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:AbstractChar}, - s::Union{String, SubString{String}}, + s::UTF8String, i::Integer, ) # TODO: Redesign these strange rules for errors, see #54584 @@ -215,44 +223,55 @@ function findprev( u = (reinterpret(UInt32, c) >> 24) % UInt8 i = Int(i)::Int return if is_standalone_byte(u) - _rsearch(s, u, i) + findprev(==(u), codeunits(s), i) else try_next(RvCharPosIter(s, c, last_utf8_byte(c)), i) end end function findprev(pred::Fix2{<:Union{typeof(isequal),typeof(==)},Int8}, a::DenseInt8, i::Integer) - @boundscheck i > lastindex(a) && throw(BoundsError(a, i)) - i < firstindex(a) && return nothing - _rsearch(a, pred.x, i) + i = Int(i)::Int + @boundscheck i > Int(lastindex(a))::Int && throw(BoundsError(a, i)) + i < Int(firstindex(a))::Int && return nothing + memrchr(a, pred.x, i) end function findprev(pred::Fix2{<:Union{typeof(isequal),typeof(==)},UInt8}, a::DenseUInt8, i::Integer) - @boundscheck i > lastindex(a) && throw(BoundsError(a, i)) - i < firstindex(a) && return nothing - _rsearch(a, pred.x, i) + i = Int(i)::Int + @boundscheck i > Int(lastindex(a))::Int && throw(BoundsError(a, i)) + i < Int(firstindex(a))::Int && return nothing + memrchr(a, pred.x, i) end -# See comments above for findfirst(::typeof(iszero)) methods -function findprev(::typeof(iszero), a::DenseUInt8OrInt8, i::Integer) - @boundscheck i > lastindex(a) && throw(BoundsError(a, i)) - i < firstindex(a) && return nothing - _rsearch(a, zero(UInt8), i) +function memrchr(a::DenseUInt8, b::UInt8, i::Int) + fst = firstindex(a) + GC.@preserve a begin + p = pointer(a) + q = ccall(:memrchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, b, i-fst+1) + end + return q == C_NULL ? nothing : (q-p+fst) % Int end -# This is essentially just a wrapper around memrchr. i must be inbounds. -function _rsearch(a::Union{String,SubString{String},DenseUInt8OrInt8}, b::Union{Int8,UInt8}, i::Integer = last_byteindex(a)) +function memrchr(a::DenseInt8, b::Int8, i::Int) fst = firstindex(a) GC.@preserve a begin - p = pointer(a) + p = Ptr{UInt8}(pointer(a)) q = ccall(:memrchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, b, i-fst+1) end return q == C_NULL ? nothing : (q-p+fst) % Int end +# See comments above for findfirst(::typeof(iszero)) methods +function findprev(::typeof(iszero), a::DenseUInt8OrInt8, i::Integer) + i = Int(i)::Int + @boundscheck i > Int(lastindex(a))::Int && throw(BoundsError(a, i)) + i < Int(firstindex(a))::Int && return nothing + memrchr(a, zero(eltype(a)), i) +end + function findall( pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:AbstractChar}, - s::Union{String, SubString{String}}, + s::UTF8String, ) iter = FwCharPosIter(s, pred.x) return if is_standalone_byte(iter.last_char_byte) @@ -336,6 +355,13 @@ end in(c::AbstractChar, s::AbstractString) = (findfirst(isequal(c),s)!==nothing) +# nothrow+foldable: iteration over `String`/`SubString{String}` is total and +# `Char` equality is total. +@assume_effects :nothrow :foldable function in(c::Char, s::Union{String,SubString{String}}) + pos = @inline findnext(==(c), s, 1) + return pos !== nothing +end + function _searchindex(s::Union{AbstractString,DenseUInt8OrInt8}, t::Union{AbstractString,AbstractChar,Int8,UInt8}, i::Integer) @@ -363,10 +389,10 @@ function _search_bloom_mask(c) UInt64(1) << (c & 63) end -_nthbyte(s::Union{String, SubString{String}}, i) = codeunit(s, i) +_nthbyte(s::DenseUTF8String, i) = codeunit(s, i) _nthbyte(t::AbstractVector, index) = t[index + (firstindex(t)-1)] -function _searchindex(s::Union{String, SubString{String}}, t::Union{String, SubString{String}}, i::Integer) +function _searchindex(s::DenseUTF8String, t::DenseUTF8String, i::Integer) # Check for fast case of a single byte lastindex(t) == 1 && return something(findnext(isequal(t[1]), s, i), 0) _searchindex(codeunits(s), codeunits(t), i) @@ -418,7 +444,7 @@ function _searchindex(s::AbstractVector{<:Union{Int8,UInt8}}, # match found if j == n - 1 - # restore in case `s` is an OffSetArray + # restore in case `s` is an OffsetArray return i+firstindex(s) end @@ -535,7 +561,7 @@ Find the last occurrence of `pattern` in `string`. Equivalent to julia> findlast("o", "Hello to the world") 15:15 -julia> findfirst("Julia", "JuliaLang") +julia> findlast("Julia", "JuliaLang") 1:5 ``` """ @@ -676,7 +702,7 @@ function _rsearchindex(s::AbstractString, end end -function _rsearchindex(s::Union{String, SubString{String}}, t::Union{String, SubString{String}}, i::Integer) +function _rsearchindex(s::DenseUTF8String, t::DenseUTF8String, i::Integer) # Check for fast case of a single byte if lastindex(t) == 1 return something(findprev(isequal(t[1]), s, i), 0) diff --git a/base/strings/string.jl b/base/strings/string.jl index 53593e3936842..e8b6af707a08f 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -1,5 +1,53 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +""" + StringView{T <: AbstractVector{UInt8}} <: AbstractString + +An `AbstractString` representation of any `vector` of `UInt8` data, +interpreted as UTF-8 encoded Unicode. +Similar to `String`, the underlying data may be invalid UTF-8. + +`StringView(v::AbstractVector{UInt8})::StringView` does not make a copy of +or modify the `v`. Use `codeunits` to get `v` from the `StringView`. +After construction, `v` may be mutated, which will be reflected in +the resulting `StringView`. + +!!! compat "Julia 1.14" + The `StringView` type requires at least Julia 1.14. + +# Examples +```jldoctest +julia> arr = [0x61, 0xf0, 0x63, 0x64]; + +julia> s = StringView(arr) +"a\\xf0cd" + +julia> codeunits(s) === arr +true + +julia> arr[2] = Int('b'); s +"abcd" +``` +""" +struct StringView{T <: AbstractVector{UInt8}} <: AbstractString + data::T + + function StringView{T}(data::T) where {T <: AbstractVector{UInt8}} + # For now, StringViews code assumes one-based indexing + require_one_based_indexing(data) + + # Prevent someone constructing e.g. a `StringView{AbstractVector{UInt8}}`, + # the existence of which will complicate the implementation and provide + # no usability benefit. + if !isconcretetype(T) + throw(ArgumentError("StringView must be parameterized with a concrete type")) + end + + new{T}(data) + end +end + + """ StringIndexError(str, i) @@ -7,11 +55,11 @@ An error occurred when trying to access `str` at index `i` that is not valid. """ struct StringIndexError <: Exception string::AbstractString - index::Integer + index::Int end -@noinline string_index_err(s::AbstractString, i::Integer) = +@noinline string_index_err((@nospecialize s::AbstractString), i::Integer) = throw(StringIndexError(s, Int(i))) -function Base.showerror(io::IO, exc::StringIndexError) +function showerror(io::IO, exc::StringIndexError) s = exc.string print(io, "StringIndexError: ", "invalid index [$(exc.index)]") if firstindex(s) <= exc.index <= ncodeunits(s) @@ -90,7 +138,7 @@ function unsafe_takestring(m::Memory{UInt8}) end """ - takestring!(x) -> String + takestring!(x)::AbstractString Create a string from the content of `x`, emptying `x`. @@ -104,11 +152,15 @@ julia> s = takestring!(v) julia> isempty(v) true ``` + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. """ takestring!(v::Vector{UInt8}) = String(v) """ unsafe_string(p::Ptr{UInt8}, [length::Integer]) + unsafe_string(p::Cstring) Copy a string from the address of a C-style (NUL-terminated) string encoded as UTF-8. (The pointer can be safely freed afterwards.) If `length` is specified @@ -157,7 +209,7 @@ pointer(s::String, i::Integer) = pointer(s) + Int(i)::Int - 1 ncodeunits(s::String) = Core.sizeof(s) codeunit(s::String) = UInt8 -codeunit(s::String, i::Integer) = codeunit(s, Int(i)) +codeunit(s::String, i::Integer) = codeunit(s, Int(i)::Int) @assume_effects :foldable @inline function codeunit(s::String, i::Int) @boundscheck checkbounds(s, i) b = GC.@preserve s unsafe_load(pointer(s, i)) @@ -192,7 +244,11 @@ typemin(::String) = typemin(String) @propagate_inbounds thisind(s::String, i::Int) = _thisind_str(s, i) -# s should be String or SubString{String} +# nothrow: i == ncodeunits(s) always satisfies the bounds check inside _thisind_str +# (it short-circuits when i == 0, otherwise 1 ≤ i ≤ n). +@assume_effects :nothrow lastindex(s::String) = thisind(s, ncodeunits(s)::Int) + +# s should be String, StringView, or SubString{String} @inline function _thisind_str(s, i::Int) i == 0 && return 0 n = ncodeunits(s) @@ -200,7 +256,7 @@ typemin(::String) = typemin(String) @boundscheck between(i, 1, n) || throw(BoundsError(s, i)) @inbounds b = codeunit(s, i) (b & 0xc0 == 0x80) & (i-1 > 0) || return i - (@noinline function _thisind_continued(s, i, n) # mark the rest of the function as a slow-path + (@noinline function _thisind_continued(s, i) # mark the rest of the function as a slow-path local b @inbounds b = codeunit(s, i-1) between(b, 0b11000000, 0b11110111) && return i-1 @@ -211,7 +267,7 @@ typemin(::String) = typemin(String) @inbounds b = codeunit(s, i-3) between(b, 0b11110000, 0b11110111) && return i-3 return i - end)(s, i, n) + end)(s, i) end @propagate_inbounds nextind(s::String, i::Int) = _nextind_str(s, i) @@ -282,7 +338,7 @@ end as seen by all 1s in that column of table below 3 -> One valid continuation byte needed to return to state 0 4,5,6 -> Two valid continuation bytes needed to return to state 0 - 7,8,9 -> Three valids continuation bytes needed to return to state 0 + 7,8,9 -> Three valid continuation bytes needed to return to state 0 Current State 0̲ 1̲ 2̲ 3̲ 4̲ 5̲ 6̲ 7̲ 8̲ 9̲ @@ -304,7 +360,7 @@ end The shifts that represent each state were derived using the SMT solver Z3, to ensure when encoded into the rows the correct shift was a result. - Each character class row is encoding 10 states with shifts as defined above. By shifting the bitsof a row by + Each character class row is encoding 10 states with shifts as defined above. By shifting the bits of a row by the current state then masking the result with 0x11110 give the shift for the new state @@ -316,7 +372,6 @@ const _UTF8DFAState = UInt32 const _UTF8_DFA_TABLE = let # let block rather than function doesn't pollute base num_classes=12 num_states=10 - bit_per_state = 6 # These shifts were derived using a SMT solver state_shifts = [0, 4, 10, 14, 18, 24, 8, 20, 12, 26] @@ -450,7 +505,7 @@ is_valid_continuation(c) = c & 0xc0 == 0x80 ## required core functionality ## -@inline function iterate(s::String, i::Int=firstindex(s)) +@inline function iterate(s::Union{String, StringView}, i::Int=firstindex(s)) (i % UInt) - 1 < ncodeunits(s) || return nothing b = @inbounds codeunit(s, i) u = UInt32(b) << 24 @@ -460,28 +515,29 @@ end # duck-type s so that external UTF-8 string packages like StringViews can hook in function iterate_continued(s, i::Int, u::UInt32) - u < 0xc0000000 && (i += 1; @goto ret) - n = ncodeunits(s) - # first continuation byte - (i += 1) > n && @goto ret - @inbounds b = codeunit(s, i) - b & 0xc0 == 0x80 || @goto ret - u |= UInt32(b) << 16 - # second continuation byte - ((i += 1) > n) | (u < 0xe0000000) && @goto ret - @inbounds b = codeunit(s, i) - b & 0xc0 == 0x80 || @goto ret - u |= UInt32(b) << 8 - # third continuation byte - ((i += 1) > n) | (u < 0xf0000000) && @goto ret - @inbounds b = codeunit(s, i) - b & 0xc0 == 0x80 || @goto ret - u |= UInt32(b); i += 1 -@label ret + @label begin + u < 0xc0000000 && (i += 1; break) + n = ncodeunits(s) + # first continuation byte + (i += 1) > n && break + @inbounds b = codeunit(s, i) + b & 0xc0 == 0x80 || break + u |= UInt32(b) << 16 + # second continuation byte + ((i += 1) > n) | (u < 0xe0000000) && break + @inbounds b = codeunit(s, i) + b & 0xc0 == 0x80 || break + u |= UInt32(b) << 8 + # third continuation byte + ((i += 1) > n) | (u < 0xf0000000) && break + @inbounds b = codeunit(s, i) + b & 0xc0 == 0x80 || break + u |= UInt32(b); i += 1 + end return reinterpret(Char, u), i end -@propagate_inbounds function getindex(s::String, i::Int) +@propagate_inbounds function getindex(s::Union{String, StringView}, i::Int) b = codeunit(s, i) u = UInt32(b) << 24 between(b, 0x80, 0xf7) || return reinterpret(Char, u) @@ -490,32 +546,36 @@ end # duck-type s so that external UTF-8 string packages like StringViews can hook in function getindex_continued(s, i::Int, u::UInt32) - if u < 0xc0000000 - # called from `getindex` which checks bounds - @inbounds isvalid(s, i) && @goto ret - string_index_err(s, i) + @label begin + if u < 0xc0000000 + # called from `getindex` which checks bounds + @inbounds isvalid(s, i) && break + string_index_err(s, i) + end + n = ncodeunits(s) + + (i += 1) > n && break + @inbounds b = codeunit(s, i) # cont byte 1 + b & 0xc0 == 0x80 || break + u |= UInt32(b) << 16 + + ((i += 1) > n) | (u < 0xe0000000) && break + @inbounds b = codeunit(s, i) # cont byte 2 + b & 0xc0 == 0x80 || break + u |= UInt32(b) << 8 + + ((i += 1) > n) | (u < 0xf0000000) && break + @inbounds b = codeunit(s, i) # cont byte 3 + b & 0xc0 == 0x80 || break + u |= UInt32(b) end - n = ncodeunits(s) - - (i += 1) > n && @goto ret - @inbounds b = codeunit(s, i) # cont byte 1 - b & 0xc0 == 0x80 || @goto ret - u |= UInt32(b) << 16 - - ((i += 1) > n) | (u < 0xe0000000) && @goto ret - @inbounds b = codeunit(s, i) # cont byte 2 - b & 0xc0 == 0x80 || @goto ret - u |= UInt32(b) << 8 - - ((i += 1) > n) | (u < 0xf0000000) && @goto ret - @inbounds b = codeunit(s, i) # cont byte 3 - b & 0xc0 == 0x80 || @goto ret - u |= UInt32(b) -@label ret return reinterpret(Char, u) end -getindex(s::String, r::AbstractUnitRange{<:Integer}) = s[Int(first(r)):Int(last(r))] +function getindex(s::Union{String, StringView}, r::AbstractUnitRange{<:Integer}) + span = (Int(first(r))::Int):(Int(last(r)))::Int + return s[span] +end @inline function getindex(s::String, r::UnitRange{Int}) isempty(r) && return "" @@ -525,18 +585,34 @@ getindex(s::String, r::AbstractUnitRange{<:Integer}) = s[Int(first(r)):Int(last( @inbounds isvalid(s, i) || string_index_err(s, i) @inbounds isvalid(s, j) || string_index_err(s, j) end - j = nextind(s, j) - 1 - n = j - i + 1 + # Safety: The boundscheck checked r is inbounds in s, + # and since we also checked r is not empty, j must be inbounds in s + j = @inbounds nextind(s, j) - 1 + n = (j - i + 1) % UInt ss = _string_n(n) GC.@preserve s ss unsafe_copyto!(pointer(ss), pointer(s, i), n) return ss end # nothrow because we know the start and end indices are valid -@assume_effects :nothrow length(s::String) = length_continued(s, 1, ncodeunits(s), ncodeunits(s)) +@assume_effects :nothrow function length(s::String) + return length_continued(s, 1, ncodeunits(s), ncodeunits(s)) +end + +function length(s::StringView) + return length_continued(s, 1, ncodeunits(s), ncodeunits(s)) +end # effects needed because @inbounds @assume_effects :consistent :effect_free @inline function length(s::String, i::Int, j::Int) + _length(s, i, j) +end + +@inline function length(s::StringView, i::Int, j::Int) + _length(s, i, j) +end + +@inline function _length(s::Union{String, StringView}, i::Int, j::Int) @boundscheck begin 0 < i ≤ ncodeunits(s)+1 || throw(BoundsError(s, i)) 0 ≤ j < ncodeunits(s)+1 || throw(BoundsError(s, j)) @@ -547,7 +623,16 @@ end @inbounds length_continued(s, i, j, c) end -@assume_effects :terminates_locally @inline @propagate_inbounds function length_continued(s::String, i::Int, n::Int, c::Int) +@assume_effects :terminates_globally @propagate_inbounds function length_continued(s::String, i::Int, n::Int, c::Int) + _length_continued(s, i, n, c) +end + +@propagate_inbounds function length_continued(s::StringView, i::Int, n::Int, c::Int) + _length_continued(s, i, n, c) +end + + +@propagate_inbounds function _length_continued(s::Union{String, StringView}, i::Int, n::Int, c::Int) i < n || return c b = codeunit(s, i) while true @@ -576,7 +661,8 @@ end isvalid(s::String, i::Int) = checkbounds(Bool, s, i) && thisind(s, i) == i -isascii(s::String) = isascii(codeunits(s)) +# `isascii(::AbstractVector)` reduces to `@inbounds codeunit(::String, ::Int)`, total. +isascii(s::String) = @assume_effects :nothrow :foldable isascii(codeunits(s)) # don't assume effects for general integers since we cannot know their implementation @assume_effects :foldable repeat(c::Char, r::BitInteger) = @invoke repeat(c::Char, r::Integer) @@ -600,6 +686,7 @@ function repeat(c::AbstractChar, r::Integer) r == 0 && return "" u = bswap(reinterpret(UInt32, c)) n = 4 - (leading_zeros(u | 0xff) >> 3) + r > typemax(UInt) ÷ UInt(n) && throw(OutOfMemoryError()) s = _string_n(n*r) p = pointer(s) GC.@preserve s if n == 1 diff --git a/base/strings/stringview.jl b/base/strings/stringview.jl new file mode 100644 index 0000000000000..0dc77037d36f1 --- /dev/null +++ b/base/strings/stringview.jl @@ -0,0 +1,155 @@ +const DenseStringView = StringView{<:Union{DenseVector{UInt8}, <:FastContiguousSubArray{UInt8, 1, <:DenseVector{UInt8}}}} +const StringAndSub = Union{String, SubString{String}} +const StringViewAndSub = Union{StringView, SubString{<:StringView}} +const DenseStringViewAndSub = Union{DenseStringView, SubString{<:DenseStringView}} +const DenseUTF8String = Union{DenseStringViewAndSub, StringAndSub} +const UTF8String = Union{StringAndSub, StringViewAndSub} + +StringView(v::AbstractVector{UInt8}) = StringView{typeof(v)}(v) +Vector{UInt8}(s::StringViewAndSub) = Vector{UInt8}(codeunits(s)) +Array{UInt8}(s::StringViewAndSub) = Vector{UInt8}(s) +String(s::StringViewAndSub) = String(copyto!(StringVector(ncodeunits(s)), codeunits(s))) +copy(s::StringView) = StringView(copy(s.data)) + +function Symbol(s::DenseStringViewAndSub) + return ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int), s, ncodeunits(s)) +end + +pointer(s::DenseStringView) = pointer(s.data) +pointer(s::DenseStringView, i::Integer) = pointer(s.data, i) +pointer(x::SubString{<:DenseStringView}) = pointer(x.string) + x.offset +pointer(x::SubString{<:DenseStringView}, i::Integer) = pointer(x.string) + x.offset + (i - 1) + +unsafe_convert(::Type{Ptr{UInt8}}, s::DenseStringViewAndSub) = pointer(s) +unsafe_convert(::Type{Ptr{Int8}}, s::DenseStringViewAndSub) = convert(Ptr{Int8}, pointer(s)) + +String(s::DenseStringViewAndSub) = GC.@preserve s unsafe_string(pointer(s), ncodeunits(s)) + +cconvert(::Type{Ptr{UInt8}}, s::DenseStringViewAndSub) = s +cconvert(::Type{Ptr{Int8}}, s::DenseStringViewAndSub) = s + +""" + reverse(s::AbstractString)::AbstractString + +Reverses a string. Technically, this function reverses the codepoints in a string and its +main utility is for reversed-order string processing, especially for reversed +regular-expression searches. See also [`reverseind`](@ref) to convert indices in `s` to +indices in `reverse(s)` and vice-versa, and `graphemes` from module `Unicode` to +operate on user-visible "characters" (graphemes) rather than codepoints. +See also [`Iterators.reverse`](@ref) for +reverse-order iteration without making a copy. Custom string types must implement the +`reverse` function themselves and should typically return a string with the same normalization +and encoding to ensure that `reverseind` works; if they return a string with a different encoding, +they must also override `reverseind` for that string type to satisfy `s[reverseind(s,i)] == reverse(s)[i]`. + +# Examples +```jldoctest +julia> reverse("JuliaLang") +"gnaLailuJ" +``` + +!!! note + The examples below may be rendered differently on different systems. + The comments indicate how they're supposed to be rendered + +Combining characters can lead to surprising results: + +```jldoctest +julia> reverse("ax̂e") # hat is above x in the input, above e in the output +"êxa" + +julia> using Unicode + +julia> join(reverse(collect(graphemes("ax̂e")))) # reverses graphemes; hat is above x in both in- and output +"ex̂a" +``` +""" +function reverse(s::UTF8String)::String + # Read characters forwards from `s` and write backwards to `out` + out = _string_n(sizeof(s)) + offs = sizeof(s) + 1 + for c in s + offs -= ncodeunits(c) + if s isa StringViewAndSub + # Since StringView is generic over the wrapped array, we could invoke UB + # if we don't validate the array behaves as expected. + offs < 1 && error("Invalid implementation of vector length") + end + __unsafe_string!(out, c, offs) + end + # note that for StringViewAndSub, we cannot return the same type of StringView + # anyway since the data type may not be mutable, so we just return String + return out +end + +sizeof(s::StringView) = length(s.data) +ncodeunits(s::StringView) = length(s.data) +codeunit(::StringView) = UInt8 +@propagate_inbounds codeunit(s::StringView, i::Integer) = s.data[i] +codeunits(s::StringView) = s.data +codeunits(s::SubString{<:StringView}) = @view s.string.data[(1 + s.offset):(s.offset + s.ncodeunits)] + +# For UTF8 encoded strings, we can operate on codeunits directly. +# For non-UTF8 strings, we use the AbstractString fallback +cmp(a::UTF8String, b::UTF8String) = cmp(codeunits(a), codeunits(b)) +==(a::UTF8String, b::UTF8String) = codeunits(a) == codeunits(b) + +# Typemin and one is the empty string (multiplicative identity) +typemin(::Type{StringView{CodeUnits{UInt8, String}}}) = StringView(CodeUnits("")) +typemin(::Type{StringView{V}}) where V = StringView(V()) +typemin(::T) where {T <: StringView} = typemin(T) +one(::Union{T, Type{T}}) where {T <: StringView} = typemin(T) +oneunit(::Union{T, Type{T}}) where {T <: StringView} = typemin(T) + +# Forward to optimised isascii(::AbstractVector{UInt8}) +isascii(s::StringViewAndSub) = isascii(codeunits(s)) + +# For dense string views, pointer-based hashing is faster than array based. +function hash(s::DenseStringViewAndSub, h::UInt) + GC.@preserve s hash_bytes(pointer(s), ncodeunits(s), UInt64(h), HASH_SECRET) % UInt +end + +# The canonical binary representation of strings is simply their byte content. +write(io::IO, s::StringViewAndSub) = write(io, codeunits(s))::Int +print(io::IO, s::StringViewAndSub) = (write(io, s); nothing) + +@propagate_inbounds thisind(s::StringViewAndSub, i::Integer) = _thisind_str(s, Int(i)::Int) +@propagate_inbounds thisind(s::StringViewAndSub, i::Int) = _thisind_str(s, i) + +@propagate_inbounds nextind(s::StringViewAndSub, i::Integer) = _nextind_str(s, Int(i)::Int) +@propagate_inbounds nextind(s::StringViewAndSub, i::Int) = _nextind_str(s, i) + +isvalid(s::StringViewAndSub, i::Int) = checkbounds(Bool, s, i) && thisind(s, i) == i + +# This is different from the String implementation, because when r is empty, +# we cannot just return the constant "". +@inline function getindex(s::StringView, r::UnitRange{Int}) + cu = codeunits(s) + isempty(r) && return StringView(cu[1:0]) + i, j = first(r), last(r) + @boundscheck begin + checkbounds(cu, r) + @inbounds isvalid(s, i) || string_index_err(s, i) + @inbounds isvalid(s, j) || string_index_err(s, j) + end + j = nextind(s, j) - 1 + return StringView(cu[i:j]) +end + +function chomp(s::StringViewAndSub) + cu = codeunits(s) + ncu = length(cu) + len = if iszero(ncu) + 0 + else + has_lf = cu[ncu] == 0x0a + two_bytes = ncu > 1 + has_cr = has_lf & two_bytes & (cu[ncu - two_bytes] == 0x0d) + ncu - (has_lf + has_cr) + end + @inbounds raw_substring(s, 1, len) +end + +function replace(io::IO, s::DenseStringViewAndSub, pat_f::Pair...; count = typemax(Int)) + return _replace_(io, s, pat_f, Int(count)) +end diff --git a/base/strings/substring.jl b/base/strings/substring.jl index 860895207f444..911f81bc17f37 100644 --- a/base/strings/substring.jl +++ b/base/strings/substring.jl @@ -1,5 +1,51 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +""" + raw_substring(s::AbstractString, first_index::Int, n_codeunits::Int)::SubString{typeof(s)} + raw_substring(s::SubString{S}, first_index::Int, n_codeunits::Int)::SubString{S} + +Create a substring of `s` spanning the codeunits `first_index:(first_index + n_codeunits - 1)`. + +If `first_index` < 1, or `first_index + n_codeunits - 1 > ncodeunits(s)`, throw a `BoundsError`. + +This function does check bounds, but does not validate that the arguments correspond to valid +start and end indices in `s`, and so the resulting substring may contain truncated characters. +The presence of truncated characters is safe and well-defined for `String`, `StringView`, and +substrings of these, but may not be permitted for custom subtypes of `AbstractString`. +Note that accessing characters that are whole in the parent string but truncated by the `SubString` +may throw a `StringIndexError`. + +!!! warning + For `AbstractString` other than `String`, `StringView` or substrings of those, callers should + ensure that the value of `n_codeunits` does not result in truncated codeunits. + +# Examples +```jldoctest +julia> s = "Hello, Bjørn!"; + +julia> ss = Base.raw_substring(s, 3, 10) +"llo, Bjør" + +julia> typeof(ss) +SubString{String} + +julia> ss2 = Base.raw_substring(ss, 3, 7) +"o, Bjø" + +julia> typeof(ss2) +SubString{String} + +julia> ss3 = Base.raw_substring(s, 11, 4); ss3[1] +ERROR: StringIndexError: +[...] +``` + +!!! compat "Julia 1.14" + This function requires at least Julia 1.14. + +""" +function raw_substring end + """ SubString(s::AbstractString, i::Integer, j::Integer=lastindex(s)) SubString(s::AbstractString, r::UnitRange{<:Integer}) @@ -36,19 +82,31 @@ struct SubString{T<:AbstractString} <: AbstractString end return new(s, i-1, nextind(s,j)-i) end - function SubString{T}(s::T, i::Int, j::Int, ::Val{:noshift}) where T<:AbstractString - @boundscheck if !(i == j == 0) - si, sj = i + 1, prevind(s, j + i + 1) - @inbounds isvalid(s, si) || string_index_err(s, si) - @inbounds isvalid(s, sj) || string_index_err(s, sj) + + global function raw_substring(s::T, first_index::Int, n_codeunits::Int) where {T <: AbstractString} + @boundscheck if n_codeunits < 0 || first_index < 1 || (n_codeunits > ncodeunits(s) - first_index + 1) + throw(BoundsError(s, first_index:(first_index+n_codeunits-1))) + end + new{T}(s, first_index - 1, n_codeunits) + end + + global function raw_substring(s::SubString{T}, first_index::Int, n_codeunits::Int) where {T <: AbstractString} + @boundscheck if n_codeunits < 0 || first_index < 1 || (n_codeunits > ncodeunits(s) - first_index + 1) + throw(BoundsError(s, first_index:(first_index+n_codeunits-1))) end - new(s, i, j) + new{T}(s.string, first_index + s.offset - 1, n_codeunits) + end + + # Unlike the un-parameterized SubString constructor, this function must allow creating + # e.g. a SubString{SubString{String}}, as this type is what the user may have explicitly + # requested. + function SubString{T}(s::T) where {T <: AbstractString} + new{T}(s, 0, ncodeunits(s)) end end @propagate_inbounds SubString(s::T, i::Int, j::Int) where {T<:AbstractString} = SubString{T}(s, i, j) -@propagate_inbounds SubString(s::T, i::Int, j::Int, v::Val{:noshift}) where {T<:AbstractString} = SubString{T}(s, i, j, v) -@propagate_inbounds SubString(s::AbstractString, i::Integer, j::Integer=lastindex(s)) = SubString(s, Int(i), Int(j)) +@propagate_inbounds SubString(s::AbstractString, i::Integer, j::Integer=lastindex(s)) = SubString(s, Int(i)::Int, Int(j)::Int) @propagate_inbounds SubString(s::AbstractString, r::AbstractUnitRange{<:Integer}) = SubString(s, first(r), last(r)) @propagate_inbounds function SubString(s::SubString, i::Int, j::Int) @@ -56,8 +114,8 @@ end SubString(s.string, s.offset+i, s.offset+j) end -SubString(s::AbstractString) = SubString(s, 1, lastindex(s)::Int) -SubString{T}(s::T) where {T<:AbstractString} = SubString{T}(s, 1, lastindex(s)::Int) +SubString(s::AbstractString) = @inbounds raw_substring(s, 1, Int(ncodeunits(s))::Int) +SubString(s::SubString) = s @propagate_inbounds view(s::AbstractString, r::AbstractUnitRange{<:Integer}) = SubString(s, r) @propagate_inbounds maybeview(s::AbstractString, r::AbstractUnitRange{<:Integer}) = view(s, r) @@ -81,6 +139,9 @@ end ncodeunits(s::SubString) = s.ncodeunits codeunit(s::SubString) = codeunit(s.string)::CodeunitType length(s::SubString) = length(s.string, s.offset+1, s.offset+s.ncodeunits) +# nothrow: SubString invariants guarantee 0 ≤ offset and offset+ncodeunits ≤ ncodeunits(string), +# so the bounds-check inside the 3-arg `length(::String, i, j)` cannot fail. +@assume_effects :nothrow length(s::SubString{String}) = length(s.string, s.offset+1, s.offset+s.ncodeunits) function codeunit(s::SubString, i::Integer) @boundscheck checkbounds(s, i) @@ -101,7 +162,8 @@ function getindex(s::SubString, i::Integer) @inbounds return getindex(s.string, s.offset + i) end -isascii(ss::SubString{String}) = isascii(codeunits(ss)) +# `isascii(::AbstractVector)` reduces to `@inbounds codeunit(::SubString{String}, ::Int)`, total. +isascii(ss::SubString{String}) = @assume_effects :nothrow :foldable isascii(codeunits(ss)) function isvalid(s::SubString, i::Integer) ib = true @@ -109,8 +171,11 @@ function isvalid(s::SubString, i::Integer) @inbounds return ib && isvalid(s.string, s.offset + i)::Bool end -thisind(s::SubString{String}, i::Int) = _thisind_str(s, i) -nextind(s::SubString{String}, i::Int) = _nextind_str(s, i) +@propagate_inbounds thisind(s::SubString{String}, i::Int) = _thisind_str(s, i) +@propagate_inbounds nextind(s::SubString{String}, i::Int) = _nextind_str(s, i) + +# nothrow: i == ncodeunits(s) always satisfies the bounds check inside _thisind_str. +@assume_effects :nothrow lastindex(s::SubString{String}) = thisind(s, ncodeunits(s)::Int) parent(s::SubString) = s.string parentindices(s::SubString) = (s.offset + 1 : thisind(s.string, s.offset + s.ncodeunits),) @@ -140,53 +205,6 @@ hash(data::SubString{String}, h::UInt) = _isannotated(::SubString{T}) where {T} = _isannotated(T) -""" - reverse(s::AbstractString)::AbstractString - -Reverses a string. Technically, this function reverses the codepoints in a string and its -main utility is for reversed-order string processing, especially for reversed -regular-expression searches. See also [`reverseind`](@ref) to convert indices in `s` to -indices in `reverse(s)` and vice-versa, and `graphemes` from module `Unicode` to -operate on user-visible "characters" (graphemes) rather than codepoints. -See also [`Iterators.reverse`](@ref) for -reverse-order iteration without making a copy. Custom string types must implement the -`reverse` function themselves and should typically return a string with the same type -and encoding. If they return a string with a different encoding, they must also override -`reverseind` for that string type to satisfy `s[reverseind(s,i)] == reverse(s)[i]`. - -# Examples -```jldoctest -julia> reverse("JuliaLang") -"gnaLailuJ" -``` - -!!! note - The examples below may be rendered differently on different systems. - The comments indicate how they're supposed to be rendered - -Combining characters can lead to surprising results: - -```jldoctest -julia> reverse("ax̂e") # hat is above x in the input, above e in the output -"êxa" - -julia> using Unicode - -julia> join(reverse(collect(graphemes("ax̂e")))) # reverses graphemes; hat is above x in both in- and output -"ex̂a" -``` -""" -function reverse(s::Union{String,SubString{String}})::String - # Read characters forwards from `s` and write backwards to `out` - out = _string_n(sizeof(s)) - offs = sizeof(s) + 1 - for c in s - offs -= ncodeunits(c) - __unsafe_string!(out, c, offs) - end - return out -end - string(a::String) = String(a) string(a::SubString{String}) = String(a) @@ -274,6 +292,8 @@ function repeat(s::Union{String, SubString{String}}, r::Integer) r == 0 && return "" r == 1 && return String(s) n = sizeof(s) + n == 0 && return "" + r > typemax(UInt) ÷ UInt(n) && throw(OutOfMemoryError()) out = _string_n(n*r) if n == 1 # common case: repeating a single-byte string @inbounds b = codeunit(s, 1) diff --git a/base/strings/unicode.jl b/base/strings/unicode.jl index 520226aacd3cb..2118477492ad8 100644 --- a/base/strings/unicode.jl +++ b/base/strings/unicode.jl @@ -6,7 +6,7 @@ module Unicode import Base: show, ==, hash, string, Symbol, isless, length, eltype, convert, isvalid, ismalformed, isoverlong, iterate, AnnotatedString, AnnotatedChar, annotated_chartransform, - @assume_effects, annotations, is_overlong_enc + @assume_effects, annotations, is_overlong_enc, unsafe_codepoint # whether codepoints are valid Unicode scalar values, i.e. 0-0xd7ff, 0xe000-0x10ffff @@ -127,6 +127,9 @@ const category_strings = [ "Invalid, too high", "Malformed, bad data", ] +# category_code returns a value in 0:31; keep category_strings sized to match so +# `category_string` remains nothrow. +@assert length(category_strings) == 32 const UTF8PROC_STABLE = (1<<1) const UTF8PROC_COMPAT = (1<<2) @@ -191,7 +194,7 @@ const _julia_charmap = Dict{UInt32,UInt32}( 0x210F => 0x0127, # hbar -> small letter h with stroke (#48870) ) -utf8proc_map(s::AbstractString, flags::Integer, chartransform::F = identity) where F = utf8proc_map(String(s), flags, chartransform) +utf8proc_map(s::AbstractString, flags::Integer, chartransform::F = identity) where F = utf8proc_map(String(s)::String, flags, chartransform) # Documented in Unicode module function normalize( @@ -271,7 +274,9 @@ function textwidth(c::Char) # We can't know a priori how terminals will render invalid UTF8 chars, # so we conservatively decide a width of 1. (ismalformed(c) || is_overlong_enc(u)) && return 1 - Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), c)) + # `unsafe_codepoint` is sound here because both malformed-encoding cases + # have been ruled out above; this lets the compiler infer `:nothrow`. + Int(@assume_effects :foldable :nothrow @ccall utf8proc_charwidth(unsafe_codepoint(c)::UInt32)::Cint) end """ @@ -286,6 +291,9 @@ julia> textwidth("March") ``` """ textwidth(s::AbstractString) = mapreduce(textwidth, +, s; init=0) +# foldable+nothrow: String iteration is total and textwidth(::Char) is nothrow. +@assume_effects :nothrow :foldable textwidth(s::String) = mapreduce(textwidth, +, s; init=0) +@assume_effects :nothrow :foldable textwidth(s::SubString{String}) = mapreduce(textwidth, +, s; init=0) textwidth(s::AnnotatedString) = textwidth(s.string) @@ -306,7 +314,7 @@ julia> lowercase('Ö') ``` """ lowercase(c::T) where {T<:AbstractChar} = isascii(c) ? ('A' <= c <= 'Z' ? c + 0x20 : c) : - T(ccall(:utf8proc_tolower, UInt32, (UInt32,), c)) + T(@assume_effects :foldable :nothrow @ccall utf8proc_tolower(c::UInt32)::UInt32) lowercase(c::AnnotatedChar) = AnnotatedChar(lowercase(c.char), annotations(c)) @@ -327,7 +335,7 @@ julia> uppercase('ê') ``` """ uppercase(c::T) where {T<:AbstractChar} = isascii(c) ? ('a' <= c <= 'z' ? c - 0x20 : c) : - T(ccall(:utf8proc_toupper, UInt32, (UInt32,), c)) + T(@assume_effects :foldable :nothrow @ccall utf8proc_toupper(c::UInt32)::UInt32) uppercase(c::AnnotatedChar) = AnnotatedChar(uppercase(c.char), annotations(c)) @@ -352,7 +360,7 @@ julia> uppercase('dž') ``` """ titlecase(c::T) where {T<:AbstractChar} = isascii(c) ? ('a' <= c <= 'z' ? c - 0x20 : c) : - T(ccall(:utf8proc_totitle, UInt32, (UInt32,), c)) + T(@assume_effects :foldable :nothrow @ccall utf8proc_totitle(c::UInt32)::UInt32) titlecase(c::AnnotatedChar) = AnnotatedChar(titlecase(c.char), annotations(c)) @@ -360,21 +368,37 @@ titlecase(c::AnnotatedChar) = AnnotatedChar(titlecase(c.char), annotations(c)) # returns UTF8PROC_CATEGORY code in 0:30 giving Unicode category function category_code(c::AbstractChar) - !ismalformed(c) ? category_code(UInt32(c)) : Cint(31) + # `unsafe_codepoint` is sound here because the `ismalformed` guard has ruled + # out the throwing path of `UInt32(::Char)`, allowing the compiler to infer + # `:nothrow` for `category_code(::Char)`. + !ismalformed(c) ? category_code(unsafe_codepoint(c)) : Cint(31) end function category_code(x::Integer) - x ≤ 0x10ffff ? (@assume_effects :foldable @ccall utf8proc_category(UInt32(x)::UInt32)::Cint) : Cint(30) + x ≤ 0x10ffff ? (@assume_effects :foldable :nothrow @ccall utf8proc_category(UInt32(x)::UInt32)::Cint) : Cint(30) end # more human-readable representations of the category code function category_abbrev(c::AbstractChar) ismalformed(c) && return "Ma" c ≤ '\U10ffff' || return "In" - unsafe_string(ccall(:utf8proc_category_string, Cstring, (UInt32,), c)) + unsafe_string(@ccall utf8proc_category_string(UInt32(c)::UInt32)::Cstring) +end +# `unsafe_string` over a `utf8proc` cstring is not inferred as `:nothrow`/`:foldable`, +# but the cstring is a static table entry inside utf8proc, and `unsafe_codepoint` +# is sound here because of the `ismalformed` guard above. +@assume_effects :nothrow :foldable function category_abbrev(c::Char) + ismalformed(c) && return "Ma" + c ≤ '\U10ffff' || return "In" + unsafe_string(@ccall utf8proc_category_string(unsafe_codepoint(c)::UInt32)::Cstring) end +# category_code(c) returns a value in 0:31 and category_strings has 32 entries +# (asserted at the top of this file), so the index is always in bounds for `Char`. category_string(c) = category_strings[category_code(c)+1] +# `getindex` on a `const` `Vector{String}` is not inferred as `:nothrow` even with +# `@inbounds`, but the bounds are guaranteed by the assert above. +@assume_effects :nothrow :foldable category_string(c::Char) = @inbounds category_strings[category_code(c)+1] isassigned(c) = UTF8PROC_CATEGORY_CN < category_code(c) <= UTF8PROC_CATEGORY_CO @@ -400,8 +424,12 @@ julia> islowercase('❤') false ``` """ -islowercase(c::AbstractChar) = ismalformed(c) ? false : - Bool(@assume_effects :foldable @ccall utf8proc_islower(UInt32(c)::UInt32)::Cint) +function islowercase(c::AbstractChar) + # `unsafe_codepoint` is sound here because of the `ismalformed` guard; + # this lets the compiler infer `:nothrow` for `islowercase(::Char)`. + ismalformed(c) ? false : + !iszero(@assume_effects :foldable :nothrow @ccall utf8proc_islower(unsafe_codepoint(c)::UInt32)::Cint) +end # true for Unicode upper and mixed case @@ -425,8 +453,12 @@ julia> isuppercase('❤') false ``` """ -isuppercase(c::AbstractChar) = ismalformed(c) ? false : - Bool(@assume_effects :foldable @ccall utf8proc_isupper(UInt32(c)::UInt32)::Cint) +function isuppercase(c::AbstractChar) + # `unsafe_codepoint` is sound here because of the `ismalformed` guard; + # this lets the compiler infer `:nothrow` for `isuppercase(::Char)`. + ismalformed(c) ? false : + !iszero(@assume_effects :foldable :nothrow @ccall utf8proc_isupper(unsafe_codepoint(c)::UInt32)::Cint) +end """ iscased(c::AbstractChar)::Bool @@ -603,8 +635,6 @@ true """ isprint(c::AbstractChar) = UTF8PROC_CATEGORY_LU <= category_code(c) <= UTF8PROC_CATEGORY_ZS -# true in principal if a printer would use ink - """ isxdigit(c::AbstractChar)::Bool @@ -639,6 +669,7 @@ julia> uppercase("Julia") """ uppercase(s::AbstractString) = map(uppercase, s) uppercase(s::AnnotatedString) = annotated_chartransform(uppercase, s) +uppercase(s::SubString{<:AnnotatedString}) = uppercase(AnnotatedString(s)) """ lowercase(s::AbstractString) @@ -655,6 +686,7 @@ julia> lowercase("STRINGS AND THINGS") """ lowercase(s::AbstractString) = map(lowercase, s) lowercase(s::AnnotatedString) = annotated_chartransform(lowercase, s) +lowercase(s::SubString{<:AnnotatedString}) = lowercase(AnnotatedString(s)) """ titlecase(s::AbstractString; [wordsep::Function], strict::Bool=true)::String @@ -720,6 +752,9 @@ function titlecase(s::AnnotatedString; wordsep::Function = !isletter, strict::Bo end end +titlecase(s::SubString{<:AnnotatedString}; wordsep::Function = !isletter, strict::Bool=true) = + titlecase(AnnotatedString(s); wordsep=wordsep, strict=strict) + """ uppercasefirst(s::AbstractString)::String @@ -754,6 +789,7 @@ function uppercasefirst(s::AnnotatedString) end end end +uppercasefirst(s::SubString{<:AnnotatedString}) = uppercasefirst(AnnotatedString(s)) """ lowercasefirst(s::AbstractString) @@ -787,6 +823,7 @@ function lowercasefirst(s::AnnotatedString) end end end +lowercasefirst(s::SubString{<:AnnotatedString}) = lowercasefirst(AnnotatedString(s)) ############################################################################ # iterators for grapheme segmentation diff --git a/base/strings/util.jl b/base/strings/util.jl index c3df790cd81e6..55b34c5d89d1c 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -66,8 +66,7 @@ function endswith(a::AbstractString, b::AbstractString) end endswith(str::AbstractString, chars::Chars) = !isempty(str) && last(str) in chars -function startswith(a::Union{String, SubString{String}}, - b::Union{String, SubString{String}}) +function startswith(a::DenseUTF8String, b::DenseUTF8String) cub = ncodeunits(b) if ncodeunits(a) < cub false @@ -78,28 +77,34 @@ function startswith(a::Union{String, SubString{String}}, end end +# nothrow+foldable: `String`/`SubString{String}` buffers are immutable, the +# byte-wise `_memcmp` is bounded by `sizeof(b) ≤ ncodeunits(a)`, and +# `nextind(a, cub)` operates on a valid index (`0 ≤ cub ≤ ncodeunits(a)`). +@assume_effects :nothrow :foldable function startswith(a::Union{String,SubString{String}}, b::Union{String,SubString{String}}) + @invoke startswith(a::DenseUTF8String, b::DenseUTF8String) +end + """ startswith(io::IO, prefix::Union{AbstractString,Base.Chars}) Check if an `IO` object starts with a prefix, which can be either a string, a character, or a tuple/vector/set of characters. See also [`peek`](@ref). """ -function Base.startswith(io::IO, prefix::Base.Chars) +function startswith(io::IO, prefix::Base.Chars) mark(io) c = read(io, Char) reset(io) return c in prefix end -function Base.startswith(io::IO, prefix::Union{String,SubString{String}}) +function startswith(io::IO, prefix::UTF8String) mark(io) s = read(io, ncodeunits(prefix)) reset(io) return s == codeunits(prefix) end -Base.startswith(io::IO, prefix::AbstractString) = startswith(io, String(prefix)) +startswith(io::IO, prefix::AbstractString) = startswith(io, String(prefix)::String) -function endswith(a::Union{String, SubString{String}}, - b::Union{String, SubString{String}}) +function endswith(a::DenseUTF8String, b::DenseUTF8String) astart = ncodeunits(a) - ncodeunits(b) + 1 if astart < 1 false @@ -110,6 +115,12 @@ function endswith(a::Union{String, SubString{String}}, end end +# nothrow+foldable: see `startswith` above; `pointer(a, astart)` is in bounds +# (`1 ≤ astart ≤ ncodeunits(a)+1`), and `thisind(a, astart)` accepts the same range. +@assume_effects :nothrow :foldable function endswith(a::Union{String,SubString{String}}, b::Union{String,SubString{String}}) + @invoke endswith(a::DenseUTF8String, b::DenseUTF8String) +end + """ contains(haystack::AbstractString, needle) @@ -232,7 +243,7 @@ end # chop(s::AbstractString) = SubString(s, firstindex(s), prevind(s, lastindex(s))) """ - chopprefix(s::AbstractString, prefix::Union{AbstractString,Regex})::SubString + chopprefix(s::AbstractString, prefix::Union{AbstractString,Regex,AbstractChar})::SubString Remove the prefix `prefix` from `s`. If `s` does not start with `prefix`, a string equal to `s` is returned. @@ -241,6 +252,9 @@ See also [`chopsuffix`](@ref). !!! compat "Julia 1.8" This function is available as of Julia 1.8. +!!! compat "Julia 1.13" + The method which accepts an `AbstractChar` prefix is available as of Julia 1.13. + # Examples ```jldoctest julia> chopprefix("Hamburger", "Ham") @@ -272,8 +286,16 @@ function chopprefix(s::Union{String, SubString{String}}, end end +function chopprefix(s::AbstractString, prefix::AbstractChar) + if !isempty(s) && first(s) == prefix + return SubString(s, nextind(s, firstindex(s))) + else + return SubString(s) + end +end + """ - chopsuffix(s::AbstractString, suffix::Union{AbstractString,Regex})::SubString + chopsuffix(s::AbstractString, suffix::Union{AbstractString,Regex,AbstractChar})::SubString Remove the suffix `suffix` from `s`. If `s` does not end with `suffix`, a string equal to `s` is returned. @@ -282,6 +304,9 @@ See also [`chopprefix`](@ref). !!! compat "Julia 1.8" This function is available as of Julia 1.8. +!!! compat "Julia 1.13" + The method which accepts an `AbstractChar` suffix is available as of Julia 1.13. + # Examples ```jldoctest julia> chopsuffix("Hamburger", "er") @@ -315,6 +340,13 @@ function chopsuffix(s::Union{String, SubString{String}}, end end +function chopsuffix(s::AbstractString, suffix::AbstractChar) + if !isempty(s) && last(s) == suffix + return SubString(s, firstindex(s), prevind(s, lastindex(s))) + else + return SubString(s) + end +end """ chomp(s::AbstractString)::SubString @@ -354,9 +386,7 @@ end has_cr = has_lf & two_bytes & (@inbounds(cu[ncu - two_bytes]) == 0x0d) ncu - (has_lf + has_cr) end - off = s isa String ? 0 : s.offset - par = s isa String ? s : s.string - @inbounds @inline SubString{String}(par, off, len, Val{:noshift}()) + @inbounds raw_substring(s, 1, len) end """ lstrip([pred=isspace,] str::AbstractString)::SubString @@ -371,7 +401,7 @@ The default behaviour is to remove leading whitespace and delimiters: see The optional `chars` argument specifies which characters to remove: it can be a single character, or a vector or set of characters. -See also [`strip`](@ref) and [`rstrip`](@ref). +See also [`strip`](@ref), [`rstrip`](@ref). # Examples ```jldoctest @@ -406,7 +436,7 @@ The default behaviour is to remove trailing whitespace and delimiters: see The optional `chars` argument specifies which characters to remove: it can be a single character, or a vector or set of characters. -See also [`strip`](@ref) and [`lstrip`](@ref). +See also [`strip`](@ref), [`lstrip`](@ref). # Examples ```jldoctest @@ -441,11 +471,11 @@ The default behaviour is to remove leading and trailing whitespace and delimiter The optional `chars` argument specifies which characters to remove: it can be a single character, vector or set of characters. -See also [`lstrip`](@ref) and [`rstrip`](@ref). - !!! compat "Julia 1.2" The method which accepts a predicate function requires Julia 1.2 or later. +See also [`lstrip`](@ref), [`rstrip`](@ref). + # Examples ```jldoctest julia> strip("{3, 5}\\n", ['{', '}', '\\n']) @@ -466,16 +496,32 @@ Stringify `s` and pad the resulting string on the left with `p` to make it `n` characters (in [`textwidth`](@ref)) long. If `s` is already `n` characters long, an equal string is returned. Pad with spaces by default. +!!! compat "Julia 1.7" + In Julia 1.7, this function was changed to use `textwidth` rather than a raw character (codepoint) count. + +See also [`rpad`](@ref). + # Examples ```jldoctest julia> lpad("March", 10) " March" ``` -!!! compat "Julia 1.7" - In Julia 1.7, this function was changed to use `textwidth` rather than a raw character (codepoint) count. """ lpad(s, n::Integer, p::Union{AbstractChar,AbstractString}=' ') = lpad(string(s)::AbstractString, n, string(p)) +# Pad for the last `0 < r < textwidth(p)` columns: the shortest prefix of a string pad +# covering `r` columns, or the whole char for a char pad. When `r` falls inside a wide +# character, the pad overshoots the requested width. +function _pad_remainder(p::AbstractString, r::Int) + width = 0 + for (i, c) in pairs(p) + width += Int(textwidth(c))::Int + width >= r && return @view p[begin:i] + end + return SubString(p) # unreachable: caller guarantees r < textwidth(p) +end +_pad_remainder(p::AbstractChar, r::Int) = p + function lpad( s::Union{AbstractChar,AbstractString}, n::Integer, @@ -493,7 +539,7 @@ function lpad( (s isa AbstractString && codeunit(s) != UInt8 ? "?" : " (bytes)?")))) end q, r = divrem(m, l) - r == 0 ? stringfn(p^q, s) : stringfn(p^q, first(p, r), s) + r == 0 ? stringfn(p^q, s) : stringfn(p^q, _pad_remainder(p, r), s) end """ @@ -503,13 +549,16 @@ Stringify `s` and pad the resulting string on the right with `p` to make it `n` characters (in [`textwidth`](@ref)) long. If `s` is already `n` characters long, an equal string is returned. Pad with spaces by default. +!!! compat "Julia 1.7" + In Julia 1.7, this function was changed to use `textwidth` rather than a raw character (codepoint) count. + +See also [`lpad`](@ref). + # Examples ```jldoctest julia> rpad("March", 20) "March " ``` -!!! compat "Julia 1.7" - In Julia 1.7, this function was changed to use `textwidth` rather than a raw character (codepoint) count. """ rpad(s, n::Integer, p::Union{AbstractChar,AbstractString}=' ') = rpad(string(s)::AbstractString, n, string(p)) @@ -530,7 +579,7 @@ function rpad( (s isa AbstractString && codeunit(s) != UInt8 ? "?" : " (bytes)?")))) end q, r = divrem(m, l) - r == 0 ? stringfn(s, p^q) : stringfn(s, p^q, first(p, r)) + r == 0 ? stringfn(s, p^q) : stringfn(s, p^q, _pad_remainder(p, r)) end """ @@ -539,6 +588,11 @@ end Truncate `str` to at most `maxwidth` columns (as estimated by [`textwidth`](@ref)), replacing the last characters with `replacement` if necessary. The default replacement string is "…". +!!! compat "Julia 1.12" + This function was added in Julia 1.12. + +See also [`ltruncate`](@ref), [`ctruncate`](@ref). + # Examples ```jldoctest julia> s = rtruncate("🍕🍕 I love 🍕", 10) @@ -550,11 +604,6 @@ julia> textwidth(s) julia> rtruncate("foo", 3) "foo" ``` - -!!! compat "Julia 1.12" - This function was added in Julia 1.12. - -See also [`ltruncate`](@ref) and [`ctruncate`](@ref). """ function rtruncate(str::AbstractString, maxwidth::Integer, replacement::Union{AbstractString,AbstractChar} = '…') ret = string_truncate_boundaries(str, Int(maxwidth), replacement, Val(:right)) @@ -572,6 +621,11 @@ end Truncate `str` to at most `maxwidth` columns (as estimated by [`textwidth`](@ref)), replacing the first characters with `replacement` if necessary. The default replacement string is "…". +!!! compat "Julia 1.12" + This function was added in Julia 1.12. + +See also [`rtruncate`](@ref), [`ctruncate`](@ref). + # Examples ```jldoctest julia> s = ltruncate("🍕🍕 I love 🍕", 10) @@ -583,11 +637,6 @@ julia> textwidth(s) julia> ltruncate("foo", 3) "foo" ``` - -!!! compat "Julia 1.12" - This function was added in Julia 1.12. - -See also [`rtruncate`](@ref) and [`ctruncate`](@ref). """ function ltruncate(str::AbstractString, maxwidth::Integer, replacement::Union{AbstractString,AbstractChar} = '…') ret = string_truncate_boundaries(str, Int(maxwidth), replacement, Val(:left)) @@ -606,6 +655,11 @@ Truncate `str` to at most `maxwidth` columns (as estimated by [`textwidth`](@ref with `replacement` if necessary. The default replacement string is "…". By default, the truncation prefers keeping chars on the left, but this can be changed by setting `prefer_left` to `false`. +!!! compat "Julia 1.12" + This function was added in Julia 1.12. + +See also [`ltruncate`](@ref), [`rtruncate`](@ref). + # Examples ```jldoctest julia> s = ctruncate("🍕🍕 I love 🍕", 10) @@ -617,11 +671,6 @@ julia> textwidth(s) julia> ctruncate("foo", 3) "foo" ``` - -!!! compat "Julia 1.12" - This function was added in Julia 1.12. - -See also [`ltruncate`](@ref) and [`rtruncate`](@ref). """ function ctruncate(str::AbstractString, maxwidth::Integer, replacement::Union{AbstractString,AbstractChar} = '…'; prefer_left::Bool = true) ret = string_truncate_boundaries(str, Int(maxwidth), replacement, Val(:center), prefer_left) @@ -700,11 +749,11 @@ The optional keyword arguments are: - `keepempty`: whether empty fields should be kept in the result. Default is `false` without a `dlm` argument, `true` with a `dlm` argument. -See also [`split`](@ref). - !!! compat "Julia 1.8" The `eachsplit` function requires at least Julia 1.8. +See also [`split`](@ref). + # Examples ```jldoctest julia> a = "Ma.rch" @@ -801,11 +850,11 @@ The optional keyword arguments are: Note that unlike [`split`](@ref), [`rsplit`](@ref) and [`eachsplit`](@ref), this function iterates the substrings right to left as they occur in the input. -See also [`eachsplit`](@ref), [`rsplit`](@ref). - !!! compat "Julia 1.11" This function requires Julia 1.11 or later. +See also [`eachsplit`](@ref), [`rsplit`](@ref). + # Examples ```jldoctest julia> a = "Ma.r.ch"; @@ -954,11 +1003,22 @@ rsplit(str::AbstractString; limit::Integer=0, keepempty::Bool=false) = rsplit(str, isspace; limit, keepempty) -_replace(io, repl, str, r, pattern) = print(io, repl) +_replace(io, repl::Union{<:AbstractString, <:AbstractChar}, str, r, pattern) = + write(io, repl) +function _replace(io, repl, str, r, pattern) + if applicable(position, io) + p1 = position(io) + print(io, repl) + p2 = position(io) + p2 - p1 + else + write(io, repr(repl)) + end +end _replace(io, repl::Function, str, r, pattern) = - print(io, repl(SubString(str, first(r), last(r)))) + _replace(io, repl(SubString(str, first(r), last(r))), str, r, pattern) _replace(io, repl::Function, str, r, pattern::Function) = - print(io, repl(str[first(r)])) + _replace(io, repl(str[first(r)]), str, r, pattern) _pat_replacer(x) = x _free_pat_replacer(x) = nothing @@ -988,38 +1048,11 @@ end function _replace_finish(io::IO, str, count::Int, e1::Int, patterns::Tuple, replaces::Tuple, rs::Tuple) n = 1 - i = a = firstindex(str) - while true - p = argmin(map(first, rs)) # TODO: or argmin(rs), to pick the shortest first match ? - r = rs[p] - j, k = first(r), last(r) - j > e1 && break - if i == a || i <= k - # copy out preserved portion - GC.@preserve str unsafe_write(io, pointer(str, i), UInt(j-i)) - # copy out replacement string - _replace(io, replaces[p], str, r, patterns[p]) - end - if k < j - i = j - j == e1 && break - k = nextind(str, j) - else - i = k = nextind(str, k) - end - n == count && break - let k = k - rs = map(patterns, rs) do p, r - if first(r) < k - r = findnext(p, str, k) - if r === nothing || first(r) == 0 - return e1+1:0 - end - r isa Int && (r = r:r) # findnext / performance fix - end - return r - end - end + i = start = firstindex(str) + while n <= count + rs, _, r, _, i = @inline _replace_once( + io, str, start, e1, patterns, replaces, rs, count, n, i) + first(r) >= e1 && break n += 1 end foreach(_free_pat_replacer, patterns) @@ -1027,6 +1060,44 @@ function _replace_finish(io::IO, str, count::Int, return io end +function _replace_once(io::IO, str, start::Int, e1::Int, + patterns::Tuple, replaces::Tuple, rs::Tuple, + count::Int, n::Int, i::Int) + x = argmin(map(first, rs)) # TODO: or argmin(rs), to pick the shortest first match ? + r = rs[x] + j, k = first(r), last(r) + j > e1 && return rs, x, r, 0, i + nb = if i == start || i <= k + # copy out preserved portion + GC.@preserve str unsafe_write(io, pointer(str, i), UInt(j-i)) + # copy out replacement string + _replace(io, replaces[x], str, r, patterns[x]) + else + 0 + end + if k < j + i = j + j == e1 && return rs, x, r, nb, i + k = nextind(str, j) + else + i = k = nextind(str, k) + end + n == count && return rs, x, r, nb, i + let k = k + rs = map(patterns, rs) do p, r + if first(r) < k + r = findnext(p, str, k) + if r === nothing || first(r) == 0 + return e1+1:0 + end + r isa Int && (r = r:r) # findnext / performance fix + end + return r + end + end + return rs, x, r, nb, i +end + # note: leave str untyped here to make it easier for packages like StringViews to hook in function _replace_(io::IO, str, pat_repl::NTuple{N, Pair}, count::Int) where N if count == 0 @@ -1178,6 +1249,8 @@ to `dest`. The length of `dest` must be half the length of `itr`. Calling hex2bytes! with iterators producing UInt8 requires version 1.7. In earlier versions, you can collect the iterable before calling instead. + +See also [`hex2bytes`](@ref), [`bytes2hex`](@ref). """ function hex2bytes!(dest::AbstractArray{UInt8}, itr) isodd(length(itr)) && throw(ArgumentError("length of iterable must be even")) @@ -1195,7 +1268,7 @@ function hex2bytes!(dest::AbstractArray{UInt8}, itr) return dest end -@inline number_from_hex(c::AbstractChar) = number_from_hex(Char(c)) +@inline number_from_hex(c::AbstractChar) = number_from_hex(Char(c)::Char) @inline number_from_hex(c::Char) = number_from_hex(UInt8(c)) @inline function number_from_hex(c::UInt8) UInt8('0') <= c <= UInt8('9') && return c - UInt8('0') @@ -1217,6 +1290,8 @@ via `bytes2hex(io, itr)`. The hexadecimal characters are all lowercase. Julia 1.7 or later. In earlier versions, you can `collect` the iterator before calling `bytes2hex`. +See also [`hex2bytes`](@ref), [`hex2bytes!`](@ref). + # Examples ```jldoctest julia> a = string(12345, base = 16) @@ -1235,12 +1310,15 @@ function bytes2hex end function bytes2hex(itr) eltype(itr) === UInt8 || throw(ArgumentError("eltype of iterator not UInt8")) - b = Base.StringMemory(2*length(itr)) - @inbounds for (i, x) in enumerate(itr) - b[2i - 1] = hex_chars[1 + x >> 4] - b[2i ] = hex_chars[1 + x & 0xf] + str = Base._string_n(2*length(itr)) + GC.@preserve str begin + p = pointer(str) + for (i, x) in enumerate(itr) + unsafe_store!(p, @inbounds(hex_chars[1 + x >> 4]), 2i - 1) + unsafe_store!(p, @inbounds(hex_chars[1 + x & 0xf]), 2i) + end end - return unsafe_takestring(b) + return str end function bytes2hex(io::IO, itr) @@ -1265,7 +1343,7 @@ end Convert a string to `String` type and check that it contains only ASCII data, otherwise throwing an `ArgumentError` indicating the position of the first non-ASCII byte. -See also the [`isascii`](@ref) predicate to filter or replace non-ASCII characters. +See also [`isascii`](@ref). # Examples ```jldoctest @@ -1278,7 +1356,7 @@ julia> ascii("abcdefgh") "abcdefgh" ``` """ -ascii(x::AbstractString) = ascii(String(x)) +ascii(x::AbstractString) = ascii(String(x)::String) Base.rest(s::Union{String,SubString{String}}, i=1) = SubString(s, i) function Base.rest(s::AbstractString, st...) diff --git a/base/subarray.jl b/base/subarray.jl index bc68d7c555d13..eda51c9fc56eb 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -33,9 +33,14 @@ function SubArray(::IndexCartesian, parent::P, indices::I, ::NTuple{N,Any}) wher end function SubArray(::IndexLinear, parent::P, indices::I, ::NTuple{N,Any}) where {P,I,N} @inline - # Compute the stride and offset - stride1 = compute_stride1(parent, indices) - SubArray{eltype(P), N, P, I, true}(parent, indices, compute_offset1(parent, stride1, indices), stride1) + offset1, stride1 = _compute_linear_layout(parent, indices) + SubArray{eltype(P), N, P, I, true}(parent, indices, offset1, stride1) +end + +function _compute_linear_layout(parent, indices) + @inline + stride1 = Int(compute_stride1(parent, indices)) + return Int(compute_offset1(parent, stride1, indices)), stride1 end check_parent_index_match(parent, indices) = check_parent_index_match(parent, index_ndims(indices...)) @@ -65,6 +70,7 @@ viewindexing(I::Tuple{AbstractArray, Vararg{Any}}) = IndexCartesian() size(V::SubArray) = (@inline; map(length, axes(V))) similar(V::SubArray, T::Type, dims::Dims) = similar(V.parent, T, dims) +similar(::Type{TA}, dims::Dims) where {T,N,P,TA<:SubArray{T,N,P}} = similar(P, dims) sizeof(V::SubArray) = length(V) * sizeof(eltype(V)) sizeof(V::SubArray{<:Any,<:Any,<:Array}) = length(V) * elsize(V.parent) @@ -116,8 +122,11 @@ function unaliascopy(V::SubArray{T,N,A,I,LD}) where {T,N,A<:Array,I<:Tuple{Varar vdest = trimmedpind isa Tuple{Vararg{Union{Slice,Colon}}} ? dest : view(dest, trimmedpind...) copyto!(vdest, view(V, _trimmedvind(V.indices...)...)) indices = map(_trimmedindex, V.indices) - stride1 = LD ? compute_stride1(dest, indices) : 0 - offset1 = LD ? compute_offset1(dest, stride1, indices) : 0 + if LD + offset1, stride1 = _compute_linear_layout(dest, indices) + else + offset1, stride1 = 0, 0 + end SubArray{T,N,A,I,LD}(dest, indices, offset1, stride1) end # Get the proper trimmed shape @@ -172,7 +181,8 @@ Calling [`getindex`](@ref) or [`setindex!`](@ref) on the returned value (often a [`SubArray`](@ref)) computes the indices to access or modify the parent array on the fly. The behavior is undefined if the shape of the parent array is changed after `view` is called because there is no bound check for the parent array; e.g., -it may cause a segmentation fault. +it may cause a segmentation fault. It is likewise undefined behavior to modify the `inds` +array(s) after construction of the view. Some immutable parent arrays (like ranges) may choose to simply recompute a new array in some circumstances instead of returning @@ -298,15 +308,8 @@ reindex(idxs::Tuple{AbstractMatrix, Vararg{Any}}, subidxs::Tuple{Any, Any, Varar (@_propagate_inbounds_meta; (idxs[1][subidxs[1], subidxs[2]], reindex(tail(idxs), tail(tail(subidxs)))...)) # In general, we index N-dimensional parent arrays with N indices -@generated function reindex(idxs::Tuple{AbstractArray{T,N}, Vararg{Any}}, subidxs::Tuple{Vararg{Any}}) where {T,N} - if length(subidxs.parameters) >= N - subs = [:(subidxs[$d]) for d in 1:N] - tail = [:(subidxs[$d]) for d in N+1:length(subidxs.parameters)] - :(@_propagate_inbounds_meta; (idxs[1][$(subs...)], reindex(tail(idxs), ($(tail...),))...)) - else - :(throw(ArgumentError("cannot re-index SubArray with fewer indices than dimensions\nThis should not occur; please submit a bug report."))) - end -end +reindex(idxs::Tuple{AbstractArray{<:Any,N}, Vararg{Any}}, subidxs::Tuple{Vararg{Any}}) where {N} = + (@_propagate_inbounds_meta; (idxs[1][subidxs[1:N]...], reindex(tail(idxs), subidxs[N+1:end])...)) # In general, we simply re-index the parent indices by the provided ones SlowSubArray{T,N,P,I} = SubArray{T,N,P,I,false} @@ -319,7 +322,7 @@ end # But SubArrays with fast linear indexing pre-compute a stride and offset FastSubArray{T,N,P,I} = SubArray{T,N,P,I,true} -# We define a convenience functions to compute the shifted parent index +# We define convenience functions to compute the shifted parent index # This differs from reindex as this accepts the view directly, instead of its indices @inline _reindexlinear(V::FastSubArray, i::Int) = V.offset1 + V.stride1*i @inline _reindexlinear(V::FastSubArray, i::AbstractUnitRange{Int}) = V.offset1 .+ V.stride1 .* i @@ -436,7 +439,7 @@ strides(V::SubArray) = substrides(strides(V.parent), V.indices) substrides(strds::Tuple{}, ::Tuple{}) = () substrides(strds::NTuple{N,Int}, I::Tuple{ScalarIndex, Vararg{Any}}) where N = (substrides(tail(strds), tail(I))...,) substrides(strds::NTuple{N,Int}, I::Tuple{Slice, Vararg{Any}}) where N = (first(strds), substrides(tail(strds), tail(I))...) -substrides(strds::NTuple{N,Int}, I::Tuple{AbstractRange, Vararg{Any}}) where N = (first(strds)*step(I[1]), substrides(tail(strds), tail(I))...) +substrides(strds::NTuple{N,Int}, I::Tuple{AbstractRange, Vararg{Any}}) where N = (first(strds)*Int(step(I[1])), substrides(tail(strds), tail(I))...) substrides(strds, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError( LazyString("strides is invalid for SubArrays with indices of type ", typeof(I[1])))) @@ -447,8 +450,8 @@ compute_stride1(parent::AbstractArray, I::NTuple{N,Any}) where {N} = compute_stride1(s, inds, I::Tuple{}) = s compute_stride1(s, inds, I::Tuple{Vararg{ScalarIndex}}) = s compute_stride1(s, inds, I::Tuple{ScalarIndex, Vararg{Any}}) = - (@inline; compute_stride1(s*length(inds[1]), tail(inds), tail(I))) -compute_stride1(s, inds, I::Tuple{AbstractRange, Vararg{Any}}) = s*step(I[1]) + (@inline; compute_stride1(s*Int(length(inds[1])), tail(inds), tail(I))) +compute_stride1(s, inds, I::Tuple{AbstractRange, Vararg{Any}}) = s * Int(step(I[1])) compute_stride1(s, inds, I::Tuple{Slice, Vararg{Any}}) = s compute_stride1(s, inds, I::Tuple{Any, Vararg{Any}}) = throw(ArgumentError(LazyString("invalid strided index type ", typeof(I[1])))) @@ -466,7 +469,7 @@ first_index(V::SubArray) = compute_linindex(parent(V), V.indices) # The running sum is `f`; the cumulative stride product is `s`. # If the parent is a vector, then we offset the parent's own indices with parameters of I compute_offset1(parent::AbstractVector, stride1::Integer, I::Tuple{AbstractRange}) = - (@inline; first(I[1]) - stride1*first(axes1(I[1]))) + (@inline; Int(first(I[1])) - stride1*Int(first(axes1(I[1])))) # If the result is one-dimensional and it's a Colon, then linear # indexing uses the indices along the given dimension. # If the result is one-dimensional and it's a range, then linear @@ -475,20 +478,25 @@ compute_offset1(parent::AbstractVector, stride1::Integer, I::Tuple{AbstractRange compute_offset1(parent, stride1::Integer, I::Tuple) = (@inline; compute_offset1(parent, stride1, find_extended_dims(1, I...), find_extended_inds(I...), I)) compute_offset1(parent, stride1::Integer, dims::Tuple{Int}, inds::Tuple{Slice}, I::Tuple) = - (@inline; compute_linindex(parent, I) - stride1*first(axes(parent, dims[1]))) # index-preserving case + (@inline; compute_linindex(parent, I) - stride1*Int(first(axes(parent, dims[1])))) # index-preserving case compute_offset1(parent, stride1::Integer, dims, inds::Tuple{AbstractRange}, I::Tuple) = - (@inline; compute_linindex(parent, I) - stride1*first(axes1(inds[1]))) # potentially index-offsetting case + (@inline; compute_linindex(parent, I) - stride1*Int(first(axes1(inds[1])))) # potentially index-offsetting case compute_offset1(parent, stride1::Integer, dims, inds, I::Tuple) = (@inline; compute_linindex(parent, I) - stride1) function compute_linindex(parent, I::NTuple{N,Any}) where N @inline IP = fill_to_length(axes(parent), OneTo(1), Val(N)) - compute_linindex(first(LinearIndices(parent)), 1, IP, I) + compute_linindex(Int(first(LinearIndices(parent))), 1, IP, I) +end +function compute_linindex(f, s, IP::Tuple, I::Tuple{Any, Any, Vararg{Any}}) + @inline + Δi = Int(first(I[1])) - Int(first(IP[1])) + compute_linindex(f + Δi*s, s*Int(length(IP[1])), tail(IP), tail(I)) end -function compute_linindex(f, s, IP::Tuple, I::Tuple{Any, Vararg{Any}}) +function compute_linindex(f, s, IP::Tuple, I::Tuple{Any}) @inline - Δi = first(I[1])-first(IP[1]) - compute_linindex(f + Δi*s, s*length(IP[1]), tail(IP), tail(I)) + Δi = Int(first(I[1])) - Int(first(IP[1])) + f + Δi*s end compute_linindex(f, s, IP::Tuple, I::Tuple{}) = f diff --git a/base/summarysize.jl b/base/summarysize.jl index 9dfd1431b84c7..9057364babc83 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -6,17 +6,21 @@ struct SummarySize frontier_i::Vector{Int} exclude::Any chargeall::Any + count::Bool end nth_pointer_isdefined(obj, i::Int) = ccall(:jl_nth_pointer_isdefined, Cint, (Any, Csize_t), obj, i-1) != 0 get_nth_pointer(obj, i::Int) = ccall(:jl_get_nth_pointer, Any, (Any, Csize_t), obj, i-1) """ - Base.summarysize(obj; exclude=Union{...}, chargeall=Union{...})::Int + Base.summarysize(obj; count = false, exclude=Union{...}, chargeall=Union{...})::Int -Compute the amount of memory, in bytes, used by all unique objects reachable from the argument. +Compute all unique objects reachable from the argument and return either their size in +memory (in bytes) or the number of allocations they span. # Keyword Arguments +- `count`: if false, return the total size of the objects in memory. if true, return the + number of allocations spanned by the object. - `exclude`: specifies the types of objects to exclude from the traversal. - `chargeall`: specifies the types of objects to always charge the size of all of their fields, even if those fields would normally be excluded. @@ -33,13 +37,17 @@ julia> Base.summarysize(Ref(rand(100))) julia> sizeof(Ref(rand(100))) 8 + +julia> Base.summarysize(Core.svec(1.0, "testing", true); count=true) +4 ``` """ function summarysize(obj; + count::Bool = false, exclude = Union{DataType, Core.TypeName, Core.MethodInstance}, chargeall = Union{Core.TypeMapEntry, Method}) @nospecialize obj exclude chargeall - ss = SummarySize(IdDict(), Any[], Int[], exclude, chargeall) + ss = SummarySize(IdDict(), Any[], Int[], exclude, chargeall, count) size::Int = ss(obj) while !isempty(ss.frontier_x) # DFS heap traversal of everything without a specialization @@ -47,14 +55,33 @@ function summarysize(obj; x = ss.frontier_x[end] i = ss.frontier_i[end] val = nothing - if isa(x, SimpleVector) + if isa(x, Core.SimpleVector) nf = length(x) if isassigned(x, i) val = x[i] end + elseif isa(x, Core.CancellationTokenSource) + # the strong references of a source are its (hidden, trailing) + # parent links, enumerated here rather than via the layout + nf = Int(x.nparents) + val = _cancel_parent(x, i) + elseif isa(x, Core.WaitEntryN) + # `task` plus the (strong) owner/next pair of each hidden + # trailing wait slot + ns = _nslots(x) + nf = 1 + 2 * ns + if i == 1 + t = @atomic :monotonic x.task + t === nothing || (val = t) + else + si, k = divrem(i - 2, 2) + slot = slots(x)[si + 1] + v = k == 0 ? slot.owner : slot.next + v === nothing || (val = v) + end elseif isa(x, GenericMemory) T = eltype(x) - if Base.allocatedinline(T) + if allocatedinline(T) np = datatype_npointers(T) nf = length(x) * np idx = (i-1) ÷ np + 1 @@ -90,9 +117,9 @@ function summarysize(obj; return size end -(ss::SummarySize)(@nospecialize obj) = _summarysize(ss, obj) +(ss::SummarySize)(@nospecialize obj) = _summarysize(ss, obj, ss.count) # define the general case separately to make sure it is not specialized for every type -@noinline function _summarysize(ss::SummarySize, @nospecialize obj) +@noinline function _summarysize(ss::SummarySize, @nospecialize(obj), count::Bool) issingletontype(typeof(obj)) && return 0 # NOTE: this attempts to discover multiple copies of the same immutable value, # and so is somewhat approximate. @@ -112,7 +139,7 @@ end # 0-field mutable structs are not unique return gc_alignment(0) end - return sz + return count ? 1 : sz end (::SummarySize)(obj::Symbol) = 0 @@ -121,14 +148,13 @@ end function (ss::SummarySize)(obj::String) key = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) - return Core.sizeof(Int) + Core.sizeof(obj) + return (ss.count ? 1 : (Core.sizeof(Int) + Core.sizeof(obj))) end function (ss::SummarySize)(obj::DataType) key = pointer_from_objref(obj) haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) - size::Int = 7 * Core.sizeof(Int) + 6 * Core.sizeof(Int32) - size += 4 * nfields(obj) + ifelse(Sys.WORD_SIZE == 64, 4, 0) + size::Int = ss.count ? 1 : sizeof(DataType) size += ss(obj.parameters)::Int if isdefined(obj, :types) size += ss(obj.types)::Int @@ -139,19 +165,23 @@ end function (ss::SummarySize)(obj::Core.TypeName) key = pointer_from_objref(obj) haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) - return Core.sizeof(obj) + return (ss.count ? 1 : Core.sizeof(obj)) end function (ss::SummarySize)(obj::GenericMemory) haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true) - headersize = 2*sizeof(Int) - size::Int = headersize + headersize = 2 * sizeof(Int) + size::Int = (ss.count ? 1 : headersize) datakey = unsafe_convert(Ptr{Cvoid}, obj) if !haskey(ss.seen, datakey) ss.seen[datakey] = true - size += sizeof(obj) + if !ss.count + size += sizeof(obj) + elseif pointer_from_objref(obj) + 16 != datakey + size += 1 + end T = eltype(obj) - if !isempty(obj) && T !== Symbol && (!Base.allocatedinline(T) || (T isa DataType && !Base.datatype_pointerfree(T))) + if !isempty(obj) && T !== Symbol && (!allocatedinline(T) || (T isa DataType && !datatype_pointerfree(T))) push!(ss.frontier_x, obj) push!(ss.frontier_i, 1) end @@ -159,10 +189,10 @@ function (ss::SummarySize)(obj::GenericMemory) return size end -function (ss::SummarySize)(obj::SimpleVector) +function (ss::SummarySize)(obj::Core.SimpleVector) key = pointer_from_objref(obj) haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) - size::Int = Core.sizeof(obj) + size::Int = (ss.count ? 1 : Core.sizeof(obj)) if !isempty(obj) push!(ss.frontier_x, obj) push!(ss.frontier_i, 1) @@ -172,7 +202,7 @@ end function (ss::SummarySize)(obj::Module) haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true) - size::Int = Core.sizeof(obj) + size::Int = (ss.count ? 1 : Core.sizeof(obj)) for binding in names(obj, all = true) if isdefined(obj, binding) && !isdeprecated(obj, binding) value = getfield(obj, binding) @@ -191,9 +221,35 @@ function (ss::SummarySize)(obj::Module) return size end +function (ss::SummarySize)(obj::Core.CancellationTokenSource) + key = pointer_from_objref(obj) + haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) + # Variable-sized: Core.sizeof includes the trailing parent link entries. + # The (strong) parent references are traversed through the iterative + # frontier (see the branch in `summarysize`), which keeps deep chains + # off the stack and honors `exclude`; the child list is weak - a source + # does not keep its children alive - so it is deliberately not followed. + if Int(obj.nparents) > 0 + push!(ss.frontier_x, obj) + push!(ss.frontier_i, 1) + end + return ss.count ? 1 : Core.sizeof(obj) +end + +function (ss::SummarySize)(obj::Core.WaitEntryN) + key = pointer_from_objref(obj) + haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) + # Variable-sized: Core.sizeof includes the trailing wait slots, whose + # strong owner/next references (invisible to the layout) are traversed + # through the iterative frontier alongside `task`. + push!(ss.frontier_x, obj) + push!(ss.frontier_i, 1) + return ss.count ? 1 : Core.sizeof(obj) +end + function (ss::SummarySize)(obj::Task) haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true) - size::Int = Core.sizeof(obj) + size::Int = (ss.count ? 1 : Core.sizeof(obj)) if isdefined(obj, :code) size += ss(obj.code)::Int end @@ -204,4 +260,4 @@ function (ss::SummarySize)(obj::Task) return size end -(ss::SummarySize)(obj::BigInt) = _summarysize(ss, obj) + obj.alloc*sizeof(Base.GMP.Limb) +(ss::SummarySize)(obj::BigInt) = _summarysize(ss, obj, ss.count) + (ss.count ? 1 : obj.alloc * sizeof(GMP.Limb)) diff --git a/base/sysimg.jl b/base/sysimg.jl index fd71544c205cc..8466400948928 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -89,7 +89,7 @@ let :SHA, # transitive through Random :Sockets, # used by stream.jl - # Transitive through LingAlg + # Transitive through LinearAlgebra # OpenBLAS_jll # libblastrampoline_jll @@ -101,10 +101,9 @@ let # PackageCompiler can filter out stdlibs so it can be empty maxlen = maximum(textwidth.(string.(stdlibs)); init=0) - tot_time_stdlib = 0.0 # use a temp module to avoid leaving the type of this closure in Main push!(empty!(LOAD_PATH), "@stdlib") - m = Core.Module() + m = Module() GC.@preserve m begin print_time = @eval m (mod, t) -> (print(rpad(string(mod) * " ", $maxlen + 3, "─")); Base.time_print(stdout, t * 10^9); println()) diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 06e5cd298caf1..68104e1d6eecc 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -module Sys +# NB: This file is `Core.eval`-uated into the (pre-existing) module Sys + @doc """ Provide methods for retrieving information about hardware and the operating system. """ Sys @@ -8,12 +9,13 @@ Provide methods for retrieving information about hardware and the operating syst export BINDIR, STDLIB, CPU_THREADS, + EFFECTIVE_CPU_THREADS, CPU_NAME, WORD_SIZE, ARCH, MACHINE, - KERNEL, JIT, + PAGESIZE, cpu_info, cpu_summary, sysimage_target, @@ -23,24 +25,13 @@ export BINDIR, total_memory, free_physical_memory, total_physical_memory, - isapple, - isbsd, - isdragonfly, - isfreebsd, - islinux, - isnetbsd, - isopenbsd, - isunix, - iswindows, - isjsvm, isexecutable, isreadable, iswritable, username, - which, - detectwsl + which -import ..Base: show +import ..Base: DATAROOTDIR, show """ Sys.BINDIR::String @@ -54,12 +45,10 @@ global BINDIR::String = ccall(:jl_get_julia_bindir, Any, ())::String A string containing the full path to the directory containing the `stdlib` packages. """ -global STDLIB::String = "$BINDIR/../share/julia/stdlib/v$(VERSION.major).$(VERSION.minor)" # for bootstrap +global STDLIB::String = "$BINDIR/$DATAROOTDIR/julia/stdlib/v$(VERSION.major).$(VERSION.minor)" # for bootstrap # In case STDLIB change after julia is built, the variable below can be used # to update cached method locations to updated ones. const BUILD_STDLIB_PATH = STDLIB -# Similarly, this is the root of the julia repo directory that julia was built from -const BUILD_ROOT_PATH = "$BINDIR/../.." # helper to avoid triggering precompile warnings @@ -72,23 +61,35 @@ CPU cores, for example, in the presence of [hyper-threading](https://en.wikipedia.org/wiki/Hyper-threading). See Hwloc.jl or CpuId.jl for extended information, including number of physical cores. + +See also: [`Sys.EFFECTIVE_CPU_THREADS`](@ref) for a container-aware CPU count that respects +cgroup limits. """ global CPU_THREADS::Int = 1 # for bootstrap, changed on startup """ - Sys.ARCH::Symbol + Sys.EFFECTIVE_CPU_THREADS::Int -A symbol representing the architecture of the build configuration. -""" -const ARCH = ccall(:jl_get_ARCH, Any, ())::Symbol +The effective number of logical CPU cores available to the Julia process, taking into +account container limits (e.g., Docker `--cpus`, Kubernetes CPU limits, cgroup quotas). +This is the minimum of the hardware CPU thread count and any imposed CPU limits. +In non-containerized environments, this typically equals `Sys.CPU_THREADS`. In containerized +environments, it respects cgroup CPU limits and provides a more accurate measure of +available parallelism. +Use this constant when determining default thread pool sizes or parallelism levels to +ensure proper behavior in containerized deployments. """ - Sys.KERNEL::Symbol +global EFFECTIVE_CPU_THREADS::Int = 1 # for bootstrap, changed on startup -A symbol representing the name of the operating system, as returned by `uname` of the build configuration. """ -const KERNEL = ccall(:jl_get_UNAME, Any, ())::Symbol + Sys.ARCH::Symbol + +A symbol representing the architecture of the build configuration. +""" +const ARCH = ccall(:jl_get_ARCH, Any, ())::Symbol + """ Sys.MACHINE::String @@ -109,22 +110,20 @@ const WORD_SIZE = Core.sizeof(Int) * 8 The number of system "clock ticks" per second, corresponding to `sysconf(_SC_CLK_TCK)` on POSIX systems, or `0` if it is unknown. - -CPU times, e.g. as returned by `Sys.cpu_info()`, are in units of ticks, i.e. units of `1 / Sys.SC_CLK_TCK` seconds if `Sys.SC_CLK_TCK > 0`. """ global SC_CLK_TCK::Clong """ Sys.CPU_NAME::String -A string representing the name of CPU. +A string representing the name of the host CPU. # Examples For example, `Sys.CPU_NAME` might equal `"tigerlake"` on an [Intel Core "Tiger Lake" CPU](https://en.wikipedia.org/wiki/Tiger_Lake), or `"apple-m1"` on an [Apple M1 CPU](https://en.wikipedia.org/wiki/Apple_M1). -Note: Included in the detailed system information via `versioninfo(verbose=true)`. +Note: Included in the output of `versioninfo()`. """ global CPU_NAME::String @@ -140,10 +139,18 @@ julia> Sys.JIT "ORCJIT" ``` -Note: Included in the detailed system information via `versioninfo(verbose=true)`. +Note: Included in the output of `versioninfo()`. """ global JIT::String +""" + Sys.PAGESIZE::Clong + +A number providing the pagesize of the given OS. Common values being 4kb or 64kb on Linux. +On Windows, this stores the allocation granularity, not the page size. +""" +global PAGESIZE::Clong + function __init__() env_threads = nothing if haskey(ENV, "JULIA_CPU_THREADS") @@ -159,9 +166,11 @@ function __init__() else Int(ccall(:jl_cpu_threads, Int32, ())) end + global EFFECTIVE_CPU_THREADS = min(CPU_THREADS, Int(ccall(:jl_effective_threads, Int32, ()))) global SC_CLK_TCK = ccall(:jl_SC_CLK_TCK, Clong, ()) global CPU_NAME = ccall(:jl_get_cpu_name, Ref{String}, ()) global JIT = ccall(:jl_get_JIT, Ref{String}, ()) + global PAGESIZE = Int(Sys.isunix() ? ccall(:jl_getpagesize, Clong, ()) : ccall(:jl_getallocationgranularity, Clong, ())) __init_build() nothing end @@ -170,7 +179,7 @@ end function __init_build() global BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String vers = "v$(string(VERSION.major)).$(string(VERSION.minor))" - global STDLIB = abspath(BINDIR, "..", "share", "julia", "stdlib", vers) + global STDLIB = abspath(BINDIR, DATAROOTDIR, "julia", "stdlib", vers) nothing end @@ -196,8 +205,7 @@ The `CPUinfo` type is a mutable struct with the following fields: - `cpu_times!idle::UInt64`: Time spent in idle mode. CPU state shows the CPU time that's not actively being used. - `cpu_times!irq::UInt64`: Time spent handling interrupts. CPU state shows the amount of time the CPU has been servicing hardware interrupts. -The times are in units of `1/Sys.SC_CLK_TCK` seconds if `Sys.SC_CLK_TCK > 0`; otherwise they are in -unknown units. +The times are in units of milliseconds. Note: Included in the detailed system information via `versioninfo(verbose=true)`. """ @@ -218,7 +226,6 @@ CPUinfo(info::UV_cpu_info_t) = CPUinfo(unsafe_string(info.model), info.speed, public CPUinfo function _show_cpuinfo(io::IO, info::Sys.CPUinfo, header::Bool=true, prefix::AbstractString=" ") - tck = SC_CLK_TCK if header println(io, info.model, ": ") print(io, " "^length(prefix)) @@ -226,16 +233,13 @@ function _show_cpuinfo(io::IO, info::Sys.CPUinfo, header::Bool=true, prefix::Abs lpad("sys", 9), " ", lpad("idle", 9), " ", lpad("irq", 9)) end print(io, prefix) - unit = tck > 0 ? " s " : " " - tc = max(tck, 1) + ms_per_s = 1000 + unit = " s " d(i, unit=unit) = lpad(string(round(Int64,i)), 9) * unit print(io, lpad(string(info.speed), 5), " MHz ", - d(info.cpu_times!user / tc), d(info.cpu_times!nice / tc), d(info.cpu_times!sys / tc), - d(info.cpu_times!idle / tc), d(info.cpu_times!irq / tc, tck > 0 ? " s" : " ")) - if tck <= 0 - print(io, "ticks") - end + d(info.cpu_times!user / ms_per_s), d(info.cpu_times!nice / ms_per_s), d(info.cpu_times!sys / ms_per_s), + d(info.cpu_times!idle / ms_per_s), d(info.cpu_times!irq / ms_per_s)) end show(io::IO, ::MIME"text/plain", info::CPUinfo) = _show_cpuinfo(io, info, true, " ") @@ -252,7 +256,7 @@ function _cpu_summary(io::IO, cpu::AbstractVector{CPUinfo}, i, j) summary = CPUinfo(cpu[i].model,0,0,0,0,0,0) count = j - i + 1 for x = i:j - summary.speed += cpu[i].speed + summary.speed += cpu[x].speed summary.cpu_times!user += cpu[x].cpu_times!user summary.cpu_times!nice += cpu[x].cpu_times!nice summary.cpu_times!sys += cpu[x].cpu_times!sys @@ -283,6 +287,7 @@ function cpu_summary(io::IO=stdout, cpu::AbstractVector{CPUinfo} = cpu_info()) if model != cpu[i].model _cpu_summary(io, cpu, first, i-1) first = i + model = cpu[i].model end end _cpu_summary(io, cpu, first, length(cpu)) @@ -333,7 +338,7 @@ end """ Sys.uptime() -Gets the current system uptime in seconds. +Get the current system uptime in seconds. """ function uptime() uptime_ = Ref{Float64}() @@ -425,157 +430,6 @@ See also: """ maxrss() = ccall(:jl_maxrss, Csize_t, ()) -""" - Sys.isunix([os]) - -Predicate for testing if the OS provides a Unix-like interface. -See documentation in [Handling Operating System Variation](@ref). -""" -function isunix(os::Symbol) - if iswindows(os) - return false - elseif islinux(os) || isbsd(os) - return true - elseif os === :Emscripten - # Emscripten implements the POSIX ABI and provides traditional - # Unix-style operating system functions such as file system support. - # Therefore, we consider it a unix, even though this need not be - # generally true for a jsvm embedding. - return true - else - throw(ArgumentError("unknown operating system \"$os\"")) - end -end - -""" - Sys.islinux([os]) - -Predicate for testing if the OS is a derivative of Linux. -See documentation in [Handling Operating System Variation](@ref). -""" -islinux(os::Symbol) = (os === :Linux) - -""" - Sys.isbsd([os]) - -Predicate for testing if the OS is a derivative of BSD. -See documentation in [Handling Operating System Variation](@ref). - -!!! note - The Darwin kernel descends from BSD, which means that `Sys.isbsd()` is - `true` on macOS systems. To exclude macOS from a predicate, use - `Sys.isbsd() && !Sys.isapple()`. -""" -isbsd(os::Symbol) = (isfreebsd(os) || isopenbsd(os) || isnetbsd(os) || isdragonfly(os) || isapple(os)) - -""" - Sys.isfreebsd([os]) - -Predicate for testing if the OS is a derivative of FreeBSD. -See documentation in [Handling Operating System Variation](@ref). - -!!! note - Not to be confused with `Sys.isbsd()`, which is `true` on FreeBSD but also on - other BSD-based systems. `Sys.isfreebsd()` refers only to FreeBSD. -!!! compat "Julia 1.1" - This function requires at least Julia 1.1. -""" -isfreebsd(os::Symbol) = (os === :FreeBSD) - -""" - Sys.isopenbsd([os]) - -Predicate for testing if the OS is a derivative of OpenBSD. -See documentation in [Handling Operating System Variation](@ref). - -!!! note - Not to be confused with `Sys.isbsd()`, which is `true` on OpenBSD but also on - other BSD-based systems. `Sys.isopenbsd()` refers only to OpenBSD. -!!! compat "Julia 1.1" - This function requires at least Julia 1.1. -""" -isopenbsd(os::Symbol) = (os === :OpenBSD) - -""" - Sys.isnetbsd([os]) - -Predicate for testing if the OS is a derivative of NetBSD. -See documentation in [Handling Operating System Variation](@ref). - -!!! note - Not to be confused with `Sys.isbsd()`, which is `true` on NetBSD but also on - other BSD-based systems. `Sys.isnetbsd()` refers only to NetBSD. -!!! compat "Julia 1.1" - This function requires at least Julia 1.1. -""" -isnetbsd(os::Symbol) = (os === :NetBSD) - -""" - Sys.isdragonfly([os]) - -Predicate for testing if the OS is a derivative of DragonFly BSD. -See documentation in [Handling Operating System Variation](@ref). - -!!! note - Not to be confused with `Sys.isbsd()`, which is `true` on DragonFly but also on - other BSD-based systems. `Sys.isdragonfly()` refers only to DragonFly. -!!! compat "Julia 1.1" - This function requires at least Julia 1.1. -""" -isdragonfly(os::Symbol) = (os === :DragonFly) - -""" - Sys.iswindows([os]) - -Predicate for testing if the OS is a derivative of Microsoft Windows NT. -See documentation in [Handling Operating System Variation](@ref). -""" -iswindows(os::Symbol) = (os === :Windows || os === :NT) - -""" - Sys.isapple([os]) - -Predicate for testing if the OS is a derivative of Apple Macintosh OS X or Darwin. -See documentation in [Handling Operating System Variation](@ref). -""" -isapple(os::Symbol) = (os === :Apple || os === :Darwin) - -""" - Sys.isjsvm([os]) - -Predicate for testing if Julia is running in a JavaScript VM (JSVM), -including e.g. a WebAssembly JavaScript embedding in a web browser. - -!!! compat "Julia 1.2" - This function requires at least Julia 1.2. -""" -isjsvm(os::Symbol) = (os === :Emscripten) - -""" - Sys.detectwsl() - -Runtime predicate for testing if Julia is running inside -Windows Subsystem for Linux (WSL). - -!!! note - Unlike `Sys.iswindows`, `Sys.islinux` etc., this is a runtime test, and thus - cannot meaningfully be used in `@static if` constructs. - -!!! compat "Julia 1.12" - This function requires at least Julia 1.12. -""" -function detectwsl() - # We use the same approach as canonical/snapd do to detect WSL - islinux() && ( - isfile("/proc/sys/fs/binfmt_misc/WSLInterop") - || isdir("/run/WSL") - ) -end - -for f in (:isunix, :islinux, :isbsd, :isapple, :iswindows, :isfreebsd, :isopenbsd, :isnetbsd, :isdragonfly, :isjsvm) - @eval $f() = $(getfield(@__MODULE__, f)(KERNEL)) -end - if iswindows() function windows_version() verinfo = ccall(:GetVersion, UInt32, ()) @@ -622,7 +476,7 @@ function which(program_name::String) push!(program_names, base_pname) end - # But also try appending .exe and .com` + # But also try appending .exe and .com for pe in (".exe", ".com") push!(program_names, string(base_pname, pe)) end @@ -676,7 +530,7 @@ function which(program_name::String) # If we couldn't find anything, don't return anything nothing end -which(program_name::AbstractString) = which(String(program_name)) +which(program_name::AbstractString) = which(String(program_name)::String) """ Sys.username()::String @@ -700,5 +554,3 @@ function username() isempty(pw.username) && Base.uv_error("username", Base.UV_ENOENT) return pw.username end - -end # module Sys diff --git a/base/task.jl b/base/task.jl index 4f17330bb4455..6d337e41b1ee2 100644 --- a/base/task.jl +++ b/base/task.jl @@ -2,7 +2,11 @@ ## basic task functions and TLS -Core.Task(@nospecialize(f), reserved_stack::Int=0) = Core._Task(f, reserved_stack, ThreadSynchronizer()) +function Core.Task(@nospecialize(f), reserved_stack::Int=0) + task = Core._task(f, reserved_stack) + task.donenotify = ThreadSynchronizer() + return task +end # Container for a captured exception and its backtrace. Can be serialized. struct CapturedException <: Exception @@ -28,7 +32,7 @@ end """ capture_exception(ex, bt)::Exception -Returns an exception, possibly incorporating information from a backtrace `bt`. Defaults to returning [`CapturedException(ex, bt)`](@ref). +Return an exception, possibly incorporating information from a backtrace `bt`. Defaults to returning [`CapturedException(ex, bt)`](@ref). Used in [`asyncmap`](@ref) and [`asyncmap!`](@ref) to capture exceptions thrown during the user-supplied function call. @@ -139,15 +143,17 @@ true ``` """ macro task(ex) - thunk = Base.replace_linenums!(:(()->$(esc(ex))), __source__) + thunk = replace_linenums!(:(()->$(esc(ex))), __source__) :(Task($thunk)) end # task states -const task_state_runnable = UInt8(0) -const task_state_done = UInt8(1) -const task_state_failed = UInt8(2) +const task_state_runnable = UInt8(0) +const task_state_done = UInt8(1) +const task_state_failed = UInt8(2) +# like _failed, but the task was forcibly abandoned and may have leaked resources +const task_state_abandoned = UInt8(3) @inline function getproperty(t::Task, field::Symbol) if field === :state @@ -159,8 +165,10 @@ const task_state_failed = UInt8(2) return :done elseif st === task_state_failed return :failed + elseif st === task_state_abandoned + return :abandoned else - @assert false + @assert false "unexpected state" end elseif field === :backtrace # TODO: this field name should be deprecated in 2.0 @@ -172,6 +180,8 @@ const task_state_failed = UInt8(2) error(""" Querying a Task's `scope` field is disallowed. The private `Core.current_scope()` function is better, though still an implementation detail.""") + elseif field === :invoked + error("Querying a Task's `invoked` field is disallowed because it is an implementation detail.") else return getfield(t, field) end @@ -180,6 +190,10 @@ end @inline function setproperty!(t::Task, field::Symbol, @nospecialize(v)) if field === :scope istaskstarted(t) && error("Setting scope on a started task directly is disallowed.") + elseif field === :invoked + error("Setting a Task's `invoked` field directly is disallowed because it is an implementation detail.") + elseif field === :result + error("Setting a Task's `result` field directly is disallowed. The result of a task is determined by the return value of its code; to pass a value to a suspended task, use `schedule(t, val)` or `yieldto(t, val)` instead.") end return @invoke setproperty!(t::Any, field::Symbol, v::Any) end @@ -250,7 +264,94 @@ true !!! compat "Julia 1.3" This function requires at least Julia 1.3. """ -istaskfailed(t::Task) = ((@atomic :acquire t._state) === task_state_failed) +function istaskfailed(t::Task) + st = @atomic :acquire t._state + return st === task_state_failed || st === task_state_abandoned +end + +""" + unsafe_abandon!(t::Task, next_task::Task) -> Bool + +Forcibly abandon task `t` and switch its thread to `next_task`, discarding +`t`'s execution. Returns `true` if the abandonment committed: `t`'s state is +`:abandoned` and it will never run another instruction. + +Returns `false` - with `t` untouched and still running - when the abandonment +could not be performed safely: `t` was not (or no longer) running on a +thread, was found holding runtime state that must not be discarded (runtime +locks, an in-flight finalizer or GC transition, a signal-deferral region), +or another abandonment was already in flight for that thread. Blocks (in a +scheduler-friendly wait) until the delivery settles the request one way or +the other; a victim thread that never services signals blocks the call +indefinitely. + +`next_task` must be a fresh, never-scheduled task; it takes over the +victim's thread. + +!!! warning + Abandonment discards the victim's execution wherever it stands. Any + non-runtime resource it holds (locks, buffers, connections) is leaked. + This is a last-resort recovery primitive. + +!!! note + The task must be currently running on a thread for this to have effect; + use [`cancel!`](@ref) with `CANCEL_REQUEST_ABANDON_ALL` to also stop + parked or queued tasks. +""" +unsafe_abandon!(t::Task, next_task::Task) = + unsafe_abandon!(t, next_task, CancellationRequest(0x4)) # CANCEL_REQUEST_ABANDON_ALL + +function unsafe_abandon!(t::Task, next_task::Task, @nospecialize(result)) + # The requester's own wakeup handle, staged with the request in the + # victim thread's abandon slot: the delivery paths ping it (from signal + # context, where uv_async_send is the one legal wakeup) when the + # request settles. One requester per slot means one consumer per + # handle, which is exactly the AsyncCondition trigger's latched, + # consume-once semantics - a settle that lands inside the check/park + # window below is caught by the latch. + async = AsyncCondition() + tid = ccall(:jl_abandon_task_request, Cint, (Any, Any, Any, Ptr{Cvoid}), + t, next_task, result, async.handle) + if tid < 0 + close(async) + return false + end + tid = tid % Int16 + ok = false + try + while true + verdict = ccall(:jl_abandon_task_poll, Cint, (Int16,), tid) + if verdict == 1 || verdict == -1 + ok = verdict == 1 + break + elseif verdict == 2 + # mid-settle: the ping was already sent (pre-terminal, so it + # can never race this handle's close below); the verdict is + # microseconds away + ccall(:jl_cpu_pause, Cvoid, ()) + else + wait(async; cancel=nothing) + end + end + finally + # Safe only after the terminal consume: pings happen strictly + # before the terminal state becomes visible. + close(async) + end + if ok + # A forcibly abandoned task never goes through the regular task + # completion path, so wake up anyone waiting on it. (The waiters + # observe the already-stored abandoned state; they do not touch the + # task's stack. The root task's donenotify may be `nothing`.) + donenotify = t.donenotify + if donenotify isa ThreadSynchronizer + lock(donenotify) + notify(donenotify) + unlock(donenotify) + end + end + return ok +end Threads.threadid(t::Task) = Int(ccall(:jl_get_task_tid, Int16, (Any,), t)+1) function Threads.threadpool(t::Task) @@ -302,26 +403,44 @@ function task_local_storage(body::Function, key, val) end # just wait for a task to be done, no error propagation -function _wait(t::Task) - t === current_task() && Core.throw(ConcurrencyViolationError("deadlock detected: cannot wait on current task")) +_wait(t::Task; cancel::CancelTokenArg=DEFAULT_CANCEL) = + _wait(t, resolve_cancel_token(cancel)) +# With `cancel_value`, a cancellation of the governing token (at or above +# `min_severity`) is returned as the `CancellationRequest` instead of being +# thrown; `nothing` means the task completed. +function _wait(t::Task, tok::MaybeToken; min_severity::UInt8=0x00, + cancel_value::Bool=false) + t === current_task() && throw(ConcurrencyViolationError("deadlock detected: cannot wait on current task")) if !istaskdone(t) donenotify = t.donenotify::ThreadSynchronizer lock(donenotify) + locked = true try while !istaskdone(t) - wait(donenotify) + locked = false + r = wait(donenotify, tok; min_severity=min_severity, + cancel_value=cancel_value) + locked = true + if cancel_value && r isa CancellationRequest + return r + end end finally - unlock(donenotify) + locked && unlock(donenotify) end end nothing end -# have `waiter` wait for `t` -function _wait2(t::Task, waiter::Task) +waitqueue(t::Task) = waitqueue(t.donenotify::ThreadSynchronizer) + +# Subscribe the not-yet-started `waiter` to `t`'s completion (see the +# GenericCondition method's contract in condition.jl: a start trigger +# governed by the waiter's birth cancellation source, not a park) +function schedule_on_notify!(t::Task, waiter::Task) + _assert_fresh_waiter(waiter) if !istaskdone(t) - # since _wait2 is similar to schedule, we should observe the sticky + # since this is similar to schedule, we should observe the sticky # bit, even if we don't call `schedule` with early-return below if waiter.sticky && Threads.threadid(waiter) == 0 && !GC.in_finalizer() # Issue #41324 @@ -335,30 +454,48 @@ function _wait2(t::Task, waiter::Task) end donenotify = t.donenotify::ThreadSynchronizer lock(donenotify) - if !istaskdone(t) - push!(donenotify.waitq, waiter) - unlock(donenotify) - return nothing - else + try + if !istaskdone(t) + schedule_on_notify!(donenotify, waiter) + return nothing + end + finally unlock(donenotify) end end - schedule(waiter) + # `t` already done: start (or, under a cancelled birth source, kill) + # the waiter now + _assert_fresh_waiter(waiter) + src = _birth_cancel_source(waiter) + if src !== nothing && iscancelled(src) + _schedule_subscription_cancelled(waiter, src) + else + schedule(waiter) + end nothing end """ - wait(t::Task; throw=true) + wait(t::Task; throw=true, cancel=Base.DEFAULT_CANCEL) Wait for a `Task` to finish. The keyword `throw` (defaults to `true`) controls whether a failed task results in an error, thrown as a [`TaskFailedException`](@ref) which wraps the failed task. +The `cancel` keyword argument controls which cancellation token may interrupt +the wait (see [`CancellationToken`](@ref)); by default the scoped token. A +cancelled wait throws the [`CancellationRequest`](@ref) and leaves `t` +unaffected: cancellation reaches `t` only through its own governing token +(e.g. when both waiter and waitee run under the same cancelled scope). + Throws a `ConcurrencyViolationError` if `t` is the currently running task, to prevent deadlocks. """ -function wait(t::Task; throw=true) - _wait(t) +wait(t::Task; throw=true, cancel::CancelTokenArg=DEFAULT_CANCEL) = + wait(t, check_cancel_arg(cancel); throw) +@noinline function wait(t::Task, tok::MaybeToken; throw=true) + # Inlining a blocking call buys nothing; this also keeps the inlineable `fetch(::Task)` small. + _wait(t, tok) if throw && istaskfailed(t) Core.throw(TaskFailedException(t)) end @@ -383,8 +520,12 @@ completed tasks, and the other consists of uncompleted tasks. each runs serially, since this needs to scan the list of `tasks` each time and synchronize with each one every time this is called. Or consider using [`waitall(tasks; failfast=true)`](@ref waitall) instead. + +!!! compat "Julia 1.12" + This function requires at least Julia 1.12. """ -waitany(tasks; throw=true) = _wait_multiple(tasks, throw) +waitany(tasks; throw=true, cancel::CancelTokenArg=DEFAULT_CANCEL) = + _wait_multiple(collect_tasks(tasks), throw, false, false, check_cancel_arg(cancel)) """ waitall(tasks; failfast=true, throw=true) -> (done_tasks, remaining_tasks) @@ -400,23 +541,75 @@ given tasks is finished by exception. If `throw` is `true`, throw The return value consists of two task vectors. The first one consists of completed tasks, and the other consists of uncompleted tasks. + +!!! compat "Julia 1.12" + This function requires at least Julia 1.12. """ -waitall(tasks; failfast=true, throw=true) = _wait_multiple(tasks, throw, true, failfast) +waitall(tasks; failfast=true, throw=true, cancel::CancelTokenArg=DEFAULT_CANCEL) = + _wait_multiple(collect_tasks(tasks), throw, true, failfast, check_cancel_arg(cancel)) -function _wait_multiple(waiting_tasks, throwexc=false, all=false, failfast=false) +function collect_tasks(waiting_tasks) tasks = Task[] - for t in waiting_tasks t isa Task || error("Expected an iterator of `Task` object") push!(tasks, t) end + return tasks +end +## Task completion as a waitable (see base/park.jl): a one-shot predicate +## kind - enqueue declines when the task is already done (its only notify +## has fired), and the recheck is membership-qualified: the slot witness, +## cleared when the completion notify pops the entry, is the +## already-delivered bit (without it a repark! would re-fire forever on +## consumed completions). +struct DoneWait + t::Task +end + +function wait_enqueue!(x::DoneWait, w::WaitEntry, first::Bool) + t = x.t + donenotify = t.donenotify::ThreadSynchronizer + lock(donenotify) + if istaskdone(t) + unlock(donenotify) + return false + end + # a duplicate of an already-registered task shares its slot + if _find_slot(w, donenotify) == 0 + push!(waitqueue(t), w) + end + unlock(donenotify) + return true +end + +function wait_recheck(x::DoneWait, w::WaitEntry) + t = x.t + istaskdone(t) || return false + return _find_slot(w, t.donenotify::ThreadSynchronizer) != 0 +end + +function wait_dequeue!(x::DoneWait, w::WaitEntry, why::UInt8) + # lazy on a normal wake: the claiming completion notify popped its own + # registration; eager everywhere else (fired slots must not re-fire, + # withdrawal and cleanup must not leave the entry reachable) + why == WAKE_VALUE && return nothing + t = x.t + donenotify = t.donenotify::ThreadSynchronizer + lock(donenotify) + list_deletefirst!(waitqueue(t), w) + unlock(donenotify) + return nothing +end + +function _wait_multiple(tasks::Vector{Task}, throwexc::Bool=false, all::Bool=false, failfast::Bool=false, + tok::MaybeToken=default_cancel_token()) if (all && !failfast) || length(tasks) <= 1 exception = false # Force everything to finish synchronously for the case of waitall # with failfast=false for t in tasks - _wait(t) + _wait(t, tok) exception |= istaskfailed(t) end if exception && throwexc @@ -435,74 +628,83 @@ function _wait_multiple(waiting_tasks, throwexc=false, all=false, failfast=false done_mask[i] = true exception |= istaskfailed(t) nremaining -= 1 - else - done_mask[i] = false end end - if nremaining == 0 - return tasks, Task[] - elseif any(done_mask) && (!all || (failfast && exception)) + # We can return early if all tasks are done, or if any is done and we only + # needed to wait for one, or if any task failed and we have failfast + if nremaining == 0 || (any(done_mask) && (!all || (failfast && exception))) if throwexc && (!all || failfast) && exception exceptions = [TaskFailedException(t) for t in tasks[done_mask] if istaskfailed(t)] throw(CompositeException(exceptions)) else - return tasks[done_mask], tasks[.~done_mask] + if nremaining == 0 + return tasks, Task[] + else + return tasks[done_mask], tasks[.~done_mask] + end end end - chan = Channel{Int}(Inf) - sentinel = current_task() - waiter_tasks = fill(sentinel, length(tasks)) - + # Re-run `park!` for pending tasks after each wake: the previous park + # may have stopped at a completed task, and notifications while unarmed + # can consume registrations. Keep registrations for pending tasks, but + # synchronize with completed tasks' notifiers before reusing their slots. + ct = current_task() + src = cancel_source(tok) + src === nothing || checkcancel(src) + ws = Vector{Union{DoneWait, SourceWait}}() for (i, done) in enumerate(done_mask) - done && continue - t = tasks[i] - if istaskdone(t) - done_mask[i] = true - exception |= istaskfailed(t) - nremaining -= 1 - exception && failfast && break - else - waiter = @task put!(chan, i) - waiter.sticky = false - _wait2(t, waiter) - waiter_tasks[i] = waiter - end + done || push!(ws, DoneWait(tasks[i])) end - - while nremaining > 0 - i = take!(chan) - t = tasks[i] - waiter_tasks[i] = sentinel - done_mask[i] = true - exception |= istaskfailed(t) - nremaining -= 1 - - # stop early if requested, unless there is something immediately - # ready to consume from the channel (using a race-y check) - if (!all || (failfast && exception)) && !isready(chan) + src === nothing || push!(ws, SourceWait(src, 0x00)) + w = acquire_wait_entry!(ct, ws) + while true + # suspend only when the park armed (a `false` park means a waitable + # fired - a completion, or the source - and the self-claim won) + park!(ws, w, false) && wait_safe_interrupt(ws, w) + # the fired-source outcome (and, level-triggered, any cancelled + # state) delivers here: withdraw and throw + if src !== nothing && iscancelled(src) + withdraw!(ws, w) + checkcancel(src) + end + # collect completions (a wake happens-after its completing notify, + # so the istaskdone reads below observe it) + for (i, done) in enumerate(done_mask) + done && continue + t = tasks[i] + if istaskdone(t) + wait_dequeue!(DoneWait(t), w, WAKE_WITHDRAWN) + done_mask[i] = true + exception |= istaskfailed(t) + nremaining -= 1 + end + end + if nremaining == 0 || (!all && any(done_mask)) || (exception && failfast) break end + empty!(ws) + for (i, done) in enumerate(done_mask) + done || push!(ws, DoneWait(tasks[i])) + end + src === nothing || push!(ws, SourceWait(src, 0x00)) end - - close(chan) + withdraw!(ws, w) if nremaining == 0 + if throwexc && exception + exceptions = [TaskFailedException(t) for t in tasks if istaskfailed(t)] + throw(CompositeException(exceptions)) + end return tasks, Task[] else - remaining_mask = .~done_mask - for i in findall(remaining_mask) - waiter = waiter_tasks[i] - donenotify = tasks[i].donenotify::ThreadSynchronizer - @lock donenotify Base.list_deletefirst!(donenotify.waitq, waiter) - end done_tasks = tasks[done_mask] if throwexc && exception exceptions = [TaskFailedException(t) for t in done_tasks if istaskfailed(t)] throw(CompositeException(exceptions)) else - return done_tasks, tasks[remaining_mask] + return done_tasks, tasks[.~done_mask] end end end @@ -521,18 +723,36 @@ Wait for a [`Task`](@ref) to finish, then return its result value. If the task fails with an exception, a [`TaskFailedException`](@ref) (which wraps the failed task) is thrown. """ -function fetch(t::Task) - wait(t) - return task_result(t) +@inline function fetch(t::Task; cancel::CancelTokenArg=DEFAULT_CANCEL) + # `cancel` governs the *wait* for the task: a cancellation unwinds this + # fetch, the fetched task keeps running. + wait(t; cancel) + # This typeassert looks redundant, but is required for soundness and must not be + # removed: `Task.code`/`Task.result` are mutable, so the precise type inference + # may derive here (via `PartialTask`) is a claim that must be re-checked at + # runtime, not a proven fact. + return task_result(t)::Core.task_result_type(t) end - ## lexically-scoped waiting for multiple items struct ScheduledAfterSyncException <: Exception values::Vector{Any} end +function showerror(io::IO, cr::CancellationRequest) + print(io, "CancellationRequest: ") + if cr === CANCEL_REQUEST_SAFE + print(io, "Safe Cancellation (CANCEL_REQUEST_SAFE)") + elseif cr === CANCEL_REQUEST_ABANDON_EXTERNAL + print(io, "Abandonment of External Resources (CANCEL_REQUEST_ABANDON_EXTERNAL)") + elseif cr === CANCEL_REQUEST_ABANDON_ALL + print(io, "Task Abandonment (CANCEL_REQUEST_ABANDON_ALL)") + else + print(io, "Unknown ($(cr.request))") + end +end + function showerror(io::IO, ex::ScheduledAfterSyncException) print(io, "ScheduledAfterSyncException: ") if isempty(ex.values) @@ -552,12 +772,20 @@ function showerror(io::IO, ex::ScheduledAfterSyncException) print(io, " registered after the end of a `@sync` block") end -function sync_end(c::Channel{Any}) +function sync_end(c::Channel{Any}, src::Union{Nothing, CancellationTokenSource}=nothing) local c_ex + tok = src === nothing ? nothing : CancellationToken(src) while isready(c) r = take!(c) if isa(r, Task) - _wait(r) + cancelled = _wait(r, tok; cancel_value=tok !== nothing) + if cancelled isa CancellationRequest + # Our own scope (or an ancestor) was cancelled. The children + # run under the same scope's token, so the tree walk already + # cancelled them all; await their teardown per severity. + return sync_cancel!(c, r, cancelled, tok, + @isdefined(c_ex) ? c_ex : CompositeException()) + end if istaskfailed(r) if !@isdefined(c_ex) c_ex = CompositeException() @@ -605,6 +833,57 @@ end const sync_varname = gensym(:sync) +# Teardown of a `@sync` block whose own scope was cancelled: the scope's +# token subtree (covering every child) is already cancelled; await the +# children's unwind per the severity policy. Our own acknowledgement of the +# request lets these teardown waits park; they are only re-woken by a +# severity escalation (`min_severity`). +@noinline function sync_cancel!(c::Channel{Any}, t::Task, cr::CancellationRequest, + tok::CancellationToken, c_ex::CompositeException) + waitees = Any[t] + while isready(c) + push!(waitees, take!(c)) + end + close(c) + sev = severity(cr) + for r in waitees + if isa(r, Task) + while sev < CANCEL_REQUEST_ABANDON_ALL.request + # Tasks are internal: their cancellation is awaited (for + # ABANDON_ALL they were frozen; there is nothing to wait + # for). A severity escalation completes the teardown wait + # (value-mode; only severities above the acknowledged one + # are admitted) - adopt the stronger request and keep + # awaiting internal tasks per its policy rather than + # unwinding out of the `@sync` while children are still + # running. + r2 = _wait(r, tok; min_severity=sev + 0x01, cancel_value=true) + r2 isa CancellationRequest || break + cr = r2 + sev = severity(r2) + end + if istaskfailed(r) + push!(c_ex, TaskFailedException(r)) + end + else + # Non-task waitees are external - the ABANDON_* severities cease + # waiting for external resources. + sev == CANCEL_REQUEST_SAFE.request || continue + try + wait(r) + catch e + push!(c_ex, e) + end + end + end + # Reporting the composite outcome constitutes delivery of the request; + # include the request itself if no child failure already records it. + if isempty(c_ex) + throw(cr) + end + throw(c_ex) +end + """ @sync @@ -628,10 +907,21 @@ Thread-id 1, task 2 """ macro sync(block) var = esc(sync_varname) + # The block runs in a new dynamic scope carrying the token of a fresh + # cancellation source linked under the enclosing scope's token, so that + # cancellation of the enclosing scope reaches every (transitively + # spawned) child through the token tree. This expands the equivalent of + # `@with CANCEL_TOKEN => token ...` manually: the ScopedValues macro API + # is not loaded yet when Base code containing `@sync` is compiled during + # bootstrap. + scoped_block = Expr(:tryfinally, esc(block), nothing, + :(Scope(Core.current_scope()::Union{Nothing, Scope}, + CANCEL_TOKEN => CancellationToken(var"#sync_src#")))) quote - let $var = Channel(Inf) - v = $(esc(block)) - sync_end($var) + let var"#sync_src#" = CancellationTokenSource(default_cancel_token()), + $var = Channel(Inf) + v = $scoped_block + sync_end($var, var"#sync_src#") v end end @@ -660,15 +950,15 @@ isolating the asynchronous code from changes to the variable's value in the curr Interpolating values via `\$` is available as of Julia 1.4. """ macro async(expr) - do_async_macro(expr, __source__) + do_async_macro(expr, __source__, identity) end # generate the code for @async, possibly wrapping the task in something before # pushing it to the wait queue. -function do_async_macro(expr, linenums; wrap=identity) - letargs = Base._lift_one_interp!(expr) +function do_async_macro(expr, linenums, wrap) + letargs = _lift_one_interp!(expr) - thunk = Base.replace_linenums!(:(()->($(esc(expr)))), linenums) + thunk = replace_linenums!(:(()->($(esc(expr)))), linenums) var = esc(sync_varname) quote let $(letargs...) @@ -708,7 +998,7 @@ fetch(t::UnwrapTaskFailedException) = unwrap_task_failed(fetch, t) # macro for running async code that doesn't throw wrapped exceptions macro async_unwrap(expr) - do_async_macro(expr, __source__, wrap=task->:(Base.UnwrapTaskFailedException($task))) + do_async_macro(expr, __source__, taskvar->:(UnwrapTaskFailedException($taskvar))) end """ @@ -725,7 +1015,10 @@ Stacktrace: ``` """ function errormonitor(t::Task) - t2 = Task() do + # the monitor is diagnostic cleanup: shield it from the constructing + # scope's cancellation so a cancelled scope still gets its report + t2 = ScopedValues.with(CANCEL_TOKEN => nothing) do + Task() do if istaskfailed(t) local errs = stderr try # try to display the failure atomically @@ -752,35 +1045,44 @@ function errormonitor(t::Task) end nothing end + end t2.sticky = false - _wait2(t, t2) + schedule_on_notify!(t, t2) return t end # Capture interpolated variables in $() and move them to let-block -function _lift_one_interp!(e) +function _lift_one_interp!(@nospecialize e) letargs = Any[] # store the new gensymed arguments - _lift_one_interp_helper(e, false, letargs) # Start out _not_ in a quote context (false) - letargs + _lift_one_interp_helper(e, false, 0, letargs) # Start out _not_ in a quote context (false) and not needing escapes + return letargs end -_lift_one_interp_helper(v, _, _) = v -function _lift_one_interp_helper(expr::Expr, in_quote_context, letargs) +_lift_one_interp_helper(@nospecialize(v), _::Bool, _::Int, _::Vector{Any}) = v +function _lift_one_interp_helper(expr::Expr, in_quote_context::Bool, escs::Int, letargs::Vector{Any}) if expr.head === :$ if in_quote_context # This $ is simply interpolating out of the quote # Now, we're out of the quote, so any _further_ $ is ours. in_quote_context = false - else + elseif escs == 0 + # if escs is non-zero, then we cannot hoist expr.args without violating hygiene rules newarg = gensym() push!(letargs, :($(esc(newarg)) = $(esc(expr.args[1])))) return newarg # Don't recurse into the lifted $() exprs end + elseif expr.head === :meta || expr.head === :inert + return expr elseif expr.head === :quote in_quote_context = true # Don't try to lift $ directly out of quotes elseif expr.head === :macrocall return expr # Don't recur into macro calls, since some other macros use $ + elseif expr.head === :var"hygienic-scope" + escs += 1 + elseif expr.head === :escape + escs == 0 && return expr + escs -= 1 end for (i,e) in enumerate(expr.args) - expr.args[i] = _lift_one_interp_helper(e, in_quote_context, letargs) + expr.args[i] = _lift_one_interp_helper(e, in_quote_context, escs, letargs) end expr end @@ -809,6 +1111,12 @@ end # runtime system hook called when a task finishes function task_done_hook(t::Task) + # a sticky cancellation-source registration of this task is garbage now + w = t.cached_cancel_entry + if w isa WaitEntry2 + o = @atomic :monotonic w.owner2 + o isa CancellationTokenSource && _note_dead_registration!(o) + end # `finish_task` sets `sigatomic` before entering this function err = istaskfailed(t) result = task_result(t) @@ -819,8 +1127,9 @@ function task_done_hook(t::Task) lock(donenotify) try if !isempty(donenotify.waitq) - handled = true - notify(donenotify) + # only wakes whose claim was won count as having consumed the + # result (a stale, already-claimed registration does not) + handled = notify(donenotify) > 0 end finally unlock(donenotify) @@ -862,12 +1171,13 @@ mutable struct IntrusiveLinkedListSynchronized{T} lock::Threads.SpinLock IntrusiveLinkedListSynchronized{T}() where {T} = new(IntrusiveLinkedList{T}(), Threads.SpinLock()) end +waitqueue(W::IntrusiveLinkedListSynchronized) = ILLRef(W.queue, W) isempty(W::IntrusiveLinkedListSynchronized) = isempty(W.queue) length(W::IntrusiveLinkedListSynchronized) = length(W.queue) function push!(W::IntrusiveLinkedListSynchronized{T}, t::T) where T lock(W.lock) try - push!(W.queue, t) + push!(waitqueue(W), t) finally unlock(W.lock) end @@ -876,7 +1186,7 @@ end function pushfirst!(W::IntrusiveLinkedListSynchronized{T}, t::T) where T lock(W.lock) try - pushfirst!(W.queue, t) + pushfirst!(waitqueue(W), t) finally unlock(W.lock) end @@ -885,7 +1195,7 @@ end function pop!(W::IntrusiveLinkedListSynchronized) lock(W.lock) try - return pop!(W.queue) + return pop!(waitqueue(W)) finally unlock(W.lock) end @@ -893,7 +1203,7 @@ end function popfirst!(W::IntrusiveLinkedListSynchronized) lock(W.lock) try - return popfirst!(W.queue) + return popfirst!(waitqueue(W)) finally unlock(W.lock) end @@ -901,7 +1211,7 @@ end function list_deletefirst!(W::IntrusiveLinkedListSynchronized{T}, t::T) where T lock(W.lock) try - list_deletefirst!(W.queue, t) + list_deletefirst!(waitqueue(W), t) finally unlock(W.lock) end @@ -915,7 +1225,17 @@ const Workqueue = Workqueues[1] # default work queue is thread 1 // TODO: deprec workqueue_for(tid::Int) = Workqueues[tid] function enq_work(t::Task) - (t._state === task_state_runnable && t.queue === nothing) || error("schedule: Task not runnable") + state = t._state + if state === task_state_abandoned + # A task frozen by forcible abandonment leaves its waitqueue + # registrations behind by design; a later notify of such a stale + # entry lands here. The wakeup is consumed by the abandoned task - + # drop it silently. + return t + end + (state === task_state_runnable && t.queue === nothing) || error("schedule: Task not runnable") + (@atomic :monotonic t.waiting_on) === nothing || + throw(ConcurrencyViolationError("schedule: Task is registered on a wait queue")) # Sticky tasks go into their thread's work queue. if t.sticky @@ -940,6 +1260,17 @@ function enq_work(t::Task) else @label not_sticky tp = Threads.threadpool(t) + if tp !== :foreign && Threads.threadpoolsize(tp) == 0 + # The task's threadpool has no threads, so it can never run; + # fail it with a ConcurrencyViolationError rather than queueing + # it (the multiqueue heaps are unsized for empty pools, and a + # task queued during sysimage bootstrap would be serialized + # into the system image). + setfield!(t, :result, ConcurrencyViolationError("deadlock detected: cannot schedule task")) + t._isexception = true + @atomic :release t._state = task_state_failed + return t + end if tp === :foreign || Threads.threadpoolsize(tp) == 1 # There's only one thread in the task's assigned thread pool; # use its work queue. @@ -949,10 +1280,27 @@ function enq_work(t::Task) else # Otherwise, put the task in the multiqueue. Partr.multiq_insert(t, t.priority) - tid = 0 + tid = Threads.threadid(t) + if tid != 0 && tid != Threads.threadid() + # The task's tid is pinned to another thread: typically it is that + # thread's current task, parked hosting its thread-sleep logic in + # wait(), and only that thread can resume it, so wake it directly + # (#58689). If that thread was already awake (busy with other work), + # this wake added no running thread; wake a pool thread too so the + # number of running threads still scales with enqueued work — the + # task is not sticky, so its tid may be cleared later, making it + # runnable by any pool thread. + if ccall(:jl_wakeup_thread, Cint, (Int16,), (tid - 1) % Int16) == 0 + ccall(:jl_wakeup_threadpool, Cvoid, (Int8,), Threads._sym_to_tpid(tp)) + end + else + # Wake one sleeping thread in the task's pool rather than all of them. See #61820, #50425. + ccall(:jl_wakeup_threadpool, Cvoid, (Int8,), Threads._sym_to_tpid(tp)) + end + return t end end - ccall(:jl_wakeup_thread, Cvoid, (Int16,), (tid - 1) % Int16) + ccall(:jl_wakeup_thread, Cint, (Int16,), (tid - 1) % Int16) return t end @@ -1007,7 +1355,12 @@ function schedule(t::Task, @nospecialize(arg); error=false) # schedule a task to be (re)started with the given value or exception t._state === task_state_runnable || Base.error("schedule: Task not runnable") if error - q = t.queue; q === nothing || Base.list_deletefirst!(q::IntrusiveLinkedList{Task}, t) + # Interrupt path: Unconditionally remove the wait (if any) + # TODO: This should use the proper cancellation system instead + w = @atomicswap t.waiting_on = nothing + w isa WaitEntry && try_unlink_claimed!(w) + q = t.queue + q === nothing || list_deletefirst!(q::StickyWorkqueue, t) setfield!(t, :result, arg) setfield!(t, :_isexception, true) else @@ -1020,6 +1373,48 @@ function schedule(t::Task, @nospecialize(arg); error=false) return t end +# Deliver `exc` into the parked wait whose wake the caller already claimed +# by CASing `t.waiting_on` from `w` to nothing (the cancellation walk). +# Unlike `schedule(t, exc, error=true)` - whose unconditional swap *takes* +# a claim - this must not re-claim: the claim was the wake ticket. If a +# claim-less wake (a raw interrupter, `throwto`) resumed the task first and +# it has since registered a new wait, the claimed park no longer exists and +# the delivery is dropped - a task still eligibly parked under the +# cancelled source is impossible (its registration recheck refuses), and +# anything else must not observe this request. +function deliver_claimed_wake!(t::Task, w::WaitEntry, @nospecialize(exc)) + (@atomic :monotonic t.waiting_on) === nothing || return nothing + # the claimed waitee-queue entry stays linked (the walk does not take + # waitee locks); collect it opportunistically like an interrupter would + try_unlink_claimed!(w) + # a pending wake somebody enqueued claim-lessly is superseded by the + # cancellation delivery, like an interrupt overriding a claimed value + q = t.queue + q === nothing || list_deletefirst!(q::StickyWorkqueue, t) + t._state === task_state_runnable || return nothing + setfield!(t, :result, exc) + setfield!(t, :_isexception, true) + maybe_record_enqueued!(t) + enq_work(t) + return nothing +end + +# The value-mode variant of `deliver_claimed_wake!`, with the same claim +# contract: the parked wait *completes*, returning `val` (the cancellation +# walk's delivery to a watcher - `wait(::CancellationToken)` - whose wait +# the cancellation is the event for). +function deliver_claimed_value_wake!(t::Task, w::WaitEntry, @nospecialize(val)) + (@atomic :monotonic t.waiting_on) === nothing || return nothing + try_unlink_claimed!(w) + q = t.queue + q === nothing || list_deletefirst!(q::StickyWorkqueue, t) + t._state === task_state_runnable || return nothing + setfield!(t, :result, val) + maybe_record_enqueued!(t) + enq_work(t) + return nothing +end + """ yield() @@ -1033,7 +1428,7 @@ function yield() try wait() catch - q = ct.queue; q === nothing || Base.list_deletefirst!(q::IntrusiveLinkedList{Task}, ct) + q = ct.queue; q === nothing || list_deletefirst!(q::StickyWorkqueue, ct) rethrow() end end @@ -1051,12 +1446,13 @@ Throws a `ConcurrencyViolationError` if `t` is the currently running task. function yield(t::Task, @nospecialize(x=nothing)) ct = current_task() t === ct && throw(ConcurrencyViolationError("Cannot yield to currently running task!")) - (t._state === task_state_runnable && t.queue === nothing) || throw(ConcurrencyViolationError("yield: Task not runnable")) + (t._state === task_state_runnable && t.queue === nothing && + (@atomic :monotonic t.waiting_on) === nothing) || throw(ConcurrencyViolationError("yield: Task not runnable")) # [task] user_time -yield-> wait_time record_running_time!(ct) # [task] created -scheduled-> wait_time maybe_record_enqueued!(t) - t.result = x + setfield!(t, :result, x) enq_work(ct) set_next_task(t) return try_yieldto(ensure_rescheduled) @@ -1076,14 +1472,14 @@ function yieldto(t::Task, @nospecialize(x=nothing)) # state error instead. if t._state === task_state_done return x - elseif t._state === task_state_failed + elseif t._state === task_state_failed || t._state === task_state_abandoned throw(t.result) end # [task] user_time -yield-> wait_time record_running_time!(ct) # [task] created -scheduled-unfairly-> wait_time maybe_record_enqueued!(t) - t.result = x + setfield!(t, :result, x) set_next_task(t) return try_yieldto(identity) end @@ -1102,12 +1498,12 @@ function try_yieldto(undo) end if ct._isexception exc = ct.result - ct.result = nothing + setfield!(ct, :result, nothing) ct._isexception = false throw(exc) end result = ct.result - ct.result = nothing + setfield!(ct, :result, nothing) return result end @@ -1118,15 +1514,54 @@ function throwto(t::Task, @nospecialize exc) record_running_time!(ct) # [task] created -scheduled-unfairly-> wait_time maybe_record_enqueued!(t) - t.result = exc + setfield!(t, :result, exc) t._isexception = true set_next_task(t) return try_yieldto(identity) end -@inline function wait_forever() +function wait_forever() while true - wait() + try + while true + wait() + end + catch e + if Threads.threadid() == 1 && isa(e, InterruptException) && isempty(Workqueue) + # An InterruptException landed on this internal scheduler task while + # the thread was idle (it parked here after running a completed task). + # N.B.: SIGINT no longer force-throws InterruptException on any + # platform (it cancels the ^C episode source instead), so this + # branch is reachable only via an explicit user/library throw + # (`throwto`, `schedule(..., error=...)`) into a scheduler + # task; kept as defense in depth. + # Forward it to a task that can observe it: the REPL backend if it is + # evaluating user code; nothing at an idle REPL prompt (drop it); the + # root task otherwise, e.g. a non-interactive script blocked in wait + # (#58689). + victim = repl_backend_task() + if !(victim isa Task) + at_repl_prompt = @isdefined(active_repl_backend) && active_repl_backend !== nothing + victim = (at_repl_prompt || istaskdone(roottask)) ? nothing : roottask + end + if victim isa Task + try + throwto(victim, e) + catch + # delivery is best-effort: the victim may have been + # rescheduled concurrently, or a second interrupt may arrive + # while this task is suspended in the switch + end + end + else + local errs = stderr + # try to display the failure atomically + errio = IOContext(PipeBuffer(), errs::IO) + emphasize(errio, "Internal Task ") + display_error(errio, current_exceptions()) + write(errs, errio) + end + end end end @@ -1151,21 +1586,40 @@ function ensure_rescheduled(othertask::Task) nothing end +function discard_stale_workqueue_task(t::Task) + # A task frozen in place by forcible abandonment is completed without + # ever leaving the queues it was registered with (a workqueue, or a + # waitqueue whose later notify re-enqueues it here); discard it. Any + # other non-runnable state means the task somehow got queued twice - + # probably broken now, but try discarding this switch and keep going. + # We can't throw here, because it's probably not the fault of the caller + # to wait, and don't want to use print() here, because that may try to + # incur a task switch. + if t._state !== task_state_abandoned + ccall(:jl_safe_printf, Cvoid, (Ptr{UInt8}, Int32...), + "\nWARNING: Workqueue inconsistency detected: popfirst!(Workqueue).state !== :runnable\n") + end + nothing +end + function trypoptask(W::StickyWorkqueue) while !isempty(W) t = popfirst!(W) if t._state !== task_state_runnable - # assume this somehow got queued twice, - # probably broken now, but try discarding this switch and keep going - # can't throw here, because it's probably not the fault of the caller to wait - # and don't want to use print() here, because that may try to incur a task switch - ccall(:jl_safe_printf, Cvoid, (Ptr{UInt8}, Int32...), - "\nWARNING: Workqueue inconsistency detected: popfirst!(Workqueue).state !== :runnable\n") + discard_stale_workqueue_task(t) + continue + end + return t + end + while true + t = Partr.multiq_deletemin() + t === nothing && return nothing + if t._state !== task_state_runnable + discard_stale_workqueue_task(t) continue end return t end - return Partr.multiq_deletemin() end checktaskempty = Partr.multiq_check_empty @@ -1176,17 +1630,23 @@ function wait() record_running_time!(ct) # let GC run GC.safepoint() - # check for libuv events - process_events() + # check for libuv events, but not on a completed task (#63048) + ct_done = istaskdone(ct) + ct_done || process_events() # get the next task to run W = workqueue_for(Threads.threadid()) task = trypoptask(W) if task === nothing - # No tasks to run; switch to the scheduler task to run the - # thread sleep logic. + # No tasks to run. If the current task is done, switch to the scheduler task + # to run the thread sleep logic, so that this task's stack can be freed + # promptly (#57544). Otherwise run the thread sleep logic in the context of + # the current task, so that an asynchronously thrown InterruptException + # is delivered to a task that can observe it, rather than swallowed by + # the internal scheduler task (#58689). sched_task = get_sched_task() - if ct !== sched_task + if ct !== sched_task && ct_done + istaskdone(sched_task) && (sched_task = @task wait()) return yieldto(sched_task) end task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), trypoptask, W, checktaskempty) @@ -1204,7 +1664,7 @@ end # update the `running_time_ns` field of `t` to include the time since it last started running. function record_running_time!(t::Task) if t.metrics_enabled && !istaskdone(t) - @atomic :monotonic t.running_time_ns += time_ns() - t.last_started_running_at + @atomic :monotonic t.running_time_ns +%= time_ns() -% t.last_started_running_at end return t end diff --git a/base/terminfo.jl b/base/terminfo.jl index be0dd53b1ac74..bece6b2322cda 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -12,7 +12,7 @@ A structured representation of a terminfo file, without any knowledge of particular capabilities, solely based on `term(5)`. !!! warning - This is not part of the public API, and thus subject to change without notice. + This is not part of the public API, and thus subject to change without notice. # Fields @@ -44,19 +44,16 @@ end A parsed terminfo paired with capability information. !!! warning - This is not part of the public API, and thus subject to change without notice. + This is not part of the public API, and thus subject to change without notice. # Fields - `names::Vector{String}`: The names this terminal is known by. -- `flags::Int`: The number of flags specified. -- `numbers::BitVector`: A mask indicating which of `TERM_NUMBERS` have been - specified. -- `strings::BitVector`: A mask indicating which of `TERM_STRINGS` have been - specified. -- `extensions::Vector{Symbol}`: A list of extended capability variable names. -- `capabilities::Dict{Symbol, Union{Bool, Int, String}}`: The capability values - themselves. +- `flags::Dict{Symbol, Bool}`: A mapping of flag capability names to their values. +- `numbers::Dict{Symbol, Int}`: A mapping of number capability names to their values. +- `strings::Dict{Symbol, String}`: A mapping of string capability names to their values. +- `extensions::Union{Nothing, Set{Symbol}}`: The set of extended capability names, or `nothing` if there are no extensions. +- `aliases::Dict{Symbol, Symbol}`: A mapping of alias names to their corresponding capability names in any of the above categories. See also: `TermInfoRaw` and `TermCapability`. """ @@ -69,7 +66,7 @@ struct TermInfo aliases::Dict{Symbol, Symbol} end -TermInfo() = TermInfo([], Dict(), Dict(), Dict(), nothing, Dict()) +TermInfo() = TermInfo(String[], Dict{Symbol, Bool}(), Dict{Symbol, Int}(), Dict{Symbol, String}(), nothing, Dict{Symbol, Symbol}()) function read(data::IO, ::Type{TermInfoRaw}) # Parse according to `term(5)` diff --git a/base/terminfo_data.jl b/base/terminfo_data.jl index caf2ff528d3e1..cbe43165041fe 100644 --- a/base/terminfo_data.jl +++ b/base/terminfo_data.jl @@ -1,9 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license # Updating this listing is fairly easy, assuming existence of a unix system, -# posix shell, and `awk`. Just update the version string in the commented out -# `NCURSES_VERSION` variable, and run this file. This works because this file is -# a bit of a quine. +# posix shell, and `awk`. Just call this file with `sh`, and it will be +# updated to the latest terminfo snapshot. This works because this file is a +# bit of a quine. #= awk '/^#=run/{flag=1;next}/=#/{flag=0}flag{gsub(/__FILE__/,"\"'"$0"'\"");print}' "$0" | \ @@ -16,7 +16,7 @@ awk '/^#=run/{flag=1;next}/=#/{flag=0}flag{gsub(/__FILE__/,"\"'"$0"'\"");print}' Specification of a single terminal capability. !!! warning - This is not part of the public API, and thus subject to change without notice. + This is not part of the public API, and thus subject to change without notice. # Fields @@ -41,9 +41,9 @@ version_info = IOBuffer() standard_caps = IOBuffer() user_caps = IOBuffer() -Downloads.download("https://raw.githubusercontent.com/mirror/ncurses/master/VERSION", version_info) -Downloads.download("https://raw.githubusercontent.com/mirror/ncurses/master/include/Caps", standard_caps) -Downloads.download("https://raw.githubusercontent.com/mirror/ncurses/master/include/Caps-ncurses", user_caps) +Downloads.download("https://raw.githubusercontent.com/ThomasDickey/ncurses-snapshots/refs/heads/master/VERSION", version_info) +Downloads.download("https://raw.githubusercontent.com/ThomasDickey/ncurses-snapshots/refs/heads/master/include/Caps", standard_caps) +Downloads.download("https://raw.githubusercontent.com/ThomasDickey/ncurses-snapshots/refs/heads/master/include/Caps-ncurses", user_caps) const TERM_FLAGS = NTuple{3, String}[] const TERM_NUMBERS = NTuple{3, String}[] @@ -67,6 +67,7 @@ for line in eachline(seekstart(standard_caps)) @warn "Unrecognised capability type: $type" continue end + description = replace(description, r"\\f[BPRI]?" => "") push!(caplist, (name, shortcode, description)) end @@ -132,7 +133,10 @@ function getcustomalias(allterms::Vector{NTuple{3, String}}, type, short, descri "csl" => ":clear_status_line", "Ms" => ":set_host_clipboard", "Tc" => ":truecolor", - "XF" => ":xterm_focus") + "XF" => ":xterm_focus", + "fd" => ":focus_disable", + "fe" => ":focus_enable", + ) if startswith(short, 'k') && !occursin("keypad", description) return ":key_" * replace(lowercase(description), r"[^a-z]" => '_') end @@ -156,11 +160,11 @@ end ## GENERATED CODE BEYOND THIS POINT ## -# Terminfo Capabilities as of NCurses 6.4-20230311 -const NCURSES_VERSION = v"6.4.20230311" +# Terminfo Capabilities as of NCurses 6.5-20251018 +const NCURSES_VERSION = v"6.5.20251018" """ -Ordered list of known terminal capability flag fields, as of NCurses 6.4-20230311. +Ordered list of known terminal capability flag fields, as of NCurses 6.5-20251018. """ const TERM_FLAGS = [ TermCapability(:auto_left_margin, :bw, "cub1 wraps from column 0 to last column"), @@ -210,7 +214,7 @@ const TERM_FLAGS = [ ] """ -Ordered list of known terminal capability number fields, as of NCurses 6.4-20230311. +Ordered list of known terminal capability number fields, as of NCurses 6.5-20251018. """ const TERM_NUMBERS = [ TermCapability(:columns, :cols, "number of columns in a line"), @@ -255,7 +259,7 @@ const TERM_NUMBERS = [ ] """ -Ordered list of known terminal capability string fields, as of NCurses 6.4-20230311. +Ordered list of known terminal capability string fields, as of NCurses 6.5-20251018. """ const TERM_STRINGS = [ TermCapability(:back_tab, :cbt, "back tab (P)"), @@ -346,8 +350,8 @@ const TERM_STRINGS = [ TermCapability(:key_sr, :kri, "scroll-backward key"), TermCapability(:key_stab, :khts, "set-tab key"), TermCapability(:key_up, :kcuu1, "up-arrow key"), - TermCapability(:keypad_local, :rmkx, "leave 'keyboard_transmit' mode"), - TermCapability(:keypad_xmit, :smkx, "enter 'keyboard_transmit' mode"), + TermCapability(:keypad_local, :rmkx, "leave keypad transmit mode"), + TermCapability(:keypad_xmit, :smkx, "enter keypad transmit mode"), TermCapability(:lab_f0, :lf0, "label on function key f0 if not f0"), TermCapability(:lab_f1, :lf1, "label on function key f1 if not f1"), TermCapability(:lab_f10, :lf10, "label on function key f10 if not f10"), @@ -529,7 +533,7 @@ const TERM_STRINGS = [ TermCapability(:key_f63, :kf63, "F63 function key"), TermCapability(:clr_bol, :el1, "Clear to beginning of line"), TermCapability(:clear_margins, :mgc, "clear right and left soft margins"), - TermCapability(:set_left_margin, :smgl, "set left soft margin at current column."), + TermCapability(:set_left_margin, :smgl, "set left soft margin at current column (not in BSD termcap)"), TermCapability(:set_right_margin, :smgr, "set right soft margin at current column"), TermCapability(:label_format, :fln, "label format"), TermCapability(:set_clock, :sclk, "set clock, #1 hrs #2 mins #3 secs"), @@ -620,7 +624,7 @@ const TERM_STRINGS = [ TermCapability(:set_a_foreground, :setaf, "Set foreground color to #1, using ANSI escape"), TermCapability(:set_a_background, :setab, "Set background color to #1, using ANSI escape"), TermCapability(:pkey_plab, :pfxl, "Program function key #1 to type string #2 and show string #3"), - TermCapability(:device_type, :devt, "Indicate language/codeset support"), + TermCapability(:device_type, :devt, "Indicate language, codeset support"), TermCapability(:code_set_init, :csin, "Init sequence for multiple codesets"), TermCapability(:set0_des_seq, :s0ds, "Shift to codeset 0 (EUC set 0, ASCII)"), TermCapability(:set1_des_seq, :s1ds, "Shift to codeset 1"), @@ -675,7 +679,7 @@ const TERM_STRINGS = [ ] """ -Terminfo extensions that NCurses 6.4-20230311 is aware of. +Terminfo extensions that NCurses 6.5-20251018 is aware of. """ const TERM_USER = Dict{Tuple{DataType, Symbol}, Union{Tuple{Nothing, String}, Tuple{Symbol, String}}}( (Int, :CO ) => (nothing, "number of indexed colors overlaying RGB space"), @@ -730,6 +734,8 @@ const TERM_USER = Dict{Tuple{DataType, Symbol}, Union{Tuple{Nothing, String}, Tu (String, :RV) => (nothing, "report terminal secondary device attributes"), (String, :XR) => (nothing, "report terminal version as a free-format string."), (Bool, :XF) => (:xterm_focus, "terminal supports xterm focus in/out"), + (String, :fd) => (:focus_disable, "disable xterm focus-events"), + (String, :fe) => (:focus_enable, "enable xterm focus-events"), (String, :rv) => (nothing, "response to RV, regular expression"), (String, :xr) => (nothing, "response to XR, regular expression"), (String, :csl) => (:clear_status_line, "clear status line"), diff --git a/base/threadcall.jl b/base/threadcall.jl index fbc1a87a20980..a7e7cdb5b98e2 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -1,9 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license const max_ccall_threads = parse(Int, get(ENV, "UV_THREADPOOL_SIZE", "4")) -const thread_notifiers = Union{Event, Nothing}[nothing for i in 1:max_ccall_threads] const threadcall_restrictor = Semaphore(max_ccall_threads) -const threadcall_lock = Threads.SpinLock() """ @threadcall((cfunc, clib), rettype, (argtypes...), argvals...) @@ -30,78 +28,82 @@ macro threadcall(f, rettype, argtypes, argvals...) argtypes = map(esc, argtypes.args) argvals = map(esc, argvals) - # construct non-allocating wrapper to call C function - wrapper = :(function (fptr::Ptr{Cvoid}, args_ptr::Ptr{Cvoid}, retval_ptr::Ptr{Cvoid}) - p = args_ptr - # the rest of the body is created below - end) - body = wrapper.args[2].args - args = Symbol[] - for (i, T) in enumerate(argtypes) - arg = Symbol("arg", i) - push!(body, :($arg = unsafe_load(convert(Ptr{$T}, p)))) - push!(body, :(p += Core.sizeof($T))) - push!(args, arg) - end - push!(body, :(ret = ccall(fptr, $rettype, ($(argtypes...),), $(args...)))) - push!(body, :(unsafe_store!(convert(Ptr{$rettype}, retval_ptr), ret))) - push!(body, :(return Int(Core.sizeof($rettype)))) + # `cconvert` and `unsafe_convert` each argument on the calling thread: + # cconvert may allocate or run arbitrary Julia code, and computing the + # unsafe_convert'd C representation here keeps all of that off the libuv + # worker thread, which only makes the raw call. The cconverted values are + # captured by the wrapper closure and GC.@preserve'd around the worker-thread + # ccall so their C representations (e.g. interior pointers) stay valid. + roots = [Symbol("root", i) for i in 1:length(argvals)] + args = [Symbol("arg", i) for i in 1:length(argvals)] + rootbinds = [:($(roots[i]) = cconvert($(argtypes[i]), $(argvals[i]))) for i in 1:length(argvals)] + argbinds = [:($(args[i]) = unsafe_convert($(argtypes[i]), $(roots[i]))) for i in 1:length(argvals)] + call = :(result[] = ccall(cfptr, $rettype, ($(argtypes...),), $(args...))) + # keep the cconverted values alive while their C representations are in use + body = isempty(roots) ? call : :(GC.@preserve $(roots...) $call) - # return code to generate wrapper function and send work request thread queue - wrapper = Expr(:var"hygienic-scope", wrapper, @__MODULE__, __source__) - return :(let fun_ptr = @cfunction($wrapper, Int, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid})) + return quote # use cglobal to look up the function on the calling thread - do_threadcall(fun_ptr, cglobal($f), $rettype, Any[$(argtypes...)], Any[$(argvals...)]) - end) + cfptr = cglobal($f) + $(rootbinds...) + $(argbinds...) + result = Ref{$rettype}() + # closure that performs the actual call on the worker thread and stores + # the result into the captured cell + wrapper = function () + $body + return + end + do_threadcall(wrapper, result) + end end -function do_threadcall(fun_ptr::Ptr{Cvoid}, cfptr::Ptr{Cvoid}, rettype::Type, argtypes::Vector, argvals::Vector) - # generate function pointer - c_notify_fun = @cfunction( - function notify_fun(idx) - global thread_notifiers - notify(thread_notifiers[idx]) - return - end, Cvoid, (Cint,)) +# call wrapper invoked on the libuv worker thread. `F` is the concrete type of +# the work closure, so `f()` is fully devirtualized; the closure is delivered +# by value through the `Ref{F}` cfunction argument in `do_threadcall`. +function threadcall_run(f::F) where F + f() + return +end - # cconvert, root and unsafe_convert arguments - roots = Any[] - args_size = isempty(argtypes) ? 0 : sum(Core.sizeof, argtypes) - args_arr = Vector{UInt8}(undef, args_size) - ptr = pointer(args_arr) - for (T, x) in zip(argtypes, argvals) - isbitstype(T) || throw(ArgumentError("threadcall requires isbits argument types")) - y = cconvert(T, x) - push!(roots, y) - unsafe_store!(convert(Ptr{T}, ptr), unsafe_convert(T, y)::T) - ptr += Core.sizeof(T) - end +# called from the libuv event loop once the queued work has finished, to wake +# up the task waiting in `do_threadcall` +function threadcall_notify(ct::Task, ctx::RefValue{Any}) + schedule(ct) + unpreserve_handle(ct) + unpreserve_handle(ctx) + return +end + +function do_threadcall(wrapper::F, result::Ref{T}) where {F, T} + # a plain (non-closure) call wrapper, specialized to `F` via the `Ref{F}` + # argument type; the closure is handed to C as an opaque context pointer + c_run = @cfunction(threadcall_run, Cvoid, (Ref{F},)) + # function pointer used to notify us when the work is done + c_notify_fun = @cfunction(threadcall_notify, Cvoid, (Ref{Task}, Ref{RefValue{Any}},)) - # create return buffer - ret_arr = Vector{UInt8}(undef, Core.sizeof(rettype)) + # box the closure so wrapper has a stable address to preserve + # pass as Ptr{Cvoid} to pass that stable address + ctx = RefValue{Any}(wrapper) # wait for a worker thread to be available acquire(threadcall_restrictor) - idx = -1 - @lock threadcall_lock begin - idx = findfirst(isequal(nothing), thread_notifiers)::Int - thread_notifiers[idx] = Event() - end - - GC.@preserve args_arr ret_arr roots begin + try + ct = current_task() + # keep the waiting task and the boxed closure alive until the worker thread + # has finished and woken us back up: the task so the notifier can find it by + # pointer, and the closure box so its captured values survive + preserve_handle(ct) + preserve_handle(ctx) # queue up the work to be done ccall(:jl_queue_work, Cvoid, - (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{UInt8}, Ptr{UInt8}, Ptr{Cvoid}, Cint), - fun_ptr, cfptr, args_arr, ret_arr, c_notify_fun, idx) + (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}, Any, Ref{RefValue{Any}}), + c_run, ctx, c_notify_fun, ct, ctx) - # wait for a result & return it - wait(thread_notifiers[idx]) - @lock threadcall_lock begin - thread_notifiers[idx] = nothing - end + # wait for a result + wait() + finally release(threadcall_restrictor) - - r = unsafe_load(convert(Ptr{rettype}, pointer(ret_arr))) end - return r + return result[] end diff --git a/base/threadingconstructs.jl b/base/threadingconstructs.jl index 9d175bf0d8e81..663df449db20e 100644 --- a/base/threadingconstructs.jl +++ b/base/threadingconstructs.jl @@ -44,7 +44,7 @@ to the Julia process, with atomic-acquire semantics. The result will always be greater than or equal to [`threadid()`](@ref) as well as `threadid(task)` for any task you were able to observe before calling `maxthreadid`. """ -maxthreadid() = Int(Core.Intrinsics.atomic_pointerref(cglobal(:jl_n_threads, Cint), :acquire)) +maxthreadid() = Int(unsafe_load(cglobal(:jl_n_threads, Cint), :acquire)) """ Threads.nthreads(:default | :interactive)::Int @@ -91,7 +91,7 @@ end """ Threads.threadpool(tid = threadid())::Symbol -Returns the specified thread's threadpool; either `:default`, `:interactive`, or `:foreign`. +Return the specified thread's threadpool; either `:default`, `:interactive`, or `:foreign`. """ function threadpool(tid = threadid()) tpid = ccall(:jl_threadpoolid, Int8, (Int16,), tid-1) @@ -101,7 +101,7 @@ end """ Threads.threadpooldescription(tid = threadid())::String -Returns the specified thread's threadpool name with extended description where appropriate. +Return the specified thread's threadpool name with extended description where appropriate. """ function threadpooldescription(tid = threadid()) threadpool_name = threadpool(tid) @@ -119,7 +119,7 @@ end """ Threads.nthreadpools()::Int -Returns the number of threadpools currently configured. +Return the number of threadpools currently configured. """ nthreadpools() = Int(unsafe_load(cglobal(:jl_n_threadpools, Cint))) @@ -147,7 +147,7 @@ end """ threadpooltids(pool::Symbol) -Returns a vector of IDs of threads in the given pool. +Return a vector of IDs of threads in the given pool. """ function threadpooltids(pool::Symbol) ni = _nthreads_in_pool(Int8(0)) @@ -163,38 +163,85 @@ end """ Threads.ngcthreads()::Int -Returns the number of GC threads currently configured. +Return the number of GC threads currently configured. This includes both mark threads and concurrent sweep threads. """ ngcthreads() = Int(unsafe_load(cglobal(:jl_n_gcthreads, Cint))) + 1 function threading_run(fun, static) + if static && ccall(:jl_in_threaded_region, Cint, ()) != 0 + error("`@threads :static` cannot be used concurrently or nested") + end ccall(:jl_enter_threaded_region, Cvoid, ()) n = threadpoolsize() tid_offset = threadpoolsize(:interactive) tasks = Vector{Task}(undef, n) - for i = 1:n - t = Task(() -> fun(i)) # pass in tid - t.sticky = static - if static - ccall(:jl_set_task_tid, Cint, (Any, Cint), t, tid_offset + i-1) - else - # TODO: this should be the current pool (except interactive) if there - # are ever more than two pools. - _result = ccall(:jl_set_task_threadpoolid, Cint, (Any, Int8), t, _sym_to_tpid(:default)) - @assert _result == 1 + # The workers run in a new dynamic scope carrying the token of a fresh + # cancellation source, so that cancellation of the enclosing scope reaches + # every worker through the token tree. + src = Base.CancellationTokenSource(Base.default_cancel_token()) + tok = Base.CancellationToken(src) + cr = nothing + try + Base.ScopedValues.with(Base.CANCEL_TOKEN => tok) do + for i = 1:n + t = Task(() -> fun(i)) # pass in tid + t.sticky = static + if static + ccall(:jl_set_task_tid, Cint, (Any, Cint), t, tid_offset + i-1) + else + # TODO: this should be the current pool (except interactive) if there + # are ever more than two pools. + _result = ccall(:jl_set_task_threadpoolid, Cint, (Any, Int8), t, _sym_to_tpid(:default)) + @assert _result == 1 "_result != 1" + end + tasks[i] = t + schedule(t) + end end - tasks[i] = t - schedule(t) - end - for i = 1:n - Base._wait(tasks[i]) + for i = 1:n + r = Base._wait(tasks[i], tok; cancel_value=true) + if r isa Base.CancellationRequest + # Our own scope was cancelled; the workers observe the same + # cancellation through the tree. Await their unwind rather + # than unwinding out of the `@threads` while workers are + # still running; a severity escalation completes these + # teardown waits (value-mode) and is adopted, re-arming + # them (at ABANDON_ALL the workers were frozen; nothing to + # wait for). + cr = r + sev = Base.severity(r) + for j = i:n + while sev < Base.CANCEL_REQUEST_ABANDON_ALL.request + r2 = Base._wait(tasks[j], tok; min_severity=sev + 0x01, + cancel_value=true) + r2 isa Base.CancellationRequest || break + cr = r2 + sev = Base.severity(r2) + end + end + break + end + end + finally + ccall(:jl_exit_threaded_region, Cvoid, ()) end - ccall(:jl_exit_threaded_region, Cvoid, ()) failed_tasks = filter!(istaskfailed, tasks) if !isempty(failed_tasks) throw(CompositeException(map(TaskFailedException, failed_tasks))) end + # The block's scope was cancelled but every worker happened to complete + # cleanly - the request must still surface at this cancellation-aware + # boundary rather than be swallowed. + cr === nothing || throw(cr) +end + +# Helper to generate the threading run call. Keep the expansion free of `ccall`s: a +# foreigncall in a top-level thunk forces the whole thunk through codegen, even under +# `--compile=min`, so a top-level `@threads` would otherwise make the interpreter +# unusable for everything around it. +function _threading_run_expr(schedule) + :(threading_run(threadsfor_fun, $(schedule === :static))) end function _threadsfor(iter, lbody, schedule) @@ -209,20 +256,225 @@ function _threadsfor(iter, lbody, schedule) quote local threadsfor_fun $func - if $(schedule === :greedy || schedule === :dynamic || schedule === :default) - threading_run(threadsfor_fun, false) - elseif ccall(:jl_in_threaded_region, Cint, ()) != 0 # :static - error("`@threads :static` cannot be used concurrently or nested") - else # :static - threading_run(threadsfor_fun, true) - end + $(_threading_run_expr(schedule)) nothing end end +function _threadsfor_multi_iterator(body, iterators, condition, schedule, dims, result_type) + vars = [iter.args[1] for iter in iterators] + ranges = [iter.args[2] for iter in iterators] + + tuple_var = gensym("iter_tuple") + assignments = [:($(vars[i]) = $(tuple_var)[$i]) for i in 1:length(vars)] + # Use let blocks so destructured variables are local to each iteration, + # avoiding data races when multiple threads execute the body concurrently. + new_body = Expr(:let, Expr(:block, assignments...), body) + new_condition = if condition === true + true + else + Expr(:let, Expr(:block, assignments...), condition) + end + + product_expr = :(Iterators.product($(ranges...))) + synthetic_iter = :($(tuple_var) = $(product_expr)) + + return _threadsfor_single_iterator(new_body, synthetic_iter, new_condition, schedule, dims; result_type) +end + +function _threadsfor_comprehension(gen::Expr, schedule, result_type=nothing) + @assert gen.head === :generator + + body = gen.args[1] + + # Check if the second arg is a filter (handles both single and multi-loop with filters) + iter_or_filter = gen.args[2] + if isa(iter_or_filter, Expr) && iter_or_filter.head === :filter + condition = iter_or_filter.args[1] + iterators = iter_or_filter.args[2:end] + + if length(iterators) == 1 + return _threadsfor_single_iterator(body, iterators[1], condition, schedule; result_type) + else + return _threadsfor_multi_iterator(body, iterators, condition, schedule, nothing, result_type) + end + elseif length(gen.args) > 2 + iterators = gen.args[2:end] + ranges = [iter.args[2] for iter in iterators] + # Use axes to preserve offset index spaces (e.g. OffsetArrays) + dims_expr = :(tuple($([:(axes($(esc(r)), 1)) for r in ranges]...))) + return _threadsfor_multi_iterator(body, iterators, true, schedule, dims_expr, result_type) + else + return _threadsfor_single_iterator(body, iter_or_filter, true, schedule; result_type) + end +end + +function _threadsfor_single_iterator(body, iterator, condition, schedule, dims=nothing; result_type=nothing) + lidx = iterator.args[1] + range = iterator.args[2] + esc_range = esc(range) + esc_lidx = esc(lidx) + esc_body = esc(body) + esc_condition = condition === true ? true : esc(condition) + + # Fast path: no filter and not greedy — pre-allocate and write directly + if condition === true && schedule !== :greedy + return _threadsfor_comprehension_fast(esc_range, esc_lidx, esc_body, schedule, dims, result_type) + end + + func = if schedule === :greedy + greedy_comprehension_func(esc_range, esc_lidx, esc_body, esc_condition) + else + default_comprehension_func(esc_range, esc_lidx, esc_body, esc_condition, result_type) + end + + result_expr = if schedule === :greedy + # Greedy: collect values in arrival order (no ordering guarantee) + if result_type !== nothing + esc_result_type = esc(result_type) + quote + close(result_channel) + vals = collect(result_channel) + isempty(vals) ? $esc_result_type[] : $esc_result_type[v for v in vals] + end + else + quote + close(result_channel) + collect(result_channel) + end + end + else + # Default/static: thread-local buffers, vcat in tid order preserves iteration order. + # For the untyped case, we try vcat first (fast bulk copies). If the buffers have + # a Union or Any element type — indicating a type-unstable body — we fall back to + # grow_to! which replicates serial's promote_typejoin widening. + if result_type !== nothing + esc_result_type = esc(result_type) + quote + vcat(result_channel...)::Vector{$esc_result_type} + end + else + quote + let _bufs = result_channel + _ET = eltype(eltype(_bufs)) + if isconcretetype(_ET) + # All buffers have a concrete element type (from promote_op inference + # or a uniform body). Use bulk vcat — same result as grow_to! here. + vcat(_bufs...) + else + # Type-unstable body: grow_to! discovers the correct widened type, + # matching the return type of the equivalent serial comprehension. + Base.grow_to!(Any[], Iterators.flatten(_bufs)) + end + end + end + end + end + + # If dims is provided, reshape the result to match original comprehension dimensions + if dims !== nothing + result_expr = quote + let flat_result = $result_expr + reshape(flat_result, $(dims)) + end + end + end + + quote + local threadsfor_fun + local result_channel = $func + $(_threading_run_expr(schedule)) + $result_expr + end +end + +# Fast path for non-filtered, non-greedy comprehensions: pre-allocate and write directly. +# Non-AbstractArray iterators (e.g. Iterators.flatten) are collected into a Vector +# because the parallel work distribution indexes into items with r[i]. +# AbstractArrays and Tuples are used directly to preserve their index space (e.g. OffsetArrays). +function _threadsfor_comprehension_fast(esc_range, esc_lidx, esc_body, schedule, dims, result_type) + work_dist = _work_distribution_code() + wrap_final = dims !== nothing ? (x -> :(reshape($x, $dims))) : identity + + if result_type !== nothing + # Typed path: pre-allocate with known element type + esc_result_type = esc(result_type) + return quote + let iter = $esc_range + local items = iter isa Union{Tuple, AbstractArray} ? iter : collect(iter) + local niter = length(items) + local result = similar(Vector{$esc_result_type}, axes(items)) + if niter > 0 + let items = items, result = result + local threadsfor_fun + function threadsfor_fun(tid = 1) + # Reads: items, tid. Defines: r, loop_first, loop_last. + $work_dist + for i = loop_first:loop_last + local $esc_lidx = @inbounds r[i] + @inbounds result[i] = $esc_body + end + end + $(_threading_run_expr(schedule)) + end + end + $(wrap_final(:(result))) + end + end + else + # Untyped path: evaluate first element to determine result type, + # then fill in parallel with the body expression inlined directly + # in the closure. This avoids boxing that occurs when calling a + # lambda whose return type is a Union across closure boundaries. + return quote + let iter = $esc_range + local items = iter isa Union{Tuple, AbstractArray} ? iter : collect(iter) + local niter = length(items) + if niter == 0 + $(wrap_final(:(similar(Vector{Any}, axes(items))))) + else + local _skip = firstindex(items) + local $esc_lidx = @inbounds items[_skip] + local _probe_val = $esc_body + local result = similar(Vector{typeof(_probe_val)}, axes(items)) + @inbounds result[_skip] = _probe_val + if niter > 1 + local _npool = threadpoolsize() + local _widen_buffers = [Pair{Int,Any}[] for _ in 1:_npool] + let items = items, result = result, _widen_buffers = _widen_buffers, + _skip = _skip + local threadsfor_fun + function threadsfor_fun(tid = 1) + # Reads: items, tid. Defines: r, loop_first, loop_last. + $work_dist + local _T = eltype(result) + local _my_widen = _widen_buffers[tid] + for i = loop_first:loop_last + i == _skip && continue + local $esc_lidx = @inbounds r[i] + local _val = $esc_body + if _val isa _T + @inbounds result[i] = _val + else + push!(_my_widen, i => _val) + end + end + end + $(_threading_run_expr(schedule)) + end + result = Base.setindices_widen_up_to(result, _widen_buffers) + end + $(wrap_final(:(result))) + end + end + end + end +end + + function greedy_func(itr, lidx, lbody) quote - let c = Channel{eltype($itr)}(0,spawn=true) do ch + let c = Channel{eltype($itr)}(threadpoolsize(), spawn=true) do ch for item in $itr put!(ch, item) end @@ -237,41 +489,66 @@ function greedy_func(itr, lidx, lbody) end end -function default_func(itr, lidx, lbody) +function greedy_comprehension_func(itr, esc_lidx, esc_body, esc_condition) quote - let range = $itr - function threadsfor_fun(tid = 1; onethread = false) - r = range # Load into local variable - lenr = length(r) - # divide loop iterations among threads - if onethread - tid = 1 - len, rem = lenr, 0 - else - len, rem = divrem(lenr, threadpoolsize()) - end - # not enough iterations for all the threads? - if len == 0 - if tid > rem - return + let c = Channel{eltype($itr)}(threadpoolsize(), spawn=true) do ch + for item in $itr + put!(ch, item) end - len, rem = 1, 0 end - # compute this thread's iterations - f = firstindex(r) + ((tid-1) * len) - l = f + len - 1 - # distribute remaining iterations evenly - if rem > 0 - if tid <= rem - f = f + (tid-1) - l = l + tid - else - f = f + rem - l = l + rem + result_channel = Channel{Any}(Inf) + + function threadsfor_fun(tid) + for item in c + local $esc_lidx = item + if $esc_condition + put!(result_channel, $esc_body) + end end end - # run this thread's iterations - for i = f:l + result_channel + end + end +end + +# Helper function to generate work distribution code +function _work_distribution_code() + quote + r = items # Load into local variable + lenr = length(r) + # divide loop iterations among threads + len, rem = divrem(lenr, threadpoolsize()) + # not enough iterations for all the threads? + if len == 0 + if tid > rem + return + end + len, rem = 1, 0 + end + # compute this thread's iterations + loop_first = firstindex(r) + ((tid-1) * len) + loop_last = loop_first + len - 1 + # distribute remaining iterations evenly + if rem > 0 + if tid <= rem + loop_first = loop_first + (tid-1) + loop_last = loop_last + tid + else + loop_first = loop_first + rem + loop_last = loop_last + rem + end + end + end +end + +function default_func(itr, lidx, lbody) + work_dist = _work_distribution_code() + quote + let items = $itr + function threadsfor_fun(tid = 1) + # Reads: items, tid. Defines: r, loop_first, loop_last. + $work_dist + for i = loop_first:loop_last local $(esc(lidx)) = @inbounds r[i] $(esc(lbody)) end @@ -280,15 +557,67 @@ function default_func(itr, lidx, lbody) end end +function default_comprehension_func(itr, esc_lidx, esc_body, esc_condition, result_type=nothing) + work_dist = _work_distribution_code() + if result_type !== nothing + # Typed comprehension: element type known at macro expansion time. + buf_init = :($(esc(result_type))[]) + buf_type_setup = :() + else + # Untyped comprehension: use promote_op to pre-type the per-task buffers, + # avoiding boxing in the parallel phase for type-stable bodies. + # The result is flattened through grow_to! so the final element type matches + # serial's runtime promote_typejoin widening rather than the static promote_op type. + _ET = gensym(:ET) + buf_type_setup = :(local $_ET = Base.promote_op($esc_lidx -> $esc_body, eltype(items))) + buf_init = :($_ET[]) + end + quote + let iter = $itr + local items = iter isa Union{Tuple, AbstractArray} ? iter : collect(iter) + local _npool = threadpoolsize() + $buf_type_setup + # One buffer per task-id; tasks process contiguous ranges so concatenating + # in tid order preserves iteration order without a sort step. + local local_bufs = [$buf_init for _ in 1:_npool] + + function threadsfor_fun(tid = 1) + # Reads: items, tid. Defines: r, loop_first, loop_last. + $work_dist + local buf = local_bufs[tid] + for i = loop_first:loop_last + local $esc_lidx = @inbounds r[i] + if $esc_condition + push!(buf, $esc_body) + end + end + end + local_bufs # Return per-task buffers to be vcat'd after threading_run + end + end +end + """ Threads.@threads [schedule] for ... end + Threads.@threads [schedule] [expr for ... end] + Threads.@threads [schedule] T[expr for ... end] -A macro to execute a `for` loop in parallel. The iteration space is distributed to +A macro to execute a `for` loop or array comprehension in parallel. The iteration space is distributed to coarse-grained tasks. This policy can be specified by the `schedule` argument. The execution of the loop waits for the evaluation of all iterations. +For `for` loops, the macro executes the loop body in parallel but does not return a value. +For array comprehensions, the macro executes the comprehension in parallel and returns +the collected results as an array. + +Tasks spawned by `@threads` are scheduled on the `:default` threadpool. This means that +`@threads` will not use threads from the `:interactive` threadpool, even if called from +the main thread or from a task in the interactive pool. The `:default` threadpool is +intended for compute-intensive parallel workloads. + See also: [`@spawn`](@ref Threads.@spawn) and `pmap` in [`Distributed`](@ref man-distributed). +For more information on threadpools, see the chapter on [threadpools](@ref man-threadpools). # Extended help @@ -371,13 +700,15 @@ thread other than 1. ## Examples +### For loops + To illustrate of the different scheduling strategies, consider the following function `busywait` containing a non-yielding timed loop that runs for a given number of seconds. ```julia-repl julia> function busywait(seconds) tstart = time_ns() - while (time_ns() - tstart) / 1e9 < seconds + while (time_ns() -% tstart) / 1e9 < seconds end end @@ -400,6 +731,81 @@ julia> @time begin The `:dynamic` example takes 2 seconds since one of the non-occupied threads is able to run two of the 1-second iterations to complete the for loop. + +### Array comprehensions + +The `@threads` macro also supports array comprehensions, which return the collected results. +Array comprehensions preserve element order for `:static` and `:dynamic` (default) scheduling. +The `:greedy` scheduler does not guarantee element order, since tasks consume work items as +they become available. Multi-dimensional comprehensions preserve the dimensions of the +original comprehension (e.g., `[f(i,j) for i in 1:n, j in 1:m]` returns an `n×m` matrix). +Typed comprehensions (`T[expr for ...]`) are also supported and return an array with the +specified element type. + +For non-filtered comprehensions with non-`:greedy` scheduling, a fast path is used that +pre-allocates the result array and writes directly by index, avoiding Channel overhead. + +!!! tip "Performance tip" + For best performance, use typed comprehensions (`T[expr for ...]`) when the element type + is known. Untyped comprehensions infer the type from the first result; if later results + have incompatible types, a type-widening path is used which may be slower. + +!!! warning + The body expression of a threaded comprehension may execute on any thread and may + migrate between threads. Do not rely on [`threadid()`](@ref Threads.threadid) or + [`task_local_storage()`](@ref) returning consistent values within the body expression. + +```julia-repl +julia> Threads.@threads [i^2 for i in 1:5] # Simple comprehension +5-element Vector{Int64}: + 1 + 4 + 9 + 16 + 25 + +julia> Threads.@threads [i^2 for i in 1:5 if iseven(i)] # Filtered comprehension +2-element Vector{Int64}: + 4 + 16 + +julia> Threads.@threads [i + j for i in 1:3, j in 1:3] # Multiple loops +3×3 Matrix{Int64}: + 2 3 4 + 3 4 5 + 4 5 6 + +julia> Threads.@threads Float64[i^2 for i in 1:5] # Typed comprehension +5-element Vector{Float64}: + 1.0 + 4.0 + 9.0 + 16.0 + 25.0 +``` + +When the iterator doesn't have a known length, such as a channel, the `:greedy` scheduling +option can be used. +```julia-repl +julia> c = Channel(5, spawn=true) do ch + foreach(i -> put!(ch, i), 1:5) + end; + +julia> Threads.@threads :greedy [i^2 for i in c if iseven(i)] +2-element Vector{Int64}: + 4 + 16 + +julia> # Non-indexable iterators are also supported + Threads.@threads [i for i in Iterators.flatten([1:3, 4:6])] +6-element Vector{Int64}: + 1 + 2 + 3 + 4 + 5 + 6 +``` """ macro threads(args...) na = length(args) @@ -420,13 +826,22 @@ macro threads(args...) else throw(ArgumentError("wrong number of arguments in @threads")) end - if !(isa(ex, Expr) && ex.head === :for) - throw(ArgumentError("@threads requires a `for` loop expression")) - end - if !(ex.args[1] isa Expr && ex.args[1].head === :(=)) - throw(ArgumentError("nested outer loops are not currently supported by @threads")) + if isa(ex, Expr) && (ex.head === :comprehension || ex.head === :typed_comprehension) + # Handle array comprehensions (typed and untyped) + if ex.head === :typed_comprehension + return _threadsfor_comprehension(ex.args[2], sched, ex.args[1]) + else + return _threadsfor_comprehension(ex.args[1], sched) + end + elseif isa(ex, Expr) && ex.head === :for + # Handle for loops + if !(ex.args[1] isa Expr && ex.args[1].head === :(=)) + throw(ArgumentError("nested outer loops are not currently supported by @threads")) + end + return _threadsfor(ex.args[1], ex.args[2], sched) + else + throw(ArgumentError("@threads requires a `for` loop or comprehension expression")) end - return _threadsfor(ex.args[1], ex.args[2], sched) end function _spawn_set_thrpool(t::Task, tp::Symbol) @@ -435,7 +850,7 @@ function _spawn_set_thrpool(t::Task, tp::Symbol) tpid = _sym_to_tpid(:default) end _result = ccall(:jl_set_task_threadpoolid, Cint, (Any, Int8), t, tpid) - @assert _result == 1 + @assert _result == 1 "_result != 1" nothing end diff --git a/base/threads_overloads.jl b/base/threads_overloads.jl index ccbc7e50d227b..011a6c303b851 100644 --- a/base/threads_overloads.jl +++ b/base/threads_overloads.jl @@ -12,12 +12,12 @@ will wait for all internally spawned tasks to complete before returning. If `schedule isa FairSchedule`, `Threads.foreach` will attempt to spawn tasks in a manner that enables Julia's scheduler to more freely load-balance work items across threads. This approach generally has higher per-item overhead, but may perform -better than `StaticSchedule` in concurrence with other multithreaded workloads. +better than `StaticSchedule` in concurrency with other multithreaded workloads. If `schedule isa StaticSchedule`, `Threads.foreach` will spawn tasks in a manner that incurs lower per-item overhead than `FairSchedule`, but is less amenable to load-balancing. This approach thus may be more suitable for fine-grained, -uniform workloads, but may perform worse than `FairSchedule` in concurrence +uniform workloads, but may perform worse than `FairSchedule` in concurrency with other multithreaded workloads. # Examples @@ -55,7 +55,7 @@ function Threads.foreach(f, channel::Channel; stop[] && break end catch - stop[] = true + @atomic stop[] = true rethrow() end end diff --git a/base/timing.jl b/base/timing.jl index 998103d1e78bc..8ffcb9d51ee74 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -2,35 +2,78 @@ # This type must be kept in sync with the C struct in src/gc-interface.h struct GC_Num - allocd ::Int64 # GC internal - deferred_alloc ::Int64 # GC internal - freed ::Int64 # GC internal - malloc ::Int64 - realloc ::Int64 - poolalloc ::Int64 - bigalloc ::Int64 - freecall ::Int64 - total_time ::Int64 - total_allocd ::Int64 # GC internal - collect ::Csize_t # GC internal - pause ::Cint - full_sweep ::Cint - max_pause ::Int64 - max_memory ::Int64 - time_to_safepoint ::Int64 - max_time_to_safepoint ::Int64 - total_time_to_safepoint ::Int64 - sweep_time ::Int64 - mark_time ::Int64 - stack_pool_sweep_time ::Int64 - total_sweep_time ::Int64 - total_sweep_page_walk_time ::Int64 - total_sweep_madvise_time ::Int64 - total_sweep_free_mallocd_memory_time ::Int64 - total_mark_time ::Int64 + # (GC Internal) Number of allocated bytes since the last collection. This field is reset + # after the end of every garbage collection cycle, so it will always be zero if observed + # during execution of Julia user code + allocd::Int64 + # (GC Internal) Number of allocated bytes within a `gc_disable/gc_enable` block. This field is + # reset after every garbage collection cycle and will always be zero in case of no use + # of `gc_disable/gc_enable` blocks + deferred_alloc::Int64 + # (GC Internal) Number of bytes freed in the current collection cycle. This field is + # reset after every garbage collection cycle and will always be zero when observed + # during execution of Julia user code. It's incremented as memory is reclaimed during a collection, + # used to gather some statistics within the collection itself and reset at the end of a GC cycle. + freed::Int64 + # Number of `malloc/calloc` calls (never reset by the runtime) + malloc::Int64 + # Number of `realloc` calls (never reset by the runtime) + realloc::Int64 + # Number of pool allocation calls (never reset by the runtime) + # NOTE: Julia's stock GC uses an internal (pool) allocator for objects up to 2032 bytes. + # Larger objects are allocated through `malloc/calloc`. + poolalloc::Int64 + # Number of allocations for "big objects" (non-array objects larger than 2032 bytes) + # (never reset by the runtime) + bigalloc::Int64 + # Number of `free` calls (never reset by the runtime) + freecall::Int64 + # Total time spent in garbage collection (never reset by the runtime) + total_time::Int64 + # (GC internal) Total number of bytes allocated since the program started + total_allocd::Int64 + # (GC internal) Per-thread allocation quota before triggering a GC + # NOTE: This field is no longer used by the heuristics in the stock GC + interval::Csize_t + # Duration of the last GC pause in nanoseconds + pause::Cint + # Number of full GC sweeps completed so far (never reset by the runtime) + full_sweep::Cint + # Maximum pause duration observed so far in nanoseconds + max_pause::Int64 + # Maximum number of bytes allocated at any point in time. + # NOTE: This is aggregated over objects, not pages + max_memory::Int64 + # Time taken to reach a safepoint in the last GC cycle in nanoseconds + time_to_safepoint::Int64 + # Maximum time taken to reach a safepoint across all GCs in nanoseconds + max_time_to_safepoint::Int64 + # Total time taken to reach safepoints across all GCs in nanoseconds + total_time_to_safepoint::Int64 + # Time spent in the last GC sweeping phase in nanoseconds + sweep_time::Int64 + # Time spent in the last GC marking phase in nanoseconds + mark_time::Int64 + # Time spent sweeping stack pools in the last GC in nanoseconds + stack_pool_sweep_time::Int64 + # Total time spent in sweeping phase across all GCs in nanoseconds + total_sweep_time::Int64 + # Total time spent walking pool allocated pages during sweeping phase across all GCs in nanoseconds + total_sweep_page_walk_time::Int64 + # Total time spent in madvise calls during sweeping phase across all GCs in nanoseconds + total_sweep_madvise_time::Int64 + # Total time spent in freeing malloc'd memory during sweeping phase across all GCs in nanoseconds + total_sweep_free_mallocd_memory_time::Int64 + # Total time spent in marking phase across all GCs in nanoseconds + total_mark_time::Int64 + # Total time spent sweeping stack pools across all GCs in nanoseconds total_stack_pool_sweep_time::Int64 - last_full_sweep ::Int64 - last_incremental_sweep ::Int64 + # Timestamp of the last full GC sweep in nanoseconds + last_full_sweep::Int64 + # Timestamp of the last incremental GC sweep in nanoseconds + last_incremental_sweep::Int64 + # Number of tracked image objects referencing non-image objects + image_remset_size::Int64 end gc_num() = ccall(:jl_gc_num, GC_Num, ()) @@ -86,7 +129,7 @@ function cumulative_compile_timing(b::Bool) return end -# total time spend in garbage collection, in nanoseconds +# total time spent in garbage collection, in nanoseconds gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ()) """ @@ -113,7 +156,7 @@ end @static if Base.USING_STOCK_GC # must be kept in sync with `src/gc-stock.h`` const FULL_SWEEP_REASONS = [:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL, :FULL_SWEEP_REASON_FORCED_FULL_SWEEP, - :FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE] + :FULL_SWEEP_REASON_USER_MAX_EXCEEDED, :FULL_SWEEP_REASON_LARGE_PROMOTION_RATE, :FULL_SWEEP_REASON_LARGE_HEAP_GROWTH] end """ @@ -220,30 +263,31 @@ function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, lock_confl print(io, timestr, " seconds") parens = bytes != 0 || allocs != 0 || gctime > 0 || lock_conflicts > 0 || compile_time > 0 parens && print(io, " (") - if bytes != 0 || allocs != 0 - allocs, ma = prettyprint_getunits(allocs, length(_cnt_units), Int64(1000)) + had_allocs = bytes != 0 || allocs != 0 + if had_allocs + allocs_scaled, ma = prettyprint_getunits(allocs, length(_cnt_units), Int64(1000)) if ma == 1 - print(io, Int(allocs), _cnt_units[ma], allocs==1 ? " allocation: " : " allocations: ") + print(io, Int(allocs_scaled), _cnt_units[ma], allocs_scaled==1 ? " allocation: " : " allocations: ") else - print(io, Ryu.writefixed(Float64(allocs), 2), _cnt_units[ma], " allocations: ") + print(io, Ryu.writefixed(Float64(allocs_scaled), 2), _cnt_units[ma], " allocations: ") end print(io, format_bytes(bytes)) end if gctime > 0 - if bytes != 0 || allocs != 0 + if had_allocs print(io, ", ") end print(io, Ryu.writefixed(Float64(100*gctime/elapsedtime), 2), "% gc time") end if lock_conflicts > 0 - if bytes != 0 || allocs != 0 || gctime > 0 + if had_allocs || gctime > 0 print(io, ", ") end plural = lock_conflicts == 1 ? "" : "s" print(io, lock_conflicts, " lock conflict$plural") end if compile_time > 0 - if bytes != 0 || allocs != 0 || gctime > 0 || lock_conflicts > 0 + if had_allocs || gctime > 0 || lock_conflicts > 0 print(io, ", ") end print(io, Ryu.writefixed(Float64(100*compile_time/elapsedtime), 2), "% compilation time") @@ -459,7 +503,7 @@ macro elapsed(ex) Experimental.@force_compile local t0 = time_ns() $(esc(ex)) - (time_ns() - t0) / 1e9 + (time_ns() -% t0) / 1e9 end end @@ -472,41 +516,101 @@ function gc_bytes() b[] end -function allocated(f, args::Vararg{Any,N}) where {N} +@constprop :none function allocated(f, args::Vararg{Any,N}) where {N} b0 = Ref{Int64}(0) b1 = Ref{Int64}(0) Base.gc_bytes(b0) - f(args...) + @noinline f(args...) Base.gc_bytes(b1) return b1[] - b0[] end only(methods(allocated)).called = 0xff -function allocations(f, args::Vararg{Any,N}) where {N} +@constprop :none function allocations(f, args::Vararg{Any,N}) where {N} stats = Base.gc_num() - f(args...) + @noinline f(args...) diff = Base.GC_Diff(Base.gc_num(), stats) return Base.gc_alloc_count(diff) end only(methods(allocations)).called = 0xff function is_simply_call(@nospecialize ex) + is_simple_atom(a) = a isa QuoteNode || a isa Symbol || !isa_ast_node(a) Meta.isexpr(ex, :call) || return false for a in ex.args - a isa QuoteNode && continue - a isa Symbol && continue - isa_ast_node(a) || continue + is_simple_atom(a) && continue + Meta.isexpr(a, :..., 1) && is_simple_atom(a.args[1]) && continue return false end + # Ensure Expr(:call, .+, ...) get wrapped + if ex.args[1] isa Symbol + sa = String(ex.args[1]::Symbol) + startswith(sa, ".") && + !endswith(sa, ".") && + isoperator(Symbol(sa[2:end])) && + return false + end return true end +function _gen_allocation_measurer(ex, fname::Symbol) + if isexpr(ex, :call) + if !is_simply_call(ex) + ex = :((() -> $ex)()) + end + pushfirst!(ex.args, GlobalRef(Base, fname)) + return quote + Experimental.@force_compile + $(esc(ex)) + end + elseif fname === :allocated + # v1.11-compatible implementation + return quote + Experimental.@force_compile + local b0 = Ref{Int64}(0) + local b1 = Ref{Int64}(0) + gc_bytes(b0) + $(esc(ex)) + gc_bytes(b1) + b1[] - b0[] + end + else + @assert fname === :allocations "unexpected fname" + return quote + Experimental.@force_compile + # Note this value is unused, but without it `allocated` and `allocations` + # are sufficiently different that the compiler can remove allocations here + # that it cannot remove there, giving inconsistent numbers. + local b1 = Ref{Int64}(0) + local stats = Base.gc_num() + $(esc(ex)) + local diff = Base.GC_Diff(Base.gc_num(), stats) + gc_bytes(b1) + Base.gc_alloc_count(diff) + end + end +end + """ @allocated A macro to evaluate an expression, discarding the resulting value, instead returning the total number of bytes allocated during evaluation of the expression. +If the expression is a function call, an effort is made to measure only allocations from +the argument expressions and during the function, excluding any overhead from calling it +and not performing constant propagation with the provided argument values. If you want to +include those effects, i.e. measuring the call site as well, use the syntax +`@allocated (()->f(1))()`. + +It is recommended to measure function calls with only simple argument expressions, e.g. +`x = []; @allocated f(x)` instead of `@allocated f([])` to clarify that only `f` is +being measured. + +For more complex expressions, the code is simply run in place and therefore may see +allocations due to the surrounding context. For example it is possible for +`@allocated f(1)` and `@allocated x = f(1)` to give different results. + See also [`@allocations`](@ref), [`@time`](@ref), [`@timev`](@ref), [`@timed`](@ref), and [`@elapsed`](@ref). @@ -516,11 +620,7 @@ julia> @allocated rand(10^6) ``` """ macro allocated(ex) - if !is_simply_call(ex) - ex = :((() -> $ex)()) - end - pushfirst!(ex.args, GlobalRef(Base, :allocated)) - return esc(ex) + _gen_allocation_measurer(ex, :allocated) end """ @@ -541,11 +641,7 @@ julia> @allocations rand(10^6) This macro was added in Julia 1.9. """ macro allocations(ex) - if !is_simply_call(ex) - ex = :((() -> $ex)()) - end - pushfirst!(ex.args, GlobalRef(Base, :allocations)) - return esc(ex) + _gen_allocation_measurer(ex, :allocations) end @@ -556,7 +652,7 @@ A macro to evaluate an expression, discard the resulting value, and instead retu total number of lock conflicts during evaluation, where a lock attempt on a [`ReentrantLock`](@ref) resulted in a wait because the lock was already held. -See also [`@time`](@ref), [`@timev`](@ref) and [`@timed`](@ref). +See also [`@time`](@ref), [`@timev`](@ref), [`@timed`](@ref). ```julia-repl julia> @lock_conflicts begin @@ -643,9 +739,9 @@ macro timed(ex) cumulative_compile_timing(true) local compile_elapsedtimes = cumulative_compile_time_ns() local val = @__tryfinally($(esc(ex)), - (elapsedtime = time_ns() - elapsedtime; + (elapsedtime = time_ns() -% elapsedtime; cumulative_compile_timing(false); - compile_elapsedtimes = cumulative_compile_time_ns() .- compile_elapsedtimes; + compile_elapsedtimes = map(-%, cumulative_compile_time_ns(), compile_elapsedtimes); lock_conflicts = Threads.LOCK_CONFLICT_COUNT[] - lock_conflicts; Threads.lock_profiling(false)) ) diff --git a/base/toml_parser.jl b/base/toml/parser.jl similarity index 75% rename from base/toml_parser.jl rename to base/toml/parser.jl index bf13fc2b0617a..5d87dae3cc688 100644 --- a/base/toml_parser.jl +++ b/base/toml/parser.jl @@ -1,12 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -""" -`Base.TOML` is an undocumented internal part of Julia's TOML parser -implementation. Users should call the documented interface in the -TOML.jl standard library instead (by `import TOML` or `using TOML`). -""" -module TOML - using Base: IdSet # we parse DateTime into these internal structs, @@ -34,6 +27,47 @@ const EOF_CHAR = typemax(Char) const TOMLDict = Dict{String, Any} +############ +# Comments # +############ + +# The empty path identifies the root table. +# TODO: Formatting choices (e.g. which tables print inline, indentation) could +# be retained for full round-tripping the same way as comments: captured into a +# sibling side-channel keyed by these item paths and passed back to `print`. +const CommentPath = Vector{String} + +# Text excludes `#`; `nothing` means no inline comment and `""` a bare `#`. +mutable struct CommentBlock + above::Vector{String} + inline::Union{String, Nothing} +end +CommentBlock() = CommentBlock(String[], nothing) + +""" + Comments() + +A container for comments captured while parsing a TOML document. Pass it via +the `comments` keyword argument to the parsing functions to populate it, and +via the `comments` keyword argument to the printing functions to write the +comments back out. See the TOML stdlib documentation for the rules of how +comments are associated with items of the document. +""" +struct Comments + items::Dict{CommentPath, CommentBlock} + floating::Dict{CommentPath, Vector{String}} +end +Comments() = Comments(Dict{CommentPath, CommentBlock}(), Dict{CommentPath, Vector{String}}()) + +Base.isempty(c::Comments) = isempty(c.items) && isempty(c.floating) +function Base.empty!(c::Comments) + empty!(c.items) + empty!(c.floating) + return c +end +Base.:(==)(a::CommentBlock, b::CommentBlock) = a.above == b.above && a.inline == b.inline +Base.:(==)(a::Comments, b::Comments) = a.items == b.items && a.floating == b.floating + ########## # Parser # ########## @@ -79,15 +113,29 @@ mutable struct Parser{Dates} # actually defined defined_tables::IdSet{TOMLDict} + # Tables implicitly created as intermediates by dotted keys + # in key-value entries (e.g. `a` in `a.b = 1`). + # These cannot be reopened by [table] headers. + implicit_tables::IdSet{TOMLDict} + # The table we will finally return to the user root::TOMLDict # Filled in in case we are parsing a file to improve error messages filepath::Union{String, Nothing} + + comments::Union{Comments, Nothing} + pending_comments::Vector{String} + captured_comment::String + active_table_path::Vector{String} + last_item_path::Union{CommentPath, Nothing} + # Paths cannot distinguish array-of-tables elements, so their comments are ignored. + in_array_table::Bool end -function Parser{Dates}(str::String; filepath=nothing) where {Dates} +function Parser{Dates}(str::String; filepath=nothing, comments::Union{Comments, Nothing}=nothing) where {Dates} root = TOMLDict() + comments === nothing || empty!(comments) l = Parser{Dates}( str, # str EOF_CHAR, # current_char @@ -102,8 +150,15 @@ function Parser{Dates}(str::String; filepath=nothing) where {Dates} IdSet{TOMLDict}(), # inline_tables IdSet{Any}(), # static_arrays IdSet{TOMLDict}(), # defined_tables + IdSet{TOMLDict}(), # implicit_tables root, - filepath + filepath, + comments, # comments + String[], # pending_comments + "", # captured_comment + String[], # active_table_path + nothing, # last_item_path + false, # in_array_table ) startup(l) return l @@ -124,7 +179,8 @@ Parser{Dates}(io::IO) where {Dates} = Parser{Dates}(read(io, String)) # Parser(...) will be defined by TOML stdlib -function reinit!(p::Parser, str::String; filepath::Union{Nothing, String}=nothing) +function reinit!(p::Parser, str::String; filepath::Union{Nothing, String}=nothing, + comments::Union{Comments, Nothing}=nothing) p.str = str p.current_char = EOF_CHAR p.pos = firstindex(str) @@ -139,7 +195,15 @@ function reinit!(p::Parser, str::String; filepath::Union{Nothing, String}=nothin empty!(p.inline_tables) empty!(p.static_arrays) empty!(p.defined_tables) + empty!(p.implicit_tables) p.filepath = filepath + comments === nothing || empty!(comments) + p.comments = comments + empty!(p.pending_comments) + p.captured_comment = "" + empty!(p.active_table_path) + p.last_item_path = nothing + p.in_array_table = false startup(p) return p end @@ -214,6 +278,7 @@ end ErrUnexpectedEndString ErrInvalidEscapeCharacter ErrInvalidUnicodeScalar + ErrMultilineStringAsKey end const err_message = Dict( @@ -247,6 +312,7 @@ const err_message = Dict( ErrOverflowError => "overflowed when parsing integer", ErrInvalidUnicodeScalar => "invalid unicode scalar", ErrInvalidEscapeCharacter => "invalid escape character", + ErrMultilineStringAsKey => "multiline strings are not allowed as keys", ErrUnexpectedEofExpectedValue => "unexpected end of file, expected a value", ErrSignInNonBase10Number => "number not in base 10 is not allowed to have a sign", ) @@ -297,7 +363,7 @@ end # used to show the interval where an error happened # Right now, it is only called with a == b function point_to_line(str::AbstractString, a::Int, b::Int, context) - @assert b >= a + @assert b >= a "invalid range" a = thisind(str, a) b = thisind(str, b) pos = something(findprev('\n', str, prevind(str, a)), 0) + 1 @@ -392,7 +458,7 @@ end # Return true if `f` was accepted `n` times @inline function accept_n(l::Parser, n, f::F)::Bool where {F} - for i in 1:n + for _ in 1:n if !accept(l, f) return false end @@ -412,6 +478,9 @@ function skip_ws_nl(l::Parser)::Bool while true skipped_ws = accept_batch(l, x -> iswhitespace(x) || isnewline(x)) skipped_comment = skip_comment(l) + if skipped_comment && l.comments !== nothing + push!(l.pending_comments, l.captured_comment) + end if !skipped_ws && !skipped_comment break end @@ -420,16 +489,97 @@ function skip_ws_nl(l::Parser)::Bool return skipped end +# A blank line makes pending comments float rather than attach to the next item. +function skip_ws_nl_toplevel(l::Parser)::Bool + l.comments === nothing && return skip_ws_nl(l) + skipped = false + nlines = 0 # newlines seen since the last comment (or the previous item) + while true + progress = false + while true + c = peek(l) + if iswhitespace(c) + eat_char(l) + progress = true + elseif isnewline(c) + if c == '\n' + nlines += 1 + nlines == 2 && flush_pending_comments!(l) + end + eat_char(l) + progress = true + else + break + end + end + if skip_comment(l) + push!(l.pending_comments, l.captured_comment) + nlines = 0 + progress = true + end + progress || break + skipped = true + end + return skipped +end + # Returns true if a comment was skipped function skip_comment(l::Parser)::Bool found_comment = accept(l, '#') if found_comment - accept_batch(l, !isnewline) + if l.comments === nothing + accept_batch(l, !isnewline) + else + start = l.prevpos + accept_batch(l, !isnewline) + l.captured_comment = String(SubString(l.str, start:prevind(l.str, l.prevpos))) + end end return found_comment end -skip_ws_comment(l::Parser) = skip_ws(l) && skip_comment(l) +function skip_ws_comment(l::Parser)::Bool + skip_ws(l) + return skip_comment(l) +end + +function flush_pending_comments!(l::Parser) + isempty(l.pending_comments) && return + comments = l.comments + if comments !== nothing && !l.in_array_table + floating = get!(() -> String[], comments.floating, copy(l.active_table_path)) + append!(floating, l.pending_comments) + end + empty!(l.pending_comments) + return +end + +function attach_pending_comments!(l::Parser, path::CommentPath) + isempty(l.pending_comments) && return + block = get!(CommentBlock, (l.comments::Comments).items, path) + append!(block.above, l.pending_comments) + empty!(l.pending_comments) + return +end + +function attach_inline_comment!(l::Parser) + path = l.last_item_path + path === nothing && return + block = get!(CommentBlock, (l.comments::Comments).items, path) + block.inline = l.captured_comment + return +end + +function path_traverses_array(l::Parser, keys::AbstractVector{String}) + d = l.root + for k in keys + v = get(d, k, nothing) + v isa Vector && return true + v isa TOMLDict || return false + d = v + end + return false +end @inline set_marker!(l::Parser) = l.marker = l.prevpos take_substring(l::Parser) = SubString(l.str, l.marker:(l.prevpos-1)) @@ -448,7 +598,7 @@ end function tryparse(l::Parser)::Err{TOMLDict} while true - skip_ws_nl(l) + skip_ws_nl_toplevel(l) peek(l) == EOF_CHAR && break v = parse_toplevel(l) if v isa ParserError @@ -461,6 +611,7 @@ function tryparse(l::Parser)::Err{TOMLDict} return v end end + l.comments === nothing || flush_pending_comments!(l) return l.root end @@ -470,33 +621,47 @@ function parse_toplevel(l::Parser)::Err{Nothing} if accept(l, '[') l.active_table = l.root @try parse_table(l) - skip_ws_comment(l) + if skip_ws_comment(l) && l.comments !== nothing + attach_inline_comment!(l) + end if !(peek(l) == '\n' || peek(l) == '\r' || peek(l) == '#' || peek(l) == EOF_CHAR) eat_char(l) return ParserError(ErrExpectedNewLineKeyValue) end else @try parse_entry(l, l.active_table) - skip_ws_comment(l) + if skip_ws_comment(l) && l.comments !== nothing + attach_inline_comment!(l) + end # SPEC: "There must be a newline (or EOF) after a key/value pair." if !(peek(l) == '\n' || peek(l) == '\r' || peek(l) == '#' || peek(l) == EOF_CHAR) - c = eat_char(l) + eat_char(l) return ParserError(ErrExpectedNewLineKeyValue) end end end -function recurse_dict!(l::Parser, d::Dict, dotted_keys::AbstractVector{String}, check=true)::Err{TOMLDict} +function recurse_dict!(l::Parser, d::Dict, dotted_keys::AbstractVector{String}, check=true, define_implicit=false)::Err{TOMLDict} for i in 1:length(dotted_keys) d = d::TOMLDict key = dotted_keys[i] d = get!(TOMLDict, d, key) if d isa Vector{Any} + isempty(d) && return ParserError(ErrArrayTreatedAsDictionary) d = d[end] elseif d isa Vector return ParserError(ErrKeyAlreadyHasValue) end - check && @try check_allowed_add_key(l, d, i == length(dotted_keys)) + if check + # When called from parse_entry (define_implicit=true), check + # defined_tables for ALL intermediates to prevent appending to + # tables that were closed by a different [table] section. + check_def = define_implicit ? true : (i == length(dotted_keys)) + @try check_allowed_add_key(l, d, check_def) + end + if define_implicit && d isa TOMLDict + push!(l.implicit_tables, d) + end end return d::TOMLDict end @@ -523,7 +688,23 @@ function parse_table(l) return ParserError(ErrExpectedEndOfTable) end l.active_table = @try recurse_dict!(l, l.root, table_key) + if l.active_table in l.implicit_tables + return ParserError(ErrDuplicatedKey) + end push!(l.defined_tables, l.active_table) + if l.comments !== nothing + l.in_array_table = path_traverses_array(l, table_key) + empty!(l.active_table_path) + append!(l.active_table_path, table_key) + if l.in_array_table + empty!(l.pending_comments) + l.last_item_path = nothing + else + path = copy(table_key) + attach_pending_comments!(l, path) + l.last_item_path = path + end + end return end @@ -544,27 +725,47 @@ function parse_array_table(l)::Union{Nothing, ParserError} return ParserError(ErrArrayTreatedAsDictionary) end d_new = TOMLDict() + first_element = isempty(old::Vector{Any}) push!(old::Vector{Any}, d_new) push!(l.defined_tables, d_new) l.active_table = d_new + if l.comments !== nothing + # Only the first header has an unambiguous path. + if first_element && !path_traverses_array(l, @view(table_key[1:end-1])) + attach_pending_comments!(l, copy(table_key)) + else + empty!(l.pending_comments) + end + empty!(l.active_table_path) + append!(l.active_table_path, table_key) + l.last_item_path = nothing + l.in_array_table = true + end return end function parse_entry(l::Parser, d)::Union{Nothing, ParserError} key = @try parse_key(l) + # `key` aliases `dotted_keys`, which parsing an inline table may overwrite. + capture_comments = l.comments !== nothing && !(d in l.inline_tables) + entry_path = (capture_comments && !l.in_array_table) ? + vcat(l.active_table_path, key) : nothing skip_ws(l) if !accept(l, '=') return ParserError(ErrExpectedEqualAfterKey) end if length(key) > 1 - d = @try recurse_dict!(l, d, @view(key[1:end-1])) + d = @try recurse_dict!(l, d, @view(key[1:end-1]), true, true) end last_key_part = l.dotted_keys[end] v = get(d, last_key_part, nothing) if v !== nothing @try check_allowed_add_key(l, v) + if v isa TOMLDict && v in l.implicit_tables + return ParserError(ErrDuplicatedKey) + end end skip_ws(l) @@ -575,6 +776,15 @@ function parse_entry(l::Parser, d)::Union{Nothing, ParserError} end # TODO: Performance, hashing `last_key_part` again here d[last_key_part] = value + if capture_comments + if entry_path === nothing + empty!(l.pending_comments) + l.last_item_path = nothing + else + attach_pending_comments!(l, entry_path) + l.last_item_path = entry_path + end + end return end @@ -607,9 +817,9 @@ function _parse_key(l::Parser) return ParserError(ErrEmptyBareKey) end keyval = if accept(l, '"') - @try parse_string_start(l, false) + @try parse_string_start(l, false; allow_multiline=false) elseif accept(l, '\'') - @try parse_string_start(l, true) + @try parse_string_start(l, true; allow_multiline=false) else set_marker!(l) if accept_batch(l, isvalid_barekey_char) @@ -716,11 +926,11 @@ function parse_array(l::Parser{Dates})::Err{Vector} where Dates copyto_typed!(new, array) elseif T === Union{} new = Any[] - elseif (T === TOMLDict) || (T == BigInt) || (T === UInt128) || (T === Int128) || (T <: Vector) || - (T === Dates.Date) || (T === Dates.Time) || (T === Dates.DateTime) + elseif (T === TOMLDict) || (T === BigInt) || (T === UInt128) || (T === Int128) || (T <: Vector) || + (Dates !== nothing && ((T === Dates.Date) || (T === Dates.Time) || (T === Dates.DateTime))) # do nothing, leave as Vector{Any} new = array - else @assert false end + else @assert false "unexpected type" end push!(l.static_arrays, new) return new end @@ -733,18 +943,17 @@ end function parse_inline_table(l::Parser)::Err{TOMLDict} dict = TOMLDict() push!(l.inline_tables, dict) - skip_ws(l) + skip_ws_nl(l) accept(l, '}') && return dict while true @try parse_entry(l, dict) - # SPEC: No newlines are allowed between the curly braces unless they are valid within a value. - skip_ws(l) + # TOML v1.1: newlines and trailing commas are allowed in inline tables + skip_ws_nl(l) accept(l, '}') && return dict if accept(l, ',') - skip_ws(l) - if accept(l, '}') - return ParserError(ErrTrailingCommaInlineTable) - end + skip_ws_nl(l) + # Trailing comma is allowed in TOML v1.1 + accept(l, '}') && return dict else return ParserError(ErrExpectedCommaBetweenItemsInlineTable) end @@ -772,7 +981,7 @@ isvalid_binary(c::Char) = '0' <= c <= '1' const ValidSigs = Union{typeof(isvalid_hex), typeof(isvalid_oct), typeof(isvalid_binary), typeof(isdigit)} # This function eats things accepted by `f` but also allows eating `_` in between -# digits. Returns if it ate at lest one character and if it ate an underscore +# digits. Returns if it ate at least one character and if it ate an underscore function accept_batch_underscore(l::Parser, f::ValidSigs, fail_if_underscore=true)::Err{Tuple{Bool, Bool}} contains_underscore = false at_least_one = false @@ -805,9 +1014,6 @@ function accept_batch_underscore(l::Parser, f::ValidSigs, fail_if_underscore=tru end function parse_number_or_date_start(l::Parser) - integer = true - read_dot = false - set_marker!(l) sgn = 1 parsed_sign = false @@ -846,7 +1052,16 @@ function parse_number_or_date_start(l::Parser) ate, contains_underscore = @try accept_batch_underscore(l, isvalid_binary) ate && return parse_bin(l, contains_underscore) elseif accept(l, isdigit) - return parse_local_time(l) + # Could be a local time (00:30) or a zero-padded date (0001-01-01). + # Consume remaining digits, then check for ':' or '-'. + accept_batch(l, isdigit) + if peek(l) == ':' + return parse_local_time(l) + elseif peek(l) == '-' + return parse_datetime(l) + else + return ParserError(ErrLeadingZeroNotAllowedInteger) + end end end @@ -887,8 +1102,7 @@ function parse_number_or_date_start(l::Parser) accept(l, x-> x == '+' || x == '-') # SPEC: (which follows the same rules as decimal integer values but may include leading zeros) read_digit = accept_batch(l, isdigit) - ate, read_underscore = @try accept_batch_underscore(l, isdigit, !read_digit) - contains_underscore |= read_underscore + _, read_underscore = @try accept_batch_underscore(l, isdigit, !read_digit) end if !ok_end_value(peek(l)) eat_char(l) @@ -900,7 +1114,7 @@ end function take_string_or_substring(l, contains_underscore)::SubString subs = take_substring(l) - # Need to pass a AbstractString to `parse` so materialize it in case it + # Need to pass an AbstractString to `parse` so materialize it in case it # contains underscore. return contains_underscore ? SubString(filter(!=('_'), subs)) : subs end @@ -983,8 +1197,10 @@ ok_end_value(c::Char) = iswhitespace(c) || c == '#' || c == EOF_CHAR || c == ']' accept_two(l, f::F) where {F} = accept_n(l, 2, f) || return(ParserError(ErrParsingDateTime)) function parse_datetime(l) # Year has already been eaten when we reach here + year_start = l.marker year = @try parse_int(l, false) - year in 0:9999 || return ParserError(ErrParsingDateTime) + # SPEC: date-fullyear = 4DIGIT (exactly 4 digits required) + (l.prevpos - year_start) == 4 || return ParserError(ErrParsingDateTime) # Month accept(l, '-') || return ParserError(ErrParsingDateTime) @@ -1060,6 +1276,8 @@ function try_return_date(p::Parser{Dates}, year, month, day) where Dates end function parse_local_time(l::Parser) + # SPEC: partial-time = time-hour ":" ... where time-hour = 2DIGIT + (l.prevpos - l.marker) == 2 || return ParserError(ErrParsingDateTime) h = @try parse_int(l, false) h in 0:23 || return ParserError(ErrParsingDateTime) _, m, s, ms = @try _parse_local_time(l, true) @@ -1101,32 +1319,37 @@ function _parse_local_time(l::Parser, skip_hour=false)::Err{NTuple{4, Int64}} minute = parse_int(l, false) minute in 0:59 || return ParserError(ErrParsingDateTime) - accept(l, ':') || return ParserError(ErrParsingDateTime) - - # second - set_marker!(l) - @try accept_two(l, isdigit) - second = parse_int(l, false) - second in 0:59 || return ParserError(ErrParsingDateTime) - - # optional fractional second - fractional_second = Int64(0) - if accept(l, '.') + # seconds are optional in TOML v1.1 + second = Int64(0) + millisecond = Int64(0) + if accept(l, ':') + # second set_marker!(l) - found_fractional_digit = false - for i in 1:3 - found_fractional_digit |= accept(l, isdigit) - end - if !found_fractional_digit - return ParserError(ErrParsingDateTime) + @try accept_two(l, isdigit) + second = parse_int(l, false) + second in 0:59 || return ParserError(ErrParsingDateTime) + + # optional fractional second + if accept(l, '.') + set_marker!(l) + found_fractional_digit = false + for _ in 1:3 + found_fractional_digit |= accept(l, isdigit) + end + if !found_fractional_digit + return ParserError(ErrParsingDateTime) + end + # DateTime in base only manages 3 significant digits in fractional + # second. Interpret parsed digits as fractional seconds and scale to + # milliseconds precision (e.g., ".2" => 200ms, ".20" => 200ms). + ndigits = l.prevpos - l.marker + fractional_second = parse_int(l, false)::Int64 + millisecond = fractional_second * 10^(3 - ndigits) + # Truncate off the rest eventual digits + accept_batch(l, isdigit) end - # DateTime in base only manages 3 significant digits in fractional - # second - fractional_second = parse_int(l, false)::Int64 - # Truncate off the rest eventual digits - accept_batch(l, isdigit) end - return hour, minute, second, fractional_second + return hour, minute, second, millisecond end @@ -1134,7 +1357,7 @@ end # String # ########## -function parse_string_start(l::Parser, quoted::Bool)::Err{String} +function parse_string_start(l::Parser, quoted::Bool; allow_multiline::Bool=true)::Err{String} # Have eaten a `'` if `quoted` is true, otherwise have eaten a `"` multiline = false c = quoted ? '\'' : '"' @@ -1142,8 +1365,11 @@ function parse_string_start(l::Parser, quoted::Bool)::Err{String} if !accept(l, c) return "" end - accept(l, '\r') # Eat third quote - accept(l, '\n') # Eat third quote + if !allow_multiline + return ParserError(ErrMultilineStringAsKey) + end + accept(l, '\r') # Eat optional carriage return after opening triple quote + accept(l, '\n') # Eat optional newline after opening triple quote multiline = true end return parse_string_continue(l, multiline, quoted) @@ -1188,8 +1414,9 @@ function parse_string_continue(l::Parser, multiline::Bool, quoted::Bool)::Err{St start_chunk = l.prevpos else c = eat_char(l) # eat the escaped character - if c == 'u' || c == 'U' - n = c == 'u' ? 4 : 6 + if c == 'x' || c == 'u' || c == 'U' + n = c == 'x' ? 2 : + c == 'u' ? 4 : 8 set_marker!(l) if !accept_n(l, n, isvalid_hex) return ParserError(ErrInvalidUnicodeScalar) @@ -1201,11 +1428,14 @@ function parse_string_continue(l::Parser, multiline::Bool, quoted::Bool)::Err{St Any Unicode code point except high-surrogate and low-surrogate code points. In other words, the ranges of integers 0 to D7FF16 and E00016 to 10FFFF16 inclusive. + For \x, the range is 0x00-0xFF (U+0000-U+00FF). =# - if !(codepoint <= 0xD7FF || 0xE000 <= codepoint <= 0x10FFFF) + if c == 'x' + # \xHH is always valid (0x00-0xFF) + elseif !(codepoint <= 0xD7FF || 0xE000 <= codepoint <= 0x10FFFF) return ParserError(ErrInvalidUnicodeScalar) end - elseif c != 'b' && c != 't' && c != 'n' && c != 'f' && c != 'r' && c != '"' && c!= '\\' + elseif c != 'b' && c != 't' && c != 'n' && c != 'f' && c != 'r' && c != 'e' && c != '"' && c!= '\\' return ParserError(ErrInvalidEscapeCharacter) end contains_backslash = true @@ -1230,5 +1460,3 @@ function take_chunks(l::Parser, unescape::Bool)::String empty!(l.chunks) return unescape ? unescape_string(str) : str end - -end diff --git a/base/toml/printer.jl b/base/toml/printer.jl new file mode 100644 index 0000000000000..1ec51292ca1c0 --- /dev/null +++ b/base/toml/printer.jl @@ -0,0 +1,389 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +import Base: @invokelatest +import ..isvalid_barekey_char # from Parser +import ..Comments, ..CommentBlock, ..CommentPath # from Parser + +function print_toml_escaped(io::IO, s::AbstractString) + for c::AbstractChar in s + if !isvalid(c) + error("TOML print: invalid character $(repr(c)) encountered when printing string") + end + if c == '\b' + Base.print(io, '\\', 'b') + elseif c == '\t' + Base.print(io, '\\', 't') + elseif c == '\n' + Base.print(io, '\\', 'n') + elseif c == '\f' + Base.print(io, '\\', 'f') + elseif c == '\r' + Base.print(io, '\\', 'r') + elseif c == '"' + Base.print(io, '\\', '"') + elseif c == '\\' + Base.print(io, "\\", '\\') + elseif Base.iscntrl(c) + Base.print(io, "\\u") + Base.print(io, string(UInt32(c), base=16, pad=4)) + else + Base.print(io, c) + end + end +end + +const BaseTOMLValue = Union{AbstractVector, AbstractDict, Bool, Integer, AbstractFloat, AbstractString, + Base.TOML.DateTime, Base.TOML.Time, Base.TOML.Date} + +is_valid_toml_value(@nospecialize(::Any)) = false +is_valid_toml_value(@nospecialize(::BaseTOMLValue)) = true + +######## +# Keys # +######## + +function printkey(io::IO, keys::Vector{String}) + for (i, k) in enumerate(keys) + i != 1 && Base.print(io, ".") + if length(k) == 0 + # empty key + Base.print(io, "\"\"") + elseif any(!isvalid_barekey_char, k) + # quoted key + Base.print(io, "\"") + print_toml_escaped(io, k) + Base.print(io, "\"") + else + Base.print(io, k) + end + end +end + +function to_toml_value(@nospecialize(f::Function), value) + if f === identity + error("type `$(typeof(value))` is not a valid TOML type, pass a conversion function to `TOML.print`") + end + toml_value = f(value) + if !is_valid_toml_value(toml_value) + error("TOML syntax function for type `$(typeof(value))` did not return a valid TOML type but a `$(typeof(toml_value))`") + end + return toml_value +end + +########## +# Values # +########## + +# Fallback +function printvalue(f::Function, io::IO, value, sorted::Bool) + toml_value = to_toml_value(f, value) + @invokelatest printvalue(f, io, toml_value, sorted) +end + +function printvalue(f::Function, io::IO, value::AbstractVector, sorted::Bool) + Base.print(io, "[") + for (i, x) in enumerate(value) + i != 1 && Base.print(io, ", ") + printvalue(f, io, x, sorted) + end + Base.print(io, "]") +end + +function printvalue(f::Function, io::IO, value::Base.TOML.DateTime, sorted::Bool) + printvalue(f, io, value.date, sorted) + Base.print(io, "T") + printvalue(f, io, value.time, sorted) + Base.print(io, "Z") +end + +function printvalue(f::Function, io::IO, value::Base.TOML.Time, sorted::Bool) + Base.print(io, string(value.hour, pad=2)) + Base.print(io, ":") + Base.print(io, string(value.minute, pad=2)) + Base.print(io, ":") + Base.print(io, string(value.second, pad=2)) + if value.ms != 0 + Base.print(io, ".") + Base.print(io, string(value.ms, pad=3)) + end +end + +function printvalue(f::Function, io::IO, value::Base.TOML.Date, sorted::Bool) + Base.print(io, string(value.year, pad=4)) + Base.print(io, "-") + Base.print(io, string(value.month, pad=2)) + Base.print(io, "-") + Base.print(io, string(value.day, pad=2)) +end + +function printvalue(f::Function, io::IO, value::Bool, sorted::Bool) + Base.print(io, value ? "true" : "false") +end + +function printvalue(f::Function, io::IO, value::Integer, sorted::Bool) + value isa Signed && return Base.show(io, value) + # unsigned integers are printed as hex + n = 2 * ndigits(value, base=256) + Base.print(io, "0x", string(value, base=16, pad=n)) + return +end + +function printvalue(f::Function, io::IO, value::AbstractFloat, sorted::Bool) + # The early conversion here avoids invalidations from isnan/isinf + value = Float64(value) + + if isnan(value) + Base.print(io, "nan") + elseif isinf(value) + Base.print(io, value > 0 ? "+inf" : "-inf") + else + Base.print(io, value) # TOML specifies IEEE 754 binary64 for float + end +end + +function printvalue(f::Function, io::IO, value::AbstractString, sorted::Bool) + qmark = Base.contains(value, "\n") ? "\"\"\"" : "\"" + Base.print(io, qmark) + print_toml_escaped(io, value) + Base.print(io, qmark) +end + +function printvalue(f::Function, io::IO, value::AbstractDict, sorted::Bool) + print_inline_table(f, io, value, sorted) +end + +function print_inline_table(f::Function, io::IO, value::AbstractDict, sorted::Bool) + vkeys = collect(keys(value))::AbstractArray + if sorted + sort!(vkeys) + end + Base.print(io, "{") + for (i, k) in enumerate(vkeys) + v = value[k] + i != 1 && Base.print(io, ", ") + printkey(io, [String(k)]) + Base.print(io, " = ") + printvalue(f, io, v, sorted) + end + Base.print(io, "}") +end + + +############ +# Comments # +############ + +# Sanitize comment text so it cannot introduce TOML syntax. + +function print_comment_line(io::IO, indentstr::String, text::AbstractString) + Base.print(io, indentstr, '#') + # Preserve captured whitespace, but format programmatic comments as `# text`. + if !(isempty(text) || first(text) == ' ' || first(text) == '\t') + Base.print(io, ' ') + end + for c::AbstractChar in text + if c == '\n' + Base.print(io, '\n', indentstr, "# ") + elseif c == '\t' || !Base.iscntrl(c) + Base.print(io, c) + end + end + Base.print(io, '\n') +end + +function print_comments_above(io::IO, comments::Comments, path::CommentPath, indentstr::String) + block = get(comments.items, path, nothing) + block === nothing && return + for line in block.above + print_comment_line(io, indentstr, line) + end +end + +function print_comment_inline(io::IO, comments::Comments, path::CommentPath) + block = get(comments.items, path, nothing) + block === nothing && return + text = block.inline + text === nothing && return + Base.print(io, " #") + if !(isempty(text) || first(text) == ' ' || first(text) == '\t') + Base.print(io, ' ') + end + for c::AbstractChar in text + if c == '\t' || (c != '\n' && !Base.iscntrl(c)) + Base.print(io, c) + else + Base.print(io, ' ') + end + end +end + +has_comments(comments::Comments, path::CommentPath) = + haskey(comments.items, path) || haskey(comments.floating, path) + +# Hoist nested comments to an inline table's owning entry. +function print_hoisted_inline_comments(io::IO, comments::Comments, a::AbstractDict, + path::CommentPath, indentstr::String, sorted::Bool) + floating = get(comments.floating, path, nothing) + if floating !== nothing + for line in floating + print_comment_line(io, indentstr, line) + end + end + vkeys = collect(keys(a)) + sorted && sort!(vkeys) + for k in vkeys + kpath = [path; String(k)] + block = get(comments.items, kpath, nothing) + if block !== nothing + for line in block.above + print_comment_line(io, indentstr, line) + end + block.inline === nothing || print_comment_line(io, indentstr, block.inline) + end + v = a[k] + v isa AbstractDict && print_hoisted_inline_comments(io, comments, v, kpath, indentstr, sorted) + end +end + +function print_comments_floating(io::IO, comments::Comments, path::CommentPath, indentstr::String) + lines = get(comments.floating, path, nothing) + lines === nothing && return false + for line in lines + print_comment_line(io, indentstr, line) + end + return !isempty(lines) +end + +########## +# Tables # +########## + +is_table(@nospecialize(value)) = isa(value, AbstractDict) +is_array_of_tables(@nospecialize(value)) = + isa(value, AbstractArray) && + length(value) > 0 && (isa(value, AbstractArray{<:AbstractDict}) || + all(v -> isa(v, AbstractDict), value)) +is_tabular(@nospecialize(value)) = is_table(value) || @invokelatest(is_array_of_tables(value)) + +function print_table(f::Function, io::IO, a::AbstractDict, + ks::Vector{String} = String[]; + indent::Int = 0, + first_block::Bool = true, + sorted::Bool = false, + inline_tables::IdSet, + by::Function = identity, + comments::Union{Comments, Nothing} = nothing, +) + + if a in inline_tables + @invokelatest print_inline_table(f, io, a, sorted) + return + end + + entry_indentstr = ' '^4max(0, indent-1) + if comments !== nothing + if print_comments_floating(io, comments, ks, entry_indentstr) + println(io) + end + end + + akeys = keys(a) + if sorted + akeys = sort!(collect(akeys); by) + end + + # First print non-tabular entries + for key in akeys + value = a[key] + if !is_valid_toml_value(value) + value = to_toml_value(f, value) + end + if is_tabular(value) && !(value in inline_tables) + continue + end + + if comments !== nothing + entry_path = [ks; String(key)] + print_comments_above(io, comments, entry_path, entry_indentstr) + if value isa AbstractDict && value in inline_tables + print_hoisted_inline_comments(io, comments, value, entry_path, entry_indentstr, sorted) + end + end + Base.print(io, entry_indentstr) + printkey(io, [String(key)]) + Base.print(io, " = ") # print separator + printvalue(f, io, value, sorted) + if comments !== nothing + print_comment_inline(io, comments, [ks; String(key)]) + end + Base.print(io, "\n") # new line? + first_block = false + end + + for key in akeys + value = a[key] + if !is_valid_toml_value(value) + value = to_toml_value(f, value) + end + if is_table(value) && !(value in inline_tables) + push!(ks, String(key)) + _values = @invokelatest values(value) + header = isempty(value) || !all(is_tabular(v) for v in _values)::Bool || any(v in inline_tables for v in _values)::Bool + if !header && comments !== nothing + # Preserve headers that anchor comments. + header = has_comments(comments, ks) + end + if header + # print table + first_block || println(io) + first_block = false + if comments !== nothing + print_comments_above(io, comments, ks, ' '^4indent) + end + Base.print(io, ' '^4indent) + Base.print(io,"[") + printkey(io, ks) + Base.print(io,"]") + if comments !== nothing + print_comment_inline(io, comments, ks) + end + Base.print(io,"\n") + end + # Use runtime dispatch here since the type of value seems not to be enforced other than as AbstractDict + @invokelatest print_table(f, io, value, ks; indent = indent + header, first_block = header, sorted, by, inline_tables, comments) + pop!(ks) + elseif @invokelatest(is_array_of_tables(value)) + # print array of tables + first_block || println(io) + first_block = false + push!(ks, String(key)) + if comments !== nothing + print_comments_above(io, comments, ks, ' '^4indent) + end + for v in value + Base.print(io, ' '^4indent) + Base.print(io,"[[") + printkey(io, ks) + Base.print(io,"]]\n") + # TODO, nicer error here + !isa(v, AbstractDict) && error("array should contain only tables") + @invokelatest print_table(f, io, v, ks; indent = indent + 1, sorted, by, inline_tables, comments) + end + pop!(ks) + end + end +end + + +####### +# API # +####### + +print(f::Function, io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}(), comments::Union{Comments, Nothing}=nothing) = + print_table(f, io, a; sorted, by, inline_tables, comments) +print(f::Function, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}(), comments::Union{Comments, Nothing}=nothing) = + print(f, stdout, a; sorted, by, inline_tables, comments) +print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}(), comments::Union{Comments, Nothing}=nothing) = + print_table(identity, io, a; sorted, by, inline_tables, comments) +print(a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}(), comments::Union{Comments, Nothing}=nothing) = + print(identity, stdout, a; sorted, by, inline_tables, comments) diff --git a/base/toml/toml.jl b/base/toml/toml.jl new file mode 100644 index 0000000000000..7426d1259869a --- /dev/null +++ b/base/toml/toml.jl @@ -0,0 +1,19 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +""" +`Base.TOML` is an undocumented internal part of Julia's TOML implementation. +Users should call the documented interface in the TOML.jl standard library +instead (by `import TOML` or `using TOML`). +""" +module TOML + +include("parser.jl") + +# We put the printing functionality in a separate module since it +# defines a function `print` and we don't want that to collide with normal +# usage of `(Base.)print` in other files +module Printer + include("printer.jl") +end + +end diff --git a/base/traits.jl b/base/traits.jl index 47ab8ddc0c7ac..808984d593914 100644 --- a/base/traits.jl +++ b/base/traits.jl @@ -13,6 +13,11 @@ OrderStyle(::Type{Symbol}) = Ordered() OrderStyle(::Type{<:Any}) = Unordered() OrderStyle(::Type{Union{}}, slurp...) = Ordered() +function OrderStyle(T::Type{<:Tuple}) + isconcretetype(T) || return Unordered() + all(map(S -> OrderStyle(S) === Ordered(), fieldtypes(T))) ? Ordered() : Unordered() +end + # trait for objects that support arithmetic abstract type ArithmeticStyle end struct ArithmeticRounds <: ArithmeticStyle end # least significant bits can be lost diff --git a/base/tuple.jl b/base/tuple.jl index ac42c667269e6..0d86b5a1c37dc 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -40,12 +40,10 @@ getindex(t::Tuple, c::Colon) = t get(t::Tuple, i::Integer, default) = i in 1:length(t) ? getindex(t, i) : default get(f::Callable, t::Tuple, i::Integer) = i in 1:length(t) ? getindex(t, i) : f() -# returns new tuple; N.B.: becomes no-op if `i` is out-of-bounds - """ setindex(t::Tuple, v, i::Integer) -Creates a new tuple similar to `t` with the value at index `i` set to `v`. +Create a new tuple similar to `t` with the value at index `i` set to `v`. Throws a `BoundsError` when out of bounds. # Examples @@ -161,7 +159,7 @@ end # this allows partial evaluation of bounded sequences of next() calls on tuples, # while reducing to plain next() for arbitrary iterables. indexed_iterate(t::Tuple, i::Int, state=1) = (@inline; (getfield(t, i), i+1)) -indexed_iterate(a::Array, i::Int, state=1) = (@inline; (a[i], i+1)) +indexed_iterate(a::Union{Array,Memory}, i::Int, state=1) = (@inline; (a[i], i+1)) function indexed_iterate(I, i) x = iterate(I) x === nothing && throw(BoundsError(I, i)) @@ -206,9 +204,12 @@ julia> first, Base.rest(a, state) """ function rest end rest(t::Tuple) = t -rest(t::Tuple, i::Int) = ntuple(x -> getfield(t, x+i-1), length(t)-i+1) -rest(a::Array, i::Int=1) = a[i:end] -rest(a::Core.SimpleVector, i::Int=1) = a[i:end] +function rest(t::Tuple, i) + let i = i::Int + ntuple(x -> getfield(t, x+i-1), length(t)-i+1) + end +end +rest(a::Union{Array,Memory,Core.SimpleVector}, i=1) = a[(i::Int):end] rest(itr, state...) = Iterators.rest(itr, state...) """ @@ -261,7 +262,9 @@ function _split_rest(a::Union{AbstractArray, Core.SimpleVector}, n::Int) return a[begin:end-n], a[end-n+1:end] end -@eval split_rest(t::Tuple, n::Int, i=1) = ($(Expr(:meta, :aggressive_constprop)); (t[i:end-n], t[end-n+1:end])) +@eval _split_tuple(t::Tuple, n::Int, i::Int=1) = ($(Expr(:meta, :aggressive_constprop)); (t[i:n], t[n+1:end])) + +@eval split_rest(t::Tuple, n::Int, i=1) = ($(Expr(:meta, :aggressive_constprop)); _split_tuple(t, length(t)-n, Int(i))) # Use dispatch to avoid a branch in first first(::Tuple{}) = throw(ArgumentError("tuple must be non-empty")) @@ -269,10 +272,22 @@ first(t::Tuple) = t[1] # eltype -eltype(::Type{Tuple{}}) = Bottom # the <: here makes the runtime a bit more complicated (needing to check isdefined), but really helps inference -eltype(t::Type{<:Tuple{Vararg{E}}}) where {E} = @isdefined(E) ? (E isa Type ? E : Union{}) : _compute_eltype(t) -eltype(t::Type{<:Tuple}) = _compute_eltype(t) +# `E` is undefined either when the `Vararg` matched vacuously (`t == Tuple{}`, +# eltype `Union{}`) or when a non-concrete element type left the diagonal var +# unpinned, where `_compute_eltype` recovers the join +_eltype_ntuple(t::Type{<:Tuple{Vararg{E}}}) where {E} = @isdefined(E) ? (E isa Type ? E : Union{}) : (t <: Tuple{} ? Union{} : _compute_eltype(t)) +const _eltype_ntuple_sig_condition = (Type{<:Tuple{Vararg{E}}} where E) +# We'd like to be able to infer eltype(::Tuple), so keep the number of eltype(::Type{<:Tuple}) methods at max_methods! +function eltype(t::Type{<:Tuple}) + if t <: Tuple{} + Bottom + elseif isa(t, _eltype_ntuple_sig_condition) + _eltype_ntuple(t) + else + _compute_eltype(t) + end +end function _compute_eltype(@nospecialize t) @_total_meta has_free_typevars(t) && return Any @@ -297,21 +312,6 @@ function _compute_eltype(@nospecialize t) return r end -# We'd like to be able to infer eltype(::Tuple), which needs to be able to -# look at these four methods: -# -# julia> methods(Base.eltype, Tuple{Type{<:Tuple}}) -# 4 methods for generic function "eltype" from Base: -# [1] eltype(::Type{Union{}}) -# @ abstractarray.jl:234 -# [2] eltype(::Type{Tuple{}}) -# @ tuple.jl:199 -# [3] eltype(t::Type{<:Tuple{Vararg{E}}}) where E -# @ tuple.jl:200 -# [4] eltype(t::Type{<:Tuple}) -# @ tuple.jl:209 -typeof(function eltype end).name.max_methods = UInt8(4) - # key/val types keytype(@nospecialize t::Tuple) = keytype(typeof(t)) keytype(@nospecialize T::Type{<:Tuple}) = Int @@ -371,6 +371,18 @@ const All32{T,N} = Tuple{T,T,T,T,T,T,T,T, T,T,T,T,T,T,T,T, T,T,T,T,T,T,T,T, Vararg{T,N}} + +function front(t::NTuple{32, Any}) + @inline + _front(t...) +end + +function front(t::Any32) + n = length(t) - 1 + r = ntuple(i -> getfield(t, i), n) + return r::Tuple{Vararg{eltype(typeof(t))}} +end + function map(f, t::Any32) n = length(t) A = Vector{Any}(undef, n) @@ -484,6 +496,8 @@ function _totuple(T::Type{All32{E,N}}, itr) where {E,N} (elts...,) end +# fast path for Array/Memory lives in reinterpretarray.jl + _totuple(::Type{Tuple{Vararg{E}}}, itr, s...) where {E} = (collect(E, Iterators.rest(itr,s...))...,) _totuple(::Type{Tuple}, itr, s...) = (collect(Iterators.rest(itr,s...))...,) @@ -648,6 +662,19 @@ revargs(x, r...) = (revargs(r...)..., x) reverse(t::Tuple) = revargs(t...) +""" + isassigned(v::Tuple, i::Integer) -> Bool + +Return `true` if index `i` is within the bounds of tuple `v`, i.e. `1 ≤ i ≤ length(v)`. + +!!! compat "Julia 1.13" + This method requires at least Julia 1.13. +""" +function isassigned(v::Tuple, i::Integer) + @boundscheck 1 <= i <= length(v) || return false + true +end + ## specialized reduction ## prod(x::Tuple{}) = 1 diff --git a/base/twiceprecision.jl b/base/twiceprecision.jl index 920ba71eba24f..dd96206c15e85 100644 --- a/base/twiceprecision.jl +++ b/base/twiceprecision.jl @@ -112,7 +112,7 @@ julia> Float64(hi) + Float64(lo) ``` """ function mul12(x::T, y::T) where {T<:AbstractFloat} - (h, l) = Math.two_mul(x, y) + (h, l) = two_mul(x, y) ifelse(!isfinite(h), (h, h), (h, l)) end mul12(x::T, y::T) where {T} = (p = x * y; (p, zero(p))) @@ -252,7 +252,7 @@ nbitslen(r::StepRangeLen) = nbitslen(eltype(r), length(r), r.offset) nbitslen(::Type{T}, len, offset) where {T<:IEEEFloat} = min(cld(precision(T), 2), nbitslen(len, offset)) # The +1 here is for safety, because the precision of the significand -# is 1 bit higher than the number that are explicitly stored. +# is 1 bit higher than the number that is explicitly stored. nbitslen(len, offset) = len < 2 ? 0 : top_set_bit(max(offset-1, len-offset) - 1) + 1 eltype(::Type{TwicePrecision{T}}) where {T} = T @@ -616,7 +616,7 @@ function sum(r::StepRangeLen{<:Any,<:TwicePrecision,<:TwicePrecision}) # Add in contributions of ref ref = r.ref * l sm_hi, sm_lo = add12(s_hi, ref.hi) - add12(sm_hi, sm_lo + ref.lo)[1] + add12(sm_hi, sm_lo + s_lo + ref.lo)[1] end # sum(1:n) as a product of two integers @@ -649,8 +649,8 @@ function range_start_stop_length(start::T, stop::T, len::Integer) where {T<:IEEE return steprangelen_hp(T, start, zero(T), 0, len, 1) end # Attempt to find exact rational approximations - start_n, start_d = rat(start) - stop_n, stop_d = rat(stop) + _, start_d = rat(start) + _, stop_d = rat(stop) if start_d != 0 && stop_d != 0 den = lcm_unchecked(start_d, stop_d) m = maxintfloat(T, Int) @@ -798,7 +798,7 @@ function _log_twice64_unchecked(x::Float64) if xu < (UInt64(1)<<52) # x is subnormal xu = reinterpret(UInt64, x * 0x1p52) # normalize x xu &= ~sign_mask(Float64) - xu -= UInt64(52) << 52 # mess with the exponent + xu -%= (UInt64(52) << 52) # mess with the exponent end TwicePrecision(Math._log_ext(xu)...) end diff --git a/base/util.jl b/base/util.jl index fa01f1bcde498..e9489081f3568 100644 --- a/base/util.jl +++ b/base/util.jl @@ -149,8 +149,8 @@ See also [`print`](@ref), [`println`](@ref), [`show`](@ref). Return a julia command similar to the one of the running process. Propagates any of the `--cpu-target`, `--sysimage`, `--compile`, `--sysimage-native-code`, `--compiled-modules`, `--pkgimages`, `--inline`, `--check-bounds`, `--optimize`, `--min-optlevel`, `-g`, -`--code-coverage`, `--track-allocation`, `--color`, `--startup-file`, and `--depwarn` -command line arguments that are not at their default values. +`--code-coverage`, `--code-coverage-mode`, `--track-allocation`, `--color`, `--startup-file`, +and `--depwarn` command line arguments that are not at their default values. Among others, `--math-mode`, `--warn-overwrite`, and `--trace-compile` are notably not propagated currently. @@ -175,11 +175,11 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio end image_file = unsafe_string(opts.image_file) addflags = String[] - let compile = if opts.compile_enabled == 0 + let compile = if opts.compile_enabled == JL_OPTIONS_COMPILE_OFF "no" - elseif opts.compile_enabled == 2 + elseif opts.compile_enabled == JL_OPTIONS_COMPILE_ALL "all" - elseif opts.compile_enabled == 3 + elseif opts.compile_enabled == JL_OPTIONS_COMPILE_MIN "min" else "" # default = "yes" @@ -227,6 +227,7 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio isempty(coverage_file) || push!(addflags, "--code-coverage=$coverage_file") end end + opts.code_coverage_mode == 1 && push!(addflags, "--code-coverage-mode=count") if opts.malloc_log == 1 push!(addflags, "--track-allocation=user") elseif opts.malloc_log == 2 @@ -245,6 +246,9 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio if opts.use_sysimage_native_code == 0 push!(addflags, "--sysimage-native-code=no") end + if opts.compress_sysimage == 1 + push!(addflags, "--compress-sysimage=yes") + end return `$julia -C $cpu_target -J$image_file $addflags` end @@ -677,7 +681,7 @@ end # testing """ - Base.runtests(tests=["all"]; ncores=ceil(Int, Sys.CPU_THREADS / 2), + Base.runtests(tests=["all"]; ncores=ceil(Int, Sys.EFFECTIVE_CPU_THREADS / 2), exit_on_error=false, revise=false, propagate_project=true, [seed], [julia_args::Cmd]) Run the Julia unit tests listed in `tests`, which can be either a string or an array of @@ -691,7 +695,7 @@ If a seed is provided via the keyword argument, it is used to seed the global RNG in the context where the tests are run; otherwise the seed is chosen randomly. The argument `julia_args` can be used to pass custom `julia` command line flags to the test process. """ -function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), +function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.EFFECTIVE_CPU_THREADS / 2), exit_on_error::Bool=false, revise::Bool=false, propagate_project::Bool=false, @@ -716,20 +720,15 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), Base.DATAROOTDIR, "julia", "test", "runtests.jl")) $tests`, ENV2)) nothing catch - buf = PipeBuffer() - let InteractiveUtils = Base.require_stdlib(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) - @invokelatest InteractiveUtils.versioninfo(buf) + # evaluate versioninfo in the test environment so the listed env vars are the same + vinfo = read(setenv(`$(julia_cmd()) -e 'let InteractiveUtils = Base.require_stdlib(Base.PkgId(Base.UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")); @invokelatest(InteractiveUtils.versioninfo()); end'`, ENV2), String) + msg = "A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" * + "including error messages above and the output of versioninfo():\n$(vinfo)" + if isinteractive() + error(msg) + else + print(stderr, "ERROR: ", msg) + exit(1) end - error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" * - "including error messages above and the output of versioninfo():\n$(read(buf, String))") end end - -""" - isdebugbuild() - -Return `true` if julia is a debug version. -""" -function isdebugbuild() - return ccall(:jl_is_debugbuild, Cint, ()) != 0 -end diff --git a/base/uuid.jl b/base/uuid.jl index 4b9bae863d926..9bdde25c9a854 100644 --- a/base/uuid.jl +++ b/base/uuid.jl @@ -92,13 +92,19 @@ let groupings = [36:-1:25; 23:-1:20; 18:-1:15; 13:-1:10; 8:-1:1] global string function string(u::UUID) u = u.value - a = Base.StringMemory(36) - for i in groupings - @inbounds a[i] = hex_chars[1 + u & 0xf] - u >>= 4 + str = Base._string_n(36) + GC.@preserve str begin + p = pointer(str) + for i in groupings + unsafe_store!(p, @inbounds(hex_chars[1 + u & 0xf]), i) + u >>= 4 + end + unsafe_store!(p, UInt8('-'), 9) + unsafe_store!(p, UInt8('-'), 14) + unsafe_store!(p, UInt8('-'), 19) + unsafe_store!(p, UInt8('-'), 24) end - @inbounds a[24] = a[19] = a[14] = a[9] = '-' - return unsafe_takestring(a) + return str end end diff --git a/base/version.jl b/base/version.jl index 71192916a5b22..ddf8be0cc0a0c 100644 --- a/base/version.jl +++ b/base/version.jl @@ -12,9 +12,12 @@ Version number type which follows the specifications of [semantic versioning (semver)](https://semver.org/spec/v2.0.0-rc.2.html), composed of major, minor and patch numeric values, followed by pre-release and build alphanumeric annotations. +As an extension to this standard, Julia also allows a single, empty prerelease annotation +if there is no build identifier present (e.g. `1.0-`), or a single, empty build annotation (e.g. `1.0+`). `VersionNumber` objects can be compared with all of the standard comparison operators (`==`, `<`, `<=`, etc.), with the result following semver v2.0.0-rc.2 rules. +Different from the semver standard, build annotations are not ignored when comparing version numbers. `VersionNumber` has the following public fields: - `v.major::Integer` @@ -97,11 +100,12 @@ function print(io::IO, v::VersionNumber) print(io, v.patch) if !isempty(v.prerelease) print(io, '-') - join(io, v.prerelease,'.') + # inline join to make call resolvable for --trim + @inline join(io, v.prerelease,'.') end if !isempty(v.build) print(io, '+') - join(io, v.build,'.') + @inline join(io, v.build,'.') end end show(io::IO, v::VersionNumber) = print(io, "v\"", v, "\"") @@ -272,3 +276,469 @@ else end libllvm_path() = ccall(:jl_get_libllvm, Any, ()) + + +################ +# VersionBound # +################ +struct VersionBound + t::NTuple{3, UInt32} + n::Int + function VersionBound(tin::NTuple{n, Integer}) where {n} + n <= 3 || throw(ArgumentError("VersionBound: you can only specify major, minor and patch versions")) + n == 0 && return new((0, 0, 0), n) + n == 1 && return new((tin[1], 0, 0), n) + n == 2 && return new((tin[1], tin[2], 0), n) + n == 3 && return new((tin[1], tin[2], tin[3]), n) + error("invalid $n") + end +end +VersionBound(t::Integer...) = VersionBound(t) +VersionBound(v::VersionNumber) = VersionBound(v.major, v.minor, v.patch) + +Base.getindex(b::VersionBound, i::Int) = b.t[i] + +function ≲(v::VersionNumber, b::VersionBound) + b.n == 0 && return true + b.n == 1 && return v.major <= b[1] + b.n == 2 && return (v.major, v.minor) <= (b[1], b[2]) + return (v.major, v.minor, v.patch) <= (b[1], b[2], b[3]) +end + +function ≲(b::VersionBound, v::VersionNumber) + b.n == 0 && return true + b.n == 1 && return v.major >= b[1] + b.n == 2 && return (v.major, v.minor) >= (b[1], b[2]) + return (v.major, v.minor, v.patch) >= (b[1], b[2], b[3]) +end + +function isless_ll(a::VersionBound, b::VersionBound) + m, n = a.n, b.n + for i in 1:min(m, n) + a[i] < b[i] && return true + a[i] > b[i] && return false + end + return m < n +end + +stricterlower(a::VersionBound, b::VersionBound) = isless_ll(a, b) ? b : a + +# Comparison between two upper bounds +function isless_uu(a::VersionBound, b::VersionBound) + m, n = a.n, b.n + for i in 1:min(m, n) + a[i] < b[i] && return true + a[i] > b[i] && return false + end + return m > n +end + +stricterupper(a::VersionBound, b::VersionBound) = isless_uu(a, b) ? a : b + +# `isjoinable` compares an upper bound of a range with the lower bound of the next range +# to determine if they can be joined, as in [1.5-2.8, 2.5-3] -> [1.5-3]. Used by `union!`. +# The equal-length-bounds case is special since e.g. `1.5` can be joined with `1.6`, +# `2.3.4` can be joined with `2.3.5` etc. + +function isjoinable(up::VersionBound, lo::VersionBound) + up.n == 0 && lo.n == 0 && return true + if up.n == lo.n + n = up.n + for i in 1:(n - 1) + up[i] > lo[i] && return true + up[i] < lo[i] && return false + end + up[n] < lo[n] - 1 && return false + return true + else + l = min(up.n, lo.n) + for i in 1:l + up[i] > lo[i] && return true + up[i] < lo[i] && return false + end + end + return true +end + +Base.hash(r::VersionBound, h::UInt) = hash(r.t, hash(r.n, h)) + +# Hot code +function VersionBound(s::AbstractString) + s = strip(s) + s == "*" && return VersionBound() + first(s) == 'v' && (s = SubString(s, 2)) + l = lastindex(s) + + p = findnext('.', s, 1) + b = p === nothing ? l : (p - 1) + i = parse(Int64, SubString(s, 1, b)) + p === nothing && return VersionBound(i) + + a = p + 1 + p = findnext('.', s, a) + b = p === nothing ? l : (p - 1) + j = parse(Int64, SubString(s, a, b)) + p === nothing && return VersionBound(i, j) + + a = p + 1 + p = findnext('.', s, a) + b = p === nothing ? l : (p - 1) + k = parse(Int64, SubString(s, a, b)) + p === nothing && return VersionBound(i, j, k) + + error("invalid VersionBound string $(repr(s))") +end + +################ +# VersionRange # +################ +struct VersionRange + lower::VersionBound + upper::VersionBound + # NOTE: ranges are allowed to be empty; they are ignored by VersionSpec anyway + function VersionRange(lo::VersionBound, hi::VersionBound) + # lo.t == hi.t implies that digits past min(lo.n, hi.n) are zero + # lo.n < hi.n example: 1.2-1.2.0 => 1.2.0 + # lo.n > hi.n example: 1.2.0-1.2 => 1.2 + lo.t == hi.t && (lo = hi) + return new(lo, hi) + end +end +VersionRange(b::VersionBound = VersionBound()) = VersionRange(b, b) +VersionRange(t::Integer...) = VersionRange(VersionBound(t...)) +VersionRange(v::VersionNumber) = VersionRange(VersionBound(v)) +VersionRange(lo::VersionNumber, hi::VersionNumber) = VersionRange(VersionBound(lo), VersionBound(hi)) + +# The vast majority of VersionRanges are in practice equal to "1" +const VersionRange_1 = VersionRange(VersionBound("1"), VersionBound("1")) +function VersionRange(s::AbstractString) + s == "1" && return VersionRange_1 + p = split(s, "-") + if length(p) != 1 && length(p) != 2 + throw(ArgumentError("invalid version range: $(repr(s))")) + end + lower = VersionBound(p[1]) + upper = length(p) == 1 ? lower : VersionBound(p[2]) + return VersionRange(lower, upper) +end + +function Base.isempty(r::VersionRange) + for i in 1:min(r.lower.n, r.upper.n) + r.lower[i] > r.upper[i] && return true + r.lower[i] < r.upper[i] && return false + end + return false +end + +function Base.print(io::IO, r::VersionRange) + m, n = r.lower.n, r.upper.n + return if (m, n) == (0, 0) + print(io, '*') + elseif m == 0 + print(io, "0 -") + join(io, r.upper.t, '.') + elseif n == 0 + join(io, r.lower.t, '.') + print(io, " - *") + else + join(io, r.lower.t[1:m], '.') + if r.lower != r.upper + print(io, " - ") + join(io, r.upper.t[1:n], '.') + end + end +end +Base.show(io::IO, r::VersionRange) = print(io, "VersionRange(\"", r, "\")") + +Base.in(v::VersionNumber, r::VersionRange) = r.lower ≲ v ≲ r.upper + +Base.intersect(a::VersionRange, b::VersionRange) = VersionRange(stricterlower(a.lower, b.lower), stricterupper(a.upper, b.upper)) + +function Base.union!(ranges::Vector{<:VersionRange}) + l = length(ranges) + l == 0 && return ranges + + sort!(ranges, lt = (a, b) -> (isless_ll(a.lower, b.lower) || (a.lower == b.lower && isless_uu(a.upper, b.upper)))) + + ks = findfirst(!isempty, ranges) + ks === nothing && return empty!(ranges) + + lo, up, k0 = ranges[ks].lower, ranges[ks].upper, 1 + for k in (ks + 1):l + isempty(ranges[k]) && continue + lo1, up1 = ranges[k].lower, ranges[k].upper + if isjoinable(up, lo1) + isless_uu(up, up1) && (up = up1) + continue + end + vr = VersionRange(lo, up) + @assert !isempty(vr) "empty VersionRange" + ranges[k0] = vr + k0 += 1 + lo, up = lo1, up1 + end + vr = VersionRange(lo, up) + if !isempty(vr) + ranges[k0] = vr + k0 += 1 + end + resize!(ranges, k0 - 1) + return ranges +end + +Base.minimum(r::VersionRange) = r.lower + +############### +# VersionSpec # +############### +struct VersionSpec + ranges::Vector{VersionRange} + VersionSpec(r::Vector{<:VersionRange}) = new(length(r) == 1 ? r : union!(r)) + VersionSpec(vs::VersionSpec) = vs +end + +VersionSpec(r::VersionRange) = VersionSpec(VersionRange[r]) +VersionSpec(v::VersionNumber) = VersionSpec(VersionRange(v)) +const _all_versionsspec = VersionSpec(VersionRange()) +VersionSpec() = _all_versionsspec +VersionSpec(s::AbstractString) = VersionSpec(VersionRange(s)) +VersionSpec(v::AbstractVector) = VersionSpec(map(VersionRange, v)) + +# Hot code +function Base.in(v::VersionNumber, s::VersionSpec) + for r in s.ranges + v in r && return true + end + return false +end + +# Optimized batch version check for version lists +# Fills dest[1:n] indicating which versions are in the VersionSpec +# Optimized for sorted version lists (but works correctly even if unsorted) +# Note: Only fills indices 1:n, leaves rest of dest unchanged +function matches_spec_range!(dest::BitVector, versions::AbstractVector{VersionNumber}, spec::VersionSpec, n::Int) + @assert length(versions) == n "invalid version list" + @assert length(dest) >= n "invalid dest length" + + # Initialize to false + dest[1:n] .= false + + isempty(spec.ranges) && return dest + + # Assumes versions are sorted (as created in Operations.jl:1002) + # If sorted, this avoids O(n*m) comparisons by scanning linearly + @inbounds for range in spec.ranges + # Find first version that could be in range + i = 1 + while i <= n && !(range.lower ≲ versions[i]) + i += 1 + end + + # Mark all versions in range + while i <= n && versions[i] ≲ range.upper + dest[i] = true + i += 1 + end + end + + return dest +end + +Base.copy(vs::VersionSpec) = VersionSpec(vs) + +const empty_versionspec = VersionSpec(VersionRange[]) +const _empty_symbol = "∅" + +Base.isempty(s::VersionSpec) = all(isempty, s.ranges) +@assert isempty(empty_versionspec) +# Hot code, measure performance before changing +function Base.intersect(A::VersionSpec, B::VersionSpec) + (isempty(A) || isempty(B)) && return copy(empty_versionspec) + ranges = Vector{VersionRange}(undef, length(A.ranges) * length(B.ranges)) + i = 1 + @inbounds for a in A.ranges, b in B.ranges + ranges[i] = intersect(a, b) + i += 1 + end + return VersionSpec(ranges) +end +Base.intersect(a::VersionNumber, B::VersionSpec) = a in B ? VersionSpec(a) : empty_versionspec +Base.intersect(A::VersionSpec, b::VersionNumber) = intersect(b, A) + +function Base.union(A::VersionSpec, B::VersionSpec) + A == B && return A + Ar = copy(A.ranges) + append!(Ar, B.ranges) + union!(Ar) + return VersionSpec(Ar) +end + +Base.:(==)(A::VersionSpec, B::VersionSpec) = A.ranges == B.ranges +Base.hash(s::VersionSpec, h::UInt) = hash(s.ranges, h +% (0x2fd2ca6efa023f44 % UInt)) + +function Base.print(io::IO, s::VersionSpec) + isempty(s) && return print(io, _empty_symbol) + length(s.ranges) == 1 && return print(io, s.ranges[1]) + print(io, '[') + for i in 1:length(s.ranges) + 1 < i && print(io, ", ") + print(io, s.ranges[i]) + end + return print(io, ']') +end + +function Base.show(io::IO, s::VersionSpec) + print(io, "VersionSpec(") + if length(s.ranges) == 1 + print(io, '"', s.ranges[1], '"') + else + print(io, "[") + for i in 1:length(s.ranges) + 1 < i && print(io, ", ") + print(io, '"', s.ranges[i], '"') + end + print(io, ']') + end + return print(io, ")") +end + +Base.minimum(v::VersionSpec) = minimum(v.ranges[1]) + +################### +# Semver notation # +################### + +function semver_spec(s::String; throw = true) + ranges = VersionRange[] + for ver in strip.(split(strip(s), ',')) + range = nothing + found_match = false + for (ver_reg, f) in ver_regs + if occursin(ver_reg, ver) + range = f(match(ver_reg, ver)) + found_match = true + break + end + end + if !found_match + if throw + error("invalid version specifier: \"$s\"") + else + return nothing + end + end + push!(ranges, range) + end + return VersionSpec(ranges) +end + +function semver_interval(m::RegexMatch) + @assert length(m.captures) == 4 "invalid match" + n_significant = count(x -> x !== nothing, m.captures) - 1 + typ, _major, _minor, _patch = m.captures + major = parse(Int, _major) + minor = (n_significant < 2) ? 0 : parse(Int, _minor) + patch = (n_significant < 3) ? 0 : parse(Int, _patch) + if n_significant == 3 && major == 0 && minor == 0 && patch == 0 + error("invalid version: \"0.0.0\"") + end + # Default type is :caret + vertyp = (typ == "" || typ == "^") ? :caret : :tilde + v0 = VersionBound((major, minor, patch)) + return if vertyp === :caret + if major != 0 + return VersionRange(v0, VersionBound((v0[1],))) + elseif minor != 0 + return VersionRange(v0, VersionBound((v0[1], v0[2]))) + else + if n_significant == 1 + return VersionRange(v0, VersionBound((0,))) + elseif n_significant == 2 + return VersionRange(v0, VersionBound((0, 0))) + else + return VersionRange(v0, VersionBound((0, 0, v0[3]))) + end + end + else + if n_significant == 3 || n_significant == 2 + return VersionRange(v0, VersionBound((v0[1], v0[2]))) + else + return VersionRange(v0, VersionBound((v0[1],))) + end + end +end + +const _inf = VersionBound("*") +function inequality_interval(m::RegexMatch) + @assert length(m.captures) == 4 "invalid match" + typ, _major, _minor, _patch = m.captures + n_significant = count(x -> x !== nothing, m.captures) - 1 + major = parse(Int, _major) + minor = (n_significant < 2) ? 0 : parse(Int, _minor) + patch = (n_significant < 3) ? 0 : parse(Int, _patch) + if n_significant == 3 && major == 0 && minor == 0 && patch == 0 + error("invalid version: 0.0.0") + end + v = VersionBound(major, minor, patch) + if occursin(r"^<\s*$", typ) + nil = VersionBound(0, 0, 0) + if v[3] == 0 + if v[2] == 0 + v1 = VersionBound(v[1] - 1) + else + v1 = VersionBound(v[1], v[2] - 1) + end + else + v1 = VersionBound(v[1], v[2], v[3] - 1) + end + return VersionRange(nil, v1) + elseif occursin(r"^=\s*$", typ) + return VersionRange(v) + elseif occursin(r"^>=\s*$", typ) || occursin(r"^≥\s*$", typ) + return VersionRange(v, _inf) + else + error("invalid prefix $typ") + end +end + +function hyphen_interval(m::RegexMatch) + @assert length(m.captures) == 6 "invalid match" + _lower_major, _lower_minor, _lower_patch, _upper_major, _upper_minor, _upper_patch = m.captures + if isnothing(_lower_minor) + lower_bound = VersionBound(parse(Int, _lower_major)) + elseif isnothing(_lower_patch) + lower_bound = VersionBound( + parse(Int, _lower_major), + parse(Int, _lower_minor) + ) + else + lower_bound = VersionBound( + parse(Int, _lower_major), + parse(Int, _lower_minor), + parse(Int, _lower_patch) + ) + end + if isnothing(_upper_minor) + upper_bound = VersionBound(parse(Int, _upper_major)) + elseif isnothing(_upper_patch) + upper_bound = VersionBound( + parse(Int, _upper_major), + parse(Int, _upper_minor) + ) + else + upper_bound = VersionBound( + parse(Int, _upper_major), + parse(Int, _upper_minor), + parse(Int, _upper_patch) + ) + end + return VersionRange(lower_bound, upper_bound) +end + +const version = "v?([0-9]+?)(?:\\.([0-9]+?))?(?:\\.([0-9]+?))?" +const ver_regs = + Pair{Regex, Any}[ + Regex("^([~^]?)?$version\$") => semver_interval, # 0.5 ^0.4 ~0.3.2 + Regex("^((?:≥\\s*)|(?:>=\\s*)|(?:=\\s*)|(?:<\\s*)|(?:=\\s*))v?$version\$") => inequality_interval, # < 0.2 >= 0.5,2 + Regex("^[\\s]*$version[\\s]*?\\s-\\s[\\s]*?$version[\\s]*\$") => hyphen_interval, # 0.7 - 1.3 +] diff --git a/base/version_git.sh b/base/version_git.sh index 76092e9800594..ab15353418465 100644 --- a/base/version_git.sh +++ b/base/version_git.sh @@ -6,7 +6,7 @@ echo "# This file was autogenerated by base/version_git.sh" echo "struct GitVersionInfo" echo " commit::String" -echo " commit_short::String" +echo " commit_short_raw::String" echo " branch::String" echo " build_number::Int" echo " date_string::String" @@ -17,6 +17,27 @@ echo " build_system_commit::String" echo " build_system_commit_short::String" echo "end" echo "" +echo "function Base.getproperty(info::GitVersionInfo, s::Symbol)" +echo " if s === :commit_short" +echo " commit = getfield(info, :commit_short_raw)" +echo " dirty_file = joinpath(Sys.BINDIR, Base.DATAROOTDIR, \"julia\", \"base\", \"version_git_dirty\")" +echo " dirty_str = try" +echo " read(dirty_file, String)" +echo " catch" +echo " \"\"" +echo " end" +echo " if strip(dirty_str) == \"true\"" +echo " return commit * \"*\"" +echo " end" +echo " return commit" +echo " else" +echo " return getfield(info, s)" +echo " end" +echo "end" +echo "" + +# Counter the user's git config to show the signature in logs. +gitnosig="-c log.showSignature=false" cd $1 @@ -32,14 +53,15 @@ origin=$(git config -l 2>/dev/null | grep 'remote\.\w*\.url.*JuliaLang/julia' | if [ -z "$origin" ]; then origin="origin/" fi -git_time=$(git log -1 --pretty=format:%ct) +git_time=$(git $gitnosig log -1 --pretty=format:%ct) #collect the contents commit=$(git rev-parse HEAD) commit_short=$(git rev-parse --short HEAD) if [ -n "$(git status --porcelain)" ]; then - # append dirty mark '*' if the repository has uncommitted changes - commit_short="$commit_short"* + dirty="true" +else + dirty="false" fi # Our CI system checks commits out as a detached head, and so we must @@ -70,7 +92,7 @@ case $(uname) in fi ;; MINGW*) - git_time=$(git log -1 --pretty=format:%ci) + git_time=$(git $gitnosig log -1 --pretty=format:%ci) date_string="$(date --date="$git_time" -u '+%Y-%m-%d %H:%M %Z')" ;; *) @@ -83,7 +105,7 @@ else tagged_commit="false" fi fork_master_distance=$(git rev-list HEAD ^"$(echo $origin)master" | wc -l | sed -e 's/[^[:digit:]]//g') -fork_master_timestamp=$(git show -s $(git merge-base HEAD $(echo $origin)master) --format=format:"%ct") +fork_master_timestamp=$(git $gitnosig show -s $(git merge-base HEAD $(echo $origin)master) --format=format:"%ct") # Check for errors and emit default value for missing numbers. if [ -z "$build_number" ]; then @@ -117,3 +139,7 @@ echo " $fork_master_timestamp.0," echo " \"$build_system_commit\"," echo " \"$build_system_commit_short\"," echo ")" + +# Write dirty status to a separate file to avoid triggering rebuilds +# when only the dirty status changes +echo "$dirty" > version_git_dirty diff --git a/base/views.jl b/base/views.jl index 6898abdda1471..cf6f95b33de97 100644 --- a/base/views.jl +++ b/base/views.jl @@ -14,64 +14,149 @@ should transform to A[B[lastindex(B)]] """ -replace_ref_begin_end!(ex) = replace_ref_begin_end_!(ex, nothing)[1] -# replace_ref_begin_end_!(ex,withex) returns (new ex, whether withex was used) -function replace_ref_begin_end_!(ex, withex) +replace_ref_begin_end!(__module__::Module, @nospecialize ex) = replace_ref_begin_end_!(__module__, ex, nothing, false, 0)[1] +# replace_ref_begin_end_!(...) returns (new ex, whether withex was used) +function replace_ref_begin_end_!(__module__::Module, ex, withex, in_quote_context::Bool, escs::Int) + @nospecialize used_withex = false - if isa(ex,Symbol) - if ex === :begin - withex === nothing && error("Invalid use of begin") - return withex[1], true - elseif ex === :end - withex === nothing && error("Invalid use of end") - return withex[2], true + function escapes(@nospecialize(ex), escs::Int) + for _ = 1:escs + ex = esc(ex) end - elseif isa(ex,Expr) - if ex.head === :ref - ex.args[1], used_withex = replace_ref_begin_end_!(ex.args[1], withex) - S = isa(ex.args[1],Symbol) ? ex.args[1]::Symbol : gensym(:S) # temp var to cache ex.args[1] if needed - used_S = false # whether we actually need S - # new :ref, so redefine withex - nargs = length(ex.args)-1 - if nargs == 0 - return ex, used_withex - elseif nargs == 1 - # replace with lastindex(S) - ex.args[2], used_S = replace_ref_begin_end_!(ex.args[2], (:($firstindex($S)),:($lastindex($S)))) - else - n = 1 - J = lastindex(ex.args) - for j = 2:J - exj, used = replace_ref_begin_end_!(ex.args[j], (:($firstindex($S,$n)),:($lastindex($S,$n)))) - used_S |= used - ex.args[j] = exj - if isa(exj,Expr) && exj.head === :... - # splatted object - exjs = exj.args[1] - n = :($n + length($exjs)) - elseif isa(n, Expr) - # previous expression splatted - n = :($n + 1) - else - # an integer - n += 1 + return ex + end + function handle_refexpr!(__module__::Module, ref_ex::Expr, main_ex::Expr, withex, in_quote_context, escs::Int) + @assert !in_quote_context "handle_refexpr! should not be called in quote context" + local used_withex + ref_ex.args[1], used_withex = replace_ref_begin_end_!(__module__, ref_ex.args[1], withex, in_quote_context, escs) + S = gensym(:S) # temp var to cache ex.args[1] if needed. if S is a global or expression, then it has side effects to use + assignments = [] + used_S = false # whether we actually need S + # new :ref, so redefine withex + nargs = length(ref_ex.args) - 1 + if nargs == 0 + return main_ex, used_withex + elseif nargs == 1 + # replace with lastindex(S) + ref_ex.args[2], used_S = replace_ref_begin_end_!(__module__, ref_ex.args[2], (:($firstindex($S)),:($lastindex($S))), in_quote_context, escs) + else + ni = 1 + nx = 0 + J = nargs + 1 + need_temps = false # whether any arg needs temporaries + + # First pass: determine if any argument will needs temporaries + for j = 2:J + exj = ref_ex.args[j] + if isexpr(exj, :...) + need_temps = true + break + end + end + + # Second pass: if any need temps, create temps for all args + temp_vars = Tuple{Int,Symbol}[] + for j = 2:J + n = nx === 0 ? ni : :($nx + $ni) + exj, used_arg = replace_ref_begin_end_!(__module__, ref_ex.args[j], (:($firstindex($S,$n)),:($lastindex($S,$n))), in_quote_context, escs) + used_S |= used_arg + ref_ex.args[j] = exj + ni += 1 + if need_temps + isva = isexpr(exj, :...) # implied need_temps + if isva + exj = exj.args[1] + end + if isa_ast_node(exj) # create temp to preserve evaluation order and count in case `used` gets set later + exj = gensym(:arg) + push!(temp_vars, (j, exj)) + end + if isva + ni -= 1 + nx = nx === 0 ? :(length($exj)) : :($nx + length($exj)) + end + end + end + + # Third pass: if `used`, need to actually make those temp assignments now + if used_S + for (j, temp_var) in temp_vars + exj = ref_ex.args[j] + isva = isexpr(exj, :...) # implied need_temps + if isva + exj = exj.args[1] end + push!(assignments, :(local $temp_var = $exj)) + ref_ex.args[j] = isva ? Expr(:..., temp_var) : temp_var end end - if used_S && S !== ex.args[1] - S0 = ex.args[1] - ex.args[1] = S - ex = Expr(:let, :($S = $S0), ex) + end + + if used_S + S0 = ref_ex.args[1] + S = escapes(S, escs) + ref_ex.args[1] = S + main_ex = :(local $S = $S0; $(assignments...); $main_ex) + end + return main_ex, used_withex + end + if ex isa Expr && ex.head === :macrocall + # Blithely modifying the arguments to another macro is unwise, so call + # macroexpand first on it. + # Unfortunately, macroexpand itself corrupts the scope of variables in + # the result by calling macroexpand.scm before returning which cannot be + # avoided since `jl_expand_macros` is private and somewhat difficult to + # reimplement correctly. + ex = macroexpand(__module__, ex) + end + if isa(ex,Symbol) + if !in_quote_context + if ex === :begin + withex === nothing && error("Invalid use of begin outside []") + return escapes((withex::NTuple{2,Expr})[1], escs), true + elseif ex === :end + withex === nothing && error("Invalid use of end outside []") + return escapes((withex::NTuple{2,Expr})[2], escs), true end - else - # recursive search + end + elseif isa(ex,Expr) + if !in_quote_context && ex.head === :ref # n.b. macroexpand.scm design is incapable of tracking :begin and :end scope, so emulate that here too and ignore escs + return handle_refexpr!(__module__, ex, ex, withex, in_quote_context, escs) + elseif ex.head === :$ + # no longer an executable expression (handle all equivalent forms of :inert, :quote, and QuoteNode the same way) + in_quote_context = false + elseif ex.head === :quote + # executable again + in_quote_context = true + elseif ex.head === :var"hygienic-scope" + # no longer our expression + escs += 1 + elseif ex.head === :escape + # our expression again once zero + escs == 0 && return ex, used_withex + escs -= 1 + elseif ex.head === :meta || ex.head === :inert + return ex, used_withex + elseif !in_quote_context && last(string(ex.head)) == '=' && Meta.isexpr(ex.args[1], :ref) for i = eachindex(ex.args) - ex.args[i], used = replace_ref_begin_end_!(ex.args[i], withex) + if i == 1 + # we'll deal with the ref expression later + continue + end + ex.args[i], used = replace_ref_begin_end_!(__module__, ex.args[i], withex, in_quote_context, escs) used_withex |= used end + ex, used = handle_refexpr!(__module__, ex.args[1]::Expr, ex, withex, in_quote_context, escs) + used_withex |= used + return ex, used_withex + end + # recursive search + for i = eachindex(ex.args) + ex.args[i], used = replace_ref_begin_end_!(__module__, ex.args[i], withex, in_quote_context, escs) + used_withex |= used end end - ex, used_withex + return ex, used_withex end """ @@ -125,17 +210,19 @@ julia> A macro view(ex) Meta.isexpr(ex, :ref) || throw(ArgumentError( "Invalid use of @view macro: argument must be a reference expression A[...].")) - ex = replace_ref_begin_end!(ex) + ex = replace_ref_begin_end!(__module__, ex) # NOTE We embed `view` as a function object itself directly into the AST. # By doing this, we prevent the creation of function definitions like # `view(A, idx) = xxx` in cases such as `@view(A[idx]) = xxx.` if Meta.isexpr(ex, :ref) ex = Expr(:call, view, ex.args...) - elseif Meta.isexpr(ex, :let) && (arg2 = ex.args[2]; Meta.isexpr(arg2, :ref)) + elseif Meta.isexpr(ex, :block) + arg2 = ex.args[end] + Meta.isexpr(arg2, :ref) || error("unsupported replace_ref_begin_end result") # ex replaced by let ...; foo[...]; end - ex.args[2] = Expr(:call, view, arg2.args...) + ex.args[end] = Expr(:call, view, arg2.args...) else - error("invalid expression") + error("unsupported replace_ref_begin_end result") end return esc(ex) end @@ -176,10 +263,7 @@ function _views(ex::Expr) # temp vars to avoid recomputing a and i, # which will be assigned in a let block: - a = gensym(:a) - i = let lhs=lhs # #15276 - [gensym(:i) for k = 1:length(lhs.args)-1] - end + i = Symbol[Symbol(:i, k) for k = 1:length(lhs.args)-1] # for splatted indices like a[i, j...], we need to # splat the corresponding temp var. @@ -194,14 +278,15 @@ function _views(ex::Expr) end end - Expr(:let, - Expr(:block, - :($a = $(_views(lhs.args[1]))), - Any[:($(i[k]) = $(_views(lhs.args[k+1]))) for k=1:length(i)]...), - Expr(first(h) == '.' ? :(.=) : :(=), :($a[$(I...)]), - Expr(:call, Symbol(h[1:end-1]), - :($maybeview($a, $(I...))), - mapany(_views, ex.args[2:end])...))) + Expr(:var"hygienic-scope", # assign a and i to the macro's scope + Expr(:let, + Expr(:block, + :(a = $(esc(_views(lhs.args[1])))), + Any[:($(i[k]) = $(esc(_views(lhs.args[k+1])))) for k=1:length(i)]...), + Expr(first(h) == '.' ? :(.=) : :(=), :(a[$(I...)]), + Expr(:call, esc(Symbol(h[1:end-1])), + :($maybeview(a, $(I...))), + mapany(e -> esc(_views(e)), ex.args[2:end])...))), Base) else exprarray(ex.head, mapany(_views, ex.args)) end @@ -245,5 +330,5 @@ julia> A ``` """ macro views(x) - esc(_views(replace_ref_begin_end!(x))) + esc(_views(replace_ref_begin_end!(__module__, x))) end diff --git a/base/weakdict.jl b/base/weakdict.jl new file mode 100644 index 0000000000000..67e7e5d54c99d --- /dev/null +++ b/base/weakdict.jl @@ -0,0 +1,444 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# weak key and weak value dictionaries + +mutable struct WeakKeyDictFinalizer{T} + const d::T +end +(d::WeakKeyDictFinalizer)(k) = d.d.dirty = true + + +""" + WeakKeyDict([itr]) + +`WeakKeyDict()` constructs a hash table where the keys are weak +references to objects which may be garbage collected even when +referenced in a hash table. + +See [`Dict`](@ref) for further help. Note, unlike [`Dict`](@ref), +`WeakKeyDict` does not convert keys on insertion, as this would imply the key +object was unreferenced anywhere before insertion. + +See also [`WeakRef`](@ref). +""" +mutable struct WeakKeyDict{K,V} <: AbstractDict{K,V} + const ht::Dict{WeakRef,V} + const lock::ReentrantLock + dirty::Bool + finalizer::WeakKeyDictFinalizer + + # Constructors mirror Dict's + function WeakKeyDict{K,V}() where {K, V} + t = new{K,V}(Dict{WeakRef,V}(), ReentrantLock(), false) + t.finalizer = WeakKeyDictFinalizer(t) + return t + end +end +function WeakKeyDict{K,V}(kv) where V where K + h = WeakKeyDict{K,V}() + for (k,v) in kv + h[k] = v + end + return h +end +WeakKeyDict{K,V}(p::Pair) where V where K = setindex!(WeakKeyDict{K,V}(), p.second, p.first) +function WeakKeyDict{K,V}(ps::Pair...) where V where K + h = WeakKeyDict{K,V}() + sizehint!(h, length(ps)) + for p in ps + h[p.first] = p.second + end + return h +end +WeakKeyDict() = WeakKeyDict{Any,Any}() + +WeakKeyDict(kv::Tuple{}) = WeakKeyDict() +copy(d::WeakKeyDict) = WeakKeyDict(d) + +WeakKeyDict(ps::Pair{K,V}...) where {K,V} = WeakKeyDict{K,V}(ps) +WeakKeyDict(ps::Pair{K}...) where {K} = WeakKeyDict{K,Any}(ps) +WeakKeyDict(ps::(Pair{K,V} where K)...) where {V} = WeakKeyDict{Any,V}(ps) +WeakKeyDict(ps::Pair...) = WeakKeyDict{Any,Any}(ps) + +WeakKeyDict(kv) = Base.dict_with_eltype((K, V) -> WeakKeyDict{K, V}, kv, eltype(kv)) + +function _cleanup_locked(h::WeakKeyDict) + if h.dirty + h.dirty = false + idx = skip_deleted_floor!(h.ht) + while idx != 0 + if h.ht.keys[idx].value === nothing + _delete!(h.ht, idx) + end + idx = skip_deleted(h.ht, idx + 1) + end + end + return h +end + +sizehint!(d::WeakKeyDict, newsz::Integer; shrink::Bool = true) = @lock d sizehint!(d.ht, newsz; shrink = shrink) +empty(d::WeakKeyDict, ::Type{K}, ::Type{V}) where {K, V} = WeakKeyDict{K, V}() + +IteratorSize(::Type{<:WeakKeyDict}) = SizeUnknown() + +islocked(wkh::WeakKeyDict) = islocked(wkh.lock) +lock(wkh::WeakKeyDict) = lock(wkh.lock) +unlock(wkh::WeakKeyDict) = unlock(wkh.lock) +lock(f, wkh::WeakKeyDict) = lock(f, wkh.lock) +trylock(f, wkh::WeakKeyDict) = trylock(f, wkh.lock) + +function setindex!(wkh::WeakKeyDict{K}, v, key) where K + !isa(key, K) && throw(ArgumentError("$(limitrepr(key)) is not a valid key for type $K")) + # 'nothing' is not valid both because 'finalizer' will reject it, + # and because we therefore use it as a sentinel value + key === nothing && throw(ArgumentError("`nothing` is not a valid WeakKeyDict key")) + lock(wkh) do + _cleanup_locked(wkh) + k = getkey(wkh.ht, key, nothing) + if k === nothing + finalizer(wkh.finalizer, key) + k = WeakRef(key) + else + k.value = key + end + wkh.ht[k] = v + end + return wkh +end +function get!(wkh::WeakKeyDict{K}, key, default) where {K} + v = lock(wkh) do + if key !== nothing && haskey(wkh.ht, key) + wkh.ht[key] + else + wkh[key] = default + end + end + return v +end +function get!(default::Callable, wkh::WeakKeyDict{K}, key) where {K} + v = lock(wkh) do + if key !== nothing && haskey(wkh.ht, key) + wkh.ht[key] + else + wkh[key] = default() + end + end + return v +end + +function getkey(wkh::WeakKeyDict{K}, kk, default) where K + k = lock(wkh) do + local k = getkey(wkh.ht, kk, nothing) + k === nothing && return nothing + return k.value + end + return k === nothing ? default : k::K +end + +map!(f, iter::ValueIterator{<:WeakKeyDict})= map!(f, values(iter.dict.ht)) + +function get(wkh::WeakKeyDict{K}, key, default) where {K} + key === nothing && throw(KeyError(nothing)) + lock(wkh) do + return get(wkh.ht, key, default) + end +end +function get(default::Callable, wkh::WeakKeyDict{K}, key) where {K} + key === nothing && throw(KeyError(nothing)) + lock(wkh) do + return get(default, wkh.ht, key) + end +end +function pop!(wkh::WeakKeyDict{K}, key) where {K} + key === nothing && throw(KeyError(nothing)) + lock(wkh) do + return pop!(wkh.ht, key) + end +end +function pop!(wkh::WeakKeyDict{K}, key, default) where {K} + key === nothing && return default + lock(wkh) do + return pop!(wkh.ht, key, default) + end +end +function delete!(wkh::WeakKeyDict, key) + key === nothing && return wkh + lock(wkh) do + delete!(wkh.ht, key) + end + return wkh +end +function empty!(wkh::WeakKeyDict) + lock(wkh) do + empty!(wkh.ht) + end + return wkh +end +function haskey(wkh::WeakKeyDict{K}, key) where {K} + key === nothing && return false + lock(wkh) do + return haskey(wkh.ht, key) + end +end +function getindex(wkh::WeakKeyDict{K}, key) where {K} + key === nothing && throw(KeyError(nothing)) + lock(wkh) do + return getindex(wkh.ht, key) + end +end +isempty(wkh::WeakKeyDict) = length(wkh) == 0 +function length(t::WeakKeyDict) + lock(t) do + _cleanup_locked(t) + return length(t.ht) + end +end + +function iterate(t::WeakKeyDict{K,V}, state...) where {K, V} + @lock t begin + while true + y = iterate(t.ht, state...) + y === nothing && return nothing + wkv, state = y + k = wkv[1].value + GC.safepoint() # ensure `k` is now gc-rooted + k === nothing && continue # indicates `k` is scheduled for deletion + kv = Pair{K,V}(k::K, wkv[2]) + return (kv, state) + end + end +end + +@propagate_inbounds Iterators.only(d::WeakKeyDict) = Iterators._only(d, first) + +filter!(f, d::WeakKeyDict) = filter_in_one_pass!(f, d) + + +# weak identity dictionaries + +mutable struct WeakValueIdDictFinalizer{T} + const d::T +end +(d::WeakValueIdDictFinalizer)(v) = d.d.dirty = true + +""" + WeakValueIdDict + +`WeakValueIdDict` is an identity-keyed dictionary like [`IdDict`](@ref). +However, unlike `IdDict`, the dictionary holds its values weakly. + +See also [`WeakKeyDict`](@ref), which differs in two ways: +1. `WeakKeyDict` holds its *keys* weakly, while `WeakValueIdDict` holds its *values* weakly. +2. `WeakKeyDict` is equality-keyed, while `WeakValueIdDict` is identity-keyed. + +See also [`WeakRef`](@ref). +""" +mutable struct WeakValueIdDict{K,V} <: AbstractDict{K,V} + const ht::IdDict{K,WeakRef} + const lock::ReentrantLock + dirty::Bool + # concretely typed (self-referentially) so that registering the finalizer + # remains statically resolvable, e.g. under `--trim` + finalizer::WeakValueIdDictFinalizer{WeakValueIdDict{K,V}} + + function WeakValueIdDict{K,V}() where {K, V} + t = new{K,V}(IdDict{K,WeakRef}(), ReentrantLock(), false) + t.finalizer = WeakValueIdDictFinalizer(t) + return t + end +end +function WeakValueIdDict{K,V}(kv) where V where K + h = WeakValueIdDict{K,V}() + for (k,v) in kv + h[k] = v + end + return h +end +WeakValueIdDict{K,V}(p::Pair) where V where K = setindex!(WeakValueIdDict{K,V}(), p.second, p.first) +function WeakValueIdDict{K,V}(ps::Pair...) where V where K + h = WeakValueIdDict{K,V}() + for p in ps + h[p.first] = p.second + end + return h +end +WeakValueIdDict() = WeakValueIdDict{Any,Any}() + +WeakValueIdDict(kv::Tuple{}) = WeakValueIdDict() +copy(d::WeakValueIdDict) = WeakValueIdDict(d) + +WeakValueIdDict(ps::Pair{K,V}...) where {K,V} = WeakValueIdDict{K,V}(ps...) +WeakValueIdDict(ps::Pair{K}...) where {K} = WeakValueIdDict{K,Any}(ps...) +WeakValueIdDict(ps::(Pair{K,V} where K)...) where {V} = WeakValueIdDict{Any,V}(ps...) +WeakValueIdDict(ps::Pair...) = WeakValueIdDict{Any,Any}(ps...) + +WeakValueIdDict(kv) = dict_with_eltype((K, V) -> WeakValueIdDict{K, V}, kv, eltype(kv)) + +# an entry is live iff its value is; a dead entry is indistinguishable from an +# absent one (its key stays pinned until the next sweep) +function _sweep_locked(h::WeakValueIdDict{K}) where {K} + h.dirty = false + dead = K[] + for (k, wr) in h.ht + (wr::WeakRef).value === nothing && push!(dead, k) + end + for k in dead + delete!(h.ht, k) + end + return h +end +_cleanup_locked(h::WeakValueIdDict) = h.dirty ? _sweep_locked(h) : h + +empty(d::WeakValueIdDict, ::Type{K}, ::Type{V}) where {K, V} = WeakValueIdDict{K, V}() + +IteratorSize(::Type{<:WeakValueIdDict}) = SizeUnknown() + +islocked(h::WeakValueIdDict) = islocked(h.lock) +lock(h::WeakValueIdDict) = lock(h.lock) +unlock(h::WeakValueIdDict) = unlock(h.lock) +lock(f, h::WeakValueIdDict) = lock(f, h.lock) +trylock(f, h::WeakValueIdDict) = trylock(f, h.lock) + +function setindex!(h::WeakValueIdDict{K,V}, v, key) where {K,V} + !isa(key, K) && throw(ArgumentError("$(limitrepr(key)) is not a valid key for type $K")) + # 'nothing' is not valid on either side: 'finalizer' would reject it as a + # value (and it is the dead-reference sentinel), and a 'nothing' key is + # reserved so that lookup errors stay consistent with WeakKeyDict + key === nothing && throw(ArgumentError("`nothing` is not a valid WeakValueIdDict key")) + v === nothing && throw(ArgumentError("`nothing` is not a valid WeakValueIdDict value")) + lock(h) do + _cleanup_locked(h) + finalizer(h.finalizer, v) + h.ht[key] = WeakRef(v) + end + return h +end + +# retrieve the live value for `key`, or `nothing` +function _getvalue(h::WeakValueIdDict, key) + wr = get(h.ht, key, nothing) + wr === nothing && return nothing + return (wr::WeakRef).value +end + +function get(h::WeakValueIdDict{K,V}, key, default) where {K,V} + key === nothing && throw(KeyError(nothing)) + lock(h) do + v = _getvalue(h, key) + return v === nothing ? default : v::V + end +end +function get(default::Callable, h::WeakValueIdDict{K,V}, key) where {K,V} + key === nothing && throw(KeyError(nothing)) + lock(h) do + v = _getvalue(h, key) + return v === nothing ? default() : v::V + end +end +function get!(h::WeakValueIdDict{K,V}, key, default) where {K,V} + lock(h) do + v = _getvalue(h, key) + v === nothing || return v::V + h[key] = default + return default::V + end +end +function get!(default::Callable, h::WeakValueIdDict{K,V}, key) where {K,V} + lock(h) do + v = _getvalue(h, key) + v === nothing || return v::V + v = default() + h[key] = v + return v::V + end +end + +function getkey(h::WeakValueIdDict{K}, kk, default) where {K} + kk === nothing && return default + lock(h) do + v = _getvalue(h, kk) + return v === nothing ? default : kk::K + end +end + +function getindex(h::WeakValueIdDict{K,V}, key) where {K,V} + key === nothing && throw(KeyError(key)) + lock(h) do + v = _getvalue(h, key) + v === nothing && throw(KeyError(key)) + return v::V + end +end + +function haskey(h::WeakValueIdDict, key) + key === nothing && return false + lock(h) do + return _getvalue(h, key) !== nothing + end +end + +function delete!(h::WeakValueIdDict, key) + key === nothing && return h + lock(h) do + delete!(h.ht, key) + end + return h +end + +function pop!(h::WeakValueIdDict{K,V}, key) where {K,V} + key === nothing && throw(KeyError(key)) + lock(h) do + v = _getvalue(h, key) + v === nothing && throw(KeyError(key)) + delete!(h.ht, key) + return v::V + end +end +function pop!(h::WeakValueIdDict{K,V}, key, default) where {K,V} + key === nothing && return default + lock(h) do + v = _getvalue(h, key) + delete!(h.ht, key) + return v === nothing ? default : v::V + end +end + +function empty!(h::WeakValueIdDict) + lock(h) do + empty!(h.ht) + h.dirty = false + end + return h +end + +function length(h::WeakValueIdDict) + lock(h) do + n = 0 + for (k, wr) in h.ht + n += ((wr::WeakRef).value !== nothing) + end + return n + end +end +isempty(h::WeakValueIdDict) = length(h) == 0 + +function iterate(h::WeakValueIdDict{K,V}, snapshot::Union{Nothing,Tuple{Vector{Pair{K,V}},Int}}=nothing) where {K,V} + # iterate over a snapshot of the live entries, so that concurrent + # collection of values (or mutation under the lock) cannot corrupt the + # walk; the snapshot's strong references keep its entries alive + if snapshot === nothing + live = Pair{K,V}[] + lock(h) do + for (k, wr) in h.ht + v = (wr::WeakRef).value + v === nothing && continue + push!(live, Pair{K,V}(k, v::V)) + end + end + snapshot = (live, 0) + end + pairs, i = snapshot + i += 1 + i > length(pairs) && return nothing + return pairs[i], (pairs, i) +end diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl deleted file mode 100644 index 1283dc9cbc8cb..0000000000000 --- a/base/weakkeydict.jl +++ /dev/null @@ -1,214 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# weak key dictionaries - -mutable struct WeakKeyDictFinalizer{T} - const d::T -end -(d::WeakKeyDictFinalizer)(k) = d.d.dirty = true - - -""" - WeakKeyDict([itr]) - -`WeakKeyDict()` constructs a hash table where the keys are weak -references to objects which may be garbage collected even when -referenced in a hash table. - -See [`Dict`](@ref) for further help. Note, unlike [`Dict`](@ref), -`WeakKeyDict` does not convert keys on insertion, as this would imply the key -object was unreferenced anywhere before insertion. - -See also [`WeakRef`](@ref). -""" -mutable struct WeakKeyDict{K,V} <: AbstractDict{K,V} - const ht::Dict{WeakRef,V} - const lock::ReentrantLock - dirty::Bool - finalizer::WeakKeyDictFinalizer - - # Constructors mirror Dict's - function WeakKeyDict{K,V}() where {K, V} - t = new{K,V}(Dict{WeakRef,V}(), ReentrantLock(), false) - t.finalizer = WeakKeyDictFinalizer(t) - return t - end -end -function WeakKeyDict{K,V}(kv) where V where K - h = WeakKeyDict{K,V}() - for (k,v) in kv - h[k] = v - end - return h -end -WeakKeyDict{K,V}(p::Pair) where V where K = setindex!(WeakKeyDict{K,V}(), p.second, p.first) -function WeakKeyDict{K,V}(ps::Pair...) where V where K - h = WeakKeyDict{K,V}() - sizehint!(h, length(ps)) - for p in ps - h[p.first] = p.second - end - return h -end -WeakKeyDict() = WeakKeyDict{Any,Any}() - -WeakKeyDict(kv::Tuple{}) = WeakKeyDict() -copy(d::WeakKeyDict) = WeakKeyDict(d) - -WeakKeyDict(ps::Pair{K,V}...) where {K,V} = WeakKeyDict{K,V}(ps) -WeakKeyDict(ps::Pair{K}...) where {K} = WeakKeyDict{K,Any}(ps) -WeakKeyDict(ps::(Pair{K,V} where K)...) where {V} = WeakKeyDict{Any,V}(ps) -WeakKeyDict(ps::Pair...) = WeakKeyDict{Any,Any}(ps) - -WeakKeyDict(kv) = Base.dict_with_eltype((K, V) -> WeakKeyDict{K, V}, kv, eltype(kv)) - -function _cleanup_locked(h::WeakKeyDict) - if h.dirty - h.dirty = false - idx = skip_deleted_floor!(h.ht) - while idx != 0 - if h.ht.keys[idx].value === nothing - _delete!(h.ht, idx) - end - idx = skip_deleted(h.ht, idx + 1) - end - end - return h -end - -sizehint!(d::WeakKeyDict, newsz::Integer; shrink::Bool = true) = @lock d sizehint!(d.ht, newsz; shrink = shrink) -empty(d::WeakKeyDict, ::Type{K}, ::Type{V}) where {K, V} = WeakKeyDict{K, V}() - -IteratorSize(::Type{<:WeakKeyDict}) = SizeUnknown() - -islocked(wkh::WeakKeyDict) = islocked(wkh.lock) -lock(wkh::WeakKeyDict) = lock(wkh.lock) -unlock(wkh::WeakKeyDict) = unlock(wkh.lock) -lock(f, wkh::WeakKeyDict) = lock(f, wkh.lock) -trylock(f, wkh::WeakKeyDict) = trylock(f, wkh.lock) - -function setindex!(wkh::WeakKeyDict{K}, v, key) where K - !isa(key, K) && throw(ArgumentError("$(limitrepr(key)) is not a valid key for type $K")) - # 'nothing' is not valid both because 'finalizer' will reject it, - # and because we therefore use it as a sentinel value - key === nothing && throw(ArgumentError("`nothing` is not a valid WeakKeyDict key")) - lock(wkh) do - _cleanup_locked(wkh) - k = getkey(wkh.ht, key, nothing) - if k === nothing - finalizer(wkh.finalizer, key) - k = WeakRef(key) - else - k.value = key - end - wkh.ht[k] = v - end - return wkh -end -function get!(wkh::WeakKeyDict{K}, key, default) where {K} - v = lock(wkh) do - if key !== nothing && haskey(wkh.ht, key) - wkh.ht[key] - else - wkh[key] = default - end - end - return v -end -function get!(default::Callable, wkh::WeakKeyDict{K}, key) where {K} - v = lock(wkh) do - if key !== nothing && haskey(wkh.ht, key) - wkh.ht[key] - else - wkh[key] = default() - end - end - return v -end - -function getkey(wkh::WeakKeyDict{K}, kk, default) where K - k = lock(wkh) do - local k = getkey(wkh.ht, kk, nothing) - k === nothing && return nothing - return k.value - end - return k === nothing ? default : k::K -end - -map!(f, iter::ValueIterator{<:WeakKeyDict})= map!(f, values(iter.dict.ht)) - -function get(wkh::WeakKeyDict{K}, key, default) where {K} - key === nothing && throw(KeyError(nothing)) - lock(wkh) do - return get(wkh.ht, key, default) - end -end -function get(default::Callable, wkh::WeakKeyDict{K}, key) where {K} - key === nothing && throw(KeyError(nothing)) - lock(wkh) do - return get(default, wkh.ht, key) - end -end -function pop!(wkh::WeakKeyDict{K}, key) where {K} - key === nothing && throw(KeyError(nothing)) - lock(wkh) do - return pop!(wkh.ht, key) - end -end -function pop!(wkh::WeakKeyDict{K}, key, default) where {K} - key === nothing && return default - lock(wkh) do - return pop!(wkh.ht, key, default) - end -end -function delete!(wkh::WeakKeyDict, key) - key === nothing && return wkh - lock(wkh) do - delete!(wkh.ht, key) - end - return wkh -end -function empty!(wkh::WeakKeyDict) - lock(wkh) do - empty!(wkh.ht) - end - return wkh -end -function haskey(wkh::WeakKeyDict{K}, key) where {K} - key === nothing && return false - lock(wkh) do - return haskey(wkh.ht, key) - end -end -function getindex(wkh::WeakKeyDict{K}, key) where {K} - key === nothing && throw(KeyError(nothing)) - lock(wkh) do - return getindex(wkh.ht, key) - end -end -isempty(wkh::WeakKeyDict) = length(wkh) == 0 -function length(t::WeakKeyDict) - lock(t) do - _cleanup_locked(t) - return length(t.ht) - end -end - -function iterate(t::WeakKeyDict{K,V}, state...) where {K, V} - return lock(t) do - while true - y = iterate(t.ht, state...) - y === nothing && return nothing - wkv, state = y - k = wkv[1].value - GC.safepoint() # ensure `k` is now gc-rooted - k === nothing && continue # indicates `k` is scheduled for deletion - kv = Pair{K,V}(k::K, wkv[2]) - return (kv, state) - end - end -end - -@propagate_inbounds Iterators.only(d::WeakKeyDict) = Iterators._only(d, first) - -filter!(f, d::WeakKeyDict) = filter_in_one_pass!(f, d) diff --git a/cli/.gitignore b/cli/.gitignore index 598b56f342fcd..3f71fc49fbb0f 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -3,3 +3,6 @@ /*.dSYM /*.obj /*.pdb +/*.lto +/*.def +/*.strip diff --git a/cli/Makefile b/cli/Makefile index 8c73d76f5020f..e3394441d520f 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -5,19 +5,21 @@ include $(JULIAHOME)/Make.inc include $(JULIAHOME)/deps/llvm-ver.make -HEADERS := $(addprefix $(SRCDIR)/,jl_exports.h loader.h) $(addprefix $(JULIAHOME)/src/,julia_fasttls.h support/platform.h support/dirpath.h jl_exported_data.inc jl_exported_funcs.inc) +HEADERS := $(addprefix $(SRCDIR)/,jl_exports.h loader.h dl-cache.h) $(addprefix $(JULIAHOME)/src/,julia_fasttls.h jloptions.h support/platform.h support/dirpath.h jl_exported_data.inc jl_exported_funcs.inc) LOADER_CFLAGS = $(JCFLAGS) -I$(BUILDROOT)/src -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) -ffreestanding -LOADER_LDFLAGS = $(JLDFLAGS) -ffreestanding -L$(build_shlibdir) -L$(build_libdir) +LOADER_LIBPATHS = -L$(build_shlibdir) -L$(build_libdir) +LOADER_LDFLAGS = $(JLDFLAGS) -ffreestanding $(LOADER_LIBPATHS) ifeq ($(OS),WINNT) LOADER_CFLAGS += -municode -mconsole -nostdlib -fno-stack-check -fno-stack-protector -mno-stack-arg-probe +LOADER_LIBPATHS := -L$(build_private_libdir) $(LOADER_LIBPATHS) else ifeq ($(OS),Linux) LOADER_CFLAGS += -DGLIBCXX_LEAST_VERSION_SYMBOL=\"$(shell echo "$(CSL_NEXT_GLIBCXX_VERSION)" | cut -d'|' -f1 | sed 's/\\//g')\" endif ifeq ($(OS),WINNT) -LOADER_LDFLAGS += -municode -mconsole -nostdlib -lntdll -lkernel32 -lpsapi +LOADER_LDFLAGS = $(LOADER_LIBPATHS) -lntdll -lkernel32 -lpsapi else ifeq ($(OS),Linux) # textoff and notext are aliases to the same option which suppress the TEXTREL warning for i686 LOADER_LDFLAGS += -Wl,--no-as-needed -ldl -lpthread -rdynamic -lc -Wl,--as-needed -Wl,-z,notext @@ -27,6 +29,21 @@ else ifeq ($(OS),OpenBSD) LOADER_LDFLAGS += -Wl,--no-as-needed -lpthread -rdynamic -lc -Wl,--as-needed endif +LOADER_EXE_LDFLAGS := $(LOADER_LDFLAGS) + +ifeq ($(OS),Darwin) +LOADER_EXE_LDFLAGS += -Wl,-export_dynamic +# The macOS app bundle (contrib/mac/app) uses a plain copy of this loader at +# Contents/MacOS/julia-terminal as its main executable (Apple's notary requires +# a real Mach-O there, not a symlink into the sealed Resources tree). From that +# location @executable_path/../lib misses the bundled julia tree, so also carry +# an rpath to the bundle's lib dir. In any other layout the path simply doesn't +# exist and dyld ignores it. Baking it in at link time keeps the .app assembly +# a pure repackage of the binary dist, needing no Mach-O tools (it runs on +# Linux CI hosts). +LOADER_EXE_LDFLAGS += -Wl,-rpath,'@executable_path/../Resources/julia/lib' +endif + # Build list of dependent libraries that must be opened SHIPFLAGS += -DDEP_LIBS=$(call shell_escape,$(call c_escape,$(LOADER_BUILD_DEP_LIBS))) DEBUGFLAGS += -DDEP_LIBS=$(call shell_escape,$(call c_escape,$(LOADER_DEBUG_BUILD_DEP_LIBS))) @@ -39,10 +56,15 @@ $(BUILDDIR)/loader_lib.o: export MSYS2_ARG_CONV_EXCL = -DDEP_LIBS= $(BUILDDIR)/loader_lib.dbg.obj: export MSYS2_ARG_CONV_EXCL = -DDEP_LIBS= endif # MSYS2 +ifeq ($(USE_RT_STATIC_LIBSTDCXX),1) +SHIPFLAGS += -DRT_STATIC_LIBSTDCXX +DEBUGFLAGS += -DRT_STATIC_LIBSTDCXX +endif # USE_RT_STATIC_LIBSTDCXX + EXE_OBJS := $(BUILDDIR)/loader_exe.o EXE_DOBJS := $(BUILDDIR)/loader_exe.dbg.obj -LIB_OBJS := $(BUILDDIR)/loader_lib.o -LIB_DOBJS := $(BUILDDIR)/loader_lib.dbg.obj +LIB_OBJS := $(BUILDDIR)/loader_lib.o $(BUILDDIR)/loader_symbol_probe.o $(BUILDDIR)/loader_library_probe.o +LIB_DOBJS := $(BUILDDIR)/loader_lib.dbg.obj $(BUILDDIR)/loader_symbol_probe.dbg.obj $(BUILDDIR)/loader_library_probe.dbg.obj # If this is an architecture that supports dynamic linking, link in a trampoline definition ifneq (,$(wildcard $(SRCDIR)/trampolines/trampolines_$(ARCH).S)) @@ -66,6 +88,14 @@ $(BUILDDIR)/loader_trampolines.o : $(SRCDIR)/trampolines/trampolines_$(ARCH).S $ @$(call PRINT_CC, $(CC) $(SHIPFLAGS) $(LOADER_CFLAGS) $< -c -o $@) $(BUILDDIR)/loader_trampolines.dbg.obj : $(SRCDIR)/trampolines/trampolines_$(ARCH).S $(HEADERS) $(SRCDIR)/trampolines/common.h @$(call PRINT_CC, $(CC) $(DEBUGFLAGS) $(LOADER_CFLAGS) $< -c -o $@) +$(BUILDDIR)/loader_library_probe.o : $(SRCDIR)/loader_library_probe.c $(HEADERS) $(JULIAHOME)/VERSION + @$(call PRINT_CC, $(CC) -DJL_LIBRARY_EXPORTS $(SHIPFLAGS) $(LOADER_CFLAGS) -c $< -o $@) +$(BUILDDIR)/loader_library_probe.dbg.obj : $(SRCDIR)/loader_library_probe.c $(HEADERS) $(JULIAHOME)/VERSION + @$(call PRINT_CC, $(CC) -DJL_LIBRARY_EXPORTS $(DEBUGFLAGS) $(LOADER_CFLAGS) -c $< -o $@) +$(BUILDDIR)/loader_symbol_probe.o : $(SRCDIR)/loader_symbol_probe.c $(HEADERS) $(JULIAHOME)/VERSION + @$(call PRINT_CC, $(CC) -DJL_LIBRARY_EXPORTS $(SHIPFLAGS) $(LOADER_CFLAGS) -c $< -o $@) +$(BUILDDIR)/loader_symbol_probe.dbg.obj : $(SRCDIR)/loader_symbol_probe.c $(HEADERS) $(JULIAHOME)/VERSION + @$(call PRINT_CC, $(CC) -DJL_LIBRARY_EXPORTS $(DEBUGFLAGS) $(LOADER_CFLAGS) -c $< -o $@) # Debugging target to help us see what kind of code is being generated for our trampolines dump-trampolines: $(SRCDIR)/trampolines/trampolines_$(ARCH).S @@ -75,30 +105,35 @@ DIRS = $(build_bindir) $(build_libdir) $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) ifeq ($(OS),WINNT) -$(BUILDDIR)/julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION +$(BUILDDIR)/julia_res.o $(BUILDDIR)/julia_res_debug.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION JLVER=`cat $(JULIAHOME)/VERSION` && \ JLVERi=`echo $$JLVER | perl -nle \ '/^(\d+)\.?(\d*)\.?(\d*)/ && \ print int $$1,",",int $$2,",",int $$3,",0"'` && \ $(CROSS_COMPILE)windres $< -O coff -o $@ -DJLVER=$$JLVERi -DJLVER_STR=\\\"$$JLVER\\\" EXE_OBJS += $(BUILDDIR)/julia_res.o -EXE_DOBJS += $(BUILDDIR)/julia_res.o +EXE_DOBJS += $(BUILDDIR)/julia_res_debug.o endif # Embed an Info.plist in the julia executable # Create an intermediate target Info.plist for Darwin code signing. ifeq ($(DARWIN_FRAMEWORK),1) -$(BUILDDIR)/Info.plist: $(JULIAHOME)/VERSION - /usr/libexec/PlistBuddy -x -c "Clear dict" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleName string julia" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleIdentifier string $(darwin_codesign_id_julia_ui)" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleInfoDictionaryVersion string 6.0" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleVersion string $(JULIA_COMMIT)" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleShortVersionString string $(JULIA_MAJOR_VERSION).$(JULIA_MINOR_VERSION).$(JULIA_PATCH_VERSION)" $@ -.INTERMEDIATE: $(BUILDDIR)/Info.plist # cleanup this file after we are done using it +.PHONY: Info.plist.phony +Info.plist.phony: + @TMPFILE=$$(mktemp -u $(abspath $(BUILDDIR)/Info.plist.XXXXXX)); \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleName string julia" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleIdentifier string $(darwin_codesign_id_julia_ui)" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleInfoDictionaryVersion string 6.0" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleVersion string $(JULIA_COMMIT)" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleShortVersionString string $(JULIA_MAJOR_VERSION).$(JULIA_MINOR_VERSION).$(JULIA_PATCH_VERSION)" $$TMPFILE; \ + if ! cmp -s $(BUILDDIR)/Info.plist $$TMPFILE; then \ + mv $$TMPFILE $(BUILDDIR)/Info.plist; \ + else \ + rm -f $$TMPFILE; \ + fi JLDFLAGS += -Wl,-sectcreate,__TEXT,__info_plist,Info.plist -$(build_bindir)/julia$(EXE): $(BUILDDIR)/Info.plist -$(build_bindir)/julia-debug$(EXE): $(BUILDDIR)/Info.plist +$(build_bindir)/julia$(EXE): Info.plist.phony +$(build_bindir)/julia-debug$(EXE): Info.plist.phony endif julia-release: $(build_bindir)/julia$(EXE) @@ -111,28 +146,42 @@ VERSIONSCRIPT := -Wl,--version-script=$(BUILDDIR)/julia.expmap endif ifeq ($(OS),WINNT) -# On Windows we need to strip out exported functions from the generated import library. -STRIP_EXPORTED_FUNCS := $(shell $(CPP_STDOUT) -I$(JULIAHOME)/src $(SRCDIR)/list_strip_symbols.h) +# The import library of the loader must not offer the runtime entry points that +# the loader forwards, or libjulia-internal would resolve its own calls through +# them. Build it from the linker's export definition with those left out, rather +# than editing the import library afterwards: not every linker writes one that +# objcopy can rewrite. +# 1: the shared library being linked +define implib_from_def + $(CPP_STDOUT) -P -I$(JULIAHOME)/src $(SRCDIR)/list_strip_symbols.h > $(BUILDDIR)/$(notdir $1).strip && \ + awk 'NR == FNR { for (i = 1; i <= NF; i++) omit[$$i] = 1; next } !($$1 in omit)' $(BUILDDIR)/$(notdir $1).strip $(BUILDDIR)/$(notdir $1).def > $(BUILDDIR)/$(notdir $1).stripped.def && \ + $(DLLTOOL) -m $(DLLTOOL_MACHINE) -D $(notdir $1) -d $(BUILDDIR)/$(notdir $1).stripped.def -l $(build_libdir)/$(notdir $1).a +endef +DEF_FLAGS = --output-def $(BUILDDIR)/$(notdir $1).def endif $(build_shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT): $(LIB_OBJS) $(SRCDIR)/list_strip_symbols.h $(BUILDDIR)/julia.expmap | $(build_shlibdir) $(build_libdir) - @$(call PRINT_LINK, $(CC) $(call IMPLIB_FLAGS,$@.tmp) $(LOADER_CFLAGS) -shared $(SHIPFLAGS) $(LIB_OBJS) $(RPATH_LIB) -o $@ \ - $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(VERSIONSCRIPT) $(call SONAME_FLAGS,libjulia.$(JL_MAJOR_SHLIB_EXT))) + @$(call PRINT_LINK, $(call link-dll,$@, \ + $(call DEF_FLAGS,$@) $(JLDFLAGS), \ + $(LIB_OBJS) $(LOADER_LDFLAGS), \ + $(CC) $(LOADER_CFLAGS) -shared $(SHIPFLAGS) $(RPATH_LIB), \ + $(LIB_OBJS) $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(VERSIONSCRIPT) $(call SONAME_FLAGS,libjulia.$(JL_MAJOR_SHLIB_EXT)) $(call LTO_OBJECT_PATH,$(BUILDDIR)/$(notdir $@)))) @$(INSTALL_NAME_CMD)libjulia.$(JL_MAJOR_SHLIB_EXT) $@ @$(DSYMUTIL) $@ ifeq ($(OS), WINNT) - @# Note that if the objcopy command starts getting too long, we can use `@file` to read - @# command-line options from `file` instead. - @$(call PRINT_ANALYZE, $(OBJCOPY) $(build_libdir)/$(notdir $@).tmp.a $(STRIP_EXPORTED_FUNCS) $(build_libdir)/$(notdir $@).a && rm $(build_libdir)/$(notdir $@).tmp.a) + @$(call PRINT_ANALYZE, $(call implib_from_def,$@)) endif $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(LIB_DOBJS) $(SRCDIR)/list_strip_symbols.h $(BUILDDIR)/julia.expmap | $(build_shlibdir) $(build_libdir) - @$(call PRINT_LINK, $(CC) $(call IMPLIB_FLAGS,$@.tmp) $(LOADER_CFLAGS) -shared $(DEBUGFLAGS) $(LIB_DOBJS) $(RPATH_LIB) -o $@ \ - $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(VERSIONSCRIPT) $(call SONAME_FLAGS,libjulia-debug.$(JL_MAJOR_SHLIB_EXT))) + @$(call PRINT_LINK, $(call link-dll,$@, \ + $(call DEF_FLAGS,$@) $(JLDFLAGS), \ + $(LIB_DOBJS) $(LOADER_LDFLAGS), \ + $(CC) $(LOADER_CFLAGS) -shared $(DEBUGFLAGS) $(RPATH_LIB), \ + $(LIB_DOBJS) $(JLIBLDFLAGS) $(LOADER_LDFLAGS) $(VERSIONSCRIPT) $(call SONAME_FLAGS,libjulia-debug.$(JL_MAJOR_SHLIB_EXT)) $(call LTO_OBJECT_PATH,$(BUILDDIR)/$(notdir $@)))) @$(INSTALL_NAME_CMD)libjulia-debug.$(JL_MAJOR_SHLIB_EXT) $@ @$(DSYMUTIL) $@ ifeq ($(OS), WINNT) - @$(call PRINT_ANALYZE, $(OBJCOPY) $(build_libdir)/$(notdir $@).tmp.a $(STRIP_EXPORTED_FUNCS) $(build_libdir)/$(notdir $@).a && rm $(build_libdir)/$(notdir $@).tmp.a) + @$(call PRINT_ANALYZE, $(call implib_from_def,$@)) endif ifneq ($(OS), WINNT) @@ -145,17 +194,30 @@ $(build_shlibdir)/libjulia.$(SHLIB_EXT) $(build_shlibdir)/libjulia-debug.$(SHLIB endif $(build_bindir)/julia$(EXE): $(EXE_OBJS) $(build_shlibdir)/libjulia.$(SHLIB_EXT) | $(build_bindir) - @$(call PRINT_LINK, $(CC) $(LOADER_CFLAGS) $(SHIPFLAGS) $(EXE_OBJS) -o $@ $(LOADER_LDFLAGS) $(RPATH) -ljulia) + @$(call PRINT_LINK, $(call link-exe,$@, \ + $(JLDFLAGS), \ + $(EXE_OBJS) $(LOADER_EXE_LDFLAGS) -ljulia, \ + $(CC) $(LOADER_CFLAGS) $(SHIPFLAGS), \ + $(EXE_OBJS) $(LOADER_EXE_LDFLAGS) $(RPATH) -ljulia)) $(build_bindir)/julia-debug$(EXE): $(EXE_DOBJS) $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT) | $(build_bindir) - @$(call PRINT_LINK, $(CC) $(LOADER_CFLAGS) $(DEBUGFLAGS) $(EXE_DOBJS) -o $@ $(LOADER_LDFLAGS) $(RPATH) -ljulia-debug) + @$(call PRINT_LINK, $(call link-exe,$@, \ + $(JLDFLAGS), \ + $(EXE_DOBJS) $(LOADER_EXE_LDFLAGS) -ljulia-debug, \ + $(CC) $(LOADER_CFLAGS) $(DEBUGFLAGS), \ + $(EXE_DOBJS) $(LOADER_EXE_LDFLAGS) $(RPATH) -ljulia-debug)) $(BUILDDIR)/julia.expmap: $(SRCDIR)/julia.expmap.in $(JULIAHOME)/VERSION - sed <'$<' >'$@' -e 's/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/' + @TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + sed <'$<' >$$TMPFILE -e 's/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/'; \ + mv $$TMPFILE $@ clean: | $(CLEAN_TARGETS) + rm -rf $(BUILDDIR)/*.lto rm -f $(BUILDDIR)/*.o $(BUILDDIR)/*.dbg.obj rm -f $(build_bindir)/julia* rm -f $(BUILDDIR)/julia.expmap + rm -f $(BUILDDIR)/*.def $(BUILDDIR)/*.strip + rm -f $(BUILDDIR)/Info.plist* .PHONY: clean release debug julia-release julia-debug diff --git a/cli/dl-cache.h b/cli/dl-cache.h new file mode 100644 index 0000000000000..eddaad088e979 --- /dev/null +++ b/cli/dl-cache.h @@ -0,0 +1,184 @@ +/* Support for reading /etc/ld.so.cache files written by Linux ldconfig. + Copyright (C) 1999-2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#define FLAG_ANY -1 +#define FLAG_TYPE_MASK 0x00ff +#define FLAG_LIBC4 0x0000 +#define FLAG_ELF 0x0001 +#define FLAG_ELF_LIBC5 0x0002 +#define FLAG_ELF_LIBC6 0x0003 +#define FLAG_REQUIRED_MASK 0xff00 +#define FLAG_SPARC_LIB64 0x0100 +#define FLAG_IA64_LIB64 0x0200 +#define FLAG_X8664_LIB64 0x0300 +#define FLAG_S390_LIB64 0x0400 +#define FLAG_POWERPC_LIB64 0x0500 +#define FLAG_MIPS64_LIBN32 0x0600 +#define FLAG_MIPS64_LIBN64 0x0700 +#define FLAG_X8664_LIBX32 0x0800 +#define FLAG_ARM_LIBHF 0x0900 +#define FLAG_AARCH64_LIB64 0x0a00 +#define FLAG_ARM_LIBSF 0x0b00 +#define FLAG_MIPS_LIB32_NAN2008 0x0c00 +#define FLAG_MIPS64_LIBN32_NAN2008 0x0d00 +#define FLAG_MIPS64_LIBN64_NAN2008 0x0e00 +#define FLAG_RISCV_FLOAT_ABI_SOFT 0x0f00 +#define FLAG_RISCV_FLOAT_ABI_DOUBLE 0x1000 + +#if defined(_CPU_X86_64_) + +#define _DL_CACHE_DEFAULT_ID 0x303 +#define _dl_cache_check_flags(flags) ((flags) == _DL_CACHE_DEFAULT_ID) + +#elif defined(_CPU_AARCH64_) + +#ifdef __LP64__ +# define _DL_CACHE_DEFAULT_ID (FLAG_AARCH64_LIB64 | FLAG_ELF_LIBC6) +#else +# define _DL_CACHE_DEFAULT_ID (FLAG_AARCH64_LIB32 | FLAG_ELF_LIBC6) +#endif + +#define _dl_cache_check_flags(flags) ((flags) == _DL_CACHE_DEFAULT_ID) + +#elif defined(_CPU_RISCV64_) + +/* For now we only support the natural XLEN ABI length on all targets, so the + only bits that need to go into ld.so.cache are the flags for ABI length. */ +#if defined __riscv_float_abi_double +# define _DL_CACHE_DEFAULT_ID (FLAG_RISCV_FLOAT_ABI_DOUBLE | FLAG_ELF_LIBC6) +#else +# define _DL_CACHE_DEFAULT_ID (FLAG_RISCV_FLOAT_ABI_SOFT | FLAG_ELF_LIBC6) +#endif + +#define _dl_cache_check_flags(flags) ((flags) == _DL_CACHE_DEFAULT_ID) + +#elif defined(_CPU_ARM_) + +/* In order to support the transition from unmarked objects + to marked objects we must treat unmarked objects as + compatible with either FLAG_ARM_LIBHF or FLAG_ARM_LIBSF. */ +#ifdef __ARM_PCS_VFP +# define _dl_cache_check_flags(flags) \ + ((flags) == (FLAG_ARM_LIBHF | FLAG_ELF_LIBC6) \ + || (flags) == FLAG_ELF_LIBC6) +#else +# define _dl_cache_check_flags(flags) \ + ((flags) == (FLAG_ARM_LIBSF | FLAG_ELF_LIBC6) \ + || (flags) == FLAG_ELF_LIBC6) +#endif + +#elif defined(_CPU_X86_) + +/* Defined as (FLAG_ELF_LIBC6 | FLAG_X8664_LIBX32). */ +#undef _DL_CACHE_DEFAULT_ID +#define _DL_CACHE_DEFAULT_ID 0x803 + +#elif defined(_CPU_PPC64_) + +#define _DL_CACHE_DEFAULT_ID 0x503 + +#define _dl_cache_check_flags(flags) \ + ((flags) == _DL_CACHE_DEFAULT_ID) + +#else + +#error "Missing CPU arch-specific definitions in dl-cache.h" + +#endif + +#ifndef _DL_CACHE_DEFAULT_ID +# define _DL_CACHE_DEFAULT_ID 3 +#endif + +#ifndef _dl_cache_check_flags +# define _dl_cache_check_flags(flags) \ + ((flags) == 1 || (flags) == _DL_CACHE_DEFAULT_ID) +#endif + +#ifndef LD_SO_CACHE +# define LD_SO_CACHE SYSCONFDIR "/ld.so.cache" +#endif + +#define CACHEMAGIC "ld.so-1.7.0" + +/* libc5 and glibc 2.0/2.1 use the same format. For glibc 2.2 another + format has been added in a compatible way: + The beginning of the string table is used for the new table: + old_magic + nlibs + libs[0] + ... + libs[nlibs-1] + pad, new magic needs to be aligned + - this is string[0] for the old format + new magic - this is string[0] for the new format + newnlibs + ... + newlibs[0] + ... + newlibs[newnlibs-1] + string 1 + string 2 + ... +*/ +struct file_entry +{ + int flags; /* This is 1 for an ELF library. */ + unsigned int key, value; /* String table indices. */ +}; + +struct cache_file +{ + char magic[sizeof CACHEMAGIC - 1]; + unsigned int nlibs; + struct file_entry libs[0]; +}; + +#define CACHEMAGIC_NEW "glibc-ld.so.cache" +#define CACHE_VERSION "1.1" +#define CACHEMAGIC_VERSION_NEW CACHEMAGIC_NEW CACHE_VERSION + + +struct file_entry_new +{ + int32_t flags; /* This is 1 for an ELF library. */ + uint32_t key, value; /* String table indices. */ + uint32_t osversion; /* Required OS version. */ + uint64_t hwcap; /* Hwcap entry. */ +}; + +struct cache_file_new +{ + char magic[sizeof CACHEMAGIC_NEW - 1]; + char version[sizeof CACHE_VERSION - 1]; + uint32_t nlibs; /* Number of entries. */ + uint32_t len_strings; /* Size of string table. */ + uint32_t unused[5]; /* Leave space for future extensions + and align to 8 byte boundary. */ + struct file_entry_new libs[0]; /* Entries describing libraries. */ + /* After this the string table of size len_strings is found. */ +}; + +/* Used to align cache_file_new. */ +#define ALIGN_CACHE(addr) \ +(((addr) + __alignof__ (struct cache_file_new) -1) \ + & (~(__alignof__ (struct cache_file_new) - 1))) + +// extern int _dl_cache_libcmp (const char *p1, const char *p2) attribute_hidden; diff --git a/cli/jl_exports.h b/cli/jl_exports.h index f1a05b504d9da..71077d50406dd 100644 --- a/cli/jl_exports.h +++ b/cli/jl_exports.h @@ -7,8 +7,9 @@ #include "../src/jl_exported_funcs.inc" // Define pointer data as `const void * $(name);` -#define XX(name) JL_DLLEXPORT const void * name; +#define XX(name, type) JL_DLLEXPORT const void * jl_##name; JL_EXPORTED_DATA_POINTERS(XX) +JL_CONST_GLOBAL_VARS(XX) #undef XX // Define symbol data as `$(type) $(name);` @@ -30,7 +31,7 @@ JL_RUNTIME_EXPORTED_FUNCS_WIN(XX) JL_CODEGEN_EXPORTED_FUNCS(XX) #undef XX -// Define holder locations for function addresses as `const void * $(name)_addr = NULL; +// Define holder locations for function addresses as `JL_HIDDEN anonfunc * $(name)_addr = NULL;` #define XX(name) JL_HIDDEN anonfunc * name##_addr = NULL; JL_RUNTIME_EXPORTED_FUNCS(XX) #ifdef _OS_WINDOWS_ diff --git a/cli/list_strip_symbols.h b/cli/list_strip_symbols.h index 5d534616e132b..7db709663c13d 100644 --- a/cli/list_strip_symbols.h +++ b/cli/list_strip_symbols.h @@ -1,8 +1,9 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "jl_exported_funcs.inc" -#include "trampolines/common.h" -#define XX(x) --strip-symbol=CNAME(x) +#include "../src/support/platform.h" +// Export definitions use undecorated names, including on Windows i686. +#define XX(x) x JL_RUNTIME_EXPORTED_FUNCS(XX) #ifdef _OS_WINDOWS_ JL_RUNTIME_EXPORTED_FUNCS_WIN(XX) diff --git a/cli/loader.h b/cli/loader.h index 310226c84f815..62e4f3d3e4b52 100644 --- a/cli/loader.h +++ b/cli/loader.h @@ -70,7 +70,10 @@ JL_DLLEXPORT extern int jl_load_repl(int, char **); JL_DLLEXPORT void jl_loader_print_stderr(const char * msg); void jl_loader_print_stderr3(const char * msg1, const char * msg2, const char * msg3); +void *jl_loader_open_via_mmap(const char *filepath, size_t *size); static void * lookup_symbol(const void * lib_handle, const char * symbol_name); +const char *jl_loader_probe_system_library(const char *libname, const char *symbol); +int jl_loader_locate_symbol(const char *library, const char *symbol); #ifdef _OS_WINDOWS_ LPWSTR *CommandLineToArgv(LPWSTR lpCmdLine, int *pNumArgs); diff --git a/cli/loader_exe.c b/cli/loader_exe.c index 5fc8e73189ac6..2fccf5b908434 100644 --- a/cli/loader_exe.c +++ b/cli/loader_exe.c @@ -14,6 +14,61 @@ extern "C" { JULIA_DEFINE_FAST_TLS +#ifdef _OS_DARWIN_ +// The macOS application bundle (contrib/mac/app) uses `julia-terminal` as its +// double-clickable entry point (CFBundleExecutable): a copy of this loader placed +// directly in Contents/MacOS/ (a real Mach-O, as Apple's notary requires of a bundle's +// main executable -- a symlink, or a binary living under Contents/Resources/ where it is +// also sealed as a resource, is rejected as an invalid signature). When the loader is +// invoked under that name, we don't start the REPL in-process (a Finder-launched bundle +// has no controlling terminal); instead we relaunch the REPL binary from the bundled +// tree inside a new Terminal.app window so the user gets an interactive session. Keying +// off the `julia-terminal` name is also how we avoid hijacking ordinary `julia` +// invocations (e.g. non-interactive `julia script.jl`). +static void maybe_launch_terminal(void) +{ + char exe_path[JL_PATH_MAX]; + uint32_t bufsize = sizeof(exe_path); + if (_NSGetExecutablePath(exe_path, &bufsize) != 0) + return; + // Resolve any symlinks to get this executable's canonical path. + char self_path[JL_PATH_MAX]; + if (realpath(exe_path, self_path) == NULL) + return; + + // Only take over when invoked as `julia-terminal`. + char * base = strrchr(self_path, '/'); + if (base == NULL) + return; + base++; + if (strcmp(base, "julia-terminal") != 0) + return; + + // Locate the REPL binary to hand to Terminal.app. In the app bundle the main + // executable is Contents/MacOS/julia-terminal and the REPL lives in the bundled tree + // at ../Resources/julia/bin/julia -- it must run from there so it finds its sysimage, + // stdlibs and share/ via its own ../lib and ../share. Rewrite self_path to that path + // in place, falling back to a sibling `julia` for any other (non-bundle) layout. + base[-1] = '\0'; // strip "/julia-terminal" -> "...MacOS" + char * dir = strrchr(self_path, '/'); + if (dir != NULL && strcmp(dir + 1, "MacOS") == 0) { + static const char rel[] = "/Resources/julia/bin/julia"; + if ((size_t)(dir - self_path) + sizeof(rel) > sizeof(self_path)) + return; + strcpy(dir, rel); // ".../Contents" + "/Resources/julia/bin/julia" + } + else { + static const char rel[] = "/julia"; + if (strlen(self_path) + sizeof(rel) > sizeof(self_path)) + return; + strcat(self_path, rel); + } + execl("/usr/bin/open", "open", "-a", "Terminal", self_path, (char *)NULL); + jl_loader_print_stderr("ERROR: Failed to launch Terminal.app!\n"); + exit(1); +} +#endif + #ifdef _COMPILER_ASAN_ENABLED_ JL_DLLEXPORT const char* __asan_default_options(void) { @@ -42,6 +97,10 @@ int main(int argc, char * argv[]) putenv("LBT_USE_RTLD_DEEPBIND=0"); #endif +#ifdef _OS_DARWIN_ + maybe_launch_terminal(); +#endif + // Convert Windows wchar_t values to UTF8 #ifdef _OS_WINDOWS_ for (int i = 0; i < argc; i++) { diff --git a/cli/loader_lib.c b/cli/loader_lib.c index 4d75cfd9563cb..0f5b356e120db 100644 --- a/cli/loader_lib.c +++ b/cli/loader_lib.c @@ -43,22 +43,29 @@ void jl_loader_print_stderr3(const char * msg1, const char * msg2, const char * * which the user can delete to save space if generating new code is not necessary. * However, if it exists and cannot be loaded, that's a problem. So, we alert the user * and abort the process. */ -static void * load_library(const char * rel_path, const char * src_dir, int err) { +/* If allow_basename, then any library already loaded with the same basename is treated + * as a duplicate and a handle to the loaded library is returned. Otherwise already-loaded + * libraries are detected by full filepath / inode. */ +static void *load_library(const char * rel_path, const char * src_dir, int allow_basename, int err) { void * handle = NULL; - // See if a handle is already open to the basename - const char *basename = rel_path + strlen(rel_path); - while (basename-- > rel_path) - if (*basename == PATHSEPSTRING[0] || *basename == '/') - break; - basename++; + if (allow_basename) { + // See if a handle is already open to the basename + const char *basename = rel_path + strlen(rel_path); + while (basename-- > rel_path) + if (*basename == PATHSEPSTRING[0] || *basename == '/') + break; + basename++; #if defined(_OS_WINDOWS_) - if ((handle = GetModuleHandleA(basename))) - return handle; + if ((handle = GetModuleHandleA(basename))) + return handle; + SetLastError(0); #else - // if err == 0 the library is optional, so don't allow global lookups to see it - if ((handle = dlopen(basename, RTLD_NOLOAD | RTLD_NOW | (err ? RTLD_GLOBAL : RTLD_LOCAL)))) - return handle; + // if err == 0 the library is optional, so don't allow global lookups to see it + if ((handle = dlopen(basename, RTLD_NOLOAD | RTLD_NOW | (err ? RTLD_GLOBAL : RTLD_LOCAL)))) + return handle; + (void) dlerror(); #endif + } char path[2*JL_PATH_MAX + 1] = {0}; strncat(path, src_dir, sizeof(path) - 1); @@ -85,7 +92,10 @@ static void * load_library(const char * rel_path, const char * src_dir, int err) else { if (!err && !PATH_EXISTS()) { #if defined(_OS_WINDOWS_) + SetLastError(0); free(wpath); +#else + (void) dlerror(); #endif return NULL; } @@ -117,16 +127,56 @@ static void * load_library(const char * rel_path, const char * src_dir, int err) return handle; } +// case-insensitive strcmp +static int istrcmp(const char *val, const char *token) { + for (; *token; val++, token++) { + char c = *val; + if (c >= 'A' && c <= 'Z') + c += 'a' - 'A'; + if (c != *token) + return 0; + } + return *val == '\0'; +} + +// intended to match Base.get_bool_env +static int env_var_bool(const char *name, int *value) { +#if defined(_OS_WINDOWS_) + char val[8]; + DWORD val_len = GetEnvironmentVariableA(name, val, sizeof(val)); + if (val_len == 0 || val_len >= sizeof(val)) /* unset, or too long to be a token */ + return 0; +#else + const char *val = getenv(name); + if (val == NULL) + return 0; +#endif + if (istrcmp(val, "t") || istrcmp(val, "true") || istrcmp(val, "y") || istrcmp(val, "yes") || istrcmp(val, "1")) { + *value = 1; + return 1; + } + if (istrcmp(val, "f") || istrcmp(val, "false") || istrcmp(val, "n") || istrcmp(val, "no") || istrcmp(val, "0")) { + *value = 0; + return 1; + } + return 0; +} + static void * lookup_symbol(const void * lib_handle, const char * symbol_name) { #ifdef _OS_WINDOWS_ - return GetProcAddress((HMODULE) lib_handle, symbol_name); + void *sym = GetProcAddress((HMODULE) lib_handle, symbol_name); + SetLastError(0); // clear error, if any + return sym; #else - return dlsym((void *)lib_handle, symbol_name); + void *sym = dlsym((void *)lib_handle, symbol_name); + if (sym == NULL) + (void) dlerror(); // clear dlerror + return sym; #endif } #if defined(_OS_WINDOWS_) -void win32_formatmessage(DWORD code, char *reason, int len) { +static char *win32_formatmessage(DWORD code) { DWORD res; LPWSTR errmsg; res = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | @@ -145,9 +195,9 @@ void win32_formatmessage(DWORD code, char *reason, int len) { NULL, code, 0, (LPWSTR)&errmsg, 0, NULL); } - res = WideCharToMultiByte(CP_UTF8, 0, errmsg, -1, reason, len, NULL, NULL); - reason[len - 1] = '\0'; + char *utf8msg = wchar_to_utf8(errmsg); LocalFree(errmsg); + return utf8msg; } #endif @@ -169,9 +219,9 @@ JL_DLLEXPORT const char * jl_get_libdir() DWORD err = GetLastError(); jl_loader_print_stderr3("ERROR: could not locate library \"", LIBJULIA_NAME, "\"\n"); - char msg[2048]; - win32_formatmessage(err, msg, sizeof(msg)); + char *msg = win32_formatmessage(err); jl_loader_print_stderr(msg); + free(msg); exit(1); } @@ -212,7 +262,7 @@ JL_DLLEXPORT const char * jl_get_libdir() // which can break loading of some system libraries: . // As a fix, on linux we probe the system libstdc++ to see if it is newer, and then load it if it is. // Otherwise, we load the bundled one. This improves compatibility with third party dynamic libs that -// may depend on symbols exported by the system libstdxc++. +// may depend on symbols exported by the system libstdc++. #ifdef _OS_LINUX_ #ifndef GLIBCXX_LEAST_VERSION_SYMBOL #warning GLIBCXX_LEAST_VERSION_SYMBOL should always be defined in the makefile. @@ -222,166 +272,17 @@ JL_DLLEXPORT const char * jl_get_libdir() #include #include -// write(), but handle errors and avoid EINTR -static void write_wrapper(int fd, const char *str, size_t len) -{ - size_t written_sofar = 0; - while (len) { - ssize_t bytes_written = write(fd, str + written_sofar, len); - if (bytes_written == -1 && errno == EINTR) continue; - if (bytes_written == -1 && errno != EINTR) { - perror("(julia) child libstdcxxprobe write"); - _exit(1); - } - len -= bytes_written; - written_sofar += bytes_written; - } -} - -// read(), but handle errors and avoid EINTR -static void read_wrapper(int fd, char **ret, size_t *ret_len) -{ - // Allocate an initial buffer - size_t len = JL_PATH_MAX; - char *buf = (char *)malloc(len + 1); - if (!buf) { - perror("(julia) malloc"); - exit(1); - } - - // Read into it, reallocating as necessary - size_t have_read = 0; - while (1) { - ssize_t n = read(fd, buf + have_read, len - have_read); - if (n == 0) break; - if (n == -1 && errno != EINTR) { - perror("(julia) libstdcxxprobe read"); - exit(1); - } - if (n == -1 && errno == EINTR) continue; - have_read += n; - if (have_read == len) { - buf = (char *)realloc(buf, 1 + (len *= 2)); - if (!buf) { - perror("(julia) realloc"); - exit(1); - } - } - } - - *ret = buf; - *ret_len = have_read; -} - // Return the path to the libstdcxx to load. // If the path is found, return it. -// Otherwise, print the error and exit. +// Otherwise, return NULL. // The path returned must be freed. -static char *libstdcxxprobe(void) +static const char *libstdcxxprobe(void) { - // Create the pipe and child process. - int fork_pipe[2]; - int ret = pipe(fork_pipe); - if (ret == -1) { - perror("(julia) Error during libstdcxxprobe: pipe"); - exit(1); - } - pid_t pid = fork(); - if (pid == -1) { - perror("Error during libstdcxxprobe:\nfork"); - exit(1); - } - if (pid == (pid_t) 0) { // Child process. - close(fork_pipe[0]); - - // Open the first available libstdc++.so. - // If it can't be found, report so by exiting zero. - // The star is there to prevent the compiler from merging constants - // with "\0*libstdc++.so.6", which we string replace inside the .so during - // make install. - void *handle = dlopen("libstdc++.so.6\0*", RTLD_LAZY); - if (!handle) { - _exit(0); - } - - // See if the version is compatible - char *dlerr = dlerror(); // clear out dlerror - void *sym = dlsym(handle, GLIBCXX_LEAST_VERSION_SYMBOL); - (void)sym; - dlerr = dlerror(); - if (dlerr) { - // We can't use the library that was found, so don't write anything. - // The main process will see that nothing was written, - // then exit the function and return null. - _exit(0); - } + void *handle = dlopen("libstdc++.so.6\0*", RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD); + if (handle != NULL) + return NULL; // libstdc++ already loaded - nothing we can do - // No error means the symbol was found, we can use this library. - // Get the path to it, and write it to the parent process. - struct link_map *lm; - ret = dlinfo(handle, RTLD_DI_LINKMAP, &lm); - if (ret == -1) { - char *errbuf = dlerror(); - char *errdesc = (char*)"Error during libstdcxxprobe in child process:\ndlinfo: "; - write_wrapper(STDERR_FILENO, errdesc, strlen(errdesc)); - write_wrapper(STDERR_FILENO, errbuf, strlen(errbuf)); - write_wrapper(STDERR_FILENO, "\n", 1); - _exit(1); - } - char *libpath = lm->l_name; - write_wrapper(fork_pipe[1], libpath, strlen(libpath)); - _exit(0); - } - else { // Parent process. - close(fork_pipe[1]); - - // Read the absolute path to the lib from the child process. - char *path; - size_t pathlen; - read_wrapper(fork_pipe[0], &path, &pathlen); - - // Close the read end of the pipe - close(fork_pipe[0]); - - // Wait for the child to complete. - while (1) { - int wstatus; - pid_t npid = waitpid(pid, &wstatus, 0); - if (npid == -1) { - if (errno == EINTR) continue; - if (errno == ECHILD) { - // SIGCHLD is set to SIG_IGN or has flag SA_NOCLDWAIT, so the child - // did not become a zombie and wait for `waitpid` - it just exited. - // - // Assume that it exited successfully and use whatever libpath we - // got out of the pipe, if any. - break; - } - perror("Error during libstdcxxprobe in parent process:\nwaitpid"); - exit(1); - } - else if (!WIFEXITED(wstatus)) { - const char *err_str = "Error during libstdcxxprobe in parent process:\n" - "The child process did not exit normally.\n"; - size_t err_strlen = strlen(err_str); - write_wrapper(STDERR_FILENO, err_str, err_strlen); - exit(1); - } - else if (WEXITSTATUS(wstatus)) { - // The child has printed an error and exited, so the parent should exit too. - exit(1); - } - break; - } - - if (!pathlen) { - free(path); - return NULL; - } - // Ensure that `path` is zero-terminated. - path[pathlen] = '\0'; - return path; - } + return jl_loader_probe_system_library("libstdc++.so.6", GLIBCXX_LEAST_VERSION_SYMBOL); } #endif @@ -472,15 +373,9 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) { int probe_successful = 0; // Check to see if the user has disabled libstdc++ probing - char *probevar = getenv("JULIA_PROBE_LIBSTDCXX"); - if (probevar) { - if (strcmp(probevar, "1") == 0 || strcmp(probevar, "yes") == 0) - do_probe = 1; - else if (strcmp(probevar, "0") == 0 || strcmp(probevar, "no") == 0) - do_probe = 0; - } + env_var_bool("JULIA_PROBE_LIBSTDCXX", &do_probe); if (do_probe) { - char *cxxpath = libstdcxxprobe(); + const char *cxxpath = libstdcxxprobe(); if (cxxpath) { void *cxx_handle = dlopen(cxxpath, RTLD_LAZY); (void)cxx_handle; @@ -490,27 +385,36 @@ __attribute__((constructor)) void jl_load_libjulia_internal(void) { jl_loader_print_stderr3("Message: ", dlr, "\n"); exit(1); } - free(cxxpath); + free((void *)cxxpath); probe_successful = 1; } } // If the probe rejected the system libstdc++ (or didn't find one!) // just load our bundled libstdc++ as identified by curr_dep; if (!probe_successful) { - load_library(curr_dep, lib_dir, 1); +# ifdef RT_STATIC_LIBSTDCXX + // If we have a statically-linked libstdc++, it is ok for + // this to fail. + load_library(curr_dep, lib_dir, /* allow_basename */ 1, /* err */ 0); +# else + load_library(curr_dep, lib_dir, /* allow_basename */ 1, /* err */ 1); +# endif } #endif } else if (special_idx == 1) { // This special library is `libjulia-internal` - libjulia_internal = load_library(curr_dep, lib_dir, 1); + libjulia_internal = load_library(curr_dep, lib_dir, /* allow_basename */ 0, /* err */ 1); } else if (special_idx == 2) { // This special library is `libjulia-codegen` - libjulia_codegen = load_library(curr_dep, lib_dir, 0); + int load_codegen = 1; + env_var_bool("JULIA_LOAD_CODEGEN_LIB", &load_codegen); + if (load_codegen) + libjulia_codegen = load_library(curr_dep, lib_dir, /* allow_basename */ 0, /* err */ 0); } special_idx++; } else { // Otherwise, just load it as "normal" - load_library(curr_dep, lib_dir, 1); + load_library(curr_dep, lib_dir, /* allow_basename */ 1, /* err */ 1); } // Skip ahead to next dependency diff --git a/cli/loader_library_probe.c b/cli/loader_library_probe.c new file mode 100644 index 0000000000000..e9b88b1c3ffb6 --- /dev/null +++ b/cli/loader_library_probe.c @@ -0,0 +1,216 @@ +#include "../src/support/platform.h" +#include "loader.h" + +#ifdef _OS_LINUX_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "dl-cache.h" + +void *jl_loader_open_via_mmap(const char *filepath, size_t *size) +{ + int fd; + while (1) { + fd = open(filepath, O_CLOEXEC | O_RDONLY); + if (fd >= 0) { + break; + } else if (errno != EINTR) { + return NULL; + } + } + + struct stat info; + while (1) { + int err = fstat(fd, &info); + if (err >= 0) { + break; + } else if (errno != EINTR) { + close(fd); + return NULL; + } + } + + void *buffer = mmap( + NULL, info.st_size, PROT_READ, MAP_PRIVATE | MAP_NORESERVE, fd, /* offset */ 0 + ); + close(fd); + + if (MAP_FAILED == buffer) + return NULL; + + *size = info.st_size; + return buffer; +} + +static const char *search_ldcache_new(struct cache_file_new *cache, const char *libname, size_t *index) +{ + if (strncmp(cache->magic, CACHEMAGIC_NEW, sizeof(CACHEMAGIC_NEW) - 1) != 0) + return NULL; + + for (; *index < cache->nlibs; (*index)++) { + struct file_entry_new *lib = &cache->libs[*index]; + + const char *strtab = (const char *)cache; + const char *key = &strtab[lib->key]; + const char *value = &strtab[lib->value]; + + if (!_dl_cache_check_flags(lib->flags)) + continue; + if (strcmp(key, libname) != 0) + continue; + (*index)++; + return value; + } + + return NULL; +} + +static const char *search_ldcache(struct cache_file *cache, size_t cachesize, const char *libname, size_t *index) +{ + if (strncmp(cache->magic, CACHEMAGIC, sizeof(CACHEMAGIC) - 1) != 0) + return search_ldcache_new((struct cache_file_new *)cache, libname, index); + + // check for an embedded / hybrid 'new'-style cache + size_t offset = ALIGN_CACHE(sizeof(struct cache_file) + cache->nlibs * sizeof(struct file_entry)); + if (cachesize >= offset + sizeof(struct cache_file_new)) { + struct cache_file_new *new_cache = (struct cache_file_new *)((void *)cache + offset); + if (strncmp(new_cache->magic, CACHEMAGIC_NEW, sizeof(CACHEMAGIC_NEW) - 1) == 0) + return search_ldcache_new(new_cache, libname, index); + } + + for (; *index < cache->nlibs; (*index)++) { + struct file_entry *lib = &cache->libs[*index]; + + const char *strtab = (const char *)&cache->libs[cache->nlibs]; + const char *key = &strtab[lib->key]; + const char *value = &strtab[lib->value]; + + if (!_dl_cache_check_flags(lib->flags)) + continue; + if (strcmp(key, libname) != 0) + continue; + (*index)++; + return value; + } + + return NULL; +} + +const char *ldcache_dirs[] = { + "/etc/ld.so.cache", +}; + +const char *default_libdirs[] = { + "/lib/", + "/usr/lib/", +#ifdef _P64 + "/lib64/", + "/usr/lib64/", +#endif +}; + +/** + * Search for a system library with the filename `libname` containing `symbol`. + * Return NULL if no matching library could be found. + * + * To emulate the Linux dynamic linker search behavior, this function scans for + * system libraries in: + * 1. LD_LIBRARY_PATH + * 2. `/etc/ld.so.cache` + * 3. "default" system libdirs (/lib, /usr/lib, etc.) + * + * This function does not consider any DT_RPATH or DT_RUNPATH entries. + * (see `ld.so(8)` manpage) + **/ +const char *jl_loader_probe_system_library(const char *libname, const char *symbol) +{ + char buf[PATH_MAX]; + + // Make a best-effort attempt to emulate the linker's use of LD_LIBRARY_PATH + char *LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH"); + if (LD_LIBRARY_PATH != NULL) { + LD_LIBRARY_PATH = strdup(LD_LIBRARY_PATH); + char *path = LD_LIBRARY_PATH; + + int last = 0; + while (!last) { + // walk to next ':' or '\0' + char *ch = path; + while (1) { + if (*ch == '\0') + last = 1; + if (*ch == ':' || *ch == '\0') + break; + ch += 1; + } + *ch = '\0'; + + if (ch == path) { + path += 1; + continue; + } + + int bytes = snprintf(buf, sizeof(buf), (ch[-1] == '/' ? "%s%s" : "%s/%s"), path, libname); + path = ch + 1; + if (bytes < 0 || sizeof(buf) < (size_t) bytes) + continue; + + if (jl_loader_locate_symbol(buf, symbol)) { + free(LD_LIBRARY_PATH); + return strdup(buf); + } + } + free(LD_LIBRARY_PATH); + } + + // Check the ld.so.cache for the library. Assuming we can find the cache, + // this is by far our best chance to locate the lib successfully. + size_t npaths = sizeof(ldcache_dirs) / sizeof(const char *); + for (size_t i = 0; i < npaths; i++) { + size_t sz; + struct cache_file *cache = + (struct cache_file *)jl_loader_open_via_mmap(ldcache_dirs[i], &sz); + + if (cache == NULL) + continue; // ld.so.cache was not found (could be NixOS) + + size_t iter = 0; + const char *library; + while ((library = search_ldcache(cache, sz, libname, &iter)) != NULL) { + if (jl_loader_locate_symbol(library, symbol)) { + library = strdup(library); + munmap((void *)cache, sz); + return library; + } + } + + munmap((void *)cache, sz); + } + + // As a last-ditch effort, try to emulate / search the "default" libdirs used + // by the GLIBC dynamic linker. + size_t ndirs = sizeof(default_libdirs) / sizeof(const char *); + for (size_t i = 0; i < ndirs; i++) { + int bytes = snprintf(buf, sizeof(buf), "%s%s", default_libdirs[i], libname); + if (bytes < 0 || sizeof(buf) < (size_t) bytes) + continue; + if (jl_loader_locate_symbol(buf, symbol)) + return strdup(buf); + } + + return NULL; +} + +#endif diff --git a/cli/loader_symbol_probe.c b/cli/loader_symbol_probe.c new file mode 100644 index 0000000000000..de4598eb6a89e --- /dev/null +++ b/cli/loader_symbol_probe.c @@ -0,0 +1,123 @@ +#include "../src/support/platform.h" +#include "loader.h" + +#ifdef _OS_LINUX_ + +#include + +#include +#include +#include +#include +#include +#include + +static Elf64_Shdr *Elf64_get_section(Elf64_Ehdr *hdr, size_t i) +{ + size_t section_header_sz = hdr->e_shentsize; + size_t byte_offset = hdr->e_shoff + i * section_header_sz; + return (Elf64_Shdr *)&((char *)hdr)[byte_offset]; +} + +static const char *Elf64_get_strtab(Elf64_Ehdr *hdr, Elf64_Shdr *section) +{ + return &((const char *)hdr)[section->sh_offset]; +} + +static Elf64_Sym *Elf64_get_symbol(Elf64_Ehdr *hdr, Elf64_Shdr *section, size_t i) +{ + size_t byte_offset = section->sh_offset + i * section->sh_entsize; + return (Elf64_Sym *)&((char *)hdr)[byte_offset]; +} + +static int Elf64_locate_symbol(Elf64_Ehdr *hdr, const char *symbol) +{ + if (hdr->e_type != ET_DYN) + return 0; + + for (size_t sect_idx = 0; sect_idx < hdr->e_shnum; sect_idx++) { + Elf64_Shdr *shdr = Elf64_get_section(hdr, sect_idx); + if (shdr->sh_type != SHT_DYNSYM) + continue; + + Elf64_Shdr *strtab_shdr = Elf64_get_section(hdr, shdr->sh_link); + const char *strtab = Elf64_get_strtab(hdr, strtab_shdr); + + size_t nsymbols = shdr->sh_size / shdr->sh_entsize; + for (size_t i = 0; i < nsymbols; i++) { + Elf64_Sym *sym = Elf64_get_symbol(hdr, shdr, i); + const char *name = &strtab[sym->st_name]; + if (strcmp(name, symbol) == 0) + return 1; + } + } + return 0; +} + +static Elf32_Shdr *Elf32_get_section(Elf32_Ehdr *hdr, size_t i) +{ + size_t section_header_sz = hdr->e_shentsize; + size_t byte_offset = hdr->e_shoff + i * section_header_sz; + return (Elf32_Shdr *)&((char *)hdr)[byte_offset]; +} + +static const char *Elf32_get_strtab(Elf32_Ehdr *hdr, Elf32_Shdr *section) +{ + return &((const char *)hdr)[section->sh_offset]; +} + +static Elf32_Sym *Elf32_get_symbol(Elf32_Ehdr *hdr, Elf32_Shdr *section, size_t i) +{ + size_t byte_offset = section->sh_offset + i * section->sh_entsize; + return (Elf32_Sym *)&((char *)hdr)[byte_offset]; +} + +static int Elf32_locate_symbol(Elf32_Ehdr *hdr, const char *symbol) +{ + if (hdr->e_type != ET_DYN) + return 0; + + for (size_t sect_idx = 0; sect_idx < hdr->e_shnum; sect_idx++) { + Elf32_Shdr *shdr = Elf32_get_section(hdr, sect_idx); + if (shdr->sh_type != SHT_DYNSYM) + continue; + + Elf32_Shdr *strtab_shdr = Elf32_get_section(hdr, shdr->sh_link); + const char *strtab = Elf32_get_strtab(hdr, strtab_shdr); + + size_t nsymbols = shdr->sh_size / shdr->sh_entsize; + for (size_t i = 0; i < nsymbols; i++) { + Elf32_Sym *sym = Elf32_get_symbol(hdr, shdr, i); + const char *name = &strtab[sym->st_name]; + if (strcmp(name, symbol) == 0) + return 1; + } + } + return 0; +} + +int jl_loader_locate_symbol(const char *library, const char *symbol) +{ + size_t library_sz; + void *elf_file = jl_loader_open_via_mmap(library, &library_sz); + if (elf_file == NULL) + return 0; + + int found = 0; + const char *hdr = (const char *)elf_file; + if (strncmp(hdr, ELFMAG, SELFMAG) != 0) + goto bail; + + assert(hdr[5] == ELFDATA2LSB); + if (hdr[4] == ELFCLASS32) { + found = Elf32_locate_symbol((Elf32_Ehdr *)hdr, symbol); + } else if (hdr[4] == ELFCLASS64) { + found = Elf64_locate_symbol((Elf64_Ehdr *)hdr, symbol); + } + +bail: + munmap(elf_file, library_sz); + return found; +} + +#endif diff --git a/cli/loader_win_utils.c b/cli/loader_win_utils.c index 34fe277fb2879..ae2be4cb4b15c 100644 --- a/cli/loader_win_utils.c +++ b/cli/loader_win_utils.c @@ -77,7 +77,7 @@ LPWSTR *CommandLineToArgv(LPWSTR lpCmdLine, int *pNumArgs) { if (backslashes % 2) *(out - 1) = '"'; // replace \ with " else - in_quotes = !in_quotes; // treat as quote delimater + in_quotes = !in_quotes; // treat as quote delimiter empty = 0; break; case '\t': @@ -116,36 +116,170 @@ void JL_HIDDEN exit(int code) { /* Utilities to convert from Windows' wchar_t stuff to UTF-8 */ -char *wchar_to_utf8(const wchar_t * wstr) { - /* Fast-path empty strings, as WideCharToMultiByte() returns zero for them. */ - if (wstr[0] == L'\0') { - char *str = malloc(1); - str[0] = '\0'; - return str; + +/* Decode one WTF-8 code point from the input string, advancing the pointer. + Based on libuv's uv__wtf8_decode1. */ +static int wtf8_decode1(const char** input) { + unsigned int code_point; + unsigned char b1; + unsigned char b2; + unsigned char b3; + unsigned char b4; + + b1 = **input; + if (b1 <= 0x7F) + return b1; /* ASCII code point */ + if (b1 < 0xC2) + return -1; /* invalid: continuation byte */ + code_point = b1; + + b2 = *++*input; + if ((b2 & 0xC0) != 0x80) + return -1; /* invalid: not a continuation byte */ + code_point = (code_point << 6) | (b2 & 0x3F); + if (b1 <= 0xDF) + return 0x7FF & code_point; /* two-byte character */ + + b3 = *++*input; + if ((b3 & 0xC0) != 0x80) + return -1; /* invalid: not a continuation byte */ + code_point = (code_point << 6) | (b3 & 0x3F); + if (b1 <= 0xEF) + return 0xFFFF & code_point; /* three-byte character */ + + b4 = *++*input; + if ((b4 & 0xC0) != 0x80) + return -1; /* invalid: not a continuation byte */ + code_point = (code_point << 6) | (b4 & 0x3F); + if (b1 <= 0xF4) { + code_point &= 0x1FFFFF; + if (code_point <= 0x10FFFF) + return code_point; /* four-byte character */ } - size_t len = WideCharToMultiByte(CP_UTF8, 0, wstr, -1, NULL, 0, NULL, NULL); - if (!len) - return NULL; - char *str = (char *)malloc(len); - if (!WideCharToMultiByte(CP_UTF8, 0, wstr, -1, str, len, NULL, NULL)) - return NULL; - return str; + + /* code point too large */ + return -1; } -wchar_t *utf8_to_wchar(const char * str) { - /* Fast-path empty strings, as MultiByteToWideChar() returns zero for them. */ - if (str[0] == '\0') { - wchar_t *wstr = malloc(sizeof(wchar_t)); - wstr[0] = L'\0'; - return wstr; +/* Get the surrogate pair value from UTF-16 input. + Based on libuv's uv__get_surrogate_value. */ +static int get_surrogate_value(const wchar_t* w_source_ptr) { + unsigned short u; + unsigned short next; + + u = w_source_ptr[0]; + if (u >= 0xD800 && u <= 0xDBFF) { + next = w_source_ptr[1]; + if (next >= 0xDC00 && next <= 0xDFFF) + return 0x10000 + ((u - 0xD800) << 10) + (next - 0xDC00); + } + return u; +} + +/* Convert UTF-16 to WTF-8, returning allocated string. + Based on libuv's uv_utf16_to_wtf8. */ +char *wchar_to_utf8(const wchar_t *wstr) { + const wchar_t *w_source_ptr; + size_t target_len; + int code_point; + char *target; + char *p; + + /* First pass: calculate required length */ + w_source_ptr = wstr; + target_len = 0; + while (1) { + code_point = get_surrogate_value(w_source_ptr); + if (code_point == 0) + break; + if (code_point < 0x80) + target_len += 1; + else if (code_point < 0x800) + target_len += 2; + else if (code_point < 0x10000) + target_len += 3; + else { + target_len += 4; + w_source_ptr++; + } + w_source_ptr++; } - size_t len = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); - if (!len) + + /* Allocate buffer */ + target = (char *)malloc(target_len + 1); + if (!target) return NULL; - wchar_t *wstr = (wchar_t *)malloc(len * sizeof(wchar_t)); - if (!MultiByteToWideChar(CP_UTF8, 0, str, -1, wstr, len)) + + /* Second pass: perform conversion */ + w_source_ptr = wstr; + p = target; + while (1) { + code_point = get_surrogate_value(w_source_ptr); + if (code_point == 0) + break; + if (code_point < 0x80) { + *p++ = code_point; + } else if (code_point < 0x800) { + *p++ = 0xC0 | (code_point >> 6); + *p++ = 0x80 | (code_point & 0x3F); + } else if (code_point < 0x10000) { + *p++ = 0xE0 | (code_point >> 12); + *p++ = 0x80 | ((code_point >> 6) & 0x3F); + *p++ = 0x80 | (code_point & 0x3F); + } else { + *p++ = 0xF0 | (code_point >> 18); + *p++ = 0x80 | ((code_point >> 12) & 0x3F); + *p++ = 0x80 | ((code_point >> 6) & 0x3F); + *p++ = 0x80 | (code_point & 0x3F); + w_source_ptr++; + } + w_source_ptr++; + } + *p = '\0'; + + return target; +} + +/* Convert WTF-8 to UTF-16, returning allocated string. + Based on libuv's uv_wtf8_to_utf16 and uv_wtf8_length_as_utf16. */ +wchar_t *utf8_to_wchar(const char *str) { + const char *source_ptr; + size_t w_target_len; + int code_point; + wchar_t *w_target; + wchar_t *p; + + /* First pass: calculate required length */ + source_ptr = str; + w_target_len = 0; + do { + code_point = wtf8_decode1(&source_ptr); + if (code_point < 0) + return NULL; + if (code_point > 0xFFFF) + w_target_len++; + w_target_len++; + } while (*source_ptr++); + + /* Allocate buffer */ + w_target = (wchar_t *)malloc(w_target_len * sizeof(wchar_t)); + if (!w_target) return NULL; - return wstr; + + /* Second pass: perform conversion */ + source_ptr = str; + p = w_target; + do { + code_point = wtf8_decode1(&source_ptr); + if (code_point > 0xFFFF) { + *p++ = (((code_point - 0x10000) >> 10) + 0xD800); + *p++ = ((code_point - 0x10000) & 0x3FF) + 0xDC00; + } else { + *p++ = code_point; + } + } while (*source_ptr++); + + return w_target; } size_t JL_HIDDEN strlen(const char * x) { diff --git a/cli/test_import_library.py b/cli/test_import_library.py new file mode 100644 index 0000000000000..6bb7134dff576 --- /dev/null +++ b/cli/test_import_library.py @@ -0,0 +1,82 @@ +"""Check loader import filtering with MinGW cross tools; no Windows execution. + +Run with python3 cli/test_import_library.py. Missing toolchains are skipped. +""" + +from pathlib import Path +import os +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent.parent + + +class ImportLibraryTests(unittest.TestCase): + def check_arch(self, arch, machine, use_lld=False): + prefix = arch + '-w64-mingw32-' + if not all(shutil.which(prefix + tool) for tool in ['gcc', 'dlltool', 'nm']): + self.skipTest(prefix + ' tools unavailable') + lld = os.environ.get('LLD', 'ld.lld') + if use_lld and not shutil.which(lld): + self.skipTest(lld + ' unavailable') + with tempfile.TemporaryDirectory() as tmp: + build = Path(tmp) + (build / 'toy.c').write_text( + '__declspec(dllexport) void jl_alloc_array_1d(void) {}\n' + '__declspec(dllexport) void jl_codegen_only(void) {}\n' + '__declspec(dllexport) int keep_data = 42;\n') + if use_lld: + subprocess.run([prefix + 'gcc', '-c', str(build / 'toy.c'), + '-o', str(build / 'toy.o')], check=True) + subprocess.run([lld, '-m', 'i386pep' if arch == 'x86_64' else 'i386pe', + '-shared', '-e', 'jl_codegen_only' if arch == 'x86_64' else '_jl_codegen_only', + str(build / 'toy.o'), '-o', str(build / 'toy.dll'), + '--output-def', str(build / 'toy.dll.def')], check=True) + else: + subprocess.run([prefix + 'gcc', '-shared', str(build / 'toy.c'), + '-o', str(build / 'toy.dll'), + '-Wl,--output-def,' + str(build / 'toy.dll.def')], check=True) + # Exercise the actual recipe with the repository's export list. + source = (ROOT / 'cli/Makefile').read_text() + macro = source[source.index('define implib_from_def'):source.index('DEF_FLAGS =')] + (build / 'Makefile').write_text( + f'CPP_STDOUT := {prefix}gcc -E\nJULIAHOME := {ROOT}\n' + f'SRCDIR := {ROOT}/cli\nBUILDDIR := {build}\nbuild_libdir := {build}\n' + f'DLLTOOL := {prefix}dlltool\nDLLTOOL_MACHINE := {machine}\n' + + macro + '\nall:\n\t$(call implib_from_def,toy.dll)\n') + subprocess.run(['make', '-s', '-C', tmp], check=True) + symbols = subprocess.check_output([prefix + 'nm', str(build / 'toy.dll.a')], text=True) + self.assertNotIn('jl_alloc_array_1d', symbols) + self.assertIn('jl_codegen_only', symbols) + self.assertIn('keep_data', symbols) + self.assertIn('jl_alloc_array_1d', (build / 'toy.dll.def').read_text()) + (build / 'consumer.c').write_text( + '__declspec(dllimport) void jl_codegen_only(void);\n' + '__declspec(dllimport) int keep_data;\n' + 'int main(void) { jl_codegen_only(); return keep_data; }\n') + subprocess.run([prefix + 'gcc', str(build / 'consumer.c'), + str(build / 'toy.dll.a'), '-o', str(build / 'consumer.exe')], check=True) + # A failed preprocessor must not silently produce an unfiltered library. + (build / 'toy.dll.a').unlink() + result = subprocess.run(['make', '-s', '-C', tmp, 'CPP_STDOUT=false'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((build / 'toy.dll.a').exists()) + + def test_i686(self): + self.check_arch('i686', 'i386') + + def test_x86_64(self): + self.check_arch('x86_64', 'i386:x86-64') + + def test_i686_lld(self): + self.check_arch('i686', 'i386', use_lld=True) + + def test_x86_64_lld(self): + self.check_arch('x86_64', 'i386:x86-64', use_lld=True) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/contrib/add_license_to_files.jl b/contrib/add_license_to_files.jl index 1d301a5455394..f32aa40a07c5d 100644 --- a/contrib/add_license_to_files.jl +++ b/contrib/add_license_to_files.jl @@ -6,12 +6,12 @@ # Option `old_license` to remove an existing license first in case one wants to change the # license text in the future. # -# Checks also if somewhere else in the file the license text is found (`copy/past error`) +# Checks also if somewhere else in the file the license text is found (`copy/paste error`) # and if possible deletes such lines - if other text is on the same line it raises an error. ### CONFIG HERE -const print_result = true # prints files which where not processed. +const print_result = true # prints files which were not processed. const rootdirs = [ "../base", @@ -103,7 +103,7 @@ license_linenum(line) = startswith(strip(line), "#!") ? 2 : 1 # Collects all absolute file paths in rootdir inclusive subdirs function getfilespaths!(filepaths::Vector, rootdir::AbstractString) - isdir(rootdir) || error(string("`rootdir` must be an directory. ")) + isdir(rootdir) || error(string("`rootdir` must be a directory. ")) abs_rootdir = abspath(rootdir) for name in readdir(abs_rootdir) path = joinpath(abs_rootdir, name) diff --git a/contrib/asan/Make.user.asan b/contrib/asan/Make.user.asan index 1ad8d3c8fb1f7..686ee165e45ef 100644 --- a/contrib/asan/Make.user.asan +++ b/contrib/asan/Make.user.asan @@ -13,8 +13,8 @@ USE_BINARYBUILDER_LLVM=1 override SANITIZE=1 override SANITIZE_ADDRESS=1 -# make the GC use regular malloc/frees, which are hooked by ASAN -override WITH_GC_DEBUG_ENV=1 +# Limit allocation stack traces to reduce ASAN memory and runtime overhead. +export ASAN_OPTIONS=allow_user_segv_handler=1:detect_leaks=0:malloc_context_size=2 # Enable Julia assertions and LLVM assertions FORCE_ASSERTIONS=1 diff --git a/contrib/asan/build.sh b/contrib/asan/build.sh index 2e7f243772c81..325d903c11244 100755 --- a/contrib/asan/build.sh +++ b/contrib/asan/build.sh @@ -5,10 +5,9 @@ # Usage: # contrib/asan/build.sh [...] # -# Build ASAN-enabled julia. Given a workspace directory , build -# ASAN-enabled julia in /asan. Required toolss are install under -# /toolchain. This scripts also takes optional arguments -# which are passed to `make`. The default make target is `debug`. +# Build ASAN-enabled julia in /asan. Required tools are installed under +# /toolchain. Optional are passed to `make`; the default +# target is `release`. set -ue diff --git a/contrib/bolt/.gitignore b/contrib/bolt/.gitignore deleted file mode 100644 index 921d429130268..0000000000000 --- a/contrib/bolt/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -profiles-bolt* -optimized.build -toolchain - -bolt -bolt_instrument -merge_data -copy_originals -stage0 -stage1 diff --git a/contrib/bolt/Makefile b/contrib/bolt/Makefile deleted file mode 100644 index 76833b9865020..0000000000000 --- a/contrib/bolt/Makefile +++ /dev/null @@ -1,136 +0,0 @@ -.PHONY: clean clean_profiles restore_originals - -# Settings taken from https://github.com/rust-lang/rust/blob/master/src/tools/opt-dist/src/bolt.rs -BOLT_ARGS := -# Reorder basic blocks within functions -BOLT_ARGS += -reorder-blocks=ext-tsp -# Reorder functions within the binary -BOLT_ARGS += -reorder-functions=cdsort -# Split function code into hot and code regions -BOLT_ARGS += -split-functions -# Split as many basic blocks as possible -BOLT_ARGS += -split-all-cold -# Move jump tables to a separate section -BOLT_ARGS += -jump-tables=move -# Use regular size pages for code alignment -BOLT_ARGS += -no-huge-pages -# Fold functions with identical code -BOLT_ARGS += -icf=1 -# Split using best available strategy (three-way splitting, Cache-Directed Sort) -# Disabled for libjulia-internal till https://github.com/llvm/llvm-project/issues/89508 is fixed -# BOLT_ARGS += -split-strategy=cdsplit -# Update DWARF debug info in the final binary -BOLT_ARGS += -update-debug-sections -# Print optimization statistics -BOLT_ARGS += -dyno-stats -# BOLT doesn't fully support computed gotos, https://github.com/llvm/llvm-project/issues/89117 -# Use escaped regex as the name BOLT recognises is often a bit different, e.g. apply_cl/1(*2) -# This doesn't actually seem to do anything, the actual mitigation is not using --use-old-text -# which we do in the bolt target -BOLT_ARGS += -skip-funcs=.\*apply_cl.\* - -# -fno-reorder-blocks-and-partition is needed on gcc >= 8. -BOLT_FLAGS := $\ - "BOLT_CFLAGS_GCC+=-fno-reorder-blocks-and-partition" $\ - "BOLT_LDFLAGS=-Wl,--emit-relocs" - -STAGE0_BUILD:=$(CURDIR)/toolchain -STAGE1_BUILD:=$(CURDIR)/optimized.build - -STAGE0_BINARIES:=$(STAGE0_BUILD)/usr/bin/ - -PROFILE_DIR:=$(CURDIR)/profiles-bolt -JULIA_ROOT:=$(CURDIR)/../.. - -LLVM_BOLT:=$(STAGE0_BINARIES)llvm-bolt -LLVM_MERGEFDATA:=$(STAGE0_BINARIES)merge-fdata - -# If you add new files to optimize, you need to add BOLT_LDFLAGS and BOLT_CFLAGS to the build of your new file. -SYMLINKS_TO_OPTIMIZE := libLLVM.so libjulia-internal.so libjulia-codegen.so -FILES_TO_OPTIMIZE := $(shell for file in $(SYMLINKS_TO_OPTIMIZE); do readlink $(STAGE1_BUILD)/usr/lib/$$file; done) - -AFTER_INSTRUMENT_MESSAGE:='Run `make finish_stage1` to finish off the build. $\ - You can now optionally collect more profiling data by running Julia with an appropriate workload, $\ - if you wish, run `make clean_profiles` before doing so to remove any profiling data generated by `make finish_stage1`. $\ - You should end up with some data in $(PROFILE_DIR). Afterwards run `make merge_data && make bolt`.' - -$(STAGE0_BUILD) $(STAGE1_BUILD): - $(MAKE) -C $(JULIA_ROOT) O=$@ configure - -stage0: | $(STAGE0_BUILD) - $(MAKE) -C $(STAGE0_BUILD)/deps install-BOLT && \ - touch $@ - -# Build with our custom flags, binary builder doesn't use them so we need to build LLVM for now. -# We manually skip package image creation so that we can profile it -$(STAGE1_BUILD): stage0 -stage1: export USE_BINARYBUILDER_LLVM=0 -stage1: | $(STAGE1_BUILD) - $(MAKE) -C $(STAGE1_BUILD) $(BOLT_FLAGS) julia-src-release julia-symlink julia-libccalltest \ - julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest && \ - touch $@ - -copy_originals: stage1 - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE1_BUILD)/usr/lib/$$file; \ - cp $$abs_file "$$abs_file.original"; \ - done && \ - touch $@ - -# I don't think there's any particular reason to have -no-huge-pages here, perhaps slightly more accurate profile data -# as the final build uses -no-huge-pages -# We reset the mtime of the files to prevent make from rebuilding targets depending on them. -bolt_instrument: copy_originals - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE1_BUILD)/usr/lib/$$file; \ - old_time=$$(stat -c %Y $$abs_file); \ - $(LLVM_BOLT) "$$abs_file.original" -o $$abs_file --instrument --instrumentation-file-append-pid --instrumentation-file="$(PROFILE_DIR)/$$file-prof" -no-huge-pages; \ - mkdir -p $$(dirname "$(PROFILE_DIR)/$$file-prof"); \ - touch -d "@$$old_time" $$abs_file; \ - printf "\n"; \ - done && \ - touch $@ - @echo $(AFTER_INSTRUMENT_MESSAGE) - -finish_stage1: stage1 - $(MAKE) -C $(STAGE1_BUILD) - -merge_data: bolt_instrument - for file in $(FILES_TO_OPTIMIZE); do \ - profiles=$(PROFILE_DIR)/$$file-prof.*.fdata; \ - $(LLVM_MERGEFDATA) $$profiles > "$(PROFILE_DIR)/$$file-prof.merged.fdata"; \ - done && \ - touch $@ - -# The --use-old-text saves about 16 MiB of libLLVM.so size. -# However, the rust folk found it succeeds very non-deterministically for them. -# It tries to reuse old text segments to reduce binary size -# BOLT doesn't fully support computed gotos https://github.com/llvm/llvm-project/issues/89117, so we cannot use --use-old-text on libjulia-internal -# That flag saves less than 1 MiB for libjulia-internal so oh well. -# We reset the mtime of the files to prevent make from rebuilding targets depending on them. -bolt: merge_data - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE1_BUILD)/usr/lib/$$file; \ - old_time=$$(stat -c %Y $$abs_file); \ - $(LLVM_BOLT) "$$abs_file.original" -data "$(PROFILE_DIR)/$$file-prof.merged.fdata" -o $$abs_file $(BOLT_ARGS) $$(if [ "$$file" != $(shell readlink $(STAGE1_BUILD)/usr/lib/libjulia-internal.so) ]; then echo "--use-old-text -split-strategy=cdsplit"; fi); \ - touch -d "@$$old_time" $$abs_file; \ - done && \ - touch $@ - -clean_profiles: - rm -rf $(PROFILE_DIR) - -clean: - rm -f stage0 stage1 bolt copy_originals merge_data bolt_instrument - -restore_originals: copy_originals - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE1_BUILD)/usr/lib/$$file; \ - cp -P "$$abs_file.original" $$abs_file; \ - done - -delete_originals: copy_originals - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE1_BUILD)/usr/lib/$$file; \ - rm "$$abs_file.original"; \ - done diff --git a/contrib/bolt/README.md b/contrib/bolt/README.md deleted file mode 100644 index 8680939ef6276..0000000000000 --- a/contrib/bolt/README.md +++ /dev/null @@ -1,17 +0,0 @@ -BOLT only works on x86_64 and arch64 on Linux. - -DO NOT STRIP THE RESULTING .so FILES, https://github.com/llvm/llvm-project/issues/56738. -If you really need to, try adding `-use-gnu-stack` to `BOLT_ARGS`. - -To build a BOLT-optimized version of Julia run the following commands (`cd` into this directory first) -```bash -make stage1 -make copy_originals -make bolt_instrument -make finish_stage1 -make merge_data -make bolt -``` -After these commands finish, the optimized version of Julia will be built in the `optimized.build` directory. - -This doesn't align the code to support huge pages as it doesn't seem that we do that currently, this decreases the size of the .so files by 2-4mb. diff --git a/contrib/bump_stdlib.sh b/contrib/bump_stdlib.sh new file mode 100755 index 0000000000000..6865dbe5318e4 --- /dev/null +++ b/contrib/bump_stdlib.sh @@ -0,0 +1,219 @@ +#!/bin/bash +# This file is a part of Julia. License is MIT: https://julialang.org/license +# +# Bump external stdlibs to the latest commit of their upstream branch. +# +# Usage: contrib/bump_stdlib.sh [-b ] ... +# contrib/bump_stdlib.sh [-b ] all +# +# The stdlib branch is picked automatically: on a release-X.Y or +# backports-release-X.Y Julia branch the stdlib's release-X.Y branch is used, +# and stdlibs without such a branch upstream are skipped; on other Julia +# branches the branch recorded in stdlib/.version is used. +# Pass -b to override. +# +# For each stdlib, updates stdlib/.version, replaces the old +# checksums in deps/checksums with freshly generated ones, and creates one +# commit per stdlib in the style of BumpStdlibs.jl. Nothing is pushed. + +set -euo pipefail + +JULIAHOME=$(git rev-parse --show-toplevel 2>/dev/null || true) +if [ -z "$JULIAHOME" ] || [ ! -f "$JULIAHOME/stdlib/Makefile" ]; then + JULIAHOME=$(cd "$(dirname "$0")"/.. && pwd) +fi + +usage() { + echo "Usage: $0 [-b ] ... | all" >&2 + exit 1 +} + +BRANCH_OVERRIDE= +while getopts "b:" opt; do + case $opt in + b) BRANCH_OVERRIDE=$OPTARG ;; + *) usage ;; + esac +done +shift $((OPTIND - 1)) + +[ $# -ge 1 ] || usage + +list_stdlibs() { + for f in "$JULIAHOME"/stdlib/*.version; do + basename "$f" .version + done +} + +if [ $# -eq 1 ] && [ "$1" = "all" ]; then + NAMES=($(list_stdlibs)) +else + NAMES=("$@") + for NAME in "${NAMES[@]}"; do + if [ ! -f "$JULIAHOME/stdlib/$NAME.version" ]; then + echo "error: no stdlib '$NAME'; available:" >&2 + list_stdlibs | sed 's/^/ /' >&2 + exit 1 + fi + done +fi + +echo "Julia repo: $JULIAHOME" + +JULIA_BRANCH=$(git -C "$JULIAHOME" symbolic-ref --short -q HEAD || echo "(detached)") + +MSGFILE=$(mktemp) +trap 'rm -f "$MSGFILE"' EXIT + +bump_one() { + local NAME=$1 + local VERSION_FILE="$JULIAHOME/stdlib/$NAME.version" + local UPPER + UPPER=$(echo "$NAME" | tr '[:lower:]' '[:upper:]') + + getvar() { + sed -n "s/^$1[[:space:]]*:\{0,1\}=[[:space:]]*//p" "$VERSION_FILE" | tail -n1 | tr -d '[:space:]' + } + + local OLD_SHA OLD_BRANCH GIT_URL + OLD_SHA=$(getvar "${UPPER}_SHA1") + OLD_BRANCH=$(getvar "${UPPER}_BRANCH") + GIT_URL=$(getvar "${UPPER}_GIT_URL") + + remote_sha() { + git ls-remote "$GIT_URL" "refs/heads/$1" | awk '{print $1}' + } + + local STDLIB_BRANCH + if [ -n "$BRANCH_OVERRIDE" ]; then + STDLIB_BRANCH=$BRANCH_OVERRIDE + elif [[ $JULIA_BRANCH =~ ^(backports-)?(release-[0-9]+\.[0-9]+)$ ]]; then + STDLIB_BRANCH=${BASH_REMATCH[2]} + if [ -z "$(remote_sha "$STDLIB_BRANCH")" ]; then + echo "note: no '$STDLIB_BRANCH' branch at $GIT_URL, skipping $NAME (pass -b to bump anyway)" >&2 + return 0 + fi + else + STDLIB_BRANCH=$OLD_BRANCH + fi + + local NEW_SHA + NEW_SHA=$(remote_sha "$STDLIB_BRANCH") + if [ -z "$NEW_SHA" ]; then + echo "error: no '$STDLIB_BRANCH' branch at $GIT_URL" >&2 + return 1 + fi + + local OLD7=${OLD_SHA:0:7} + local NEW7=${NEW_SHA:0:7} + + if [ "$NEW_SHA" = "$OLD_SHA" ]; then + echo "$NAME is already at the latest commit of '$STDLIB_BRANCH' ($NEW7)" + return 0 + fi + + if [ -n "$(git -C "$JULIAHOME" status --porcelain -- "deps/checksums/$NAME-*" "stdlib/$NAME.version")" ]; then + echo "error: stdlib/$NAME.version or deps/checksums/$NAME-* has uncommitted changes, commit or stash them first" >&2 + return 1 + fi + + echo "Bumping $NAME: $OLD7 -> $NEW7 (branch $STDLIB_BRANCH)" + + # Generate checksums first (the make vars override the .version file), so + # a failed download leaves the repo unchanged + local NEW_CHECKSUM_DIR="$JULIAHOME/deps/checksums/$NAME-$NEW_SHA.tar.gz" + checksum_failed() { + rm -rf "$NEW_CHECKSUM_DIR" + echo "error: failed to generate checksums for $NAME at $NEW7" >&2 + return 1 + } + make -s -C "$JULIAHOME/stdlib" DEPS_GIT=0 \ + "${UPPER}_SHA1=$NEW_SHA" "${UPPER}_BRANCH=$STDLIB_BRANCH" \ + "checksum-$NAME" || { checksum_failed; return 1; } + for type in md5 sha512; do + [ -f "$NEW_CHECKSUM_DIR/$type" ] || { checksum_failed; return 1; } + done + + sed -i.bak -E "s|^(${UPPER}_SHA1[[:space:]]*:?=[[:space:]]*).*|\1${NEW_SHA}|" "$VERSION_FILE" + sed -i.bak -E "s|^(${UPPER}_BRANCH[[:space:]]*:?=[[:space:]]*).*|\1${STDLIB_BRANCH}|" "$VERSION_FILE" + rm -f "$VERSION_FILE.bak" + local f + for f in "$JULIAHOME/deps/checksums/$NAME-"*; do + [ "$f" = "$NEW_CHECKSUM_DIR" ] || rm -rf "$f" + done + + # Commit message extras, all optional (github.com URLs only) + local OWNER_REPO JULIA_VERSION STDLIB_VERSION GITLOG JQEXPR + OWNER_REPO=$(echo "$GIT_URL" | sed -nE 's|^https://github\.com/([^/]+/[^/]+)/?$|\1|p') + OWNER_REPO=${OWNER_REPO%.git} + JULIA_VERSION=$(cat "$JULIAHOME/VERSION") + STDLIB_VERSION="" + GITLOG="" + if [ -n "$OWNER_REPO" ]; then + STDLIB_VERSION=$(curl -fsSL "https://raw.githubusercontent.com/$OWNER_REPO/$NEW_SHA/Project.toml" 2>/dev/null | + sed -nE 's/^version[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' | head -n1 || true) + JQEXPR='.commits | reverse | .[] | "\(.sha[0:7]) \(.commit.message | split("\n")[0])"' + if command -v gh >/dev/null 2>&1; then + GITLOG=$(gh api "repos/$OWNER_REPO/compare/$OLD_SHA...$NEW_SHA" --jq "$JQEXPR" 2>/dev/null || true) + fi + if [ -z "$GITLOG" ] && command -v jq >/dev/null 2>&1; then + GITLOG=$(curl -fsSL "https://api.github.com/repos/$OWNER_REPO/compare/$OLD_SHA...$NEW_SHA" 2>/dev/null | + jq -r "$JQEXPR" || true) + fi + fi + + { + echo "[$JULIA_BRANCH] Bump $NAME stdlib $OLD7 → $NEW7" + echo + echo "Stdlib: $NAME" + echo "URL: $GIT_URL" + echo "Stdlib branch: $STDLIB_BRANCH" + echo "Julia branch: $JULIA_BRANCH" + echo "Old commit: $OLD7" + echo "New commit: $NEW7" + echo "Julia version: $JULIA_VERSION" + if [ -n "$STDLIB_VERSION" ]; then + if [ "$STDLIB_VERSION" = "$JULIA_VERSION" ]; then + echo "$NAME version: $STDLIB_VERSION" + else + echo "$NAME version: $STDLIB_VERSION (Does not match)" + fi + fi + if [ -n "$OWNER_REPO" ]; then + echo + echo "Diff:" + echo "https://github.com/$OWNER_REPO/compare/$OLD_SHA...$NEW_SHA" + fi + if [ -n "$GITLOG" ]; then + echo + echo '```' + echo "\$ git log --oneline $OLD7..$NEW7" + echo "$GITLOG" + echo '```' + fi + } > "$MSGFILE" + + git -C "$JULIAHOME" add -A -- "deps/checksums/$NAME-*" "stdlib/$NAME.version" || return 1 + git -C "$JULIAHOME" commit --quiet -F "$MSGFILE" -- "deps/checksums/$NAME-*" "stdlib/$NAME.version" || return 1 + git -C "$JULIAHOME" --no-pager log -1 --format="committed %h: %s" +} + +FAILED=() +for NAME in "${NAMES[@]}"; do + if [ ${#NAMES[@]} -gt 1 ]; then + echo + echo "=== $NAME ===" + fi + # a plain `bump_one || ...` would disable `set -e` inside the function + set +e + (set -e; bump_one "$NAME") + STATUS=$? + set -e + [ "$STATUS" -eq 0 ] || FAILED+=("$NAME") +done + +if [ ${#FAILED[@]} -gt 0 ]; then + echo + echo "error: failed to bump: ${FAILED[*]}" >&2 + exit 1 +fi diff --git a/contrib/check-whitespace.jl b/contrib/check-whitespace.jl index d7e04512e153d..e92642e41ade7 100755 --- a/contrib/check-whitespace.jl +++ b/contrib/check-whitespace.jl @@ -33,11 +33,14 @@ allow_tabs(path) = endswith(path, "test/triplequote.jl") function check_whitespace() - # Get file list from ARGS if provided, otherwise use git ls-files errors = Set{Tuple{String,Int,String}}() - files_to_check = filter(arg -> arg != "--fix", ARGS) + files_to_check = filter(arg -> !startswith(arg, "-"), ARGS) if isempty(files_to_check) - files_to_check = eachline(`git ls-files -- $patterns`) + if "--stdin" in ARGS + files_to_check = collect(eachline(stdin)) + else + files_to_check = collect(eachline(`git ls-files -- $patterns`)) + end end files_fixed = 0 diff --git a/contrib/ci-timing/Project.toml b/contrib/ci-timing/Project.toml new file mode 100644 index 0000000000000..a1bcab04c6e93 --- /dev/null +++ b/contrib/ci-timing/Project.toml @@ -0,0 +1,6 @@ +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" diff --git a/contrib/ci-timing/ci_timing_compare.jl b/contrib/ci-timing/ci_timing_compare.jl new file mode 100644 index 0000000000000..9488e51dfd383 --- /dev/null +++ b/contrib/ci-timing/ci_timing_compare.jl @@ -0,0 +1,135 @@ +#!/usr/bin/env julia +# +# Compare one Buildkite build's job durations against the last N days of Julia CI history. +# +# Usage (from the julia repo root): +# julia --project=contrib/ci-timing contrib/ci-timing +# [--days 7] [--include-failed] [--min-seconds 0] +# julia --project=contrib/ci-timing contrib/ci-timing --build julia-pr/853 [...] +# +# Prints a markdown table (one row per job, sorted slowest first) plus a TOTAL row. +# No Buildkite token needed: job data comes from the build page's public JSON +# endpoint, history from https://perf.julialang.org/data/timing_summary.json.gz +# (the JuliaCI/julia-ci-timing dataset, cached in the temp dir for 6h). +# Resolving a PR number to a build needs the `gh` CLI; --build does not. + +using Pkg +Pkg.instantiate() + +using JSON, Downloads, Dates, Statistics, Printf + +const HISTORY_URL = "https://perf.julialang.org/data/timing_summary.json.gz" +const CACHE = joinpath(tempdir(), "julia_timing_summary.json.gz") +const CACHE_MAX_AGE = 6 * 3600 + +getjson(url) = JSON.parse(sprint(io -> Downloads.download(url, io; + headers = ["Accept" => "application/json"]))) + +"Find the newest Buildkite build on a JuliaLang/julia PR, plus its title and sha." +function build_from_pr(pr) + out = read(`gh pr view $pr --repo JuliaLang/julia --json statusCheckRollup,title,headRefOid`, String) + data = JSON.parse(out) + builds = Set{Tuple{String, Int}}() + for c in data["statusCheckRollup"] + m = match(r"buildkite\.com/julialang/([\w-]+)/builds/(\d+)", something(get(c, "targetUrl", nothing), "")) + m === nothing || push!(builds, (m[1], parse(Int, m[2]))) + end + isempty(builds) && error("no Buildkite build found on PR #$pr") + pipeline, number = last(sort!(collect(builds); by = last)) + return pipeline, number, data["title"], data["headRefOid"][1:9] +end + +parsetime(s) = DateTime(s, dateformat"yyyy-mm-dd\THH:MM:SS.sssZ") + +"(name, duration, failed) for every finished job of a build, newest attempt only." +function fetch_jobs(pipeline, number) + records = getjson("https://buildkite.com/julialang/$pipeline/builds/$number/data/jobs")["records"] + jobs = Tuple{String, Float64, Bool}[] + for j in records + # skip unnamed steps, broken (never ran) and running jobs, and superseded retries + (get(j, "name", nothing) isa String && j["state"] == "finished" && + j["started_at"] isa String && j["finished_at"] isa String && + get(j, "retried_at", nothing) === nothing) || continue + dur = Dates.value(parsetime(j["finished_at"]) - parsetime(j["started_at"])) / 1000 + push!(jobs, (j["name"], dur, get(j, "exit_status", 1) != 0)) + end + return jobs +end + +function history() + if !isfile(CACHE) || time() - mtime(CACHE) > CACHE_MAX_AGE + Downloads.download(HISTORY_URL, CACHE) + end + return JSON.parse(read(pipeline(`gzip -dc $CACHE`), String))["jobs"] +end + +fmt(sec) = (m = round(Int, sec); @sprintf("%d:%02d", m ÷ 60, m % 60)) + +function main(args) + pr = nothing; build = nothing; days = 7; include_failed = false; min_seconds = 0.0 + i = 1 + while i <= length(args) + a = args[i] + if a == "--build"; build = args[i += 1] + elseif a == "--days"; days = parse(Int, args[i += 1]) + elseif a == "--min-seconds"; min_seconds = parse(Float64, args[i += 1]) + elseif a == "--include-failed"; include_failed = true + elseif !startswith(a, "-"); pr = parse(Int, a) + else error("unknown argument $a") + end + i += 1 + end + + title = sha = nothing + if build !== nothing + p, n = split(build, '/') + pipeline, number = String(p), parse(Int, n) + elseif pr !== nothing + pipeline, number, title, sha = build_from_pr(pr) + else + error("give a PR number or --build /") + end + + jobs = fetch_jobs(pipeline, number) + hist = history() + latest = maximum(DateTime(r["date"], dateformat"yyyy-mm-dd HH:MM") + for v in values(hist) for r in v["recent"]) + cut = latest - Day(days) + + rows = NamedTuple[]; failed_out = String[]; nobase = String[] + for (name, dur, failed) in jobs + if failed && !include_failed + push!(failed_out, name); continue + end + vals = Float64[r["duration"] for r in get(get(hist, name, Dict()), "recent", []) + if r["state"] == "passed" && r["duration"] !== nothing && + DateTime(r["date"], dateformat"yyyy-mm-dd HH:MM") >= cut] + if isempty(vals) + push!(nobase, name); continue + end + m = mean(vals) + m < min_seconds && continue + push!(rows, (; name = name * (failed ? " *(failed)*" : ""), dur, + mean = m, mn = minimum(vals), mx = maximum(vals), n = length(vals))) + end + sort!(rows; by = r -> -r.dur) + + label = pr === nothing ? "$pipeline#$number" : "PR #$pr" + title === nothing || println("") + println("| Job | $label | Mean | Min | Max | N | Δ vs mean |") + println("|---|---:|---:|---:|---:|---:|---:|") + for r in rows + flag = r.dur > r.mx ? " ⚠️" : r.dur < r.mn ? " ✅" : "" + @printf("| %s | %s | %s | %s | %s | %d | %+.1f%%%s |\n", r.name, fmt(r.dur), + fmt(r.mean), fmt(r.mn), fmt(r.mx), r.n, 100 * (r.dur - r.mean) / r.mean, flag) + end + tp = sum(r -> r.dur, rows); tm = sum(r -> r.mean, rows) + @printf("| **TOTAL (%d jobs)** | **%s** | **%s** | — | — | — | **%+.1f%%** |\n", + length(rows), fmt(tp), fmt(tm), 100 * (tp - tm) / tm) + isempty(failed_out) || println("") + isempty(nobase) || println("") +end + +main(ARGS) diff --git a/contrib/download_cmake.sh b/contrib/download_cmake.sh index 5cf3c579ed052..0142526c7253d 100755 --- a/contrib/download_cmake.sh +++ b/contrib/download_cmake.sh @@ -8,17 +8,17 @@ mkdir -p "$(dirname "$0")"/../deps/scratch cd "$(dirname "$0")"/../deps/scratch CMAKE_VERSION_MAJOR=3 -CMAKE_VERSION_MINOR=30 -CMAKE_VERSION_PATCH=1 +CMAKE_VERSION_MINOR=31 +CMAKE_VERSION_PATCH=6 CMAKE_VERSION_MAJMIN=$CMAKE_VERSION_MAJOR.$CMAKE_VERSION_MINOR CMAKE_VERSION=$CMAKE_VERSION_MAJMIN.$CMAKE_VERSION_PATCH # listed at https://cmake.org/files/v$CMAKE_VERSION_MAJMIN/cmake-$CMAKE_VERSION-SHA-256.txt # for the files cmake-$CMAKE_VERSION-macos-universal.tar.gz # cmake-$CMAKE_VERSION-Linux-x86_64.tar.gz and cmake-$CMAKE_VERSION-Linux-aarch64.tar.gz -CMAKE_SHA256_DARWIN=51e12618829b811bba6f033ee8f39f6192da1b6abb20d82a7899d5134e879a4c -CMAKE_SHA256_LINUX_X86_64=ac31f077ef3378641fa25a3cb980d21b2f083982d3149a8f2eb9154f2b53696b -CMAKE_SHA256_LINUX_AARCH64=ad234996f8750f11d7bd0d17b03f55c434816adf1f1671aab9e8bab21a43286a +CMAKE_SHA256_DARWIN=330b9514f5112e5ed4fb08b8b05803b776fd9b539a6ae12927d14dcc0ee2ba8d +CMAKE_SHA256_LINUX_X86_64=5a1133ff103c71eb5120e2cc3de922733e7d8a26a98ae716397e8676adb367bf +CMAKE_SHA256_LINUX_AARCH64=b4cc788d63112b2749b40627e719eb5d3b8ed8f00c36d77189f4019cfe64bc9e PLATFORM="$(uname)-$(uname -m)" case $PLATFORM in diff --git a/contrib/escape_json.sh b/contrib/escape_json.sh new file mode 100755 index 0000000000000..fd7c4c28bd971 --- /dev/null +++ b/contrib/escape_json.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# print arguments escaped as json list elements + +first=true +for n in "$@"; do + $first && first=false || printf ', ' + n=$(printf '%s' "$n" | sed 's/\\/\\\\/g' | sed 's/"/\\"/g') + printf '"%s"' "$n" +done diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 578e27110f436..64824d8879481 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -15,24 +15,34 @@ end using Base.Meta ## Debugging options -# Disable parallel precompiles generation by setting `false` -const PARALLEL_PRECOMPILATION = true - -# View the code sent to the repl by setting this to `stdout` +# View the code sent to the subprocesses by setting this to `stdout` const debug_output = devnull # or stdout - -# Disable fancy printing -const fancyprint = (stdout isa Base.TTY) && Base.get_bool_env("CI", false) !== true ## -CTRL_C = '\x03' -CTRL_R = '\x12' -UP_ARROW = "\e[A" -DOWN_ARROW = "\e[B" +if Sys.isunix() + include(joinpath(@__DIR__, "..", "test", "testhelpers", "FakePTYs.jl")) + import .FakePTYs: open_fake_pty +end hardcoded_precompile_statements = """ precompile(Base.unsafe_string, (Ptr{UInt8},)) precompile(Base.unsafe_string, (Ptr{Int8},)) +precompile(Base._atexit, (Cint,)) + +# used by REPL +precompile(Tuple{typeof(Base.getproperty), Base.Terminals.TTYTerminal, Symbol}) +precompile(Tuple{typeof(Base.reseteof), Base.Terminals.TTYTerminal}) +precompile(Tuple{typeof(Base.Terminals.enable_bracketed_paste), Base.Terminals.TTYTerminal}) +precompile(Tuple{typeof(Base.Terminals.width), Base.Terminals.TTYTerminal}) +precompile(Tuple{typeof(Base.Terminals.height), Base.Terminals.TTYTerminal}) +precompile(Tuple{typeof(Base.write), Base.Terminals.TTYTerminal, Array{UInt8, 1}}) +precompile(Tuple{typeof(Base.isempty), Base.AnnotatedString{String}}) +# The owner-carrying write paths (String/Array) call _unsafe_write_owned +# directly, so the workload session does not compile the raw-pointer +# unsafe_write entry points on its own; interactive startup still reaches +# them through generic IO plumbing +precompile(Tuple{typeof(Base.unsafe_write), Base.TTY, Ptr{UInt8}, UInt}) +precompile(Tuple{typeof(Base.unsafe_write), Base.GenericIOBuffer{Memory{UInt8}}, Ptr{UInt8}, UInt}) # loading.jl - without these each precompile worker would precompile these because they're hit before pkgimages are loaded precompile(Base.__require, (Module, Symbol)) @@ -43,12 +53,13 @@ precompile(Tuple{typeof(Base.Threads.atomic_add!), Base.Threads.Atomic{Int}, Int precompile(Tuple{typeof(Base.Threads.atomic_sub!), Base.Threads.Atomic{Int}, Int}) precompile(Tuple{Type{Pair{A, B} where B where A}, Base.PkgId, UInt128}) precompile(Tuple{typeof(Base.in!), Tuple{Module, String, UInt64, UInt32, Float64}, Base.Set{Any}}) -precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf_result), Tuple{Bool, Nothing}}, typeof(Base.Compiler.handle_match!), Array{Base.Compiler.InliningCase, 1}, Core.MethodMatch, Array{Any, 1}, Base.Compiler.CallInfo, UInt32, Base.Compiler.InliningState{Base.Compiler.NativeInterpreter}}) precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) -precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf_result), Tuple{Bool, Base.Compiler.VolatileInferenceResult}}, typeof(Base.Compiler.handle_match!), Array{Base.Compiler.InliningCase, 1}, Core.MethodMatch, Array{Any, 1}, Base.Compiler.CallInfo, UInt32, Base.Compiler.InliningState{Base.Compiler.NativeInterpreter}}) precompile(Tuple{typeof(Base.getindex), Type{Pair{Base.PkgId, UInt128}}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Vararg{Pair{Base.PkgId, UInt128}}}) precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}, Core.SimpleVector}) -precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) +precompile(Base.include_package_for_output, (Base.PkgId, String, VersionNumber, Vector{String}, Vector{String}, Vector{String}, typeof(Base._concrete_dependencies), Nothing)) +precompile(Base.include_package_for_output, (Base.PkgId, String, VersionNumber, Vector{String}, Vector{String}, Vector{String}, typeof(Base._concrete_dependencies), String)) +precompile(Base.create_expr_cache, (Base.PkgId, Base.PkgLoadSpec, String, String, typeof(Base._concrete_dependencies), Cmd, Base.CacheFlags, IO, IO)) +precompile(Base.create_expr_cache, (Base.PkgId, Base.PkgLoadSpec, String, Nothing, typeof(Base._concrete_dependencies), Cmd, Base.CacheFlags, IO, IO)) # LazyArtifacts (but more generally helpful) precompile(Tuple{Type{Base.Val{x} where x}, Module}) @@ -69,6 +80,7 @@ precompile(Base.check_open, (Base.TTY,)) precompile(Base.getproperty, (Base.TTY, Symbol)) precompile(write, (Base.TTY, String)) precompile(Tuple{typeof(Base.get), Base.TTY, Symbol, Bool}) +precompile(Tuple{typeof(Base.eof), Base.TTY}) precompile(Tuple{typeof(Base.hashindex), String, Int}) precompile(Tuple{typeof(Base.write), Base.GenericIOBuffer{Array{UInt8, 1}}, String}) precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Nothing, Int}, Int}) @@ -79,6 +91,9 @@ precompile(Tuple{typeof(Base.promoteK), Type, Base.Dict{String, Any}}) precompile(Tuple{typeof(Base.promoteV), Type, Base.Dict{String, Any}, Base.Dict{String, Any}}) precompile(Tuple{typeof(Base.eval_user_input), Base.PipeEndpoint, Any, Bool}) precompile(Tuple{typeof(Base.get), Base.PipeEndpoint, Symbol, Bool}) +precompile(Tuple{Core.TypeEgal{Base.IOContext{IO_t} where IO_t<:IO}, Base.PipeEndpoint, Pair{Symbol, Bool}}) +precompile(Tuple{typeof(Base.print), Base.IOContext{Base.PipeEndpoint}, String}) +precompile(Tuple{typeof(Base.HashArrayMappedTries.next), Base.HashArrayMappedTries.HashState{Base.ScopedValues.ScopedValue{Any}}}) # used by Revise.jl precompile(Tuple{typeof(Base.parse_cache_header), String}) @@ -94,11 +109,16 @@ precompile(Tuple{typeof(push!), Vector{Function}, Function}) precompile(Base.get_preferences, (Base.UUID,)) precompile(Base.record_compiletime_preference, (Base.UUID, String)) +# Threads.@threads +# threading_run ends with filter!(istaskfailed, tasks), whose sizehint! call pulls the +# @noinline _growbeg_internal! into the compile unit of whoever runs the first threaded region +precompile(Tuple{typeof(Base.sizehint!), Vector{Task}, Int}) +precompile(Tuple{typeof(Base._growbeg_internal!), Vector{Task}, Int, Int}) + # miscellaneous precompile(Tuple{typeof(Base.exit)}) precompile(Tuple{typeof(Base.require), Base.PkgId}) -precompile(Tuple{typeof(Base.recursive_prefs_merge), Base.Dict{String, Any}}) -precompile(Tuple{typeof(Base.recursive_prefs_merge), Base.Dict{String, Any}, Base.Dict{String, Any}, Vararg{Base.Dict{String, Any}}}) +precompile(Tuple{typeof(Base.recursive_prefs_merge), Base.Dict{String, Any}, Vector{Base.Dict{String, Any}}}) precompile(Tuple{typeof(Base.hashindex), Tuple{Base.PkgId, Nothing}, Int}) precompile(Tuple{typeof(Base.hashindex), Tuple{Base.PkgId, String}, Int}) precompile(Tuple{typeof(isassigned), Core.SimpleVector, Int}) @@ -134,9 +154,12 @@ precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, precompile(Tuple{Type{Base.UUID}, Base.UUID}) """ -for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base.TTY, IOContext{Base.TTY}) - global hardcoded_precompile_statements - hardcoded_precompile_statements *= "precompile(Tuple{typeof(show), $IO, $T})\n" +if Sys.iswindows() + # on Unix these are covered (with the real terminal types) by `tty_script` below + for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base.TTY, IOContext{Base.TTY}) + global hardcoded_precompile_statements + hardcoded_precompile_statements *= "precompile(Tuple{typeof(show), $IO, $T})\n" + end end # Precompiles for Revise and other packages @@ -200,6 +223,22 @@ write(IOBuffer(), "") @time @eval Base.Experimental.@force_compile """ +# Runs in a subprocess with stdout/stderr attached to a pty, so that the traced +# signatures use the concrete `Base.TTY` types seen in a terminal session +# (tracing these in the pipe-attached process above would compile the wrong +# specializations). Ends by throwing, to trace the error-report path for an +# uncaught exception in a script. +tty_script = """ +for x in (Float16(1.0), 1.0f0, 1.0) + show(stdout, x); println() + show(IOContext(stdout, :compact => true), x); println() + buf = IOBuffer() + show(buf, x) + show(IOContext(buf, :compact => true), x) +end +throw(InterruptException()) +""" + julia_exepath() = joinpath(Sys.BINDIR, Base.julia_exename()) Artifacts = get(Base.loaded_modules, @@ -247,81 +286,26 @@ if Libdl !== nothing """ end -# Printing the current state -let - global print_state - print_lk = ReentrantLock() - status = Dict{String, String}( - "step1" => "W", - "step3" => "W", - "clock" => "◐", - ) - function print_status(key::String) - txt = status[key] - if startswith(txt, "W") # Waiting - printstyled("? ", color=Base.warn_color()); print(txt[2:end]) - elseif startswith(txt, "R") # Running - print(status["clock"], " ", txt[2:end]) - elseif startswith(txt, "F") # Finished - printstyled("✓ ", color=:green); print(txt[2:end]) - else - print(txt) - end - end - function print_state(args::Pair{String,String}...) - lock(print_lk) do - isempty(args) || push!(status, args...) - print("\r└ Collect (Basic: ") - print_status("step1") - print(") => Execute ") - print_status("step3") - end - end -end - -ansi_enablecursor = "\e[?25h" -ansi_disablecursor = "\e[?25l" -blackhole = Sys.isunix() ? "/dev/null" : "nul" procenv = Dict{String,Any}( - "JULIA_HISTORY" => blackhole, "JULIA_LOAD_PATH" => "@$(Sys.iswindows() ? ";" : ":")@stdlib", "JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":", "TERM" => "", - # "JULIA_DEBUG" => "precompilation", + # a piped-stdin child runs the REPL machinery; the REPL stdlib's pkgimage + # does not exist yet at sysimage-build time, so it must not be loaded "JULIA_FALLBACK_REPL" => "true") -generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed +function generate_precompile_statements() start_time = time_ns() sysimg = Base.unsafe_string(Base.JLOptions().image_file) - # Extract the precompile statements from the precompile file - statements_step1 = Channel{String}(Inf) - - # From hardcoded statements - for statement in split(hardcoded_precompile_statements::String, '\n') - push!(statements_step1, statement) - end + statements = String[] + append!(statements, split(hardcoded_precompile_statements::String, '\n')) - println("Collecting and executing precompile statements") - fancyprint && print(ansi_disablecursor) - print_state() - clock = @async begin - t = Timer(0; interval=1/10) - anim_chars = ["◐","◓","◑","◒"] - current = 1 - if fancyprint - while isopen(statements_step1) || !isempty(statements_step1) - print_state("clock" => anim_chars[current]) - wait(t) - current = current == 4 ? 1 : current + 1 - end - end - close(t) - end - - # Collect statements from running the script - step1 = @async mktempdir() do prec_path - print_state("step1" => "R") + # Collect statements from running the script in a fresh process: only such a + # process compiles (and therefore traces) the specializations that startup and + # code loading hit, since this process has already run them. + println("Collecting precompile statements") + mktempdir() do prec_path # Also precompile a package here pkgname = "__PackagePrecompilationStatementModule" pkguuid = "824efdaf-a0e9-431c-8ee7-3d356b2531c2" @@ -339,30 +323,57 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe uuid = "$pkguuid" """) touch(joinpath(pkgpath, "Manifest.toml")) - tmp_prec = tempname(prec_path) - tmp_proc = tempname(prec_path) + tmp_prec = tempname(prec_path; cleanup=false) + tmp_proc = tempname(prec_path; cleanup=false) + tmp_tty = tempname(prec_path; cleanup=false) s = """ pushfirst!(DEPOT_PATH, $(repr(joinpath(prec_path,"depot")))); Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp_prec)); Base.Precompilation.precompilepkgs(;fancyprint=true); $precompile_script """ - p = run(pipeline(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg + run(pipeline(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg --cpu-target=native --startup-file=no --color=yes --project=$(pkgpath)`, procenv), stdin=IOBuffer(s), stderr=debug_output, stdout=debug_output)) - n_step1 = 0 - for f in (tmp_prec, tmp_proc) - isfile(f) || continue - for statement in split(read(f, String), '\n') - push!(statements_step1, statement) - n_step1 += 1 + if Sys.isunix() + try + script = joinpath(prec_path, "tty_script.jl") + write(script, tty_script) + pts, ptm = open_fake_pty() + outbuf = IOBuffer() + drain = @async try + while !eof(ptm) + write(outbuf, readavailable(ptm)) + end + catch # ignore EIO when the child exits + end + p = run(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_tty --sysimage $sysimg + --cpu-target=native --startup-file=no --color=yes $script`, procenv), + devnull, pts, pts; wait=false) + Base.close_stdio(pts) + wait(p) + wait(drain) + close(ptm) + output = String(take!(outbuf)) + # the script must reach its final `throw` for the error path to be traced + occursin("InterruptException", output) || + error("unexpected output from the pty-attached process:\n$output") + catch ex + @warn "Failed to collect precompile statements from the pty-attached process" exception=(ex, catch_backtrace()) + end + end + for (name, f) in (("package precompilation", tmp_prec), ("script", tmp_proc)) + n = isfile(f) ? countlines(f) : 0 + if n == 0 + # in a real sysimage build (bare sysimage) both processes trace plenty; + # when run standalone against a finished sysimage they may trace nothing + msg = "no precompile statements were traced from the $name process" + Base.get_bool_env("CI", false) ? error(msg) : @warn(msg) end + n > 0 && append!(statements, eachline(f)) end - close(statements_step1) - print_state("step1" => "F$n_step1") - return :ok + isfile(tmp_tty) && append!(statements, eachline(tmp_tty)) end - PARALLEL_PRECOMPILATION ? bind(statements_step1, step1) : wait(step1) # Create a staging area where all the loaded packages are available PrecompileStagingArea = Module() @@ -373,59 +384,40 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe end Core.eval(PrecompileStagingArea, :(const Compiler = Base.Compiler)) + println("Executing precompile statements") n_succeeded = 0 - # Make statements unique - statements = Set{String}() - # Execute the precompile statements - for statement in statements_step1 + executed = Set{String}() + for statement in statements + statement = strip(statement) + # the hardcoded statements are written as a commented block + (isempty(statement) || startswith(statement, '#')) && continue # Main should be completely clean occursin("Main.", statement) && continue - Base.in!(statement, statements) && continue - # println(statement) + Base.in!(statement, executed) && continue try ps = Meta.parse(statement) - if !isexpr(ps, :call) - # these are typically comments - @debug "skipping statement because it does not parse as an expression" statement - delete!(statements, statement) - continue - end + # a malformed statement used to be skipped silently here, allowing + # hardcoded statements to rot unnoticed; make it loud instead + isexpr(ps, :call) || error("statement does not parse to a call") popfirst!(ps.args) # precompile(...) ps.head = :tuple - # println(ps) ps = Core.eval(PrecompileStagingArea, ps) - if precompile(ps...) - n_succeeded += 1 - else - Base.get_bool_env("CI", false) && error("Precompilation failed for $statement") - @warn "Failed to precompile expression" form=statement _module=nothing _file=nothing _line=0 - end - failed = length(statements) - n_succeeded - yield() # Make clock spinning - print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : "")) + precompile(ps...) || error("precompile returned false") + n_succeeded += 1 catch ex # See #28808 Base.get_bool_env("CI", false) && error("Precompilation failed for $statement") @warn "Failed to precompile expression" form=statement exception=(ex,catch_backtrace()) _module=nothing _file=nothing _line=0 end end - wait(clock) # Stop asynchronous printing - failed = length(statements) - n_succeeded - print_state("step3" => string("F$n_succeeded", failed > 0 ? " ($failed failed)" : "")) - println() - # Seems like a reasonable number right now, adjust as needed - # comment out if debugging script - have_repl = false - n_succeeded > (have_repl ? 650 : 90) || @warn "Only $n_succeeded precompile statements" - - fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.") + failed = length(executed) - n_succeeded + println("Executed $n_succeeded precompile statements", failed > 0 ? " ($failed failed)" : "") tot_time = time_ns() - start_time println("Precompilation complete. Summary:") print("Total ─────── "); Base.time_print(stdout, tot_time); println() -finally - fancyprint && print(ansi_enablecursor) GC.gc(true); GC.gc(false); # reduce memory footprint + return end generate_precompile_statements() diff --git a/contrib/github_source_release.sh b/contrib/github_source_release.sh new file mode 100755 index 0000000000000..ca135e7cbc1cb --- /dev/null +++ b/contrib/github_source_release.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# This file is a part of Julia. License is MIT: https://julialang.org/license +# +# Attach the CI-signed source dists to a release's GitHub release. +# +# The pipeline builds, KMS-signs, and publishes the source tarballs +# (julia-buildkite#593): julia-publish puts them at bin/src// in +# the nightlies bucket and julia-promote copies them to the release +# bucket. This script does the last step no pipeline does: download the +# four files from the release CDN, verify the GPG signatures against the +# published keyring and the in-tarball julia-/ prefix, and +# attach them to the GitHub release (creating it as a prerelease if it +# does not exist). +# +# No signing happens here; the .asc files were produced in CI by the KMS +# key. Nothing secret is needed beyond `gh auth` with release write +# access to JuliaLang/julia. +# +# Usage: ./github_source_release.sh 1.13.0-rc2 [--execute] +# Dry-run by default: downloads and verifies, then only prints the gh +# commands it would run. +set -euo pipefail + +VERSION="${1:?usage: $0 [--execute]}" +VERSION="${VERSION#v}" +TAG="v${VERSION}" +MAJMIN="$(cut -d. -f1-2 <<<"${VERSION}")" +EXECUTE="${2:-}" + +CDN="${CDN:-https://julialang-s3.julialang.org}" +REPO="${REPO:-JuliaLang/julia}" +KEYRING_URL="${KEYRING_URL:-https://raw.githubusercontent.com/JuliaCI/julia-buildkite/main/signing-pubkeys/tarball_signing.pub.asc}" + +FILES=( + "julia-${VERSION}.tar.gz" + "julia-${VERSION}.tar.gz.asc" + "julia-${VERSION}-full.tar.gz" + "julia-${VERSION}-full.tar.gz.asc" +) + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT +cd "${WORK}" + +echo "--- Download source dists from ${CDN}/bin/src/${MAJMIN}/" +for f in "${FILES[@]}"; do + curl -fSsLO "${CDN}/bin/src/${MAJMIN}/${f}" +done +if command -v sha256sum >/dev/null; then + sha256sum ./*.tar.gz +else + shasum -a 256 ./*.tar.gz +fi + +echo "--- Verify GPG signatures (keyring: ${KEYRING_URL})" +export GNUPGHOME="${WORK}/gnupg" +mkdir -m 700 "${GNUPGHOME}" +curl -fSsL "${KEYRING_URL}" | gpg --quiet --import +for f in julia-"${VERSION}".tar.gz julia-"${VERSION}"-full.tar.gz; do + gpg --verify "${f}.asc" "${f}" +done + +echo "--- Verify in-tarball prefix is julia-${VERSION}/" +for f in julia-"${VERSION}".tar.gz julia-"${VERSION}"-full.tar.gz; do + first="$( (tar -tzf "${f}" || true) | head -1)" + if [[ "${first}" != "julia-${VERSION}/"* ]]; then + echo "ERROR: ${f} first entry is '${first}'" >&2 + exit 1 + fi +done + +PRERELEASE_ARGS=() +[[ "${VERSION}" == *-* ]] && PRERELEASE_ARGS=( --prerelease ) +NOTES="See [NEWS.md](https://github.com/${REPO}/blob/${TAG}/NEWS.md) for what will be new in ${MAJMIN}." +if [[ "${VERSION}" =~ -rc([0-9]+)$ ]]; then + NOTES="Release candidate ${BASH_REMATCH[1]} for the upcoming ${MAJMIN} release. ${NOTES}" +fi + +if ! gh release view "${TAG}" --repo "${REPO}" >/dev/null 2>&1; then + CREATE=( gh release create "${TAG}" --repo "${REPO}" --verify-tag + "${PRERELEASE_ARGS[@]}" --title "${TAG}" --notes "${NOTES}" ) +else + echo "Release ${TAG} already exists; will only upload assets." + CREATE=() +fi +UPLOAD=( gh release upload "${TAG}" --repo "${REPO}" --clobber "${FILES[@]}" ) + +if [[ "${EXECUTE}" == "--execute" ]]; then + [[ "${#CREATE[@]}" -gt 0 ]] && "${CREATE[@]}" + "${UPLOAD[@]}" + echo "+++ Uploaded: https://github.com/${REPO}/releases/tag/${TAG}" +else + echo "--- DRY RUN, would execute:" + [[ "${#CREATE[@]}" -gt 0 ]] && printf ' %q ' "${CREATE[@]}" && echo + printf ' %q ' "${UPLOAD[@]}" && echo + echo "Re-run with --execute to do it." +fi diff --git a/contrib/juliac/Artifacts.toml b/contrib/juliac/Artifacts.toml deleted file mode 100644 index 54771b41b21f7..0000000000000 --- a/contrib/juliac/Artifacts.toml +++ /dev/null @@ -1,19 +0,0 @@ -[[mingw-w64]] -arch = "x86_64" -git-tree-sha1 = "b17bda08a19173572926f43a48aad5ef3d845e7c" -os = "windows" -lazy = true - - [[mingw-w64.download]] - sha256 = "53645e06775a55733580426341395c67dda20a664af83bcda76a1d052b618b59" - url = "https://github.com/JuliaLang/PackageCompiler.jl/releases/download/v2.1.24/x86_64-14.2.0-release-posix-seh-msvcrt-rt_v12-rev0.tar.gz" - -[[mingw-w64]] -arch = "i686" -git-tree-sha1 = "76b9f278e7de1d7dfdfe3a786afbe9c1e29003ea" -os = "windows" -lazy = true - - [[mingw-w64.download]] - sha256 = "d049bd771e01b02f2ca9274435f0e6f9f4f295bf2af72a8059dd851c52144910" - url = "https://github.com/JuliaLang/PackageCompiler.jl/releases/download/v2.1.24/i686-14.2.0-release-posix-dwarf-msvcrt-rt_v12-rev0.tar.gz" diff --git a/contrib/juliac/juliac-buildscript.jl b/contrib/juliac/juliac-buildscript.jl deleted file mode 100644 index 80a3fd756bcd0..0000000000000 --- a/contrib/juliac/juliac-buildscript.jl +++ /dev/null @@ -1,100 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# Script to run in the process that generates juliac's object file output - -# Run the verifier in the current world (before modifications), so that error -# messages and types print in their usual way. -Core.Compiler._verify_trim_world_age[] = Base.get_world_counter() - -# Initialize some things not usually initialized when output is requested -Sys.__init__() -Base.init_depot_path() -Base.init_load_path() -Base.init_active_project() -task = current_task() -task.rngState0 = 0x5156087469e170ab -task.rngState1 = 0x7431eaead385992c -task.rngState2 = 0x503e1d32781c2608 -task.rngState3 = 0x3a77f7189200c20b -task.rngState4 = 0x5502376d099035ae -uuid_tuple = (UInt64(0), UInt64(0)) -ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) -if Base.get_bool_env("JULIA_USE_FLISP_PARSER", false) === false - Base.JuliaSyntax.enable_in_core!() -end - -if Base.JLOptions().trim != 0 - include(joinpath(@__DIR__, "juliac-trim-base.jl")) -end - -# Load user code - -import Base.Experimental.entrypoint - -# for use as C main if needed -function _main(argc::Cint, argv::Ptr{Ptr{Cchar}})::Cint - args = ccall(:jl_set_ARGS, Any, (Cint, Ptr{Ptr{Cchar}}), argc, argv)::Vector{String} - return Main.main(args) -end - -let include_result = Base.include(Main, ARGS[1]) - Core.@latestworld - if ARGS[2] == "--output-exe" - have_cmain = false - if isdefined(Main, :main) - for m in methods(Main.main) - if isdefined(m, :ccallable) - # TODO: possibly check signature and return type - have_cmain = true - break - end - end - elseif include_result isa Module && isdefined(include_result, :main) - error(""" - The `main` function must be defined in `Main`. If you are defining it inside a - module, try adding `import .$(nameof(include_result)).main` to $(ARGS[1]). - """) - end - if !have_cmain - if Base.should_use_main_entrypoint() - if hasmethod(Main.main, Tuple{Vector{String}}) - entrypoint(_main, (Cint, Ptr{Ptr{Cchar}})) - Base._ccallable("main", Cint, Tuple{typeof(_main), Cint, Ptr{Ptr{Cchar}}}) - else - error("`@main` must accept a `Vector{String}` argument.") - end - else - error("To generate an executable a `@main` function must be defined in the `Main` module.") - end - end - end - #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{Base.SubString{String}, 1}, String)) - #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char)) - entrypoint(Base.task_done_hook, (Task,)) - entrypoint(Base.wait, ()) - entrypoint(Base.wait_forever, ()) - entrypoint(Base.trypoptask, (Base.StickyWorkqueue,)) - entrypoint(Base.checktaskempty, ()) - if ARGS[3] == "true" - ccall(:jl_add_ccallable_entrypoints, Cvoid, ()) - end -end - -if Base.JLOptions().trim != 0 - include(joinpath(@__DIR__, "juliac-trim-stdlib.jl")) -end - -empty!(Core.ARGS) -empty!(Base.ARGS) -empty!(LOAD_PATH) -empty!(DEPOT_PATH) -empty!(Base.TOML_CACHE.d) -Base.TOML.reinit!(Base.TOML_CACHE.p, "") -Base.ACTIVE_PROJECT[] = nothing -@eval Base begin - PROGRAM_FILE = "" -end -@eval Sys begin - BINDIR = "" - STDLIB = "" -end diff --git a/contrib/juliac/juliac-trim-base.jl b/contrib/juliac/juliac-trim-base.jl deleted file mode 100644 index 1d4d9558e0bf5..0000000000000 --- a/contrib/juliac/juliac-trim-base.jl +++ /dev/null @@ -1,111 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# Patches to Base needed for trimming - -@eval Core begin - DomainError(@nospecialize(val), @nospecialize(msg::AbstractString)) = (@noinline; $(Expr(:new, :DomainError, :val, :msg))) -end - -(f::Base.RedirectStdStream)(io::Core.CoreSTDOUT) = Base._redirect_io_global(io, f.unix_fd) - -@eval Base begin - depwarn(msg, funcsym; force::Bool=false) = nothing - _assert_tostring(msg) = "" - reinit_stdio() = nothing - JuliaSyntax.enable_in_core!() = nothing - init_active_project() = ACTIVE_PROJECT[] = nothing - set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile - disable_library_threading() = nothing - start_profile_listener() = nothing - invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) - const invokelatest = invokelatest_trimmed - function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} - s = IOBuffer(sizehint=sizehint) - if context isa Tuple - f(IOContext(s, context...), args...) - elseif context !== nothing - f(IOContext(s, context), args...) - else - f(s, args...) - end - String(_unsafe_take!(s)) - end - function show_typeish(io::IO, @nospecialize(T)) - if T isa Type - show(io, T) - elseif T isa TypeVar - print(io, (T::TypeVar).name) - else - print(io, "?") - end - end - function show(io::IO, T::Type) - if T isa DataType - print(io, T.name.name) - if T !== T.name.wrapper && length(T.parameters) > 0 - print(io, "{") - first = true - for p in T.parameters - if !first - print(io, ", ") - end - first = false - if p isa Int - show(io, p) - elseif p isa Type - show(io, p) - elseif p isa Symbol - print(io, ":") - print(io, p) - elseif p isa TypeVar - print(io, p.name) - else - print(io, "?") - end - end - print(io, "}") - end - elseif T isa Union - print(io, "Union{") - show_typeish(io, T.a) - print(io, ", ") - show_typeish(io, T.b) - print(io, "}") - elseif T isa UnionAll - print(io, T.body::Type) - print(io, " where ") - print(io, T.var.name) - end - end - show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) -end -@eval Base.Sys begin - __init_build() = nothing # VersionNumber parsing is not supported yet -end -@eval Base.GMP begin - function __init__() # VersionNumber parsing is not supported yet - try - ccall((:__gmp_set_memory_functions, libgmp), Cvoid, - (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), - cglobal(:jl_gc_counted_malloc), - cglobal(:jl_gc_counted_realloc_with_old_size), - cglobal(:jl_gc_counted_free_with_size)) - ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL - ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) - catch ex - Base.showerror_nostdio(ex, "WARNING: Error during initialization of module GMP") - end - # This only works with a patched version of GMP, ignore otherwise - try - ccall((:__gmp_set_alloc_overflow_function, libgmp), Cvoid, - (Ptr{Cvoid},), - cglobal(:jl_throw_out_of_memory_error)) - ALLOC_OVERFLOW_FUNCTION[] = true - catch ex - # ErrorException("ccall: could not find function...") - if typeof(ex) != ErrorException - rethrow() - end - end - end -end diff --git a/contrib/juliac/juliac-trim-stdlib.jl b/contrib/juliac/juliac-trim-stdlib.jl deleted file mode 100644 index 0cc3f01aa92f8..0000000000000 --- a/contrib/juliac/juliac-trim-stdlib.jl +++ /dev/null @@ -1,83 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# Patches to stdlib needed for trimming - -let - find_loaded_root_module(key::Base.PkgId) = Base.maybe_root_module(key) - - SparseArrays = find_loaded_root_module(Base.PkgId( - Base.UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf"), "SparseArrays")) - if SparseArrays !== nothing - @eval SparseArrays.CHOLMOD begin - function __init__() - ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_malloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_calloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_calloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_realloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_realloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_free_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_free, Ptr{Cvoid})) - end - end - end - - Artifacts = find_loaded_root_module(Base.PkgId( - Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts")) - if Artifacts !== nothing - @eval Artifacts begin - function _artifact_str( - __module__, - artifacts_toml, - name, - path_tail, - artifact_dict, - hash, - platform, - _::Val{LazyArtifacts} - ) where LazyArtifacts - # If the artifact exists, we're in the happy path and we can immediately - # return the path to the artifact: - dirs = artifacts_dirs(bytes2hex(hash.bytes)) - for dir in dirs - if isdir(dir) - return jointail(dir, path_tail) - end - end - error("Artifact not found") - end - end - end - - Pkg = find_loaded_root_module(Base.PkgId( - Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) - if Pkg !== nothing - @eval Pkg begin - __init__() = nothing # Assume the Pkg is not actually used - end - end - - StyledStrings = find_loaded_root_module(Base.PkgId( - Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) - if StyledStrings !== nothing - @eval StyledStrings begin - __init__() = nothing # Assume that StyledStrings are not actually used - end - end - - Markdown = find_loaded_root_module(Base.PkgId( - Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) - if Markdown !== nothing - @eval Markdown begin - __init__() = nothing # Assume that Markdown is not actually used with StyledStrings - end - end - - JuliaSyntaxHighlighting = find_loaded_root_module(Base.PkgId( - Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) - if JuliaSyntaxHighlighting !== nothing - @eval JuliaSyntaxHighlighting begin - __init__() = nothing # Assume the JuliaSyntaxHighlighting is not actually used with StyledStrings - end - end -end diff --git a/contrib/juliac/juliac.jl b/contrib/juliac/juliac.jl deleted file mode 100644 index eb6277785c789..0000000000000 --- a/contrib/juliac/juliac.jl +++ /dev/null @@ -1,211 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# Julia compiler wrapper script -# NOTE: The interface and location of this script are considered unstable/experimental - -using LazyArtifacts - -module JuliaConfig - include(joinpath(@__DIR__, "..", "julia-config.jl")) -end - -julia_cmd = `$(Base.julia_cmd()) --startup-file=no --history-file=no` -cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) -julia_cmd_target = `$(Base.julia_cmd(;cpu_target)) --startup-file=no --history-file=no` -output_type = nothing # exe, sharedlib, sysimage -outname = nothing -file = nothing -add_ccallables = false -relative_rpath = false -verbose = false - -help = findfirst(x->x == "--help", ARGS) -if help !== nothing - println( - """ - Usage: julia juliac.jl [--output-exe | --output-lib | --output-sysimage] [options] - --experimental --trim= Only output code statically determined to be reachable - --compile-ccallable Include all methods marked `@ccallable` in output - --relative-rpath Configure the library / executable to lookup all required libraries in an adjacent "julia/" folder - --verbose Request verbose output - """) - exit(0) -end - -# Copied from PackageCompiler -# https://github.com/JuliaLang/PackageCompiler.jl/blob/1c35331d8ef81494f054bbc71214811253101993/src/PackageCompiler.jl#L147-L190 -function get_compiler_cmd(; cplusplus::Bool=false) - cc = get(ENV, "JULIA_CC", nothing) - path = nothing - @static if Sys.iswindows() - path = joinpath(LazyArtifacts.artifact"mingw-w64", - "extracted_files", - (Int==Int64 ? "mingw64" : "mingw32"), - "bin", - cplusplus ? "g++.exe" : "gcc.exe") - compiler_cmd = `$path` - end - if cc !== nothing - compiler_cmd = Cmd(Base.shell_split(cc)) - path = nothing - elseif !Sys.iswindows() - compilers_cpp = ("g++", "clang++") - compilers_c = ("gcc", "clang") - found_compiler = false - if cplusplus - for compiler in compilers_cpp - if Sys.which(compiler) !== nothing - compiler_cmd = `$compiler` - found_compiler = true - break - end - end - end - if !found_compiler - for compiler in compilers_c - if Sys.which(compiler) !== nothing - compiler_cmd = `$compiler` - found_compiler = true - if cplusplus && !WARNED_CPP_COMPILER[] - @warn "could not find a c++ compiler (g++ or clang++), falling back to $compiler, this might cause link errors" - WARNED_CPP_COMPILER[] = true - end - break - end - end - end - found_compiler || error("could not find a compiler, looked for ", - join(((cplusplus ? compilers_cpp : ())..., compilers_c...), ", ", " and ")) - end - if path !== nothing - compiler_cmd = addenv(compiler_cmd, "PATH" => string(ENV["PATH"], ";", dirname(path))) - end - return compiler_cmd -end - -# arguments to forward to julia compilation process -julia_args = [] -enable_trim::Bool = false -project::String = "--project=$(Base.active_project())" - -let i = 1 - while i <= length(ARGS) - arg = ARGS[i] - if arg == "--output-exe" || arg == "--output-lib" || arg == "--output-sysimage" - isnothing(output_type) || error("Multiple output types specified") - global output_type = arg - i == length(ARGS) && error("Output specifier requires an argument") - global outname = ARGS[i+1] - i += 1 - elseif arg == "--compile-ccallable" - global add_ccallables = true - elseif arg == "--verbose" - global verbose = true - elseif arg == "--relative-rpath" - global relative_rpath = true - elseif startswith(arg, "--trim") - global enable_trim = arg != "--trim=no" - push!(julia_args, arg) # forwarded arg - elseif arg == "--experimental" - push!(julia_args, arg) # forwarded arg - elseif startswith(arg, "--proj") - global project = arg - else - if arg[1] == '-' || !isnothing(file) - println("Unexpected argument `$arg`") - exit(1) - end - global file = arg - end - i += 1 - end -end - -isnothing(outname) && error("No output file specified") -isnothing(file) && error("No input file specified") - -function get_rpath(; relative::Bool = false) - if relative - if Sys.isapple() - return "-Wl,-rpath,'@loader_path/julia/' -Wl,-rpath,'@loader_path/'" - elseif Sys.islinux() - return "-Wl,-rpath,'\$ORIGIN/julia/' -Wl,-rpath,'\$ORIGIN/'" - else - error("unimplemented") - end - else - return JuliaConfig.ldrpath() - end -end - -cc = get_compiler_cmd() -absfile = abspath(file) -cflags = JuliaConfig.cflags(; framework=false) -cflags = Base.shell_split(cflags) -allflags = JuliaConfig.allflags(; framework=false, rpath=false) -allflags = Base.shell_split(allflags) -rpath = get_rpath(; relative = relative_rpath) -rpath = Base.shell_split(rpath) -tmpdir = mktempdir(cleanup=false) -img_path = joinpath(tmpdir, "img.a") -bc_path = joinpath(tmpdir, "img-bc.a") - -function precompile_env() - # Pre-compile the environment - # (otherwise obscure error messages will occur) - cmd = addenv(`$julia_cmd $project -e "using Pkg; Pkg.precompile()"`) - verbose && println("Running: $cmd") - if !success(pipeline(cmd; stdout, stderr)) - println(stderr, "\nError encountered during pre-compilation of environment.") - exit(1) - end -end - -function compile_products(enable_trim::Bool) - - # Only strip IR / metadata if not `--trim=no` - strip_args = String[] - if enable_trim - push!(strip_args, "--strip-ir") - push!(strip_args, "--strip-metadata") - end - - # Compile the Julia code - cmd = addenv(`$julia_cmd_target $project --output-o $img_path --output-incremental=no $strip_args $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) - verbose && println("Running: $cmd") - if !success(pipeline(cmd; stdout, stderr)) - println(stderr, "\nFailed to compile $file") - exit(1) - end -end - -function link_products() - global outname - if output_type == "--output-lib" || output_type == "--output-sysimage" - of, ext = splitext(outname) - soext = "." * Base.BinaryPlatforms.platform_dlext() - if ext == "" - outname = of * soext - end - end - - julia_libs = Base.shell_split(Base.isdebugbuild() ? "-ljulia-debug -ljulia-internal-debug" : "-ljulia -ljulia-internal") - try - if output_type == "--output-lib" - cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` - elseif output_type == "--output-sysimage" - cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` - else - cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` - end - verbose && println("Running: $cmd2") - run(cmd2) - catch e - println("\nCompilation failed: ", e) - exit(1) - end -end - -precompile_env() -compile_products(enable_trim) -link_products() diff --git a/contrib/mac/app/Info.plist.in b/contrib/mac/app/Info.plist.in new file mode 100644 index 0000000000000..3a0b9e2a6bbf6 --- /dev/null +++ b/contrib/mac/app/Info.plist.in @@ -0,0 +1,32 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + Julia + CFBundleExecutable + julia-terminal + CFBundleIconFile + julia.icns + CFBundleIdentifier + @APP_ID@ + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Julia + CFBundlePackageType + APPL + CFBundleShortVersionString + @JULIA_VERSION_MAJOR_MINOR_PATCH@ + CFBundleVersion + @JULIA_VERSION_OPT_COMMIT@ + LSMinimumSystemVersion + 10.9 + NSHighResolutionCapable + + NSHumanReadableCopyright + @APP_COPYRIGHT@ + + diff --git a/contrib/mac/app/Makefile b/contrib/mac/app/Makefile index 70436a857c265..b3f123decc312 100644 --- a/contrib/mac/app/Makefile +++ b/contrib/mac/app/Makefile @@ -33,22 +33,32 @@ dmg/Applications: -mkdir -p dmg ln -fs /Applications $@ -dmg/$(APP_NAME): startup.applescript julia.icns - -mkdir -p dmg - osacompile -o $@ startup.applescript - rm $@/Contents/Resources/applet.icns - cp julia.icns $@/Contents/Resources/ - plutil -replace CFBundleDevelopmentRegion -string "en" $@/Contents/Info.plist - plutil -insert CFBundleDisplayName -string "Julia" $@/Contents/Info.plist - plutil -replace CFBundleIconFile -string "julia.icns" $@/Contents/Info.plist - plutil -insert CFBundleIdentifier -string "$(APP_ID)" $@/Contents/Info.plist - plutil -replace CFBundleName -string "Julia" $@/Contents/Info.plist - plutil -insert CFBundleShortVersionString -string "$(JULIA_VERSION_MAJOR_MINOR_PATCH)" $@/Contents/Info.plist - plutil -insert CFBundleVersion -string "$(JULIA_VERSION_OPT_COMMIT)" $@/Contents/Info.plist - plutil -insert NSHumanReadableCopyright -string "$(APP_COPYRIGHT)" $@/Contents/Info.plist +dmg/$(APP_NAME): Info.plist.in julia.icns + -mkdir -p $@/Contents/MacOS -mkdir -p $@/Contents/Resources/julia + # Author the bundle's Info.plist from the template, substituting version info. + sed -e 's|@APP_ID@|$(APP_ID)|g' \ + -e 's|@APP_COPYRIGHT@|$(APP_COPYRIGHT)|g' \ + -e 's|@JULIA_VERSION_MAJOR_MINOR_PATCH@|$(JULIA_VERSION_MAJOR_MINOR_PATCH)|g' \ + -e 's|@JULIA_VERSION_OPT_COMMIT@|$(JULIA_VERSION_OPT_COMMIT)|g' \ + Info.plist.in > $@/Contents/Info.plist + printf 'APPL????' > $@/Contents/PkgInfo + cp julia.icns $@/Contents/Resources/ $(MAKE) -C $(JULIAHOME) binary-dist $(TAR) -xzf $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz -C $@/Contents/Resources/julia --strip-components 1 + # The app's double-clickable entry point is `julia-terminal` (CFBundleExecutable): + # a real copy of the julia loader placed directly in Contents/MacOS/. Apple's notary + # requires the bundle's main executable to be a regular Mach-O in Contents/MacOS/ -- + # a symlink, or a binary living under Contents/Resources/ (where it is also sealed as + # a resource), is rejected as an invalid signature. When the loader is invoked under + # this name it relaunches the REPL from the bundled tree in a new Terminal.app window + # (see cli/loader_exe.c) instead of starting the REPL in the windowless bundle + # process; the distinct name is also how it avoids hijacking ordinary `julia` runs. + # A plain `cp` suffices: the loader is linked with an extra rpath into the bundled + # tree (@executable_path/../Resources/julia/lib, see cli/Makefile) precisely so this + # copy can resolve @rpath/libjulia from Contents/MacOS/ -- keeping this rule free of + # Mach-O tooling so it can run on any host (CI assembles the .app on Linux). + cp $@/Contents/Resources/julia/bin/julia $@/Contents/MacOS/julia-terminal find $@/Contents/Resources/julia -type f -exec chmod -w {} \; # Even though the tarball may already be signed, we re-sign here to make it easier to add # unsigned executables (like the app launcher) and whatnot, without needing to maintain lists @@ -58,9 +68,9 @@ dmg/$(APP_NAME): startup.applescript julia.icns MACHO_FILES=$$(find "$@" -type f -perm -0111 | cut -d: -f1); \ for f in $${MACHO_FILES}; do \ echo "Codesigning $${f}..."; \ - codesign -s "$$MACOS_CODESIGN_IDENTITY" --option=runtime --entitlements Entitlements.plist -vvv --timestamp --deep --force "$${f}"; \ + codesign -s "$$MACOS_CODESIGN_IDENTITY" --option=runtime --entitlements Entitlements.plist -vvv --timestamp --force "$${f}"; \ done; \ - codesign -s "$$MACOS_CODESIGN_IDENTITY" --option=runtime --entitlements Entitlements.plist -vvv --timestamp --deep --force "$@"; \ + codesign -s "$$MACOS_CODESIGN_IDENTITY" --option=runtime --entitlements Entitlements.plist -vvv --timestamp --force "$@"; \ else \ true; \ fi diff --git a/contrib/mac/app/README.md b/contrib/mac/app/README.md index 501754312ab9e..44d4e6fc51b55 100644 --- a/contrib/mac/app/README.md +++ b/contrib/mac/app/README.md @@ -4,13 +4,27 @@ Julia OS X packaging This builds the Julia OS X application bundle (.app folder), and stores it in a disk image (.dmg file). -The application bundle is actually just a bundled applet which opens Terminal.app and -executes the julia binary (which opens the REPL). All the Julia binary files and their -dependencies are bundled inside this. +The application bundle opens Terminal.app and executes the julia binary (which opens +the REPL). All the Julia binary files and their dependencies are bundled inside this. + +Its double-clickable entry point is `julia-terminal` (the bundle's `CFBundleExecutable`): +a real copy of the julia loader placed directly at `Contents/MacOS/julia-terminal`. When +the loader is invoked under that name it relaunches the REPL binary from the bundled tree +(`Contents/Resources/julia/bin/julia`) inside a new Terminal.app window rather than +starting the REPL directly in the (windowless) bundle process. See `cli/loader_exe.c` for +that logic. + +The main executable is a real file in `Contents/MacOS/` -- not a symlink, and not inside +`Contents/Resources/` -- because Apple's notary rejects a bundle main executable that is a +symlink or that lives among the sealed resources. Because this copy runs from +`Contents/MacOS/` rather than the tree's `bin/`, the loader is linked with an extra rpath +into the bundled tree so dyld can resolve `libjulia` from there (see `cli/Makefile`); +assembling the bundle is then a plain `cp` needing no Mach-O tools, so it runs on any +host (CI assembles the .app on Linux). Run `make` to build. Other files in this directory -* `startup.applescript` is the script which is compiled to the applet. +* `Info.plist.in` is the template for the bundle's `Info.plist`. * `julia.icns` is the Julia icon file. diff --git a/contrib/mac/app/startup.applescript b/contrib/mac/app/startup.applescript deleted file mode 100644 index d7b46cec1a89d..0000000000000 --- a/contrib/mac/app/startup.applescript +++ /dev/null @@ -1,3 +0,0 @@ -set RootPath to (path to me) -set JuliaPath to POSIX path of ((RootPath as text) & "Contents:Resources:julia:bin:julia") -do shell script "open -a Terminal '" & JuliaPath & "'" diff --git a/contrib/mac/libSystem.tbd b/contrib/mac/libSystem.tbd new file mode 100644 index 0000000000000..c00b2739464ad --- /dev/null +++ b/contrib/mac/libSystem.tbd @@ -0,0 +1,3828 @@ +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/libSystem.B.dylib' +current-version: 1356 +reexported-libraries: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + libraries: [ '/usr/lib/system/libcache.dylib', '/usr/lib/system/libcommonCrypto.dylib', + '/usr/lib/system/libcompiler_rt.dylib', '/usr/lib/system/libcopyfile.dylib', + '/usr/lib/system/libcorecrypto.dylib', '/usr/lib/system/libdispatch.dylib', + '/usr/lib/system/libdyld.dylib', '/usr/lib/system/libkeymgr.dylib', + '/usr/lib/system/libmacho.dylib', '/usr/lib/system/libquarantine.dylib', + '/usr/lib/system/libremovefile.dylib', '/usr/lib/system/libsystem_asl.dylib', + '/usr/lib/system/libsystem_blocks.dylib', '/usr/lib/system/libsystem_c.dylib', + '/usr/lib/system/libsystem_collections.dylib', '/usr/lib/system/libsystem_configuration.dylib', + '/usr/lib/system/libsystem_containermanager.dylib', '/usr/lib/system/libsystem_coreservices.dylib', + '/usr/lib/system/libsystem_darwin.dylib', '/usr/lib/system/libsystem_darwindirectory.dylib', + '/usr/lib/system/libsystem_dnssd.dylib', '/usr/lib/system/libsystem_eligibility.dylib', + '/usr/lib/system/libsystem_featureflags.dylib', '/usr/lib/system/libsystem_info.dylib', + '/usr/lib/system/libsystem_kernel.dylib', '/usr/lib/system/libsystem_m.dylib', + '/usr/lib/system/libsystem_malloc.dylib', '/usr/lib/system/libsystem_networkextension.dylib', + '/usr/lib/system/libsystem_notify.dylib', '/usr/lib/system/libsystem_platform.dylib', + '/usr/lib/system/libsystem_pthread.dylib', '/usr/lib/system/libsystem_sandbox.dylib', + '/usr/lib/system/libsystem_sanitizers.dylib', '/usr/lib/system/libsystem_secinit.dylib', + '/usr/lib/system/libsystem_symptoms.dylib', '/usr/lib/system/libsystem_trace.dylib', + '/usr/lib/system/libsystem_trial.dylib', '/usr/lib/system/libunwind.dylib', + '/usr/lib/system/libxpc.dylib' ] +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst ] + symbols: [ 'R8289209$_close', 'R8289209$_fork', 'R8289209$_fsync', 'R8289209$_getattrlist', + 'R8289209$_getrlimit', 'R8289209$_getxattr', 'R8289209$_open', + 'R8289209$_pthread_attr_destroy', 'R8289209$_pthread_attr_init', + 'R8289209$_pthread_attr_setdetachstate', 'R8289209$_pthread_create', + 'R8289209$_pthread_mutex_lock', 'R8289209$_pthread_mutex_unlock', + 'R8289209$_pthread_self', 'R8289209$_ptrace', 'R8289209$_read', + 'R8289209$_setattrlist', 'R8289209$_setrlimit', 'R8289209$_sigaction', + 'R8289209$_stat', 'R8289209$_sysctl', 'R8289209$_time', 'R8289209$_unlink', + 'R8289209$_write' ] + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___crashreporter_info__, _libSystem_init_after_boot_tasks_4launchd, + _mach_init_routine ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libcache.dylib' +current-version: 95 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _cache_create, _cache_destroy, _cache_get, _cache_get_and_retain, + _cache_get_cost_hint, _cache_get_count_hint, _cache_get_info, + _cache_get_info_for_key, _cache_get_info_for_keys, _cache_get_minimum_values_hint, + _cache_get_name, _cache_hash_byte_string, _cache_invoke, _cache_key_hash_cb_cstring, + _cache_key_hash_cb_integer, _cache_key_is_equal_cb_cstring, + _cache_key_is_equal_cb_integer, _cache_print, _cache_print_stats, + _cache_release, _cache_release_cb_free, _cache_release_value, + _cache_remove, _cache_remove_all, _cache_remove_with_block, + _cache_retain, _cache_set_and_retain, _cache_set_cost_hint, + _cache_set_count_hint, _cache_set_minimum_values_hint, _cache_set_name, + _cache_simulate_memory_warning_event, _cache_value_make_nonpurgeable_cb, + _cache_value_make_purgeable_cb ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libcommonCrypto.dylib' +current-version: 65535 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _CCAESCmac, _CCAESCmacCreate, _CCAESCmacDestroy, _CCAESCmacFinal, + _CCAESCmacOutputSizeFromContext, _CCAESCmacUpdate, _CCBigNumAdd, + _CCBigNumAddI, _CCBigNumBitCount, _CCBigNumByteCount, _CCBigNumClear, + _CCBigNumCompare, _CCBigNumCompareI, _CCBigNumCopy, _CCBigNumCreateRandom, + _CCBigNumDiv, _CCBigNumFree, _CCBigNumFromData, _CCBigNumFromDecimalString, + _CCBigNumFromHexString, _CCBigNumGetI, _CCBigNumIsNegative, + _CCBigNumIsPrime, _CCBigNumIsZero, _CCBigNumLeftShift, _CCBigNumMod, + _CCBigNumModExp, _CCBigNumModI, _CCBigNumMul, _CCBigNumMulI, + _CCBigNumMulMod, _CCBigNumRightShift, _CCBigNumSetI, _CCBigNumSetNegative, + _CCBigNumSub, _CCBigNumSubI, _CCBigNumToData, _CCBigNumToDecimalString, + _CCBigNumToHexString, _CCBigNumZeroLSBCount, _CCCKG2ContributorCommit, + _CCCKG2ContributorCreate, _CCCKG2ContributorDestroy, _CCCKG2ContributorFinish, + _CCCKG2GetCommitmentSize, _CCCKG2GetOpeningSize, _CCCKG2GetShareSize, + _CCCKG2OwnerCreate, _CCCKG2OwnerDestroy, _CCCKG2OwnerFinish, + _CCCKG2OwnerGenerateShare, _CCCKG2ParamsP224Sha256Version2, + _CCCKGContributorCommit, _CCCKGContributorCreate, _CCCKGContributorDestroy, + _CCCKGContributorFinish, _CCCKGGetCommitmentSize, _CCCKGGetOpeningSize, + _CCCKGGetShareSize, _CCCKGOwnerCreate, _CCCKGOwnerDestroy, + _CCCKGOwnerFinish, _CCCKGOwnerGenerateShare, _CCCalibratePBKDF, + _CCCreateBigNum, _CCCrypt, _CCCryptorAddParameter, _CCCryptorChaCha20, + _CCCryptorChaCha20Poly1305OneshotDecrypt, _CCCryptorChaCha20Poly1305OneshotEncrypt, + _CCCryptorCreate, _CCCryptorCreateFromData, _CCCryptorCreateFromDataWithMode, + _CCCryptorCreateWithMode, _CCCryptorDecryptDataBlock, _CCCryptorEncryptDataBlock, + _CCCryptorFinal, _CCCryptorGCM, _CCCryptorGCMAddAAD, _CCCryptorGCMAddADD, + _CCCryptorGCMAddIV, _CCCryptorGCMDecrypt, _CCCryptorGCMEncrypt, + _CCCryptorGCMFinal, _CCCryptorGCMFinalize, _CCCryptorGCMOneshotDecrypt, + _CCCryptorGCMOneshotEncrypt, _CCCryptorGCMReset, _CCCryptorGCMSetIV, + _CCCryptorGCMaddAAD, _CCCryptorGetIV, _CCCryptorGetOutputLength, + _CCCryptorGetParameter, _CCCryptorRelease, _CCCryptorReset, + _CCCryptorReset_binary_compatibility, _CCCryptorUpdate, _CCDHComputeKey, + _CCDHCreate, _CCDHGenerateKey, _CCDHRelease, _CCDeriveKey, + _CCDesCBCCksum, _CCDesIsWeakKey, _CCDesSetOddParity, _CCDigest, + _CCDigestBlockSize, _CCDigestCreate, _CCDigestCreateByOID, + _CCDigestDestroy, _CCDigestFinal, _CCDigestGetBlockSize, _CCDigestGetBlockSizeFromRef, + _CCDigestGetOutputSize, _CCDigestGetOutputSizeFromRef, _CCDigestInit, + _CCDigestOID, _CCDigestOIDLen, _CCDigestOutputSize, _CCDigestReset, + _CCDigestUpdate, _CCECCryptorBlind, _CCECCryptorBlindingKeysRelease, + _CCECCryptorComputeSharedSecret, _CCECCryptorCreateFromData, + _CCECCryptorExportKey, _CCECCryptorExportPublicKey, _CCECCryptorGenerateBlindingKeys, + _CCECCryptorGeneratePair, _CCECCryptorGetKeyComponents, _CCECCryptorGetPublicKeyFromPrivateKey, + _CCECCryptorH2C, _CCECCryptorImportKey, _CCECCryptorImportPublicKey, + _CCECCryptorRelease, _CCECCryptorSignHash, _CCECCryptorTwinDiversifyEntropySize, + _CCECCryptorTwinDiversifyKey, _CCECCryptorUnblind, _CCECCryptorUnwrapKey, + _CCECCryptorVerifyHash, _CCECCryptorWrapKey, _CCECGetKeySize, + _CCECGetKeyType, _CCHKDFExpand, _CCHKDFExtract, _CCHmac, _CCHmacClone, + _CCHmacCreate, _CCHmacDestroy, _CCHmacFinal, _CCHmacInit, + _CCHmacOneShot, _CCHmacOutputSize, _CCHmacOutputSizeFromRef, + _CCHmacUpdate, _CCKDFParametersCreateAnsiX963, _CCKDFParametersCreateCtrHmac, + _CCKDFParametersCreateCtrHmacFixed, _CCKDFParametersCreateHkdf, + _CCKDFParametersCreatePbkdf2, _CCKDFParametersDestroy, _CCKeyDerivationHMac, + _CCKeyDerivationPBKDF, _CCRSACryptorCreateFromData, _CCRSACryptorCreatePublicKeyFromPrivateKey, + _CCRSACryptorCrypt, _CCRSACryptorDecrypt, _CCRSACryptorEncrypt, + _CCRSACryptorExport, _CCRSACryptorGeneratePair, _CCRSACryptorGetPublicKeyFromPrivateKey, + _CCRSACryptorImport, _CCRSACryptorRecoverPrivateKey, _CCRSACryptorRelease, + _CCRSACryptorSign, _CCRSACryptorVerify, _CCRSAGetCRTComponents, + _CCRSAGetCRTComponentsSizes, _CCRSAGetKeyComponents, _CCRSAGetKeySize, + _CCRSAGetKeyType, _CCRandomCopyBytes, _CCRandomGenerateBytes, + _CCRandomUniform, _CCSymmetricKeyUnwrap, _CCSymmetricKeyWrap, + _CCSymmetricUnwrappedSize, _CCSymmetricWrappedSize, _CC_MD2, + _CC_MD2_Final, _CC_MD2_Init, _CC_MD2_Update, _CC_MD4, _CC_MD4_Final, + _CC_MD4_Init, _CC_MD4_Update, _CC_MD5, _CC_MD5_Final, _CC_MD5_Init, + _CC_MD5_Update, _CC_SHA1, _CC_SHA1_Final, _CC_SHA1_Init, _CC_SHA1_Update, + _CC_SHA224, _CC_SHA224_Final, _CC_SHA224_Init, _CC_SHA224_Update, + _CC_SHA256, _CC_SHA256_Final, _CC_SHA256_Init, _CC_SHA256_Update, + _CC_SHA384, _CC_SHA384_Final, _CC_SHA384_Init, _CC_SHA384_Update, + _CC_SHA512, _CC_SHA512_Final, _CC_SHA512_Init, _CC_SHA512_Update, + _CCrfc3394_iv, _CCrfc3394_ivLen, _CNCRC, _CNCRCDumpTable, + _CNCRCFinal, _CNCRCInit, _CNCRCRelease, _CNCRCUpdate, _CNCRCWeakTest, + _CNEncode, _CNEncoderBlocksize, _CNEncoderBlocksizeFromRef, + _CNEncoderCreate, _CNEncoderCreateCustom, _CNEncoderFinal, + _CNEncoderGetOutputLength, _CNEncoderGetOutputLengthFromEncoding, + _CNEncoderRelease, _CNEncoderUpdate, _MD5Final, _ccDRBGGetRngState, + _ccDevRandomGetRngState, _kCCRandomDefault, _kCCRandomDevRandom ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, arm64-macos, arm64e-macos ] +install-name: '/usr/lib/system/libcompiler_rt.dylib' +current-version: 103.3 +parent-umbrella: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + umbrella: System +exports: + - targets: [ x86_64-macos ] + symbols: [ '$ld$hide$os10.4$___absvti2', '$ld$hide$os10.4$___addvti3', + '$ld$hide$os10.4$___ashlti3', '$ld$hide$os10.4$___ashrti3', + '$ld$hide$os10.4$___cmpti2', '$ld$hide$os10.4$___ctzti2', + '$ld$hide$os10.4$___ffsti2', '$ld$hide$os10.4$___fixunsxfti', + '$ld$hide$os10.4$___fixxfti', '$ld$hide$os10.4$___floattixf', + '$ld$hide$os10.4$___floatuntixf', '$ld$hide$os10.4$___lshrti3', + '$ld$hide$os10.4$___multi3', '$ld$hide$os10.4$___mulvti3', + '$ld$hide$os10.4$___negti2', '$ld$hide$os10.4$___negvti2', + '$ld$hide$os10.4$___parityti2', '$ld$hide$os10.4$___popcountti2', + '$ld$hide$os10.4$___subvti3', '$ld$hide$os10.4$___ucmpti2', + '$ld$hide$os10.5$___absvti2', '$ld$hide$os10.5$___addvti3', + '$ld$hide$os10.5$___ashlti3', '$ld$hide$os10.5$___ashrti3', + '$ld$hide$os10.5$___cmpti2', '$ld$hide$os10.5$___ctzti2', + '$ld$hide$os10.5$___ffsti2', '$ld$hide$os10.5$___fixunsxfti', + '$ld$hide$os10.5$___fixxfti', '$ld$hide$os10.5$___floattixf', + '$ld$hide$os10.5$___floatuntixf', '$ld$hide$os10.5$___lshrti3', + '$ld$hide$os10.5$___multi3', '$ld$hide$os10.5$___mulvti3', + '$ld$hide$os10.5$___negti2', '$ld$hide$os10.5$___negvti2', + '$ld$hide$os10.5$___parityti2', '$ld$hide$os10.5$___popcountti2', + '$ld$hide$os10.5$___subvti3', '$ld$hide$os10.5$___ucmpti2', + ___absvdi2, ___absvsi2, ___absvti2, ___addvdi3, ___addvsi3, + ___addvti3, ___ashldi3, ___ashlti3, ___ashrdi3, ___ashrti3, + ___clzdi2, ___clzsi2, ___cmpdi2, ___cmpti2, ___ctzdi2, ___ctzsi2, + ___ctzti2, ___divdc3, ___divdi3, ___divsc3, ___divxc3, ___ffsdi2, + ___ffsti2, ___fixdfdi, ___fixsfdi, ___fixunsdfdi, ___fixunsdfsi, + ___fixunssfdi, ___fixunssfsi, ___fixunsxfdi, ___fixunsxfsi, + ___fixunsxfti, ___fixxfdi, ___fixxfti, ___floatdidf, ___floatdisf, + ___floatdixf, ___floattixf, ___floatundidf, ___floatundisf, + ___floatundixf, ___floatuntixf, ___lshrdi3, ___lshrti3, ___moddi3, + ___muldi3, ___mulodi4, ___mulosi4, ___muloti4, ___multi3, + ___mulvdi3, ___mulvsi3, ___mulvti3, ___mulxc3, ___negdi2, + ___negti2, ___negvdi2, ___negvsi2, ___negvti2, ___paritydi2, + ___paritysi2, ___parityti2, ___popcountdi2, ___popcountsi2, + ___popcountti2, ___powixf2, ___subvdi3, ___subvsi3, ___subvti3, + ___ucmpdi2, ___ucmpti2, ___udivdi3, ___udivmoddi4, ___umoddi3 ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ '$ld$hide$os10.10$___chkstk_darwin', '$ld$hide$os10.10$___extendhfsf2', + '$ld$hide$os10.10$___truncdfhf2', '$ld$hide$os10.10$___truncsfhf2', + '$ld$hide$os10.10$_atomic_flag_clear', '$ld$hide$os10.10$_atomic_flag_clear_explicit', + '$ld$hide$os10.10$_atomic_flag_test_and_set', '$ld$hide$os10.10$_atomic_flag_test_and_set_explicit', + '$ld$hide$os10.10$_atomic_signal_fence', '$ld$hide$os10.10$_atomic_thread_fence', + '$ld$hide$os10.11$___chkstk_darwin', '$ld$hide$os10.12$___chkstk_darwin', + '$ld$hide$os10.13$___chkstk_darwin', '$ld$hide$os10.14$___chkstk_darwin', + '$ld$hide$os10.4$___absvdi2', '$ld$hide$os10.4$___absvsi2', + '$ld$hide$os10.4$___addvdi3', '$ld$hide$os10.4$___addvsi3', + '$ld$hide$os10.4$___ashldi3', '$ld$hide$os10.4$___ashrdi3', + '$ld$hide$os10.4$___clear_cache', '$ld$hide$os10.4$___clzdi2', + '$ld$hide$os10.4$___clzsi2', '$ld$hide$os10.4$___clzti2', + '$ld$hide$os10.4$___cmpdi2', '$ld$hide$os10.4$___ctzdi2', + '$ld$hide$os10.4$___ctzsi2', '$ld$hide$os10.4$___divdc3', + '$ld$hide$os10.4$___divdi3', '$ld$hide$os10.4$___divsc3', + '$ld$hide$os10.4$___divti3', '$ld$hide$os10.4$___divxc3', + '$ld$hide$os10.4$___enable_execute_stack', '$ld$hide$os10.4$___ffsdi2', + '$ld$hide$os10.4$___fixdfdi', '$ld$hide$os10.4$___fixdfti', + '$ld$hide$os10.4$___fixsfdi', '$ld$hide$os10.4$___fixsfti', + '$ld$hide$os10.4$___fixunsdfdi', '$ld$hide$os10.4$___fixunsdfsi', + '$ld$hide$os10.4$___fixunsdfti', '$ld$hide$os10.4$___fixunssfdi', + '$ld$hide$os10.4$___fixunssfsi', '$ld$hide$os10.4$___fixunssfti', + '$ld$hide$os10.4$___fixunsxfdi', '$ld$hide$os10.4$___fixunsxfsi', + '$ld$hide$os10.4$___fixxfdi', '$ld$hide$os10.4$___floatdidf', + '$ld$hide$os10.4$___floatdisf', '$ld$hide$os10.4$___floatdixf', + '$ld$hide$os10.4$___floattidf', '$ld$hide$os10.4$___floattisf', + '$ld$hide$os10.4$___floatundidf', '$ld$hide$os10.4$___floatundisf', + '$ld$hide$os10.4$___floatundixf', '$ld$hide$os10.4$___floatuntidf', + '$ld$hide$os10.4$___floatuntisf', '$ld$hide$os10.4$___gcc_personality_v0', + '$ld$hide$os10.4$___lshrdi3', '$ld$hide$os10.4$___moddi3', + '$ld$hide$os10.4$___modti3', '$ld$hide$os10.4$___muldc3', + '$ld$hide$os10.4$___muldi3', '$ld$hide$os10.4$___mulsc3', + '$ld$hide$os10.4$___mulvdi3', '$ld$hide$os10.4$___mulvsi3', + '$ld$hide$os10.4$___mulxc3', '$ld$hide$os10.4$___negdi2', + '$ld$hide$os10.4$___negvdi2', '$ld$hide$os10.4$___negvsi2', + '$ld$hide$os10.4$___paritydi2', '$ld$hide$os10.4$___paritysi2', + '$ld$hide$os10.4$___popcountdi2', '$ld$hide$os10.4$___popcountsi2', + '$ld$hide$os10.4$___powidf2', '$ld$hide$os10.4$___powisf2', + '$ld$hide$os10.4$___powixf2', '$ld$hide$os10.4$___subvdi3', + '$ld$hide$os10.4$___subvsi3', '$ld$hide$os10.4$___ucmpdi2', + '$ld$hide$os10.4$___udivdi3', '$ld$hide$os10.4$___udivmoddi4', + '$ld$hide$os10.4$___udivmodti4', '$ld$hide$os10.4$___udivti3', + '$ld$hide$os10.4$___umoddi3', '$ld$hide$os10.4$___umodti3', + '$ld$hide$os10.5$___absvdi2', '$ld$hide$os10.5$___absvsi2', + '$ld$hide$os10.5$___addvdi3', '$ld$hide$os10.5$___addvsi3', + '$ld$hide$os10.5$___ashldi3', '$ld$hide$os10.5$___ashrdi3', + '$ld$hide$os10.5$___clear_cache', '$ld$hide$os10.5$___clzdi2', + '$ld$hide$os10.5$___clzsi2', '$ld$hide$os10.5$___clzti2', + '$ld$hide$os10.5$___cmpdi2', '$ld$hide$os10.5$___ctzdi2', + '$ld$hide$os10.5$___ctzsi2', '$ld$hide$os10.5$___divdc3', + '$ld$hide$os10.5$___divdi3', '$ld$hide$os10.5$___divsc3', + '$ld$hide$os10.5$___divti3', '$ld$hide$os10.5$___divxc3', + '$ld$hide$os10.5$___enable_execute_stack', '$ld$hide$os10.5$___ffsdi2', + '$ld$hide$os10.5$___fixdfdi', '$ld$hide$os10.5$___fixdfti', + '$ld$hide$os10.5$___fixsfdi', '$ld$hide$os10.5$___fixsfti', + '$ld$hide$os10.5$___fixunsdfdi', '$ld$hide$os10.5$___fixunsdfsi', + '$ld$hide$os10.5$___fixunsdfti', '$ld$hide$os10.5$___fixunssfdi', + '$ld$hide$os10.5$___fixunssfsi', '$ld$hide$os10.5$___fixunssfti', + '$ld$hide$os10.5$___fixunsxfdi', '$ld$hide$os10.5$___fixunsxfsi', + '$ld$hide$os10.5$___fixxfdi', '$ld$hide$os10.5$___floatdidf', + '$ld$hide$os10.5$___floatdisf', '$ld$hide$os10.5$___floatdixf', + '$ld$hide$os10.5$___floattidf', '$ld$hide$os10.5$___floattisf', + '$ld$hide$os10.5$___floatundidf', '$ld$hide$os10.5$___floatundisf', + '$ld$hide$os10.5$___floatundixf', '$ld$hide$os10.5$___floatuntidf', + '$ld$hide$os10.5$___floatuntisf', '$ld$hide$os10.5$___gcc_personality_v0', + '$ld$hide$os10.5$___lshrdi3', '$ld$hide$os10.5$___moddi3', + '$ld$hide$os10.5$___modti3', '$ld$hide$os10.5$___muldc3', + '$ld$hide$os10.5$___muldi3', '$ld$hide$os10.5$___mulsc3', + '$ld$hide$os10.5$___mulvdi3', '$ld$hide$os10.5$___mulvsi3', + '$ld$hide$os10.5$___mulxc3', '$ld$hide$os10.5$___negdi2', + '$ld$hide$os10.5$___negvdi2', '$ld$hide$os10.5$___negvsi2', + '$ld$hide$os10.5$___paritydi2', '$ld$hide$os10.5$___paritysi2', + '$ld$hide$os10.5$___popcountdi2', '$ld$hide$os10.5$___popcountsi2', + '$ld$hide$os10.5$___powidf2', '$ld$hide$os10.5$___powisf2', + '$ld$hide$os10.5$___powixf2', '$ld$hide$os10.5$___subvdi3', + '$ld$hide$os10.5$___subvsi3', '$ld$hide$os10.5$___ucmpdi2', + '$ld$hide$os10.5$___udivdi3', '$ld$hide$os10.5$___udivmoddi4', + '$ld$hide$os10.5$___udivmodti4', '$ld$hide$os10.5$___udivti3', + '$ld$hide$os10.5$___umoddi3', '$ld$hide$os10.5$___umodti3', + '$ld$hide$os10.6$___atomic_compare_exchange', '$ld$hide$os10.6$___atomic_compare_exchange_1', + '$ld$hide$os10.6$___atomic_compare_exchange_2', '$ld$hide$os10.6$___atomic_compare_exchange_4', + '$ld$hide$os10.6$___atomic_compare_exchange_8', '$ld$hide$os10.6$___atomic_exchange', + '$ld$hide$os10.6$___atomic_exchange_1', '$ld$hide$os10.6$___atomic_exchange_2', + '$ld$hide$os10.6$___atomic_exchange_4', '$ld$hide$os10.6$___atomic_exchange_8', + '$ld$hide$os10.6$___atomic_fetch_add_1', '$ld$hide$os10.6$___atomic_fetch_add_2', + '$ld$hide$os10.6$___atomic_fetch_add_4', '$ld$hide$os10.6$___atomic_fetch_add_8', + '$ld$hide$os10.6$___atomic_fetch_and_1', '$ld$hide$os10.6$___atomic_fetch_and_2', + '$ld$hide$os10.6$___atomic_fetch_and_4', '$ld$hide$os10.6$___atomic_fetch_and_8', + '$ld$hide$os10.6$___atomic_fetch_or_1', '$ld$hide$os10.6$___atomic_fetch_or_2', + '$ld$hide$os10.6$___atomic_fetch_or_4', '$ld$hide$os10.6$___atomic_fetch_or_8', + '$ld$hide$os10.6$___atomic_fetch_sub_1', '$ld$hide$os10.6$___atomic_fetch_sub_2', + '$ld$hide$os10.6$___atomic_fetch_sub_4', '$ld$hide$os10.6$___atomic_fetch_sub_8', + '$ld$hide$os10.6$___atomic_fetch_xor_1', '$ld$hide$os10.6$___atomic_fetch_xor_2', + '$ld$hide$os10.6$___atomic_fetch_xor_4', '$ld$hide$os10.6$___atomic_fetch_xor_8', + '$ld$hide$os10.6$___atomic_load', '$ld$hide$os10.6$___atomic_load_1', + '$ld$hide$os10.6$___atomic_load_2', '$ld$hide$os10.6$___atomic_load_4', + '$ld$hide$os10.6$___atomic_load_8', '$ld$hide$os10.6$___atomic_store', + '$ld$hide$os10.6$___atomic_store_1', '$ld$hide$os10.6$___atomic_store_2', + '$ld$hide$os10.6$___atomic_store_4', '$ld$hide$os10.6$___atomic_store_8', + '$ld$hide$os10.6$___chkstk_darwin', '$ld$hide$os10.6$___extendhfsf2', + '$ld$hide$os10.6$___truncdfhf2', '$ld$hide$os10.6$___truncsfhf2', + '$ld$hide$os10.6$_atomic_flag_clear', '$ld$hide$os10.6$_atomic_flag_clear_explicit', + '$ld$hide$os10.6$_atomic_flag_test_and_set', '$ld$hide$os10.6$_atomic_flag_test_and_set_explicit', + '$ld$hide$os10.6$_atomic_signal_fence', '$ld$hide$os10.6$_atomic_thread_fence', + '$ld$hide$os10.7$___atomic_compare_exchange', '$ld$hide$os10.7$___atomic_compare_exchange_1', + '$ld$hide$os10.7$___atomic_compare_exchange_2', '$ld$hide$os10.7$___atomic_compare_exchange_4', + '$ld$hide$os10.7$___atomic_compare_exchange_8', '$ld$hide$os10.7$___atomic_exchange', + '$ld$hide$os10.7$___atomic_exchange_1', '$ld$hide$os10.7$___atomic_exchange_2', + '$ld$hide$os10.7$___atomic_exchange_4', '$ld$hide$os10.7$___atomic_exchange_8', + '$ld$hide$os10.7$___atomic_fetch_add_1', '$ld$hide$os10.7$___atomic_fetch_add_2', + '$ld$hide$os10.7$___atomic_fetch_add_4', '$ld$hide$os10.7$___atomic_fetch_add_8', + '$ld$hide$os10.7$___atomic_fetch_and_1', '$ld$hide$os10.7$___atomic_fetch_and_2', + '$ld$hide$os10.7$___atomic_fetch_and_4', '$ld$hide$os10.7$___atomic_fetch_and_8', + '$ld$hide$os10.7$___atomic_fetch_or_1', '$ld$hide$os10.7$___atomic_fetch_or_2', + '$ld$hide$os10.7$___atomic_fetch_or_4', '$ld$hide$os10.7$___atomic_fetch_or_8', + '$ld$hide$os10.7$___atomic_fetch_sub_1', '$ld$hide$os10.7$___atomic_fetch_sub_2', + '$ld$hide$os10.7$___atomic_fetch_sub_4', '$ld$hide$os10.7$___atomic_fetch_sub_8', + '$ld$hide$os10.7$___atomic_fetch_xor_1', '$ld$hide$os10.7$___atomic_fetch_xor_2', + '$ld$hide$os10.7$___atomic_fetch_xor_4', '$ld$hide$os10.7$___atomic_fetch_xor_8', + '$ld$hide$os10.7$___atomic_load', '$ld$hide$os10.7$___atomic_load_1', + '$ld$hide$os10.7$___atomic_load_2', '$ld$hide$os10.7$___atomic_load_4', + '$ld$hide$os10.7$___atomic_load_8', '$ld$hide$os10.7$___atomic_store', + '$ld$hide$os10.7$___atomic_store_1', '$ld$hide$os10.7$___atomic_store_2', + '$ld$hide$os10.7$___atomic_store_4', '$ld$hide$os10.7$___atomic_store_8', + '$ld$hide$os10.7$___chkstk_darwin', '$ld$hide$os10.7$___extendhfsf2', + '$ld$hide$os10.7$___truncdfhf2', '$ld$hide$os10.7$___truncsfhf2', + '$ld$hide$os10.7$_atomic_flag_clear', '$ld$hide$os10.7$_atomic_flag_clear_explicit', + '$ld$hide$os10.7$_atomic_flag_test_and_set', '$ld$hide$os10.7$_atomic_flag_test_and_set_explicit', + '$ld$hide$os10.7$_atomic_signal_fence', '$ld$hide$os10.7$_atomic_thread_fence', + '$ld$hide$os10.8$___atomic_compare_exchange', '$ld$hide$os10.8$___atomic_compare_exchange_1', + '$ld$hide$os10.8$___atomic_compare_exchange_2', '$ld$hide$os10.8$___atomic_compare_exchange_4', + '$ld$hide$os10.8$___atomic_compare_exchange_8', '$ld$hide$os10.8$___atomic_exchange', + '$ld$hide$os10.8$___atomic_exchange_1', '$ld$hide$os10.8$___atomic_exchange_2', + '$ld$hide$os10.8$___atomic_exchange_4', '$ld$hide$os10.8$___atomic_exchange_8', + '$ld$hide$os10.8$___atomic_fetch_add_1', '$ld$hide$os10.8$___atomic_fetch_add_2', + '$ld$hide$os10.8$___atomic_fetch_add_4', '$ld$hide$os10.8$___atomic_fetch_add_8', + '$ld$hide$os10.8$___atomic_fetch_and_1', '$ld$hide$os10.8$___atomic_fetch_and_2', + '$ld$hide$os10.8$___atomic_fetch_and_4', '$ld$hide$os10.8$___atomic_fetch_and_8', + '$ld$hide$os10.8$___atomic_fetch_or_1', '$ld$hide$os10.8$___atomic_fetch_or_2', + '$ld$hide$os10.8$___atomic_fetch_or_4', '$ld$hide$os10.8$___atomic_fetch_or_8', + '$ld$hide$os10.8$___atomic_fetch_sub_1', '$ld$hide$os10.8$___atomic_fetch_sub_2', + '$ld$hide$os10.8$___atomic_fetch_sub_4', '$ld$hide$os10.8$___atomic_fetch_sub_8', + '$ld$hide$os10.8$___atomic_fetch_xor_1', '$ld$hide$os10.8$___atomic_fetch_xor_2', + '$ld$hide$os10.8$___atomic_fetch_xor_4', '$ld$hide$os10.8$___atomic_fetch_xor_8', + '$ld$hide$os10.8$___atomic_load', '$ld$hide$os10.8$___atomic_load_1', + '$ld$hide$os10.8$___atomic_load_2', '$ld$hide$os10.8$___atomic_load_4', + '$ld$hide$os10.8$___atomic_load_8', '$ld$hide$os10.8$___atomic_store', + '$ld$hide$os10.8$___atomic_store_1', '$ld$hide$os10.8$___atomic_store_2', + '$ld$hide$os10.8$___atomic_store_4', '$ld$hide$os10.8$___atomic_store_8', + '$ld$hide$os10.8$___chkstk_darwin', '$ld$hide$os10.8$___extendhfsf2', + '$ld$hide$os10.8$___truncdfhf2', '$ld$hide$os10.8$___truncsfhf2', + '$ld$hide$os10.8$_atomic_flag_clear', '$ld$hide$os10.8$_atomic_flag_clear_explicit', + '$ld$hide$os10.8$_atomic_flag_test_and_set', '$ld$hide$os10.8$_atomic_flag_test_and_set_explicit', + '$ld$hide$os10.8$_atomic_signal_fence', '$ld$hide$os10.8$_atomic_thread_fence', + '$ld$hide$os10.9$___chkstk_darwin', '$ld$hide$os10.9$___extendhfsf2', + '$ld$hide$os10.9$___truncdfhf2', '$ld$hide$os10.9$___truncsfhf2', + '$ld$hide$os10.9$_atomic_flag_clear', '$ld$hide$os10.9$_atomic_flag_clear_explicit', + '$ld$hide$os10.9$_atomic_flag_test_and_set', '$ld$hide$os10.9$_atomic_flag_test_and_set_explicit', + '$ld$hide$os10.9$_atomic_signal_fence', '$ld$hide$os10.9$_atomic_thread_fence', + ___atomic_compare_exchange, ___atomic_compare_exchange_1, + ___atomic_compare_exchange_2, ___atomic_compare_exchange_4, + ___atomic_compare_exchange_8, ___atomic_exchange, ___atomic_exchange_1, + ___atomic_exchange_2, ___atomic_exchange_4, ___atomic_exchange_8, + ___atomic_fetch_add_1, ___atomic_fetch_add_2, ___atomic_fetch_add_4, + ___atomic_fetch_add_8, ___atomic_fetch_and_1, ___atomic_fetch_and_2, + ___atomic_fetch_and_4, ___atomic_fetch_and_8, ___atomic_fetch_or_1, + ___atomic_fetch_or_2, ___atomic_fetch_or_4, ___atomic_fetch_or_8, + ___atomic_fetch_sub_1, ___atomic_fetch_sub_2, ___atomic_fetch_sub_4, + ___atomic_fetch_sub_8, ___atomic_fetch_xor_1, ___atomic_fetch_xor_2, + ___atomic_fetch_xor_4, ___atomic_fetch_xor_8, ___atomic_is_lock_free, + ___atomic_load, ___atomic_load_1, ___atomic_load_2, ___atomic_load_4, + ___atomic_load_8, ___atomic_store, ___atomic_store_1, ___atomic_store_2, + ___atomic_store_4, ___atomic_store_8, ___chkstk_darwin, ___clear_cache, + ___clzti2, ___divti3, ___enable_execute_stack, ___extendhfsf2, + ___fixdfti, ___fixsfti, ___fixunsdfti, ___fixunssfti, ___floattidf, + ___floattisf, ___floatuntidf, ___floatuntisf, ___gcc_personality_v0, + ___gnu_f2h_ieee, ___gnu_h2f_ieee, ___modti3, ___muldc3, ___mulsc3, + ___powidf2, ___powisf2, ___truncdfhf2, ___truncsfhf2, ___udivmodti4, + ___udivti3, ___umodti3, _atomic_flag_clear, _atomic_flag_clear_explicit, + _atomic_flag_test_and_set, _atomic_flag_test_and_set_explicit, + _atomic_signal_fence, _atomic_thread_fence ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libcopyfile.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _copyfile, _copyfile_state_alloc, _copyfile_state_free, _copyfile_state_get, + _copyfile_state_set, _fcopyfile, _xattr_flags_from_name, _xattr_intent_with_flags, + _xattr_name_with_flags, _xattr_name_without_flags, _xattr_preserve_for_intent ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libcorecrypto.dylib' +current-version: 1922.40.14 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst ] + symbols: [ _ccaes_intel_cbc_decrypt_aesni_mode, _ccaes_intel_cbc_decrypt_opt_mode, + _ccaes_intel_cbc_encrypt_aesni_mode, _ccaes_intel_cbc_encrypt_opt_mode, + _ccaes_intel_ecb_decrypt_aesni_mode, _ccaes_intel_ecb_decrypt_opt_mode, + _ccaes_intel_ecb_encrypt_aesni_mode, _ccaes_intel_ecb_encrypt_opt_mode, + _ccaes_intel_xts_decrypt_aesni_mode, _ccaes_intel_xts_decrypt_opt_mode, + _ccaes_intel_xts_encrypt_aesni_mode, _ccaes_intel_xts_encrypt_opt_mode, + _ccsha1_vng_intel_SupplementalSSE3_di, _ccsha224_vng_intel_SupplementalSSE3_di, + _ccsha256_vng_intel_SupplementalSSE3_di ] + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _CCEC_FAULT_CANARY, _CCMLDSA_FAULT_CANARY, _CCRSA_PKCS1_FAULT_CANARY, + _CCRSA_PSS_FAULT_CANARY, _CCSS_PRIME_P192, _CCSS_PRIME_P224, + _CCSS_PRIME_P256, _CCSS_PRIME_P384, _CCSS_PRIME_P521, _cc_abort, + _cc_atfork_child, _cc_atfork_parent, _cc_atfork_prepare, _cc_clear, + _cc_cmp_safe, _cc_impl_name, _cc_try_abort, _cc_try_abort_if, + _ccaes_cbc_decrypt_mode, _ccaes_cbc_encrypt_mode, _ccaes_ccm_decrypt_mode, + _ccaes_ccm_encrypt_mode, _ccaes_cfb8_decrypt_mode, _ccaes_cfb8_encrypt_mode, + _ccaes_cfb_decrypt_mode, _ccaes_cfb_encrypt_mode, _ccaes_ctr_crypt_mode, + _ccaes_ecb_decrypt_mode, _ccaes_ecb_encrypt_mode, _ccaes_gcm_decrypt_mode, + _ccaes_gcm_encrypt_mode, _ccaes_gladman_cbc_decrypt_mode, + _ccaes_gladman_cbc_encrypt_mode, _ccaes_ltc_ecb_decrypt_mode, + _ccaes_ltc_ecb_encrypt_mode, _ccaes_ofb_crypt_mode, _ccaes_siv_decrypt_mode, + _ccaes_siv_encrypt_mode, _ccaes_siv_hmac_sha256_decrypt_mode, + _ccaes_siv_hmac_sha256_encrypt_mode, _ccaes_unwind, _ccaes_xts_decrypt_mode, + _ccaes_xts_encrypt_mode, _ccansikdf_x963, _ccapsic_client_check_intersect_response, + _ccapsic_client_generate_match_response, _ccapsic_client_init, + _ccapsic_client_state_sizeof, _ccapsic_server_determine_intersection, + _ccapsic_server_encode_element, _ccapsic_server_init, _ccapsic_server_state_sizeof, + _ccascon_ascon128a, _ccascon_ascon128a_cmac, _ccascon_ascon128a_cmac_ref, + _ccascon_ascon128a_decrypt, _ccascon_ascon128a_encrypt, _ccascon_ascon128a_ref, + _ccascon_cmac_init, _ccascon_cmac_process, _ccascon_cmac_tag, + _ccascon_cmac_verify, _ccblowfish_cbc_decrypt_mode, _ccblowfish_cbc_encrypt_mode, + _ccblowfish_cfb8_decrypt_mode, _ccblowfish_cfb8_encrypt_mode, + _ccblowfish_cfb_decrypt_mode, _ccblowfish_cfb_encrypt_mode, + _ccblowfish_ctr_crypt_mode, _ccblowfish_ecb_decrypt_mode, + _ccblowfish_ecb_encrypt_mode, _ccblowfish_ofb_crypt_mode, + _cccast_cbc_decrypt_mode, _cccast_cbc_encrypt_mode, _cccast_cfb8_decrypt_mode, + _cccast_cfb8_encrypt_mode, _cccast_cfb_decrypt_mode, _cccast_cfb_encrypt_mode, + _cccast_ctr_crypt_mode, _cccast_ecb_decrypt_mode, _cccast_ecb_encrypt_mode, + _cccast_ofb_crypt_mode, _cccbc_block_size, _cccbc_clear_iv, + _cccbc_context_size, _cccbc_copy_iv, _cccbc_init, _cccbc_one_shot, + _cccbc_one_shot_explicit, _cccbc_set_iv, _cccbc_update, _ccccm_aad, + _ccccm_block_size, _ccccm_cbcmac, _ccccm_context_size, _ccccm_decrypt, + _ccccm_encrypt, _ccccm_finalize, _ccccm_finalize_and_generate_tag, + _ccccm_finalize_and_verify_tag, _ccccm_init, _ccccm_nonce_size, + _ccccm_one_shot, _ccccm_one_shot_decrypt, _ccccm_one_shot_encrypt, + _ccccm_reset, _ccccm_set_iv, _ccccm_update, _cccfb8_block_size, + _cccfb8_context_size, _cccfb8_init, _cccfb8_one_shot, _cccfb8_update, + _cccfb_block_size, _cccfb_context_size, _cccfb_init, _cccfb_one_shot, + _cccfb_update, _ccchacha20, _ccchacha20_final, _ccchacha20_init, + _ccchacha20_reset, _ccchacha20_setcounter, _ccchacha20_setnonce, + _ccchacha20_update, _ccchacha20poly1305_aad, _ccchacha20poly1305_decrypt, + _ccchacha20poly1305_decrypt_oneshot, _ccchacha20poly1305_encrypt, + _ccchacha20poly1305_encrypt_oneshot, _ccchacha20poly1305_finalize, + _ccchacha20poly1305_incnonce, _ccchacha20poly1305_info, _ccchacha20poly1305_init, + _ccchacha20poly1305_reset, _ccchacha20poly1305_setnonce, _ccchacha20poly1305_verify, + _ccckg2_contributor_commit, _ccckg2_contributor_finish, _ccckg2_ctx_cp, + _ccckg2_ctx_di, _ccckg2_init, _ccckg2_owner_finish, _ccckg2_owner_generate_share, + _ccckg2_params_p224_sha256_v2, _ccckg2_sizeof_commitment, + _ccckg2_sizeof_ctx, _ccckg2_sizeof_opening, _ccckg2_sizeof_share, + _ccckg_contributor_commit, _ccckg_contributor_finish, _ccckg_ctx_cp, + _ccckg_ctx_di, _ccckg_init, _ccckg_owner_finish, _ccckg_owner_generate_share, + _ccckg_sizeof_commitment, _ccckg_sizeof_ctx, _ccckg_sizeof_opening, + _ccckg_sizeof_share, _cccmac_final_generate, _cccmac_final_verify, + _cccmac_init, _cccmac_one_shot_generate, _cccmac_one_shot_verify, + _cccmac_update, _ccctr_block_size, _ccctr_context_size, _ccctr_init, + _ccctr_one_shot, _ccctr_update, _cccurve25519, _cccurve25519_make_key_pair, + _cccurve25519_make_priv, _cccurve25519_make_pub, _cccurve25519_make_pub_with_rng, + _cccurve25519_with_rng, _cccurve448, _cccurve448_make_key_pair, + _cccurve448_make_priv, _cccurve448_make_pub, _ccder_blob_decode_bitstring, + _ccder_blob_decode_eckey, _ccder_blob_decode_len, _ccder_blob_decode_len_strict, + _ccder_blob_decode_oid, _ccder_blob_decode_range, _ccder_blob_decode_range_strict, + _ccder_blob_decode_seqii, _ccder_blob_decode_seqii_strict, + _ccder_blob_decode_sequence_tl, _ccder_blob_decode_sequence_tl_strict, + _ccder_blob_decode_tag, _ccder_blob_decode_tl, _ccder_blob_decode_tl_strict, + _ccder_blob_decode_uint, _ccder_blob_decode_uint64, _ccder_blob_decode_uint_n, + _ccder_blob_decode_uint_strict, _ccder_blob_encode_body, _ccder_blob_encode_body_tl, + _ccder_blob_encode_eckey, _ccder_blob_encode_implicit_integer, + _ccder_blob_encode_implicit_octet_string, _ccder_blob_encode_implicit_raw_octet_string, + _ccder_blob_encode_implicit_uint64, _ccder_blob_encode_integer, + _ccder_blob_encode_len, _ccder_blob_encode_octet_string, _ccder_blob_encode_oid, + _ccder_blob_encode_raw_octet_string, _ccder_blob_encode_tag, + _ccder_blob_encode_tl, _ccder_blob_encode_uint64, _ccder_blob_reserve, + _ccder_blob_reserve_tl, _ccder_decode_bitstring, _ccder_decode_constructed_tl, + _ccder_decode_constructed_tl_strict, _ccder_decode_dhparam_n, + _ccder_decode_dhparams, _ccder_decode_eckey, _ccder_decode_len, + _ccder_decode_len_strict, _ccder_decode_oid, _ccder_decode_rsa_priv, + _ccder_decode_rsa_priv_n, _ccder_decode_rsa_pub, _ccder_decode_rsa_pub_n, + _ccder_decode_rsa_pub_x509, _ccder_decode_rsa_pub_x509_n, + _ccder_decode_seqii, _ccder_decode_seqii_strict, _ccder_decode_sequence_tl, + _ccder_decode_sequence_tl_strict, _ccder_decode_tag, _ccder_decode_tl, + _ccder_decode_tl_strict, _ccder_decode_uint, _ccder_decode_uint64, + _ccder_decode_uint_n, _ccder_decode_uint_strict, _ccder_encode_body, + _ccder_encode_body_nocopy, _ccder_encode_constructed_tl, _ccder_encode_dhparams, + _ccder_encode_dhparams_size, _ccder_encode_eckey, _ccder_encode_eckey_size, + _ccder_encode_implicit_integer, _ccder_encode_implicit_octet_string, + _ccder_encode_implicit_raw_octet_string, _ccder_encode_implicit_uint64, + _ccder_encode_integer, _ccder_encode_len, _ccder_encode_octet_string, + _ccder_encode_oid, _ccder_encode_raw_octet_string, _ccder_encode_rsa_priv, + _ccder_encode_rsa_priv_size, _ccder_encode_rsa_pub, _ccder_encode_rsa_pub_size, + _ccder_encode_tag, _ccder_encode_tl, _ccder_encode_uint64, + _ccder_sizeof, _ccder_sizeof_eckey, _ccder_sizeof_implicit_integer, + _ccder_sizeof_implicit_octet_string, _ccder_sizeof_implicit_raw_octet_string, + _ccder_sizeof_implicit_raw_octet_string_overflow, _ccder_sizeof_implicit_uint64, + _ccder_sizeof_integer, _ccder_sizeof_len, _ccder_sizeof_octet_string, + _ccder_sizeof_oid, _ccder_sizeof_overflow, _ccder_sizeof_raw_octet_string, + _ccder_sizeof_tag, _ccder_sizeof_uint64, _ccdes3_cbc_decrypt_mode, + _ccdes3_cbc_encrypt_mode, _ccdes3_cfb8_decrypt_mode, _ccdes3_cfb8_encrypt_mode, + _ccdes3_cfb_decrypt_mode, _ccdes3_cfb_encrypt_mode, _ccdes3_ctr_crypt_mode, + _ccdes3_ecb_decrypt_mode, _ccdes3_ecb_encrypt_mode, _ccdes3_ltc_ecb_decrypt_mode, + _ccdes3_ltc_ecb_encrypt_mode, _ccdes3_ofb_crypt_mode, _ccdes_cbc_cksum, + _ccdes_cbc_decrypt_mode, _ccdes_cbc_encrypt_mode, _ccdes_cfb8_decrypt_mode, + _ccdes_cfb8_encrypt_mode, _ccdes_cfb_decrypt_mode, _ccdes_cfb_encrypt_mode, + _ccdes_ctr_crypt_mode, _ccdes_ecb_decrypt_mode, _ccdes_ecb_encrypt_mode, + _ccdes_key_is_weak, _ccdes_key_set_odd_parity, _ccdes_ofb_crypt_mode, + _ccdh_ccn_size, _ccdh_compute_shared_secret, _ccdh_ctx_init, + _ccdh_ctx_public, _ccdh_export_pub, _ccdh_export_pub_size, + _ccdh_generate_key, _ccdh_gp_apple768, _ccdh_gp_g, _ccdh_gp_l, + _ccdh_gp_n, _ccdh_gp_order, _ccdh_gp_order_bitlen, _ccdh_gp_prime, + _ccdh_gp_rfc2409group02, _ccdh_gp_rfc3526group05, _ccdh_gp_rfc3526group14, + _ccdh_gp_rfc3526group15, _ccdh_gp_rfc3526group16, _ccdh_gp_rfc3526group17, + _ccdh_gp_rfc3526group18, _ccdh_gp_rfc5114_MODP_1024_160, _ccdh_gp_rfc5114_MODP_2048_224, + _ccdh_gp_rfc5114_MODP_2048_256, _ccdh_gp_size, _ccdh_import_full, + _ccdh_import_priv, _ccdh_import_pub, _ccdh_init_gp_from_bytes, + _ccdigest, _ccdigest_init, _ccdigest_oid_lookup, _ccdigest_parallel, + _ccdigest_update, _ccdrbg_context_size, _ccdrbg_df_bc_init, + _ccdrbg_done, _ccdrbg_factory_nistctr, _ccdrbg_factory_nisthmac, + _ccdrbg_generate, _ccdrbg_init, _ccdrbg_must_reseed, _ccdrbg_reseed, + _ccec_affinify, _ccec_blind, _ccec_compact_export, _ccec_compact_export_pub, + _ccec_compact_generate_key, _ccec_compact_generate_key_init, + _ccec_compact_generate_key_step, _ccec_compact_import_priv, + _ccec_compact_import_priv_size, _ccec_compact_import_pub, + _ccec_compact_import_pub_size, _ccec_compact_transform_key, + _ccec_compressed_x962_export_pub, _ccec_compressed_x962_export_pub_size, + _ccec_compressed_x962_import_pub, _ccec_compute_key, _ccec_cp_192, + _ccec_cp_224, _ccec_cp_256, _ccec_cp_384, _ccec_cp_521, _ccec_curve_for_length_lookup, + _ccec_der_export_diversified_pub, _ccec_der_export_diversified_pub_size, + _ccec_der_export_priv, _ccec_der_export_priv_size, _ccec_der_import_diversified_pub, + _ccec_der_import_priv, _ccec_der_import_priv_keytype, _ccec_diversify_min_entropy_len, + _ccec_diversify_priv_twin, _ccec_diversify_pub, _ccec_diversify_pub_twin, + _ccec_export_affine_point, _ccec_export_affine_point_size, + _ccec_export_pub, _ccec_extract_rs, _ccec_full_add, _ccec_full_sub, + _ccec_generate_blinding_keys, _ccec_generate_key, _ccec_generate_key_deterministic, + _ccec_generate_key_fips, _ccec_generate_key_legacy, _ccec_generate_scalar_fips_retry, + _ccec_get_cp, _ccec_get_fullkey_components, _ccec_get_pubkey_components, + _ccec_import_affine_point, _ccec_import_pub, _ccec_is_compactable_pub, + _ccec_keysize_is_supported, _ccec_make_priv, _ccec_make_pub, + _ccec_mult_blinded, _ccec_pairwise_consistency_check, _ccec_print_full_key, + _ccec_print_public_key, _ccec_projectify, _ccec_raw_import_priv_only, + _ccec_raw_import_pub, _ccec_rfc6637_dh_curve_p256, _ccec_rfc6637_dh_curve_p521, + _ccec_rfc6637_unwrap_key, _ccec_rfc6637_unwrap_sha256_kek_aes128, + _ccec_rfc6637_unwrap_sha512_kek_aes256, _ccec_rfc6637_wrap_key, + _ccec_rfc6637_wrap_key_diversified, _ccec_rfc6637_wrap_key_size, + _ccec_rfc6637_wrap_sha256_kek_aes128, _ccec_rfc6637_wrap_sha512_kek_aes256, + _ccec_sign, _ccec_sign_composite, _ccec_sign_composite_msg, + _ccec_sign_msg, _ccec_signature_r_s_size, _ccec_unblind, _ccec_validate_pub, + _ccec_verify, _ccec_verify_composite, _ccec_verify_composite_digest, + _ccec_verify_composite_msg, _ccec_verify_digest, _ccec_verify_msg, + _ccec_x963_export, _ccec_x963_import_priv, _ccec_x963_import_priv_size, + _ccec_x963_import_pub, _ccec_x963_import_pub_size, _ccecb_block_size, + _ccecb_context_size, _ccecb_init, _ccecb_one_shot, _ccecb_one_shot_explicit, + _ccecb_update, _ccecdh_compute_shared_secret, _ccecdh_generate_key, + _ccecies_decrypt_gcm, _ccecies_decrypt_gcm_composite, _ccecies_decrypt_gcm_from_shared_secret, + _ccecies_decrypt_gcm_plaintext_size, _ccecies_decrypt_gcm_plaintext_size_cp, + _ccecies_decrypt_gcm_setup, _ccecies_encrypt_gcm, _ccecies_encrypt_gcm_ciphertext_size, + _ccecies_encrypt_gcm_composite, _ccecies_encrypt_gcm_from_shared_secret, + _ccecies_encrypt_gcm_setup, _ccecies_import_eph_pub, _ccecies_pub_key_size, + _ccecies_pub_key_size_cp, _cced25519_make_key_pair, _cced25519_make_pub, + _cced25519_make_pub_with_rng, _cced25519_sign, _cced25519_sign_with_rng, + _cced25519_verify, _cced448_make_key_pair, _cced448_make_pub, + _cced448_sign, _cced448_verify, _ccentropy_add_entropy, _ccentropy_digest_init, + _ccentropy_get_seed, _ccentropy_list_init, _ccentropy_lock_init, + _ccentropy_reset, _ccentropy_rng_init, _ccgcm_aad, _ccgcm_block_size, + _ccgcm_context_size, _ccgcm_finalize, _ccgcm_gmac, _ccgcm_inc_iv, + _ccgcm_init, _ccgcm_init_with_iv, _ccgcm_one_shot, _ccgcm_one_shot_legacy, + _ccgcm_reset, _ccgcm_set_iv, _ccgcm_set_iv_legacy, _ccgcm_update, + _cch2c, _cch2c_name, _cch2c_p256_sha256_sae_compat_info, _cch2c_p256_sha256_sswu_ro_info, + _cch2c_p384_sha384_sae_compat_info, _cch2c_p384_sha512_sswu_ro_info, + _cch2c_p521_sha512_sswu_ro_info, _cche_bytes_to_coeffs, _cche_ciphertext_apply_galois, + _cche_ciphertext_coeff_compose, _cche_ciphertext_coeff_dcrt_plaintext_mul, + _cche_ciphertext_coeff_decompose, _cche_ciphertext_coeff_decompose_nptexts, + _cche_ciphertext_coeff_plaintext_mul, _cche_ciphertext_correction_factor, + _cche_ciphertext_eval_dcrt_plaintext_mul, _cche_ciphertext_eval_plaintext_mul, + _cche_ciphertext_fresh_correction_factor, _cche_ciphertext_fresh_npolys, + _cche_ciphertext_fwd_ntt, _cche_ciphertext_galois_elt_rotate_rows_left, + _cche_ciphertext_galois_elt_rotate_rows_right, _cche_ciphertext_galois_elt_swap_columns, + _cche_ciphertext_inv_ntt, _cche_ciphertext_plaintext_add, + _cche_ciphertext_sizeof, _cche_coeffs_to_bytes, _cche_crt_compose, + _cche_dcrt_plaintext_encode, _cche_dcrt_plaintext_sizeof, + _cche_decode_poly_uint64, _cche_decode_simd_int64, _cche_decode_simd_uint64, + _cche_decrypt, _cche_deserialize_ciphertext_coeff, _cche_deserialize_ciphertext_eval, + _cche_deserialize_seeded_ciphertext_coeff, _cche_deserialize_seeded_ciphertext_eval, + _cche_encode_poly_uint64, _cche_encode_simd_int64, _cche_encode_simd_reduced_int64, + _cche_encode_simd_uint64, _cche_encrypt_symmetric, _cche_encryption_params_coefficient_moduli, + _cche_encryption_params_coefficient_nmoduli, _cche_encryption_params_plaintext_modulus, + _cche_encryption_params_polynomial_degree, _cche_galois_key_generate, + _cche_galois_key_load, _cche_galois_key_save, _cche_galois_key_sizeof, + _cche_param_ctx_ciphertext_ctx_nmoduli, _cche_param_ctx_coefficient_moduli, + _cche_param_ctx_he_scheme, _cche_param_ctx_init, _cche_param_ctx_key_ctx_nmoduli, + _cche_param_ctx_key_ctx_poly_nbytes, _cche_param_ctx_plaintext_modulus, + _cche_param_ctx_plaintext_modulus_inverse, _cche_param_ctx_polynomial_degree, + _cche_param_ctx_sizeof, _cche_param_ctx_supports_simd_encoding, + _cche_plaintext_sizeof, _cche_relin_key_generate, _cche_relin_key_load, + _cche_relin_key_save, _cche_relin_key_sizeof, _cche_rng_seed_sizeof, + _cche_secret_key_generate, _cche_secret_key_generate_from_seed, + _cche_secret_key_sizeof, _cche_serialize_ciphertext_coeff, + _cche_serialize_ciphertext_coeff_max_nskip_lsbs, _cche_serialize_ciphertext_coeff_nbytes, + _cche_serialize_ciphertext_eval, _cche_serialize_ciphertext_eval_nbytes, + _cche_serialize_seeded_ciphertext_coeff, _cche_serialize_seeded_ciphertext_coeff_nbytes, + _cche_serialize_seeded_ciphertext_eval, _cche_serialize_seeded_ciphertext_eval_nbytes, + _cchkdf, _cchkdf_expand, _cchkdf_extract, _cchmac, _cchmac_final, + _cchmac_init, _cchmac_update, _cchpke_initiator_encrypt, _cchpke_initiator_export, + _cchpke_initiator_seal, _cchpke_initiator_setup, _cchpke_kem_generate_key_pair, + _cchpke_params_sizeof_aead_key, _cchpke_params_sizeof_aead_nonce, + _cchpke_params_sizeof_aead_tag, _cchpke_params_sizeof_kdf_hash, + _cchpke_params_sizeof_kem_enc, _cchpke_params_sizeof_kem_pk, + _cchpke_params_sizeof_kem_pk_marshalled, _cchpke_params_sizeof_kem_shared_secret, + _cchpke_params_sizeof_kem_sk, _cchpke_params_x25519_AESGCM128_HKDF_SHA256, + _cchpke_params_x25519_AESGCM256_HKDF_SHA256, _cchpke_params_xwing_AESGCM128_HKDF_SHA256, + _cchpke_responder_decrypt, _cchpke_responder_export, _cchpke_responder_open, + _cchpke_responder_setup, _cckem_decapsulate, _cckem_derive_key_from_seed, + _cckem_encapsulate, _cckem_encapsulated_key_nbytes_ctx, _cckem_encapsulated_key_nbytes_info, + _cckem_export_privkey, _cckem_export_pubkey, _cckem_full_ctx_init, + _cckem_generate_key, _cckem_generate_key_with_seed, _cckem_import_privkey, + _cckem_import_pubkey, _cckem_kyber1024, _cckem_kyber768, _cckem_mlkem1024, + _cckem_mlkem768, _cckem_privkey_nbytes_ctx, _cckem_privkey_nbytes_info, + _cckem_pub_ctx_init, _cckem_pubkey_nbytes_ctx, _cckem_pubkey_nbytes_info, + _cckem_public_ctx, _cckem_seed_nbytes_ctx, _cckem_seed_nbytes_info, + _cckem_shared_key_nbytes_ctx, _cckem_shared_key_nbytes_info, + _cckem_sizeof_full_ctx, _cckem_sizeof_pub_ctx, _cckem_xwing_mlkem768x25519, + _cclr_aes_init, _cclr_block_nbytes, _cclr_decrypt_block, _cclr_encrypt_block, + _ccmd2_ltc_di, _ccmd4_ltc_di, _ccmd5_di, _ccmd5_ltc_di, _ccmgf, + _ccmldsa65, _ccmldsa87, _ccmldsa_derive_key_from_seed, _ccmldsa_export_privkey, + _ccmldsa_export_pubkey, _ccmldsa_full_ctx_init, _ccmldsa_generate_key, + _ccmldsa_generate_key_with_seed, _ccmldsa_hash_nbytes_ctx, + _ccmldsa_hash_nbytes_params, _ccmldsa_import_privkey, _ccmldsa_import_pubkey, + _ccmldsa_prehash, _ccmldsa_prehash_with_context, _ccmldsa_privkey_nbytes_ctx, + _ccmldsa_privkey_nbytes_params, _ccmldsa_pub_ctx_init, _ccmldsa_pubkey_nbytes_ctx, + _ccmldsa_pubkey_nbytes_params, _ccmldsa_public_ctx, _ccmldsa_seed_nbytes_ctx, + _ccmldsa_seed_nbytes_params, _ccmldsa_sign, _ccmldsa_sign_prehashed, + _ccmldsa_sign_with_context, _ccmldsa_signature_nbytes_ctx, + _ccmldsa_signature_nbytes_params, _ccmldsa_sizeof_full_ctx, + _ccmldsa_sizeof_pub_ctx, _ccmldsa_verify, _ccmldsa_verify_prehashed, + _ccmldsa_verify_prehashed_with_canary, _ccmldsa_verify_with_canary, + _ccmldsa_verify_with_context, _ccmldsa_verify_with_context_and_canary, + _ccmode_factory_cbc_decrypt, _ccmode_factory_cbc_encrypt, + _ccmode_factory_ccm_decrypt, _ccmode_factory_ccm_encrypt, + _ccmode_factory_cfb8_decrypt, _ccmode_factory_cfb8_encrypt, + _ccmode_factory_cfb_decrypt, _ccmode_factory_cfb_encrypt, + _ccmode_factory_ctr_crypt, _ccmode_factory_gcm_decrypt, _ccmode_factory_gcm_encrypt, + _ccmode_factory_ofb_crypt, _ccmode_factory_siv_decrypt, _ccmode_factory_siv_encrypt, + _ccmode_factory_xts_decrypt, _ccmode_factory_xts_encrypt, + _ccn_add, _ccn_add1, _ccn_bitlen, _ccn_cmp, _ccn_cmpn, _ccn_lprint, + _ccn_print, _ccn_read_uint, _ccn_set_bit, _ccn_seti, _ccn_sub, + _ccn_swap, _ccn_write_int, _ccn_write_int_size, _ccn_write_uint, + _ccn_write_uint_padded, _ccn_write_uint_padded_ct, _ccn_write_uint_size, + _ccn_xor, _ccn_zero, _ccnistkdf_ctr_cmac, _ccnistkdf_ctr_cmac_fixed, + _ccnistkdf_ctr_hmac, _ccnistkdf_ctr_hmac_fixed, _ccofb_block_size, + _ccofb_context_size, _ccofb_init, _ccofb_one_shot, _ccofb_update, + _ccoid_equal, _ccoid_payload, _ccoid_size, _ccpad_cts1_decrypt, + _ccpad_cts1_encrypt, _ccpad_cts2_decrypt, _ccpad_cts2_encrypt, + _ccpad_cts3_decrypt, _ccpad_cts3_encrypt, _ccpad_pkcs7_decode, + _ccpad_pkcs7_decrypt, _ccpad_pkcs7_ecb_decrypt, _ccpad_pkcs7_ecb_encrypt, + _ccpad_pkcs7_encrypt, _ccpad_xts_decrypt, _ccpad_xts_encrypt, + _ccpbkdf2_hmac, _ccpoly1305, _ccpoly1305_final, _ccpoly1305_init, + _ccpoly1305_update, _ccpost, _ccrc2_cbc_decrypt_mode, _ccrc2_cbc_encrypt_mode, + _ccrc2_cfb8_decrypt_mode, _ccrc2_cfb8_encrypt_mode, _ccrc2_cfb_decrypt_mode, + _ccrc2_cfb_encrypt_mode, _ccrc2_ctr_crypt_mode, _ccrc2_ecb_decrypt_mode, + _ccrc2_ecb_encrypt_mode, _ccrc2_ofb_crypt_mode, _ccrc4, _ccrc4_eay, + _ccrmd160_ltc_di, _ccrng, _ccrng_drbg_done, _ccrng_drbg_init, + _ccrng_drbg_init_withdrbg, _ccrng_drbg_reseed, _ccrng_ecfips_test_init, + _ccrng_pbkdf2_prng_init, _ccrng_prng, _ccrng_rsafips_test_init, + _ccrng_rsafips_test_set_next, _ccrng_sequence_init, _ccrng_system_done, + _ccrng_system_init, _ccrng_test_done, _ccrng_test_init, _ccrng_trng, + _ccrng_uniform, _ccrsa_block_size, _ccrsa_block_start, _ccrsa_ctx_private_zp, + _ccrsa_ctx_public, _ccrsa_decrypt_eme_pkcs1v15, _ccrsa_decrypt_oaep, + _ccrsa_dump_full_key, _ccrsa_dump_public_key, _ccrsa_eme_pkcs1v15_decode, + _ccrsa_eme_pkcs1v15_decode_safe, _ccrsa_eme_pkcs1v15_encode, + _ccrsa_emsa_pkcs1v15_encode, _ccrsa_emsa_pkcs1v15_verify, + _ccrsa_emsa_pss_decode, _ccrsa_emsa_pss_encode, _ccrsa_encrypt_eme_pkcs1v15, + _ccrsa_encrypt_oaep, _ccrsa_export_priv, _ccrsa_export_priv_size, + _ccrsa_export_pub, _ccrsa_export_pub_size, _ccrsa_generate_fips186_key, + _ccrsa_generate_key, _ccrsa_generate_key_deterministic, _ccrsa_get_fullkey_components, + _ccrsa_get_pubkey_components, _ccrsa_import_priv, _ccrsa_import_priv_n, + _ccrsa_import_pub, _ccrsa_import_pub_n, _ccrsa_init_pub, _ccrsa_make_priv, + _ccrsa_make_pub, _ccrsa_n_from_size, _ccrsa_oaep_decode, _ccrsa_oaep_decode_parameter, + _ccrsa_oaep_encode, _ccrsa_oaep_encode_parameter, _ccrsa_priv_crypt, + _ccrsa_pub_crypt, _ccrsa_pubkeylength, _ccrsa_recover_priv, + _ccrsa_sign_pkcs1v15, _ccrsa_sign_pkcs1v15_msg, _ccrsa_sign_pss, + _ccrsa_sign_pss_msg, _ccrsa_sizeof_n_from_size, _ccrsa_verify_pkcs1v15, + _ccrsa_verify_pkcs1v15_allowshortsigs, _ccrsa_verify_pkcs1v15_digest, + _ccrsa_verify_pkcs1v15_msg, _ccrsa_verify_pss_digest, _ccrsa_verify_pss_msg, + _ccrsabssa_blind_message, _ccrsabssa_ciphersuite_rsa2048_sha384, + _ccrsabssa_ciphersuite_rsa3072_sha384, _ccrsabssa_ciphersuite_rsa4096_sha384, + _ccrsabssa_sign_blinded_message, _ccrsabssa_unblind_signature, + _ccsae_generate_commitment, _ccsae_generate_commitment_finalize, + _ccsae_generate_commitment_init, _ccsae_generate_commitment_partial, + _ccsae_generate_confirmation, _ccsae_generate_h2c_commit, + _ccsae_generate_h2c_commit_finalize, _ccsae_generate_h2c_commit_init, + _ccsae_generate_h2c_pt, _ccsae_get_keys, _ccsae_init, _ccsae_init_p256_sha256, + _ccsae_init_p384_sha384, _ccsae_lexographic_order_key, _ccsae_sizeof_commitment, + _ccsae_sizeof_confirmation, _ccsae_sizeof_ctx, _ccsae_sizeof_kck, + _ccsae_sizeof_kck_h2c, _ccsae_sizeof_pt, _ccsae_verify_commitment, + _ccsae_verify_commitment_with_rejected_groups, _ccsae_verify_confirmation, + _ccscrypt, _ccscrypt_storage_size, _ccsha1_di, _ccsha1_eay_di, + _ccsha1_ltc_di, _ccsha224_di, _ccsha224_ltc_di, _ccsha256_di, + _ccsha256_ltc_di, _ccsha384_di, _ccsha384_ltc_di, _ccsha3_224_di, + _ccsha3_256_di, _ccsha3_384_di, _ccsha3_512_di, _ccsha512_256_di, + _ccsha512_256_ltc_di, _ccsha512_di, _ccsha512_ltc_di, _ccsigma_clear, + _ccsigma_clear_key, _ccsigma_compute_mac, _ccsigma_derive_session_keys, + _ccsigma_exclave_pairing_info, _ccsigma_export_key_share, + _ccsigma_import_peer_key_share, _ccsigma_import_peer_verification_key, + _ccsigma_import_signing_key, _ccsigma_init, _ccsigma_kex_init_ctx, + _ccsigma_kex_resp_ctx, _ccsigma_mfi_info, _ccsigma_mfi_nvm_info, + _ccsigma_open, _ccsigma_peer_role, _ccsigma_seal, _ccsigma_set_signing_function, + _ccsigma_sign, _ccsigma_verify, _ccsiv_aad, _ccsiv_block_size, + _ccsiv_ciphertext_size, _ccsiv_context_size, _ccsiv_crypt, + _ccsiv_hmac_aad, _ccsiv_hmac_block_size, _ccsiv_hmac_ciphertext_size, + _ccsiv_hmac_context_size, _ccsiv_hmac_crypt, _ccsiv_hmac_init, + _ccsiv_hmac_one_shot, _ccsiv_hmac_plaintext_size, _ccsiv_hmac_reset, + _ccsiv_hmac_set_nonce, _ccsiv_init, _ccsiv_one_shot, _ccsiv_plaintext_size, + _ccsiv_reset, _ccsiv_set_nonce, _ccspake_cp_256, _ccspake_cp_256_rfc, + _ccspake_cp_384, _ccspake_cp_384_rfc, _ccspake_cp_521, _ccspake_cp_521_rfc, + _ccspake_generate_L, _ccspake_get_session_key, _ccspake_kex_generate, + _ccspake_kex_process, _ccspake_mac_compute, _ccspake_mac_hkdf_cmac_aes128_sha256, + _ccspake_mac_hkdf_hmac_sha256, _ccspake_mac_hkdf_hmac_sha512, + _ccspake_mac_verify_and_get_session_key, _ccspake_prover_init, + _ccspake_prover_initialize, _ccspake_reduce_w, _ccspake_reduce_w_RFC9383, + _ccspake_sizeof_ctx, _ccspake_sizeof_point, _ccspake_sizeof_w, + _ccspake_verifier_init, _ccspake_verifier_initialize, _ccsrp_client_process_challenge, + _ccsrp_client_set_noUsernameInX, _ccsrp_client_start_authentication, + _ccsrp_client_verify_session, _ccsrp_ctx_init, _ccsrp_ctx_init_option, + _ccsrp_ctx_init_with_size_option, _ccsrp_exchange_size, _ccsrp_generate_salt_and_verification, + _ccsrp_generate_verifier, _ccsrp_get_premaster_secret, _ccsrp_get_session_key, + _ccsrp_get_session_key_length, _ccsrp_gp_rfc5054_1024, _ccsrp_gp_rfc5054_2048, + _ccsrp_gp_rfc5054_3072, _ccsrp_gp_rfc5054_4096, _ccsrp_gp_rfc5054_8192, + _ccsrp_is_authenticated, _ccsrp_server_compute_session, _ccsrp_server_generate_public_key, + _ccsrp_server_start_authentication, _ccsrp_server_verify_session, + _ccsrp_session_size, _ccsrp_sizeof_M_HAMK, _ccsrp_sizeof_public_key, + _ccsrp_sizeof_session_key, _ccsrp_sizeof_verifier, _ccss_shamir_parameters_init, + _ccss_shamir_parameters_maximum_secret_length, _ccss_shamir_share_bag_add_share, + _ccss_shamir_share_bag_init, _ccss_shamir_share_bag_recover_secret, + _ccss_shamir_share_export, _ccss_shamir_share_generator_deserialize, + _ccss_shamir_share_generator_generate_share, _ccss_shamir_share_generator_init, + _ccss_shamir_share_generator_init_with_secrets_less_than_prime, + _ccss_shamir_share_generator_serialize, _ccss_shamir_share_import, + _ccss_shamir_share_init, _ccss_shamir_share_sizeof_y, _ccss_sizeof_generator, + _ccss_sizeof_parameters, _ccss_sizeof_shamir_share_generator_serialization, + _ccss_sizeof_share, _ccss_sizeof_share_bag, _ccvrf_derive_public_key, + _ccvrf_factory_irtfdraft03, _ccvrf_factory_irtfdraft03_default, + _ccvrf_proof_to_hash, _ccvrf_prove, _ccvrf_sizeof_hash, _ccvrf_sizeof_proof, + _ccvrf_sizeof_public_key, _ccvrf_sizeof_secret_key, _ccvrf_verify, + _ccwrap_auth_decrypt, _ccwrap_auth_decrypt_withiv, _ccwrap_auth_encrypt, + _ccwrap_auth_encrypt_withiv, _ccwrap_unwrapped_size, _ccwrap_wrapped_size, + _ccxts_block_size, _ccxts_context_size, _ccxts_init, _ccxts_one_shot, + _ccxts_set_tweak, _ccxts_update, _ccz_add, _ccz_addi, _ccz_bit, + _ccz_bitlen, _ccz_capacity, _ccz_cmp, _ccz_cmpi, _ccz_divmod, + _ccz_expmod, _ccz_free, _ccz_init, _ccz_is_negative, _ccz_is_one, + _ccz_is_prime, _ccz_is_zero, _ccz_lsl, _ccz_lsr, _ccz_mod, + _ccz_mul, _ccz_muli, _ccz_mulmod, _ccz_n, _ccz_neg, _ccz_random_bits, + _ccz_read_radix, _ccz_read_uint, _ccz_set, _ccz_set_bit, _ccz_set_capacity, + _ccz_set_n, _ccz_set_sign, _ccz_seti, _ccz_sign, _ccz_size, + _ccz_sub, _ccz_subi, _ccz_trailing_zeros, _ccz_write_int, + _ccz_write_int_size, _ccz_write_radix, _ccz_write_radix_size, + _ccz_write_uint, _ccz_write_uint_size, _ccz_zero, _cczp_add, + _cczp_bitlen, _cczp_inv, _cczp_mod, _cczp_mul, _cczp_n, _cczp_prime, + _cczp_sub, _csss_shamir_share_bag_can_recover_secret, _fipspost_post, + _fipspost_trace_vtable, _map_to_curve_sswu ] + - targets: [ arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + symbols: [ _ccaes_arm_cbc_decrypt_mode, _ccaes_arm_cbc_encrypt_mode, + _ccaes_arm_cfb_decrypt_mode, _ccaes_arm_cfb_encrypt_mode, + _ccaes_arm_ecb_decrypt_mode, _ccaes_arm_ecb_encrypt_mode, + _ccaes_arm_ofb_crypt_mode, _ccaes_arm_xts_decrypt_mode, _ccaes_arm_xts_encrypt_mode, + _ccsha1_vng_arm_di, _ccsha224_vng_arm_di, _ccsha256_vng_arm64neon_di, + _ccsha256_vng_arm_di, _ccsha384_vng_arm_di, _ccsha384_vng_arm_hw_di, + _ccsha512_256_vng_arm_di, _ccsha512_256_vng_arm_hw_di, _ccsha512_vng_arm_di, + _ccsha512_vng_arm_hw_di ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libdispatch.dylib' +current-version: 1542.0.4 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ '$ld$hide$os10.12$_dispatch_assert_queue', '$ld$hide$os10.12$_dispatch_assert_queue_not', + '$ld$hide$os10.12$_dispatch_queue_create_with_target', __dispatch_begin_NSAutoReleasePool, + __dispatch_bug, __dispatch_data_destructor_free, __dispatch_data_destructor_munmap, + __dispatch_data_destructor_none, __dispatch_data_destructor_vm_deallocate, + __dispatch_data_empty, __dispatch_data_format_type_base32, + __dispatch_data_format_type_base32hex, __dispatch_data_format_type_base64, + __dispatch_data_format_type_none, __dispatch_data_format_type_utf16be, + __dispatch_data_format_type_utf16le, __dispatch_data_format_type_utf8, + __dispatch_data_format_type_utf_any, __dispatch_end_NSAutoReleasePool, + __dispatch_get_main_queue_handle_4CF, __dispatch_get_main_queue_port_4CF, + __dispatch_iocntl, __dispatch_is_fork_of_multithreaded_parent, + __dispatch_is_multithreaded, __dispatch_log, __dispatch_mach_hooks_install_default, + __dispatch_main_q, __dispatch_main_queue_callback_4CF, __dispatch_poll_for_events_4launchd, + __dispatch_prohibit_transition_to_multithreaded, __dispatch_pthread_root_queue_create_with_observer_hooks_4IOHID, + __dispatch_queue_attr_concurrent, __dispatch_queue_is_exclusively_owned_by_current_thread_4IOHID, + __dispatch_runloop_root_queue_create_4CF, __dispatch_runloop_root_queue_get_port_4CF, + __dispatch_runloop_root_queue_perform_4CF, __dispatch_runloop_root_queue_wakeup_4CF, + __dispatch_source_set_runloop_timer_4CF, __dispatch_source_type_data_add, + __dispatch_source_type_data_or, __dispatch_source_type_data_replace, + __dispatch_source_type_exclaves_notification, __dispatch_source_type_interval, + __dispatch_source_type_mach_recv, __dispatch_source_type_mach_send, + __dispatch_source_type_memorypressure, __dispatch_source_type_memorystatus, + __dispatch_source_type_nw_channel, __dispatch_source_type_proc, + __dispatch_source_type_read, __dispatch_source_type_signal, + __dispatch_source_type_sock, __dispatch_source_type_timer, + __dispatch_source_type_vfs, __dispatch_source_type_vm, __dispatch_source_type_vnode, + __dispatch_source_type_write, __dispatch_source_will_reenable_kevent_4NW, + __dispatch_wait_for_enqueuer, __dispatch_workloop_set_observer_hooks_4IOHID, + __dispatch_workloop_should_yield_4NW, __firehose_spi_version, + __os_object_alloc, __os_object_alloc_bridged, __os_object_alloc_realized, + __os_object_dealloc, __os_object_release, __os_object_release_internal, + __os_object_release_internal_n, __os_object_release_without_xref_dispose, + __os_object_retain, __os_object_retain_internal, __os_object_retain_internal_n, + __os_object_retain_with_resurrect, _dispatch_activate, _dispatch_after, + _dispatch_after_f, _dispatch_allocator_layout, _dispatch_allow_send_signals, + _dispatch_apply, _dispatch_apply_attr_destroy, _dispatch_apply_attr_init, + _dispatch_apply_attr_query, _dispatch_apply_attr_set_parallelism, + _dispatch_apply_f, _dispatch_apply_with_attr, _dispatch_apply_with_attr_f, + _dispatch_assert_queue, '_dispatch_assert_queue$V2', _dispatch_assert_queue_barrier, + _dispatch_assert_queue_not, '_dispatch_assert_queue_not$V2', + _dispatch_async, _dispatch_async_and_wait, _dispatch_async_and_wait_f, + _dispatch_async_enforce_qos_class_f, _dispatch_async_f, _dispatch_async_swift_job, + _dispatch_atfork_child, _dispatch_atfork_parent, _dispatch_atfork_prepare, + _dispatch_barrier_async, _dispatch_barrier_async_and_wait, + _dispatch_barrier_async_and_wait_f, _dispatch_barrier_async_f, + _dispatch_barrier_sync, _dispatch_barrier_sync_f, _dispatch_benchmark, + _dispatch_benchmark_f, _dispatch_block_cancel, _dispatch_block_create, + _dispatch_block_create_with_qos_class, _dispatch_block_create_with_voucher, + _dispatch_block_create_with_voucher_and_qos_class, _dispatch_block_notify, + _dispatch_block_perform, _dispatch_block_testcancel, _dispatch_block_wait, + _dispatch_channel_async, _dispatch_channel_async_f, _dispatch_channel_cancel, + _dispatch_channel_create, _dispatch_channel_drain, _dispatch_channel_drain_f, + _dispatch_channel_enqueue, _dispatch_channel_foreach_work_item_peek, + _dispatch_channel_foreach_work_item_peek_f, _dispatch_channel_testcancel, + _dispatch_channel_wakeup, _dispatch_data_apply, _dispatch_data_apply_f, + _dispatch_data_copy_region, _dispatch_data_create, _dispatch_data_create_alloc, + _dispatch_data_create_concat, _dispatch_data_create_f, _dispatch_data_create_map, + _dispatch_data_create_subrange, _dispatch_data_create_with_transform, + _dispatch_data_get_flattened_bytes_4libxpc, _dispatch_data_get_size, + _dispatch_data_make_memory_entry, _dispatch_debug, _dispatch_debugv, + _dispatch_get_context, _dispatch_get_current_queue, _dispatch_get_global_queue, + _dispatch_get_specific, _dispatch_group_async, _dispatch_group_async_f, + _dispatch_group_create, _dispatch_group_enter, _dispatch_group_leave, + _dispatch_group_notify, _dispatch_group_notify_f, _dispatch_group_wait, + _dispatch_io_barrier, _dispatch_io_barrier_f, _dispatch_io_close, + _dispatch_io_create, _dispatch_io_create_f, _dispatch_io_create_with_io, + _dispatch_io_create_with_io_f, _dispatch_io_create_with_path, + _dispatch_io_create_with_path_f, _dispatch_io_get_descriptor, + _dispatch_io_read, _dispatch_io_read_f, _dispatch_io_set_high_water, + _dispatch_io_set_interval, _dispatch_io_set_low_water, _dispatch_io_write, + _dispatch_io_write_f, _dispatch_lock_override_end, _dispatch_lock_override_start_with_debounce, + _dispatch_mach_can_handoff_4libxpc, _dispatch_mach_cancel, + _dispatch_mach_connect, _dispatch_mach_create, _dispatch_mach_create_4libxpc, + _dispatch_mach_create_f, _dispatch_mach_get_checkin_port, + _dispatch_mach_handoff_reply, _dispatch_mach_handoff_reply_f, + _dispatch_mach_hooks_install_4libxpc, _dispatch_mach_mig_demux, + _dispatch_mach_mig_demux_get_context, _dispatch_mach_msg_create, + _dispatch_mach_msg_get_context, _dispatch_mach_msg_get_filter_policy_id, + _dispatch_mach_msg_get_msg, _dispatch_mach_notify_no_senders, + _dispatch_mach_receive_barrier, _dispatch_mach_receive_barrier_f, + _dispatch_mach_reconnect, _dispatch_mach_request_no_senders, + _dispatch_mach_send, _dispatch_mach_send_and_wait_for_reply, + _dispatch_mach_send_barrier, _dispatch_mach_send_barrier_f, + _dispatch_mach_send_with_result, _dispatch_mach_send_with_result_and_async_reply_4libxpc, + _dispatch_mach_send_with_result_and_wait_for_reply, _dispatch_mach_set_flags, + _dispatch_main, _dispatch_mig_server, _dispatch_once, _dispatch_once_f, + _dispatch_pthread_root_queue_copy_current, _dispatch_pthread_root_queue_create, + _dispatch_queue_attr_make_initially_inactive, _dispatch_queue_attr_make_with_autorelease_frequency, + _dispatch_queue_attr_make_with_overcommit, _dispatch_queue_attr_make_with_qos_class, + _dispatch_queue_create, _dispatch_queue_create_with_accounting_override_voucher, + _dispatch_queue_create_with_target, '_dispatch_queue_create_with_target$V2', + _dispatch_queue_get_label, _dispatch_queue_get_qos_class, + _dispatch_queue_get_specific, _dispatch_queue_get_threadid_4wdt, + _dispatch_queue_offsets, _dispatch_queue_set_label_nocopy, + _dispatch_queue_set_specific, _dispatch_queue_set_width, _dispatch_read, + _dispatch_read_f, _dispatch_release, _dispatch_resume, _dispatch_retain, + _dispatch_semaphore_create, _dispatch_semaphore_signal, _dispatch_semaphore_wait, + _dispatch_set_context, _dispatch_set_finalizer_f, _dispatch_set_qos_class, + _dispatch_set_qos_class_fallback, _dispatch_set_qos_class_floor, + _dispatch_set_target_queue, _dispatch_source_cancel, _dispatch_source_cancel_and_wait, + _dispatch_source_create, _dispatch_source_get_data, _dispatch_source_get_extended_data, + _dispatch_source_get_handle, _dispatch_source_get_mask, _dispatch_source_merge_data, + _dispatch_source_set_cancel_handler, _dispatch_source_set_cancel_handler_f, + _dispatch_source_set_event_handler, _dispatch_source_set_event_handler_f, + _dispatch_source_set_mandatory_cancel_handler, _dispatch_source_set_mandatory_cancel_handler_f, + _dispatch_source_set_registration_handler, _dispatch_source_set_registration_handler_f, + _dispatch_source_set_timer, _dispatch_source_testcancel, _dispatch_suspend, + _dispatch_swift_job_should_yield, _dispatch_sync, _dispatch_sync_f, + _dispatch_thread_get_current_override_qos_floor, _dispatch_thread_override_self, + _dispatch_time, _dispatch_time_from_nsec, _dispatch_time_to_nsec, + _dispatch_time_to_nsecs, _dispatch_tsd_indexes, _dispatch_verify_current_queue_4swiftonly, + _dispatch_walltime, _dispatch_workloop_copy_current, _dispatch_workloop_create, + _dispatch_workloop_create_inactive, _dispatch_workloop_is_current, + _dispatch_workloop_set_autorelease_frequency, _dispatch_workloop_set_cpupercent, + _dispatch_workloop_set_os_workgroup, _dispatch_workloop_set_qos_class, + _dispatch_workloop_set_qos_class_floor, _dispatch_workloop_set_scheduler_priority, + _dispatch_workloop_set_uses_bound_thread, _dispatch_write, + _dispatch_write_f, _libdispatch_init, _mach_voucher_persona_for_originator, + _mach_voucher_persona_self, _os_eventlink_activate, _os_eventlink_associate, + _os_eventlink_cancel, _os_eventlink_create, _os_eventlink_create_remote_with_eventlink, + _os_eventlink_create_with_port, _os_eventlink_disassociate, + _os_eventlink_extract_remote_port, _os_eventlink_signal, _os_eventlink_signal_and_wait, + _os_eventlink_signal_and_wait_until, _os_eventlink_wait, _os_eventlink_wait_until, + _os_release, _os_retain, _os_workgroup_attr_set_flags, _os_workgroup_attr_set_interval_type, + _os_workgroup_attr_set_telemetry_flavor, _os_workgroup_cancel, + _os_workgroup_copy_port, _os_workgroup_create, _os_workgroup_create_with_port, + _os_workgroup_create_with_workgroup, _os_workgroup_create_with_workload_id, + _os_workgroup_create_with_workload_id_and_port, _os_workgroup_create_with_workload_id_and_workgroup, + _os_workgroup_get_working_arena, _os_workgroup_interval_copy_current_4AudioToolbox, + _os_workgroup_interval_create, _os_workgroup_interval_create_with_workload_id, + _os_workgroup_interval_data_set_complexity, _os_workgroup_interval_data_set_flags, + _os_workgroup_interval_data_set_telemetry, _os_workgroup_interval_finish, + _os_workgroup_interval_start, _os_workgroup_interval_update, + _os_workgroup_join, _os_workgroup_join_self, _os_workgroup_leave, + _os_workgroup_leave_self, _os_workgroup_max_parallel_threads, + _os_workgroup_parallel_create, _os_workgroup_set_working_arena, + _os_workgroup_testcancel, _voucher_activity_create_with_data, + _voucher_activity_create_with_data_2, _voucher_activity_create_with_location, + _voucher_activity_flush, _voucher_activity_get_logging_preferences, + _voucher_activity_get_logging_preferences_with_port, _voucher_activity_get_metadata_buffer, + _voucher_activity_id_allocate, _voucher_activity_initialize_4libtrace, + _voucher_activity_should_send_strings, _voucher_activity_trace, + _voucher_activity_trace_v, _voucher_activity_trace_v_2, _voucher_adopt, + _voucher_copy, _voucher_copy_with_persona_mach_voucher, _voucher_copy_with_persona_mach_voucher_and_error, + _voucher_copy_without_importance, _voucher_create_with_mach_msg, + _voucher_decrement_importance_count4CF, _voucher_get_activity_id, + _voucher_get_activity_id_and_creator, _voucher_get_current_persona, + _voucher_get_current_persona_originator_info, _voucher_get_current_persona_proximate_info, + _voucher_kvoucher_debug, _voucher_process_can_use_arbitrary_personas, + _voucher_release, _voucher_replace_default_voucher, _voucher_retain ] + objc-classes: [ OS_dispatch_channel, OS_dispatch_data, OS_dispatch_disk, OS_dispatch_group, + OS_dispatch_io, OS_dispatch_mach, OS_dispatch_mach_msg, OS_dispatch_object, + OS_dispatch_operation, OS_dispatch_queue, OS_dispatch_queue_attr, + OS_dispatch_queue_concurrent, OS_dispatch_queue_global, OS_dispatch_queue_main, + OS_dispatch_queue_serial, OS_dispatch_queue_serial_executor, + OS_dispatch_semaphore, OS_dispatch_source, OS_dispatch_workloop, + OS_object, OS_os_eventlink, OS_os_workgroup, OS_os_workgroup_interval, + OS_os_workgroup_parallel, OS_voucher ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libdyld.dylib' +current-version: 0 +compatibility-version: 0 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _NSVersionOfLinkTimeLibrary, _NSVersionOfRunTimeLibrary, _NXArgc, + _NXArgv, __NSGetExecutablePath, ___progname, __dyld_atfork_parent, + __dyld_atfork_prepare, __dyld_call_with_writable_tpro_memory, + __dyld_dlopen_atfork_child, __dyld_dlopen_atfork_parent, __dyld_dlopen_atfork_prepare, + __dyld_dlsym_blocked, __dyld_find_foreign_type_protocol_conformance, + __dyld_find_foreign_type_protocol_conformance_on_disk, __dyld_find_pointer_hash_table_entry, + __dyld_find_protocol_conformance, __dyld_find_protocol_conformance_on_disk, + __dyld_find_unwind_sections, __dyld_for_each_objc_class, __dyld_for_each_objc_protocol, + __dyld_for_each_prewarming_range, __dyld_for_objc_header_opt_ro, + __dyld_for_objc_header_opt_rw, __dyld_fork_child, __dyld_get_dlopen_image_header, + __dyld_get_image_header, __dyld_get_image_name, __dyld_get_image_slide, + __dyld_get_image_uuid, __dyld_get_image_vmaddr_slide, __dyld_get_objc_selector, + __dyld_get_prog_image_header, __dyld_get_shared_cache_range, + __dyld_get_shared_cache_uuid, __dyld_get_swift_prespecialized_data, + __dyld_has_fix_for_radar, __dyld_has_preoptimized_swift_protocol_conformances, + __dyld_image_count, __dyld_images_for_addresses, __dyld_initializer, + __dyld_is_memory_immutable, __dyld_is_objc_constant, __dyld_is_preoptimized_objc_image_loaded, + __dyld_is_pseudodylib, __dyld_launch_mode, __dyld_lookup_section_info, + __dyld_objc_class_count, __dyld_objc_notify_register, __dyld_objc_register_callbacks, + __dyld_objc_uses_large_shared_cache, __dyld_process_info_create, + __dyld_process_info_for_each_image, __dyld_process_info_for_each_segment, + __dyld_process_info_get_aot_cache, __dyld_process_info_get_cache, + __dyld_process_info_get_platform, __dyld_process_info_get_state, + __dyld_process_info_notify, __dyld_process_info_notify_main, + __dyld_process_info_notify_release, __dyld_process_info_notify_retain, + __dyld_process_info_release, __dyld_process_info_retain, __dyld_pseudodylib_deregister, + __dyld_pseudodylib_deregister_callbacks, __dyld_pseudodylib_register, + __dyld_pseudodylib_register_callbacks, __dyld_register_dlsym_notifier, + __dyld_register_driverkit_main, __dyld_register_for_bulk_image_loads, + __dyld_register_for_image_loads, __dyld_register_func_for_add_image, + __dyld_register_func_for_remove_image, __dyld_shared_cache_contains_path, + __dyld_shared_cache_is_locally_built, __dyld_shared_cache_optimized, + __dyld_shared_cache_real_path, __dyld_stack_bottom, __dyld_stack_range, + __dyld_stack_top, __dyld_swift_optimizations_version, __dyld_visit_objc_classes, + __tlv_atexit, __tlv_bootstrap, __tlv_exit, _dladdr, _dlclose, + _dlerror, _dlopen, _dlopen_audited, _dlopen_from, _dlopen_preflight, + _dlsym, _dyldVersionNumber, _dyldVersionString, _dyld_dynamic_interpose, + _dyld_for_each_installed_shared_cache, _dyld_for_each_installed_shared_cache_with_system_path, + _dyld_get_active_platform, _dyld_get_base_platform, _dyld_get_image_versions, + _dyld_get_min_os_version, _dyld_get_program_min_os_version, + _dyld_get_program_minos_version_token, _dyld_get_program_sdk_version, + _dyld_get_program_sdk_version_token, _dyld_get_sdk_version, + _dyld_has_inserted_or_interposing_libraries, _dyld_image_content_for_section, + _dyld_image_content_for_segment, _dyld_image_copy_uuid, _dyld_image_for_each_section_info, + _dyld_image_for_each_segment_info, _dyld_image_get_file_path, + _dyld_image_get_installname, _dyld_image_header_containing_address, + _dyld_image_local_nlist_content_4Symbolication, _dyld_image_path_containing_address, + _dyld_is_simulator_platform, _dyld_minos_at_least, _dyld_need_closure, + _dyld_process_create_for_current_task, _dyld_process_create_for_task, + _dyld_process_dispose, _dyld_process_has_objc_patches, _dyld_process_is_restricted, + _dyld_process_register_for_event_notification, _dyld_process_register_for_image_notifications, + _dyld_process_snapshot_create_for_process, _dyld_process_snapshot_create_from_data, + _dyld_process_snapshot_dispose, _dyld_process_snapshot_for_each_image, + _dyld_process_snapshot_get_shared_cache, _dyld_process_unregister_for_notification, + _dyld_program_minos_at_least, _dyld_program_sdk_at_least, + _dyld_sdk_at_least, _dyld_shared_cache_copy_uuid, _dyld_shared_cache_file_path, + _dyld_shared_cache_find_iterate_text, _dyld_shared_cache_for_each_file, + _dyld_shared_cache_for_each_image, _dyld_shared_cache_for_file, + _dyld_shared_cache_get_base_address, _dyld_shared_cache_get_mapped_size, + _dyld_shared_cache_is_mapped_private, _dyld_shared_cache_iterate_text, + _dyld_shared_cache_pin_mapping, _dyld_shared_cache_some_image_overridden, + _dyld_shared_cache_unpin_mapping, _dyld_version_token_at_least, + _dyld_version_token_get_platform, _environ, _macho_arch_name_for_cpu_type, + _macho_arch_name_for_mach_header, _macho_best_slice, _macho_best_slice_in_fd, + _macho_cpu_type_for_arch_name, _macho_dylib_install_name, + _macho_for_each_defined_rpath, _macho_for_each_dependent_dylib, + _macho_for_each_exported_symbol, _macho_for_each_imported_symbol, + _macho_for_each_runnable_arch_name, _macho_for_each_slice, + _macho_for_each_slice_in_fd, _macho_source_version, dyld_stub_binder ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ _NSAddImage, _NSAddLibrary, _NSAddLibraryWithSearching, _NSAddressOfSymbol, + _NSCreateObjectFileImageFromFile, _NSCreateObjectFileImageFromMemory, + _NSDestroyObjectFileImage, _NSGetSectionDataInObjectFileImage, + _NSInstallLinkEditErrorHandlers, _NSIsSymbolDefinedInObjectFileImage, + _NSIsSymbolNameDefined, _NSIsSymbolNameDefinedInImage, _NSIsSymbolNameDefinedWithHint, + _NSLibraryNameForModule, _NSLinkEditError, _NSLinkModule, + _NSLookupAndBindSymbol, _NSLookupAndBindSymbolWithHint, _NSLookupSymbolInImage, + _NSLookupSymbolInModule, _NSModuleForSymbol, _NSNameOfModule, + _NSNameOfSymbol, _NSSymbolDefinitionCountInObjectFileImage, + _NSSymbolDefinitionNameInObjectFileImage, _NSSymbolReferenceCountInObjectFileImage, + _NSSymbolReferenceNameInObjectFileImage, _NSUnLinkModule, + __dyld_all_twolevel_modules_prebound, __dyld_bind_fully_image_containing_address, + __dyld_get_image_header_containing_address, __dyld_image_containing_address, + __dyld_launched_prebound, __dyld_lookup_and_bind, __dyld_lookup_and_bind_fully, + __dyld_lookup_and_bind_with_hint, __dyld_present, __dyld_process_info_for_each_aot_image, + _dyld_shared_cache_for_each_subcache4Rosetta, _dyld_shared_subcache_get_info4Rosetta ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libkeymgr.dylib' +current-version: 31 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___keymgr_dwarf2_register_sections, ___keymgr_initializer, + __keymgr_get_and_lock_processwide_ptr, __keymgr_get_and_lock_processwide_ptr_2, + __keymgr_get_lock_count_processwide_ptr, __keymgr_get_lockmode_processwide_ptr, + __keymgr_set_and_unlock_processwide_ptr, __keymgr_set_lockmode_processwide_ptr, + __keymgr_unlock_processwide_ptr, _keymgrVersionNumber, _keymgrVersionString ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, arm64-macos, arm64e-macos ] +install-name: '/usr/lib/system/libmacho.dylib' +current-version: 1030.6.3 +parent-umbrella: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + umbrella: System +exports: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ _NXCombineCpuSubtypes, _NXFindBestFatArch, _NXFindBestFatArch_64, + _NXFreeArchInfo, _NXGetAllArchInfos, _NXGetArchInfoFromCpuType, + _NXGetArchInfoFromName, _NXGetLocalArchInfo, _get_edata, _get_end, + _get_etext, _getsectbyname, _getsectbynamefromheader, _getsectbynamefromheader_64, + _getsectbynamefromheaderwithswap, _getsectbynamefromheaderwithswap_64, + _getsectdata, _getsectdatafromFramework, _getsectdatafromheader, + _getsectdatafromheader_64, _getsectiondata, _getsegbyname, + _getsegmentdata, _slot_name, _swap_build_tool_version, _swap_build_version_command, + _swap_dyld_info_command, _swap_dylib_command, _swap_dylib_module, + _swap_dylib_module_64, _swap_dylib_reference, _swap_dylib_table_of_contents, + _swap_dylinker_command, _swap_dysymtab_command, _swap_encryption_command, + _swap_encryption_command_64, _swap_entry_point_command, _swap_fat_arch, + _swap_fat_arch_64, _swap_fat_header, _swap_fileset_entry_command, + _swap_fvmfile_command, _swap_fvmlib_command, _swap_i386_exception_state, + _swap_i386_float_state, _swap_i386_thread_state, _swap_ident_command, + _swap_indirect_symbols, _swap_linkedit_data_command, _swap_linker_option_command, + _swap_load_command, _swap_mach_header, _swap_mach_header_64, + _swap_nlist, _swap_nlist_64, _swap_note_command, _swap_prebind_cksum_command, + _swap_prebound_dylib_command, _swap_ranlib, _swap_ranlib_64, + _swap_relocation_info, _swap_routines_command, _swap_routines_command_64, + _swap_rpath_command, _swap_section, _swap_section_64, _swap_segment_command, + _swap_segment_command_64, _swap_source_version_command, _swap_sub_client_command, + _swap_sub_framework_command, _swap_sub_library_command, _swap_sub_umbrella_command, + _swap_symseg_command, _swap_symtab_command, _swap_thread_command, + _swap_twolevel_hint, _swap_twolevel_hints_command, _swap_uuid_command, + _swap_version_min_command, _swap_x86_debug_state, _swap_x86_debug_state32, + _swap_x86_debug_state64, _swap_x86_exception_state, _swap_x86_exception_state64, + _swap_x86_float_state, _swap_x86_float_state64, _swap_x86_state_hdr, + _swap_x86_thread_state, _swap_x86_thread_state64 ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libquarantine.dylib' +current-version: 196.40.3 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __qtn_error, __qtn_file_alloc, __qtn_file_apply_to_fd, __qtn_file_apply_to_mount_point, + __qtn_file_apply_to_path, __qtn_file_clone, __qtn_file_free, + __qtn_file_get_flags, __qtn_file_get_identifier, __qtn_file_get_metadata, + __qtn_file_get_metadata_size, __qtn_file_get_timestamp, __qtn_file_init, + __qtn_file_init_with_data, __qtn_file_init_with_disk_image_backing_store, + __qtn_file_init_with_fd, __qtn_file_init_with_mount_point, + __qtn_file_init_with_path, __qtn_file_set_flags, __qtn_file_set_identifier, + __qtn_file_set_metadata, __qtn_file_set_timestamp, __qtn_file_to_data, + __qtn_label_name, __qtn_proc_alloc, __qtn_proc_apply_to_pid, + __qtn_proc_apply_to_self, __qtn_proc_clone, __qtn_proc_free, + __qtn_proc_get_flags, __qtn_proc_get_identifier, __qtn_proc_get_metadata, + __qtn_proc_get_metadata_size, __qtn_proc_get_path_exclusion_pattern, + __qtn_proc_get_tracking_data, __qtn_proc_get_tracking_size, + __qtn_proc_init, __qtn_proc_init_with_data, __qtn_proc_init_with_self, + __qtn_proc_set_flags, __qtn_proc_set_identifier, __qtn_proc_set_metadata, + __qtn_proc_set_path_exclusion_pattern, __qtn_proc_set_tracking_data, + __qtn_proc_to_data, __qtn_xattr_name, _qtn_proc_init_with_pid, + _responsibility_get_pid_responsible_for_pid, _responsibility_get_responsible_for_pid, + _responsibility_get_uniqueid_responsible_for_pid, _responsibility_init, + _responsibility_set_pid_responsible_for_pid ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ _qtn_spawnattrs_get_tracking_data, _qtn_spawnattrs_set_tracking_data, + _responsibility_get_attribution_for_audittoken, _responsibility_get_responsible_audit_token_for_audit_token, + _responsibility_identity_get_binary_entitlement_data, _responsibility_identity_get_binary_is_platform, + _responsibility_identity_get_binary_offset, _responsibility_identity_get_binary_path, + _responsibility_identity_get_binary_signing_id, _responsibility_identity_get_binary_team_id, + _responsibility_identity_get_csflags, _responsibility_identity_get_hosted_path, + _responsibility_identity_get_hosted_team_id, _responsibility_identity_get_persistent_identifier, + _responsibility_identity_get_platform, _responsibility_identity_get_sdk, + _responsibility_identity_get_user_uuid, _responsibility_identity_open_binary_fd, + _responsibility_identity_release, _responsibility_set_audittoken_responsible_for_caller, + _responsibility_set_audittoken_responsible_for_self, _responsibility_set_caller_responsible_for_self, + _responsibility_set_hosted_path, _responsibility_set_hosted_team_id, + _responsibility_spawnattrs_getdisclaim, _responsibility_spawnattrs_setdisclaim ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libremovefile.dylib' +current-version: 84 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___removefile_init_random, ___removefile_random_char, ___removefile_randomize_buffer, + ___removefile_rename_unlink, ___removefile_sunlink, ___removefile_tree_walker, + ___removefile_tree_walker_slim, _removefile, _removefile_cancel, + _removefile_state_alloc, _removefile_state_free, _removefile_state_get, + _removefile_state_set, _removefileat ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_asl.dylib' +current-version: 404 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _ASL_LEVEL_TO_STRING, __asl_evaluate_send, __asl_fork_child, + __asl_lib_log, __asl_log_args_to_xpc, __asl_msg_dump, __asl_mt_shim_fork_child, + __asl_mt_shim_send_message, __asl_server_cancel_direct_watch, + __asl_server_control_query, __asl_server_create_aux_link, + __asl_server_match, __asl_server_message, __asl_server_query_2, + __asl_server_register_direct_watch, _asl_add_log_file, _asl_add_output, + _asl_add_output_file, _asl_append, _asl_base_msg, _asl_client_add_output_file, + _asl_client_get_control, _asl_client_kvdict, _asl_client_match, + _asl_client_open, _asl_client_open_from_file, _asl_client_release, + _asl_client_remove_output_file, _asl_client_retain, _asl_client_search, + _asl_client_send, _asl_client_set_control, _asl_client_set_filter, + _asl_client_set_output_file_filter, _asl_close, _asl_close_auxiliary_file, + _asl_core_check_access, _asl_core_decode_buffer, _asl_core_encode_buffer, + _asl_core_error, _asl_core_get_service_port, _asl_core_htonq, + _asl_core_level_to_string, _asl_core_new_msg_id, _asl_core_ntohq, + _asl_core_parse_time, _asl_core_str_match, _asl_core_str_match_c_time, + _asl_core_str_match_char, _asl_core_str_to_size, _asl_core_str_to_time, + _asl_core_str_to_uint32, _asl_core_string_hash, _asl_core_time_to_str, + _asl_count, _asl_create_auxiliary_file, _asl_fetch_key_val_op, + _asl_file_close, _asl_file_compact, _asl_file_ctime, _asl_file_fetch, + _asl_file_fetch_next, _asl_file_fetch_previous, _asl_file_filter, + _asl_file_filter_level, _asl_file_list_add, _asl_file_list_close, + _asl_file_list_match, _asl_file_list_match_end, _asl_file_list_match_next, + _asl_file_list_match_start, _asl_file_match, _asl_file_open_read, + _asl_file_open_write, _asl_file_read_set_position, _asl_file_release, + _asl_file_retain, _asl_file_save, _asl_file_size, _asl_filesystem_path, + _asl_format, _asl_format_message, _asl_free, _asl_get, _asl_get_filter, + _asl_get_index, _asl_get_local_control, _asl_get_type, _asl_get_value_for_key, + _asl_key, _asl_legacy1_close, _asl_legacy1_fetch, _asl_legacy1_match, + _asl_legacy1_open, _asl_list_from_string, _asl_log, _asl_log_auxiliary_location, + _asl_log_descriptor, _asl_log_message, _asl_match, _asl_msg_cmp, + _asl_msg_cmp_list, _asl_msg_copy, _asl_msg_count, _asl_msg_fetch, + _asl_msg_from_string, _asl_msg_get_val_for_key, _asl_msg_key, + _asl_msg_list_append, _asl_msg_list_count, _asl_msg_list_from_string, + _asl_msg_list_get_index, _asl_msg_list_insert, _asl_msg_list_match, + _asl_msg_list_new, _asl_msg_list_new_count, _asl_msg_list_next, + _asl_msg_list_prepend, _asl_msg_list_prev, _asl_msg_list_release, + _asl_msg_list_remove_index, _asl_msg_list_reset_iteration, + _asl_msg_list_retain, _asl_msg_list_search, _asl_msg_list_to_asl_string, + _asl_msg_list_to_string, _asl_msg_lookup, _asl_msg_merge, + _asl_msg_new, _asl_msg_release, _asl_msg_retain, _asl_msg_set_key_val, + _asl_msg_set_key_val_op, _asl_msg_to_string, _asl_msg_type, + _asl_msg_unset, _asl_msg_unset_index, _asl_new, _asl_next, + _asl_object_append, _asl_object_count, _asl_object_get_key_val_op_at_index, + _asl_object_get_object_at_index, _asl_object_get_val_op_for_key, + _asl_object_match, _asl_object_next, _asl_object_prepend, + _asl_object_prev, _asl_object_remove_object_at_index, _asl_object_search, + _asl_object_set_iteration_index, _asl_object_set_key_val_op, + _asl_object_unset_key, _asl_open, _asl_open_from_file, _asl_open_path, + _asl_parse_time, _asl_prepend, _asl_prev, _asl_release, _asl_remote_notify_name, + _asl_remove_log_file, _asl_remove_output, _asl_remove_output_file, + _asl_reset_iteration, _asl_retain, _asl_search, _asl_send, + _asl_set, _asl_set_filter, _asl_set_key_val_op, _asl_set_local_control, + _asl_set_output_file_filter, _asl_set_query, _asl_store_close, + _asl_store_location, _asl_store_match, _asl_store_match_next, + _asl_store_match_start, _asl_store_match_timeout, _asl_store_max_file_size, + _asl_store_open_aux, _asl_store_open_read, _asl_store_open_write, + _asl_store_release, _asl_store_retain, _asl_store_save, _asl_store_set_flags, + _asl_store_statistics, _asl_store_sweep_file_cache, _asl_string_allocated_size, + _asl_string_append, _asl_string_append_asl_key, _asl_string_append_asl_msg, + _asl_string_append_char_no_encoding, _asl_string_append_no_encoding, + _asl_string_append_no_encoding_len, _asl_string_append_op, + _asl_string_append_xml_tag, _asl_string_bytes, _asl_string_length, + _asl_string_new, _asl_string_release, _asl_string_release_return_bytes, + _asl_string_retain, _asl_syslog_faciliy_name_to_num, _asl_syslog_faciliy_num_to_name, + _asl_trigger_aslmanager, _asl_unset, _asl_unset_key, _asl_vlog, + _aslresponse_free, _aslresponse_next, _closelog, _openlog, + _setlogmask, _syslog, '_syslog$DARWIN_EXTSN', _vsyslog ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_blocks.dylib' +current-version: 96 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _Block_size, __Block_copy, __Block_extended_layout, __Block_has_signature, + __Block_isDeallocating, __Block_layout, __Block_object_assign, + __Block_object_dispose, __Block_release, __Block_signature, + __Block_tryRetain, __Block_use_RR2, __Block_use_stret, __NSConcreteAutoBlock, + __NSConcreteFinalizingBlock, __NSConcreteGlobalBlock, __NSConcreteMallocBlock, + __NSConcreteStackBlock, __NSConcreteWeakBlockVariable ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_c.dylib' +current-version: 1725.40.4 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst ] + symbols: [ '___opendir2$INODE64', '__readdir_unlocked$INODE64', '__seekdir$INODE64', + '_alphasort$INODE64', '_daemon$1050', '_fdopendir$INODE64', + _fstatx64_np, '_fstatx_np$INODE64', '_fts_children$INODE64', + '_fts_close$INODE64', '_fts_open$INODE64', '_fts_open_b$INODE64', + '_fts_read$INODE64', '_fts_set$INODE64', '_ftw$INODE64', '_getmntinfo$INODE64', + _getmntinfo64, '_getmntinfo_r_np$INODE64', '_glob$INODE64', + '_glob_b$INODE64', _lstatx64_np, '_lstatx_np$INODE64', '_nftw$INODE64', + '_opendir$INODE64', '_readdir$INODE64', '_readdir_r$INODE64', + '_rewinddir$INODE64', '_scandir$INODE64', '_scandir_b$INODE64', + '_seekdir$INODE64', _statx64_np, '_statx_np$INODE64', _strtoencf80_l, + '_telldir$INODE64', mcount ] + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ '$ld$weak$os10.11$_basename_r', '$ld$weak$os10.11$_clock_getres', + '$ld$weak$os10.11$_clock_gettime', '$ld$weak$os10.11$_clock_settime', + '$ld$weak$os10.11$_dirname_r', '$ld$weak$os10.11$_getentropy', + '$ld$weak$os10.11$_mkostemp', '$ld$weak$os10.11$_mkostemps', + '$ld$weak$os10.11$_timingsafe_bcmp', '$ld$weak$os10.12$_utimensat', + __CurrentRuneLocale, __DefaultRuneLocale, __Exit, __NSGetArgc, + __NSGetArgv, __NSGetEnviron, __NSGetMachExecuteHeader, __NSGetProgname, + __PathLocale, __Read_RuneMagi, ___Balloc_D2A, ___Bfree_D2A, + ____mb_cur_max, ____mb_cur_max_l, ____runetype, ____runetype_l, + ____tolower, ____tolower_l, ____toupper, ____toupper_l, ___add_ovflpage, + ___addel, ___any_on_D2A, ___assert_rtn, ___b2d_D2A, ___big_delete, + ___big_insert, ___big_keydata, ___big_return, ___big_split, + ___bigtens_D2A, ___bt_close, ___bt_cmp, ___bt_defcmp, ___bt_defpfx, + ___bt_delete, ___bt_dleaf, ___bt_fd, ___bt_free, ___bt_get, + ___bt_new, ___bt_open, ___bt_pgin, ___bt_pgout, ___bt_put, + ___bt_ret, ___bt_search, ___bt_seq, ___bt_setcur, ___bt_split, + ___bt_sync, ___buf_free, ___call_hash, ___cleanup, ___cmp_D2A, + ___collate_equiv_match, ___collate_load_error, ___collate_lookup, + ___collate_lookup_l, ___copybits_D2A, ___cxa_atexit, ___cxa_finalize, + ___cxa_finalize_ranges, ___cxa_thread_atexit, ___d2b_D2A, + ___dbpanic, ___default_hash, ___default_utx, ___delpair, ___diff_D2A, + ___dtoa, ___expand_table, ___fflush, ___fgetwc, ___find_bigpair, + ___find_last_page, ___fix_locale_grouping_str, ___fread, ___free_ovflpage, + ___freedtoa, ___gdtoa, ___gdtoa_locks, ___get_buf, ___get_page, + ___getonlyClocaleconv, ___hash_open, ___hdtoa, ___hi0bits_D2A, + ___hldtoa, ___i2b_D2A, ___ibitmap, ___isctype, ___istype, + ___istype_l, ___ldtoa, ___libc_init, ___lo0bits_D2A, ___log2, + ___lshift_D2A, ___maskrune, ___maskrune_l, ___match_D2A, ___mb_cur_max, + ___mb_sb_limit, ___memccpy_chk, ___memcpy_chk, ___memmove_chk, + ___memset_chk, ___mult_D2A, ___multadd_D2A, ___nrv_alloc_D2A, + ___opendir2, ___ovfl_delete, ___ovfl_get, ___ovfl_put, ___pow5mult_D2A, + ___put_page, ___quorem_D2A, ___ratio_D2A, ___rec_close, ___rec_delete, + ___rec_dleaf, ___rec_fd, ___rec_fmap, ___rec_fpipe, ___rec_get, + ___rec_iput, ___rec_open, ___rec_put, ___rec_ret, ___rec_search, + ___rec_seq, ___rec_sync, ___rec_vmap, ___rec_vpipe, ___reclaim_buf, + ___rshift_D2A, ___rv_alloc_D2A, ___s2b_D2A, ___sF, ___sclose, + ___sdidinit, ___setonlyClocaleconv, ___sflags, ___sflush, + ___sfp, ___sfvwrite, ___sglue, ___sinit, ___slbexpand, ___smakebuf, + ___snprintf_chk, ___snprintf_object_size_chk, ___split_page, + ___sprintf_chk, ___sprintf_object_size_chk, ___sread, ___srefill, + ___srget, ___sseek, ___stack_chk_fail, ___stack_chk_guard, + ___stderrp, ___stdinp, ___stdoutp, ___stpcpy_chk, ___stpncpy_chk, + ___strcat_chk, ___strcp_D2A, ___strcpy_chk, ___strlcat_chk, + ___strlcpy_chk, ___strncat_chk, ___strncpy_chk, ___sum_D2A, + ___svfscanf, ___swbuf, ___swhatbuf, ___swrite, ___swsetup, + ___tens_D2A, ___tinytens_D2A, ___tolower, ___tolower_l, ___toupper, + ___toupper_l, ___trailz_D2A, ___ulp_D2A, ___ungetc, ___ungetwc, + ___vsnprintf_chk, ___vsprintf_chk, ___wcwidth, ___wcwidth_l, + ___xlocale_C_collate, __allocenvstate, __atexit_receipt, __c_locale, + __cleanup, __closeutx, __collate_lookup, __copyenv, __cthread_init_routine, + __deallocenvstate, __endutxent, __flockfile_debug_stub, __fseeko, + __ftello, __fwalk, __getenvp, __getutxent, __getutxid, __getutxline, + __inet_aton_check, __init_clock_port, __int_to_time, __libc_fork_child, + __libc_fork_parent, __libc_fork_prepare, __libc_initializer, + __long_to_time, __mkpath_np, __mktemp, __openutx, __os_assert_log, + __os_assert_log_ctx, __os_assumes_log, __os_assumes_log_ctx, + __os_avoid_tail_call, __os_crash, __os_crash_callback, __os_crash_fmt, + __os_crash_msg, __os_debug_log, __os_debug_log_error_offset, + __os_debug_log_error_str, __putenvp, __pututxline, __rand48_add, + __rand48_mult, __rand48_seed, __readdir_unlocked, __reclaim_telldir, + __seekdir, __setenvp, __setutxent, __sigaction_nobind, __sigintr, + __signal_nobind, __sigvec_nobind, __sread, __sseek, __subsystem_init, + __swrite, __time32_to_time, __time64_to_time, __time_to_int, + __time_to_long, __time_to_time32, __time_to_time64, __unsetenvp, + __utmpxname, _a64l, _abort, _abort_report_np, _abs, _acl_add_flag_np, + _acl_add_perm, _acl_calc_mask, _acl_clear_flags_np, _acl_clear_perms, + _acl_copy_entry, _acl_copy_ext, _acl_copy_ext_native, _acl_copy_int, + _acl_copy_int_native, _acl_create_entry, _acl_create_entry_np, + _acl_delete_def_file, _acl_delete_entry, _acl_delete_fd_np, + _acl_delete_file_np, _acl_delete_flag_np, _acl_delete_link_np, + _acl_delete_perm, _acl_dup, _acl_free, _acl_from_text, _acl_get_entry, + _acl_get_fd, _acl_get_fd_np, _acl_get_file, _acl_get_flag_np, + _acl_get_flagset_np, _acl_get_link_np, _acl_get_perm_np, _acl_get_permset, + _acl_get_permset_mask_np, _acl_get_qualifier, _acl_get_tag_type, + _acl_init, _acl_maximal_permset_mask_np, _acl_set_fd, _acl_set_fd_np, + _acl_set_file, _acl_set_flagset_np, _acl_set_link_np, _acl_set_permset, + _acl_set_permset_mask_np, _acl_set_qualifier, _acl_set_tag_type, + _acl_size, _acl_to_text, _acl_valid, _acl_valid_fd_np, _acl_valid_file_np, + _acl_valid_link, _addr2ascii, _alarm, _alphasort, _arc4random, + _arc4random_addrandom, _arc4random_buf, _arc4random_stir, + _arc4random_uniform, _ascii2addr, _asctime, _asctime_r, _asprintf, + _asprintf_l, _asxprintf, _asxprintf_exec, _at_quick_exit, + _atexit, _atexit_b, _atof, _atof_l, _atoi, _atoi_l, _atol, + _atol_l, _atoll, _atoll_l, _backtrace, _backtrace_async, _backtrace_from_fp, + _backtrace_image_offsets, _backtrace_set_pcs_func, _backtrace_symbols, + _backtrace_symbols_fd, _basename, _basename_r, _bcopy, _brk, + _bsd_signal, _bsearch, _bsearch_b, _btowc, _btowc_l, _catclose, + _catgets, _catopen, _cfgetispeed, _cfgetospeed, _cfmakeraw, + _cfsetispeed, _cfsetospeed, _cfsetspeed, _cgetcap, _cgetclose, + _cgetent, _cgetfirst, _cgetmatch, _cgetnext, _cgetnum, _cgetset, + _cgetstr, _cgetustr, _chmodx_np, _clearerr, _clearerr_unlocked, + _clock, _clock_getres, _clock_gettime, _clock_gettime_nsec_np, + _clock_port, _clock_sem, _clock_settime, _closedir, _compat_mode, + _confstr, _copy_printf_domain, _creat, '_creat$NOCANCEL', + _crypt, _ctermid, _ctermid_r, _ctime, _ctime_r, _daemon, _daylight, + _dbm_clearerr, _dbm_close, _dbm_delete, _dbm_dirfno, _dbm_error, + _dbm_fetch, _dbm_firstkey, _dbm_nextkey, _dbm_open, _dbm_store, + _dbopen, _devname, _devname_r, _difftime, _digittoint, _digittoint_l, + _dirfd, _dirname, _dirname_r, _div, _dprintf, _dprintf_l, + _drand48, _duplocale, _dxprintf, _dxprintf_exec, _ecvt, _encrypt, + _endttyent, _endusershell, _endutxent, _endutxent_wtmp, _environ_lock_np, + _environ_unlock_np, _erand48, _err, _err_set_exit, _err_set_exit_b, + _err_set_file, _errc, _errx, _execl, _execle, _execlp, _execv, + _execvP, _execvp, _exit, _f_prealloc, _fchmodx_np, _fclose, + _fcvt, _fdopen, '_fdopen$DARWIN_EXTSN', _fdopendir, _feof, + _feof_unlocked, _ferror, _ferror_unlocked, _fflagstostr, _fflush, + _fgetc, _fgetln, _fgetpos, _fgetrune, _fgets, _fgetwc, _fgetwc_l, + _fgetwln, _fgetwln_l, _fgetws, _fgetws_l, _fileno, _fileno_unlocked, + _filesec_dup, _filesec_free, _filesec_get_property, _filesec_init, + _filesec_query_property, _filesec_set_property, _filesec_unset_property, + _flockfile, _fmemopen, _fmtcheck, _fmtmsg, _fnmatch, _fopen, + '_fopen$DARWIN_EXTSN', _fork, _forkpty, _fparseln, _fprintf, + _fprintf_l, _fpurge, _fputc, _fputrune, _fputs, _fputwc, _fputwc_l, + _fputws, _fputws_l, _fread, _free_printf_comp, _free_printf_domain, + _freelocale, _freopen, _fscanf, _fscanf_l, _fseek, _fseeko, + _fsetpos, _fstatvfs, _fstatx_np, _fsync_volume_np, _ftell, + _ftello, _ftime, _ftok, _ftrylockfile, _fts_children, _fts_close, + _fts_open, _fts_open_b, _fts_read, _fts_set, _ftw, _fungetrune, + _funlockfile, _funopen, _fwide, _fwprintf, _fwprintf_l, _fwrite, + _fwscanf, _fwscanf_l, _fxprintf, _fxprintf_exec, _gcvt, _getbsize, + _getc, _getc_unlocked, _getchar, _getchar_unlocked, _getcwd, + _getdate, _getdate_err, _getdelim, _getdiskbyname, _getenv, + _getenv_copy_np, '_getgroups$DARWIN_EXTSN', _gethostid, _gethostname, + _getipv4sourcefilter, _getlastlogx, _getlastlogxbyname, _getline, + _getloadavg, _getlogin, _getlogin_r, _getmntinfo, _getmntinfo_r_np, + _getmode, _getopt, _getopt_long, _getopt_long_only, _getpagesize, + _getpass, _getpeereid, _getprogname, _gets, _getsourcefilter, + _getsubopt, _gettimeofday, _getttyent, _getttynam, _getusershell, + _getutmp, _getutmpx, _getutxent, _getutxent_wtmp, _getutxid, + _getutxline, _getvfsbyname, _getw, _getwc, _getwc_l, _getwchar, + _getwchar_l, _getwd, _glob, _glob_b, _globfree, _gmtime, _gmtime_r, + _grantpt, _hash_create, _hash_destroy, _hash_purge, _hash_search, + _hash_stats, _hash_traverse, _hcreate, _hdestroy, _heapsort, + _heapsort_b, _hsearch, _imaxabs, _imaxdiv, _inet_addr, _inet_aton, + _inet_lnaof, _inet_makeaddr, _inet_net_ntop, _inet_net_pton, + _inet_neta, _inet_netof, _inet_network, _inet_nsap_addr, _inet_nsap_ntoa, + _inet_ntoa, _inet_ntop, _inet_ntop4, _inet_ntop6, _inet_pton, + _initstate, _insque, _isalnum, _isalnum_l, _isalpha, _isalpha_l, + _isascii, _isatty, _isblank, _isblank_l, _iscntrl, _iscntrl_l, + _isdigit, _isdigit_l, _isgraph, _isgraph_l, _ishexnumber, + _ishexnumber_l, _isideogram, _isideogram_l, _islower, _islower_l, + _isnumber, _isnumber_l, _isphonogram, _isphonogram_l, _isprint, + _isprint_l, _ispunct, _ispunct_l, _isrune, _isrune_l, _isspace, + _isspace_l, _isspecial, _isspecial_l, _isupper, _isupper_l, + _iswalnum, _iswalnum_l, _iswalpha, _iswalpha_l, _iswascii, + _iswblank, _iswblank_l, _iswcntrl, _iswcntrl_l, _iswctype, + _iswctype_l, _iswdigit, _iswdigit_l, _iswgraph, _iswgraph_l, + _iswhexnumber, _iswhexnumber_l, _iswideogram, _iswideogram_l, + _iswlower, _iswlower_l, _iswnumber, _iswnumber_l, _iswphonogram, + _iswphonogram_l, _iswprint, _iswprint_l, _iswpunct, _iswpunct_l, + _iswrune, _iswrune_l, _iswspace, _iswspace_l, _iswspecial, + _iswspecial_l, _iswupper, _iswupper_l, _iswxdigit, _iswxdigit_l, + _isxdigit, _isxdigit_l, _jrand48, _kOSThermalNotificationPressureLevelName, + _killpg, _l64a, _labs, _lchflags, _lchmod, _lcong48, _ldiv, + _lfind, _libc_set_introspection_hooks, _link_addr, _link_ntoa, + _llabs, _lldiv, _localeconv, _localeconv_l, _localtime, _localtime_r, + _lockf, '_lockf$NOCANCEL', _login, _login_tty, _logout, _logwtmp, + _lrand48, _lsearch, _lstatx_np, _lutimes, _mblen, _mblen_l, + _mbmb, _mbrlen, _mbrlen_l, _mbrrune, _mbrtowc, _mbrtowc_l, + _mbrune, _mbsinit, _mbsinit_l, _mbsnrtowcs, _mbsnrtowcs_l, + _mbsrtowcs, _mbsrtowcs_l, _mbstowcs, _mbstowcs_l, _mbtowc, + _mbtowc_l, _memmem, _memset_s, _mergesort, _mergesort_b, _mkdirx_np, + _mkdtemp, _mkdtempat_np, _mkfifox_np, _mkostemp, _mkostemps, + _mkostempsat_np, _mkpath_np, _mkpathat_np, _mkstemp, _mkstemp_dprotected_np, + _mkstemps, _mkstempsat_np, _mktemp, _mktime, _monaddition, + _moncontrol, _moncount, _moninit, _monitor, _monoutput, _monreset, + _monstartup, _mpool_close, _mpool_filter, _mpool_get, _mpool_new, + _mpool_open, _mpool_put, _mpool_sync, _mrand48, _nanosleep, + '_nanosleep$NOCANCEL', _new_printf_comp, _new_printf_domain, + _newlocale, _nextwctype, _nextwctype_l, _nftw, _nice, _nl_langinfo, + _nl_langinfo_l, _nrand48, _nvis, _offtime, _open_memstream, + _open_with_subsystem, _open_wmemstream, _opendev, _opendir, + _openpty, _openx_np, _optarg, _opterr, _optind, _optopt, _optreset, + _pause, '_pause$NOCANCEL', _pclose, _perror, _popen, '_popen$DARWIN_EXTSN', + _posix2time, _posix_openpt, _posix_spawnp, _printf, _printf_l, + _psignal, _psort, _psort_b, _psort_r, _ptsname, _ptsname_r, + _putc, _putc_unlocked, _putchar, _putchar_unlocked, _putenv, + _puts, _pututxline, _putw, _putwc, _putwc_l, _putwchar, _putwchar_l, + _qsort, _qsort_b, _qsort_r, _querylocale, _quick_exit, _radixsort, + _raise, _rand, _rand_r, _random, _rb_tree_count, _rb_tree_find_node, + _rb_tree_find_node_geq, _rb_tree_find_node_leq, _rb_tree_init, + _rb_tree_insert_node, _rb_tree_iterate, _rb_tree_remove_node, + _readdir, _readdir_r, _readpassphrase, _realpath, '_realpath$DARWIN_EXTSN', + _recv, '_recv$NOCANCEL', _regcomp, _regcomp_l, _regerror, + _regexec, _regfree, _register_printf_domain_function, _register_printf_domain_render_std, + _regncomp, _regncomp_l, _regnexec, _regwcomp, _regwcomp_l, + _regwexec, _regwncomp, _regwncomp_l, _regwnexec, _remove, + _remque, _rewind, _rewinddir, _rindex, _rpmatch, _sbrk, _scandir, + _scandir_b, _scanf, _scanf_l, _seed48, _seekdir, _send, '_send$NOCANCEL', + _setbuf, _setbuffer, _setenv, _sethostid, _sethostname, _setinvalidrune, + _setipv4sourcefilter, _setkey, _setlinebuf, _setlocale, _setlogin, + _setmode, _setpgrp, _setprogname, _setrgid, _setruid, _setrunelocale, + _setsourcefilter, _setstate, _settimeofday, _setttyent, _setusershell, + _setutxent, _setutxent_wtmp, _setvbuf, _sigaction, _sigaddset, + _sigaltstack, _sigblock, _sigdelset, _sigemptyset, _sigfillset, + _sighold, _sigignore, _siginterrupt, _sigismember, _signal, + _sigpause, '_sigpause$NOCANCEL', _sigrelse, _sigset, _sigsetmask, + _sigvec, _sl_add, _sl_find, _sl_free, _sl_init, _sleep, '_sleep$NOCANCEL', + _snprintf, _snprintf_l, _snvis, _sockatmark, _sprintf, _sprintf_l, + _sradixsort, _srand, _srand48, _sranddev, _srandom, _srandomdev, + _sscanf, _sscanf_l, _stat_with_subsystem, _statvfs, _statx_np, + _stpcpy, _stpncpy, _stravis, _strcasecmp, _strcasecmp_l, _strcasestr, + _strcasestr_l, _strcat, _strchrnul, _strcoll, _strcoll_l, + _strcspn, _strdup, _strenvisx, _strerror, _strerror_r, _strfmon, + _strfmon_l, _strftime, _strftime_l, _strmode, _strncasecmp, + _strncasecmp_l, _strncat, _strndup, _strnstr, _strnunvis, + _strnunvisx, _strnvis, _strnvisx, _strpbrk, _strptime, _strptime_l, + _strrchr, _strsenvisx, _strsep, _strsignal, _strsignal_r, + _strsnvis, _strsnvisx, _strspn, _strsvis, _strsvisx, _strtod, + _strtod_l, _strtoencf16, _strtoencf32, _strtoencf64, _strtoencf64x, + _strtof, _strtof_l, _strtofflags, _strtoimax, _strtoimax_l, + _strtok, _strtok_r, _strtol, _strtol_l, _strtold, _strtold_l, + _strtoll, _strtoll_l, _strtonum, _strtoq, _strtoq_l, _strtoul, + _strtoul_l, _strtoull, _strtoull_l, _strtoumax, _strtoumax_l, + _strtouq, _strtouq_l, _strunvis, _strunvisx, _strvis, _strvisx, + _strxfrm, _strxfrm_l, _suboptarg, _svis, _swab, _swprintf, + _swprintf_l, _swscanf, _swscanf_l, _sxprintf, _sxprintf_exec, + _sync_volume_np, _sys_errlist, _sys_nerr, _sys_siglist, _sys_signame, + _sysconf, _sysctl, _sysctlbyname, _sysctlnametomib, _system, + '_system$NOCANCEL', _tcdrain, '_tcdrain$NOCANCEL', _tcflow, + _tcflush, _tcgetattr, _tcgetpgrp, _tcgetsid, _tcsendbreak, + _tcsetattr, _tcsetpgrp, _tdelete, _telldir, _tempnam, _tfind, + _thread_stack_async_pcs, _thread_stack_pcs, _time, _time2posix, + _timegm, _timelocal, _timeoff, _times, _timespec_get, _timezone, + _timingsafe_bcmp, _tmpfile, _tmpnam, _toascii, _tolower, _tolower_l, + _toupper, _toupper_l, _towctrans, _towctrans_l, _towlower, + _towlower_l, _towupper, _towupper_l, _tre_ast_new_catenation, + _tre_ast_new_iter, _tre_ast_new_literal, _tre_ast_new_node, + _tre_ast_new_union, _tre_compile, _tre_fill_pmatch, _tre_free, + _tre_mem_alloc_impl, _tre_mem_destroy, _tre_mem_new_impl, + _tre_parse, _tre_stack_destroy, _tre_stack_new, _tre_stack_num_objects, + _tre_tnfa_run_backtrack, _tre_tnfa_run_parallel, _tsearch, + _ttyname, _ttyname_r, _ttyslot, _twalk, _tzname, _tzset, _tzsetwall, + _ualarm, _ulimit, _umaskx_np, _uname, _ungetc, _ungetwc, _ungetwc_l, + _unlockpt, _unsetenv, _unvis, _uselocale, _usleep, '_usleep$NOCANCEL', + _utime, _utmpxname, _uuid_clear, _uuid_compare, _uuid_copy, + _uuid_generate, _uuid_generate_random, _uuid_generate_time, + _uuid_is_null, _uuid_pack, _uuid_parse, _uuid_unpack, _uuid_unparse, + _uuid_unparse_lower, _uuid_unparse_upper, _vasprintf, _vasprintf_l, + _vasxprintf, _vasxprintf_exec, _vdprintf, _vdprintf_l, _vdxprintf, + _vdxprintf_exec, _verr, _verrc, _verrx, _vfork, _vfprintf, + _vfprintf_l, _vfscanf, _vfscanf_l, _vfwprintf, _vfwprintf_l, + _vfwscanf, _vfwscanf_l, _vfxprintf, _vfxprintf_exec, _vis, + _vprintf, _vprintf_l, _vscanf, _vscanf_l, _vsnprintf, _vsnprintf_l, + _vsprintf, _vsprintf_l, _vsscanf, _vsscanf_l, _vswprintf, + _vswprintf_l, _vswscanf, _vswscanf_l, _vsxprintf, _vsxprintf_exec, + _vwarn, _vwarnc, _vwarnx, _vwprintf, _vwprintf_l, _vwscanf, + _vwscanf_l, _vxprintf, _vxprintf_exec, _wait, '_wait$NOCANCEL', + _wait3, _waitpid, '_waitpid$NOCANCEL', _warn, _warnc, _warnx, + _wcpcpy, _wcpncpy, _wcrtomb, _wcrtomb_l, _wcscasecmp, _wcscasecmp_l, + _wcscat, _wcschr, _wcscmp, _wcscoll, _wcscoll_l, _wcscpy, + _wcscspn, _wcsdup, _wcsftime, _wcsftime_l, _wcslcat, _wcslcpy, + _wcslen, _wcsncasecmp, _wcsncasecmp_l, _wcsncat, _wcsncmp, + _wcsncpy, _wcsnlen, _wcsnrtombs, _wcsnrtombs_l, _wcspbrk, + _wcsrchr, _wcsrtombs, _wcsrtombs_l, _wcsspn, _wcsstr, _wcstod, + _wcstod_l, _wcstof, _wcstof_l, _wcstoimax, _wcstoimax_l, _wcstok, + _wcstol, _wcstol_l, _wcstold, _wcstold_l, _wcstoll, _wcstoll_l, + _wcstombs, _wcstombs_l, _wcstoul, _wcstoul_l, _wcstoull, _wcstoull_l, + _wcstoumax, _wcstoumax_l, _wcswidth, _wcswidth_l, _wcsxfrm, + _wcsxfrm_l, _wctob, _wctob_l, _wctomb, _wctomb_l, _wctrans, + _wctrans_l, _wctype, _wctype_l, _wcwidth, _wcwidth_l, _wmemchr, + _wmemcmp, _wmemcpy, _wmemmove, _wmemset, _wordexp, _wordfree, + _wprintf, _wprintf_l, _wscanf, _wscanf_l, _wtmpxname, _xprintf, + _xprintf_exec ] + weak-symbols: [ __os_debug_log_redirect_func ] +reexports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _bcmp, _bzero, _index, _memccpy, _memchr, _memcmp, _memcpy, + _memmove, _memset, _memset_pattern16, _memset_pattern4, _memset_pattern8, + _strchr, _strcmp, _strcpy, _strlcat, _strlcpy, _strlen, _strncmp, + _strncpy, _strnlen, _strstr ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_collections.dylib' +current-version: 1725.40.4 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _os_map_128_clear, _os_map_128_count, _os_map_128_delete, + _os_map_128_destroy, _os_map_128_find, _os_map_128_foreach, + _os_map_128_init, _os_map_128_insert, _os_map_32_clear, _os_map_32_count, + _os_map_32_delete, _os_map_32_destroy, _os_map_32_find, _os_map_32_foreach, + _os_map_32_init, _os_map_32_insert, _os_map_64_clear, _os_map_64_count, + _os_map_64_delete, _os_map_64_destroy, _os_map_64_find, _os_map_64_foreach, + _os_map_64_init, _os_map_64_insert, _os_map_str_clear, _os_map_str_count, + _os_map_str_delete, _os_map_str_destroy, _os_map_str_entry, + _os_map_str_find, _os_map_str_foreach, _os_map_str_init, _os_map_str_insert, + _os_set_32_ptr_clear, _os_set_32_ptr_count, _os_set_32_ptr_delete, + _os_set_32_ptr_destroy, _os_set_32_ptr_find, _os_set_32_ptr_foreach, + _os_set_32_ptr_init, _os_set_32_ptr_insert, _os_set_64_ptr_clear, + _os_set_64_ptr_count, _os_set_64_ptr_delete, _os_set_64_ptr_destroy, + _os_set_64_ptr_find, _os_set_64_ptr_foreach, _os_set_64_ptr_init, + _os_set_64_ptr_insert, _os_set_str_ptr_clear, _os_set_str_ptr_count, + _os_set_str_ptr_delete, _os_set_str_ptr_destroy, _os_set_str_ptr_find, + _os_set_str_ptr_foreach, _os_set_str_ptr_init, _os_set_str_ptr_insert ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_configuration.dylib' +current-version: 1385.40.9 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __dns_configuration_ack, __libSC_info_fork_child, __libSC_info_fork_parent, + __libSC_info_fork_prepare, __nwi_config_agent_copy_data, __nwi_state_ack, + __nwi_state_force_refresh, _config_agent_copy_dns_information, + _config_agent_copy_proxy_information, _config_agent_free_dns_information, + _config_agent_free_proxy_information, _config_agent_get_dns_nameservers, + _config_agent_get_dns_searchdomains, _config_agent_update_proxy_information, + _dns_configuration_copy, _dns_configuration_free, _dns_configuration_notify_key, + _is_config_agent_type_dns, _is_config_agent_type_proxy, _nwi_ifstate_compare_rank, + _nwi_ifstate_get_dns_signature, _nwi_ifstate_get_flags, _nwi_ifstate_get_generation, + _nwi_ifstate_get_ifname, _nwi_ifstate_get_next, _nwi_ifstate_get_reachability_flags, + _nwi_ifstate_get_signature, _nwi_ifstate_get_vpn_server, _nwi_state_copy, + _nwi_state_get_first_ifstate, _nwi_state_get_generation, _nwi_state_get_ifstate, + _nwi_state_get_interface_names, _nwi_state_get_notify_key, + _nwi_state_get_reachability_flags, _nwi_state_release ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_containermanager.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _CMCONTAINERSEAM_DEFAULT, _CMDISPATCHSEAM_DEFAULT, _CMFSSEAM_DEFAULT, + _CMNOTIFYSEAM_DEFAULT, _CMPWDSEAM_DEFAULT, _CMQUARANTINESEAM_DEFAULT, + _CMSANDBOXSEAM_DEFAULT, _CONTAINER_CLASS_NAMES, _CONTAINER_NOTIFY_GENERATION_INITIAL, + _CONTAINER_NOTIFY_GENERATION_INVALID, _CONTAINER_NOTIFY_USER_INVALIDATED, + _CONTAINER_PERSONA_CURRENT, _CONTAINER_PERSONA_PRIMARY, __container_init, + __container_notify_copy_notify_name, __container_query_get_next_result_sync, + __container_query_get_result_at_index, __container_query_get_servicing_pid, + __container_query_operation_set_private_flags, __container_query_reset_iterator, + __container_references_get_servicing_pid, __container_traverse_continuation_create, + __container_traverse_continuation_free, __container_traverse_continuation_pop, + __container_traverse_continuation_push, __container_traverse_get_last_path_component, + __container_traverse_path_pop, __container_traverse_path_push, + __container_traverse_state_deinit, __container_traverse_state_init, + _container_acquire_sandbox_extension, _container_audit_token_copy_codesign_hash, + _container_audit_token_copy_codesign_identifier, _container_audit_token_copy_codesign_team_identifier, + _container_audit_token_copy_entitlement, _container_audit_token_copy_executable_name, + _container_audit_token_copy_executable_name_with_pid, _container_audit_token_for_pid, + _container_audit_token_get_codesign_status, _container_audit_token_get_egid, + _container_audit_token_get_euid, _container_audit_token_get_pid, + _container_audit_token_get_platform, _container_audit_token_is_valid, + _container_authorize, _container_authorize_container, _container_authorize_container_for_self, + _container_authorize_get_result_description, _container_base64_decode, + _container_base64_decode_string, _container_base64_encode, + _container_base64_encode_string, _container_bundle_copy_data_container, + _container_bundle_copy_data_container_path, _container_class_for_each_normalized_class, + _container_class_normalized, _container_class_supports_data_subdirectory, + _container_class_supports_randomized_path, _container_class_supports_randomized_path_on_current_platform, + _container_client_copy_decoded_from_xpc_object, _container_client_copy_encoded_xpc_object, + _container_client_copy_entitlement, _container_client_create_from_audit_token, + _container_client_get_audit_token, _container_client_get_codesign_identifier, + _container_client_get_codesign_team_identifier, _container_client_get_egid, + _container_client_get_euid, _container_client_get_persona_unique_string, + _container_client_get_pid, _container_client_get_platform, + _container_client_initializer, _container_client_is_alive, + _container_client_is_platform_binary, _container_client_is_sandboxed, + _container_client_is_signature_valid, _container_client_is_signed, + _container_client_is_test_client, _container_codesign_copy_cdhash, + _container_codesign_copy_cs_identity, _container_codesign_copy_cs_team_identifier, + _container_codesign_get_current_identifier, _container_codesign_get_self_audit_token, + _container_codesign_get_status, _container_copy_client, _container_copy_code_signing_info_for_identifier, + _container_copy_from_path, _container_copy_info, _container_copy_info_value_for_key, + _container_copy_object, _container_copy_path, _container_copy_persona_unique_strings, + _container_copy_sandbox_token, _container_copy_unlocalized_description, + _container_create_merged_array, _container_create_or_lookup_app_group_path_by_app_group_identifier, + _container_create_or_lookup_app_group_paths_for_current_user, + _container_create_or_lookup_app_group_paths_for_platform, + _container_create_or_lookup_app_group_paths_from_entitlements, + _container_create_or_lookup_app_group_paths_from_entitlements_4ls, + _container_create_or_lookup_for_current_user, _container_create_or_lookup_for_platform, + _container_create_or_lookup_path_for_current_user, _container_create_or_lookup_path_for_platform, + _container_create_or_lookup_system_group_paths, _container_create_or_lookup_user_managed_assets_path, + _container_create_or_lookup_user_managed_assets_relative_path, + _container_delete, _container_delete_all_container_content, + _container_delete_all_data_container_content_for_current_user, + _container_delete_array_of_containers, _container_delete_user_managed_assets, + _container_delete_with_class_and_identifier_for_current_user, + _container_disk_usage, _container_disposition, _container_disposition_for_array, + _container_entitlements_copy_container_identifiers, _container_error_copy, + _container_error_copy_unlocalized_description, _container_error_create, + _container_error_create_with_message, _container_error_free, + _container_error_get_category, _container_error_get_message, + _container_error_get_path, _container_error_get_posix_errno, + _container_error_get_type, _container_error_is_fatal, _container_error_is_file_system_error, + _container_error_reinitialize, _container_flush_container_cache, + _container_flush_persona_cache, _container_free_array_of_containers, + _container_free_client, _container_free_object, _container_frozenset_copyout_external_bytes, + _container_frozenset_create, _container_frozenset_create_from_external_bytes, + _container_frozenset_destroy, _container_frozenset_enumerate_matches, + _container_frozenset_get_container_class_of_container_at_index, + _container_frozenset_get_count, _container_frozenset_get_creator_of_container_at_index, + _container_frozenset_get_generation, _container_frozenset_get_identifier_of_container_at_index, + _container_frozenset_get_is_new_of_container_at_index, _container_frozenset_get_is_transient_of_container_at_index, + _container_frozenset_get_path_of_container_at_index, _container_frozenset_get_persona_unique_string_of_container_at_index, + _container_frozenset_get_stored_string, _container_frozenset_get_uid_of_container_at_index, + _container_frozenset_get_uma_relative_path_of_container_at_index, + _container_frozenset_get_unique_path_component_of_container_at_index, + _container_frozenset_get_uuid_of_container_at_index, _container_fs_add_path_component, + _container_fs_append_trailing_slash, _container_fs_item_exists, + _container_fs_item_exists_at, _container_fs_load_plist_at, + _container_fs_path_at, _container_fs_resolve_dirent_type_at, + _container_get_all_with_class_for_current_user, _container_get_class, + _container_get_creator_codesign_identifier, _container_get_error_description, + _container_get_identifier, _container_get_info, _container_get_info_value_for_key, + _container_get_path, _container_get_persona_unique_string, + _container_get_uid, _container_get_unique_path_component, + _container_get_user_managed_assets_relative_path, _container_group_container_identifiers_for_current_user, + _container_info_delete, _container_info_modify, _container_info_put, + _container_internal_get_first_boot_uuid, _container_invalidate_code_signing_cache, + _container_is_equal, _container_is_new, _container_is_transient, + _container_log_client_fault_logging_is_enabled, _container_log_error, + _container_log_error_with_faults, _container_log_ext_error, + _container_log_ext_error_with_faults, _container_log_handle_for_category, + _container_log_replication_disable, _container_log_replication_enable_to_uid_relative_path, + _container_log_replication_prune_for_uid, _container_log_set_client_fault_logging, + _container_notify_create, _container_notify_create_with_class, + _container_notify_create_with_initial_gen_count, _container_notify_free, + _container_notify_get_generation, _container_notify_get_last_error, + _container_notify_has_changed, _container_notify_increment_generation, + _container_notify_post, _container_notify_set_class, _container_notify_set_event_handler, + _container_notify_set_flags, _container_notify_set_generation, + _container_notify_set_queue, _container_notify_set_uid, _container_notify_start, + _container_notify_stop, _container_object_copy, _container_object_create, + _container_object_create_blank, _container_object_free, _container_object_get_class, + _container_object_get_creator_codesign_identifier, _container_object_get_identifier, + _container_object_get_info, _container_object_get_path, _container_object_get_persona_unique_string, + _container_object_get_query, _container_object_get_sandbox_token, + _container_object_get_uid, _container_object_get_unique_path_component, + _container_object_get_user_managed_assets_relative_path, _container_object_get_uuid, + _container_object_is_new, _container_object_is_transient, + _container_object_sandbox_extension_activate, _container_object_set_backing_store_from_query, + _container_object_set_class, _container_object_set_creator_codesign_identifier, + _container_object_set_info, _container_object_set_is_new, + _container_object_set_path, _container_object_set_sandbox_token, + _container_object_set_transient, _container_object_set_unique_path_component, + _container_object_set_user_managed_assets_relative_path, _container_object_set_uuid, + _container_object_update_metadata, _container_operation_complete_background_tasks, + _container_operation_delete, _container_operation_delete_array, + _container_operation_delete_reclaim_disk_space, _container_paths_context_create, + _container_paths_context_free, _container_paths_context_set_class, + _container_paths_context_set_flags, _container_paths_context_set_persona_unique_string, + _container_paths_context_set_transient, _container_paths_context_set_uid, + _container_paths_copy_container_at, _container_paths_copy_container_from_path, + _container_paths_copy_container_root_path_for_context, _container_paths_copy_part_subpath, + _container_paths_copy_uid_home_relative, _container_paths_create_uid_home_relative, + _container_paths_enumerate_containers_at, _container_perfect_hash_copyout_external_bytes, + _container_perfect_hash_count, _container_perfect_hash_create, + _container_perfect_hash_create_from_external_bytes, _container_perfect_hash_destroy, + _container_perfect_hash_index_of, _container_perfect_hash_num_indexes, + _container_perform_with_client_context, _container_persona_collect_all_ids, + _container_persona_convert_unique_string_to_persona_uid, _container_persona_foreach, + _container_process_restored_container, _container_pwd_copy_user_home_path, + _container_pwd_for_name, _container_pwd_for_uid, _container_pwd_get_cached_current_user_home_path, + _container_pwd_get_mobile_user_uid, _container_query_copy, + _container_query_count_results, _container_query_create, _container_query_create_from_container, + _container_query_free, _container_query_get_last_error, _container_query_get_single_result, + _container_query_iterate_results_sync, _container_query_iterate_results_with_identifier_sync, + _container_query_iterate_results_with_subquery_sync, _container_query_operation_set_client, + _container_query_operation_set_flags, _container_query_operation_set_part, + _container_query_operation_set_part_domain, _container_query_operation_set_platform, + _container_query_set_class, _container_query_set_group_identifiers, + _container_query_set_identifiers, _container_query_set_include_other_owners, + _container_query_set_persona_unique_string, _container_query_set_transient, + _container_query_set_uid, _container_realpath, _container_realpathat, + _container_recreate_structure, _container_references_get_last_error, + _container_regenerate_uuid, _container_repair_user_data, _container_replace, + _container_retry_test, _container_sandbox_extension_consume, + _container_sandbox_extension_revoke, _container_sandbox_issue_custom_extension, + _container_sandbox_issue_extension, _container_seam_container_reset, + _container_seam_container_set_common, _container_seam_dispatch_reset, + _container_seam_dispatch_set_common, _container_seam_fs_ensure_lazy_loaded, + _container_seam_fs_reset, _container_seam_fs_set_common, _container_seam_notify_reset, + _container_seam_notify_set_common, _container_seam_pwd_reset, + _container_seam_pwd_set_common, _container_seam_quarantine_reset, + _container_seam_quarantine_set_common, _container_seam_sandbox_reset, + _container_seam_sandbox_set_common, _container_serialize_copy_deserialized_reference, + _container_serialize_copy_serialized_reference, _container_set_code_signing_info_for_identifier, + _container_set_data_protection_for_current_user, _container_set_info_value, + _container_set_test_lock, _container_stage_shared_system_content, + _container_string_rom_copyout_external_bytes, _container_string_rom_count, + _container_string_rom_create, _container_string_rom_create_from_external_bytes, + _container_string_rom_destroy, _container_string_rom_index_of, + _container_string_rom_string_at_index, _container_subdirectories_for_class, + _container_system_group_path_for_identifier, _container_system_path_for_identifier, + _container_test_set_locks_and_restart, _container_traverse_directory, + _container_traverse_node_copy, _container_traverse_node_copy_unlocalized_description, + _container_traverse_node_free, _container_traverse_node_get_depth_from_origin, + _container_traverse_node_get_name, _container_traverse_node_get_optional_alloc_size, + _container_traverse_node_get_optional_dp_class, _container_traverse_node_get_optional_file_id, + _container_traverse_node_get_optional_flags, _container_traverse_node_get_optional_gid, + _container_traverse_node_get_optional_link_count, _container_traverse_node_get_optional_mode, + _container_traverse_node_get_optional_parent_fd, _container_traverse_node_get_optional_uid, + _container_traverse_node_get_path, _container_traverse_node_get_subordinate_count, + _container_traverse_node_is_directory, _container_traverse_node_is_regular_file, + _container_traverse_node_is_symlink, _container_traverse_node_optional_has_acl, + _container_update_with_container, _container_user_managed_assets_path, + _container_user_managed_assets_relative_path, _container_xpc_connection_is_shared, + _container_xpc_create_connection, _container_xpc_decode_container_object, + _container_xpc_decode_create_container_object_array, _container_xpc_decode_error, + _container_xpc_encode_client_onto_message, _container_xpc_encode_container_array, + _container_xpc_encode_container_as_object, _container_xpc_encode_container_metadata_as_object, + _container_xpc_encode_container_object, _container_xpc_encode_error, + _container_xpc_encode_error_as_object, _container_xpc_get_incoming_reply_preprocess_block, + _container_xpc_get_outgoing_message_postprocess_block, _container_xpc_get_outgoing_message_send_block, + _container_xpc_get_raw_connection, _container_xpc_invalidate_connection, + _container_xpc_release, _container_xpc_send_message, _container_xpc_send_sync_message, + _container_xpc_send_sync_message_with_object, _container_xpc_set_client_context_during_block, + _container_xpc_set_incoming_reply_preprocess_block, _container_xpc_set_outgoing_message_postprocess_block, + _container_xpc_set_outgoing_message_send_block, _container_xpc_set_use_shared_connection, + _container_xpc_shared_copy_connection, _gCMContainerSeam, + _gCMDispatchSeam, _gCMFSSeam, _gCMNotifySeam, _gCMPWDSeam, + _gCMQuarantineSeam, _gCMSandboxSeam ] + - targets: [ x86_64-maccatalyst, arm64-maccatalyst, arm64e-maccatalyst ] + symbols: [ _CONTAINER_CURRENT_MOBILE_UID, _CONTAINER_INSTALLATION_UID, + _CONTAINER_SYSTEM_UID, _container_create_or_lookup, _container_create_or_lookup_app_group_paths, + _container_create_or_lookup_group_container_paths_for_current_user, + _container_create_or_lookup_path, _container_delete_all_data_container_content, + _container_delete_with_uid_class_and_identifier, _container_get_all_with_class, + _container_perform_data_migration, _container_perform_data_migration_for_current_user, + _container_references_add, _container_references_create, _container_references_free, + _container_references_iterate_by_group_sync, _container_references_iterate_by_owner_sync, + _container_references_operation_set_flags, _container_references_remove, + _container_references_set_class, _container_references_set_persona_unique_string, + _container_references_set_uid ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_coreservices.dylib' +current-version: 129 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _NSGetNextSearchPathEnumeration, _NSStartSearchPathEnumeration, + _NSStartSearchPathEnumerationPrivate, ___user_local_dirname, + ___user_relative_dirname, __dirhelper, __dirhelper_relative, + __get_user_dir_suffix, __libcoreservices_fork_child, __set_user_dir_suffix, + _sysdir_get_next_search_path_enumeration, _sysdir_start_search_path_enumeration, + _sysdir_start_search_path_enumeration_private ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ ___user_relative_dirname_with_hints, __dirhelper_relative_with_hints, + __dirhelper_remove_test, __dirhelper_test, __idle_exit ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_darwin.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___libdarwin_init, ___os_temporary_resource_shortage, _claimfd_np, + _close_drop_np, _close_drop_optional_np, _crfprintf_np, _dirstat_np, + _dirstatat_np, _dup_np, _err_np, _errc_np, _fcheck_np, _memdup2_np, + _memdup_np, _os_assert_mach, _os_assert_mach_port_status, + _os_boot_arg_string_to_int, _os_crash_get_reporter_port_array, + _os_crash_port_array_deallocate, _os_crash_set_reporter_port, + _os_crash_spawnattr_set_reporter_port, _os_enumerate_boot_args, + _os_enumerate_boot_args_b, _os_enumerate_boot_args_from_buffer, + _os_enumerate_boot_args_from_buffer_b, _os_flagset_copy_string, + _os_localtime_file, _os_mach_msg_copy_description, _os_mach_msg_get_audit_trailer, + _os_mach_msg_get_context_trailer, _os_mach_msg_get_trailer, + _os_mach_msg_trailer_copy_description, _os_mach_port_copy_description, + _os_parse_boot_arg_from_buffer_int, _os_parse_boot_arg_from_buffer_string, + _os_parse_boot_arg_int, _os_parse_boot_arg_string, _os_simple_hash, + _os_simple_hash_string, _os_simple_hash_string_with_seed, + _os_simple_hash_with_seed, _os_subcommand_fprintf, _os_subcommand_main, + _os_subcommand_vfprintf, _os_variant_allows_internal_security_policies, + _os_variant_allows_security_research, _os_variant_check, _os_variant_copy_description, + _os_variant_has_factory_content, _os_variant_has_internal_content, + _os_variant_has_internal_diagnostics, _os_variant_has_internal_ui, + _os_variant_init_4launchd, _os_variant_is_darwinos, _os_variant_is_recovery, + _os_variant_uses_ephemeral_storage, _realpath_np, _strerror_np, + _strexit_np, _symerror_np, _symexit_np, _sysctl_get_data_np, + _sysctlbyname_get_data_np, _sysexit_np, _vcrfprintf_np, _verr_np, + _verrc_np, _vwarn_np, _vwfprintf_np, _warn_np, _wfprintf_np, + _xferfd_np, _zsnprintf_np ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ _os_boot_mode_query, _os_variant_is_basesystem ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_darwindirectory.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _DarwinDirectoryGetGeneration, _DarwinDirectoryRecordStoreApply, + _DarwinDirectoryRecordStoreApplyWithFilter ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_dnssd.dylib' +current-version: 2881.40.18 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _DNSServiceAddRecord, _DNSServiceAttrCreate, _DNSServiceAttrFree, + _DNSServiceAttrSetFailoverPolicy, _DNSServiceAttrSetValidationData, + _DNSServiceAttrSetValidationPolicy, _DNSServiceAttributeCreate, + _DNSServiceAttributeDeallocate, _DNSServiceAttributeSetAAAAPolicy, + _DNSServiceAttributeSetHostKeyHash, _DNSServiceAttributeSetResolverOverride, + _DNSServiceAttributeSetTimestamp, _DNSServiceBrowse, _DNSServiceBrowseEx, + _DNSServiceConstructFullName, _DNSServiceCreateConnection, + _DNSServiceCreateDelegateConnection, _DNSServiceEnumerateDomains, + _DNSServiceErrorCodeToString, _DNSServiceGetAddrInfo, _DNSServiceGetAddrInfoEx, + _DNSServiceGetPID, _DNSServiceGetProperty, _DNSServiceGetValidationData, + _DNSServiceNATPortMappingCreate, _DNSServiceProcessResult, + _DNSServiceQueryRecord, _DNSServiceQueryRecordEx, _DNSServiceQueryRecordWithAttribute, + _DNSServiceReconfirmRecord, _DNSServiceRefDeallocate, _DNSServiceRefSockFD, + _DNSServiceRegister, _DNSServiceRegisterRecord, _DNSServiceRegisterRecordWithAttribute, + _DNSServiceRegisterWithAttribute, _DNSServiceRemoveRecord, + _DNSServiceResolve, _DNSServiceResolveEx, _DNSServiceSendQueuedRequests, + _DNSServiceSetDefaultDomainForUser, _DNSServiceSetDispatchQueue, + _DNSServiceSetResolverDefaults, _DNSServiceSleepKeepalive, + _DNSServiceSleepKeepalive_sockaddr, _DNSServiceUpdateRecord, + _DNSServiceUpdateRecordWithAttribute, _PeerConnectionRelease, + _TXTRecordContainsKey, _TXTRecordCreate, _TXTRecordDeallocate, + _TXTRecordGetBytesPtr, _TXTRecordGetCount, _TXTRecordGetItemAtIndex, + _TXTRecordGetLength, _TXTRecordGetValuePtr, _TXTRecordRemoveValue, + _TXTRecordSetValue, _kDNSServiceAttrAllowFailover, _kDNSServiceAttrValidationRequired, + _kDNSServiceAttributeAAAAFallback ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, arm64-macos, arm64e-macos ] +install-name: '/usr/lib/system/libsystem_eligibility.dylib' +current-version: 286.40.9 +parent-umbrella: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + umbrella: System +exports: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ _os_eligibility_domain_for_name, _os_eligibility_dump_sysdiagnose_data_to_dir, + _os_eligibility_force_domain_answer, _os_eligibility_force_domain_set_answers, + _os_eligibility_get_all_domain_answers, _os_eligibility_get_domain_answer, + _os_eligibility_get_domain_notification_name, _os_eligibility_get_internal_state, + _os_eligibility_get_state_dump, _os_eligibility_precise_locations, + _os_eligibility_reset_all_domains, _os_eligibility_reset_domain, + _os_eligibility_set_input, _os_eligibility_set_test_mode ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_featureflags.dylib' +current-version: 101 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __os_feature_enabled_impl, __os_feature_enabled_simple_impl ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_info.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _LI_get_thread_item, _LI_get_thread_list, _LI_ils_create, + _LI_set_thread_item, _LI_set_thread_list, ___dn_skipname, + __authenticate, __ds_running, __gai_nat64_can_v4_address_be_synthesized, + __gai_serv_to_port, __gai_simple, __getaddrinfo_interface_async_call, + __getlong, __getnameinfo_interface_async_call, __getshort, + __null_auth, __res, __seterr_reply, __si_disable_opendirectory, + __yp_dobind, _alias_endent, _alias_getbyname, _alias_getent, + _alias_setent, _authnone_create, _authunix_create, _authunix_create_default, + _bindresvport, _bindresvport_sa, _bootparams_endent, _bootparams_getbyname, + _bootparams_getent, _bootparams_setent, _clnt_broadcast, _clnt_create, + _clnt_pcreateerror, _clnt_perrno, _clnt_perror, _clnt_spcreateerror, + _clnt_sperrno, _clnt_sperror, _clntraw_create, _clnttcp_create, + _clntudp_bufcreate, _clntudp_create, _configuration_profile_copy_property_list, + _configuration_profile_create_notification_key, _dn_expand, + _endfsent, _endgrent, _endhostent, _endnetent, _endnetgrent, + _endprotoent, _endpwent, _endrpcent, _endservent, _ether_aton, + _ether_hostton, _ether_line, _ether_ntoa, _ether_ntohost, + _freeaddrinfo, _freehostent, _freeifaddrs, _freeifmaddrs, + _gL1CacheEnabled, _gai_strerror, _getaddrinfo, _getaddrinfo_async_cancel, + _getaddrinfo_async_handle_reply, _getaddrinfo_async_receive, + _getaddrinfo_async_send, _getaddrinfo_async_start, _getdomainname, + _getfsent, _getfsfile, _getfsspec, _getgrent, _getgrgid, _getgrgid_r, + _getgrnam, _getgrnam_r, _getgroupcount, _getgrouplist, _getgrouplist_2, + _getgruuid, _getgruuid_r, _gethostbyaddr, _gethostbyaddr_async_cancel, + _gethostbyaddr_async_handleReply, _gethostbyaddr_async_start, + _gethostbyname, _gethostbyname2, _gethostbyname_async_cancel, + _gethostbyname_async_handleReply, _gethostbyname_async_start, + _gethostent, _getifaddrs, _getifmaddrs, _getipnodebyaddr, + _getipnodebyname, _getnameinfo, _getnameinfo_async_cancel, + _getnameinfo_async_handle_reply, _getnameinfo_async_send, + _getnameinfo_async_start, _getnetbyaddr, _getnetbyname, _getnetent, + _getnetgrent, _getprotobyname, _getprotobynumber, _getprotoent, + _getpwent, _getpwnam, _getpwnam_r, _getpwuid, _getpwuid_r, + _getpwuuid, _getpwuuid_r, _getrpcbyname, _getrpcbynumber, + _getrpcent, _getrpcport, _getservbyname, _getservbyport, _getservent, + _group_from_gid, _h_errno, _herror, _hstrerror, _htonl, _htons, + _if_freenameindex, _if_indextoname, _if_nameindex, _if_nametoindex, + _in6addr_any, _in6addr_linklocal_allnodes, _in6addr_linklocal_allrouters, + _in6addr_linklocal_allv2routers, _in6addr_loopback, _in6addr_nodelocal_allnodes, + _inet6_opt_append, _inet6_opt_find, _inet6_opt_finish, _inet6_opt_get_val, + _inet6_opt_init, _inet6_opt_next, _inet6_opt_set_val, _inet6_option_alloc, + _inet6_option_append, _inet6_option_find, _inet6_option_init, + _inet6_option_next, _inet6_option_space, _inet6_rth_add, _inet6_rth_getaddr, + _inet6_rth_init, _inet6_rth_reverse, _inet6_rth_segments, + _inet6_rth_space, _inet6_rthdr_add, _inet6_rthdr_getaddr, + _inet6_rthdr_getflags, _inet6_rthdr_init, _inet6_rthdr_lasthop, + _inet6_rthdr_segments, _inet6_rthdr_space, _initgroups, _innetgr, + _iruserok, _iruserok_sa, _kvarray_free, _kvbuf_add_dict, _kvbuf_add_key, + _kvbuf_add_val, _kvbuf_add_val_len, _kvbuf_append_kvbuf, _kvbuf_decode, + _kvbuf_free, _kvbuf_get_len, _kvbuf_init, _kvbuf_init_zone, + _kvbuf_make_nonpurgeable, _kvbuf_make_purgeable, _kvbuf_new, + _kvbuf_new_zone, _kvbuf_next_dict, _kvbuf_next_key, _kvbuf_next_val, + _kvbuf_next_val_len, _kvbuf_query, _kvbuf_query_key_int, _kvbuf_query_key_uint, + _kvbuf_query_key_val, _kvbuf_reset, _lookup_close_connections, + _mbr_check_membership, _mbr_check_membership_by_id, _mbr_check_membership_ext, + _mbr_check_membership_refresh, _mbr_check_service_membership, + _mbr_close_connections, _mbr_gid_to_uuid, _mbr_group_name_to_uuid, + _mbr_identifier_to_uuid, _mbr_identifier_translate, _mbr_reset_cache, + _mbr_set_identifier_ttl, _mbr_sid_to_string, _mbr_sid_to_uuid, + _mbr_string_to_sid, _mbr_string_to_uuid, _mbr_uid_to_uuid, + _mbr_user_name_to_uuid, _mbr_uuid_to_id, _mbr_uuid_to_sid, + _mbr_uuid_to_sid_type, _mbr_uuid_to_string, _ntohl, _ntohs, + _pmap_getmaps, _pmap_getport, _pmap_rmtcall, _pmap_set, _pmap_unset, + _prdb_end, _prdb_get, _prdb_getbyname, _prdb_set, _rcmd, _rcmd_af, + _res_init, _res_query, _res_search, _rpc_createerr, _rresvport, + _rresvport_af, _ruserok, _setdomainname, _setfsent, _setgrent, + _setgroupent, _sethostent, _setnetent, _setnetgrent, _setpassent, + _setprotoent, _setpwent, _setrpcent, _setservent, _si_addrinfo, + _si_addrinfo_list, _si_addrinfo_list_from_hostent, _si_addrinfo_v4, + _si_addrinfo_v4_mapped, _si_addrinfo_v6, _si_alias_all, _si_alias_byname, + _si_async_call, _si_async_cancel, _si_async_handle_reply, + _si_destination_compare, _si_destination_compare_no_dependencies, + _si_fs_all, _si_fs_byfile, _si_fs_byspec, _si_group_all, _si_group_bygid, + _si_group_byname, _si_group_byuuid, _si_grouplist, _si_host_all, + _si_host_byaddr, _si_host_byname, _si_in_netgroup, _si_ipnode_byname, + _si_item_call, _si_item_is_valid, _si_item_match, _si_item_release, + _si_item_retain, _si_list_add, _si_list_call, _si_list_concat, + _si_list_next, _si_list_release, _si_list_reset, _si_list_retain, + _si_mac_all, _si_mac_bymac, _si_mac_byname, _si_module_allows_caching, + _si_module_name, _si_module_release, _si_module_retain, _si_module_vers, + _si_module_with_name, _si_nameinfo, _si_netgroup_byname, _si_network_all, + _si_network_byaddr, _si_network_byname, _si_protocol_all, + _si_protocol_byname, _si_protocol_bynumber, _si_rpc_all, _si_rpc_byname, + _si_rpc_bynumber, _si_search, _si_search_module_set_flags, + _si_service_all, _si_service_byname, _si_service_byport, _si_set_nat64_v4_requires_synthesis, + _si_set_nat64_v4_synthesize, _si_set_path_check, _si_srv_byname, + _si_standardize_mac_address, _si_user_all, _si_user_byname, + _si_user_byuid, _si_user_byuuid, _si_wants_addrinfo, _svc_fdset, + _svc_getreq, _svc_getreqset, _svc_register, _svc_run, _svc_sendreply, + _svc_unregister, _svcerr_auth, _svcerr_decode, _svcerr_noproc, + _svcerr_noprog, _svcerr_progvers, _svcerr_systemerr, _svcerr_weakauth, + _svcfd_create, _svcraw_create, _svctcp_create, _svcudp_bufcreate, + _svcudp_create, _user_from_uid, _xdr_array, _xdr_authunix_parms, + _xdr_bool, _xdr_bytes, _xdr_callhdr, _xdr_callmsg, _xdr_char, + _xdr_des_block, _xdr_domainname, _xdr_double, _xdr_enum, _xdr_float, + _xdr_free, _xdr_hyper, _xdr_int, _xdr_int16_t, _xdr_int32_t, + _xdr_int64_t, _xdr_keydat, _xdr_long, _xdr_longlong_t, _xdr_mapname, + _xdr_netobj, _xdr_opaque, _xdr_peername, _xdr_pmap, _xdr_pmaplist, + _xdr_pointer, _xdr_reference, _xdr_replymsg, _xdr_rmtcall_args, + _xdr_rmtcallres, _xdr_short, _xdr_string, _xdr_u_char, _xdr_u_hyper, + _xdr_u_int, _xdr_u_int16_t, _xdr_u_int32_t, _xdr_u_int64_t, + _xdr_u_long, _xdr_u_longlong_t, _xdr_u_short, _xdr_union, + _xdr_valdat, _xdr_vector, _xdr_void, _xdr_wrapstring, _xdr_x_passwd, + _xdr_ypbind_binding, _xdr_ypbind_resp, _xdr_ypbind_resptype, + _xdr_ypbind_setdom, _xdr_ypmaplist, _xdr_yppasswd, _xdr_ypreq_key, + _xdr_ypreq_nokey, _xdr_ypresp_all, _xdr_ypresp_all_seq, _xdr_ypresp_key_val, + _xdr_ypresp_maplist, _xdr_ypresp_master, _xdr_ypresp_order, + _xdr_ypresp_val, _xdr_ypstat, _xdrmem_create, _xdrrec_create, + _xdrrec_endofrecord, _xdrrec_eof, _xdrrec_skiprecord, _xdrstdio_create, + _xprt_register, _xprt_unregister, _yp_all, _yp_bind, _yp_first, + _yp_get_default_domain, _yp_maplist, _yp_master, _yp_match, + _yp_next, _yp_order, _yp_unbind, _yperr_string, _ypprot_err ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_kernel.dylib' +current-version: 12377.41.6 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst ] + symbols: [ '_fstat$INODE64', '_fstatat$INODE64', '_fstatfs$INODE64', + '_getfsstat$INODE64', _i386_get_ldt, _i386_set_ldt, '_lstat$INODE64', + '_pselect$1050', '_select$1050', '_stat$INODE64', '_statfs$INODE64' ] + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _NDR_record, _____sigwait_nocancel, ____kernelVersionNumber, + ____kernelVersionString, ___abort_with_payload, ___accept, + ___accept_nocancel, ___access_extended, ___aio_suspend_nocancel, + ___bind, ___bsdthread_create, ___bsdthread_ctl, ___bsdthread_register, + ___bsdthread_terminate, ___carbon_delete, ___channel_get_info, + ___channel_get_opt, ___channel_open, ___channel_set_opt, ___channel_sync, + ___chmod, ___chmod_extended, ___close_nocancel, ___coalition, + ___coalition_info, ___coalition_ledger, ___commpage_gettimeofday, + ___connect, ___connect_nocancel, ___copyfile, ___crossarch_trap, + ___csrctl, ___darwin_check_fd_set_overflow, ___debug_syscall_reject, + ___debug_syscall_reject_config, ___delete, ___disable_threadsignal, + ___error, ___execve, ___exit, ___fchmod, ___fchmod_extended, + ___fcntl, ___fcntl_nocancel, ___fork, ___fs_snapshot, ___fstat64_extended, + ___fstat_extended, ___fsync_nocancel, ___get_remove_counter, + ___getattrlist, ___getdirentries64, ___gethostuuid, ___getlogin, + ___getpeername, ___getpid, ___getrlimit, ___getsgroups, ___getsockname, + ___gettid, ___gettimeofday, ___getwgroups, ___guarded_open_dprotected_np, + ___guarded_open_np, ___identitysvc, ___inc_remove_counter, + ___initgroups, ___ioctl, ___iopolicysys, ___kdebug_trace, + ___kdebug_trace64, ___kdebug_trace_string, ___kdebug_typefilter, + ___kill, ___kqueue_workloop_ctl, ___lchown, ___libkernel_init, + ___libkernel_init_after_boot_tasks, ___libkernel_init_late, + ___libkernel_platform_init, ___libkernel_voucher_init, ___listen, + ___log_data, ___lseek, ___lstat64_extended, ___lstat_extended, + ___mac_execve, ___mac_get_fd, ___mac_get_file, ___mac_get_link, + ___mac_get_mount, ___mac_get_pid, ___mac_get_proc, ___mac_getfsstat, + ___mac_mount, ___mac_set_fd, ___mac_set_file, ___mac_set_link, + ___mac_set_proc, ___mac_syscall, ___mach_bridge_remote_time, + ___mach_eventlink_signal, ___mach_eventlink_signal_wait_until, + ___mach_eventlink_wait_until, ___map_with_linking_np, ___memorystatus_available_memory, + ___microstackshot, ___mkdir_extended, ___mkfifo_extended, + ___mmap, ___mprotect, ___msgctl, ___msgrcv_nocancel, ___msgsnd_nocancel, + ___msgsys, ___msync, ___msync_nocancel, ___munmap, ___nexus_create, + ___nexus_deregister, ___nexus_destroy, ___nexus_get_opt, ___nexus_open, + ___nexus_register, ___nexus_set_opt, ___open, ___open_dprotected_np, + ___open_extended, ___open_nocancel, ___openat, ___openat_dprotected_np, + ___openat_nocancel, ___os_nexus_flow_add, ___os_nexus_flow_del, + ___os_nexus_get_llink_info, ___os_nexus_ifattach, ___os_nexus_ifdetach, + ___oslog_coproc, ___oslog_coproc_reg, ___persona, ___pipe, + ___poll_nocancel, ___posix_spawn, ___pread_nocancel, ___preadv_nocancel, + ___proc_info, ___proc_info_extended_id, ___process_policy, + ___pselect, ___pselect_nocancel, ___psynch_cvbroad, ___psynch_cvclrprepost, + ___psynch_cvsignal, ___psynch_cvwait, ___psynch_mutexdrop, + ___psynch_mutexwait, ___psynch_rw_downgrade, ___psynch_rw_longrdlock, + ___psynch_rw_rdlock, ___psynch_rw_unlock, ___psynch_rw_unlock2, + ___psynch_rw_upgrade, ___psynch_rw_wrlock, ___psynch_rw_yieldwrlock, + ___pthread_canceled, ___pthread_chdir, ___pthread_fchdir, + ___pthread_kill, ___pthread_markcancel, ___pthread_sigmask, + ___ptrace, ___pwrite_nocancel, ___pwritev_nocancel, ___read_nocancel, + ___readv_nocancel, ___reboot, ___record_system_event, ___recvfrom, + ___recvfrom_nocancel, ___recvmsg, ___recvmsg_nocancel, ___rename, + ___renameat, ___renameatx_np, ___rmdir, ___sandbox_me, ___sandbox_mm, + ___sandbox_ms, ___sandbox_msp, ___select, ___select_nocancel, + ___sem_open, ___sem_wait_nocancel, ___semctl, ___semsys, ___semwait_signal, + ___semwait_signal_nocancel, ___sendmsg, ___sendmsg_nocancel, + ___sendto, ___sendto_nocancel, ___setattrlist, ___setlogin, + ___setpriority, ___setregid, ___setreuid, ___setrlimit, ___setsgroups, + ___settid, ___settid_with_pid, ___settimeofday, ___setwgroups, + ___sfi_ctl, ___sfi_pidctl, ___shared_region_check_np, ___shared_region_map_and_slide_2_np, + ___shm_open, ___shmctl, ___shmsys, ___sigaction, ___sigaltstack, + ___sigreturn, ___sigsuspend, ___sigsuspend_nocancel, ___sigwait, + ___socketpair, ___stack_snapshot_with_config, ___stat64_extended, + ___stat_extended, ___syscall, ___syscall_logger, ___sysctl, + ___sysctlbyname, ___telemetry, ___terminate_with_payload, + ___thread_selfid, ___thread_selfusage, ___ulock_wait, ___ulock_wait2, + ___ulock_wake, ___umask_extended, ___unlink, ___unlinkat, + ___vfork, ___wait4, ___wait4_nocancel, ___waitid_nocancel, + ___work_interval_ctl, ___workq_kernreturn, ___workq_open, + ___write_nocancel, ___writev_nocancel, __cpu_capabilities, + __cpu_has_altivec, __exclaves_ctl_trap, __exit, __get_cpu_capabilities, + __getprivatesystemidentifier, __host_page_size, __init_cpu_capabilities, + __kernelrpc_host_create_mach_voucher, __kernelrpc_mach_port_allocate, + __kernelrpc_mach_port_allocate_full, __kernelrpc_mach_port_allocate_name, + __kernelrpc_mach_port_allocate_qos, __kernelrpc_mach_port_allocate_trap, + __kernelrpc_mach_port_assert_attributes, __kernelrpc_mach_port_construct, + __kernelrpc_mach_port_construct_trap, __kernelrpc_mach_port_deallocate, + __kernelrpc_mach_port_deallocate_trap, __kernelrpc_mach_port_destroy, + __kernelrpc_mach_port_destruct, __kernelrpc_mach_port_destruct_trap, + __kernelrpc_mach_port_dnrequest_info, __kernelrpc_mach_port_extract_member, + __kernelrpc_mach_port_extract_member_trap, __kernelrpc_mach_port_extract_right, + __kernelrpc_mach_port_get_attributes, __kernelrpc_mach_port_get_attributes_trap, + __kernelrpc_mach_port_get_context, __kernelrpc_mach_port_get_refs, + __kernelrpc_mach_port_get_service_port_info, __kernelrpc_mach_port_get_set_status, + __kernelrpc_mach_port_get_srights, __kernelrpc_mach_port_guard, + __kernelrpc_mach_port_guard_trap, __kernelrpc_mach_port_guard_with_flags, + __kernelrpc_mach_port_insert_member, __kernelrpc_mach_port_insert_member_trap, + __kernelrpc_mach_port_insert_right, __kernelrpc_mach_port_insert_right_trap, + __kernelrpc_mach_port_is_connection_for_service, __kernelrpc_mach_port_kernel_object, + __kernelrpc_mach_port_kobject, __kernelrpc_mach_port_kobject_description, + __kernelrpc_mach_port_mod_refs, __kernelrpc_mach_port_mod_refs_trap, + __kernelrpc_mach_port_move_member, __kernelrpc_mach_port_move_member_trap, + __kernelrpc_mach_port_names, __kernelrpc_mach_port_peek, __kernelrpc_mach_port_rename, + __kernelrpc_mach_port_request_notification, __kernelrpc_mach_port_request_notification_trap, + __kernelrpc_mach_port_set_attributes, __kernelrpc_mach_port_set_context, + __kernelrpc_mach_port_set_mscount, __kernelrpc_mach_port_set_seqno, + __kernelrpc_mach_port_space_basic_info, __kernelrpc_mach_port_space_info, + __kernelrpc_mach_port_special_reply_port_reset_link, __kernelrpc_mach_port_swap_guard, + __kernelrpc_mach_port_type, __kernelrpc_mach_port_type_trap, + __kernelrpc_mach_port_unguard, __kernelrpc_mach_port_unguard_trap, + __kernelrpc_mach_ports_lookup3, __kernelrpc_mach_ports_register3, + __kernelrpc_mach_task_is_self, __kernelrpc_mach_vm_allocate, + __kernelrpc_mach_vm_allocate_trap, __kernelrpc_mach_vm_deallocate, + __kernelrpc_mach_vm_deallocate_trap, __kernelrpc_mach_vm_map, + __kernelrpc_mach_vm_map_trap, __kernelrpc_mach_vm_protect, + __kernelrpc_mach_vm_protect_trap, __kernelrpc_mach_vm_purgable_control, + __kernelrpc_mach_vm_purgable_control_trap, __kernelrpc_mach_vm_read, + __kernelrpc_mach_vm_remap, __kernelrpc_mach_vm_remap_new, + __kernelrpc_mach_vm_update_pointers_with_remote_tags, __kernelrpc_mach_voucher_extract_attr_recipe, + __kernelrpc_task_set_port_space, __kernelrpc_thread_policy, + __kernelrpc_thread_policy_set, __kernelrpc_thread_set_policy, + __kernelrpc_vm_map, __kernelrpc_vm_purgable_control, __kernelrpc_vm_read, + __kernelrpc_vm_remap, __kernelrpc_vm_remap_new, __mach_errors, + __mach_fork_child, __mach_snprintf, __mach_vsnprintf, __os_alloc_once_table, + __os_xbs_chrooted, __register_gethostuuid_callback, __thread_set_tsd_base, + _abort_with_payload, _abort_with_reason, _accept, '_accept$NOCANCEL', + _access, _accessx_np, _acct, _act_get_state, _act_set_state, + _adjtime, _aio_cancel, _aio_error, _aio_fsync, _aio_read, + _aio_return, _aio_suspend, '_aio_suspend$NOCANCEL', _aio_write, + _audit, _audit_session_join, _audit_session_port, _audit_session_self, + _auditctl, _auditon, _bind, _bootstrap_port, _cerror, _cerror_nocancel, + _change_fdguard_np, _chdir, _chflags, _chmod, _chown, _chroot, + _clock_alarm, _clock_alarm_reply, _clock_get_attributes, _clock_get_time, + _clock_set_attributes, _clock_set_time, _clock_sleep, _clock_sleep_trap, + _clonefile, _clonefileat, _close, '_close$NOCANCEL', _coalition_create, + _coalition_info_debug_info, _coalition_info_pid_list, _coalition_info_resource_usage, + _coalition_info_set_efficiency, _coalition_info_set_name, + _coalition_ledger_set_logical_writes_limit, _coalition_policy_get, + _coalition_policy_set, _coalition_reap, _coalition_terminate, + _connect, '_connect$NOCANCEL', _connectx, _csops, _csops_audittoken, + _csr_check, _csr_get_active_config, _debug_control_port_for_pid, + _debug_syscall_reject, _debug_syscall_reject_config, _denap_boost_assertion_token, + _disconnectx, _dup, _dup2, _errno, _etap_trace_thread, _exc_server, + _exc_server_routine, _exception_raise, _exception_raise_state, + _exception_raise_state_identity, _exchangedata, _exclaves_aoe_message_loop, + _exclaves_aoe_setup, _exclaves_aoe_work_loop, _exclaves_audio_buffer_copyout, + _exclaves_audio_buffer_copyout_with_status, _exclaves_audio_buffer_create, + _exclaves_boot, _exclaves_endpoint_call, _exclaves_inbound_buffer_copyin, + _exclaves_inbound_buffer_create, _exclaves_indicator_min_on_time, + _exclaves_launch_conclave, _exclaves_lookup_service, _exclaves_named_buffer_copyin, + _exclaves_named_buffer_copyout, _exclaves_named_buffer_create, + _exclaves_notification_create, _exclaves_outbound_buffer_copyout, + _exclaves_outbound_buffer_create, _exclaves_sensor_create, + _exclaves_sensor_start, _exclaves_sensor_status, _exclaves_sensor_stop, + _execve, _faccessat, _fchdir, _fchflags, _fchmod, _fchmodat, + _fchown, _fchownat, _fclonefileat, _fcntl, '_fcntl$NOCANCEL', + _fdatasync, _ffsctl, _fgetattrlist, _fgetxattr, _fhopen, _fileport_makefd, + _fileport_makeport, _flistxattr, _flock, _fmount, _fpathconf, + _freadlink, _fremovexattr, _fs_snapshot_create, _fs_snapshot_delete, + _fs_snapshot_list, _fs_snapshot_mount, _fs_snapshot_rename, + _fs_snapshot_revert, _fs_snapshot_root, _fsctl, _fsetattrlist, + _fsetxattr, _fsgetpath, _fsgetpath_ext, _fstat, _fstat64, + _fstatat, _fstatat64, _fstatfs, _fstatfs64, _fstatfs_ext, + _fsync, '_fsync$NOCANCEL', _ftruncate, _funmount, _futimens, + _futimes, _getattrlist, _getattrlistat, _getattrlistbulk, + _getaudit, _getaudit_addr, _getauid, _getdirentries, _getdirentriesattr, + _getdtablesize, _getegid, _getentropy, _geteuid, _getfh, _getfsstat, + _getfsstat64, _getgid, _getgroups, _gethostuuid, _getiopolicy_np, + _getitimer, _getpeername, _getpgid, _getpgrp, _getpid, _getppid, + _getpriority, _getrlimit, _getrusage, _getsgroups_np, _getsid, + _getsockname, _getsockopt, _getuid, _getwgroups_np, _getxattr, + _grab_pgo_data, _graftdmg, _guarded_close_np, _guarded_kqueue_np, + _guarded_open_dprotected_np, _guarded_open_np, _guarded_pwrite_np, + _guarded_write_np, _guarded_writev_np, _host_check_multiuser_mode, + _host_create_mach_voucher, _host_create_mach_voucher_trap, + _host_default_memory_manager, _host_get_UNDServer, _host_get_atm_diagnostic_flag, + _host_get_boot_info, _host_get_clock_control, _host_get_clock_service, + _host_get_exception_ports, _host_get_io_main, _host_get_io_master, + _host_get_multiuser_config_flags, _host_get_special_port, + _host_info, _host_kernel_version, _host_lockgroup_info, _host_page_size, + _host_priv_statistics, _host_processor_info, _host_processor_set_priv, + _host_processor_sets, _host_processors, _host_reboot, _host_register_mach_voucher_attr_manager, + _host_register_well_known_mach_voucher_attr_manager, _host_request_notification, + _host_security_create_task_token, _host_security_set_task_token, + _host_self, _host_self_trap, _host_set_UNDServer, _host_set_atm_diagnostic_flag, + _host_set_exception_ports, _host_set_multiuser_config_flags, + _host_set_special_port, _host_statistics, _host_statistics64, + _host_swap_exception_ports, _host_virtual_physical_table_info, + _important_boost_assertion_token, _internal_catch_exc_subsystem, + _ioctl, _issetugid, _kas_info, _kdebug_is_enabled, _kdebug_signpost, + _kdebug_signpost_end, _kdebug_signpost_start, _kdebug_timestamp, + _kdebug_timestamp_from_absolute, _kdebug_timestamp_from_continuous, + _kdebug_trace, _kdebug_trace_string, _kdebug_typefilter, _kdebug_using_continuous_time, + _kevent, _kevent64, _kevent_id, _kevent_qos, _kext_request, + _kill, _kmod_control, _kmod_create, _kmod_destroy, _kmod_get_info, + _kpersona_alloc, _kpersona_dealloc, _kpersona_find, _kpersona_find_by_type, + _kpersona_get, _kpersona_getpath, _kpersona_info, _kpersona_palloc, + _kpersona_pidinfo, _kqueue, _lchown, _ledger, _link, _linkat, + _lio_listio, _listen, _listxattr, _lock_set_create, _lock_set_destroy, + _log_data_as_kernel, _lseek, _lstat, _lstat64, _mach_absolute_time, + _mach_approximate_time, _mach_boottime_usec, _mach_continuous_approximate_time, + _mach_continuous_time, _mach_error, _mach_error_full_diag, + _mach_error_string, _mach_error_type, _mach_eventlink_associate, + _mach_eventlink_create, _mach_eventlink_destroy, _mach_eventlink_disassociate, + _mach_eventlink_signal, _mach_eventlink_signal_wait_until, + _mach_eventlink_wait_until, _mach_generate_activity_id, _mach_get_times, + _mach_host_self, _mach_host_special_port_description, _mach_host_special_port_for_id, + _mach_init, _mach_make_memory_entry, _mach_make_memory_entry_64, + _mach_memory_entry_access_tracking, _mach_memory_entry_get_page_counts, + _mach_memory_entry_ownership, _mach_memory_entry_purgable_control, + _mach_memory_info, _mach_memory_object_memory_entry, _mach_memory_object_memory_entry_64, + _mach_msg, _mach_msg2_internal, _mach_msg2_trap, _mach_msg_destroy, + _mach_msg_overwrite, _mach_msg_overwrite_trap, _mach_msg_priority_encode, + _mach_msg_priority_is_pthread_priority, _mach_msg_priority_overide_qos, + _mach_msg_priority_qos, _mach_msg_priority_relpri, _mach_msg_receive, + _mach_msg_send, _mach_msg_server, _mach_msg_server_importance, + _mach_msg_server_once, _mach_msg_trap, _mach_notify_dead_name, + _mach_notify_no_senders, _mach_notify_port_deleted, _mach_notify_port_destroyed, + _mach_notify_send_once, _mach_port_allocate, _mach_port_allocate_full, + _mach_port_allocate_name, _mach_port_allocate_qos, _mach_port_assert_attributes, + _mach_port_construct, _mach_port_deallocate, _mach_port_destroy, + _mach_port_destruct, _mach_port_dnrequest_info, _mach_port_extract_member, + _mach_port_extract_right, _mach_port_get_attributes, _mach_port_get_context, + _mach_port_get_refs, _mach_port_get_service_port_info, _mach_port_get_set_status, + _mach_port_get_srights, _mach_port_guard, _mach_port_guard_with_flags, + _mach_port_insert_member, _mach_port_insert_right, _mach_port_is_connection_for_service, + _mach_port_kernel_object, _mach_port_kobject, _mach_port_kobject_description, + _mach_port_mod_refs, _mach_port_move_member, _mach_port_names, + _mach_port_peek, _mach_port_rename, _mach_port_request_notification, + _mach_port_set_attributes, _mach_port_set_context, _mach_port_set_mscount, + _mach_port_set_seqno, _mach_port_space_basic_info, _mach_port_space_info, + _mach_port_swap_guard, _mach_port_type, _mach_port_unguard, + _mach_ports_lookup, _mach_ports_register, _mach_reply_port, + _mach_right_recv_construct, _mach_right_recv_destruct, _mach_right_send_create, + _mach_right_send_once_consume, _mach_right_send_once_create, + _mach_right_send_release, _mach_right_send_retain, _mach_sync_ipc_link_monitoring_start, + _mach_sync_ipc_link_monitoring_stop, _mach_task_is_self, _mach_task_self, + _mach_task_self_, _mach_task_special_port_description, _mach_task_special_port_for_id, + _mach_thread_self, _mach_thread_special_port_description, + _mach_thread_special_port_for_id, _mach_timebase_info, _mach_timebase_info_trap, + _mach_vm_allocate, _mach_vm_behavior_set, _mach_vm_copy, _mach_vm_deallocate, + _mach_vm_deferred_reclamation_buffer_allocate, _mach_vm_deferred_reclamation_buffer_flush, + _mach_vm_deferred_reclamation_buffer_query, _mach_vm_deferred_reclamation_buffer_resize, + _mach_vm_inherit, _mach_vm_machine_attribute, _mach_vm_map, + _mach_vm_msync, _mach_vm_page_info, _mach_vm_page_query, _mach_vm_page_range_query, + _mach_vm_protect, _mach_vm_purgable_control, _mach_vm_range_create, + _mach_vm_read, _mach_vm_read_list, _mach_vm_read_overwrite, + _mach_vm_reclaim_copied_ring_free, _mach_vm_reclaim_copied_ring_query, + _mach_vm_reclaim_get_rings_for_task, _mach_vm_reclaim_is_reusable, + _mach_vm_reclaim_query_state, _mach_vm_reclaim_ring_allocate, + _mach_vm_reclaim_ring_capacity, _mach_vm_reclaim_ring_copy, + _mach_vm_reclaim_ring_flush, _mach_vm_reclaim_ring_resize, + _mach_vm_reclaim_round_capacity, _mach_vm_reclaim_try_cancel, + _mach_vm_reclaim_try_enter, _mach_vm_reclaim_update_kernel_accounting, + _mach_vm_reclaim_update_kernel_accounting_trap, _mach_vm_region, + _mach_vm_region_recurse, _mach_vm_remap, _mach_vm_remap_new, + _mach_vm_tag_describe, _mach_vm_update_pointers_with_remote_tags, + _mach_vm_wire, _mach_vm_write, _mach_voucher_attr_command, + _mach_voucher_deallocate, _mach_voucher_debug_info, _mach_voucher_extract_all_attr_recipes, + _mach_voucher_extract_attr_content, _mach_voucher_extract_attr_recipe, + _mach_voucher_extract_attr_recipe_trap, _mach_wait_until, + _mach_zone_force_gc, _mach_zone_get_btlog_records, _mach_zone_get_zlog_zones, + _mach_zone_info, _mach_zone_info_for_largest_zone, _mach_zone_info_for_zone, + _macx_backing_store_recovery, _macx_backing_store_suspend, + _macx_swapoff, _macx_swapon, _macx_triggers, _madvise, _memorystatus_control, + _memorystatus_get_level, _mig_allocate, _mig_dealloc_reply_port, + _mig_dealloc_special_reply_port, _mig_deallocate, _mig_get_reply_port, + _mig_get_special_reply_port, _mig_put_reply_port, _mig_reply_setup, + _mig_strncpy, _mig_strncpy_zerofill, _mincore, _minherit, + _mk_timer_arm, _mk_timer_arm_leeway, _mk_timer_cancel, _mk_timer_create, + _mk_timer_destroy, _mkdir, _mkdirat, _mkfifo, _mkfifoat, _mknod, + _mknodat, _mlock, _mlockall, _mmap, _mount, _mprotect, _mremap_encrypted, + _msg_receive, _msg_rpc, _msg_send, _msgctl, _msgget, _msgrcv, + '_msgrcv$NOCANCEL', _msgsnd, '_msgsnd$NOCANCEL', _msgsys, + _msync, '_msync$NOCANCEL', _munlock, _munlockall, _munmap, + _necp_client_action, _necp_match_policy, _necp_open, _necp_session_action, + _necp_session_open, _net_qos_guideline, _netagent_trigger, + _netname_check_in, _netname_check_out, _netname_look_up, _netname_version, + _nfsclnt, _nfssvc, _non_boost_assertion_token, _normal_boost_assertion_token, + _ntp_adjtime, _ntp_gettime, _objc_bp_assist_cfg_np, _open, + '_open$NOCANCEL', _open_dprotected_np, _openat, '_openat$NOCANCEL', + _openat_authenticated_np, _openat_dprotected_np, _openbyid_np, + _os_buflet_get_data_address, _os_buflet_get_data_length, _os_buflet_get_data_limit, + _os_buflet_get_data_offset, _os_buflet_get_object_address, + _os_buflet_get_object_limit, _os_buflet_set_data_length, _os_buflet_set_data_offset, + _os_channel_advance_slot, _os_channel_attr_clone, _os_channel_attr_create, + _os_channel_attr_destroy, _os_channel_attr_get, _os_channel_attr_get_key, + _os_channel_attr_set, _os_channel_attr_set_key, _os_channel_available_slot_count, + _os_channel_buflet_alloc, _os_channel_buflet_free, _os_channel_configure_interface_advisory, + _os_channel_create, _os_channel_create_extended, _os_channel_destroy, + _os_channel_event_free, _os_channel_event_get_event_data, + _os_channel_event_get_next_event, _os_channel_flow_admissible, + _os_channel_flow_adv_get_ce_count, _os_channel_flow_adv_get_feedback, + _os_channel_get_advisory_region, _os_channel_get_fd, _os_channel_get_interface_advisory, + _os_channel_get_next_event_handle, _os_channel_get_next_slot, + _os_channel_get_stats_region, _os_channel_get_upp_buffer_stats, + _os_channel_is_defunct, _os_channel_large_packet_alloc, _os_channel_packet_alloc, + _os_channel_packet_free, _os_channel_packet_pool_purge, _os_channel_pending, + _os_channel_read_attr, _os_channel_read_nexus_extension_info, + _os_channel_ring_id, _os_channel_ring_notify_time, _os_channel_ring_sync_time, + _os_channel_rx_ring, _os_channel_set_slot_properties, _os_channel_slot_attach_packet, + _os_channel_slot_detach_packet, _os_channel_slot_get_packet, + _os_channel_sync, _os_channel_tx_ring, _os_channel_write_attr, + _os_copy_and_inet_checksum, _os_cpu_copy_in_cksum, _os_cpu_in_cksum, + _os_cpu_in_cksum_mbuf, _os_fault_with_payload, _os_inet_checksum, + _os_log_coprocessor_as_kernel, _os_log_coprocessor_register_as_kernel, + _os_nexus_attr_clone, _os_nexus_attr_create, _os_nexus_attr_destroy, + _os_nexus_attr_get, _os_nexus_attr_set, _os_nexus_controller_add_traffic_rule, + _os_nexus_controller_alloc_provider_instance, _os_nexus_controller_bind_provider_instance, + _os_nexus_controller_create, _os_nexus_controller_deregister_provider, + _os_nexus_controller_destroy, _os_nexus_controller_free_provider_instance, + _os_nexus_controller_get_fd, _os_nexus_controller_iterate_traffic_rules, + _os_nexus_controller_read_provider_attr, _os_nexus_controller_register_provider, + _os_nexus_controller_remove_traffic_rule, _os_nexus_controller_unbind_provider_instance, + _os_nexus_flow_set_connection_idle, _os_nexus_flow_set_wake_from_sleep, + _os_packet_add_buflet, _os_packet_add_inet_csum_flags, _os_packet_clear_flow_uuid, + _os_packet_decrement_use_count, _os_packet_finalize, _os_packet_get_aggregation_type, + _os_packet_get_buflet_count, _os_packet_get_compression_generation_count, + _os_packet_get_data_length, _os_packet_get_expire_time, _os_packet_get_expiry_action, + _os_packet_get_flow_uuid, _os_packet_get_group_end, _os_packet_get_group_start, + _os_packet_get_headroom, _os_packet_get_inet_checksum, _os_packet_get_keep_alive, + _os_packet_get_link_broadcast, _os_packet_get_link_ethfcs, + _os_packet_get_link_header_length, _os_packet_get_link_multicast, + _os_packet_get_next_buflet, _os_packet_get_packetid, _os_packet_get_segment_count, + _os_packet_get_service_class, _os_packet_get_token, _os_packet_get_trace_id, + _os_packet_get_traffic_class, _os_packet_get_transport_retransmit, + _os_packet_get_transport_traffic_background, _os_packet_get_transport_traffic_realtime, + _os_packet_get_truncated, _os_packet_get_vlan_id, _os_packet_get_vlan_priority, + _os_packet_get_vlan_tag, _os_packet_get_wake_flag, _os_packet_increment_use_count, + _os_packet_set_app_metadata, _os_packet_set_compression_generation_count, + _os_packet_set_expire_time, _os_packet_set_expiry_action, + _os_packet_set_flow_uuid, _os_packet_set_group_end, _os_packet_set_group_start, + _os_packet_set_headroom, _os_packet_set_inet_checksum, _os_packet_set_keep_alive, + _os_packet_set_l4s_flag, _os_packet_set_link_broadcast, _os_packet_set_link_ethfcs, + _os_packet_set_link_header_length, _os_packet_set_link_multicast, + _os_packet_set_packetid, _os_packet_set_protocol_segment_size, + _os_packet_set_service_class, _os_packet_set_token, _os_packet_set_trace_id, + _os_packet_set_traffic_class, _os_packet_set_transport_last_packet, + _os_packet_set_transport_retransmit, _os_packet_set_transport_traffic_background, + _os_packet_set_transport_traffic_realtime, _os_packet_set_tso_flags, + _os_packet_set_tx_timestamp, _os_packet_set_vlan_tag, _os_packet_set_wake_flag, + _os_packet_trace_event, _os_proc_available_memory, _panic, + _panic_init, _panic_with_data, _pathconf, _peeloff, _pid_for_task, + _pid_hibernate, _pid_resume, _pid_shutdown_networking, _pid_shutdown_sockets, + _pid_suspend, _pipe, _pivot_root, _poll, '_poll$NOCANCEL', + _port_obj_init, _port_obj_table, _port_obj_table_size, _posix_madvise, + _posix_spawn, _posix_spawn_file_actions_add_fileportdup2_np, + _posix_spawn_file_actions_addchdir, _posix_spawn_file_actions_addchdir_np, + _posix_spawn_file_actions_addclose, _posix_spawn_file_actions_adddup2, + _posix_spawn_file_actions_addfchdir, _posix_spawn_file_actions_addfchdir_np, + _posix_spawn_file_actions_addinherit_np, _posix_spawn_file_actions_addopen, + _posix_spawn_file_actions_destroy, _posix_spawn_file_actions_init, + _posix_spawnattr_destroy, _posix_spawnattr_disable_ptr_auth_a_keys_np, + _posix_spawnattr_get_darwin_role_np, _posix_spawnattr_get_qos_clamp_np, + _posix_spawnattr_getarchpref_np, _posix_spawnattr_getbinpref_np, + _posix_spawnattr_getcpumonitor, _posix_spawnattr_getflags, + _posix_spawnattr_getmacpolicyinfo_np, _posix_spawnattr_getpcontrol_np, + _posix_spawnattr_getpgroup, _posix_spawnattr_getprocesstype_np, + _posix_spawnattr_getsigdefault, _posix_spawnattr_getsigmask, + _posix_spawnattr_init, _posix_spawnattr_set_alt_rosetta_np, + _posix_spawnattr_set_conclave_id_np, _posix_spawnattr_set_conclavememlimit_ext, + _posix_spawnattr_set_crash_behavior_deadline_np, _posix_spawnattr_set_crash_behavior_np, + _posix_spawnattr_set_crash_count_np, _posix_spawnattr_set_csm_np, + _posix_spawnattr_set_darwin_role_np, _posix_spawnattr_set_filedesclimit_ext, + _posix_spawnattr_set_gid_np, _posix_spawnattr_set_groups_np, + _posix_spawnattr_set_importancewatch_port_np, _posix_spawnattr_set_jetsam_ttr_np, + _posix_spawnattr_set_kqworklooplimit_ext, _posix_spawnattr_set_launch_type_np, + _posix_spawnattr_set_login_np, _posix_spawnattr_set_max_addr_np, + _posix_spawnattr_set_persona_gid_np, _posix_spawnattr_set_persona_groups_np, + _posix_spawnattr_set_persona_np, _posix_spawnattr_set_persona_uid_np, + _posix_spawnattr_set_platform_np, _posix_spawnattr_set_portlimits_ext, + _posix_spawnattr_set_ptrauth_task_port_np, _posix_spawnattr_set_qos_clamp_np, + _posix_spawnattr_set_registered_ports_np, _posix_spawnattr_set_subsystem_root_path_np, + _posix_spawnattr_set_threadlimit_ext, _posix_spawnattr_set_uid_np, + _posix_spawnattr_set_use_sec_transition_shims_np, _posix_spawnattr_setarchpref_np, + _posix_spawnattr_setauditsessionport_np, _posix_spawnattr_setbinpref_np, + _posix_spawnattr_setcoalition_np, _posix_spawnattr_setcpumonitor, + _posix_spawnattr_setcpumonitor_default, _posix_spawnattr_setdataless_iopolicy_np, + _posix_spawnattr_setexceptionports_np, _posix_spawnattr_setflags, + _posix_spawnattr_setjetsam_ext, _posix_spawnattr_setmacpolicyinfo_np, + _posix_spawnattr_setnosmt_np, _posix_spawnattr_setpcontrol_np, + _posix_spawnattr_setpgroup, _posix_spawnattr_setprocesstype_np, + _posix_spawnattr_setsigdefault, _posix_spawnattr_setsigmask, + _posix_spawnattr_setspecialport_np, _pread, '_pread$NOCANCEL', + _preadv, '_preadv$NOCANCEL', _proc_clear_cpulimits, _proc_clear_delayidlesleep, + _proc_clear_dirty, _proc_clear_vmpressure, _proc_current_thread_schedinfo, + _proc_denap_assertion_begin_with_msg, _proc_denap_assertion_complete, + _proc_disable_apptype, _proc_disable_cpumon, _proc_disable_wakemon, + _proc_donate_importance_boost, _proc_enable_apptype, _proc_get_cpumon_params, + _proc_get_dirty, _proc_get_wakemon_params, _proc_importance_assertion_begin_with_msg, + _proc_importance_assertion_complete, _proc_kmsgbuf, _proc_libversion, + _proc_list_dynkqueueids, _proc_list_uptrs, _proc_listallpids, + _proc_listchildpids, _proc_listcoalitions, _proc_listpgrppids, + _proc_listpids, _proc_listpidspath, _proc_name, _proc_pid_rusage, + _proc_piddynkqueueinfo, _proc_pidfdinfo, _proc_pidfileportinfo, + _proc_pidinfo, _proc_pidoriginatorinfo, _proc_pidpath, _proc_pidpath_audittoken, + _proc_regionfilename, _proc_reset_footprint_interval, _proc_resume_cpumon, + _proc_rlimit_control, _proc_set_cpumon_defaults, _proc_set_cpumon_params, + _proc_set_cpumon_params_fatal, _proc_set_csm, _proc_set_delayidlesleep, + _proc_set_dirty, _proc_set_no_smt, _proc_set_owner_vmpressure, + _proc_set_wakemon_defaults, _proc_set_wakemon_params, _proc_setcpu_percentage, + _proc_setpcontrol, _proc_setthread_cpupercent, _proc_setthread_csm, + _proc_setthread_no_smt, _proc_signal_delegate, _proc_signal_with_audittoken, + _proc_suppress, _proc_terminate, _proc_terminate_all_rsr, + _proc_terminate_delegate, _proc_terminate_with_audittoken, + _proc_trace_log, _proc_track_dirty, _proc_udata_info, _proc_uuid_policy, + _processor_assign, _processor_control, _processor_exit, _processor_get_assignment, + _processor_info, _processor_set_create, _processor_set_default, + _processor_set_destroy, _processor_set_info, _processor_set_max_priority, + _processor_set_policy_control, _processor_set_policy_disable, + _processor_set_policy_enable, _processor_set_stack_usage, + _processor_set_statistics, _processor_set_tasks, _processor_set_tasks_with_flavor, + _processor_set_threads, _processor_start, _pselect, '_pselect$DARWIN_EXTSN', + '_pselect$DARWIN_EXTSN$NOCANCEL', '_pselect$NOCANCEL', _pthread_getugid_np, + _pthread_setugid_np, _ptrace, _pwrite, '_pwrite$NOCANCEL', + _pwritev, '_pwritev$NOCANCEL', _quota, _quotactl, _read, '_read$NOCANCEL', + _readlink, _readlinkat, _readv, '_readv$NOCANCEL', _reboot, + _reboot_np, _record_system_event_as_kernel, _recvfrom, '_recvfrom$NOCANCEL', + _recvmsg, '_recvmsg$NOCANCEL', _recvmsg_x, _register_uexc_handler, + _removexattr, _rename, _rename_ext, _renameat, _renameatx_np, + _renamex_np, _revoke, _rmdir, _searchfs, _select, '_select$DARWIN_EXTSN', + '_select$DARWIN_EXTSN$NOCANCEL', '_select$NOCANCEL', _sem_close, + _sem_destroy, _sem_getvalue, _sem_init, _sem_open, _sem_post, + _sem_trywait, _sem_unlink, _sem_wait, '_sem_wait$NOCANCEL', + _semaphore_create, _semaphore_destroy, _semaphore_signal, + _semaphore_signal_all, _semaphore_signal_all_trap, _semaphore_signal_thread, + _semaphore_signal_thread_trap, _semaphore_signal_trap, _semaphore_timedwait, + _semaphore_timedwait_signal, _semaphore_timedwait_signal_trap, + _semaphore_timedwait_trap, _semaphore_wait, _semaphore_wait_signal, + _semaphore_wait_signal_trap, _semaphore_wait_trap, _semctl, + _semget, _semop, _semsys, _sendfile, _sendmsg, '_sendmsg$NOCANCEL', + _sendmsg_x, _sendto, '_sendto$NOCANCEL', _setattrlist, _setattrlistat, + _setaudit, _setaudit_addr, _setauid, _setegid, _seteuid, _setgid, + _setgroups, _setiopolicy_np, _setitimer, _setpgid, _setpriority, + _setprivexec, _setquota, _setregid, _setreuid, _setrlimit, + _setsgroups_np, _setsid, _setsockopt, _setuid, _setwgroups_np, + _setxattr, _sfi_get_class_offtime, _sfi_process_get_flags, + _sfi_process_set_flags, _sfi_set_class_offtime, _shm_open, + _shm_unlink, _shmat, _shmctl, _shmdt, _shmget, _shmsys, _shutdown, + _sigpending, _sigprocmask, _sigsuspend, '_sigsuspend$NOCANCEL', + _socket, _socket_delegate, _socketpair, _stackshot_capture_with_config, + _stackshot_config_create, _stackshot_config_dealloc, _stackshot_config_dealloc_buffer, + _stackshot_config_get_stackshot_buffer, _stackshot_config_get_stackshot_size, + _stackshot_config_set_delta_timestamp, _stackshot_config_set_flags, + _stackshot_config_set_pagetable_mask, _stackshot_config_set_pid, + _stackshot_config_set_size_hint, _stat, _stat64, _statfs, + _statfs64, _statfs_ext, _swapon, _swtch, _swtch_pri, _symlink, + _symlinkat, _sync, _syscall, _syscall_thread_switch, _system_get_sfi_window, + _system_override, _system_set_sfi_window, _system_version_compat_mode, + _task_assign, _task_assign_default, _task_create, _task_create_identity_token, + _task_dyld_process_info_notify_deregister, _task_dyld_process_info_notify_get, + _task_dyld_process_info_notify_register, _task_for_pid, _task_generate_corpse, + _task_get_assignment, _task_get_dyld_image_infos, _task_get_emulation_vector, + _task_get_exc_guard_behavior, _task_get_exception_ports, _task_get_exception_ports_info, + _task_get_mach_voucher, _task_get_special_port, _task_get_state, + _task_identity_token_get_task_port, _task_info, _task_inspect, + _task_inspect_for_pid, _task_map_corpse_info, _task_map_corpse_info_64, + _task_map_kcdata_object_64, _task_name_for_pid, _task_policy, + _task_policy_get, _task_policy_set, _task_purgable_info, _task_read_for_pid, + _task_register_dyld_get_process_state, _task_register_dyld_image_infos, + _task_register_dyld_set_dyld_state, _task_register_dyld_shared_cache_image_info, + _task_register_hardened_exception_handler, _task_restartable_ranges_register, + _task_restartable_ranges_synchronize, _task_resume, _task_resume2, + _task_sample, _task_self_, _task_self_trap, _task_set_corpse_forking_behavior, + _task_set_emulation, _task_set_emulation_vector, _task_set_exc_guard_behavior, + _task_set_exception_ports, _task_set_info, _task_set_mach_voucher, + _task_set_phys_footprint_limit, _task_set_policy, _task_set_port_space, + _task_set_ras_pc, _task_set_special_port, _task_set_state, + _task_suspend, _task_suspend2, _task_swap_exception_ports, + _task_swap_mach_voucher, _task_terminate, _task_test_async_upcall_propagation, + _task_test_sync_upcall, _task_threads, _task_unregister_dyld_image_infos, + _task_zone_info, _terminate_with_payload, _terminate_with_reason, + _thread_abort, _thread_abort_safely, _thread_adopt_exception_handler, + _thread_assign, _thread_assign_default, _thread_convert_thread_state, + _thread_create, _thread_create_running, _thread_depress_abort, + _thread_destruct_special_reply_port, _thread_get_assignment, + _thread_get_exception_ports, _thread_get_exception_ports_info, + _thread_get_mach_voucher, _thread_get_register_pointer_values, + _thread_get_special_port, _thread_get_special_reply_port, + _thread_get_state, _thread_info, _thread_policy, _thread_policy_get, + _thread_policy_set, _thread_resume, _thread_sample, _thread_self_trap, + _thread_selfcounts, _thread_set_exception_ports, _thread_set_mach_voucher, + _thread_set_policy, _thread_set_special_port, _thread_set_state, + _thread_suspend, _thread_swap_exception_ports, _thread_swap_mach_voucher, + _thread_switch, _thread_terminate, _thread_wire, _tracker_action, + _truncate, _umask, _undelete, _ungraftdmg, _unlink, _unlinkat, + _unmount, _usrctl, _utimensat, _utimes, _vfs_purge, _vm_allocate, + _vm_allocate_cpm, _vm_behavior_set, _vm_copy, _vm_deallocate, + _vm_inherit, _vm_kernel_page_mask, _vm_kernel_page_shift, + _vm_kernel_page_size, _vm_machine_attribute, _vm_map, _vm_map_page_query, + _vm_msync, _vm_page_mask, _vm_page_shift, _vm_page_size, _vm_pressure_monitor, + _vm_protect, _vm_purgable_control, _vm_read, _vm_read_list, + _vm_read_overwrite, _vm_region_64, _vm_region_recurse_64, + _vm_remap, _vm_remap_new, _vm_wire, _vm_write, _voucher_mach_msg_adopt, + _voucher_mach_msg_clear, _voucher_mach_msg_revert, _voucher_mach_msg_set, + _vprintf_stderr_func, _wait4, _waitid, '_waitid$NOCANCEL', + _work_interval_copy_port, _work_interval_create, _work_interval_destroy, + _work_interval_get_flags_from_port, _work_interval_instance_alloc, + _work_interval_instance_clear, _work_interval_instance_finish, + _work_interval_instance_free, _work_interval_instance_get_complexity, + _work_interval_instance_get_deadline, _work_interval_instance_get_finish, + _work_interval_instance_get_id, _work_interval_instance_get_start, + _work_interval_instance_get_telemetry_data, _work_interval_instance_set_complexity, + _work_interval_instance_set_deadline, _work_interval_instance_set_finish, + _work_interval_instance_set_start, _work_interval_instance_start, + _work_interval_instance_update, _work_interval_join, _work_interval_join_port, + _work_interval_leave, _work_interval_notify, _work_interval_notify_simple, + _write, '_write$NOCANCEL', _writev, '_writev$NOCANCEL' ] + - targets: [ arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___fstat, ___fstatat, ___fstatfs, ___getfsstat, ___lstat, + ___stat, ___statfs, __current_pid, _mach_absolute_time_kernel, + _mach_continuous_time_kernel ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, x86_64h-macos, x86_64h-maccatalyst, + arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_m.dylib' +current-version: 3309 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, x86_64h-macos, x86_64h-maccatalyst, + arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, x86_64h-macos, x86_64h-maccatalyst ] + symbols: [ __FE_DFL_DISABLE_SSE_DENORMS_ENV, ___cos_d4, ___cos_f8, ___sin_d4, + ___sin_f8, __simd_acos_d4, __simd_acos_d8, __simd_acos_f16, + __simd_acos_f8, __simd_acosh_d4, __simd_acosh_d8, __simd_acosh_f16, + __simd_acosh_f8, __simd_asin_d4, __simd_asin_d8, __simd_asin_f16, + __simd_asin_f8, __simd_asinh_d4, __simd_asinh_d8, __simd_asinh_f16, + __simd_asinh_f8, __simd_atan2_d4, __simd_atan2_d8, __simd_atan2_f16, + __simd_atan2_f8, __simd_atan_d4, __simd_atan_d8, __simd_atan_f16, + __simd_atan_f8, __simd_atanh_d4, __simd_atanh_d8, __simd_atanh_f16, + __simd_atanh_f8, __simd_cbrt_d4, __simd_cbrt_d8, __simd_cbrt_f16, + __simd_cbrt_f8, __simd_cos_d4, __simd_cos_d8, __simd_cos_f16, + __simd_cos_f8, __simd_cosh_d4, __simd_cosh_d8, __simd_cosh_f16, + __simd_cosh_f8, __simd_cospi_d4, __simd_cospi_d8, __simd_cospi_f16, + __simd_cospi_f8, __simd_erf_d4, __simd_erf_d8, __simd_erf_f16, + __simd_erf_f8, __simd_erfc_d4, __simd_erfc_d8, __simd_erfc_f16, + __simd_erfc_f8, __simd_exp10_d4, __simd_exp10_d8, __simd_exp10_f16, + __simd_exp10_f8, __simd_exp2_d4, __simd_exp2_d8, __simd_exp2_f16, + __simd_exp2_f8, __simd_exp_d4, __simd_exp_d8, __simd_exp_f16, + __simd_exp_f8, __simd_expm1_d4, __simd_expm1_d8, __simd_expm1_f16, + __simd_expm1_f8, __simd_fmod_d4, __simd_fmod_d8, __simd_fmod_f16, + __simd_fmod_f8, __simd_hypot_d4, __simd_hypot_d8, __simd_hypot_f16, + __simd_hypot_f8, __simd_lgamma_d4, __simd_lgamma_d8, __simd_lgamma_f16, + __simd_lgamma_f8, __simd_log10_d4, __simd_log10_d8, __simd_log10_f16, + __simd_log10_f8, __simd_log1p_d4, __simd_log1p_d8, __simd_log1p_f16, + __simd_log1p_f8, __simd_log2_d4, __simd_log2_d8, __simd_log2_f16, + __simd_log2_f8, __simd_log_d4, __simd_log_d8, __simd_log_f16, + __simd_log_f8, __simd_nextafter_d4, __simd_nextafter_d8, __simd_nextafter_f16, + __simd_nextafter_f8, __simd_pow_d4, __simd_pow_d8, __simd_pow_f16, + __simd_pow_f8, __simd_remainder_d4, __simd_remainder_d8, __simd_remainder_f16, + __simd_remainder_f8, __simd_round_d4, __simd_round_d8, __simd_round_f16, + __simd_round_f8, __simd_sin_d4, __simd_sin_d8, __simd_sin_f16, + __simd_sin_f8, __simd_sinh_d4, __simd_sinh_d8, __simd_sinh_f16, + __simd_sinh_f8, __simd_sinpi_d4, __simd_sinpi_d8, __simd_sinpi_f16, + __simd_sinpi_f8, __simd_tan_d4, __simd_tan_d8, __simd_tan_f16, + __simd_tan_f8, __simd_tanh_d4, __simd_tanh_d8, __simd_tanh_f16, + __simd_tanh_f8, __simd_tanpi_d4, __simd_tanpi_d8, __simd_tanpi_f16, + __simd_tanpi_f8, __simd_tgamma_d4, __simd_tgamma_d8, __simd_tgamma_f16, + __simd_tgamma_f8 ] + - targets: [ x86_64-macos, x86_64-maccatalyst, x86_64h-macos, x86_64h-maccatalyst, + arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + symbols: [ __FE_DFL_ENV, ___Libm_version, ___ceilf16, ___copysignf16, + ___cos_d2, ___cos_f4, ___cospi, ___cospif, ___exp10, ___exp10f, + ___fabsf16, ___fegetfltrounds, ___floorf16, ___fmaf16, ___fmaxf16, + ___fminf16, ___fpclassifyd, ___fpclassifyf, ___fpclassifyl, + ___hypotf16, ___inline_isfinited, ___inline_isfinitef, ___inline_isfinitel, + ___inline_isinfd, ___inline_isinff, ___inline_isinfl, ___inline_isnand, + ___inline_isnanf, ___inline_isnanl, ___inline_isnormald, ___inline_isnormalf, + ___inline_isnormall, ___inline_signbitd, ___inline_signbitf, + ___inline_signbitl, ___invert_d2, ___invert_d3, ___invert_d4, + ___invert_f2, ___invert_f3, ___invert_f4, ___invert_h2, ___invert_h3, + ___invert_h4, ___isfinited, ___isfinitef, ___isfinitel, ___isinfd, + ___isinff, ___isinfl, ___isnand, ___isnanf, ___isnanl, ___isnormald, + ___isnormalf, ___isnormall, ___math_errhandling, ___nextafterf16, + ___rintf16, ___roundf16, ___signbitd, ___signbitf, ___signbitl, + ___sin_d2, ___sin_f4, ___sincos, ___sincos_stret, ___sincosf, + ___sincosf_stret, ___sincospi, ___sincospi_stret, ___sincospif, + ___sincospif_stret, ___sinpi, ___sinpif, ___sqrtf16, ___tanpi, + ___tanpif, ___truncf16, __simd_acos_d2, __simd_acos_f4, __simd_acosh_d2, + __simd_acosh_f4, __simd_asin_d2, __simd_asin_f4, __simd_asinh_d2, + __simd_asinh_f4, __simd_atan2_d2, __simd_atan2_f4, __simd_atan_d2, + __simd_atan_f4, __simd_atanh_d2, __simd_atanh_f4, __simd_cbrt_d2, + __simd_cbrt_f4, __simd_cos_d2, __simd_cos_f4, __simd_cosh_d2, + __simd_cosh_f4, __simd_cospi_d2, __simd_cospi_f4, __simd_erf_d2, + __simd_erf_f4, __simd_erfc_d2, __simd_erfc_f4, __simd_exp10_d2, + __simd_exp10_f4, __simd_exp2_d2, __simd_exp2_f4, __simd_exp_d2, + __simd_exp_f4, __simd_expm1_d2, __simd_expm1_f4, __simd_fma_d2, + __simd_fma_f4, __simd_fmod_d2, __simd_fmod_f4, __simd_hypot_d2, + __simd_hypot_f4, __simd_incircle_pd2, __simd_incircle_pf2, + __simd_incircle_ph2, __simd_insphere_pd3, __simd_insphere_pf3, + __simd_insphere_ph3, __simd_lgamma_d2, __simd_lgamma_f4, __simd_log10_d2, + __simd_log10_f4, __simd_log1p_d2, __simd_log1p_f4, __simd_log2_d2, + __simd_log2_f4, __simd_log_d2, __simd_log_f4, __simd_nextafter_d2, + __simd_nextafter_f4, __simd_orient_pd2, __simd_orient_pd3, + __simd_orient_pf2, __simd_orient_pf3, __simd_orient_ph2, __simd_orient_ph3, + __simd_orient_vd2, __simd_orient_vd3, __simd_orient_vf2, __simd_orient_vf3, + __simd_orient_vh2, __simd_orient_vh3, __simd_pow_d2, __simd_pow_f4, + __simd_remainder_d2, __simd_remainder_f4, __simd_round_d2, + __simd_round_f4, __simd_sin_d2, __simd_sin_f4, __simd_sincos_d2, + __simd_sincos_f4, __simd_sincospi_d2, __simd_sincospi_f4, + __simd_sinh_d2, __simd_sinh_f4, __simd_sinpi_d2, __simd_sinpi_f4, + __simd_tan_d2, __simd_tan_f4, __simd_tanh_d2, __simd_tanh_f4, + __simd_tanpi_d2, __simd_tanpi_f4, __simd_tgamma_d2, __simd_tgamma_f4, + _acos, _acosf, _acosh, _acoshf, _acoshl, _acosl, _asin, _asinf, + _asinh, _asinhf, _asinhl, _asinl, _atan, _atan2, _atan2f, + _atan2l, _atanf, _atanh, _atanhf, _atanhl, _atanl, _cabs, + _cabsf, _cabsl, _cacos, _cacosf, _cacosh, _cacoshf, _cacoshl, + _cacosl, _carg, _cargf, _cargl, _casin, _casinf, _casinh, + _casinhf, _casinhl, _casinl, _catan, _catanf, _catanh, _catanhf, + _catanhl, _catanl, _cbrt, _cbrtf, _cbrtl, _ccos, _ccosf, _ccosh, + _ccoshf, _ccoshl, _ccosl, _ceil, _ceilf, _ceill, _cexp, _cexpf, + _cexpl, _cimag, _cimagf, _cimagl, _clog, _clogf, _clogl, _conj, + _conjf, _conjl, _copysign, _copysignf, _copysignl, _cos, _cosf, + _cosh, _coshf, _coshl, _cosl, _cpow, _cpowf, _cpowl, _cproj, + _cprojf, _cprojl, _creal, _crealf, _creall, _csin, _csinf, + _csinh, _csinhf, _csinhl, _csinl, _csqrt, _csqrtf, _csqrtl, + _ctan, _ctanf, _ctanh, _ctanhf, _ctanhl, _ctanl, _erf, _erfc, + _erfcf, _erfcl, _erff, _erfl, _exp, _exp2, _exp2f, _exp2l, + _expf, _expl, _expm1, _expm1f, _expm1l, _fabs, _fabsf, _fabsl, + _fdim, _fdimf, _fdiml, _feclearexcept, _fegetenv, _fegetexceptflag, + _fegetround, _feholdexcept, _feraiseexcept, _fesetenv, _fesetexceptflag, + _fesetround, _fetestexcept, _feupdateenv, _floor, _floorf, + _floorl, _fma, _fmaf, _fmal, _fmax, _fmaxf, _fmaxl, _fmin, + _fminf, _fminl, _fmod, _fmodf, _fmodl, _frexp, _frexpf, _frexpl, + _hypot, _hypotf, _hypotl, _ilogb, _ilogbf, _ilogbl, _isinf, + _isnan, _j0, _j1, _jn, _ldexp, _ldexpf, _ldexpl, _lgamma, + _lgamma_r, _lgammaf, _lgammaf_r, _lgammal, _lgammal_r, _llrint, + _llrintf, _llrintl, _llround, _llroundf, _llroundl, _log, + _log10, _log10f, _log10l, _log1p, _log1pf, _log1pl, _log2, + _log2f, _log2l, _logb, _logbf, _logbl, _logf, _logl, _lrint, + _lrintf, _lrintl, _lround, _lroundf, _lroundl, _matrix_identity_double2x2, + _matrix_identity_double3x3, _matrix_identity_double4x4, _matrix_identity_float2x2, + _matrix_identity_float3x3, _matrix_identity_float4x4, _matrix_identity_half2x2, + _matrix_identity_half3x3, _matrix_identity_half4x4, _modf, + _modff, _modfl, _nan, _nanf, _nanl, _nearbyint, _nearbyintf, + _nearbyintl, _nextafter, _nextafterf, _nextafterl, _nexttoward, + _nexttowardf, _nexttowardl, _pow, _powf, _powl, _remainder, + _remainderf, _remainderl, _remquo, _remquof, _remquol, _rint, + _rintf, _rintl, _round, _roundf, _roundl, _scalb, _scalbln, + _scalblnf, _scalblnl, _scalbn, _scalbnf, _scalbnl, _signgam, + _sin, _sinf, _sinh, _sinhf, _sinhl, _sinl, _sqrt, _sqrtf, + _sqrtl, _tan, _tanf, _tanh, _tanhf, _tanhl, _tanl, _tgamma, + _tgammaf, _tgammal, _trunc, _truncf, _truncl, _y0, _y1, _yn ] + - targets: [ x86_64-macos, x86_64h-macos ] + symbols: [ ___fpclassify, ___inf, ___inff, ___infl, ___isfinite, ___isinf, + ___isnan, ___isnormal, ___nan, ___signbit, '_acos$fenv_access_off', + '_acosf$fenv_access_off', '_acosh$fenv_access_off', '_acoshf$fenv_access_off', + '_acoshl$fenv_access_off', '_acosl$fenv_access_off', '_asin$fenv_access_off', + '_asinf$fenv_access_off', '_asinh$fenv_access_off', '_asinhf$fenv_access_off', + '_asinhl$fenv_access_off', '_asinl$fenv_access_off', '_atan$fenv_access_off', + '_atan2$fenv_access_off', '_atan2f$fenv_access_off', '_atan2l$fenv_access_off', + '_atanf$fenv_access_off', '_atanh$fenv_access_off', '_atanhf$fenv_access_off', + '_atanhl$fenv_access_off', '_atanl$fenv_access_off', '_cabs$fenv_access_off', + '_cabsf$fenv_access_off', '_cabsl$fenv_access_off', '_cacos$fenv_access_off', + '_cacosf$fenv_access_off', '_cacosh$fenv_access_off', '_cacoshf$fenv_access_off', + '_cacoshl$fenv_access_off', '_cacosl$fenv_access_off', '_carg$fenv_access_off', + '_cargf$fenv_access_off', '_cargl$fenv_access_off', '_casin$fenv_access_off', + '_casinf$fenv_access_off', '_casinh$fenv_access_off', '_casinhf$fenv_access_off', + '_casinhl$fenv_access_off', '_casinl$fenv_access_off', '_catan$fenv_access_off', + '_catanf$fenv_access_off', '_catanh$fenv_access_off', '_catanhf$fenv_access_off', + '_catanhl$fenv_access_off', '_catanl$fenv_access_off', '_cbrt$fenv_access_off', + '_cbrtf$fenv_access_off', '_cbrtl$fenv_access_off', '_ccos$fenv_access_off', + '_ccosf$fenv_access_off', '_ccosh$fenv_access_off', '_ccoshf$fenv_access_off', + '_ccoshl$fenv_access_off', '_ccosl$fenv_access_off', '_ceil$fenv_access_off', + '_ceilf$fenv_access_off', '_ceill$fenv_access_off', '_cexp$fenv_access_off', + '_cexpf$fenv_access_off', '_cexpl$fenv_access_off', '_cimag$fenv_access_off', + '_cimagf$fenv_access_off', '_cimagl$fenv_access_off', '_clog$fenv_access_off', + '_clogf$fenv_access_off', '_clogl$fenv_access_off', '_conj$fenv_access_off', + '_conjf$fenv_access_off', '_conjl$fenv_access_off', '_copysign$fenv_access_off', + '_copysignf$fenv_access_off', '_copysignl$fenv_access_off', + '_cos$fenv_access_off', '_cosf$fenv_access_off', '_cosh$fenv_access_off', + '_coshf$fenv_access_off', '_coshl$fenv_access_off', '_cosl$fenv_access_off', + '_cpow$fenv_access_off', '_cpowf$fenv_access_off', '_cpowl$fenv_access_off', + '_cproj$fenv_access_off', '_cprojf$fenv_access_off', '_cprojl$fenv_access_off', + '_creal$fenv_access_off', '_crealf$fenv_access_off', '_creall$fenv_access_off', + '_csin$fenv_access_off', '_csinf$fenv_access_off', '_csinh$fenv_access_off', + '_csinhf$fenv_access_off', '_csinhl$fenv_access_off', '_csinl$fenv_access_off', + '_csqrt$fenv_access_off', '_csqrtf$fenv_access_off', '_csqrtl$fenv_access_off', + '_ctan$fenv_access_off', '_ctanf$fenv_access_off', '_ctanh$fenv_access_off', + '_ctanhf$fenv_access_off', '_ctanhl$fenv_access_off', '_ctanl$fenv_access_off', + _drem, '_erf$fenv_access_off', '_erfc$fenv_access_off', '_erfcf$fenv_access_off', + '_erfcl$fenv_access_off', '_erff$fenv_access_off', '_erfl$fenv_access_off', + '_exp$fenv_access_off', '_exp2$fenv_access_off', '_exp2f$fenv_access_off', + '_exp2l$fenv_access_off', '_expf$fenv_access_off', '_expl$fenv_access_off', + '_expm1$fenv_access_off', '_expm1f$fenv_access_off', '_expm1l$fenv_access_off', + '_fabs$fenv_access_off', '_fabsf$fenv_access_off', '_fabsl$fenv_access_off', + '_fdim$fenv_access_off', '_fdimf$fenv_access_off', '_fdiml$fenv_access_off', + _fegetexcept, _fesetexcept, _finite, '_floor$fenv_access_off', + '_floorf$fenv_access_off', '_floorl$fenv_access_off', '_fma$fenv_access_off', + '_fmaf$fenv_access_off', '_fmal$fenv_access_off', '_fmax$fenv_access_off', + '_fmaxf$fenv_access_off', '_fmaxl$fenv_access_off', '_fmin$fenv_access_off', + '_fminf$fenv_access_off', '_fminl$fenv_access_off', '_fmod$fenv_access_off', + '_fmodf$fenv_access_off', '_fmodl$fenv_access_off', '_frexp$fenv_access_off', + '_frexpf$fenv_access_off', '_frexpl$fenv_access_off', _gamma, + '_gamma$fenv_access_off', '_hypot$fenv_access_off', '_hypotf$fenv_access_off', + '_hypotl$fenv_access_off', '_ilogb$fenv_access_off', '_ilogbf$fenv_access_off', + '_ilogbl$fenv_access_off', '_ldexp$fenv_access_off', '_ldexpf$fenv_access_off', + '_ldexpl$fenv_access_off', '_lgamma$fenv_access_off', '_lgamma_r$fenv_access_off', + '_lgammaf$fenv_access_off', '_lgammaf_r$fenv_access_off', + '_lgammal$fenv_access_off', '_lgammal_r$fenv_access_off', + '_llrint$fenv_access_off', '_llrintf$fenv_access_off', '_llrintl$fenv_access_off', + '_llround$fenv_access_off', '_llroundf$fenv_access_off', '_llroundl$fenv_access_off', + '_log$fenv_access_off', '_log10$fenv_access_off', '_log10f$fenv_access_off', + '_log10l$fenv_access_off', '_log1p$fenv_access_off', '_log1pf$fenv_access_off', + '_log1pl$fenv_access_off', '_log2$fenv_access_off', '_log2f$fenv_access_off', + '_log2l$fenv_access_off', '_logb$fenv_access_off', '_logbf$fenv_access_off', + '_logbl$fenv_access_off', '_logf$fenv_access_off', '_logl$fenv_access_off', + '_lrint$fenv_access_off', '_lrintf$fenv_access_off', '_lrintl$fenv_access_off', + '_lround$fenv_access_off', '_lroundf$fenv_access_off', '_lroundl$fenv_access_off', + _matherr, '_modf$fenv_access_off', '_modff$fenv_access_off', + '_modfl$fenv_access_off', '_nearbyint$fenv_access_off', '_nearbyintf$fenv_access_off', + '_nearbyintl$fenv_access_off', '_nextafter$fenv_access_off', + _nextafterd, '_nextafterf$fenv_access_off', '_nextafterl$fenv_access_off', + '_nexttoward$fenv_access_off', '_nexttowardf$fenv_access_off', + '_nexttowardl$fenv_access_off', '_pow$fenv_access_off', '_powf$fenv_access_off', + '_powl$fenv_access_off', '_remainder$fenv_access_off', '_remainderf$fenv_access_off', + '_remainderl$fenv_access_off', '_remquo$fenv_access_off', + '_remquof$fenv_access_off', '_remquol$fenv_access_off', '_rint$fenv_access_off', + '_rintf$fenv_access_off', '_rintl$fenv_access_off', _rinttol, + '_round$fenv_access_off', '_roundf$fenv_access_off', '_roundl$fenv_access_off', + _roundtol, '_scalbln$fenv_access_off', '_scalblnf$fenv_access_off', + '_scalblnl$fenv_access_off', '_scalbn$fenv_access_off', '_scalbnf$fenv_access_off', + '_scalbnl$fenv_access_off', _significand, '_sin$fenv_access_off', + '_sinf$fenv_access_off', '_sinh$fenv_access_off', '_sinhf$fenv_access_off', + '_sinhl$fenv_access_off', '_sinl$fenv_access_off', '_sqrt$fenv_access_off', + '_sqrtf$fenv_access_off', '_sqrtl$fenv_access_off', '_tan$fenv_access_off', + '_tanf$fenv_access_off', '_tanh$fenv_access_off', '_tanhf$fenv_access_off', + '_tanhl$fenv_access_off', '_tanl$fenv_access_off', '_tgamma$fenv_access_off', + '_tgammaf$fenv_access_off', '_tgammal$fenv_access_off', '_trunc$fenv_access_off', + '_truncf$fenv_access_off', '_truncl$fenv_access_off' ] + - targets: [ arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + symbols: [ __FE_DFL_DISABLE_DENORMS_ENV ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_malloc.dylib' +current-version: 792.41.1 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___mach_stack_logging_shared_memory_address, ___malloc_init, + ___malloc_late_init, __malloc_fork_child, __malloc_fork_parent, + __malloc_fork_prepare, __malloc_no_asl_log, __os_cpu_number_override, + _aligned_alloc, _calloc, _free, _mag_set_thread_index, _malloc, + _malloc_allows_internal_security_4test, _malloc_check_counter, + _malloc_check_each, _malloc_check_start, _malloc_claimed_address, + _malloc_create_legacy_default_zone, _malloc_create_zone, _malloc_debug, + _malloc_default_purgeable_zone, _malloc_default_zone, _malloc_destroy_zone, + _malloc_engaged_nano, _malloc_engaged_secure_allocator, _malloc_enter_process_memory_limit_warn_mode, + _malloc_error, _malloc_freezedry, _malloc_get_all_zones, _malloc_get_thread_options, + _malloc_get_wrapped_zone, _malloc_get_zone_name, _malloc_good_size, + _malloc_jumpstart, _malloc_logger, _malloc_make_nonpurgeable, + _malloc_make_purgeable, _malloc_memory_event_handler, _malloc_memorypressure_mask_default_4libdispatch, + _malloc_memorypressure_mask_msl_4libdispatch, _malloc_num_zones, + _malloc_num_zones_allocated, _malloc_printf, _malloc_register_stack_logger, + _malloc_sanitizer_get_functions, _malloc_sanitizer_is_enabled, + _malloc_sanitizer_set_functions, _malloc_set_thread_options, + _malloc_set_zone_name, _malloc_singlethreaded, _malloc_size, + _malloc_type_aligned_alloc, _malloc_type_calloc, _malloc_type_free, + _malloc_type_malloc, _malloc_type_posix_memalign, _malloc_type_realloc, + _malloc_type_valloc, _malloc_type_zone_calloc, _malloc_type_zone_free, + _malloc_type_zone_malloc, _malloc_type_zone_malloc_with_options, + _malloc_type_zone_malloc_with_options_internal, _malloc_type_zone_memalign, + _malloc_type_zone_realloc, _malloc_type_zone_valloc, _malloc_variant_is_debug_4test, + _malloc_zero_on_free_disable, _malloc_zone_batch_free, _malloc_zone_batch_malloc, + _malloc_zone_calloc, _malloc_zone_check, _malloc_zone_claimed_address, + _malloc_zone_disable_discharge_checking, _malloc_zone_discharge, + _malloc_zone_enable_discharge_checking, _malloc_zone_enumerate_discharged_pointers, + _malloc_zone_free, _malloc_zone_from_ptr, _malloc_zone_log, + _malloc_zone_malloc, _malloc_zone_malloc_with_options, _malloc_zone_malloc_with_options_np, + _malloc_zone_memalign, _malloc_zone_pressure_relief, _malloc_zone_print, + _malloc_zone_print_ptr_info, _malloc_zone_realloc, _malloc_zone_register, + _malloc_zone_statistics, _malloc_zone_unregister, _malloc_zone_valloc, + _malloc_zones, _mstats, _pgm_extract_report_from_corpse, _posix_memalign, + _realloc, '_reallocarray$DARWIN_EXTSN', '_reallocarrayf$DARWIN_EXTSN', + _reallocf, _sanitizer_diagnose_fault_from_crash_reporter, + _scalable_zone_info, _scalable_zone_statistics, _set_malloc_singlethreaded, + _stack_logging_enable_logging, _szone_check_counter, _szone_check_modulo, + _szone_check_start, _tiny_print_region_free_list, _turn_off_stack_logging, + _turn_on_stack_logging, _valloc, _vfree, _xzm_malloc_zone_introspect, + _xzm_ptr_lookup_4test, _xzm_type_choose_ptr_bucket_4test, + _zeroify_scalable_zone ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_networkextension.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _NEHelperCacheAddRedirectedAddress, _NEHelperCacheClearRedirectedAddresses, + _NEHelperCacheClearUUIDs, _NEHelperCacheClearUUIDsForBundleID, + _NEHelperCacheCopyAppUUIDMapping, _NEHelperCacheCopyAppUUIDMappingExtended, + _NEHelperCacheCopyAppUUIDMappingForUIDExtended, _NEHelperCacheCopySigningIdentifierMapping, + _NEHelperCachePopulateUUIDsForConfiguration, _NEHelperCacheSetDomainDictionaries, + _NEHelperCacheSetMatchDomains, _NEHelperCacheSetRoutes, _NEHelperCopyAggregatePathRules, + _NEHelperCopyAppInfo, _NEHelperCopyCurrentNetworkAsync, _NEHelperCopyCurrentNetworkInfo, + _NEHelperCopyDataForCertificate, _NEHelperCopyPerAppDomains, + _NEHelperCopyResponse, _NEHelperCopyXPCEndpointForIdentityProxy, + _NEHelperCopyXPCEndpointForIdentityProxyExtended, _NEHelperGetAppTrackerDomains, + _NEHelperGetIKESocket, _NEHelperGetIKESocketWithResult, _NEHelperGetKernelControlSocket, + _NEHelperGetKernelControlSocketExtended, _NEHelperGetNECPSessionFD, + _NEHelperGetPFKeySocket, _NEHelperGetPid, _NEHelperGetRawSocket, + _NEHelperHandleConfigurationsChangedBySC, _NEHelperInit, _NEHelperInterfaceCreate, + _NEHelperInterfaceDestroy, _NEHelperInterfaceRemoveAddress, + _NEHelperInterfaceSetAddress, _NEHelperInterfaceSetAddressWithLifetime, + _NEHelperInterfaceSetDelegate, _NEHelperInterfaceSetDescription, + _NEHelperInterfaceSetMTU, _NEHelperInterfaceSetOption, _NEHelperSendRequest, + _NEHelperSettingsRemove, _NEHelperSettingsSetArray, _NEHelperSettingsSetBool, + _NEHelperSettingsSetNumber, _NEHelperVPNConfigurationExists, + _NEHelperVPNSetEnabled, _g_ne_read_uuid_cache, _g_ne_uuid_cache_hit, + _ne_copy_cached_bundle_identifier_for_uuid, _ne_copy_cached_uuids_for_bundle_identifier, + _ne_copy_signature_info_for_pid, _ne_copy_signing_identifier_for_pid, + _ne_copy_signing_identifier_for_pid_with_audit_token, _ne_copy_uuid_cache, + _ne_force_reset_uuid_cache, _ne_get_configuration_generation, + _ne_is_sockaddr_valid, _ne_log_large_obj, _ne_log_obj, _ne_print_backtrace, + _ne_privacy_dns_netagent_id, _ne_privacy_proxy_netagent_id, + _ne_session_add_necp_drop_dest_from_dest_list, _ne_session_add_necp_drop_dest_from_path, + _ne_session_address_matches_subnets, _ne_session_agent_get_advisory, + _ne_session_agent_get_advisory_interface_index, _ne_session_always_on_vpn_configs_present, + _ne_session_always_on_vpn_configs_present_at_boot, _ne_session_app_vpn_configs_present, + _ne_session_applications_have_local_network_entitlements, + _ne_session_cancel, _ne_session_clear_caches, _ne_session_content_filter_configs_present, + _ne_session_copy_app_data_from_flow_divert_socket, _ne_session_copy_app_data_from_flow_divert_token, + _ne_session_copy_os_version_string, _ne_session_copy_policy_match, + _ne_session_copy_security_session_info, _ne_session_copy_socket_attributes, + _ne_session_copy_socket_domain_attributes, _ne_session_create, + _ne_session_disable_restrictions, _ne_session_dns_proxy_configs_present, + _ne_session_dns_settings_configs_present, _ne_session_establish_ipc, + _ne_session_fallback_advisory, _ne_session_fallback_default, + _ne_session_fetch_server_parameters, _ne_session_get_boot_session_uuid, + _ne_session_get_config_id_from_network_agent, _ne_session_get_configuration_id, + _ne_session_get_info, _ne_session_get_info2, _ne_session_get_status, + _ne_session_get_type, _ne_session_info_type_to_string, _ne_session_initialize_necp_drop_all, + _ne_session_is_always_on_vpn_enabled, _ne_session_is_safeboot, + _ne_session_local_communication_configs_present, _ne_session_local_communication_send_info, + _ne_session_manager_get_pid, _ne_session_manager_has_active_sessions, + _ne_session_manager_is_running, _ne_session_map_interface_to_provider_uuid, + _ne_session_on_demand_configs_present, _ne_session_path_controller_configs_present, + _ne_session_policy_copy_flow_divert_token, _ne_session_policy_copy_flow_divert_token_with_key, + _ne_session_policy_match_get_service, _ne_session_policy_match_get_service_action, + _ne_session_policy_match_get_service_type, _ne_session_relay_configs_present, + _ne_session_release, _ne_session_reset_cache, _ne_session_retain, + _ne_session_send_barrier, _ne_session_service_copy_cached_match_domains, + _ne_session_service_get_dns_service_id, _ne_session_service_get_dns_service_id_for_interface, + _ne_session_service_matches_address, _ne_session_service_matches_address_for_interface, + _ne_session_set_device_communication_exception, _ne_session_set_event_handler, + _ne_session_set_socket_attributes, _ne_session_set_socket_context_attribute, + _ne_session_set_socket_tracker_attributes, _ne_session_should_disable_nexus, + _ne_session_start, _ne_session_start_on_behalf_of, _ne_session_start_with_options, + _ne_session_status_to_string, _ne_session_stop, _ne_session_stop_all_with_plugin_type, + _ne_session_stop_reason_to_string, _ne_session_type_to_string, + _ne_session_urlfilter_configs_present, _ne_session_use_as_system_vpn, + _ne_session_vod_evaluate_connection_present, _ne_session_vpn_configs_present, + _ne_session_vpn_include_all_networks_configs_present, _ne_socket_set_attribution, + _ne_socket_set_domains, _ne_socket_set_is_app_initiated, _ne_socket_set_website_attribution, + _ne_tracker_build_cache, _ne_tracker_build_trie, _ne_tracker_check_info_changed, + _ne_tracker_check_is_hostname_blocked, _ne_tracker_check_tcc, + _ne_tracker_clear_cache, _ne_tracker_context_can_block_request, + _ne_tracker_context_get_domain, _ne_tracker_context_get_domain_owner, + _ne_tracker_context_is_from_app_list, _ne_tracker_context_is_from_web_list, + _ne_tracker_copy_current_stacktrace, _ne_tracker_create_xcode_issue, + _ne_tracker_get_ddg_dictionary, _ne_tracker_get_disposition, + _ne_tracker_lookup_app_domains, _ne_tracker_set_test_domains, + _ne_tracker_should_save_stacktrace, _ne_tracker_validate_domain, + _ne_trie_free, _ne_trie_has_high_ascii, _ne_trie_init, _ne_trie_init_from_file, + _ne_trie_insert, _ne_trie_save_to_file, _ne_trie_search, _necp_drop_dest_copy_dest_entry_list, + _nehelper_copy_connection_for_delegate_class, _nehelper_queue, + _nelog_is_debug_logging_enabled, _nelog_is_extra_vpn_logging_enabled, + _nelog_is_info_logging_enabled ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_notify.dylib' +current-version: 344.0.1 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __notify_fork_child, _notify_cancel, _notify_check, _notify_dump_status, + _notify_get_event, _notify_get_state, _notify_is_valid_token, + _notify_monitor_file, _notify_peek, _notify_post, _notify_register_check, + _notify_register_dispatch, _notify_register_file_descriptor, + _notify_register_mach_port, _notify_register_plain, _notify_register_signal, + _notify_resume, _notify_resume_pid, _notify_set_options, _notify_set_state, + _notify_simple_post, _notify_suspend, _notify_suspend_pid ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_platform.dylib' +current-version: 359.40.3 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst ] + symbols: [ '_OSAtomicFifoDequeue$VARIANT$PFZ', '_OSAtomicFifoDequeue$VARIANT$UnfairLock', + '_OSAtomicFifoEnqueue$VARIANT$PFZ', '_OSAtomicFifoEnqueue$VARIANT$UnfairLock', + ___no_overread_variant_setup, __sigtramp, _bzero_sse_np, _longjmperror, + _memmove_sse_np, _memset_pattern16_sse_np, _memset_pattern4_sse_np, + _memset_pattern8_sse_np, _memset_sse_np ] + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _OSAtomicAdd32, _OSAtomicAdd32Barrier, _OSAtomicAdd64, _OSAtomicAdd64Barrier, + _OSAtomicAnd32, _OSAtomicAnd32Barrier, _OSAtomicAnd32Orig, + _OSAtomicAnd32OrigBarrier, _OSAtomicCompareAndSwap32, _OSAtomicCompareAndSwap32Barrier, + _OSAtomicCompareAndSwap64, _OSAtomicCompareAndSwap64Barrier, + _OSAtomicCompareAndSwapInt, _OSAtomicCompareAndSwapIntBarrier, + _OSAtomicCompareAndSwapLong, _OSAtomicCompareAndSwapLongBarrier, + _OSAtomicCompareAndSwapPtr, _OSAtomicCompareAndSwapPtrBarrier, + _OSAtomicDecrement32, _OSAtomicDecrement32Barrier, _OSAtomicDecrement64, + _OSAtomicDecrement64Barrier, _OSAtomicDequeue, _OSAtomicEnqueue, + _OSAtomicFifoDequeue, _OSAtomicFifoEnqueue, _OSAtomicIncrement32, + _OSAtomicIncrement32Barrier, _OSAtomicIncrement64, _OSAtomicIncrement64Barrier, + _OSAtomicOr32, _OSAtomicOr32Barrier, _OSAtomicOr32Orig, _OSAtomicOr32OrigBarrier, + _OSAtomicTestAndClear, _OSAtomicTestAndClearBarrier, _OSAtomicTestAndSet, + _OSAtomicTestAndSetBarrier, _OSAtomicXor32, _OSAtomicXor32Barrier, + _OSAtomicXor32Orig, _OSAtomicXor32OrigBarrier, _OSMemoryBarrier, + _OSSpinLockLock, _OSSpinLockTry, _OSSpinLockUnlock, __OSSpinLockLockSlow, + ___bzero, ___libplatform_init, ___os_log_simple_offset, ___os_once_reset, + ___platform_sigaction, __longjmp, __os_alloc_once, __os_lock_type_eliding, + __os_lock_type_handoff, __os_lock_type_nospin, __os_lock_type_spin, + __os_lock_type_transactional, __os_lock_type_unfair, __os_log_simple, + __os_log_simple_parse, __os_log_simple_parse_identifiers, + __os_log_simple_parse_message, __os_log_simple_parse_subsystem, + __os_log_simple_parse_timestamp, __os_log_simple_parse_type, + __os_log_simple_reinit_4launchd, __os_log_simple_send, __os_log_simple_shim, + __os_nospin_lock_lock, __os_nospin_lock_trylock, __os_nospin_lock_unlock, + __os_once, __os_semaphore_create, __os_semaphore_dispose, + __os_semaphore_signal, __os_semaphore_wait, __platform_bzero, + __platform_memccpy, __platform_memchr, __platform_memcmp, + __platform_memcmp_zero_aligned8, __platform_memmove, __platform_memset, + __platform_memset_pattern16, __platform_memset_pattern4, __platform_memset_pattern8, + __platform_strchr, __platform_strcmp, __platform_strcpy, __platform_strlcat, + __platform_strlcpy, __platform_strlen, __platform_strncmp, + __platform_strncpy, __platform_strnlen, __platform_strstr, + __setjmp, __simple_asl_get_fd, __simple_asl_log, __simple_asl_log_prog, + __simple_asl_msg_new, __simple_asl_msg_set, __simple_asl_send, + __simple_dprintf, __simple_esappend, __simple_esprintf, __simple_getenv, + __simple_put, __simple_putline, __simple_salloc, __simple_sappend, + __simple_sfree, __simple_snprintf, __simple_sprintf, __simple_sresize, + __simple_string, __simple_vdprintf, __simple_vesprintf, __simple_vsnprintf, + __simple_vsprintf, __spin_lock, __spin_lock_try, __spin_unlock, + _ffs, _ffsl, _ffsll, _fls, _flsl, _flsll, _getcontext, _longjmp, + _makecontext, _os_apt_msg_async_task_running_4swift, _os_apt_msg_async_task_waiting_on_4swift, + _os_lock_lock, _os_lock_trylock, _os_lock_unlock, _os_log_simple_now, + _os_log_simple_type_from_asl, _os_security_config_get, _os_security_config_get_for_proc, + _os_security_config_get_for_task, _os_sync_wait_on_address, + _os_sync_wait_on_address_with_deadline, _os_sync_wait_on_address_with_timeout, + _os_sync_wake_by_address_all, _os_sync_wake_by_address_any, + _os_unfair_lock_assert_not_owner, _os_unfair_lock_assert_owner, + _os_unfair_lock_lock, _os_unfair_lock_lock_no_tsd, _os_unfair_lock_lock_with_flags, + _os_unfair_lock_lock_with_options, _os_unfair_lock_trylock, + _os_unfair_lock_trylock_with_options, _os_unfair_lock_unlock, + _os_unfair_lock_unlock_no_tsd, _os_unfair_recursive_lock_lock_with_options, + _os_unfair_recursive_lock_owned, _os_unfair_recursive_lock_trylock, + _os_unfair_recursive_lock_tryunlock4objc, _os_unfair_recursive_lock_unlock, + _os_unfair_recursive_lock_unlock_forked_child, _setcontext, + _setjmp, _siglongjmp, _sigsetjmp, _spin_lock, _spin_lock_try, + _spin_unlock, _swapcontext, _sys_cache_control, _sys_dcache_flush, + _sys_icache_invalidate, _timingsafe_enable_if_supported, _timingsafe_restore_if_supported ] + - targets: [ arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___sme_memchr, ___sme_memcpy, ___sme_memmove, ___sme_memset, + __ctx_done ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_pthread.dylib' +current-version: 539 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ ____chkstk_darwin, ___is_threaded, ___pthread_init, ___pthread_late_init, + ___pthread_workqueue_setkill, ___unix_conforming, __pthread_atfork_child, + __pthread_atfork_child_handlers, __pthread_atfork_parent, + __pthread_atfork_parent_handlers, __pthread_atfork_prepare, + __pthread_atfork_prepare_handlers, __pthread_clear_qos_tsd, + __pthread_exit_if_canceled, __pthread_fork_child, __pthread_fork_child_postinit, + __pthread_fork_parent, __pthread_fork_prepare, __pthread_is_threaded, + __pthread_override_qos_class_end_direct, __pthread_override_qos_class_start_direct, + __pthread_qos_class_and_override_decode, __pthread_qos_class_and_override_encode, + __pthread_qos_class_decode, __pthread_qos_class_encode, __pthread_qos_class_encode_workqueue, + __pthread_qos_override_end_direct, __pthread_qos_override_start_direct, + __pthread_sched_pri_decode, __pthread_sched_pri_encode, __pthread_self, + __pthread_set_properties_self, __pthread_set_self, __pthread_setspecific_static, + __pthread_start, __pthread_tsd_shared_cache_first, __pthread_tsd_shared_cache_last, + __pthread_workloop_create, __pthread_workloop_destroy, __pthread_workqueue_add_cooperativethreads, + __pthread_workqueue_addthreads, __pthread_workqueue_allow_send_signals, + __pthread_workqueue_asynchronous_override_add, __pthread_workqueue_asynchronous_override_reset_all_self, + __pthread_workqueue_asynchronous_override_reset_self, __pthread_workqueue_init, + __pthread_workqueue_init_with_kevent, __pthread_workqueue_init_with_workloop, + __pthread_workqueue_override_reset, __pthread_workqueue_override_start_direct, + __pthread_workqueue_override_start_direct_check_owner, __pthread_workqueue_set_event_manager_priority, + __pthread_workqueue_should_narrow, __pthread_workqueue_supported, + __pthread_wqthread, __pthread_yield_to_enqueuer_4dispatch, + _cthread_yield, _posix_spawnattr_get_qos_class_np, _posix_spawnattr_set_qos_class_np, + _pthread_atfork, _pthread_attr_destroy, _pthread_attr_get_qos_class_np, + _pthread_attr_getdetachstate, _pthread_attr_getguardsize, + _pthread_attr_getinheritsched, _pthread_attr_getschedparam, + _pthread_attr_getschedpolicy, _pthread_attr_getscope, _pthread_attr_getstack, + _pthread_attr_getstackaddr, _pthread_attr_getstacksize, _pthread_attr_init, + _pthread_attr_set_qos_class_np, _pthread_attr_setcpupercent_np, + _pthread_attr_setdetachstate, _pthread_attr_setguardsize, + _pthread_attr_setinheritsched, _pthread_attr_setschedparam, + _pthread_attr_setschedpolicy, _pthread_attr_setscope, _pthread_attr_setstack, + _pthread_attr_setstackaddr, _pthread_attr_setstacksize, _pthread_attr_setworkinterval_np, + _pthread_cancel, _pthread_chdir_np, _pthread_cond_broadcast, + _pthread_cond_destroy, _pthread_cond_init, _pthread_cond_signal, + _pthread_cond_signal_thread_np, _pthread_cond_timedwait, '_pthread_cond_timedwait$NOCANCEL', + _pthread_cond_timedwait_relative_np, _pthread_cond_wait, '_pthread_cond_wait$NOCANCEL', + _pthread_condattr_destroy, _pthread_condattr_getpshared, _pthread_condattr_init, + _pthread_condattr_setpshared, _pthread_cpu_number_np, _pthread_create, + _pthread_create_from_mach_thread, _pthread_create_suspended_np, + _pthread_create_with_workgroup_np, _pthread_current_stack_contains_np, + _pthread_dependency_fulfill_np, _pthread_dependency_init_np, + _pthread_dependency_wait_np, _pthread_detach, _pthread_equal, + _pthread_exit, _pthread_fchdir_np, _pthread_from_mach_thread_np, + _pthread_get_qos_class_np, _pthread_get_stackaddr_np, _pthread_get_stacksize_np, + _pthread_getconcurrency, _pthread_getname_np, _pthread_getschedparam, + _pthread_getspecific, _pthread_install_workgroup_functions_np, + _pthread_introspection_getspecific_np, _pthread_introspection_hook_install, + _pthread_introspection_setspecific_np, _pthread_is_threaded_np, + _pthread_jit_write_freeze_callbacks_np, _pthread_jit_write_protect_supported_np, + _pthread_jit_write_with_callback_np, _pthread_join, '_pthread_join$NOCANCEL', + _pthread_key_create, _pthread_key_delete, _pthread_key_init_np, + _pthread_kill, _pthread_layout_offsets, _pthread_mach_thread_np, + _pthread_main_np, _pthread_main_thread_np, _pthread_mutex_destroy, + _pthread_mutex_getprioceiling, _pthread_mutex_init, _pthread_mutex_lock, + _pthread_mutex_setprioceiling, _pthread_mutex_trylock, _pthread_mutex_unlock, + _pthread_mutexattr_destroy, _pthread_mutexattr_getpolicy_np, + _pthread_mutexattr_getprioceiling, _pthread_mutexattr_getprotocol, + _pthread_mutexattr_getpshared, _pthread_mutexattr_gettype, + _pthread_mutexattr_init, _pthread_mutexattr_setpolicy_np, + _pthread_mutexattr_setprioceiling, _pthread_mutexattr_setprotocol, + _pthread_mutexattr_setpshared, _pthread_mutexattr_settype, + _pthread_once, _pthread_override_qos_class_end_np, _pthread_override_qos_class_start_np, + _pthread_prefer_alternate_amx_self, _pthread_prefer_alternate_cluster_self, + _pthread_qos_max_parallelism, _pthread_rwlock_destroy, _pthread_rwlock_init, + _pthread_rwlock_rdlock, _pthread_rwlock_tryrdlock, _pthread_rwlock_trywrlock, + _pthread_rwlock_unlock, _pthread_rwlock_wrlock, _pthread_rwlockattr_destroy, + _pthread_rwlockattr_getpshared, _pthread_rwlockattr_init, + _pthread_rwlockattr_setpshared, _pthread_self, _pthread_self_is_exiting_np, + _pthread_set_fixedpriority_self, _pthread_set_qos_class_np, + _pthread_set_qos_class_self_np, _pthread_set_timeshare_self, + _pthread_setcancelstate, _pthread_setcanceltype, _pthread_setconcurrency, + _pthread_setname_np, _pthread_setschedparam, _pthread_setspecific, + _pthread_sigmask, _pthread_stack_frame_decode_np, _pthread_testcancel, + _pthread_threadid_np, _pthread_time_constraint_max_parallelism, + _pthread_workqueue_addthreads_np, _pthread_workqueue_setdispatch_np, + _pthread_workqueue_setdispatchoffset_np, _pthread_workqueue_setup, + _pthread_yield_np, _qos_class_main, _qos_class_self, _sched_get_priority_max, + _sched_get_priority_min, _sched_yield, _sigwait, '_sigwait$NOCANCEL', + _start_wqthread, _thread_chkstk_darwin, _thread_start ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ __pthread_mutex_enable_legacy_mode, _pthread_jit_write_protect_np ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_sandbox.dylib' +current-version: 2680.41.1 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _APP_SANDBOX_IOKIT_CLIENT, _APP_SANDBOX_MACH, _APP_SANDBOX_READ, + _APP_SANDBOX_READ_WRITE, _IOS_SANDBOX_APPLICATION_GROUP, _IOS_SANDBOX_CONTAINER, + _SANDBOX_CHECK_ALLOW_APPROVAL, _SANDBOX_CHECK_CANONICAL, _SANDBOX_CHECK_NOFOLLOW, + _SANDBOX_CHECK_NO_APPROVAL, _SANDBOX_CHECK_NO_REPORT, _SANDBOX_CHECK_POSIX_READABLE, + _SANDBOX_CHECK_POSIX_WRITEABLE, _SANDBOX_EXTENSION_CANONICAL, + _SANDBOX_EXTENSION_DEFAULT, _SANDBOX_EXTENSION_NOFOLLOW, _SANDBOX_EXTENSION_NOFOLLOW_ANY, + _SANDBOX_EXTENSION_NO_REPORT, _SANDBOX_EXTENSION_NO_STORAGE_CLASS, + _SANDBOX_EXTENSION_PREFIXMATCH, _SANDBOX_PROFILE_TYPE_AUTOBOX, + _SANDBOX_PROFILE_TYPE_BASTION, _SANDBOX_PROFILE_TYPE_GLOBAL_OVERRIDE, + _SANDBOX_PROFILE_TYPE_PLATFORM, _SANDBOX_PROFILE_TYPE_PROCESS, + _SANDBOX_STORAGE_CLASS_GROUP_ANY, _SANDBOX_STORAGE_CLASS_PROPERTY_ACCEPTS_USER_APPROVAL, + _SANDBOX_STORAGE_CLASS_PROPERTY_READ_RESTRICTED, _SANDBOX_STORAGE_CLASS_PROPERTY_REPLACEMENT_RESTRICTED, + _SANDBOX_STORAGE_CLASS_PROPERTY_WRITE_RESTRICTED, __amkrtemp, + __sandbox_enter_notify_libxpc, __sandbox_in_a_container, __sandbox_register_app_bundle_0, + __sandbox_register_app_bundle_1, _kSBXProfileNoInternet, _kSBXProfileNoNetwork, + _kSBXProfileNoWrite, _kSBXProfileNoWriteExceptTemporary, _kSBXProfilePureComputation, + _kSandboxAppBundleAnySigningId, _kSandboxAppBundlePlatformTeamId, + _kSandboxAppContainerAnySigningId, _kSandboxAppContainerPlatformTeamId, + _rootless_allows_task_for_pid, _rootless_check_datavault_flag, + _rootless_check_restricted_flag, _rootless_check_trusted, + _rootless_check_trusted_class, _rootless_check_trusted_fd, + _rootless_convert_to_datavault, _rootless_mkdir_datavault, + _rootless_mkdir_nounlink, _rootless_mkdir_restricted, _rootless_protected_volume, + _rootless_protected_volume_fd, _rootless_register_trusted_storage_class, + _rootless_remove_datavault_in_favor_of_static_storage_class, + _rootless_remove_restricted_in_favor_of_static_storage_class, + _rootless_restricted_environment, _rootless_suspend, _rootless_trusted_by_self_token, + _rootless_verify_trusted_by_self_token, _sandbox_apply_bytecode, + _sandbox_builtin_query, _sandbox_check, _sandbox_check_bulk, + _sandbox_check_by_audit_token, _sandbox_check_by_reference, + _sandbox_check_by_uniqueid, _sandbox_check_finder_automation_for_path, + _sandbox_check_message_filter_integer, _sandbox_check_message_filter_string, + _sandbox_check_process_signal_target, _sandbox_check_protected_app_container, + _sandbox_check_self_signal_target, _sandbox_check_storage_class, + _sandbox_check_with_attribution, _sandbox_consume_extension, + _sandbox_consume_fs_extension, _sandbox_consume_mach_extension, + _sandbox_container_path_for_audit_token, _sandbox_container_path_for_pid, + _sandbox_enable_local_state_flag, _sandbox_enable_root_translation, + _sandbox_enable_state_flag, _sandbox_extension_consume, _sandbox_extension_issue_file, + _sandbox_extension_issue_file_to_process, _sandbox_extension_issue_file_to_process_by_pid, + _sandbox_extension_issue_file_to_self, _sandbox_extension_issue_generic, + _sandbox_extension_issue_generic_to_process, _sandbox_extension_issue_generic_to_process_by_pid, + _sandbox_extension_issue_iokit_registry_entry_class, _sandbox_extension_issue_iokit_registry_entry_class_to_process, + _sandbox_extension_issue_iokit_registry_entry_class_to_process_by_pid, + _sandbox_extension_issue_mach, _sandbox_extension_issue_mach_to_process, + _sandbox_extension_issue_mach_to_process_by_pid, _sandbox_extension_issue_related_file_to_process, + _sandbox_extension_reap, _sandbox_extension_release, _sandbox_extension_release_and_detect_last_reference, + _sandbox_extension_release_file, _sandbox_extension_update_file, + _sandbox_extension_update_file_by_fileid, _sandbox_free_error, + _sandbox_get_container_expected, _sandbox_init, _sandbox_init_from_pid, + _sandbox_init_with_parameters, _sandbox_issue_fs_extension, + _sandbox_issue_mach_extension, _sandbox_message_filter_query, + _sandbox_message_filter_release, _sandbox_message_filter_retain, + _sandbox_note, _sandbox_passthrough_access, _sandbox_proc_getcontainer, + _sandbox_proc_getprofilename, _sandbox_query_approval_policy_for_path, + _sandbox_query_user_intent_for_process_with_audit_token, _sandbox_reference_release, + _sandbox_reference_retain_by_audit_token, _sandbox_register_app_bundle, + _sandbox_register_app_bundle_exception, _sandbox_register_app_bundle_package_exception, + _sandbox_register_app_container, _sandbox_register_app_container_exception, + _sandbox_register_app_container_package_exception, _sandbox_register_bastion_profile, + _sandbox_register_disk_image_backing_store, _sandbox_register_sync_root, + _sandbox_release_fs_extension, _sandbox_requests_integrity_protection_for_preference_domain, + _sandbox_set_container_path_for_application_group, _sandbox_set_container_path_for_application_group_with_persona, + _sandbox_set_container_path_for_audit_token, _sandbox_set_container_path_for_signing_id, + _sandbox_set_container_path_for_signing_id_with_persona, _sandbox_spawnattrs_getcontainer, + _sandbox_spawnattrs_getprofilename, _sandbox_spawnattrs_init, + _sandbox_spawnattrs_setcontainer, _sandbox_spawnattrs_setprofilename, + _sandbox_suspend, _sandbox_unregister_app_bundle, _sandbox_unregister_app_container, + _sandbox_unregister_bastion_profile, _sandbox_unregister_disk_image_backing_store, + _sandbox_unsuspend ] + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ _SANDBOX_EXTENSION_MACL_LEARNING, _SANDBOX_EXTENSION_USER_INTENT, + _gpu_bundle_find_trusted, _gpu_bundle_is_path_trusted, _rootless_apply, + _rootless_apply_internal, _rootless_apply_relative, _rootless_manifest_free, + _rootless_manifest_parse, _rootless_preflight ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, arm64-macos, arm64e-macos ] +install-name: '/usr/lib/system/libsystem_sanitizers.dylib' +parent-umbrella: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + umbrella: System +exports: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ ___asan_abi_addr_is_in_fake_stack, ___asan_abi_address_is_poisoned, + ___asan_abi_after_dynamic_init, ___asan_abi_alloca_poison, + ___asan_abi_allocas_unpoison, ___asan_abi_before_dynamic_init, + ___asan_abi_exp_load_n, ___asan_abi_exp_store_n, ___asan_abi_get_current_fake_stack, + ___asan_abi_handle_no_return, ___asan_abi_init, ___asan_abi_load_cxx_array_cookie, + ___asan_abi_load_n, ___asan_abi_memcpy, ___asan_abi_memmove, + ___asan_abi_memset, ___asan_abi_poison_cxx_array_cookie, ___asan_abi_poison_intra_object_redzone, + ___asan_abi_poison_memory_region, ___asan_abi_poison_stack_memory, + ___asan_abi_region_is_poisoned, ___asan_abi_register_elf_globals, + ___asan_abi_register_globals, ___asan_abi_register_image_globals, + ___asan_abi_report_exp_load_n, ___asan_abi_report_exp_store_n, + ___asan_abi_report_load_n, ___asan_abi_report_store_n, ___asan_abi_set_shadow_xx_n, + ___asan_abi_stack_free_n, ___asan_abi_stack_malloc_always_n, + ___asan_abi_stack_malloc_n, ___asan_abi_store_n, ___asan_abi_unpoison_intra_object_redzone, + ___asan_abi_unpoison_memory_region, ___asan_abi_unpoison_stack_memory, + ___asan_abi_unregister_elf_globals, ___asan_abi_unregister_globals, + ___asan_abi_unregister_image_globals, ___asan_get_alloc_stack, + ___asan_get_free_stack, ___asan_get_report_access_size, ___asan_get_report_access_type, + ___asan_get_report_address, ___asan_get_report_bp, ___asan_get_report_description, + ___asan_get_report_pc, ___asan_get_report_sp, ___asan_get_shadow_mapping, + ___asan_locate_address, ___asan_report_present, __sanitizers_init, + _sanitizers_address_on_report, _sanitizers_diagnose_memory_error, + _sanitizers_report_globals, _sanitizers_testonly_diagnose_error, + _sanitizers_testonly_get_shadow_address ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_secinit.dylib' +current-version: 168.40.2 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __libsecinit_initializer, _libsecinit_delete_all_data_container_content_for_current_user, + _libsecinit_fileoperation_save, _libsecinit_fileoperation_set_attributes, + _libsecinit_fileoperation_symlink ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_symptoms.dylib' +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __symptoms_daemon_fallback_initial_disposition, __symptoms_daemon_fallback_subseq_disposition, + __symptoms_is_daemon_fallback_blacklisted, _symptom_framework_init, + _symptom_framework_set_version, _symptom_new, _symptom_send, + _symptom_send_immediate, _symptom_set_additional_qualifier, + _symptom_set_qualifier ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libsystem_trace.dylib' +current-version: 1815.40.20 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _RTBinIndexForType, _RTLogBufferAddResource, _RTLogBufferAllocateResource, + _RTLogBufferCheckStatus, _RTLogBufferGetHeader, _RTLogBufferGetResource, + _RTLogBufferGetResourceSize, _RTLogBufferInitialize, _RTLogBufferIterate, + _RTLogBufferRequiredStorageSize, _RTLogConnect, _RTLogConnectMemoryConfigFromString, + _RTLogConnectRingBuffer, _RTLogDisconnect, _RTLogRingBufferCreateManaged, + _RTLogRingBufferDataSize, _RTLogRingBufferGetSegmentCount, + _RTLogRingBufferGetSegmentSize, _RTLogRingBufferInit, _RTLogRingBufferIsDataAvailable, + _RTLogRingBufferIterate, _RTLogRingBufferIterateFrom, _RTLogRingBufferJoinManaged, + _RTLogRingBufferReadAt, _RTLogRingBufferWriteBuffer, _RTLogRingBufferWriteMessage, + _RTLogRingBufferWriteWithCallback, __libtrace_fork_child, + __libtrace_init, __os_activity_create, __os_activity_current, + __os_activity_initiate, __os_activity_initiate_f, __os_activity_label_useraction, + __os_activity_none, __os_activity_set_breadcrumb, __os_activity_start, + __os_activity_stream_entry_encode, __os_log_create, __os_log_debug, + __os_log_debug_impl, __os_log_default, __os_log_disabled, + __os_log_error, __os_log_error_impl, __os_log_fault, __os_log_fault_impl, + __os_log_get_nscf_formatter, __os_log_impl, __os_log_internal, + __os_log_pack_fill, __os_log_pack_size, __os_log_preferences_compute, + __os_log_preferences_copy_cache, __os_log_preferences_load, + __os_log_preferences_load_sysprefs, __os_log_preferences_merge, + __os_log_release, __os_log_send_and_compose_impl, __os_log_set_nscf_formatter, + __os_log_unreliable_impl, __os_metric_double_create_impl, + __os_metric_double_op_impl, __os_metric_int64_create_impl, + __os_metric_int64_op_impl, __os_metric_label_create_impl, + __os_metric_label_create_v, __os_metric_reset_impl, __os_metric_set_scale_impl, + __os_metric_set_unit_impl, __os_metric_uint64_create_impl, + __os_metric_uint64_op_impl, __os_signpost_emit_impl, __os_signpost_emit_unreliably_with_name_impl, + __os_signpost_emit_with_name_impl, __os_signpost_pack_fill, + __os_signpost_pack_send, __os_state_request_for_pidlist, __os_trace_app_cryptex_sysprefsdir_path, + __os_trace_atm_diagnostic_config, __os_trace_basesystem_storage_available, + __os_trace_calloc_typed, __os_trace_commpage_compute, __os_trace_fdscandir_b, + __os_trace_get_boot_uuid, __os_trace_get_image_info, __os_trace_get_mode_for_pid, + __os_trace_get_times_now, __os_trace_getxattr_at, __os_trace_intprefsdir_path, + __os_trace_is_development_build, __os_trace_lazy_init_completed_4libxpc, + __os_trace_lazy_init_completed_4swift, __os_trace_log_simple, + __os_trace_macho_for_each_slice, __os_trace_malloc_typed, + __os_trace_memdup, __os_trace_mmap, __os_trace_mmap_at, __os_trace_mmap_offset, + __os_trace_mode_match_4tests, __os_trace_os_cryptex_sysprefsdir_path, + __os_trace_prefs_latest_version_4tests, __os_trace_prefsdir_path, + __os_trace_read_file_at, __os_trace_read_plist_at, __os_trace_realloc_typed, + __os_trace_scandir_free_namelist, __os_trace_sect_names, __os_trace_set_diagnostic_flags, + __os_trace_set_mode_for_pid, __os_trace_strdup, __os_trace_sysprefsdir_path, + __os_trace_update_with_datavolume_4launchd, __os_trace_with_buffer, + __os_trace_write, __os_trace_writev, __os_trace_zalloc_typed, + _amfi_check_dyld_policy_for_pid, _amfi_check_dyld_policy_self, + _amfi_load_trust_cache, _amfi_unload_trust_cache, _os_activity_apply, + _os_activity_apply_f, _os_activity_diagnostic_for_pid, _os_activity_end, + _os_activity_for_task_thread, _os_activity_for_thread, _os_activity_get_active, + _os_activity_get_identifier, _os_activity_iterate_activities, + _os_activity_iterate_breadcrumbs, _os_activity_iterate_messages, + _os_activity_iterate_processes, _os_activity_messages_for_thread, + _os_activity_scope_enter, _os_activity_scope_leave, _os_log_backtrace_copy_description, + _os_log_backtrace_copy_serialized_buffer, _os_log_backtrace_create_from_buffer, + _os_log_backtrace_create_from_current, _os_log_backtrace_create_from_pcs, + _os_log_backtrace_create_from_return_address, _os_log_backtrace_destroy, + _os_log_backtrace_get_frames, _os_log_backtrace_get_length, + _os_log_backtrace_print_to_blob, _os_log_backtrace_serialize_to_blob, + _os_log_compare_enablement, _os_log_copy_decorated_message, + _os_log_copy_message_string, _os_log_create, _os_log_errors_count, + _os_log_fault_default_callback, _os_log_faults_count, _os_log_fmt_compose, + _os_log_fmt_convert_trace, _os_log_fmt_extract_pubdata, _os_log_fmt_get_plugin, + _os_log_get_type, _os_log_is_debug_enabled, _os_log_is_enabled, + _os_log_pack_compose, _os_log_pack_send, _os_log_pack_send_and_compose, + _os_log_set_client_type, _os_log_set_enabled, _os_log_set_fault_callback, + _os_log_set_hook, _os_log_set_hook_with_params, _os_log_set_test_callback, + _os_log_shim_enabled, _os_log_shim_legacy_logging_enabled, + _os_log_shim_to_stdout, _os_log_shim_with_CFString, _os_log_shim_with_CFString_4NSLog, + _os_log_type_enabled, _os_log_type_get_name, _os_log_with_args, + _os_log_with_args_4syslog, _os_metric_dimensions_add, _os_metric_dimensions_create, + _os_metric_group_create, _os_set_logging_unreliable_for_current_thread, + _os_signpost_enabled, _os_signpost_id_generate, _os_signpost_id_make_with_pointer, + _os_signpost_set_introspection_hook_4Perf, _os_state_add_handler, + _os_state_remove_handler, _os_trace_debug_enabled, _os_trace_get_mode, + _os_trace_get_type, _os_trace_info_enabled, _os_trace_set_mode ] + objc-classes: [ OS_os_log ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, arm64-macos, arm64e-macos ] +install-name: '/usr/lib/system/libsystem_trial.dylib' +current-version: 474.3 +parent-umbrella: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + umbrella: System +exports: + - targets: [ x86_64-macos, arm64-macos, arm64e-macos ] + symbols: [ __os_trial_factor_get_bool_impl, __os_trial_factor_get_long_impl, + __os_trial_factor_has_impl ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libunwind.dylib' +current-version: 1900.125 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ __Unwind_Backtrace, __Unwind_DeleteException, __Unwind_FindEnclosingFunction, + __Unwind_Find_FDE, __Unwind_ForcedUnwind, __Unwind_GetCFA, + __Unwind_GetDataRelBase, __Unwind_GetGR, __Unwind_GetIP, __Unwind_GetIPInfo, + __Unwind_GetLanguageSpecificData, __Unwind_GetRegionStart, + __Unwind_GetTextRelBase, __Unwind_RaiseException, __Unwind_Resume, + __Unwind_Resume_or_Rethrow, __Unwind_SetGR, __Unwind_SetIP, + __ZN9libunwind25findDynamicUnwindSectionsEPvP27unw_dynamic_unwind_sections, + ___deregister_frame, ___register_frame, ___unw_add_dynamic_eh_frame_section, + ___unw_add_dynamic_fde, ___unw_add_find_dynamic_unwind_sections, + ___unw_remove_dynamic_eh_frame_section, ___unw_remove_dynamic_fde, + ___unw_remove_find_dynamic_unwind_sections, _unw_get_fpreg, + _unw_get_proc_info, _unw_get_proc_name, _unw_get_reg, _unw_getcontext, + _unw_init_local, _unw_is_fpreg, _unw_is_signal_frame, _unw_iterate_dwarf_unwind_cache, + _unw_local_addr_space, _unw_regname, _unw_resume, _unw_resume_with_frames_walked, + _unw_set_fpreg, _unw_set_reg, _unw_step ] + - targets: [ arm64-macos, arm64-maccatalyst, arm64e-macos, arm64e-maccatalyst ] + symbols: [ ___unw_is_pointer_auth_enabled ] +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +install-name: '/usr/lib/system/libxpc.dylib' +current-version: 3089.41.2 +parent-umbrella: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + umbrella: System +exports: + - targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] + symbols: [ _XPC_ACTIVITY_ALLOW_BATTERY, _XPC_ACTIVITY_APP_REFRESH, _XPC_ACTIVITY_CHECK_IN, + _XPC_ACTIVITY_COMMUNICATES_WITH_PAIRED_DEVICE, _XPC_ACTIVITY_CPU_INTENSIVE, + _XPC_ACTIVITY_DELAY, _XPC_ACTIVITY_DESIRED_MOTION_STATE, _XPC_ACTIVITY_DISK_INTENSIVE, + _XPC_ACTIVITY_DO_IT_LATER, _XPC_ACTIVITY_DUET_ACTIVITY_SCHEDULER_DATA, + _XPC_ACTIVITY_DUET_ATTRIBUTE_COST, _XPC_ACTIVITY_DUET_ATTRIBUTE_NAME, + _XPC_ACTIVITY_DUET_ATTRIBUTE_VALUE, _XPC_ACTIVITY_DUET_RELATED_APPLICATIONS, + _XPC_ACTIVITY_EXCLUSIVE, _XPC_ACTIVITY_EXPECTED_DURATION, + _XPC_ACTIVITY_GRACE_PERIOD, _XPC_ACTIVITY_GROUP_CONCURRENCY_LIMIT, + _XPC_ACTIVITY_GROUP_NAME, _XPC_ACTIVITY_INTERVAL, _XPC_ACTIVITY_INTERVAL_15_MIN, + _XPC_ACTIVITY_INTERVAL_1_DAY, _XPC_ACTIVITY_INTERVAL_1_HOUR, + _XPC_ACTIVITY_INTERVAL_1_MIN, _XPC_ACTIVITY_INTERVAL_30_MIN, + _XPC_ACTIVITY_INTERVAL_4_HOURS, _XPC_ACTIVITY_INTERVAL_5_MIN, + _XPC_ACTIVITY_INTERVAL_7_DAYS, _XPC_ACTIVITY_INTERVAL_8_HOURS, + _XPC_ACTIVITY_INVOLVED_PROCESSES, _XPC_ACTIVITY_MAY_REBOOT_DEVICE, + _XPC_ACTIVITY_MEMORY_INTENSIVE, _XPC_ACTIVITY_MOTION_STATE_AUTOMOTIVE, + _XPC_ACTIVITY_MOTION_STATE_AUTOMOTIVE_MOVING, _XPC_ACTIVITY_MOTION_STATE_AUTOMOTIVE_STATIONARY, + _XPC_ACTIVITY_MOTION_STATE_CYCLING, _XPC_ACTIVITY_MOTION_STATE_RUNNING, + _XPC_ACTIVITY_MOTION_STATE_STATIONARY, _XPC_ACTIVITY_MOTION_STATE_WALKING, + _XPC_ACTIVITY_NETWORK_DOWNLOAD_SIZE, _XPC_ACTIVITY_NETWORK_TRANSFER_BIDIRECTIONAL, + _XPC_ACTIVITY_NETWORK_TRANSFER_DIRECTION, _XPC_ACTIVITY_NETWORK_TRANSFER_DIRECTION_DOWNLOAD, + _XPC_ACTIVITY_NETWORK_TRANSFER_DIRECTION_UPLOAD, _XPC_ACTIVITY_NETWORK_TRANSFER_ENDPOINT, + _XPC_ACTIVITY_NETWORK_TRANSFER_PARAMETERS, _XPC_ACTIVITY_NETWORK_TRANSFER_SIZE, + _XPC_ACTIVITY_NETWORK_UPLOAD_SIZE, _XPC_ACTIVITY_POST_INSTALL, + _XPC_ACTIVITY_POWER_NAP, _XPC_ACTIVITY_PREVENT_DEVICE_SLEEP, + _XPC_ACTIVITY_PRIORITY, _XPC_ACTIVITY_PRIORITY_MAINTENANCE, + _XPC_ACTIVITY_PRIORITY_UTILITY, _XPC_ACTIVITY_RANDOM_INITIAL_DELAY, + _XPC_ACTIVITY_REPEATING, _XPC_ACTIVITY_REPLY_ENDPOINT, _XPC_ACTIVITY_REQUIRES_BUDDY_COMPLETE, + _XPC_ACTIVITY_REQUIRES_CLASS_A, _XPC_ACTIVITY_REQUIRES_CLASS_B, + _XPC_ACTIVITY_REQUIRES_CLASS_C, _XPC_ACTIVITY_REQUIRE_BATTERY_LEVEL, + _XPC_ACTIVITY_REQUIRE_HDD_SPINNING, _XPC_ACTIVITY_REQUIRE_INEXPENSIVE_NETWORK_CONNECTIVITY, + _XPC_ACTIVITY_REQUIRE_NETWORK_CONNECTIVITY, _XPC_ACTIVITY_REQUIRE_SCREEN_SLEEP, + _XPC_ACTIVITY_REQUIRE_SIGNIFICANT_USER_INACTIVITY, _XPC_ACTIVITY_RUN_WHEN_APP_FOREGROUNDED, + _XPC_ACTIVITY_SEQUENCE_NUMBER, _XPC_ACTIVITY_SHOULD_WAKE_DEVICE, + _XPC_ACTIVITY_USER_REQUESTED_BACKUP_TASK, _XPC_ACTIVITY_USES_DATA_BUDGETING, + _XPC_ACTIVITY_USES_DUET_POWER_BUDGETING, _XPC_COALITION_INFO_KEY_BUNDLE_IDENTIFIER, + _XPC_COALITION_INFO_KEY_CID, _XPC_COALITION_INFO_KEY_LEADER_PID, + _XPC_COALITION_INFO_KEY_NAME, _XPC_COALITION_INFO_KEY_RESOURCE_USAGE_BLOB, + ___xpc_connection_set_logging, __availability_version_check, + __launch_job_query_routine, __launch_job_routine, __launch_job_routine_async, + __launch_msg2, __launch_server_test_routine, __launch_service_stats_copy_4ppse_impl, + __launch_service_stats_copy_impl, __launch_trial_factors_routine, + __libxpc_initializer, __spawn_via_launchd, __system_ios_support_version_copy_string_sysctl, + __system_version_copy_string_plist, __system_version_copy_string_sysctl, + __system_version_fallback, __system_version_parse_string, + __system_version_plist_path, __vproc_get_last_exit_status, + __vproc_grab_subset, __vproc_kickstart_by_label, __vproc_log, + __vproc_log_error, __vproc_logv, __vproc_pid_is_managed, __vproc_post_fork_ping, + __vproc_send_signal_by_label, __vproc_set_global_on_demand, + __vproc_standby_begin, __vproc_standby_count, __vproc_standby_end, + __vproc_standby_timeout, __vproc_transaction_begin, __vproc_transaction_count, + __vproc_transaction_count_for_pid, __vproc_transaction_end, + __vproc_transaction_set_clean_callback, __vproc_transaction_try_exit, + __vproc_transactions_enable, __vprocmgr_detach_from_console, + __vprocmgr_getsocket, __vprocmgr_init, __vprocmgr_log_drain, + __vprocmgr_log_forward, __vprocmgr_move_subset_to_user, __vprocmgr_switch_to_session, + __xpc_bool_create_distinct, __xpc_bool_false, __xpc_bool_set_value, + __xpc_bool_true, __xpc_connection_create_internal_listener, + __xpc_connection_get_parent_4test, __xpc_connection_get_recvp_4test, + __xpc_connection_set_event_handler_f, __xpc_data_set_value, + __xpc_dictionary_create_reply_with_port, __xpc_dictionary_extract_mach_send, + __xpc_dictionary_extract_reply_msg_id, __xpc_dictionary_extract_reply_port, + __xpc_dictionary_get_reply_msg_id, __xpc_dictionary_get_transaction, + __xpc_dictionary_set_remote_connection, __xpc_dictionary_set_reply_msg_id, + __xpc_domain_routine, __xpc_double_set_value, __xpc_error_connection_interrupted, + __xpc_error_connection_invalid, __xpc_error_key_description, + __xpc_error_peer_code_signing_requirement, __xpc_error_termination_imminent, + __xpc_event_key_name, __xpc_event_key_stream_name, __xpc_fd_get_port, + __xpc_int64_set_value, __xpc_payload_create_from_mach_msg, + __xpc_peer_requirement_match_token, __xpc_pipe_handle_mig, + __xpc_pipe_interface_routine, __xpc_pipe_interface_routine_async, + __xpc_pipe_interface_simpleroutine, __xpc_runtime_get_entitlements_data, + __xpc_runtime_get_self_entitlements, __xpc_runtime_is_app_sandboxed, + __xpc_runtime_process_has_entered_sandbox, __xpc_service_last_xref_cancel, + __xpc_service_routine, __xpc_session_create_from_connection_4SWIFT, + __xpc_session_extract_connection_4SWIFT, __xpc_session_get_peer_audit_token_4SWIFT, + __xpc_shmem_get_mach_port, __xpc_spawnattr_binprefs_pack, + __xpc_spawnattr_binprefs_size, __xpc_spawnattr_binprefs_unpack, + __xpc_spawnattr_pack_bytes, __xpc_spawnattr_pack_string, __xpc_spawnattr_pack_string_fragment, + __xpc_spawnattr_unpack_bytes, __xpc_spawnattr_unpack_string, + __xpc_spawnattr_unpack_strings, __xpc_string_set_value, __xpc_type_activity, + __xpc_type_array, __xpc_type_base, __xpc_type_bool, __xpc_type_bundle, + __xpc_type_connection, __xpc_type_data, __xpc_type_date, __xpc_type_dictionary, + __xpc_type_double, __xpc_type_endpoint, __xpc_type_error, + __xpc_type_fd, __xpc_type_file_transfer, __xpc_type_int64, + __xpc_type_mach_recv, __xpc_type_mach_send, __xpc_type_mach_send_once, + __xpc_type_null, __xpc_type_pipe, __xpc_type_pointer, __xpc_type_rich_error, + __xpc_type_serializer, __xpc_type_service, __xpc_type_service_instance, + __xpc_type_session, __xpc_type_shmem, __xpc_type_string, __xpc_type_string_cache, + __xpc_type_uint64, __xpc_type_uuid, __xpc_vtables, _amfi_developer_mode_resolved, + _amfi_developer_mode_status, _amfi_interface_authorize_local_signing, + _amfi_interface_cdhash_in_trustcache, _amfi_interface_get_local_signing_private_key, + _amfi_interface_get_local_signing_public_key, _amfi_interface_query_bootarg_state, + _amfi_interface_set_local_signing_public_key, _amfi_launch_constraint_matches_process, + _amfi_launch_constraint_set_spawnattr, _amfi_restricted_execution_mode_enable, + _amfi_restricted_execution_mode_status, _bootstrap_check_in, + _bootstrap_check_in2, _bootstrap_check_in3, _bootstrap_create_server, + _bootstrap_create_service, _bootstrap_get_root, _bootstrap_info, + _bootstrap_init, _bootstrap_look_up, _bootstrap_look_up2, + _bootstrap_look_up3, _bootstrap_look_up_per_user, _bootstrap_lookup_children, + _bootstrap_parent, _bootstrap_register, _bootstrap_register2, + _bootstrap_status, _bootstrap_strerror, _bootstrap_subset, + _bootstrap_unprivileged, _create_and_switch_to_per_session_launchd, + _launch_activate_socket, _launch_active_user_login, _launch_active_user_logout, + _launch_add_external_service, _launch_bootout_user_service_4coresim, + _launch_bootout_user_service_4coresim_with_flags, _launch_copy_busy_extension_instances, + _launch_copy_endpoints_properties_for_pid, _launch_copy_extension_properties, + _launch_copy_extension_properties_for_pid, _launch_copy_properties_for_pid_4assertiond, + _launch_create_persona, _launch_cryptex_terminate, _launch_data_alloc, + _launch_data_array_get_count, _launch_data_array_get_index, + _launch_data_array_set_index, _launch_data_copy, _launch_data_dict_get_count, + _launch_data_dict_insert, _launch_data_dict_iterate, _launch_data_dict_lookup, + _launch_data_dict_remove, _launch_data_free, _launch_data_get_bool, + _launch_data_get_errno, _launch_data_get_fd, _launch_data_get_integer, + _launch_data_get_machport, _launch_data_get_opaque, _launch_data_get_opaque_size, + _launch_data_get_real, _launch_data_get_string, _launch_data_get_type, + _launch_data_new_bool, _launch_data_new_errno, _launch_data_new_fd, + _launch_data_new_integer, _launch_data_new_machport, _launch_data_new_opaque, + _launch_data_new_real, _launch_data_new_string, _launch_data_pack, + _launch_data_set_bool, _launch_data_set_errno, _launch_data_set_fd, + _launch_data_set_integer, _launch_data_set_machport, _launch_data_set_opaque, + _launch_data_set_real, _launch_data_set_string, _launch_data_unpack, + _launch_destroy_persona, _launch_disable_directory, _launch_enable_directory, + _launch_extension_check_in_live_4UIKit, _launch_extension_property_bundle_id, + _launch_extension_property_host_bundle_id, _launch_extension_property_host_pid, + _launch_extension_property_path, _launch_extension_property_pid, + _launch_extension_property_version, _launch_extension_property_xpc_bundle, + _launch_get_fd, _launch_get_service_enabled, _launch_get_system_service_enabled, + _launch_job_query_get_additional_job_properties, _launch_load_mounted_jetsam_properties, + _launch_msg, _launch_path_for_user_service_4coresim, _launch_perfcheck_property_endpoint_active, + _launch_perfcheck_property_endpoint_event, _launch_perfcheck_property_endpoint_name, + _launch_perfcheck_property_endpoint_needs_activation, _launch_perfcheck_property_endpoints, + _launch_remove_external_service, _launch_service_instance_copy_uuids, + _launch_service_instance_create, _launch_service_instance_remove, + _launch_service_stats_disable, _launch_service_stats_disable_4ppse, + _launch_service_stats_enable, _launch_service_stats_enable_4ppse, + _launch_service_stats_is_enabled, _launch_service_stats_is_enabled_4ppse, + _launch_set_service_enabled, _launch_set_system_service_enabled, + _launch_socket_service_check_in, _launch_trial_factors_active_reload, + _launch_urgent_log_submission_completed, _launch_userspace_reboot_with_purpose, + _launch_version_for_user_service_4coresim, _launch_wait, _launchd_close, + _launchd_fdopen, _launchd_getfd, _launchd_msg_recv, _launchd_msg_send, + _load_launchd_jobs_at_loginwindow_prompt, _mpm_uncork_fork, + _mpm_wait, _os_system_version_get_current_version, _os_system_version_get_ios_support_version, + _os_system_version_sim_get_current_host_version, _os_transaction_copy_description, + _os_transaction_create, _os_transaction_get_description, _os_transaction_get_ra, + _os_transaction_get_timestamp, _os_transaction_log_active, + _os_transaction_needs_more_time, _place_hold_on_real_loginwindow, + _reboot2, _reboot3, _vproc_release, _vproc_retain, _vproc_standby_begin, + _vproc_standby_end, _vproc_swap_complex, _vproc_swap_integer, + _vproc_swap_string, _vproc_transaction_begin, _vproc_transaction_end, + _vprocmgr_lookup_vproc, _xpc_activity_add_eligibility_changed_handler, + _xpc_activity_copy_criteria, _xpc_activity_copy_dispatch_queue, + _xpc_activity_copy_identifier, _xpc_activity_debug, _xpc_activity_defer_until_network_change, + _xpc_activity_defer_until_percentage, _xpc_activity_get_percentage, + _xpc_activity_get_state, _xpc_activity_list, _xpc_activity_register, + _xpc_activity_remove_eligibility_changed_handler, _xpc_activity_run, + _xpc_activity_set_completion_status, _xpc_activity_set_criteria, + _xpc_activity_set_network_threshold, _xpc_activity_set_state, + _xpc_activity_should_be_data_budgeted, _xpc_activity_should_defer, + _xpc_activity_unregister, _xpc_add_bundle, _xpc_add_bundle_with_lwcr, + _xpc_add_bundles_for_domain, _xpc_array_append_value, _xpc_array_apply, + _xpc_array_apply_f, _xpc_array_copy_mach_send, _xpc_array_create, + _xpc_array_create_connection, _xpc_array_create_empty, _xpc_array_dup_fd, + _xpc_array_get_array, _xpc_array_get_bool, _xpc_array_get_count, + _xpc_array_get_data, _xpc_array_get_date, _xpc_array_get_dictionary, + _xpc_array_get_double, _xpc_array_get_int64, _xpc_array_get_pointer, + _xpc_array_get_string, _xpc_array_get_uint64, _xpc_array_get_uuid, + _xpc_array_get_value, _xpc_array_set_bool, _xpc_array_set_connection, + _xpc_array_set_data, _xpc_array_set_date, _xpc_array_set_double, + _xpc_array_set_fd, _xpc_array_set_int64, _xpc_array_set_mach_send, + _xpc_array_set_pointer, _xpc_array_set_string, _xpc_array_set_uint64, + _xpc_array_set_uuid, _xpc_array_set_value, _xpc_atfork_child, + _xpc_atfork_parent, _xpc_atfork_prepare, _xpc_binprefs_add, + _xpc_binprefs_alloc, _xpc_binprefs_copy, _xpc_binprefs_copy_description, + _xpc_binprefs_count, _xpc_binprefs_cpu_subtype, _xpc_binprefs_cpu_type, + _xpc_binprefs_equal, _xpc_binprefs_init, _xpc_binprefs_is_noop, + _xpc_binprefs_set_psattr, _xpc_bool_create, _xpc_bool_get_value, + _xpc_bs_main, _xpc_bundle_copy_info_dictionary, _xpc_bundle_copy_normalized_cryptex_path, + _xpc_bundle_copy_services, _xpc_bundle_create, _xpc_bundle_create_from_origin, + _xpc_bundle_create_from_origin_with_string_cache, _xpc_bundle_create_main, + _xpc_bundle_create_with_string_cache, _xpc_bundle_get_error, + _xpc_bundle_get_executable_path, _xpc_bundle_get_info_dictionary, + _xpc_bundle_get_path, _xpc_bundle_get_property, _xpc_bundle_get_xpcservice_dictionary, + _xpc_bundle_populate, _xpc_bundle_resolve, _xpc_bundle_resolve_on_queue, + _xpc_bundle_resolve_sync, _xpc_coalition_copy_info, _xpc_coalition_history_pipe_async, + _xpc_connection_activate, _xpc_connection_bs_seal_listener, + _xpc_connection_cancel, _xpc_connection_copy_bundle_id, _xpc_connection_copy_entitlement_value, + _xpc_connection_copy_invalidation_reason, _xpc_connection_create, + _xpc_connection_create_bs_service_listener, _xpc_connection_create_from_endpoint, + _xpc_connection_create_listener, _xpc_connection_create_mach_service, + _xpc_connection_enable_sim2host_4sim, _xpc_connection_enable_termination_imminent_event, + _xpc_connection_get_asid, _xpc_connection_get_audit_token, + _xpc_connection_get_bs_type, _xpc_connection_get_context, + _xpc_connection_get_egid, _xpc_connection_get_euid, _xpc_connection_get_filter_policy_id_4test, + _xpc_connection_get_instance, _xpc_connection_get_name, _xpc_connection_get_peer_instance, + _xpc_connection_get_pid, _xpc_connection_is_extension, _xpc_connection_kill, + _xpc_connection_resume, _xpc_connection_send_barrier, _xpc_connection_send_message, + _xpc_connection_send_message_with_reply, _xpc_connection_send_message_with_reply_sync, + _xpc_connection_send_notification, _xpc_connection_set_bootstrap, + _xpc_connection_set_bs_type, _xpc_connection_set_context, + _xpc_connection_set_distorter, _xpc_connection_set_event_channel, + _xpc_connection_set_event_handler, _xpc_connection_set_finalizer_f, + _xpc_connection_set_instance, _xpc_connection_set_instance_binpref, + _xpc_connection_set_legacy, _xpc_connection_set_non_launching, + _xpc_connection_set_oneshot_instance, _xpc_connection_set_peer_code_signing_requirement, + _xpc_connection_set_peer_entitlement_exists_requirement, _xpc_connection_set_peer_entitlement_matches_value_requirement, + _xpc_connection_set_peer_lightweight_code_requirement, _xpc_connection_set_peer_platform_identity_requirement, + _xpc_connection_set_peer_requirement, _xpc_connection_set_peer_team_identity_requirement, + _xpc_connection_set_privileged, _xpc_connection_set_qos_class_fallback, + _xpc_connection_set_qos_class_floor, _xpc_connection_set_target_queue, + _xpc_connection_set_target_uid, _xpc_connection_set_target_user_session_uid, + _xpc_connection_suspend, _xpc_copy, _xpc_copy_bootstrap, _xpc_copy_clean_description, + _xpc_copy_code_signing_identity_for_token, _xpc_copy_debug_description, + _xpc_copy_description, _xpc_copy_domain, _xpc_copy_entitlement_for_self, + _xpc_copy_entitlement_for_token, _xpc_copy_entitlements_data_for_token, + _xpc_copy_entitlements_for_pid, _xpc_copy_entitlements_for_self, + _xpc_copy_event, _xpc_copy_event_entitlements, _xpc_copy_short_description, + _xpc_create_from_ce_der, _xpc_create_from_ce_der_with_key, + _xpc_create_from_plist, _xpc_create_from_plist_descriptor, + _xpc_create_from_plist_with_string_cache, _xpc_create_from_serialization, + _xpc_create_from_serialization_with_ool, _xpc_create_from_serialization_with_string_cache, + _xpc_create_reply_with_format, _xpc_create_reply_with_format_and_arguments, + _xpc_create_with_format, _xpc_create_with_format_and_arguments, + _xpc_data_create, _xpc_data_create_with_dispatch_data, _xpc_data_get_bytes, + _xpc_data_get_bytes_ptr, _xpc_data_get_bytes_ptr_and_length, + _xpc_data_get_inline_max, _xpc_data_get_length, _xpc_date_create, + _xpc_date_create_absolute, _xpc_date_create_from_current, + _xpc_date_get_value, _xpc_date_get_value_absolute, _xpc_date_is_int64_range, + _xpc_dictionary_apply, _xpc_dictionary_apply_f, _xpc_dictionary_copy_basic_description, + _xpc_dictionary_copy_mach_send, _xpc_dictionary_create, _xpc_dictionary_create_connection, + _xpc_dictionary_create_empty, _xpc_dictionary_create_reply, + _xpc_dictionary_dup_fd, _xpc_dictionary_expects_reply, _xpc_dictionary_extract_mach_recv, + _xpc_dictionary_get_array, _xpc_dictionary_get_audit_token, + _xpc_dictionary_get_bool, _xpc_dictionary_get_connection, + _xpc_dictionary_get_count, _xpc_dictionary_get_data, _xpc_dictionary_get_date, + _xpc_dictionary_get_dictionary, _xpc_dictionary_get_double, + _xpc_dictionary_get_int64, _xpc_dictionary_get_pointer, _xpc_dictionary_get_remote_connection, + _xpc_dictionary_get_string, _xpc_dictionary_get_uint64, _xpc_dictionary_get_uuid, + _xpc_dictionary_get_value, _xpc_dictionary_handoff_reply, + _xpc_dictionary_handoff_reply_f, _xpc_dictionary_send_reply, + _xpc_dictionary_send_reply_4SWIFT, _xpc_dictionary_set_bool, + _xpc_dictionary_set_connection, _xpc_dictionary_set_data, + _xpc_dictionary_set_date, _xpc_dictionary_set_double, _xpc_dictionary_set_fd, + _xpc_dictionary_set_int64, _xpc_dictionary_set_mach_recv, + _xpc_dictionary_set_mach_send, _xpc_dictionary_set_pointer, + _xpc_dictionary_set_string, _xpc_dictionary_set_uint64, _xpc_dictionary_set_uuid, + _xpc_dictionary_set_value, _xpc_dictionary_set_value_with_key_string_cache, + _xpc_double_create, _xpc_double_get_value, _xpc_endpoint_compare, + _xpc_endpoint_copy_listener_port_4sim, _xpc_endpoint_create, + _xpc_endpoint_create_bs_from_port, _xpc_endpoint_create_bs_named, + _xpc_endpoint_create_bs_named_user, _xpc_endpoint_create_bs_service, + _xpc_endpoint_create_mach_port_4sim, _xpc_endpoint_get_bs_job_handle, + _xpc_equal, _xpc_event_publisher_activate, _xpc_event_publisher_copy_event, + _xpc_event_publisher_create, _xpc_event_publisher_create_subscription, + _xpc_event_publisher_fire, _xpc_event_publisher_fire_barrier, + _xpc_event_publisher_fire_noboost, _xpc_event_publisher_fire_with_reply, + _xpc_event_publisher_fire_with_reply_sync, _xpc_event_publisher_get_subscriber_asid, + _xpc_event_publisher_set_error_handler, _xpc_event_publisher_set_event, + _xpc_event_publisher_set_handler, _xpc_event_publisher_set_initial_load_completed_handler_4remoted, + _xpc_event_publisher_set_subscriber_keepalive, _xpc_event_publisher_set_throttling, + _xpc_exit_reason_get_label, _xpc_extension_type_init, _xpc_fd_create, + _xpc_fd_dup, _xpc_file_transfer_cancel, _xpc_file_transfer_copy_io, + _xpc_file_transfer_create_with_fd, _xpc_file_transfer_create_with_path, + _xpc_file_transfer_get_size, _xpc_file_transfer_get_transfer_id, + _xpc_file_transfer_send_finished, _xpc_file_transfer_set_transport_writing_callbacks, + _xpc_file_transfer_write_finished, _xpc_file_transfer_write_to_fd, + _xpc_file_transfer_write_to_path, _xpc_generate_audit_token, + _xpc_get_attachment_endpoint, _xpc_get_class4NSXPC, _xpc_get_event_name, + _xpc_get_instance, _xpc_get_service_identifier_for_token, + _xpc_get_service_name_from_pid, _xpc_get_service_uid_for_token, + _xpc_get_type, _xpc_handle_service, _xpc_handle_subservice, + _xpc_hash, _xpc_impersonate_user, _xpc_init_services, _xpc_inspect_copy_description, + _xpc_inspect_copy_description_local, _xpc_inspect_copy_short_description, + _xpc_inspect_copy_short_description_local, _xpc_install_remote_hooks, + _xpc_int64_create, _xpc_int64_get_value, _xpc_is_kind_of_xpc_object4NSXPC, + _xpc_is_system_session, _xpc_is_xpcservice, _xpc_listener_activate, + _xpc_listener_cancel, _xpc_listener_copy_description, _xpc_listener_create, + _xpc_listener_create_anonymous, _xpc_listener_create_endpoint, + _xpc_listener_reject_peer, _xpc_listener_set_incoming_session_handler, + _xpc_listener_set_peer_code_signing_requirement, _xpc_listener_set_peer_requirement, + _xpc_mach_recv_create, _xpc_mach_recv_extract_right, _xpc_mach_send_copy_right, + _xpc_mach_send_create, _xpc_mach_send_create_with_disposition, + _xpc_mach_send_get_right, _xpc_mach_send_once_create, _xpc_mach_send_once_extract_right, + _xpc_main, _xpc_make_serialization, _xpc_make_serialization_with_ool, + _xpc_null_create, _xpc_peer_requirement_create_entitlement_exists, + _xpc_peer_requirement_create_entitlement_matches_value, _xpc_peer_requirement_create_lwcr, + _xpc_peer_requirement_create_platform_identity, _xpc_peer_requirement_create_team_identity, + _xpc_peer_requirement_match_received_message, _xpc_pipe_create, + _xpc_pipe_create_from_port, _xpc_pipe_create_reply_from_port, + _xpc_pipe_create_with_user_session_uid, _xpc_pipe_invalidate, + _xpc_pipe_receive, _xpc_pipe_routine, _xpc_pipe_routine_async, + _xpc_pipe_routine_forward, _xpc_pipe_routine_reply, _xpc_pipe_routine_with_flags, + _xpc_pipe_simpleroutine, _xpc_pipe_try_receive, _xpc_pointer_create, + _xpc_pointer_get_value, _xpc_receive_mach_msg, _xpc_receive_mach_msg_validate_hdr, + _xpc_receive_remote_msg, _xpc_release, _xpc_retain, _xpc_rich_error_can_retry, + _xpc_rich_error_copy_description, _xpc_service_attach, _xpc_service_attach_with_flags, + _xpc_service_create, _xpc_service_create_from_specifier, _xpc_service_get_rendezvous_token, + _xpc_service_instance_dup2, _xpc_service_instance_get_context, + _xpc_service_instance_get_host_pid, _xpc_service_instance_get_pid, + _xpc_service_instance_get_type, _xpc_service_instance_is_configurable, + _xpc_service_instance_run, _xpc_service_instance_set_archpref, + _xpc_service_instance_set_binpref, _xpc_service_instance_set_context, + _xpc_service_instance_set_cwd, _xpc_service_instance_set_endpoint, + _xpc_service_instance_set_environment, _xpc_service_instance_set_finalizer_f, + _xpc_service_instance_set_jetsam_properties, _xpc_service_instance_set_path, + _xpc_service_instance_set_start_suspended, _xpc_service_instance_set_use_sec_transition_shims, + _xpc_service_kickstart, _xpc_service_kickstart_with_flags, + _xpc_service_set_attach_handler, _xpc_session_activate, _xpc_session_cancel, + _xpc_session_copy_description, _xpc_session_create_mach_service, + _xpc_session_create_xpc_endpoint, _xpc_session_create_xpc_service, + _xpc_session_send_message, _xpc_session_send_message_with_reply_async, + _xpc_session_send_message_with_reply_sync, _xpc_session_set_cancel_handler, + _xpc_session_set_incoming_message_handler, _xpc_session_set_instance, + _xpc_session_set_peer_code_signing_requirement, _xpc_session_set_peer_requirement, + _xpc_session_set_target_queue, _xpc_session_set_target_user_session_uid, + _xpc_set_event, _xpc_set_event_state, _xpc_set_event_stream_handler, + _xpc_set_event_with_flags, _xpc_set_idle_handler, _xpc_shmem_create, + _xpc_shmem_create_readonly, _xpc_shmem_get_length, _xpc_shmem_map, + _xpc_strerror, _xpc_string_cache_create, _xpc_string_cache_for_each, + _xpc_string_cache_get_count, _xpc_string_cache_get_name, _xpc_string_create, + _xpc_string_create_cached, _xpc_string_create_no_copy, _xpc_string_create_with_format, + _xpc_string_create_with_format_and_arguments, _xpc_string_get_length, + _xpc_string_get_string_ptr, _xpc_test_symbols_exported, _xpc_track_activity, + _xpc_transaction_begin, _xpc_transaction_end, _xpc_transaction_exit_clean, + _xpc_transaction_interrupt_clean_exit, _xpc_transaction_try_exit_clean, + _xpc_transactions_enable, _xpc_traverse_serialized_data, _xpc_type_get_name, + _xpc_uint64_create, _xpc_uint64_get_value, _xpc_user_sessions_enabled, + _xpc_user_sessions_get_foreground_uid, _xpc_user_sessions_get_session_uid, + _xpc_uuid_create, _xpc_uuid_get_bytes ] + objc-classes: [ OS_xpc_object ] +... diff --git a/contrib/normalize_triplet.py b/contrib/normalize_triplet.py index 833b725480996..6f7f7c60cdac1 100755 --- a/contrib/normalize_triplet.py +++ b/contrib/normalize_triplet.py @@ -21,13 +21,15 @@ 'darwin': "-apple-darwin[\\d\\.]*", 'freebsd': "-(.*-)?freebsd[\\d\\.]*", 'openbsd': "-(.*-)?openbsd[\\d\\.]*", - 'windows': "-w64-mingw32", + 'windows': "-w64", 'linux': "-(.*-)?linux", } libc_mapping = { 'blank_libc': "", 'gnu': "-gnu", 'musl': "-musl", + 'msvcrt': "-mingw32", + 'ucrt': "-ucrt-mingw32", } call_abi_mapping = { 'blank_call_abi': "", @@ -88,7 +90,10 @@ def r(x): x = x.replace("blank_call_abi", "") x = x.replace("blank_libgfortran", "") x = x.replace("blank_cxx_abi", "") - x = x.replace("blank_libc", "") + # We combine platform and libc below, since the windows mapping + # needs to know the platform for the correct default libc, so + # replace this one with `-` included. + x = x.replace("-blank_libc", "") return x def p(x): @@ -96,6 +101,8 @@ def p(x): # capture group names, unfortunately: os_remapping = { 'darwin': 'apple-darwin', + 'windows-msvcrt': 'w64-mingw32', + 'windows-ucrt': 'w64-ucrt-mingw32', 'windows': 'w64-mingw32', 'freebsd': 'unknown-freebsd', 'openbsd': 'unknown-openbsd', @@ -141,7 +148,7 @@ def p(x): "": "", }[sys.argv[3]] -print(arch+p(platform)+p(libc)+r(call_abi)+p(libgfortran_version)+p(cxx_abi)) +print(arch+p(platform+"-"+libc)+r(call_abi)+p(libgfortran_version)+p(cxx_abi)) # Testing suite: # triplets="i686-w64-mingw32 x86_64-pc-linux-musl arm-linux-musleabihf x86_64-linux-gnu arm-linux-gnueabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1" diff --git a/contrib/pgo-lto-bolt/.gitignore b/contrib/optimized/.gitignore similarity index 51% rename from contrib/pgo-lto-bolt/.gitignore rename to contrib/optimized/.gitignore index 1b29279acc0da..30c884ad131c7 100644 --- a/contrib/pgo-lto-bolt/.gitignore +++ b/contrib/optimized/.gitignore @@ -1,10 +1,11 @@ -stage0* -stage1* -stage2* +stage0 +stage1 +stage2 bolt -bolt_instrument -merge_data -copy_originals +bolt-originals +bolt-instrument +bolt-train +bolt-merge profiles profiles-bolt diff --git a/contrib/optimized/Makefile b/contrib/optimized/Makefile new file mode 100644 index 0000000000000..6086e081b6265 --- /dev/null +++ b/contrib/optimized/Makefile @@ -0,0 +1,422 @@ +.PHONY: all clean clean-profiles clean-pgo-profiles clean-bolt-profiles restore-originals delete-originals \ + print-profile-artifacts top + +# Abort recipes on the first failed command, including inside loops. +.SHELLFLAGS := -ec + +STAGE0_BUILD:=$(CURDIR)/toolchain +STAGE1_BUILD:=$(CURDIR)/pgo-instrumented.build +STAGE2_BUILD:=$(CURDIR)/optimized.build + +STAGE0_BINARIES:=$(STAGE0_BUILD)/usr/bin/ +STAGE0_TOOLS:=$(STAGE0_BUILD)/usr/tools/ + +BOLT_PROFILE_DIR:=$(CURDIR)/profiles-bolt +PGO_PROFILE_DIR:=$(CURDIR)/profiles +PGO_PROFILE_FILE:=$(PGO_PROFILE_DIR)/merged.prof +PGO_PROFRAW_FILES:=$(wildcard $(PGO_PROFILE_DIR)/*.profraw) +BOLT_PROFRAW_FILES:=$(wildcard $(BOLT_PROFILE_DIR)/*-prof.[0-9]*.fdata) +JULIA_ROOT:=$(CURDIR)/../.. +CONTRIB_DIR_REL:=contrib/$(notdir $(CURDIR)) + +# Stage 1 only exists to profile the compiler, so it does not need a system +# image for every CPU target of a release build. Stage 2 uses whatever +# JULIA_CPU_TARGET the caller passes. +STAGE1_CPU_TARGET ?= generic + +# Ask the regular build system about the platform it would build for, so that +# the staged builds agree with it. BUILD_MACHINE is the target triple recorded +# in the system image (Sys.MACHINE); the stage-0 clang spells it differently +# than the default compiler (e.g. aarch64-apple-darwin vs arm64-apple-darwin), +# and a plain `make binary-dist` of the optimized build would then consider +# build_h.jl stale and rebuild the system image. +# Preserve command-line variables, but omit flags such as -n from the probe. +JULIA_BUILD_VARS:=$(shell MAKEFLAGS='-- $(subst ','"'"',$(MAKEOVERRIDES))' $(MAKE) -s --no-print-directory -C $(JULIA_ROOT) print-OS print-ARCH print-BINARY print-EXE print-BUILD_MACHINE) +HOST_OS:=$(patsubst OS=%,%,$(filter OS=%,$(JULIA_BUILD_VARS))) +HOST_ARCH:=$(patsubst ARCH=%,%,$(filter ARCH=%,$(JULIA_BUILD_VARS))) +HOST_BINARY:=$(patsubst BINARY=%,%,$(filter BINARY=%,$(JULIA_BUILD_VARS))) +EXE:=$(patsubst EXE=%,%,$(filter EXE=%,$(JULIA_BUILD_VARS))) +BUILD_MACHINE:=$(patsubst BUILD_MACHINE=%,%,$(filter BUILD_MACHINE=%,$(JULIA_BUILD_VARS))) +ifeq (,$(HOST_OS)) +$(error Could not determine the build platform: `$(MAKE) -C $(JULIA_ROOT) print-OS` \ + failed. Run it to see why; a XC_HOST without a matching OS is one way.) +endif + +# Which optimizations to apply. PGO and ThinLTO work wherever the stage-0 +# toolchain does. BOLT rewrites ELF binaries only, and only for x86-64 and +# AArch64, so it defaults to off everywhere else; set USE_BOLT explicitly to +# override. USE_PGO and USE_LTO are mostly there to bisect build problems. +USE_PGO ?= 1 +USE_LTO ?= 1 +ifeq ($(HOST_OS),Linux) +ifneq (,$(filter $(HOST_ARCH),x86_64 aarch64)) +USE_BOLT ?= 1 +endif +endif +USE_BOLT ?= 0 + +# Each concurrent ThinLTO backend holds a module and the PGO profile. +# Limit their number on 32-bit targets so wide hosts do not exhaust the +# linker's address space. Empty leaves the linker's default. +ifeq ($(HOST_BINARY),32) +LTO_JOBS ?= 8 +endif +LTO_JOBS ?= + +# Settings taken from https://github.com/rust-lang/rust/blob/master/src/tools/opt-dist/src/bolt.rs +BOLT_ARGS := +# Reorder basic blocks within functions +BOLT_ARGS += -reorder-blocks=ext-tsp +# Reorder functions within the binary +BOLT_ARGS += -reorder-functions=cdsort +# Split function code into hot and cold regions +BOLT_ARGS += -split-functions +# Split as many basic blocks as possible +BOLT_ARGS += -split-all-cold +# Move jump tables to a separate section (only supported on x86-64). +ifeq ($(HOST_ARCH),x86_64) +BOLT_ARGS += -jump-tables=move +endif +# Use regular size pages for code alignment +BOLT_ARGS += -no-huge-pages +# Preserve function addresses, including those obtained through dlsym. +BOLT_ARGS += -icf=none +# Cache-directed splitting is incompatible with AArch64's default code model. +# On x86-64, apply it below except for libjulia-internal until +# https://github.com/llvm/llvm-project/issues/89508 is fixed. +ifeq ($(HOST_ARCH),x86_64) +BOLT_SPLIT_STRATEGY := -split-strategy=cdsplit +endif +# Update DWARF debug info in the final binary +BOLT_ARGS += -update-debug-sections +# Print optimization statistics +BOLT_ARGS += -dyno-stats +# BOLT doesn't fully support computed gotos, https://github.com/llvm/llvm-project/issues/89117 +# Use escaped regex as the name BOLT recognises is often a bit different, e.g. apply_cl/1(*2) +# This doesn't actually seem to do anything, the actual mitigation is not using --use-old-text +# which we do in the bolt target +BOLT_ARGS += -skip-funcs=.\*apply_cl.\* + +# BOLT needs relocations in the libraries it rewrites. Only pass this when BOLT +# is enabled: --emit-relocs is an ELF linker flag. +# -fno-reorder-blocks-and-partition is needed on gcc >= 8. +ifeq ($(USE_BOLT),1) +BOLT_FLAGS := $\ + "BOLT_CFLAGS_GCC+=-fno-reorder-blocks-and-partition" $\ + "BOLT_LDFLAGS=-Wl,--emit-relocs" +else +BOLT_FLAGS := +endif + +LLVM_BOLT:=$(STAGE0_BINARIES)llvm-bolt +LLVM_MERGEFDATA:=$(STAGE0_BINARIES)merge-fdata +LLVM_CXXFILT:=$(STAGE0_TOOLS)llvm-cxxfilt +LLVM_PROFDATA:=$(STAGE0_TOOLS)llvm-profdata +LLVM_OBJCOPY:=$(STAGE0_TOOLS)llvm-objcopy + +ifeq ($(USE_BOLT),1) +# Stop stage 2 after the libraries that BOLT rewrites, so that building the +# system image and the stdlib package images afterwards exercises the +# instrumented libraries and produces the BOLT profile. +STAGE2_TARGETS := julia-src-release julia-symlink julia-libccalltest \ + julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest +# If you add new files to optimize, you need to add BOLT_LDFLAGS and BOLT_CFLAGS to the build of your new file. +# Resolve these symlinks in the recipes, after stage 2 creates them. +SYMLINKS_TO_OPTIMIZE := libLLVM.so libjulia-internal.so libjulia-codegen.so +else +# Without BOLT, stage 2 is the complete build. +STAGE2_TARGETS := +SYMLINKS_TO_OPTIMIZE := +endif + +AFTER_INSTRUMENT_MESSAGE:='Run `make bolt-train` to build the rest of Julia with the $\ + instrumented libraries. You can then optionally collect more profiling data by running Julia $\ + with an appropriate workload; if you wish, run `make clean-bolt-profiles` before doing so to remove $\ + any profiling data generated by `make bolt-train`. You should end up with some data in $\ + $(BOLT_PROFILE_DIR). Afterwards run `make bolt`.' + +# When building a single libLLVM.so we need to increase -vp-counters-per-site +# significantly +COUNTERS_PER_SITE:=6 +# Note: profile counters are not atomic by default, https://discourse.llvm.org/t/profile-guided-optimization-pgo-related-questions-and-suggestions/75232/5 + +ifeq ($(USE_LTO),1) +LTO_FLAGS:=-flto=thin +else +LTO_FLAGS:= +endif + +ifeq ($(USE_PGO),1) +PGO_GENERATE_FLAGS:=-fprofile-generate=$(PGO_PROFILE_DIR) +PGO_COUNTER_FLAGS:=-Xclang -mllvm -Xclang -vp-counters-per-site=$(COUNTERS_PER_SITE) +# Configure probes can collide with profiled functions such as main. Suppress +# missing/stale-profile warnings so CMake's -Werror checks test the requested +# option, not the profile's coverage. Otherwise LLVM can silently drop -fPIC +# and -fno-semantic-interposition after a profile hash mismatch. +PGO_USE_FLAGS:=-fprofile-use=$(PGO_PROFILE_FILE) +PGO_WARNING_FLAGS:=-Wno-backend-plugin -Wno-profile-instr-unprofiled $\ + -Wno-profile-instr-out-of-date +else +PGO_GENERATE_FLAGS:= +PGO_COUNTER_FLAGS:= +PGO_USE_FLAGS:= +PGO_WARNING_FLAGS:= +endif + +AFTER_STAGE1_MESSAGE:='You can now optionally collect more profiling data for use in PGO by running Julia $\ + with an appropriate workload. If you wish, run `make clean-pgo-profiles` before doing so to remove any profiling data $\ + generated by building Julia. The profiles are written to $(PGO_PROFILE_DIR). $\ + Note that running extensive scripts may result in counter overflows, which can be detected by running $\ + `make top`. Afterwards run `make stage2`.' + +# Native programs need Windows paths in LLVM_PROFILE_FILE. Keep compiler +# arguments in MSYS2 form so its argument conversion does not mangle them. +ifeq ($(HOST_OS),WINNT) +native_path = $(shell cygpath -m '$(1)') +else +native_path = $(1) +endif + +# Platform hooks for toolchain setup, stage flags and DLL lookup. +STAGE0_EXTRA_TARGETS:= +STAGE0_EXTRA_FLAGS:= +# Turn [cd]tors into init/fini_array sections in libclang_rt, since lld doesn't +# do that, and otherwise the profile constructor is not executed. +STAGE0_POST = find $(STAGE0_BUILD) -name 'libclang_rt.profile-*.a' -exec $(LLVM_OBJCOPY) --rename-section .ctors=.init_array --rename-section .dtors=.fini_array {} + +STAGE1_PLATFORM_FLAGS:= +# How a stage's own shared libraries are made findable by the tools built +# during it. 1: that stage's build directory. +STAGE_PATH = + +# How to drive the toolchain on this platform. Everything else below is +# platform independent. +ifeq ($(HOST_OS),Darwin) +# The stage-0 clang does not come with Xcode, so tell it where the SDK is. +export SDKROOT ?= $(shell xcrun --show-sdk-path) +# Link with Xcode's ld64: the stage-0 ld64.lld cannot always read the text stubs +# of newer SDKs. ld64 has to use the stage-0 libLTO, since Xcode's cannot read +# the stage-0 bitcode, and clang only passes it along when it knows the linker +# version. Select ld64 explicitly for links driven by clang as well as make. +LINKER:=$(shell xcrun --find ld) +LINKER_FLAGS:=--ld-path=$(LINKER) -Wl,-lto_library,$(STAGE0_BUILD)/usr/lib/libLTO.dylib +LINK_ONLY_FLAGS:= +# For the same reason, LLVM's static libraries cannot be created with Xcode's +# libtool, which does not index the stage-0 bitcode. +PLATFORM_FLAGS:="LLVM_LIBTOOL=$(STAGE0_TOOLS)llvm-libtool-darwin" +else ifeq ($(HOST_OS),WINNT) +# Julia invokes LD directly; ld.lld selects lld's MinGW driver. +LINKER:=$(STAGE0_TOOLS)ld.lld$(EXE) +# MSYS2's GCC links winpthread through its spec file, which is what its libstdc++ +# expects; clang only does so for -pthread. +LINKER_FLAGS:=-fuse-ld=lld -pthread +# The ELF-only flags do not apply to PE/COFF. +LINK_ONLY_FLAGS:= +# Apply the export map through a COFF definition for lld's MinGW driver. +# Keep the caller's linker flags, but not the driver flags added by this flow. +PLATFORM_FLAGS:="WIN_LD_USE_DEF=1" "LINK_LDFLAGS=$(LDFLAGS)" +# BinaryBuilder's tools need its compiler support libraries (MSYS2's +# libstdc++ lacks some of their imports), plus libLLVM's compression DLLs. +# Place these beside the tools before running any of them. Skip the ELF +# constructor fixup: COFF's profile runtime registers through .CRT$XCU. +STAGE0_EXTRA_TARGETS:=install-zlib install-csl +STAGE0_EXTRA_FLAGS:=USE_BINARYBUILDER_CSL=1 +STAGE0_POST = cp $(STAGE0_BUILD)/usr/bin/*.dll $(STAGE0_TOOLS) +# Those direct linker invocations also miss the profile runtime that +# -fprofile-generate would otherwise have contributed to the link. +STAGE1_PLATFORM_FLAGS = "WIN_LD_EXTRA_LIBS=$(wildcard $(STAGE0_BUILD)/usr/lib/clang/*/lib/windows/libclang_rt.profile-*.a)" +# Let newly built tools such as llvm-tblgen find the stage's zstd/zlib DLLs. +# Append so the sysroot's compiler support libraries retain PATH precedence. +STAGE_PATH = PATH="$$PATH:$(1)/usr/bin" +else +LINKER:=$(STAGE0_TOOLS)ld.lld$(EXE) +LINKER_FLAGS:=-fuse-ld=lld +# Julia's version script names symbols that not every configuration defines. +LINK_ONLY_FLAGS:=-Wl,--undefined-version +ifneq (,$(LTO_JOBS)) +LINK_ONLY_FLAGS+=-Wl,--thinlto-jobs=$(LTO_JOBS) +endif +PLATFORM_FLAGS:= +endif + +TOOLCHAIN_FLAGS = $\ + "CC=$(STAGE0_TOOLS)clang$(EXE)" $\ + "BUILD_MACHINE=$(BUILD_MACHINE)" $\ + "CXX=$(STAGE0_TOOLS)clang++$(EXE)" $\ + "LD=$(LINKER)" $\ + "AR=$(STAGE0_TOOLS)llvm-ar$(EXE)" $\ + "RANLIB=$(STAGE0_TOOLS)llvm-ranlib$(EXE)" $\ + "CFLAGS+=$(STAGE_CFLAGS)" $\ + "CXXFLAGS+=$(STAGE_CXXFLAGS)" $\ + "LDFLAGS+=$(LINKER_FLAGS) $(LINK_ONLY_FLAGS) $(STAGE_LDFLAGS)" $\ + $(PLATFORM_FLAGS) + +# Profile data worth keeping, as globs relative to the Julia source root; CI +# archives these so that a build can be reproduced or a regression bisected. +PROFILE_ARTIFACTS := +ifeq ($(USE_PGO),1) +PROFILE_ARTIFACTS += $(CONTRIB_DIR_REL)/$(notdir $(PGO_PROFILE_DIR))/$(notdir $(PGO_PROFILE_FILE)) +endif +ifeq ($(USE_BOLT),1) +PROFILE_ARTIFACTS += $(CONTRIB_DIR_REL)/$(notdir $(BOLT_PROFILE_DIR))/*.merged.fdata +endif + +# The whole flow, in one invocation. Every step below is reachable on its own +# too, e.g. to collect extra profile data between stages. The stages announce +# themselves with a line that buildkite renders as a collapsible log section. +ifeq ($(USE_BOLT),1) +all: bolt +else +all: stage2 +endif + +$(STAGE0_BUILD) $(STAGE1_BUILD) $(STAGE2_BUILD): + $(MAKE) -C $(JULIA_ROOT) O=$@ configure + +# Stage 0: download the toolchain. +stage0: export USE_BINARYBUILDER_LLVM=1 +stage0: | $(STAGE0_BUILD) + @echo "--- Stage 0: toolchain" + # The toolchain links against zstd, which macOS does not provide. + $(MAKE) -C $(STAGE0_BUILD)/deps $(STAGE0_EXTRA_FLAGS) install-clang install-llvm install-lld install-llvm-tools install-zstd $(STAGE0_EXTRA_TARGETS) +ifeq ($(USE_BOLT),1) + $(MAKE) -C $(STAGE0_BUILD)/deps install-BOLT +endif + $(STAGE0_POST) + touch $@ + +# Stage 1: an instrumented Julia. Building its system image and package images +# is the workload that the PGO profile is collected from. +stage1: STAGE_CFLAGS:=$(PGO_GENERATE_FLAGS) $(PGO_COUNTER_FLAGS) +stage1: STAGE_CXXFLAGS:=$(PGO_GENERATE_FLAGS) $(PGO_COUNTER_FLAGS) +stage1: STAGE_LDFLAGS:=$(LTO_FLAGS) $(PGO_GENERATE_FLAGS) +stage1: export USE_BINARYBUILDER_LLVM=0 +stage1: stage0 | $(STAGE1_BUILD) + @echo "--- Stage 1: instrumented build" + # Building the dependencies runs instrumented build tools, e.g. llvm-tblgen, + # which share code with libLLVM but not Julia's workload. Keep their + # profiles out of the profile directory. + $(call STAGE_PATH,$(STAGE1_BUILD)) LLVM_PROFILE_FILE=$(call native_path,$(STAGE1_BUILD))/deps-profiles/%m.profraw $(MAKE) -C $(STAGE1_BUILD) $(TOOLCHAIN_FLAGS) $(STAGE1_PLATFORM_FLAGS) JULIA_CPU_TARGET=$(STAGE1_CPU_TARGET) julia-deps + $(call STAGE_PATH,$(STAGE1_BUILD)) $(MAKE) -C $(STAGE1_BUILD) $(TOOLCHAIN_FLAGS) $(STAGE1_PLATFORM_FLAGS) JULIA_CPU_TARGET=$(STAGE1_CPU_TARGET) + # Remove the empty profile emitted by the instrumented launcher. + rm -f $(JULIA_ROOT)/base/default.profraw + touch $@ + @echo $(AFTER_STAGE1_MESSAGE) + +# Merge whatever profiles are around. The glob is expanded by the shell, not by +# make, so that this also works in the same invocation that runs stage 1. +$(PGO_PROFILE_FILE): stage1 $(PGO_PROFRAW_FILES) + $(LLVM_PROFDATA) merge -output=$@ $(PGO_PROFILE_DIR)/*.profraw + +# show top 50 functions +top: $(PGO_PROFILE_FILE) + $(LLVM_PROFDATA) show --topn=50 $< | $(LLVM_CXXFILT) + +# Stage 2: the optimized Julia. With BOLT this stops after the libraries that +# get rewritten (see STAGE2_TARGETS); bolt-train builds the rest. +# +# Emit bitcode and ensure configure checks link it with the stage-0 LTO support. +# The linker flags only matter when linking, so compile-only invocations need +# -Qunused-arguments or -Werror'd compile probes fail (e.g. LLVM's -fPIC check, +# which would silently disable PIC and break the libLLVM link). +stage2 bolt-train: STAGE_CFLAGS:=$(PGO_USE_FLAGS) $(PGO_WARNING_FLAGS) $(LTO_FLAGS) $(LINKER_FLAGS) -Qunused-arguments +stage2 bolt-train: STAGE_CXXFLAGS:=$(PGO_USE_FLAGS) $(PGO_WARNING_FLAGS) $(LTO_FLAGS) $(LINKER_FLAGS) -Qunused-arguments +stage2 bolt-train: STAGE_LDFLAGS:=$(LTO_FLAGS) $(PGO_USE_FLAGS) +stage2 bolt-train: export USE_BINARYBUILDER_LLVM=0 +ifeq ($(USE_PGO),1) +stage2: $(PGO_PROFILE_FILE) +else +stage2: stage0 +endif +stage2: | $(STAGE2_BUILD) + @echo "--- Stage 2: optimized build" + $(call STAGE_PATH,$(STAGE2_BUILD)) $(MAKE) -C $(STAGE2_BUILD) $(TOOLCHAIN_FLAGS) $(BOLT_FLAGS) $(STAGE2_TARGETS) + touch $@ + +# Keep a copy of the libraries as linked, since both the instrumentation and +# the final rewrite below start from them. +bolt-originals: stage2 + for link in $(SYMLINKS_TO_OPTIMIZE); do \ + file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \ + abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ + cp -a $$abs_file "$$abs_file.original"; \ + done + touch $@ + +# Match the page alignment of the final rewrite. Preserve library mtimes so +# that make does not rebuild their dependents solely due to instrumentation. +bolt-instrument: bolt-originals + @echo "--- BOLT: instrument" + mkdir -p $(BOLT_PROFILE_DIR) + for link in $(SYMLINKS_TO_OPTIMIZE); do \ + file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \ + abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ + $(LLVM_BOLT) "$$abs_file.original" -o $$abs_file --instrument --instrumentation-file-append-pid --instrumentation-file="$(BOLT_PROFILE_DIR)/$$file-prof" -no-huge-pages; \ + touch -r "$$abs_file.original" $$abs_file; \ + printf "\n"; \ + done + touch $@ + @echo $(AFTER_INSTRUMENT_MESSAGE) + +# Build the rest of Julia against the instrumented libraries: the system image +# and the stdlib package images are the BOLT training workload. +bolt-train: bolt-instrument + @echo "--- BOLT: training build" + # Keep $(BOLT_FLAGS) here: anything relinked during this stage must carry + # --emit-relocs, or the later BOLT rewrite of it fails. + $(call STAGE_PATH,$(STAGE2_BUILD)) $(MAKE) -C $(STAGE2_BUILD) $(TOOLCHAIN_FLAGS) $(BOLT_FLAGS) + touch $@ + +bolt-merge: bolt-train $(BOLT_PROFRAW_FILES) + for link in $(SYMLINKS_TO_OPTIMIZE); do \ + file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \ + $(LLVM_MERGEFDATA) "$(BOLT_PROFILE_DIR)/$$file-prof."[0-9]*.fdata > "$(BOLT_PROFILE_DIR)/$$file-prof.merged.fdata"; \ + done + touch $@ + +# Reuse old text segments to reduce library size, except for libjulia-internal: +# BOLT does not fully support its computed gotos (llvm/llvm-project#89117). +# Preserve mtimes so the rewrite does not trigger dependent builds. +bolt: bolt-merge + @echo "--- BOLT: rewrite the libraries" + for link in $(SYMLINKS_TO_OPTIMIZE); do \ + file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \ + abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ + if [ "$$link" = libjulia-internal.so ]; then extra_args=; else extra_args="--use-old-text $(BOLT_SPLIT_STRATEGY)"; fi; \ + $(LLVM_BOLT) "$$abs_file.original" -data "$(BOLT_PROFILE_DIR)/$$file-prof.merged.fdata" -o $$abs_file $(BOLT_ARGS) $$extra_args; \ + touch -r "$$abs_file.original" $$abs_file; \ + done + touch $@ + +# The pre-BOLT copies are only needed to redo a rewrite; drop them before +# packaging the build, or restore them to undo the rewrite. +restore-originals: + for link in $(SYMLINKS_TO_OPTIMIZE); do \ + file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \ + abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ + cp -pP "$$abs_file.original" $$abs_file; \ + done + rm -f bolt + +delete-originals: + for link in $(SYMLINKS_TO_OPTIMIZE); do \ + file=$$(readlink $(STAGE2_BUILD)/usr/lib/$$link); \ + rm -f "$(STAGE2_BUILD)/usr/lib/$$file.original"; \ + done + +print-profile-artifacts: + @echo '$(PROFILE_ARTIFACTS)' + +clean-profiles: clean-pgo-profiles clean-bolt-profiles + +clean-pgo-profiles: + rm -rf $(PGO_PROFILE_DIR) + +clean-bolt-profiles: + rm -rf $(BOLT_PROFILE_DIR) + mkdir -p $(BOLT_PROFILE_DIR) + rm -f bolt-merge bolt + +# Only the stamps: this makes the next build redo the stages incrementally. +clean: + rm -f stage0 stage1 stage2 bolt bolt-originals bolt-instrument bolt-train bolt-merge + rm -f $(PGO_PROFILE_FILE) diff --git a/contrib/optimized/README.md b/contrib/optimized/README.md new file mode 100644 index 0000000000000..51462526ff11a --- /dev/null +++ b/contrib/optimized/README.md @@ -0,0 +1,96 @@ +# Optimized Julia builds + +This directory builds Julia with profile-guided optimization (PGO), ThinLTO and, +where supported, BOLT. It replaces `contrib/pgo-lto`, `contrib/bolt` and +`contrib/pgo-lto-bolt`. + +```sh +make -C contrib/optimized -j8 all +``` + +The result is in `contrib/optimized/optimized.build`. The build first downloads +clang, lld, LLVM tools and (when enabled) BOLT from BinaryBuilder. It then +builds an instrumented Julia, using the system image and package image builds as +the PGO workload, and builds Julia again with the resulting +profile and ThinLTO. With BOLT enabled, it also instruments `libLLVM`, +`libjulia-internal` and `libjulia-codegen`, builds the system image and package +images against them, and rewrites the libraries using that profile. + +## Options + +| Variable | Default | Meaning | +| --- | --- | --- | +| `USE_BOLT` | 1 on Linux x86-64 and AArch64, else 0 | Run the BOLT stages | +| `USE_PGO` | 1 | Build and profile stage 1, then optimize with it | +| `USE_LTO` | 1 | Build stage 2 with ThinLTO | +| `LTO_JOBS` | 8 on 32-bit targets, else the linker's default | ThinLTO backend threads per link (Linux only) | +| `STAGE1_CPU_TARGET` | `generic` | CPU target of the instrumented build | +| `STAGE0_BUILD` | `$(CURDIR)/toolchain` | Toolchain build directory | +| `STAGE1_BUILD` | `$(CURDIR)/pgo-instrumented.build` | Instrumented build directory | +| `STAGE2_BUILD` | `$(CURDIR)/optimized.build` | Optimized build directory | + +For PGO+LTO alone, set `USE_BOLT=0`. For BOLT alone, set `USE_PGO=0 USE_LTO=0`; +this skips stage 1. Command-line build variables are passed to the staged builds, +including `JULIA_CPU_TARGET` for stage 2. Use absolute paths when overriding the +build directories. `STAGE2_BUILD` can also point at the source checkout for +packaging there. `make print-profile-artifacts` prints the profile globs to +archive, relative to the source root. + +Stages use stamp files to resume completed builds. They do not track changes to +options: use fresh build directories and remove the stamps with `make clean` +when changing optimization settings. `make clean` only removes stamps and the +merged PGO profile; it does not clean compiled objects or undo a BOLT rewrite. + +## Custom workloads + +Run the stages separately to collect additional profile data. From this directory, +with PGO enabled: + +```sh +make stage1 +# Optional: make clean-pgo-profiles to discard the build's PGO workload. +./pgo-instrumented.build/julia my-workload.jl +make top # inspect the merged PGO profile +make stage2 +``` + +If BOLT is enabled, stage 2 stops after building the libraries. Complete the +training build before running your own workload: + +```sh +make bolt-train +# Optional: make clean-bolt-profiles to discard the build's BOLT workload. +./optimized.build/julia my-workload.jl +make bolt # merge profiles and rewrite the libraries +``` + +Collect BOLT profiles before the final rewrite, while the libraries are still +instrumented. The separate cleanup targets preserve the other optimization's +profiles; `make clean-profiles` clears both. New or updated raw profiles cause +the corresponding merge to run again. + +`make restore-originals` restores the libraries saved before BOLT and preserves +their mtimes. A subsequent `make bolt` reapplies the rewrite. Once satisfied, +`make delete-originals` removes those saved copies before packaging; restoring +or rewriting again then requires rebuilding the libraries. + +## Platforms + +The flow supports PGO+ThinLTO on macOS and Windows x86-64. On Windows it uses +BinaryBuilder's Clang and lld's MinGW driver with an MSYS2 mingw64 sysroot. +Stage 0 places the matching support DLLs beside the tools; the instrumented +stage supplies Clang's profile runtime to Julia's direct linker invocations. +The runtime export map is applied through a COFF export definition. A discovery +link identifies exports from the objects and archive members actually used; +the final link exports the matching names and explicitly exported symbols. + +Windows i686 optimized builds are not supported. Its native BinaryBuilder +toolchain is 32-bit, making address space a constraint for ThinLTO links of +libLLVM; supporting it would require a separate cross-toolchain setup. +BOLT defaults to Linux x86-64 and AArch64, where it can rewrite ELF libraries. +It cannot rewrite Windows PE/COFF binaries. An optimization setting does not +replace validation on the target platform. + +Do not strip shared libraries rewritten by BOLT; see +. The rewrite uses regular page +alignment, as does the normal Julia build. diff --git a/contrib/optimized/test_makefile.py b/contrib/optimized/test_makefile.py new file mode 100644 index 0000000000000..c864fab11ff7d --- /dev/null +++ b/contrib/optimized/test_makefile.py @@ -0,0 +1,237 @@ +"""Check stage ordering and flags with mocked builds and LLVM tools. + +Run with python3 contrib/optimized/test_makefile.py. No Julia build is started. +""" + +import itertools +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +SOURCE = Path(__file__).with_name("Makefile") +HELPER = r''' +import json, os, pathlib, sys +root = pathlib.Path(__file__).parent +args = sys.argv[1:] +if args[0] == 'build': + args.append({k: os.getenv(k) for k in ['PATH', 'LLVM_PROFILE_FILE']}) +with (root / 'events').open('a') as f: + f.write(json.dumps([pathlib.Path.cwd().name, args]) + '\n') +if args[0] == 'configure': + p = pathlib.Path(args[1]) + if p.exists(): + sys.exit('configure called on existing directory') + (p / 'deps').mkdir(parents=True) + recipe = 'all julia-deps julia-src-release julia-symlink julia-libccalltest julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest:\n\t@python3 ' + str(root / 'helper.py') + ' build $@ "$(CFLAGS)" "$(LDFLAGS)" "$(JULIA_CPU_TARGET)" "$(USE_BINARYBUILDER_LLVM)" "$(LD)" "$(WIN_LD_USE_DEF)" "$(WIN_LD_EXTRA_LIBS)" "$(LINK_LDFLAGS)"\n' + (p / 'Makefile').write_text(recipe) + (p / 'deps/Makefile').write_text('%:\n\t@python3 ' + str(root / 'helper.py') + ' install $@ "$(USE_BINARYBUILDER_LLVM)" "$(OS)" "$(USE_BINARYBUILDER_CSL)"\n') +elif args[0] == 'install' and args[3] == 'WINNT': + stage = pathlib.Path.cwd().parent + (stage / 'usr/bin').mkdir(parents=True, exist_ok=True) + (stage / 'usr/tools').mkdir(parents=True, exist_ok=True) + (stage / 'usr/bin/support.dll').write_text('support') + runtime = stage / 'usr/lib/clang/22/lib/windows' + runtime.mkdir(parents=True, exist_ok=True) + (runtime / 'libclang_rt.profile-x86_64.a').write_text('runtime') +elif args[0] == 'build': + p = pathlib.Path.cwd() + if p.name == 'pgo-instrumented.build' and args[1] == 'all': + (root / 'contrib/optimized/profiles').mkdir(exist_ok=True) + (root / 'contrib/optimized/profiles/test.profraw').write_text('profile') + elif (p.name == 'optimized.build' or p == root) and args[1] in ['all', 'julia-src-release']: + lib = p / 'usr/lib' + lib.mkdir(parents=True, exist_ok=True) + for name in ['libLLVM.so', 'libjulia-internal.so', 'libjulia-codegen.so']: + if not (lib / name).exists(): + (lib / (name + '.1')).write_text('original') + (lib / name).symlink_to(name + '.1') + if args[1] == 'all' and (root / 'contrib/optimized/bolt-instrument').exists(): + (root / ('contrib/optimized/profiles-bolt/' + name + '.1-prof.123.fdata')).write_text('raw\n') +elif args[0] == 'merge': + assert args[-1].endswith('.profraw') and pathlib.Path(args[-1]).exists(), args + pathlib.Path(args[1].split('=', 1)[1]).write_text('merged') +elif args[0] == 'fdata': + assert all('.merged.' not in x and pathlib.Path(x).is_file() for x in args[1:]), args + print(''.join(pathlib.Path(x).read_text() for x in args[1:]), end='') +elif args[0] == 'bolt': + if os.getenv('FAIL_BOLT'): + sys.exit(1) + assert pathlib.Path(args[1]).read_text() == 'original' + pathlib.Path(args[args.index('-o') + 1]).write_text('instrumented' if '--instrument' in args else 'optimized') +''' + +class FlowTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix='opt-unify-review-') + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.flow = self.root / 'contrib/optimized' + self.flow.mkdir(parents=True) + shutil.copyfile(SOURCE, self.flow / 'Makefile') + (self.root / 'helper.py').write_text(HELPER) + (self.root / 'Makefile').write_text('OS := Linux\nARCH := x86_64\nBINARY := 64\nBUILD_MACHINE := test-triple\nprint-%:\n\t@echo "$*=$($*)"\nconfigure:\n\t@python3 helper.py configure $(O)\n') + for name, mode in [('profdata', 'merge'), ('mergefdata', 'fdata'), ('bolt-tool', 'bolt')]: + p = self.root / name + # profdata already gets 'merge' as the first argument. + prefix = '' if name == 'profdata' else mode + ' ' + p.write_text('#!/bin/sh\nexec python3 ' + str(self.root / 'helper.py') + ' ' + prefix + '"$@"\n') + p.chmod(0o755) + self.args = ['LLVM_PROFDATA=' + str(self.root / 'profdata'), 'LLVM_MERGEFDATA=' + str(self.root / 'mergefdata'), 'LLVM_BOLT=' + str(self.root / 'bolt-tool')] + + def make(self, *args, ok=True, env=None): + p = subprocess.run(['make', '--no-print-directory', '-s', '-j4', '-C', str(self.flow), *self.args, *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) + if ok: + self.assertEqual(p.returncode, 0, p.stdout) + else: + self.assertNotEqual(p.returncode, 0, p.stdout) + return p.stdout + + def events(self): + return [json.loads(x) for x in (self.root / 'events').read_text().splitlines()] + + def test_matrix(self): + for pgo, lto, bolt in itertools.product('01', repeat=3): + with self.subTest(pgo=pgo, lto=lto, bolt=bolt): + # Each configuration needs its own outputs and stamps. + for child in self.flow.iterdir(): + if child.name != 'Makefile': + shutil.rmtree(child) if child.is_dir() else child.unlink() + (self.root / 'events').unlink(missing_ok=True) + knobs = [f'USE_PGO={pgo}', f'USE_LTO={lto}', f'USE_BOLT={bolt}', 'JULIA_CPU_TARGET=generic;haswell'] + self.make('all', *knobs) + events = self.events() + builds = [(d, a) for d, a in events if a[0] == 'build'] + self.assertEqual(any(d == 'pgo-instrumented.build' for d, a in builds), pgo == '1') + for d, a in builds: + if d == 'pgo-instrumented.build': + self.assertIn('-fprofile-generate', a[2]) + self.assertNotIn('-fprofile-use', a[2]) + self.assertEqual(a[4], 'generic') + else: + self.assertEqual('-fprofile-use' in a[2], pgo == '1') + self.assertEqual('-flto=thin' in a[2], lto == '1') + self.assertEqual(a[4], 'generic;haswell') + self.assertEqual(a[5], '0') + self.assertTrue(all(a[2] == '1' for d, a in events if a[0] == 'install')) + self.assertEqual(any(a[:2] == ['install', 'install-BOLT'] for d, a in events), bolt == '1') + if bolt == '1': + self.assertEqual((self.flow / 'optimized.build/usr/lib/libLLVM.so').read_text(), 'optimized') + self.assertEqual(len([a for d,a in events if a[0] == 'bolt']), 6) + count = len(events) + self.make('all', *knobs) + self.assertEqual(len(self.events()), count) + + def test_override_and_dry_run(self): + target = '--eval=review:;@echo $(HOST_OS) $(HOST_ARCH) $(USE_BOLT) $(BUILD_MACHINE) [$(LINK_ONLY_FLAGS)]' + for osname, arch, binary, bolt, jobs in [('Linux','i686','32','0','8'), ('Linux','aarch64','64','1',''), ('WINNT','x86_64','64','0',''), ('Darwin','aarch64','64','0','')]: + for dry in [[], ['-n']]: + out = self.make(*dry, target, 'review', 'OS='+osname, 'ARCH='+arch, 'BINARY='+binary, 'SDKROOT=/sdk', 'LINKER=/ld') + self.assertIn(f'{osname} {arch} {bolt} test-triple', out) + self.assertEqual('--thinlto-jobs=' + jobs in out, jobs != '') + out = self.make(target, 'review', 'OS=Linux', 'ARCH=i686', 'BINARY=32', 'LTO_JOBS=', 'LINKER=/ld') + self.assertNotIn('--thinlto-jobs', out) + + def test_windows_stages(self): + cygpath = self.root / 'cygpath' + cygpath.write_text('#!/bin/sh\n[ "$1" = -m ] || exit 1\nprintf "C:%s\\n" "$2"\n') + cygpath.chmod(0o755) + env = dict(os.environ, PATH=str(self.root) + os.pathsep + os.environ['PATH']) + self.make('all', 'OS=WINNT', 'EXE=.exe', 'LDFLAGS=--no-insert-timestamp', env=env) + events = self.events() + installs = [a for d, a in events if a[0] == 'install'] + self.assertTrue({'install-csl', 'install-zlib', 'install-zstd'} <= {a[1] for a in installs}) + self.assertTrue(all(a[4] == '1' for a in installs)) + self.assertFalse(any(a[1] == 'install-BOLT' for a in installs)) + stage0 = self.flow / 'toolchain' + self.assertEqual((stage0 / 'usr/tools/support.dll').read_text(), 'support') + for directory, args in events: + if args[0] != 'build': + continue + self.assertEqual(args[6], str(stage0 / 'usr/tools/ld.lld.exe')) + self.assertEqual(args[7], '1') + self.assertEqual(args[9], '--no-insert-timestamp') + self.assertIn('-pthread', args[3]) + self.assertNotIn('--undefined-version', args[3]) + self.assertNotIn('--emit-relocs', args[3]) + self.assertEqual(args[-1]['PATH'], env['PATH'] + ':' + str(self.flow / directory / 'usr/bin')) + if directory == 'pgo-instrumented.build': + self.assertEqual(args[8], str(stage0 / 'usr/lib/clang/22/lib/windows/libclang_rt.profile-x86_64.a')) + self.assertNotIn('-flto', args[2]) + if args[1] == 'julia-deps': + self.assertEqual(args[-1]['LLVM_PROFILE_FILE'], 'C:' + str(self.flow / directory / 'deps-profiles/%m.profraw')) + else: + self.assertEqual(args[8], '') + self.assertNotIn('-gline-tables-only', args[2]) + self.assertIn('-flto=thin', args[2]) + self.assertIn('-fprofile-use=', args[2]) + + def test_bolt_flags_per_architecture(self): + # cdsplit and the jump table mode are x86-only; BOLT rejects the former + # on AArch64 and ignores the latter. + target = '--eval=review:;@echo "[$(BOLT_ARGS)][$(BOLT_SPLIT_STRATEGY)]"' + for arch, x86_only in [('x86_64', True), ('aarch64', False)]: + out = self.make(target, 'review', 'OS=Linux', 'ARCH=' + arch) + self.assertEqual('-jump-tables=move' in out, x86_only, out) + self.assertEqual('-split-strategy=cdsplit' in out, x86_only, out) + self.assertIn('-reorder-blocks=ext-tsp', out) + + def test_custom_bolt_profiles(self): + self.make('bolt-train') + profile = self.flow / 'profiles/merged.prof' + mtime = profile.stat().st_mtime_ns + self.make('clean-bolt-profiles') + self.assertTrue(profile.exists()) + self.assertTrue((self.flow / 'profiles-bolt').is_dir()) + for name in ['libLLVM.so', 'libjulia-internal.so', 'libjulia-codegen.so']: + (self.flow / f'profiles-bolt/{name}.1-prof.456.fdata').write_text('custom\n') + self.make('bolt') + merged = self.flow / 'profiles-bolt/libLLVM.so.1-prof.merged.fdata' + self.assertEqual(merged.read_text(), 'custom\n') + new = self.flow / 'profiles-bolt/libLLVM.so.1-prof.789.fdata' + new.write_text('extra\n') + self.make('bolt') + self.assertEqual(merged.read_text(), 'custom\nextra\n') + self.assertEqual(profile.stat().st_mtime_ns, mtime) + + def test_missing_library_fails(self): + self.make('stage2') + (self.flow / 'optimized.build/usr/lib/libLLVM.so').unlink() + self.make('bolt-originals', ok=False) + self.assertFalse((self.flow / 'bolt-originals').exists()) + + def test_instrument_failure_stops_chain(self): + self.make('all', ok=False, env=dict(os.environ, FAIL_BOLT='1')) + self.assertFalse((self.flow / 'bolt-instrument').exists()) + self.assertFalse((self.flow / 'bolt-train').exists()) + + def test_restore_preserves_mtime_and_all_rewrites_again(self): + self.make('all') + lib = self.flow / 'optimized.build/usr/lib/libLLVM.so.1' + mtime = lib.stat().st_mtime_ns + self.make('restore-originals') + self.assertEqual(lib.read_text(), 'original') + self.assertEqual(lib.stat().st_mtime_ns, mtime) + self.make('all') + self.assertEqual(lib.read_text(), 'optimized') + self.assertEqual(lib.stat().st_mtime_ns, mtime) + + def test_source_checkout_as_stage2(self): + with (self.root / 'Makefile').open('a') as f: + f.write('.DEFAULT_GOAL := all\nall julia-src-release julia-symlink julia-libccalltest ' + 'julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest:\n' + '\t@python3 helper.py build $@ "$(CFLAGS)" "$(LDFLAGS)" ' + '"$(JULIA_CPU_TARGET)" "$(USE_BINARYBUILDER_LLVM)"\n') + self.make('all', 'STAGE2_BUILD=' + str(self.root)) + self.assertEqual((self.root / 'usr/lib/libLLVM.so').read_text(), 'optimized') + + def test_clean_does_not_reconfigure_existing_directories(self): + self.make('all', 'USE_BOLT=0') + self.make('clean') + self.make('all', 'USE_BOLT=0') + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/contrib/pgo-lto-bolt/Makefile b/contrib/pgo-lto-bolt/Makefile deleted file mode 100644 index ce1b8b04f68c9..0000000000000 --- a/contrib/pgo-lto-bolt/Makefile +++ /dev/null @@ -1,190 +0,0 @@ -.PHONY: clean clean_profiles restore_originals - -# See the makefiles in contrib/bolt and contrib/pgo-lto for more information. - -# Settings taken from https://github.com/rust-lang/rust/blob/master/src/tools/opt-dist/src/bolt.rs -BOLT_ARGS := -# Reorder basic blocks within functions -BOLT_ARGS += -reorder-blocks=ext-tsp -# Reorder functions within the binary -BOLT_ARGS += -reorder-functions=cdsort -# Split function code into hot and code regions -BOLT_ARGS += -split-functions -# Split as many basic blocks as possible -BOLT_ARGS += -split-all-cold -# Move jump tables to a separate section -BOLT_ARGS += -jump-tables=move -# Use regular size pages for code alignment -BOLT_ARGS += -no-huge-pages -# Fold functions with identical code -BOLT_ARGS += -icf=1 -# Split using best available strategy (three-way splitting, Cache-Directed Sort) -# Disabled for libjulia-internal till https://github.com/llvm/llvm-project/issues/89508 is fixed -# BOLT_ARGS += -split-strategy=cdsplit -# Update DWARF debug info in the final binary -BOLT_ARGS += -update-debug-sections -# Print optimization statistics -BOLT_ARGS += -dyno-stats -# BOLT doesn't fully support computed gotos, https://github.com/llvm/llvm-project/issues/89117 -# Use escaped regex as the name BOLT recognises is often a bit different, e.g. apply_cl/1(*2) -# This doesn't actually seem to do anything, the actual mitigation is not using --use-old-text -# which we do in the bolt target -BOLT_ARGS += -skip-funcs=.\*apply_cl.\* - -# -fno-reorder-blocks-and-partition is needed on gcc >= 8. -BOLT_FLAGS := $\ - "BOLT_CFLAGS_GCC+=-fno-reorder-blocks-and-partition" $\ - "BOLT_LDFLAGS=-Wl,--emit-relocs" - -STAGE0_BUILD:=$(CURDIR)/toolchain -STAGE1_BUILD:=$(CURDIR)/pgo-instrumented.build -STAGE2_BUILD:=$(CURDIR)/optimized.build - -STAGE0_BINARIES:=$(STAGE0_BUILD)/usr/bin/ -STAGE0_TOOLS:=$(STAGE0_BUILD)/usr/tools/ - -BOLT_PROFILE_DIR:=$(CURDIR)/profiles-bolt -PGO_PROFILE_DIR:=$(CURDIR)/profiles -PGO_PROFILE_FILE:=$(PGO_PROFILE_DIR)/merged.prof -PGO_PROFRAW_FILES:=$(wildcard $(PGO_PROFILE_DIR)/*.profraw) -JULIA_ROOT:=$(CURDIR)/../.. - -LLVM_BOLT:=$(STAGE0_BINARIES)llvm-bolt -LLVM_MERGEFDATA:=$(STAGE0_BINARIES)merge-fdata -LLVM_CXXFILT:=$(STAGE0_TOOLS)llvm-cxxfilt -LLVM_PROFDATA:=$(STAGE0_TOOLS)llvm-profdata -LLVM_OBJCOPY:=$(STAGE0_TOOLS)llvm-objcopy - -# If you add new files to optimize, you need to add BOLT_LDFLAGS and BOLT_CFLAGS to the build of your new file. -SYMLINKS_TO_OPTIMIZE := libLLVM.so libjulia-internal.so libjulia-codegen.so -FILES_TO_OPTIMIZE := $(shell for file in $(SYMLINKS_TO_OPTIMIZE); do readlink $(STAGE1_BUILD)/usr/lib/$$file; done) - -AFTER_INSTRUMENT_MESSAGE:='Run `make finish_stage2` to finish off the build. $\ - You can now optionally collect more profiling data by running Julia with an appropriate workload, $\ - if you wish, run `make clean_profiles` before doing so to remove any profiling data generated by `make finish_stage2`. $\ - You should end up with some data in $(BOLT_PROFILE_DIR). Afterwards run `make merge_data && make bolt`.' - -# When building a single libLLVM.so we need to increase -vp-counters-per-site -# significantly -COUNTERS_PER_SITE:=6 -# Note: profile counters are not atomic by default, https://discourse.llvm.org/t/profile-guided-optimization-pgo-related-questions-and-suggestions/75232/5 - -AFTER_STAGE1_MESSAGE:='You can now optionally collect more profiling data for use in PGO by running Julia $\ - with an appropriate workload. If you wish, run `make clean_profiles` before doing so to remove any profiling data $\ - generated by building Julia. You should end up with about 15MB of data in $(PGO_PROFILE_DIR). $\ - Note that running extensive scripts may result in counter overflows, which can be detected by running $\ - `make top`. Afterwards run `make stage2`.' - -TOOLCHAIN_FLAGS = $\ - "CC=$(STAGE0_TOOLS)clang" $\ - "CXX=$(STAGE0_TOOLS)clang++" $\ - "LD=$(STAGE0_TOOLS)ld.lld" $\ - "AR=$(STAGE0_TOOLS)llvm-ar" $\ - "RANLIB=$(STAGE0_TOOLS)llvm-ranlib" $\ - "CFLAGS+=$(PGO_CFLAGS)" $\ - "CXXFLAGS+=$(PGO_CXXFLAGS)" $\ - "LDFLAGS+=-fuse-ld=lld $(PGO_LDFLAGS)" - -$(STAGE0_BUILD) $(STAGE1_BUILD) $(STAGE2_BUILD): - $(MAKE) -C $(JULIA_ROOT) O=$@ configure - -stage0: export USE_BINARYBUILDER_LLVM=1 -stage0: | $(STAGE0_BUILD) - # Turn [cd]tors into init/fini_array sections in libclang_rt, since lld - # doesn't do that, and otherwise the profile constructor is not executed - $(MAKE) -C $(STAGE0_BUILD)/deps install-clang install-llvm install-lld install-llvm-tools install-BOLT && \ - find $< -name 'libclang_rt.profile-*.a' -exec $(LLVM_OBJCOPY) --rename-section .ctors=.init_array --rename-section .dtors=.fini_array {} + && \ - touch $@ - -$(STAGE1_BUILD): stage0 -stage1: PGO_CFLAGS:=-fprofile-generate=$(PGO_PROFILE_DIR) -Xclang -mllvm -Xclang -vp-counters-per-site=$(COUNTERS_PER_SITE) -stage1: PGO_CXXFLAGS:=-fprofile-generate=$(PGO_PROFILE_DIR) -Xclang -mllvm -Xclang -vp-counters-per-site=$(COUNTERS_PER_SITE) -stage1: PGO_LDFLAGS:=-flto=thin -fprofile-generate=$(PGO_PROFILE_DIR) -stage1: export USE_BINARYBUILDER_LLVM=0 -stage1: | $(STAGE1_BUILD) - $(MAKE) -C $(STAGE1_BUILD) $(TOOLCHAIN_FLAGS) && touch $@ - @echo $(AFTER_STAGE1_MESSAGE) - -stage2: PGO_CFLAGS:=-fprofile-use=$(PGO_PROFILE_FILE) -stage2: PGO_CXXFLAGS:=-fprofile-use=$(PGO_PROFILE_FILE) -stage2: PGO_LDFLAGS:=-flto=thin -fprofile-use=$(PGO_PROFILE_FILE) -Wl,--icf=safe -stage2: export USE_BINARYBUILDER_LLVM=0 -stage2: $(PGO_PROFILE_FILE) | $(STAGE2_BUILD) - $(MAKE) -C $(STAGE2_BUILD) $(TOOLCHAIN_FLAGS) $(BOLT_FLAGS) julia-src-release julia-symlink julia-libccalltest \ - julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest && \ - touch $@ - -copy_originals: stage2 - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ - cp $$abs_file "$$abs_file.original"; \ - done && \ - touch $@ - -# I don't think there's any particular reason to have -no-huge-pages here, perhaps slightly more accurate profile data -# as the final build uses -no-huge-pages -# We reset the mtime of the files to prevent make from rebuilding targets depending on them. -bolt_instrument: copy_originals - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ - old_time=$$(stat -c %Y $$abs_file); \ - $(LLVM_BOLT) "$$abs_file.original" -o $$abs_file --instrument --instrumentation-file-append-pid --instrumentation-file="$(BOLT_PROFILE_DIR)/$$file-prof" -no-huge-pages; \ - mkdir -p $$(dirname "$(BOLT_PROFILE_DIR)/$$file-prof"); \ - touch -d "@$$old_time" $$abs_file; \ - printf "\n"; \ - done && \ - touch $@ - @echo $(AFTER_INSTRUMENT_MESSAGE) - -finish_stage2: PGO_CFLAGS:=-fprofile-use=$(PGO_PROFILE_FILE) -finish_stage2: PGO_CXXFLAGS:=-fprofile-use=$(PGO_PROFILE_FILE) -finish_stage2: PGO_LDFLAGS:=-flto=thin -fprofile-use=$(PGO_PROFILE_FILE) -Wl,--icf=safe -finish_stage2: stage2 - $(MAKE) -C $(STAGE2_BUILD) $(TOOLCHAIN_FLAGS) - -merge_data: bolt_instrument - for file in $(FILES_TO_OPTIMIZE); do \ - profiles=$(BOLT_PROFILE_DIR)/$$file-prof.*.fdata; \ - $(LLVM_MERGEFDATA) $$profiles > "$(BOLT_PROFILE_DIR)/$$file-prof.merged.fdata"; \ - done && \ - touch $@ - -# The --use-old-text saves about 16 MiB of libLLVM.so size. -# However, the rust folk found it succeeds very non-deterministically for them. -# It tries to reuse old text segments to reduce binary size -# BOLT doesn't fully support computed gotos https://github.com/llvm/llvm-project/issues/89117, so we cannot use --use-old-text on libjulia-internal -# That flag saves less than 1 MiB for libjulia-internal so oh well. -# We reset the mtime of the files to prevent make from rebuilding targets depending on them. -bolt: merge_data - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ - old_time=$$(stat -c %Y $$abs_file); \ - $(LLVM_BOLT) "$$abs_file.original" -data "$(BOLT_PROFILE_DIR)/$$file-prof.merged.fdata" -o $$abs_file $(BOLT_ARGS) $$(if [ "$$file" != $(shell readlink $(STAGE2_BUILD)/usr/lib/libjulia-internal.so) ]; then echo "--use-old-text -split-strategy=cdsplit"; fi); \ - touch -d "@$$old_time" $$abs_file; \ - done && \ - touch $@ - -clean_profiles: - rm -rf $(PGO_PROFILE_DIR) $(BOLT_PROFILE_DIR) - -clean: - rm -f stage0 stage1 stage2 $(PGO_PROFILE_FILE) bolt copy_originals merge_data bolt_instrument - -restore_originals: copy_originals - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ - cp -P "$$abs_file.original" $$abs_file; \ - done - -delete_originals: copy_originals - for file in $(FILES_TO_OPTIMIZE); do \ - abs_file=$(STAGE2_BUILD)/usr/lib/$$file; \ - rm "$$abs_file.original"; \ - done - -$(PGO_PROFILE_FILE): stage1 $(PGO_PROFRAW_FILES) - $(LLVM_PROFDATA) merge -output=$@ $(PGO_PROFRAW_FILES) - -# show top 50 functions -top: $(PGO_PROFILE_FILE) - $(LLVM_PROFDATA) show --topn=50 $< | $(LLVM_CXXFILT) diff --git a/contrib/pgo-lto-bolt/README.md b/contrib/pgo-lto-bolt/README.md deleted file mode 100644 index ab574907c292f..0000000000000 --- a/contrib/pgo-lto-bolt/README.md +++ /dev/null @@ -1,18 +0,0 @@ -BOLT only works on x86_64 and arch64 on Linux. - -DO NOT STRIP THE RESULTING .so FILES, https://github.com/llvm/llvm-project/issues/56738. -If you really need to, try adding `-use-gnu-stack` to `BOLT_ARGS`. - -To build a PGO+LTO+BOLT version of Julia run the following commands (`cd` into this directory first) -```bash -make stage1 -make stage2 -make copy_originals -make bolt_instrument -make finish_stage2 -make merge_data -make bolt -``` -After these commands finish, the optimized version of Julia will be built in the `optimized.build` directory. - -This doesn't align the code to support huge pages as it doesn't seem that we do that currently, this decreases the size of the .so files by 2-4mb. diff --git a/contrib/pgo-lto/.gitignore b/contrib/pgo-lto/.gitignore deleted file mode 100644 index 978d8f2ca86dd..0000000000000 --- a/contrib/pgo-lto/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -profiles -stage0* -stage1* -stage2* diff --git a/contrib/pgo-lto/Makefile b/contrib/pgo-lto/Makefile deleted file mode 100644 index 5902d4ad08151..0000000000000 --- a/contrib/pgo-lto/Makefile +++ /dev/null @@ -1,74 +0,0 @@ -.PHONY: top clean clean-profiles - -STAGE0_BUILD:=$(CURDIR)/stage0.build -STAGE1_BUILD:=$(CURDIR)/stage1.build -STAGE2_BUILD:=$(CURDIR)/stage2.build - -STAGE0_TOOLS:=$(STAGE0_BUILD)/usr/tools/ - -PROFILE_DIR:=$(CURDIR)/profiles -PROFILE_FILE:=$(PROFILE_DIR)/merged.prof -JULIA_ROOT:=$(CURDIR)/../.. - -LLVM_CXXFILT:=$(STAGE0_TOOLS)llvm-cxxfilt -LLVM_PROFDATA:=$(STAGE0_TOOLS)llvm-profdata -LLVM_OBJCOPY:=$(STAGE0_TOOLS)llvm-objcopy - -# When building a single libLLVM.so we need to increase -vp-counters-per-site -# significantly -COUNTERS_PER_SITE:=6 -# Note: profile counters are not atomic by default, https://discourse.llvm.org/t/profile-guided-optimization-pgo-related-questions-and-suggestions/75232/5 - -AFTER_STAGE1_MESSAGE:='You can now optionally collect more profiling data for use in PGO by running Julia $\ - with an appropriate workload. If you wish, run `make clean_profiles` before doing so to remove any profiling data $\ - generated by building Julia. You should end up with about 15MB of data in $(PGO_PROFILE_DIR). $\ - Note that running extensive scripts may result in counter overflows, which can be detected by running $\ - `make top`. Afterwards run `make stage2`.' - -STAGE1_FLAGS:=LDFLAGS="-fuse-ld=lld -flto=thin -Wl,--undefined-version -fprofile-generate=$(PROFILE_DIR)" $\ - CFLAGS="-fprofile-generate=$(PROFILE_DIR) -Xclang -mllvm -Xclang -vp-counters-per-site=$(COUNTERS_PER_SITE)" $\ - CXXFLAGS="-fprofile-generate=$(PROFILE_DIR) -Xclang -mllvm -Xclang -vp-counters-per-site=$(COUNTERS_PER_SITE)" -STAGE2_FLAGS:=LDFLAGS="-fuse-ld=lld -flto=thin -Wl,--undefined-version -fprofile-use=$(PROFILE_FILE) -Wl,--icf=safe" $\ - CFLAGS="-fprofile-use=$(PROFILE_FILE)" $\ - CXXFLAGS="-fprofile-use=$(PROFILE_FILE)" - -COMMON_FLAGS:=USE_BINARYBUILDER_LLVM=0 - -all: stage2 # Default target as first in file - -$(STAGE0_BUILD) $(STAGE1_BUILD) $(STAGE2_BUILD): - $(MAKE) -C $(JULIA_ROOT) O=$@ configure - -stage0: export USE_BINARYBUILDER_LLVM=1 -stage0: | $(STAGE0_BUILD) - # Turn [cd]tors into init/fini_array sections in libclang_rt, since lld - # doesn't do that, and otherwise the profile constructor is not executed - $(MAKE) -C $(STAGE0_BUILD)/deps install-clang install-llvm install-lld install-llvm-tools && \ - find $< -name 'libclang_rt.profile-*.a' -exec $(LLVM_OBJCOPY) --rename-section .ctors=.init_array --rename-section .dtors=.fini_array {} + && \ - touch $@ - -$(STAGE1_BUILD): stage0 -stage1: | $(STAGE1_BUILD) - @echo "--- Build Julia Stage 1 - with instrumentation" - PATH=$(STAGE0_TOOLS):$$PATH $(STAGE1_FLAGS) $(MAKE) -C $(STAGE1_BUILD) $(COMMON_FLAGS) && touch $@ - @echo $(AFTER_STAGE1_MESSAGE) - -stage2: $(PROFILE_FILE) | $(STAGE2_BUILD) - @echo "--- Build Julia Stage 2 - PGO + LTO optimised" - PATH=$(STAGE0_TOOLS):$$PATH $(STAGE2_FLAGS) $(MAKE) -C $(STAGE2_BUILD) $(COMMON_FLAGS) && touch $@ - -.DEFAULT: stage2 - PATH=$(STAGE0_TOOLS):$$PATH $(STAGE2_FLAGS) $(MAKE) -C $(STAGE2_BUILD) $(COMMON_FLAGS) $@ - -$(PROFILE_FILE): stage1 $(wildcard $(PROFILE_DIR)/*.profraw) - $(LLVM_PROFDATA) merge -output=$@ $(PROFILE_DIR)/*.profraw - -# show top 50 functions -top: $(PROFILE_FILE) - $(LLVM_PROFDATA) show --topn=50 $< | $(LLVM_CXXFILT) - -clean-profiles: - rm -rf $(PROFILE_DIR) - -clean: - rm -f stage0 stage1 stage2 $(PROFILE_FILE) diff --git a/contrib/prepare_release.sh b/contrib/prepare_release.sh deleted file mode 100755 index 2772e44a858f1..0000000000000 --- a/contrib/prepare_release.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/sh -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# script to prepare binaries and source tarballs for a Julia release -# aka "bucket dance" julianightlies -> julialang -set -e # stop on failure -cd "$(dirname "$0")"/.. # run in top-level directory - -shashort=$(git rev-parse --short=10 HEAD) -tag=$(git tag --points-at $shashort) -if [ -z "$tag" ]; then - echo "error: this script must be run with a tagged commit checked out" >&2 - exit 1 -fi -version=$(cat VERSION) -majmin=$(cut -d. -f1-2 VERSION) -# remove -rc# if present -majminpatch=$(cut -d- -f1 VERSION) -if [ "$tag" != "v$version" ]; then - echo "error: tagged commit does not match content of VERSION file" >&2 - exit 1 -fi - -# create full-source-dist and light-source-dist tarballs from a separate -# clone to ensure the directory name in them is julia-$version -git clone https://github.com/JuliaLang/julia -b $tag julia-$version -cd julia-$version -make full-source-dist -make light-source-dist -mv julia-${version}_$shashort-full.tar.gz ../julia-$version-full.tar.gz -mv julia-${version}_$shashort.tar.gz ../julia-$version.tar.gz -cd .. -rm -rf julia-$version - -# download and rename binaries, with -latest copies -julianightlies="https://julialangnightlies-s3.julialang.org/bin" -curl -L -o julia-$version-linux-x86_64.tar.gz \ - $julianightlies/linux/x64/$majmin/julia-$majminpatch-$shashort-linux64.tar.gz -cp julia-$version-linux-x86_64.tar.gz julia-$majmin-latest-linux-x86_64.tar.gz -curl -L -o julia-$version-linux-i686.tar.gz \ - $julianightlies/linux/x86/$majmin/julia-$majminpatch-$shashort-linux32.tar.gz -cp julia-$version-linux-i686.tar.gz julia-$majmin-latest-linux-i686.tar.gz -curl -L -o julia-$version-linux-arm.tar.gz \ - $julianightlies/linux/arm/$majmin/julia-$majminpatch-$shashort-linuxarm.tar.gz -cp julia-$version-linux-arm.tar.gz julia-$majmin-latest-linux-arm.tar.gz -curl -L -o julia-$version-linux-ppc64le.tar.gz \ - $julianightlies/linux/ppc64le/$majmin/julia-$majminpatch-$shashort-linuxppc64le.tar.gz -cp julia-$version-linux-ppc64le.tar.gz julia-$majmin-latest-linux-ppc64le.tar.gz -curl -L -o "julia-$version-mac64.dmg" \ - $julianightlies/mac/x64/$majmin/julia-$majminpatch-$shashort-mac64.dmg -cp "julia-$version-mac64.dmg" "julia-$majmin-latest-mac64.dmg" -curl -L -o julia-$version-win64.exe \ - $julianightlies/winnt/x64/$majmin/julia-$majminpatch-$shashort-win64.exe -cp julia-$version-win64.exe julia-$majmin-latest-win64.exe -curl -L -o julia-$version-win32.exe \ - $julianightlies/winnt/x86/$majmin/julia-$majminpatch-$shashort-win32.exe -cp julia-$version-win32.exe julia-$majmin-latest-win32.exe - -shasum -a 256 julia-$version* | grep -v -e sha256 -e md5 -e asc > julia-$version.sha256 -md5sum julia-$version* | grep -v -e sha256 -e md5 -e asc > julia-$version.md5 - -gpg -u julia --armor --detach-sig julia-$version-full.tar.gz -gpg -u julia --armor --detach-sig julia-$version.tar.gz -gpg -u julia --armor --detach-sig julia-$version-linux-x86_64.tar.gz -gpg -u julia --armor --detach-sig julia-$version-linux-i686.tar.gz -gpg -u julia --armor --detach-sig julia-$version-linux-arm.tar.gz -gpg -u julia --armor --detach-sig julia-$version-linux-ppc64le.tar.gz - -aws configure -aws s3 cp --acl public-read julia-$version.sha256 s3://julialang/bin/checksums/ -aws s3 cp --acl public-read julia-$version.md5 s3://julialang/bin/checksums/ -for plat in x86_64 i686 arm ppc64le; do - platshort=$(echo $plat | sed -e 's/x86_64/x64/' -e 's/i686/x86/') - aws s3 cp --acl public-read julia-$version-linux-$plat.tar.gz \ - s3://julialang/bin/linux/$platshort/$majmin/ - aws s3 cp --acl public-read julia-$version-linux-$plat.tar.gz.asc \ - s3://julialang/bin/linux/$platshort/$majmin/ - aws s3 cp --acl public-read julia-$majmin-latest-linux-$plat.tar.gz \ - s3://julialang/bin/linux/$platshort/$majmin/ - curl -X PURGE -L "https://julialang-s3.julialang.org/bin/linux/$platshort/$majmin/julia-$majmin-latest-linux-$plat.tar.gz" -done -aws s3 cp --acl public-read "julia-$version-mac64 .dmg" \ - s3://julialang/bin/mac/x64/$majmin/ -aws s3 cp --acl public-read "julia-$majmin-latest-mac64.dmg" \ - s3://julialang/bin/mac/x64/$majmin/ -curl -X PURGE -L "https://julialang-s3.julialang.org/bin/mac/x64/$majmin/julia-$majmin-latest-mac64.dmg" -aws s3 cp --acl public-read "julia-$version-win64.exe" \ - s3://julialang/bin/winnt/x64/$majmin/ -aws s3 cp --acl public-read "julia-$majmin-latest-win64.exe" \ - s3://julialang/bin/winnt/x64/$majmin/ -curl -X PURGE -L "https://julialang-s3.julialang.org/bin/winnt/x64/$majmin/julia-$majmin-latest-win64.exe" -aws s3 cp --acl public-read "julia-$version-win32.exe" \ - s3://julialang/bin/winnt/x86/$majmin/ -aws s3 cp --acl public-read "julia-$majmin-latest-win32.exe" \ - s3://julialang/bin/winnt/x86/$majmin/ -curl -X PURGE -L "https://julialang-s3.julialang.org/bin/winnt/x86/$majmin/julia-$majmin-latest-win32.exe" - - -echo "All files prepared. Attach julia-$version.tar.gz" -echo "and julia-$version-full.tar.gz to github releases." diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index 77921858f2b6e..34d95dd3dc476 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,11 +24,11 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia +BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia compilerrt BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps -NON_BB_PROJECTS=patchelf mozillacert lapack libwhich utf8proc ittapi +NON_BB_PROJECTS=patchelf mozillacert lapack libwhich utf8proc ittapi cpufeatures lmdb ifneq ($(VERBOSE),1) QUIET_MAKE := -s @@ -103,9 +103,15 @@ pack-checksum-llvm: | checksum-llvm-tools pack-checksum-csl: | pack-checksum-compilersupportlibraries @# nothing to do but disable the prefix rule pack-checksum-compilersupportlibraries: | checksum-csl +.PHONY: checksum-compilersupportlibraries compilersupportlibraries +checksum-compilersupportlibraries: checksum-csl +compilersupportlibraries: csl pack-checksum-libsuitesparse: | pack-checksum-suitesparse @# nothing to do but disable the prefix rule pack-checksum-suitesparse: | checksum-libsuitesparse +.PHONY: checksum-suitesparse suitesparse +checksum-suitesparse: checksum-libsuitesparse +suitesparse: libsuitesparse # This is a bit tricky: we want llvmunwind, clang, and lld to be separate from unwind and llvm, # so we add a rule to process those first pack-checksum-llvm pack-checksum-unwind: | pack-checksum-llvmunwind @@ -117,7 +123,7 @@ pack-checksum-llvmunwind: | pack-checksum-llvm.*unwind # override general rule b cd "$(JULIAHOME)/deps/checksums" && mv 'llvm.*unwind' llvmunwind clean-%: FORCE - -rm "$(JULIAHOME)/deps/checksums"/'$*' + rm -f "$(JULIAHOME)/deps/checksums"/'$*' # define how to pack parallel checksums into a single file format pack-checksum-%: FORCE | checksum-% diff --git a/contrib/tsan/ignorelist.txt b/contrib/tsan/ignorelist.txt new file mode 100644 index 0000000000000..34c9c2aced7d8 --- /dev/null +++ b/contrib/tsan/ignorelist.txt @@ -0,0 +1 @@ +mainfile:*/gc-*.c diff --git a/contrib/tsan/suppressions.txt b/contrib/tsan/suppressions.txt new file mode 100644 index 0000000000000..706f258e65d71 --- /dev/null +++ b/contrib/tsan/suppressions.txt @@ -0,0 +1 @@ +deadlock:invalidate_code_instance diff --git a/contrib/updateSPDX.jl b/contrib/updateSPDX.jl index 94b428ac70748..ceeeb6b6b5b4f 100644 --- a/contrib/updateSPDX.jl +++ b/contrib/updateSPDX.jl @@ -14,7 +14,7 @@ spdxData= JSON.parsefile(spdxDocument; dicttype=OrderedDict{String, Any}) # At the moment we can only update a few items automatically with each release. # These are the crucial elements to make a new version of the SPDX file. -# Any other changes (ex. Adding or removing of external dependencies, updating copyright text, etc.) must be performed manually +# Any other changes (ex. adding or removing of external dependencies, updating copyright text, etc.) must be performed manually spdxData["documentNamespace"]= "https://julialang.org/spdxdocs/julia-spdx-" * string(uuid4()) spdxData["creationInfo"]["created"]= Dates.format(now(tz"UTC"), "yyyy-mm-ddTHH:MM:SS") * "Z" diff --git a/contrib/windows/build-installer.iss b/contrib/windows/build-installer.iss index a63cf853d4373..79cf2d077d172 100644 --- a/contrib/windows/build-installer.iss +++ b/contrib/windows/build-installer.iss @@ -109,7 +109,7 @@ Source: "{#SourceDir}\{#AppMainExeName}"; DestDir: "{app}\bin"; Flags: ignorever [Icons] Name: "{autodesktop}\{#AppNameLong}"; Filename: "{app}\{#AppMainExeName}"; WorkingDir: "{%USERPROFILE}"; Tasks: desktopicon -Name: "{autostartmenu}\{#AppNameLong}"; Filename: "{app}\{#AppMainExeName}"; WorkingDir: "{%USERPROFILE}"; Tasks: startmenu +Name: "{autostartmenu}\Programs\{#AppNameLong}"; Filename: "{app}\{#AppMainExeName}"; WorkingDir: "{%USERPROFILE}"; Tasks: startmenu [Run] diff --git a/contrib/windows/link_with_exports.py b/contrib/windows/link_with_exports.py new file mode 100644 index 0000000000000..11f89f5e4300d --- /dev/null +++ b/contrib/windows/link_with_exports.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Apply Julia's export map when linking PE images with lld's MinGW driver.""" + +import fnmatch +from pathlib import Path +import re +import subprocess +import sys +import tempfile + + +def export_patterns(path): + # Julia's map contains named version blocks, global patterns and local: *. + # PE has no symbol versions; only the export selection applies here. + text = re.sub(r'/\*.*?\*/', '', Path(path).read_text(), flags=re.S) + block = r'[^{};\s]*\s*\{([^{}]*)\}\s*;' + if re.sub(block, '', text).strip(): + raise ValueError('unsupported export map: ' + path) + patterns = [] + for body in re.findall(block, text): + sections = re.split(r'\b(global|local)\s*:', body) + if sections[0].strip(): + raise ValueError('unsupported export map: ' + path) + for kind, entries in zip(sections[1::2], sections[2::2]): + names = [name.strip() for name in entries.split(';') if name.strip()] + if kind == 'global': + patterns.extend(names) + elif names != ['*']: + raise ValueError('unsupported local export rule: ' + entries) + if not patterns: + raise ValueError('empty export map: ' + path) + return patterns + + +def link(linker, args): + maps = [arg for arg in args if arg.startswith('--version-script=')] + if not maps: + subprocess.run([linker, *args], check=True) + return + if len(maps) != 1: + raise ValueError('expected one --version-script') + patterns = export_patterns(maps[0].split('=', 1)[1]) + args = [arg for arg in args if arg != maps[0]] + output_index = args.index('-o') + 1 + output = Path(args[output_index]) + + # Discover exports after archive selection. Enumerating every archive member + # up front would force otherwise unused members into the final library. + with tempfile.TemporaryDirectory(prefix=output.name + '.', dir=output.parent) as tmp: + definition = Path(tmp) / 'exports.def' + probe = args.copy() + probe[output_index] = str(Path(tmp) / output.name) + # Do not overwrite the real import library during the discovery link. + inputs = iter(probe) + probe = [] + for arg in inputs: + if arg in ('--out-implib', '--output-def'): + next(inputs) + elif not arg.startswith(('--out-implib=', '--output-def=')): + probe.append(arg) + subprocess.run([linker, *probe, '--output-def', str(definition)], check=True) + exports = [] + for line in definition.read_text().splitlines(): + if line.strip() == 'EXPORTS': + continue + match = re.fullmatch(r'\s*(\S+)\s+@\d+(\s+DATA)?\s*', line) + if not match: + raise ValueError('unexpected lld export definition: ' + line) + name, data = match.groups() + if any(fnmatch.fnmatchcase(name, pattern) for pattern in patterns): + # Let the final link assign ordinals after filtering. + exports.append(' ' + name + (' DATA' if data else '')) + definition.write_text('EXPORTS\n' + '\n'.join(exports) + '\n') + # Explicit dllexport directives remain effective, just as with GNU ld's + # version script. Disable only the unfiltered automatic exports. + args = [arg for arg in args if arg != '--export-all-symbols'] + subprocess.run([linker, *args, '--exclude-all-symbols', str(definition)], check=True) + + +if __name__ == '__main__': + try: + link(sys.argv[1], sys.argv[2:]) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + sys.exit(str(error)) diff --git a/contrib/windows/test_exports.py b/contrib/windows/test_exports.py new file mode 100644 index 0000000000000..008b40b550c01 --- /dev/null +++ b/contrib/windows/test_exports.py @@ -0,0 +1,90 @@ +"""Compare GNU ld and lld exports, including ThinLTO and archive selection. + +Run with python3 contrib/windows/test_exports.py. CLANG and LLD can select tools. +""" + +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +class ExportTests(unittest.TestCase): + def check_exports(self, arch, thinlto): + prefix = arch + '-w64-mingw32-' + clang = os.environ.get('CLANG', 'clang') + lld = os.environ.get('LLD', 'ld.lld') + for tool in (prefix + 'gcc', prefix + 'ld', prefix + 'ar', clang, lld, 'llvm-ar'): + if not shutil.which(tool): + self.skipTest(tool + ' unavailable') + with tempfile.TemporaryDirectory() as tmp: + build = Path(tmp) + export_map = build / 'julia.expmap' + export_map.write_text((ROOT / 'src/julia.expmap.in').read_text() + .replace('@JULIA_SHLIB_SYMBOL_VERSION@', 'JULIA_TEST') + .replace('@LLVM_SHLIB_SYMBOL_VERSION@', 'LLVM_TEST')) + (build / 'entry.c').write_text( + 'extern int jl_from_archive(void);\n' + '__declspec(dllexport) int jl_entry(void) { return jl_from_archive(); }\n' + '__declspec(dllexport) int explicit_unlisted(void) { return 2; }\n' + 'int jl_unannotated(void) { return 3; }\n' + 'int jl_data = 4;\n' + 'int unrelated(void) { return 5; }\n') + (build / 'used.c').write_text( + 'int jl_from_archive(void) { return 6; }\n' + 'int archive_internal(void) { return 7; }\n') + (build / 'unused.c').write_text('int jl_unused_archive_member(void) { return 8; }\n') + definitions = [] + for compiler in ('gnu', 'llvm'): + cc = [prefix + 'gcc', '-fvisibility=hidden'] if compiler == 'gnu' else [clang, '--target=' + arch + '-w64-mingw32'] + if compiler == 'llvm' and thinlto: + cc += ['-flto=thin'] + for source in ('entry', 'used', 'unused'): + subprocess.run([*cc, '-O2', '-c', str(build / (source + '.c')), + '-o', str(build / (source + '.o'))], check=True) + archive = build / (compiler + '.a') + ar = prefix + 'ar' if compiler == 'gnu' else 'llvm-ar' + subprocess.run([ar, 'rcs', str(archive), str(build / 'used.o'), str(build / 'unused.o')], check=True) + dll = build / (compiler + '.dll') + definition = build / (compiler + '.def') + implib = build / (compiler + '.dll.a') + linker = [prefix + 'ld'] if compiler == 'gnu' else [sys.executable, str(ROOT / 'contrib/windows/link_with_exports.py'), lld] + subprocess.run([*linker, '-m', 'i386pep' if arch == 'x86_64' else 'i386pe', + '-shared', '-e', 'jl_entry' if arch == 'x86_64' else '_jl_entry', + '-o', str(dll), str(build / 'entry.o'), str(archive), + '--export-all-symbols', '--version-script=' + str(export_map), + '--out-implib', str(implib), '--output-def', str(definition)], check=True) + self.assertTrue(implib.is_file()) + exports = {} + for line in definition.read_text().splitlines(): + match = re.fullmatch(r'\s*(\S+)\s+@(\d+)(\s+DATA)?\s*', line) + if match: + name, ordinal, data = match.groups() + exports[name] = (int(ordinal), bool(data)) + definitions.append(exports) + self.assertEqual(definitions[0], definitions[1]) + self.assertEqual(set(definitions[1]), {'jl_entry', 'explicit_unlisted', + 'jl_unannotated', 'jl_data', 'jl_from_archive'}) + self.assertTrue(definitions[1]['jl_data'][1]) + + def test_x86_64(self): + self.check_exports('x86_64', False) + + def test_x86_64_thinlto(self): + self.check_exports('x86_64', True) + + def test_i686(self): + self.check_exports('i686', False) + + def test_i686_thinlto(self): + self.check_exports('i686', True) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/deps/BOLT.mk b/deps/BOLT.mk index 34391ab10f716..330f3de96e3cc 100644 --- a/deps/BOLT.mk +++ b/deps/BOLT.mk @@ -8,81 +8,79 @@ $(eval $(call git-external,BOLT,BOLT,CMakeLists.txt,,$(SRCCACHE))) BOLT_BUILDDIR := $(BUILDDIR)/$(BOLT_SRC_DIR)/build -LLVM_ENABLE_PROJECTS := bolt +BOLT_BUILD_PROJECTS := bolt -LLVM_CFLAGS := -LLVM_CXXFLAGS := -LLVM_CPPFLAGS := -LLVM_LDFLAGS := -LLVM_CMAKE := +BOLT_BUILD_CFLAGS := $(CFLAGS) +BOLT_BUILD_CXXFLAGS := $(CXXFLAGS) +BOLT_BUILD_CPPFLAGS := $(CPPFLAGS) +BOLT_BUILD_LDFLAGS := $(LDFLAGS) +BOLT_BUILD_CMAKE := -LLVM_CMAKE += -DLLVM_ENABLE_PROJECTS="$(LLVM_ENABLE_PROJECTS)" +BOLT_BUILD_CMAKE += -DLLVM_ENABLE_PROJECTS="$(BOLT_BUILD_PROJECTS)" # Otherwise LLVM will translate \\ to / on mingw -LLVM_CMAKE += -DLLVM_WINDOWS_PREFER_FORWARD_SLASH=False +BOLT_BUILD_CMAKE += -DLLVM_WINDOWS_PREFER_FORWARD_SLASH=False -# Allow adding LLVM specific flags -LLVM_CFLAGS += $(CFLAGS) -LLVM_CXXFLAGS += $(CXXFLAGS) -LLVM_CXXFLAGS += $(LLVM_CXXFLAGS) -LLVM_CPPFLAGS += $(CPPFLAGS) -LLVM_LDFLAGS += $(LDFLAGS) -LLVM_LDFLAGS += $(LLVM_LDFLAGS) -LLVM_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING=host -DCMAKE_BUILD_TYPE=Release -LLVM_CMAKE += -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_HOST_TRIPLE="$(or $(XC_HOST),$(BUILD_MACHINE))" -LLVM_CMAKE += -DLLVM_ENABLE_ZLIB=FORCE_ON -DZLIB_ROOT="$(build_prefix)" +BOLT_BUILD_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING=host -DCMAKE_BUILD_TYPE=Release +BOLT_BUILD_CMAKE += -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_HOST_TRIPLE="$(or $(XC_HOST),$(BUILD_MACHINE))" +BOLT_BUILD_CMAKE += -DLLVM_ENABLE_ZLIB=FORCE_ON -DZLIB_ROOT="$(build_prefix)" -LLVM_CMAKE += -DLLVM_BINDINGS_LIST="" -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INCLUDE_DOCS=Off -DLLVM_ENABLE_TERMINFO=Off -DHAVE_LIBEDIT=Off +BOLT_BUILD_CMAKE += -DLLVM_BINDINGS_LIST="" -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INCLUDE_DOCS=Off -DLLVM_ENABLE_TERMINFO=Off -DHAVE_LIBEDIT=Off ifeq ($(OS), WINNT) -LLVM_CPPFLAGS += -D__USING_SJLJ_EXCEPTIONS__ -D__CRT__NO_INLINE +BOLT_BUILD_CPPFLAGS += -D__USING_SJLJ_EXCEPTIONS__ -D__CRT__NO_INLINE endif # OS == WINNT ifneq ($(HOSTCC),$(CC)) -LLVM_CMAKE += -DCROSS_TOOLCHAIN_FLAGS_NATIVE="-DCMAKE_C_COMPILER=$$(which $(HOSTCC));-DCMAKE_CXX_COMPILER=$$(which $(HOSTCXX))" +BOLT_BUILD_CMAKE += -DCROSS_TOOLCHAIN_FLAGS_NATIVE="-DCMAKE_C_COMPILER=$$(which $(HOSTCC));-DCMAKE_CXX_COMPILER=$$(which $(HOSTCXX))" # Defaults to off when crosscompiling, starting from LLVM 18 -LLVM_CMAKE += -DBOLT_ENABLE_RUNTIME=ON +BOLT_BUILD_CMAKE += -DBOLT_ENABLE_RUNTIME=ON endif ifeq ($(OS), emscripten) -LLVM_CMAKE += -DCMAKE_TOOLCHAIN_FILE=$(EMSCRIPTEN)/cmake/Modules/Platform/Emscripten.cmake -DLLVM_INCLUDE_TOOLS=OFF -DLLVM_BUILD_TOOLS=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_ENABLE_THREADS=OFF -DLLVM_BUILD_UTILS=OFF +BOLT_BUILD_CMAKE += -DCMAKE_TOOLCHAIN_FILE=$(EMSCRIPTEN)/cmake/Modules/Platform/Emscripten.cmake -DLLVM_INCLUDE_TOOLS=OFF -DLLVM_BUILD_TOOLS=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_ENABLE_THREADS=OFF -DLLVM_BUILD_UTILS=OFF endif # OS == emscripten ifneq (,$(filter $(ARCH), powerpc64le ppc64le)) ifeq (${USECLANG},0) -LLVM_CXXFLAGS += -mminimal-toc +BOLT_BUILD_CXXFLAGS += -mminimal-toc endif endif ifeq ($(fPIC),) -LLVM_CMAKE += -DLLVM_ENABLE_PIC=OFF +BOLT_BUILD_CMAKE += -DLLVM_ENABLE_PIC=OFF endif -LLVM_CMAKE += -DCMAKE_C_FLAGS="$(LLVM_CPPFLAGS) $(LLVM_CFLAGS)" \ - -DCMAKE_CXX_FLAGS="$(LLVM_CPPFLAGS) $(LLVM_CXXFLAGS)" +BOLT_BUILD_CMAKE += -DCMAKE_C_FLAGS="$(BOLT_BUILD_CPPFLAGS) $(BOLT_BUILD_CFLAGS)" \ + -DCMAKE_CXX_FLAGS="$(BOLT_BUILD_CPPFLAGS) $(BOLT_BUILD_CXXFLAGS)" ifeq ($(OS),Darwin) # Explicitly use the default for -mmacosx-version-min=10.9 and later -LLVM_CMAKE += -DLLVM_ENABLE_LIBCXX=ON +BOLT_BUILD_CMAKE += -DLLVM_ENABLE_LIBCXX=ON endif -LLVM_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(LLVM_LDFLAGS)" \ - -DCMAKE_SHARED_LINKER_FLAGS="$(LLVM_LDFLAGS)" +BOLT_BUILD_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(BOLT_BUILD_LDFLAGS)" \ + -DCMAKE_SHARED_LINKER_FLAGS="$(BOLT_BUILD_LDFLAGS)" ifeq ($(USE_SYSTEM_ZLIB), 0) $(BOLT_BUILDDIR)/build-configured: | $(build_prefix)/manifest/zlib endif -$(BOLT_BUILDDIR)/build-configured: $(SRCCACHE)/$(BOLT_SRC_DIR)/source-extracted +# Backport of llvm/llvm-project#215415, which the BOLT_jll build also carries: +# without it, BOLT cannot rewrite a ThinLTO-built libLLVM on AArch64. +$(SRCCACHE)/$(BOLT_SRC_DIR)/BOLT-aarch64-adr-relaxation-non-simple.patch-applied: $(SRCCACHE)/$(BOLT_SRC_DIR)/source-extracted + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/BOLT-aarch64-adr-relaxation-non-simple.patch + echo 1 > $@ + +$(BOLT_BUILDDIR)/build-configured: $(SRCCACHE)/$(BOLT_SRC_DIR)/BOLT-aarch64-adr-relaxation-non-simple.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(SRCCACHE)/$(BOLT_SRC_DIR)/llvm $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) $(LLVM_CMAKE) \ + $(CMAKE) $(SRCCACHE)/$(BOLT_SRC_DIR)/llvm $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) $(BOLT_BUILD_CMAKE) \ || { echo '*** To install a newer version of cmake, run contrib/download_cmake.sh ***' && false; } echo 1 > $@ $(BOLT_BUILDDIR)/build-compiled: $(BOLT_BUILDDIR)/build-configured cd $(BOLT_BUILDDIR) && \ - $(if $(filter $(CMAKE_GENERATOR),make), \ - $(MAKE), \ - $(CMAKE) --build . --target bolt) + $(CMAKE) --build . --target bolt echo 1 > $@ $(BOLT_BUILDDIR)/build-checked: $(BOLT_BUILDDIR)/build-compiled @@ -102,7 +100,7 @@ $(eval $(call staged-install, \ clean-bolt: -rm -f $(BOLT_BUILDDIR)/build-configured $(BOLT_BUILDDIR)/build-compiled - -$(MAKE) -C $(BOLT_BUILDDIR) clean + -if [ -d $(BOLT_BUILDDIR) ]; then $(MAKE) -C $(BOLT_BUILDDIR) clean; fi get-bolt: $(BOLT_SRC_FILE) extract-bolt: $(SRCCACHE)/$(BOLT_SRC_DIR)/source-extracted diff --git a/deps/BOLT.version b/deps/BOLT.version index 6a785041e163f..41a88aa0ac77d 100644 --- a/deps/BOLT.version +++ b/deps/BOLT.version @@ -1,11 +1,11 @@ # -*- makefile -*- -BOLT_VER := 18.1.4 -BOLT_JLL_VER := 18.1.4+0 +BOLT_VER := 23.1.2 +BOLT_JLL_VER := 23.1.2+0 ## jll artifact BOLT_JLL_NAME := BOLT ## source build BOLT_BRANCH=llvmorg-$(BOLT_VER) -BOLT_SHA1=e6c3289804a67ea0bb6a86fadbe454dd93b8d855 +BOLT_SHA1=85ac560262434c9ccfc0c183ec22d4138ed647fb diff --git a/deps/JuliaSyntax.mk b/deps/JuliaSyntax.mk deleted file mode 100644 index 4a8afa8fbd53c..0000000000000 --- a/deps/JuliaSyntax.mk +++ /dev/null @@ -1,16 +0,0 @@ -$(eval $(call git-external,JuliaSyntax,JULIASYNTAX,,,$(BUILDDIR))) - -$(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/source-extracted - @# no build steps - echo 1 > $@ - -$(eval $(call symlink_install,JuliaSyntax,$$(JULIASYNTAX_SRC_DIR),$$(BUILDROOT)/base)) - -clean-JuliaSyntax: - -rm -f $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled -get-JuliaSyntax: $(JULIASYNTAX_SRC_FILE) -extract-JuliaSyntax: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/source-extracted -configure-JuliaSyntax: extract-JuliaSyntax -compile-JuliaSyntax: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled -fastcheck-JuliaSyntax: check-JuliaSyntax -check-JuliaSyntax: compile-JuliaSyntax diff --git a/deps/JuliaSyntax.version b/deps/JuliaSyntax.version deleted file mode 100644 index 9487754d8a617..0000000000000 --- a/deps/JuliaSyntax.version +++ /dev/null @@ -1,4 +0,0 @@ -JULIASYNTAX_BRANCH = main -JULIASYNTAX_SHA1 = 46723f071d5b2efcb21ca6757788028afb91cc13 -JULIASYNTAX_GIT_URL := https://github.com/JuliaLang/JuliaSyntax.jl.git -JULIASYNTAX_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntax.jl/tarball/$1 diff --git a/deps/Makefile b/deps/Makefile index 392b4fad2b2e2..bef476ed57763 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -26,7 +26,7 @@ BUILDDIR := $(BUILDDIR)$(MAYBE_HOST) # custom Makefile rules: openlibm dsfmt libsuitesparse lapack blastrampoline openblas utf8proc objconv libwhich # CMake libs: llvm llvmunwind libgit2 libssh2 libtracyclient # -# downloadable via git: llvm-svn, libuv, libopenlibm, utf8proc, libgit2, libssh2, libtracyclient, mmtk_julia +# downloadable via git: llvm-svn, libuv, libopenlibm, utf8proc, libgit2, libssh2, libtracyclient # # to debug 'define' rules, replace eval at the usage site with info or error @@ -36,8 +36,6 @@ BUILDDIR := $(BUILDDIR)$(MAYBE_HOST) # prevent installing libs into usr/lib64 on opensuse unexport CONFIG_SITE -DEP_LIBS := JuliaSyntax - ifeq ($(USE_SYSTEM_LIBBLASTRAMPOLINE), 0) DEP_LIBS += blastrampoline endif @@ -97,6 +95,8 @@ ifeq ($(USE_SYSTEM_DSFMT), 0) DEP_LIBS += dsfmt endif +DEP_LIBS += cpufeatures + ifeq ($(USE_SYSTEM_LLVM), 0) DEP_LIBS += llvm endif @@ -164,6 +164,10 @@ ifeq ($(USE_SYSTEM_ZSTD), 0) DEP_LIBS += zstd endif +ifeq ($(USE_SYSTEM_LMDB), 0) +DEP_LIBS += lmdb +endif + ifeq ($(USE_SYSTEM_P7ZIP), 0) DEP_LIBS += p7zip endif @@ -202,18 +206,14 @@ DEP_LIBS += libwhich endif endif -ifeq (${USE_THIRD_PARTY_GC},mmtk) -DEP_LIBS += mmtk_julia -endif - DEP_LIBS_STAGED := $(DEP_LIBS) # list all targets DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \ openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \ - objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \ - sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax \ - terminfo mmtk_julia + objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib zstd lmdb p7zip csl \ + sanitizers libsuitesparse lld libtracyclient ittapi nvtx \ + terminfo cpufeatures DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL) ifneq ($(USE_BINARYBUILDER_OPENBLAS),0) @@ -247,6 +247,22 @@ distcleanall: $(addprefix distclean-, $(DEP_LIBS_ALL)) rm -rf $(build_prefix) getall: $(addprefix get-, $(DEP_LIBS_ALL)) +.PHONY: default +.PHONY: get +.PHONY: extract +.PHONY: configure +.PHONY: compile +.PHONY: check +.PHONY: fastcheck +.PHONY: stage +.PHONY: install +.PHONY: version-check + +.PHONY: uninstall +.PHONY: cleanall +.PHONY: distcleanall +.PHONY: getall + include $(SRCDIR)/BOLT.mk include $(SRCDIR)/csl.mk include $(SRCDIR)/sanitizers.mk @@ -264,10 +280,12 @@ include $(SRCDIR)/utf8proc.mk include $(SRCDIR)/libsuitesparse.mk include $(SRCDIR)/zlib.mk include $(SRCDIR)/zstd.mk +include $(SRCDIR)/lmdb.mk include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk include $(SRCDIR)/patchelf.mk +include $(SRCDIR)/cpufeatures.mk include $(SRCDIR)/openssl.mk include $(SRCDIR)/libssh2.mk include $(SRCDIR)/nghttp2.mk @@ -278,10 +296,4 @@ include $(SRCDIR)/p7zip.mk include $(SRCDIR)/libtracyclient.mk include $(SRCDIR)/terminfo.mk -# MMTk -include $(SRCDIR)/mmtk_julia.mk - -# vendored Julia libs -include $(SRCDIR)/JuliaSyntax.mk - include $(SRCDIR)/tools/uninstallers.mk diff --git a/deps/blastrampoline.mk b/deps/blastrampoline.mk index cfa28a4d8b88f..0aac8f006cc5a 100644 --- a/deps/blastrampoline.mk +++ b/deps/blastrampoline.mk @@ -8,6 +8,7 @@ $(eval $(call git-external,blastrampoline,BLASTRAMPOLINE,,,$(BUILDDIR))) BLASTRAMPOLINE_BUILD_OPTS := $(MAKE_COMMON) CC="$(CC) $(SANITIZE_OPTS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" BLASTRAMPOLINE_BUILD_OPTS += ARCH="$(ARCH)" OS="$(OS)" +BLASTRAMPOLINE_BUILD_OPTS += LBT_THREADSAFE=1 $(BUILDDIR)/$(BLASTRAMPOLINE_SRC_DIR)/build-configured: $(BUILDDIR)/$(BLASTRAMPOLINE_SRC_DIR)/source-extracted mkdir -p $(dir $@) @@ -33,7 +34,7 @@ $(eval $(call staged-install, \ $$(INSTALL_NAME_CMD)libblastrampoline.$$(SHLIB_EXT) $$(build_shlibdir)/libblastrampoline.$$(SHLIB_EXT))) clean-blastrampoline: - -$(MAKE) -C $(BLASTRAMPOLINE_BUILD_ROOT) clean + -if [ -d $(BLASTRAMPOLINE_BUILD_ROOT) ]; then $(MAKE) -C $(BLASTRAMPOLINE_BUILD_ROOT) clean; fi -$(RM) $(BUILDDIR)/$(BLASTRAMPOLINE_SRC_DIR)/build-compiled \ $(BUILDDIR)/$(BLASTRAMPOLINE_SRC_DIR)/build-configured diff --git a/deps/blastrampoline.version b/deps/blastrampoline.version index bb711cfcd67ec..7029040df93c6 100644 --- a/deps/blastrampoline.version +++ b/deps/blastrampoline.version @@ -4,6 +4,6 @@ BLASTRAMPOLINE_JLL_NAME := libblastrampoline ## source build -BLASTRAMPOLINE_VER := 5.13.1 -BLASTRAMPOLINE_BRANCH=v5.13.1 -BLASTRAMPOLINE_SHA1=f26278e83ddc9035ae7695da597f1a5b26a4c62b +BLASTRAMPOLINE_VER := 5.16.0 +BLASTRAMPOLINE_BRANCH=v5.16.0 +BLASTRAMPOLINE_SHA1=59700e7c5fb1c7de16e3d3b533c967c7bbea417a diff --git a/deps/checksums/ArgTools-1314758ad02ff5e9e5ca718920c6c633b467a84a.tar.gz/md5 b/deps/checksums/ArgTools-1314758ad02ff5e9e5ca718920c6c633b467a84a.tar.gz/md5 deleted file mode 100644 index e172379604478..0000000000000 --- a/deps/checksums/ArgTools-1314758ad02ff5e9e5ca718920c6c633b467a84a.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -d3209f45b8ea01a22ac7e9b265e3b84f diff --git a/deps/checksums/ArgTools-1314758ad02ff5e9e5ca718920c6c633b467a84a.tar.gz/sha512 b/deps/checksums/ArgTools-1314758ad02ff5e9e5ca718920c6c633b467a84a.tar.gz/sha512 deleted file mode 100644 index 991a457654113..0000000000000 --- a/deps/checksums/ArgTools-1314758ad02ff5e9e5ca718920c6c633b467a84a.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -314981eee11356f14b6dc9e07389c51432e7862d6c767d87d6679385f5a36faef34902954a5dfa6b37d8f3f25eaa4f23ba9431cc78acd3513377955e7d73f210 diff --git a/deps/checksums/ArgTools-f77276a25334585e7adce14491f281404cdc23d7.tar.gz/md5 b/deps/checksums/ArgTools-f77276a25334585e7adce14491f281404cdc23d7.tar.gz/md5 new file mode 100644 index 0000000000000..375e6435491f9 --- /dev/null +++ b/deps/checksums/ArgTools-f77276a25334585e7adce14491f281404cdc23d7.tar.gz/md5 @@ -0,0 +1 @@ +cf05ecfaa3fe25f8e17261a300485970 diff --git a/deps/checksums/ArgTools-f77276a25334585e7adce14491f281404cdc23d7.tar.gz/sha512 b/deps/checksums/ArgTools-f77276a25334585e7adce14491f281404cdc23d7.tar.gz/sha512 new file mode 100644 index 0000000000000..38ee753c88e7f --- /dev/null +++ b/deps/checksums/ArgTools-f77276a25334585e7adce14491f281404cdc23d7.tar.gz/sha512 @@ -0,0 +1 @@ +8b6021fa5dcd7e99a4602cbcc86d1e17d6bc940fc07874496646e9accc15da3cea288b65ebf01ff68fa0f809d41139371c47cb8ac57a6276e21a7957fddc47b6 diff --git a/deps/checksums/BOLT-85ac560262434c9ccfc0c183ec22d4138ed647fb.tar.gz/md5 b/deps/checksums/BOLT-85ac560262434c9ccfc0c183ec22d4138ed647fb.tar.gz/md5 new file mode 100644 index 0000000000000..d1f01489a73f2 --- /dev/null +++ b/deps/checksums/BOLT-85ac560262434c9ccfc0c183ec22d4138ed647fb.tar.gz/md5 @@ -0,0 +1 @@ +c9d0b4d9024e6ea4a305bf421a673953 diff --git a/deps/checksums/BOLT-85ac560262434c9ccfc0c183ec22d4138ed647fb.tar.gz/sha512 b/deps/checksums/BOLT-85ac560262434c9ccfc0c183ec22d4138ed647fb.tar.gz/sha512 new file mode 100644 index 0000000000000..dbbee6422d566 --- /dev/null +++ b/deps/checksums/BOLT-85ac560262434c9ccfc0c183ec22d4138ed647fb.tar.gz/sha512 @@ -0,0 +1 @@ +4497c19bf7091ab02b33b5542ff1847c5e50433157e2e2bacf4fed05ad3983a0c5fb826ee2ab7973ee8cd647ec84bb7e50991782554e1b6bbe322f4b1d74abb3 diff --git a/deps/checksums/BOLT.v18.1.4+0.x86_64-linux-gnu-cxx11.tar.gz/md5 b/deps/checksums/BOLT.v18.1.4+0.x86_64-linux-gnu-cxx11.tar.gz/md5 deleted file mode 100644 index 62e63ff3174d6..0000000000000 --- a/deps/checksums/BOLT.v18.1.4+0.x86_64-linux-gnu-cxx11.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c12540d5889cef05bc87183a4ce5a54c diff --git a/deps/checksums/BOLT.v18.1.4+0.x86_64-linux-gnu-cxx11.tar.gz/sha512 b/deps/checksums/BOLT.v18.1.4+0.x86_64-linux-gnu-cxx11.tar.gz/sha512 deleted file mode 100644 index 0635e180ac9a5..0000000000000 --- a/deps/checksums/BOLT.v18.1.4+0.x86_64-linux-gnu-cxx11.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -61cc7cc42b925f37502eed0d31eafadbfdc24a9ebc892c9b8d96a27b004cbccf2e5da7face5c8d9c9db57fac1b5cf662d890a67337436c5d4aa3373256638ab1 diff --git a/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-gnu-cxx11.tar.gz/md5 b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-gnu-cxx11.tar.gz/md5 new file mode 100644 index 0000000000000..e3a7f54584552 --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-gnu-cxx11.tar.gz/md5 @@ -0,0 +1 @@ +1dab81b68b526f7c2a870255e38204a7 diff --git a/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-gnu-cxx11.tar.gz/sha512 b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-gnu-cxx11.tar.gz/sha512 new file mode 100644 index 0000000000000..2df4f375820f0 --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-gnu-cxx11.tar.gz/sha512 @@ -0,0 +1 @@ +57049cdbb9d59d998308d2500bbdcd6d9547f415a689f1dd36d1c47f00ff0cdc2aeff8ffd85080f0bde016e65b880c68432cf539d8a1528093bee7b97c0e023b diff --git a/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-musl-cxx11.tar.gz/md5 b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-musl-cxx11.tar.gz/md5 new file mode 100644 index 0000000000000..e8e563ce1a3b5 --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-musl-cxx11.tar.gz/md5 @@ -0,0 +1 @@ +6333071903b599b2fc989209c6c080e9 diff --git a/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-musl-cxx11.tar.gz/sha512 b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-musl-cxx11.tar.gz/sha512 new file mode 100644 index 0000000000000..a1be53b49548f --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.aarch64-linux-musl-cxx11.tar.gz/sha512 @@ -0,0 +1 @@ +e4c8b0d1582fd26cdf0dbe9edc07e2633fcefb5ca34c01cc78a600ca6c597a17707f7b023de46c4f6e325907acffcdfd72fe317916178c8ce97389972b4557b4 diff --git a/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-gnu-cxx11.tar.gz/md5 b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-gnu-cxx11.tar.gz/md5 new file mode 100644 index 0000000000000..0ce8d5e7feb77 --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-gnu-cxx11.tar.gz/md5 @@ -0,0 +1 @@ +01fd2d336b5bda452eb769ff3af9c14d diff --git a/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-gnu-cxx11.tar.gz/sha512 b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-gnu-cxx11.tar.gz/sha512 new file mode 100644 index 0000000000000..388e47c7ee86d --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-gnu-cxx11.tar.gz/sha512 @@ -0,0 +1 @@ +1327421ef3b2bcfffd9e5df23dcda9ab57d993333edbf5a5cc78680753f546b52d3f5e732bac27379eb8832b6b20f1a467c5c1774dbef9d7a603fa329bd5424d diff --git a/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-musl-cxx11.tar.gz/md5 b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-musl-cxx11.tar.gz/md5 new file mode 100644 index 0000000000000..498eebca0804b --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-musl-cxx11.tar.gz/md5 @@ -0,0 +1 @@ +b633b1dbac34b6e1756eb884fc824320 diff --git a/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-musl-cxx11.tar.gz/sha512 b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-musl-cxx11.tar.gz/sha512 new file mode 100644 index 0000000000000..90aad0f21ed30 --- /dev/null +++ b/deps/checksums/BOLT.v23.1.2+0.x86_64-linux-musl-cxx11.tar.gz/sha512 @@ -0,0 +1 @@ +3041564ca2515d22a0e5f7615e0574c54d5c479d4f81ba0e161f56ce763350176e588fa0fd62965b72c7a5404ce73dad3363c08c362a133c15af99383ebb0237 diff --git a/deps/checksums/DelimitedFiles-26972625510129cead74dbd652eb2168ca67f7db.tar.gz/md5 b/deps/checksums/DelimitedFiles-26972625510129cead74dbd652eb2168ca67f7db.tar.gz/md5 new file mode 100644 index 0000000000000..ca4ccd6d6c8a9 --- /dev/null +++ b/deps/checksums/DelimitedFiles-26972625510129cead74dbd652eb2168ca67f7db.tar.gz/md5 @@ -0,0 +1 @@ +051446351d3cfc675c1974e00356bad6 diff --git a/deps/checksums/DelimitedFiles-26972625510129cead74dbd652eb2168ca67f7db.tar.gz/sha512 b/deps/checksums/DelimitedFiles-26972625510129cead74dbd652eb2168ca67f7db.tar.gz/sha512 new file mode 100644 index 0000000000000..ea57cf75c7882 --- /dev/null +++ b/deps/checksums/DelimitedFiles-26972625510129cead74dbd652eb2168ca67f7db.tar.gz/sha512 @@ -0,0 +1 @@ +358f548751481268330b6573953be8e23000e17a172c7b51834cde25c1b155bb3b73edeace20af47c24919293b24417f3adaa0e98c392bd3323a4e17bc190b9e diff --git a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 deleted file mode 100644 index 092778b7bb675..0000000000000 --- a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -eef0f8463d4a7ba1c697f666e815b0da diff --git a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 deleted file mode 100644 index abbec567981d6..0000000000000 --- a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -e119db9546e698872e5e4cdf027a1c31e694f36c1e775a7d9361ad0db5d6a8e1804732090e68b40765a519ec3f7aa45d40c8c5e22275262303c100367bd681b3 diff --git a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 deleted file mode 100644 index ca3d9730181f4..0000000000000 --- a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -fdf2e62fcaed6aa5ad69bca405329675 diff --git a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 deleted file mode 100644 index 186dfb34221b6..0000000000000 --- a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -2361fc4ccad83139cf728f14fa38466f075fbf93dddfac533af8f5c0c35d6b86511881b7b97a95ee59f858ba9a33428d3514ac3bd605b745b002a673acfc3190 diff --git a/deps/checksums/Distributed-a7ea563843cb8b5950d0d5bb6756b2f85af9dc75.tar.gz/md5 b/deps/checksums/Distributed-a7ea563843cb8b5950d0d5bb6756b2f85af9dc75.tar.gz/md5 new file mode 100644 index 0000000000000..733efb1b297ca --- /dev/null +++ b/deps/checksums/Distributed-a7ea563843cb8b5950d0d5bb6756b2f85af9dc75.tar.gz/md5 @@ -0,0 +1 @@ +1bb4ffb4961d3ecb317887b41cf53847 diff --git a/deps/checksums/Distributed-a7ea563843cb8b5950d0d5bb6756b2f85af9dc75.tar.gz/sha512 b/deps/checksums/Distributed-a7ea563843cb8b5950d0d5bb6756b2f85af9dc75.tar.gz/sha512 new file mode 100644 index 0000000000000..9648e8a2f717a --- /dev/null +++ b/deps/checksums/Distributed-a7ea563843cb8b5950d0d5bb6756b2f85af9dc75.tar.gz/sha512 @@ -0,0 +1 @@ +4c95494d1d281dc0e3ecaa993a66dc0f4856ad4d06908f06da5af8b87a67e2920f38e8b81a5d15e284fdfd75495390cec9e77838f7da1e6002e1ee32d3755a1b diff --git a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 deleted file mode 100644 index ed84729b00910..0000000000000 --- a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ac2209576b09a9c7a4da02a058e9ec87 diff --git a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 deleted file mode 100644 index aa81c3a943f3f..0000000000000 --- a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -0636b4e7f17d8747408949b77470f5f7f1b4c4b618dc905e20d84d0a5ca4713834e8a4d6a10bcccf7a28982edefe03eeff11bb0aca7d8cb49cb230b7e1ddc06f diff --git a/deps/checksums/Downloads-66706fe293b33fb126b4ae47ff4dd3d774173826.tar.gz/md5 b/deps/checksums/Downloads-66706fe293b33fb126b4ae47ff4dd3d774173826.tar.gz/md5 new file mode 100644 index 0000000000000..ed949d6173062 --- /dev/null +++ b/deps/checksums/Downloads-66706fe293b33fb126b4ae47ff4dd3d774173826.tar.gz/md5 @@ -0,0 +1 @@ +5acb32eaab2f7ee62db6f456d76f93ee diff --git a/deps/checksums/Downloads-66706fe293b33fb126b4ae47ff4dd3d774173826.tar.gz/sha512 b/deps/checksums/Downloads-66706fe293b33fb126b4ae47ff4dd3d774173826.tar.gz/sha512 new file mode 100644 index 0000000000000..3252b63eeeebc --- /dev/null +++ b/deps/checksums/Downloads-66706fe293b33fb126b4ae47ff4dd3d774173826.tar.gz/sha512 @@ -0,0 +1 @@ +83678d0b71ff339bd200a6075f614146e5ff5c21038611a0e919a7af4e0e5cf9a4f5104cbd634f0693126b46a7df88c5e47846ba79ad881433df5372dd36d480 diff --git a/deps/checksums/JuliaSyntax-46723f071d5b2efcb21ca6757788028afb91cc13.tar.gz/md5 b/deps/checksums/JuliaSyntax-46723f071d5b2efcb21ca6757788028afb91cc13.tar.gz/md5 deleted file mode 100644 index ff40f520dfe85..0000000000000 --- a/deps/checksums/JuliaSyntax-46723f071d5b2efcb21ca6757788028afb91cc13.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -2a0921e59edfab54554aa173f091c5b7 diff --git a/deps/checksums/JuliaSyntax-46723f071d5b2efcb21ca6757788028afb91cc13.tar.gz/sha512 b/deps/checksums/JuliaSyntax-46723f071d5b2efcb21ca6757788028afb91cc13.tar.gz/sha512 deleted file mode 100644 index 64e90d0edaba0..0000000000000 --- a/deps/checksums/JuliaSyntax-46723f071d5b2efcb21ca6757788028afb91cc13.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -17050e23216335f6599f009f71e9614a11b6686e455554b1efd287cd8526a7ebece06dc473e34cd50f61bf52085ff72bb4279144a9fdb3a234d3d589a10fddaf diff --git a/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/md5 b/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/md5 new file mode 100644 index 0000000000000..12fce1e97c1db --- /dev/null +++ b/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/md5 @@ -0,0 +1 @@ +ecef4caa8b237a51f92d5622b811a0c3 diff --git a/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/sha512 b/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/sha512 new file mode 100644 index 0000000000000..f042854e27a47 --- /dev/null +++ b/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/sha512 @@ -0,0 +1 @@ +56dc5158ebfaf0d5e3e5002dfeb322a137f0866add071cfa9f7a0d9ef2d40859e4c6131358c5aeaf0e9e39fe77a94ba88022028092230b059099cd87e2b795ac diff --git a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 deleted file mode 100644 index 4a586ce45dbcc..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -778d62517cab8b4a95920337631f9439 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 deleted file mode 100644 index 5a88e30a10a3f..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -95db08cd6775920271e347bae3ee4a68ef532ec6dceb834a63bc8f918b785c042c2bed9babf9ca76ff610ee2782130a69b4ecf158e4beae6361115cdef57dc51 diff --git a/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 new file mode 100644 index 0000000000000..ae83b1b01fc4c --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 @@ -0,0 +1 @@ +e01101fae581c424f776e145a0998196 diff --git a/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 new file mode 100644 index 0000000000000..f033fd54e4442 --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 @@ -0,0 +1 @@ +b94bdd91573a26046ec2120ca55de049189850627cae18cd7e66c9c66e42b7523d3a34dab124770b9b5d686304443457de304cdc4f4df1d631265c5601af1946 diff --git a/deps/checksums/LazyArtifacts-bdb8ac40a19c45aae550b79993222ce5e2e14ecf.tar.gz/md5 b/deps/checksums/LazyArtifacts-bdb8ac40a19c45aae550b79993222ce5e2e14ecf.tar.gz/md5 new file mode 100644 index 0000000000000..2d25635dae876 --- /dev/null +++ b/deps/checksums/LazyArtifacts-bdb8ac40a19c45aae550b79993222ce5e2e14ecf.tar.gz/md5 @@ -0,0 +1 @@ +1cc0f4faea68e20b30f40b8fb2dc27b3 diff --git a/deps/checksums/LazyArtifacts-bdb8ac40a19c45aae550b79993222ce5e2e14ecf.tar.gz/sha512 b/deps/checksums/LazyArtifacts-bdb8ac40a19c45aae550b79993222ce5e2e14ecf.tar.gz/sha512 new file mode 100644 index 0000000000000..57f8638577a81 --- /dev/null +++ b/deps/checksums/LazyArtifacts-bdb8ac40a19c45aae550b79993222ce5e2e14ecf.tar.gz/sha512 @@ -0,0 +1 @@ +dc479af6852961b03a7424874228f195b2d103f5aa4100cf5598416c76eedffb126896bc0f8d7a020a9204fd32dd599aa1e4e79917218eafc256f9e47b99a5ab diff --git a/deps/checksums/LazyArtifacts-e4cfc39598c238f75bdfdbdb3f82c9329a5af59c.tar.gz/md5 b/deps/checksums/LazyArtifacts-e4cfc39598c238f75bdfdbdb3f82c9329a5af59c.tar.gz/md5 deleted file mode 100644 index 48bd7a8a7fa25..0000000000000 --- a/deps/checksums/LazyArtifacts-e4cfc39598c238f75bdfdbdb3f82c9329a5af59c.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -405faa2237105ff823e80e759b2df17a diff --git a/deps/checksums/LazyArtifacts-e4cfc39598c238f75bdfdbdb3f82c9329a5af59c.tar.gz/sha512 b/deps/checksums/LazyArtifacts-e4cfc39598c238f75bdfdbdb3f82c9329a5af59c.tar.gz/sha512 deleted file mode 100644 index 9fa6aec4d1939..0000000000000 --- a/deps/checksums/LazyArtifacts-e4cfc39598c238f75bdfdbdb3f82c9329a5af59c.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9bd2bdd5a83df28a26ebfb0d4e59b50584962e07b1364e6fd76bc7a6a7b109f1facaa04366beaa9f340192ea9efa540decde1393ddd50dc3efa13937deeb5d7f diff --git a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 deleted file mode 100644 index 264c8e5ea6293..0000000000000 --- a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -b3a67e92f5a9d5832699623e34abf1b2 diff --git a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 deleted file mode 100644 index 78f04bb4becb8..0000000000000 --- a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -d558321aa6099ddb61cc402f85d3ea7f0d5c875aee1371f10e6b50c53d8b578045a132a3fd361b96fabf44658b364496d0377371d1ec232962fc8450eb242217 diff --git a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 deleted file mode 100644 index 63847447093fd..0000000000000 --- a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -6ea57484fcb60da4bf9d7c646fe73b2e diff --git a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 deleted file mode 100644 index fd0479374f265..0000000000000 --- a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7b75d79804e73a3fc7140fe0636b06a4d0267c96ee6d6b1dbe8eedeefd79a1f8a82501bed4a3038ba309284dc0291e7424463ced1f1b76eb20f5bd57c7c0a7b5 diff --git a/deps/checksums/LinearAlgebra-802d661bfc25a7467520e6638702698342079b09.tar.gz/md5 b/deps/checksums/LinearAlgebra-802d661bfc25a7467520e6638702698342079b09.tar.gz/md5 new file mode 100644 index 0000000000000..b6d50115097d2 --- /dev/null +++ b/deps/checksums/LinearAlgebra-802d661bfc25a7467520e6638702698342079b09.tar.gz/md5 @@ -0,0 +1 @@ +7465ada67dbb2fedf7ef4aec63e65c0b diff --git a/deps/checksums/LinearAlgebra-802d661bfc25a7467520e6638702698342079b09.tar.gz/sha512 b/deps/checksums/LinearAlgebra-802d661bfc25a7467520e6638702698342079b09.tar.gz/sha512 new file mode 100644 index 0000000000000..03c20d953d770 --- /dev/null +++ b/deps/checksums/LinearAlgebra-802d661bfc25a7467520e6638702698342079b09.tar.gz/sha512 @@ -0,0 +1 @@ +52bed7c67640f0db3020280e1f6fc8d072f747b38ba2cfc9a5e30acb8028cf8708cce7e69c768048244bcf44dd52c3d82f2e58051cfb2e93138a0bda6758de3f diff --git a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 deleted file mode 100644 index fbb7d5b86627f..0000000000000 --- a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -afab093d162a62d5a488894f33d3b396 diff --git a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 deleted file mode 100644 index 146d3a3d1bad8..0000000000000 --- a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -14b41cc9c93e2f9eeaa9499d65b8c42ee80691cbd533ef6cafabdb6e94c7cf31eee00fb603ca70dfe86930c871419cf17a8f05c0a76bd379a8bbf705b875dfe2 diff --git a/deps/checksums/NetworkOptions-7034c55dbf52ee959cabd63bcbe656df658f5bda.tar.gz/md5 b/deps/checksums/NetworkOptions-7034c55dbf52ee959cabd63bcbe656df658f5bda.tar.gz/md5 new file mode 100644 index 0000000000000..8f918b40b2ffb --- /dev/null +++ b/deps/checksums/NetworkOptions-7034c55dbf52ee959cabd63bcbe656df658f5bda.tar.gz/md5 @@ -0,0 +1 @@ +1fbf59e3052ec0d40a195d935b3d4a96 diff --git a/deps/checksums/NetworkOptions-7034c55dbf52ee959cabd63bcbe656df658f5bda.tar.gz/sha512 b/deps/checksums/NetworkOptions-7034c55dbf52ee959cabd63bcbe656df658f5bda.tar.gz/sha512 new file mode 100644 index 0000000000000..feeb688dcb7e2 --- /dev/null +++ b/deps/checksums/NetworkOptions-7034c55dbf52ee959cabd63bcbe656df658f5bda.tar.gz/sha512 @@ -0,0 +1 @@ +4ec9724062d97a9d400bfb4a672ed5ce52999738ddbc01d2892d97df3fd256fe03bb5ea69f2ebbbbdbfef91edc24d82e54df48f997781eba2c6cd8e1c36d046c diff --git a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 deleted file mode 100644 index f1a89122167f9..0000000000000 --- a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -474392cbf2fe976bca300944fc62f5ab diff --git a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 deleted file mode 100644 index e3a190082f455..0000000000000 --- a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -99c1b57efd80eade5f89f350d244bedc80717809c108056bf28b2f6213dba040bbed2ef4869f97a624efa1ca8474504c087481860758bb366025093a8bec34b5 diff --git a/deps/checksums/Pkg-dbbd4046ec8604c89268e8db16ed776bfab16cd6.tar.gz/md5 b/deps/checksums/Pkg-dbbd4046ec8604c89268e8db16ed776bfab16cd6.tar.gz/md5 new file mode 100644 index 0000000000000..e31b2f36ea8fb --- /dev/null +++ b/deps/checksums/Pkg-dbbd4046ec8604c89268e8db16ed776bfab16cd6.tar.gz/md5 @@ -0,0 +1 @@ +60446eecba3f10c72dd08bb36c096f1e diff --git a/deps/checksums/Pkg-dbbd4046ec8604c89268e8db16ed776bfab16cd6.tar.gz/sha512 b/deps/checksums/Pkg-dbbd4046ec8604c89268e8db16ed776bfab16cd6.tar.gz/sha512 new file mode 100644 index 0000000000000..8b360aa9477a0 --- /dev/null +++ b/deps/checksums/Pkg-dbbd4046ec8604c89268e8db16ed776bfab16cd6.tar.gz/sha512 @@ -0,0 +1 @@ +950e60fda7abc74ff4e810920b85b84416c6f758e495807cecf05955453bdb32a935664e7409085cf89dd2547a4e1f5240548d51a65b076524e7bd3654ee2fad diff --git a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 deleted file mode 100644 index ef6edc13c3a2c..0000000000000 --- a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ab91b3b95af44071020c564f6cb2b83f diff --git a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 deleted file mode 100644 index 26889e062b48d..0000000000000 --- a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -ea90fdf05a0c345ff697b91bea45ab5458123cb7d7bad53cebeea2304822545ec422012f116090d2c9491c185a9de61ad44b2eb71237a69e959737a140d8cf71 diff --git a/deps/checksums/SHA-b1aa926a09b6ecfbf293ff243496d9614b1e2768.tar.gz/md5 b/deps/checksums/SHA-b1aa926a09b6ecfbf293ff243496d9614b1e2768.tar.gz/md5 new file mode 100644 index 0000000000000..6f72ce38ea435 --- /dev/null +++ b/deps/checksums/SHA-b1aa926a09b6ecfbf293ff243496d9614b1e2768.tar.gz/md5 @@ -0,0 +1 @@ +e7ada5be0e0c219916a495ebc0bf4e0a diff --git a/deps/checksums/SHA-b1aa926a09b6ecfbf293ff243496d9614b1e2768.tar.gz/sha512 b/deps/checksums/SHA-b1aa926a09b6ecfbf293ff243496d9614b1e2768.tar.gz/sha512 new file mode 100644 index 0000000000000..bf72df8598288 --- /dev/null +++ b/deps/checksums/SHA-b1aa926a09b6ecfbf293ff243496d9614b1e2768.tar.gz/sha512 @@ -0,0 +1 @@ +86f900d65deccafcc1abc055abe93b331338477027e9baa6b8900eed7766e7779b0598fe10fff84c1fca474048c98a507a4806ce8e9cb6f353a6b2c301016400 diff --git a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 deleted file mode 100644 index a660fb96bd9d8..0000000000000 --- a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -3615e2464cfab2c7d00ae53dedc0510b diff --git a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 deleted file mode 100644 index be8fbd8d2e1a3..0000000000000 --- a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -1998b694bd9451ac92517b70c25fc2683b1482ca1d547500cec8e09eecbb615167aea94c16cae899a32b5cb2fcb842ce081fa90641a2921ea3cfa0666e9ed385 diff --git a/deps/checksums/SparseArrays-f1177baed7cbb0097cc6dbb02725b07a457bb1b5.tar.gz/md5 b/deps/checksums/SparseArrays-f1177baed7cbb0097cc6dbb02725b07a457bb1b5.tar.gz/md5 new file mode 100644 index 0000000000000..c718cce8aebc9 --- /dev/null +++ b/deps/checksums/SparseArrays-f1177baed7cbb0097cc6dbb02725b07a457bb1b5.tar.gz/md5 @@ -0,0 +1 @@ +1dbf12b391b186b752c052277d8e3279 diff --git a/deps/checksums/SparseArrays-f1177baed7cbb0097cc6dbb02725b07a457bb1b5.tar.gz/sha512 b/deps/checksums/SparseArrays-f1177baed7cbb0097cc6dbb02725b07a457bb1b5.tar.gz/sha512 new file mode 100644 index 0000000000000..787a1e6486163 --- /dev/null +++ b/deps/checksums/SparseArrays-f1177baed7cbb0097cc6dbb02725b07a457bb1b5.tar.gz/sha512 @@ -0,0 +1 @@ +b5d0bb8431fcae67573eff2f0ef182dff3724e0e55356eed69d3fd53a8ae8460d64904640a30302d88c3d1ac76cffe526656aeceac92b2b62d9a1bdf21856688 diff --git a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 deleted file mode 100644 index c5f56d9064e92..0000000000000 --- a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -0b60da1286ca8a978cf3c27b8fbc0601 diff --git a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 deleted file mode 100644 index 8cd97202f18ba..0000000000000 --- a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -2e03fe3b79dfb299caa0ac23e045bf26addeb3d38ef0b5e5430966dc227e771cfd84722d8bb80aaaedc0a988fbd2228bebf56c3e7d80b3e8993c623d8436660c diff --git a/deps/checksums/Statistics-e7bf7a47d467256c00147807df8defbb8b2dbee6.tar.gz/md5 b/deps/checksums/Statistics-e7bf7a47d467256c00147807df8defbb8b2dbee6.tar.gz/md5 new file mode 100644 index 0000000000000..9aaefe94e1a71 --- /dev/null +++ b/deps/checksums/Statistics-e7bf7a47d467256c00147807df8defbb8b2dbee6.tar.gz/md5 @@ -0,0 +1 @@ +ce94318e8626fc9ffa7400a069377ae6 diff --git a/deps/checksums/Statistics-e7bf7a47d467256c00147807df8defbb8b2dbee6.tar.gz/sha512 b/deps/checksums/Statistics-e7bf7a47d467256c00147807df8defbb8b2dbee6.tar.gz/sha512 new file mode 100644 index 0000000000000..363c64b66de05 --- /dev/null +++ b/deps/checksums/Statistics-e7bf7a47d467256c00147807df8defbb8b2dbee6.tar.gz/sha512 @@ -0,0 +1 @@ +53f689680fc6e0dc5ab9c45b8e9bd9b4384573ec150c95029ec8b6bb1db89904fd4b759501cc3fc54b52b94c8df19024ed66e975599040cc8d5cf21a9d3d0cce diff --git a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 deleted file mode 100644 index 0285925aaaa2a..0000000000000 --- a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -1762267d32633457c7c64067c1c9d871 diff --git a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 deleted file mode 100644 index f7d817c2f1e84..0000000000000 --- a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -72ca1b52b8f0c4e3d3ca58526cd34a3f1486c4fe373930bf513570f056700cddc65a4666d5b7767305b115ec0352639c4caaf03b9378911d596f6add3cbb762b diff --git a/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 b/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 new file mode 100644 index 0000000000000..9144e5269d019 --- /dev/null +++ b/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 @@ -0,0 +1 @@ +628eee7616058469a454fe3698557d09 diff --git a/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 b/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 new file mode 100644 index 0000000000000..df42d7384c552 --- /dev/null +++ b/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 @@ -0,0 +1 @@ +5346ec19f852e544a7c748b746bd2b8f7284a76d326eb6686c8f5a11dd595e0078fd4df5040f42eccee4f5df4e41935dd6ede25beab8ac9957ed892edb171f72 diff --git a/deps/checksums/Tar-1114260f5c7a7b59441acadca2411fa227bb8a3b.tar.gz/md5 b/deps/checksums/Tar-1114260f5c7a7b59441acadca2411fa227bb8a3b.tar.gz/md5 deleted file mode 100644 index 3c7510a592760..0000000000000 --- a/deps/checksums/Tar-1114260f5c7a7b59441acadca2411fa227bb8a3b.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -1d606dfc60d2af892009213650169129 diff --git a/deps/checksums/Tar-1114260f5c7a7b59441acadca2411fa227bb8a3b.tar.gz/sha512 b/deps/checksums/Tar-1114260f5c7a7b59441acadca2411fa227bb8a3b.tar.gz/sha512 deleted file mode 100644 index ec49e695cbb3a..0000000000000 --- a/deps/checksums/Tar-1114260f5c7a7b59441acadca2411fa227bb8a3b.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -6e60d74d00ffc2e1a5a9c13f59b3e3fc4360e641b9f0e3e4797c8b524288e779397bd56a8e57f47d5a06d1e6f359c86917164ec7f6e0ac3d6e876dfa09d2b0c8 diff --git a/deps/checksums/Tar-1ea3c30a1c91020ee4d850cd908fde8c21f14441.tar.gz/md5 b/deps/checksums/Tar-1ea3c30a1c91020ee4d850cd908fde8c21f14441.tar.gz/md5 new file mode 100644 index 0000000000000..e259f6d44eb19 --- /dev/null +++ b/deps/checksums/Tar-1ea3c30a1c91020ee4d850cd908fde8c21f14441.tar.gz/md5 @@ -0,0 +1 @@ +a7143cd56fe455977b3124b535c05d79 diff --git a/deps/checksums/Tar-1ea3c30a1c91020ee4d850cd908fde8c21f14441.tar.gz/sha512 b/deps/checksums/Tar-1ea3c30a1c91020ee4d850cd908fde8c21f14441.tar.gz/sha512 new file mode 100644 index 0000000000000..6abf839e0b633 --- /dev/null +++ b/deps/checksums/Tar-1ea3c30a1c91020ee4d850cd908fde8c21f14441.tar.gz/sha512 @@ -0,0 +1 @@ +5febb8edf1c706b20c27b65151ef8077a06682398ad8e0c8bb0173a4ce38eb6d5ea7892e6104725e21e149451293a61812e9bf3419dd4cd66825836af74e89c1 diff --git a/deps/checksums/UnicodeData-16.0.0.txt/md5 b/deps/checksums/UnicodeData-16.0.0.txt/md5 deleted file mode 100644 index 79ae5d27eff0e..0000000000000 --- a/deps/checksums/UnicodeData-16.0.0.txt/md5 +++ /dev/null @@ -1 +0,0 @@ -f50a0495d2000b7d6dd979cb40e00ba2 diff --git a/deps/checksums/UnicodeData-16.0.0.txt/sha512 b/deps/checksums/UnicodeData-16.0.0.txt/sha512 deleted file mode 100644 index 05b998b3724ba..0000000000000 --- a/deps/checksums/UnicodeData-16.0.0.txt/sha512 +++ /dev/null @@ -1 +0,0 @@ -963e5a1e7a480873c6e66d53e9288232b5029942477a694a0bfafa7e994c55189cb9c2f8d00255de84b82b72ff6066932e5531e3664fb422eeef9c69ea25d80e diff --git a/deps/checksums/UnicodeData-17.0.0.txt/md5 b/deps/checksums/UnicodeData-17.0.0.txt/md5 new file mode 100644 index 0000000000000..389a8decab6e2 --- /dev/null +++ b/deps/checksums/UnicodeData-17.0.0.txt/md5 @@ -0,0 +1 @@ +959cb19fcda0240caef8c02953e3d771 diff --git a/deps/checksums/UnicodeData-17.0.0.txt/sha512 b/deps/checksums/UnicodeData-17.0.0.txt/sha512 new file mode 100644 index 0000000000000..1a037aa3f6b81 --- /dev/null +++ b/deps/checksums/UnicodeData-17.0.0.txt/sha512 @@ -0,0 +1 @@ +7aa597dead08f862295dac2cbcee4c994d847c374910ec23f90a34a413ab9420828d3038d72a75b3f2c583ffcdb6f7b75205ffcde3309fec6310e494c9252798 diff --git a/deps/checksums/blastrampoline b/deps/checksums/blastrampoline index 7870242560f34..7cd3ab2e89497 100644 --- a/deps/checksums/blastrampoline +++ b/deps/checksums/blastrampoline @@ -1,38 +1,38 @@ -blastrampoline-f26278e83ddc9035ae7695da597f1a5b26a4c62b.tar.gz/md5/855b7723a6e9eb8885876eb675d48329 -blastrampoline-f26278e83ddc9035ae7695da597f1a5b26a4c62b.tar.gz/sha512/29cbd060c8f5eb17ef486d0a10ee4b221eeceec3a2ab0f9f98f60880f3d19a2247d93ac0dc0d32ec568ef876acd30f6c0642aaf704757580c2e17884e425607f -libblastrampoline.v5.13.1+0.aarch64-apple-darwin.tar.gz/md5/d8dc0f092f86b379b2fb9da97382be70 -libblastrampoline.v5.13.1+0.aarch64-apple-darwin.tar.gz/sha512/d9fc0439565afaabe53f56f64c20aeddb846c991dafeafdef6c2369bd7a359c1a6b49cdf8d63eaae2730a336509854b5c306e630eb520445712efc4e41c0263e -libblastrampoline.v5.13.1+0.aarch64-linux-gnu.tar.gz/md5/c181e51a6ca4cde0da3d036d561e24dc -libblastrampoline.v5.13.1+0.aarch64-linux-gnu.tar.gz/sha512/fe4a86bb4c94ef86c2307adad528bb58d0508a33c194c64190fffe7902f5b915592567d9e0cc35414633c5ab9067def2fa20cf669a2f4309265744180a5ec51a -libblastrampoline.v5.13.1+0.aarch64-linux-musl.tar.gz/md5/6f9eb8d73a0e61f3a2b97dba7105086e -libblastrampoline.v5.13.1+0.aarch64-linux-musl.tar.gz/sha512/9c3db080155729a91b5dd47df91d3852539aefc331d4dc51167fccaf3b01e601b36911ec259c53e211fe192c108e839a1f14b837009fa4f7d88ed82d658f80ff -libblastrampoline.v5.13.1+0.aarch64-unknown-freebsd.tar.gz/md5/68f65db9da9938929d510eea3540335b -libblastrampoline.v5.13.1+0.aarch64-unknown-freebsd.tar.gz/sha512/2fc7b375a751f3bb201504e0417828602fe014a2c8626137779c09ca7264ac6d39d44db0d1d32e0dc506284f56b49e23791922b0cc1237021473fb505fbf06bd -libblastrampoline.v5.13.1+0.armv6l-linux-gnueabihf.tar.gz/md5/a377fa4e5751fbeb3c42f319cb6341de -libblastrampoline.v5.13.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/9ddb1e2f4daab45d65b66dafc00df6ca7f788cb919cd6699c4aa0deca3e99a86d9ced10c3741610a6e480093d483e8a02c1d9165f91a7179632c1e2ae1abcfb7 -libblastrampoline.v5.13.1+0.armv6l-linux-musleabihf.tar.gz/md5/42c841baa05f80f17ea1b1d4f3405bef -libblastrampoline.v5.13.1+0.armv6l-linux-musleabihf.tar.gz/sha512/0c3ed42bd48f8f1ee9b1dc18faa7afa6e2fb27cffc59b9a420e29b5e6cdf8fb3bde36b82f3086075f8f7f329614aeb91ca5f173b1683e30e9530076f341ea2e0 -libblastrampoline.v5.13.1+0.armv7l-linux-gnueabihf.tar.gz/md5/61e515ec1223c99705175a26e6fbaf87 -libblastrampoline.v5.13.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/92260dcc563ece74719f21921a7cb51266884ed01b50c97fa997b4a98737e900ec9eaa8012d2c4c67ab479c4080bd1cf2708612eaaaddbba28e4f9147f3708ea -libblastrampoline.v5.13.1+0.armv7l-linux-musleabihf.tar.gz/md5/d45816d705dd46572d85105567bc060e -libblastrampoline.v5.13.1+0.armv7l-linux-musleabihf.tar.gz/sha512/45cba07050b818cd85c67acdfc29515e1fe416eb4e0c219171f2c0c026f7412903c3a9367d48258259a16e89f36c1e8f9fa054e455759720f1c6c5e8e27be476 -libblastrampoline.v5.13.1+0.i686-linux-gnu.tar.gz/md5/c8d3fd5f314353133934396361857c92 -libblastrampoline.v5.13.1+0.i686-linux-gnu.tar.gz/sha512/a949b3c0655ad9d6f8d53fd8a3f0b4ab504046e49a373039defc94e832b7faf90c77520f3912c4d6db8b0829951d85b4fc2a4021b3d8bb2c399d1ad04ce77ab0 -libblastrampoline.v5.13.1+0.i686-linux-musl.tar.gz/md5/a7bbd2233366d180ce8aa61fd3568c11 -libblastrampoline.v5.13.1+0.i686-linux-musl.tar.gz/sha512/e78cbef5b3bcfa93a86e14eebf0d704a94ac7b1f5c7030706d1f4a960de888c42e3daddb65395c7102e08dfd444efbfb40273e58a5f1de199d44ad55fd3ae658 -libblastrampoline.v5.13.1+0.i686-w64-mingw32.tar.gz/md5/4ca5cf3f855d04d3e8bdbd15321944ad -libblastrampoline.v5.13.1+0.i686-w64-mingw32.tar.gz/sha512/33160caa000c6c44cedd594195e1f2efefb950459653ee12ad2be4cedf0b833874772512f1812948d753f075ee7b8fe5629e5f9bd753a3da7804c4a6e1b0e0a8 -libblastrampoline.v5.13.1+0.powerpc64le-linux-gnu.tar.gz/md5/8be947c20f7d35ec22247f9a11ccce43 -libblastrampoline.v5.13.1+0.powerpc64le-linux-gnu.tar.gz/sha512/56f4246f96d2f49b03f5e5f3b996660a48d50b3784f89df7cd1dc52bab6efea0c120a65015040041a51d18fc6f361f89486f77317d771fbf588a1ba7565d77a2 -libblastrampoline.v5.13.1+0.riscv64-linux-gnu.tar.gz/md5/85e1f70a3235097158b4884a58a58154 -libblastrampoline.v5.13.1+0.riscv64-linux-gnu.tar.gz/sha512/11f1f5c2a409dbdab11d6bc968610b5700e9b0cb95094e348fe43ddca5586eda47bda1c382fb1f4b5a15aa741a6fc2b31f58f9b08bfe46631b5471e864bc009b -libblastrampoline.v5.13.1+0.x86_64-apple-darwin.tar.gz/md5/c6756ca8b6778ce2a4a440f63355c32e -libblastrampoline.v5.13.1+0.x86_64-apple-darwin.tar.gz/sha512/895d9bba75a9a0861809dca48b3dae7b5ffc5d866a518729ffd52f70fa1742a41a4b8b4e03bb354cba12d9ad11a33f3f112fa69a30ab3f945a9dede0d59d92b3 -libblastrampoline.v5.13.1+0.x86_64-linux-gnu.tar.gz/md5/1326a406aa98b6045f7459d7fb237894 -libblastrampoline.v5.13.1+0.x86_64-linux-gnu.tar.gz/sha512/4965baa1de5532425ea57b8100e369cf44b55963340cd144c0359f845560f27a1bea1597e4c72ec541917f71aaff8a4863f47d01a095c2e761a68212bfb08d1e -libblastrampoline.v5.13.1+0.x86_64-linux-musl.tar.gz/md5/5103983b7fecc7b87f495cd3b6c4d7a5 -libblastrampoline.v5.13.1+0.x86_64-linux-musl.tar.gz/sha512/f3243d84a0a0a191abad9e3850c37be78892eb5905b63b47bfb3e5a4148e0dae672ee72d311c5c764ad0fffe57d39c10dfd2086466efd76b5030118941d36a00 -libblastrampoline.v5.13.1+0.x86_64-unknown-freebsd.tar.gz/md5/a001ecd07b5178ce724a4f78996dc43e -libblastrampoline.v5.13.1+0.x86_64-unknown-freebsd.tar.gz/sha512/508866d54a9a49df2ef7eaa5d807173016c6dfaec59c4c89d5b37cd3faa7384302d2d4d39aca1975d79a948414657b7ec048a3ebdf6bf5c938037aa89303013a -libblastrampoline.v5.13.1+0.x86_64-w64-mingw32.tar.gz/md5/14fc4ec99e72e5bb646f5e6e8410fe01 -libblastrampoline.v5.13.1+0.x86_64-w64-mingw32.tar.gz/sha512/b7d07218047917fe217736b3c97d2b0565f6c904cd9cf6de96e38c66552aeec13b3cde714775fce1eb5a230db0ec0f2822572de8f0e166cb042552a16beb2b79 +blastrampoline-59700e7c5fb1c7de16e3d3b533c967c7bbea417a.tar.gz/md5/d5fac42d20e2ed62de0901459b1db604 +blastrampoline-59700e7c5fb1c7de16e3d3b533c967c7bbea417a.tar.gz/sha512/c7ab602419a7bccca62a50f138521e2baf1f5e3a514e1a0b81916c35ed5c46b812228f7593ac189850befbcb2eb7779eae51679e376e8b14bdcffe304bdf4471 +libblastrampoline.v5.16.0+0.aarch64-apple-darwin.tar.gz/md5/feb0a18e83a6e5259ac8d9cbe62b645f +libblastrampoline.v5.16.0+0.aarch64-apple-darwin.tar.gz/sha512/f770282ecc85d715a6fb7a1b1b115a8283140d041f5aecc1eb6decf52c9ccda460d7eb87787266ed3c91fea5d17b970c751f8618a4a039ec94870ca052a76e19 +libblastrampoline.v5.16.0+0.aarch64-linux-gnu.tar.gz/md5/9a447de8109e16a04642924539b51ac1 +libblastrampoline.v5.16.0+0.aarch64-linux-gnu.tar.gz/sha512/a690f6c948f3acc9d91f7591a8dbdb9e8f4f7f6fcb6501901a270d11d1da77b40e7907d6e1a71b8fbe7aeadaad92a23dca812b4cc7cb6b8adf112d25a32c9917 +libblastrampoline.v5.16.0+0.aarch64-linux-musl.tar.gz/md5/3cdbb6be01209e493d832fde53ad3c2a +libblastrampoline.v5.16.0+0.aarch64-linux-musl.tar.gz/sha512/4953e062df1356335f43e3219b5b297aecfa49f9d7686ee82c7148fb19b2da8da9a2f1b3130f200b554da561921f8e1d22387380a913c3ab5e26ea543de57bee +libblastrampoline.v5.16.0+0.aarch64-unknown-freebsd.tar.gz/md5/8cbbadef2a2a49a42306ab36a9286b00 +libblastrampoline.v5.16.0+0.aarch64-unknown-freebsd.tar.gz/sha512/de2596ac6b3e7a973d8e60195e2dd8e39e69e0ab4837179e7451d90176e5c6b69634c26ca6d39c3c925d1332ea9d4dfe1f3c7899ed9c122e62a6beba38a127a7 +libblastrampoline.v5.16.0+0.armv6l-linux-gnueabihf.tar.gz/md5/48b6cfdd2872ff105d9ec081d75f8a57 +libblastrampoline.v5.16.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/15af33cac3c3c2f9bb372a42cf7f501adbb7df1fe70ab991a589ade3dee79451638aac89ab8905cdc3da76a162f8dc17d9eeb213e1a4ff791552a7e2f40b1ac3 +libblastrampoline.v5.16.0+0.armv6l-linux-musleabihf.tar.gz/md5/f9b84577e582e3b6bf59deb9ca2c5080 +libblastrampoline.v5.16.0+0.armv6l-linux-musleabihf.tar.gz/sha512/1982ac1a9f061a5a09fd0f3f343926088231f1da6f79fd748a3e42a9b2a0a702ef1c79689c64bcab3342c47760c92ccd44a5ee0a032ceafade86948a066304ae +libblastrampoline.v5.16.0+0.armv7l-linux-gnueabihf.tar.gz/md5/5d3919ee3664123d79ea9f9b14598161 +libblastrampoline.v5.16.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/f1889dfb2a406ae8dfbfe91ba2979419b61bc7be4f7db253c861c5bed484805a7c27ea03205d42db6e5e5b4535447e72e46b47b2847f35364cad93134147f452 +libblastrampoline.v5.16.0+0.armv7l-linux-musleabihf.tar.gz/md5/a67a9b827b84a842f5fdf1eb497ef09a +libblastrampoline.v5.16.0+0.armv7l-linux-musleabihf.tar.gz/sha512/079cacada542d1ef4286f768f84a1add0e7e3a8eaf11bba26bb72f33238a898aeb6659db56b94d14084bd9c5525a9f9face1a5e02c1410b7215b8764f6b4b92c +libblastrampoline.v5.16.0+0.i686-linux-gnu.tar.gz/md5/22111d217269c0e3c2965d8dcf2e2b44 +libblastrampoline.v5.16.0+0.i686-linux-gnu.tar.gz/sha512/180820dd6695298b0fefa8fc1b67c1903b42d39665fc520a83024b6ae9b8258811ec6279b9adf54248ad5f62f8a3890176c8598fe9744db466be049453b63b35 +libblastrampoline.v5.16.0+0.i686-linux-musl.tar.gz/md5/6e28d603cdccf9c777c07ef1aeef8f97 +libblastrampoline.v5.16.0+0.i686-linux-musl.tar.gz/sha512/0b787d73a9371c9271ae50b6395d1faecd6120cd596e2ff9937ccf7b10d6483149cadef1aa4c9a66049ba28456292549c18a3da69d9d9c4b97a172fde30e40b4 +libblastrampoline.v5.16.0+0.i686-w64-mingw32.tar.gz/md5/ba2a596203a222791a6c6e9722f62849 +libblastrampoline.v5.16.0+0.i686-w64-mingw32.tar.gz/sha512/efb962c34740e4ed3346d94cc8473918d71edb15ae6fa41c62c0f1b0c381910273a8d1de9e2107469458f181daad62baea74ebd0dd5ee745caa1b62b0e4c00e5 +libblastrampoline.v5.16.0+0.powerpc64le-linux-gnu.tar.gz/md5/e1996eb15953ea214490b0d6305141b4 +libblastrampoline.v5.16.0+0.powerpc64le-linux-gnu.tar.gz/sha512/994f15f6381b2628d610fc28673a7c3f9537c7f98738b29895b8f81e9dfde39633bae5b20f9016433174f95c2e6fa2067f2fc8dbc39ba1c2bd8a564675590ae3 +libblastrampoline.v5.16.0+0.riscv64-linux-gnu.tar.gz/md5/d809fbc72b367b7af67cd309f9d18efc +libblastrampoline.v5.16.0+0.riscv64-linux-gnu.tar.gz/sha512/cf33f55e7b817be22fc2ab039e3bfa9f3ce101ba498ab9b47ebd1ec22982c669133b4d507c83795753866549ad74135ae8e3748db03ae74835756a0346302881 +libblastrampoline.v5.16.0+0.x86_64-apple-darwin.tar.gz/md5/9ccbe079af5758a33095fa473a6bf016 +libblastrampoline.v5.16.0+0.x86_64-apple-darwin.tar.gz/sha512/050c1ed7f6560ebca7c4ec0c1880012c84c3e018fc249eca11e7f7618119abec4c19e57c4ce18f7c9c0c4b5809940d7059dd1ddbff8672249580612bdfcce25e +libblastrampoline.v5.16.0+0.x86_64-linux-gnu.tar.gz/md5/1bd38055e5ef904d3218a9e9e6d9c37f +libblastrampoline.v5.16.0+0.x86_64-linux-gnu.tar.gz/sha512/2f101e7e755d13db40afcda40dcb7c543cd6d96771559c4d968c3d856b4fba55afe5358634af2c126d065c9a74ef795fa562916dd0bfe81ce12173faa986bcb5 +libblastrampoline.v5.16.0+0.x86_64-linux-musl.tar.gz/md5/07a0c82bc85a98343caf3e24ab9078b2 +libblastrampoline.v5.16.0+0.x86_64-linux-musl.tar.gz/sha512/33475409cf9034b099c78682b436b5f57c460dacb81e970c66d668cf5cd14a1988e4e79298e3e0658355ca421da1285eecf394e4b6d45fdfd10cdfaf459c8238 +libblastrampoline.v5.16.0+0.x86_64-unknown-freebsd.tar.gz/md5/9096434ef9adcb47b3edc3cdf54e469c +libblastrampoline.v5.16.0+0.x86_64-unknown-freebsd.tar.gz/sha512/09117a9c2893e68581978c23d31e70e1a5b82ac9f9b67836ecd627cfc5f3179a12096c1a14245baeb3b93458a5091c2932845b5e8763f97e8bc9b85fc7d2ffe4 +libblastrampoline.v5.16.0+0.x86_64-w64-mingw32.tar.gz/md5/ccf51f658adacc35f4aeec50d96f739a +libblastrampoline.v5.16.0+0.x86_64-w64-mingw32.tar.gz/sha512/a37b2100f0e40b691a6a216c6bac5b0d43cddf659af8b2a8b3d89ed7e84e5e873a0b387040629d69af1173b399e572ed017103514d936ecbc8b15540b8a2fbe6 diff --git a/deps/checksums/cacert-2025-07-15.pem/md5 b/deps/checksums/cacert-2025-07-15.pem/md5 deleted file mode 100644 index 084a7e65d544f..0000000000000 --- a/deps/checksums/cacert-2025-07-15.pem/md5 +++ /dev/null @@ -1 +0,0 @@ -ce594ef75f07eed66c538f7d4d83eafd diff --git a/deps/checksums/cacert-2025-07-15.pem/sha512 b/deps/checksums/cacert-2025-07-15.pem/sha512 deleted file mode 100644 index 7fb658508c700..0000000000000 --- a/deps/checksums/cacert-2025-07-15.pem/sha512 +++ /dev/null @@ -1 +0,0 @@ -743e96c112787f9b415310cb762ef9fec845534974e7e27bcec07472c346b1085fcbe89dad1f97699123800b37e29cb97f3fa1ffa6bb005af9743b4c74771fae diff --git a/deps/checksums/cacert-2026-08-13.pem/md5 b/deps/checksums/cacert-2026-08-13.pem/md5 new file mode 100644 index 0000000000000..69415e685f06f --- /dev/null +++ b/deps/checksums/cacert-2026-08-13.pem/md5 @@ -0,0 +1 @@ +b7a0a25670b6987c14b542d83212af4e diff --git a/deps/checksums/cacert-2026-08-13.pem/sha512 b/deps/checksums/cacert-2026-08-13.pem/sha512 new file mode 100644 index 0000000000000..0516413b9c821 --- /dev/null +++ b/deps/checksums/cacert-2026-08-13.pem/sha512 @@ -0,0 +1 @@ +5d9a91833fddac0f48bad1167d46dbcd882c43dc8ed1aba675b70aa444f8cbe74ae002c357b71a1d9771c60a0c01122c3531039da910a0242c50c9bad1b7b7e0 diff --git a/deps/checksums/clang b/deps/checksums/clang index 2d1b19d45c84f..ceb25e651e4fd 100644 --- a/deps/checksums/clang +++ b/deps/checksums/clang @@ -1,120 +1,68 @@ -Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/744ad2230a1594acee8781be442cbc65 -Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/118f67c2fbd6b11e3d850a38ea331f10403cbef563f22bb1102ff5960d1dcb3e264dcb36e71534828d422e669de1495acdc14c9d452991fb74b68810acc721b2 -Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/6b208a76bb52b83bdd3239c93fc37876 -Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/8eb5d93c894a830321f58ab18e4aad58a52895bfe8976728249758f41b439e249beb35990daa2fc635a4fd5369aa89e710771d06d7242ee7c34002a18c939629 -Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/69d3fd7ba2103fc9074af330e30b0397 -Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/f29d2552626a7f3a91a50984bdd396beae9beca65d119e3e25f427334723d40dea4672e27e6b2057bc1ce354eacda57dd06b6718347d9d075f455dbffbd972f3 -Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/778dcf8e95ff94a946726eee2987f3f8 -Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/31ac240f96d0081a20284de2c18c919ef74cb1734426051567eabedc18752f18550a1fe48ccb66fd89f3605a50ded57b61631aa48ff528c20a07dfb73b2bb063 -Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/4239b43accecdfc6fcb96317a84f267a -Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1df612a15d6e9f59235a0d97f8aec1b1252320077b8b3ddc5623bbfa531b7b089663f4c826e1b881401715a288cc170d79b9953ad987caabda234b3c1fbf9efb -Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/bc3e688489a6440a4eef0cb3731d979f -Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/7c0b96a9b927e3679ec79a45db388a3bf79d773ddc3d85da9f42a92c197c5e9886d13cbf00e7189e24b1d3a2ad942ce7c0cc50d3666a14c6e051b1d01e639145 -Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/770f427ecd0f6900eaab9bdcdb7b1964 -Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/8263c16d644c94ee21d5b46c0b515fad5b73dab7bd0fe658eef239bdd83b9c10e26546706cd22a892ea9d8d43e054123b04c68f586407634a4eb2826ea93946c -Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/c02058697c68d62d94b68392b00da335 -Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/ca4de5088f9f016ad0b782b9816a06ddd2dd2413e28c587cb8968308b89ef1295c7432de271c1839a0d436d670e06c64b7989f44f53ffbfd1ad82ca35ef910f7 -Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/edba25d64e92d9bcdb6380c92779ff68 -Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/2d080e0f159ebb3e91ade714343f597d0da9733c361ca998dca9cb436f5702604ca0c20b6ec547b230f77f963d232e38cd139f7ed4203c3fca96bfbe9818bbee -Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/414dd96ffffc43202293d0f2987ce478 -Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/41aeff9a58036e38c5d714ef03a7850b52b93dd84e78327075dade971027a3d714d0f3ede85144a4ec7b10449ce5d5dfccf6c1123c04a7da74abb5b25a38ca79 -Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/7c8b3eb507228e61f7251d7e8b6dd578 -Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/0acbd2fc5cc95830ba1cb2efa24b0a7b0bfe66c9f2592a8fd166a2318727346fa4827097601dbb9d915473b90e8ef5cd88c8d0c73947e27f8ce1df9a74879d03 -Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/74f4afcdc240f3e3c56edc6058f04530 -Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/b2b815a323994b3bf5a386d10d126f5bd5572ea4e883c5b28ad68cdadab4c8c5d4eb7ba9799eff97abe556de65f725c62426dfc44492240f444206f7ae666abb -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/ffa2f021083b5d27e9de06b618364079 -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/9295f6f080eeeba21570165435e937a7347867977b6a5e5abbc4be7150396808e938b27e7cef5b0ddd9d1e538a04e17af795fa3c3508f179ac8916157fc61f52 -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/4b33633c5f9dbdede7721f815c39b9e7 -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/a358b4898a34a2882029652769faa8a4ce0a2f5ed36c853ed40a5875797dcd14ece37dadd620a5f7a25afec1bbc426fcf5710eb9be54023c03cb0c50087ead9c -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/25076268c657945ed492d46c2dd819eb -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/e8bc9533ef8fdbee5eb938f9a764d69de52b47220d8ed97b9a23a7ddcb62206e54e479abeedc14339fb2082ed451f861d1b19eb93830dd0a06d609c8b85fdec4 -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/e0a95901306ab2c66a6cbdf86ebb0c5c -Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/8713ac54a84e44b1d565db8334befc271a8d5ca8646f46f75b43754c4681f433b8ecc239b3103d917ae28ad258c33f6e656db895d449a0b97c3af3adb3893ac9 -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/c52e8e1667745e3886223c214abe8191 -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/abb67bf8041cf61a81f37f109a30fafc595def65b11d5628a7a7cfd095b0c6b2303913a9149df7f1a2996893233f6f8152a33a26943368c62763efced7ef8e83 -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/e51f6b1a3bdc815c164440976b3eb380 -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/2633d7c1bde3bae99d81c2865644f15e02e935587b7cbadd63b3bc90eb9539dd455dfe592283574aa56030b70681b0e4e1950f17643b352e4053d63e61cfc93e -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/b1c191536aa62bb9f514b0f5615bd322 -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0973319c7d8a3f3058cf8b710afc0efc7d28679cc3182982bfb1feee431fb52eab88c3387aed5e351294e2bf4155775bf2b20ae6c35e21c5f123937140bfbf40 -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/2ddbb22f341c0fc79e02ddd2bc60abae -Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/e829147ede7726dfd018183b1c89cf976e2160cf33f5f39eda0f0c9389e31dbe52cc127cbc3f43c19253fcde84ba6d8bfa4eecf6d6a3d6704519b578e77778ab -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/739d03b8f5f11ef6ac54054fc767548b -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/af3fdce8bc0920e028adf5126dcc442f8f8658707f1f5b58ce4179dffe3351b3337e91908ab50a309789a568e0110930fe6a64ace2384e5631f960847ee696ff -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/1c1a6e5abd46f5b82d16af33f1f04055 -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/d2758bdc071eb5b5cd9bd6e0fb215aca8989429b6fbdb27a2874b1b3c7988624c311d9f71cdc76d8fda2fae113ada8d18212180ffdef80071ea25123686d16b7 -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/fc073b15200b2a4514289c1670761a32 -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/7be72926f46764b7b106e4a7623ec7cf54da57feb9b12b4310ca565324bd277b06197fadb3cdf4aeb8ae804b5e68c66f8d18d0b9e8c07a0b5f2acef6befd7001 -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/894be9034932a5ddd8868030470aa1fe -Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/7460e69dac9c61942494d4f2a48d3cc52e4741395b4185fbad72b418a86a825f4b872e80b89324e465a2b83e8fcf3532aa081f9706c29c7040c61ac6fb1071cb -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/1ad205872d5fee1e4d5821c6569ed8b8 -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/e3004ec1a7e1424e1dfc44f3b8f4e87edc1c9d490866bfb22fcbfc79d9494e5cbb73b9d7859de061ce2c8bd08715eb1b5d9b0ba23e0231f4a607ba7097370b88 -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/2ff49e5d12a9f5db5758deb116fe62ee -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/29ee0cb52dfe643f99f92a50e0f60d6fc35aae8311e0419106862b9d14c08ee162fe7b0a2b04a291176f72cb789d006eae3aa41a50e9383d6acb03aee99c2c9e -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/89c6e3312234835ae172ec9f7697b888 -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/44e20f9ef9dae0e961bd5b469e930cd4d701f228a48c867d0655a7eff220bc060a8e769a520af7dcfa1019bb058516c95ed9bd25564488264604609f76f60a68 -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/ed6b213ee57408a819912740dd5d2248 -Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/43d68c8712bff0fc260be31df1ccfc657d725971f5b115645d5efeafeaf10209401752c5d5fe0b0f3e4be003d68a2d75c08f97df7fb8d8aa0ed1a49c2b148a75 -Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/f813d4d256ec68b000be8e7dd24172bf -Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6a433e3459ae36b6a131aed5fde1bf35af7abf825d5d5014cd82be0117087fe138a3d3ee883c12a56cbfe5a0ccd797508dd38587abcf80bd1e06847a522c6acf -Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/1fe9b72f78ac84c375cc5ecbae608e14 -Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/51663a78e81635ae0a65fb0c7c4ebf66080fc919271736ab0895d3661686d500c6dfcf4181865ff6e0fa645a359a24224f84e0d2aa6c9b9a81431e4905f81cf1 -Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a3bfe33b1a5ecc815b81cc5719c5df3a -Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c7d341cc38fc895b6cd884bffdbeb1d2cdeda6cdd90fe3e85ee2fc909ea6e6c2267c1d165035f8f8c1ae7e5d96eeec61c65e803149f267cc1ea54599c49463ac -Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/b77d8afd01a62b31eb5f62a39ea7c834 -Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/141248c50ecb5178722c2be6540e12e7e62f99903d06bf7f1e401f9ea27ccb42dd146aa66e08c790e9fe663c167a3ea48809023daf95cf7e32fb0f177147f4a6 -Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/08b9ef36900ec33066898bcd173a5d33 -Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/1e2b9f4011873cdabfcd3cd154bf47b93da682f4ffc1bd41cadcee47e693f921ea62d7792f23b07793b2731434cf6cee91bd11432bab6f6b0ac8969aef507e84 -Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/c48a8ee67bcbb13a5263f79e757d2fbd -Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/5f24c4d8b76fbf9036bcc3344436a5906eb025a9949118780e4586a5da6afe3d5a326011fdc01bdca7595940d2eb94d668dabf2df65af33e31637d9ce9ca374a -Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/ec43a0ba8d6957e071936d85eb5e360e -Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/9a145a4ec60f3a3826001cbe4b089e7053e10a9443182145b44fb9b0a0c9051c0d277632e17eb989cfac9746441cbe3fd3d8e68726e48727d63aefdcdc1d012b -Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/08507753c774304274f1531c36f12426 -Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/73fde12aec5fa8a6cd1037767a24dbed2c496071b2dac6df98624d01316c3cde85f7fa176e9572b5b74c2f2a7010fb2545bc4b949d27291c926e749284ddead6 -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/a118c6ef27c7b69f2877b10f532bf2e5 -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/620727a97254c81d16ef4e1bf3ca4013aeff75c069f12de4b5d013e1df28e9d1804f2dcac95c116c19a537db39ce190064a2e39db9d2f10f5b83e93145ebf7f3 -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/13a9e37303fe1e793d2782690159403a -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/4a24a1cf630dc16c9dce2b9eedb7ef6327b82eed288cb76e42d227db9651694acb52b9c59b96736af7a19a5d4be122935e5694a0384184bf28d2b4419de8a9c6 -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/402a534e78daddcdf42966ba51696663 -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c3835b05b621b9a4f0b7a2aae44a7d53e64164010a054785ff4264a2a369d1f5b9df84c1a4e2522df55f240790cf1a939becadd13893c70c3a9139d2703fcf5b -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/9b7a6b511b728972c0ccc3e1b33855c4 -Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/cd9fa85402fbe04ee55f6b69ffb87d3a4923374c40176199d4ffe10f8e8aba54e24645ac30ad0e74ed60e9ab5119bd75f089d8dec4a132ba29a6960fbd0338be -Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/fadabaefeabb7c1df445473ed5ece280 -Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/aa6556a9d27703fa8b49d682c60a1ca843ae887b400c3ecdd3a78e8ef456b1feac5b1b46cedc6124f04760ca485d028defb404da808a96bdeb0b65a704928d06 -Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/c7dfffb39c3d193785b4b31ce58dab0a -Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/3c64e13b5846f21ab9796326f2858eadd77bb8825a840ed14383012e651f93c6d74fbed3cee9c324cd3d9256feb243dce15e10bd7962ffd668fe83c9772c6beb -Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a4596e099c5edeee4a204305b74be573 -Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/bae493823a3eacf6a6f7c0c53a78903425d8ea71e846e5d28b583ee067cf9c8dc7c0103f53f30b08d271f4867a2f3513a187c04b4e9136eb7c4fbe1334be5240 -Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/f3d54eaf5770a465a510c4370ee96548 -Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/cbdb8e0c1592f9c4bd8b1e6a15ed15decc92502009d11c75be7ca8840bea066f341e6bfdd4b60dc51e2098103425b324096870358bbca36f8e3af9eb4b1b9559 -Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/d697a3ea0444d2a583ee37b609ddd545 -Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/49651c7e38e41aac7645c7f8e8fd5f789b742ee8e945b59181675b40bcbfcba4aea8d8d54e7dd703ee40fc91cd76d02f1f7999ca040beaf49c2d605a1f76b818 -Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/43a0fa128b939182277f285b9d6f4007 -Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/c7ebdadf877fa46f55e2366b5ccc18499a15fb01888e6373f1596ba10530c2c7d60f248def10b6c196fcb5d50c9bf9e27bad4b7a0c3e5291f411ef36a604328a -Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/1a43bec0e1fc4cb6b94885d4201e75f2 -Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/76295abc2d2e1c4c2f56719f6926dee6002f09472266f77be2d097d2f2f1da706174215d9de59cf2f88807e6df3dd38050935e82f8e66b0c41f5cd8a1dc2ba2b -Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/c3ac86b91ee81032410e732722be1301 -Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/dd370f92c522ef8fcaf85c2a87cddfdff74fee92bcff8c3b1d5ae8e12bb3f90186ead91d22808f96d1262ca84806938506333fb3dcb5bbab51ca845de6d256e9 -Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/60d67e563b4f8d5660692da70a399b92 -Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/55c135642fb4b9e61eac6fca3cee9b2d0f9f83456f9670bb90428e1fc0c28642f285a2dfce9cfa32a585f9174645ab6b2ff8943367b9653792233f4c32a78b0a -Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/2448604ea7a346df8532511e36d9a59d -Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/6627693edb541e700596212ea580d18a064d8e765642353d4b5e474f394bd6d9120b781f5c02840bd327b084ce0228b05ff17a58bafb6161b933a789759eb0ac -Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/de3ab95295e91a8dc55beece1c094489 -Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/4b2f1d6026fd8680a5929de517a42bf84f66fc6cbd32f8becc8b0601d3b67de66c78ab67a94662184ba5becc28bc4fcc2644f17731b838b8de46f909e4773252 -Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/7d3eec07c5d71a2a90fff278116f5b09 -Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/8d837e4f24fc80f3860716aceb23961ef33d499cbb6ffb5bc89b422b795ef46c60a787038e79a1728471ee2f0807906b28e9aed7fdf3772fa13d99afda0d2332 -Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/82298efca03959f3c01c91d0da4b8b07 -Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/90bdf33dfb8eff0a3ad9cec730067900283a5e31f05a008a873c8673a603e8a3e704048729c15b841f0adc29c96daaf1085cdcb3029cffd7d42f8b99116dcc86 -Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/c9671bf1a659a52fa7866716472dacf9 -Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/7d6296bc6c44311b763d70a3a2617c7d1e7835a0df1c232b5de453611d71eb42dd43f659fa2bfb320a100d87594e71647b17c7f108772d444581daae567b152d -Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/e9457d3c788a66624a2d34c20d1c300c -Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/4aa67eec6c4f8db62fd88e568f7ac5a5b24ba5ae92ce6a779dac570a4b95a373bce2ca70b46ae1f919459e44697f9aecc861e9375dfa39511bfdb43b8bca0539 -Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/21dc84915705793f2c0045db6430391b -Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/d2a1d55d1cc04f4ea0c890dd52ac2c0961d64c3f9295a2ddc8b681f8a13cfd32b0b05d34aabbba5708fa9b0e68f5802e33267456dbe42154771d5c610d5d1f02 -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/37091fed96267a3865f837d637ce9140 -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/046ac9f2414eb9e163e7520d7d57e2a6caf5279ca6bb7297eaddf5b36b2218ad83ea222c276455111bf1b32e16a56e40ea419698ba2f18f93ec271ffda9ae433 -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/4fb68bfc1f784051b212f13f1c766aed -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/c32936a923b4af1106293e24ea07311bfa728b97609540f55bb65acca8d2fc937fb718796cc785fad2260c22631e09e96782ceff8a7e3b2b2659f6eb55e3c74f -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/edfca70811c32f08b758cab451b6d254 -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/8eec4b27eea12a40c77c47f19fc0ce50f88aaf83822dc76c274ee9b051da8b41d37bf4a7fd3d0a6124f1a6fd58c52ec86cd8ef72f7844a688554744add4daf1f -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/97ecfeef24bf469e594360dd1cbcc177 -Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/e6ee100c2a944dd7d6cca577f8cd01fd1e0236720310667191bfe09990a516b09d663889ab29d8819f8341906d397d3871c28f351c88c144de9d238215c985df +Clang.v22.1.8+2.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/fe44c16a8529027006734ce0e7cf044c +Clang.v22.1.8+2.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/97b711e34ab3b2f4d58f1e62ca4d38b66bf1ab0190fd927aec3b504d1f37e80d5ff7ed76637192aac2c113585aa4e14d2fbd85d601c49e5b95bb431c8e6a6d4c +Clang.v22.1.8+2.aarch64-apple-darwin-llvm_version+22.tar.gz/md5/73f6a4dc4556f7d4fb7da9dac4e02528 +Clang.v22.1.8+2.aarch64-apple-darwin-llvm_version+22.tar.gz/sha512/b7241661b137cae132b6576ecaad6aa37b7285158200433ca5cb638f0cb50bbe505333af3ad77f9df6538fda2c41d9940c256c015590809a79655d9d9a44a254 +Clang.v22.1.8+2.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/055ca4ec8415710a742c5e99f9173dd8 +Clang.v22.1.8+2.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/839cf77252d357573876b1f6a792d72a33a7e7ee8e5af0842a905834da3c0ed2a07bc4eb1765176bd2ae3fc3c93c1078297b713fa0b189119db1ad4adfa9c5e9 +Clang.v22.1.8+2.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/b361143722df8bb188e3332bd0fbc0bf +Clang.v22.1.8+2.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/583589feecd73513553028d308a0ee044909d21d5b1b2a8f9f19e598658a9ef87ef2b0d926a0cb63e2f982d3a38d51ed288a6623db7108297c5da25873cc912f +Clang.v22.1.8+2.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/4287698fb7d1886c44dd90b7e6ee5d36 +Clang.v22.1.8+2.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/5b23a52ed92313a59073864c7d86b9844d4594c215b554e3715696af57f74433665d2c6005f0eb6727611869e93e4eb114f529d28bea30c4dcb20926baa4623e +Clang.v22.1.8+2.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/646e4c8c7fd44ff7262f609372722737 +Clang.v22.1.8+2.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/022a4679a4bddb0bc983ced2f0999694e2e201b461abe67b7e3a0305e1e73194f35a1cc2283f48484f04c1ab685ec3f2342b825e678de1735429f6453a12112c +Clang.v22.1.8+2.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/4e10b2cc56e5a55f32ff6e0b7c3e4957 +Clang.v22.1.8+2.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/27d81c7a21fc759847faca21778d09afe528ff56155998d03bb032b5b49f91c3d2caf7f07cca660f48587a93fc00d40116468727cfcdd48ee605c30e0d58c682 +Clang.v22.1.8+2.aarch64-unknown-freebsd-llvm_version+22.tar.gz/md5/ccf14004e35e7ed29eec3a2fd452eecf +Clang.v22.1.8+2.aarch64-unknown-freebsd-llvm_version+22.tar.gz/sha512/11529ac6d9c37c09d78a9b9effd61820ea4b9ddb3b589891aded17bd07540ed7663e94e427e2f4812160425c5122141aea0c9a5d3719c7797dcb125fd2798bae +Clang.v22.1.8+2.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/87dd527806bc8386a2fa1f656a3e9a8f +Clang.v22.1.8+2.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/471effc21a39c4068f95c397a23f736ab3e4617b717470cb1d308d422bf2e3710ab8c81afb10306f620c74d3aaf078bcdbd22171fd7e6886b5d033a9e76b3560 +Clang.v22.1.8+2.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/0991558832546a494014c6393a672bd1 +Clang.v22.1.8+2.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/fab2701ec1d05fac40308d012c3f9d1f83544429c29679b7c93d4c0f236561454a15356bc743e6271655ddb449f9cc97c6c9decbd247261f72e83477060990c9 +Clang.v22.1.8+2.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/d0360dddd89ca73a1b71ed5fc666ed3f +Clang.v22.1.8+2.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/7c22c661400e9753266af93599e489ae3af22f75cbb193c31baf0f626761700f197a07a33bddf5da9c5b90c128003d54f2a87286d0a1058df5d74c63fd0ce657 +Clang.v22.1.8+2.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/e8f607a264b60702f82f163417a5c163 +Clang.v22.1.8+2.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/fe33cd9edd0a3ed025149dcb1fb93fefd0c1007645865115aee7f7b5962cb77da39f039c64ed395fa71c937bf84ef010767c829b9e373b8b50d45cb8651e92eb +Clang.v22.1.8+2.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/fd3df317543a40e0ff61449361cddd40 +Clang.v22.1.8+2.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/2abb304cce7615b0291a9b36f656b17719f28db0c16e74aa93938eb395f870c13b1fc757ebe9562c9a334003be3aeb54009e8606916f31b141fc1300cebabd11 +Clang.v22.1.8+2.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/dda2aa21ecb75c9bd756956bd022cbf0 +Clang.v22.1.8+2.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/b76c59623d87d655e14424906e328103a53ced8e5df72f05d9755cdcff881d50d4a6a7b2d5c6f5d1398f5ad8b5e48ec22c79c927918cbc6dc1e485fe3a15d1d5 +Clang.v22.1.8+2.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/cf50a11e791634fa4a0b10c591c60336 +Clang.v22.1.8+2.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/17863cef56a04a7d3f9352161687081048cbf498e6ed13aeb83551be3b2b05763c18b78170bb38defbf4ee05048b9049af6d680d7128e6b32400927edec7448e +Clang.v22.1.8+2.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/c120f179a633dcc671939a9ac9f91d62 +Clang.v22.1.8+2.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/dc393cc0bd560018bb31eb0a831c1b197d9b1f9a47fe1915e84ebcf6f586d7bf21846bab445b35eec3079693bcbcb2edf8d78d60c74d56662fa720772909add0 +Clang.v22.1.8+2.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/6127aaa1ca2f3281a693c06fd387a6c8 +Clang.v22.1.8+2.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/20c34b589d8850e8b124e4da9217f95e6a72e214cebe49041b0d66f3555140b5c49e3ea5d4854dbf91a1a303fb9afc61fca6ed64428f1d185432c79741aebe79 +Clang.v22.1.8+2.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/557cd74376ab41f1dbfb67406aafe3a6 +Clang.v22.1.8+2.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/1286c05467debc88a33319ae0f36e9e49557d09475150572d7cd9f489897ad73c3fcf5fc5b069b5ce8df65212c9e563a2434229945d52a1e957af2743ea2703c +Clang.v22.1.8+2.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/74ce3637a39663e6947da80f57e4ceda +Clang.v22.1.8+2.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/c6bb14ade57dcc7623e3392add3bcc53e82535199c5c7dcf1fc5d4c6a9a5f223b8ed4c90d69f35cd45ae552fccd6dbf0201a6697489f8e125cf6591e05723a77 +Clang.v22.1.8+2.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/d029c8f7f5776657753f7112eef5817d +Clang.v22.1.8+2.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/f1663017f8386c836ebb9d4a58ed52c3b181bb198be193e251aba190928edccc490b555169234b44b58c9721aa76c2e5c157b5d7e4a7072e428a217047dd9a24 +Clang.v22.1.8+2.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/250a6523e0741ac7fb8de8c1f7fed381 +Clang.v22.1.8+2.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/bccbcaed12b0fc9df71964eed55e28de1593f68570b4ed3eb95024682c4e7aaf0aadccf4d13a8e3992e109ffbb4934ddccaacc0993b217ffcdb3a2986678cf4e +Clang.v22.1.8+2.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/2e022758021b7f50a09d467041b5f5db +Clang.v22.1.8+2.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/9bc91036f4df8b2a177bb07ab0cf637f825723ebef3649e53eed2137098933751866948fe26d03abacb2350cbe404374e42845a604566a5f3a790652511786f7 +Clang.v22.1.8+2.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/e3174bb3f678fd85189fbeb10305dfd0 +Clang.v22.1.8+2.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/f6fb9d4f825a30dee4ee29f7a22ef23b1a77b6ea8ae1de1c1bf78658f4a5a61948b66837ce633a0b7cbe6ed98a3f17791f8976239565efa0f524bb158ec2e97a +Clang.v22.1.8+2.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/dd791babe9127ec76c730a50182dfe62 +Clang.v22.1.8+2.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/89466a0d0e4d9b47ebd62c17e1de4ac72debe4b8761f1765fd2c799d1576e9507bb9cf794d3cd474d0f77435e63d6fcdfed39ba2d3babaa232ad93e2b9a9f1b0 +Clang.v22.1.8+2.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/761d5c467a2631ddeb411be3708ebe4b +Clang.v22.1.8+2.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/108a772bf6a014bb83c9800e4ceb302b4f97af533a0ccdb0d55f93c1148bf0da9c7a0c37f1fe5e35d7fdbe844ef0c118d7efd71f432a35b0edd50437819ba591 +Clang.v22.1.8+2.x86_64-apple-darwin-llvm_version+22.tar.gz/md5/373bf066633261c19f3998557d4b680c +Clang.v22.1.8+2.x86_64-apple-darwin-llvm_version+22.tar.gz/sha512/94d266d3b8c3507a2fdcdd439d49a1fcd72b8bc3dbdac86399047edfd70ba882560bc92051951280205e93af0e73a6afba00132232f71fa14c4a78cf492af72f +Clang.v22.1.8+2.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/8b1f2377441698c8506a4a7482f6f7c3 +Clang.v22.1.8+2.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/5702f0a8d3afd39c4aebe09faf8b0f29087c299ec7ab30a5ea086be069f1f93558b054a800411ff963f0066c0d4fe15a36259709e2c56307271ea7fa60fe14fc +Clang.v22.1.8+2.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/c02e8f5e1acbf0c28e690d7e7999bbe4 +Clang.v22.1.8+2.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/09760bbfe05f35f64f014c37247c979e261adac18e89920bd9044bee83b6bac9b5a0395bae23bb5f0148dc0357bcf7a5d052f9ecd3fa7093e7565f22c75aa68d +Clang.v22.1.8+2.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/034a8fca8713116ed677b484f97a2df0 +Clang.v22.1.8+2.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/040d24c9c9117b075f2fab2a452768d8c96c8bde901e422ab61a2e968c68c6315e34fece60242d0cccdc7226279adf267cece901b2b8b64b2e17606e7f612776 +Clang.v22.1.8+2.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/4f1594f30b05b21631469ac8361561ff +Clang.v22.1.8+2.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/419f6e97ec1fd1f0557cf03a0aa2c26bdb35212ef4c357310b49228e04fd6edd260c46e2e3d51e22f038eb31c7cc7186fe3520db924aff2e1903d5680368421c +Clang.v22.1.8+2.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/90e70bec6b6107371260eb605e4c4131 +Clang.v22.1.8+2.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/29f178d7d4b4f52af88c97cc1e8a0fc8145afba4000c373502ab3d6cc9b5cde1727587b7a79b592661942dc98575c61f309f9aa06f763fed8774f4d11e06ba4d +Clang.v22.1.8+2.x86_64-unknown-freebsd-llvm_version+22.tar.gz/md5/cd593aae23b92ec66857ee10510dad93 +Clang.v22.1.8+2.x86_64-unknown-freebsd-llvm_version+22.tar.gz/sha512/10e39327a3e1a5e9745945df34588079713a107150bb79376a1e3048c5823caa745b2643adc4200a5289def091a2f1c6568504aace89d5bbc419d0e8adfb6c1a +Clang.v22.1.8+2.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/6f77687fe73ece835b36e7729a93c695 +Clang.v22.1.8+2.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/4515bd0f839d2be7235c5ae5160d4a2c0ef27965366d88a025a25cc62a2c7ca368deea6bef29e65429f39c98ef1e5ccf1cb101d23f25cadefc2869fe3e5f4f56 +Clang.v22.1.8+2.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/9608e992e690cd702b861e10220d5e5a +Clang.v22.1.8+2.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/43df39821fca64dbdb84bdeeab02978386d0e77f8206318df981920c54fb4bd16ddcdc48a978d1b43f744ad2b7a7752c41c2b15aa6759c886d3142e222120c0d diff --git a/deps/checksums/compilerrt b/deps/checksums/compilerrt new file mode 100644 index 0000000000000..eb81dce083286 --- /dev/null +++ b/deps/checksums/compilerrt @@ -0,0 +1,26 @@ +LLVMCompilerRT.v17.0.6+0.aarch64-apple-darwin.tar.gz/md5/b508ae202b6ea65272b2bbb7411bf54d +LLVMCompilerRT.v17.0.6+0.aarch64-apple-darwin.tar.gz/sha512/6d611fed19a2a35e5f7577ed3fe5c26f34dc27cef901e9363eb71f93d9568a2a013822e01c349ae02c5b2b021c5c7f20e251a85469761237a0eff319ce0cc97b +LLVMCompilerRT.v17.0.6+0.aarch64-linux-gnu.tar.gz/md5/70e3d4a12f88e30f3681634eb398a636 +LLVMCompilerRT.v17.0.6+0.aarch64-linux-gnu.tar.gz/sha512/3030713241c71b199d03a69495a9b3fd2eb42a51750df736cf884f375dabee3d3f87a9c4278f4d4292ae87248669e093594c0b557fa3e19cef6df5a1e0ecd9ea +LLVMCompilerRT.v17.0.6+0.aarch64-linux-musl.tar.gz/md5/70e3d4a12f88e30f3681634eb398a636 +LLVMCompilerRT.v17.0.6+0.aarch64-linux-musl.tar.gz/sha512/3030713241c71b199d03a69495a9b3fd2eb42a51750df736cf884f375dabee3d3f87a9c4278f4d4292ae87248669e093594c0b557fa3e19cef6df5a1e0ecd9ea +LLVMCompilerRT.v17.0.6+0.armv6l-linux-gnueabihf.tar.gz/md5/70e3d4a12f88e30f3681634eb398a636 +LLVMCompilerRT.v17.0.6+0.armv6l-linux-gnueabihf.tar.gz/sha512/3030713241c71b199d03a69495a9b3fd2eb42a51750df736cf884f375dabee3d3f87a9c4278f4d4292ae87248669e093594c0b557fa3e19cef6df5a1e0ecd9ea +LLVMCompilerRT.v17.0.6+0.armv6l-linux-musleabihf.tar.gz/md5/70e3d4a12f88e30f3681634eb398a636 +LLVMCompilerRT.v17.0.6+0.armv6l-linux-musleabihf.tar.gz/sha512/3030713241c71b199d03a69495a9b3fd2eb42a51750df736cf884f375dabee3d3f87a9c4278f4d4292ae87248669e093594c0b557fa3e19cef6df5a1e0ecd9ea +LLVMCompilerRT.v17.0.6+0.armv7l-linux-gnueabihf.tar.gz/md5/70e3d4a12f88e30f3681634eb398a636 +LLVMCompilerRT.v17.0.6+0.armv7l-linux-gnueabihf.tar.gz/sha512/3030713241c71b199d03a69495a9b3fd2eb42a51750df736cf884f375dabee3d3f87a9c4278f4d4292ae87248669e093594c0b557fa3e19cef6df5a1e0ecd9ea +LLVMCompilerRT.v17.0.6+0.armv7l-linux-musleabihf.tar.gz/md5/70e3d4a12f88e30f3681634eb398a636 +LLVMCompilerRT.v17.0.6+0.armv7l-linux-musleabihf.tar.gz/sha512/3030713241c71b199d03a69495a9b3fd2eb42a51750df736cf884f375dabee3d3f87a9c4278f4d4292ae87248669e093594c0b557fa3e19cef6df5a1e0ecd9ea +LLVMCompilerRT.v17.0.6+0.i686-linux-gnu.tar.gz/md5/c6dddb7fd42b5a2dc1c0d62d09225f07 +LLVMCompilerRT.v17.0.6+0.i686-linux-gnu.tar.gz/sha512/715fb9a6c2b190e44853104b1a5086578d9c2a85fca8122a53b7af4b4bdff00184784458409afc24e9154b4d0edc2984079fe64835f305cca6ac69a3ca3c92f5 +LLVMCompilerRT.v17.0.6+0.i686-w64-mingw32.tar.gz/md5/5d7d9af75d21571718d4168c731076ce +LLVMCompilerRT.v17.0.6+0.i686-w64-mingw32.tar.gz/sha512/f9af3b8bffcd2a75d2afb4dc048032031f3b032534e47945d6c601ace7b843e1debe2aa32e41a13be58442363dc59ff9bb867217e653f1f2fd6ef2958ff113b5 +LLVMCompilerRT.v17.0.6+0.x86_64-apple-darwin.tar.gz/md5/4f9841236585cabfe9aff7ab2b04d9b4 +LLVMCompilerRT.v17.0.6+0.x86_64-apple-darwin.tar.gz/sha512/eb9c93e9e153cb647c186c091e7836c59a4608c020bdc5de029c803efbe2913ea034c8858e683d6b45baf7fca2839a3f71806ad78879839153cfc8fc4888629d +LLVMCompilerRT.v17.0.6+0.x86_64-linux-gnu.tar.gz/md5/2862138606c4b3002691b03e7c74ff91 +LLVMCompilerRT.v17.0.6+0.x86_64-linux-gnu.tar.gz/sha512/365d0af5da59a47c83746fa16bf2e2e9f18a76b9b679ce4ed9ad154260b5c1708dbc69bf24572f3a51ddeab664f58a3ad660cd076c5af7725c280e1bc6677945 +LLVMCompilerRT.v17.0.6+0.x86_64-unknown-freebsd.tar.gz/md5/cacc06839d78420b4ca6ef20684430b3 +LLVMCompilerRT.v17.0.6+0.x86_64-unknown-freebsd.tar.gz/sha512/a9bc032109fabfb4ee262b5d4dc1a91fa791190cfcb17a22ca2f931c74270e9125e9d7fb122d253a42b5854f1a978a32b2213fae83701028dc45f8275ad8e624 +LLVMCompilerRT.v17.0.6+0.x86_64-w64-mingw32.tar.gz/md5/48059987872c5102516a549ff816c38f +LLVMCompilerRT.v17.0.6+0.x86_64-w64-mingw32.tar.gz/sha512/2711c719ae098ac35a894a91f0ab3cd817b29784241786ba7d94af3b6a335fe835fb4e195b25d875eb2c992d3f2725552421e72678613b3fe9715d5fe8822a99 diff --git a/deps/checksums/compilersupportlibraries b/deps/checksums/compilersupportlibraries index 08802ca1f4de3..32980ed3d7108 100644 --- a/deps/checksums/compilersupportlibraries +++ b/deps/checksums/compilersupportlibraries @@ -1,98 +1,36 @@ -CompilerSupportLibraries.v1.3.0+1.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 -CompilerSupportLibraries.v1.3.0+1.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-gnu-libgfortran3.tar.gz/md5/c679907ddce62f21bc30667cc40d8d52 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/284b17b6634087f0b969d3e99b2e4152667ab5eb9e6b5813f9739bd14ae1c25dba01f15488e901ca5fcfd780b02bc02b6bff670fefed7d965dcb585e81b03782 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-gnu-libgfortran4.tar.gz/md5/1b4f6efeb83f5f3e27c42eddeafe993a -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/b1d5aa122b2bd25bcd1ce47e000f71785d617d77f44acda56f9f5ad77101a0c54f6c6a4c5560a7c12ffb8c89ae325d4f056bd92f893d219385c3d5c85aa457e9 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-gnu-libgfortran5.tar.gz/md5/834adb105f78ac1bb223ef309dbf7cdc -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/dd0440805145f1a8e8521633a955317567606bf2e3725a5a7eb90515128b077f2163832ab608022fab152526f2a55991f50256ab92104d5d62bbb8a740e25009 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-musl-libgfortran3.tar.gz/md5/d613881e48181bb8ac0bf34a456c9736 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-musl-libgfortran3.tar.gz/sha512/34214bca9f7c66e3c508b2f9d88cb296695721cfba0c001660e2edb0387a2efbb2fecb0360f8eb2b1d0ec502480fe63e802f350367498a342c455c0f58aadd82 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-musl-libgfortran4.tar.gz/md5/97e4ea4394df1d784ce4de3f75aed580 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-musl-libgfortran4.tar.gz/sha512/a072ceece6600b704dae5a7491f1ead9b4e11da3d4438b7056f2c71e59b0a37d3023fb812cbae205a4f1fcaf18a4b223a5ba2cea32131c5eda0d55b1f0649c23 -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-musl-libgfortran5.tar.gz/md5/df09c5b33b2e307e0d9c2b39b450c0eb -CompilerSupportLibraries.v1.3.0+1.aarch64-linux-musl-libgfortran5.tar.gz/sha512/d0a8dc03ea1667d90bd58c2376b575a1090a54a4412bc53b311a3ea910c76dc698be5ca1078e6ca8341244f1fd6b84201ba10c10baba194c1d6c3ffb7e69563c -CompilerSupportLibraries.v1.3.0+1.aarch64-unknown-freebsd-libgfortran4.tar.gz/md5/4f9b257eabaf0a817755495cfbf75088 -CompilerSupportLibraries.v1.3.0+1.aarch64-unknown-freebsd-libgfortran4.tar.gz/sha512/876036a8b599008512ab7010f4bc5f11fbf963bb9b9f77499adcca21fcad89f94180f653dce3121e5c1206f4fd4ace717ef8f3b40d8009a71039a84ae7272588 -CompilerSupportLibraries.v1.3.0+1.aarch64-unknown-freebsd-libgfortran5.tar.gz/md5/0d296a080921b54d959a2a60884b7938 -CompilerSupportLibraries.v1.3.0+1.aarch64-unknown-freebsd-libgfortran5.tar.gz/sha512/d8232dd9131c2890ea2f5c90f62c646ea1dc93a0a6de4af0a98c7e69928c5ca5698e79ff9d23bdcf47de1f5670467c9e8fed5f01e82e009696641896f0658030 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/3e0727a3813c699b6daa041e336d6e13 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/00cc2c34af7c4a5df06deaff27dff5b94b231ede4afe7a47b7b783a8d2e62158c0ba1b2062d40df949fdc0a21ac703f8c9011f998ab032bac265aef153cea012 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/b7cb328b5e5fae5b5e456d058f5c18b7 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/ff191595bdf4dfb2cdd77d42e591adc0b27ca0e1055efa7fb25fc06784f26add83e6c5c7594405bdfd715f9c8e6ae3f2171a50ae218b4b691099da754fe9bedd -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/46a3fc18a65e223ba59d984f99d42979 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/cb470147f6695b101d497bf2d84caeb1f97d967bf23d1844ad70be47505588d981df096378136a98c35cda5aec090255d60cf7c1c8def9801233c72ca002b563 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/28e1bc0fb0ac1512a8598f26ee3f376a -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/7ec17859790cd08942929281906918e4c69d7f306a8302dcd591a4a67b3d95f7f72f7afbeea3a86a0d94ca5b608b3bda00ce43b594e9f173edb0228c0f79ba49 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/f9bff1a49d95fc0f3ad3d4a90b259c87 -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/8ad503a213b949f569c5b9eac28e33ed51cc55298bb66b147375dc12cb9ed90e60165aa2dca8e3d28f1a2c153894a9e4672bdb2ae3cfb3a67b1e06b345cb454f -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/d550a4dac1b20606681a56acc00c01ad -CompilerSupportLibraries.v1.3.0+1.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/14ca10ad8809dfd2434e300ad5280915f21cc1ba159a9f4aed7aa2164ae624687a2a7a9e6dd99abcfe95f40cb037c72292c992f4483fa1affcf8a9b5cf29c9bf -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/3e0727a3813c699b6daa041e336d6e13 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/00cc2c34af7c4a5df06deaff27dff5b94b231ede4afe7a47b7b783a8d2e62158c0ba1b2062d40df949fdc0a21ac703f8c9011f998ab032bac265aef153cea012 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/b7cb328b5e5fae5b5e456d058f5c18b7 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/ff191595bdf4dfb2cdd77d42e591adc0b27ca0e1055efa7fb25fc06784f26add83e6c5c7594405bdfd715f9c8e6ae3f2171a50ae218b4b691099da754fe9bedd -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/46a3fc18a65e223ba59d984f99d42979 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/cb470147f6695b101d497bf2d84caeb1f97d967bf23d1844ad70be47505588d981df096378136a98c35cda5aec090255d60cf7c1c8def9801233c72ca002b563 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/28e1bc0fb0ac1512a8598f26ee3f376a -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/7ec17859790cd08942929281906918e4c69d7f306a8302dcd591a4a67b3d95f7f72f7afbeea3a86a0d94ca5b608b3bda00ce43b594e9f173edb0228c0f79ba49 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/f9bff1a49d95fc0f3ad3d4a90b259c87 -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/8ad503a213b949f569c5b9eac28e33ed51cc55298bb66b147375dc12cb9ed90e60165aa2dca8e3d28f1a2c153894a9e4672bdb2ae3cfb3a67b1e06b345cb454f -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/d550a4dac1b20606681a56acc00c01ad -CompilerSupportLibraries.v1.3.0+1.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/14ca10ad8809dfd2434e300ad5280915f21cc1ba159a9f4aed7aa2164ae624687a2a7a9e6dd99abcfe95f40cb037c72292c992f4483fa1affcf8a9b5cf29c9bf -CompilerSupportLibraries.v1.3.0+1.i686-linux-gnu-libgfortran3.tar.gz/md5/73e14b94dc74d17aca38a51ad402f836 -CompilerSupportLibraries.v1.3.0+1.i686-linux-gnu-libgfortran3.tar.gz/sha512/d37263a216fb3e9b94dd032642ed6bf5be154a5c66de3e4bd74e5e2059d9740958a673796eb652ca9ebea8ec09a7eec837d8906a50775913325899aa190808db -CompilerSupportLibraries.v1.3.0+1.i686-linux-gnu-libgfortran4.tar.gz/md5/23996e5c6690b35e7c36bff245f6f4d1 -CompilerSupportLibraries.v1.3.0+1.i686-linux-gnu-libgfortran4.tar.gz/sha512/660dc4866a13f9a1ae98424b605723b250218a034e02151d4160d58ca07bba4fa1390e99e7fe2f31eccdd518d1ac4c5f5454968ce52525e3a2d21918b6b5bba8 -CompilerSupportLibraries.v1.3.0+1.i686-linux-gnu-libgfortran5.tar.gz/md5/af836562cfaf76f0728be0d875d29ae1 -CompilerSupportLibraries.v1.3.0+1.i686-linux-gnu-libgfortran5.tar.gz/sha512/a2b10c2f72d1e84c7b496b7ad6d38629342c93cd6a7f691e5bbe96ce28ef40fd38509d382d22208e40cc4953e7b93d1c211bf59529db0ad1a77b684ba75bc68a -CompilerSupportLibraries.v1.3.0+1.i686-linux-musl-libgfortran3.tar.gz/md5/502f089e5ee03b3a290ee6e18577a22f -CompilerSupportLibraries.v1.3.0+1.i686-linux-musl-libgfortran3.tar.gz/sha512/13a97c2386f37aba2416ec35fe67b99a1eccb880b0254ff0a70f2ba01a01a15c80251606ec7eb0503d59a7723542b6b9778d6c9d9e4ba66ae5cce51e46a9cb40 -CompilerSupportLibraries.v1.3.0+1.i686-linux-musl-libgfortran4.tar.gz/md5/221aa40c278faee74ab6af46686d68d6 -CompilerSupportLibraries.v1.3.0+1.i686-linux-musl-libgfortran4.tar.gz/sha512/9e4e598c8acdecebc812555de9631f022f6158d679c329537e37f83c76c818f31476a5827924b5ac12978515d64a7e913f220ca75314f41d3227573e9a2ac9af -CompilerSupportLibraries.v1.3.0+1.i686-linux-musl-libgfortran5.tar.gz/md5/c364ec196e66dd5eadc3932b208a0385 -CompilerSupportLibraries.v1.3.0+1.i686-linux-musl-libgfortran5.tar.gz/sha512/3f7b80fb35a967d9354c2f4c40bb6d62751a0d791aeec09817cdc278393cacef089214f61d8338c0981f7a4ed3144d37bc9267cf0e7ce6c4cf651bc67c431b70 -CompilerSupportLibraries.v1.3.0+1.i686-w64-mingw32-libgfortran3.tar.gz/md5/4177f1ede00d81472bb69888f5b3e26f -CompilerSupportLibraries.v1.3.0+1.i686-w64-mingw32-libgfortran3.tar.gz/sha512/09ab710599d237ee35fca6a39b4d67b36bbadb7d127797724743026eae72319faa161755b03f4cb67c83f801aa4132968b561245487b2c2c0836d0ff867c0e83 -CompilerSupportLibraries.v1.3.0+1.i686-w64-mingw32-libgfortran4.tar.gz/md5/9402d280886784bc245096bdc838fbc6 -CompilerSupportLibraries.v1.3.0+1.i686-w64-mingw32-libgfortran4.tar.gz/sha512/60e72336efdd307b88b1e6db5234388ac1892504ac858b412d18f072a33ca1aeaf1b8621ccf43027508b7a4653150f0849a89c57164beb1e7f24ef32f7fb7f11 -CompilerSupportLibraries.v1.3.0+1.i686-w64-mingw32-libgfortran5.tar.gz/md5/310b163628e7defdfa6a293360b203db -CompilerSupportLibraries.v1.3.0+1.i686-w64-mingw32-libgfortran5.tar.gz/sha512/02e9a797246feb9c4b09b0c67c773dac5c3bb61568bdd48be147adeb2dc08fd2bd7151f2293e2756685d011e463e39dc5ca0f79593dda7501cacbc15adfc74e0 -CompilerSupportLibraries.v1.3.0+1.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/c139a9d54f39701e805d2af185a6f17c -CompilerSupportLibraries.v1.3.0+1.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/0d1f29cb04b42b276edd7998a02f6166295f6b7a2a8ffdf6b2986145476385b19c2f93b012974835363ef57f2018bdb80814adef3b72b9378f0d2c6a8805c43e -CompilerSupportLibraries.v1.3.0+1.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/3ab360133835e1d0a6a24bb2de1dde02 -CompilerSupportLibraries.v1.3.0+1.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/9c2f765b58a73b3705f787f68c995d8f2cbd211978c0ec8ac2adbfec6685f4b3a02aa63bf75b9dbf0a2a5c048e35536929d04b89c120671174d76132cbd2c7ed -CompilerSupportLibraries.v1.3.0+1.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/6ce9e27ab33b35900d8f81c2ad05eec2 -CompilerSupportLibraries.v1.3.0+1.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/1d8af2664e68d18ef4f68b9fed28979af0acf3dd09c8064c4b25b3268808bc6901ce727b5b3ec3c27e37914a7c1f8c92e5ce35de093d66cb6a2e98ad59c2252b -CompilerSupportLibraries.v1.3.0+1.riscv64-linux-gnu-libgfortran5.tar.gz/md5/6c292cf98c6b4cbf10aeb4f0af383222 -CompilerSupportLibraries.v1.3.0+1.riscv64-linux-gnu-libgfortran5.tar.gz/sha512/1497789d918d633f319f89a04241678602d3b0f441ca6f8f6d756f6d1fba59d5eca54fd24183e39e9b956cd3c053afd747dc03a9a1e2d4819d26de3539c5eb07 -CompilerSupportLibraries.v1.3.0+1.x86_64-apple-darwin-libgfortran3.tar.gz/md5/0aae7ac19dade024e0228bb1a3565edf -CompilerSupportLibraries.v1.3.0+1.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/b779badad7e6021875b5df04793445b4056d84cc217f389f9496d8ca61af71d98a667ec29b912131c83319be4d6e82c59e7c3f409f302cc3691899f0e77edd46 -CompilerSupportLibraries.v1.3.0+1.x86_64-apple-darwin-libgfortran4.tar.gz/md5/6fcb9749463a96504f1e23cd97695f60 -CompilerSupportLibraries.v1.3.0+1.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/66d4cb8237859234f8fd49461b5976a7f155e02fb93c765208701c43c041dc8693f3f8b868ba74bd28614586c0f5109a5b5aa0d0d69ac38732ad6d84d2635e04 -CompilerSupportLibraries.v1.3.0+1.x86_64-apple-darwin-libgfortran5.tar.gz/md5/af01aefc789a0388df504abae68fc01f -CompilerSupportLibraries.v1.3.0+1.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/444d1d54fb6ef95f2093894c685a4065e9708504b820bd9325bdf32619eac8b2972b1601e788ff8f1ee2759117b726c04c8bb395820359bdc737bdfdc3c4026b -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-gnu-libgfortran3.tar.gz/md5/df1c55a47f9faebf09ea093d5d1ee344 -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/03477fdf14d8dfce204999e6825d9ad94c2c78686d59b251f39d1bb357b3c9d9a74339c4d5f5e97420870d44f7bc2fceca637fbf7b862d0d1cf04a19a2a0b036 -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-gnu-libgfortran4.tar.gz/md5/8812418d84c2ac289d64a597d4968704 -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/4da50ea541c13a98ae05c6ff67b8021496b871a205f994605f0230a67eb6c58ede55aa3a471df8bbdd5618177d34914186cfae106664b80a7fef795e5fe97e8f -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-gnu-libgfortran5.tar.gz/md5/55bd8dacbc4afff6196494542ea13eec -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/2bb63e68f56818c0a3bb988f395ebcbe99af2740f806e324c385c1dcd7a5dbb058afd286fb6d85a1621ca668aba962a8701bef96a4547b0d22f92d9e4f4b51cc -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-musl-libgfortran3.tar.gz/md5/1e06592e53de4448b0712a79e61b9a51 -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-musl-libgfortran3.tar.gz/sha512/cf92bbc217a51b9a18e07c5b5248ac5f59f92a7924c5fc566a1bd5b87a1acd36ec9c1d64871b273f80670596c05c1795cec91294f32f8dc1490633ea6d543037 -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-musl-libgfortran4.tar.gz/md5/fa81135fc7e697eb8409baf3fcafdcb6 -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-musl-libgfortran4.tar.gz/sha512/32ab98be0521f2451ce9b71c5ce7dfc70094583df80ed8db3990a2041594839f065abcf6c847fe6b8293eac3b3395da16ab3d24cf5e15c962aa320b28a6cd4be -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-musl-libgfortran5.tar.gz/md5/d6f0a90da74eaf2f9bf4f7b884231a2a -CompilerSupportLibraries.v1.3.0+1.x86_64-linux-musl-libgfortran5.tar.gz/sha512/cbbbd0284799f78cf20a41f1b2d110651ee0460f0191d519d522a5034a31edaaf62ef130e7ef42c28882e224a4f997f0bead5b569254cdda7100b1f41e286b78 -CompilerSupportLibraries.v1.3.0+1.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/c525e70e726f0fc1c49deedd08ab6026 -CompilerSupportLibraries.v1.3.0+1.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/eb50d1443e1d13b892c141ac579b2e807f346d98a75e2ce9a0a23494c754b7149d1900046f5c39e324b48bfeedc6bee590a7e2c182e6f0e3c07b9f816fcb9d6d -CompilerSupportLibraries.v1.3.0+1.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/9777c3216792efd8e8625f5f72442be6 -CompilerSupportLibraries.v1.3.0+1.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/215398a9c893a5298101d98a3cf3df1e59e6dd4b0d66b3cdcd9decd8725541ae33c30d1e391fb51d7aaaa33dc5911511257f7ee7e3ea6350a8942ae70fcb3ada -CompilerSupportLibraries.v1.3.0+1.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/c7571567040d646935234b51c121745b -CompilerSupportLibraries.v1.3.0+1.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/159900879d46eb2a2e45f0bfbf6eb7b03c1e28705d576ad712f67a3ae242e7e4642c08f3be181b9fbac659e1c76de6ca278ad3662fd15e8371adc7bf19e9e9b3 -CompilerSupportLibraries.v1.3.0+1.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/63187354746bbcfd43c11b8047595d21 -CompilerSupportLibraries.v1.3.0+1.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/7c004e5ee255a9cc410b2f8f8836d0dffae8f4e35552c57a74a9c2eb8dadd6f0966ffceb296fd61c5c0ad7a0ea25c80ee2d7bd80ed3ccf1305f236b64e2dad5a -CompilerSupportLibraries.v1.3.0+1.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/64f5d316b2d694dbdb2c96e557482de8 -CompilerSupportLibraries.v1.3.0+1.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/e1b3be2e6e9d4ccae55ec131f6cd51a7c4391639365057f7c8ecde539c9f5fa4d73942cbc2d06c62f43c2e1bca0469862a9ac6dc064536400ec09f37a20e2b1d -CompilerSupportLibraries.v1.3.0+1.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/d10bb5d9facb9428c22f920798876f9b -CompilerSupportLibraries.v1.3.0+1.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/7671d0a7e1d965d0dfd56e3f037dbb47a2748cbff2656be26741e1b15687b3ba48bb44e7d43e005cd610257c94ffa8e71eb3e3ade772ee5c6f6aeee4535f04ce +CompilerSupportLibraries.v1.5.7+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/c3c792846973ffb1c9a01451fc35e878 +CompilerSupportLibraries.v1.5.7+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/e46c02f7e2a74b4fd04be60584b6d0678ebc25340691a135452eef116cbce8242139d9e57ecf1ebc00ea2e2b4386b7fa768785516b35f213e888efa913a6d08d +CompilerSupportLibraries.v1.5.7+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/8ca881f5e037d1433de8438ef4328105 +CompilerSupportLibraries.v1.5.7+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/8d1884d527214d8803db1e06469c0bd370b2dd6d7860e00c50b07d27bfdf10a6b444fe2750e2cfcb0712febb4f964f64bb5e7d22145d35001f3880c2b4d9c39e +CompilerSupportLibraries.v1.5.7+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/1b82da33b8f4d46de9c6ed61808a4ca7 +CompilerSupportLibraries.v1.5.7+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/b8c2cb28ceee7fe4fb70c87a670d591d93df20ba71816b29ae18fe9c62e8b19c42279be45d49d18e2ed751e885346cd169c533a13495d3abefb05828af929691 +CompilerSupportLibraries.v1.5.7+0.aarch64-unknown-freebsd-libgfortran5.tar.gz/md5/68ab7542c325f9ab578b20453a88e870 +CompilerSupportLibraries.v1.5.7+0.aarch64-unknown-freebsd-libgfortran5.tar.gz/sha512/83d9c12ba47dbdcd42825a8f6e74725ddae00d46a1adcb4bbb5901888258716311961b840fe5522f61cce068ad57ba9d4508eb55ccdca9d7d0fb590132fb6f68 +CompilerSupportLibraries.v1.5.7+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/5fe7028d81f5c4fe9b271bd7d70b67e0 +CompilerSupportLibraries.v1.5.7+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/3daeca55ccff9057ac00fccd9acc568b5170e57df2f60ec35b13c2efe8bf7d0c48124289014775a645b0bd43648b1d1581383315dc1cb7878596feccda70edcb +CompilerSupportLibraries.v1.5.7+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/b2c913a54035ddfba443774550171af4 +CompilerSupportLibraries.v1.5.7+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/9bba395a287cc6622b8d3f2b9215d6536e02d109641dd4bd608ac01d632f555ec21c0c283becedd32c39d4695121f9a53469abd689f7a5d9d03c88f80d43d57d +CompilerSupportLibraries.v1.5.7+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/5fe7028d81f5c4fe9b271bd7d70b67e0 +CompilerSupportLibraries.v1.5.7+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/3daeca55ccff9057ac00fccd9acc568b5170e57df2f60ec35b13c2efe8bf7d0c48124289014775a645b0bd43648b1d1581383315dc1cb7878596feccda70edcb +CompilerSupportLibraries.v1.5.7+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/b2c913a54035ddfba443774550171af4 +CompilerSupportLibraries.v1.5.7+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/9bba395a287cc6622b8d3f2b9215d6536e02d109641dd4bd608ac01d632f555ec21c0c283becedd32c39d4695121f9a53469abd689f7a5d9d03c88f80d43d57d +CompilerSupportLibraries.v1.5.7+0.i686-linux-gnu-libgfortran5.tar.gz/md5/910a2be6942586cbd6dc28b1f1299e59 +CompilerSupportLibraries.v1.5.7+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/75c5cf940d18dbfb3e3924ba97b02c17a6a507f9376acd76181269d37889d2d91fa0249c18d43bbe492c8b79234da8aab83a7eb9f3b01fd8f2bdeee5496150ed +CompilerSupportLibraries.v1.5.7+0.i686-linux-musl-libgfortran5.tar.gz/md5/0257b0a8d52a57019515d4c7e9b6c6da +CompilerSupportLibraries.v1.5.7+0.i686-linux-musl-libgfortran5.tar.gz/sha512/33f6856ebeb896e43bcdf7b44ccbf344a58c33dfe404ca270734c60bca966f57ec43866ca9a256959e3db39e0be9a6a0fd89ed2fed0257b98e335c7bc24c2a4a +CompilerSupportLibraries.v1.5.7+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/9bdb9e6f31893d1f03ee3e63d33f9d1e +CompilerSupportLibraries.v1.5.7+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/2c543b42632ad301eb7010caaa04dff311a7029fecfd7ace117e82483b6e0b9eaa76c1032b593a8c1d7c3678281fef9fcf3fc9f5b9db404e6a3db4e605eef26a +CompilerSupportLibraries.v1.5.7+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/dea6f66fb0b94c0295a898c5866334e5 +CompilerSupportLibraries.v1.5.7+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/145c190d38209fa404d0a639b7fec9ea13e0e38580ccecedec942390d2eb56cb2eff032205b4117f2cea0570695b8a5c9529966d2e0649b5f54a17795e9b19f9 +CompilerSupportLibraries.v1.5.7+0.riscv64-linux-gnu-libgfortran5.tar.gz/md5/5f05012536e1e5a9167d6d00410e24fa +CompilerSupportLibraries.v1.5.7+0.riscv64-linux-gnu-libgfortran5.tar.gz/sha512/d2a040e8235f5c4f0944105453d0980ca01226e2d5e98e54652a50ac4cc37f1450f11b78f1b3b5bb33d974a5e873352d642abc90550f5df334279d1429b3da4d +CompilerSupportLibraries.v1.5.7+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/7e9bfbfce1f636fab2e21ee5407fa43f +CompilerSupportLibraries.v1.5.7+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/824d3561dc8a7b0e3fc8dc46630dda4612ef9de5eb9d9d83036d19d1f5e9da2c1f5480cd1a6273152e2916f0c9b3a7c7eb43d7bb55852cba1324ef5ad16da4df +CompilerSupportLibraries.v1.5.7+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/e44c78ed5c380ad4b0456f4978f1740b +CompilerSupportLibraries.v1.5.7+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/56f3963515bbf0ba13dfcf601689f25a17f532b54930dbd63aa6364597ce5d7f923fc359d7ff1e8a9e269e1f8df2c5398f74300cb8845d97e98893c5698d33a7 +CompilerSupportLibraries.v1.5.7+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/d6ad6eac45a292c66b4da0a91b44a257 +CompilerSupportLibraries.v1.5.7+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/733fc1e866af7b0d65a843af127bfde27ca8f31498264ae06aac889309f84023db813cb759c6b9d4d422fd986b8c1d0784dde41764765b941546f140bc6caf7e +CompilerSupportLibraries.v1.5.7+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/bc3885ab47161710c1d2d282b625b8a7 +CompilerSupportLibraries.v1.5.7+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/76dcadc3e566a677e273c1b699430e8dafe27414f6692b5d1b0beed41a85d03c16d43c763291dfe0180a6f9c31773a3c11593f466d39f30b05b2896a19515943 +CompilerSupportLibraries.v1.5.7+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/bf33a4af73539b182d85a5743b2faef9 +CompilerSupportLibraries.v1.5.7+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/7d540d5731908ccfdb5aa95a317fa022b81e14c8c587c1b7f8d544b0a5ac2025131695d40f8812b7d7816252ee029ef61b3a40462f26fb7e531f859a15729886 diff --git a/deps/checksums/cpufeatures b/deps/checksums/cpufeatures new file mode 100644 index 0000000000000..9703fd0778966 --- /dev/null +++ b/deps/checksums/cpufeatures @@ -0,0 +1,2 @@ +cpufeatures-6bb9489f4edddfca6e2478b8bfac4acedc349b89.tar.gz/md5/66659f750a9fa120fadcc564f61a246a +cpufeatures-6bb9489f4edddfca6e2478b8bfac4acedc349b89.tar.gz/sha512/559752623d0963d87644f7c04bc08983876230107efecd0833798e3c9376b9a7167949971fd80d33c722adf8edfcb3e1f7be52b7c6a4bf7959aeac0906bb9a09 diff --git a/deps/checksums/curl b/deps/checksums/curl index c7f63bfb9dc2d..0053a8185bf6c 100644 --- a/deps/checksums/curl +++ b/deps/checksums/curl @@ -1,38 +1,40 @@ -LibCURL.v8.15.0+1.aarch64-apple-darwin.tar.gz/md5/6e0789ffb84bb7baae49814c36db354f -LibCURL.v8.15.0+1.aarch64-apple-darwin.tar.gz/sha512/9d64ef5d61db74f3cc67cdcaf9b41d3e7e6a0a3b51deea97dbb3da1928adc9d8aa9558de786af1497de42df43dd184af057465f57471fd01cc1a83020dddbd6f -LibCURL.v8.15.0+1.aarch64-linux-gnu.tar.gz/md5/54214152cfef71c9296a150594c8c405 -LibCURL.v8.15.0+1.aarch64-linux-gnu.tar.gz/sha512/9a1c96e943313af3151763104bb2bed465722d81a4a5c97f8eefbcb4c566718e2a426bb9e81fafb17129a9cc5de93fd8cba5f83f7314dfa4aca6ad817011b627 -LibCURL.v8.15.0+1.aarch64-linux-musl.tar.gz/md5/be2288fc32572680434756ac1e9774e3 -LibCURL.v8.15.0+1.aarch64-linux-musl.tar.gz/sha512/d15a5a7e0f08fe9a0ad1b95f86c53167e3f26356be8010fa332a1a7003d879b5506a8029b7ab36e221821033012cd3a5581c30023fc950c711b517b49d891a3f -LibCURL.v8.15.0+1.aarch64-unknown-freebsd.tar.gz/md5/a18e1e712923540b1e8ccf8c67e0e735 -LibCURL.v8.15.0+1.aarch64-unknown-freebsd.tar.gz/sha512/cb0b2f4041abdcd3b8f15f8fa1a894f8c4aacaaea9f5b4d5213cee53bee2dfe8884299df2cc1f51c31f058b19c57c6b9e3c3cfd63f84ce2db27af14ebc4c0c5f -LibCURL.v8.15.0+1.armv6l-linux-gnueabihf.tar.gz/md5/389f85ab11b49689a941de10853072a3 -LibCURL.v8.15.0+1.armv6l-linux-gnueabihf.tar.gz/sha512/2e1f70f804b540159baf720953716badea9854e7e92783b2f3a8805057c430b2701cd662c24f0cf4a8d11e79cd80a352fd59f23cbf705dd09ffd9ab7dcabef8e -LibCURL.v8.15.0+1.armv6l-linux-musleabihf.tar.gz/md5/4b071a0e0af2a64ad1cde498f54a7177 -LibCURL.v8.15.0+1.armv6l-linux-musleabihf.tar.gz/sha512/90eaf004296fd1ebb3dd02120e92ef574a8c37456c5c99477764e91cdcf0326d92de852d0c8261693403958a3d0e40c4b6b3cffda79f517ecb4f1f852a812c02 -LibCURL.v8.15.0+1.armv7l-linux-gnueabihf.tar.gz/md5/b3da6e896d2fc03ff384abda6f27baf1 -LibCURL.v8.15.0+1.armv7l-linux-gnueabihf.tar.gz/sha512/21c7bb94b94bc1840c5bcda0cd3fd01e4850e3d9e43804c7f2ba49f2f97793603191476c2f764bd5e23f4288c1b04d1645a1c55655cc09971ec15925aef7fd77 -LibCURL.v8.15.0+1.armv7l-linux-musleabihf.tar.gz/md5/37d6ff69f83338c222d65f5132025112 -LibCURL.v8.15.0+1.armv7l-linux-musleabihf.tar.gz/sha512/f483d2fce926272fa7cabbcec8a269f681150b4bdb1cd25e638d9cff358e770a5014ada6394a2b42a1fabe6f6830d131924442003eeb3841dafbc8f2b9288112 -LibCURL.v8.15.0+1.i686-linux-gnu.tar.gz/md5/24bf5ca0680cb664b2a9d11e5356c4b7 -LibCURL.v8.15.0+1.i686-linux-gnu.tar.gz/sha512/21b000f346458079d7091f3766a2ee06dda90b1c4e48d236e768181013088b9ee8daf51eaf966c6c88cf3d9380b204ea7f9f827336de841f4b5efb38644ea5c6 -LibCURL.v8.15.0+1.i686-linux-musl.tar.gz/md5/c158455a2fb3e75d8ddc08e253bafb82 -LibCURL.v8.15.0+1.i686-linux-musl.tar.gz/sha512/8b59acb7bc33cc04fa39107d85c5ef3296a901d138bf4951001b7cd7d5780d84e6728b68862173501f9f4720c30736611f24c6334a01d271649ab1a03ca7a05c -LibCURL.v8.15.0+1.i686-w64-mingw32.tar.gz/md5/e681eea3a492df7cfb0c0e8d1152a56c -LibCURL.v8.15.0+1.i686-w64-mingw32.tar.gz/sha512/41b9c183099b00eadefd7f121d17611972f201cf21ecdebec538c5ee9620646bd8fb7610888df4b9d1a1ab3055d9edb6d345e8f787d9df4c1b739cf72312d5dc -LibCURL.v8.15.0+1.powerpc64le-linux-gnu.tar.gz/md5/9c71b40f94b7ffd65c182dd96da76d04 -LibCURL.v8.15.0+1.powerpc64le-linux-gnu.tar.gz/sha512/763cf590b7fc8cc509b4762f8c3fcd1f0534f756407472bc86063b70a8a144fd651990d95f9300bc7eb30682875abafab399a19449af6a8a4d9d9fb548b2a5d8 -LibCURL.v8.15.0+1.riscv64-linux-gnu.tar.gz/md5/d84d6ecb606d7de45a7e0e91d29f8e06 -LibCURL.v8.15.0+1.riscv64-linux-gnu.tar.gz/sha512/5c786779c838797182824849f682d54cf57f248ea7b7f80c21d34540324df16996796647f82e3b3d1cb274278be6c935c080658edb780fd76dbfe1d04928c74b -LibCURL.v8.15.0+1.x86_64-apple-darwin.tar.gz/md5/2738d7814b7e75a292301c5bce181a0e -LibCURL.v8.15.0+1.x86_64-apple-darwin.tar.gz/sha512/7ffe3b616f6a6591608dcd3a17cf67420b830b17e4d5cd9bd0441d104910d1a5ce28484238fa7c32dee0306c8f2ac9ffaf673a28ceca646e6879e5a552e3065d -LibCURL.v8.15.0+1.x86_64-linux-gnu.tar.gz/md5/a945b3ed85f8fb6ec27c551fbfe2946b -LibCURL.v8.15.0+1.x86_64-linux-gnu.tar.gz/sha512/1646773b43433ea9f403c0242fe0efb0595f16507554cfe326d775ef3f0bcd400a04a7a7db5b184649f84531d47938ec03d79d335c53929647a50cef9011fbcc -LibCURL.v8.15.0+1.x86_64-linux-musl.tar.gz/md5/6aa66d59e057fdb7da74a359648fa41d -LibCURL.v8.15.0+1.x86_64-linux-musl.tar.gz/sha512/1a422ca310ffea3eb70d27472440631031a2f7c8f089e261cecbb3bcb1bc7161fa41fcad91d3eb01cd688af30c8b43ee75b565f09bdff8c2fb0ab64fab571eb9 -LibCURL.v8.15.0+1.x86_64-unknown-freebsd.tar.gz/md5/d95c02c6c541689250e0015eec5a8577 -LibCURL.v8.15.0+1.x86_64-unknown-freebsd.tar.gz/sha512/0b6e0790779370f721ba251f6cd5f3464b62361fdc23bc1ca9ae22c82c4efbf2c7fc8409277b90c1b2e73996eb4378c188924b083d4b9ba255eaaeeff339e84d -LibCURL.v8.15.0+1.x86_64-w64-mingw32.tar.gz/md5/690e63d716b19e109b5487dfb4ef580d -LibCURL.v8.15.0+1.x86_64-w64-mingw32.tar.gz/sha512/2e763b8acbfc622dac431fa742c9cfdb12cb95436b0eba2e409a7a1dede107e99952f394c92d8f3aa55f3febf039f5c16058ee122d88e0eac5a75ae98f2e9018 -curl-8.15.0.tar.bz2/md5/8b475c3eec74c5e78a9fb45a902dae76 -curl-8.15.0.tar.bz2/sha512/797fc9af599de88ceb36c8bc284d3f1a2a1ca0703c7bdd377d67ce6da4317ca673ba4a946c61f3bd5a66febee37b5aa88826f26fbbb398f5e20630769a0de033 +LibCURL-b16e4d0c99b0399094da7fa4563ec075b634ef9a.tar.gz/md5/f4fed5675759fa3dfa1a0e20bad62160 +LibCURL-b16e4d0c99b0399094da7fa4563ec075b634ef9a.tar.gz/sha512/deed18e9de12192d791a7d5724a6eabe195f4bb1e8c86c4fdfbdd907206282fa54db4e0aaa169f73f324c68fbe25cdc14796ff2b280bc91037657c6b10369e4f +LibCURL.v8.22.0+0.aarch64-apple-darwin.tar.gz/md5/64365e85c9aa3448d55f973e7acd7b32 +LibCURL.v8.22.0+0.aarch64-apple-darwin.tar.gz/sha512/61092d2be7a1540f8a417585c4ccf191dae46885ed3ee18caa7e47eb29206ee1587935c1f94e528d97a488624c80a5c747fd2e1c65d05d3a3586e173c36aceb8 +LibCURL.v8.22.0+0.aarch64-linux-gnu.tar.gz/md5/f3949df9582d453e382049d489bbcf37 +LibCURL.v8.22.0+0.aarch64-linux-gnu.tar.gz/sha512/1869c91df2665a3638d3d5bf364621b51cf60863ef676d01dbff15404b95b3669170383b07cb91c3abc5d1d6c268b59d8847eeb803650117178ae528703cc7b1 +LibCURL.v8.22.0+0.aarch64-linux-musl.tar.gz/md5/623eeee0b42328e750d15d113fb21486 +LibCURL.v8.22.0+0.aarch64-linux-musl.tar.gz/sha512/310fedc0463a15fb7b90e2db9569e4b395b3074d5de3c58314b8ce1d14783b90326e7111b05aedf4debe2716bf25f6bfaf55f14dc5db6bca658f1baf4b527802 +LibCURL.v8.22.0+0.aarch64-unknown-freebsd.tar.gz/md5/eda72e60a89cd342b66a7c4f4aeafc34 +LibCURL.v8.22.0+0.aarch64-unknown-freebsd.tar.gz/sha512/dc26d4e033959fae7f3808c8ce9baa17b12667975456b920afa84c4fa638db4426ceffe70849ed54c71c9cc48a56f10872026ab46cd1a1de05f277ce49af75c9 +LibCURL.v8.22.0+0.armv6l-linux-gnueabihf.tar.gz/md5/d88d74ee33a404b881f65b3ca7be02b0 +LibCURL.v8.22.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/0c47f91141f78039d8299e0c10a1e4493d8316daf77ef7a14d368d97ab65e35826e8bde7017626416c3a787150062211cdfeb17b60c7ae0d6d087473797e5255 +LibCURL.v8.22.0+0.armv6l-linux-musleabihf.tar.gz/md5/428df3d5e0ece4e2affbaf33816c8621 +LibCURL.v8.22.0+0.armv6l-linux-musleabihf.tar.gz/sha512/f6234b20384fea6047ec9c332afdfa6b66307d3599d6851872c3bd2705f732b278c567132d0d01e56ee7754bd2df00b46a21a31a08c6c60c29779986adfdd734 +LibCURL.v8.22.0+0.armv7l-linux-gnueabihf.tar.gz/md5/4d51f0e22b06eec93915fd104040cff7 +LibCURL.v8.22.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/425612c42bafb17728c41e5a9e443696263269df027a509e1dcac011f98435f728ee710844ead04adac52c2e8e8be1757d301c69391abc8698461e505dd3925e +LibCURL.v8.22.0+0.armv7l-linux-musleabihf.tar.gz/md5/e85c6c65ade338046f330bfd3039920b +LibCURL.v8.22.0+0.armv7l-linux-musleabihf.tar.gz/sha512/3efd9d78349ac54cbbe41f31dda505074be4305e2b4c23d7007e19bd8789c547c3e924ffef24049fad1372e17287737a4b8ecd9461fc39f8b5931ef302a924e4 +LibCURL.v8.22.0+0.i686-linux-gnu.tar.gz/md5/399482c7eed08d09aaf197480062bee2 +LibCURL.v8.22.0+0.i686-linux-gnu.tar.gz/sha512/b35a01ab6880541f8cb0604bc729d802c92727969669c44139da12401317c3c57f1b2ed16d4e09370ff0c71d33b2f2b2c2fb2f566889872ab4be3dc1d7895887 +LibCURL.v8.22.0+0.i686-linux-musl.tar.gz/md5/fb93f698c451fce8016e21f0b0ff1068 +LibCURL.v8.22.0+0.i686-linux-musl.tar.gz/sha512/e40ba8146b4a713b6d9df7cdaec1de659221d6e8fe566f815005836b13d35dca4b4626389b3816e040a8581242e6dae49dff413a2515265948a10ffba89cc1b4 +LibCURL.v8.22.0+0.i686-w64-mingw32.tar.gz/md5/f702bf0194429b0890970808f1d1fdee +LibCURL.v8.22.0+0.i686-w64-mingw32.tar.gz/sha512/b0d89296f9bd0d7406ee9d2c66f03f77d2a766b783f758a379761c7ea87d8ad175219696293ff7fd947c8465834e2fc4708f8a1954060db5cb93d56947283712 +LibCURL.v8.22.0+0.powerpc64le-linux-gnu.tar.gz/md5/cb030aec6c15a3ace3d4a07459f3f89b +LibCURL.v8.22.0+0.powerpc64le-linux-gnu.tar.gz/sha512/4469e2ef3e88cb120d4f4980ec5241d2152cc4a40270ab70450d2563fa748cd4b2d980cc3ff1d4ab51e4c9af416982fd31bd078cfef428adbf988f8b7f3dc6a7 +LibCURL.v8.22.0+0.riscv64-linux-gnu.tar.gz/md5/c599017149423a8ef3142ab2c0979676 +LibCURL.v8.22.0+0.riscv64-linux-gnu.tar.gz/sha512/2c85826ca77996a4d227187f7090660fafbf18aa9ff92872ebce6402b65e74c0c95bf74d096516f15453f252ff572a65d68dea334c36530c12565eef0e1a36d4 +LibCURL.v8.22.0+0.x86_64-apple-darwin.tar.gz/md5/982820e7094473001b8f063a52e8e9c4 +LibCURL.v8.22.0+0.x86_64-apple-darwin.tar.gz/sha512/500ba1f6ad461cffe3b042a5dcd3b2223a1fe4016f9c294921d9e199174102e291545b37bfb666b4b2495e8fa2c0f111a883e5203b3921cf58ed2b14dc4762b9 +LibCURL.v8.22.0+0.x86_64-linux-gnu.tar.gz/md5/a9420eaf1b3155cd391c5d1b67c76e59 +LibCURL.v8.22.0+0.x86_64-linux-gnu.tar.gz/sha512/0699ecec5907ecc2d8189a733b691113838b2de5d0ccd1bc577a0e8b9211e2b69ce6e2516856d614bdbaab5e8375e37c1a36402ae10aef605f28396db590f015 +LibCURL.v8.22.0+0.x86_64-linux-musl.tar.gz/md5/cc691973be847ff57531f6d9198ccf1e +LibCURL.v8.22.0+0.x86_64-linux-musl.tar.gz/sha512/5d82d40353c3118d33b15621a9d6970561af0d8ebb43daa710ede85012cfdc950a0fb75ab598e3eb96ef1ed05844d766310e4ef48faed7fad195f642aeb6db41 +LibCURL.v8.22.0+0.x86_64-unknown-freebsd.tar.gz/md5/ab9ef3299e54bba1967b54e33d04eb44 +LibCURL.v8.22.0+0.x86_64-unknown-freebsd.tar.gz/sha512/711cd0ff80bffc3d152c86b84b3cc5bc3359f596a74cfbe7c92e073748ed98cb9269f67a43737c49f53a6b78bf69aca26a3fb63be17e1a590d0638a8c8c7514a +LibCURL.v8.22.0+0.x86_64-w64-mingw32.tar.gz/md5/8d4e3607eb0e13fda3fb524219d8b482 +LibCURL.v8.22.0+0.x86_64-w64-mingw32.tar.gz/sha512/d193f4bac8430bb5b7b41607feb137878162703cd9784eb65ea4e0d92dde854abf1040608d28f3c91ca7b87e5e8b70c05aacabcef6df4d261e4c7ae41c078bb9 +curl-8.22.0.tar.bz2/md5/3f6862e123fa4203667d1253ceb2d89c +curl-8.22.0.tar.bz2/sha512/15aa511bc370913c076237aa5888a2b354334a536fdbd6eab43f566d0be6acfe1687025a886eaddb8b87965b395b1241e168cfe2df9ede2eee05a5676d9b322b diff --git a/deps/checksums/gmp b/deps/checksums/gmp index 949e4d738a472..038e229b1f5ff 100644 --- a/deps/checksums/gmp +++ b/deps/checksums/gmp @@ -1,66 +1,38 @@ -GMP.v6.3.0+2.aarch64-apple-darwin.tar.gz/md5/3fb601fcf70024fcc40889cf1b958441 -GMP.v6.3.0+2.aarch64-apple-darwin.tar.gz/sha512/7ecc97c1f22287e9d7f3e8073e1cc3c6b3c75aa4a350a55a0b6f92c5bf60339b52f8866994f5973077e1026b9d3b10a7bcd71ec2abf25c3cc1bf6ca1041c3e73 -GMP.v6.3.0+2.aarch64-linux-gnu-cxx03.tar.gz/md5/10581945c01bac319c9c2d76f1f7052c -GMP.v6.3.0+2.aarch64-linux-gnu-cxx03.tar.gz/sha512/3aa2799ef7783a4edb767a695bd2797776def8ce1b2dc471b2cc733371db9981d6c3f395fee2fb50b13c7ef74c1521d2787c29dc60a75e1b92652b94819b5364 -GMP.v6.3.0+2.aarch64-linux-gnu-cxx11.tar.gz/md5/c1f9765fccec8ec131faa5e31b7ac28f -GMP.v6.3.0+2.aarch64-linux-gnu-cxx11.tar.gz/sha512/aebde82400544dc7a2aef0a4531cee78f9abcac9352dfd5d86472a70d704b281de03325cc609583169ecbe4cb64623ab04a3d7fff9cf24c70991530fe530aa05 -GMP.v6.3.0+2.aarch64-linux-musl-cxx03.tar.gz/md5/b1f771c79f3b380555c1c96232074523 -GMP.v6.3.0+2.aarch64-linux-musl-cxx03.tar.gz/sha512/daca9d3b4179e99da8e61f4010f5965718c79d02627e0b3272e4d20c34dac0d933408dc7d760a6d6fa09546e436c800ad5da4a1d34283eac9558f3d2f97bebce -GMP.v6.3.0+2.aarch64-linux-musl-cxx11.tar.gz/md5/523c386457e9d48430b83f2db85ac10f -GMP.v6.3.0+2.aarch64-linux-musl-cxx11.tar.gz/sha512/18155dd92641bf6240606d23b0d3cab16bb9b63b6034a7c7c61f3728fb48a6b710fdc21c6477145c015c648557e97003b0cc6087b4b36a691daecb87272cd51a -GMP.v6.3.0+2.aarch64-unknown-freebsd.tar.gz/md5/7dd3f2813fd7e9e620a8123ae2340ab2 -GMP.v6.3.0+2.aarch64-unknown-freebsd.tar.gz/sha512/375b12dee41285b65b5cdd55f6b000a90fd431c3eeb788a928396a102594fb6fad257f2c4e707f11ce7d0e4d45bc82a77ac85d8a48fa0a42f969b48b8b2c1c23 -GMP.v6.3.0+2.armv6l-linux-gnueabihf-cxx03.tar.gz/md5/7d23f84102362ec3974ca2d84da33c4a -GMP.v6.3.0+2.armv6l-linux-gnueabihf-cxx03.tar.gz/sha512/51e419159fad75ca0ab12c31db29259be6fa280e66e2b980df4c99a0558615297741f633322978a409fbc071ec71834214b12d27d04ced0c043c569438dabd12 -GMP.v6.3.0+2.armv6l-linux-gnueabihf-cxx11.tar.gz/md5/5f809ffa56ec07cc04e3c4cb155faad0 -GMP.v6.3.0+2.armv6l-linux-gnueabihf-cxx11.tar.gz/sha512/e394afb93a2c0aebe0ac7887bb2610720cb926256f0f5e7b05f3b1a805d3f7967fb97f4227ccec049df554c6cd1c4d4e9414fc4fea33f201204dd87e207e33ff -GMP.v6.3.0+2.armv6l-linux-musleabihf-cxx03.tar.gz/md5/494564a56197edc5b8772c15eca7b117 -GMP.v6.3.0+2.armv6l-linux-musleabihf-cxx03.tar.gz/sha512/a7bd8bc19a030c56edd4d91e3cff16d78d4a9c1c1bec99897e55cfaca7e14cb99cee32e220473e207b78f0b5e0c0bf188c679d1748c010380485fad4d89758c5 -GMP.v6.3.0+2.armv6l-linux-musleabihf-cxx11.tar.gz/md5/751c36d4975d6ff88eb968123afc1845 -GMP.v6.3.0+2.armv6l-linux-musleabihf-cxx11.tar.gz/sha512/af471834ba32a970b4f358a263434b03e169dc48445aa5af412ec51e70668a41699f9c408d90f64b06dc9233360f70a03df859428fdc0d759e5696a3ae32f3f4 -GMP.v6.3.0+2.armv7l-linux-gnueabihf-cxx03.tar.gz/md5/ea9c867ae191a29647e8ccfb67947bc6 -GMP.v6.3.0+2.armv7l-linux-gnueabihf-cxx03.tar.gz/sha512/d6c44c945d1ef869155be087320d7750be549399b186aad8c92bba32ff5312bf09cbb2fb57be91be237be7d50f8f6ef0aea67070f50c024e6f5302485f405d5e -GMP.v6.3.0+2.armv7l-linux-gnueabihf-cxx11.tar.gz/md5/ee5becfac9fe3c448a5de322ddee66d7 -GMP.v6.3.0+2.armv7l-linux-gnueabihf-cxx11.tar.gz/sha512/bc9bb2ad83644cf0b9f2bb0bfce28938ee6e82dbc0de74d1f411a8eb5ab96c5ec00c648019384ec07f34a469bd984d6c62eac1bcb803eaa013b6c85547ec3277 -GMP.v6.3.0+2.armv7l-linux-musleabihf-cxx03.tar.gz/md5/23962e487398f02c8d660724d88bf7f6 -GMP.v6.3.0+2.armv7l-linux-musleabihf-cxx03.tar.gz/sha512/4c561053f79ed976a698c7382c5c94ebcbcd25ed27c939016bbb4af59948fd6bfb82e494e18fc7b4969941a7756c33afd2f177b3158f1b3d659215c25c958d2c -GMP.v6.3.0+2.armv7l-linux-musleabihf-cxx11.tar.gz/md5/4734feb61dd3f2a4e6e395f9ac7ccf57 -GMP.v6.3.0+2.armv7l-linux-musleabihf-cxx11.tar.gz/sha512/088a52c372681b4853fe7c4c70eb8625b58df6d79eea2a8982bd781458188930aa31dd9a121ff7a6d00cd8165f5d126155d7f931100aeff256b55a2281d44a90 -GMP.v6.3.0+2.i686-linux-gnu-cxx03.tar.gz/md5/e229a7a09d6c843f03028b036a54b786 -GMP.v6.3.0+2.i686-linux-gnu-cxx03.tar.gz/sha512/d92cccfdd7abe3ca5c6ee1eecfe3f7aebe875ca6b9f6257bf1181dc5ee9c873a930ebb2accc825596ee26dc45bd290a482f0405cfd7a3a1b0eb606f5ca897b70 -GMP.v6.3.0+2.i686-linux-gnu-cxx11.tar.gz/md5/01dbe43b15197cd39351dce91b3a62c9 -GMP.v6.3.0+2.i686-linux-gnu-cxx11.tar.gz/sha512/d6e7ea99f76e10b4f7733d8c7f4af3fb2fc09618510c222da1fb95e8b4c83b0aa7c5d2f896bb620546bf39041d6dc1b32ca74ddf5024ef1beb5526b374ba885c -GMP.v6.3.0+2.i686-linux-musl-cxx03.tar.gz/md5/ce2f8d8b59228888cb7f03da0c1aca70 -GMP.v6.3.0+2.i686-linux-musl-cxx03.tar.gz/sha512/cc024a2ca4b4f042c19f667c4c3c08e3041d9b9ea0279cc668a3c0212103e86444abbdb323304e05c506b44b3c1b32a55f90c04cc32e9d26ac013336821c9ac1 -GMP.v6.3.0+2.i686-linux-musl-cxx11.tar.gz/md5/c37741b3a03ef2e705d45124eae25afa -GMP.v6.3.0+2.i686-linux-musl-cxx11.tar.gz/sha512/c343ad2ea47d5775e6e4c50fd8d46745d39f3632f4ad479199f7583fd02b08a0126048625d3999b23a0534e4f5c2bf19d021436229689da7c794427102c7780b -GMP.v6.3.0+2.i686-w64-mingw32-cxx03.tar.gz/md5/52a773a2111f7b1f938e78263c4608b0 -GMP.v6.3.0+2.i686-w64-mingw32-cxx03.tar.gz/sha512/6ef89b7eda8f0709315c1080e4d57810f976939c755f160e34b04e4c199409c8c707036fae5a73fca3a16813cb4ceff8daca38d1ead73e36d7ff23506e5bb4b1 -GMP.v6.3.0+2.i686-w64-mingw32-cxx11.tar.gz/md5/88b1ff47d913fa301c95e9e2aecf42ce -GMP.v6.3.0+2.i686-w64-mingw32-cxx11.tar.gz/sha512/3d631ee81906627a8bd9194fa8f18b634467565c10e5e08db7d1a4b0943bae9391ae15a1c39533c9796edf24e1f0210d082e44dc7c1fbd9f93855f37e207da07 -GMP.v6.3.0+2.powerpc64le-linux-gnu-cxx03.tar.gz/md5/0b2c73cf7936500ce0f07577c4c76ba5 -GMP.v6.3.0+2.powerpc64le-linux-gnu-cxx03.tar.gz/sha512/30e099bd6384e801fb28b4741810122f82ab0760a4e09d6ab28559b72feff278a48150579907cb2920a624fc85287a197743331bc1808353d0855c198341bfa1 -GMP.v6.3.0+2.powerpc64le-linux-gnu-cxx11.tar.gz/md5/f496279b474948435f836ba39291c708 -GMP.v6.3.0+2.powerpc64le-linux-gnu-cxx11.tar.gz/sha512/c37d4fbba284af87fc16a24bf1fdfe80b42c84bd44f1859d1c9ee97fdbb489817b58db80a078729e19c8a5b8448f9234408a8e477fd15acf15521f3129e86acd -GMP.v6.3.0+2.riscv64-linux-gnu-cxx03.tar.gz/md5/f07fc6751104a407ea2515fda3f26880 -GMP.v6.3.0+2.riscv64-linux-gnu-cxx03.tar.gz/sha512/435b375da747d2dfba06a303b55118471c6ef705cc65afeabb5a59477cc98aa9a956b31c5e8b571126f63d922498b9a66510f8f6810a60f6a4fabba5ec368cdf -GMP.v6.3.0+2.riscv64-linux-gnu-cxx11.tar.gz/md5/493c24a7a7370f308f0da2955f40b5d5 -GMP.v6.3.0+2.riscv64-linux-gnu-cxx11.tar.gz/sha512/2e1a7562b759219d1a4283372e66fa1e907279c5b5feb8a858f6bd8de8b9c2ef3ddd09d5e812d93813fa781090574fd26d0cec85b211274db628681301a206f9 -GMP.v6.3.0+2.x86_64-apple-darwin.tar.gz/md5/c3bb785e10fe19cf1c47db6bc5e98fdd -GMP.v6.3.0+2.x86_64-apple-darwin.tar.gz/sha512/5280896654e1c7864d770ecbfc853a1c7837c2b1dd369047432d10f831762a26fdaeac4201ca419d8bf7c545c107800b892660f4484b5eb87bfaf42c919fb640 -GMP.v6.3.0+2.x86_64-linux-gnu-cxx03.tar.gz/md5/0fd62bb914554c3cb6b5dc0f5ec0d330 -GMP.v6.3.0+2.x86_64-linux-gnu-cxx03.tar.gz/sha512/78cdf0cdcdca4a0ddc87755f4afdb8f290fa946b3c5541a3e31145f8bd905884d59f38e9f5ee4fe96ceaedaf90881af795f4e3ecf1be922103b838964da101cf -GMP.v6.3.0+2.x86_64-linux-gnu-cxx11.tar.gz/md5/02f54f8895bae0d7a824374888300744 -GMP.v6.3.0+2.x86_64-linux-gnu-cxx11.tar.gz/sha512/83c865f6164400e56c28949c680cf92457daa270b745d89034e1bcc46af1eb93c96bce708561dee03b58162191f6448e4325e921daec11083bbc42dcf3a1ffda -GMP.v6.3.0+2.x86_64-linux-musl-cxx03.tar.gz/md5/8f3f26422f8bd0889b5c2ecd22d97101 -GMP.v6.3.0+2.x86_64-linux-musl-cxx03.tar.gz/sha512/680beb99936433bc1c3367e85f3a4129c5a99d4c4031a1da919293819f6d3f1b85be801a2f48af352c47d7cb6f394534333f1a0d0404ff41899952d55c4b1f75 -GMP.v6.3.0+2.x86_64-linux-musl-cxx11.tar.gz/md5/7ec0e3e9125c14a20d6d0044036f0996 -GMP.v6.3.0+2.x86_64-linux-musl-cxx11.tar.gz/sha512/c22e6a25ec854f9c199d5e76bc1dbcbe57c4cc219eb2b5f24418729252eee1a5c1d3e8bbf5b62d148cb408595e96f448f68a29a9425a902952bee666b6f051f6 -GMP.v6.3.0+2.x86_64-unknown-freebsd.tar.gz/md5/6782d7fd0bd15c189c4a1753ee0fb0eb -GMP.v6.3.0+2.x86_64-unknown-freebsd.tar.gz/sha512/04d7a95337e832f7ec228f160a09b74ed7908ef9cef1bd392555392a24ff63ce4a88b616b5426cd710dcb581e164bb94c04fe17f0b599adf3c3bc33106bcd886 -GMP.v6.3.0+2.x86_64-w64-mingw32-cxx03.tar.gz/md5/b4cb31e93c85cd453b7d8d392a365088 -GMP.v6.3.0+2.x86_64-w64-mingw32-cxx03.tar.gz/sha512/3bd84fa8f580b272eecb06077ef710ae8df661126e86afa2c901b298a2598975a07f840b922da0066dbf555f03376cba1b7e4915cd37617341fd420b6707276d -GMP.v6.3.0+2.x86_64-w64-mingw32-cxx11.tar.gz/md5/2342842254e7b47b26836366d29d6802 -GMP.v6.3.0+2.x86_64-w64-mingw32-cxx11.tar.gz/sha512/fb12be14433763d9de689a5df222802cd79d5c990da9a53855fd2f6f8e663a9838b444a310318c059cdb4962eb87d0d4cc2b54d163cf82b09377339c8e45510f +GMP.v6.3.0+5.aarch64-apple-darwin.tar.gz/md5/446a6ce6a7947a95dfce0d32d9b196af +GMP.v6.3.0+5.aarch64-apple-darwin.tar.gz/sha512/22920e0043ea866d89cdbcabb98a14abc17b62c7c3d8f1cd3628febe0ea9665c20d2e7bc1f515673b936f1c0496a696823f30477ee88da494913aa35f048e5c8 +GMP.v6.3.0+5.aarch64-linux-gnu-cxx11.tar.gz/md5/f4d37e03adbd955fd53fd9b7d65b22c1 +GMP.v6.3.0+5.aarch64-linux-gnu-cxx11.tar.gz/sha512/108cdadd3f471c72f61966e54fb2db7524d3dac70f6a907d6a7b7e43aedf3a1e426a5bcfa2f3148904f9d2697948d26577e7ed412f69d2c68a2fda3f20bafacc +GMP.v6.3.0+5.aarch64-linux-musl-cxx11.tar.gz/md5/0d0855b64cfcdad64f8068aa105345d9 +GMP.v6.3.0+5.aarch64-linux-musl-cxx11.tar.gz/sha512/9ae5a545bc724203a581ce2a86d82ef69f31352472485ac0cb4e059ba6abccf1d6799b91700d1c155ee888a1814bbf33f263aace19265e7d8f70c0a29003f630 +GMP.v6.3.0+5.aarch64-unknown-freebsd.tar.gz/md5/d438f6f1c5b31dc4c5c04b5899c47b6c +GMP.v6.3.0+5.aarch64-unknown-freebsd.tar.gz/sha512/70c6cf716efe476cd9d78ed2b12994a903b219d31a2e69d14ae977f5942a5bfde5cbd48dde545ffd3d064de84364188712ced7361caf4b3404098fbddf8f6fd0 +GMP.v6.3.0+5.armv6l-linux-gnueabihf-cxx11.tar.gz/md5/78074744e7d3b3f76cc0ef9daf3cfa2b +GMP.v6.3.0+5.armv6l-linux-gnueabihf-cxx11.tar.gz/sha512/232146e73b3a553792f4c59d32fcd2dcf278d6a861d3bc51651aab11de4772599c40ef07332affa940ba849d7a73f1c755ec7d7928a37ff511c5a4c61ea986df +GMP.v6.3.0+5.armv6l-linux-musleabihf-cxx11.tar.gz/md5/85d0884cdd3a1c66627d40bfef36ed16 +GMP.v6.3.0+5.armv6l-linux-musleabihf-cxx11.tar.gz/sha512/bae6f5134c27c669c02655c8ca82d3de21b60bbde0478a53b5879955ffa0fd19cf20fb0e6651ece6244eb2e489bcd57ccf2952401dd3b41232306a7f08380a9c +GMP.v6.3.0+5.armv7l-linux-gnueabihf-cxx11.tar.gz/md5/5917d14c174b1d138294acaa19b3773c +GMP.v6.3.0+5.armv7l-linux-gnueabihf-cxx11.tar.gz/sha512/b787d407ea08fe0ee337137d0eb5f066c9ab8ebbc423361186670fc24e7f0d3496cb2b19953194576bd245644b0330c317706b7c29ee788beb77b7dba99bdc5a +GMP.v6.3.0+5.armv7l-linux-musleabihf-cxx11.tar.gz/md5/8a7df39576f7b16d5174520fb33c13fc +GMP.v6.3.0+5.armv7l-linux-musleabihf-cxx11.tar.gz/sha512/3591b9298b067db89beabac0994012762426fbafd335efa454c21eaee74d5580a09c4c8df339482d3741ba37ddf2730e17c4bbbbed330415cd78ef3d646d1ced +GMP.v6.3.0+5.i686-linux-gnu-cxx11.tar.gz/md5/183093de0b7dfa737927266de0d7e7db +GMP.v6.3.0+5.i686-linux-gnu-cxx11.tar.gz/sha512/81ee69e3f0a264eccb98e689cf046c1e2557ffe1e3f1b1c67efdb8bcb5c7c986057482e4a57057c16ce35b7135b7e2bef9a17ef42d5c34413edcdaab60b008f6 +GMP.v6.3.0+5.i686-linux-musl-cxx11.tar.gz/md5/a5886612c4c410eae12ba9971d30294f +GMP.v6.3.0+5.i686-linux-musl-cxx11.tar.gz/sha512/3289c51309f8266bbe81693b4ec464ddbc40782d3b8e5c8a8ca0f490b5af2c32ce4ba601ad35f8d92a6a7a8e8a81a3c28c5dd006d7708469c5478f9669c9c495 +GMP.v6.3.0+5.i686-w64-mingw32-cxx11.tar.gz/md5/e59cee40d22693f8b9dd3cbd9fbc3932 +GMP.v6.3.0+5.i686-w64-mingw32-cxx11.tar.gz/sha512/e338337c7348a8ab39ce3e0520183281862932f32704d4fc4be372b12bc633080bd79d1776e260acf1fe65d9dcdeb3f55b74cabf5645d586c91929eb4122fbf1 +GMP.v6.3.0+5.powerpc64le-linux-gnu-cxx11.tar.gz/md5/0ff529ce76f47add354860cac07a0f58 +GMP.v6.3.0+5.powerpc64le-linux-gnu-cxx11.tar.gz/sha512/37147e5e2f77cb91abc3c5462d8645642b6933b92fbc0748a46a06a1dbe8c765f0ec1d40bb94074f574dbd10574dfcb4caf1013c582ce27faa3a9e2c2eb0ccf7 +GMP.v6.3.0+5.riscv64-linux-gnu-cxx11.tar.gz/md5/f7dbd01d580c20f1dbfbf8fa86a223fd +GMP.v6.3.0+5.riscv64-linux-gnu-cxx11.tar.gz/sha512/a895be298c1c05da0369741b596114c4a888906c8057f0b21c71477a425d07173dc12292881ec6770a81ef4148d82ac63820a7999cbf9fe494beeb35cd59ac7b +GMP.v6.3.0+5.x86_64-apple-darwin.tar.gz/md5/95041d42f5f042f5ea162fda991af80c +GMP.v6.3.0+5.x86_64-apple-darwin.tar.gz/sha512/09fe3d6a8b9ca7a699b011c616014616bd01e7815e78af4c4f2e0794e0842d893698e43c597ab92a12d6681c345ad821ac0cb387664f61d9549f43594ff48fef +GMP.v6.3.0+5.x86_64-linux-gnu-cxx11.tar.gz/md5/205687fd8ad5d28f23a1bf93829075a8 +GMP.v6.3.0+5.x86_64-linux-gnu-cxx11.tar.gz/sha512/89d7cc0f3b96c132ab5cd47e838f8adb438b5b53bef83df9f8a5ce547fc70ddd3d4580564848c51840977f8f4d96c49b2275f3b94dbe77b65a916bbc546c9ba0 +GMP.v6.3.0+5.x86_64-linux-musl-cxx11.tar.gz/md5/6262b88949dd41d18c0c2bd8c8947c9f +GMP.v6.3.0+5.x86_64-linux-musl-cxx11.tar.gz/sha512/8bc99bcabba850b23711c255623599dd57a181a71903b9cfa12628efd9a618dbca48ad736139c6ec8b3286f0b9bcad547518f77879766251e0e2a9b2f18b2517 +GMP.v6.3.0+5.x86_64-unknown-freebsd.tar.gz/md5/974f5704034fec741f50c55ff36d22da +GMP.v6.3.0+5.x86_64-unknown-freebsd.tar.gz/sha512/f74cac7109526ae102c57bdccf3dd1a1984e1c1fc0398b074303ee8e881874fc8bcee53ef3a3866edc2b7bb063f674d32391544956cd1ee6043933833e192535 +GMP.v6.3.0+5.x86_64-w64-mingw32-cxx11.tar.gz/md5/e71f7a54e120dcc93000d82072b37426 +GMP.v6.3.0+5.x86_64-w64-mingw32-cxx11.tar.gz/sha512/ee2730aabc674008465162ccd28f6890babd2abbea2f2efab5be4a26b0a32498dde40c6a2b29496d27e0dd11bc3d2ded0da794761d1bf827f83253266bd6c7ea gmp-6.3.0.tar.bz2/md5/c1cd6ef33085e9cb818b9b08371f9000 gmp-6.3.0.tar.bz2/sha512/3b684c9bcb9ede2b7e54d0ba4c9764bfa17c20d4f3000017c553b6f1e135b536949580ff37341680c25dc236cfe0ba1db8cfdfe619ce013656189ef0871b89f8 diff --git a/deps/checksums/lapack b/deps/checksums/lapack index 997b522f7ef84..5171646a3200d 100644 --- a/deps/checksums/lapack +++ b/deps/checksums/lapack @@ -1,2 +1,2 @@ -lapack-3.9.0.tgz/md5/0d39aa430ac2716d88b45224f4de2c8c -lapack-3.9.0.tgz/sha512/90c2c8372c2567b15c25653ed64e7c86a2cb0f1dda116f33716c6877490415210217af5badb67cb50e9d428f56ff83d33a3dad60b6ed7d31919164f6e7e98e0f +lapack-3.12.1.tar.gz/md5/9acad9d1a8946268310c613daaac9bed +lapack-3.12.1.tar.gz/sha512/88e2a87177ff0ed821932cb7cc43f91b6bed71e8a6647df427ffa62554890e5fc8090669849d5ad71b0d5a2e727f8009b4dc73e811a3be36eb677c813f907047 diff --git a/deps/checksums/libgit2 b/deps/checksums/libgit2 index 2339d381a7079..800428399c801 100644 --- a/deps/checksums/libgit2 +++ b/deps/checksums/libgit2 @@ -1,38 +1,38 @@ -LibGit2.v1.9.1+0.aarch64-apple-darwin.tar.gz/md5/1281d4cfc44ab26054b83355a053c42b -LibGit2.v1.9.1+0.aarch64-apple-darwin.tar.gz/sha512/d4040d56c588333c222d66488944c2988a3ed1bb096ffe2bac67061b0ce20c44c5a803741fdc53a14a2523d0bd3fe5b3a6f247b0c366d0d3676c48bac44d3c05 -LibGit2.v1.9.1+0.aarch64-linux-gnu.tar.gz/md5/b2571cd9b20a307fc12c9d7979f42e81 -LibGit2.v1.9.1+0.aarch64-linux-gnu.tar.gz/sha512/52a2301f1311a8fb3c091b158e99d29267175156979779896d346cff5a5dac8b784f113793ca86bbd41f92c937c4eb64b8b8238df55ae95a6b1320cc3455b379 -LibGit2.v1.9.1+0.aarch64-linux-musl.tar.gz/md5/b53fa9546706075f41cf12c39c4969ed -LibGit2.v1.9.1+0.aarch64-linux-musl.tar.gz/sha512/9d818007520f1780eff4d0b6b2d8c7c1fc09763f36280b55d3795895dfa908d87db1699e9dc265162f831b9b1f03879cd187c020f8082c77eda57320eca14ff0 -LibGit2.v1.9.1+0.aarch64-unknown-freebsd.tar.gz/md5/5975e2abf13b3cefbc8cfd3d1d24a956 -LibGit2.v1.9.1+0.aarch64-unknown-freebsd.tar.gz/sha512/05337be9595c5d94816d168cf245289f2c7b2717c49ea000ba872b582c919b327be3a8cfa877eaf7c541a38897db3f57e11a8f49df9b9bfbf8f3110b39389e0e -LibGit2.v1.9.1+0.armv6l-linux-gnueabihf.tar.gz/md5/a4de1b1a529f130d85a75b56f82551e8 -LibGit2.v1.9.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/cfdb3aef71f88405e7ec3c31df1f84617ad01dd227d0d6dde9a7dcd5e4a59590437f4ae90b91b63ab14a5716b366e554ed8d2e403eca2bcc9f4a3f7b813d3df3 -LibGit2.v1.9.1+0.armv6l-linux-musleabihf.tar.gz/md5/6a5deeaf7dc9f4352dc018df12c1ec61 -LibGit2.v1.9.1+0.armv6l-linux-musleabihf.tar.gz/sha512/cec900743c68c1b2580d21e51abe1852ae44ecadd682e48300d0b656ce955f71c4706c1462c35d301e331d79c37330bb4abface88709dee8b4154d1b65ea1217 -LibGit2.v1.9.1+0.armv7l-linux-gnueabihf.tar.gz/md5/90d0cab30f58b6e1d75b006c12f3ce0f -LibGit2.v1.9.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/735174f87677795e313b8338a50f97ce438058386ff594ae91f7b8bed3085e8579c4da5e305e3e70821e30cbf6d878deb1c161b695b1f19f58bc924663bf2949 -LibGit2.v1.9.1+0.armv7l-linux-musleabihf.tar.gz/md5/56c495c7cd018e3942ee5879d0e397fe -LibGit2.v1.9.1+0.armv7l-linux-musleabihf.tar.gz/sha512/2ea7cafb587c2f2f693789f37c1962fdcd7d4815b064e676c876178f42f8dab400a91d48767bbfd1758646174bf060cad2978cd0cc74fa810955acb1e96b6ba0 -LibGit2.v1.9.1+0.i686-linux-gnu.tar.gz/md5/d8fd641059379d0a153aefc806adc12a -LibGit2.v1.9.1+0.i686-linux-gnu.tar.gz/sha512/bbe3b2903c981037d71d841a1376d744068a5f1ac9d476f833e3f31d7c22284d35262f04ce0fd885d63102c0bb68558e29ec2743adafa6dc618b13b061be402a -LibGit2.v1.9.1+0.i686-linux-musl.tar.gz/md5/c7e31adcdb6735e4a658e8c75d5cf213 -LibGit2.v1.9.1+0.i686-linux-musl.tar.gz/sha512/1f1ec4a3f1cbe14ff1b94ddb1d4c693b0fefec4bff51ab18920ed84eb12406d778af624baebf75a82c6411540cb62bbd3e00d975b38281a7ef51d7b7cfe255ff -LibGit2.v1.9.1+0.i686-w64-mingw32.tar.gz/md5/d28b2507115bcbceab8fe2c1a3141e72 -LibGit2.v1.9.1+0.i686-w64-mingw32.tar.gz/sha512/42d3ba8a06207dbe4a9862981f18a08c77fbf7eb0da047b2631d4cc46352e6403c8016fa6490b6732165c8286e341bd167f1d4953a77607102fceec276f37eed -LibGit2.v1.9.1+0.powerpc64le-linux-gnu.tar.gz/md5/ef21df7cd15d2eb71e9b1ed00d30a008 -LibGit2.v1.9.1+0.powerpc64le-linux-gnu.tar.gz/sha512/1b6b76d098d1b3bb48ece5c95b529e067b815aa95115f8df604a9169ed243d8b1b1c86598425f34193dce594dba19d7094ae95860ad7fed723d891df3cbf53c1 -LibGit2.v1.9.1+0.riscv64-linux-gnu.tar.gz/md5/52eb7e4c1057c704a5abf4a3526ff6f9 -LibGit2.v1.9.1+0.riscv64-linux-gnu.tar.gz/sha512/1d1378143ca1f1cb61335785b6d7db655d00da721ba5a5837c735b3dbd2a5be3207ee1e8bd2cfbcbd08e41f8aa61c028335affad7854cfe499b0210b5642c124 -LibGit2.v1.9.1+0.x86_64-apple-darwin.tar.gz/md5/a571908803d937204a84d0418c49284e -LibGit2.v1.9.1+0.x86_64-apple-darwin.tar.gz/sha512/bc3f19cc22889f7d082363274e6fd70487e0e5c1c0e156098d6f4f1350b78b327f8d7891f3c5c476628e85649a44ef0b2ec51899afa8721fbb26363fd52c1586 -LibGit2.v1.9.1+0.x86_64-linux-gnu.tar.gz/md5/9823d346d9cfebb7d14f537e7d4f66da -LibGit2.v1.9.1+0.x86_64-linux-gnu.tar.gz/sha512/5ac7a780861907554e3586ec2bd2109603af93ef3a6a3d8a9746af65013d3d745f00289dd24312b6b6bbe6154f0e66fd7925e4907873e712f3e765b960a36771 -LibGit2.v1.9.1+0.x86_64-linux-musl.tar.gz/md5/d12787cdb2407c9668dd1c9fe5cef246 -LibGit2.v1.9.1+0.x86_64-linux-musl.tar.gz/sha512/d7b0a08bc154639ca0c3f12ebf3b5d1725edc45a177ac27e91abb83add19c3e3b2c414c955477d9ace6ceeddb5e1f7257a56baccee8134812b5d47932162089c -LibGit2.v1.9.1+0.x86_64-unknown-freebsd.tar.gz/md5/0aecfa9999032119fe0ee6aee13a55c1 -LibGit2.v1.9.1+0.x86_64-unknown-freebsd.tar.gz/sha512/b2b42ff038abf32fb4e3c15d4b4760578410218685fcd2761c91efb366d5430e2dcad267f51c28205040b2714dbe91aaa060742ac66b77690744367de14484d7 -LibGit2.v1.9.1+0.x86_64-w64-mingw32.tar.gz/md5/d610537c9abd49eec7d861c814226c2d -LibGit2.v1.9.1+0.x86_64-w64-mingw32.tar.gz/sha512/5bea03406ef4d2bf89bd82fa6cdd8fa52fb3cc6efdc7e5016a1e3a33e60603be23f867630d7a5b14d2fe435fcade5229e10067a86945a5e46abaa20ce1f7d9dd -libgit2-0060d9cf5666f015b1067129bd874c6cc4c9c7ac.tar.gz/md5/904b3287acfbd1098a447e50b957f76d -libgit2-0060d9cf5666f015b1067129bd874c6cc4c9c7ac.tar.gz/sha512/e3f49faa709456d506993328177226932e3673fce3cf9ac0d0b0063206aa9ce0a6d780348c35f5ad28f59ae50f777c9f4fa54ec35af04582742cf7e36af09b56 +LibGit2.v1.9.7+0.aarch64-apple-darwin.tar.gz/md5/2ede4c6d3d3a844b39434ea2021d2d23 +LibGit2.v1.9.7+0.aarch64-apple-darwin.tar.gz/sha512/b5cd41f99b8a7b8ab3f5de5e096ab1757f0a414c1a7a2af0d212e0fa364fa540b9e1a7b0daeb33b1fd2b2a768f72c4dd362a991121d660dc6bfa6f0b7ed78a8a +LibGit2.v1.9.7+0.aarch64-linux-gnu.tar.gz/md5/c01d3d494281f773b590bec01ab9e6c2 +LibGit2.v1.9.7+0.aarch64-linux-gnu.tar.gz/sha512/073ee25f532517a26c301688b4ae9245ac00c34262d2f9dd89f6eb8bb81a6bb2050ca42fe7b7898230df8df775a02111911fecc6111a97480d9b400c2e6debb0 +LibGit2.v1.9.7+0.aarch64-linux-musl.tar.gz/md5/296c5b3f95ab75c1b64784a8de5388b8 +LibGit2.v1.9.7+0.aarch64-linux-musl.tar.gz/sha512/88abef08d58507a3cc88d5cad27b57caf1da697ddfe7058fc30fed074cc27dd171dacbb0c339bb09ec0b7286f3a000622da357adb06d4e8be86d502e04660a94 +LibGit2.v1.9.7+0.aarch64-unknown-freebsd.tar.gz/md5/ead8796d3f3b99d5ba325b344627e622 +LibGit2.v1.9.7+0.aarch64-unknown-freebsd.tar.gz/sha512/925829de3e8daffcbfbdb37c11d62f9ecd34ee37fd4cd2bbe2a1e2edb7eb02163d29865c8d2fd402d8988d1e85840357c6834b19569dee0e637cf5a010ff4c47 +LibGit2.v1.9.7+0.armv6l-linux-gnueabihf.tar.gz/md5/c94e7f2798a7d4d7686039612d5c50c5 +LibGit2.v1.9.7+0.armv6l-linux-gnueabihf.tar.gz/sha512/19e9c5233a9e1ecbb0b5486977e46bf82f7d3aa2b8cbfc3169894530e0baad20f01345540ba802fcea50dd8e0e83f4a8cad45fb4655647f9993d3b09f8896b1c +LibGit2.v1.9.7+0.armv6l-linux-musleabihf.tar.gz/md5/70484a44931e60012e733209a76b2de1 +LibGit2.v1.9.7+0.armv6l-linux-musleabihf.tar.gz/sha512/978641303ff7f84f133d11001f62e78430d175339357281b89592f2c767dea9f643aa6ada0c5820975d53fae1b943f94223399c85c79111304b319a9f59ac539 +LibGit2.v1.9.7+0.armv7l-linux-gnueabihf.tar.gz/md5/cd2ce158ebf57ff1bf45dba2eefb6246 +LibGit2.v1.9.7+0.armv7l-linux-gnueabihf.tar.gz/sha512/c9b95db04b6b7a2f51e908d4fb0d01b1897af66348489ce02fa5cef8b29a9486f22576024560be03fa8c62a6b7fac20092366499e54b4ab28d2ce33ce393c53e +LibGit2.v1.9.7+0.armv7l-linux-musleabihf.tar.gz/md5/320caec68d5d6d524ef9a3818c8b9d84 +LibGit2.v1.9.7+0.armv7l-linux-musleabihf.tar.gz/sha512/0468dd551a620da6da4ee19637e8dd59e6e614138344cb7b587d6d21e82df442fbe0ca04c4f5d5a41f58d80dcc602a01594de3a88d07ebcdd0a5687d4c4da3a2 +LibGit2.v1.9.7+0.i686-linux-gnu.tar.gz/md5/1944a1c2747574361f3f0b0adfcb7166 +LibGit2.v1.9.7+0.i686-linux-gnu.tar.gz/sha512/19d5b722ca96d3fe56b55b0da0fd6e8d696c87600172f9d273c0097455637283bfc2d40ec24dd06c5834f2289ca9b9182b20d245b3a4b3f9c5411947200715f6 +LibGit2.v1.9.7+0.i686-linux-musl.tar.gz/md5/23c7502426aa1e47a996009a6ffdc64f +LibGit2.v1.9.7+0.i686-linux-musl.tar.gz/sha512/cac8423bb37fab9f1cae8a0d69a47bedf27e09bbf536f624f598fb95f00c21d9759b484677b27dc8cc549fbcb668037dfb46bc1a0d96c6dd9e3bdff3dd5f7a51 +LibGit2.v1.9.7+0.i686-w64-mingw32.tar.gz/md5/712c33c44bb842db7dfd88d3d0dd1fcc +LibGit2.v1.9.7+0.i686-w64-mingw32.tar.gz/sha512/fa66278e1e4abdcbc40d0a704f25111ad556d0b587adc4fa612b891907f1c80cb14f993373ee923a4a700ecc6c9eef160851e86563e3913c7d477ae400d2f2e1 +LibGit2.v1.9.7+0.powerpc64le-linux-gnu.tar.gz/md5/a5c18d7b87292f36504f67ee73c309ac +LibGit2.v1.9.7+0.powerpc64le-linux-gnu.tar.gz/sha512/8bb497025cdd31ca3ec929d4dddbf40f30f38bd528219719026c78c18d0bb2921f857383cfe587c6aec80e4efffdb211b1180f79bd695d35b454aa4cdda82e5e +LibGit2.v1.9.7+0.riscv64-linux-gnu.tar.gz/md5/29f146a982a0f4fca2c4cd464c5857f5 +LibGit2.v1.9.7+0.riscv64-linux-gnu.tar.gz/sha512/25201f3deb17af1ef7306ee4b364c1243144d2c0c74d9b1a74dd5c1c0ae2a08f7fcb6f65238a1ef7cb0eaaadbcc5f6d0a48426b224b9ef7e72c4e1b159bd5f44 +LibGit2.v1.9.7+0.x86_64-apple-darwin.tar.gz/md5/b9578ee9c58ffdb429cd61631f7e38ef +LibGit2.v1.9.7+0.x86_64-apple-darwin.tar.gz/sha512/097872bcbefe6977e3bd3888ba78e3dd5d69e49bbe81bdb4f1a51788921d5a97990f95c2073c25ed798bb87e0fd0346a5297a59e933a19fdbac562394a2b6d4e +LibGit2.v1.9.7+0.x86_64-linux-gnu.tar.gz/md5/492632ae807bac1ce2cd69e10d800911 +LibGit2.v1.9.7+0.x86_64-linux-gnu.tar.gz/sha512/9e066ec3e2f0d5c80119387490ca8937d664531794478aa9af673f69d03d59f42ffdf5b8d79153a1596a12eac5fdf268898fa7ae7a0db6fd947746e9e7ce38f1 +LibGit2.v1.9.7+0.x86_64-linux-musl.tar.gz/md5/547726e95a4f1c8586b760a5ec6dd34b +LibGit2.v1.9.7+0.x86_64-linux-musl.tar.gz/sha512/fb374cd81f1a7b443a826dbd749ef12acd4c3338460461566752fdd4552082ba2debdc586d4dd8d9620ea6ad083394ed5c4866c67198bae7eda6d8daa0b55c02 +LibGit2.v1.9.7+0.x86_64-unknown-freebsd.tar.gz/md5/b7081776c53ff270bd06077d3b8fb07f +LibGit2.v1.9.7+0.x86_64-unknown-freebsd.tar.gz/sha512/1910c8e5e597824d4e7c492ef02a10312d26da031c5d35c9b4381b4d0d3aff1e87965114f4e764e76ab5558389c8e6cf4087dc70b2c1f30f71b9ce58c719770b +LibGit2.v1.9.7+0.x86_64-w64-mingw32.tar.gz/md5/badbfaa784a935254ab028b8ff3644ab +LibGit2.v1.9.7+0.x86_64-w64-mingw32.tar.gz/sha512/e5a6c91ea7a90c958cdcdfefca3f799a290bd569e393ab51a52486e26eb4b61010678619327898d0cde208672d36ad6ab5df1e7f632b6db03b66a4ffafac74e8 +libgit2-49e408b3208bc3093757a1c2db938d3590f3f412.tar.gz/md5/d3f739472ef58577c1e74fe2f40151f5 +libgit2-49e408b3208bc3093757a1c2db938d3590f3f412.tar.gz/sha512/7e284a261cf81003cd1bd43adaa5e7fab0d44929929a510b48ea3bcd46b95a7e54c327ac899619df4acfd690a6bb4e933f9dd181a45788b392fd51fa7b4cda2e diff --git a/deps/checksums/libssh2 b/deps/checksums/libssh2 index 056d373656d98..d8f58f3816589 100644 --- a/deps/checksums/libssh2 +++ b/deps/checksums/libssh2 @@ -1,38 +1,38 @@ -LibSSH2.v1.11.3+1.aarch64-apple-darwin.tar.gz/md5/87ba86e78421d6195aa6a46129ff61d4 -LibSSH2.v1.11.3+1.aarch64-apple-darwin.tar.gz/sha512/2b7129be9e9518337f59857474882a6a3448f358c931c66ab9f9ec67506c68d2356df591bd45925154844ca0d6f6e1f071d4c54d62039c5078b468fcb356187b -LibSSH2.v1.11.3+1.aarch64-linux-gnu.tar.gz/md5/84c6eb68e7797038d0863513fa4e292f -LibSSH2.v1.11.3+1.aarch64-linux-gnu.tar.gz/sha512/3012beb35fdf94136907037e8f5261a5cc94d102f461172321d4ed8f328da3789d521513dd03cb344c6fcb73675cd1d3ede606bf9a904fb811d40c43fd09d8aa -LibSSH2.v1.11.3+1.aarch64-linux-musl.tar.gz/md5/5a49057201e779f3427b794b72bf07a2 -LibSSH2.v1.11.3+1.aarch64-linux-musl.tar.gz/sha512/62a812efb4ad7b24bfeeb3bb89756004215c09a1cc01e0530f14ce4b8546f1dcbbac18155ac2ce08311c1790d659b14674e3bb3549ff68d1209d52b5e5986fff -LibSSH2.v1.11.3+1.aarch64-unknown-freebsd.tar.gz/md5/a5129167b7be7ac8ba2c873e164afb1b -LibSSH2.v1.11.3+1.aarch64-unknown-freebsd.tar.gz/sha512/f8d9cc5098a3b401fbbe98a24efaca0ea46f533ecaf11dbfe8f7e7e3853363af19914de62bd1cb5a573e55e90d5c6074532ddc6d64723c9e235b277f438ce6ef -LibSSH2.v1.11.3+1.armv6l-linux-gnueabihf.tar.gz/md5/5c59c95612bf9aa172e5d487002db509 -LibSSH2.v1.11.3+1.armv6l-linux-gnueabihf.tar.gz/sha512/5ba41e49365c2018d55c92e4a23d806ca9ab960a448593b08380527da21eec03f76cab89c34befbc56f4104002aa189d5cae6f655797f1447f395b51a14d40e2 -LibSSH2.v1.11.3+1.armv6l-linux-musleabihf.tar.gz/md5/4bc27411f0eddf82a787d1ede17ce2c3 -LibSSH2.v1.11.3+1.armv6l-linux-musleabihf.tar.gz/sha512/d6024b6949ac6867c56c66defbb99300a5661e0c73da6c330165bceba78d64063986c8851601ca74554b27944d5b02e3f602b1e71781097bbb8b12effc0cbbdb -LibSSH2.v1.11.3+1.armv7l-linux-gnueabihf.tar.gz/md5/40e1a0d323969b96ab121eb5a3ecc874 -LibSSH2.v1.11.3+1.armv7l-linux-gnueabihf.tar.gz/sha512/67ce15a5b1c1fe0fd1096ed5d2d9f44d83983de11c1bc651f5914d70d387a99ee6bde31716031b758f48981e2a9383599f077f02d61a5c783ee6d09a7bf445db -LibSSH2.v1.11.3+1.armv7l-linux-musleabihf.tar.gz/md5/9453c52394b1b06bd36c43e461a3b48f -LibSSH2.v1.11.3+1.armv7l-linux-musleabihf.tar.gz/sha512/c62068ecb1b88dbd08a2474e0b93cd313bdc4e1407a22cd9164a73b2d897564f12a3c34f6fc492b264af579b00e9335a0fe1fa853fbe0fbb18d8335b77d409b2 -LibSSH2.v1.11.3+1.i686-linux-gnu.tar.gz/md5/992453b1c59033aefa8d98b89f491ff6 -LibSSH2.v1.11.3+1.i686-linux-gnu.tar.gz/sha512/ebf14565d614086c4401e1a997a3aacc83f8e499ed836c429f87c4f95f1c8409713fad47f1c34a2b1cd23f90de3daf14caafba3c82b15642018592213607c874 -LibSSH2.v1.11.3+1.i686-linux-musl.tar.gz/md5/e0cb0566c724c107f4f04619080d4c0c -LibSSH2.v1.11.3+1.i686-linux-musl.tar.gz/sha512/af7d08dba5bb06eaf7ce8aeb12b69701d3c2829996a1c8e68510c106402a1166ad060687987df49365c26d30e8d6511c66f2a50ec810a493d2c090931ccf05a5 -LibSSH2.v1.11.3+1.i686-w64-mingw32.tar.gz/md5/c5e8d3145deb56d6df008522a5d3ea6f -LibSSH2.v1.11.3+1.i686-w64-mingw32.tar.gz/sha512/47f3c36747d2e42a4c0669ef468d395078328235d30056b7d67d76bd737b5118c1bbc720aef455c4d9017e7b9350e8cc043ed28264ea8a9ecb6833ca517f82aa -LibSSH2.v1.11.3+1.powerpc64le-linux-gnu.tar.gz/md5/12eba4aec5e320a4d0cf09225bca3f7c -LibSSH2.v1.11.3+1.powerpc64le-linux-gnu.tar.gz/sha512/d6b8413d77d8af3d29b867692f6c02b63e793f5e8f17c4777756d247c8e602b3ab87380031aefa60f2c3ddae5a3c7a1f1c739439f149db34a32c79f32e08048b -LibSSH2.v1.11.3+1.riscv64-linux-gnu.tar.gz/md5/cc11dd403ecaa373241b3c30cd16bd24 -LibSSH2.v1.11.3+1.riscv64-linux-gnu.tar.gz/sha512/d195ad62cde58dfa1e3546efd70a5f6b8a0762a2a933c637120aa71eda45dc6dc4213e87f9f401e2e148bbd5fb10638e429ae514bcda5bada0940c70cb7ff15e -LibSSH2.v1.11.3+1.x86_64-apple-darwin.tar.gz/md5/f6e7cd35e16290b198c80c61a0fca5e5 -LibSSH2.v1.11.3+1.x86_64-apple-darwin.tar.gz/sha512/2c83814ef6ae78ec94a43f2997151dd7195c0a0f9cf456fcd3f780268bd1cbdd7ea55182fc5a1f8e1413c26889e54fccb01964b0b91dd4b925ecaa16b7df8d07 -LibSSH2.v1.11.3+1.x86_64-linux-gnu.tar.gz/md5/95aa96befc9f9007e6a000a95c1b7572 -LibSSH2.v1.11.3+1.x86_64-linux-gnu.tar.gz/sha512/6058dca6d933afb7fe5fc3374937b4432f202a5dfe3ebcc2f91f65777230c18d76801c38071f84f8362527ee08656a97f79da234ab5481265a7ccf29e94c20c5 -LibSSH2.v1.11.3+1.x86_64-linux-musl.tar.gz/md5/88b69d889d602bc3df420535dba30f9e -LibSSH2.v1.11.3+1.x86_64-linux-musl.tar.gz/sha512/7335954124074e7df786989db86e86e3bcf41f503b8e3b27d6ac18032c8025bec26180bd2c537b23349bcf5673eb67245531479b939670e620faf5aa13c8c4ab -LibSSH2.v1.11.3+1.x86_64-unknown-freebsd.tar.gz/md5/6d5f6e9455c35c5f6655cb4d46797db0 -LibSSH2.v1.11.3+1.x86_64-unknown-freebsd.tar.gz/sha512/9515d11bb5686e29eb5a37bbcb7ab07574da0869c82e5b3f0cf282bbc56792af31e6174521d58133968b997caa6db75ac9b195024144fd2c95fd1bbf689ebbf6 -LibSSH2.v1.11.3+1.x86_64-w64-mingw32.tar.gz/md5/e66cdac0c2d5ce2d160e482d780ad0c3 -LibSSH2.v1.11.3+1.x86_64-w64-mingw32.tar.gz/sha512/2dabb1e8da5ea496898751d5517ca37178e1a44c78c26fe33f87487a0b4acf7185f686ce8d6ea0e65e38a8fd56e5ff09fd70becda402a942b5e459707eb2a44e +LibSSH2.v1.11.104+0.aarch64-apple-darwin.tar.gz/md5/c80c3a6d9438c4d8c039c8dbd910719f +LibSSH2.v1.11.104+0.aarch64-apple-darwin.tar.gz/sha512/584cfa26448cf6ad9ff6649fbd67931e6a450753ee3abf93e2b1e21b71efc633b9a205a8b689e7dc68ac1ed290abb76e6d1520150af13240ae9e70d8f33b272c +LibSSH2.v1.11.104+0.aarch64-linux-gnu.tar.gz/md5/736fc613e088caa8b32155f3967e5c61 +LibSSH2.v1.11.104+0.aarch64-linux-gnu.tar.gz/sha512/befc5cc82f952370a92fba06a73a99c9902f107ab6d2221bc99bc81a69d638bf2c7b0d4d1fb9798e68c07ecf138b3643bd81468ce5f1fdc7ffcd3037f6e93641 +LibSSH2.v1.11.104+0.aarch64-linux-musl.tar.gz/md5/61b8210e45460273ed93a5e4f8a7cb8e +LibSSH2.v1.11.104+0.aarch64-linux-musl.tar.gz/sha512/bf8afcda7042f7011741d3b7523272300ddb065209f1a74e4025bff34119f3940e8541a024bb1bbc2c3d0ebb2307322a5916a2fbbecafd3c8867c3c452011806 +LibSSH2.v1.11.104+0.aarch64-unknown-freebsd.tar.gz/md5/5acc317322895743e1a835a3e4b140e5 +LibSSH2.v1.11.104+0.aarch64-unknown-freebsd.tar.gz/sha512/8880f2e4d1fa6e46e26566dd15c2a60ad5b3cf3a139f290654bbca9912d0f4c0b80286b89ea57cf5dc3e17a70a933a57f4de96fb63e519f08de7ad5f72e72157 +LibSSH2.v1.11.104+0.armv6l-linux-gnueabihf.tar.gz/md5/777a4814e584b32693eb30bda0358b85 +LibSSH2.v1.11.104+0.armv6l-linux-gnueabihf.tar.gz/sha512/c5ea3a33987cda5e52c5544956c22c11e315312fbe9c417e33449b1e169b7403b384a36f75a7647572ea385d5f1ab0414280429033739a766b13d5ffd772d586 +LibSSH2.v1.11.104+0.armv6l-linux-musleabihf.tar.gz/md5/62cd1829dde7245638cc78431661a0e9 +LibSSH2.v1.11.104+0.armv6l-linux-musleabihf.tar.gz/sha512/b7436082ffa3737916210c3354bffeefa8cba4398af8b6ea4caf5ccc1736026682ba234a31a58e687a5493dffef4386a64c90374c375f01f916237d6f0c4c670 +LibSSH2.v1.11.104+0.armv7l-linux-gnueabihf.tar.gz/md5/8f75fa1d0032543d28c59e244c66a08c +LibSSH2.v1.11.104+0.armv7l-linux-gnueabihf.tar.gz/sha512/bc47d7acc6b820bab92db3c852ea1603242720e5ff332b58f9d19ef7938f478a23e6db18bb9ea1e6992064d2b271e74eb53e9258fb8c5bd7e139024dd63e0e4a +LibSSH2.v1.11.104+0.armv7l-linux-musleabihf.tar.gz/md5/f0dc168cbb2b7026f13ff0846a44f069 +LibSSH2.v1.11.104+0.armv7l-linux-musleabihf.tar.gz/sha512/ca5bf1fc1b378c8ae00a9345d0a0db96920ac480c7be169292c1c681ea4990579e0d349d3f0163a84ce863ac658e55f4b6867ad2be86060e3944374576b0e6d7 +LibSSH2.v1.11.104+0.i686-linux-gnu.tar.gz/md5/8d86f81eb69d005e2cdb24b306e5a0c9 +LibSSH2.v1.11.104+0.i686-linux-gnu.tar.gz/sha512/6faaf8989eca814921bd5cb453132b42d7681d811332984ca9a007add5b12241dba080dfba498ac349d33855c0f8198d7fb1190683e73bb7154d14dd599c8ee0 +LibSSH2.v1.11.104+0.i686-linux-musl.tar.gz/md5/cfc1a41e856cde7c27024266a5c6e99d +LibSSH2.v1.11.104+0.i686-linux-musl.tar.gz/sha512/13a084bd0fcbd05b26f481213c078f7d97b5883d9f6e7cc1ce2d933d19a208a67418fa971b0ac618b1cd86c8597a3c8558b278748d1c74277d8d19c547d78566 +LibSSH2.v1.11.104+0.i686-w64-mingw32.tar.gz/md5/bdc82199535a44d12e02057b3233f722 +LibSSH2.v1.11.104+0.i686-w64-mingw32.tar.gz/sha512/08abb65d7a890f82bfd2865abf66f104d531d9fe611383fa9d3013dc62919737134162ded1b932d61792b0ee45f546ffb3af39259943b1b081c82dda8c31607f +LibSSH2.v1.11.104+0.powerpc64le-linux-gnu.tar.gz/md5/8d20e5281ba4cb27d6982e03458df3bd +LibSSH2.v1.11.104+0.powerpc64le-linux-gnu.tar.gz/sha512/d6e716683a4f34abe416eb98e58ec4146bff479da768a77ed15eb0c379b9b07bf55ee5a6335773a504633c75712f38ea51d8caef0f72546af9e27ddd0fa4a1f7 +LibSSH2.v1.11.104+0.riscv64-linux-gnu.tar.gz/md5/848644503dbbc1d8bb10532f5a9c16dd +LibSSH2.v1.11.104+0.riscv64-linux-gnu.tar.gz/sha512/4dbd9d29f0b17492808dc7a1899f00f100e0970c388e019da6060bfd779dea03239fb2a1111e6e2619f5a99050a8e15792885187a71abbc244510beca139c916 +LibSSH2.v1.11.104+0.x86_64-apple-darwin.tar.gz/md5/f7e379473de8225ff24549a7214bc334 +LibSSH2.v1.11.104+0.x86_64-apple-darwin.tar.gz/sha512/443977b263834d83b7d7f8b0b3073dd7b4d39f972dc40224a6514837cefd7c736fc2e09e6914558382b7df9f66ef0367ea6af4972ccfa77c78f57f687cbf8ab5 +LibSSH2.v1.11.104+0.x86_64-linux-gnu.tar.gz/md5/84533c6f0454fcf3ff2935c9f2aac41d +LibSSH2.v1.11.104+0.x86_64-linux-gnu.tar.gz/sha512/1aee8b4bada0f95f2cbf44e97ed42ccf07ae22edc10c097c08e593fc196c294062b2e1c54d034609f83e17752168467f5e7040b7d72736d88cc3e98985311667 +LibSSH2.v1.11.104+0.x86_64-linux-musl.tar.gz/md5/2b3ec00615dfacc2bf35e260e59a1fc2 +LibSSH2.v1.11.104+0.x86_64-linux-musl.tar.gz/sha512/ff8cf9b6b205fe5bfb0f28fb1dd33a69b0bad7ee495b516762db42942c29347afa1028bd08897b601d42638d59cb9c21f24d388cdff4f57e3b9b1a31aa6a8aeb +LibSSH2.v1.11.104+0.x86_64-unknown-freebsd.tar.gz/md5/6882b80e65a546a9a98e257c57f041a5 +LibSSH2.v1.11.104+0.x86_64-unknown-freebsd.tar.gz/sha512/d08e663684c6007167ab23417135262775b47e144c9d1c102f7214c882faf98e30a8734c7e90b3f9b32b26437928f37fc5dccb168bcdd76342f3e87de0140c1b +LibSSH2.v1.11.104+0.x86_64-w64-mingw32.tar.gz/md5/7b301030186b141d20be22fbbb9ef61a +LibSSH2.v1.11.104+0.x86_64-w64-mingw32.tar.gz/sha512/a5e8816d323d063706346780e76169954f78249974ff5f50ee5789f5af278bfed1ef98868297c9fa7cdfab978fe6b859812316c1f65124e68e923f11b9c2b86c libssh2-a312b43325e3383c865a87bb1d26cb52e3292641.tar.gz/md5/06d5e2881ac023583c7fd6665d628a87 libssh2-a312b43325e3383c865a87bb1d26cb52e3292641.tar.gz/sha512/5dee8cce91853eb8c9968d7453b1ad0c3cd1411901d288f1731b7c7e4adf380313f61c2a66eee0d3b89eba79e420e13269bb3738bcf2c59f0b88276aa785fa8c diff --git a/deps/checksums/libtracyclient b/deps/checksums/libtracyclient index a212c415f09e5..3d423b67dae52 100644 --- a/deps/checksums/libtracyclient +++ b/deps/checksums/libtracyclient @@ -1,34 +1,38 @@ -LibTracyClient.v0.9.1+5.aarch64-apple-darwin.tar.gz/md5/c6768380fef203f5310d2cb3ab6fe509 -LibTracyClient.v0.9.1+5.aarch64-apple-darwin.tar.gz/sha512/309216fbc5be52319241ccdd2360c8960ffecf50c963bf248ee6aab6a43a6cb4c3a703391c7d3e1b07cb077badce930673f85f28c0924975b75909a4654ca3a6 -LibTracyClient.v0.9.1+5.aarch64-linux-gnu.tar.gz/md5/20d6c3ef5032d458817f18aa7f92b44b -LibTracyClient.v0.9.1+5.aarch64-linux-gnu.tar.gz/sha512/d2e341ff18bd06b57094f2356fdb36a3f9dcf56f3340b83006dc02d41d6e5040f145a23a06f86ccd1c9800e93fc4461ddd7902b6eccb93b78b365c824e5d764c -LibTracyClient.v0.9.1+5.aarch64-linux-musl.tar.gz/md5/58cd88ab771a31326fb4e3448325a17f -LibTracyClient.v0.9.1+5.aarch64-linux-musl.tar.gz/sha512/dfa11ac4fa5261bad557a244458b2f67c20c761c5d49d31488a9b944345e32e55f1a40515097b4f5f4896fe5046e3bbc70faa40ff4dd504d4d97cfa42e46bc33 -LibTracyClient.v0.9.1+5.armv6l-linux-gnueabihf.tar.gz/md5/cce5ef56b53255494b334df157e285bd -LibTracyClient.v0.9.1+5.armv6l-linux-gnueabihf.tar.gz/sha512/1287f734669b7a7a6f43b6cf3b725de650e64edcd4284d51120e7371f91ca18c63d4981d58c28ce2da9355eefa5a02e0bc0b35303b096733a7842bc952da2df6 -LibTracyClient.v0.9.1+5.armv6l-linux-musleabihf.tar.gz/md5/60508f4f9c757678b8e52ed0342079eb -LibTracyClient.v0.9.1+5.armv6l-linux-musleabihf.tar.gz/sha512/92d2d8521a8b6c6901f36ad71f59fa1b2eb58d6dd06da02140cc26b6c44d2741ddd0cd3107bbd3e70ca1713a33fda48d5288bec59304264d0afc20d4e0167a50 -LibTracyClient.v0.9.1+5.armv7l-linux-gnueabihf.tar.gz/md5/64563cbf8fe18fe84a2ec9e9fda19e6b -LibTracyClient.v0.9.1+5.armv7l-linux-gnueabihf.tar.gz/sha512/b0158367834f32fd7449d45b983f37bcfed71727bcea3febf382a779265ce4b99845ae32f5c3969d83089bbf5e072680b11138a45b7b7030364ce341f285e220 -LibTracyClient.v0.9.1+5.armv7l-linux-musleabihf.tar.gz/md5/1d272a82701889fb695edde6bdde21bc -LibTracyClient.v0.9.1+5.armv7l-linux-musleabihf.tar.gz/sha512/1539f06593eb769ba35ef198f90b9fa6c11d7146124f21e35c8fee577d8fcff3d71f4e38e6d26d84dc8f66b06a26a130f4bc740a201cb27573ec8e6816d489e2 -LibTracyClient.v0.9.1+5.i686-linux-gnu.tar.gz/md5/4e14c36ea3b1e54a26897767d4a010d6 -LibTracyClient.v0.9.1+5.i686-linux-gnu.tar.gz/sha512/4c1d9cda642a4ea3084b73c0b536edd0f33a216aa02c59f914ab428e0e97120ba3f81e7eb2262a2242884f553fd319b80ea7b013344e87e173dc1ee9b3421ef0 -LibTracyClient.v0.9.1+5.i686-linux-musl.tar.gz/md5/75ced50efcc6ce1c17064a3447933fb1 -LibTracyClient.v0.9.1+5.i686-linux-musl.tar.gz/sha512/969c41de91d288e4e37a36f990341c2f71d6788d93bb34eb6708532ea60bfa1bae6760871de33b42cca50b61dbf8028f639538f34ab9bebef2312d449c259f4c -LibTracyClient.v0.9.1+5.i686-w64-mingw32.tar.gz/md5/0f8c6cd2e1aa738b340e10e1ce81732b -LibTracyClient.v0.9.1+5.i686-w64-mingw32.tar.gz/sha512/76824a28f16650e14b06051f49a5c42cd8692dbcf808c2d47ab21ac986cf49e17508062ce5e0e71f2a689112098897dd7fcd47b819cab6967080b7e23224bf1e -LibTracyClient.v0.9.1+5.powerpc64le-linux-gnu.tar.gz/md5/573e5c6aca49845b7e9e881c7bc8f755 -LibTracyClient.v0.9.1+5.powerpc64le-linux-gnu.tar.gz/sha512/7451731c6f0bf0ac08c966f925b0dd628f6f4c0ff7e2d65e8bd3d27231e3fbb2512169c917431baeca3fe8e66af4bbbea7ca4ba79dd7d3b9e1d43b08a580dc76 -LibTracyClient.v0.9.1+5.x86_64-apple-darwin.tar.gz/md5/b037ea1027e6466d5dd9c0fb41f65ded -LibTracyClient.v0.9.1+5.x86_64-apple-darwin.tar.gz/sha512/81e2d00bd8eaa1cbcbd5c0ee4552028ccedffcc072beea3dc08ac3181677da93406e8dfc581a78434175fa5bb861df06848dd3012f8adbbb6dc72efcbb5094a0 -LibTracyClient.v0.9.1+5.x86_64-linux-gnu.tar.gz/md5/886d5b76711252176eaf7e41dd2db0a5 -LibTracyClient.v0.9.1+5.x86_64-linux-gnu.tar.gz/sha512/783c6469c586520c7f1206f5c2eae6a909a2dac7c3f726f439da91b0f3dde970fc4f17c6e3b54aa8924ae537b1b6a14729cd1305e6488c458db68ffe973c2ced -LibTracyClient.v0.9.1+5.x86_64-linux-musl.tar.gz/md5/170a53b0c63e7e130bf3d71590e27193 -LibTracyClient.v0.9.1+5.x86_64-linux-musl.tar.gz/sha512/ccda22a897358d9eb55b500dbeb35c8d136f484a649c29066d5f2d0665d5b71526502237dbc374c2d018fa212896fa6a6c6903fc7c4833f814d1e60c3f12fa83 -LibTracyClient.v0.9.1+5.x86_64-unknown-freebsd.tar.gz/md5/58fdabdbbdaa7b4f53bc4249a7c57059 -LibTracyClient.v0.9.1+5.x86_64-unknown-freebsd.tar.gz/sha512/ad0d4b74c707da86fbcf7210cbcc9a3ebdef770ad0bf12f33e3476c2e455d48e389f021fc47e50b85d1c4741072396b71dc034cb95bfd38f775f4879d06b5998 -LibTracyClient.v0.9.1+5.x86_64-w64-mingw32.tar.gz/md5/cf5b40edd556a7b8ab23d28bf0eecb12 -LibTracyClient.v0.9.1+5.x86_64-w64-mingw32.tar.gz/sha512/8ece28dae598418c3435cfd323609e6e615ce8299370040fdd3a000007faecf87c5ffdfebcca80ea9b4180ad9bf13d3f3d65fed2fafe6f54d4a6068f9cae61ca -libtracyclient-897aec5b062664d2485f4f9a213715d2e527e0ca.tar.gz/md5/51986311723ba88ac305ad2c1e3e86c6 -libtracyclient-897aec5b062664d2485f4f9a213715d2e527e0ca.tar.gz/sha512/f92c5bd71fd3e933f03e3535c0668a9afddc7ea19531aaee11b22bde09c57cc8a555f7f17f489d4221645fb6d73ecf9299d5bb11949d7529987beec3e7d91763 +LibTracyClient.v0.13.1+0.aarch64-apple-darwin.tar.gz/md5/a69b8e4c1ebe997a4c4f4fb22105c39e +LibTracyClient.v0.13.1+0.aarch64-apple-darwin.tar.gz/sha512/5e7a9d6d9bb8e9904823bdcf8e3fe83c263bbacdbd4dc77bb27b3dd930235c74646b9ffb10c228936d5df8dd084434aefb5190f3ea818986daf7233b0f3efe8b +LibTracyClient.v0.13.1+0.aarch64-linux-gnu.tar.gz/md5/cc909cdc323596b2db3b2a7cb41ab59e +LibTracyClient.v0.13.1+0.aarch64-linux-gnu.tar.gz/sha512/704d533abf7893460d6f3ae3a3bdde4c004eae1b4455b330f6189a36a2c5537f3346a6f949265429520f5aca285ade8daf2c5056b1e326de794158cb9b7f78e4 +LibTracyClient.v0.13.1+0.aarch64-linux-musl.tar.gz/md5/9bf6431d384f81771eb7c9b0dac08071 +LibTracyClient.v0.13.1+0.aarch64-linux-musl.tar.gz/sha512/5fbab846dcc18e1d3ee93abf21996437270e8c691b121d1c4eeabc904e17650d978dc4e240cb4aa1bad0a1b538b780ac241e3cb3b20ec730e20b8479619f9dd3 +LibTracyClient.v0.13.1+0.aarch64-unknown-freebsd.tar.gz/md5/89bb60b17b668dcdd5efd711d90c6ea8 +LibTracyClient.v0.13.1+0.aarch64-unknown-freebsd.tar.gz/sha512/44c0e46da41b271b5b0476422d17a76b097f461f13a5d3fca0c36fa67136417087d0c605e08839a009f0d1555ba4e04b7d31112d2340c387bb61bf658467f34e +LibTracyClient.v0.13.1+0.armv6l-linux-gnueabihf.tar.gz/md5/32bdadcb863b33948e34cf035ed7c502 +LibTracyClient.v0.13.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/746aa2f34b2844eee02eaae6d2072c729a5f22defd6f4dcf525cdabc8c092dfe8fd8540f03403a2c58f049f2457eb2e55d60f1a7efd70711f49e7f0cb847d36c +LibTracyClient.v0.13.1+0.armv6l-linux-musleabihf.tar.gz/md5/968b65c444b0569e1b78dc082940e5b6 +LibTracyClient.v0.13.1+0.armv6l-linux-musleabihf.tar.gz/sha512/49257f0852963fde93cade8b82acf4a1a41f31dcfd3f76d1d7fa182083bbac90cde94a6118462d299d854eb8e740b91834a675b337ecd4c42f2b3ad94a9f7c8c +LibTracyClient.v0.13.1+0.armv7l-linux-gnueabihf.tar.gz/md5/8ece285dc42589227e89a6302db679b4 +LibTracyClient.v0.13.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/1244c49fea5c5ebe9ceb7bd20f36e379f4a1956e560aa6070cca4f19759098718a7c92c1ccf3521c0b62128690cb8ad972b8aa38e3ee8b06a0db0671b9f9b370 +LibTracyClient.v0.13.1+0.armv7l-linux-musleabihf.tar.gz/md5/45482778d809184e6984ffc417c004bd +LibTracyClient.v0.13.1+0.armv7l-linux-musleabihf.tar.gz/sha512/4e9fa294b95a7ec62973dbcf5c66f481daf6f39d666f1a5bec8fa13ab607deb1655a8757e23a33e96b0b620f4d4cc1883677f015d92bdb9f6a1a915d45820776 +LibTracyClient.v0.13.1+0.i686-linux-gnu.tar.gz/md5/d9b7dc454a5a58cd55224849946e3938 +LibTracyClient.v0.13.1+0.i686-linux-gnu.tar.gz/sha512/5df33eae00290ae3ed3c0db0b451ae939a731e61700640021231a15f35a045f58ab2b9be16e9ffefcd1f5bfc4dd72808cdc1a20a22d26c1fa249bae0278f780a +LibTracyClient.v0.13.1+0.i686-linux-musl.tar.gz/md5/09a8e53e72eb595ba95c6e529a241f2c +LibTracyClient.v0.13.1+0.i686-linux-musl.tar.gz/sha512/9de8b174bf20dc472e6aa074b673eab6384f97cecf362ffe60f578269bbf98cf495ec52148924dae3ad6cbd8dcb6a5c75e5e90a9930030db31cf0530e09579db +LibTracyClient.v0.13.1+0.i686-w64-mingw32.tar.gz/md5/c50574edf249c8b7c70f2135e3e0e859 +LibTracyClient.v0.13.1+0.i686-w64-mingw32.tar.gz/sha512/43877511f328387368979cb35bc56cfb554e249c8f7f2c12c86d5270747854918014acf7d27da69a3f83c81cb9f1e8a174a659688a7e71003c0ab4e5bb533f36 +LibTracyClient.v0.13.1+0.powerpc64le-linux-gnu.tar.gz/md5/8e53205c65b831f308e616f80de6dd08 +LibTracyClient.v0.13.1+0.powerpc64le-linux-gnu.tar.gz/sha512/c983067fdbb6fe3ada30861f3a33c451a0ab481d3505da720ab4d9acff8edce2d67ef3c112c2ac4910b698856627568d268c025fe1273f62d2b3e35a20c061c5 +LibTracyClient.v0.13.1+0.riscv64-linux-gnu.tar.gz/md5/6297980941b8f3546f896ad80ba5b6e4 +LibTracyClient.v0.13.1+0.riscv64-linux-gnu.tar.gz/sha512/e2dce0e784c2ed1eb60394f3df9a3d0804ccb4bdf568a1f818269e309b097223a01ba65d7bd5bb814eff89ffdf907067b483e54528b71e601eba53e902bb2717 +LibTracyClient.v0.13.1+0.x86_64-apple-darwin.tar.gz/md5/9e9b09f24221aad2b8c5c4ab5e9e20b9 +LibTracyClient.v0.13.1+0.x86_64-apple-darwin.tar.gz/sha512/027aa83b23b7421724f5cb6b305c4c46e6831adcb1a419a2161b3c90c1d0ec1a8a15b6a795113c0995037a9e04ca324781fe72c750dfaaf6f5dec7dcfc9e38c7 +LibTracyClient.v0.13.1+0.x86_64-linux-gnu.tar.gz/md5/dd7e713a546df8a8ee2b41be711de5c0 +LibTracyClient.v0.13.1+0.x86_64-linux-gnu.tar.gz/sha512/5a97797ec28140d3dd878287defca386f5a4f4976c815eb5edc38ad9da6e7817c7378625b8f38943bf8e6d6775106d2f8458458f35a0346360114d7f565e777c +LibTracyClient.v0.13.1+0.x86_64-linux-musl.tar.gz/md5/fc002c515d678d5063c681a0519b2ac7 +LibTracyClient.v0.13.1+0.x86_64-linux-musl.tar.gz/sha512/6cfee9c4576d340a999321b03e0ca9837317408a6ef5ae27fdb43ddfad7a1dd63f1c1c58a2b34e4a27df411af3fb512a6fb2a3280230ee2f35819f98eb040d2c +LibTracyClient.v0.13.1+0.x86_64-unknown-freebsd.tar.gz/md5/a531faa71f0e3aa30f79481f8c322c09 +LibTracyClient.v0.13.1+0.x86_64-unknown-freebsd.tar.gz/sha512/73bc8429bfef8278bd8dc49b381e98b28a764b5248baedfc005c057882b2d1406f87f11d8876bb9e42f5f12866bb73bdfdb3877f5bdeefe0b219e47ffdb61648 +LibTracyClient.v0.13.1+0.x86_64-w64-mingw32.tar.gz/md5/063952da04ffe3eb563d61a732609ec5 +LibTracyClient.v0.13.1+0.x86_64-w64-mingw32.tar.gz/sha512/129f939bec924dc2c09d3f646ce66b11e54f0c6a14c7b0742a23cba5eea8b1bbc1e33ca8793b453f81fd8d0e608d746c12697ee9ddf3b9f25922030677449529 +libtracyclient-6cd7751479d4efd5c35f39e856891570a89dd060.tar.gz/md5/3cc78417ed1223f8cd96fc5c4f51c0c5 +libtracyclient-6cd7751479d4efd5c35f39e856891570a89dd060.tar.gz/sha512/01c06025eb9f3f1c7b27042c2b0d725527133098556ebb34a8b9583e8d1add770bffbdb93df0625e35316f5e96f53a603ff6fefa18cb766c8bc702e79e594623 diff --git a/deps/checksums/libuv b/deps/checksums/libuv index fb2904b308a90..33eb47d43dd76 100644 --- a/deps/checksums/libuv +++ b/deps/checksums/libuv @@ -1,38 +1,38 @@ -LibUV.v2.0.1+20.aarch64-apple-darwin.tar.gz/md5/7b889e32bcb01afc19f9f3801b28a0fd -LibUV.v2.0.1+20.aarch64-apple-darwin.tar.gz/sha512/cfa58e835512957171c7f2dcc9171bd9ea1717f71ed6920b6cac2560da3c5b13440df0d14c5aee210df3346743d3605dec22d78891e8237f5c3867d5cb6d4f56 -LibUV.v2.0.1+20.aarch64-linux-gnu.tar.gz/md5/696ae3e79f9b838a98dae8152d980ff4 -LibUV.v2.0.1+20.aarch64-linux-gnu.tar.gz/sha512/74ac009cebfa3ec67209921296f5d395c2f888d825b0f3d3f3bad5432819deaf5ee758f88030e620530e94f4861734d7984b8ef981ae4eebc356d96e274d678d -LibUV.v2.0.1+20.aarch64-linux-musl.tar.gz/md5/d23a45e4d9cefad93e3e83cf990c095a -LibUV.v2.0.1+20.aarch64-linux-musl.tar.gz/sha512/80d06afae0b5ab657c5c743beeaff112c3cddabd2a2604f8fc16a50cab5db878b4ea8941496a11004c9464bcada13844528cc4fee209bdd8ba374d9b13351991 -LibUV.v2.0.1+20.aarch64-unknown-freebsd.tar.gz/md5/7957f7740cfe6dd5ccb4ff4cd2811b45 -LibUV.v2.0.1+20.aarch64-unknown-freebsd.tar.gz/sha512/923fc895d3fe41005e47d6af422ba450b32e94210c393065a891b44f83f016104d3073a9faa609cd7979bfeca9d0ed0c7164b37de5da92deeb4e8676311cc57f -LibUV.v2.0.1+20.armv6l-linux-gnueabihf.tar.gz/md5/c2ec51470a4d66e3bd23bed67c109cc9 -LibUV.v2.0.1+20.armv6l-linux-gnueabihf.tar.gz/sha512/d139d5a6a141933a83f0d10f8da9366d709013bd2ef005d2b783716fb13d165b87640b7d9f51dc2772a69fc6f63e8545901c96da2343506a2f940edf36332164 -LibUV.v2.0.1+20.armv6l-linux-musleabihf.tar.gz/md5/a1a432902cd687f692c5619e72de241b -LibUV.v2.0.1+20.armv6l-linux-musleabihf.tar.gz/sha512/209207dde41fa699adb72af9a6211d74366d393d335c9b0d4c9c9509c832123707bca27e8410c7b1c63f89fbae77dc15eba55031701e307f88d5c183b929d9f3 -LibUV.v2.0.1+20.armv7l-linux-gnueabihf.tar.gz/md5/bdb8124a2b3c9e42b1b9dc8ce813e664 -LibUV.v2.0.1+20.armv7l-linux-gnueabihf.tar.gz/sha512/2f8879b4f41aa6cab3b195a76dd02376bf5d47f51ac157541b0c8453d03cd2f51fac83f59b2cd2fa49a2395262d18d636251715f1a4912750aa3de56eab4d6f3 -LibUV.v2.0.1+20.armv7l-linux-musleabihf.tar.gz/md5/91ddead3be8fa8b06b37983cba074615 -LibUV.v2.0.1+20.armv7l-linux-musleabihf.tar.gz/sha512/3be790d1c580e2a69d76171b82cfd2f594135920e68f7f4ff7a6fdc42918130e628458492fa2a157947c25effd2de0a71d434fcc1c6fb1d741985bbbfcfac3c5 -LibUV.v2.0.1+20.i686-linux-gnu.tar.gz/md5/c906674ba1bffffb685f0f00189187c1 -LibUV.v2.0.1+20.i686-linux-gnu.tar.gz/sha512/c3e5b394959dc76e2abd51fe59f7e8bbb1755b3a008f019ad05c41ffe8fd9f42d0bf262a506b36f26a2f8f4b14c937eff70a9e1ba2c55f19fbc57e5ba9c2dacf -LibUV.v2.0.1+20.i686-linux-musl.tar.gz/md5/977204bc42355bbdb908693b3baa8e10 -LibUV.v2.0.1+20.i686-linux-musl.tar.gz/sha512/5fb2717575ee97545026b79c2acc0660eaa04827637138896aabbe69bffa0c11732de4f9aad9dd78ba68db265ccf5ff3aef244d7da0008cafc4a417423db361e -LibUV.v2.0.1+20.i686-w64-mingw32.tar.gz/md5/ae698bbab57855ad41bd850ef2ccc695 -LibUV.v2.0.1+20.i686-w64-mingw32.tar.gz/sha512/9c6530404babe8383c6a1db7fa1e81b40b08de0dc2d2be3507a6466c150acc842cca277e39680b21a6c7f5a6dbae618bd3f5c3ac8f11882898cc116d5e13e7d9 -LibUV.v2.0.1+20.powerpc64le-linux-gnu.tar.gz/md5/e68314bb638f210d2ec9326c617752ca -LibUV.v2.0.1+20.powerpc64le-linux-gnu.tar.gz/sha512/f74ce6b21cd2776cdf49b4c6c2ad551c0bf55951f8bd9090020e71d2b233f72907a3e145b9a95715c391b82ad36ab1a069bb9f87d54c219179021cc26902dd22 -LibUV.v2.0.1+20.riscv64-linux-gnu.tar.gz/md5/d8e1ffb730c784df14faff06027b724d -LibUV.v2.0.1+20.riscv64-linux-gnu.tar.gz/sha512/60e7699ac4dc353d0b9fbd34952bd68185ab301a449354b7e805b6759d3866ffa5906041cd9e6ff299cb9fe3f5a92f4c5bfd9c441210125d52f06d614afc84a5 -LibUV.v2.0.1+20.x86_64-apple-darwin.tar.gz/md5/15d8197dea20880edb96a8bf643fe95e -LibUV.v2.0.1+20.x86_64-apple-darwin.tar.gz/sha512/7d65d4d2e0720f997c164234b78a729f4d4239fbb0b01634f23081e2209ab010ef27deca1cc3824fd8e17630370efa86f1567aae035a246ab9f60a6c14ea6d3b -LibUV.v2.0.1+20.x86_64-linux-gnu.tar.gz/md5/013be6d2673a59cd00b2ea62d4e34e21 -LibUV.v2.0.1+20.x86_64-linux-gnu.tar.gz/sha512/f466af2a1f9ff83d887ecaa200d3042bd5685d6cd487af00bdf8c92bf1d4256017f2757084de3b7331071c473b254df43b03f580de09db3bb9268af759a5b0c7 -LibUV.v2.0.1+20.x86_64-linux-musl.tar.gz/md5/21099b0c3ad76c3d67fb24260ec39836 -LibUV.v2.0.1+20.x86_64-linux-musl.tar.gz/sha512/03279a4d29072246dd806d800b80d9db14b637235e211294d5840104056cd206b370a987a2b771216e762549d13b13432f1e1893510e4fba6c4b111bb3330a05 -LibUV.v2.0.1+20.x86_64-unknown-freebsd.tar.gz/md5/dfcce3d6c2c42f419987f8289b1ace02 -LibUV.v2.0.1+20.x86_64-unknown-freebsd.tar.gz/sha512/8ad3c51f43124b7ad43cbdfe92685ce448d3195eeff5838387ef3145f1bec89851106293eca501ab6f986c0714f9bf9ecbb5a7ef44935a76a95bbdecd4fd2fba -LibUV.v2.0.1+20.x86_64-w64-mingw32.tar.gz/md5/7c37d147586c06f00f6dea947d7e912d -LibUV.v2.0.1+20.x86_64-w64-mingw32.tar.gz/sha512/58762e5a7a8cfd4ee8f0c7ba2c2919fc3b922f673e9b6138ee3714062d8088cac8e3cd5bd244d262426260ac55cef609abb30c25b1a5e38123fb61476a522a53 -libuv-af4172ec713ee986ba1a989b9e33993a07c60c9e.tar.gz/md5/c1a7d3c74ef3999052f3bfe426264353 -libuv-af4172ec713ee986ba1a989b9e33993a07c60c9e.tar.gz/sha512/a3f16863b711ddeeb5ab8d135d7df7a4be19cc2b9821fc78c8cd3ba421231d39b7d8bd9965321455094fda01584842a58f60612d93082b4fe32210b8aa44d999 +LibUV.v2.0.1+23.aarch64-apple-darwin.tar.gz/md5/039e899082deefbfbeaba7a8d24793e0 +LibUV.v2.0.1+23.aarch64-apple-darwin.tar.gz/sha512/275840a1a6f2e11684df9c77f5eba9f30c9ada2b8fb57ddd4c0b136d0661d3b1099b3c2fe080052b9f7365e4866b1681ff830669198720ec53b04277596cb9c4 +LibUV.v2.0.1+23.aarch64-linux-gnu.tar.gz/md5/a5af58a8fb6d3ef7bc60e87f7db1be60 +LibUV.v2.0.1+23.aarch64-linux-gnu.tar.gz/sha512/c2939581abb304b4d4a78b4fc2f2df84efdf7cc475ff3fe627a0081271eab24911ef418d56b7a76d5fc89db7a07366d81482277a3373e427f5086a71f3b7c7db +LibUV.v2.0.1+23.aarch64-linux-musl.tar.gz/md5/9d0f524a08ca8e3ab4d944a714d08583 +LibUV.v2.0.1+23.aarch64-linux-musl.tar.gz/sha512/535b572059f06c49e5303ed576818baba7f6594969433e240b2d361ed14fc986bdbbd02b793290cdcb8dd82eaace4e86acd834986b0e2ceddc124c3b7ef95018 +LibUV.v2.0.1+23.aarch64-unknown-freebsd.tar.gz/md5/5f57f7d01b42671eab99245235b16db4 +LibUV.v2.0.1+23.aarch64-unknown-freebsd.tar.gz/sha512/173cc54512ef7dd862fad27cfed9c58c257ff808fc276a8ceee99217084d919a0e88cad75c9ddc5e7d330e6662e04e4ad1d2d48fa170693c4939b3ea5b7182ad +LibUV.v2.0.1+23.armv6l-linux-gnueabihf.tar.gz/md5/27828aac426c695ab818b89d6186812d +LibUV.v2.0.1+23.armv6l-linux-gnueabihf.tar.gz/sha512/ab6abc54171651edb719a1306c9cd1ac5e1bc102d666cf6566906624156220cd0a40eb75c3ef1a6f89b7a71084ffb26718bb913130ec02db61852126615ebe7e +LibUV.v2.0.1+23.armv6l-linux-musleabihf.tar.gz/md5/0a9b9878c3d5a06fcd7b45942d818f95 +LibUV.v2.0.1+23.armv6l-linux-musleabihf.tar.gz/sha512/0ee1bc92b8755462fc68da20c9e1e8b6d79c469d31d6c04ef56c9a96b7e849f59f874fd72476c33954741578479b3cf2e98dde45914380c92d7c95b66462e02f +LibUV.v2.0.1+23.armv7l-linux-gnueabihf.tar.gz/md5/8653d26820503809f17bf4277ea62e08 +LibUV.v2.0.1+23.armv7l-linux-gnueabihf.tar.gz/sha512/d5c1a8f6cd474f5a125bf3ec82667be48c2a25f6a889b85379890015f3e9a5cc1f9d554bd24126a395414ae2a5c6fa6a7df516138ae62f2a0d10b5c0e7fe76d3 +LibUV.v2.0.1+23.armv7l-linux-musleabihf.tar.gz/md5/c00de0121e2b72d345149d4b600f4710 +LibUV.v2.0.1+23.armv7l-linux-musleabihf.tar.gz/sha512/e1e2eb400070bcbafb8d42555a4e18efaeb350727fd3fab8c12d4183bd38bd88b05c9db478d4c937807aa020ba75d20632c0844334317aee814efb62d9988e80 +LibUV.v2.0.1+23.i686-linux-gnu.tar.gz/md5/106b54f0441c91a10fc6289eb7ee34bd +LibUV.v2.0.1+23.i686-linux-gnu.tar.gz/sha512/9602e929e8477de2b0e979a57e2e4bda50fbf01dac8aa74b3d628abc8cb654b14b8e33806a8dd972cbc88565a066ab33528361797f1e07e64139c22ba1c6b8f3 +LibUV.v2.0.1+23.i686-linux-musl.tar.gz/md5/01ccb0286114f35c94eaffb072829706 +LibUV.v2.0.1+23.i686-linux-musl.tar.gz/sha512/2e099162bb6468cd319dac38db1aae3373fd70a8fd9c97baaecab72acf520354553e8df908add23b1fae10610e649bb400a867961f3419254a2a5cf67176e5f0 +LibUV.v2.0.1+23.i686-w64-mingw32.tar.gz/md5/c59482a9575c0341c094c6e5fae0ef9d +LibUV.v2.0.1+23.i686-w64-mingw32.tar.gz/sha512/67e56b09ac267532e412472939626819ec85615ce359fab0d0eca9e130743bf2684bcfde8e531730e4220b8c86ae5a46cd6c0f5bad62b6cb2d3539106e1d438d +LibUV.v2.0.1+23.powerpc64le-linux-gnu.tar.gz/md5/508aeb28a9416668113f201cbe359517 +LibUV.v2.0.1+23.powerpc64le-linux-gnu.tar.gz/sha512/cbce72d3aa14bf1eb09eea7a1317e6ea299d96d5816ed681115b9a280fa0982bd2a90959e2597b0e1589cb3b2f0efa0523fb059106278939894b5cdb3ad1a1d7 +LibUV.v2.0.1+23.riscv64-linux-gnu.tar.gz/md5/f6477f94167269ab442607996adb232c +LibUV.v2.0.1+23.riscv64-linux-gnu.tar.gz/sha512/cb9a7babbb847a3622f9f127276ddbe776d284d1ed1313d558ccfa1416caa023d8bd9f3af6aa3b5b768bf2d3c2f8b500e0f142bde975a8aa06fb8ddc20fb0c79 +LibUV.v2.0.1+23.x86_64-apple-darwin.tar.gz/md5/d96429509769a1bcb6d17768dab5d53c +LibUV.v2.0.1+23.x86_64-apple-darwin.tar.gz/sha512/51947166da338d09a769da4b84671be199c64423e79323ced047925df3d3497c39f02162b63ea15b4d6aa82cc5a981631bac0cf3eeaf560e22ad9d977d9ce010 +LibUV.v2.0.1+23.x86_64-linux-gnu.tar.gz/md5/3ad3d2e9f5d0a0454ae9178693a9109c +LibUV.v2.0.1+23.x86_64-linux-gnu.tar.gz/sha512/40746a18cc0b4a2ffd7301f7aedb4d5247dd5db4abb212e1c82339d10b7139f884d259de674d8304fbff14601e037d99ab4d67173df03ea78bbd8ecb42348bf0 +LibUV.v2.0.1+23.x86_64-linux-musl.tar.gz/md5/7c8e16673067bbbdbd7193e9e8f9cfec +LibUV.v2.0.1+23.x86_64-linux-musl.tar.gz/sha512/ba4da32ef4e367f11cb73e3158299f4cdb6f71fd937c7ab048702cc80288e2352de03f0e6ac4efb252a71f9aaf4a74cfaae257e2ecae791d98604537ae543e59 +LibUV.v2.0.1+23.x86_64-unknown-freebsd.tar.gz/md5/25d1f9da2b0fa82b616c70f80bb9f712 +LibUV.v2.0.1+23.x86_64-unknown-freebsd.tar.gz/sha512/eed0a58e78c10a46d0e336267c8b9d6089bf10ed91f6698e4e09b7eeb92f8b601ab231491e927bc8e0346662f14846c423ca0b467e9cd8146290a2c2b53875b0 +LibUV.v2.0.1+23.x86_64-w64-mingw32.tar.gz/md5/843c8e9d6249009c3694ab5ddac4d69c +LibUV.v2.0.1+23.x86_64-w64-mingw32.tar.gz/sha512/ea0c619ce793fbf7b4773b56a7b71867ff873167db70a22266044dbe6d5b2845fe212b55cb058fad2a9f07875608e14164ad6b457550c7328dd78601f6692aa5 +libuv-e6b9850fbef2b4778e6a81bb277d3e59a2779402.tar.gz/md5/1892f9de3e822ef5cd910cd0c82c579e +libuv-e6b9850fbef2b4778e6a81bb277d3e59a2779402.tar.gz/sha512/48c2ed9433e4aa419e304eb3d0a405b8010742adc9d7efd87c1093c030b4e0aefdda6c973bc77161379a7e5ec44de19d9af4e3b645657d1cc08cebd7972bb724 diff --git a/deps/checksums/lld b/deps/checksums/lld index 6e8fab41edbcb..4bac264ec50e5 100644 --- a/deps/checksums/lld +++ b/deps/checksums/lld @@ -1,120 +1,68 @@ -LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/8cdfcae09219dd5c42a575ee95dbe933 -LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/050783b4d922a806a566e6d5e5afc77c5d0bd935cc8c5538f388b6221d37f387bc8e81682196c8f07b56100ab2a0aec331738dc5815925cd09bb1b5b0cd68ebd -LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/4beca49829fe5fdce2674c778df340c8 -LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/c7593ade99be7e6da301845525428ac8feab063b2349bce266e02325dca806e875fd8586ba1f9252bb5e7f701c1e750dc26dd7ddb4fffce3f48be918d89152a9 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/34d2f188679bf5abf4c28c7ff4e30703 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/e89e8156ac291b3373f5b723638affefe43a7a6845ea49720d6e39387e3cdeb34e4790de5221df1ccb50939eb8b41141af94fd4436bcafca49e24e3007bf1c16 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/3a431153ad307ff728b3c07f10aaeb17 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/5526f8e004d93b54689f47b62067d689ffc5471289d9f43f2ddacc6c17275d7133e86837de1895b40d0cf969006d07b8fed2ecc96b7b65472d6fd4972f4ac082 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/e9d0903886a7be9fdf5421540899418d -LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/42f29306911257a2982b0cb9630a4ea81b50add6567d7135c9f3098162c699c2d234256cff80ef789a0cdc6ebafdc2ceed9009bb84aa18bafdba5d5d69874210 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/97ef4ddf834a725a7d13faa1ddcc3906 -LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/09a557558f51b3e22fb8a93c5dd458c5d0b397858df2acfd2cfcca11e8fce11666eae867b8e3f51c3b296a175d72fa5716135f0c0612c859bc293d8fbf98d226 -LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/8e5b12a175c78b24ce43e442c735924e -LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/00492672666c6ece6a8273811e1cd0ff1a4fac3f32ae33ff003a978c44d2f6e1024e24f9e29bb970882f766f922987f7fcacc79d2c81c29acea5b397406fbba4 -LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/b68c5ed9e9f52c3954d1664c49ad1755 -LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/5a2cb3cd347ebaa05c0aeef636e8dac0878114e73004f2f903608a87567a2a4bfd36d633df7cf48e20b60965fbb2c8e9760180f0ab03e51101775c75029f5a97 -LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/53f810e9d3d0442f2290954c70f93f37 -LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/417dff21f10eed36ed0b911341903914658240c157ffece632b2e0d28b65b0aeda376b1dbeb1f4677ee0b434c63d763fd44c9d6195b192a94ca575314fbfbbd7 -LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/25713a33485613c69d718518aeb49259 -LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/4542b3d1f98a6357ff7fbe686803103223c82c56096696bc90d89e1378455d3eda19fe45835f6e6d5035e754ad5aa51f24678ad20ec24fcbaee873cc729fd189 -LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/978b490710efe5019486c88d19dece8c -LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/7b47a5216b765f7c54a8f48006d351a01ee38ff379b003f62880894b4a5523c5842fb1205edfc2dc71420f7d19cf31f417e6aea4f6b30fcd905671dafb73dd2e -LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/52562639d81bfca12e3586740f4b35bd -LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/b230117d657a8d18d0c43a2d3da025890f4df3b11ceabd7fd883621c3baff6ce1e8dcbce213edc2fdc733a59022889619a56fa7c1d773d6039150b6e2e44eea2 -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/01efa3cd3ecdfe45803800b8b8bcddf8 -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/8af2ed095fb79e831b613cb5ad83393b5299cd9df1748f1a6b5771c51d3df2d14af286d00e0fff432b2e97be5991bc6d515fbf2089f3f58ea0cefdf87eb0622e -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/1f5c758df3e347960a5429ec7cb84849 -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/1a38fd706b55e202044db984be82f96fe21da67ae39edd396e49a5dc86daaad47185877a1863f7ec21e1303b103fc533b0729221fc87533d3c02efb9e05338c6 -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/bc0e9ff37b2bd42460b2627e973f115d -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1926cdf82fb5807019647133a602ad34a57dc6b45ef072443ce0a5eca02ff7466d5daa93b45e5fc861113e2aa8818f789afabf86a99a431d73bf29e5c0951a10 -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/5426e6022fb0646d61a8b48f4e91f50c -LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/b4a0dd3f2b3b9d4d643405ac9ccac3448ebfcf199568ddf7079b6b77c5f29b42d9210ac2a6881416449b50d4d52b822dbba3ffaa007cc79028c8ef49c07f0c31 -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/08563d88b2ced2f5ff6f2a876b10e88a -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/4bb9d42394c5d6895dae473257af00950692be4d07a7d393e70d8a7397535389f095c4c65d5b54342e51068c76157d9737c404b01e18f7648d54ee4aaf1b36d8 -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/2c20b768d5c77175613b59104b848086 -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/122177d274de1867fd383a4d90e25564d5079f9e7cd09e5167fd449ab0d0c5cefaca867623921a6163872fccf6b65dd0640a3b22d33f30412e3e2f458f9f4f9e -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/d301952a3da16a4d6a32e5027d946982 -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/90dde9fa203766ac128c80a167046508b9a1e31121b5498e986e469559876a6c587c01016aef6e1ebc328fd15f8f479dfeff94042f8e172b6a4e8582c3696995 -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/364812da3ab7e29e11c14503d676d01d -LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/0a0a819023643ab2d06800e7dc75bbd0b738b91a11e2257dc752f56b8d7e6438b85ab5695daa72672d4797d871b912c210b6df2c80f516429f9e8335a97a18bf -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/7d77b4217d5bdb192dc545749cba9fe7 -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/16cc0e199d37d4537505b1cca4e0c648d5e9e49b2135dbb2b96355501d9121ffab32ba1a076ae1b77a57378be3bc32822d2109bd85a4c6a5747e50e1ccac3965 -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/efd07e543cc5bbb3767f9d64c83eb385 -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/5ecd6029bec97fe1cb56dbbc1b1eea39df5c049b23465c8eb1232c281e733d59f730dc38107368468ad56a07b3c864ae4a0d51fd2503d3d4077923e7f1ba49b8 -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/cca1d7d104f95c432ddde95969f43c09 -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1ba73da3f31e1fdbcde0289e399fcd34b7064f626d32cab724559f8d0a893287a1aa67965be5ef85c9f7e4ba645f338e852e0f93ae8942706deb5e7654a6e38c -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/9414dd3e1cd61b52f3ee312e5fa4faec -LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/d60b7482c276a76e288360a37c885689dcbea4a00b555768cad635a5d04227f294e85fbca6f586036ee3a33ec77bf9f9f3bfd110fcc697cee8f4586935067ddb -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/22fee78e5d5fbfc99d426382d1f1d7df -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/195c19aa3ebff8795cc6c1862b9d2c8a577809a94da20a860de57e4200baac1c2e76683c278eac6d54db4afd976e11dfbf055a008e7e44d22579847cab7f2eb6 -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/356ea4ed4353c3d7245987e55a5f6897 -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/fc071585b97db5900a6d72c109d78596c731a2a0971f46ffe26b4423bb1da3488df350aa5eb9e7ee03b5b2fd66c3c80950c764e0703a7c6fd6a4015af05b994e -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/3dfb7a907a0b18a971e4227f82009dbd -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/bfe5601d4d4654f15330eae9168327f9dcb91bc658cf7514526776d13f4b74ce95f99a5dafe48d59deca645f22a4f5b6103bbb0e35ebfcec85e82767ff5543a4 -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/fcc8d611a6bb893408da646268faa061 -LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/7d09101bbd6ee53bcc4ecbeb0883a5a4fe440de49902595c53e6f56d8e0c67a350b923210e17d22618569ec087ecabc097719307ce18db0cb7cea7c16e0b805e -LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/74faaf27eddb69690ab6bfd66479ab05 -LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/23aca5a1f80fe35fd1ce7a3ea563bf490215c9e2fb82dbbe7fcfbecb6b5a73dd1643e019d0446c57c6f31e9f7a1f5c52c8264555f23ba3c5fa8275edb6715ad9 -LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/6ce4b175bfd0242be28b02d22b8da351 -LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/e0b0e9f27f0546df44985e6b1cce85711efa9c3023d9bad88c25111ae86107a72af5c2921014173f8449eebe9f57ecdf6e1afb0b10d5593ce9b90e0415aa4954 -LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/83c135cffa8563f67f0ec1bd8cf271c5 -LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1ab03926e7be1691e9e79b15785caacd00de08ee666a0fca264fff5fab1c277e368d7c6eb77beca32709fecdac730f7d9df03191ec2240d0ba8bbf6ad9494330 -LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5242e12e1c74e2dae73c6b4b7d4307e8 -LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/0f88cb79e838b2b92111473638fc7f2e7973b6301597f80caf66e378077cd306768d6cd18035c26262eb6e79525f1690dedc6d6db2b7dfaa1483ab2008bc7d9a -LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/7290bc875a2fcac989aa8fc3122ae6b6 -LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/a332d69b92ee11fa2ab5ba909c8d3632ebda2bb86d74b1a4a93caa62d6cdff5e21356d92b6e5a5744a5d0b8bd94c0fa1a4e69ada9c5e006cd35ddb658e37352b -LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/645a495655eeed199af92f26c0c5325e -LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/5c6b16c22322bd92a0de52a393a87085d1a3836f85a75257675194a6fdf6783aea9e77670e176371cceb637b9450c7c5b2b4d03f82e1580b813548b078f7b254 -LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/f060e69be07766a078cc9d8158abdde5 -LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/e4f006aa955304144983cd3437fa2ed213e0e27d904d69deb916ce82f352809311e443522d31d928e7f4c1907c0bc82828d422f5b1f0c9647abdc0c4fae7c6b4 -LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/ee06aa4b93bcb60c4434339eb68715ea -LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/cc6f319855af8e35c282428bb3c9012f0c4c9c4a8dcca1065d2d88c732bbeef6cd21cd42708968335ef4625fcc50e80d126e1caf545dd79cac9a23b4753bb16d -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/80757f0836a949560b66c03891472b07 -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/ab3f0bad55b902837cffc1e747a39a7a0887ac5dccbfc6fe31571de58bc88340fd2b848d7cf806768e0dba4b895ae9c945214757789c9ef134f4b7e48afdbee2 -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/716af34d7c11db49857a31b685bf30df -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/9e05fd855b39a8b33781902d801244471f38bcb766b234dfa15f71f3a3bc0a1500b2f2a56c137b71edc4c1f3bd456145ec589a188640a2d0eb2ece0c3c361075 -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/44f9e0d5ca28b85641b6c06daab74733 -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/778d85ab80755b95a6b2b61b676026487cb8dccde3c68a71935e9b3db4cd113577d5072d4db1d09f0c7233a24bbca5f1e4bc1fc192b3a7839fbdc0a68b649b2d -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/d2b52996d60e0e4f3b3012d481840842 -LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/082980ba92a796db4d9b757445ed9400099f1b2167e79b868da3aaa5d3d2e627fb0980b8e9351d8f2fce9845839aca6d386ea59a1639f5aeb5f7060904f6cc1f -LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/df3f2fc3722ddf530462247bc484e6cf -LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/cc881793eb7f2ebd9bd37058b50e87afb3fccebfa545a3422ba2019d51d17457d4a0cb6a49b8c91e74e76ec9cf923aa1d8ec0d8e8e285d149a521622b4c0add8 -LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/05cebf19f1eb5c1871efd39de205d490 -LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/0cfb6fdeada331e5d0ede9a7c50b1677d3b4bb5a36149a24f957cca4944e6852bf016f4ca2aa99dc7d9a590a28dfc2e02399feb32846064bf3e3065920400198 -LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/bf911ffeef7eb44907d288b0cf18f434 -LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/66fe8f97f4e200eed12d3e7dfd79f1062318af5f0d4d96fadd389f7582146fc45a6dab593d8346ecf3bc23f02a84035a22ed42cabb7222baf406890a5f80856b -LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/45556dcd1655ae34564a17bd4fec150e -LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/bbee6388e40007eec26c766dda6a6e4f3ccd84656157e1e81ff103295cc1dbbb42db6c858f9f5519434945c27807d880934ad791eeb8e13579003edc065a272d -LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/0647f954526b0ff65dc35a1b65b4b756 -LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/d614f0ae0da90505f6d8710077c7e9a06e815f4a7192c167f791488ce5fb8d0ed2e6e3cba1262c8d78917217315e820dd180f8838cc9fd739eb4dfe8b7b7793b -LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/e13e78f28632eb19ed1451ef1f243328 -LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/d035ee204120915c9ff7fe3385f1066993cb7c0b21df970be62ff573a4adda395341b0539984f579c236a973e18f05ab8e8a4befc2bae34fde08b03e9b975807 -LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/6442326b50f9b7cca3edcce9e671e544 -LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/294b2bb952ebb724c57c9f3f25e66a87224d9686053b3e4d517ca82ad728093371d193852450b579a586bbf2c2e9b4fbbc703657335524253a23a3106ebab9cb -LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/51b96db9df5fed624bff8a89bca2e96e -LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/27086b42e8ccbbfad398250853cfb6941f5e6a2d81d6dfeb99b6a08e456c017cba3ec7978e4cdf2f48b3ce771b7015bac7e5953648db0a1e65c601e9dca76b56 -LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/9485f990275be736e6ed8fe9c905db2a -LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/58e4f4ae61ac789030322125ffb6b11a16eda792d1f9e8d01aadf5e399375c40d263184f05143e2d01402bb60df068c6028ba2c3bc9757037f776c0f63de1be8 -LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/1bccb2ceb7070060122dc2c10811be4b -LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/06adc0285afd6f5de86b533ab73cc5005769d44562e748b55062ef0c195e8d6b0c287a20b99582e6096ab0c989a072a1c7bb1d347a37cbd9634d165fdb78c7ac -LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/ad160553bf0f20984759686db7b87243 -LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/f64d9d3ba6b9595ec745f9326eb7f1564d99330975e2997f387f969c21bbe757fa8f63faf583a115887a311f8806e70ec3b9a87ba6a8894a09c0e9fb8496c357 -LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/0e5433d428daca35c954fe261c82a874 -LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/67de9856e05cdfca1ec25ce9661337a04b01ee49f6f2eb05ab970d19ee585beea425c5d8c6cd72a95c8d5da48ac7968a1ac2ffeca9817fc695d2be020149e76b -LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/c8a10b058232cef8e9ffd0d1ccfa6c4b -LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/15d54ad7337f337a340a8ed0b2ef2bf9b75687ce691369fd9780e50a3e064e1866295cfc9460889f095501a3d48b7f891e1af22d8d197097640ceb94bd257aa4 -LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/4eda180a515b58400fe5230fa309bc54 -LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/87c0a59e425768fdcda295ca8323eb6eec2690aa9784dfd8d6200a69820c2dab8d8fdd7497c2b53e255a6a5bd855ec3ada9d8e1f65fd12322833c58363772137 -LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/f971f5bb4255f544511e689bde1a66ef -LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/43367f52d2278cf8f96bfa60aedca4174e9a97770de4b23f4a586d644439bcd437ae9cbf4a802ddb22ebb6f5411084d1fbada762f29a34ba8ea5d0859f1100f9 -LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/6ff7a32c9089f1c0ddd192ff037fe1fa -LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/af9e2fd3f6cf68d45a6469f1458e2b3b21ddf8522fe96a6060a511e80e9da562c7c13ddd4ecac0f77d2e432618f617f829514684f9f7cab197adb9ef8f68e01b -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/d3cfeb84b4b46d430a859cd2460f1a89 -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/bdd8c1503844f8c99960fd7d72faad2993ea56ba1d0b3af65b37a4fb3df586779e864379cc53a1387b50be55259a8f2f6d9f1f2544c1e09365d2ae65528139e4 -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/32d94428edf18af0a887570b1a757a3a -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/55df0a8f4dc2ca2df5926562e40065fc01ef1f9a3f88d5ec0e6dd40458ac66922a6292091c26b48bd1bbb599a081c1e73ccf34093b11f2888b97ff979dc75dae -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/462735ba1292bb622c82ca8652d0d587 -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/f0fdb9764db4870e4db9bdf2fde99caea1016c3b499dcd406a188a5c973a5d3e804625fabeab8bcbba51a45a503fa9157a02aa9c772538d5a0b8b7953642fc0a -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/0c86dee0baa0e9e71c5c703e45d1d726 -LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/28f9d913805287661765bbd24c15466a39c6485ab96699db27cf359c18165dc8121e95359065c16f16376369fc76a63ac67a451ab2c53f189b064c637dfdd0a7 +LLD.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/a704e33e3032096f14d192117f1d695e +LLD.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/26e49f991a703c46e3380021441746668439c37ca7ed12a55ab4666b76ddc5cc1aa009b62670bf3d6bd93a2991c301187ebd6853852096ad25f63862fada706c +LLD.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.tar.gz/md5/82d5980f4c4567489ca6da6938e935af +LLD.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.tar.gz/sha512/a1510ecde56b9686f9e6c2eff0cd6b59a3abc6d07823a5cc0b74f70c7812091af2797fe154922925ad9de2135c8e8ca82731e8c8788fb308c6ca5467308889ab +LLD.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/2508c614a43989cbb5a899f3b425da06 +LLD.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/e964a58b5278099f6741123c5432dee359afd2af6a63ac30d8f3253c1a976ca60f16e1a4da93bffc96b833c4e0c95c6e2f9db426e5812ac15acfa69282fec50a +LLD.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/515bcbd00bcc06b0e4c89df00e09e8c1 +LLD.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/bc4d9236bade7ea31b8728c2dd3ee0e0d7dc76b8af1ac0b23d17482df449e25934201a5a4b0b4a1ab9eeff80a6662d3d39d7293af617d6489f2516c3fb46235f +LLD.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/a7205c2ba7abc33db044b6c53bdb4a45 +LLD.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/0e447d978357ccf30385defdf90d783ed8e56504e31cfd9809cc767e23a56ddaea7648ae9e55ffbadcda7cf205c999dcb3c8400665446dca8d7bc182fcb45bf0 +LLD.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/0c8c2b8472445cf41ce9c3aa8780f710 +LLD.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/022929603dec3f4b2c30f84c414a157a6423ef421840107dab5be31b874a2d7239706e65f855188862c7819769d4bff93579365f60b0a4d13bce8400b0b2b601 +LLD.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/a63699f3aa61dccd1405f4450e310319 +LLD.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/676b26eff2bf9c25ff4d86a1dd7d2dca3ab1b1ebb6af2033964a98edf292b922f52091adb2113262472e26a1458d6ddefd92ef80a83d61cbed86aa50a76ca27b +LLD.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.tar.gz/md5/a7e49005e10a17d2938c47dfeab3b50e +LLD.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.tar.gz/sha512/c7366c73f3aaadca321215471e7bca9d4057ba5037a7541dafdd86ff90f6ee07d61ae3d8db07126d4b055f3ec0c4052fb367bbcb296c2708806d2be44a89f6e9 +LLD.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/5b379afb2bdef203403c19157dbb60d7 +LLD.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/4016555ae3f48cf714c2cc207afdbf2701a2802c92180d555803c6ad51c9583cf26d5e98f65cb6039b906220d7a041259ea77741356bfb143841465d0e14573e +LLD.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/140c2e393576ce3bb0cc9a736de4e0e6 +LLD.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/68b82d25e60d94316b6e686c08800914d3edc45d62f8a4cade05edfcce3fcb1e2a25aed559a57ea0837c8aa3b9a379722d4450d8e0ee9e84c7cbc7395d89b2b1 +LLD.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/d489ef109706b3bb30e813acf5a996b6 +LLD.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/94d448ee52638f3a00594c9d31b164a6bfb19576cb184b63642c5e2dfc0466fd0f141a36ad283e8570e6fcd59fd706ae51854c859ad5a993bfce0ada3e1e119d +LLD.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/43e4c953a77400380f0b01567abe6b3b +LLD.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/e0ae4f14737c78ffd03efa041a7994337f9c465bc52bb290bf536ad93a502ce54b46e62b4433f72d485fd91f10acce26327afa9beee0d743db7aa7072a750a6f +LLD.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/bc8ee0c5649e08f4c07c5dd708a52b7f +LLD.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/0c78033d6f7b20d1774de50801ee07b2f4a8651d4d9f5bbf16d0ca5e13c6476d2f9da44e4a6bd6231563b59e59d067ed00b3d6d3a34e031699eb8b96382c7294 +LLD.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/7e45f640d95e5908f49f573206aa1c56 +LLD.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/1ff6113d46f412877e5e59f986ab6d2b8ba8c008a9bc7fb747aaa672674d89c298a23f29b35ca18abc7e92d8a152bf2fa93a43db93d17abd44e956b95374d51c +LLD.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/a0c6f7bc8f937c0cc9618298f908abce +LLD.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/bcf4b91f303c14e04e700a0830acb7311071fba99b3e0b6bce6418c9ce1e379c86a6e3e2117a831eb4c32ca438973f4f1311618b90c25399845345c048a00628 +LLD.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/57f6605f4456d9d6873d8c2c7924c3d7 +LLD.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/5450c8835cc0c97477868168ba96ae910566fb83fbbd81a567d9f02ec5fe495fbd4c6867c3e326fc94306fce9b40b3e9b7ecd45e96e75d428cefaa06d56f8071 +LLD.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/1bc450042ded3f4f196e9cefefb96be0 +LLD.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/e47d42b4f6aeef572884146866630ff93fa056c26683ab18df3db408d6d99f8079bb4c71d1eee13a6428eec6b44c8b34f47052fd8f7ee92dafd59dfce7009220 +LLD.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/d69b898909c7282a2bac1a463cc4bc9f +LLD.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/35e29acfdbd1b4f9594179f18333dcf8b622e55d5b257d7cdb2f1bdc459893b095708897814a4565b6ace60320a98bfa5e47de8931cefc397a4b01c95b52d406 +LLD.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/5e9082a3f1d64b466c87035d9d1724b1 +LLD.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/361666f210ecb7b076c8a12c9cc1822c7334c2a9c65f8650d2641951618fc3eb95644e4a5d0e986780eabaf884e8d5f4315461e1c760c99f4f1412aad769634f +LLD.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/f512bbab39e35be6e47d57b64406912a +LLD.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/8f428f0691b95f89b3568cc65f06c990dc8fbc473b784dbbdad773a2dc34adb078fedcb006b359d9f2b8965111dc507401f6275b3ded3ffb6e2d281a2ba9fef9 +LLD.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/bc72674b6d0244f353ca1b147d67277c +LLD.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/b1b374122e782b92bb4133c2140a7abd852094071bd4c69b7f3b8719f9f247716533dae46168a9be4ebaaef57e79caf7bb5efab1082f1f1a08548142b513e9d2 +LLD.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/6e383c2519e82351aeb8ffc9df66ed84 +LLD.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/7917206019146a274970d4d8672f4f41e9ba8cde4b56ad05a7d8f356f9e604c60d3783ec707398efb074f28ca19e7582a97253d817f84e43c5ea93b53e60a287 +LLD.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/dd41c2660f00469f18493458c3503445 +LLD.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/0e2e7af44dc855730511b06ce2cd1fec1ccd6eaba4bfb2e9c0eb296e4ee0f729f7e973c182ca9f022170694c1feb2af16aed4ab56aa9b61fe6411acc37a6c70e +LLD.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/b91bf4994dfcae75660670ee3c66a9f2 +LLD.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/4950aadab99020e9462323a70534175937a05bf64e9c89ff579a1653bfadd05626d68d8d391e3b363164a045261ff67ee2f6d32921e260b5936f9da922195dc6 +LLD.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/a72416631f10c7366857bdab37558582 +LLD.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/a0842fdb7f73a5a2cc6593b330199e128e14060e257743a526a2d793a971bafe71b13439e7ce489b54da16c388adcd0817b94c9d8827e04254d9cbaad66fcdd5 +LLD.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.tar.gz/md5/86e20b1d1d41041fb7c623b30e05c905 +LLD.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.tar.gz/sha512/359051f57627d01ac57267f15bf286482ad51debfbcd8a4eea09a822b09a2e06b326f8bfcf3612909cc14ae32f2b1f20ab39696bd425265a06eadc3305ff13ca +LLD.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/f4f2600c3c709978ebd780c72902edfe +LLD.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/90e2d8c59ffee8498e7673f0524945d8199279ead3c11f835b8a9f3e682a7b498784e421af3b3327e089262fa1946ce7bd3d4732de11f825526adc9e0e387ed4 +LLD.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/35bcfe1b968b6e2619bd20edc9e71162 +LLD.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/eae9d2cf17acdb6084af3b4eebaaa09961e8cf684b476973e572f6ae2913fa36acbb448c84ad52fdf4275c41b5d038eff2cac9269eee971628f3441a2418ab50 +LLD.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/999fc2acb66bd415719e19f6f888ad56 +LLD.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/59d552f4cabec432111aa56ef85a8c3cc357ca7e607d97d5985dfdf50ab77636ba75e5f45cf9cd315e0878a1dba7f07391e19ba920b35ab4b4495a4b38b935db +LLD.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/bbc625adccc9ab4b809d81724973ae84 +LLD.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/bffbb2ba7fa24c402a78eb3ab9529ccb900cbdb8791d3edbd17c94916addb759653298227b8bcb7255566c2c1eeefe9f99aad0ae69dfe4665831c0efc4db6f82 +LLD.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/c0533b7732eeb2b9bdfd6b6bc58abc7c +LLD.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/b5380ca65bae56679e65813fcef5146d370789a131564ff67710affa7e8b8901d3c42bb81be482d9bd49f8cb2201f9b0d7731e3c5bb6dd6afaca5217d1736723 +LLD.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.tar.gz/md5/5394e3d92c9a7e75ef741db50b0cb575 +LLD.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.tar.gz/sha512/f02661b1d3f7510c2742403380a60e1952d01863ae8b50d6a5e234ed23d1b48fb1430444dc93f79162ed2df604b7d19f87fd538bd53f891280d6393f3aa8f55f +LLD.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/4af7012afccf2ece6c0be948ff1bfee6 +LLD.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/0d1f635379fa2b9eaa701266f6ff5c8099e795ff3a8202861181f6673b2e8e9825a93fcc4eecaa5c94472a9b06918b50f5b13219fe0d1bc3f1e155b97b228617 +LLD.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/9f9d1dfb553a933adad3d08649fe8100 +LLD.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/6d515280e71410d8bff6d1c791b74e9caa5522a3aecadb24aa32f173fda3ba051cdb58b086bf8ff3e04e768294972a2de8eae134ad98e777854e269a730a0f71 diff --git a/deps/checksums/llvm b/deps/checksums/llvm index 552904b4cc204..6a579432d04c9 100644 --- a/deps/checksums/llvm +++ b/deps/checksums/llvm @@ -1,244 +1,140 @@ -LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/038cfa3a9136493d533a122a0da0ba1c -LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/dd7921a4a056cdae21a7b04c862d5056776a42e91e29799c1daf48e12ddc049140469ae12d342a87931b7b83f139555675051ceb89bc9930e6683a08f82d4da8 -LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/26dcdd037bb2049b555f4b7da48b6ab2 -LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/e39100ceabd0304eb6fe1b02292a9c25854c8e4229e36bf9831bb386bf08a2d2cc40719163111cfc624b6da0498e59b853624aa1916505d4db9fed7d799c9ac8 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/ce2172cfae9bc13e28f503ffed4c2bdb -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/b4a53a1a1ff9aab16fd7304772096d73d9716831ec5accab6016fcb9e51e5ea83f8f17d9342a0b03837e710c4f88f418c300366624924df1784faa35c9adcf92 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/6c0bd66796ef1982cfa3ac1cbbbf0e39 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/ed2d9753e7e885a6eb9d6bd5de186df8082280aeeb797e7c38dfa3821e1743ca9abc2c735b917720841c54a722ab44507bb125bb1c1e26f1bf2524b0ab0e5743 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/22fa2895daac28d10ad9403ca77d5693 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/29ce790dc2fc9e4c0079ecd5902c1cad16cfe2b176c9172656d14f9792f48ee3064bae94479fdf0f41c062842325a6c7ecf6f453c50304983ff48a2f5794e155 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/00aced1879f83691170582b689219827 -LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/b0f61eb2a84b8a445a066abd54cb5c26dfeeafe02f2980bbb3216e8bcf562ac9890bd4dce89940b36937150e6f1d0be3986e3943b9240ac73df83e147dd1c2fa -LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/e4f0d850f1ddf3c3ded9f9c783749005 -LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/31761e04c462a7bda09e6c85239ee356318cfdbfb353350faaca6563e60698a9916e362b560c8f28b88c3bbfd556d7fa1631232307a9c67129b12184f6b34a10 -LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/51603be4b6e55ec5395624a1ad89461c -LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/b219ff439fb028a1f5c360c3dc38c7332b6abe107f9fb13ba22cf756bd3ebef10d000737d8be1569d0d81294bbc9af6f6e5eb8aea98e90882775523baf20151b -LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/217b079b37b2038dff9905039ba12984 -LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/a5ffb0fca566384f014fd0ea0fb07977eea1f96c0e76a213864be472ad6021a8861055a1f8353dbe4cf85dba161d113240e732eb812ee1b821a3332e24d1147c -LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/7fdd788e1993ecede1e7e1cfa66ce1c1 -LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/8483d21dbdbab20b40f453e776ebaed3e0ebde517646469e3848e066b6a69c39b9102e1a37e6d97725048423e6335313339b945f446cc0cb0f2b5c9cd336a742 -LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/7bf432f28eccb363d91411b2cb7e0dbf -LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/d374ee3a252a09dbcb5f9ce516e1dbe1c5fd1489d07312213f57b690d245fd06d9e3ebec3ee399fd24222fb6fb9bdee92e7aa58896d2954d7f6c821ccde2751d -LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/8be5b43865cdc03d55fe62166b10696d -LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/6bf57f66b04d58643f0fbb2cee3c4138205e95f4978136b5b472450a5a6481fb69e968e84b63f2c7a4c6e4a4107e96742d04c30bfc7ca1fb3d64d0247c524993 -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/c1e18ad4763fedad78b138c8361f18df -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/a7bc2154bf8d9ade8d346a3402995eeac3352dc7f3d058850d34df538fe004f8121fc91dbe1ceaf059b8215a5499bd03c71f2ac5db0a14325c3886e482c2a260 -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/0720fb4c94a4d13d5eef479b99544311 -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/c253aed3e4f7d0704083417631a9b64610db9a81c636c37050b194084631f1d918217dee69a78c517aa4972c22c68d96c64f1d3db067cd61ed7106f2e548cce4 -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/df1648374d8feb6beb345bb3d048697e -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/74854e8f5fcdb068e13b6b3bf83369888d16e603a9cc4b524c7edeee9c95501e7bc39dfdb119696c0665914947b8363dae72949f76fd74942e15476cb5e5ed55 -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/cd02804f5eb841693da41db5d6b61472 -LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/af22ffb9becb1dfbe529dd30975b258fd1a7bf17f416ca326834f5a7ec534e15a8d74f08b98087de457492cda5c7f2bffb0ea57bae5025cd992324582f2cb3f6 -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/4519bd2b723309739ac6b1b8ec9b40e6 -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/a184299ad255e79b91232ee079fc8617bfd82b3b72f46c62e39a964b1a7bd664e134f0c565585c10cb7b78472417eb8fee0be833155bf2f2a42acc73f8f4f9be -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/bd8ff4a34fcc829c83ee9d634b2925fd -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/576771e02790b532f89b23a6c9df3434628d0bb48527e39a071bc458dc7483a040a6b0c136de183f3bf42fd95785418aac6cb85b8dc15c69245a8f6ed44effcc -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/4b9f611840f96c4dbc30771c72a93739 -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/c10735cb77fbbcbf05e89811c87090fcfe19fd19f86cd2eebf8d09ff6d02f0b23c41ecc6a6c0205cf889047b3b5332aa0e362818369069e35212c2927a49a226 -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/f5bd1acb396b3e8d87aacd01dc00a700 -LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/0b0647af108b854e760ddd606ea9fcc0576793d71b6fe39e0f355fe03538e4ba2c8eb821a7468faf382de92adfeb2f12a07d47fa9f7b28cbe304d1e399bb0720 -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/f3ff46e4cf9aea3fc8834d3b9e777048 -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/3eb7d36b99e30c44cf189df640ae42316e664e6f4ad20dc83da4d1fe87a2a9375bdc4f54c0e751a6a84d97937c9b5add880b7413000d12e3ec34daf0830efbc9 -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/d1644c95cd080315f31a7848f07f1866 -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/22a438b6133ce7aa3aecad386e8e446b8373315bce6224fb41f183f2eca40f9c2a5d529221d3224f645526efc3f112b1dc6d3fec0c56cfa6b373d9f7667d217e -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/88e5f3943425417e11e137ecf45b7441 -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0d1b59de138a317a2e19860cdf4fbdef0e536cd3d22d1b8f3fe3e2737e08fde3521cc8e40562c5a304fe6e78ad4fade85af16bf7dae9665ce8b3e0b5c4212eec -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/7d57a0cd005d10393c27414e5294ae26 -LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/eed2f929a73f4cb17c0e338f7e06b10f9ca0973a53c0b8459a3ec60330c25f4b582bcf0908bc3f193853e7a849ac3da973165c71032b0f99ca80997575fd2f0d -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/8947b6739ff8ba38c5afba85bbb3626b -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/303094d1223c7cadc9f3507f79df7c97c3ff53c40d93136b5ce622403d6625753329b718f23a74fe66d1d31c1517c7ae3574c86c88edefa15671cf236e6f7a00 -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/c72a78b88c7272823f5fda84ed05b379 -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/a19cc0c639b4724d00cefba060207db252794bb3e14dcf0ffaef6787d6a4c3e68248cb62197fe4c9177098c760024ff3271e3660599d25f6e336d714ff2318eb -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/07e0a1e3f46e3601216188222c585f8b -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/e56c24feaad72b16d80e63509449501d37ff5c31a84ebffbc4eb31608511a7ca6a30f6eeefefdb82558ec2378fc74d105acde60d5b930b4fd1bcff35c52ec210 -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/8831ec22c84c9737661aee6684b7951a -LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/c1205e9fe613538bde8a90ffddd6e21ac849b601e55cf48cd257eb323f2fa52a9ecf0a1bced274efe5d78ea9fdf280b6bc3a21943d2ccf47475f474bef1f59bd -LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/1804610fc03ee2f414b68722b1cb4346 -LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6f36e726a78f02507c93cb75ca1112ec41e99b2fddef8ebf31090996d9b4a5992186062f1fc120b67b19c6da0c905ea1bbe404774eaf4811bf782d3150160554 -LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/bda9d895d9ee5ab01b12c830b31f7b65 -LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/f72fc2e9d434b0de616a8decdd43145849220d230583284d9ac4497ecd4c06a77589db63290fcc3aa080364533d32c60cdd63f412795f8bf09a452f643447906 -LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/2d32e54cd77a9ebdc4e8ad09226139f1 -LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/2ba917662ee8a2a25827394207b25ec313d515ef4c942dba14d1ea8abcfc90c4adb472c557ff61c8f9f9655a0780d6addeef00a689852a5add01fb7e1e2f5b34 -LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5d96535fd770ba7c782ee5ebb14a9341 -LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/04127a1e3dfbfdeb1b7f644adc5832062f649068d161002dff376670dab7b1cef2d8ce91d2c7c1254b935f4ffda1699cbfa30f1b189eeec8c662dc6a00a351d3 -LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/5ab83d6a3b2a80a0ce26473403e9d6fd -LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/57541c1956b6db41d6da6ee4f9ed8e81946c6f50e6e0e9e2d88f5c2049c13ec2599794f92a0f87794ebe13ab7bbf345e1b5978920bd4b4a9419a0360944703fa -LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/1df5afd05dce8bfe48b4de3c83a1c169 -LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/8f6711e77fea3aa28c6c752a93b676b062cbe95b682c7834a7d290a9f2d37c8abe1a44daca86c73f31dc8d1fad5314866c63e766e8cb581a63a390e49027479e -LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/9f7ac8c4df65e684f31a77c6458cbd8f -LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/f01b4df2f56b3a7fce3ddc2400a71a7795a5e6cc1392f63ee2d164f35acc678db4b13ec6ecd73b716e6405212f15362b751b5e4fc6ebadb037577906421d3728 -LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/b6c3ec59f73a15d6d884b75089752009 -LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/c2d79fef639b06a24ffec333f7748298b19b40f3c1dbadadd609bc1c6a699c8ca3950a7f4d9715af66c01a74ea22addafa642509e61f71a13c7303ba97fa1f70 -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/a47722f625d9ed01e4e8c89c7e5ff361 -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/8a581faac501cfe8cff7b27502e8bf5108135eb60544ae2988790fe856caea26151cd00c430c12bac15f56d41357d862b97daf34b7630bf6f3883824fb1700c2 -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/0deee3e8f23bb78aef9ab92901635db1 -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/7c9cb3c1cd8b7b1f777d1ecc6bf3fa5b3536ef4bc74c2b7ac8dde9c82b52653890ea0c398ff4a53c9545381a4c1534ee6b1be3bbeeadd8b1d39d34ecbe66a432 -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/4cdeeebe845a999b0af9ac967cba2c32 -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c1cda095c549fe4f1e8a14aa9318b0a151653daa10f45598fd1e3f72441ae98f69f3bfc1ee1ad1ab15f88243081ba7a65315ce2996510d09f3541af4a243d63e -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5ff186b5b66d3023298b02d3d1ba1ecb -LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/733ce7757004f1dd06ee768389edb13dc86fcffcfd0678b90bf296205492ccb5206042f103ea65330aa291b0ab46208906d8fe513557bbf8da7b05c4e098a07c -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/07a298aa555905dc38733075e37bacfd -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/135deff2656e728dfacff4383e095876ce85da4f8de5ed178cc8919a949627f1d5ab61b2c4dab7fdc98e363b0fe630632d5a9bf9d6d074d33f0bc7ef62eb7ba1 -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/0ad39270c0136606cde9bc37c6f84540 -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/64e3d2ef2130e83eee773798c968e65dad02cb0313e48946e6f8e3cbff98100d246b0acdaf5e23cc4d2a61cffca2aa237a7909887dbefbf79ca48ad6f7a80572 -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/3e5b8898f2c67fec7ad5eada36156686 -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/860dff38be4614e21e2e12349d4f0a776cc96aee3870254b42a0f2851e0a3186188486d47b1f804b9c1670b6eef2501a66b758ad6c3f3823b10b65dc7128579d -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/b0c9231cceb48572296606fa1ec3b763 -LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/db16e2d9dab7d88c27b33218f075442fe4627f00ce32af2ebe3f1b65bd5da852f91047e97abe99e1e3541fb42e592b638d314300ec870646af823e29afe3bcab -LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/16aefb3daade4fc7838c7f40b4a9717f -LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/54e8c25636ebb8a86edef578d039fd4d23859ed2c012025b872af5d05f6b485b3825d88cf6fc8027dc44b881c84bbd911753d6d70a963facf7708b92109707e0 -LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/4533027acd5733377fd24a971b1f0987 -LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/8c4cfcf4f1961eccf3ecfa48ce55ae6d5b812afbb636dd59c1889a0cef60c59828e42b601528258584785fc0ca357e1b57ecd8cbcd2eef0aef9735b63ad7a778 -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/eb1c74229fb43d9f8f5b3917222b37ee -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6fef5953f3d03e9b1172feb297c8cec7d487cf38dd7d88085ef6e41c719920d2fbe4c62a1fea175600fc3e839c719385d7ae6ae62c606e0294d5df0555d5cec0 -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/86cf13ddeb512f99030957c9d223b411 -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/59b701fa8b2dbf3afddbc846f6e4953f1b18abd74ac263cfbad815c997d57048217a672a9b34cafa9aaf6f87a636d400d008762d80ce35de8ddf61978d1bafae -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/73c8986b390269b2f2a6e14e1b60c7dd -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/af8338185d68774d690ab0b9b3725d5752ff26a32139a0f596780dccd7b124661c38561958bbe5b3c26258a5587eb7a40ed67aebbf0273d238b90d12020f8c65 -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/4ca9c2962f62b45c82104dff43f7864b -LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/63e580c4431d86925652374e25e00cb5d76fa72e48a2a1d6dcd99c2a6c7236382bf6ff696a12869f4d008ab2e9ffb10b84f7faab1b107816a8cf01dc1ef1b133 -LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/eba4632089f7cf8785e8aa122068eee2 -LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/adaf858e0362ac2813f555e46da82427e2366d90fa0e6b7b103cb57a98cda6586284923f58e91eb0d3f888fc0af17d0ccfa7a58c5432ba1f99cd90143febcac9 -LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/c12059240e4f2d78f99a0e0e912bad66 -LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/c14865f4088e4dd31c89738f21cafa6f34122a82bb7c20c9ca8f184d3b9c76c2bba6dbb3c9caa1b1bb7183517bc46e23f070741f0083c8bbf7cd4f3d5bfe89d1 -LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/4c2e4479005f4767ed04b216926031a8 -LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/fbfdc5451a652bcedfdd6a870b550a0c9dbcd8feae81a10792922f1972420408d5e3d69d675768d45cb7b893098b46493982669af55e6892b45654d2dd759caa -LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/1c78eb7776082c7555ec61975df84e6d -LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/5fc325293747d8bc4508256f016f5ac62342e1dcb58f3d47bfe9027dd127564dbb7c321f1ee689ed2983d24abeaf79909156167774c3ae20ca7697647f7d6387 -LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/6850d32e14a28529daded9a0a74b2f23 -LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/9c837940c27307e67fab0a57b5619a44429ac85edd0e04c4ea5fc2991312f2534f606c2a7b263dc5e82b67b8e33447e3e29081862208e2fb19998d9bf59769a9 -LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/ffe2beee649f7d2624a04616f7bdba37 -LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/e637f58309e61c568a2e3d28440330f0a8dca3a4005e171e9fe14808c1f25df498ee63d46de54a8827b99a78bf55b0323272bc1e82356227a26b2f86a5e8dcc2 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/fea1960f440c1f44161f7a2bee756073 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/97ab7fc9977fa3d4a7442554385218d56479f5a87420fc75a8827d9044db5ab52f3a46a9ad71769658ca08abac9e25df15f456d80c25d3c76afb98ab2855abf9 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/f71a246f402b1f4f5efe60e09e2f44af -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/568a39ccf791eb983b0d1a94a7188986477398308712425ac1007387a2fa0df99e00836046533d3891ff2b208ed4218289e0953ef0f1b4ec1f6127af76b3f066 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/b66193be08fbb0b07181e333a8857f08 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/17c9e77e907f40e799da4b2d21cb36e6e10bde6ceda6ff29c5941e5f671c8625401f6079f3be0de710b57475f4e7beccc1e7e64824c334286efad7b21919a7a8 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/122ab2567ca9d8824f94299b45e8b1f3 -LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/78420eaba62e18541443f4b1d846fecaf95b58e93ee7cc08c4818339d3a0705b0dd82dadae2024f6be65b3b2e26011269e319eaa9739af599b640486031b60ce -libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/291723bdec338b312764cc6e33a76007 -libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/105282bd4f63d3c6fbf4e45f5a72a52f1a6154e46e6ba595c70aeb763afe27dea688884e6448d4ef502cc4598bb377f10f172747cfc05b42495ba7ab4222307c -libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/289aede4d44649a0d265abba59041ec4 -libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/76f215d8e45bff747109a6aac5338ac09c5c89ab4af9b82c1575bc96908ba9a4675fc18a034bb222435a362f244b69abfb4282d0bc0c71f84ee9f715f281ec7f -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/267645d41e71d3f3a5a19cc70c63e573 -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/14d79aa893acca711baa5a20354bedec51e8c77a90f52639279d9b4453efb670b647dd0d1d60820b9c49033e848f90efdc25e87cb5aa65a9ac35aa8f6f71ac54 -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/2f03e683eb1631503e28117ffd6b0bfd -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/d0fafe8643cf0e1b702d4022fcc3c4a0c35389a65cca9490770d1a053a9ad8102dbb51e8a01fea46ef47d222074fefac66fdfdd4a34c2805a7761f43281acc0d -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/7729756ba2186da8578b5247e1ece99c -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/134058fecbdb9150435f1b3a9c6d208838562e0af8b3aac5c519502644b37b0d2093c2171a21ebd653ace9d51ca9bd4825343a60d80678496a5ccf876c951b71 -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/55e222b181a94df519477c3d9450ef1f -libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/6958f4972db0871b6005d24a8dc426b86504aa604a75a3cb18472dc4000477284e5c76c9477490252b521e40b17b9f587a7c6c3657e02e562858cbaae601260f -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/70deee22fe83f5f7234e5b2849579522 -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/24e956e1fc8431838d2f4d6b1d12bd90250e30acef64bda2330e542753ea7c60f7233fed0110a6166cd7510d65a0512b0115edead427924e5509f3443632716a -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/1082d39fd25ed971e1ed3f43882cdf77 -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/2fde64b294b1144af2754a543b5a9f63d3d161fe7de5c50061160ac845e4b80f8c447618a90768cc0a239546196a67e89f1a3295bc9cce6fd3ece36749cab871 -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/279ab853b812c2ae99aefbd301699e49 -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/f1f6ff89876bef3d2da28fcb230399378bab1df532a2ee31b83ae4cde8e5689e1bff2d44bb87b61c649055823fa8477fe8bb3445c8c8ed12a0b6cddffba5f26b -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/1ba2e81cc47c6600a25a507286bc74ae -libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/f8e88b9064b14e7565063d93630f67da9a927945e12b4234fce2335fcbc4c365528273058fde86cd046e2b63e0edc666641d5db6123a94865e1450f298468966 -libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/b722ae106708f785689f294f061105c8 -libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/82bda3b9668775da095c9078680042926e76c30ec3c18c893d16319cc740b7cc8e2f52f3a0c2f8f59ee84637be0c80bb11eefde531d5ae88d1d51d14f387d0b0 -libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/ee1c39eb3fd4ab6b6cf81dec58cd0f1e -libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/4500d7b795dace2ab6c9279fd62c9a68376fc9c1cb000450dc63547d8c7f2b4a41f523130ab824923ad73efe890fe5497a486748d9efa6feb523b29640693b88 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/87a0be634e7f789b04aab50580b8c670 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/442fa7f7702c2c6e80682d943bef8df3fd4867e0f4d871a3bbb9ad015b53fce4c1f9d38f8fe80a7805d3955060031efe569c0e909ea326969923dcf923180a86 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/e50d255bfbec40a6d4e97cc810aa4497 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/9108628863586f4ff4099d04b45aea75ca51d6540191d4471910a32f2ccb89f4fda388968ee7011932affbfc1cf8d33a02e28e81d572ecdc6c96f2e2fe982a62 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/88260fa90d1a3f9bf0b3f7fe37af52ba -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/b2455517b2c5a698b465b04305b547352e641ce96079d42542278d3b4f92ca9e181585ab6fa42f2bb4b5407b4c72249af5a5fab131efa74fc79a043d975cc102 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/78cf5b861f6f159ca48335b6a927e759 -libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/b11274b7ae94edba3e12004df5a95d24004d9ef81e5af7037741caeaa44b3d27e300565ba82d120f4943ed1f401f5d306318cf20bef6cecba80344c71881456b -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/7f87487fce6b663644013178394c8194 -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/6bceaa61a716c0950b176141eb838cd809e46a1ca79ec278f43b8d44bbe1f6cd22efbf2049ffda7bdcc521a5407cbe813aa07408e929a50bed5bfc0e19907391 -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/e9fc6ba0213cc88f15282a946779dddb -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/ed893995aadc5cbf611bfd18e9e1a06d4a82da69414529f88ef1facffa3c1adeaf9952b9a75e1b9e40b15141b9fe1eee33f9fdf6e84145b51f5da1c0751d83fa -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/43a4ed1c5818dcf25f1b828f86ae2959 -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/f834da31818afff29ff02a305569a3e9928789fd749e5c18a5df1d8173768ea9803c94ed73e0b362cb9111d0822794d084a4285a4ac4d8ce8bceea6a4c276e51 -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/7aa67860d888970dcd8c441a8995e6a8 -libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/265f9ef484b09fb38929a9d713cf17ce295796da64eb9bb3c56fc5eb17159d730fcfcfca29b4555dbe39345216ecc12d22f0808b0d855e96fa42f46e4020978a -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/d59c841e3d0129e8e036282fcdb25a48 -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/88753f91f8c488ef851f2bf4548f3e4b25053ecbeaa950b24b334bf9f9ab78152d99cb4b1e88831e257990d1e8f750405f5b185fec61c81a87f6dab6e5b451da -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/7f1dab964b8d008377f42db85e200ba1 -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/b407984e1b4d11d1bc3db3d094697972f34790466caa9fdc70b85ffd5b823e08b2335eb4cac1d7e67d1fe74cdc7ef29ead9a9e39f69d8cf55312d560ebd7c9e7 -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/29b4d1de6645ead45db6b9f17f4fc09b -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/509e2dcd9c5d2262fff70ceeae54f6d67995bb7cec316e199fc8c7032c74ea8896ffa3431245a33c28f3f846ef267a21c5f798b2ec8f761e6b3b4af4d4412899 -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/9f079ae058a98362d0ee50d938a3125c -libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/7452b65b750ffdf596fd0cb0740b459e18b23586b833f296f4db23cc5b69a72f6744ae85a7c5e1bf77a07563addcb17480b6feab5d2314dc74feb6c80b398978 -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/2bebe715afc61928505ee08cd8ed3e9b -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/c5ae5c79ef4d73d8a9d8bc99d0d418d1f0e7503c7a9e90f6e40b51bece4a33a077acd1e91d1ea355361d9911089179f93e1a5068532646cede01fa6941100d65 -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/b6db0d56617db57ee07a90ea4f778b32 -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/3aa62ee8df72ff360a660eb9b282f00c1690c181f739757423df618e224bfa8601e93eaf4690ab7527fa6097f1c0663cbfec7f0616ba897943a4a95bbc2d4abd -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/8d945748dd65b28134943e6f8b101dd4 -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/85371381137581a804b96c4864b6bda27aab0bad39626f2e33bc69bdf1ace5ccfac9a2e2255c1ddc09fcc67aa90b25be27fe8f88bca9e0e4a0bca284cb653473 -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/2628e4c8be69d729c063a91a7db8493b -libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/8dba6382ef1faee5fd80729e0754f288397a8dd20eb540536edbd952be6d43287aaebdb22576ffa4879a76577f275ebde9c87fc67af1064fcef815cc7b59f8f1 -libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/d8e406a6745deac26a671c9d4c567ee1 -libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6e615de0bd6e3020d232af587aa09d380e7d0538daaade75ee06d99851e2d899c926ce2f7316bdad7b1e4de985a99cb0b7bc37cdea4c365156346896aa256197 -libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/f09676e4ee1c2c18b8906ccdc7fe1833 -libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/5aabe655e54b1045f94327f6358003ea13c435c313dd75c9c0a870a76fe47416d86d6f768fda3e4b69ff7c3040eebdfd8b7ed43edeaf45ec2e4adf438b10005f -libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a7d246bcd0fa0d1425e1a405df00ef3c -libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/b27be72f5269096bfde5c26f15eed7285137dd83b802d8129b35557caf4e575619e66cef31a758323b20336d3003e11320b60e3188e4826c6802dc062b236fef -libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/7a4a7852ce434bdb89a202c8ea8079a1 -libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/b3e6c6337fa388a095e51d03ec235964cd92de809db8269f14f346c7cdeed8e27423c36f72aaa09e6c6d97c552830791b842c317779e986c13d489600eaa71f4 -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/15ca3900fc28ac7f59142b003411398c -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/5a25f51486a42ffcb62cd5904b09017bcbff9e2d2b1ae4460989dedeed736ac62c32e5d0c3168aa1340f541bb9558eb4c4ef70435cbcf08a92635efe4de37a43 -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/bcd5a589ff854f7a0f51d4809a5c90fc -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/2bc56d8899993b7dd0243eebdba4d45d869d237cc8c8c89e2505de34d5752636c9e55bbfe60d7a91fbec8b37e14f8fe736c7f97859122285b90d6d399e38f582 -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/9915cafbe3063f5d579d408385d8fc41 -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/7a8aef9f82e6a4e7e0ab75a912841b4c02219f86cd5afba0aefb32f1060b343425b3e4ab54a89557686ffa0557bc4cbeede0219bda980e2f0d84a401892ed433 -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/45a3430d5e980eaedf0e3d149a83f985 -libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/ff849bce1450e67f9a94676999914f0a7c8aae60f0cf1037b70a727e6587cab77dce8f549dd6531d0a20c87682780c5bd6abdf301d3c70b7a12ca1db23a5a420 -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/efa6087f7827c57d282c35125d5697e5 -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/ca154f18d4d2861026f2d59c92a09da25794f30913efc764108427a8e33da6170c11bf4c229cba4c3ae87e05efa6f18af3a1a5b5bcbe94e96b1d52e036c65d75 -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/cf88d74b55eab615c04412c1ea582fdf -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/58bce04a429a7fc893567c2c3ca5954d92183043fe5f71b7180a8bbc480dfc355bfc1fb1d5e71cb9de7806c37ad472c6fe4c7eb341fb48c856b93ab4d3d5ef97 -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/25c8b00a4452cd9d66ef53edbb747276 -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/25716e63d7c28600324ee68664f0457167638fa46d91a5e4a3fa5001d08fb816e0b3e0402ffa5f45b7e7f1efc014976094fa090fc4f10d71ca753da3f2a73a3d -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/de3025ca30ed8df01e8f358fd868c3ec -libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/517dc881378677bdd6011b349d88fbc51cf6e64cc67542f8cc5e4382deb5d0c16177990ed67890a797fd078c298a615b0ecac94788a2bf27ba6ea353361d4e95 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/85defcd1027bf22359db1c8dbbfa5f66 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/0d3995021935454a11d27da18ad41c2c7b3e86745d9f978ef321a16a312e73d0545482e34e6cb10ed9dd255bf0d5686c240aa4ea795d56c8d55b8207bd5b66c9 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/7a30a754501c41f5efe28c0a1d5dfff5 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/26cb6342efdba3e53305330355bf266e40b1300879011215ecf2311e4fe516d65a4793aa209e7eff96354d59ea962a0b58fb1b9544559786e569fba0fb393cc3 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/92d7c4e18a1ee1c32d7f25361b63d828 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/dd64730725a3ed2bc26262b661c565fee9c2a4a8660109707c773437615cb92a148c1d196700b254d6d7950de0de7177af7802963674021da9ee2d0650de95c6 -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/fbf6174730f401f14bfccdf34c97518c -libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/f3d78b63241bb142787d1dead601bc2b5f06d5b9dc2b2dd78da015df6d50d9f9a7cb478db2084157eba65a57f3328e9f728ddbfa3b7d760eda1aab0f97cb916b -libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/fb1c596f521fc005e3626cee940fc4bd -libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/7b4ec22c2e882644aa8377e10c8ee18fa2f880e2afd1f49548c5251a74636a9933af104ab0d1cf114b1ec474342c2c296b9e5eacb681aec202868929283aea92 -libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/ace5ea53262082e1962cd1e6068fdfd1 -libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/2391e569a172e200ec9b1384c5e6f2f04807c8b47dcf4938243872e23867ad2b53057f511ccf9e3fd2817696af322e6285764b75e2e72cef9da06cee2650be25 -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/9e207cccaf077025faf4ffbfbe911b26 -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/b1fd95de38aa698e519316ed14e3ad4aff6e0e11a8415edfd85b2bf4e2504915d76cffcd631b69b3c949f10ae087c4fe43ecc7b44a390646e764f44100e3039b -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/5d221a750e203aa06b6e4a6d22730acf -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/71b18114084096a3199d866f05df8741049d1f536b0ef156d90f18194a0f78262f04e8ac68e7e5e7f95bc2aef34dd524d06ab5c1f41524ff2ddad17e7002c3b3 -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/c3ea41e66f7d0749e1f45b7e9d2e32ae -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/a85cfc811ff0081bf565e922d337c56544ee3f7bd10d8db51f5bc26c45928b725bd1f3a57760c868b90b0ed1a7b949afd1087331dce995cbec73f627b1fe8d87 -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/a739244415d4ea0f2c0ae3a96ce9b3c3 -libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/8cd8d5985172dfb3558d1b5d1658b3a01dfd75327badb0dea8582d51bdeeb77db0af6890a5dfb8ae25f3611ddfd734666ac08347c2d2d63f81bed6e9796b364f -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/2d69f3b14ce0fe480dd6b3a8c030a27d -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/99f4c9e6fb8114b61aaa642fd85b945dbf47328b2eaa60cf5845b3883c2917e5b5e239a8fac13005271fd018d0f4b23d51934f5894ce3c994bc185d8eebfca17 -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/5096e6347bb08fd53b0416b4d4338405 -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/bcc0848e319846f930c8b89008030f1ad124cd2caa7cae11a63d33d4ccde78f044e65857a4e11f7e25ffc0e785af8c9e7083d289273d79181aee366c1f3754fb -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/022d407dbd191ba036f73bd986b067be -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/0c793bb350eb1540c09a70743801ba1b76053f07f64a57d133a1158fcbb057c52a62bc2d08403c605aef30a4dfbd7268fbcc83e18af8406523b114f939f7e830 -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/644e82b0e190856d95d6af1fd1bc84ea -libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/bf736aa4b882ead66b02afb504cb23ad7f77e8674d9ab9fdd769e62c7328b0f9112a360967d495a208e597f9a2f20f9cf436acf2706e6d09847d65638a3cb482 -libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/e90086cb8edaa9a4066b58b0bb8e2636 -libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/38908f3054bee0fc7b9f6c257cdae59dc641cef19d48793faf76a027aeba774fcf9ed9d820766bdc31d3301f4ddd5135c3c328f9db9a39c613ce925db0a166be -libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/98be4b9ddd1d044cd7ad6250b38fccad -libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/18e881e0d752ce0334d3c37e4cd0a41db4172e376410c860fb29e31d37ad5d95f9598f1bf0280c81fabf88129a17535c4f22e7f80f519c58e3da521923667fb6 -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/281234060bf9f417fbae340cafd20309 -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/e9e1c01dc849473f8885c2534ecb1391186cf283076402be9ba05bb5eb8633734e9b1f03af39218eb7cc40f59b727d8feae3a82fc60d593c86b5909f316d7c8b -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/aab22266d8f08325697cfe2b8083895b -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/295a4d82a4671023df12f8faef8593863a2ca970f59a1946605add0f80650a220fa75bb250b7d3df05070b23bd10f11daba4594fc0d5a16db09a2a7b737f4bb8 -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/da928189b76c03126b556d7141bb4adb -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/fe0cd8f8791bfba761ba43374c9618c06813828421e9b2a0093bb132b6dabf41b732413e0d59a6e2a97e42733d9cef8293086ec1d33bdac031bb0d528f424aa9 -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/4f4c5a6edaf01f8d6b98fde4e27e73ac -libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/0f83c130e14afcfd3e25b3f6200b0dc909d3f8ad498e8580c480121e7ee313c9c33972cabda9d6b25f548a1ec42e19f7df9300b1717dcae352ebb5b8004b05ea -llvm-julia-20.1.2-0.tar.gz/md5/aebf84a1b2ea2384d8f7d89ae6e06a29 -llvm-julia-20.1.2-0.tar.gz/sha512/0023c0ba21df6f130b543e58c4934a4061c4fdba9525867da3b7f1dd7a9aa2189eca90c8600d93958da5694780cb4aa2c05683d19817d8298305ec81d88cdc4a +LLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/e4091b029f66d30ef5af0602d572a0fa +LLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/c0a623391bdd65bf8ec8d9b001888b4dba62e29bf1f52f616bfc5630dd329b761bdc87d6b1a53581b82d50eb5006901f69452b332987eff64a29802e6aa6a514 +LLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.tar.gz/md5/9bc3d746aad25c4b3db3650c8ae61829 +LLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.tar.gz/sha512/c7f7f1d82e4a19b7ac79ec0b83d1139ade78d26571a0640c5989e4aea39804bad88ff3a17e43410eb34168db4bbc4546f06932ca9a78584ff405074cfcee3131 +LLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/e973025786ff769ee1053c83af7bc674 +LLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/b043d70b0eccd578341c44546e8b10160805b10f8b482b830c72ff2a8da2a81da2ffc7576ed9084f1c456c5b4e78de9c1a28d560922907676915554981f8b274 +LLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/3608dac1e98b95091b2879c2e39a2092 +LLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/670a6a312b147f4eb11ad9faf4ba3633104306b589eb6c4625dd8c5fe18e3e6612f93afe1c761911b96c0ea714335747c209a5130053570ddfcbe18ccd24a1cb +LLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/f983b532ef429df5257b5f04e3067433 +LLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/c72bb0985ef107a17582c86fc1ed207ad05046a7f25cac04ff590e0156809d1d4e8c8b72a7dd3f3cd9a2d76083b205d02c4868e6b5ceb1b50d4a5a13e8f60a75 +LLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/699b6424bf932b06ede8b12edeec9f6a +LLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/10b60c16a8831b572bb189780a0e917a27b057ad0bdf80ab280a4cccbc0f0d6af157f4dd70a18a9ef9345ab5ca8a644c45baf7f3b2143642b9e6b1e5f856b3b5 +LLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/e6a6badb75ff5282fd10433e57eab0b1 +LLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/b18c56ad53332668c54248e5bfed7ad4a80a83044780cb259fd47f171d75813661dd7991438f0dddac27bdc8ecc6913fc70757816ff933ecc2cde6e67c95aef9 +LLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.tar.gz/md5/27bdb14318dadf1ecff9116b8fe807a7 +LLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.tar.gz/sha512/838a8487ad3f0c47261ee8e19c8a64eb8c9a506ac29124099c99d5f3a4ce5c6f9bfbaf0b08651611e509fa4c75f371e2e5b7ee10a0ffd1f1e7c3a7514bc2b35e +LLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/e2173bd9dfc72c0cba7d368d0b170a11 +LLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/dc4c4581f9320629f2cda05f9f43d008dec8a4d2285f3184116c8bdae657ddabb0858a3412f4bc69ff59b8124a76fa2f3f0bca100ebd0b6a48e0da29103145dd +LLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/83919a183e029645c9bbfa4d7a01a0d2 +LLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/c71415b49d27187a35f5eab089cfe6b84940545825db4ada9cb48572b98ea965aebd570ee4a2b2a68aee0536d663780d16bc36538a49f6adf04ea65c1c313e2f +LLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/6b3486514307ca0d03a12f5246bbf0f1 +LLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/be9ca91de38a3dbf46ffffffb416aaac393319cae80e09e46b09b08f8e108b85035be1a918ebadaf0aec52bf4e8f244a1280ffdd53a0538c6044bf4a993a96dc +LLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/5c73bed8fb52f7871a91e3d6fb1727ca +LLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/464a7552c66f491fa4f23cd8f80cf155c106db0e608a16ae92a9e28afc0508addb8df5013b9aa02404a8525ea99c6ec10a711cca534d28a9abceda93654aa1e2 +LLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/cc11ba4c4b08fdd608e6c5a33e8ec41e +LLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/db57f78581c7ee63acf9e406fa7a9004d2141e5f97cb42c449b496680642cb0cb05e4c3df1e09d6ccfb504d7e768c524922d4d4b4373e5f0a5c8ff0601a3630b +LLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/d2791b50858b1d5297014c0733b17407 +LLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/85cf8ea177e4552cdb1d0f0c5ea2ee825f871efc8686b2cabd75ab2e33b50adcbdf54723a58b48ed233bcfafd44fa3a68886e15362c367688090609ccda3ac36 +LLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/c28c72768ceb0eccf433636c3449323d +LLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/311c5e742f5c8cbf2976161f2863e5491f9dce6e2c5d0439240839a70afa22f468d2d417b670ae699ea76c712c5d91646d86085f8ba2da129e9d59d3480b3012 +LLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/d898b1006b95e8dcd75635921be57368 +LLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/04a12e26e4ca833c04796a66fa7aa76e51e251a42c4c5b8825b9d45d25c85d4d51d1a91e9c6798498abe9b903da333b84c579b11eca227319e5dcedd863a1b98 +LLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/792ec532ef95780e564de898e58190ed +LLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/6c87248314a8763e547daa59fc496fc1c04840b8a1118fe4308da260ad3ae50717a08f7932c1fd66213230514d5ef8c441937ac2c45ebc8bde174c6d9fdbc730 +LLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/8f6d6560f6b367c8b4ac4dbcc174a9e2 +LLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/e145fc8fbf8f4eac820f09e0004b4d0393d771ceb33e8ffe3ebcc6661c3db7a28b8a97250a864d73d1cd0b1b58c67004a8b605bfbde085045fe812be2f9dd047 +LLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/5c124e731b10eb76dd8e96b74ad345f8 +LLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/9df82a6e8c66fc34156e32cb07f281612ff35c77e586418661df4e955d6102b9b24370504ddad77f54c03b8acb01f6dd105420b4fb58a3f1b09bb6dc41cd4883 +LLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/9f81a518640b1fdfb6844dfbcaec4853 +LLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/2856c1b27e166e7bcf9ee06894dfc4f1bb7a3fb4c1ba0062679b89ac899ab7d906908c19eaa2f210a71163d2b92daa455245ad4ddaecef161eaade8f0592d8cc +LLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/eeb679f3c18531c37e63fd6a0d062b59 +LLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/6a3538a73ec611f70c3336f4d173327a8255c5b9c835beae41a08f1863130541def857d49239865fc55d58816f10556afb2f827264ead8b41cea19fbbbc9c17f +LLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/8d677f589860fad471b3480c7332983f +LLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/9741b2114cc63e7689c09d17f1e1d49daed282edcda8ed74eeedc56d29eda9594ccd78c3068f4c9e887bdfb6381423d85388931cb2105d22ebc226155bcb2797 +LLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/1fd11960cdbf292d767c5b462d790d18 +LLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/a0e37c5cc0c53cf52fbd76b53f675cba38abe11a6ced7767d54e984e0b48ccae10de79325bdad1806a89d0907e4e5ee3c8e20ff7536b49088dc107c4596f929a +LLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/e4cdfdfffb215cb9f7b6d74699faa266 +LLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/c6aed091c45589b0bb79b02354ad911f1254bc0b1056d90a38cc7ddcb87c0cffb0076345cbb16628f206744b0cdd442f1be09773bcda9f74970cfe0c3fdae2a6 +LLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/0ba1173b587ad6fa63893d6d94c969ee +LLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/94957c25ddad4fbbcd6dc6c345b281f7a7c912d745faecfd4ae368db48c59bbb24f8ace4f51b00e1eaf187a6ec957b163e2affe2ffe3e9be290cb56384a8d44a +LLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.tar.gz/md5/9c3b6a934f227de12ee909e40a3b38ba +LLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.tar.gz/sha512/793235365fe4287cdbcc72d878df7c3de7f25d3641c00b4eeef838c71820b9e4d0c0fdea01dc38aaa1a631db724885f0ddb210ace6cbc166475949c67e9536d5 +LLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/67b022425f92455d76d7af2cf305cf6f +LLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/b23e36e63c5a093992ad6eda0e3130adeed7a17491dcc30775fe4401b7a84b750a1e5eb6582eb61b82eaef4064183616eead352b57af4faaf627025017ccb9a0 +LLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/02eb0e794fe5a23c30899e7e122934d3 +LLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/62b15639a5448c05bb7ed90ad8226c4cd1da54132a4ad4f6af08016a795139647195bf6b2fec4dba2c4a607a8bcd42bcd7b835459cc10d7f2851ea222c50fb0c +LLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/ea886c135c12accb833aab4cb5bb5dd4 +LLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/e2d035f4e4e590adf4caaa75731c4009c2f34db75513b6da0e24c2fb22313094a8160e5321419f05c2f8d7e697135a21002555c7ad99addb38da566b9999d2c6 +LLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/ab088c1bbb67675a1e6bedcdbaa39d92 +LLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/c88bd0b6e5ca7cc4353d77a8a3839758d46d441f5a5904e2e906d3e9e54945c6999abb55d849f5420c12384b5b92f08b551b091f178ab11dc1b1fa98b0285552 +LLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/bbc7fc324660063fd624d0810c28d173 +LLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/908f64a74d32b230a49291d9fefcfad607f7f1772212b069e8ad152531c6659c458077e80fde2d3b7d20916576f534e42e7b8da71355615087cb75c7f6217b17 +LLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.tar.gz/md5/6a3c8c10c305c7ad2f326eeaf8e23c54 +LLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.tar.gz/sha512/cfe8680d4365ab9596b41871b9b1e48dab9501480277e757a569d52277b1886766faf6ea39912d7672ec80527eb1ed369e9bc1863650b5428b423bfbba0233c9 +LLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/b416bd42e4d5b9b78772574ed61686ce +LLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/30b5d58a400854b3d9e015ab2a18046f1baff318969f4093a3aab1344ede25d4d066d888f08461026a8a10baf5bbe2424592a4f703fcac6a06575ecc593b7ab5 +LLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/212e92eddfcf070b02a15e400b63c77e +LLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/00d1d726aaecc268bf5b63ed6f5d199bedf031e3a380cb943d6b21309fa27ed44620b28050fb604c055163da03cf3c26cf2c6969202472a109c4b2649c9b4bb2 +libLLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/1aa0a11c13302384a189adcc8393ee01 +libLLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/715a650cfebe5abb133a2853ab3b6a38a61bc408bcc1a6c166c2d74de33e6cc0cc2687a58372f8d45673192ca809fc5935e6f7289c7fc8d4669320e646254890 +libLLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.tar.gz/md5/b17c7ea776241805f72f0c1435f1af75 +libLLVM.v22.1.8+1.aarch64-apple-darwin-llvm_version+22.tar.gz/sha512/2f0fa272a0495b9b56b55202ada58b56907c5ef24fcf567e924faf012bd95317f026512552b1904cbd4c2a26e83ecad2365722b36fb79375877cdebf5ac9aff6 +libLLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/e1c4333c1679795091f1eab46524cc43 +libLLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/b5310d1dcefa8674e4c068755fafe702ee2c3dfbf8afa24d80c0e1380f22bce96cd9739b06ef76c4dcc1ba53615759634a9f0668e4193b7cc778ca7f5f0cfb0c +libLLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/2af1147a151c6efa1313b5bfaca5afac +libLLVM.v22.1.8+1.aarch64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/d9dabf51ab5cabea6d9183094b1409612e6b92385ef14864f65d2eb1752fe55ac206db15b9901e6b5f969b606baa3e61e67e3edae7efde8bc5d4729ab9cb2083 +libLLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/b240431d8ad997bc632da46d87760fbe +libLLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/5e117defbe26dce8c28ac13c5bc55f11ed7e8a92a76ca7fb9e63309f24231240efa68ff43f6d048e4c3a285765b0c0e9a4c0a81501212494c2da8a854b12d75f +libLLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/17be3dee4b8ce9d4c1d861975c8cbe7b +libLLVM.v22.1.8+1.aarch64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/5c2cce26419788a3ec4a0b7dc7c431bc271c8bf81aaf7d881241e2c56ab1941303adc46f882626a753e9a39df6dace1826b0b73358672466560badd1d667c421 +libLLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/8d0fd0b2f074b8a65257428f2af7ec42 +libLLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/ab9966e9eefb4e7a2efef058157882c9d0ac29427a807a463ad9417ae093092bf84894364c6d5b8730f8daf230a72d171d84c751e82c4170163ec5cedd5bd577 +libLLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.tar.gz/md5/8af4895ff2848982e4fb792f2ec5a447 +libLLVM.v22.1.8+1.aarch64-unknown-freebsd-llvm_version+22.tar.gz/sha512/7588bff65d9326b41d3223491910e04c41d95ab7a01d58fdca69d4d0a8d68d25a9e37e7386b9c493f73023e72037149e1df91c88606b3b5b3447fedaea6a48d8 +libLLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/8cb1cc4f4a4cb1b3eda8efec856ccfa4 +libLLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/9e4f331d9759c54740f93a39f484e5ead64ed2b414130a0e400c1ee9cfa0c8a7388efecaba3bc97ff18c0a13f1d29999541bf11c2ea8408c60f129220f949a0c +libLLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/f78a82646713ddc0c77268247c39463c +libLLVM.v22.1.8+1.armv6l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/7669a920934c837c00b541035bfd3bbee73eea1fe6ae58be629b4c2a63f5a3531a18b2a46bcf22900b2f885306b0f3770cc1fbadd4ea49935c22a0a0a913c7ca +libLLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/3bde5d294dac68942190f58613b0a0a7 +libLLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/1d9e5d7fc49e7eab219630689d27c91223dfe670c69df741a232dae237c215a9ea8ac8922aca4089fa3d4ab2748fb6912f42aecf80398c3ed13242bc72c1c541 +libLLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/3c53451ea7406b9f28de4ac2f2330ee1 +libLLVM.v22.1.8+1.armv6l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/f3f6861a80148a15e97b82e88cfd8fb069643b91c9428a2fd0f497dcb466ed12c8cf96f4a549fa0be171c5d80d0e8da8bf3f7d131456a28dc364c3befe1839dc +libLLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/a2b4f05ed2e0f13488cdb65bf16b1ee7 +libLLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/328680073c45fd546e78e1836f2b74b6d21c2d4640dcd9bf4718a07422ef0cfd6577af48bb56d58ecb155aab97a7a01eb94fee801abed63267fed0f4231ee6cb +libLLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/md5/7e3a8aa03f9b79d960a3072d60d61452 +libLLVM.v22.1.8+1.armv7l-linux-gnueabihf-cxx11-llvm_version+22.tar.gz/sha512/7fa46fe3813bf469e3bbe542e99144ddb3b4bae1335dc6c0268c96e7cb26a75dabbcd096ab4fac3fb16354fa6768314a65678b5e21ba63119d31aebf73671cd5 +libLLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/md5/6eeff9959566da25f7833c8773fec061 +libLLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.asserts.tar.gz/sha512/f88fbdd6d17260d362630775303a1fbc6e9f3aaff38fd476e6a8a56d524e02e4d5e5eedbe21d5d4ab9b25e50ce27f8a458222c58b855e76d903823fe40c20be0 +libLLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/md5/fee6c5c661ffb64e398bfbc47d153a9f +libLLVM.v22.1.8+1.armv7l-linux-musleabihf-cxx11-llvm_version+22.tar.gz/sha512/628859e5707e2ff3296bd65a6eb95201aafcc9e632b41b8d05345a5e8d7631150795b2a42cb2d90b82485867dce55c784299d93b6f5dbbfe3ca605cdae58a426 +libLLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/f6a2888273b117ffe0945b02d5806020 +libLLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/93351a2d61be343ce259e7a21176bea324f6d0f3bff2579b88a833f19ef4a18850b1a993700f074d87e974e1203923c745157a672c414c99476bf184ec44e5b9 +libLLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/7c2eed3475691df0a1d88e4a03ac1485 +libLLVM.v22.1.8+1.i686-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/44141b4ac40bbc9f0a32f35f02159d8e2f7d04f2bd614d66cc052420ead3b4f454d91b1b373c934149bad5e3dcc9c28d283389c563632bff2d217d0562660eca +libLLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/9dc6c933531f9281bc9ad6cd5c249046 +libLLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/b93df1fe1206264d27757ee955de6be9f1c6c45d58c5248b4014e1fbe5f4dbb18e35d08aa9e93479b2640d0584f6bda7bc4960d7e4cab6b7ff597951dcd6292c +libLLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/6cb267f7feee342decd5b1d94048e1cb +libLLVM.v22.1.8+1.i686-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/8b8277c0cfa48f88977e0e1098eebb93c525b554049ef23c344cbfb2c554c6a5dd059e3a54a30ca382255d69dd138a8f7faa9129e425db30bf39b25226be0783 +libLLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/63a07628d199c985e62e3b67b80a3fce +libLLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/a237dbba9182a3b348e56e0077a2d90ebc2904dec5c9232fcb97fa0fb84471c29b7ce3849d571aebd3f3560071b7feaf3185a0690fbe1e67ad0579bb44eed14b +libLLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/ffb7aed4055a784cfa25e38a2643500e +libLLVM.v22.1.8+1.powerpc64le-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/1ad97ef5e3ae7517f54c31c14f4cc5107766a1e4157d04832d6e6780034238773df461b36486903c8fa8977f47a1ff0115a5ce7cdd5bac068a57d0cad1e200af +libLLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/dd5469050b556d356209c2edcd844e96 +libLLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/13a7bc7879245d098a54391e4ad7656bc2fd0a2c7e32742f628ca1a48bc6132a380093922de5d33383fdff7d27dbe640ae850ff98a5480c69119dd4960a8b1c5 +libLLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/3cb6609ebfe23a275ed47da8d6e96915 +libLLVM.v22.1.8+1.riscv64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/5930e95e79419fa5bd056a61d12e951955e3f7296f36d176db3f7a211fe535dc559d1fd0ec1bd416cf8131a5c65c776f0f539fdb3a09ef2b939b5b4920b4c01a +libLLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/md5/5f91864a236aedfe23df520b5e384632 +libLLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.asserts.tar.gz/sha512/13ab9c9f0739ed362f9f6ebe0e9c433597257cce71ed1083747724b02a832b28a6db86f597987124cde80fe59cbded7ed45da1685acb00c6c117f25ba6d027fe +libLLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.tar.gz/md5/60c927be03ef8b228e0a88116ffbfcbb +libLLVM.v22.1.8+1.x86_64-apple-darwin-llvm_version+22.tar.gz/sha512/c63506f5d08397121cb2d8ac54c20b3dec1779376989d51062957e1f0fd5aef24a7756142fefcac7107ad9ce306ddba7a42b3619b55691a385f94f350ad61285 +libLLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/md5/1e1e43a593c991d502a897f38dacad80 +libLLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.asserts.tar.gz/sha512/6b8dee10851f4a2c9dd09fba01cbd7f3a627846cfd4e4950161266f94dc0e7aaa84f07c376055f854c7cc88b11549ab633a71cf9c244979345a2bf94f2fd38e7 +libLLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/md5/8beb0bd8040996764b566b7f168be470 +libLLVM.v22.1.8+1.x86_64-linux-gnu-cxx11-llvm_version+22.tar.gz/sha512/2181efa67b7a237ea5a32e8837763dc4f9e4fc5812b2d9fcf76200019dd720e4c968b2cadbddd7aaf9578b67593ea2c2f0538af24e8550892a9cc019aa1bfa24 +libLLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/md5/dac4c2ba5c8b24a2741d3721b1e9c4b8 +libLLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.asserts.tar.gz/sha512/78cf6afc7405062d750449b6a65fa604742384f3a632e52da842e8c455d613eff82c550a32ca577272e7f6b50b57b1b2e52e64aacbb8d24e393c26aae4239338 +libLLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/md5/fe2bff0c6eeadf2df8e5fd19c8dd1f06 +libLLVM.v22.1.8+1.x86_64-linux-musl-cxx11-llvm_version+22.tar.gz/sha512/5fa2fd5a94a88a7b1ee01a536c2a18a2f86321b6205618583c2d2adc3748515c12d6fcbdf36b5089c1cef7d9e1e10eb836d36641bd6359d85ad9a4c3036d1ae7 +libLLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/md5/67e5207bc439bc49bcc9628aea6341ff +libLLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.asserts.tar.gz/sha512/beea143378486d8e515f9e6b25e31d40fef4a0806c7a98e193cb917b991903c8feea5f572d00420d3656085912563c5899315767708df3f75cad50d9b323177a +libLLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.tar.gz/md5/703d560fdb9e031cef5baf00d6501608 +libLLVM.v22.1.8+1.x86_64-unknown-freebsd-llvm_version+22.tar.gz/sha512/6a0926eb6424fcd2c6938d6a1b1d8acc1fe28884026230fd24f619794f6875794e084cc4b48989dba219065e42215f632ece3dddce3d79e2b2812397cda298a9 +libLLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/md5/5e3335778ee064a17902858d3b0f7979 +libLLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.asserts.tar.gz/sha512/234c5c5950966bdc04de67aea0a127ea8e7d31d7a5a61133d056e947a621b957fcf833eae5ef16b4c6adfd11800c4b75c868527918f4f4d026b14ce5db9213c0 +libLLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/md5/d1b4699badfac55b2aac2fd06c9ac0ee +libLLVM.v22.1.8+1.x86_64-w64-mingw32-cxx11-llvm_version+22.tar.gz/sha512/ceb50c7ee396b85bec116baa0d00fd75e5b0900583a11555dc5e23b8355ace271453234731cd01d3ff078fb08245071123e25c76256d766dd9911b518f1cf071 +llvm-julia-22.1.8-1.tar.gz/md5/f9ed1b9b5f36e8b87a057b2b7f9f6741 +llvm-julia-22.1.8-1.tar.gz/sha512/9325d87db877dfdacf130d27c6eb593be893966d436ed2476f18a85f77c4b1215ab378d93ad10b5e10007987a7641e5ad9025a371084c0da976901d576348716 llvm-project-19.1.4.tar.xz/md5/1e13043b18558e4346ea3769094c9737 llvm-project-19.1.4.tar.xz/sha512/a586f8a41dde5e0d9ca6d8c58e9ef2a2e59b70a86d2e2c46106dc31b5c096bb80af0cdbdb486179e9cc676a540099f49a1c2db9e5e84c50362db1f72e9af6906 diff --git a/deps/checksums/lmdb b/deps/checksums/lmdb new file mode 100644 index 0000000000000..1b826b5e6effd --- /dev/null +++ b/deps/checksums/lmdb @@ -0,0 +1,2 @@ +lmdb-2562c3297402d82bbc049c7e645515edb4079eba.tar.gz/md5/7bd6f9a0410d872279e034ffd47e7f41 +lmdb-2562c3297402d82bbc049c7e645515edb4079eba.tar.gz/sha512/83be75634b59481fa846d017a0f6979f66d23c4c32c72c501ca7e729d164bf7a873195013c5e727518039856ec4a0d2587df1946c98b8badfb5e9fc31088e27d diff --git a/deps/checksums/mmtk_julia b/deps/checksums/mmtk_julia deleted file mode 100644 index f28a602271270..0000000000000 --- a/deps/checksums/mmtk_julia +++ /dev/null @@ -1,4 +0,0 @@ -mmtk_julia-4933fb41ba5d1d21f720e2ca1bfa5bc938b73b12.tar.gz/md5/e27debba476479953bdc23eb970f297f -mmtk_julia-4933fb41ba5d1d21f720e2ca1bfa5bc938b73b12.tar.gz/sha512/6541bacfd16cbf1969103fea612c2f8b2011c3e38748bf82c41633c6084c2bf981bca9fb0d9f736832b0d7ef505271ba48465bcd27e8cd1fa45f7f4c5e36f246 -mmtk_julia.v0.30.6+0.x86_64-linux-gnu.tar.gz/md5/9b0c3e1ae95d82aa2184cbe889def0da -mmtk_julia.v0.30.6+0.x86_64-linux-gnu.tar.gz/sha512/04fd29468fea544f18622046c5d237d335c80e7c1c3b968f8445e439449f94afd4b82266ee49fa8c2b69b41776062959346a06e916d6b525a15f449529013311 diff --git a/deps/checksums/mpfr b/deps/checksums/mpfr index 7f0de6099713c..7c73169135676 100644 --- a/deps/checksums/mpfr +++ b/deps/checksums/mpfr @@ -1,38 +1,38 @@ -MPFR.v4.2.2+0.aarch64-apple-darwin.tar.gz/md5/01a13215fd646c761e469f36f693fdc8 -MPFR.v4.2.2+0.aarch64-apple-darwin.tar.gz/sha512/da473776ac8c687ab34792235ee5e1e08dc6a2e29b73620bd6dac93db32397037ae502b8ac3a35e020f722dae7da007a060e5e11e3287c4cdb846bf7e5168297 -MPFR.v4.2.2+0.aarch64-linux-gnu.tar.gz/md5/58ca9f3e08a388c3e40692e623f3884e -MPFR.v4.2.2+0.aarch64-linux-gnu.tar.gz/sha512/c6846d982ce1211791b466ed6fed2aad9e5f9a4866c48db99eb288dcbb1480660772010869fdea66d6453c8c140c92e367cfe55f6087fe41ea040fbd77eafe34 -MPFR.v4.2.2+0.aarch64-linux-musl.tar.gz/md5/2ff7e1400f27d049e3274a6277322860 -MPFR.v4.2.2+0.aarch64-linux-musl.tar.gz/sha512/388f7050288be9d30c4a2e772c0859e414b0cf6dbc845eec0eb6aeda53595df94a4e3001d02fa04c173fcf74e00c2552a8880b62ebf5adf443da2a95497be891 -MPFR.v4.2.2+0.aarch64-unknown-freebsd.tar.gz/md5/d1e6c477ab9678d1cd1dfa7e00366e69 -MPFR.v4.2.2+0.aarch64-unknown-freebsd.tar.gz/sha512/897174756651d01272d86bb147f5dda9f84f8f1bf1fe02b8505e141df3cc38523019f85cbe538fcc6ea8073d7743fc6428a06271107b059de80cd8f959c52daa -MPFR.v4.2.2+0.armv6l-linux-gnueabihf.tar.gz/md5/5213b0ef1b191c529e3335e05b918003 -MPFR.v4.2.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/bbcdb90f80d8cb826cd055eb41f051890c7847fc0887389b61bd24c051d35873af36672e5f1956cc3fb23b8e3ee50ee069c185fc2faabe302787d70210bd5b07 -MPFR.v4.2.2+0.armv6l-linux-musleabihf.tar.gz/md5/9a9d9207a6b52b6e84b1b2b1c631e0f2 -MPFR.v4.2.2+0.armv6l-linux-musleabihf.tar.gz/sha512/fd40d16a40b1db2b441339e5c8cb3f8a1810d2889713b0504f9bfd5451f4f4c2dd0ca35a4b2922feca9cf50e4a9b3bf8cf2c088655dd85a23c33ee67c12e0a72 -MPFR.v4.2.2+0.armv7l-linux-gnueabihf.tar.gz/md5/44532dd5607ced01a8ba0856c3bfdbc3 -MPFR.v4.2.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/469fc030f458bd52f6bdffc442ceaaf8659f0f1e40d581eb1303fd4753d2c665fcb75bc6c54d04eb53d77b1945d67f48a5ea5614f2ee82cc7fd27e89859b45f4 -MPFR.v4.2.2+0.armv7l-linux-musleabihf.tar.gz/md5/fbd13b054b8d27be6bc836283f7846bf -MPFR.v4.2.2+0.armv7l-linux-musleabihf.tar.gz/sha512/926dc03f99a6827c833614d17c5ef4f80fb862bdf4397db9aaf8ae9b3a66e8b9121cfa044b18db46f5774abbd7e9c129363183ccb2ae3192084711e7ff9d6382 -MPFR.v4.2.2+0.i686-linux-gnu.tar.gz/md5/da6fbb90dc20830af9325cfaf3544e4c -MPFR.v4.2.2+0.i686-linux-gnu.tar.gz/sha512/d235884e1d1bef406b1e5ceb9c34aab68c1a8040b2022964105238ef8cdfd4af7aebe474fef80849689ca88d9168697fd55e8d6ab92b6641a1f37c431d5e3ff3 -MPFR.v4.2.2+0.i686-linux-musl.tar.gz/md5/fc885092e1469a06aaaaf24168e8fafe -MPFR.v4.2.2+0.i686-linux-musl.tar.gz/sha512/5307926e1222b302e48e2f5c08479b920279d15b95937a245e16ac1dfd5c6206cb64fe4b6ca4cb7d6be847d8cc01a04d2661a630b978dc2dbd60605d222b8b21 -MPFR.v4.2.2+0.i686-w64-mingw32.tar.gz/md5/55f129d5b5b849b3bc018e68ccf14914 -MPFR.v4.2.2+0.i686-w64-mingw32.tar.gz/sha512/9a24e4616e05f5c1fb53e7a12167f7a55d05ec1895124d6ee23b2efd548f49e4c7995c16d240ec803f352d586ae4667027ee0bdeefa520e0c1f581fcc338dc44 -MPFR.v4.2.2+0.powerpc64le-linux-gnu.tar.gz/md5/6f47e4cde45ddf0cb2ea4f31ef9c9e04 -MPFR.v4.2.2+0.powerpc64le-linux-gnu.tar.gz/sha512/4fd8fbe166e719c636e430d4d5c938231fa9126b29eacbc678d2eb50d3d4b95cf6ccef155ce401c6d33b9730c2f89c0c77ec8fb39254483c2e4004639c503c1c -MPFR.v4.2.2+0.riscv64-linux-gnu.tar.gz/md5/c4736705ff2a55cf8206c3af84bfc417 -MPFR.v4.2.2+0.riscv64-linux-gnu.tar.gz/sha512/e1e77d64ee88de2990fbc791d7307afe859cfbdc1ac67e7bdfa633627b5542ce2e3ee0cd9fe4036abfaf60509277a43f263e2155665ac2c5e38b8627e470f399 -MPFR.v4.2.2+0.x86_64-apple-darwin.tar.gz/md5/c3e983178a1e9600f42714d4cc1ecdf6 -MPFR.v4.2.2+0.x86_64-apple-darwin.tar.gz/sha512/c5c6cebcdfc5b7b84e9e217a81d99e5af78d163949745d570af5689210b3eedeb9de3c11991b1b36d8fdbee17b550a4072af951d19c3f863cf24cda7d9c12950 -MPFR.v4.2.2+0.x86_64-linux-gnu.tar.gz/md5/61fc7c7aa676d0a07e1709b433a8e423 -MPFR.v4.2.2+0.x86_64-linux-gnu.tar.gz/sha512/74bdefa72c51c82ca709e3494cd664a6593173bbfbe0198f18f4c0add06ce4c1217e4dd49e99cb151d71c85cd696ae2147aed29ed2cf3f1ca0e5b40582abb571 -MPFR.v4.2.2+0.x86_64-linux-musl.tar.gz/md5/207ee8ad2293ba36d3d7bb845ab346e0 -MPFR.v4.2.2+0.x86_64-linux-musl.tar.gz/sha512/63325e6595861a324f3c299d8c51b1d665197217c8fc9a5ae627b624037394f050bb08a9acd14e9809f982942c066f1185dded0fa493f360bcd3baae17a05f92 -MPFR.v4.2.2+0.x86_64-unknown-freebsd.tar.gz/md5/74e5a5ce0ea84959ccec7b7f7ab22c66 -MPFR.v4.2.2+0.x86_64-unknown-freebsd.tar.gz/sha512/411dbb339218669af6181fdf1e17f926abb9830ae54a8f9ef1b7df53021e8da01a41fda13067731afaf9b803324d5f82c060ef5b5b91045625188458b99dcc75 -MPFR.v4.2.2+0.x86_64-w64-mingw32.tar.gz/md5/2de84b494ea832147be4f9bfa786cd19 -MPFR.v4.2.2+0.x86_64-w64-mingw32.tar.gz/sha512/5f86aef6ab4fd7517cb23ad9a32ae21954a3ce1f27f5cbd28abe038271e20197b7c241055092a4aa6d5391f012bdee10465c58b53acd64bb5b99fd754c75ad29 +MPFR.v4.2.2+1.aarch64-apple-darwin.tar.gz/md5/9e5c782d761722a937f6ea52faa87649 +MPFR.v4.2.2+1.aarch64-apple-darwin.tar.gz/sha512/504893776029ae886b8e780289f02514bdfcb4143d5ec64f9a35238f0d4f28c425ad31276bb95c8e9006a024b80185b65d2757a8fd2743c76ac0c29d88f6d18b +MPFR.v4.2.2+1.aarch64-linux-gnu.tar.gz/md5/a7fc71b4d1881ff9d0480c571626656b +MPFR.v4.2.2+1.aarch64-linux-gnu.tar.gz/sha512/cbaab79b5f92db3325ced4415cf2734d3d38ca7a9719789140bbb7a3619d70f4fc181fdc43545da141371faf99a45f7aa26771195056aba33173bc6572776516 +MPFR.v4.2.2+1.aarch64-linux-musl.tar.gz/md5/516a52753cba5e384eeb00be93b87ada +MPFR.v4.2.2+1.aarch64-linux-musl.tar.gz/sha512/27bf0ab3f4560739f2d9523e1c08d646d233a0a5cc923b1df31c899591909c0e8cc91652ece0d39ac69da2c74239f8754c2a05b5e34adf221646803c38f02ff2 +MPFR.v4.2.2+1.aarch64-unknown-freebsd.tar.gz/md5/c23b4646ef4a1b20cbda78978c651b98 +MPFR.v4.2.2+1.aarch64-unknown-freebsd.tar.gz/sha512/6b93a8645530f1d4e2ae30847d61a89788dd865520625075e45f9c95df124b91db7dc0b5ff6a6be22f1c8bf8f4a7d92c69b77013efafc53e49b2a66fca08b2c6 +MPFR.v4.2.2+1.armv6l-linux-gnueabihf.tar.gz/md5/d1ccb63bbe073318f02afe2b26e59200 +MPFR.v4.2.2+1.armv6l-linux-gnueabihf.tar.gz/sha512/a17190cca51a8030e942aa3d54b14b02acaa178bf861a93b5154948b871805f90b55f7c58edc2ed51bfa680bcab2a2b986083b650b756192b21b8bb72d1b82b3 +MPFR.v4.2.2+1.armv6l-linux-musleabihf.tar.gz/md5/2a9589d9b7b9d3d13d873cb28c99c5a1 +MPFR.v4.2.2+1.armv6l-linux-musleabihf.tar.gz/sha512/48b30807a57e7447161bb7ba74d2456b75f3a0985926ce7de3718b0f7399e45a24c50fd64c66e6d05bab7203c4dd52d2c667ebd2eb9660a55089b5fae43f7294 +MPFR.v4.2.2+1.armv7l-linux-gnueabihf.tar.gz/md5/c02c7790569e4a0a044a60e723acbc0b +MPFR.v4.2.2+1.armv7l-linux-gnueabihf.tar.gz/sha512/a7cb7e45b2b4b334983b4155fd2c2c6ee352189eedaaed5b3e19a03587fad7cc6427570fdfce909563e9022fbf90e3923a50558f8275c9fe7807c88cc2539b94 +MPFR.v4.2.2+1.armv7l-linux-musleabihf.tar.gz/md5/b3bdcb75356c46f10e19ef6f322cf7ff +MPFR.v4.2.2+1.armv7l-linux-musleabihf.tar.gz/sha512/9ddb373b5a0e7f1bb42fddf3aacbcdf113145392de1b76abb2da5f8a9f2c908be93185566c1bde8ccc53abd71811f263b55be082104176d99fa8038d9e3d1bd7 +MPFR.v4.2.2+1.i686-linux-gnu.tar.gz/md5/aa1cae51c7242dc317361dbcef5f8b54 +MPFR.v4.2.2+1.i686-linux-gnu.tar.gz/sha512/e64279469c46c8c3a79d489d6d95c69699688beb036a53351a677d66ef3f0a56c9258047ef3883c19d56458d7388bc4edd19e7f384e898e4015185f250f7c428 +MPFR.v4.2.2+1.i686-linux-musl.tar.gz/md5/afdf57a7a99c44972a06e07439a1d1a5 +MPFR.v4.2.2+1.i686-linux-musl.tar.gz/sha512/fee07a481388a094c48ea4d271143310ddf52847a490a226307ded046f5705d2820989aa73eef669ecef82be7a7c20a3478dc7f879e584479b30a583d57b5507 +MPFR.v4.2.2+1.i686-w64-mingw32.tar.gz/md5/9c3bdd42c7c309891ebb4a0db39d3e81 +MPFR.v4.2.2+1.i686-w64-mingw32.tar.gz/sha512/9f1056307950264720d4cce80ac9bbb0bc479512ea3a3f9d146f5ed4724717f8f5507f4b376fc003008d07fb5c77530dd2897e57e9ddaf1b1c2434c14621eec0 +MPFR.v4.2.2+1.powerpc64le-linux-gnu.tar.gz/md5/d824d84f512cc2bc0f7b49a34a36275a +MPFR.v4.2.2+1.powerpc64le-linux-gnu.tar.gz/sha512/74044edee50ee81533d6fda457bf361dca0d2f75cb336108124bb4c50679998aba8ce24d14c7609a0d982b5cfcd4fd027bdb02967c13c32922bd6731eea6b97d +MPFR.v4.2.2+1.riscv64-linux-gnu.tar.gz/md5/75bec3fff53f380800576414cce76ff9 +MPFR.v4.2.2+1.riscv64-linux-gnu.tar.gz/sha512/33aebe221efe78418b1d324052be5bab04a16c110b0b0c52afa924a87b4c9259abafd599c0f768f70bf89dc856639b62df54dfed4cb98f60c4bb0c8a81ed6f0f +MPFR.v4.2.2+1.x86_64-apple-darwin.tar.gz/md5/afb6f5317aea6bf1a2fd4cfe2190e8b3 +MPFR.v4.2.2+1.x86_64-apple-darwin.tar.gz/sha512/c372fa6305e315cbc63008fa265112dfc34ee8bd9e4ca4f4c7e00bf1897a0f84664d0ba832404e2d8987f57c8e78a3c8e751b29c0dde888ac37c60d1f6344996 +MPFR.v4.2.2+1.x86_64-linux-gnu.tar.gz/md5/66c72019eef9c96fc5b0ec34382d8693 +MPFR.v4.2.2+1.x86_64-linux-gnu.tar.gz/sha512/ebe2865322c389a07dd6f07016c43938c1622ad763e0ea67d3f2dc70da54c1856500cba31588db6028d37abab05c7ab623b7fdca9706eb8e63cdc09059fbccc2 +MPFR.v4.2.2+1.x86_64-linux-musl.tar.gz/md5/0ee5cd2ef36147c4204fd8574a921bf2 +MPFR.v4.2.2+1.x86_64-linux-musl.tar.gz/sha512/10386371a2898baa3a6442e2e93bdeeaf243a19698c440266f9b100a7c583244bea26e245a0d4c8f4f9f79bd9b70bee5b6aceb827efe425ee5daa7686e78943d +MPFR.v4.2.2+1.x86_64-unknown-freebsd.tar.gz/md5/538d36ad9242e304602eff5d61a14b4e +MPFR.v4.2.2+1.x86_64-unknown-freebsd.tar.gz/sha512/10e21ad9949e49c84934329032531c5605573761f839ae77fa4b0b504a0dd66733d70f99a204c7fa3a20b1c85441ed236549ecf532ed48e873fa6e6322323278 +MPFR.v4.2.2+1.x86_64-w64-mingw32.tar.gz/md5/1a479ba48476f6200a291ad7fdd24cf2 +MPFR.v4.2.2+1.x86_64-w64-mingw32.tar.gz/sha512/b2859ac13656266296959dcd9dfd98bbd3821e0e4cd62c27c30c6c924d986d861cd845a8209026c971ac0d670d266bb09c03df0151e29b28144229e5c8b697c3 mpfr-4.2.2.tar.bz2/md5/afe8268360bc8702fbc8297d351c8b5e mpfr-4.2.2.tar.bz2/sha512/0176e50808dcc07afbf5bc3e38bf9b7b21918e5f194aa0bfd860d99b00c470630aef149776c4be814a61c44269c3a5b9a4b0b1c0fcd4c9feb1459d8466452da8 diff --git a/deps/checksums/nghttp2 b/deps/checksums/nghttp2 index 24ffbdd7c51f6..0cae99b46488a 100644 --- a/deps/checksums/nghttp2 +++ b/deps/checksums/nghttp2 @@ -1,38 +1,38 @@ -nghttp2-1.65.0.tar.bz2/md5/7af53ab8c3d41874708a364a1940d22e -nghttp2-1.65.0.tar.bz2/sha512/dc9f8137929c9669935fa589243e49a45a85e8d2026801eb489738c8a23e0c8e412a692c2c90a38f184e403f5a7bf9341eba2dfb24ccf94c1326e0a9787f258d -nghttp2.v1.65.0+0.aarch64-apple-darwin.tar.gz/md5/6d7ef75bb41338b9c31d8002ef0de891 -nghttp2.v1.65.0+0.aarch64-apple-darwin.tar.gz/sha512/5c430ec968114bcfb02abf0ba454842980bb613c8d520d0a455361c6184b580b661e81565a51daf3e996a5c9bd2e104876ca7f5d2f2ae81d2a67993bf5cb722f -nghttp2.v1.65.0+0.aarch64-linux-gnu.tar.gz/md5/d96313e3b0caed58a596853f59f5d9d3 -nghttp2.v1.65.0+0.aarch64-linux-gnu.tar.gz/sha512/a36b77dc5e5a10e7e7df991a7c24dda81c29905a66920b71648c0845e42c35590ec055d5000ffd8885ccc7a83b3237e35369a4bef2f6d354aa8084972b4ec659 -nghttp2.v1.65.0+0.aarch64-linux-musl.tar.gz/md5/c8931f131fcdd08cb267fc602b57274f -nghttp2.v1.65.0+0.aarch64-linux-musl.tar.gz/sha512/e2157bec6055bb647bcfa2a7d0156f10fa5c125bec4d85c2a35593cb49c66597d1e90605c7385177a5b37e99b8f2f45e2d82c07412fea4f13e795ffd27d97738 -nghttp2.v1.65.0+0.aarch64-unknown-freebsd.tar.gz/md5/430793be9b5c8ee3396dba3d8586aa9a -nghttp2.v1.65.0+0.aarch64-unknown-freebsd.tar.gz/sha512/fc1ab985ec69f5f01fdee7e12c21dd70c3695c42ffb80466dfd46a6b3c727ab868beff7ff04f12f590f4ea63509b07e61eb289e451af2422a273d8f14ba60ae9 -nghttp2.v1.65.0+0.armv6l-linux-gnueabihf.tar.gz/md5/714bf2461a511391a6c229371c988d90 -nghttp2.v1.65.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/405b201d4b59b2cbd189b26d32f92360240e0a6824ef3f3f86c9a1a259d198bf2ab1dde1da591313784ea86ff27b2190ee614c11089c3304c46d7aeee9838296 -nghttp2.v1.65.0+0.armv6l-linux-musleabihf.tar.gz/md5/052e0cfef92fec9245fb564c1a7b3bb3 -nghttp2.v1.65.0+0.armv6l-linux-musleabihf.tar.gz/sha512/583948f7f83ecc7c9c22601033f67ec9db9bcb2bf04e45e3426771b78c800537a51c9bc1642456457781745621ccfe6c89874fa5132c54551b74211c511daccc -nghttp2.v1.65.0+0.armv7l-linux-gnueabihf.tar.gz/md5/7460bf5c5e51161f68c4bc6be7dad974 -nghttp2.v1.65.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/5d98dc646647e4477a976fa7b131419ab00d6d80a07a84e7e34bc2f12b071176986536567018b70bfce28cfe250901d803bcf9c92b67abe98484f5b846aa21d2 -nghttp2.v1.65.0+0.armv7l-linux-musleabihf.tar.gz/md5/89b1f46dd339c99e7d0b9a5e35a646c3 -nghttp2.v1.65.0+0.armv7l-linux-musleabihf.tar.gz/sha512/d213c6445474a38a9aa1acf275d184a6c35f5320a93d182fb619e46be41cb99704ed0d64b5269cb88f2f0c774533568564b2468fe5c909cd9dc77d1794c0b256 -nghttp2.v1.65.0+0.i686-linux-gnu.tar.gz/md5/aab9d9d3f9cdaa473a4ca8be7100a843 -nghttp2.v1.65.0+0.i686-linux-gnu.tar.gz/sha512/028d328ddb923ff472cbf3580e5b8b277439834642085aaaf0919971cd276a2fc6a58e05f5316f4b0a86080d5ed3d0678cc4f222d6fac99a83d6f461ef115a88 -nghttp2.v1.65.0+0.i686-linux-musl.tar.gz/md5/13ba56f991242da75bdd12e7f17828d6 -nghttp2.v1.65.0+0.i686-linux-musl.tar.gz/sha512/3ec51341bef5cadfc6f07f84ffa9871b2b4ea408da0d2be7a84dec21b75d550d6c883b24f2d9fbe7bafd1bb562452b14934eee2c4ebe27670c1f74525fcb7c97 -nghttp2.v1.65.0+0.i686-w64-mingw32.tar.gz/md5/297e11d7bd22caa0c44f34408e8da225 -nghttp2.v1.65.0+0.i686-w64-mingw32.tar.gz/sha512/571f767eecc2bc416429ae3f323c7e4ca249dfee4a79262e92a98514d5200efbf56058f229d074d04fc7842e0f8cf490cf6b2b814bd191a0a94541e471d6811e -nghttp2.v1.65.0+0.powerpc64le-linux-gnu.tar.gz/md5/403838ae480ed172b46df1e682fa7030 -nghttp2.v1.65.0+0.powerpc64le-linux-gnu.tar.gz/sha512/94e902c0aeb745a7826d9ba75069b0a1d4a32512ad45ba07df8a6d9c38dcecdb952ebcb63713abb93335c78fc129271eaa439514bd13acbf17785b8577387953 -nghttp2.v1.65.0+0.riscv64-linux-gnu.tar.gz/md5/3e0e0af6c0736d90607f06ea2b4b97e0 -nghttp2.v1.65.0+0.riscv64-linux-gnu.tar.gz/sha512/c6ed39ec45d4453fe2d3f030ad61acc3430971ecbdd716f8ff9b152cca4d8a28e23615423c307cae8eccf5f29e204982b7ebf81711e4dcb1168be2a74133152c -nghttp2.v1.65.0+0.x86_64-apple-darwin.tar.gz/md5/44e9d920146035834c00310d3be9b07d -nghttp2.v1.65.0+0.x86_64-apple-darwin.tar.gz/sha512/557b7592b2e2baf066da571f919ef0fcc9bc55aceed14ee2ff17a7e2b42c2a8bb70b1eece5db9df0171fb1d26cc419e232bc4ba6442b8c09d22782a303efae17 -nghttp2.v1.65.0+0.x86_64-linux-gnu.tar.gz/md5/9ecd26ca3489a890cb2f3fa3b85b4175 -nghttp2.v1.65.0+0.x86_64-linux-gnu.tar.gz/sha512/e9ddcb84d4e4e471b890ba0a49465ce3e87e55e0b549d24bf0575db52af5f09a3d0f665749c7f44b7248115cd14843dac859f485838fd01c6360c33dc707ee4c -nghttp2.v1.65.0+0.x86_64-linux-musl.tar.gz/md5/377efb2190c4937fd99211c9eafb2b07 -nghttp2.v1.65.0+0.x86_64-linux-musl.tar.gz/sha512/b68cb00e21879efe43b5bf82b1161e081ac8a1f9f75bb8b67c805019d0213f8891874469e289211a2719a00db30fb34acb20da22bb17773124127974d160a296 -nghttp2.v1.65.0+0.x86_64-unknown-freebsd.tar.gz/md5/2af524ec6455349603ea2fb40d948d3e -nghttp2.v1.65.0+0.x86_64-unknown-freebsd.tar.gz/sha512/45e2134f7010c171f2eded76df8497cbbadd63b1fca7b1e0587ae71e82ed1a9175f553d0cdf6a27908ccc19e687472eead92d01962e8df15ec26ff01fb1ce7dc -nghttp2.v1.65.0+0.x86_64-w64-mingw32.tar.gz/md5/a9ffe97f65c970ea8db686eda36ebfb4 -nghttp2.v1.65.0+0.x86_64-w64-mingw32.tar.gz/sha512/57f13c387bf2fd5eeffe643201a1e8eeac5999b86a4505be03462c3cc4c13c50dcc173303082bc4a1996bab614a8f8c61a55a5a3326a3f32086d29ac82ab9652 +nghttp2-1.70.0.tar.bz2/md5/77c68e6a8aca5da47d590abbf58eef34 +nghttp2-1.70.0.tar.bz2/sha512/f847642fc393c9e7f82a5cc9c0e331fabf68efc0b52b33a38e9d4ee1fbfe443c76c3436fed0afe8d3e14a52adce068b82f54fa3fca51d23ba392c5c71adb2d3b +nghttp2.v1.70.0+0.aarch64-apple-darwin.tar.gz/md5/b778dbaa39f4bc20263cd7416528399c +nghttp2.v1.70.0+0.aarch64-apple-darwin.tar.gz/sha512/0bf6cfb609b2a75490a62956e11d70305f2559114304cb3332ac721ce02d6d4c7107f589a2382c0d6c98a12a237850e50b025e6f463950534096cf93f152e891 +nghttp2.v1.70.0+0.aarch64-linux-gnu.tar.gz/md5/2fc25fb8126258e62ccbf2041615a86b +nghttp2.v1.70.0+0.aarch64-linux-gnu.tar.gz/sha512/57b55f6cf95669175d49083fc0df8d1eee652f2b954e2daf286b34b8137a3b57508d0ce9450034d806b8080b8f7b90abf6db57b768d76fe77359f9ad92284181 +nghttp2.v1.70.0+0.aarch64-linux-musl.tar.gz/md5/e533afef5c6fc950d10ada024aa906ea +nghttp2.v1.70.0+0.aarch64-linux-musl.tar.gz/sha512/54b051fd86a54ac02cdefc3a890bfa20431bfb8a5f0744cbea581571b99c7f9998aaba0053220aaecf9b6ef7b5f72ff721b2af1ecb2afca26aab695bf85e29e5 +nghttp2.v1.70.0+0.aarch64-unknown-freebsd.tar.gz/md5/f4bd787523d3d0383b55a01a8b696f8b +nghttp2.v1.70.0+0.aarch64-unknown-freebsd.tar.gz/sha512/a2277ddfcae320a7ca017893d4f86d3539ca9b3dd35ff2dc6537b2c955abe1ec7a949a414166cfb0151f674d927cf51fe8db77e92a4ebec8b8dcf3d2f2af48a2 +nghttp2.v1.70.0+0.armv6l-linux-gnueabihf.tar.gz/md5/10588226ed68f506dce5443f84881bc2 +nghttp2.v1.70.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/320d2070aa12b7e394484c68abc3d831b22c4c3a2e52ab5c795288ca17addfd2b35cbfcf8fc521e0e938b0a75c33dd477acb80fd730143c3083ac3f034b4dcbc +nghttp2.v1.70.0+0.armv6l-linux-musleabihf.tar.gz/md5/758d0239bd015c2b377177e0b3b0f494 +nghttp2.v1.70.0+0.armv6l-linux-musleabihf.tar.gz/sha512/a32754917bb8d4e3e43334e67b0df9a72a435f93ec6d3016c2dad83474be41c2c348a13042559650e05e57f364d24cd5ad04e884ef66aad22178ca4fd8d318ec +nghttp2.v1.70.0+0.armv7l-linux-gnueabihf.tar.gz/md5/c98258ab01ef4c2bacc85f121ca7974e +nghttp2.v1.70.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/c87e234b96c9d038b0353920f3215195776dbf30dfa3b34810b6dbc536fc58355a932d4b7b9f52e499c53927f905af2f79b78a04f21c35a769dd821f5f0039af +nghttp2.v1.70.0+0.armv7l-linux-musleabihf.tar.gz/md5/8bb70e512f0f947892437ee1e3e18f08 +nghttp2.v1.70.0+0.armv7l-linux-musleabihf.tar.gz/sha512/0075436bc916b4d2b541b42de9041f7879015e26b808a7c8fef429fd6520279cd7d4f909311060128b5aa23fe7b0b24131aa0e9e8f027009dd32a52e37f3247a +nghttp2.v1.70.0+0.i686-linux-gnu.tar.gz/md5/684206af6bda5861738731e61ce1f23d +nghttp2.v1.70.0+0.i686-linux-gnu.tar.gz/sha512/816f2c3e1fa1c39bed80e47b39851086b47b49b955d6865c58fd6dc24bcf942fad408c5afc82bcb38e059dbb908a8ca01180e2591619a310af999f9df7384d59 +nghttp2.v1.70.0+0.i686-linux-musl.tar.gz/md5/b6443cb57233dd0c052fb88feddfbc2a +nghttp2.v1.70.0+0.i686-linux-musl.tar.gz/sha512/46245627be923378a1444085290e83fc9623049f7bfe90218a981ef928721b75b0180df77adae6295fb88c6c0ce792eb054ed9c2f023252ed3bc36570f51a3a3 +nghttp2.v1.70.0+0.i686-w64-mingw32.tar.gz/md5/fed2569296d4a9e8f63f43f1f152e908 +nghttp2.v1.70.0+0.i686-w64-mingw32.tar.gz/sha512/9d4c0857b899f0b91ca38909c11e0941a2deb0bee0dd1715c10983ba4b5bcd8bbb3d0ed32d40a6d19c041dfdd793fc51ac746a6b1b61a1b555d5cda48bd8192e +nghttp2.v1.70.0+0.powerpc64le-linux-gnu.tar.gz/md5/774f137200f8a461205e532e04c2edf5 +nghttp2.v1.70.0+0.powerpc64le-linux-gnu.tar.gz/sha512/11c1403dc62cb981f4070b2c1221c99fbae8a700596fde9ae751f71598ff0b2e27a2fcd02689cd350c8bc21ceb48baa9ddc51a13db4ee64cbae10139258fca7b +nghttp2.v1.70.0+0.riscv64-linux-gnu.tar.gz/md5/40787dadd04c5b04996544b509e19fe0 +nghttp2.v1.70.0+0.riscv64-linux-gnu.tar.gz/sha512/e26522133d357bf6843422efe75c346c0ca9bc720799682474ae0936af479d7453b7ea9a7d073c6d2eb9f7ccd6aa4c6303bd15161436af48304694357c25a042 +nghttp2.v1.70.0+0.x86_64-apple-darwin.tar.gz/md5/cb35dfa922714c516d7c73db79608796 +nghttp2.v1.70.0+0.x86_64-apple-darwin.tar.gz/sha512/41df94d2dbd87583bd281ba4ad3e12cecfe732c1ef8014defb3bc3cd639fdd33d5809c4700917f56fd47169cbed470e5ae026d12ac811114a966a2997bec845a +nghttp2.v1.70.0+0.x86_64-linux-gnu.tar.gz/md5/968cb87a62e057d4aaf915c1db56fbfd +nghttp2.v1.70.0+0.x86_64-linux-gnu.tar.gz/sha512/4f015898a9b983b1e31451ba3d604a91cc028f661c5a583131f1821b7d3c6b65564cd5d159e96e185ba0be00f61288a7ac1562aeedfe3c33a661653f1275e3f9 +nghttp2.v1.70.0+0.x86_64-linux-musl.tar.gz/md5/45a6b6b42ddc2337f1069f91b95d3ce6 +nghttp2.v1.70.0+0.x86_64-linux-musl.tar.gz/sha512/5afcf06cf59f8a0573e018e5610cd16b8d2f4d1529bad63cb8f092d61936333253ca74e8f36033d9cae25c7b314011a265c1148b25513348f3ad91e48bbf6537 +nghttp2.v1.70.0+0.x86_64-unknown-freebsd.tar.gz/md5/574f276017cb501619c9357361a3d901 +nghttp2.v1.70.0+0.x86_64-unknown-freebsd.tar.gz/sha512/a7e31bae4043be41d68b2d5d1895196dc4242b903a9acce500971642e2e63c0013c69ce7234cbc74484a3f7a9965660747faafa37ffa17b5e613b3b2d2ed3eb4 +nghttp2.v1.70.0+0.x86_64-w64-mingw32.tar.gz/md5/5b1a64e33cc0874ae1e184747fc2b5d1 +nghttp2.v1.70.0+0.x86_64-w64-mingw32.tar.gz/sha512/a9c822f5110ced69ecbefc6acb77266c2c309805ac38ee2e4c92195af99e28e303fc4e249cfd86cee6d4c2e6205491d934acf2ffd17087c572cbb46b5de117b8 diff --git a/deps/checksums/objconv b/deps/checksums/objconv index ce20ae4acf76b..09db43a8b7bf7 100644 --- a/deps/checksums/objconv +++ b/deps/checksums/objconv @@ -1,32 +1,36 @@ -Objconv.v2.53.0+0.aarch64-apple-darwin.tar.gz/md5/ff9f237208e8bb48daa5eb4d18526f24 -Objconv.v2.53.0+0.aarch64-apple-darwin.tar.gz/sha512/5960cc291fd551dc288cba4cfbe9c2448ebb67da72487a52d364419199e91541a427c70d73c610e054b7fe6617c9ca715ee5d3e6e694d49f16331a067bb7df73 -Objconv.v2.53.0+0.aarch64-linux-gnu.tar.gz/md5/e9ead2b4711ff50a9c88272fc5b4f0a5 -Objconv.v2.53.0+0.aarch64-linux-gnu.tar.gz/sha512/9eeab07437e61af46d884d4d3bfa6bf7538c9bd996a8b217748ed70a2d53df25c20235af78d8767106603a3c92955a1f1870533d018d35e55ba8088b14e685cc -Objconv.v2.53.0+0.aarch64-linux-musl.tar.gz/md5/c506fb9e5701a416c2204dd42a0e84b1 -Objconv.v2.53.0+0.aarch64-linux-musl.tar.gz/sha512/b6e2d91cddfe6d8c42d233e9aca1f322a4f4e12480ed2a0d148a327955b4dfcf049cb05e2151bc7a64d46ca3e3507eaae8e77416eb895bac16f394217f3e83ca -Objconv.v2.53.0+0.armv6l-linux-gnueabihf.tar.gz/md5/2f353785310504e7cbed6bc8bad2cc64 -Objconv.v2.53.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/153e595c0385395d90cafd57d08ab1ab0cae9f0376c13f139c9dc6132763cfd19c64e74e8d50f33aa50e74173f3f57f11a11ad0afeef20c2f38c00badd339068 -Objconv.v2.53.0+0.armv6l-linux-musleabihf.tar.gz/md5/a5413f1376abb962de5665cebea77c39 -Objconv.v2.53.0+0.armv6l-linux-musleabihf.tar.gz/sha512/78e6778bdc4d31905a585ecdf87c499f7befad4ee4c46a2d26e6b1e9330d9175d3def536291a4bea004dd65cf95b8b751557afc40a1513a786502000c5a3769e -Objconv.v2.53.0+0.armv7l-linux-gnueabihf.tar.gz/md5/9008ac2cbe75388fcd34cbaf523512f3 -Objconv.v2.53.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/ba877ea596965ab1ec419cc46e59d9010a96c52fba8036374e9df78157812926a34ceb3d29e997a5a27d38143d404343cbcf15c1e1b136835235b9a3c8543c77 -Objconv.v2.53.0+0.armv7l-linux-musleabihf.tar.gz/md5/86d14f65fb4f3c0dc0848a9d683f0fa9 -Objconv.v2.53.0+0.armv7l-linux-musleabihf.tar.gz/sha512/bd3a29982a16437936481b7244d237d9ec8d0cc7d78e50cc8f30c7c3f58475ba7a35e1fc0f1d1540d780b2573cfc2de2e59c8dd8d1f90de7b7fbb81a74f526a6 -Objconv.v2.53.0+0.i686-linux-gnu.tar.gz/md5/d22c35a1b47c64a291903f0ca5acc297 -Objconv.v2.53.0+0.i686-linux-gnu.tar.gz/sha512/b431d44839a628b75c189c5188f15462eeaf666a868e33c56e1c5f41041f1903e6daa3c4538b95a96a26c5842501a0055c7f2f12585e4f341ee2c8fb452855de -Objconv.v2.53.0+0.i686-linux-musl.tar.gz/md5/dd583694f915309c8e7868cebe27f20d -Objconv.v2.53.0+0.i686-linux-musl.tar.gz/sha512/279f36b7d1bda71ac37a1cd9c9e7776808499066fde94b456b000c187afa173fe7858894bcb4666831af2948c4363a15955740da77c01d63f6007627b61c1c0b -Objconv.v2.53.0+0.i686-w64-mingw32.tar.gz/md5/42c7264dac94b87556dfb2d469c5c60a -Objconv.v2.53.0+0.i686-w64-mingw32.tar.gz/sha512/a7eda3e7c5b3b19321d49264c6a7c35c526955ea7fc6b0201062f7e136eea6799a4d9269d78bd84770146d79fa3e49c1251a4568b5ffd1675c952e9544566553 -Objconv.v2.53.0+0.powerpc64le-linux-gnu.tar.gz/md5/33cbf415f36c0be59f34bc2e9b4c1226 -Objconv.v2.53.0+0.powerpc64le-linux-gnu.tar.gz/sha512/71cb373ade619cd431d64a8e097ff18e4979b39cff8b6be71b6ce77dc2dc4f663b25b1ddcd74e8d60834574cc3a5ed512618e6205df757d1f9752d78e35a0ac8 -Objconv.v2.53.0+0.x86_64-apple-darwin.tar.gz/md5/036603ce6fe365ccb838aebb6479ac58 -Objconv.v2.53.0+0.x86_64-apple-darwin.tar.gz/sha512/86f2f3e551bdc0595dd096925115671558970b4750c13d1ceec28fa47a3f61459cb1c4c57a5f930670035136f8cbd6073b1aced0dcc245f3a7815c26f14ad6c0 -Objconv.v2.53.0+0.x86_64-linux-gnu.tar.gz/md5/6a3ed48d4c8181934c02de60902b9f4f -Objconv.v2.53.0+0.x86_64-linux-gnu.tar.gz/sha512/94f306e8053ac94d9c679a403fcecd0a5d3bbc7f62409886cdf45342fba3fff7c22a30679aec0bd9d76262a3ae753b8a68c243b32e5966f83557bde5d988d80c -Objconv.v2.53.0+0.x86_64-linux-musl.tar.gz/md5/093aa158739b90fd5f5583868b5c898e -Objconv.v2.53.0+0.x86_64-linux-musl.tar.gz/sha512/715ea896c0cfbd2505d1ae39f2d8c6950d56f0b262c108a0e378163a209e4a1581e2d7d1f3a7677d489baa21dea9e46695415edf7615ec66078c1a7978cc578e -Objconv.v2.53.0+0.x86_64-unknown-freebsd.tar.gz/md5/a56c36c004cfb70f7feb7be0b681af19 -Objconv.v2.53.0+0.x86_64-unknown-freebsd.tar.gz/sha512/ea40c235145b00b730ea81e946a0be247a10950a564a3ac33a493c0fbcaa866f039ccc47b852d436b13fe3c52b7436f4903621a8c91e48d947cddfda42e6a482 -Objconv.v2.53.0+0.x86_64-w64-mingw32.tar.gz/md5/291a044f511f9529e9e2425aef8a7c16 -Objconv.v2.53.0+0.x86_64-w64-mingw32.tar.gz/sha512/4c43ed6842f53ab6c081bff2392e65b5295acffc7940caaa8d36a2f845368d37aa40259d0825f0ff08fad2ba58d5accd78dd96d51c8992396571fb96c81b1555 +Objconv.v2.55.0+0.aarch64-apple-darwin.tar.gz/md5/8ba9de3951c3406881dc9b9a08f6d9f9 +Objconv.v2.55.0+0.aarch64-apple-darwin.tar.gz/sha512/ef58b22ae077c9ec01d2ce0c7ef3a15ead5457399fc058f2a8333b44602bf0df033b421113934c75ac4d9edbb48ebcdaf5e08608829199fc5590d2807dcc2e2e +Objconv.v2.55.0+0.aarch64-linux-gnu.tar.gz/md5/11dbe5a12f73f5d8fe04b563cd3fc457 +Objconv.v2.55.0+0.aarch64-linux-gnu.tar.gz/sha512/371d8542892070aacd06dc44915edb053f8df71a992d11cedaa6fb496a10d4c2f50e36ce45892af2409b422f5218c16dd95cdaf25a3f6642d5c817bd660e8017 +Objconv.v2.55.0+0.aarch64-linux-musl.tar.gz/md5/2a8c213058a8288099630fc201fb31f9 +Objconv.v2.55.0+0.aarch64-linux-musl.tar.gz/sha512/90c83ad869ff4cb08248adf469019896fc99b17bf1d010db9aca965b942a853fbf2e7fb6d788bdf7096a6c5be462fc48a206e003097abf3374a6e5cf40217401 +Objconv.v2.55.0+0.aarch64-unknown-freebsd.tar.gz/md5/ae34a491f753ab3065c55010c37eef54 +Objconv.v2.55.0+0.aarch64-unknown-freebsd.tar.gz/sha512/dd6da052ffa879beb92459caa471af2d67a86c16940f3134e40ef51720fccd1a84a3db5f52d6f84c306c5ea83b223db380c898c298d917237920415350d598dc +Objconv.v2.55.0+0.armv6l-linux-gnueabihf.tar.gz/md5/cad9ce64eee92b0656663760c0a79969 +Objconv.v2.55.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/4e5788efc851519af3230553a4765193a0696c16deaa4ce531ed9ca9e5e71aba86aae0a993c82b2fa220bebcddcb1f691a82e0f2e2db3608358afa122edddf8c +Objconv.v2.55.0+0.armv6l-linux-musleabihf.tar.gz/md5/a963aafa8c1cc3e9b29c147ed07694ba +Objconv.v2.55.0+0.armv6l-linux-musleabihf.tar.gz/sha512/96ff5c1abdc6d939cdd94eb47c33c23656b655796c34de35881f531a7b18a454ece39c90ddb21e2a8e5611a922d6804b18043e6421e7e07d16a9bee10ed2b7e2 +Objconv.v2.55.0+0.armv7l-linux-gnueabihf.tar.gz/md5/25ce7fbd3926eff25dff2971b74eba67 +Objconv.v2.55.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/7656afd0f4e5b96b26f596e0d9d23701a017c22cfd0aae9153f4aae3355c2f3bda3f6530c2daf60eedd877d80c941790ee4aad0f2fa1283f62b463efb1c2d222 +Objconv.v2.55.0+0.armv7l-linux-musleabihf.tar.gz/md5/0983c6acb87ced5f4a23e3aad0adb5ae +Objconv.v2.55.0+0.armv7l-linux-musleabihf.tar.gz/sha512/98ddf8eac980f42260d5fa5f5739e3bc3aee040ad3b6af3fa6b148b657509940c2bdbc7760118f1e7e0a86ae6ca45372eae8600808c29a52a367ad9a8bbd8062 +Objconv.v2.55.0+0.i686-linux-gnu.tar.gz/md5/3252f86019ee64092fb061c1e8f42cc8 +Objconv.v2.55.0+0.i686-linux-gnu.tar.gz/sha512/07e36ad4c54c16dd0d3eacacdc79beba69ab602e0b5a4654fe218ef3eab340af5beb21cf7cc6f3bcf9af38a77eab5893ab3c38d52388154fe66d6dd149e4e2ce +Objconv.v2.55.0+0.i686-linux-musl.tar.gz/md5/75f9a0ab44e9c5202b2a6e3433b5f345 +Objconv.v2.55.0+0.i686-linux-musl.tar.gz/sha512/626a5daa02203306fabd2348833fba4476dab9166091f86393d32eb01bdaf33f73fa6e377ff8e12ddd67c3aed1597def01e2c974c8aa5c498f36054eb07594ec +Objconv.v2.55.0+0.i686-w64-mingw32.tar.gz/md5/c0ed10fe824e5d49436bde2843e02594 +Objconv.v2.55.0+0.i686-w64-mingw32.tar.gz/sha512/89bbc593bb977d10757acb4fd273e0d92d0c8d6cd4df88aa24b41434b80ef71de9df2edf6f21968777ef275a58e6182e674b273cdd5d94a24e04926c35a30633 +Objconv.v2.55.0+0.powerpc64le-linux-gnu.tar.gz/md5/9ea42b486d436bd9bf9f1244c4e4d9a8 +Objconv.v2.55.0+0.powerpc64le-linux-gnu.tar.gz/sha512/c14490521259ff3164521e861c6c6c9766785f201b1fd9e71c851d99b859405e5397ecb56a0dc7a8acfef779e56ca933be7319ca7fb2713f213669149f259847 +Objconv.v2.55.0+0.riscv64-linux-gnu.tar.gz/md5/a43272b0a5470143148c948260ad541f +Objconv.v2.55.0+0.riscv64-linux-gnu.tar.gz/sha512/9b0c24aa917f7e6750c64bbe01a0c0857fb72e389b05d4b8d38aeddc41bd9546d3731f79b698ac0ee3cac2e6362ded4efc3fe0f99a842e303d91146e3e45b65e +Objconv.v2.55.0+0.x86_64-apple-darwin.tar.gz/md5/4c656ec2e49be66271a5dd05181b2caf +Objconv.v2.55.0+0.x86_64-apple-darwin.tar.gz/sha512/f20d4cbb3e6ddd4ad3d4e8c8f6be79ac6c28e1981552a2bfcb4683a78bfa242e28db2d6e0e9d1913ec71b3ed1cbb1ca14ac4321328d7a7fc878734624b78d6db +Objconv.v2.55.0+0.x86_64-linux-gnu.tar.gz/md5/e1e427f0dcefe285eaeaabe8a947ea95 +Objconv.v2.55.0+0.x86_64-linux-gnu.tar.gz/sha512/96405d3a90918a978c772b0ab0452679dd91bd51ba9f2a06e8b5fdb2ab6608ba18cd74acd6167709af45efc5147f42a958fb1dac3d5b75cd270b13adf40e2f4c +Objconv.v2.55.0+0.x86_64-linux-musl.tar.gz/md5/69c46df368e12f0ea046eb3fd5da743c +Objconv.v2.55.0+0.x86_64-linux-musl.tar.gz/sha512/74bd1db59fcd016d1bb06e8445d54c3589fbb979ddb1eba2c99d8b366e27aed8f1d830c896acc89eb4458afd7870da5475a95e8679059186fe9481210d77c182 +Objconv.v2.55.0+0.x86_64-unknown-freebsd.tar.gz/md5/1db670d15bd5aae1810414a9b010fcd3 +Objconv.v2.55.0+0.x86_64-unknown-freebsd.tar.gz/sha512/dc1edaadb2423cf48403036c19fa61fc4f2cc9a3a6b2efe4dc37b0a1073870b10e6e78e5eca862167005b6101cc1e23806addbbe9be036a6efcef376abcae889 +Objconv.v2.55.0+0.x86_64-w64-mingw32.tar.gz/md5/d5a050f359a21e921a8fbf1285c31793 +Objconv.v2.55.0+0.x86_64-w64-mingw32.tar.gz/sha512/ec78583b8522bf56a5048e7ac95f5460f0b67d1d9cec5c3cea297fe9e036e5248ee27f1ca916a8dc26252fe2f1dc9e3200af397112e351ca8909a8583e1fee17 diff --git a/deps/checksums/openblas b/deps/checksums/openblas index f1bcf3f322d8c..57a226937599e 100644 --- a/deps/checksums/openblas +++ b/deps/checksums/openblas @@ -1,96 +1,38 @@ -OpenBLAS.v0.3.29+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/227fc95ef10e30698aade797ebd8b685 -OpenBLAS.v0.3.29+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/49a932f0c1c2d1087d20a3de2940733ed6a944284e1cf2a384a7401c5ca6bd90a35e9679b4f19bac176923aa170427e7514a47fc16261413ee03a59bbb301bd0 -OpenBLAS.v0.3.29+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/77acdfde5dc6f05629f3fb68a95b78f8 -OpenBLAS.v0.3.29+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/f28187213eac2d481bc12263fe13fcb35f4771084bacaa42b0b149ac15cf89d033910519ecc5cada77915a48c95a2de3ea4a476c0c6bc3f154e7f2ceb4bf3ffd -OpenBLAS.v0.3.29+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/4fb2bd80d3e4ad8ce04fa33c9a2aaa19 -OpenBLAS.v0.3.29+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/8cc2aee3b351cc5c78e494efededdf98f65ce8942453bb3a55c90e0822ddcc07bc7716d0746bbc16701eca458b7a7aa933e9363f71bd56788c9fab36bd9bcf6d -OpenBLAS.v0.3.29+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/3a3ef97dc80dec3d0debade503ca2232 -OpenBLAS.v0.3.29+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/53d707f9bf57c1a19279f0146e767d779280f922ef621b5f372cedc018efb2798adabbd762324819f342d0fd98ec17c68badc50da7b6e9aa3e57c3a3c045dab2 -OpenBLAS.v0.3.29+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/463cb6b46091f4b4b4f2535b9f38f11d -OpenBLAS.v0.3.29+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/5a5a7d5a7ca5e619d5af9bcbab7cfffcb4b7954005cb4a2d03f4cd0ef29c95707e830ad0b0303d694cace557cb1e9973c0244ae1f635249a313fb9f9cdfaacd9 -OpenBLAS.v0.3.29+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/699ca0247ec7cccec0d9d2801b5a35a7 -OpenBLAS.v0.3.29+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/3bb2926d2d2a43c280bb947063dd74b65194118edbd99df820bef56a546648ed903245e0947ebc31765ff43784b11349bf86cd592c78d143c0627d692162b344 -OpenBLAS.v0.3.29+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/2ab069e5abd5014495b849bfbaabbd3a -OpenBLAS.v0.3.29+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/fd10e4ce326c524d97e69e50342ab63b8298c796faab8f4512772fbb9c4ae1ddc85d54643c868f3b2dc8084af974430e1f8751576bedfdc88af2ba0d2affba1a -OpenBLAS.v0.3.29+0.aarch64-unknown-freebsd-libgfortran4.tar.gz/md5/ce5d04e041e9447529ad8e043e45895c -OpenBLAS.v0.3.29+0.aarch64-unknown-freebsd-libgfortran4.tar.gz/sha512/eaf521d3957713e9d22b2c0b991f5eb846096891dc15bc42ad0817c32e6a1343617d28afe739dce0e39c185d022d3cdd44db2610635691990003b1b0a29f4657 -OpenBLAS.v0.3.29+0.aarch64-unknown-freebsd-libgfortran5.tar.gz/md5/00b3a4433f93a56fa8b0f17acc254865 -OpenBLAS.v0.3.29+0.aarch64-unknown-freebsd-libgfortran5.tar.gz/sha512/a9845380778ec15642d74a46dfa65f8a325929f8ec8d61915941f6e228bb1ed29310f86f20ec559fdc2d5dac98a780f71a1b3116676a34e18ee7c0cb86cb7124 -OpenBLAS.v0.3.29+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/1be6fa7ef684733faab744fdec6c8dbd -OpenBLAS.v0.3.29+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/7c7803a0484b8c8e343ff5049e52fe81b76e43f0aaca7a5ad0134079147d2311cb5b159738486dcdd7ec69eb42cb0eea738741401179499a53fead2fbd8dba3b -OpenBLAS.v0.3.29+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/451dad687dd26a299e4a44db37a8db2a -OpenBLAS.v0.3.29+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/ea73ee91896203566dd7510303c73d77189afec913ac1de3b7c7935dc2c460f87c83a8ddd272d9542b619e419b9392479f02540ef1c8d3daa528bf05aaf5c3f1 -OpenBLAS.v0.3.29+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/07ca32f715981570f2e1a5ac6721e569 -OpenBLAS.v0.3.29+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/c6ece0dac375fd66a303ca6f503e46f78472a59dc13381e8462e3e9c29e133cbe87ee77f6144a80924ae286162620c4395f5217e4f9ba379a471409085950427 -OpenBLAS.v0.3.29+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/49ac07fcdf0d7ce221051d089b408e05 -OpenBLAS.v0.3.29+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/6c9c379473f1bb5f202ca183c6ef4d43b442c867e67712e6ec2936790c282143c1edae0a1385e366f729c952e02fca13604f6b51d778dabb28ca7be0f359281e -OpenBLAS.v0.3.29+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/87e3dea9e115fbc9a0c7f64020c41f74 -OpenBLAS.v0.3.29+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/63a37a9cc882562978460e1e0f603177921a64ece7d4050b0b7a584e05d80f58314e7f8e988ea5446945d7009620c4f746ce547fe7dcb77a0707d54fd830983e -OpenBLAS.v0.3.29+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/8c85e7ce9bd702438c548bdae54f5c32 -OpenBLAS.v0.3.29+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/3dbaa326944d79688fa167c968a7e2660bf3b94c2e052755cc8b1ede853c02364edb7fa974880c37c60ee6e6f84c75848eb4d999c5c1e8881441191dbab056e2 -OpenBLAS.v0.3.29+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/1be6fa7ef684733faab744fdec6c8dbd -OpenBLAS.v0.3.29+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/7c7803a0484b8c8e343ff5049e52fe81b76e43f0aaca7a5ad0134079147d2311cb5b159738486dcdd7ec69eb42cb0eea738741401179499a53fead2fbd8dba3b -OpenBLAS.v0.3.29+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/451dad687dd26a299e4a44db37a8db2a -OpenBLAS.v0.3.29+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/ea73ee91896203566dd7510303c73d77189afec913ac1de3b7c7935dc2c460f87c83a8ddd272d9542b619e419b9392479f02540ef1c8d3daa528bf05aaf5c3f1 -OpenBLAS.v0.3.29+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/07ca32f715981570f2e1a5ac6721e569 -OpenBLAS.v0.3.29+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/c6ece0dac375fd66a303ca6f503e46f78472a59dc13381e8462e3e9c29e133cbe87ee77f6144a80924ae286162620c4395f5217e4f9ba379a471409085950427 -OpenBLAS.v0.3.29+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/49ac07fcdf0d7ce221051d089b408e05 -OpenBLAS.v0.3.29+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/6c9c379473f1bb5f202ca183c6ef4d43b442c867e67712e6ec2936790c282143c1edae0a1385e366f729c952e02fca13604f6b51d778dabb28ca7be0f359281e -OpenBLAS.v0.3.29+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/87e3dea9e115fbc9a0c7f64020c41f74 -OpenBLAS.v0.3.29+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/63a37a9cc882562978460e1e0f603177921a64ece7d4050b0b7a584e05d80f58314e7f8e988ea5446945d7009620c4f746ce547fe7dcb77a0707d54fd830983e -OpenBLAS.v0.3.29+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/8c85e7ce9bd702438c548bdae54f5c32 -OpenBLAS.v0.3.29+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/3dbaa326944d79688fa167c968a7e2660bf3b94c2e052755cc8b1ede853c02364edb7fa974880c37c60ee6e6f84c75848eb4d999c5c1e8881441191dbab056e2 -OpenBLAS.v0.3.29+0.i686-linux-gnu-libgfortran3.tar.gz/md5/86834236dee3db3affb38b8cdcf59681 -OpenBLAS.v0.3.29+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/6731b4ea22a0f3d25f9d041e2baa6d66f1027dce49931a334a33711fc4c6de5da368274c9328618ed78158855c5d38524b917447d1aafb5c551934cf982505d2 -OpenBLAS.v0.3.29+0.i686-linux-gnu-libgfortran4.tar.gz/md5/c63c2fb1bda01456d99590e9aec3b45f -OpenBLAS.v0.3.29+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/437c260499f4a28db9efb4bbdff31c0f675f3ccef1bd48fd2dfbb8c8897fc75608bd7247293bd3eae129b133cb05c3c8150dd19c243faa09b6506688f57c633a -OpenBLAS.v0.3.29+0.i686-linux-gnu-libgfortran5.tar.gz/md5/376567d56bf4314f8a4adcfc4d1baa66 -OpenBLAS.v0.3.29+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/c4952874b19af4fd0d5541999d07094f7e7e983124964405a4756b9adf619172b7128e11557e64a80bc4eadaf76c783609a75f25ccfc44fc4f181886a0c8ca18 -OpenBLAS.v0.3.29+0.i686-linux-musl-libgfortran3.tar.gz/md5/8f7abbc6d5cefdbefb2b9499ec8874c9 -OpenBLAS.v0.3.29+0.i686-linux-musl-libgfortran3.tar.gz/sha512/b8c39674df9400efecbe4ac740f0c3ef11a04dd852f31774d63db3ca6583a21c8e0a0b80aa4e7b82be7a8fa3de38892d4fbca34244acef7fb49e8ffc0e1eed09 -OpenBLAS.v0.3.29+0.i686-linux-musl-libgfortran4.tar.gz/md5/6b0f0544fe45de9d2dea946c7f55cc40 -OpenBLAS.v0.3.29+0.i686-linux-musl-libgfortran4.tar.gz/sha512/8c21df39a8ee99614ef0880706c1497d032f68dfc332cc5ee111f69bfc818db4896115a964f16115ac49b01b31713037c905792d9586dd05471efdb21dd0be88 -OpenBLAS.v0.3.29+0.i686-linux-musl-libgfortran5.tar.gz/md5/aa343048c35c5227a4bcc37f25ddfacb -OpenBLAS.v0.3.29+0.i686-linux-musl-libgfortran5.tar.gz/sha512/af6c9d15d9d5a4901d228522d2e20da5276f1bf35d7f34648697ba7a39153a9152dc17f5f0d360593e733ef3e3317df29581cb86fdd9fe8d6e6093592a6240bb -OpenBLAS.v0.3.29+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/8595dda5ee1f15b2070d8ac20077f389 -OpenBLAS.v0.3.29+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/df7d7ad68b47f8865d01f6edd6ba44587c6563ebc4a1900f92210b5117fc7c581e6145f95e10fe7a3db48eda9805330073c8cbeec7eb8a19978ec33f2528cef8 -OpenBLAS.v0.3.29+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/4e67905ab599f24327e9726f70d261cf -OpenBLAS.v0.3.29+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/13ba78e98d7c2cda62a6ca9226365e90fa8a5404e4006ae5e49030b314b762a37d78977f14c72448c844e68a6b83ecd679c60362fde023c9052b9b8597d7775c -OpenBLAS.v0.3.29+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/e78c5005d9ee57ab464fca86c6d6fff1 -OpenBLAS.v0.3.29+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/8ceb9527461136cd4f4d02f10c241f5e7070991f73c974389acedb1d9d7be4bade592bc021ba1001c5ac148ea580cf8355fb89c88438820bfa665bf3e72392fa -OpenBLAS.v0.3.29+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/421d93da4cfab0df79569e09dff1015b -OpenBLAS.v0.3.29+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/351174d948626ad36daf40c45672cd1ac40bbe4af25c28332fbea62a7ba89188a7d33836d327d31ce99b9a9334c6053366d33b58f588355c2818e332e46b34d0 -OpenBLAS.v0.3.29+0.riscv64-linux-gnu-libgfortran5.tar.gz/md5/34cc0b3260d9471bc8fb32005e3c5043 -OpenBLAS.v0.3.29+0.riscv64-linux-gnu-libgfortran5.tar.gz/sha512/5eec279c5eead55d099d8db4a75dd4a3f2bcbc8bb22b33884a89d678e4eebf87c6dece1aa4c24374d0162b35f376648a473c2d6d7866583e61016e37f4262820 -OpenBLAS.v0.3.29+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/f921a0ad6ebf91f444cb8d927886e573 -OpenBLAS.v0.3.29+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/5cc98edf9fa8ba8981ce78b2595fd27645c783561ff19d0fd25ecc927f63492437a4b9b80d5caf51ad619b7ca5d24cb43e153156921f9f03c64741014b686196 -OpenBLAS.v0.3.29+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/0126b52c134954e63ab8f9197afebd7a -OpenBLAS.v0.3.29+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/6d1e37009e6831a26f25bfd3e95dbcc841ee50a3f84dc4355d7fd528cd74a400138955558306345e986a732d0d1ef9294c4f5be457d05119a8e1e5851cc8ca20 -OpenBLAS.v0.3.29+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/e8c7bd786672a9caf989dbe4fcef896a -OpenBLAS.v0.3.29+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/2e708fddfa8e5821d2e44bbc00a86df83b09cdfc0054d7c2bbb2a8de52ed80c95973e6602048335a60b54be1baeb617121b605644daf50579b2044d0c5766063 -OpenBLAS.v0.3.29+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/b1efd957a2a63f814168bd318381812e -OpenBLAS.v0.3.29+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/097a750b7f381089588498e52a2b07a67202bfd4bc2e38f5abbbeb372129e392fcd53beade2fa7cb60ef0038f2baf61d57fab40b5585806d3ddb1fcdad73bbe3 -OpenBLAS.v0.3.29+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/c3560828f503962c6ae94135c4f00ac5 -OpenBLAS.v0.3.29+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/1ad514172e51a5d3eb6fea03182e3eb9c6db99d9d11c430e3d8542a9ce0f5d6967e623b9c0951535b683210ce0b02460358c67520b06363594f6063f8f012396 -OpenBLAS.v0.3.29+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/07a9c3050824bbc6a96efdb333fff0ea -OpenBLAS.v0.3.29+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/b737ab1fc8c5ffd1494804c59f8fd3e5d3d8a063a89fbbc29cbd75d43af233ddf77f63d0e514059164517f408ea340ffe95c020a7c696af8c52be3a7259922ab -OpenBLAS.v0.3.29+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/9a4a828a1b58737c79eb170c94021c52 -OpenBLAS.v0.3.29+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/785443a38cda87a63ee4268cdaa51bbc2c4662de27e0695cd7e21ffe55c3bddb1fa1a399edec39c3466f2ea0bd5ce727daca2eb381213059419c2e8371b5a733 -OpenBLAS.v0.3.29+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/cd4afdd6f6ba06c7541e7124316802b3 -OpenBLAS.v0.3.29+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/55796fdb52e1ac92750dfc2233d3feb37b53920b12024be605bf6c7322153c4dbeb650f16d6def4f0fac685733a04a1c4cacb1fc4e562a27a00b4f44484a4715 -OpenBLAS.v0.3.29+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/8cd55ac7a7f0a7bda80b44171793718e -OpenBLAS.v0.3.29+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/728991a4c39d691abebac3ebbb2dbe093f3a4acd2d3aefb5c7c08bccf0dc1fd5aaa24de6367961d278d448b76a4ddacab36b7be15128f7ccec5049eab83828da -OpenBLAS.v0.3.29+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/c2dda93a61e02812831b6a6e33f7d2ca -OpenBLAS.v0.3.29+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/bd62e44f266b834c6dfab068841506a83eaf510eefbcf8896dfca36671321430293dc251885af108d94affc5b193919e0e29c965fef3ce6d994df37324aef013 -OpenBLAS.v0.3.29+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/8cbd64d2ce4e3944e702696839a4ad3a -OpenBLAS.v0.3.29+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/3621dfa5cf8bca62bb8f2a049acdc0ed4e02cb2b5585758e6e1173e61b3a5f0e1655a10f2feb2f0e70a098b00181d0b24dcd61e1205324d436b712f58e58df5d -OpenBLAS.v0.3.29+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/476f1ebfb93baad6fac778fa00c4f99e -OpenBLAS.v0.3.29+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/06aa18da572b0904e5d8ec0823626d0af02a29224aba98efd43d8fbf4636d2625ece9f88f9a86d2e493f016c106f2ae71422191afc16dda2b26bbc81eb09d901 -OpenBLAS.v0.3.29+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/8c55d04d9def74f6bc2cc0d03b764975 -OpenBLAS.v0.3.29+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/d6196a339a263d80c05b94596ec5acfeff6e3ce93fafee348a864f760aa1239aa59ee294cab29fd730dcf7974ac6dcb230433184be093612bad3bc3edc067649 -OpenBLAS.v0.3.29+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/8427f098a44457ba65b21a16439ee6c0 -OpenBLAS.v0.3.29+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/4855321b2a16d55e1c6e830e33d0a199286002798c0f33c7f594a55626b5a502df94c172de4fd0a38ab6ba92f384abbbc3ef06123c3115a3f290f50a9d43ae9d -OpenBLAS.v0.3.29+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/9d1636bb7500d9ba15ed703231f8def2 -OpenBLAS.v0.3.29+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/0b3530fd95e01d58b85157d7bb75e44ee7b2f0c5a912920ff0763f404e1ab28d16a624463f3f20241c7baea57e00fca3f896d6e0befb6a1c9e5ece4264b87e35 -openblas-8795fc7985635de1ecf674b87e2008a15097ffab.tar.gz/md5/095d293409140dd8eee500eb92372eb7 -openblas-8795fc7985635de1ecf674b87e2008a15097ffab.tar.gz/sha512/7b10d4c2bef68159e0a88fb6d4fd0ecca17b4c6394479e8f838f5078d9d5acef24c6bd44777d43c03859c952d4612d76b57aa0bff367b197920ea16eb3839144 +OpenBLAS.v0.3.34+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/3c0a21bae02e14d96ff5e17f2f7ca34d +OpenBLAS.v0.3.34+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/64b3650fa4d1596571794146937547ea632be11df81273c78346e82b0c18a800962120a1b6072c924bb385c52b8f5b65bb2c1a57cd4da4e1cf6458d007ba4da2 +OpenBLAS.v0.3.34+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/83420e4303cdc70a014cde3213757609 +OpenBLAS.v0.3.34+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/882e9676ce5fc75109deccb499450d32aa4a68e563ac49b91de33085486258e7f8bc5f6600e8e4a6faf8e86df8e02e576efffd0b9b255cc742168d0e8b663e75 +OpenBLAS.v0.3.34+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/023462d5b711fdc497398839ed445e62 +OpenBLAS.v0.3.34+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/826340cf4cfc610aabcde6869aab0b5091b8e723b5ea2559b2398e31e9b71eebae877928afcf6aba6ee54ccfba65e747b8c32f7a41bcd9d9e681925687095004 +OpenBLAS.v0.3.34+0.aarch64-unknown-freebsd-libgfortran5.tar.gz/md5/a91fae1616576d588c56860805c5a699 +OpenBLAS.v0.3.34+0.aarch64-unknown-freebsd-libgfortran5.tar.gz/sha512/5423f6d0f6e21ba6a2fe6f153a113d25592b547618c201b1fcb4cef40f8463afbf4b2c47daa097548d455f028658de0184dd2911b7d88ac6ea8999aa147428bc +OpenBLAS.v0.3.34+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/8e2f82df9e3d9c0873ae23fcb37576db +OpenBLAS.v0.3.34+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/397b28df3a2a5277033ce8afe5512d1f40559fcc1b01fc51a81047a53fd8588b3b1c064692adf9557dd50f1aeb3dc170df1670a3062da629a1f2980a6b82d8a0 +OpenBLAS.v0.3.34+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/1a9e27fd69aefc081d33bdc0afff5872 +OpenBLAS.v0.3.34+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/c51d78ae2748bc6d4acea0aa738e6961f82e4bf3694a1ea645dfd2c4ab4833de7e707d9e13703835a6754500b16f9934ddac4a409ab1ea885a602341eb5ca9ea +OpenBLAS.v0.3.34+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/8e2f82df9e3d9c0873ae23fcb37576db +OpenBLAS.v0.3.34+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/397b28df3a2a5277033ce8afe5512d1f40559fcc1b01fc51a81047a53fd8588b3b1c064692adf9557dd50f1aeb3dc170df1670a3062da629a1f2980a6b82d8a0 +OpenBLAS.v0.3.34+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/1a9e27fd69aefc081d33bdc0afff5872 +OpenBLAS.v0.3.34+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/c51d78ae2748bc6d4acea0aa738e6961f82e4bf3694a1ea645dfd2c4ab4833de7e707d9e13703835a6754500b16f9934ddac4a409ab1ea885a602341eb5ca9ea +OpenBLAS.v0.3.34+0.i686-linux-gnu-libgfortran5.tar.gz/md5/31084d05e8967e7bb61de8ad6a1be710 +OpenBLAS.v0.3.34+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/895e6d1472ec38c413a9109746e8a6d255f1575a0c5a7530f7692e5ed1fd8f32011d31d71ad79ef9adc8ee798e8ff628bbf2fdc9ede37a00fbe94e6bf97b1a1a +OpenBLAS.v0.3.34+0.i686-linux-musl-libgfortran5.tar.gz/md5/ca1d4c4c8fea235c4a2da6c9dc0d7b32 +OpenBLAS.v0.3.34+0.i686-linux-musl-libgfortran5.tar.gz/sha512/8f454a7e7c03dc18b9b5e66f08baaee2cb433dc9f2ef45f33cf0cfec38a32813cbfd8f07dfdf19df464870e2648bdde8fce1d4ca94f9ba18f4e9f81de731499f +OpenBLAS.v0.3.34+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/2106ac87f89f181df679726ddfd973a6 +OpenBLAS.v0.3.34+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/4852bd099ec0c2374173b0d7531a500023a731ed6dff2ee0a0bab6dea39067791c75b72d04595d6dc8c8531df52c8bc1731415b9354bd140b702349afd431cff +OpenBLAS.v0.3.34+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/500a962bb317ff18c361ca93a0f6c4da +OpenBLAS.v0.3.34+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/bb693b3b3743a7e9fc8e0e9f0f517e5f5f768f29f274dd2316a8594d6ab74de793051241777a644dc7b7decd9f03e5c576da3b3181c35b4f51b073b4cc4c065e +OpenBLAS.v0.3.34+0.riscv64-linux-gnu-libgfortran5.tar.gz/md5/f907d52481c2fc5acfb985dbe09f990b +OpenBLAS.v0.3.34+0.riscv64-linux-gnu-libgfortran5.tar.gz/sha512/64461626e97a27d090d9981127bfd0769fe7b5d181439839b892527b9f052c858a80739043f79bebe241c84133bd64135ff33c771058dc53fea7ad243efe9d5e +OpenBLAS.v0.3.34+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/1d7372af294ddc3f4f42b74c374006ec +OpenBLAS.v0.3.34+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/34910b72042ebecbe3eba104d9c0a0c3707a8f761b2362333fa35c5edbbaf9ed41fc55ecf359d2e78e24b44f2ce71760d870bc659db02772a93d18b291f2a847 +OpenBLAS.v0.3.34+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/da8216507e911094e5e94d6aa5bf4cf0 +OpenBLAS.v0.3.34+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/0aacb0c5f9befc6ecd625b4f1adadad0e3f51d19abd2d8a20b92dd3e06c0fc0831e458885332c0ec493aa904233699848d362a254fccb6e9c20cb26d4a2af36c +OpenBLAS.v0.3.34+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/4530e6003cc47645dd700526d74a99d0 +OpenBLAS.v0.3.34+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/8f3204924ce93e94d9c97f0c9051c8c202c42bf55822873b04822a91ad4eb707cafa4f7d245d2178f0e7f658df6cd08c2b600334c2b736772760486089c41404 +OpenBLAS.v0.3.34+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/b9fc55a43fa9b3bcb498adadcd545acd +OpenBLAS.v0.3.34+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/b571bafde1b083ec1aa770a1715ea7898252fa2cf53f2e6a5f39629891d3e28e96c9b5a5cd6bf461976b71f41e07f05ed2f4920c72ebf1768d97efaec081d1eb +OpenBLAS.v0.3.34+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/fdcf915c77dcc89ee7ebfe0d84e4778e +OpenBLAS.v0.3.34+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/825c81bd5f9de3a18ac5e19e40a8d905a3790a818c917a9a40e580d8f2f6165ce5d0f257e2cfb0ef3156c2f52bb66d390c94a056241e7b6286ceaa60c86901b0 +openblas-e0166008be8e466242aa76b2ff75ce3f0fbf574a.tar.gz/md5/18edf735ecbbb21e32becfeb4481e41b +openblas-e0166008be8e466242aa76b2ff75ce3f0fbf574a.tar.gz/sha512/f0075a97f5b70d135575f3d80b8b78b6754e8befa40e113133a1ef331d6db714f4b76e27f183b071f4d63457352b7dd3afe68b8e97857dc66ca78deb0584fca9 diff --git a/deps/checksums/openlibm b/deps/checksums/openlibm index cad61fd42cf94..e3cd48920e12a 100644 --- a/deps/checksums/openlibm +++ b/deps/checksums/openlibm @@ -1,38 +1,38 @@ -OpenLibm.v0.8.5+0.aarch64-apple-darwin.tar.gz/md5/5fcbd746e90712e396e76dc4e76724d0 -OpenLibm.v0.8.5+0.aarch64-apple-darwin.tar.gz/sha512/f4ac2bc38bdc723384b67119daa2974fb43da34b2e45cea2029ea48f92c84c4cad6dfb43521b09a1e89ddf8c5b8cc22a38fa4b78ba39ac7524fd6bd1ba897aa9 -OpenLibm.v0.8.5+0.aarch64-linux-gnu.tar.gz/md5/4d1b4cd566805b5179c5ecdd060da473 -OpenLibm.v0.8.5+0.aarch64-linux-gnu.tar.gz/sha512/a9fe1a3d2e3898c017eb8615b2f3dbb514995ff041ac964c931c99c60d8cfe4eab7563a9cd65058f42f83c812f33d998573a7c5cc56a2e3960a4657e459ed321 -OpenLibm.v0.8.5+0.aarch64-linux-musl.tar.gz/md5/413be59af62b3ce0ebafeca093e3179e -OpenLibm.v0.8.5+0.aarch64-linux-musl.tar.gz/sha512/7bd76373e047ba854066af61f1c56b2e3a4d28c266228d7b30f596eadbaec52b070548ae60d41840c425ad5d0829c6c0cdaf326f2f160ed7508877ab5ec1a4b1 -OpenLibm.v0.8.5+0.aarch64-unknown-freebsd.tar.gz/md5/80736f9022c695eb1198e0b591a8fa63 -OpenLibm.v0.8.5+0.aarch64-unknown-freebsd.tar.gz/sha512/c633644578265e7ccc259ceb0442457b8c09290b4861b66c86dd6be7b30c4e394e70728142798097d6fe3afcfb4d9d1bd7ef58513fe8eed5684a4fba51bf185a -OpenLibm.v0.8.5+0.armv6l-linux-gnueabihf.tar.gz/md5/8fe0900a318393a290907f016bc654c3 -OpenLibm.v0.8.5+0.armv6l-linux-gnueabihf.tar.gz/sha512/167100a2d46e68462ef9a66915ced881d6358f05337bd38f2f77176f41cfd5be37e3c5226dd5d7d59147bd3e1aa7fb0893c1c81e9516134d3ab663b5752c4969 -OpenLibm.v0.8.5+0.armv6l-linux-musleabihf.tar.gz/md5/e8566719387984604f19dc5f9354a783 -OpenLibm.v0.8.5+0.armv6l-linux-musleabihf.tar.gz/sha512/532dd2b764fa15f7a838fb14cccafd2d4fe8fa4a132ea8394479a719c7aee11442f1b8a18e5d4a26ca820fa696d9d2afc7f5ec63dd96fa3b6763cea72b7026c3 -OpenLibm.v0.8.5+0.armv7l-linux-gnueabihf.tar.gz/md5/8fe0900a318393a290907f016bc654c3 -OpenLibm.v0.8.5+0.armv7l-linux-gnueabihf.tar.gz/sha512/167100a2d46e68462ef9a66915ced881d6358f05337bd38f2f77176f41cfd5be37e3c5226dd5d7d59147bd3e1aa7fb0893c1c81e9516134d3ab663b5752c4969 -OpenLibm.v0.8.5+0.armv7l-linux-musleabihf.tar.gz/md5/e8566719387984604f19dc5f9354a783 -OpenLibm.v0.8.5+0.armv7l-linux-musleabihf.tar.gz/sha512/532dd2b764fa15f7a838fb14cccafd2d4fe8fa4a132ea8394479a719c7aee11442f1b8a18e5d4a26ca820fa696d9d2afc7f5ec63dd96fa3b6763cea72b7026c3 -OpenLibm.v0.8.5+0.i686-linux-gnu.tar.gz/md5/9580d34e69d6067427b9c33db631cfd3 -OpenLibm.v0.8.5+0.i686-linux-gnu.tar.gz/sha512/46934f82791f69ac5f5da0dab7dcc6e3e9a4577c3bb529e9c0519c38f140c7b54517c55ff3579cd4ed4df68f0863e006aa98e51873f1dab452ce9f853996429a -OpenLibm.v0.8.5+0.i686-linux-musl.tar.gz/md5/66bfc9611d04c5d609e7824cb076d24b -OpenLibm.v0.8.5+0.i686-linux-musl.tar.gz/sha512/1bda2395d44c22aba3d1aab2b08ae06f763d3755037d454aa73f8e8134289a1ab5d65862bbc5a17a7a6b9f2918eb87e926b21527ddc4471e2ea20d605ba14e2d -OpenLibm.v0.8.5+0.i686-w64-mingw32.tar.gz/md5/0e97311b2f08b57d79085635f01ccced -OpenLibm.v0.8.5+0.i686-w64-mingw32.tar.gz/sha512/ae061ea406c06969332af58ed6fdfce2825326d771d30274d90775a1709b0361b7ca1dc7e6b0b76b93e4dd7a81d1842510a2c835251ee0a0978d6c839d96070e -OpenLibm.v0.8.5+0.powerpc64le-linux-gnu.tar.gz/md5/8ecfff7db76eee29591a654871e88855 -OpenLibm.v0.8.5+0.powerpc64le-linux-gnu.tar.gz/sha512/af03993b162316dd581f6ba5d1c23bca4c26cb22356ab229f326c42e111acbdf7ef45c9ad05894fe2d68794a63670cf89888653f788192a38b9255ce4bc72e28 -OpenLibm.v0.8.5+0.riscv64-linux-gnu.tar.gz/md5/69e06de135940666791c984941e9c4ad -OpenLibm.v0.8.5+0.riscv64-linux-gnu.tar.gz/sha512/2ac84deb7eb80a6a6237eff6fe861fd2907b3c95d1a76366dea062f3f35228dbc67aa40bd982e646508b4ff7cb6ef029111e2c0325039e60679800d6c6886be5 -OpenLibm.v0.8.5+0.x86_64-apple-darwin.tar.gz/md5/bd671ab9fe01835cab3e42e7cfa790fb -OpenLibm.v0.8.5+0.x86_64-apple-darwin.tar.gz/sha512/8bf2e66df17effc1e8778453904ffc20127f785bf096873289e8fdd8b17069ca844faffbd9f7621b87a7cb0a0051037eb9402360f2a03cf8794fbac8f7719777 -OpenLibm.v0.8.5+0.x86_64-linux-gnu.tar.gz/md5/df7fab134fbce3b625e9a82376f23e79 -OpenLibm.v0.8.5+0.x86_64-linux-gnu.tar.gz/sha512/64d07434e0db79833f84a2225838456eb9532617d377a776b3a534a908b1673bc4f890903f95350e4045e05c29539d993a18ecadeb879761e279ec3947f74390 -OpenLibm.v0.8.5+0.x86_64-linux-musl.tar.gz/md5/ebef6bb7651d116b397e035f39adfb1b -OpenLibm.v0.8.5+0.x86_64-linux-musl.tar.gz/sha512/de9036073e5dba2721b4119ecbbd21a0c9f75b65aff9392b7e88e464da35b97135d62404477441d0dadd3a2f8d49f1082291b35bf4b626fb1096d36d401980bf -OpenLibm.v0.8.5+0.x86_64-unknown-freebsd.tar.gz/md5/1115497539f00a37af18aa6516d52268 -OpenLibm.v0.8.5+0.x86_64-unknown-freebsd.tar.gz/sha512/71a2c06d141b3671fd220f2d88d72e845848b6d2b08a7b3a6c4bb1d5cc27cc450e1e681647bb583e7ed6375d5a70748401e95e61dc95d7808f33a9aa06755337 -OpenLibm.v0.8.5+0.x86_64-w64-mingw32.tar.gz/md5/b6b5335f4c83f7ebf0f74cf753358f00 -OpenLibm.v0.8.5+0.x86_64-w64-mingw32.tar.gz/sha512/e8351ddda305b757f337bb7ea26c441968843b23861676f0bdd7bcf83bb3969af790d4112307d3204eb87fac044dda9be305f349700ebe9ba2bfe3d6df24fde8 -openlibm-db24332879c320606c37f77fea165e6ecb49153c.tar.gz/md5/2375dd448e77e59152442a4b33abda01 -openlibm-db24332879c320606c37f77fea165e6ecb49153c.tar.gz/sha512/36054e7051990d04913f054a0542e2e104273f61308e9a442c2dab3dd392d40c03f264fbeca93c4296218eed85dad71028989a225088254013d752f4407d57ef +OpenLibm.v0.8.8+0.aarch64-apple-darwin.tar.gz/md5/bb50c9ed9fb4d5999fa63a6969b14696 +OpenLibm.v0.8.8+0.aarch64-apple-darwin.tar.gz/sha512/4e91f3bdc87dbee7f102644d6ac7290ff8393b64c682f4d907b2a1320c0feb33df810db04d305bc15f46315504e5f4d1d5d6b7f00de477fdfa3ecb25996c5765 +OpenLibm.v0.8.8+0.aarch64-linux-gnu.tar.gz/md5/5f77190c8da55a909668438ffd487fb4 +OpenLibm.v0.8.8+0.aarch64-linux-gnu.tar.gz/sha512/7013de04380e55d8057a1ba0ea6a57a4ef9d596d13b1c1c5df38885aba34892422a4764f2e11e4fa8c217904964020b2e5e960a7192bcbb56e8b0df67eb77092 +OpenLibm.v0.8.8+0.aarch64-linux-musl.tar.gz/md5/3970c75df912b17e64e9e909292518bf +OpenLibm.v0.8.8+0.aarch64-linux-musl.tar.gz/sha512/752e36b6abcb9685ec1bc7f185d0e6addd087482624787c777eb0c0968888f4f9477628a53a89529d1a4f96c96bbc2500f071d6088654cbc66449693e2e0ef2a +OpenLibm.v0.8.8+0.aarch64-unknown-freebsd.tar.gz/md5/31c38ceaa924815ccdced233f36c4f9f +OpenLibm.v0.8.8+0.aarch64-unknown-freebsd.tar.gz/sha512/75df16f4c6374efe2e685faec57427b9341edc3b23b42ff41a284cf4b382a5a7ae961808101083f69d4cca643062fa4d6a0b40c334d510da553513a6f288bb48 +OpenLibm.v0.8.8+0.armv6l-linux-gnueabihf.tar.gz/md5/65feb2118687a16f4a40f47f3ff66710 +OpenLibm.v0.8.8+0.armv6l-linux-gnueabihf.tar.gz/sha512/2b1b0eceb5378129fab2e213f044c7bc1e11807e2d7ab532c20c27aeb17771a55aa19e3630c094c96b5254cbdc3d1f4e00d2f1ffa83a63951aee0e01a1505285 +OpenLibm.v0.8.8+0.armv6l-linux-musleabihf.tar.gz/md5/b5871377182bf964f5d5ee6fc525a02c +OpenLibm.v0.8.8+0.armv6l-linux-musleabihf.tar.gz/sha512/1ce362934a7e6aa6e12c206dbcaf5227bb3ad55636e0fb429f4ee87d2b7cab84d0df44d3fae97ce42f2c99e959f69924a28d5941d22f225b4fa0bdea21e0bce7 +OpenLibm.v0.8.8+0.armv7l-linux-gnueabihf.tar.gz/md5/65feb2118687a16f4a40f47f3ff66710 +OpenLibm.v0.8.8+0.armv7l-linux-gnueabihf.tar.gz/sha512/2b1b0eceb5378129fab2e213f044c7bc1e11807e2d7ab532c20c27aeb17771a55aa19e3630c094c96b5254cbdc3d1f4e00d2f1ffa83a63951aee0e01a1505285 +OpenLibm.v0.8.8+0.armv7l-linux-musleabihf.tar.gz/md5/b5871377182bf964f5d5ee6fc525a02c +OpenLibm.v0.8.8+0.armv7l-linux-musleabihf.tar.gz/sha512/1ce362934a7e6aa6e12c206dbcaf5227bb3ad55636e0fb429f4ee87d2b7cab84d0df44d3fae97ce42f2c99e959f69924a28d5941d22f225b4fa0bdea21e0bce7 +OpenLibm.v0.8.8+0.i686-linux-gnu.tar.gz/md5/035348e0febfcb7e0c81b74a1b4f2719 +OpenLibm.v0.8.8+0.i686-linux-gnu.tar.gz/sha512/678044471913b2ccc2aba43d31a2a8b4978911570025bc3b461006a989d221a42794d718ff5dc109ef724530a2f32eff9e8fdb2748f569da923c2cf7dc44ce0c +OpenLibm.v0.8.8+0.i686-linux-musl.tar.gz/md5/032bc47d090a5184580efebb8638be05 +OpenLibm.v0.8.8+0.i686-linux-musl.tar.gz/sha512/1fb47e692cb87fca29f5708212720cf829f2372e0e31c2555c97a657ab25aac16afebec46c771a081887056f8589cd1ed6a07efdc8647c3643e3eb659f3060dc +OpenLibm.v0.8.8+0.i686-w64-mingw32.tar.gz/md5/446e7b2d3dcbbf0325650ed021127bca +OpenLibm.v0.8.8+0.i686-w64-mingw32.tar.gz/sha512/f6ee66f2607ed2488598306731a51a821ab868db002dac548bbe3a796bd0f2e508220bc93aaa573d4fd151c2afec9c4a2673de7e229b894763b2d13c0295c17b +OpenLibm.v0.8.8+0.powerpc64le-linux-gnu.tar.gz/md5/e17ab89fca960909910835dfa44c5af4 +OpenLibm.v0.8.8+0.powerpc64le-linux-gnu.tar.gz/sha512/cea70f6fdfe64ce92a18aa5f288a1f6c6b0d3ba8bd0c289a982247b1cd234e255c10cfd775d264ce420188196d5d3a6e47673734a58eb766903ed95478eca665 +OpenLibm.v0.8.8+0.riscv64-linux-gnu.tar.gz/md5/93ef1e09412792a4d36601df648e7be8 +OpenLibm.v0.8.8+0.riscv64-linux-gnu.tar.gz/sha512/31d391c7808f43ec0e3d5ec592c8f22e2423f33cc3f5f6d487815f26c97b07f2cb26d60033adab26cda81c18105480db2e70b63e8866492eb23110729d94612e +OpenLibm.v0.8.8+0.x86_64-apple-darwin.tar.gz/md5/5293adfc5fb82b60755465deb31c40ea +OpenLibm.v0.8.8+0.x86_64-apple-darwin.tar.gz/sha512/8c0385ff4307bdb7705be731faebf0fae7137d62385b99b4d0dbe4daff1a399b36bd55c77f7bb8400a291e96d13f099e1a42dc4042fd4caad5a1acb9af6cb533 +OpenLibm.v0.8.8+0.x86_64-linux-gnu.tar.gz/md5/eb91e714666acdaba71dfbadb0b12f71 +OpenLibm.v0.8.8+0.x86_64-linux-gnu.tar.gz/sha512/3dd856b8a95dd5b0c08f03179929b9168f6f37f1f6f65c637e7c88c2a265e66a77cf93e3b1961371ece19c7812d8b3892ed955a3060e8ad799a14142369880f5 +OpenLibm.v0.8.8+0.x86_64-linux-musl.tar.gz/md5/9b9c8c22d9838287b453016cc0a3f6a3 +OpenLibm.v0.8.8+0.x86_64-linux-musl.tar.gz/sha512/fe15a56d70f0d05ec8f6f9ffe1ad88f6ad7ed687b58129bca485ba98cbc52ee21994c0f98e8250b81354f7dd9ca3f5d6e182d538fd07132c3a61a7c42f69f463 +OpenLibm.v0.8.8+0.x86_64-unknown-freebsd.tar.gz/md5/feb6e3f15b7b753b804b01228d71c165 +OpenLibm.v0.8.8+0.x86_64-unknown-freebsd.tar.gz/sha512/3172fe2c33990a6f4c9d168b9e21c4c7e1a8b2f868adf2914e9b027bcabc4fccb591d0e837fa731a53ded1edc456a412f5bf322d15f4526e3ff26777468b8479 +OpenLibm.v0.8.8+0.x86_64-w64-mingw32.tar.gz/md5/fd294a2b1e5befe55371c2cf3ff34f4a +OpenLibm.v0.8.8+0.x86_64-w64-mingw32.tar.gz/sha512/74408b2aae0d746975b564c199f21f3ae43e6a5aeb192c6ba07ccf99ad65b3f56de994d815472bd84a8bb84a79249ad029d15538bd80bd2732f8d835a5783db3 +openlibm-5fe399749f9276eaa0b8403e507470da05cbbb3f.tar.gz/md5/ab2e73e6c03adf532c3b0dfb430117ae +openlibm-5fe399749f9276eaa0b8403e507470da05cbbb3f.tar.gz/sha512/0ab35df3acf63805f587d47495d3d4f982952766bd0ec20e7b1dc78885fdad73d1dab78d95c3876322922ec7606771caf195421d74d61e5e94b01a46b1c0f668 diff --git a/deps/checksums/openssl b/deps/checksums/openssl index cca21ccd8c5a5..6fd14baa17ad7 100644 --- a/deps/checksums/openssl +++ b/deps/checksums/openssl @@ -1,38 +1,38 @@ -OpenSSL.v3.5.1+0.aarch64-apple-darwin.tar.gz/md5/b19522093c25c50685002ad48933a835 -OpenSSL.v3.5.1+0.aarch64-apple-darwin.tar.gz/sha512/afa6363f8396deac5131f5efbe92d5b60f4d6982d279d63b5847e80ac4717d89e32edcc9bc7a5fbaab95e03908a6e3e9b386a3931effb0a7163b947b38ed2cd5 -OpenSSL.v3.5.1+0.aarch64-linux-gnu.tar.gz/md5/60af2cb22b7d5f4fddd94bd196f86ad2 -OpenSSL.v3.5.1+0.aarch64-linux-gnu.tar.gz/sha512/3d384f5da4be3af848b47f48f2438dbda8cdb228b8569d01bd4fbd6feea9f494ecafd3cab6e7b0bbed596746aa2614826971133a2b6dea02836c0904ce870760 -OpenSSL.v3.5.1+0.aarch64-linux-musl.tar.gz/md5/5f96641ec5256a547e03cd6028892a50 -OpenSSL.v3.5.1+0.aarch64-linux-musl.tar.gz/sha512/668c08f2a08f9d65b2e5c1ca4db8f74932995d0fa97c4737a2d9cedb3548482f85fddd478fad37325e2d48f76259fd8f7e003d31fc2a9ecfdb88c4748f90e1d6 -OpenSSL.v3.5.1+0.aarch64-unknown-freebsd.tar.gz/md5/377bd17ae448f4394b3100b290602f35 -OpenSSL.v3.5.1+0.aarch64-unknown-freebsd.tar.gz/sha512/f989a15062b47f059086d4dc8fd53af00717ca622ef8c475a11f6e62a29d8ec4a80159d93a683e8729da66c4bda4c46b7647754dc996ed2ff5635cbbdaf702aa -OpenSSL.v3.5.1+0.armv6l-linux-gnueabihf.tar.gz/md5/83bcc0b545bea092a0a5de9e64cbcbf1 -OpenSSL.v3.5.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/c589119945ff6c1341bc229a2e61096c630288f7d483ea9538202915f8ee1a9e26cd53efc479f1e92a83a75aa6c7453ceba446832678ffed340a4bec13fefbfc -OpenSSL.v3.5.1+0.armv6l-linux-musleabihf.tar.gz/md5/3b2e34e506960259dbb40a36fed26ffe -OpenSSL.v3.5.1+0.armv6l-linux-musleabihf.tar.gz/sha512/735f22fe1202818f64f369471292bb8fdf8cf1f3395d01e70ddf8f65efc5430aec54a63fe950e52625f2c8a5dbd59ed0175f088144af8d99c7db1967ed0e5aeb -OpenSSL.v3.5.1+0.armv7l-linux-gnueabihf.tar.gz/md5/aedb37bde1b3fad428987745dc1dd095 -OpenSSL.v3.5.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/1919823df3c0de01c59a5f9cf42b912a1d56fe7de903c4e7cbcd54603760783a99fe34cd1c108e954d5fe41502c1791b803d67742d70abae64d316c3656b7280 -OpenSSL.v3.5.1+0.armv7l-linux-musleabihf.tar.gz/md5/573a752ca28fd62644208a4c0b32eaa4 -OpenSSL.v3.5.1+0.armv7l-linux-musleabihf.tar.gz/sha512/3385b170973a9e50919981e66e591077479ae7561368941f018aca6f42c86b3d01aa1d9896d4d5f6deb69760fa42f535f6aaa076b75a15f207328ba6f0a32660 -OpenSSL.v3.5.1+0.i686-linux-gnu.tar.gz/md5/fcdb2ab108900c412abf154a6cbd46e7 -OpenSSL.v3.5.1+0.i686-linux-gnu.tar.gz/sha512/b558e6c23809f702a7388dba7031a9df577e1a2eb1ca86b7cf0dcd9809973dff1c9b56d4a09c315b17dcc9860e7f89604513a2d022117d9145f2bc81befa094b -OpenSSL.v3.5.1+0.i686-linux-musl.tar.gz/md5/0192e44a52d9518d712db58019ace62c -OpenSSL.v3.5.1+0.i686-linux-musl.tar.gz/sha512/fe9740850e6eb32eb539d16303b39d9ad1d3e8cc2e5a742304013890a0e1e8af93e131a5393c3c817b5723680425947d6954455dd715cc83589fd097c517b5c2 -OpenSSL.v3.5.1+0.i686-w64-mingw32.tar.gz/md5/51b5546301f8c474bcc9c97b625df2c1 -OpenSSL.v3.5.1+0.i686-w64-mingw32.tar.gz/sha512/47874ce005e6944f3a4d482f3edf44bcaa3724338230d68fff22c484c0620fe857a11bdc515ef9154522a2884f64bacadfd1fddb1430a45c7722a6a4799107f6 -OpenSSL.v3.5.1+0.powerpc64le-linux-gnu.tar.gz/md5/1aeaa0660006b4b8c13cd1cb45b2acfc -OpenSSL.v3.5.1+0.powerpc64le-linux-gnu.tar.gz/sha512/dff025feb0d1ae96a7c33f1beff5e6f76d5a85833314470f59d75bf791e90363145ae79f3ed82c5c40e36416b75fa9deb5807911c8133fe11f31b4705737f0bc -OpenSSL.v3.5.1+0.riscv64-linux-gnu.tar.gz/md5/160065eb12650c504fd40a25e4bae2ba -OpenSSL.v3.5.1+0.riscv64-linux-gnu.tar.gz/sha512/68951cf98c4eb778d372e239d14497405e6161461a623135a5254c3fd65bc3a12fe3df1ecce88909cb05dc29104b5b18caafea115799c5abf2505afe75be3207 -OpenSSL.v3.5.1+0.x86_64-apple-darwin.tar.gz/md5/7e5903d1d051de70a93a9b801ce274db -OpenSSL.v3.5.1+0.x86_64-apple-darwin.tar.gz/sha512/729b33cc208b8646394bcf0029a276ad41cf2e9d44737dbc1e15dca794cc55a52e2b782b0c72ef57b5580b84a93b25133788424f1b16ef2b784d409bca598150 -OpenSSL.v3.5.1+0.x86_64-linux-gnu.tar.gz/md5/9cee745524f41dc21af2f460ac2f1293 -OpenSSL.v3.5.1+0.x86_64-linux-gnu.tar.gz/sha512/6949c7f19b7919073542af903019ec0d8fd5172450141a3f69f0c325f0c5cc19618f1959b96380719538c5a1a5a388165a0db8e6eab041d0a5874a627820212b -OpenSSL.v3.5.1+0.x86_64-linux-musl.tar.gz/md5/e55565c84e5cff597ea490e02c559d1a -OpenSSL.v3.5.1+0.x86_64-linux-musl.tar.gz/sha512/c27930401c72b6be94ba7717f7b3be0025b09138e146f3d2a761e805308ee51f4ca871459941448e102f64b0e3c1aa479f39ee77f3234321890fa7105418ed44 -OpenSSL.v3.5.1+0.x86_64-unknown-freebsd.tar.gz/md5/276d97e2d573977727ca8d2113335fac -OpenSSL.v3.5.1+0.x86_64-unknown-freebsd.tar.gz/sha512/71f72a82c590542928660f004f38f84ea335b303e7da53578d712994fff9e84a3b69fc836c08d03dd8310d17c9edc63e5f6975e6d26e67124124763633ab1b59 -OpenSSL.v3.5.1+0.x86_64-w64-mingw32.tar.gz/md5/cebdbbf8a8a301e332d75c46dcdb1af0 -OpenSSL.v3.5.1+0.x86_64-w64-mingw32.tar.gz/sha512/0e141d7317ac8f5c43d1ecc9d161b00345f99145af785d7554f750b5787ea69969f785e7a1305059137271d26450835c25dd126bf9e5aef2cdf7dcbbdebb6911 -openssl-3.5.1.tar.gz/md5/562a4e8d14ee5272f677a754b9c1ca5c -openssl-3.5.1.tar.gz/sha512/0fa152ae59ab5ea066319de039dfb1d24cbb247172d7512feb5dd920db3740f219d76b0195ea562f84fe5eae36c23772302eddfbb3509df13761452b4dafb9d3 +OpenSSL.v3.5.8+0.aarch64-apple-darwin.tar.gz/md5/d0fb486c759bcc080e1c76a30d5ff2fb +OpenSSL.v3.5.8+0.aarch64-apple-darwin.tar.gz/sha512/83658bcfc768f34a3ab543dcfb71fd814fbf6b1040e74802f3cb69e6fce5d20d39ac3a80ab5d41e234cbac07399de7a95d7c3f7d8dc4ff909339bdb6974e526c +OpenSSL.v3.5.8+0.aarch64-linux-gnu.tar.gz/md5/e638569b81bf06b7b0f91c6ae9ccae7e +OpenSSL.v3.5.8+0.aarch64-linux-gnu.tar.gz/sha512/51de1c0d64dc87622768d00baf27df3b698e2cba55889af208086dab174e46a18716a03b3a2aebfd6d465b62aae47b533d7e7a1fefead0f50461dbf6f7e5ea28 +OpenSSL.v3.5.8+0.aarch64-linux-musl.tar.gz/md5/db1488815119516a92e15f7440cdc47b +OpenSSL.v3.5.8+0.aarch64-linux-musl.tar.gz/sha512/c50bb403c75466a974c2e8ddbab5d4d5438fe60673c23c8143080da3d5da0e64be68eb84908729ebace196ee147f9ad01d094c3f45498b198b9140f4f4ee89ea +OpenSSL.v3.5.8+0.aarch64-unknown-freebsd.tar.gz/md5/ced518eb5bc286fe204b9e7d173ea3fd +OpenSSL.v3.5.8+0.aarch64-unknown-freebsd.tar.gz/sha512/8046455f54ddeb47c1276ae945c7d8b886399515e249aef75751068657a528224e9661ad2ab24586a10b513c9f4851133c40b1b3d8015bfea98f39fc356f51d0 +OpenSSL.v3.5.8+0.armv6l-linux-gnueabihf.tar.gz/md5/0259df03d7b953b322462817966fe1cb +OpenSSL.v3.5.8+0.armv6l-linux-gnueabihf.tar.gz/sha512/2d7b04ef16064eb5e694ce488e216d98cfc902b0b589315a635ca1f8ab6553e8a21abfc0c1c9c80d97ca57b0ec499372f9261803cce1a15d9ab668a5c10f2b70 +OpenSSL.v3.5.8+0.armv6l-linux-musleabihf.tar.gz/md5/e2dec980af240ab8a8f6631e1152c243 +OpenSSL.v3.5.8+0.armv6l-linux-musleabihf.tar.gz/sha512/373fbd6fd901a5cc0bd6111aa12e27414a44b4d2b4000c5b1a2222bde55555ea7f8a09c227b65c56911f5d1e51ab8ec646acd63972d5072179d66545caa034c4 +OpenSSL.v3.5.8+0.armv7l-linux-gnueabihf.tar.gz/md5/e0c0b5f39b720200ed86f6e7fcbe3710 +OpenSSL.v3.5.8+0.armv7l-linux-gnueabihf.tar.gz/sha512/b6cead655631f53b9e87ea26bb2789d3c42e8552717daa605cda1fecfd5044e7c44bf4fdc6d9342db9e4021643bf516e95ab55c4e2e6005c2dc47907f90f0946 +OpenSSL.v3.5.8+0.armv7l-linux-musleabihf.tar.gz/md5/17655d517e376e2e7d57385e23b187d6 +OpenSSL.v3.5.8+0.armv7l-linux-musleabihf.tar.gz/sha512/27db1f9eae629599ee0bc60ec0bb4e6de530a9ba1905ff533cc32337b9c4f44f91c033ad609efd81758712088198132dfea0147d9a607adfb30ce64b2d4cea8d +OpenSSL.v3.5.8+0.i686-linux-gnu.tar.gz/md5/fa7c9139ffdae92384f953d8db7dfa0e +OpenSSL.v3.5.8+0.i686-linux-gnu.tar.gz/sha512/88420b64a32d3df61dce54e3c17372ea224c172539d64a1971c8a24a37efdf0056a75600f3a5d9acd0d83b681f75e3787d7f05aec285e82ec0e36fba113eb0d4 +OpenSSL.v3.5.8+0.i686-linux-musl.tar.gz/md5/63c2f4724649738170086cab3f652997 +OpenSSL.v3.5.8+0.i686-linux-musl.tar.gz/sha512/683d8584922157fa2199cb4108988a7c508817048e36567551ee764ea4c19d5e6a42755cf97d632c5ee23f443beed5d385a07965f1e6dd60e0cb3233ec91a6a0 +OpenSSL.v3.5.8+0.i686-w64-mingw32.tar.gz/md5/ab95a3aba8a820d49462657c670d6842 +OpenSSL.v3.5.8+0.i686-w64-mingw32.tar.gz/sha512/9ca6b48dfe2265898abff143d8c21d855a93553c84be35912124b7e315aab7aa9383e146547625d4502c4f30161b3c9e6c74f629913b49f99d47917a9a02e392 +OpenSSL.v3.5.8+0.powerpc64le-linux-gnu.tar.gz/md5/7acd3297f1d7dc9345d1e970d4fa832d +OpenSSL.v3.5.8+0.powerpc64le-linux-gnu.tar.gz/sha512/b37d376baa7f28124751e9cd4de487f54bc19edd87daf85bc5f815e8c952d2c647ba51b2d71477ffabdd5a232bed344e7670dd3d84dd97ab349ebaec7293225d +OpenSSL.v3.5.8+0.riscv64-linux-gnu.tar.gz/md5/ce19793b40eb23f7285e60fd60944be4 +OpenSSL.v3.5.8+0.riscv64-linux-gnu.tar.gz/sha512/02a2cee7521428efd421ece2edd8793fa0f7257ff9f5c9865e83f07ecebb7ad70794ff7d916a984e21ef2f59ee7f262f9d4cca208ff5ed36d3db7ad8e905036c +OpenSSL.v3.5.8+0.x86_64-apple-darwin.tar.gz/md5/df2b6a893a5c2ebd95ab04701bb82aa6 +OpenSSL.v3.5.8+0.x86_64-apple-darwin.tar.gz/sha512/65b3adb272324b5d184a24381919afe72a3556f8ba3acf595711451e946162e6473e84a8995e8341bd7fb4f4569dc7144369f2855ad8e1f99ca2ffdf4979b409 +OpenSSL.v3.5.8+0.x86_64-linux-gnu.tar.gz/md5/1d42f490737661f346a339519955fa64 +OpenSSL.v3.5.8+0.x86_64-linux-gnu.tar.gz/sha512/3f64e548e8b03cc3038e84f09e9b1faf00a70bbed37c91e688024502b12ca5e60ec0a8a91012177f51b4d8a821af9e9e7e1ba8d1119e9d8d8aafba4a0bee2ff6 +OpenSSL.v3.5.8+0.x86_64-linux-musl.tar.gz/md5/683728d63dce0368a9883598534ca086 +OpenSSL.v3.5.8+0.x86_64-linux-musl.tar.gz/sha512/80d79db9a4a3196494482dba6bedd6f01a5ebdf8f4e29acba042c206c4a24cf18849eeddaf2d8737635edaa6467c6c14dc3c3796eaa17f2996334e4498a091f4 +OpenSSL.v3.5.8+0.x86_64-unknown-freebsd.tar.gz/md5/a46eded6ac0b0416440ce8976d4c5342 +OpenSSL.v3.5.8+0.x86_64-unknown-freebsd.tar.gz/sha512/1e47b19561a7579eb396ffb786e8f72f368d01c192c3dcdf48874f2118f8d42fb8c6b18decb2a5b82a9243b626defe8466f964ca18d681ef66b7a5363229edca +OpenSSL.v3.5.8+0.x86_64-w64-mingw32.tar.gz/md5/fb2381d75118dc6a360d3b4579688bd9 +OpenSSL.v3.5.8+0.x86_64-w64-mingw32.tar.gz/sha512/ef7aa5379e7a1823e6990a78b74d90d7fea8cbb68e9aaac8b9a6fc6dea891fe5836c5798f7fee6279256af605564f491420132e9de73e173403d119fa9ab0c09 +openssl-3.5.8.tar.gz/md5/88975626b10916f46c14ce74d69482d0 +openssl-3.5.8.tar.gz/sha512/62a1dbed0fad75245b332e41b85a1f7c2379189525e7628a7cf68947d115e90a47f179e3f87d27641e5b2d357c357292179fc0e64eccecdebc81c083f7a8ebe4 diff --git a/deps/checksums/p7zip b/deps/checksums/p7zip index 6850967ace1b5..6d48c29f8209b 100644 --- a/deps/checksums/p7zip +++ b/deps/checksums/p7zip @@ -1,38 +1,38 @@ -p7zip-17.05.tar.gz/md5/de921a08f37242a8eed8e4a758fbcb58 -p7zip-17.05.tar.gz/sha512/97a7cfd15287998eb049c320548477be496c4ddf6b45c833c42adca4ab88719b07a442ae2e71cf2dc3b30a0777a3acab0a1a30f01fd85bacffa3fa9bd22c3f7d -p7zip.v17.5.0+2.aarch64-apple-darwin.tar.gz/md5/2a254e251901b3d1ddfd7aff23a6e5eb -p7zip.v17.5.0+2.aarch64-apple-darwin.tar.gz/sha512/8efb9a2c9bcab388e523adba3dc0b876e8ae34e2440c3eee01fd780eb87c8619c7a7bbdc46d703ccefff6aa6ad64c4e4b45b723136ab1f6fd6de4f52e75ebbbf -p7zip.v17.5.0+2.aarch64-linux-gnu.tar.gz/md5/bb1f3773fd409dbb91a10f7d9d2e99b5 -p7zip.v17.5.0+2.aarch64-linux-gnu.tar.gz/sha512/e95ccc342be644570d218d25403b91a7db9ee983fbf8cce3deff453355d68d426f9301eaac865a98691025b596b8cd77ebebf6184c0eaf8b2f294bc6763b9a4b -p7zip.v17.5.0+2.aarch64-linux-musl.tar.gz/md5/3fac518a6a70412294d71ca510958cf2 -p7zip.v17.5.0+2.aarch64-linux-musl.tar.gz/sha512/fc127790739bf8a8b918b2e83753d86f5e79ee8706bde4cc79d74d9f7d846aae99a109da4b2b3cc92ccedc1eef4d52a555a65a95f588e173e0fecc11f2ca21e6 -p7zip.v17.5.0+2.aarch64-unknown-freebsd.tar.gz/md5/4190f8d7d42572b3fdab0fa382417d43 -p7zip.v17.5.0+2.aarch64-unknown-freebsd.tar.gz/sha512/5b0cb08374b8561873f76cb2b8bcbb8de1ff4c91bde23222cc1b650c6ea2fff265e48b6190551ed136324a47d25e1d357a754295b674e74b4628b20223ad067d -p7zip.v17.5.0+2.armv6l-linux-gnueabihf.tar.gz/md5/355410848192de3b02d12fd663867f4b -p7zip.v17.5.0+2.armv6l-linux-gnueabihf.tar.gz/sha512/8f103b41e755d157d70dacca89a0ef4610bea109686b4005e8edd5f79ed2e6419c00c2625d0ab90e6e33fa389e670490d8de263c0bdae952cc34cbbf440e275f -p7zip.v17.5.0+2.armv6l-linux-musleabihf.tar.gz/md5/34363b227306fce34a728af54b71064f -p7zip.v17.5.0+2.armv6l-linux-musleabihf.tar.gz/sha512/8dd7b37ce6223c9fedcaa999eb806eb6dec8c4a3133d3c07e2456cb8543b8e4f5b881c1bff2d2e25f19b1312b18673e9013aeff87d6a274eec6c451b1ba0d6b9 -p7zip.v17.5.0+2.armv7l-linux-gnueabihf.tar.gz/md5/dbb1fc0cf3bea674442ff8cc932a94cd -p7zip.v17.5.0+2.armv7l-linux-gnueabihf.tar.gz/sha512/c4d71d905fa420391417786ed206a0c334475dd0df8baa1fc3f6560ce548db11805003d0d0b35bb622fe818c761f2b0abe0796d1cbfce2a922da69e697f056a2 -p7zip.v17.5.0+2.armv7l-linux-musleabihf.tar.gz/md5/d188b5dd453faedb616ba9c48fdeab6b -p7zip.v17.5.0+2.armv7l-linux-musleabihf.tar.gz/sha512/ea30a775370502ca9e271b87cbda528d0c51d63ce0df41883d4dbc1527a32f251d797f3692fcf9b883b5fbaaad80515b971a8f8fe09ba102978b19a0ecb58528 -p7zip.v17.5.0+2.i686-linux-gnu.tar.gz/md5/dc02bdde045a0b6b22cf14d6960e63ed -p7zip.v17.5.0+2.i686-linux-gnu.tar.gz/sha512/d2d0dd14a5fc1163fea2276e0925bfa8d075d5dba1d8018e4e3160977d3b09642b2e521d8e57d049abaf0e2ea391a846f0b0136b3c59e8b476c8c52ac5210447 -p7zip.v17.5.0+2.i686-linux-musl.tar.gz/md5/0b8658147938a8ec109ee2b3b0a0665f -p7zip.v17.5.0+2.i686-linux-musl.tar.gz/sha512/411b2950f5928c537b87ba0651c09c08e57afed765db9fee89eda8b12939ef0da94c8ba38c0a24ba46b4513a0e4cca798eb09f2b20a011099ed3cf14455dd19e -p7zip.v17.5.0+2.i686-w64-mingw32.tar.gz/md5/98bdd8767c77a35f71303ff490a3d363 -p7zip.v17.5.0+2.i686-w64-mingw32.tar.gz/sha512/14f08071af74297df8bfe1d9f7efa3c0212e62ace573848f17b729e4c36dc3861110f3c5cc9315364c318e5b040736443a24492e86d76161993653a309996eb3 -p7zip.v17.5.0+2.powerpc64le-linux-gnu.tar.gz/md5/b18c917b9852898a9b9d6d24bcc6863e -p7zip.v17.5.0+2.powerpc64le-linux-gnu.tar.gz/sha512/0148dc8a0bc9c95212d7f8e2f92ee24e968eb7290fe72c7ae02e286bf5c05dd6b1f10b32350a7ff37777ed5a8cc21f3303f464620f3394c7a4690ae98bf77299 -p7zip.v17.5.0+2.riscv64-linux-gnu.tar.gz/md5/8d5f804091c2d21b2c35121d40d1024f -p7zip.v17.5.0+2.riscv64-linux-gnu.tar.gz/sha512/68042f32b8b9f8d422dc0390efa2502d4a1a816daf4adf1133128f9366ec93ee1c1dda699844c0c3c3649a6b55a16312bd6b8fe4aedd6780e6faf11509932a9a -p7zip.v17.5.0+2.x86_64-apple-darwin.tar.gz/md5/da31752a2556644d39e48649bb0111de -p7zip.v17.5.0+2.x86_64-apple-darwin.tar.gz/sha512/0695ad111263d2fadfdf9a46ce7ee80def0bf60db7d1c2585ed2af6fc945fb169311a9f1ffc6f95fb43b0b03694d2d1be9136d3d78ba2ef2b19228987883a385 -p7zip.v17.5.0+2.x86_64-linux-gnu.tar.gz/md5/2fb55d86e4eaccb0488bd637d088b996 -p7zip.v17.5.0+2.x86_64-linux-gnu.tar.gz/sha512/38ac355157d59c09f308fc29964d0e9c1466c9633efd8d3c6ff3c738abce2af45ebc6b92a29f56d5e7baa4871f9f39b14ecfcbedd4e2f4ca7c0fe6627c6b13e7 -p7zip.v17.5.0+2.x86_64-linux-musl.tar.gz/md5/f0bd567a851d2dd9d306552ffafbca3a -p7zip.v17.5.0+2.x86_64-linux-musl.tar.gz/sha512/e60047a6e7e3496cb6658f87c8c88676f399cd9f3d0d7daa880b6be09cd5525f7f22776896f1375722b47555514ff8c018f02ce800ec3fd0ed922e16e8a6d657 -p7zip.v17.5.0+2.x86_64-unknown-freebsd.tar.gz/md5/d37bd26e39a3ec84f262636f70624341 -p7zip.v17.5.0+2.x86_64-unknown-freebsd.tar.gz/sha512/0604a880c19f9d72d5828edd75be641625c29f230b3a5e7d70ec3812c014c96b76ee7b45e0e80f49be63f109a48700e75d1e5be01b5ae7b46d42dafda9885e8c -p7zip.v17.5.0+2.x86_64-w64-mingw32.tar.gz/md5/f02c7b2481dee880b096340a8735350f -p7zip.v17.5.0+2.x86_64-w64-mingw32.tar.gz/sha512/08b717c1b072d1309f6af8973eb09b1a482abb7ae7d01fba79873d4310a7c11292e2e8779029f99cc60627ed0d064224bc87782e587c520f970b840b7b838052 +7z2602-src.tar.xz/md5/3c022d210622d70523aad67faea21759 +7z2602-src.tar.xz/sha512/85449360904b295cb47a86597cca5e6623f4119e4e5ad3491d67bea8cd416bad68c4bfb69b9ec521ccee6e431a080d9e8aa8f5eecb69d8305c66973048e56a16 +p7zip.v17.8.2+0.aarch64-apple-darwin.tar.gz/md5/f545fb605fd071a1eb0894d81e234351 +p7zip.v17.8.2+0.aarch64-apple-darwin.tar.gz/sha512/8c44dcf6cb76b606870ed4b0def87ebeba2b5e20100c55c28e61abcb4a198203e9648b60a4bc0b10f1f0378185d7413a7634d519873e68cdd839c33d9e67176f +p7zip.v17.8.2+0.aarch64-linux-gnu.tar.gz/md5/5fe3171e8a8e86f3c40c6eeae0f87c59 +p7zip.v17.8.2+0.aarch64-linux-gnu.tar.gz/sha512/b7562ba24b31432124c5964c77e68ed4e1521b64baa6611fd3e2cd1fbb00cfd000b0350b0e25dd50cc95fd7f64c1b9454a7963bc44f918ef322dfd56e4eb73b6 +p7zip.v17.8.2+0.aarch64-linux-musl.tar.gz/md5/1f5648ff9419f7c132a3e172d32d9d1a +p7zip.v17.8.2+0.aarch64-linux-musl.tar.gz/sha512/49f108fa4b1fae04df80d0000ef89bea0af8b6ded421ff81b69d491620b3d5178a0fc19de69414476fe244f658bf9ca8d0d73afb4bf5c322c94ce761390eef51 +p7zip.v17.8.2+0.aarch64-unknown-freebsd.tar.gz/md5/8614371d48b6bc091f5bfe13c0c036fa +p7zip.v17.8.2+0.aarch64-unknown-freebsd.tar.gz/sha512/036a7e2c3822b1e3a3c61d0b69b4bc7f1c4e53d3f1c55d9ec4cfc9f8b1c201b653890d628be9be415f17ada30ceedb101f1475cd1b083f845887d71fe840dd41 +p7zip.v17.8.2+0.armv6l-linux-gnueabihf.tar.gz/md5/57e8d53c6177dbd779c69befe241a967 +p7zip.v17.8.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/e6b6428d1ad2606f73a8a6e633a821bd7cba6355f98d8ed94414ffe926c950ac17b5085346f46c7cbcec5e41a6b95bd4e83a77105365f70ef46269d18fe265de +p7zip.v17.8.2+0.armv6l-linux-musleabihf.tar.gz/md5/ce8b45062665efe8bfcfd0147d2b7512 +p7zip.v17.8.2+0.armv6l-linux-musleabihf.tar.gz/sha512/6947e4ea82b67e1135e44d8cbcc748c20f91e2a83d19a3b127e40d39ce650fd03e8e22dfaec76aa830bceca2914db7e25a78682a1b5bb5a0fe5d7a37977c976f +p7zip.v17.8.2+0.armv7l-linux-gnueabihf.tar.gz/md5/acb20a82452566a68a598951307095f5 +p7zip.v17.8.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/78e95d8a4585ab62b1873751d62760a9bb9ef573bb2ba29218bfca4790755ec4c782394ce2e9279bc08b8a663445d6b1afc53298791ef1b9b788301a637e0445 +p7zip.v17.8.2+0.armv7l-linux-musleabihf.tar.gz/md5/ffc3b551e9ee24428d3f28be9d6874e8 +p7zip.v17.8.2+0.armv7l-linux-musleabihf.tar.gz/sha512/deb1651dc95a3f134d9363b5ddfcf4ff0e25d8e23676c7f7f56584c62c6cd092aba0171cacf8c28a41711d9aa3bedbf3a828b3613fc60d9d2cb9e33c647f156d +p7zip.v17.8.2+0.i686-linux-gnu.tar.gz/md5/366e3ac3ccd572939a557e467ab10d4e +p7zip.v17.8.2+0.i686-linux-gnu.tar.gz/sha512/a9f89296259dec5826ca3ed09c80acd78f7f4de083331ccc6cc8079d009baca5313afd509cd37b6207077b88b32cb3570016edcf0f00701249b9965c95d67461 +p7zip.v17.8.2+0.i686-linux-musl.tar.gz/md5/3f7d011a9d75ad645ad49d69793f1633 +p7zip.v17.8.2+0.i686-linux-musl.tar.gz/sha512/e42ebce35135b96b04cc7f072e624223e3bf19516b55459ac02f88d25f5497fefd331d0f31d1c04b1e4b8d5b1ba4036e3fb3465d75ea9ffefd297a65af970afb +p7zip.v17.8.2+0.i686-w64-mingw32.tar.gz/md5/10d9a33b29677ad1f4611414b3caca3d +p7zip.v17.8.2+0.i686-w64-mingw32.tar.gz/sha512/b534d0bef553843f4696d055dad1b04bad1204c684f2c0a11318a18008c14ff87d4f0f61f68237c4182344d1e2add1596fc116c08d970684d47dce577518e5d3 +p7zip.v17.8.2+0.powerpc64le-linux-gnu.tar.gz/md5/5d483aac12f06883566cd12ee836c9ac +p7zip.v17.8.2+0.powerpc64le-linux-gnu.tar.gz/sha512/60c353ad525488f485c91173f871398dfbdcc8081bd8fd11ae1c8b7eebb1d1743151ca45adb584905c070b7ff9d90eac6081a538c9ec3e79bd0245a5575a814b +p7zip.v17.8.2+0.riscv64-linux-gnu.tar.gz/md5/c5a747d75c232e3dd12ea28ebe64700a +p7zip.v17.8.2+0.riscv64-linux-gnu.tar.gz/sha512/7e048c7e5182be3f49247596152b8e88cf2a8d1c6dd8b546e501b4b5bb77172c84f1623a1e6184f6f7410db2a119dbfc7443a32577a2769a8755864736435da0 +p7zip.v17.8.2+0.x86_64-apple-darwin.tar.gz/md5/ff3ed47aa357dfda027b77c9bed1b737 +p7zip.v17.8.2+0.x86_64-apple-darwin.tar.gz/sha512/58ef3feab5729f65b2c743d8647cc5a5dad8fab3e0df935b90ad6b2644166957f0a92c13ebfae8a5e6626acca23c586a20c6f83a1e0cec44355721c0532ad2af +p7zip.v17.8.2+0.x86_64-linux-gnu.tar.gz/md5/693805d864719d6a3c56609096b7e4ce +p7zip.v17.8.2+0.x86_64-linux-gnu.tar.gz/sha512/802c80befed0cd299af5810f3dabb02bc2fff2b6280517658dba030640b1533a16562798f42f6cb518b44dd0ad180a4cd754bf48db25804e73fdbfa5b58887c6 +p7zip.v17.8.2+0.x86_64-linux-musl.tar.gz/md5/a0d3fd55c8135ae63eb04dc051cdbb81 +p7zip.v17.8.2+0.x86_64-linux-musl.tar.gz/sha512/b90a9dddea62cc725ac568571c82dea3908b2e3aac8bd9be71f2cd4e41960c4020310381e7a182f71666c37fa176efc6d8812cabcc0e5ac053ddefb76804e39f +p7zip.v17.8.2+0.x86_64-unknown-freebsd.tar.gz/md5/802ee53c832ecb828cc9ba021569b7f5 +p7zip.v17.8.2+0.x86_64-unknown-freebsd.tar.gz/sha512/f21b5b0dc90e695c302de5072c374b53a2affe6d3520a009ade91faee1f449cf58d7a9536e195e0518ae988e1d5aadcabce28ab76e4b3c7e6ccd6ff8439ae386 +p7zip.v17.8.2+0.x86_64-w64-mingw32.tar.gz/md5/dc8e839cebee5b7a8fe97f7c981cfd62 +p7zip.v17.8.2+0.x86_64-w64-mingw32.tar.gz/sha512/499936c9922da1b530ccefccb895858a58cc88de6cd0aff5e57b92895d3e1fadc8cecaffe4227dad2f178a6c0011a1e669aea4816256a2a8aff8065b7d960d03 diff --git a/deps/checksums/patchelf b/deps/checksums/patchelf index 6392e44d8f2e8..c1b97c70b4c35 100644 --- a/deps/checksums/patchelf +++ b/deps/checksums/patchelf @@ -1,2 +1,2 @@ -patchelf-0.17.2.tar.bz2/md5/d76db4f1a27b0934d0b0d0585b081c0f -patchelf-0.17.2.tar.bz2/sha512/8277adf95513f88fb190536a38bdfdf438a4cc7685d8a130bdffbe064441f0f25095b6c83bbb190133e1a138963776d15b46c247dd2f1a073a1bfe1d1dbdd503 +patchelf-0.19.1.tar.bz2/md5/fe09ac93e481b596921e7bea47290321 +patchelf-0.19.1.tar.bz2/sha512/a6ef811f983f02abd7571e60d5bc45a44fab91d000d39f7b1a1283c2c54cc0374a0d26d927956c0723ee74a0105927482c5af6dde400740b064030a460841b9a diff --git a/deps/checksums/pcre b/deps/checksums/pcre index dd22c2b91576c..1a6236c86cda6 100644 --- a/deps/checksums/pcre +++ b/deps/checksums/pcre @@ -1,38 +1,38 @@ -PCRE2.v10.45.0+0.aarch64-apple-darwin.tar.gz/md5/896991347ac3198a2cc84a1c39b8bf2e -PCRE2.v10.45.0+0.aarch64-apple-darwin.tar.gz/sha512/92ad56e63f7efe6511092826a994cb5d1ac3cfb20f86db557042bb734df8786d9f2f592c7cfea695955494696274916bea5af90116fd549ece49dd142d2de5f5 -PCRE2.v10.45.0+0.aarch64-linux-gnu.tar.gz/md5/dceb9622c7845494635226627c5d9809 -PCRE2.v10.45.0+0.aarch64-linux-gnu.tar.gz/sha512/d2688bf19ccc7f0567e9d9926be1f95b3e5aa8c85800fc40aae6feda81eda4e59bbff03f8a8f08cd21ac14956c44e7b6077ef5b8eba61fb0fa3541144ba23f47 -PCRE2.v10.45.0+0.aarch64-linux-musl.tar.gz/md5/5caf1b5a65e01ee25272b0125168c6aa -PCRE2.v10.45.0+0.aarch64-linux-musl.tar.gz/sha512/e0bf9422cce467373df64cba82d411768a2bb7e1b03b28c16267f49a56cf259450b34a47ecfa18051ba12a6b1bf2126a28df18508c25e0e09a35040502bb95c8 -PCRE2.v10.45.0+0.aarch64-unknown-freebsd.tar.gz/md5/5e1fc216d3051103911b7958aeb9ca8f -PCRE2.v10.45.0+0.aarch64-unknown-freebsd.tar.gz/sha512/de892eae2c3f815f676cc05771a54e84c2f6fc9b8cf6994b6953ee19eec7160950c80a6a3ff2d9974c4d42a23fccd0d24cd0fc569c87263f3b9dd05e86a66527 -PCRE2.v10.45.0+0.armv6l-linux-gnueabihf.tar.gz/md5/d00c7585e21c9c35288c9ad1ab86f6db -PCRE2.v10.45.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/4fba211d5cb526fc313d62b6eeaa49d5daa471f507a4649cb6331686b762c951217e54017ebc745214811ca7a26a621e850d0092f9e88f1bbf9e1dbb451776c9 -PCRE2.v10.45.0+0.armv6l-linux-musleabihf.tar.gz/md5/67e2cf530ea4f54065e8218da5735de5 -PCRE2.v10.45.0+0.armv6l-linux-musleabihf.tar.gz/sha512/5097fcf75be76e62f95f066c65a2e87f33bf8897cde9951889349dd2f0b62b609dafbe5f4b0039d3acfb79eb5a18606bc5f10e591ff0d8520c37e32b487bf7cf -PCRE2.v10.45.0+0.armv7l-linux-gnueabihf.tar.gz/md5/87b587f70b5fea9a1623f836d810c73c -PCRE2.v10.45.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/07f00365f81d6e27dc3aab0ee9f5c47ec612a09e271baa9f703e40d6a5394e631c01cc171ced248f24dcf561d9ee95e54d879d4458db9809755887a799e0b97b -PCRE2.v10.45.0+0.armv7l-linux-musleabihf.tar.gz/md5/efdb80dd89b059f50e425e5788873422 -PCRE2.v10.45.0+0.armv7l-linux-musleabihf.tar.gz/sha512/9fcd05831e9581b8ef5c85bbf13d90ec11268a60b474237b3679c71d804176d7816750aa65b1d02652ab599814f61f3023fa4c57e4e0bc29a9667b928f906b0a -PCRE2.v10.45.0+0.i686-linux-gnu.tar.gz/md5/6d79e55c8e3461400eee5a280d1a7bf2 -PCRE2.v10.45.0+0.i686-linux-gnu.tar.gz/sha512/a500f6fbf3952497539b6dd1d3c01fe4ca713eec838de3162854183b559100077f43537021224121abbdd72836dd0ed599a7ef7d2728848bf32c5af344d8deaf -PCRE2.v10.45.0+0.i686-linux-musl.tar.gz/md5/f820b790aba3c4dcbf0dd43c3570f94f -PCRE2.v10.45.0+0.i686-linux-musl.tar.gz/sha512/b9daa0df01701b1c4b97bdeaab113226ee085f4af49298ca93f4bab29e94781c2cc10442342a0205f52f089937efe01900f4ed75360cb8174f70aeec48e864e9 -PCRE2.v10.45.0+0.i686-w64-mingw32.tar.gz/md5/fd5bd53baab8ce75026722fc88a5f804 -PCRE2.v10.45.0+0.i686-w64-mingw32.tar.gz/sha512/117bf87ced28c2c85e5a90e83af14bd75a314bcb0628a71a86d7103a8a098f43be168b6093bd6c0439eeec940ff7327e20a5cfcc57060b9a6383090b03e947b5 -PCRE2.v10.45.0+0.powerpc64le-linux-gnu.tar.gz/md5/b6d67f0a9710b6531ac5c5aa32559a24 -PCRE2.v10.45.0+0.powerpc64le-linux-gnu.tar.gz/sha512/a287b1ced53d8fac0f6970d140dcbac7677cde241382f4d33617d4b0370ab5b65d498218962891a46b6a9e80ed7b2bd361675d263b4606fdac336606cc80d36f -PCRE2.v10.45.0+0.riscv64-linux-gnu.tar.gz/md5/bd06447be24fe3b9eaa8258664684d6d -PCRE2.v10.45.0+0.riscv64-linux-gnu.tar.gz/sha512/a8c5aa1011daf811c9431122a7221e1ed98e1a593861f21f8ce06593428d6b024633b7fc1b31c89b06a69f03e0da7c99ea8c6abc4ab8c9c47ffef786fcab0de4 -PCRE2.v10.45.0+0.x86_64-apple-darwin.tar.gz/md5/f4cafdaaa4cbb224d35337fce37a5fa0 -PCRE2.v10.45.0+0.x86_64-apple-darwin.tar.gz/sha512/6f891e3fbcc0a3e9c906d645cf98cc15eb224addb31d1a31770e41aa5fd1d84b8979f8f8f76c9ed9b2a34e97e137f5404a7d3d49a1750c1e633cf9f03415b580 -PCRE2.v10.45.0+0.x86_64-linux-gnu.tar.gz/md5/26a732774abff1d4c301640f513f4e8c -PCRE2.v10.45.0+0.x86_64-linux-gnu.tar.gz/sha512/ee89bfedd7a79dbe85c83460bb0b3834606af3f950d26223676733417ee4028d9b5a9fe3f2851f42c55b3a4c800ee48e25afdf7ae7bdbe7df021c38cffa7c7fc -PCRE2.v10.45.0+0.x86_64-linux-musl.tar.gz/md5/e4b0e47adc80128317697edab7312130 -PCRE2.v10.45.0+0.x86_64-linux-musl.tar.gz/sha512/521cfe714c1ff1604e86b24935060b12fbb3ab62af9b0c59121f0c26627bc24f5d612f4758739e068f3f7974c3890043c55c6f28e500c9de6e5550705c45c982 -PCRE2.v10.45.0+0.x86_64-unknown-freebsd.tar.gz/md5/c6655a1f349c51ac898878836b72fa1a -PCRE2.v10.45.0+0.x86_64-unknown-freebsd.tar.gz/sha512/69b48b9b6872f0c6b61a8a3ae81ba103fb49887c392497e5a387de888b35fc45c716b99c972670ee5edf0255db361051f265e30ea8f509b790ab9edd50bcb6db -PCRE2.v10.45.0+0.x86_64-w64-mingw32.tar.gz/md5/cb5c7a9a56e1f2209b2e227c598336c4 -PCRE2.v10.45.0+0.x86_64-w64-mingw32.tar.gz/sha512/ff5a982d90089316ccc1e6a9dc2c17cc61b348df0db16aaa4bb7cf1787701acf31a65ffcf51be48abc8cb4436afe94984086fead84580b3de3509d111ba71992 -pcre2-10.45.tar.bz2/md5/f71abbe1b5adf25cd9af5d26ef223b66 -pcre2-10.45.tar.bz2/sha512/4c1f0cf793624516d7eeb15745d6c07c9f678dd2c2b349062c6b614e88bf42262972d133576e85140dee2a882984aaf2d688953fc9c69ec7105b2daaeae89845 +PCRE2.v10.48.0+0.aarch64-apple-darwin.tar.gz/md5/d2c7e95fde90752ffdb1401fd6e2855b +PCRE2.v10.48.0+0.aarch64-apple-darwin.tar.gz/sha512/5995ca91170fc4f6712be3438fee7ed370415c44ec5f0735480f0a195be7d12dab05becfaffbc2765ea0aab7a000bc717d9c21f29c51b8e2454fe91ee2fc9446 +PCRE2.v10.48.0+0.aarch64-linux-gnu.tar.gz/md5/77e34b1aebde62b35052e1a1118f56b1 +PCRE2.v10.48.0+0.aarch64-linux-gnu.tar.gz/sha512/0985cb79a8f4602ac77f9cbfdb6671c1aca2fd9f2ccccd0b29502e2678a0a4166198fb71000832bedefa9cd9feec557738dd86570f74e12a2e79e763523eb1df +PCRE2.v10.48.0+0.aarch64-linux-musl.tar.gz/md5/1c5c295f4810a40bfe5a744a6d56d6f6 +PCRE2.v10.48.0+0.aarch64-linux-musl.tar.gz/sha512/7da95aa44116d44cc16559b2f4f33649c6d401dfab29b25c2bde8542460bd3b4064468f896ce0ddc5872b9995ae7affdf67e9e2e0645473bb44d36bebd132aef +PCRE2.v10.48.0+0.aarch64-unknown-freebsd.tar.gz/md5/41a562bb8818b7caca836be9cf64cedf +PCRE2.v10.48.0+0.aarch64-unknown-freebsd.tar.gz/sha512/3fb64528dbdcaaa0a842f168b9570c3ac9f0665a8977153e12d0c54c798bed99c45b398200c99849b2e50494b577b7a60a7ec7553578489ffac5110bf8fe4f09 +PCRE2.v10.48.0+0.armv6l-linux-gnueabihf.tar.gz/md5/0d1e6556718ea4c2b2f5af90ac17a30b +PCRE2.v10.48.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/94e7706fc1e85a28553c2b6bf0d28bf65a886f833d96fce474462f54938dbcf2679b5da30817e76d2c2a501b95dc8edbd4f541e02bb93747092f4f5abd680f2e +PCRE2.v10.48.0+0.armv6l-linux-musleabihf.tar.gz/md5/71a71de107dc50d7f5e0ae2615774de5 +PCRE2.v10.48.0+0.armv6l-linux-musleabihf.tar.gz/sha512/387a99101c5bd3dd1a258ff32aaefdd721c3f4c9f34dd4710ce36da05112ca30ccc2b62a5795f83bf07d9f22f2e4ff891f7a9dccb79bf3e75966a989bce75df1 +PCRE2.v10.48.0+0.armv7l-linux-gnueabihf.tar.gz/md5/eef01f65c02933681d9b1141b65c2c09 +PCRE2.v10.48.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/cb47498db81c1626557695d3d5878dea88773298ead310ba8a9e2bd2098ac7453d905952e11128b380009279590d322881df69496cb512ad8e19085ac7df5493 +PCRE2.v10.48.0+0.armv7l-linux-musleabihf.tar.gz/md5/eecf3b64c57581398df23e6fa61fe9f3 +PCRE2.v10.48.0+0.armv7l-linux-musleabihf.tar.gz/sha512/b4e8c46b2e51031e8eac3e448ce700f7aaafb5d3c0c10717476379f85800a95ca8d679f3f9edcccdefab83a1ef4cb200c694221eedd90d7536487b5bf4e0b29f +PCRE2.v10.48.0+0.i686-linux-gnu.tar.gz/md5/d13deff96cdf5dcd3eedc694fb266032 +PCRE2.v10.48.0+0.i686-linux-gnu.tar.gz/sha512/54e03cde8ad7fd69511613252b900f86d4f8c2745b04e55de828ff24dcac91292f9033efe9ae53532d560368ef0bfec16a5d9b88b9f05af7d15d5d713f6f0911 +PCRE2.v10.48.0+0.i686-linux-musl.tar.gz/md5/93174beb09c7455ab2e906b948e4a6a9 +PCRE2.v10.48.0+0.i686-linux-musl.tar.gz/sha512/b3d6dd8e90db856b7205cf4383b473cd01f0adf48f27cc1010b4181f1bde49c4dc581dde41ab20d0a43f75674e1027f745dadb8653d84750471d98547562a485 +PCRE2.v10.48.0+0.i686-w64-mingw32.tar.gz/md5/96b471636bb2796681b56933acffd38f +PCRE2.v10.48.0+0.i686-w64-mingw32.tar.gz/sha512/62b5de267f10912ff88d672209b1c86030f4118ba4d5d58ef0e92f64c1ee973a56303d51b880ecba58cfeeec98113e988a04d60df6c21c23d519100252ddc889 +PCRE2.v10.48.0+0.powerpc64le-linux-gnu.tar.gz/md5/e97e552282dcf9a0c15086c0fd1dbb85 +PCRE2.v10.48.0+0.powerpc64le-linux-gnu.tar.gz/sha512/3b3b64f1f2d0eb565999a6495e8c3f5b4696517fbbe0e95e596b94400dd67537defd43a27948d64e233925dc8e24edbbd12454260ce091b62f25677397d7e943 +PCRE2.v10.48.0+0.riscv64-linux-gnu.tar.gz/md5/19f61041c47bc734fa4b85dd95187d7d +PCRE2.v10.48.0+0.riscv64-linux-gnu.tar.gz/sha512/a70ca06e48d35745d7c64b46c6a43d9a650a48943b2ac4c06d116b08074c094373b00f29e4012d3a0a7bfa045038ba22cfd9ccc3371bb6364c7dc0b5cbe95965 +PCRE2.v10.48.0+0.x86_64-apple-darwin.tar.gz/md5/f1aa5c8b7b4f50851c2cd1591e6cf07f +PCRE2.v10.48.0+0.x86_64-apple-darwin.tar.gz/sha512/a50b45c10bcde248878aee8318f320199ba0ba15c943cf981b3d2b4b01030c848161603797b2e3a51673f344d9b8dbba116b94adbb0cb53d9202e81de9401e25 +PCRE2.v10.48.0+0.x86_64-linux-gnu.tar.gz/md5/4913a5037e0c7cbcdbe2677a7c23d4b5 +PCRE2.v10.48.0+0.x86_64-linux-gnu.tar.gz/sha512/d300c22f736f6edc8239e5a17f154813a96e1efc0d566cb605a6a9d62e87f6a93509eceb0465e8431b4f8529bd92009e782058bd7b627ab76f9c7e65467e6196 +PCRE2.v10.48.0+0.x86_64-linux-musl.tar.gz/md5/c83715237065ed53d8edaa8dc7273bfd +PCRE2.v10.48.0+0.x86_64-linux-musl.tar.gz/sha512/8b503c97a97398a2b9e001afe61a0a36e56c4ebda1801f4b458717f8d98f90eab876ca71907075d84569590d8b84f425dc76ca3800e9cc0b19ce9677283a5952 +PCRE2.v10.48.0+0.x86_64-unknown-freebsd.tar.gz/md5/4e160d11094d268ce30765b55438400f +PCRE2.v10.48.0+0.x86_64-unknown-freebsd.tar.gz/sha512/0e719f565e0ff8f07d198b2f02ef56495cc4247d703627ff5ffe4d3291a3b13aa395da6667392ee4b313ddd532f0f9881c4690e62179aa3b2a603a6b0d042e00 +PCRE2.v10.48.0+0.x86_64-w64-mingw32.tar.gz/md5/1fe88c4ae5b354fd2defd93e98be7f43 +PCRE2.v10.48.0+0.x86_64-w64-mingw32.tar.gz/sha512/c7dd509c89b9bcde9ed0d5d06f612374070d4252400e220288f94ac0f7543dbcaa78682c55c329d1990e1b213c66133214734717884b8d41167977f163d1861c +pcre2-10.48.tar.bz2/md5/98013efc5ebfe0a7b82ae53a43b8fd72 +pcre2-10.48.tar.bz2/sha512/b350b8bfc909f3ebf9dfe39c535f04cc4618997f0fa025d12b4d92aaaac3c15ae93b6def62ed220eabbe08df9277cda0b80de4eedb7ec7376daf7e6a78a868b8 diff --git a/deps/checksums/suitesparse b/deps/checksums/suitesparse index 30ac49aa2994b..a0f6e4516754f 100644 --- a/deps/checksums/suitesparse +++ b/deps/checksums/suitesparse @@ -1,40 +1,38 @@ -SuiteSparse-7.10.1.tar.gz/md5/3b8c4f18091589da221a1acc5a331edb -SuiteSparse-7.10.1.tar.gz/sha512/893f6a51df9a30793db67baa068a7c10237bc96bab1ed534293b09fb1c87ed4b2ec5fa3f6c285977582f9cba9b7a5947aa31e65d52ed2af67c93ac2ce64b55b3 -SuiteSparse-e8285dd13a6d5b5cf52d8124793fc4d622d07554.tar.gz/md5/46541001073d1c3c85e18d910f8308f3 -SuiteSparse-e8285dd13a6d5b5cf52d8124793fc4d622d07554.tar.gz/sha512/f7470a447b934ca9315e216a07b97e363f11bc93186f9aa057b20b2d05092c58ae4f1b733de362de4a0730861c00be4ca5588d0b3ba65f018c1798b9122b9672 -SuiteSparse.v7.10.1+0.aarch64-apple-darwin.tar.gz/md5/cd62af0918c511728741613e81df1c00 -SuiteSparse.v7.10.1+0.aarch64-apple-darwin.tar.gz/sha512/a8f5a357c42de61fa71c8f275412b0c5743a275ffb46f95d73ba4135be0e221176a1b7f7d98d39b4ff5a8a2890601e888b0a2e87e18da958ad5e78290f2a816b -SuiteSparse.v7.10.1+0.aarch64-linux-gnu.tar.gz/md5/2019565921ffb5ebb77dc997a41985f9 -SuiteSparse.v7.10.1+0.aarch64-linux-gnu.tar.gz/sha512/2e5243dc943ec4e00d5210d4e201a8163bbcfa8ced216b1737c515b0343e87cacf7eca69dc45b0a3893c6daa157f70c6179388efd90a2412887132c7f1725526 -SuiteSparse.v7.10.1+0.aarch64-linux-musl.tar.gz/md5/f9e904e29fdc5f65af179d2a1b315ceb -SuiteSparse.v7.10.1+0.aarch64-linux-musl.tar.gz/sha512/c3335b9902ececf78e6eec3930cbe52cedc0428f2dc20295ebc940b32177d40ca255cc252469a4b18e3c21fe2a1f4ec3d2a75a4d1ffc9dc60febcffd6625afa7 -SuiteSparse.v7.10.1+0.aarch64-unknown-freebsd.tar.gz/md5/91cda0a046879b44da31352e42b6a678 -SuiteSparse.v7.10.1+0.aarch64-unknown-freebsd.tar.gz/sha512/9e292c7484b348950817f8b298ce76de6c6f3ba2fedb91518f9d050eea076849db64b1ee3b0f20ffa37cb35ddf3fbb25b8e72f8647d24d87f9757b2771a14979 -SuiteSparse.v7.10.1+0.armv6l-linux-gnueabihf.tar.gz/md5/00a30f24ebf6c78f989ce123b8160757 -SuiteSparse.v7.10.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/eb23943cef8c540d15f47d49442d612a43b4f1247b0190294bac64ef0a95ffabb355eedfd15fb0e6bf8d01eb490a3f3f0ef80ab04ad2c1e3460a1a7729e6a059 -SuiteSparse.v7.10.1+0.armv6l-linux-musleabihf.tar.gz/md5/975429f8462de4f9bd7929e070fbd3e2 -SuiteSparse.v7.10.1+0.armv6l-linux-musleabihf.tar.gz/sha512/a616124a4085be7100ef1a3ec030e2cd4c0eae961a4aa5986c5abd63c9d85397f7cc84ee83e6f8a28754834139f5794a6393c2d20d9600c5cdc107e9c24049d3 -SuiteSparse.v7.10.1+0.armv7l-linux-gnueabihf.tar.gz/md5/8261f8c08e5243d30980ea9c989bd793 -SuiteSparse.v7.10.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/3d9fa542789d2ab5e8eb7b541b184d572d14d6980741cd0156f3fb74de710e7f21ebe524dfcb8c777b93a005ea66d9ce0768883d63fcb3a90583db7feffe596b -SuiteSparse.v7.10.1+0.armv7l-linux-musleabihf.tar.gz/md5/2c0ab3267dceb40d38473f24da54da69 -SuiteSparse.v7.10.1+0.armv7l-linux-musleabihf.tar.gz/sha512/c8092ac3b1303bbdb8f551609bc2cb6e0871bc201e90e8eda7c1dccc3de916ea932237cfbb1f738d12f4515d72d1cc03466a3d391e1dda5c09b1a1923ced42c4 -SuiteSparse.v7.10.1+0.i686-linux-gnu.tar.gz/md5/5c9f9e8ef3ff88eb35cb4e9c20810fdc -SuiteSparse.v7.10.1+0.i686-linux-gnu.tar.gz/sha512/4ea534ca7764253ea92c20a002927687d5d49279065a76853f36557ef030a839d9f6a99f3aa35e1144dd405a9acf9f098c45812f6e61e9a73caab4a40d9b6062 -SuiteSparse.v7.10.1+0.i686-linux-musl.tar.gz/md5/f70262bc5f0dfade2b71dc1ab1394b17 -SuiteSparse.v7.10.1+0.i686-linux-musl.tar.gz/sha512/8ac49fec2c6103b5aa87990d2245bacf1a6805ff89fedb390b332803419bdb59a808b1252db55f9ea802589ed5b87d33c3d42af2d613b04fb0c4b88c2bed4e56 -SuiteSparse.v7.10.1+0.i686-w64-mingw32.tar.gz/md5/3d34fee6e16d8df340a5af96794cb415 -SuiteSparse.v7.10.1+0.i686-w64-mingw32.tar.gz/sha512/b9f092973a0b5705f084a81e4814ce943dcb3e212bd9a5d23a7f6987ab67303ae3533b83e4bb07259c631c9a05c72da657e9a473549d50478c6bc91d3303cf68 -SuiteSparse.v7.10.1+0.powerpc64le-linux-gnu.tar.gz/md5/351afb17c410d18a0147ac6127e78692 -SuiteSparse.v7.10.1+0.powerpc64le-linux-gnu.tar.gz/sha512/d9eef86c1528c5c1be9a88e20cce1a79a88cbdc9b5848398ed1ad44986d43f836299ecaf94c36d390a5fe6dd9fb891c74e963560f673f08f2b9115f12f13a2c6 -SuiteSparse.v7.10.1+0.riscv64-linux-gnu.tar.gz/md5/1d6716f75d958423f94f9e9bd9e73006 -SuiteSparse.v7.10.1+0.riscv64-linux-gnu.tar.gz/sha512/12603984ab46cc0d1eab43e6ef9156448d0ec46306a98a0d970eb26db8d1ddb240e40c108d7aa0198f3c14242804efa50500d2aa8034c71578a43117e97f9dfa -SuiteSparse.v7.10.1+0.x86_64-apple-darwin.tar.gz/md5/97384c46a7273e90f1a42a98a28ce787 -SuiteSparse.v7.10.1+0.x86_64-apple-darwin.tar.gz/sha512/e1eab80f99062f343461e536c83dbf39fb305ac4a109e3fc92fe8999afd90b0f469e94d8400c451c20aeafc1f0b4f3efbbaf8c6cbc97096c94cd2dd6cb00141b -SuiteSparse.v7.10.1+0.x86_64-linux-gnu.tar.gz/md5/19b877aa692b5aaf832f8a72b90407c3 -SuiteSparse.v7.10.1+0.x86_64-linux-gnu.tar.gz/sha512/62f4ebbcdf3a432588deb84d58201e939761b6e1dcde229e499a4cf0a5198130a83552709909c9b6a7483c0d9b165db1a80a6c85ec998d2569849d05236abfbe -SuiteSparse.v7.10.1+0.x86_64-linux-musl.tar.gz/md5/c0cc0294e4fcdfa2bafc3945ff745bd2 -SuiteSparse.v7.10.1+0.x86_64-linux-musl.tar.gz/sha512/b66c6b67b4e2dcb02e4b9d3319f70ca13f16c381ce51559032316c69840dd9f545819579dc0fef987f7e11ca075bbfa8f7fa081eb65c7c1c3acd3bd742466e81 -SuiteSparse.v7.10.1+0.x86_64-unknown-freebsd.tar.gz/md5/d6480c9e4ee22fc52d0c78151abd1720 -SuiteSparse.v7.10.1+0.x86_64-unknown-freebsd.tar.gz/sha512/4828e8a68eccdbd020635c7a491afcbf96d7d83e231eb98d1bd325fae4daa396daf47adf4ba7e620ae30567a2d148892b47744668d4195e4b6c90ca2fbc3fc6a -SuiteSparse.v7.10.1+0.x86_64-w64-mingw32.tar.gz/md5/ce32fda49383eca94f4a3ad56b4424a4 -SuiteSparse.v7.10.1+0.x86_64-w64-mingw32.tar.gz/sha512/7aa8f123e5486c336043b554c7e5ea2c8143a8f9ed47ae2146f3cb908d8834c3bca3c0cff73e14460c29015a031aeccd324ea7ff66a83fdce48d0283cb345b5d +SuiteSparse-7.14.0.tar.gz/md5/4f1de135ee1afb0d5383f59316c26489 +SuiteSparse-7.14.0.tar.gz/sha512/4cd00b0625ef8081703139cf15c45c32a90e101bcd9c6ba38b87b988c2e76f51a73da0c312581b87e0892098c717d14782edfd6f19c92b0b0db196044cfe8337 +SuiteSparse.v7.14.0+1.aarch64-apple-darwin.tar.gz/md5/5979cf690b83dca32890d365b673db61 +SuiteSparse.v7.14.0+1.aarch64-apple-darwin.tar.gz/sha512/07d2eb2ba84d561c2b32244cf23aa25d0f0e6142e0623d5d5844e55ae8477efd8bde3971b14fdf58d4a4f9242136e9e6b6cbb2ffa74ee615dd6afe985531233a +SuiteSparse.v7.14.0+1.aarch64-linux-gnu.tar.gz/md5/b9dc09d0b8b0cb2c0f56bb05bdfbfeb4 +SuiteSparse.v7.14.0+1.aarch64-linux-gnu.tar.gz/sha512/52a247c68c31a4e44a5f350f5f87b403a4832180723d13dd8a1892a91223696b150f54f7737c0ca5a455a2a6e0d19b2a283f1ecc79faed72fac1faeedba266d4 +SuiteSparse.v7.14.0+1.aarch64-linux-musl.tar.gz/md5/428b29b316e18e0d8e7bdfddac733bd4 +SuiteSparse.v7.14.0+1.aarch64-linux-musl.tar.gz/sha512/dd101d7d1be9e03191564303459e223ea7315f415c41fb5b82d6c626f6aae3f0ed800053be75749f1d6895ffb3e15bf5d5d18c3fc9bcfd770180b92d6845cfec +SuiteSparse.v7.14.0+1.aarch64-unknown-freebsd.tar.gz/md5/8ce9ff5dd055ac6fd14560a064c5b169 +SuiteSparse.v7.14.0+1.aarch64-unknown-freebsd.tar.gz/sha512/c5eb31e2cfc6b1b34f19c3de2eb349c4582cc0a0d51427fc05527a6c07d44cdab49d7323f4e9848dbaa7eae514ad5647c568858fc37322c18edc10d2fced5342 +SuiteSparse.v7.14.0+1.armv6l-linux-gnueabihf.tar.gz/md5/bbdf2396ca40065a398c9c2f5c3c9ec4 +SuiteSparse.v7.14.0+1.armv6l-linux-gnueabihf.tar.gz/sha512/3ef9f100a6c40468ee2f29670f4fae145ee55b2b80814617fcbed2279d3283c29b60330bac87c764977f19071d2694c636742da8f5fd91695aa31867fdfd0ed9 +SuiteSparse.v7.14.0+1.armv6l-linux-musleabihf.tar.gz/md5/c7289b859b4fcf070085f5b289c9cbfc +SuiteSparse.v7.14.0+1.armv6l-linux-musleabihf.tar.gz/sha512/d8f37a8379dbb1e4e504b7ca662ca8c098e9bca27786bc2c299b442ff78a302b772d25db7c44cca802f5f34cd5142d78a7b423cb48b8a02e7c9808ed28683fcf +SuiteSparse.v7.14.0+1.armv7l-linux-gnueabihf.tar.gz/md5/b7bec1ca6fb528a041a97238d2a81179 +SuiteSparse.v7.14.0+1.armv7l-linux-gnueabihf.tar.gz/sha512/a726133480cfb4f689a98644d9adb3ade23fea7287a6f381a90b176a5b74a8b4c918e979b16a464310e20d01fb2a8acfcb915f9e2fd345f48139b0e64f35612a +SuiteSparse.v7.14.0+1.armv7l-linux-musleabihf.tar.gz/md5/bd07b70a968855e3e4cee2146efa6890 +SuiteSparse.v7.14.0+1.armv7l-linux-musleabihf.tar.gz/sha512/e75ef5e7cfa1b54cb52ad24c2db301eff5b17fef82af3276ae8d5274601b91af2389ee54e7b6ca78318c62a594e865852b738a02a7143792ad301176678c17b8 +SuiteSparse.v7.14.0+1.i686-linux-gnu.tar.gz/md5/e0e301aa39358b98d68760a55384146b +SuiteSparse.v7.14.0+1.i686-linux-gnu.tar.gz/sha512/80028373e7e67e4f9433f21ddf6383c66e9ff1d70a2e77c856dbb51180f6c77b825a220d4b988453ec73393231efd993f605a205a0e9a995edbc5515f2b41a79 +SuiteSparse.v7.14.0+1.i686-linux-musl.tar.gz/md5/aa83662b1b3bd570008b8ba63930358e +SuiteSparse.v7.14.0+1.i686-linux-musl.tar.gz/sha512/7041f90bedd914d7c4ae1af7a56804225a4cda7f4bd71ca5d7c6b0fb82a9894eb686c53f4d62565c6f78926c06bdfac50907190fb3d9f91d43b34fc4b13002b3 +SuiteSparse.v7.14.0+1.i686-w64-mingw32.tar.gz/md5/c1e8237b49ace72a21793b39fef7505b +SuiteSparse.v7.14.0+1.i686-w64-mingw32.tar.gz/sha512/d9243836031f2cf4cb0acb63fddcc8cc75752d96bd970a8f6663b36a65312594c2fcfa934273749f0d59c5c3e52c39ae06aad796052a35eaeec4d73fea4b95b7 +SuiteSparse.v7.14.0+1.powerpc64le-linux-gnu.tar.gz/md5/a0139a7fae5149960fa7d696113316f4 +SuiteSparse.v7.14.0+1.powerpc64le-linux-gnu.tar.gz/sha512/bf3187dc9b8353c906b803e8c2a6005524c6393709db515615cae2923467db0c31bdd727dca9be7ebd97fa25e68d73b3682fcbc4e2415e9602df0fd0f71e26a8 +SuiteSparse.v7.14.0+1.riscv64-linux-gnu.tar.gz/md5/d44a7fd036dc412789f557519183193e +SuiteSparse.v7.14.0+1.riscv64-linux-gnu.tar.gz/sha512/7b196f716287f234f804327c3560bc7cde1ab3cbbec7651b64f2f48140480d24b96d0931a52ec4dad53532644062b3456b3aa649806666b370f2cf49d6d19456 +SuiteSparse.v7.14.0+1.x86_64-apple-darwin.tar.gz/md5/bb2e73688a8aaa50c378308b5cfd5a7f +SuiteSparse.v7.14.0+1.x86_64-apple-darwin.tar.gz/sha512/3b418f9e1bbcaa4e4810f941b41fae702c040f879c68f2c348eb04e4da0c04f837527b8645cc61556c00e72a028078cfd438bc9fcd2d67f559b14fb33aed33d4 +SuiteSparse.v7.14.0+1.x86_64-linux-gnu.tar.gz/md5/6f4e09cc42fee091c45a39a87b1c6246 +SuiteSparse.v7.14.0+1.x86_64-linux-gnu.tar.gz/sha512/2e3067a668a4e858e0127a1ea47b1b9b0eadaf5ba92ce816bb575388ab2731a29028246a3bb158a0a1deef1610eb3535c13a4f4e924de0de110a272768134353 +SuiteSparse.v7.14.0+1.x86_64-linux-musl.tar.gz/md5/5b20fc5d179f6e5a0b48f34874431b1c +SuiteSparse.v7.14.0+1.x86_64-linux-musl.tar.gz/sha512/86a2be09da6e2378812dcfdeb779f833b792f3e3d507a3e96f48aa5e34438fc6337cc8c44f5a2221f235c7d0c67b235ab009b26160be9731f5db6806a20a1d32 +SuiteSparse.v7.14.0+1.x86_64-unknown-freebsd.tar.gz/md5/34b81a99300b0386fcab60076e865a65 +SuiteSparse.v7.14.0+1.x86_64-unknown-freebsd.tar.gz/sha512/ce16206146bfd72ab675fe2237f100356c91d3beeb9afdb24f35c769ddd327242b4987e870394b04dcd89276ed2ff03b98e10f3aa050186fa2b821ffd4028476 +SuiteSparse.v7.14.0+1.x86_64-w64-mingw32.tar.gz/md5/92f5564edeb5260cdbb6ad68f598e8c0 +SuiteSparse.v7.14.0+1.x86_64-w64-mingw32.tar.gz/sha512/bb5c7fcb029a5b1fa40bd22583a7a8259762e8f6d16f9165d4bb902b994a3c6f971638989caedfb37c2135d66467bd32cb011a02a967be85a2e1334961e92c70 diff --git a/deps/checksums/unwind b/deps/checksums/unwind index c7dbc28dd3976..9156874a8cc50 100644 --- a/deps/checksums/unwind +++ b/deps/checksums/unwind @@ -1,30 +1,30 @@ -LibUnwind.v1.8.2+1.aarch64-linux-gnu.tar.gz/md5/f23ea74855dd2db466170f170a8eea7a -LibUnwind.v1.8.2+1.aarch64-linux-gnu.tar.gz/sha512/27b4c37d225c1632e1d8814989d768b0fda2675552c611cb9fad1eea158e4b3815e0a8127c2eea97bcbfd23685d970cf1c478d24f6b5a8abe8357e12d8762ce7 -LibUnwind.v1.8.2+1.aarch64-linux-musl.tar.gz/md5/5073e00eeacd4f5921164090966f5ae8 -LibUnwind.v1.8.2+1.aarch64-linux-musl.tar.gz/sha512/0499cdd22510745e3e1f7da5c4c937a79b2eb07e2bf97afa5db9ce7e91aa2bab85bef2dc3ad8b859ac147ca8da908047bf12daf6e3d3fcb2c7a865402dd90beb -LibUnwind.v1.8.2+1.aarch64-unknown-freebsd.tar.gz/md5/b4aa1f850db793ffe9c1e0bf0524da65 -LibUnwind.v1.8.2+1.aarch64-unknown-freebsd.tar.gz/sha512/92bbb42a742427d4a4bfb67530191093ee55a210a99abe0145bdf4ccdc3e583a9852a8045dfdd20002a8fd8eaf9feb6245f84a4995fee4a097a33a2b64998dcd -LibUnwind.v1.8.2+1.armv6l-linux-gnueabihf.tar.gz/md5/a6eff40a5bef97c13fed1546dd77a503 -LibUnwind.v1.8.2+1.armv6l-linux-gnueabihf.tar.gz/sha512/b0e687036c080cf5a421a924e070e57d9f52cee1e67e6c9558d586edc58296d15c9e10e2e46c27e6d290c866b84edeee11023077fb2481dfc9f08fd08540326a -LibUnwind.v1.8.2+1.armv6l-linux-musleabihf.tar.gz/md5/19f8f17923dc1b87b3e89480b1eeaedb -LibUnwind.v1.8.2+1.armv6l-linux-musleabihf.tar.gz/sha512/23251a9fe459242589797a364ff69e38711ebf93d4f047acc7d302c8bc669d3b0546a318568787f0f9bf4d1e633670e0941887cbac1c34e34d86e217040533af -LibUnwind.v1.8.2+1.armv7l-linux-gnueabihf.tar.gz/md5/74bd5a25811e4977386ba3aefe98bd8c -LibUnwind.v1.8.2+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d41f653640ffaebb7295d49a533efebf2ffca7314a181e6e669c399e65b49d176915afeb60b4702632d65aa20aad772052373efbd3c6b9fdc0c9fd772986b2a -LibUnwind.v1.8.2+1.armv7l-linux-musleabihf.tar.gz/md5/7d882db1897c88c7e2eb7cf94e3b3f9e -LibUnwind.v1.8.2+1.armv7l-linux-musleabihf.tar.gz/sha512/dad5bdbccd630eb27af69a32f01245865fc9cf494f1e2ab4cf9e06d78985d997accbac7db6c947bb1964f0aa98f7c977cdb9288e4453fa22d47eb99d73144536 -LibUnwind.v1.8.2+1.i686-linux-gnu.tar.gz/md5/74076851e0da8ebf94fefa35718b3ec8 -LibUnwind.v1.8.2+1.i686-linux-gnu.tar.gz/sha512/4549cbe8da9b0ff5cb62c7cdc24534e4fd0f48354902082212d9c559efbdb02017884b41554432e4f16886bafe27b737f4264c05ddba4b14166212290d97f37c -LibUnwind.v1.8.2+1.i686-linux-musl.tar.gz/md5/9a91cf5da60391e2bbae8ec77ef158c6 -LibUnwind.v1.8.2+1.i686-linux-musl.tar.gz/sha512/5dd44a67c259cbd629d8b17375e00a1e18f9fcf2d9bb86f26eefc31c52cef0ae7ebfd279e37c6b2ef428df98af30348c65a0760e27409ba90614f3cbb80a8c08 -LibUnwind.v1.8.2+1.powerpc64le-linux-gnu.tar.gz/md5/052e192212dbf6412a4ea12ca5a8b40f -LibUnwind.v1.8.2+1.powerpc64le-linux-gnu.tar.gz/sha512/2d985c65453935393de449133dbbfabab68c832ecb6243100e1514ba7a613716dfa2bf690bf1df9a86d9c45eb683e7c61c04ff187347d61b17055964ee7944af -LibUnwind.v1.8.2+1.riscv64-linux-gnu.tar.gz/md5/bfd355c19743b5b8121005f41a7a1eaa -LibUnwind.v1.8.2+1.riscv64-linux-gnu.tar.gz/sha512/4659d1ddf80cf41c99ee374d98a56264706fd81836ce6cc5e71bfa547bf181cf7775d017d9c3b925a678e33446ca09bb8cda057ed48fdb702e338c9c1adc5aa4 -LibUnwind.v1.8.2+1.x86_64-linux-gnu.tar.gz/md5/9c0d4c4c8d96a3b64d1edeba61df80d9 -LibUnwind.v1.8.2+1.x86_64-linux-gnu.tar.gz/sha512/2290ca45c950dc6c0ef2f3f915386879e179f98e08b29a5f3e55e8c5fb646389e8dc09dbd1f8f91c382e623a167996322624dfe035142aea11f8e30fa643ec2e -LibUnwind.v1.8.2+1.x86_64-linux-musl.tar.gz/md5/19a6b5638f1c12732a4ea29dc2ba45ca -LibUnwind.v1.8.2+1.x86_64-linux-musl.tar.gz/sha512/834f2e5a2a98e11af20e8d0c3ec91a6e72a60573e8cd556d6c5d2321ecda94afe90911f1b9b6b313140f13433a5c3b1b30d4951ee40ffc862500eea88a44c8b8 -LibUnwind.v1.8.2+1.x86_64-unknown-freebsd.tar.gz/md5/cc97cb986489fb1aaa157490986da567 -LibUnwind.v1.8.2+1.x86_64-unknown-freebsd.tar.gz/sha512/7ca93ec2eaec8f1c22104459ad2db49d5f1303686e8f686312296287b37f51cbde3fb605be84297848fc0997836735e59914f2f8c86fe72bb9ec7c44c35ee8f9 -libunwind-1.8.2.tar.gz/md5/0124a38fb752aa5492635f35d089f6b7 -libunwind-1.8.2.tar.gz/sha512/f1ff26763c1b2e68948413c4aec22303b6c886425a8264eb65fbd58fc202f79c7b04bd4784bd8499850d08933f0e363cfa3a7d177efdadc223ed0254bc381345 +LibUnwind.v1.8.3+1.aarch64-linux-gnu.tar.gz/md5/42933c5396d2a0f9c99ad508ba6f9769 +LibUnwind.v1.8.3+1.aarch64-linux-gnu.tar.gz/sha512/61d5afc81628b96e87f7cd55636f42f96918a2f1f757e48b51f0f0604b7b48b32c12e2cac78357ce589852a50058c25d577e2a62d3588c73c1c69e238995dc4f +LibUnwind.v1.8.3+1.aarch64-linux-musl.tar.gz/md5/1a84ed9c3a364d771a285cd8589a9f4e +LibUnwind.v1.8.3+1.aarch64-linux-musl.tar.gz/sha512/7103c2307b228c608b67aaa78f2541cef26a588dd1b08034bf19e8e27d3d14a09e1392193d3afa9200c23b277ce14df39f8a99578ef2c53a966365383e599722 +LibUnwind.v1.8.3+1.aarch64-unknown-freebsd.tar.gz/md5/74a0def1b15ecbb50830e20b374b4070 +LibUnwind.v1.8.3+1.aarch64-unknown-freebsd.tar.gz/sha512/53515362c4e0de0eadbf7fc852402ff8ddc230464825ba8ec2d92ad1f6c2464a40429f346987b250dc69de29ed36aa67181c24c18c113b67e2691349235c8a94 +LibUnwind.v1.8.3+1.armv6l-linux-gnueabihf.tar.gz/md5/2d249112bf38318f2b2e687454db33fb +LibUnwind.v1.8.3+1.armv6l-linux-gnueabihf.tar.gz/sha512/8c1d82d37dce7a9908c60c1ada49f49fd55dd9a1c894ae64da70338c89e9c0a6f2a86875e8b5a14d271c6233a351ee8be53ac597a451b227d1af339f224152a1 +LibUnwind.v1.8.3+1.armv6l-linux-musleabihf.tar.gz/md5/a6eac99ae0c8bf9fe65964c41cc73f02 +LibUnwind.v1.8.3+1.armv6l-linux-musleabihf.tar.gz/sha512/174a88054bca1dff413d160e609d2c4ed4c0fb2a88f7284ca799d6599e4eda15a7639b6abad549ed5420ad4c03936241a842350f71a8291193a33a3649df6078 +LibUnwind.v1.8.3+1.armv7l-linux-gnueabihf.tar.gz/md5/859b2b6c825e819ef856c6d44c6759e9 +LibUnwind.v1.8.3+1.armv7l-linux-gnueabihf.tar.gz/sha512/9f27a84e2098517d0f9850af1ce3e6ed75aee2743c473c9eca83ff62e0d88a3d617cc0223b42987f264346652efdc637992cfb33bf498cb21e1a18be8885f90f +LibUnwind.v1.8.3+1.armv7l-linux-musleabihf.tar.gz/md5/86f27fe4cff459a885c62230df750983 +LibUnwind.v1.8.3+1.armv7l-linux-musleabihf.tar.gz/sha512/cc755e5ebbd7b8c591a80b8c8fe4147b6d6edf74955cbdf5a6bd54af9a4a76930cc557686f7c804c8ea4603e5c8fc5dd0a436da460f3234d82edb72b281e7bb5 +LibUnwind.v1.8.3+1.i686-linux-gnu.tar.gz/md5/b0123a757efeaac50e5406c763ca9489 +LibUnwind.v1.8.3+1.i686-linux-gnu.tar.gz/sha512/0eaff35a4bcd3a2b7d5219966b14a9bf40d4a80d771bc4f96202c7c2f106b15eac3e11c987bcb4ca5f7d96f439377117bd003e8553f9981d0fd991ebd81fe550 +LibUnwind.v1.8.3+1.i686-linux-musl.tar.gz/md5/7c2ad85168d82a16fb404cad28fa22c4 +LibUnwind.v1.8.3+1.i686-linux-musl.tar.gz/sha512/e3ba5eb3bdf8b862270ad676a04088d7b06a409df36a20647090b64d539a181951156d6a13d41b1d58881bd3e9dda95202753071aa1d25a4af58842bdad1d325 +LibUnwind.v1.8.3+1.powerpc64le-linux-gnu.tar.gz/md5/23cfa2f753df224d0a2854d126406ba3 +LibUnwind.v1.8.3+1.powerpc64le-linux-gnu.tar.gz/sha512/a7add97dfa82e54afd150b5da6eeff58bad0010e24ff80db5c8791ffa538ddfa23eb2b2b5cfd3e2b4a8b8c05b9d77ec47848a55df665ee602be990d8f5e6a995 +LibUnwind.v1.8.3+1.riscv64-linux-gnu.tar.gz/md5/823817e06b13b3f90869896de7f2410f +LibUnwind.v1.8.3+1.riscv64-linux-gnu.tar.gz/sha512/19787fd62d7a4fbaeb672712155ffa9f9d05b0276128c0d1293890e2e79257d37a116370b78f3bfd178072030b0a579f64752592883ed63c1ace82aad8d8ef6f +LibUnwind.v1.8.3+1.x86_64-linux-gnu.tar.gz/md5/2df1dd005ae3ba5592c6b406420dbd6c +LibUnwind.v1.8.3+1.x86_64-linux-gnu.tar.gz/sha512/dd4d6ef149cd04bafd7278abcde018822adaf6274afb07a32c8f8529eaf26fbf87563fab19f2775b5dd043362b1e016becf4987037a27e00f20b2c077c2f0bfb +LibUnwind.v1.8.3+1.x86_64-linux-musl.tar.gz/md5/8ef5e92d9ebdcb21a3e06e504d12c5b4 +LibUnwind.v1.8.3+1.x86_64-linux-musl.tar.gz/sha512/a804f731151491afa37a78aaf7e8e7fd892e487813fa2415a1fe563ade463535690428c2c96caeb4e6a9faca06e05bb8a90dd37b2c2f6e7052353b31060c8a68 +LibUnwind.v1.8.3+1.x86_64-unknown-freebsd.tar.gz/md5/d8d5a3fb82d3edee48fb478c92fce09c +LibUnwind.v1.8.3+1.x86_64-unknown-freebsd.tar.gz/sha512/bb7af9da6f4475711daef9f3238de34d7088e7f9f32e264183b6819ad2d4ecad9ac68d07df09e35fc1f2852b9fc00d9809fea11f4c425244ab64956ce3876d02 +libunwind-1.8.3.tar.gz/md5/13bc7b41462ac6ea157d350eaf6c1503 +libunwind-1.8.3.tar.gz/sha512/732c9b74635cf4e65bc90fd5b9a4b90fe5968c7fcce6cc704295038f0d41b848f78a59d725e813072e78df72ff8fc77001096abdfb9ef1c73e3bd82433987679 diff --git a/deps/checksums/utf8proc b/deps/checksums/utf8proc index 2055d3323b7e1..769514c34a243 100644 --- a/deps/checksums/utf8proc +++ b/deps/checksums/utf8proc @@ -1,2 +1,2 @@ -utf8proc-a1b99daa2a3393884220264c927a48ba1251a9c6.tar.gz/md5/2c404870fdc19982ec5313ee78e478d7 -utf8proc-a1b99daa2a3393884220264c927a48ba1251a9c6.tar.gz/sha512/a6652f5840439fe051d973d9467ca9805dcea8d0ac75a2d35e3f8041c513d6ccd5d205a3873f28d7cb5e33ce6471165850164997f188ca359111963b3aac9a16 +utf8proc-e5e799221b45bbb90f5fdc5c69b6b8dfbf017e78.tar.gz/md5/749152abfd6122032b02e7fb484f5b64 +utf8proc-e5e799221b45bbb90f5fdc5c69b6b8dfbf017e78.tar.gz/sha512/717552acf99d5a9df54c1378749dc3e816a1d6ab6a66704325c3715b00096f891480456c7fa09bd97d32f08d392ed4ef5b7b65da401dd26804cb8431e6a04c07 diff --git a/deps/checksums/zlib b/deps/checksums/zlib index bd651003399b9..4063b37b6a60d 100644 --- a/deps/checksums/zlib +++ b/deps/checksums/zlib @@ -1,38 +1,38 @@ -Zlib.v1.3.1+2.aarch64-apple-darwin.tar.gz/md5/938c376c7513fa48d4b8b78cea741260 -Zlib.v1.3.1+2.aarch64-apple-darwin.tar.gz/sha512/ccece3f5618efe3e3699eb521167e2ee768932ea6f4e411d36619a941af3e9e32394beb260171d557930382f412f9be70f4c69215d3f7e448d4446b1690111ee -Zlib.v1.3.1+2.aarch64-linux-gnu.tar.gz/md5/44a14273caeea9c5cb34ce3e0ba9d1fc -Zlib.v1.3.1+2.aarch64-linux-gnu.tar.gz/sha512/8977bdc225404a01746fc14885e4823b4e2781c73a75e0ee0c8d9ca58b706c6cf9f98647b4e22bb09e7e09640caf4643e5210054a4624e06c76fc3eb2c2a2728 -Zlib.v1.3.1+2.aarch64-linux-musl.tar.gz/md5/dcef6c714555de9b2181b8c5b0a2c668 -Zlib.v1.3.1+2.aarch64-linux-musl.tar.gz/sha512/499701cc0fd1e52f3952da1b3c4377662c54390db9ebd6f5be82ecc0ba8754d2ca42b2f572b3a78ccdef30e527b7bed22c15511944f1299398587c529f8f4619 -Zlib.v1.3.1+2.aarch64-unknown-freebsd.tar.gz/md5/166f8a076a01a6f0979c712d7cec44e8 -Zlib.v1.3.1+2.aarch64-unknown-freebsd.tar.gz/sha512/7a1546b614cb5e2c0566774247269740d881c0a6d22ef6dca8010d77829b4e64594f4e609bb83299fa239d66909a4eb046d6d078268006723747f86e6c733e6b -Zlib.v1.3.1+2.armv6l-linux-gnueabihf.tar.gz/md5/1f0bcb50b545badbc9de1569f51c4668 -Zlib.v1.3.1+2.armv6l-linux-gnueabihf.tar.gz/sha512/1e4bea6fa41300ec770822dcd9335d1393c087db45d128e2c60d9315db01a69c984c98304b83af0725a99ae3a5cac4a273f4eea8a4213454608edbe0e55c74ce -Zlib.v1.3.1+2.armv6l-linux-musleabihf.tar.gz/md5/3a78103181bf8a74dfc0c6f7681bd3de -Zlib.v1.3.1+2.armv6l-linux-musleabihf.tar.gz/sha512/2a7c70266fd5928e46c8d71d95884054eaff2432d9fbce37eef67eb62af2b087f5f9fa3752a5d14f50cd058519d39a1b81450b30786a4f66eafbd16d18ef7b6b -Zlib.v1.3.1+2.armv7l-linux-gnueabihf.tar.gz/md5/4e202f829e7f478451e93da7be2b6f98 -Zlib.v1.3.1+2.armv7l-linux-gnueabihf.tar.gz/sha512/0734bc8a84b039b971a15620adb9b5da77d1b1992fb4c6adf9031fa8c592512645d424d2ce752efdda1f300f871c3d4f3b159794c3725fd113e1acd5512aed59 -Zlib.v1.3.1+2.armv7l-linux-musleabihf.tar.gz/md5/5000d1941b7e32dec4a2d125bbd22fff -Zlib.v1.3.1+2.armv7l-linux-musleabihf.tar.gz/sha512/6abd69ef6878fa6cdcf7fe94e4d7aedaae58d961122e131a414f3aea43b401a3812d9d847ab4b1690e9faf89d577935d7f547484edb6cb2814cbc1156159e8ed -Zlib.v1.3.1+2.i686-linux-gnu.tar.gz/md5/7a5de529294b9d8dba4ac1eeb4cbcbdc -Zlib.v1.3.1+2.i686-linux-gnu.tar.gz/sha512/72d52c4e2f01fe1436b22c854efff83068f6a65a9280556018c77bb843f81902c0c96f30132123d4dd6a66041e9391a418ceec227b2b2411f99a26df76d21c74 -Zlib.v1.3.1+2.i686-linux-musl.tar.gz/md5/d18b442e4d108397482fd852deb4241e -Zlib.v1.3.1+2.i686-linux-musl.tar.gz/sha512/6c367f7c522167db510cf42a84dfcce75fc129bb49800c05b90dfdfa5fb31fa20ed020e165f1b283b81f4568c9bf79d4c41f7ad4e42a3513cb13149a50707114 -Zlib.v1.3.1+2.i686-w64-mingw32.tar.gz/md5/b2c8af112298ae9e635054e4ba22e5ae -Zlib.v1.3.1+2.i686-w64-mingw32.tar.gz/sha512/0a28076fc8cf8daa925f4be76dd0025d01d8ad6bc269f87164749da0c3bea6f4b404ef097a2907ce2c009211d9f8387f844fe5e5b1bd2f6d77c9b35b2b1c7548 -Zlib.v1.3.1+2.powerpc64le-linux-gnu.tar.gz/md5/9ae4feb621ae088c323ff12235bdf5db -Zlib.v1.3.1+2.powerpc64le-linux-gnu.tar.gz/sha512/912134f741fe96217d1b8465510ac82d238d3d8a21519534fb88c568092dcc9eb8e51ef66b7ca56a2a7a881474f04edb7a6f7bf1ebf06bfff7708b3edd3487c0 -Zlib.v1.3.1+2.riscv64-linux-gnu.tar.gz/md5/43b61473a02e492f932ce60f726966a8 -Zlib.v1.3.1+2.riscv64-linux-gnu.tar.gz/sha512/4742503831da6a6b7945c6e5efd96bd6b03b8a63b73e68181e961b6f326ac5d805350219c43f4537165d1af0ac2ed496d5a72edd0c1d11e012ef12385a8f2e5f -Zlib.v1.3.1+2.x86_64-apple-darwin.tar.gz/md5/347a92686d81ed7c022d2e7310babb77 -Zlib.v1.3.1+2.x86_64-apple-darwin.tar.gz/sha512/a59b9c4f63124c893a2a11b886bbe03bfc75846599eb21997652bd62a2f28afe754d16127e964683552423bf1c0da160e46c32d0b584ed07e28f4e91904b9c67 -Zlib.v1.3.1+2.x86_64-linux-gnu.tar.gz/md5/0630f603e35ab56efdef331e29db436b -Zlib.v1.3.1+2.x86_64-linux-gnu.tar.gz/sha512/b936c328fad685c22473ff4cbfcc8bc48772ed9927c5b29c5d0503b95888efd0ca8d891f53cca45da7b5768ed4a1f6994f9e211167f4704c15c688cced90cac5 -Zlib.v1.3.1+2.x86_64-linux-musl.tar.gz/md5/252d8542bb5a53c479f4ffc067847e27 -Zlib.v1.3.1+2.x86_64-linux-musl.tar.gz/sha512/449809acbbff1fcbd89b9689e803f69d1f9cb49860f0b508b69c560cfcb51232640fcff17ede6ea75d9906edb5a8f38139afd890f18a34260ef5dbb5d167af36 -Zlib.v1.3.1+2.x86_64-unknown-freebsd.tar.gz/md5/79fa906629dff81c38b93001a7798040 -Zlib.v1.3.1+2.x86_64-unknown-freebsd.tar.gz/sha512/53dbcce99b2d6ec82ef86d76f3f574db304ab90f1b131c49b2c06f89bd2426afa4a31bfa8455e2ecdad64d4da71fef1b2d79f471efd55a8bbdc29e95c952a289 -Zlib.v1.3.1+2.x86_64-w64-mingw32.tar.gz/md5/92b083205ca44db131b7cf6b9c09eb21 -Zlib.v1.3.1+2.x86_64-w64-mingw32.tar.gz/sha512/1c3b7b414f09b1896c5a08f156c5e55f07ee012cf3f6fe50d5ba116405dcd9a80e5004ee7c774f7cc981e4d1b099efee85e16b8417cef2734cb7c12ec440d09a -zlib-51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf.tar.gz/md5/7ce1b2766499af7d948130113b649028 -zlib-51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf.tar.gz/sha512/79d032b8c93260ce6b9806f2289cdccce67e9d80865b5bb39ac46dadffc8ee009da51c551eead59c56249c7adfa164c1d5ebcf2b10a8645e0b11b5650176cb24 +Zlib.v1.3.2+0.aarch64-apple-darwin.tar.gz/md5/0fe3b69212c48e72cca7d3e0703b6fae +Zlib.v1.3.2+0.aarch64-apple-darwin.tar.gz/sha512/16d10a87358b6406ab2191bf57859ecc414a0a7b07e7e5b1957162eccc512a25def6beb19337167fb7bf2d3e0131d26f8d4cb6b3c3a198b0fd1ef504e92300dd +Zlib.v1.3.2+0.aarch64-linux-gnu.tar.gz/md5/60e72f0455aff281f49d31f1652039c7 +Zlib.v1.3.2+0.aarch64-linux-gnu.tar.gz/sha512/96653ede02fa30c91287cd9adffba3c5ec108d4b2e1837c3d416d90d773d74b3fb6bba99c5f0e362baf1b3a1f717bead74c2df64b3392d0a8da6c754bc873696 +Zlib.v1.3.2+0.aarch64-linux-musl.tar.gz/md5/1834bef7f8974e9d2f38ca5353d12d78 +Zlib.v1.3.2+0.aarch64-linux-musl.tar.gz/sha512/cb6b0812b0a94048e3b3bb62c517454f90f185e203bc3c338dc2a5c7e592a9752f55889c52fb0e733fb494a7fac6800f2fa485a981d36233e77e99f0dda5e964 +Zlib.v1.3.2+0.aarch64-unknown-freebsd.tar.gz/md5/a0e49399e7312c1e8d5dfd9c04401e67 +Zlib.v1.3.2+0.aarch64-unknown-freebsd.tar.gz/sha512/19ce4097446ce7edc879ef42532dab91c6432f13ab89fc65ea61cf33e0d3285940b78b934537c2176e51b2398848db54cf246463e2ff418e8ca57b9256656af4 +Zlib.v1.3.2+0.armv6l-linux-gnueabihf.tar.gz/md5/1d7636c42d90d48af1ffb563c73c3e07 +Zlib.v1.3.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/e230d761aa7d59d610d311a8fb1872ab22e4a81205ecaaa1de996d20962270bed7490a3e009ec288c31967058ddee227fa91e0a55721e7de4d5efd7316f8cec8 +Zlib.v1.3.2+0.armv6l-linux-musleabihf.tar.gz/md5/5336c605bd8098e04f9ff8ebc1a144c5 +Zlib.v1.3.2+0.armv6l-linux-musleabihf.tar.gz/sha512/8ff0576c80762c60f668580d647da23796684a22f767f935768446c41b2fdb2d4f05f78909722a48395fcf64bdfc7bbd6809933a425ed0fbc37140f32a164036 +Zlib.v1.3.2+0.armv7l-linux-gnueabihf.tar.gz/md5/87847d8af9c07d954d3513c7619eddf2 +Zlib.v1.3.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/86865771e52493c59e92c9924a9ec8a6d89cc200d2101956d3171f28a4e43311daee19297a4d17861a8c403bf3d21007b349dae9bf687823f937a90cb9c273cb +Zlib.v1.3.2+0.armv7l-linux-musleabihf.tar.gz/md5/8a36e7392f283da92ddc5aba74d66177 +Zlib.v1.3.2+0.armv7l-linux-musleabihf.tar.gz/sha512/27199aefdabed11182f2753a23fc9bc6c63433cd48ef3873863756d65a891dfe16a36624dd7825032340f20904f2a531b0f5d70c6b8562aab490da503825ac7a +Zlib.v1.3.2+0.i686-linux-gnu.tar.gz/md5/325a535f7d48a83742fb9f0b083824b3 +Zlib.v1.3.2+0.i686-linux-gnu.tar.gz/sha512/cd7b5f354f33fc2da97c7a699316c4de4532983c578c0eb564785e9ef8ad53a2a9a0374d3813106761744a374ac22af2873e21f2502225843bb4ebb3c6fe604a +Zlib.v1.3.2+0.i686-linux-musl.tar.gz/md5/65d34493321b15146b7dc7de20e2e31b +Zlib.v1.3.2+0.i686-linux-musl.tar.gz/sha512/9d942e226d4d16b494e6fee4e4ea81b4d668fe6a39ddfe18163f0a0aaa0072a09707851cee056b7a826bada964624b7388aaa42bd3497e4d128e61f360707534 +Zlib.v1.3.2+0.i686-w64-mingw32.tar.gz/md5/0e434ac0cef64f9aaeae2f23379e4d18 +Zlib.v1.3.2+0.i686-w64-mingw32.tar.gz/sha512/46ef23ce0a6e1acacabaf28b51ac5786ff4306dbcc653b8c0368dbc9d4faf3d2c760326c3d60b58d4cfcff886c395bf4cb3838f6a426ef30005333ed072c8964 +Zlib.v1.3.2+0.powerpc64le-linux-gnu.tar.gz/md5/aa31344defa122e30c47ee809867542d +Zlib.v1.3.2+0.powerpc64le-linux-gnu.tar.gz/sha512/1e9ca7dc2098011e61271c0acb5d83c443fe7cb35effd342df9dcd26fe226f759501af10acb0d82b06718274a06da44cafc7c3a018ea1b480ee6c80b7526db34 +Zlib.v1.3.2+0.riscv64-linux-gnu.tar.gz/md5/406086586a756180ce57376eba522142 +Zlib.v1.3.2+0.riscv64-linux-gnu.tar.gz/sha512/361b1e47b26b0875393b387b93799f44b9dfa0934c3f419a75799c5b5da5680090c81096a7734ca8221065391583537894b862517955553672b0b5d9a1a7b188 +Zlib.v1.3.2+0.x86_64-apple-darwin.tar.gz/md5/107bfcb5fee055c880d8f48734a3b782 +Zlib.v1.3.2+0.x86_64-apple-darwin.tar.gz/sha512/0833a2fdfba45b4e56cb49faf31dad88419cae2492c71d50a9f9f4be7dfd73a185675c11485221bbd6e5dc5f82b8fbc1b3924dfbebbe7df72c0b64531e5c4f32 +Zlib.v1.3.2+0.x86_64-linux-gnu.tar.gz/md5/27239beb46334b85ccf5fa9ef10c1133 +Zlib.v1.3.2+0.x86_64-linux-gnu.tar.gz/sha512/59554aa02d6c150e5c639179aa8e30f70c48ed5cd0208f63254028989697e43a793456163da46913c7f4d2a20f6779b7076d22ba7fe0947dd56abbed1ae31397 +Zlib.v1.3.2+0.x86_64-linux-musl.tar.gz/md5/bcc690fb6c43a73f7a887c39fbba2e29 +Zlib.v1.3.2+0.x86_64-linux-musl.tar.gz/sha512/eef2d2973ac2e10dadef13be6011a49d566a26736bd2be68fe68cc412d191a3aa67aa73921313b563daa716345374acc83973f6f4075e831c2d397d7264b145e +Zlib.v1.3.2+0.x86_64-unknown-freebsd.tar.gz/md5/dc2f717d559f4094d9277d03780af515 +Zlib.v1.3.2+0.x86_64-unknown-freebsd.tar.gz/sha512/d785a829483fcc5b9ccbfef6033c3abf79a744922b4e73f2a3d0e68fc8e53ce8adb3baa88e597fb74df586b8f1b5941d7f2af1bb3b64af8f2b3c82ef5fbf9c2f +Zlib.v1.3.2+0.x86_64-w64-mingw32.tar.gz/md5/96a709d238511d11422d4ddf99d7e837 +Zlib.v1.3.2+0.x86_64-w64-mingw32.tar.gz/sha512/39014563d36407fe2730bdbec6de52433a4c5ebf979a4778d1c8d1b4d60189e2b5e1309e74246df1a308eeca5f76c33dcea480e281071961db7cb112d6758638 +zlib-da607da739fa6047df13e66a2af6b8bec7c2a498.tar.gz/md5/057e303ffc34f861f05a5c4f81fc19b7 +zlib-da607da739fa6047df13e66a2af6b8bec7c2a498.tar.gz/sha512/9ab998e7ec0f5d5b32652ec60500791cbf9a5209453600a077e900ec3f63a9e5c134db75ab602d065665938c46ffef7f4cb6d2588bd51c0eec1d5da46c8b0486 diff --git a/deps/clang.version b/deps/clang.version index 68dafab189f7e..4b315e567f41f 100644 --- a/deps/clang.version +++ b/deps/clang.version @@ -3,4 +3,4 @@ ## jll artifact # Clang (paired with LLVM, only here as a JLL download) CLANG_JLL_NAME := Clang -CLANG_JLL_VER := 20.1.2+0 +CLANG_JLL_VER := 22.1.8+2 diff --git a/deps/compilerrt.version b/deps/compilerrt.version new file mode 100644 index 0000000000000..0275a6da252b5 --- /dev/null +++ b/deps/compilerrt.version @@ -0,0 +1,6 @@ +# -*- makefile -*- + +## jll artifact +# Bundled compiler-rt builtins archive (libclang_rt.osx.a on macOS) +COMPILERRT_JLL_NAME := LLVMCompilerRT +COMPILERRT_JLL_VER := 17.0.6+0 diff --git a/deps/cpufeatures.mk b/deps/cpufeatures.mk new file mode 100644 index 0000000000000..51e2c47658d4f --- /dev/null +++ b/deps/cpufeatures.mk @@ -0,0 +1,34 @@ +## CPUFEATURES ## +CPUFEATURES_GIT_URL := https://github.com/JuliaLang/cpufeatures.git +CPUFEATURES_TAR_URL = https://api.github.com/repos/JuliaLang/cpufeatures/tarball/$1 +$(eval $(call git-external,cpufeatures,CPUFEATURES,,,$(BUILDDIR))) + +CPUFEATURES_CFLAGS := -O2 -Wall -Wextra $(fPIC) $(SANITIZE_OPTS) +CPUFEATURES_CXXFLAGS := -std=c++17 -O2 -Wall -Wextra -fno-exceptions -fno-rtti $(fPIC) $(SANITIZE_OPTS) +CPUFEATURES_MFLAGS := CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS) $(CPUFEATURES_CFLAGS)" CXXFLAGS="$(CXXFLAGS) $(CPUFEATURES_CXXFLAGS)" AR="$(AR)" ARCH=$(ARCH) +CPUFEATURES_BUILDDIR := $(BUILDDIR)/$(CPUFEATURES_SRC_DIR) + +$(CPUFEATURES_BUILDDIR)/build-compiled: $(CPUFEATURES_BUILDDIR)/source-extracted + $(MAKE) -C $(dir $<) $(CPUFEATURES_MFLAGS) lib + echo 1 > $@ + +define CPUFEATURES_INSTALL + mkdir -p $2/$$(build_includedir)/cpufeatures + mkdir -p $2/$$(build_libdir) + cp $1/include/*.h $2/$$(build_includedir)/cpufeatures/ + cp $1/generated/target_tables_*.h $2/$$(build_includedir)/cpufeatures/ + cp $1/build/libtarget_parsing.a $2/$$(build_libdir)/ +endef +$(eval $(call staged-install, \ + cpufeatures,$(CPUFEATURES_SRC_DIR), \ + CPUFEATURES_INSTALL,,,)) + +clean-cpufeatures: + -rm -f $(CPUFEATURES_BUILDDIR)/build-compiled + +get-cpufeatures: $(CPUFEATURES_SRC_FILE) +extract-cpufeatures: $(CPUFEATURES_BUILDDIR)/source-extracted +configure-cpufeatures: extract-cpufeatures +compile-cpufeatures: $(CPUFEATURES_BUILDDIR)/build-compiled +fastcheck-cpufeatures: check-cpufeatures +check-cpufeatures: compile-cpufeatures diff --git a/deps/cpufeatures.version b/deps/cpufeatures.version new file mode 100644 index 0000000000000..90d9808057ebb --- /dev/null +++ b/deps/cpufeatures.version @@ -0,0 +1,4 @@ +# -*- makefile -*- + +CPUFEATURES_BRANCH=main +CPUFEATURES_SHA1=6bb9489f4edddfca6e2478b8bfac4acedc349b89 diff --git a/deps/csl.mk b/deps/csl.mk index 51368187c55fc..67de8c2eb0be1 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -1,6 +1,15 @@ -# Interrogate the fortran compiler (which is always GCC based) on where it is keeping its libraries -STD_LIB_PATH := $(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^programs: =' | sed -e "s/^programs: =//") -STD_LIB_PATH += $(PATHSEP)$(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^libraries: =' | sed -e "s/^libraries: =//") +# Interrogate the fortran compiler (which is always GCC based) on where it is keeping its libraries. +# If there is no functioning fortran compiler, fall back to the C compiler, since many distros +# install the GCC runtime libraries even when gfortran itself is not installed. A GCC-based C +# compiler reports the same search directories; clang (e.g. `CC=clang`) also answers +# `-print-search-dirs` and includes the GCC installation it found, so the lookup works there too. +ifneq ($(shell LANG=C $(FC) -print-search-dirs 2>/dev/null),) +CSL_COMPILER := $(FC) +else +CSL_COMPILER := $(CC) +endif +STD_LIB_PATH := $(shell LANG=C $(CSL_COMPILER) -print-search-dirs 2>/dev/null | grep '^programs: =' | sed -e "s/^programs: =//") +STD_LIB_PATH += $(PATHSEP)$(shell LANG=C $(CSL_COMPILER) -print-search-dirs 2>/dev/null | grep '^libraries: =' | sed -e "s/^libraries: =//") ifeq ($(BUILD_OS),WINNT) # the mingw compiler lies about it search directory paths STD_LIB_PATH += $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g") endif @@ -54,11 +63,20 @@ $$(build_shlibdir)/$(1): | $$(build_shlibdir) [ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_shlibdir)' endef -define copy_csl_static_lib +define copy_csl_static +install-csl: | $$(build_libdir) $$(build_libdir)/$(1) +$$(build_libdir)/$(1): | $$(build_libdir) + -@SRC='$$(call pathsearch,$(1),$$(STD_LIB_PATH))'; \ + [ -n "$$$${SRC}" ] && cp "$$$${SRC}" '$$(build_libdir)' +endef + +# Like copy_csl_static, but stages into $(build_private_libdir) instead of +# $(build_libdir). See JuliaLang/julia#51698. +define copy_csl_static_private install-csl: | $$(build_private_libdir) $$(build_private_libdir)/$(1) $$(build_private_libdir)/$(1): | $$(build_private_libdir) - -@SRC_LIB='$$(call pathsearch,$(1),$$(STD_LIB_PATH))'; \ - [ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_private_libdir)' + -@SRC='$$(call pathsearch,$(1),$$(STD_LIB_PATH))'; \ + [ -n "$$$${SRC}" ] && cp "$$$${SRC}" '$$(build_private_libdir)' endef # libgfortran has multiple names; we're just going to copy any version we can find @@ -71,19 +89,43 @@ $(eval $(call copy_csl,$(call versioned_libname,libgfortran,5))) $(eval $(call copy_csl,$(call versioned_libname,libquadmath,0))) $(eval $(call copy_csl,$(call versioned_libname,libstdc++,6))) $(eval $(call copy_csl,$(call versioned_libname,libatomic,1))) +$(eval $(call copy_csl,libatomic.$(SHLIB_EXT))) $(eval $(call copy_csl,$(call versioned_libname,libgomp,1))) # Configurable either a static or dynamic library depending on the system $(eval $(call copy_csl,$(call versioned_libname,libssp,0))) -$(eval $(call copy_csl_static_lib,libssp.a)) ifeq ($(OS),WINNT) -# On windows we need the static gcc runtime libraries for linking pkgimages -$(eval $(call copy_csl_static_lib,libgcc.a)) -$(eval $(call copy_csl_static_lib,libgcc_s.a)) -$(eval $(call copy_csl_static_lib,libmsvcrt.a)) -$(eval $(call copy_csl_static_lib,libmingwex.a)) -$(eval $(call copy_csl_static_lib,libkernel32.a)) +# These are static gcc runtime libraries / objects for direct Windows links +$(eval $(call copy_csl_static_private,libgcc.a)) +$(eval $(call copy_csl_static_private,libgcc_s.a)) +$(eval $(call copy_csl_static_private,libmsvcrt.a)) +$(eval $(call copy_csl_static_private,libmsvcrt-os.a)) +$(eval $(call copy_csl_static_private,libmingwex.a)) +$(eval $(call copy_csl_static_private,libkernel32.a)) +$(eval $(call copy_csl_static_private,libmingw32.a)) +$(eval $(call copy_csl_static_private,libmoldname.a)) +$(eval $(call copy_csl_static_private,libntdll.a)) +$(eval $(call copy_csl_static_private,libpsapi.a)) +$(eval $(call copy_csl_static_private,libws2_32.a)) +$(eval $(call copy_csl_static_private,libiphlpapi.a)) +$(eval $(call copy_csl_static_private,libwinmm.a)) +$(eval $(call copy_csl_static_private,libdbghelp.a)) +$(eval $(call copy_csl_static_private,libuserenv.a)) +$(eval $(call copy_csl_static_private,libsecur32.a)) +$(eval $(call copy_csl_static_private,libsynchronization.a)) +$(eval $(call copy_csl_static_private,libole32.a)) +$(eval $(call copy_csl_static_private,libuuid.a)) +$(eval $(call copy_csl_static_private,libadvapi32.a)) +$(eval $(call copy_csl_static_private,libshell32.a)) +$(eval $(call copy_csl_static_private,libuser32.a)) +$(eval $(call copy_csl_static_private,libpthread.dll.a)) +$(eval $(call copy_csl_static_private,libssp.dll.a)) +$(eval $(call copy_csl_static_private,crt2.o)) +$(eval $(call copy_csl_static_private,crt2u.o)) +$(eval $(call copy_csl_static_private,dllcrt2.o)) +$(eval $(call copy_csl_static_private,crtbegin.o)) +$(eval $(call copy_csl_static_private,crtend.o)) # Windows has special gcc_s names ifeq ($(ARCH),i686) $(eval $(call copy_csl,$(call versioned_libname,libgcc_s_sjlj,1))) @@ -106,13 +148,24 @@ endif else # Other targets just use libgcc_s.1 $(eval $(call copy_csl,$(call versioned_libname,libgcc_s,1))) +$(eval $(call copy_csl,libgcc_s.$(SHLIB_EXT))) +# These are static gcc runtime libraries / objects for linking pkgimages +$(eval $(call copy_csl_static,libgcc.a)) +$(eval $(call copy_csl_static,crti.o)) +$(eval $(call copy_csl_static,crtn.o)) +$(eval $(call copy_csl_static,crtbeginS.o)) +$(eval $(call copy_csl_static,crtendS.o)) +ifeq ($(LIBC),glibc) +$(eval $(call copy_csl_static,libc_nonshared.a)) +endif endif endif -# winpthread is only Windows, pthread is only others +# winpthread is only Windows, pthread is only others. Not on Linux, where libpthread is +# part of glibc: a copy from a toolchain's sysroot would not match the system's libc. ifeq ($(OS),WINNT) $(eval $(call copy_csl,$(call versioned_libname,libwinpthread,1))) -else +else ifneq ($(OS),Linux) $(eval $(call copy_csl,$(call versioned_libname,libpthread,0))) endif @@ -132,16 +185,50 @@ distclean-csl: clean-csl else $(eval $(call bb-install,csl,CSL,true)) +GCC_VERSION = 15 ifeq ($(OS),WINNT) -GCC_VERSION = 14 install-csl: mkdir -p $(build_private_libdir)/ cp -a $(build_shlibdir)/$(call versioned_libname,libstdc++,6) $(build_shlibdir)/libstdc++.$(SHLIB_EXT) cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc_s.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmsvcrt.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmsvcrt-os.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmingwex.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libkernel32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmingw32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmoldname.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libntdll.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libpsapi.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libws2_32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libiphlpapi.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libwinmm.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libdbghelp.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libuserenv.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libsecur32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libsynchronization.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libole32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libuuid.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libadvapi32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libshell32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libuser32.a $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libpthread.dll.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libssp.dll.a $(build_private_libdir)/ - cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libssp.dll.a $(build_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crt2.o $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crt2u.o $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/dllcrt2.o $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crtbegin.o $(build_private_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crtend.o $(build_private_libdir)/ +else ifneq ($(OS),Darwin) +install-csl: + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc.a $(build_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crti.o $(build_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crtn.o $(build_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crtbeginS.o $(build_libdir)/ + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/crtendS.o $(build_libdir)/ +ifeq ($(LIBC),glibc) + cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libc_nonshared.a $(build_libdir)/ +endif endif endif @@ -152,7 +239,42 @@ uninstall-gcc-libraries: -rm -f $(build_private_libdir)/libgcc_s.a -rm -f $(build_private_libdir)/libgcc.a -rm -f $(build_private_libdir)/libmsvcrt.a + -rm -f $(build_private_libdir)/libmsvcrt-os.a + -rm -f $(build_private_libdir)/libmingwex.a + -rm -f $(build_private_libdir)/libkernel32.a + -rm -f $(build_private_libdir)/libmingw32.a + -rm -f $(build_private_libdir)/libmoldname.a + -rm -f $(build_private_libdir)/libntdll.a + -rm -f $(build_private_libdir)/libpsapi.a + -rm -f $(build_private_libdir)/libws2_32.a + -rm -f $(build_private_libdir)/libiphlpapi.a + -rm -f $(build_private_libdir)/libwinmm.a + -rm -f $(build_private_libdir)/libdbghelp.a + -rm -f $(build_private_libdir)/libuserenv.a + -rm -f $(build_private_libdir)/libsecur32.a + -rm -f $(build_private_libdir)/libsynchronization.a + -rm -f $(build_private_libdir)/libole32.a + -rm -f $(build_private_libdir)/libuuid.a + -rm -f $(build_private_libdir)/libadvapi32.a + -rm -f $(build_private_libdir)/libshell32.a + -rm -f $(build_private_libdir)/libuser32.a + -rm -f $(build_private_libdir)/libpthread.dll.a -rm -f $(build_private_libdir)/libssp.dll.a - -rm -f $(build_libdir)/libssp.dll.a + -rm -f $(build_private_libdir)/crt2.o + -rm -f $(build_private_libdir)/crt2u.o + -rm -f $(build_private_libdir)/dllcrt2.o + -rm -f $(build_private_libdir)/crtbegin.o + -rm -f $(build_private_libdir)/crtend.o +.PHONY: uninstall-gcc-libraries +else ifneq ($(OS),Darwin) +uninstall-csl: uninstall-gcc-libraries +uninstall-gcc-libraries: + -rm -f $(build_libdir)/libgcc.a + -rm -f $(build_libdir)/libc_nonshared.a + -rm -f $(build_shlibdir)/libgcc_s.$(SHLIB_EXT) + -rm -f $(build_libdir)/crti.o + -rm -f $(build_libdir)/crtn.o + -rm -f $(build_libdir)/crtbeginS.o + -rm -f $(build_libdir)/crtendS.o .PHONY: uninstall-gcc-libraries endif diff --git a/deps/csl.version b/deps/csl.version index 51af26c566c92..948a9c559bb88 100644 --- a/deps/csl.version +++ b/deps/csl.version @@ -1,2 +1,4 @@ +# -*- makefile -*- + ## jll artifact CSL_JLL_NAME := CompilerSupportLibraries diff --git a/deps/curl.mk b/deps/curl.mk index 1f650dce9370e..231be2835acd0 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -37,6 +37,9 @@ $(SRCCACHE)/curl-$(CURL_VER)/source-extracted: $(SRCCACHE)/curl-$(CURL_VER).tar. checksum-curl: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2 $(JLCHECKSUM) $< +$(SRCCACHE)/curl-$(CURL_VER)/source-patched: $(SRCCACHE)/curl-$(CURL_VER)/source-extracted + echo 1 > $@ + ## xref: https://github.com/JuliaPackaging/Yggdrasil/blob/master/L/LibCURL/common.jl # Disable....almost everything CURL_CONFIGURE_FLAGS := $(CONFIGURE_COMMON) \ @@ -54,15 +57,19 @@ CURL_CONFIGURE_FLAGS += \ # We use different TLS libraries on different platforms. # On Windows, we use schannel +# On macOS, we use OpenSSL and validate using SecTrust by default # On other platforms, we use OpenSSL ifeq ($(OS), WINNT) CURL_TLS_CONFIGURE_FLAGS := --with-schannel +else ifeq ($(OS), Darwin) +CURL_TLS_CONFIGURE_FLAGS := --with-openssl +CURL_TLS_CONFIGURE_FLAGS += --with-apple-sectrust else CURL_TLS_CONFIGURE_FLAGS := --with-openssl endif CURL_CONFIGURE_FLAGS += $(CURL_TLS_CONFIGURE_FLAGS) -$(BUILDDIR)/curl-$(CURL_VER)/build-configured: $(SRCCACHE)/curl-$(CURL_VER)/source-extracted +$(BUILDDIR)/curl-$(CURL_VER)/build-configured: $(SRCCACHE)/curl-$(CURL_VER)/source-patched mkdir -p $(dir $@) cd $(dir $@) && \ $(dir $<)/configure $(CURL_CONFIGURE_FLAGS) \ @@ -86,7 +93,7 @@ $(eval $(call staged-install, \ clean-curl: -rm -f $(BUILDDIR)/curl-$(CURL_VER)/build-configured $(BUILDDIR)/curl-$(CURL_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/curl-$(CURL_VER) clean + -if [ -d $(BUILDDIR)/curl-$(CURL_VER) ]; then $(MAKE) -C $(BUILDDIR)/curl-$(CURL_VER) clean; fi distclean-curl: rm -rf $(SRCCACHE)/curl-$(CURL_VER).tar.bz2 $(SRCCACHE)/curl-$(CURL_VER) $(BUILDDIR)/curl-$(CURL_VER) diff --git a/deps/curl.version b/deps/curl.version index ecd8089d9fd1f..7a305ea1e6f52 100644 --- a/deps/curl.version +++ b/deps/curl.version @@ -3,4 +3,4 @@ CURL_JLL_NAME := LibCURL ## source build -CURL_VER := 8.15.0 +CURL_VER := 8.22.0 diff --git a/deps/gfortblas.c b/deps/gfortblas.c index 321fe124d7e87..cbf31fe8f2e98 100644 --- a/deps/gfortblas.c +++ b/deps/gfortblas.c @@ -31,7 +31,7 @@ * they should. This breaks them in 64-bit builds on the x86_64 * architecture. * - * Newer gfortran compoilers no longer default to the F2C calling + * Newer gfortran compilers no longer default to the F2C calling * convention. These wrappers map the F2C conformant functions in * libBLAS and libLAPACK to the native gfortran calling convention, so * that the libraries can be used with software built for x86_64 diff --git a/deps/gmp.mk b/deps/gmp.mk index 23075c861cd35..6942d2e3db3a3 100644 --- a/deps/gmp.mk +++ b/deps/gmp.mk @@ -6,6 +6,8 @@ ifneq ($(USE_BINARYBUILDER_GMP),1) GMP_CONFIGURE_OPTS := $(CONFIGURE_COMMON) GMP_CONFIGURE_OPTS += --enable-cxx --enable-shared --disable-static GMP_CONFIGURE_OPTS += CC_FOR_BUILD="$(HOSTCC)" +# gmp 6.3.0's configure probes are not C23-clean and reject GCC >= 15 as "not working" +GMP_CONFIGURE_OPTS += CC="$(CC) $(SANITIZE_OPTS) -std=gnu11" ifeq ($(BUILD_ARCH),x86_64) GMP_CONFIGURE_OPTS += --enable-fat @@ -45,7 +47,12 @@ $(SRCCACHE)/gmp-$(GMP_VER)/gmp-alloc_overflow.patch-applied: $(SRCCACHE)/gmp-$(G patch -p1 -f < $(SRCDIR)/patches/gmp-alloc_overflow.patch echo 1 > $@ -$(SRCCACHE)/gmp-$(GMP_VER)/source-patched: $(SRCCACHE)/gmp-$(GMP_VER)/gmp-alloc_overflow.patch-applied +$(SRCCACHE)/gmp-$(GMP_VER)/gmp-mpz_realloc.patch-applied: $(SRCCACHE)/gmp-$(GMP_VER)/gmp-alloc_overflow.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/gmp-mpz_realloc.patch + echo 1 > $@ + +$(SRCCACHE)/gmp-$(GMP_VER)/source-patched: $(SRCCACHE)/gmp-$(GMP_VER)/gmp-mpz_realloc.patch-applied echo 1 > $@ $(BUILDDIR)/gmp-$(GMP_VER)/build-configured: $(SRCCACHE)/gmp-$(GMP_VER)/source-patched @@ -72,7 +79,7 @@ $(eval $(call staged-install, \ clean-gmp: -rm -f $(BUILDDIR)/gmp-$(GMP_VER)/build-configured $(BUILDDIR)/gmp-$(GMP_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/gmp-$(GMP_VER) clean + -if [ -d $(BUILDDIR)/gmp-$(GMP_VER) ]; then $(MAKE) -C $(BUILDDIR)/gmp-$(GMP_VER) clean; fi distclean-gmp: rm -rf $(SRCCACHE)/gmp-$(GMP_VER).tar.bz2 \ diff --git a/deps/ittapi.mk b/deps/ittapi.mk index f27ae2d6f77d5..788f60c156b0e 100644 --- a/deps/ittapi.mk +++ b/deps/ittapi.mk @@ -5,7 +5,7 @@ ITTAPI_GIT_URL := https://github.com/intel/ittapi.git ITTAPI_TAR_URL = https://api.github.com/repos/intel/ittapi/tarball/$1 $(eval $(call git-external,ittapi,ITTAPI,CMakeLists.txt,,$(SRCCACHE))) -ITTAPI_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DITT_API_IPT_SUPPORT= -DITT_API_FORTRAN_SUPPORT=0 +ITTAPI_OPTS := $(CMAKE_COMMON) -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_BUILD_TYPE=Release -DITT_API_IPT_SUPPORT= -DITT_API_FORTRAN_SUPPORT=0 $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-configured: $(SRCCACHE)/$(ITTAPI_SRC_DIR)/source-extracted mkdir -p $(dir $@) diff --git a/deps/jlutilities/documenter/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml index 55ff511a39daa..5315f75fca0e7 100644 --- a/deps/jlutilities/documenter/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -1,295 +1,535 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.13.0-DEV" -manifest_format = "2.0" +julia_version = "1.14.0-DEV" +manifest_format = "2.1" project_hash = "1e9ffa7d4739f7d125a5e2c66af8747a8effd889" [[deps.ANSIColoredPrinters]] git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" +registries = "General" uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" version = "0.0.1" + [deps.ANSIColoredPrinters.syntax] + julia_version = "1.0.0" + [[deps.AbstractTrees]] git-tree-sha1 = "2d9c9a55f9c93e8887ad391fbae72f8ef55e1177" +registries = "General" uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" version = "0.4.5" + [deps.AbstractTrees.syntax] + julia_version = "1.0.0" + [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" +version = "1.2.0" + + [deps.ArgTools.syntax] + julia_version = "1.3.0" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" + [deps.Artifacts.syntax] + julia_version = "1.14.0-DEV" + [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" + [deps.Base64.syntax] + julia_version = "1.14.0-DEV" + [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" +git-tree-sha1 = "970758a3d591a2a5c2a907c53f2e2f8c1b1d3537" +registries = "General" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.8" +version = "0.7.9" + + [deps.CodecZlib.syntax] + julia_version = "1.6.0" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.5.7+0" + + [deps.CompilerSupportLibraries_jll.syntax] + julia_version = "1.6.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" + [deps.Dates.syntax] + julia_version = "1.14.0-DEV" + [[deps.DocStringExtensions]] -git-tree-sha1 = "e7b7e6f178525d17c720ab9c081e4ef04429f860" +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +registries = "General" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.4" +version = "0.9.5" + + [deps.DocStringExtensions.syntax] + julia_version = "1.0.0" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "6c182d0bd94142d7cbc3ae8a1e74668f15d0dd65" +git-tree-sha1 = "191e6bef0cf32cac3a3913cd4787d851715254c2" +registries = "General" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.11.4" +version = "1.19.0" + + [deps.Documenter.syntax] + julia_version = "1.6.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.7.0" + [deps.Downloads.syntax] + julia_version = "1.10.0" + [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" +git-tree-sha1 = "f4d39eee89f1e58c26bf447f1d4156c0125d6838" +registries = "General" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.5+0" +version = "2.8.3+0" + + [deps.Expat_jll.syntax] + julia_version = "1.6.0" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" + [deps.FileWatching.syntax] + julia_version = "1.14.0-DEV" + [[deps.Git]] -deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] -git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" +deps = ["Git_LFS_jll", "Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "824a1890086880696fc908fe12a17bcf61738bd8" +registries = "General" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.4.0" +version = "1.5.0" + + [deps.Git.syntax] + julia_version = "1.6.6" + +[[deps.Git_LFS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8c66e385d631bb934ff05e76d4a566c640c8df69" +registries = "General" +uuid = "020c3dae-16b3-5ae5-87b3-4cb189e250b2" +version = "3.7.1+0" + + [deps.Git_LFS_jll.syntax] + julia_version = "1.6.0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "2f6d6f7e6d6de361865d4394b802c02fc944fc7c" +git-tree-sha1 = "7b16700f9e313c0d972d1222ede50a2076aa0770" +registries = "General" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.49.0+0" +version = "2.55.0+0" + + [deps.Git_jll.syntax] + julia_version = "1.6.0" [[deps.IOCapture]] deps = ["Logging", "Random"] -git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770" +git-tree-sha1 = "0ee181ec08df7d7c911901ea38baf16f755114dc" +registries = "General" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.5" +version = "1.0.0" + + [deps.IOCapture.syntax] + julia_version = "1.0.0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" + [deps.InteractiveUtils.syntax] + julia_version = "1.14.0-DEV" + [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" +git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" +registries = "General" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.0" +version = "1.8.0" + + [deps.JLLWrappers.syntax] + julia_version = "1.0.0" [[deps.JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "c7345ab1a7ca4dc8a02c9f6510da0d9857bbe513" +registries = "General" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.4" +version = "1.7.1" + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + + [deps.JSON.syntax] + julia_version = "1.9.0" + + [deps.JSON.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" [[deps.JuliaSyntaxHighlighting]] deps = ["StyledStrings"] uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" -version = "1.12.0" +version = "1.13.0" + + [deps.JuliaSyntaxHighlighting.syntax] + julia_version = "1.12.0" [[deps.LazilyInitializedFields]] git-tree-sha1 = "0f2da712350b020bc3957f269c9caad516383ee0" +registries = "General" uuid = "0e77f7df-68c5-4e49-93ce-4cd80f5598bf" version = "1.3.0" + [deps.LazilyInitializedFields.syntax] + julia_version = "1.6.0" + [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" +version = "1.0.0" + + [deps.LibCURL.syntax] + julia_version = "1.3.0" [[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "Zstd_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.14.1+1" +version = "8.22.0+0" + + [deps.LibCURL_jll.syntax] + julia_version = "1.11.0" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" + [deps.LibGit2.syntax] + julia_version = "1.14.0-DEV" + [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.1+0" +version = "1.9.7+0" + + [deps.LibGit2_jll.syntax] + julia_version = "1.9.0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.3+1" +version = "1.11.104+0" + + [deps.LibSSH2_jll.syntax] + julia_version = "1.8.0" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" + [deps.Libdl.syntax] + julia_version = "1.14.0-DEV" + [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809" +registries = "General" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" version = "1.18.0+0" + [deps.Libiconv_jll.syntax] + julia_version = "1.6.0" + [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" + [deps.Logging.syntax] + julia_version = "1.14.0-DEV" + [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" version = "1.11.0" + [deps.Markdown.syntax] + julia_version = "1.14.0-DEV" + [[deps.MarkdownAST]] deps = ["AbstractTrees", "Markdown"] -git-tree-sha1 = "465a70f0fc7d443a00dcdc3267a497397b8a3899" +git-tree-sha1 = "93c718d892e73931841089cdc0e982d6dd9cc87b" +registries = "General" uuid = "d0879d2d-cac2-40c8-9cee-1863dc0c7391" -version = "0.1.2" +version = "0.1.3" -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" + [deps.MarkdownAST.syntax] + julia_version = "1.0.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2025.5.20" +version = "2026.8.13" + + [deps.MozillaCACerts_jll.syntax] + julia_version = "1.14.0-DEV" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" + [deps.NetworkOptions.syntax] + julia_version = "1.12.0" + [[deps.OpenSSH_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] -git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +git-tree-sha1 = "2da18ab26a6eb38b374c16b157d5a4cc3ab74e02" +registries = "General" uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" -version = "10.0.1+0" +version = "10.5.1+0" + + [deps.OpenSSH_jll.syntax] + julia_version = "1.6.0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.0+0" +version = "3.5.8+0" + + [deps.OpenSSL_jll.syntax] + julia_version = "1.6.0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.45.0+0" +version = "10.48.0+0" + + [deps.PCRE2_jll.syntax] + julia_version = "1.6.0" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" +git-tree-sha1 = "3de8f5e6e90ebfa8d6d1f86997d6cdcd6a912ff3" +registries = "General" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.3" +version = "2.8.7" + + [deps.Parsers.syntax] + julia_version = "1.6.0" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["ArgTools", "Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "Zstd_jll", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.13.0" +version = "1.14.0" weakdeps = ["REPL"] [deps.Pkg.extensions] REPLExt = "REPL" + [deps.Pkg.syntax] + julia_version = "1.12.0" + [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "516f18f048a195409d6e072acf879a9f017d3900" +git-tree-sha1 = "edbeefc7a4889f528644251bdb5fc9ab5348bc2c" +registries = "General" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.3.2" +version = "1.3.4" + + [deps.PrecompileTools.syntax] + julia_version = "1.12.0" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +registries = "General" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" +version = "1.5.2" + + [deps.Preferences.syntax] + julia_version = "1.0.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" + [deps.Printf.syntax] + julia_version = "1.14.0-DEV" + [[deps.REPL]] -deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +deps = ["Base64", "Dates", "FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" + [deps.REPL.syntax] + julia_version = "1.14.0-DEV" + [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" version = "1.11.0" + [deps.Random.syntax] + julia_version = "1.14.0-DEV" + [[deps.RegistryInstances]] deps = ["LazilyInitializedFields", "Pkg", "TOML", "Tar"] git-tree-sha1 = "ffd19052caf598b8653b99404058fce14828be51" +registries = "General" uuid = "2792f1a3-b283-48e8-9a74-f99dce5104f3" version = "0.1.0" + [deps.RegistryInstances.syntax] + julia_version = "1.6.0" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" +version = "1.13.0" + + [deps.SHA.syntax] + julia_version = "1.0.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" + [deps.Serialization.syntax] + julia_version = "1.14.0-DEV" + [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" + [deps.Sockets.syntax] + julia_version = "1.14.0-DEV" + +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "2d0fc55c61321ba245c47be599570d11bac50303" +registries = "General" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.8.5" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsStaticArraysCoreExt = ["StaticArraysCore"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.syntax] + julia_version = "1.9.0" + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + [[deps.StyledStrings]] uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" +version = "1.13.0" + + [deps.StyledStrings.syntax] + julia_version = "1.11.0" [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" version = "1.0.3" + [deps.TOML.syntax] + julia_version = "1.6.0" + [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" version = "1.10.0" + [deps.Tar.syntax] + julia_version = "1.3.0" + [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" version = "1.11.0" + [deps.Test.syntax] + julia_version = "1.14.0-DEV" + [[deps.TranscodingStreams]] git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +registries = "General" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" version = "0.11.3" + [deps.TranscodingStreams.syntax] + julia_version = "1.6.0" + [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" version = "1.11.0" + [deps.UUIDs.syntax] + julia_version = "1.14.0-DEV" + [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" version = "1.11.0" + [deps.Unicode.syntax] + julia_version = "1.14.0-DEV" + [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.3.1+2" +version = "1.3.2+0" + + [deps.Zlib_jll.syntax] + julia_version = "1.6.0" + +[[deps.Zstd_jll]] +deps = ["CompilerSupportLibraries_jll", "Libdl"] +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + + [deps.Zstd_jll.syntax] + julia_version = "1.6.0" [[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.65.0+0" +version = "1.70.0+0" + + [deps.nghttp2_jll.syntax] + julia_version = "1.11.0" [[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.5.0+2" +version = "17.8.2+0" + + [deps.p7zip_jll.syntax] + julia_version = "1.6.0" + +[registries.General] +url = "https://github.com/JuliaRegistries/General.git" +uuid = "23338594-aafe-5451-b93e-139f81909106" diff --git a/deps/jlutilities/juliac/Manifest.toml b/deps/jlutilities/juliac/Manifest.toml new file mode 100644 index 0000000000000..5fb5335128262 --- /dev/null +++ b/deps/jlutilities/juliac/Manifest.toml @@ -0,0 +1,399 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.14.0-DEV" +manifest_format = "2.1" +project_hash = "d3f4758b27fc8d19cd1b7603061a3b69d4285c69" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.2.0" + + [deps.ArgTools.syntax] + julia_version = "1.3.0" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + + [deps.Artifacts.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + + [deps.Base64.syntax] + julia_version = "1.14.0-DEV" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.5.7+0" + + [deps.CompilerSupportLibraries_jll.syntax] + julia_version = "1.6.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + + [deps.Dates.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + + [deps.Downloads.syntax] + julia_version = "1.10.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + + [deps.FileWatching.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Glob]] +git-tree-sha1 = "246c628cec062230b7d183aab88841fa94fcabe9" +registries = "General" +uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" +version = "1.5.0" + + [deps.Glob.syntax] + julia_version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" +registries = "General" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.8.0" + + [deps.JLLWrappers.syntax] + julia_version = "1.0.0" + +[[deps.JuliaC]] +deps = ["LIEF_Patchelf_jll", "LazyArtifacts", "Libdl", "Mmap", "ObjectFile", "PackageCompiler", "Pkg", "Preferences", "RelocatableFolders", "SHA", "StructIO", "TOML"] +git-tree-sha1 = "b23c19d50a0f47d4cab16299c760ca842cd47599" +repo-rev = "main" +repo-url = "https://github.com/JuliaLang/JuliaC.jl.git" +uuid = "acedd4c2-ced6-4a15-accc-2607eb759ba2" +version = "0.3.10" + + [deps.JuliaC.syntax] + julia_version = "1.10.0" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.13.0" + + [deps.JuliaSyntaxHighlighting.syntax] + julia_version = "1.12.0" + +[[deps.LIEF_Patchelf_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "dcc5301d81b1e7f1c9768aa2658fcd7ca36ff231" +registries = "General" +uuid = "d7f80cea-2d23-50ed-9ab7-0409554cddf2" +version = "1.0.0+1" + + [deps.LIEF_Patchelf_jll.syntax] + julia_version = "1.6.0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + + [deps.LazyArtifacts.syntax] + julia_version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "1.0.0" + + [deps.LibCURL.syntax] + julia_version = "1.3.0" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "Zstd_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.22.0+0" + + [deps.LibCURL_jll.syntax] + julia_version = "1.11.0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + + [deps.LibGit2.syntax] + julia_version = "1.14.0-DEV" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.7+0" + + [deps.LibGit2_jll.syntax] + julia_version = "1.9.0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.104+0" + + [deps.LibSSH2_jll.syntax] + julia_version = "1.8.0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + + [deps.Libdl.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + + [deps.Logging.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + + [deps.Markdown.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Mmap]] +deps = ["Serialization"] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + + [deps.Mmap.syntax] + julia_version = "1.14.0-DEV" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2026.8.13" + + [deps.MozillaCACerts_jll.syntax] + julia_version = "1.14.0-DEV" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + + [deps.NetworkOptions.syntax] + julia_version = "1.12.0" + +[[deps.ObjectFile]] +deps = ["Reexport", "StructIO"] +git-tree-sha1 = "22faba70c22d2f03e60fbc61da99c4ebfc3eb9ba" +registries = "General" +uuid = "d8793406-e978-5875-9003-1fc021f44a92" +version = "0.5.0" + + [deps.ObjectFile.syntax] + julia_version = "1.6.0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.8+0" + + [deps.OpenSSL_jll.syntax] + julia_version = "1.6.0" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.48.0+0" + + [deps.PCRE2_jll.syntax] + julia_version = "1.6.0" + +[[deps.PackageCompiler]] +deps = ["Artifacts", "Glob", "LazyArtifacts", "Libdl", "Pkg", "Printf", "RelocatableFolders", "TOML", "UUIDs", "p7zip_jll"] +git-tree-sha1 = "513193b976208b1b646ad4303323153e8bff0997" +registries = "General" +uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d" +version = "2.4.0" + + [deps.PackageCompiler.syntax] + julia_version = "1.10.0" + +[[deps.Pkg]] +deps = ["ArgTools", "Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "Zstd_jll", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.14.0" + + [deps.Pkg.extensions] + REPLExt = "REPL" + + [deps.Pkg.syntax] + julia_version = "1.12.0" + + [deps.Pkg.weakdeps] + REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +registries = "General" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + + [deps.Preferences.syntax] + julia_version = "1.0.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + + [deps.Printf.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + + [deps.Random.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +registries = "General" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + + [deps.Reexport.syntax] + julia_version = "1.0.0" + +[[deps.RelocatableFolders]] +deps = ["SHA", "Scratch"] +git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864" +registries = "General" +uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" +version = "1.0.1" + + [deps.RelocatableFolders.syntax] + julia_version = "1.0.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "1.13.0" + + [deps.SHA.syntax] + julia_version = "1.0.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" +registries = "General" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.3.0" + + [deps.Scratch.syntax] + julia_version = "1.9.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + + [deps.Serialization.syntax] + julia_version = "1.14.0-DEV" + +[[deps.StructIO]] +git-tree-sha1 = "c581be48ae1cbf83e899b14c07a807e1787512cc" +registries = "General" +uuid = "53d494c1-5632-5724-8f4c-31dff12d585f" +version = "0.3.1" + + [deps.StructIO.syntax] + julia_version = "1.0.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.13.0" + + [deps.StyledStrings.syntax] + julia_version = "1.11.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + + [deps.TOML.syntax] + julia_version = "1.6.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + + [deps.Tar.syntax] + julia_version = "1.3.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + + [deps.UUIDs.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + + [deps.Unicode.syntax] + julia_version = "1.14.0-DEV" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.2+0" + + [deps.Zlib_jll.syntax] + julia_version = "1.6.0" + +[[deps.Zstd_jll]] +deps = ["CompilerSupportLibraries_jll", "Libdl"] +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + + [deps.Zstd_jll.syntax] + julia_version = "1.6.0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.70.0+0" + + [deps.nghttp2_jll.syntax] + julia_version = "1.11.0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.8.2+0" + + [deps.p7zip_jll.syntax] + julia_version = "1.6.0" + +[registries.General] +url = "https://github.com/JuliaRegistries/General.git" +uuid = "23338594-aafe-5451-b93e-139f81909106" diff --git a/deps/jlutilities/juliac/Project.toml b/deps/jlutilities/juliac/Project.toml new file mode 100644 index 0000000000000..3b1e7f84d6215 --- /dev/null +++ b/deps/jlutilities/juliac/Project.toml @@ -0,0 +1,5 @@ +[deps] +JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2" + +[sources] +JuliaC = {rev = "main", url = "https://github.com/JuliaLang/JuliaC.jl.git"} diff --git a/deps/jlutilities/objectfile/Manifest.toml b/deps/jlutilities/objectfile/Manifest.toml new file mode 100644 index 0000000000000..af968af762f1c --- /dev/null +++ b/deps/jlutilities/objectfile/Manifest.toml @@ -0,0 +1,37 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.14.0-DEV" +manifest_format = "2.1" +project_hash = "23b8253b8eadb7ba5d7489bb56f38819b7150654" + +[[deps.ObjectFile]] +deps = ["Reexport", "StructIO"] +git-tree-sha1 = "22faba70c22d2f03e60fbc61da99c4ebfc3eb9ba" +registries = "General" +uuid = "d8793406-e978-5875-9003-1fc021f44a92" +version = "0.5.0" + + [deps.ObjectFile.syntax] + julia_version = "1.6.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +registries = "General" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + + [deps.Reexport.syntax] + julia_version = "1.0.0" + +[[deps.StructIO]] +git-tree-sha1 = "c581be48ae1cbf83e899b14c07a807e1787512cc" +registries = "General" +uuid = "53d494c1-5632-5724-8f4c-31dff12d585f" +version = "0.3.1" + + [deps.StructIO.syntax] + julia_version = "1.0.0" + +[registries.General] +url = "https://github.com/JuliaRegistries/General.git" +uuid = "23338594-aafe-5451-b93e-139f81909106" diff --git a/deps/jlutilities/objectfile/Project.toml b/deps/jlutilities/objectfile/Project.toml new file mode 100644 index 0000000000000..acdde4be0965e --- /dev/null +++ b/deps/jlutilities/objectfile/Project.toml @@ -0,0 +1,2 @@ +[deps] +ObjectFile = "d8793406-e978-5875-9003-1fc021f44a92" diff --git a/deps/jlutilities/revise/Manifest.toml b/deps/jlutilities/revise/Manifest.toml index 143f20b08e9c1..44ad3cabcba51 100644 --- a/deps/jlutilities/revise/Manifest.toml +++ b/deps/jlutilities/revise/Manifest.toml @@ -1,159 +1,284 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.13.0-DEV" -manifest_format = "2.0" -project_hash = "6de5e8b1c4d9b467a5c126490dbc755dc0575a9c" +julia_version = "1.14.0-DEV" +manifest_format = "2.1" +project_hash = "6ffe32f6ba6335cf51104ef8b025301b81ced713" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" + [deps.Artifacts.syntax] + julia_version = "1.14.0-DEV" + [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" + [deps.Base64.syntax] + julia_version = "1.14.0-DEV" + +[[deps.CRC32c]] +uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" +version = "1.11.0" + + [deps.CRC32c.syntax] + julia_version = "1.14.0-DEV" + [[deps.CodeTracking]] -deps = ["InteractiveUtils", "UUIDs"] -git-tree-sha1 = "062c5e1a5bf6ada13db96a4ae4749a4c2234f521" +deps = ["InteractiveUtils", "REPL", "UUIDs"] +git-tree-sha1 = "cfb7a2e89e245a9d5016b70323db412b3a7438d5" +registries = "General" uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" -version = "1.3.9" +version = "3.0.2" + + [deps.CodeTracking.syntax] + julia_version = "1.10.0" [[deps.Compiler]] git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +registries = "General" uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" version = "0.1.1" + [deps.Compiler.syntax] + julia_version = "1.10.0" + [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.3.0+1" +version = "1.5.7+0" + + [deps.CompilerSupportLibraries_jll.syntax] + julia_version = "1.6.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + + [deps.Dates.syntax] + julia_version = "1.14.0-DEV" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" + [deps.FileWatching.syntax] + julia_version = "1.14.0-DEV" + [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" + [deps.InteractiveUtils.syntax] + julia_version = "1.14.0-DEV" + [[deps.JuliaInterpreter]] deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] -git-tree-sha1 = "6ac9e4acc417a5b534ace12690bc6973c25b862f" +git-tree-sha1 = "521a88f82e52920a3c72b8f43090e23ab3687657" +repo-rev = "kf/define_method" +repo-url = "https://github.com/KenoAIStaging/JuliaInterpreter.jl.git" uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" -version = "0.10.3" +version = "0.11.4" + + [deps.JuliaInterpreter.syntax] + julia_version = "1.10.0" [[deps.JuliaSyntaxHighlighting]] deps = ["StyledStrings"] uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" -version = "1.12.0" +version = "1.13.0" + + [deps.JuliaSyntaxHighlighting.syntax] + julia_version = "1.12.0" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" + [deps.LibGit2.syntax] + julia_version = "1.14.0-DEV" + [[deps.LibGit2_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.1+0" +version = "1.9.7+0" + + [deps.LibGit2_jll.syntax] + julia_version = "1.9.0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.3+1" +version = "1.11.104+0" + + [deps.LibSSH2_jll.syntax] + julia_version = "1.8.0" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" + [deps.Libdl.syntax] + julia_version = "1.14.0-DEV" + [[deps.LoweredCodeUtils]] -deps = ["Compiler", "JuliaInterpreter"] -git-tree-sha1 = "b882a7dd7ef37643066ae8f9380beea8fdd89cae" +deps = ["CodeTracking", "Compiler", "JuliaInterpreter"] +git-tree-sha1 = "1d4c737ab26f51ceed52ab2019c09b7660eb7440" +repo-rev = "kf/define_method" +repo-url = "https://github.com/KenoAIStaging/LoweredCodeUtils.jl.git" uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" -version = "3.4.2" +version = "3.8.0" + + [deps.LoweredCodeUtils.syntax] + julia_version = "1.10.0" [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" version = "1.11.0" + [deps.Markdown.syntax] + julia_version = "1.14.0-DEV" + [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" + [deps.NetworkOptions.syntax] + julia_version = "1.12.0" + [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" +version = "3.5.8+0" + + [deps.OpenSSL_jll.syntax] + julia_version = "1.6.0" [[deps.OrderedCollections]] -git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +git-tree-sha1 = "05f45c2e0de6259db764adbfd2f1dc6d3f8de13c" +registries = "General" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.8.1" +version = "2.0.1" + + [deps.OrderedCollections.syntax] + julia_version = "1.7.1" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.45.0+0" +version = "10.48.0+0" + + [deps.PCRE2_jll.syntax] + julia_version = "1.6.0" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +registries = "General" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + + [deps.Preferences.syntax] + julia_version = "1.0.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" + [deps.Printf.syntax] + julia_version = "1.14.0-DEV" + [[deps.REPL]] -deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +deps = ["Base64", "Dates", "FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" + [deps.REPL.syntax] + julia_version = "1.14.0-DEV" + [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" version = "1.11.0" -[[deps.Requires]] -deps = ["UUIDs"] -git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.1" + [deps.Random.syntax] + julia_version = "1.14.0-DEV" [[deps.Revise]] -deps = ["CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] -git-tree-sha1 = "82dc140c7f52e4daeeec3675a411d48167a85a87" -repo-rev = "master" -repo-url = "https://github.com/timholy/Revise.jl.git" +deps = ["CRC32c", "CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Preferences", "REPL", "UUIDs"] +git-tree-sha1 = "6098400ed73008c45f5f47b35ae114475436ad37" +repo-rev = "kf/define_method" +repo-url = "https://github.com/KenoAIStaging/Revise.jl.git" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" -version = "3.8.0" +version = "3.16.2" [deps.Revise.extensions] DistributedExt = "Distributed" + [deps.Revise.syntax] + julia_version = "1.10.0" + [deps.Revise.weakdeps] Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" +version = "1.13.0" + + [deps.SHA.syntax] + julia_version = "1.0.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" + [deps.Sockets.syntax] + julia_version = "1.14.0-DEV" + [[deps.StyledStrings]] uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" +version = "1.13.0" + + [deps.StyledStrings.syntax] + julia_version = "1.11.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + + [deps.TOML.syntax] + julia_version = "1.6.0" [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" version = "1.11.0" + [deps.UUIDs.syntax] + julia_version = "1.14.0-DEV" + [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" version = "1.11.0" + [deps.Unicode.syntax] + julia_version = "1.14.0-DEV" + [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.3.1+2" +version = "1.3.2+0" + + [deps.Zlib_jll.syntax] + julia_version = "1.6.0" + +[registries.General] +url = "https://github.com/JuliaRegistries/General.git" +uuid = "23338594-aafe-5451-b93e-139f81909106" diff --git a/deps/jlutilities/revise/Project.toml b/deps/jlutilities/revise/Project.toml index 7f7ec8b06162c..a40e6250ca2d5 100644 --- a/deps/jlutilities/revise/Project.toml +++ b/deps/jlutilities/revise/Project.toml @@ -1,5 +1,9 @@ [deps] +JuliaInterpreter = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +LoweredCodeUtils = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" [sources] -Revise = {rev = "master", url = "https://github.com/timholy/Revise.jl.git"} +JuliaInterpreter = {rev = "kf/define_method", url = "https://github.com/KenoAIStaging/JuliaInterpreter.jl.git"} +LoweredCodeUtils = {rev = "kf/define_method", url = "https://github.com/KenoAIStaging/LoweredCodeUtils.jl.git"} +Revise = {rev = "kf/define_method", url = "https://github.com/KenoAIStaging/Revise.jl.git"} diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 85bc0629f6f28..48c9e9ead04b4 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -86,7 +86,7 @@ $(eval $(call staged-install, \ clean-libgit2: -rm -f $(build_datarootdir)/julia/cert.pem -rm -f $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean + -if [ -d $(BUILDDIR)/$(LIBGIT2_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean; fi get-libgit2: $(LIBGIT2_SRC_FILE) extract-libgit2: $(SRCCACHE)/$(LIBGIT2_SRC_DIR)/source-extracted diff --git a/deps/libgit2.version b/deps/libgit2.version index 90cdd89b83a29..2d82c455929c2 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -3,12 +3,12 @@ LIBGIT2_JLL_NAME := LibGit2 ## source build -LIBGIT2_BRANCH=v1.9.1 -LIBGIT2_SHA1=0060d9cf5666f015b1067129bd874c6cc4c9c7ac +LIBGIT2_BRANCH=v1.9.7 +LIBGIT2_SHA1=49e408b3208bc3093757a1c2db938d3590f3f412 ## Other deps # Specify the version of the Mozilla CA Certificate Store to obtain. # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes. # See https://curl.haxx.se/docs/caextract.html for more details. # Keep in sync with `stdlib/MozillaCACerts_jll/Project.toml`. -MOZILLA_CACERT_VERSION := 2025-07-15 +MOZILLA_CACERT_VERSION := 2026-08-13 diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 05cc12b6e159b..325b134402167 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -32,10 +32,68 @@ endif LIBSSH2_SRC_PATH := $(SRCCACHE)/$(LIBSSH2_SRC_DIR) -$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: $(LIBSSH2_SRC_PATH)/source-extracted +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-7598-256d04b60d80bf1190e96b0ad1e91b2174d744b1.patch-applied: $(LIBSSH2_SRC_PATH)/source-extracted + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-7598-256d04b60d80bf1190e96b0ad1e91b2174d744b1.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2025-15661.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-7598-256d04b60d80bf1190e96b0ad1e91b2174d744b1.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2025-15661.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-55199.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2025-15661.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-55199.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-55200.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-55199.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-55200.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-58050-3449752.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-55200.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-58050-3449752.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-58051-a9758da.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-58050-3449752.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-58051-a9758da.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-unconst-backport.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-58051-a9758da.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-unconst-backport.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66032.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-unconst-backport.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-66032.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66033.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66032.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-66033.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66034.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66033.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-66034.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66035.patch-applied: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66034.patch-applied + cd $(dir $@) && \ + patch -p1 -f < $(SRCDIR)/patches/libssh2-CVE-2026-66035.patch + echo 1 > $@ + +$(LIBSSH2_SRC_PATH)/source-patched: $(LIBSSH2_SRC_PATH)/libssh2-CVE-2026-66035.patch-applied + echo 1 > $@ + +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: $(LIBSSH2_SRC_PATH)/source-patched mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(CMAKE_GENERATOR_COMMAND) $(dir $<) $(LIBSSH2_OPTS) + $(CMAKE) -G"Unix Makefiles" $(dir $<) $(LIBSSH2_OPTS) echo 1 > $@ $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured @@ -55,7 +113,7 @@ $(eval $(call staged-install, \ clean-libssh2: -rm -f $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) clean + -if [ -d $(BUILDDIR)/$(LIBSSH2_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) clean; fi get-libssh2: $(LIBSSH2_SRC_FILE) diff --git a/deps/libsuitesparse.mk b/deps/libsuitesparse.mk index 4bfeb0742fb7e..6e361906017c7 100644 --- a/deps/libsuitesparse.mk +++ b/deps/libsuitesparse.mk @@ -83,7 +83,7 @@ clean-libsuitesparse: uninstall-libsuitesparse -rm -f $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled -rm -fr $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/lib -rm -fr $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/include - -$(MAKE) -C $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER) clean + -if [ -d $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER) ]; then $(MAKE) -C $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER) clean; fi distclean-libsuitesparse: rm -rf $(SRCCACHE)/SuiteSparse-$(LIBSUITESPARSE_VER).tar.gz \ diff --git a/deps/libsuitesparse.version b/deps/libsuitesparse.version index dbb4d1eb713a7..6c2cb46d1db28 100644 --- a/deps/libsuitesparse.version +++ b/deps/libsuitesparse.version @@ -4,5 +4,5 @@ LIBSUITESPARSE_JLL_NAME := SuiteSparse ## source build -LIBSUITESPARSE_VER := 7.10.1 -LIBSUITESPARSE_SHA1=d3c4926d2c47fd6ae558e898bfc072ade210a2a1 +LIBSUITESPARSE_VER := 7.14.0 +LIBSUITESPARSE_SHA1=4d40960f58fada6113b3bcf715ae504a43ec4f5f diff --git a/deps/libtracyclient.mk b/deps/libtracyclient.mk index 92d6bee4caea6..90556f9acf164 100644 --- a/deps/libtracyclient.mk +++ b/deps/libtracyclient.mk @@ -36,17 +36,7 @@ $(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-freebsd-elfw.patch-applied: $(LIBTRACY patch -p1 -f < $(SRCDIR)/patches/libTracyClient-freebsd-elfw.patch echo 1 > $@ -$(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-no-sampling.patch-applied: $(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-freebsd-elfw.patch-applied - cd $(LIBTRACYCLIENT_BUILDDIR) && \ - patch -p1 -f < $(SRCDIR)/patches/libTracyClient-no-sampling.patch - echo 1 > $@ - -$(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-plot-config.patch-applied: $(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-no-sampling.patch-applied - cd $(LIBTRACYCLIENT_BUILDDIR) && \ - patch -p1 -f < $(SRCDIR)/patches/libTracyClient-plot-config.patch - echo 1 > $@ - -$(LIBTRACYCLIENT_BUILDDIR)/build-configured: $(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-plot-config.patch-applied +$(LIBTRACYCLIENT_BUILDDIR)/build-configured: $(LIBTRACYCLIENT_BUILDDIR)/libTracyClient-freebsd-elfw.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) . $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) $(LIBTRACYCLIENT_CMAKE) \ @@ -62,12 +52,12 @@ $(LIBTRACYCLIENT_BUILDDIR)/build-compiled: $(LIBTRACYCLIENT_BUILDDIR)/build-conf $(eval $(call staged-install, \ libtracyclient,$$(LIBTRACYCLIENT_SRC_DIR), \ - MAKE_INSTALL,,, \ + CMAKE_INSTALL,,, \ $$(INSTALL_NAME_CMD)libtracyclient.$$(SHLIB_EXT) $$(build_shlibdir)/libtracyclient.$$(SHLIB_EXT))) clean-libtracyclient: rm -rf $(LIBTRACYCLIENT_BUILDDIR)/build-configured $(LIBTRACYCLIENT_BUILDDIR)/build-compiled - -$(MAKE) -C $(LIBTRACYCLIENT_BUILDDIR) clean + -if [ -d $(LIBTRACYCLIENT_BUILDDIR) ]; then $(MAKE) -C $(LIBTRACYCLIENT_BUILDDIR) clean; fi get-libtracyclient: $(LIBTRACYCLIENT_SRC_FILE) extract-libtracyclient: $(LIBTRACYCLIENT_BUILDDIR)/source-extracted diff --git a/deps/libtracyclient.version b/deps/libtracyclient.version index 60b5a3e8ce630..54e5f53abfe17 100644 --- a/deps/libtracyclient.version +++ b/deps/libtracyclient.version @@ -1,8 +1,8 @@ ## jll artifact LIBTRACYCLIENT_JLL_NAME := LibTracyClient -LIBTRACYCLIENT_JLL_VER := 0.9.1+5 +LIBTRACYCLIENT_JLL_VER := 0.13.1+0 ## source build -LIBTRACYCLIENT_VER := 0.9.1 -LIBTRACYCLIENT_BRANCH=v0.9.1 -LIBTRACYCLIENT_SHA1=897aec5b062664d2485f4f9a213715d2e527e0ca +LIBTRACYCLIENT_VER := 0.13.1 +LIBTRACYCLIENT_BRANCH=v0.13.1 +LIBTRACYCLIENT_SHA1=6cd7751479d4efd5c35f39e856891570a89dd060 diff --git a/deps/libuv.mk b/deps/libuv.mk index 993aa4fc144da..62a41e1509905 100644 --- a/deps/libuv.mk +++ b/deps/libuv.mk @@ -60,7 +60,7 @@ $(eval $(call staged-install, \ clean-libuv: rm -rf $(LIBUV_BUILDDIR)/build-configured $(LIBUV_BUILDDIR)/build-compiled - -$(MAKE) -C $(LIBUV_BUILDDIR) clean + -if [ -d $(LIBUV_BUILDDIR) ]; then $(MAKE) -C $(LIBUV_BUILDDIR) clean; fi get-libuv: $(LIBUV_SRC_FILE) diff --git a/deps/libuv.version b/deps/libuv.version index ebfc63927d9db..b8a8338b6115c 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -6,4 +6,4 @@ LIBUV_JLL_NAME := LibUV ## source build LIBUV_VER := 2 LIBUV_BRANCH=julia-uv2-1.48.0 -LIBUV_SHA1=af4172ec713ee986ba1a989b9e33993a07c60c9e +LIBUV_SHA1=e6b9850fbef2b4778e6a81bb277d3e59a2779402 diff --git a/deps/libwhich.mk b/deps/libwhich.mk index 79017838193d2..df77bdbe11d10 100644 --- a/deps/libwhich.mk +++ b/deps/libwhich.mk @@ -26,7 +26,7 @@ $(eval $(call staged-install, \ clean-libwhich: -rm -f $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(LIBWHICH_SRC_DIR) clean + -if [ -d $(BUILDDIR)/$(LIBWHICH_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(LIBWHICH_SRC_DIR) clean; fi get-libwhich: $(LIBWHICH_SRC_FILE) extract-libwhich: $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/source-extracted diff --git a/deps/lld.version b/deps/lld.version index 023122efc4596..f08287dd679a4 100644 --- a/deps/lld.version +++ b/deps/lld.version @@ -2,4 +2,4 @@ ## jll artifact LLD_JLL_NAME := LLD -LLD_JLL_VER := 20.1.2+0 +LLD_JLL_VER := 22.1.8+1 diff --git a/deps/llvm-tools.version b/deps/llvm-tools.version index 285079ca412e5..e86ed0ea01497 100644 --- a/deps/llvm-tools.version +++ b/deps/llvm-tools.version @@ -3,5 +3,5 @@ ## jll artifact # LLVM_tools (downloads LLVM_jll to get things like `lit` and `opt`) LLVM_TOOLS_JLL_NAME := LLVM -LLVM_TOOLS_JLL_VER := 20.1.2+0 -LLVM_TOOLS_ASSERT_JLL_VER := 20.1.2+0 +LLVM_TOOLS_JLL_VER := 22.1.8+1 +LLVM_TOOLS_ASSERT_JLL_VER := 22.1.8+1 diff --git a/deps/llvm.mk b/deps/llvm.mk index e3303aba55afd..4f9d3f4311fc8 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -46,7 +46,7 @@ LLVM_ENABLE_PROJECTS := LLVM_EXTERNAL_PROJECTS := LLVM_ENABLE_RUNTIMES := ifeq ($(BUILD_LLVM_CLANG), 1) -LLVM_ENABLE_PROJECTS := $(LLVM_ENABLE_PROJECTS);clang +LLVM_ENABLE_PROJECTS := $(LLVM_ENABLE_PROJECTS);clang;clang-tools-extra LLVM_ENABLE_RUNTIMES := $(LLVM_ENABLE_RUNTIMES);compiler-rt endif ifeq ($(USE_POLLY), 1) @@ -82,7 +82,13 @@ LLVM_EXPERIMENTAL_TARGETS := LLVM_CFLAGS := LLVM_CXXFLAGS := LLVM_CPPFLAGS := -LLVM_LDFLAGS := "-L$(build_shlibdir)" # hacky way to force zlib to be found when linking against libLLVM and sysroot is set +# Find our zlib/zstd when linking against libLLVM with a sysroot toolchain: `-L` for +# lld, `-rpath-link` for GNU ld (which does not search `-L` paths for the dependencies +# of shared libraries). The latter is ELF-only, as in Make.inc's RPATH. +LLVM_LDFLAGS := "-L$(build_shlibdir)" +ifeq (,$(filter $(OS),WINNT emscripten Darwin)) +LLVM_LDFLAGS += "-Wl,-rpath-link,$(build_shlibdir)" +endif LLVM_CMAKE := LLVM_CMAKE += -DLLVM_ENABLE_PROJECTS="$(LLVM_ENABLE_PROJECTS)" @@ -111,8 +117,7 @@ LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=FORCE_ON -DZSTD_ROOT="$(build_prefix)" ifeq ($(USE_POLLY_ACC),1) LLVM_CMAKE += -DPOLLY_ENABLE_GPGPU_CODEGEN=ON endif -LLVM_CMAKE += -DLLVM_TOOLS_INSTALL_DIR=$(call rel_path,$(build_prefix),$(build_depsbindir)) -LLVM_CMAKE += -DLLVM_UTILS_INSTALL_DIR=$(call rel_path,$(build_prefix),$(build_depsbindir)) +LLVM_CMAKE += -DCMAKE_INSTALL_BINDIR=$(call rel_path,$(build_prefix),$(build_depsbindir)) LLVM_CMAKE += -DLLVM_INCLUDE_UTILS=ON -DLLVM_INSTALL_UTILS=ON LLVM_CMAKE += -DLLVM_BINDINGS_LIST="" -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INCLUDE_DOCS=Off -DLLVM_ENABLE_TERMINFO=Off -DHAVE_LIBEDIT=Off -DLLVM_ENABLE_LIBEDIT=OFF ifeq ($(LLVM_ASSERTIONS), 1) @@ -207,6 +212,11 @@ LLVM_CMAKE += -DCMAKE_C_FLAGS="$(LLVM_CPPFLAGS) $(LLVM_CFLAGS)" \ ifeq ($(OS),Darwin) # Explicitly use the default for -mmacosx-version-min=10.9 and later LLVM_CMAKE += -DLLVM_ENABLE_LIBCXX=ON +# LLVM archives its static libraries with Xcode's libtool, which cannot index +# LTO bitcode from a different LLVM; allow using e.g. llvm-libtool-darwin. +ifneq ($(LLVM_LIBTOOL),) +LLVM_CMAKE += -DCMAKE_LIBTOOL="$(LLVM_LIBTOOL)" +endif endif ifeq ($(BUILD_LLVM_CLANG),0) @@ -221,6 +231,12 @@ endif LLVM_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(LLVM_LDFLAGS)" \ -DCMAKE_SHARED_LINKER_FLAGS="$(LLVM_LDFLAGS)" +ifeq ($(OS),Darwin) +# Build-tree tools such as tblgen link against the bundled zlib and zstd, whose +# install names are `@rpath/...`, so they need a build rpath to find them. +# CMake replaces it with the install rpath on install. +LLVM_CMAKE += -DCMAKE_BUILD_RPATH="$(build_shlibdir)" +endif # change the SONAME of Julia's private LLVM # i.e. libLLVM-14jl.so @@ -263,6 +279,9 @@ ifeq ($(USE_SYSTEM_ZSTD), 0) $(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zstd endif +# LLVM's configure checks link against libraries installed by csl. +$(LLVM_BUILDDIR_withtype)/build-configured: | install-csl + # NOTE: LLVM 12 and 13 have their patches applied to JuliaLang/llvm-project @@ -311,21 +330,70 @@ ifeq ($(OS),$(BUILD_OS)) endif echo 1 > $@ +ifeq ($(OS),Darwin) +ifneq ($(BUILD_LLVM_CLANG), 1) +# The LLVM runtimes build of compiler-rt requires clang, so build the builtins +# standalone with the host compiler instead. Only the host architecture is built, +# and not the kernel extension variant, which Julia does not use: building several +# arm64 variants in parallel races on the generated outline atomics. +LLVM_COMPILERRT_BUILDDIR := $(LLVM_BUILDDIR)/build_compiler-rt +ifeq ($(ARCH),aarch64) +LLVM_COMPILERRT_ARCH := arm64 +else +LLVM_COMPILERRT_ARCH := $(ARCH) +endif + +$(LLVM_COMPILERRT_BUILDDIR)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted + mkdir -p $(dir $@) + cd $(dir $@) && \ + $(CMAKE) $(SRCCACHE)/$(LLVM_SRC_DIR)/compiler-rt/lib/builtins $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) \ + -DCMAKE_BUILD_TYPE=Release -DCOMPILER_RT_STANDALONE_BUILD=ON \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=$(MACOSX_VERSION_MIN) \ + -DCOMPILER_RT_ENABLE_IOS=OFF -DCOMPILER_RT_ENABLE_WATCHOS=OFF \ + -DCOMPILER_RT_ENABLE_TVOS=OFF -DCOMPILER_RT_ENABLE_XROS=OFF \ + -DDARWIN_osx_BUILTIN_ARCHS=$(LLVM_COMPILERRT_ARCH) -DDARWIN_osx_SKIP_CC_KEXT=ON + echo 1 > $@ + +$(LLVM_COMPILERRT_BUILDDIR)/build-compiled: $(LLVM_COMPILERRT_BUILDDIR)/build-configured + cd $(dir $@) && \ + $(if $(filter $(CMAKE_GENERATOR),make), \ + $(MAKE), \ + $(CMAKE) --build .) + echo 1 > $@ + +$(LLVM_BUILDDIR_withtype)/build-compiled: $(LLVM_COMPILERRT_BUILDDIR)/build-compiled +endif +endif + +# CMake runs on the build host, even when targeting Windows from Unix. +# MSYS2 excludes CMAKE_INSTALL_PREFIX from conversion; LLVM's prefix is absolute. +ifeq ($(BUILD_OS),WINNT) +LLVM_INSTALL_PREFIX = $(call cygpath_w,$1) +else +LLVM_INSTALL_PREFIX = $1 +endif + LLVM_INSTALL = \ cd $1 && mkdir -p $2$$(build_depsbindir)/lit && \ cp $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/*.py $2$$(build_depsbindir)/lit/ && \ cp $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/*.toml $2$$(build_depsbindir)/lit/ && \ cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/lit $2$$(build_depsbindir)/lit/ && \ - $$(CMAKE) -DCMAKE_INSTALL_PREFIX="$2$$(build_prefix)" -P cmake_install.cmake + $$(CMAKE) -DCMAKE_INSTALL_PREFIX="$$(call LLVM_INSTALL_PREFIX,$2$$(build_prefix))" -P cmake_install.cmake ifeq ($(OS), WINNT) -LLVM_INSTALL += && cp $2$$(build_shlibdir)/$(LLVM_SHARED_LIB_NAME).dll $2$$(build_depsbindir) +# CMAKE_INSTALL_BINDIR puts the DLL in build_depsbindir alongside the tools, +# but Julia loads it out of build_shlibdir, so it has to be in both places. +# Staged installs have no directory there yet, and cp would make the DLL one. +LLVM_INSTALL += && mkdir -p $2$$(build_shlibdir) && cp $2$$(build_depsbindir)/$(LLVM_SHARED_LIB_NAME).dll $2$$(build_shlibdir) endif ifeq ($(OS),Darwin) # https://github.com/JuliaLang/julia/issues/29981 LLVM_INSTALL += && ln -s libLLVM.dylib $2$$(build_shlibdir)/libLLVM-$$(LLVM_VER_SHORT).dylib +# compiler-rt is required for linking sysimages on Darwin +ifeq ($(BUILD_LLVM_CLANG), 1) +LLVM_INSTALL += && install -m 0644 $2$$(build_prefix)/lib/clang/$$(LLVM_VER_MAJ)/lib/darwin/libclang_rt.osx.a $2$$(build_libdir)/libclang_rt.osx.a +else +LLVM_INSTALL += && install -m 0644 $$(abspath $$(LLVM_COMPILERRT_BUILDDIR))/lib/darwin/libclang_rt.osx.a $2$$(build_libdir)/libclang_rt.osx.a endif -ifeq ($(BUILD_LLD), 1) -LLVM_INSTALL += && cp $2$$(build_bindir)/lld$$(EXE) $2$$(build_depsbindir) endif $(eval $(call staged-install, \ @@ -334,7 +402,7 @@ $(eval $(call staged-install, \ clean-llvm: -rm -f $(LLVM_BUILDDIR_withtype)/build-configured $(LLVM_BUILDDIR_withtype)/build-compiled - -$(MAKE) -C $(LLVM_BUILDDIR_withtype) clean + -if [ -d $(LLVM_BUILDDIR_withtype) ]; then $(MAKE) -C $(LLVM_BUILDDIR_withtype) clean; fi get-llvm: $(LLVM_SRC_FILE) extract-llvm: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted @@ -377,6 +445,17 @@ $(eval $(call bb-install,llvm-tools,LLVM_TOOLS,false,true)) $(build_prefix)/manifest/llvm-tools uninstall-llvm-tools: \ TAR:=$(TAR) --exclude=llvm-config.exe +ifeq ($(OS),Darwin) +# compiler-rt is required for linking sysimages on Darwin +$(eval $(call bb-install,compilerrt,COMPILERRT,false,false)) +$(build_libdir)/libclang_rt.osx.a: $(build_prefix)/manifest/compilerrt + mv $(build_libdir)/darwin/libclang_rt.osx.a $@ + touch $@ + rm -rf $(build_libdir)/darwin +install-compilerrt: $(build_libdir)/libclang_rt.osx.a +install-llvm: install-compilerrt +endif + endif # USE_BINARYBUILDER_LLVM get-lld: get-llvm diff --git a/deps/llvm.version b/deps/llvm.version index c68198b6d1729..fd0e925dbdc87 100644 --- a/deps/llvm.version +++ b/deps/llvm.version @@ -2,14 +2,14 @@ ## jll artifact LLVM_JLL_NAME := libLLVM -LLVM_ASSERT_JLL_VER := 20.1.2+1 +LLVM_ASSERT_JLL_VER := 22.1.8+1 ## source build # Version number of LLVM -LLVM_VER := 20.1.2 +LLVM_VER := 22.1.8 # Git branch name in `LLVM_GIT_URL` repository -LLVM_BRANCH=julia-20.1.2-0 +LLVM_BRANCH=julia-22.1.8-1 # Git ref in `LLVM_GIT_URL` repository -LLVM_SHA1=julia-20.1.2-0 +LLVM_SHA1=julia-22.1.8-1 ## Following options are used to automatically fetch patchset from Julia's fork. This is ## useful if you want to build an external LLVM while still applying Julia's patches. @@ -18,6 +18,6 @@ LLVM_APPLY_JULIA_PATCHES := 0 # GitHub repository to use for fetching the Julia patches to apply to LLVM source code. LLVM_JULIA_DIFF_GITHUB_REPO := https://github.com/llvm/llvm-project # Base GitHub ref for generating the diff. -LLVM_BASE_REF := llvm:llvmorg-20.1.2 +LLVM_BASE_REF := llvm:llvmorg-22.1.8 # Julia fork's GitHub ref for generating the diff. -LLVM_JULIA_REF := JuliaLang:julia-20.1.2-0 +LLVM_JULIA_REF := JuliaLang:$(LLVM_SHA1) diff --git a/deps/lmdb.mk b/deps/lmdb.mk new file mode 100644 index 0000000000000..3df44d09caa7a --- /dev/null +++ b/deps/lmdb.mk @@ -0,0 +1,65 @@ +## LMDB ## +# LMDB is statically linked into libjulia-codegen (see src/objcache.cpp), so we +# always build it from source as a static archive rather than consuming the +# shared library shipped by LMDB_jll. +LMDB_GIT_URL := https://github.com/LMDB/lmdb.git +LMDB_TAR_URL = https://api.github.com/repos/LMDB/lmdb/tarball/$1 +# LMDB builds in its source tree, so each build needs its own extraction. +$(eval $(call git-external,lmdb,LMDB,,,$(BUILDDIR))) + +LMDB_SRC_SUBDIR := libraries/liblmdb +LMDB_BUILD_OPTS := CC="$(CC)" AR="$(AR)" prefix=$(abspath $(build_prefix)) +# -fPIC so the static archive can be linked into the libjulia-codegen shared library +LMDB_BUILD_OPTS += XCFLAGS="$(CFLAGS) $(fPIC)" +# we pass our flags to LMDB via XCFLAGS, but this only works as long as +# nothing overrides CFLAGS; so restore the default LMDB CFLAGS here +# (this is important for libjulia_jll) +LMDB_BUILD_OPTS += CFLAGS='$$(THREADS) $$(OPT) $$(W) $$(XCFLAGS)' +# Select LMDB's robust lock backend, including SysV semaphores on Apple/BSD. +LMDB_BUILD_OPTS += CPPFLAGS="-DMDB_USE_ROBUST=1" + +$(BUILDDIR)/$(LMDB_SRC_DIR)/mingw-types.patch-applied: $(BUILDDIR)/$(LMDB_SRC_DIR)/source-extracted + cd $(BUILDDIR)/$(LMDB_SRC_DIR) && \ + patch -p1 -f < $(SRCDIR)/patches/0001-Fix-MinGW-Windows-type-errors.patch + echo 1 > $@ + +$(BUILDDIR)/$(LMDB_SRC_DIR)/remote-fs.patch-applied: $(BUILDDIR)/$(LMDB_SRC_DIR)/mingw-types.patch-applied + cd $(BUILDDIR)/$(LMDB_SRC_DIR) && \ + patch -p1 -f < $(SRCDIR)/patches/0002-Refuse-to-open-environments-on-network-filesystems.patch + echo 1 > $@ + +$(BUILDDIR)/$(LMDB_SRC_DIR)/pidns.patch-applied: $(BUILDDIR)/$(LMDB_SRC_DIR)/remote-fs.patch-applied + cd $(BUILDDIR)/$(LMDB_SRC_DIR) && \ + patch -p1 -f < $(SRCDIR)/patches/0003-Refuse-to-open-environments-across-pid-namespaces.patch + echo 1 > $@ + +$(BUILDDIR)/$(LMDB_SRC_DIR)/source-patched: $(BUILDDIR)/$(LMDB_SRC_DIR)/pidns.patch-applied + echo 1 > $@ + +$(BUILDDIR)/$(LMDB_SRC_DIR)/build-configured: $(BUILDDIR)/$(LMDB_SRC_DIR)/source-patched + echo 1 > $@ + +$(BUILDDIR)/$(LMDB_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LMDB_SRC_DIR)/build-configured + $(MAKE) -C $(BUILDDIR)/$(LMDB_SRC_DIR)/$(LMDB_SRC_SUBDIR) $(MAKE_COMMON) $(LMDB_BUILD_OPTS) liblmdb.a + echo 1 > $@ + +define LMDB_INSTALL + mkdir -p $2/$$(build_libdir) + mkdir -p $2/$$(build_includedir) + cp $1/$(LMDB_SRC_SUBDIR)/liblmdb.a $2/$$(build_libdir)/ + cp $1/$(LMDB_SRC_SUBDIR)/lmdb.h $2/$$(build_includedir)/ +endef +$(eval $(call staged-install, \ + lmdb,$(LMDB_SRC_DIR), \ + LMDB_INSTALL,,,)) + +clean-lmdb: + -rm -f $(BUILDDIR)/$(LMDB_SRC_DIR)/build-configured $(BUILDDIR)/$(LMDB_SRC_DIR)/build-compiled + -if [ -d $(BUILDDIR)/$(LMDB_SRC_DIR)/$(LMDB_SRC_SUBDIR) ]; then $(MAKE) -C $(BUILDDIR)/$(LMDB_SRC_DIR)/$(LMDB_SRC_SUBDIR) $(MAKE_COMMON) clean; fi + +get-lmdb: $(LMDB_SRC_FILE) +extract-lmdb: $(BUILDDIR)/$(LMDB_SRC_DIR)/source-extracted +configure-lmdb: $(BUILDDIR)/$(LMDB_SRC_DIR)/build-configured +compile-lmdb: $(BUILDDIR)/$(LMDB_SRC_DIR)/build-compiled +fastcheck-lmdb: check-lmdb +check-lmdb: compile-lmdb diff --git a/deps/lmdb.version b/deps/lmdb.version new file mode 100644 index 0000000000000..e1cef54ec21e5 --- /dev/null +++ b/deps/lmdb.version @@ -0,0 +1,6 @@ +# -*- makefile -*- + +## source build +LMDB_VER := 1.0.0 +LMDB_BRANCH=LMDB_1.0.0 +LMDB_SHA1=2562c3297402d82bbc049c7e645515edb4079eba diff --git a/deps/mmtk_julia.mk b/deps/mmtk_julia.mk deleted file mode 100644 index b443f9b6a25da..0000000000000 --- a/deps/mmtk_julia.mk +++ /dev/null @@ -1,99 +0,0 @@ -## MMTK ## - -# Both MMTK_MOVING and MMTK_PLAN should be specified in the Make.user file. -# FIXME: By default we do a non-moving build. We should change the default to 1 -# once we support moving plans. -MMTK_MOVING ?= 0 -MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING) - -ifneq ($(USE_BINARYBUILDER_MMTK_JULIA),1) -$(eval $(call git-external,mmtk_julia,MMTK_JULIA,,,$(BUILDDIR))) -get-mmtk_julia: $(MMTK_JULIA_SRC_FILE) - -# Download the binding, build it from source -ifeq (${MMTK_JULIA_DIR},$(BUILDROOT)/usr/lib/mmtk_julia) - -MMTK_JULIA_DIR=$(BUILDROOT)/deps/$(BUILDDIR)/$(MMTK_JULIA_SRC_DIR) -MMTK_JULIA_LIB_PATH=$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD) -PROJECT_DIRS := JULIA_PATH=$(JULIAHOME) JULIA_BUILDROOT=$(BUILDROOT) MMTK_JULIA_DIR=$(MMTK_JULIA_DIR) - -$(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/build-compiled: $(BUILDROOT)/usr/lib/libmmtk_julia.so - @echo 1 > $@ - -# NB: use the absolute dir when creating the symlink -$(BUILDROOT)/usr/lib/libmmtk_julia.so: $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so - @ln -sf $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so $@ - -$(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so: $(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/source-extracted - @$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD) - -extract-mmtk_julia: $(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/source-extracted -configure-mmtk_julia: extract-mmtk_julia -compile-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so -fastcheck-mmtk_julia: #none -check-mmtk_julia: compile-mmtk_julia - -$(eval $(call symlink_install,mmtk_julia,$$(MMTK_JULIA_SRC_DIR),$$(BUILDROOT)/usr/lib)) - -# In this case, there is a custom version of the binding in MMTK_JULIA_DIR -# Build it and symlink libmmtk_julia.so file into $(BUILDROOT)/usr/lib -else - -PROJECT_DIRS := JULIA_PATH=$(JULIAHOME) JULIA_BUILDROOT=$(BUILDROOT) MMTK_JULIA_DIR=$(MMTK_JULIA_DIR) -MMTK_JULIA_LIB_PATH=$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD) - -install-mmtk_julia: compile-mmtk_julia $(build_prefix)/manifest/mmtk_julia - -compile-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so - -version-check-mmtk_julia: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so - -# NB: This will NOT run `cargo build` if there are changes in the Rust source files -# inside the binding repo. However the target below should remake the symlink if there -# are changes in the libmmtk_julia.so from the custom MMTK_JULIA_DIR folder -$(BUILDROOT)/usr/lib/libmmtk_julia.so: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so - @ln -sf $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so $@ - -$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so: - @$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD) - -MMTK_JULIA_VER := mmtk_julia_custom - -UNINSTALL_mmtk_julia := $(MMTK_JULIA_VER) manual_mmtk_julia - -define manual_mmtk_julia -uninstall-mmtk_julia: - -rm -f $(build_prefix)/manifest/mmtk_julia - -rm -f $(BUILDROOT)/usr/lib/libmmtk_julia.so -endef - -$(build_prefix)/manifest/mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so - @echo $(UNINSTALL_mmtk_julia) > $@ - -endif # MMTK_JULIA_DIR - -else -# We are building using the BinaryBuilder version of the binding - -# This will download all the versions of the binding that are available in the BinaryBuilder -$(eval $(call bb-install,mmtk_julia,MMTK_JULIA,false)) - -# Make sure we use the right version of $MMTK_PLAN, $MMTK_MOVING and $MMTK_BUILD -ifeq (${MMTK_PLAN},Immix) -LIB_PATH_PLAN = immix -else ifeq (${MMTK_PLAN},StickyImmix) -LIB_PATH_PLAN = sticky -endif - -ifeq ($(MMTK_MOVING), 0) -LIB_PATH_MOVING := non_moving -else -LIB_PATH_MOVING := moving -endif - -version-check-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so - -$(BUILDROOT)/usr/lib/libmmtk_julia.so: get-mmtk_julia - @ln -sf $(BUILDROOT)/usr/lib/$(LIB_PATH_PLAN)/$(LIB_PATH_MOVING)/$(MMTK_BUILD)/libmmtk_julia.so $@ - -endif # USE_BINARYBUILDER_MMTK_JULIA diff --git a/deps/mmtk_julia.version b/deps/mmtk_julia.version deleted file mode 100644 index 6b8ef2f816354..0000000000000 --- a/deps/mmtk_julia.version +++ /dev/null @@ -1,6 +0,0 @@ -MMTK_JULIA_BRANCH = master -MMTK_JULIA_SHA1 = 4933fb41ba5d1d21f720e2ca1bfa5bc938b73b12 -MMTK_JULIA_GIT_URL := https://github.com/mmtk/mmtk-julia.git -MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.30.6.tar.gz -MMTK_JULIA_JLL_VER := 0.30.6+0 -MMTK_JULIA_JLL_NAME := mmtk_julia diff --git a/deps/mpfr.mk b/deps/mpfr.mk index 5a0605ba6b601..0310ae8c6321f 100644 --- a/deps/mpfr.mk +++ b/deps/mpfr.mk @@ -60,7 +60,7 @@ $(eval $(call staged-install, \ clean-mpfr: -rm -f $(BUILDDIR)/mpfr-$(MPFR_VER)/build-configured $(BUILDDIR)/mpfr-$(MPFR_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/mpfr-$(MPFR_VER) clean + -if [ -d $(BUILDDIR)/mpfr-$(MPFR_VER) ]; then $(MAKE) -C $(BUILDDIR)/mpfr-$(MPFR_VER) clean; fi distclean-mpfr: rm -rf $(SRCCACHE)/mpfr-$(MPFR_VER).tar.bz2 \ diff --git a/deps/nghttp2.mk b/deps/nghttp2.mk index 5c12a0155c017..f85ce6b4748ff 100644 --- a/deps/nghttp2.mk +++ b/deps/nghttp2.mk @@ -38,7 +38,7 @@ $(eval $(call staged-install, \ clean-nghttp2: -rm -f $(BUILDDIR)/nghttp2-$(NGHTTP2_VER)/build-configured $(BUILDDIR)/nghttp2-$(NGHTTP2_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/nghttp2-$(NGHTTP2_VER) clean + -if [ -d $(BUILDDIR)/nghttp2-$(NGHTTP2_VER) ]; then $(MAKE) -C $(BUILDDIR)/nghttp2-$(NGHTTP2_VER) clean; fi distclean-nghttp2: rm -rf $(SRCCACHE)/nghttp2-$(NGHTTP2_VER).tar.bz2 \ diff --git a/deps/nghttp2.version b/deps/nghttp2.version index 9a7f5588b2bc0..ad1dbc1241b8a 100644 --- a/deps/nghttp2.version +++ b/deps/nghttp2.version @@ -3,4 +3,4 @@ NGHTTP2_JLL_NAME := nghttp2 ## source build -NGHTTP2_VER := 1.65.0 +NGHTTP2_VER := 1.70.0 diff --git a/deps/objconv.mk b/deps/objconv.mk index 70c7289b07bfa..d0bce5afd300d 100644 --- a/deps/objconv.mk +++ b/deps/objconv.mk @@ -3,18 +3,17 @@ include $(SRCDIR)/objconv.version ifneq ($(USE_BINARYBUILDER_OBJCONV),1) -$(SRCCACHE)/objconv.zip: | $(SRCCACHE) - $(JLDOWNLOAD) $@ https://www.agner.org/optimize/objconv.zip - -$(BUILDDIR)/objconv/source-extracted: $(SRCCACHE)/objconv.zip - rm -rf $(dir $@) - mkdir -p $(BUILDDIR) - unzip -d $(dir $@) $< - cd $(dir $@) && unzip source.zip +$(SRCCACHE)/objconv-$(OBJCONV_VER).tar.gz: | $(SRCCACHE) + $(JLDOWNLOAD) $@ https://github.com/staticfloat/objconv/archive/refs/tags/v$(OBJCONV_VER).tar.gz + +$(BUILDDIR)/objconv/source-extracted: $(SRCCACHE)/objconv-$(OBJCONV_VER).tar.gz + -rm -rf $(dir $@) + mkdir -p $(dir $@) + $(TAR) -C $(dir $@) --strip-components 1 -zxf $< echo 1 > $@ $(BUILDDIR)/objconv/build-compiled: $(BUILDDIR)/objconv/source-extracted - cd $(dir $<) && $(CXX) -o objconv -O2 *.cpp + cd $(dir $<) && $(CXX) -o objconv -O2 src/*.cpp echo 1 > $@ $(eval $(call staged-install, \ @@ -25,10 +24,9 @@ clean-objconv: -rm -f $(BUILDDIR)/objconv/build-compiled $(build_depsbindir)/objconv distclean-objconv: - rm -rf $(SRCCACHE)/objconv.zip $(BUILDDIR)/objconv - + rm -rf $(SRCCACHE)/objconv-$(OBJCONV_VER).tar.gz $(BUILDDIR)/objconv -get-objconv: $(SRCCACHE)/objconv.zip +get-objconv: $(SRCCACHE)/objconv-$(OBJCONV_VER).tar.gz extract-objconv: $(BUILDDIR)/objconv/source-extracted configure-objconv: extract-objconv compile-objconv: $(BUILDDIR)/objconv/build-compiled diff --git a/deps/objconv.version b/deps/objconv.version index 185354e23b9e1..377d2d9cd692b 100644 --- a/deps/objconv.version +++ b/deps/objconv.version @@ -1,7 +1,7 @@ ## jll artifact # Objconv (we don't ship this, so no need for a fake JLL; therefore we specify the JLL_VER here instead of in a `stdlib/Objconv_jll/Project.toml` file) OBJCONV_JLL_NAME := Objconv -OBJCONV_JLL_VER := 2.53.0+0 +OBJCONV_JLL_VER := 2.55.0+0 ## source build -OBJCONV_VER := 2.53.0 +OBJCONV_VER := 2.55 diff --git a/deps/openblas.mk b/deps/openblas.mk index e5a988ba84df2..65af1150527b8 100644 --- a/deps/openblas.mk +++ b/deps/openblas.mk @@ -90,12 +90,12 @@ $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-winexit.patch-applied: $(BUILDDIR)/$(OP patch -p1 -f < $(SRCDIR)/patches/openblas-winexit.patch echo 1 > $@ -$(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-ofast-power.patch-applied: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-winexit.patch-applied +$(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-cancellation.patch-applied: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-winexit.patch-applied cd $(BUILDDIR)/$(OPENBLAS_SRC_DIR) && \ - patch -p1 -f < $(SRCDIR)/patches/openblas-ofast-power.patch + patch -p1 -f < $(SRCDIR)/patches/openblas-cancellation.patch echo 1 > $@ -$(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-ofast-power.patch-applied +$(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-cancellation.patch-applied echo 1 > $@ $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-compiled: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-configured @@ -116,7 +116,7 @@ $(eval $(call staged-install, \ clean-openblas: -rm -f $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(OPENBLAS_SRC_DIR) clean + -if [ -d $(BUILDDIR)/$(OPENBLAS_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(OPENBLAS_SRC_DIR) clean; fi get-openblas: $(OPENBLAS_SRC_FILE) @@ -145,17 +145,17 @@ LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(USE_BLAS_FFLAGS) -O0" \ OPTS="$(FFLAGS) $(JFFLAGS) $(USE_BLAS_FFLAGS)" FORTRAN="$(FC)" \ LOADER="$(FC)" BLASLIB="$(RPATH_ESCAPED_ORIGIN) $(LIBBLAS)" -$(SRCCACHE)/lapack-$(LAPACK_VER).tgz: | $(SRCCACHE) +$(SRCCACHE)/lapack-$(LAPACK_VER).tar.gz: | $(SRCCACHE) $(JLDOWNLOAD) $@ https://www.netlib.org/lapack/$(notdir $@) -$(BUILDDIR)/lapack-$(LAPACK_VER)/source-extracted: $(SRCCACHE)/lapack-$(LAPACK_VER).tgz +$(BUILDDIR)/lapack-$(LAPACK_VER)/source-extracted: $(SRCCACHE)/lapack-$(LAPACK_VER).tar.gz $(JLCHECKSUM) $< mkdir -p $(BUILDDIR) cd $(BUILDDIR) && $(TAR) -zxf $< cp $(dir $@)INSTALL/make.inc.gfortran $(dir $@)make.inc echo 1 > $@ -checksum-lapack: $(SRCCACHE)/lapack-$(LAPACK_VER).tgz +checksum-lapack: $(SRCCACHE)/lapack-$(LAPACK_VER).tar.gz $(JLCHECKSUM) $< ifeq ($(USE_SYSTEM_BLAS), 0) @@ -187,13 +187,13 @@ $(eval $(call staged-install, \ clean-lapack: -rm -f $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled0 $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/lapack-$(LAPACK_VER) clean + -if [ -d $(BUILDDIR)/lapack-$(LAPACK_VER) ]; then $(MAKE) -C $(BUILDDIR)/lapack-$(LAPACK_VER) clean; fi distclean-lapack: - rm -rf $(SRCCACHE)/lapack-$(LAPACK_VER).tgz $(BUILDDIR)/lapack-$(LAPACK_VER) + rm -rf $(SRCCACHE)/lapack-$(LAPACK_VER).tar.gz $(BUILDDIR)/lapack-$(LAPACK_VER) -get-lapack: $(SRCCACHE)/lapack-$(LAPACK_VER).tgz +get-lapack: $(SRCCACHE)/lapack-$(LAPACK_VER).tar.gz extract-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/source-extracted configure-lapack: extract-lapack compile-lapack: $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled diff --git a/deps/openblas.version b/deps/openblas.version index f9729639c67ab..8f64896e62bcb 100644 --- a/deps/openblas.version +++ b/deps/openblas.version @@ -3,9 +3,9 @@ OPENBLAS_JLL_NAME := OpenBLAS ## source build -OPENBLAS_VER := 0.3.29 -OPENBLAS_BRANCH=v0.3.29 -OPENBLAS_SHA1=8795fc7985635de1ecf674b87e2008a15097ffab +OPENBLAS_VER := 0.3.34 +OPENBLAS_BRANCH=v0.3.34 +OPENBLAS_SHA1=e0166008be8e466242aa76b2ff75ce3f0fbf574a # LAPACK, source-only -LAPACK_VER := 3.9.0 +LAPACK_VER := 3.12.1 diff --git a/deps/openlibm.mk b/deps/openlibm.mk index f99cdade47b91..e4177f8819ce5 100644 --- a/deps/openlibm.mk +++ b/deps/openlibm.mk @@ -17,7 +17,7 @@ $(eval $(call staged-install, \ clean-openlibm: -rm -f $(BUILDDIR)/$(OPENLIBM_SRC_DIR)/build-compiled $(build_libdir)/libopenlibm.a - -$(MAKE) -C $(BUILDDIR)/$(OPENLIBM_SRC_DIR) distclean $(OPENLIBM_FLAGS) + -if [ -d $(BUILDDIR)/$(OPENLIBM_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(OPENLIBM_SRC_DIR) distclean $(OPENLIBM_FLAGS); fi get-openlibm: $(OPENLIBM_SRC_FILE) diff --git a/deps/openlibm.version b/deps/openlibm.version index 788701a66301b..482de709f20e5 100644 --- a/deps/openlibm.version +++ b/deps/openlibm.version @@ -4,6 +4,6 @@ OPENLIBM_JLL_NAME := OpenLibm ## source build -OPENLIBM_VER := 0.8.5 -OPENLIBM_BRANCH=v0.8.5 -OPENLIBM_SHA1=db24332879c320606c37f77fea165e6ecb49153c +OPENLIBM_VER := 0.8.8 +OPENLIBM_BRANCH=v0.8.8 +OPENLIBM_SHA1=5fe399749f9276eaa0b8403e507470da05cbbb3f diff --git a/deps/openssl.mk b/deps/openssl.mk index ab6bd94921562..6ffd1bb4133d2 100644 --- a/deps/openssl.mk +++ b/deps/openssl.mk @@ -89,13 +89,13 @@ $(eval $(call staged-install, \ OPENSSL_INSTALL,,,$(OPENSSL_POST_INSTALL))) clean-openssl: - -rm -f $(BUILDDIR)/-openssl-$(OPENSSL_VER)/build-configured $(BUILDDIR)/-openssl-$(OPENSSL_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/-openssl-$(OPENSSL_VER) clean + -rm -f $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-configured $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-compiled + -if [ -d $(BUILDDIR)/openssl-$(OPENSSL_VER) ]; then $(MAKE) -C $(BUILDDIR)/openssl-$(OPENSSL_VER) clean; fi distclean-openssl: - rm -rf $(SRCCACHE)/-openssl-$(OPENSSL_VER).tar.gz \ - $(SRCCACHE)/-openssl-$(OPENSSL_VER) \ - $(BUILDDIR)/-openssl-$(OPENSSL_VER) + rm -rf $(SRCCACHE)/openssl-$(OPENSSL_VER).tar.gz \ + $(SRCCACHE)/openssl-$(OPENSSL_VER) \ + $(BUILDDIR)/openssl-$(OPENSSL_VER) get-openssl: $(SRCCACHE)/openssl-$(OPENSSL_VER).tar.gz extract-openssl: $(SRCCACHE)/openssl-$(OPENSSL_VER)/source-extracted diff --git a/deps/openssl.version b/deps/openssl.version index 2313ae5ffe116..b2d1f42ad626c 100644 --- a/deps/openssl.version +++ b/deps/openssl.version @@ -3,4 +3,4 @@ OPENSSL_JLL_NAME := OpenSSL ## source build -OPENSSL_VER := 3.5.1 +OPENSSL_VER := 3.5.8 diff --git a/deps/p7zip.mk b/deps/p7zip.mk index cbc850a1d5280..5a2842cffbd98 100644 --- a/deps/p7zip.mk +++ b/deps/p7zip.mk @@ -3,28 +3,26 @@ include $(SRCDIR)/p7zip.version ifneq ($(USE_BINARYBUILDER_P7ZIP),1) -P7ZIP_BUILD_OPTS := bindir=$(build_private_libexecdir) CC="$(CC)" CXX="$(CXX)" +$(SRCCACHE)/7z$(subst .,,$(P7ZIP_VER))-src.tar.xz: | $(SRCCACHE) + $(JLDOWNLOAD) $@ https://github.com/ip7z/7zip/releases/download/$(P7ZIP_VER)/7z$(subst .,,$(P7ZIP_VER))-src.tar.xz -$(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz: | $(SRCCACHE) - $(JLDOWNLOAD) $@ https://github.com/p7zip-project/p7zip/archive/refs/tags/v$(P7ZIP_VER).tar.gz - -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz +$(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted: $(SRCCACHE)/7z$(subst .,,$(P7ZIP_VER))-src.tar.xz $(JLCHECKSUM) $< mkdir -p $(dir $@) - cd $(dir $@) && $(TAR) --strip-components 1 -zxf $< + cd $(dir $@) && $(TAR) -Jxf $< echo 1 > $@ -checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz +checksum-p7zip: $(SRCCACHE)/7z$(subst .,,$(P7ZIP_VER))-src.tar.xz $(JLCHECKSUM) $< $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured - $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za + $(MAKE) -C $(dir $<)CPP/7zip/Bundles/Alone -f makefile.gcc echo 1 > $@ define P7ZIP_INSTALL mkdir -p $2/$$(build_private_libexecdir)/ - cp -a $1/bin/7za$(EXE) $2/$$(build_private_libexecdir)/7z$(EXE) + cp -a $1/CPP/7zip/Bundles/Alone/_o/7za$(EXE) $2/$$(build_private_libexecdir)/7z$(EXE) endef $(eval $(call staged-install, \ p7zip,p7zip-$(P7ZIP_VER), \ @@ -33,13 +31,13 @@ $(eval $(call staged-install, \ clean-p7zip: -rm -f $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled -rm -f $(build_bindir)/7z$(EXE) $(build_bindir)/7z$(EXE) $(build_private_libexecdir)/7z$(EXE) - -$(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) clean + -if [ -d $(BUILDDIR)/p7zip-$(P7ZIP_VER) ]; then $(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) clean; fi distclean-p7zip: - rm -rf $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(SRCCACHE)/p7zip-$(P7ZIP_VER) $(BUILDDIR)/p7zip-$(P7ZIP_VER) + rm -rf $(SRCCACHE)/7z$(subst .,,$(P7ZIP_VER))-src.tar.xz $(SRCCACHE)/p7zip-$(P7ZIP_VER) $(BUILDDIR)/p7zip-$(P7ZIP_VER) -get-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz +get-p7zip: $(SRCCACHE)/7z$(subst .,,$(P7ZIP_VER))-src.tar.xz extract-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER)/source-extracted configure-p7zip: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured compile-p7zip: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled @@ -53,19 +51,16 @@ $(eval $(call bb-install,p7zip,P7ZIP,false)) # move from bindir to shlibdir, where we expect to install it install-p7zip: post-install-p7zip uninstall-p7zip: pre-uninstall-p7zip -post-install-p7zip: $(build_prefix)/manifest/p7zip +post-install-p7zip: $(build_prefix)/manifest/p7zip $(PATCHELF_MANIFEST) mkdir -p $(build_private_libexecdir)/ [ ! -e $(build_bindir)/7z$(EXE) ] || mv $(build_bindir)/7z$(EXE) $(build_private_libexecdir)/7z$(EXE) [ -e $(build_private_libexecdir)/7z$(EXE) ] -ifeq ($(OS),WINNT) - [ ! -e $(build_bindir)/7z.dll ] || mv $(build_bindir)/7z.dll $(build_private_libexecdir)/7z.dll - [ -e $(build_private_libexecdir)/7z.dll ] +ifneq (,$(findstring $(OS),Linux FreeBSD)) + [ -L $(build_private_libexecdir)/7z ] || \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_build_private_libexecdir_rel)' $(build_private_libexecdir)/7z$(EXE) endif pre-uninstall-p7zip: -rm -f $(build_private_libexecdir)/7z$(EXE) -ifeq ($(OS),WINNT) - -rm -f $(build_private_libexecdir)/7z.dll -endif .PHONY: post-install-p7zip pre-uninstall-p7zip diff --git a/deps/p7zip.version b/deps/p7zip.version index 0fcde938eeb95..3cd8ac6ee5418 100644 --- a/deps/p7zip.version +++ b/deps/p7zip.version @@ -1,5 +1,7 @@ +# -*- makefile -*- + ## jll artifact P7ZIP_JLL_NAME := p7zip ## source build -P7ZIP_VER := 17.05 +P7ZIP_VER := 26.02 diff --git a/deps/patchelf.mk b/deps/patchelf.mk index aaf0ecb313b80..d74db8662c323 100644 --- a/deps/patchelf.mk +++ b/deps/patchelf.mk @@ -14,10 +14,13 @@ $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted: $(SRCCACHE)/patchelf-$(PA checksum-patchelf: $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2 $(JLCHECKSUM) $< +$(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-patched: $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted + echo 1 > $@ + $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured: CC:=$(HOSTCC) $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured: CXX:=$(HOSTCXX) $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured: XC_HOST:=$(BUILD_MACHINE) -$(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured: $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted +$(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured: $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-patched mkdir -p $(dir $@) cd $(dir $@) && \ $(dir $<)/configure $(CONFIGURE_COMMON) LDFLAGS="$(CXXLDFLAGS)" CPPFLAGS="$(CPPFLAGS)" MAKE=$(MAKE) @@ -41,7 +44,7 @@ $(eval $(call staged-install, \ clean-patchelf: -rm -f $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured \ $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/patchelf-$(PATCHELF_VER) clean + -if [ -d $(BUILDDIR)/patchelf-$(PATCHELF_VER) ]; then $(MAKE) -C $(BUILDDIR)/patchelf-$(PATCHELF_VER) clean; fi distclean-patchelf: rm -rf $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2 \ @@ -51,6 +54,7 @@ distclean-patchelf: get-patchelf: $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2 extract-patchelf: $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted +patch-patchelf: $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-patched configure-patchelf: $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-configured compile-patchelf: $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-compiled check-patchelf: $(BUILDDIR)/patchelf-$(PATCHELF_VER)/build-checked diff --git a/deps/patchelf.version b/deps/patchelf.version index 6e4f32a0c2fe4..1f5a7a114d97c 100644 --- a/deps/patchelf.version +++ b/deps/patchelf.version @@ -1,4 +1,3 @@ ## source build # Patchelf (we don't ship this or even use a JLL, we just always build it) -# NOTE: Do not upgrade this to 0.18+ until https://github.com/NixOS/patchelf/issues/492 is fixed -PATCHELF_VER := 0.17.2 +PATCHELF_VER := 0.19.1 diff --git a/deps/patches/0001-Fix-MinGW-Windows-type-errors.patch b/deps/patches/0001-Fix-MinGW-Windows-type-errors.patch new file mode 100644 index 0000000000000..0861e0d773a36 --- /dev/null +++ b/deps/patches/0001-Fix-MinGW-Windows-type-errors.patch @@ -0,0 +1,118 @@ +From cb6acd394ee3d852b94ccd5e6236c15168992c96 Mon Sep 17 00:00:00 2001 +From: Keno Fischer +Date: Mon, 13 Jul 2026 02:26:43 +0000 +Subject: [PATCH] Fix MinGW Windows type errors + +Access LARGE_INTEGER through QuadPart, use SIZE_T for mapped view sizes, and pass the address type required by NtMapViewOfSection so the Windows remapping and incremental-copy code compiles as C with MinGW. + +Co-authored-by: Codex +--- + libraries/liblmdb/mdb.c | 30 ++++++++++++++++++++++++------ + 1 file changed, 24 insertions(+), 6 deletions(-) + +diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c +index 6802cc7..4da2542 100644 +--- a/libraries/liblmdb/mdb.c ++++ b/libraries/liblmdb/mdb.c +@@ -66,7 +66,7 @@ typedef enum _SECTION_INHERIT { + } SECTION_INHERIT; + + typedef NTSTATUS (WINAPI NtMapViewOfSectionFunc) +- (IN PHANDLE sh, IN HANDLE ph, ++ (IN HANDLE sh, IN HANDLE ph, + IN OUT PVOID *addr, IN ULONG_PTR zbits, + IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL, + IN OUT PSIZE_T vs, IN SECTION_INHERIT ih, +@@ -6928,7 +6928,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret, MDB_page **e + #define SET_OFF(off,val) off.QuadPart = val + #define MAP(rc,env,addr,len,off) \ + addr = NULL; \ +- rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \ ++ rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), (PVOID *)&(addr), 0, \ + len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \ + if (rc) rc = mdb_nt2win32(rc) + #else +@@ -11608,13 +11608,16 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + char *ptr; + #ifdef _WIN32 + DWORD len, w2; ++#if MDB_RPAGE_CACHE ++ SIZE_T mlen; ++#endif + #else + ssize_t len; + size_t w2; + #endif + #if MDB_RPAGE_CACHE + #ifdef _WIN32 +- LARGE_INTEGER off = 0; ++ LARGE_INTEGER off; + #else + off_t off = 0; + #endif +@@ -11678,7 +11681,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + } + #if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { +- off = wsize; ++ SET_OFF(off, wsize); + } + #endif + wsize = w3 - wsize; +@@ -11686,7 +11689,12 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize; + #if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { ++#ifdef _WIN32 ++ mlen = w2; ++ MAP(rc, env, ptr, mlen, off); ++#else + MAP(rc, env, ptr, w2, off); ++#endif + } + #endif + DO_WRITE(rc, fd, ptr, w2, len); +@@ -11704,7 +11712,11 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + wsize -= len; + #if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { ++#ifdef _WIN32 ++ off.QuadPart += len; ++#else + off += len; ++#endif + } + #endif + continue; +@@ -11946,7 +11958,8 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) + return ENOMEM; + + #ifdef _WIN32 +- SetFilePointerEx(env->me_fd, 0, NULL, FILE_BEGIN); ++ off.QuadPart = 0; ++ SetFilePointerEx(env->me_fd, off, NULL, FILE_BEGIN); + #else + lseek(env->me_fd, 0, SEEK_SET); + #endif +@@ -12011,12 +12024,17 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) + ptr += rlen; + rsize -= rlen; + } ++#ifdef _WIN32 ++ off.QuadPart = (LONGLONG)(pg-prevpg-numprev) * env->me_psize; ++#else + off = (pg-prevpg-numprev) * env->me_psize; ++#endif + rsize = numpgs * env->me_psize; +- if (off) { + #ifdef _WIN32 ++ if (off.QuadPart) { + SetFilePointerEx(env->me_fd, off, NULL, FILE_CURRENT); + #else ++ if (off) { + lseek(env->me_fd, off, SEEK_CUR); + #endif + } +-- +2.54.0 + diff --git a/deps/patches/0002-Refuse-to-open-environments-on-network-filesystems.patch b/deps/patches/0002-Refuse-to-open-environments-on-network-filesystems.patch new file mode 100644 index 0000000000000..cf073e20e3d00 --- /dev/null +++ b/deps/patches/0002-Refuse-to-open-environments-on-network-filesystems.patch @@ -0,0 +1,137 @@ +From f95cc314e8044f30ef370e56d20d4a2c2df21342 Mon Sep 17 00:00:00 2001 +From: Keno Fischer +Date: Sun, 19 Jul 2026 02:02:09 +0000 +Subject: [PATCH 2/3] Refuse to open environments on network filesystems + +LMDB environments rely on coherent shared memory mappings of the +lockfile and on advisory file locks on the data and lock files. +Network filesystems provide neither reliably: mmap coherence between +clients is not guaranteed (or MAP_SHARED may not even be supported), +and lock implementations are frequently broken or unavailable, which +can corrupt the database or wedge every client of the environment. + +Detect well-known network filesystems in mdb_env_open() and refuse to +open the environment with a new MDB_REMOTE_FS error code. Detection +uses statfs(2) f_type magic numbers on Linux, the MNT_LOCAL mount flag +on macOS and the BSDs, and GetDriveType() on Windows. Filesystems +that cannot be identified are presumed to be local. + +Co-Authored-By: Claude Fable 5 +--- + libraries/liblmdb/lmdb.h | 4 ++- + libraries/liblmdb/mdb.c | 72 ++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 75 insertions(+), 1 deletion(-) + +diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h +index 641a3d5..1e34fd7 100644 +--- a/libraries/liblmdb/lmdb.h ++++ b/libraries/liblmdb/lmdb.h +@@ -540,8 +540,10 @@ typedef enum MDB_cursor_op { + #define MDB_IS_READONLY (-30770) + /** Requested map address is unavailable */ + #define MDB_ADDR_BUSY (-30769) ++ /** Environment resides on a network filesystem */ ++#define MDB_REMOTE_FS (-30768) + /** The last defined error code */ +-#define MDB_LAST_ERRCODE MDB_ADDR_BUSY ++#define MDB_LAST_ERRCODE MDB_REMOTE_FS + /** @} */ + + /** @brief Statistics for a database in the environment */ +diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c +index 4da2542..985323a 100644 +--- a/libraries/liblmdb/mdb.c ++++ b/libraries/liblmdb/mdb.c +@@ -1925,6 +1925,7 @@ static char *const mdb_errstr[] = { + "MDB_ENV_BUSY: Environment is busy, can't use previous snapshot", + "MDB_IS_READONLY: Can't write in readonly txn or environment", + "MDB_ADDR_BUSY: Requested map address is unavailable", ++ "MDB_REMOTE_FS: Environment resides on a network filesystem", + }; + + char * +@@ -6227,6 +6228,73 @@ mdb_env_envflags(MDB_env *env) + # error "Persistent DB flags & env flags overlap, but both go in mm_flags" + #endif + ++#if defined(__linux__) ++# include ++#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) ++# include /* statfs(), MNT_LOCAL */ ++#endif ++ ++/** Check if the environment resides on a network (remote) filesystem. ++ * LMDB requires coherent shared memory maps and working file locks, ++ * neither of which can be relied upon on network filesystems, so ++ * #mdb_env_open() refuses such paths with #MDB_REMOTE_FS. ++ * Filesystems that cannot be identified are presumed to be local. ++ * @param[in] path The path passed to #mdb_env_open() (used on POSIX). ++ * @param[in] fname The converted filename (used on Windows). ++ * @return 1 if the filesystem is known to be remote, 0 otherwise. ++ */ ++static int ESECT ++mdb_env_fs_is_remote(const char *path, const MDB_name *fname) ++{ ++#if defined(_WIN32) ++ WCHAR vol[MAX_PATH+1]; ++ (void)path; ++ if (!GetVolumePathNameW(fname->mn_val, vol, MAX_PATH+1)) ++ return 0; ++ return GetDriveTypeW(vol) == DRIVE_REMOTE; ++#elif defined(__linux__) ++ /* Linux statfs(2) has no local/remote flag; compare f_type against ++ * the known network filesystem magic numbers from statfs(2). ++ */ ++ static const unsigned long remote_magics[] = { ++ 0x6969, /* NFS */ ++ 0x517b, /* SMB */ ++ 0xfe534d42, /* SMB2 */ ++ 0xff534d42, /* CIFS */ ++ 0x73757245, /* Coda */ ++ 0x5346414f, /* AFS */ ++ 0x6b414653, /* kAFS */ ++ 0x564c, /* NCP */ ++ 0x01021997, /* 9P (VM/container shared folders) */ ++ 0x0bd00bd0, /* Lustre */ ++ 0x00c36400, /* Ceph */ ++ 0x01161970, /* GFS2 */ ++ 0x7461636f, /* OCFS2 */ ++ 0x786f4256, /* VirtualBox shared folders */ ++ }; ++ struct statfs st; ++ unsigned i; ++ (void)fname; ++ if (statfs(path, &st)) ++ return 0; ++ for (i = 0; i < sizeof(remote_magics)/sizeof(remote_magics[0]); i++) ++ if ((unsigned long)st.f_type == remote_magics[i]) ++ return 1; ++ return 0; ++#elif defined(MNT_LOCAL) ++ /* macOS and the BSDs flag local mounts */ ++ struct statfs st; ++ (void)fname; ++ if (statfs(path, &st)) ++ return 0; ++ return !(st.f_flags & MNT_LOCAL); ++#else ++ (void)path; ++ (void)fname; ++ return 0; ++#endif ++} ++ + int ESECT + mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) + { +@@ -6282,6 +6350,10 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode + flags |= MDB_RAWPART | MDB_NOSUBDIR; + } + #endif ++ if (!(flags & MDB_RAWPART) && mdb_env_fs_is_remote(path, &fname)) { ++ rc = MDB_REMOTE_FS; ++ goto leave; ++ } + flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close_active() to clean up */ + + if (flags & MDB_RDONLY) { diff --git a/deps/patches/0003-Refuse-to-open-environments-across-pid-namespaces.patch b/deps/patches/0003-Refuse-to-open-environments-across-pid-namespaces.patch new file mode 100644 index 0000000000000..de6ec2675fe29 --- /dev/null +++ b/deps/patches/0003-Refuse-to-open-environments-across-pid-namespaces.patch @@ -0,0 +1,154 @@ +From d307fe69fd3f8e178a106f5b929991f55abb1681 Mon Sep 17 00:00:00 2001 +From: Keno Fischer +Date: Sun, 19 Jul 2026 02:04:15 +0000 +Subject: [PATCH 3/3] Refuse to open environments across pid namespaces + +LMDB identifies other users of an environment by pid: the reader +table records reader pids, F_SETLK pid locks are used for liveness +checks, and mdb_reader_check() clears slots whose pid is gone. All +of this assumes every user of the environment observes the same pid +numbers, which is not true across pid namespaces: the same numeric +pid can name different processes, so a reader in another namespace +can be judged dead (its slot recycled and its pages reclaimed) while +it is still active, or a stale slot can pin the environment because +its pid happens to be taken in the checking process's namespace. + +Record the pid namespace of the process that initializes the lock +region (the exclusive-lock holder, i.e. the first active opener) in +a new lockfile header field, and refuse to join an active lock +region initialized from a different pid namespace with a new +MDB_PIDNS_MISMATCH error code. Once all users are gone, the next +opener re-initializes the lock region and stamps its own namespace. +Pid namespaces are identified by the inode of /proc/self/ns/pid on +Linux; on other platforms (and if /proc is unavailable) the field is +0 and the check is disabled. + +Since this changes the lockfile layout, bump MDB_LOCK_VERSION to 902 +so unpatched LMDB rejects these lockfiles instead of misreading them. + +Co-Authored-By: Claude Fable 5 +--- + libraries/liblmdb/lmdb.h | 4 +++- + libraries/liblmdb/mdb.c | 51 ++++++++++++++++++++++++++++++++++++++-- + 2 files changed, 52 insertions(+), 3 deletions(-) + +diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h +index 1e34fd7..b7733c1 100644 +--- a/libraries/liblmdb/lmdb.h ++++ b/libraries/liblmdb/lmdb.h +@@ -542,8 +542,10 @@ typedef enum MDB_cursor_op { + #define MDB_ADDR_BUSY (-30769) + /** Environment resides on a network filesystem */ + #define MDB_REMOTE_FS (-30768) ++ /** Environment is in use from a different pid namespace */ ++#define MDB_PIDNS_MISMATCH (-30767) + /** The last defined error code */ +-#define MDB_LAST_ERRCODE MDB_REMOTE_FS ++#define MDB_LAST_ERRCODE MDB_PIDNS_MISMATCH + /** @} */ + + /** @brief Statistics for a database in the environment */ +diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c +index 985323a..9cfb1b7 100644 +--- a/libraries/liblmdb/mdb.c ++++ b/libraries/liblmdb/mdb.c +@@ -683,8 +683,10 @@ static txnid_t mdb_debug_start; + + /** The version number for a database's datafile format. */ + #define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 3) +- /** The version number for a database's lockfile format. */ +-#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 2) ++ /** The version number for a database's lockfile format. ++ * 902 = upstream version 2, plus the #MDB_txbody.%mtb_pidns field. ++ */ ++#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 902) + /** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT. + * The remaining bits must leave room for #MDB_lock_desc. + */ +@@ -908,6 +910,17 @@ typedef struct MDB_txbody { + * when readers release their slots. + */ + volatile unsigned mtb_numreaders; ++ /** The pid namespace of the process that initialized the lock ++ * region, or 0 if it could not be determined (also on platforms ++ * without pid namespaces). Liveness detection of other users of ++ * the environment (reader table pids, pid locks and, depending ++ * on the platform, robust mutex recovery) relies on pids being ++ * comparable between all users, which does not hold across pid ++ * namespaces, so #mdb_env_open() refuses to join a lock region ++ * initialized in a different pid namespace with ++ * #MDB_PIDNS_MISMATCH. ++ */ ++ uint64_t mtb_pidns; + #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + /** Binary form of names of the reader/writer locks */ + mdb_hash_t mtb_mutexid; +@@ -931,6 +944,7 @@ typedef struct MDB_txninfo { + #define mti_rmutex mt1.mtb.mtb_rmutex + #define mti_txnid mt1.mtb.mtb_txnid + #define mti_numreaders mt1.mtb.mtb_numreaders ++#define mti_pidns mt1.mtb.mtb_pidns + #define mti_mutexid mt1.mtb.mtb_mutexid + #ifdef MDB_USE_SYSV_SEM + #define mti_semid mt1.mtb.mtb_semid +@@ -1926,6 +1940,7 @@ static char *const mdb_errstr[] = { + "MDB_IS_READONLY: Can't write in readonly txn or environment", + "MDB_ADDR_BUSY: Requested map address is unavailable", + "MDB_REMOTE_FS: Environment resides on a network filesystem", ++ "MDB_PIDNS_MISMATCH: Environment is in use from a different pid namespace", + }; + + char * +@@ -5911,6 +5926,23 @@ mdb_env_mname_init(MDB_env *env) + + #endif + ++/** Return an identifier for the pid namespace this process runs in, ++ * or 0 if it cannot be determined. Non-zero only on Linux; other ++ * supported platforms have no pid namespaces. ++ */ ++static uint64_t ESECT ++mdb_env_pidns(void) ++{ ++#ifdef __linux__ ++ struct stat st; ++ if (stat("/proc/self/ns/pid", &st)) ++ return 0; ++ return (uint64_t)st.st_ino; ++#else ++ return 0; ++#endif ++} ++ + /** Open and/or initialize the lock region for the environment. + * @param[in] env The LMDB environment. + * @param[in] fname Filename + scratch area, from #mdb_fname_init(). +@@ -6099,6 +6131,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) + env->me_txns->mti_format = MDB_LOCK_FORMAT; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; ++ env->me_txns->mti_pidns = mdb_env_pidns(); + + } else { + #ifdef MDB_USE_SYSV_SEM +@@ -6115,6 +6148,20 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) + rc = MDB_VERSION_MISMATCH; + goto fail; + } ++ { ++ /* The exclusive-lock holder that initialized the lock region ++ * is (or was, while any of the openers it admitted are still ++ * alive) in charge of pid-based liveness detection, which ++ * breaks down if users of the environment disagree about pid ++ * namespaces. Refuse to join from a different pid namespace. ++ */ ++ uint64_t pidns = mdb_env_pidns(); ++ if (pidns && env->me_txns->mti_pidns && ++ env->me_txns->mti_pidns != pidns) { ++ rc = MDB_PIDNS_MISMATCH; ++ goto fail; ++ } ++ } + #ifdef _WIN32 + mdb_env_mname_init(env); + env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'r')); diff --git a/deps/patches/BOLT-aarch64-adr-relaxation-non-simple.patch b/deps/patches/BOLT-aarch64-adr-relaxation-non-simple.patch new file mode 100644 index 0000000000000..fb1f185b16ce3 --- /dev/null +++ b/deps/patches/BOLT-aarch64-adr-relaxation-non-simple.patch @@ -0,0 +1,80 @@ +From c74738e6a59d86db31cff48591adc7f59cace00b Mon Sep 17 00:00:00 2001 +From: Adam Bzowski +Date: Wed, 12 Aug 2026 08:43:53 +0100 +Subject: [PATCH] [BOLT] Avoid unnecessary AArch64 ADR relaxation (#215415) + +This is a temporary fix for an AArch64 ADR/LDR relaxation issue. The +correct treatment likely requires merging AArch64RelaxationPass with +LongJmpPass, so address materialization and branch/stub insertion can +use one final layout model. + +The relaxation pass currently tries to relax every ADR/LDR in functions +at least 1 MiB, including same-fragment references in non-simple +functions. Since BOLT cannot grow non-simple functions without an +adjacent NOP, this can fail even though the preserved layout leaves the +original displacement valid. + +Skip relaxation for same-fragment references in non-simple functions, +while retaining the size check for reorderable simple functions. +--- + bolt/lib/Passes/AArch64RelaxationPass.cpp | 8 +++-- + .../AArch64/adr-relaxation-large-non-simple.s | 30 +++++++++++++++++++ + 2 files changed, 35 insertions(+), 3 deletions(-) + create mode 100644 bolt/test/AArch64/adr-relaxation-large-non-simple.s + +diff --git a/bolt/lib/Passes/AArch64RelaxationPass.cpp b/bolt/lib/Passes/AArch64RelaxationPass.cpp +index 2b7384dc848dd..51260ceadf918 100644 +--- a/bolt/lib/Passes/AArch64RelaxationPass.cpp ++++ b/bolt/lib/Passes/AArch64RelaxationPass.cpp +@@ -59,10 +59,12 @@ void AArch64RelaxationPass::runOnFunction(BinaryFunction &BF) { + continue; + } + +- // Don't relax ADR/LDR if it points to the same function and is in the +- // main fragment and BF initial size is < 1MB. ++ // The layout of a non-simple function is preserved, so references within ++ // the same fragment retain their original in-range displacement. For ++ // simple functions, basic blocks can move, but an initial size below 1MiB ++ // guarantees that internal references remain in range after reordering. + const unsigned OneMB = 0x100000; +- if (BF.getSize() < OneMB) { ++ if (!BF.isSimple() || BF.getSize() < OneMB) { + BinaryFunction *TargetBF = BC.getFunctionForSymbol(Symbol); + if (TargetBF == &BF && !BB.isSplit()) + continue; +diff --git a/bolt/test/AArch64/adr-relaxation-large-non-simple.s b/bolt/test/AArch64/adr-relaxation-large-non-simple.s +new file mode 100644 +index 0000000000000..babcdb5304b98 +--- /dev/null ++++ b/bolt/test/AArch64/adr-relaxation-large-non-simple.s +@@ -0,0 +1,30 @@ ++## Check that an ADR targeting the same fragment is not relaxed in a large ++## non-simple function. BOLT preserves the layout of non-simple functions, so ++## the ADR displacement cannot change even when the function is larger than the ++## instruction's 1MiB range. ++ ++# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o ++# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q ++# RUN: llvm-bolt %t.exe -o %t.bolt --lite=false ++ ++ .text ++ .globl _start ++ .type _start, %function ++_start: ++ .cfi_startproc ++.Ladr: ++ adr x1, .Ladr ++ br x0 ++ ++ // Make the function's code larger than 1MiB. The unknown indirect branch ++ // makes the function non-simple, while the self-referential ADR remains in ++ // range. ++ .rept 0x40000 ++ nop ++ .endr ++ ret ++ .cfi_endproc ++ .size _start, .-_start ++ ++ // Force BOLT's relocation mode. ++ .reloc 0, R_AARCH64_NONE diff --git a/deps/patches/SuiteSparse-shlib.patch b/deps/patches/SuiteSparse-shlib.patch deleted file mode 100644 index fe25aa1ef38f1..0000000000000 --- a/deps/patches/SuiteSparse-shlib.patch +++ /dev/null @@ -1,43 +0,0 @@ ---- a/SuiteSparse_config/SuiteSparse_config.mk 2018-12-22 22:15:58.000000000 -0500 -+++ b/SuiteSparse_config/SuiteSparse_config.mk 2019-02-28 01:40:51.768394370 -0500 -@@ -357,8 +357,8 @@ - #--------------------------------------------------------------------------- - - ifeq ($(UNAME),Linux) -- # add the realtime library, librt, and SuiteSparse/lib -- LDLIBS += -lrt -Wl,-rpath=$(INSTALL_LIB) -+ # add the posix realtime extensions library: librt -+ LDLIBS += -lrt - endif - - #--------------------------------------------------------------------------- -@@ -447,11 +447,14 @@ - SO_OPTS = $(LDFLAGS) - - ifeq ($(UNAME),Windows) -- # Cygwin Make on Windows (untested) -+ # Cygwin Make on Windows - AR_TARGET = $(LIBRARY).lib -- SO_PLAIN = $(LIBRARY).dll -+ SO_TARGET = $(LIBRARY).dll -+ # The following two links are just garbage copies of the real target -+ # they aren't actually supported by this OS - SO_MAIN = $(LIBRARY).$(SO_VERSION).dll -- SO_TARGET = $(LIBRARY).$(VERSION).dll -+ SO_PLAIN = $(LIBRARY).$(VERSION).dll -+ SO_OPTS += -shared - SO_INSTALL_NAME = echo - else - # Mac or Linux/Unix -@@ -472,8 +475,9 @@ - SO_PLAIN = $(LIBRARY).so - SO_MAIN = $(LIBRARY).so.$(SO_VERSION) - SO_TARGET = $(LIBRARY).so.$(VERSION) -- SO_OPTS += -shared -Wl,-soname -Wl,$(SO_MAIN) -Wl,--no-undefined -- # Linux/Unix *.so files can be moved without modification: -+ SO_OPTS += -shared -Wl,-soname -Wl,$(SO_MAIN) -Wl,--no-undefined \ -+ -Wl,-rpath,'$$ORIGIN' -Wl,-z,origin -+ # Use rpath ORIGIN so that Linux/Unix *.so files can be moved without modification: - SO_INSTALL_NAME = echo - endif - endif diff --git a/deps/patches/gmp-mpz_realloc.patch b/deps/patches/gmp-mpz_realloc.patch new file mode 100644 index 0000000000000..5d074531a45c6 --- /dev/null +++ b/deps/patches/gmp-mpz_realloc.patch @@ -0,0 +1,102 @@ +--- a/mpz/realloc.c ++++ b/mpz/realloc.c +@@ -50,21 +50,40 @@ + __GMP_ALLOC_OVERFLOW_FUNC (); + } + +- if (ALLOC (m) == 0) ++ /* Allocate-copy-install-free-last (rather than realloc-in-place), so the ++ mpz is structurally valid at every interruptible point: an asynchronous ++ unwind (e.g. Julia task cancellation delivered inside GMP) between any ++ two statements leaves PTR/ALLOC/SIZ mutually consistent, at worst ++ leaking the block that was not yet freed. The realloc-based version ++ has a window where PTR (m) dangles - the old block is freed inside the ++ allocation hook before the new pointer is stored - which a later ++ mpz_clear would double free. ALLOC is stepped through MIN (old, new) ++ so that capacity is understated, never overstated, in every window. */ ++ mp = __GMP_ALLOCATE_FUNC_LIMBS (new_alloc); ++ if (ALLOC (m) != 0) + { +- mp = __GMP_ALLOCATE_FUNC_LIMBS (new_alloc); +- } +- else +- { +- mp = __GMP_REALLOCATE_FUNC_LIMBS (PTR (m), ALLOC (m), new_alloc); ++ mp_ptr old_mp = PTR (m); ++ mp_size_t old_alloc = ALLOC (m); ++ mp_size_t copy_size = MIN (ABSIZ (m), MIN (old_alloc, new_alloc)); ++ if (copy_size > new_alloc) ++ copy_size = new_alloc; ++ MPN_COPY (mp, old_mp, copy_size); + + /* Don't create an invalid number; if the current value doesn't fit after + reallocation, clear it to 0. */ + if (UNLIKELY (ABSIZ (m) > new_alloc)) + SIZ (m) = 0; +- } + +- PTR (m) = mp; +- ALLOC(m) = new_alloc; ++ if (new_alloc < old_alloc) ++ ALLOC (m) = new_alloc; ++ PTR (m) = mp; ++ ALLOC (m) = new_alloc; ++ __GMP_FREE_FUNC_LIMBS (old_mp, old_alloc); ++ } ++ else ++ { ++ PTR (m) = mp; ++ ALLOC (m) = new_alloc; ++ } + return (void *) mp; + } +diff --git a/mpz/mul.c b/mpz/mul.c +--- a/mpz/mul.c ++++ b/mpz/mul.c +@@ -103,20 +103,37 @@ + wsize = usize + vsize; + if (ALLOC (w) < wsize) + { +- if (ALLOC (w) != 0) ++ /* Allocate-install-free-last (mirroring _mpz_realloc), so W stays ++ structurally valid at every interruptible point: an asynchronous ++ unwind (e.g. Julia task cancellation delivered inside the ++ allocation hook) between any two statements leaves PTR/ALLOC/SIZ ++ mutually consistent, at worst leaking the just-allocated block. ++ The previous order freed the old limbs and raised ALLOC before the ++ new block was installed, so an unwind there left W pointing at ++ freed memory with an overstated capacity, which a later mpz_clear ++ double-freed. */ ++ mp_ptr old_wp = wp; ++ mp_size_t old_alloc = ALLOC (w); ++ wp = __GMP_ALLOCATE_FUNC_LIMBS (wsize); ++ PTR (w) = wp; ++ /* The old value is not copied; don't let SIZ describe uninitialized ++ limbs if the operation is unwound before the final SIZ store. ++ (usize/vsize were captured above - clearing SIZ is safe even when ++ W aliases an input.) */ ++ SIZ (w) = 0; ++ ALLOC (w) = wsize; ++ if (old_alloc != 0) + { +- if (wp == up || wp == vp) ++ if (old_wp == up || old_wp == vp) + { +- free_me = wp; +- free_me_size = ALLOC (w); ++ /* W is also an input: its old limbs are still read by the ++ multiplication below - free them only afterwards. */ ++ free_me = old_wp; ++ free_me_size = old_alloc; + } + else +- (*__gmp_free_func) (wp, (size_t) ALLOC (w) * GMP_LIMB_BYTES); ++ (*__gmp_free_func) (old_wp, (size_t) old_alloc * GMP_LIMB_BYTES); + } +- +- ALLOC (w) = wsize; +- wp = __GMP_ALLOCATE_FUNC_LIMBS (wsize); +- PTR (w) = wp; + } + else + { diff --git a/deps/patches/libTracyClient-no-sampling.patch b/deps/patches/libTracyClient-no-sampling.patch deleted file mode 100644 index c4c8576099348..0000000000000 --- a/deps/patches/libTracyClient-no-sampling.patch +++ /dev/null @@ -1,79 +0,0 @@ -commit 6249999153a9497b32bc84e9dc95a1537a0af714 -Author: Cody Tapscott -Date: Tue Apr 4 15:20:46 2023 -0400 - - linux: respect `TRACY_NO_SAMPLING` for sys-tracing - - This compile-time flag was being ignored on Linux. This change adds - gating for software-sampled stack trace sampling following the same - pattern as other `TRACY_NO_SAMPLE_*` options. - - If `TRACY_NO_SAMPLING=1` is provided as an environment variable, - software stack sampling is also disabled. - -diff --git a/public/client/TracySysTrace.cpp b/public/client/TracySysTrace.cpp -index 4a562eaa..af0641fe 100644 ---- a/public/client/TracySysTrace.cpp -+++ b/public/client/TracySysTrace.cpp -@@ -770,6 +770,13 @@ bool SysTraceStart( int64_t& samplingPeriod ) - TracyDebug( "sched_wakeup id: %i\n", wakeupId ); - TracyDebug( "drm_vblank_event id: %i\n", vsyncId ); - -+#ifdef TRACY_NO_SAMPLING -+ const bool noSoftwareSampling = true; -+#else -+ const char* noSoftwareSamplingEnv = GetEnvVar( "TRACY_NO_SAMPLING" ); -+ const bool noSoftwareSampling = noSoftwareSamplingEnv && noSoftwareSamplingEnv[0] == '1'; -+#endif -+ - #ifdef TRACY_NO_SAMPLE_RETIREMENT - const bool noRetirement = true; - #else -@@ -839,28 +846,31 @@ bool SysTraceStart( int64_t& samplingPeriod ) - pe.clockid = CLOCK_MONOTONIC_RAW; - #endif - -- TracyDebug( "Setup software sampling\n" ); -- ProbePreciseIp( pe, currentPid ); -- for( int i=0; i -Date: Wed Mar 8 23:18:36 2023 +0100 - - Add support for configuring plots to C API. - -diff --git a/public/client/TracyProfiler.cpp b/public/client/TracyProfiler.cpp -index 6104a7ed..38b5ea13 100644 ---- a/public/client/TracyProfiler.cpp -+++ b/public/client/TracyProfiler.cpp -@@ -4149,6 +4149,7 @@ TRACY_API void ___tracy_emit_frame_image( const void* image, uint16_t w, uint16_ - TRACY_API void ___tracy_emit_plot( const char* name, double val ) { tracy::Profiler::PlotData( name, val ); } - TRACY_API void ___tracy_emit_plot_float( const char* name, float val ) { tracy::Profiler::PlotData( name, val ); } - TRACY_API void ___tracy_emit_plot_int( const char* name, int64_t val ) { tracy::Profiler::PlotData( name, val ); } -+TRACY_API void ___tracy_emit_plot_config( const char* name, int type, int step, int fill, uint32_t color ) { tracy::Profiler::ConfigurePlot( name, tracy::PlotFormatType(type), step, fill, color ); } - TRACY_API void ___tracy_emit_message( const char* txt, size_t size, int callstack ) { tracy::Profiler::Message( txt, size, callstack ); } - TRACY_API void ___tracy_emit_messageL( const char* txt, int callstack ) { tracy::Profiler::Message( txt, callstack ); } - TRACY_API void ___tracy_emit_messageC( const char* txt, size_t size, uint32_t color, int callstack ) { tracy::Profiler::MessageColor( txt, size, color, callstack ); } -diff --git a/public/tracy/TracyC.h b/public/tracy/TracyC.h -index bedf5e16..736b51ed 100644 ---- a/public/tracy/TracyC.h -+++ b/public/tracy/TracyC.h -@@ -11,6 +11,13 @@ - extern "C" { - #endif - -+enum TracyPlotFormatEnum -+{ -+ TracyPlotFormatNumber, -+ TracyPlotFormatMemory, -+ TracyPlotFormatPercentage, -+}; -+ - TRACY_API void ___tracy_set_thread_name( const char* name ); - - #define TracyCSetThreadName( name ) ___tracy_set_thread_name( name ); -@@ -60,6 +67,8 @@ typedef const void* TracyCZoneCtx; - #define TracyCPlot(x,y) - #define TracyCPlotF(x,y) - #define TracyCPlotI(x,y) -+#define TracyCPlotConfig(x,y,z,w,a) -+ - #define TracyCMessage(x,y) - #define TracyCMessageL(x) - #define TracyCMessageC(x,y,z) -@@ -289,11 +298,13 @@ TRACY_API void ___tracy_emit_frame_image( const void* image, uint16_t w, uint16_ - TRACY_API void ___tracy_emit_plot( const char* name, double val ); - TRACY_API void ___tracy_emit_plot_float( const char* name, float val ); - TRACY_API void ___tracy_emit_plot_int( const char* name, int64_t val ); -+TRACY_API void ___tracy_emit_plot_config( const char* name, int type, int step, int fill, uint32_t color ); - TRACY_API void ___tracy_emit_message_appinfo( const char* txt, size_t size ); - - #define TracyCPlot( name, val ) ___tracy_emit_plot( name, val ); - #define TracyCPlotF( name, val ) ___tracy_emit_plot_float( name, val ); - #define TracyCPlotI( name, val ) ___tracy_emit_plot_int( name, val ); -+#define TracyCPlotConfig( name, type, step, fill, color ) ___tracy_emit_plot_config( name, type, step, fill, color ); - #define TracyCAppInfo( txt, size ) ___tracy_emit_message_appinfo( txt, size ); diff --git a/deps/patches/libssh2-CVE-2025-15661.patch b/deps/patches/libssh2-CVE-2025-15661.patch new file mode 100644 index 0000000000000..8e3ab286bcd77 --- /dev/null +++ b/deps/patches/libssh2-CVE-2025-15661.patch @@ -0,0 +1,111 @@ +From 2dae3024897e1898d389835151f4e9606227721d Mon Sep 17 00:00:00 2001 +From: Will Cosgrove +Date: Fri, 10 Oct 2025 08:26:20 -0700 +Subject: [PATCH] Update sftp_symlink to avoid out of bounds read on malformed + packet #1705 (#1717) + +--- libssh2-1.11.1.orig/src/sftp.c ++++ libssh2-1.11.1/src/sftp.c +@@ -3795,15 +3795,19 @@ static int sftp_symlink(LIBSSH2_SFTP *sf + { + LIBSSH2_CHANNEL *channel = sftp->channel; + LIBSSH2_SESSION *session = channel->session; +- size_t data_len = 0, link_len; ++ size_t data_len = 0, lk_len; + /* 13 = packet_len(4) + packet_type(1) + request_id(4) + path_len(4) */ + ssize_t packet_len = + path_len + 13 + + ((link_type == LIBSSH2_SFTP_SYMLINK) ? (4 + target_len) : 0); + unsigned char *s, *data = NULL; ++ struct string_buf buf; + static const unsigned char link_responses[2] = + { SSH_FXP_NAME, SSH_FXP_STATUS }; + int retcode; ++ unsigned char packet_type; ++ uint32_t tmp_u32; ++ unsigned char *lk_target; + + if(sftp->symlink_state == libssh2_NB_state_idle) { + sftp->last_errno = LIBSSH2_FX_OK; +@@ -3891,8 +3895,25 @@ static int sftp_symlink(LIBSSH2_SFTP *sf + + sftp->symlink_state = libssh2_NB_state_idle; + +- if(data[0] == SSH_FXP_STATUS) { +- retcode = _libssh2_ntohu32(data + 5); ++ buf.data = (unsigned char *)LIBSSH2_UNCONST(data); ++ buf.dataptr = buf.data; ++ buf.len = data_len; ++ ++ if(_libssh2_get_byte(&buf, &packet_type)) { ++ LIBSSH2_FREE(session, data); ++ return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, ++ "SFTP Protocol Error (type)"); ++ } ++ ++ if(packet_type == SSH_FXP_STATUS) { ++ if(_libssh2_get_u32(&buf, &tmp_u32)) { ++ LIBSSH2_FREE(session, data); ++ return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, ++ "SFTP Protocol Error (code)"); ++ } ++ ++ retcode = (int)tmp_u32; ++ + LIBSSH2_FREE(session, data); + if(retcode == LIBSSH2_FX_OK) + return LIBSSH2_ERROR_NONE; +@@ -3903,30 +3924,37 @@ static int sftp_symlink(LIBSSH2_SFTP *sf + } + } + +- if(_libssh2_ntohu32(data + 5) < 1) { ++ /* advance past id */ ++ if(_libssh2_get_u32(&buf, &tmp_u32)) { + LIBSSH2_FREE(session, data); + return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, +- "Invalid READLINK/REALPATH response, " +- "no name entries"); ++ "SFTP Protocol Error (id)"); + } + +- if(data_len < 13) { +- if(data_len > 0) { +- LIBSSH2_FREE(session, data); +- } ++ /* look for at least one link */ ++ if(_libssh2_get_u32(&buf, &tmp_u32) || tmp_u32 < 1) { ++ LIBSSH2_FREE(session, data); + return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, +- "SFTP stat packet too short"); ++ "Invalid READLINK/REALPATH response, " ++ "no name entries"); + } + +- /* this reads a u32 and stores it into a signed 32bit value */ +- link_len = _libssh2_ntohu32(data + 9); +- if(link_len < target_len) { +- memcpy(target, data + 13, link_len); +- target[link_len] = 0; +- retcode = (int)link_len; ++ if(_libssh2_get_string(&buf, &lk_target, &lk_len) == LIBSSH2_ERROR_NONE) { ++ if(lk_len < target_len) { ++ memcpy(target, lk_target, lk_len); ++ target[lk_len] = '\0'; ++ retcode = (int)lk_len; ++ } ++ else { ++ retcode = LIBSSH2_ERROR_BUFFER_TOO_SMALL; ++ } + } +- else +- retcode = LIBSSH2_ERROR_BUFFER_TOO_SMALL; ++ else { ++ LIBSSH2_FREE(session, data); ++ return _libssh2_error(session, LIBSSH2_ERROR_SFTP_PROTOCOL, ++ "SFTP Protocol Error (filename)"); ++ } ++ + LIBSSH2_FREE(session, data); + + return retcode; diff --git a/deps/patches/libssh2-CVE-2026-55199.patch b/deps/patches/libssh2-CVE-2026-55199.patch new file mode 100644 index 0000000000000..5ef23eec86b9d --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-55199.patch @@ -0,0 +1,21 @@ +From 17626857d20b3c9a1addfa45979dadcee1cd84a4 Mon Sep 17 00:00:00 2001 +From: TristanInSec +Date: Wed, 15 Apr 2026 14:51:08 -0400 +Subject: [PATCH] packet: check `_libssh2_get_string()` return in `EXT_INFO` + handler + +--- libssh2-1.11.1.orig/src/packet.c ++++ libssh2-1.11.1/src/packet.c +@@ -868,8 +868,10 @@ _libssh2_packet_add(LIBSSH2_SESSION * se + + nr_extensions -= 1; + +- _libssh2_get_string(&buf, &name, &name_len); +- _libssh2_get_string(&buf, &value, &value_len); ++ if(_libssh2_get_string(&buf, &name, &name_len)) ++ break; ++ if(_libssh2_get_string(&buf, &value, &value_len)) ++ break; + + if(name && value) { + _libssh2_debug((session, diff --git a/deps/patches/libssh2-CVE-2026-55200.patch b/deps/patches/libssh2-CVE-2026-55200.patch new file mode 100644 index 0000000000000..0adb94cb1a221 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-55200.patch @@ -0,0 +1,22 @@ +From 97acf3dfda80c91c3a8c9f2372546301d4a1a7a8 Mon Sep 17 00:00:00 2001 +From: Will Cosgrove +Date: Fri, 12 Jun 2026 15:57:44 -0700 +Subject: [PATCH] transport.c: Additional boundary checks for packet length + (#2052) + +--- libssh2-1.11.1.orig/src/transport.c ++++ libssh2-1.11.1/src/transport.c +@@ -639,8 +639,12 @@ int _libssh2_transport_read(LIBSSH2_SESS + total_num = 4; + + p->packet_length = _libssh2_ntohu32(block); +- if(p->packet_length < 1) ++ if(p->packet_length < 1) { + return LIBSSH2_ERROR_DECRYPT; ++ } ++ else if(p->packet_length > LIBSSH2_PACKET_MAXPAYLOAD) { ++ return LIBSSH2_ERROR_OUT_OF_BOUNDARY; ++ } + + /* total_num may include size field, however due to existing + * logic it needs to be removed after the entire packet is read diff --git a/deps/patches/libssh2-CVE-2026-58050-3449752.patch b/deps/patches/libssh2-CVE-2026-58050-3449752.patch new file mode 100644 index 0000000000000..a9dbcbac6e033 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-58050-3449752.patch @@ -0,0 +1,32 @@ +commit 34497525929b9a47f03dfb81887ac896202b7e12 +Author: Viktor Szakats +Date: Sun Jun 28 02:12:52 2026 +0200 + + publickey: fix potential multiplication overflow in 32-bit `libssh2_publickey_list_fetch()` + + Cap list size at 1024 elements. + + Reported-and-initial-patch-by: Mateusz Gierblinski + Reported-and-initial-patch-by: Behzod Abdullayev + Reported-by: Sharique Raza + + Follow-up to e15f5d97a04cc676ce117dd324fef85b046207a9 + + Closes #2128 + +diff --git a/src/publickey.c b/src/publickey.c +index a0d31000..3a6dd116 100644 +--- a/src/publickey.c ++++ b/src/publickey.c +@@ -1048,6 +1048,11 @@ int libssh2_publickey_list_fetch(LIBSSH2_PUBLICKEY *pkey, + } + + if(list[keys].num_attrs) { ++ if(list[keys].num_attrs > 1024) { ++ _libssh2_error(session, LIBSSH2_ERROR_OUT_OF_BOUNDARY, ++ "Too many publickey attributes"); ++ goto err_exit; ++ } + list[keys].attrs = + SSH2_ALLOC(session, + list[keys].num_attrs * diff --git a/deps/patches/libssh2-CVE-2026-58051-a9758da.patch b/deps/patches/libssh2-CVE-2026-58051-a9758da.patch new file mode 100644 index 0000000000000..c7160b5c1d139 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-58051-a9758da.patch @@ -0,0 +1,25 @@ +commit a9758da45a52bc8c630ec9493804d0c6ea30b24a +Author: Viktor Szakats +Date: Mon Jun 29 19:12:21 2026 +0200 + + publickey: fix potential arbitrary free in `libssh2_publickey_list_fetch()` (#2127) + + Due to uninitialized list entry. + + Reported-and-patch-by: Behzod Abdullayev + Reported-by: Sharique Raza + + Follow-up to e15f5d97a04cc676ce117dd324fef85b046207a9 + +diff --git a/src/publickey.c b/src/publickey.c +index b7f38431..a0d31000 100644 +--- a/src/publickey.c ++++ b/src/publickey.c +@@ -905,6 +905,7 @@ int libssh2_publickey_list_fetch(LIBSSH2_PUBLICKEY *pkey, + goto err_exit; + } + list = newlist; ++ memset(&list[keys], 0, sizeof(list[keys])); + } + if(pkey->version == 1) { + unsigned long comment_len; diff --git a/deps/patches/libssh2-CVE-2026-66032.patch b/deps/patches/libssh2-CVE-2026-66032.patch new file mode 100644 index 0000000000000..4e84a98b149b7 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-66032.patch @@ -0,0 +1,24 @@ +From 5e4776146552d898b9c0e1b313cd093fa8dc92d0 Mon Sep 17 00:00:00 2001 +From: Will Cosgrove +Date: Thu, 2 Jul 2026 11:00:23 -0700 +Subject: [PATCH] Prevent dangling pointer by nullifying data (#2180) + +Set data to NULL after freeing it to avoid dangling pointer. fixes +GHSA-px3w-7g75-hg7w. + +Credit: VladimirEliTokarev +Forwarded: not-needed +--- + src/sftp.c | 1 + + 1 file changed, 1 insertion(+) + +--- a/src/sftp.c ++++ b/src/sftp.c +@@ -1279,6 +1279,7 @@ + "got HANDLE FXOK")); + + LIBSSH2_FREE(session, data); ++ data = NULL; + + /* silly situation, but check for a HANDLE */ + rc = sftp_packet_require(sftp, SSH_FXP_HANDLE, diff --git a/deps/patches/libssh2-CVE-2026-66033.patch b/deps/patches/libssh2-CVE-2026-66033.patch new file mode 100644 index 0000000000000..a312ac83b0271 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-66033.patch @@ -0,0 +1,40 @@ +From a2ed82d40964bbc0d64cd717aa0a5a892117d2e6 Mon Sep 17 00:00:00 2001 +From: Viktor Szakats +Date: Thu, 23 Jul 2026 10:32:04 +0200 +Subject: [PATCH] openssl: fix potential OOB read/write with AES-GCM in + `ssh2_cipher_crypt()` + +By applying two bounds checks to non-debug builds. + +Reported-by: Vladimir Eli Tokarev +Fixes GHSA-c4f7-cvfc-33j7 +Follow-up to 3c953c05d67eb1ebcfd3316f279f12c4b1d600b4 #797 + +Closes #2401 +Forwarded: not-needed +--- + src/openssl.c | 10 ++++++---- + 1 file changed, 6 insertions(+), 4 deletions(-) + +--- a/src/openssl.c ++++ b/src/openssl.c +@@ -1042,13 +1042,15 @@ + const int aadlen = (is_aesgcm && IS_FIRST(firstlast)) ? 4 : 0; + /* size of AT, if present */ + const int authenticationtag = IS_LAST(firstlast) ? authlen : 0; +- /* length to encrypt */ +- const int cryptlen = (unsigned int)blocksize - aadlen - authenticationtag; ++ unsigned int cryptlen; /* length to encrypt */ + + (void)algo; + +- assert(blocksize <= sizeof(buf)); +- assert(cryptlen >= 0); ++ if(blocksize > sizeof(buf) || ++ blocksize < (size_t)(aadlen + authenticationtag)) ++ return 1; ++ ++ cryptlen = (unsigned int)blocksize - aadlen - authenticationtag; + + #if LIBSSH2_AES_GCM + /* First block */ diff --git a/deps/patches/libssh2-CVE-2026-66034.patch b/deps/patches/libssh2-CVE-2026-66034.patch new file mode 100644 index 0000000000000..c0d06df375727 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-66034.patch @@ -0,0 +1,31 @@ +From a13bb6c773f0d55ad1628cede57e99803cd898d9 Mon Sep 17 00:00:00 2001 +From: Viktor Szakats +Date: Sat, 4 Jul 2026 11:19:49 +0200 +Subject: [PATCH] publickey: fix potential OOB read in + `libssh2_publickey_list_fetch()` + +Reported-by: Vladimir Eli Tokarev +Fixes GHSA-w6g9-cpfp-22gc + +Closes #2202 +Forwarded: not-needed +--- + src/publickey.c | 7 +++++++ + 1 file changed, 7 insertions(+) + +--- a/src/publickey.c ++++ b/src/publickey.c +@@ -988,6 +988,13 @@ + } + + if(comment_len) { ++ if(pkey->listFetch_s + comment_len > ++ pkey->listFetch_data + pkey->listFetch_data_len) { ++ _libssh2_error(session, LIBSSH2_ERROR_BUFFER_TOO_SMALL, ++ "ListFetch data too short"); ++ goto err_exit; ++ } ++ + list[keys].num_attrs = 1; + list[keys].attrs = + LIBSSH2_ALLOC(session, diff --git a/deps/patches/libssh2-CVE-2026-66035.patch b/deps/patches/libssh2-CVE-2026-66035.patch new file mode 100644 index 0000000000000..2008ea4035679 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-66035.patch @@ -0,0 +1,37 @@ +From 42e33d81577ed4b95d4b4f6f845e5ee8efe5eeb4 Mon Sep 17 00:00:00 2001 +From: Viktor Szakats +Date: Fri, 3 Jul 2026 18:22:55 +0200 +Subject: [PATCH] transport: fix potential heap overflow on ETM decrypt + +Reported-by: Vladimir Eli Tokarev +Fixes GHSA-6c79-444r-wx26 + +Closes #2198 +Forwarded: not-needed +--- + src/transport.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +--- a/src/transport.c ++++ b/src/transport.c +@@ -242,6 +242,12 @@ + unsigned char *decrypt_buffer; + int blocksize = session->remote.crypt->blocksize; + ++ if(p->total_num < mac_len + 4 + (size_t)blocksize) { ++ LIBSSH2_FREE(session, p->payload); ++ return LIBSSH2_ERROR_DECRYPT; ++ } ++ decrypt_size = (ssize_t)(p->total_num - mac_len - 4); ++ + rc = decrypt(session, p->payload + 4, + first_block, blocksize, FIRST_BLOCK); + if(rc) { +@@ -249,7 +255,6 @@ + } + + /* we need buffer for decrypt */ +- decrypt_size = p->total_num - mac_len - 4; + decrypt_buffer = LIBSSH2_ALLOC(session, decrypt_size); + if(!decrypt_buffer) { + return LIBSSH2_ERROR_ALLOC; diff --git a/deps/patches/libssh2-CVE-2026-7598-256d04b60d80bf1190e96b0ad1e91b2174d744b1.patch b/deps/patches/libssh2-CVE-2026-7598-256d04b60d80bf1190e96b0ad1e91b2174d744b1.patch new file mode 100644 index 0000000000000..1b85be2569a94 --- /dev/null +++ b/deps/patches/libssh2-CVE-2026-7598-256d04b60d80bf1190e96b0ad1e91b2174d744b1.patch @@ -0,0 +1,52 @@ +From 256d04b60d80bf1190e96b0ad1e91b2174d744b1 Mon Sep 17 00:00:00 2001 +From: Will Cosgrove +Date: Mon, 13 Apr 2026 11:18:25 -0700 +Subject: [PATCH] userauth.c: username_len bounds checking (#1858) + +Return errors when username_len will exceed bounds, fix existing bounds +check. + +Credit: +[dapickle](https://github.com/dapickle) +--- + src/userauth.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/src/userauth.c b/src/userauth.c +index f8e02651c4..43d9ab9b9d 100644 +--- a/src/userauth.c ++++ b/src/userauth.c +@@ -80,6 +80,12 @@ static char *userauth_list(LIBSSH2_SESSION *session, const char *username, + memset(&session->userauth_list_packet_requirev_state, 0, + sizeof(session->userauth_list_packet_requirev_state)); + ++ if(username_len > UINT32_MAX - 27) { ++ _libssh2_error(session, LIBSSH2_ERROR_PROTO, ++ "username_len out of bounds"); ++ return NULL; ++ } ++ + session->userauth_list_data_len = username_len + 27; + + if(session->userauth_list_data) { +@@ -316,6 +322,11 @@ userauth_password(LIBSSH2_SESSION *session, + * 40 = packet_type(1) + username_len(4) + service_len(4) + + * service(14)"ssh-connection" + method_len(4) + method(8)"password" + + * chgpwdbool(1) + password_len(4) */ ++ if(username_len > UINT32_MAX - 40) { ++ return _libssh2_error(session, LIBSSH2_ERROR_PROTO, ++ "username_len out of bounds"); ++ } ++ + session->userauth_pswd_data_len = username_len + 40; + + session->userauth_pswd_data0 = +@@ -456,7 +467,7 @@ userauth_password(LIBSSH2_SESSION *session, + } + + /* basic data_len + newpw_len(4) */ +- if(username_len + password_len + 44 <= UINT_MAX) { ++ if(username_len <= UINT32_MAX - password_len - 44) { + session->userauth_pswd_data_len = + username_len + password_len + 44; + s = session->userauth_pswd_data = diff --git a/deps/patches/libssh2-unconst-backport.patch b/deps/patches/libssh2-unconst-backport.patch new file mode 100644 index 0000000000000..4d75169fa43ab --- /dev/null +++ b/deps/patches/libssh2-unconst-backport.patch @@ -0,0 +1,24 @@ +Needed by the fix for CVE-2025-15661 + +Cherrypicked from +commit 606c102e52f8447de2b745dd6c5ddf418defc519 +Author: Viktor Szakats +Date: Thu Jan 30 21:18:23 2025 +0100 + +--- libssh2-1.11.1.orig/src/libssh2_priv.h ++++ libssh2-1.11.1/src/libssh2_priv.h +@@ -117,6 +117,14 @@ + #define UINT32_MAX 0xffffffffU + #endif + ++#ifdef _WIN64 ++#define LIBSSH2_UNCONST(p) ((void *)(libssh2_uint64_t)(const void *)(p)) ++#elif defined(_MSC_VER) ++#define LIBSSH2_UNCONST(p) ((void *)(unsigned int)(const void *)(p)) ++#else ++#define LIBSSH2_UNCONST(p) ((void *)(uintptr_t)(const void *)(p)) ++#endif ++ + #if (defined(__GNUC__) || defined(__clang__)) && \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(LIBSSH2_NO_FMT_CHECKS) diff --git a/deps/patches/libunwind-missing-parameter-names.patch b/deps/patches/libunwind-missing-parameter-names.patch deleted file mode 100644 index 59a33f1a56880..0000000000000 --- a/deps/patches/libunwind-missing-parameter-names.patch +++ /dev/null @@ -1,28 +0,0 @@ -From 9ebe5e12b8a0063953f9ef196b2433eca9933559 Mon Sep 17 00:00:00 2001 -From: Stephen Webb -Date: Tue, 15 Apr 2025 10:48:20 -0400 -Subject: [PATCH] Fix FTBFS in src/ptrace/_UPT_ptrauth_insn_mask.c - -Added missing parameter names to make C code comply to ISO/IEC 9899. ---- - src/ptrace/_UPT_ptrauth_insn_mask.c | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/src/ptrace/_UPT_ptrauth_insn_mask.c b/src/ptrace/_UPT_ptrauth_insn_mask.c -index dcc512370..e7b3a514b 100644 ---- a/src/ptrace/_UPT_ptrauth_insn_mask.c -+++ b/src/ptrace/_UPT_ptrauth_insn_mask.c -@@ -49,9 +49,10 @@ unw_word_t _UPT_ptrauth_insn_mask (UNUSED unw_addr_space_t as, void *arg) - - #else - --unw_word_t _UPT_ptrauth_insn_mask (unw_addr_space_t, void *) -+unw_word_t _UPT_ptrauth_insn_mask (UNUSED unw_addr_space_t as, UNUSED void *arg) - { - return 0; - } - --#endif -\ No newline at end of file -+#endif -+ diff --git a/deps/patches/openblas-cancellation.patch b/deps/patches/openblas-cancellation.patch new file mode 100644 index 0000000000000..84c6aead2af1e --- /dev/null +++ b/deps/patches/openblas-cancellation.patch @@ -0,0 +1,550 @@ +diff --git a/cblas.h b/cblas.h +index 8395f1b8..b3911c97 100644 +--- a/cblas.h ++++ b/cblas.h +@@ -59,6 +59,25 @@ typedef void (*openblas_dojob_callback)(int thread_num, void *jobdata, int dojob + typedef void (*openblas_threads_callback)(int sync, openblas_dojob_callback dojob, int numjobs, size_t jobdata_elsize, void *jobdata, int dojob_data); + void openblas_set_threads_callback_function(openblas_threads_callback callback); + ++/* Cooperative cancellation of in-flight operations. ++ * ++ * Every thread owns a pointer-sized generation slot; ++ * openblas_cancel_token() returns its (stable) address for the calling ++ * thread. Supported compute drivers (currently the level-3 GEMM drivers) ++ * advance the slot to a fresh even generation at operation entry on the ++ * issuing thread and poll it during the computation. To cancel the ++ * operation in flight on a thread, load that thread's slot value, verify ++ * the operation you mean to cancel is still the current one, and call ++ * openblas_cancel(token, loaded_value): the cancel bit (bit 0) is set iff ++ * the slot still holds the loaded value, so stale requests cannot affect ++ * later operations. A cancelled operation leaves its output in an ++ * unspecified, partially-updated state; the caller must discard the ++ * result. The library itself stays consistent and can service further ++ * calls. These symbols are exported without any symbol prefix/suffix ++ * decoration. */ ++size_t *openblas_cancel_token(void); ++void openblas_cancel(size_t *token, size_t loaded_token); ++ + #ifdef OPENBLAS_OS_LINUX + /* Sets thread affinity for OpenBLAS threads. `thread_idx` is in [0, openblas_get_num_threads()-1]. */ + int openblas_setaffinity(int thread_idx, size_t cpusetsize, cpu_set_t* cpu_set); +diff --git a/common.h b/common.h +index 4746633c..def82b87 100644 +--- a/common.h ++++ b/common.h +@@ -720,6 +720,17 @@ int get_node_equal (void); + + void goto_set_num_threads(int); + ++/* Cooperative cancellation of in-flight operations ++ * (implemented in driver/others/openblas_cancel.c). These symbols are ++ * exported without SYMBOLPREFIX/SYMBOLSUFFIX decoration. */ ++size_t *openblas_cancel_token(void); ++void openblas_cancel(size_t *token, size_t loaded_token); ++ ++/* Internal helpers for the instrumented compute drivers. */ ++size_t openblas_cancel_begin(void); ++size_t *openblas_cancel_self(void); ++int openblas_cancel_poll(size_t *slot, size_t gen); ++ + void gotoblas_affinity_init(void); + void gotoblas_affinity_quit(void); + void gotoblas_dynamic_init(void); +diff --git a/common_macro.h b/common_macro.h +index 4051cb37..a3274bf6 100644 +--- a/common_macro.h ++++ b/common_macro.h +@@ -2763,6 +2763,13 @@ typedef struct { + void * routine; + int routine_mode; + ++ /* Generation slot of the thread that issued this operation and the ++ generation it runs as (see driver/others/openblas_cancel.c), or ++ NULL/0. Consulted only by the level-3 thread drivers; must be set ++ explicitly wherever it is to be observed. */ ++ size_t * cancel_slot; ++ size_t cancel_gen; ++ + } blas_arg_t; + #endif + +diff --git a/driver/level3/level3.c b/driver/level3/level3.c +index 78bc6aa5..b1451b09 100644 +--- a/driver/level3/level3.c ++++ b/driver/level3/level3.c +@@ -214,6 +214,15 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + double total; + #endif + ++ /* Begin a cancellable generation on the calling thread and poll it at ++ * block granularity; openblas_cancel() against this thread's slot makes ++ * the operation return early, leaving C partially updated. (When this ++ * driver runs on a BLAS worker thread on behalf of another operation, ++ * the generation is the worker's own, which nobody cancels; the outer ++ * operation's cancellation is handled by the threading driver.) */ ++ size_t cancel_gen = openblas_cancel_begin(); ++ size_t *cancel_slot = openblas_cancel_self(); ++ + k = K; + + a = (IFLOAT *)A; +@@ -303,11 +312,15 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + #endif + + for(js = n_from; js < n_to; js += GEMM_R){ ++ if (openblas_cancel_poll(cancel_slot, cancel_gen)) return 0; ++ + min_j = n_to - js; + if (min_j > GEMM_R) min_j = GEMM_R; + + for(ls = 0; ls < k; ls += min_l){ + ++ if (openblas_cancel_poll(cancel_slot, cancel_gen)) return 0; ++ + min_l = k - ls; + + if (min_l >= GEMM_Q * 2) { +@@ -354,6 +367,8 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + + #else + for(jjs = js; jjs < js + min_j; jjs += min_jj){ ++ if (openblas_cancel_poll(cancel_slot, cancel_gen)) return 0; ++ + min_jj = min_j + js - jjs; + #if defined(SKYLAKEX) || defined(COOPERLAKE) || defined(SAPPHIRERAPIDS) + /* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve best performance */ +@@ -391,6 +406,8 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + #endif + + for(is = m_from + min_i; is < m_to; is += min_i){ ++ if (openblas_cancel_poll(cancel_slot, cancel_gen)) return 0; ++ + min_i = m_to - is; + + if (min_i >= GEMM_P * 2) { +diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c +index 2657bbcf..1c8d2a11 100644 +--- a/driver/level3/level3_thread.c ++++ b/driver/level3/level3_thread.c +@@ -250,6 +250,17 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + FLOAT *c; + job_t *job = (job_t *)args -> common; + ++ /* Cancellation token forwarded (by gemm_driver) from the thread that ++ * issued this operation, or NULL. Once cancellation is observed the ++ * remaining compute (copy/kernel calls) is skipped, but every ++ * synchronization point - buffer publication and flag clearing - is ++ * still executed, so sibling threads never deadlock and the operation ++ * finishes quickly with C left in an unspecified, partially-updated ++ * state. */ ++ size_t *cancel_slot = args -> cancel_slot; ++ size_t cancel_gen = args -> cancel_gen; ++ int cancelled = 0; ++ + BLASLONG nthreads_m; + BLASLONG mypos_m, mypos_n; + BLASLONG divide_rate = DIVIDE_RATE; +@@ -314,8 +325,10 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + n_to = range_n[mypos + 1]; + } + +- /* Multiply C by beta if needed */ +- if (beta) { ++ cancelled = openblas_cancel_poll(cancel_slot, cancel_gen); ++ ++ /* Multiply C by beta if needed (skipped when cancelled) */ ++ if (beta && !cancelled) { + #ifndef COMPLEX + if (beta[0] != ONE) + #else +@@ -342,6 +355,9 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + /* Iterate through steps of k */ + for(ls = 0; ls < k; ls += min_l){ + ++ /* Poll for cancellation at block granularity */ ++ if (!cancelled) cancelled = openblas_cancel_poll(cancel_slot, cancel_gen); ++ + /* Determine step size in k */ + min_l = k - ls; + if (min_l >= GEMM_Q * 2) { +@@ -371,10 +387,12 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + } + } + +- /* Copy local region of A into workspace */ +- START_RPCC(); +- ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_from, sa); +- STOP_RPCC(copy_A); ++ /* Copy local region of A into workspace (skipped when cancelled) */ ++ if (!cancelled) { ++ START_RPCC(); ++ ICOPY_OPERATION(min_l, min_i, a, lda, ls, m_from, sa); ++ STOP_RPCC(copy_A); ++ } + + /* Copy local region of B into workspace and apply kernel */ + div_n = (n_to - n_from + divide_rate - 1) / divide_rate; +@@ -389,9 +407,13 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + + #if defined(FUSED_GEMM) && !defined(TIMING) + +- /* Fused operation to copy region of B into workspace and apply kernel */ +- FUSED_KERNEL_OPERATION(min_i, MIN(n_to, js + div_n) - js, min_l, alpha, +- sa, buffer[bufferside], b, ldb, c, ldc, m_from, js, ls); ++ /* Fused operation to copy region of B into workspace and apply kernel ++ * (skipped when cancelled; the buffer is still published below so ++ * sibling threads do not stall) */ ++ if (!cancelled) cancelled = openblas_cancel_poll(cancel_slot, cancel_gen); ++ if (!cancelled) ++ FUSED_KERNEL_OPERATION(min_i, MIN(n_to, js + div_n) - js, min_l, alpha, ++ sa, buffer[bufferside], b, ldb, c, ldc, m_from, js, ls); + + #else + +@@ -410,6 +432,12 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + */ + if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N; + #endif ++ /* Poll for cancellation between blocks; when cancelled, skip the ++ * copy and kernel but keep iterating so the buffer is still ++ * published below and sibling threads do not stall */ ++ if (!cancelled) cancelled = openblas_cancel_poll(cancel_slot, cancel_gen); ++ if (!cancelled) { ++ + /* Copy part of local region of B into workspace */ + START_RPCC(); + OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs, +@@ -427,6 +455,8 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + ops += 2 * min_i * min_jj * min_l; + #endif + ++ } ++ + } + #endif + +@@ -456,7 +486,10 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + STOP_RPCC(waiting2); + MB; + +- /* Apply kernel with local region of A and part of other region of B */ ++ /* Apply kernel with local region of A and part of other region of B ++ * (skipped when cancelled; the flag is still cleared below) */ ++ if (!cancelled) cancelled = openblas_cancel_poll(cancel_slot, cancel_gen); ++ if (!cancelled) { + START_RPCC(); + KERNEL_OPERATION(min_i, MIN(range_n[current + 1] - js, div_n), min_l, alpha, + sa, (IFLOAT *)job[current].working[mypos][CACHE_LINE_SIZE * bufferside], +@@ -466,6 +499,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + #ifdef TIMING + ops += 2 * min_i * MIN(range_n[current + 1] - js, div_n) * min_l; + #endif ++ } + } + + /* Clear synchronization flag if this thread is done with other region of B */ +@@ -476,9 +510,12 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + } + } while (current != mypos); + +- /* Iterate through steps of m ++ /* Iterate through steps of m + * Note: First step has already been finished */ + for(is = m_from + min_i; is < m_to; is += min_i){ ++ /* Poll for cancellation between blocks */ ++ if (!cancelled) cancelled = openblas_cancel_poll(cancel_slot, cancel_gen); ++ + min_i = m_to - is; + if (min_i >= GEMM_P * 2) { + min_i = GEMM_P; +@@ -487,10 +524,12 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + min_i = (((min_i + 1) / 2 + GEMM_UNROLL_M - 1)/GEMM_UNROLL_M) * GEMM_UNROLL_M; + } + +- /* Copy local region of A into workspace */ +- START_RPCC(); +- ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); +- STOP_RPCC(copy_A); ++ /* Copy local region of A into workspace (skipped when cancelled) */ ++ if (!cancelled) { ++ START_RPCC(); ++ ICOPY_OPERATION(min_l, min_i, a, lda, ls, is, sa); ++ STOP_RPCC(copy_A); ++ } + + /* Get regions of B and apply kernel */ + current = mypos; +@@ -500,17 +539,20 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, + div_n = (range_n[current + 1] - range_n[current] + divide_rate - 1) / divide_rate; + for (js = range_n[current], bufferside = 0; js < range_n[current + 1]; js += div_n, bufferside ++) { + +- /* Apply kernel with local region of A and part of region of B */ ++ /* Apply kernel with local region of A and part of region of B ++ * (skipped when cancelled; the flag is still cleared below) */ ++ if (!cancelled) { + START_RPCC(); + KERNEL_OPERATION(min_i, MIN(range_n[current + 1] - js, div_n), min_l, alpha, + sa, (IFLOAT *)job[current].working[mypos][CACHE_LINE_SIZE * bufferside], + c, ldc, is, js); + STOP_RPCC(kernel); +- ++ + #ifdef TIMING + ops += 2 * min_i * MIN(range_n[current + 1] - js, div_n) * min_l; + #endif +- ++ } ++ + /* Clear synchronization flag if this thread is done with region of B */ + if (is + min_i >= m_to) { + WMB; +@@ -688,6 +730,12 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG + newarg.beta = args -> beta; + newarg.nthreads = args -> nthreads; + newarg.common = (void *)job; ++ ++ /* Begin a cancellable generation on this (the calling, i.e. issuing) ++ * thread and forward it to the worker threads through the job ++ * arguments. */ ++ newarg.cancel_gen = openblas_cancel_begin(); ++ newarg.cancel_slot = openblas_cancel_self(); + #ifdef PARAMTEST + newarg.gemm_p = args -> gemm_p; + newarg.gemm_q = args -> gemm_q; +@@ -791,6 +839,11 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG + WMB; + /* Execute parallel computation */ + exec_blas(nthreads, queue); ++ ++ /* Skip the remaining column blocks once this operation has been ++ * cancelled. Each exec_blas() round is a full barrier, so bailing ++ * out between rounds leaves the library in a consistent state. */ ++ if (openblas_cancel_poll(newarg.cancel_slot, newarg.cancel_gen)) break; + } + + #ifdef USE_ALLOC_HEAP +diff --git a/driver/others/CMakeLists.txt b/driver/others/CMakeLists.txt +index 72abdc4e..b25337db 100644 +--- a/driver/others/CMakeLists.txt ++++ b/driver/others/CMakeLists.txt +@@ -41,6 +41,7 @@ set(COMMON_SOURCES + openblas_env.c + openblas_get_num_procs.c + openblas_get_num_threads.c ++ openblas_cancel.c + blas_server_callback.c + ) + +diff --git a/driver/others/Makefile b/driver/others/Makefile +index 0a1bcff9..52a893e7 100644 +--- a/driver/others/Makefile ++++ b/driver/others/Makefile +@@ -1,7 +1,7 @@ + TOPDIR = ../.. + include ../../Makefile.system + +-COMMONOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) c_abs.$(SUFFIX) z_abs.$(SUFFIX) openblas_set_num_threads.$(SUFFIX) openblas_get_num_threads.$(SUFFIX) openblas_get_num_procs.$(SUFFIX) openblas_get_config.$(SUFFIX) openblas_get_parallel.$(SUFFIX) openblas_error_handle.$(SUFFIX) openblas_env.$(SUFFIX) blas_server_callback.$(SUFFIX) ++COMMONOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) c_abs.$(SUFFIX) z_abs.$(SUFFIX) openblas_set_num_threads.$(SUFFIX) openblas_get_num_threads.$(SUFFIX) openblas_get_num_procs.$(SUFFIX) openblas_get_config.$(SUFFIX) openblas_get_parallel.$(SUFFIX) openblas_error_handle.$(SUFFIX) openblas_env.$(SUFFIX) openblas_cancel.$(SUFFIX) blas_server_callback.$(SUFFIX) + + #COMMONOBJS += slamch.$(SUFFIX) slamc3.$(SUFFIX) dlamch.$(SUFFIX) dlamc3.$(SUFFIX) + +@@ -172,6 +172,9 @@ openblas_error_handle.$(SUFFIX) : openblas_error_handle.c + openblas_env.$(SUFFIX) : openblas_env.c + $(CC) $(CFLAGS) -c $< -o $(@F) + ++openblas_cancel.$(SUFFIX) : openblas_cancel.c ../../common.h ++ $(CC) $(CFLAGS) -c $< -o $(@F) ++ + blasL1thread.$(SUFFIX) : blas_l1_thread.c ../../common.h ../../common_thread.h + $(CC) $(CFLAGS) -c $< -o $(@F) + +diff --git a/driver/others/openblas_cancel.c b/driver/others/openblas_cancel.c +new file mode 100644 +index 00000000..6e8c3a68 +--- /dev/null ++++ b/driver/others/openblas_cancel.c +@@ -0,0 +1,130 @@ ++/***************************************************************************** ++Copyright (c) 2026, The OpenBLAS Project ++All rights reserved. ++ ++Redistribution and use in source and binary forms, with or without ++modification, are permitted provided that the following conditions are ++met: ++ ++ 1. Redistributions of source code must retain the above copyright ++ notice, this list of conditions and the following disclaimer. ++ ++ 2. Redistributions in binary form must reproduce the above copyright ++ notice, this list of conditions and the following disclaimer in ++ the documentation and/or other materials provided with the ++ distribution. ++ 3. Neither the name of the OpenBLAS project nor the names of ++ its contributors may be used to endorse or promote products ++ derived from this software without specific prior written ++ permission. ++ ++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ++AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ++IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ++ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE ++LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL ++DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR ++SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ++CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE ++USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++**********************************************************************************/ ++ ++#include ++#include "common.h" ++ ++/* Cooperative cancellation of in-flight OpenBLAS operations. ++ * ++ * Every thread owns a pointer-sized generation slot in thread-local ++ * storage: ++ * ++ * - openblas_cancel_token() returns the address of the calling thread's ++ * slot. The address is stable for the lifetime of the thread. ++ * - Instrumented compute drivers advance the slot to a fresh even value ++ * at operation entry (on the issuing thread) and poll it at block ++ * granularity: any change - the cancel bit or a later generation - ++ * makes the operation abandon its remaining work and return early. ++ * - openblas_cancel(token, loaded_token) requests cancellation of the ++ * operation that was in flight when the caller loaded *token: it ++ * atomically sets the cancel bit iff the slot still holds ++ * loaded_token. A racing or stale request (the operation completed; ++ * a new one may have started) either fails the compare-exchange or ++ * dirties an already-dead generation - both harmless. May be called ++ * from any thread. ++ * ++ * A cancelled operation returns quickly but leaves its output buffers in ++ * an unspecified, partially-updated state; the caller is responsible for ++ * discarding the result. The library itself remains in a consistent ++ * state and can service further calls. ++ * ++ * These symbols are deliberately exported without any SYMBOLPREFIX / ++ * SYMBOLSUFFIX decoration, like the other openblas_* utility functions ++ * that are not per-precision entry points. ++ */ ++ ++#if defined(_MSC_VER) && !defined(__clang__) ++#define OPENBLAS_CANCEL_TLS __declspec(thread) ++#else ++#define OPENBLAS_CANCEL_TLS __thread ++#endif ++ ++/* This thread's generation slot. Bit 0 is the cancel flag; the remaining ++ * bits count operations issued by this thread. */ ++static OPENBLAS_CANCEL_TLS size_t openblas_cancel_slot = 0; ++ ++#if defined(__GNUC__) || defined(__clang__) ++#define OPENBLAS_CANCEL_LOAD(PTR) __atomic_load_n((PTR), __ATOMIC_RELAXED) ++#define OPENBLAS_CANCEL_STORE(PTR, VAL) __atomic_store_n((PTR), (VAL), __ATOMIC_RELEASE) ++#define OPENBLAS_CANCEL_CAS(PTR, EXPP, VAL) \ ++ __atomic_compare_exchange_n((PTR), (EXPP), (VAL), 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED) ++#else ++/* Fallback for compilers without the GNU atomic builtins. The slot is ++ * only advanced by its owning thread; the cancel request degrades to a ++ * check-then-store, which can at worst dirty a dead generation. */ ++#define OPENBLAS_CANCEL_LOAD(PTR) (*(volatile size_t *)(PTR)) ++#define OPENBLAS_CANCEL_STORE(PTR, VAL) do { WMB; *(volatile size_t *)(PTR) = (VAL); } while (0) ++static int openblas_cancel_cas_fallback(size_t *ptr, size_t *expp, size_t val) { ++ if (*(volatile size_t *)ptr != *expp) return 0; ++ WMB; ++ *(volatile size_t *)ptr = val; ++ return 1; ++} ++#define OPENBLAS_CANCEL_CAS(PTR, EXPP, VAL) openblas_cancel_cas_fallback((PTR), (EXPP), (VAL)) ++#endif ++ ++/* Address of the calling thread's generation slot. */ ++size_t *openblas_cancel_token(void) { ++ return (size_t *)&openblas_cancel_slot; ++} ++ ++/* Request cancellation of the operation that was in flight on the token's ++ * thread when the caller loaded LOADED_TOKEN from it. May be called from ++ * any thread. */ ++void openblas_cancel(size_t *token, size_t loaded_token) { ++ if (token == NULL) return; ++ OPENBLAS_CANCEL_CAS(token, &loaded_token, loaded_token | (size_t)1); ++} ++ ++/* --- internal helpers (not part of the public API) --------------------- */ ++ ++/* Begin an instrumented operation on the calling thread: advance the slot ++ * past any stale cancel bit to a fresh even generation and return it. */ ++size_t openblas_cancel_begin(void) { ++ size_t gen = (openblas_cancel_slot | (size_t)1) + 1; ++ OPENBLAS_CANCEL_STORE(&openblas_cancel_slot, gen); ++ return gen; ++} ++ ++/* The calling thread's slot address (same as openblas_cancel_token, for ++ * internal use without going through the exported symbol). */ ++size_t *openblas_cancel_self(void) { ++ return (size_t *)&openblas_cancel_slot; ++} ++ ++/* Cheap poll usable from compute drivers: nonzero once GEN is no longer ++ * the live value of SLOT (cancelled, or superseded). SLOT may be NULL. */ ++int openblas_cancel_poll(size_t *slot, size_t gen) { ++ if (slot == NULL) return 0; ++ return OPENBLAS_CANCEL_LOAD(slot) != gen; ++} +diff --git a/exports/gensymbol b/exports/gensymbol +index 6284e6b6..a8189a6f 100755 +--- a/exports/gensymbol ++++ b/exports/gensymbol +@@ -186,6 +186,15 @@ misc_no_underscore_objs=" + + misc_underscore_objs="" + ++# Symbols that are exported exactly as spelled, without any SYMBOLPREFIX / ++# SYMBOLSUFFIX decoration (like the openblas_* utility functions, these are ++# not per-precision entry points). They are therefore deliberately left out ++# of the objcopy/objconv rename lists. ++misc_no_suffix_objs=" ++ openblas_cancel ++ openblas_cancel_token ++" ++ + # These routines are provided by OpenBLAS. + lapackobjss=" + sgesv +@@ -4010,6 +4019,10 @@ case "$p1" in + for obj in $no_underscore_objs; do + printf '_%s%s%s\n' "$symbolprefix" "$obj" "$symbolsuffix" + done ++ ++ for obj in $misc_no_suffix_objs; do ++ printf '_%s\n' "$obj" ++ done + ;; + + aix) +@@ -4115,6 +4128,11 @@ case "$p1" in + "$symbolprefix" "$obj" "$symbolsuffix" "$obj" "$count" + count=`expr $count + 1` + done ++ ++ for obj in $misc_no_suffix_objs; do ++ printf '\t%s @%s\n' "$obj" "$count" ++ count=`expr $count + 1` ++ done + ;; + + win2khpl) +@@ -4167,6 +4185,10 @@ case "$p1" in + "$symbolprefix" "$uppercase" "$symbolsuffix" "$obj" "$count" + count=`expr $count + 1` + done ++ ++ for obj in $misc_no_suffix_objs; do ++ printf '\t%s\n' "$obj" ++ done + ;; + + linktest) diff --git a/deps/patches/openblas-ofast-power.patch b/deps/patches/openblas-ofast-power.patch deleted file mode 100644 index 01089286257f7..0000000000000 --- a/deps/patches/openblas-ofast-power.patch +++ /dev/null @@ -1,43 +0,0 @@ -diff --git a/Makefile.power b/Makefile.power -index aa1ca080a..42c417a78 100644 ---- a/Makefile.power -+++ b/Makefile.power -@@ -13,16 +13,16 @@ ifeq ($(CORE), POWER10) - ifneq ($(C_COMPILER), PGI) - ifeq ($(C_COMPILER), GCC)) - ifeq ($(GCCVERSIONGTEQ10), 1) --CCOMMON_OPT += -Ofast -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math -+CCOMMON_OPT += -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math - else ifneq ($(GCCVERSIONGT4), 1) - $(warning your compiler is too old to fully support POWER9, getting a newer version of gcc is recommended) --CCOMMON_OPT += -Ofast -mcpu=power8 -mtune=power8 -mvsx -fno-fast-math -+CCOMMON_OPT += -mcpu=power8 -mtune=power8 -mvsx -fno-fast-math - else - $(warning your compiler is too old to fully support POWER10, getting a newer version of gcc is recommended) --CCOMMON_OPT += -Ofast -mcpu=power9 -mtune=power9 -mvsx -fno-fast-math -+CCOMMON_OPT += -mcpu=power9 -mtune=power9 -mvsx -fno-fast-math - endif - else --CCOMMON_OPT += -Ofast -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math -+CCOMMON_OPT += -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math - endif - ifeq ($(F_COMPILER), IBM) - FCOMMON_OPT += -O2 -qrecur -qnosave -qarch=pwr10 -qtune=pwr10 -qfloat=nomaf -qzerosize -@@ -34,7 +34,7 @@ endif - - ifeq ($(CORE), POWER9) - ifneq ($(C_COMPILER), PGI) --CCOMMON_OPT += -Ofast -mvsx -fno-fast-math -+CCOMMON_OPT += -mvsx -fno-fast-math - ifeq ($(C_COMPILER), GCC) - ifneq ($(GCCVERSIONGT4), 1) - $(warning your compiler is too old to fully support POWER9, getting a newer version of gcc is recommended) -@@ -70,7 +70,7 @@ endif - - ifeq ($(CORE), POWER8) - ifneq ($(C_COMPILER), PGI) --CCOMMON_OPT += -Ofast -mcpu=power8 -mtune=power8 -mvsx -fno-fast-math -+CCOMMON_OPT += -mcpu=power8 -mtune=power8 -mvsx -fno-fast-math - else - CCOMMON_OPT += -fast -Mvect=simd -Mcache_align - endif diff --git a/deps/patches/openblas-winexit.patch b/deps/patches/openblas-winexit.patch index 33389f34a4074..2f4acf53ee450 100644 --- a/deps/patches/openblas-winexit.patch +++ b/deps/patches/openblas-winexit.patch @@ -1,8 +1,11 @@ -diff --git a/driver/others/memory.c b/driver/others/memory.c -index 6e654ccf..1d2f9f12 100644 +Remove OpenBLAS's exit-time cleanup on Windows (MSVC): julia coordinates +shutdown itself, and the automatic DllMain/atexit-style cleanup deadlocks +against julia's own exit sequence. Rebased onto OpenBLAS 0.3.33 (upstream +renamed DllMain to OpenBLASDllMain and registers it via TLS callbacks). + --- a/driver/others/memory.c +++ b/driver/others/memory.c -@@ -1534,7 +1534,7 @@ void CONSTRUCTOR gotoblas_init(void) { +@@ -1567,7 +1567,7 @@ } @@ -11,11 +14,11 @@ index 6e654ccf..1d2f9f12 100644 if (gotoblas_initialized == 0) return; -@@ -1572,75 +1572,11 @@ void DESTRUCTOR gotoblas_quit(void) { +@@ -1605,75 +1605,11 @@ } #if defined(_MSC_VER) && !defined(__clang__) --BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) +-BOOL APIENTRY OpenBLASDllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) -{ - switch (ul_reason_for_call) - { @@ -64,10 +67,10 @@ index 6e654ccf..1d2f9f12 100644 -#endif - -#ifdef _WIN64 --static const PIMAGE_TLS_CALLBACK dll_callback(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = DllMain; +-static const PIMAGE_TLS_CALLBACK dll_callback(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = OpenBLASDllMain; -#pragma const_seg() -#else --static void (APIENTRY *dll_callback)(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = DllMain; +-static void (APIENTRY *dll_callback)(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = OpenBLASDllMain; -#pragma data_seg() -#endif - @@ -87,7 +90,7 @@ index 6e654ccf..1d2f9f12 100644 #endif #if (defined(C_PGI) || (!defined(C_SUN) && defined(F_INTERFACE_SUN))) && (defined(ARCH_X86) || defined(ARCH_X86_64)) -@@ -3146,7 +3082,7 @@ void CONSTRUCTOR gotoblas_init(void) { +@@ -3456,7 +3392,7 @@ } @@ -96,12 +99,12 @@ index 6e654ccf..1d2f9f12 100644 if (gotoblas_initialized == 0) return; -@@ -3175,71 +3111,6 @@ void DESTRUCTOR gotoblas_quit(void) { +@@ -3485,71 +3421,6 @@ #endif } -#if defined(_MSC_VER) && !defined(__clang__) --BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) +-BOOL APIENTRY OpenBLASDllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved) -{ - switch (ul_reason_for_call) - { @@ -145,7 +148,7 @@ index 6e654ccf..1d2f9f12 100644 -#else -#pragma data_seg(".CRT$XLB") -#endif --static void (APIENTRY *dll_callback)(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = DllMain; +-static void (APIENTRY *dll_callback)(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = OpenBLASDllMain; -#ifdef _WIN64 -#pragma const_seg() -#else diff --git a/deps/pcre.mk b/deps/pcre.mk index 8fbdb1ba79a35..f0ff80d6dbf92 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -53,7 +53,7 @@ $(eval $(call staged-install, \ clean-pcre: -rm -f $(BUILDDIR)/pcre2-$(PCRE_VER)/build-configured $(BUILDDIR)/pcre2-$(PCRE_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/pcre2-$(PCRE_VER) clean + -if [ -d $(BUILDDIR)/pcre2-$(PCRE_VER) ]; then $(MAKE) -C $(BUILDDIR)/pcre2-$(PCRE_VER) clean; fi distclean-pcre: rm -rf $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 $(SRCCACHE)/pcre2-$(PCRE_VER) $(BUILDDIR)/pcre2-$(PCRE_VER) diff --git a/deps/pcre.version b/deps/pcre.version index 6810ee36fe83f..fad8ec848c10a 100644 --- a/deps/pcre.version +++ b/deps/pcre.version @@ -3,4 +3,4 @@ PCRE_JLL_NAME := PCRE2 ## source build -PCRE_VER := 10.45 +PCRE_VER := 10.48 diff --git a/deps/tools/common.mk b/deps/tools/common.mk index b7ebc39169221..172ece82409d6 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -181,6 +181,7 @@ upper = $(shell echo $1 | tr a-z A-Z) # so it's harder to get half-installed (or half-reinstalled) dependencies # # and enables sharing deps compiles, uninstall, and fast reinstall MAKE_INSTALL = MSYS2_ARG_CONV_EXCL="prefix=" $$(MAKE) -C $1 install $$(MAKE_COMMON) $3 DESTDIR="$2" +CMAKE_INSTALL = DESTDIR="$2" $$(CMAKE) --install $1 $3 define SHLIBFILE_INSTALL mkdir -p $2/$$(build_shlibdir) @@ -199,9 +200,10 @@ install-$(strip $1): $$(build_prefix)/manifest/$(strip $1) ifeq (exists, $$(shell [ -e $$(build_staging)/$2.tar ] && echo exists )) # clean depends on uninstall only if the staged file exists distclean-$(strip $1) clean-$(strip $1): uninstall-$(strip $1) -else +else ifeq (exists, $$(shell [ -e $$(build_prefix)/manifest/$(strip $1) ] && echo exists )) # uninstall depends on staging only if the staged file doesn't exist -# otherwise, uninstall doesn't actually want the file to be updated first +# but the dep is installed; otherwise, uninstall doesn't actually want +# the file to be updated first (or has nothing to uninstall) uninstall-$(strip $1): | $$(build_staging)/$2.tar endif @@ -289,6 +291,26 @@ endif ## phony targets ## -.PHONY: default get extract configure compile fastcheck check install uninstall reinstall cleanall distcleanall version-check \ - get-* extract-* configure-* compile-* fastcheck-* check-* install-* uninstall-* reinstall-* clean-* distclean-* \ - update-llvm +.PHONY: check +.PHONY: check-* +.PHONY: clean-* +.PHONY: cleanall +.PHONY: compile +.PHONY: compile-* +.PHONY: configure +.PHONY: configure-* +.PHONY: default +.PHONY: distclean-* +.PHONY: distcleanall +.PHONY: extract +.PHONY: extract-* +.PHONY: fastcheck +.PHONY: fastcheck-* +.PHONY: get +.PHONY: get-* +.PHONY: install +.PHONY: install-* +.PHONY: reinstall +.PHONY: reinstall-* +.PHONY: uninstall +.PHONY: uninstall-* diff --git a/deps/tools/git-external.mk b/deps/tools/git-external.mk index cf1610ac1bf5d..1ed1b1173dae5 100644 --- a/deps/tools/git-external.mk +++ b/deps/tools/git-external.mk @@ -1,4 +1,8 @@ ## A rule for making a git-external dependency ## +# For a user-facing overview of building a dependency from a Git checkout (DEPS_GIT, +# USE_BINARYBUILDER_, the .version file, and editing the worktree in place), see +# "Building a dependency from a Git checkout" in doc/src/devdocs/build/build.md +# # call syntax: # $(eval $(call git-external,dirname,VARNAME,file_from_download,file_from_compile,SRCDIR) # dirname is the folder name to create @@ -29,12 +33,18 @@ ifneq (,$$(filter $1 1,$$(DEPS_GIT))) $2_SRC_DIR := $1 $2_SRC_FILE := $$(SRCCACHE)/$1.git $$($2_SRC_FILE)/HEAD: | $$(SRCCACHE) - git clone -q --mirror --branch $$($2_BRANCH) $$($2_GIT_URL) $$(dir $$@) -$5/$1/.git/HEAD: | $$($2_SRC_FILE)/HEAD + # this repo also backs the worktree the user edits, so it gets the remote-tracking + # refs of an ordinary clone, leaving refs/heads/ and `git push` to the user + git clone -q --bare -c remote.origin.fetch='+refs/heads/*:refs/remotes/origin/*' $$($2_GIT_URL) $$(dir $$@) +$5/$1/.git: | $$($2_SRC_FILE)/HEAD # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) -cd $$($2_SRC_FILE) && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) - git clone -q --depth=10 --branch $$($2_BRANCH) $$($2_SRC_FILE) $5/$1 - cd $5/$1 && git remote set-url origin $$($2_GIT_URL) + # if a checkout is already present (e.g. an old-style local clone from a previous + # build), keep it and proceed as before; otherwise add a worktree sharing the bare + # cache's object store instead of making a full local clone + -git -C $$($2_SRC_FILE) worktree prune + # the path must be absolute: git -C resolves it against the bare cache, not $$(CURDIR) + [ -e $5/$1/.git ] || git -C $$($2_SRC_FILE) worktree add --detach $$(abspath $5/$1) $$($2_BRANCH) #ifneq ($3,) touch -c $5/$1/$3 # old target #endif @@ -44,14 +54,15 @@ $$(BUILDDIR)/$1: mkdir -p $$@ $5/$1/source-extracted: | $$(BUILDDIR)/$1 endif -$5/$1/source-extracted: $$(SRCDIR)/$1.version | $5/$1/.git/HEAD +$5/$1/source-extracted: $$(SRCDIR)/$1.version | $5/$1/.git # try to update the cache, if that fails, attempt to continue anyways (the ref might already be local) -cd $$(SRCCACHE)/$1.git && git fetch -q $$($2_GIT_URL) $$($2_BRANCH):remotes/origin/$$($2_BRANCH) - cd $5/$1 && git fetch -q $$(SRCCACHE)/$1.git remotes/origin/$$($2_BRANCH):remotes/origin/$$($2_BRANCH) + # pull the branch into the checkout: harmless for a worktree (it already shares the + # cache's object store), and required to update an old-style local clone + -cd $5/$1 && git fetch -q $$(SRCCACHE)/$1.git remotes/origin/$$($2_BRANCH):remotes/origin/$$($2_BRANCH) cd $5/$1 && git checkout -q --detach $$($2_SHA1) @[ '$$($2_SHA1)' = "$$$$(cd $5/$1 && git show -s --format='%H' HEAD)" ] || echo $$(WARNCOLOR)'==> warning: SHA1 hash did not match $1.version file'$$(ENDCOLOR) echo 1 > $$@ -$5/$1/source-compiled: $5/$1/.git/HEAD $$($2_SRC_FILE): | $$($2_SRC_FILE)/HEAD touch -c $$@ diff --git a/deps/tools/jldownload b/deps/tools/jldownload index 99822195fe303..933eb3e64aff7 100755 --- a/deps/tools/jldownload +++ b/deps/tools/jldownload @@ -10,8 +10,9 @@ CURL=$(which curl 2>/dev/null) FETCH=$(which fetch 2>/dev/null) TIMEOUT=15 # seconds -WGET_OPTS="--no-check-certificate --tries=1 --timeout=$TIMEOUT" -CURL_OPTS="-fkL --connect-timeout $TIMEOUT -y $TIMEOUT" +RETRIES=3 +WGET_OPTS="--no-check-certificate --waitretry=$TIMEOUT --tries=$RETRIES --timeout=$TIMEOUT" +CURL_OPTS="-fkL --connect-timeout $TIMEOUT -y $TIMEOUT --retry $RETRIES --retry-max-time $TIMEOUT" FETCH_OPTS="-T $TIMEOUT" if [ $# -eq 1 ]; then diff --git a/deps/tools/uninstallers.mk b/deps/tools/uninstallers.mk index 0051786ed1d0a..a7157b16b76d5 100644 --- a/deps/tools/uninstallers.mk +++ b/deps/tools/uninstallers.mk @@ -9,7 +9,10 @@ ifeq (undefined,$$(flavor $$(word 2,$$(MANIFEST_$1)))) MANIFEST_$1 := $$(UNINSTALL_$1) endif UNINST_HOW_$1 := $$(word 2,$$(MANIFEST_$1)) -ifneq ($$(UNINST_HOW_$1),) +ifeq (,$$(wildcard $$(build_prefix)/manifest/$1)) +uninstall-$1: + @echo "skipping uninstall: $1 not installed" +else ifneq ($$(UNINST_HOW_$1),) UNINST_WHO_$1 := $$(firstword $$(MANIFEST_$1)) UNINST_WHERE_$1 := $$(wordlist 3,99,$$(MANIFEST_$1)) $$(eval $$(call $$(UNINST_HOW_$1),$1,$$(UNINST_WHO_$1),$$(UNINST_WHERE_$1))) diff --git a/deps/unwind.mk b/deps/unwind.mk index 2b49b3cf0973a..dd4c71f91f1bf 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -39,14 +39,10 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-app cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-disable-initial-exec-tls.patch echo 1 > $@ -$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied - cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-missing-parameter-names.patch - echo 1 > $@ - # note minidebuginfo requires liblzma, which we do not have a source build for # (it will be enabled in BinaryBuilder-based downloads however) # since https://github.com/JuliaPackaging/Yggdrasil/commit/0149e021be9badcb331007c62442a4f554f3003c -$(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied +$(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(dir $<)/configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" LDFLAGS="$(LDFLAGS) $(LIBUNWIND_LDFLAGS)" --enable-shared --disable-minidebuginfo --disable-tests --enable-debug_frame --enable-zlibdebuginfo --disable-conservative-checks --enable-per-thread-cache @@ -68,7 +64,7 @@ $(eval $(call staged-install, \ clean-unwind: -rm -f $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-compiled - -$(MAKE) -C $(BUILDDIR)/libunwind-$(UNWIND_VER) clean + -if [ -d $(BUILDDIR)/libunwind-$(UNWIND_VER) ]; then $(MAKE) -C $(BUILDDIR)/libunwind-$(UNWIND_VER) clean; fi distclean-unwind: rm -rf $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz \ @@ -146,7 +142,7 @@ $(eval $(call staged-install, \ clean-llvmunwind: -rm -f $(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER)/build-configured $(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER)/build-compiled rm -rf $(build_includedir)/mach-o/ $(build_includedir)/unwind.h $(build_includedir)/libunwind.h - -$(MAKE) -C $(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER) clean + -if [ -d $(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER) ]; then $(MAKE) -C $(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER) clean; fi distclean-llvmunwind: rm -rf $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).tar.xz \ diff --git a/deps/unwind.version b/deps/unwind.version index d2709230d18fc..682182179825c 100644 --- a/deps/unwind.version +++ b/deps/unwind.version @@ -4,5 +4,5 @@ UNWIND_JLL_NAME := LibUnwind ## source build -UNWIND_VER_TAG := 1.8.2 -UNWIND_VER := 1.8.2 +UNWIND_VER_TAG := 1.8.3 +UNWIND_VER := 1.8.3 diff --git a/deps/utf8proc.mk b/deps/utf8proc.mk index cc526d40250c5..2b3d627390cdd 100644 --- a/deps/utf8proc.mk +++ b/deps/utf8proc.mk @@ -5,7 +5,7 @@ $(eval $(call git-external,utf8proc,UTF8PROC,,,$(BUILDDIR))) UTF8PROC_OBJ_LIB := $(build_libdir)/libutf8proc.a UTF8PROC_OBJ_HEADER := $(build_includedir)/utf8proc.h -UTF8PROC_CFLAGS := -O2 $(SANITIZE_OPTS) +UTF8PROC_CFLAGS := -O2 $(SANITIZE_OPTS) $(LINKER_GC_CFLAGS) UTF8PROC_MFLAGS := CC="$(CC)" CFLAGS="$(CFLAGS) $(UTF8PROC_CFLAGS)" PICFLAG="$(fPIC)" AR="$(AR)" UTF8PROC_BUILDDIR := $(BUILDDIR)/$(UTF8PROC_SRC_DIR) @@ -29,8 +29,8 @@ $(eval $(call staged-install, \ UTF8PROC_INSTALL,,,)) clean-utf8proc: - -rm -f $(BUILDDIR)/$(UTF8PROC_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(UTF8PROC_SRC_DIR) clean + -rm -f $(UTF8PROC_BUILDDIR)/build-compiled + -if [ -d $(UTF8PROC_BUILDDIR) ]; then $(MAKE) -C $(UTF8PROC_BUILDDIR) clean; fi get-utf8proc: $(UTF8PROC_SRC_FILE) extract-utf8proc: $(UTF8PROC_BUILDDIR)/source-extracted diff --git a/deps/utf8proc.version b/deps/utf8proc.version index c880d6561ce09..d98ec9b0268b6 100644 --- a/deps/utf8proc.version +++ b/deps/utf8proc.version @@ -1,2 +1,2 @@ -UTF8PROC_BRANCH=v2.10.0 -UTF8PROC_SHA1=a1b99daa2a3393884220264c927a48ba1251a9c6 +UTF8PROC_BRANCH=v2.11.3 +UTF8PROC_SHA1=e5e799221b45bbb90f5fdc5c69b6b8dfbf017e78 diff --git a/deps/zlib.mk b/deps/zlib.mk index 347b8d4cf53b6..7dc58705de53f 100644 --- a/deps/zlib.mk +++ b/deps/zlib.mk @@ -6,7 +6,7 @@ $(eval $(call git-external,zlib,ZLIB,,,$(SRCCACHE))) # use `-DUNIX=true` to ensure that it is always named `libz` ZLIB_BUILD_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUNIX=true -ZLIB_BUILD_OPTS += -DCMAKE_POSITION_INDEPENDENT_CODE=ON +ZLIB_BUILD_OPTS += -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZLIB_BUILD_TESTING=OFF -DZLIB_BUILD_STATIC=OFF $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-configured: $(SRCCACHE)/$(ZLIB_SRC_DIR)/source-extracted mkdir -p $(dir $@) @@ -24,7 +24,7 @@ $(eval $(call staged-install, \ clean-zlib: -rm -f $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-configured $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(ZLIB_SRC_DIR) clean + -if [ -d $(BUILDDIR)/$(ZLIB_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(ZLIB_SRC_DIR) clean; fi get-zlib: $(ZLIB_SRC_FILE) extract-zlib: $(BUILDDIR)/$(ZLIB_SRC_DIR)/source-extracted diff --git a/deps/zlib.version b/deps/zlib.version index 27d862a4cc35b..11bd35ff03219 100644 --- a/deps/zlib.version +++ b/deps/zlib.version @@ -3,6 +3,6 @@ ZLIB_JLL_NAME := Zlib ## source build -ZLIB_VER := 1.3.1 -ZLIB_BRANCH=v1.3.1 -ZLIB_SHA1=51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf +ZLIB_VER := 1.3.2 +ZLIB_BRANCH=v1.3.2 +ZLIB_SHA1=da607da739fa6047df13e66a2af6b8bec7c2a498 diff --git a/deps/zstd.mk b/deps/zstd.mk index ecce416ab3f38..7fce0396ab0a8 100644 --- a/deps/zstd.mk +++ b/deps/zstd.mk @@ -22,11 +22,11 @@ $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-co $(eval $(call staged-install, \ zstd,$(ZSTD_SRC_DIR), \ MAKE_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \ - $(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_private_libexecdir)/libzstd.$(SHLIB_EXT))) + $(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_shlibdir)/libzstd.$(SHLIB_EXT))) clean-zstd: -rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled - -$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean + -if [ -d $(BUILDDIR)/$(ZSTD_SRC_DIR) ]; then $(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean; fi get-zstd: $(ZSTD_SRC_FILE) extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted @@ -48,10 +48,13 @@ post-install-zstd: $(build_prefix)/manifest/zstd $(PATCHELF_MANIFEST) [ -e $(build_private_libexecdir)/zstd$(EXE) ] [ -e $(build_private_libexecdir)/zstdmt$(EXE) ] ifeq ($(OS), Darwin) + # zstd is relocated from bin/ to libexec/julia/, so its JLL rpath must be updated to keep finding ../libzstd. + # but this invalidates the existing signature, and macOS can then refuse to launch zstd, so also call codesign. for j in zstd zstdmt ; do \ [ -L $(build_private_libexecdir)/$$j ] && continue; \ install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @loader_path/$(build_libdir_rel) $(build_private_libexecdir)/$$j 2>/dev/null || true; \ install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_build_private_libexecdir_rel) $(build_private_libexecdir)/$$j || exit 1; \ + codesign -s - -f $(build_private_libexecdir)/$$j || exit 1; \ done else ifneq (,$(findstring $(OS),Linux FreeBSD)) for j in zstd zstdmt ; do \ diff --git a/doc/.gitignore b/doc/.gitignore index 4e9962dc08c2b..b6e93fec91fc7 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -3,4 +3,5 @@ _build/ UnicodeData.txt src/stdlib/ src/NEWS.md +src/devdocs/agents/skills/*/index.md diff --git a/doc/Makefile b/doc/Makefile index 177eeac5e6a85..0c5d94d0100fb 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,5 +1,6 @@ # Makefile for building documentation +.PHONY: default default: html # You can set these variables from the command line. @@ -9,8 +10,7 @@ SRCCACHE := $(abspath $(JULIAHOME)/deps/srccache) include $(JULIAHOME)/Make.inc JULIA_EXECUTABLE := $(call spawn,$(build_bindir)/julia) --startup-file=no -.PHONY: help clean cleanall html pdf deps deploy - +.PHONY: help help: @echo "Please use 'make ' where is one of" @echo " html to make standalone HTML files" @@ -25,7 +25,7 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) \ texplatform=$(texplatform) revise=$(revise) stdlibdir=$(call cygpath_w,$(build_datarootdir)/julia/stdlib/$(VERSDIR)/) -UNICODE_DATA_VERSION=16.0.0 +UNICODE_DATA_VERSION=17.0.0 $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt: @mkdir -p "$(SRCCACHE)" $(JLDOWNLOAD) "$@" https://www.unicode.org/Public/$(UNICODE_DATA_VERSION)/ucd/UnicodeData.txt @@ -35,39 +35,49 @@ DOC_FILES=$(filter-out NEWS.md stdlib,$(notdir $(wildcard $(SRCDIR)/src/*))) src/%: @mkdir -p src ln -s $(SRCDIR)/src/$* $@ +.PHONY: src src: $(addprefix src/,$(DOC_FILES)) +.PHONY: deps deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" cp "$<" UnicodeData.txt +.PHONY: alldeps alldeps: deps $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) deps $(DOCUMENTER_OPTIONS) +.PHONY: checksum-unicodedata checksum-unicodedata: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" +.PHONY: clean clean: rm -rf _build/* deps/* docbuild.log UnicodeData.txt +.PHONY: cleanall cleanall: clean +.PHONY: html html: deps @echo "Building HTML documentation." $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) $(DOCUMENTER_OPTIONS) @echo "Build finished. The HTML pages are in _build/html." +.PHONY: pdf pdf: deps @echo "Building PDF documentation." $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) -- pdf $(DOCUMENTER_OPTIONS) @echo "Build finished." # The deploy target should only be called in CI builds +.PHONY: deploy deploy: deps @echo "Deploying HTML documentation." $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) -- deploy $(DOCUMENTER_OPTIONS) @echo "Build & deploy of docs finished." +.PHONY: update-documenter update-documenter: @echo "Updating Documenter." - JULIA_PKG_PRECOMPILE_AUTO=0 $(JULIA_EXECUTABLE) --project=$(call cygpath_w,$(SRCDIR)/../deps/jlutilities/documenter/) --color=yes -e 'using Pkg; Pkg.update("Documenter")' + JULIA_PKG_PRECOMPILE_AUTO=0 $(JULIA_EXECUTABLE) --project=$(call cygpath_w,$(SRCDIR)/../deps/jlutilities/documenter/) --color=yes -e 'using Pkg; Pkg.update()' diff --git a/doc/NEWS-update.jl b/doc/NEWS-update.jl index 4014c67645eb2..a5ffcd5753314 100644 --- a/doc/NEWS-update.jl +++ b/doc/NEWS-update.jl @@ -5,7 +5,7 @@ NEWS = get(ARGS, 1, "NEWS.md") s = read(NEWS, String) -m = match(r"\[#[0-9]+\]:", s) +m = match(r"^\[#[0-9]+\]:"m, s) if m !== nothing s = s[1:m.offset-1] end diff --git a/doc/make.jl b/doc/make.jl index 2604c801f19aa..65d0687cda48f 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -25,7 +25,7 @@ end documenter_project_dir = joinpath(@__DIR__, "..", "deps", "jlutilities", "documenter") empty!(DEPOT_PATH) push!(DEPOT_PATH, joinpath(buildroot, "deps", "jlutilities", "depot")) -push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) +push!(DEPOT_PATH, abspath(Sys.BINDIR, Base.DATAROOTDIR, "julia")) using Pkg Pkg.activate(documenter_project_dir) Pkg.instantiate() @@ -158,6 +158,179 @@ function generate_markdown(basename) end generate_markdown("NEWS") +function split_skill_markdown(path) + lines = split(read(path, String), '\n'; keepempty=true) + if isempty(lines) || strip(lines[1]) != "---" + error("Agent Skill $path does not start with YAML frontmatter") + end + closing = nothing + for i in 2:length(lines) + if strip(lines[i]) == "---" + closing = i + break + end + end + closing === nothing && error("Agent Skill $path has no closing frontmatter delimiter") + frontmatter = join(lines[2:closing-1], "\n") + body = join(lines[closing+1:end], "\n") + return frontmatter, replace(body, r"^(\r?\n)+" => "") +end + +function parse_skill_frontmatter(frontmatter, path) + metadata = Pair{String,String}[] + for line in split(frontmatter, '\n') + isempty(strip(line)) && continue + parts = split(line, ':'; limit=2) + length(parts) == 2 || error("Unable to parse frontmatter line in $path: $line") + push!(metadata, strip(parts[1]) => strip(parts[2])) + end + return metadata +end + +function skill_metadata_value(metadata, key, path) + i = findfirst(pair -> first(pair) == key, metadata) + i === nothing && error("Agent Skill $path is missing required frontmatter field: $key") + return last(metadata[i]) +end + +function render_skill_metadata(metadata) + lines = ["!!! note \"Agent Skill metadata\""] + for (key, value) in metadata + rendered = key == "name" ? "`$value`" : value + rendered = replace(rendered, "\n" => "\n ") + push!(lines, " - `$key`: $rendered") + end + return join(lines, "\n") +end + +function insert_skill_metadata(body, metadata) + lines = split(body, '\n'; keepempty=true) + if !isempty(lines) && startswith(lines[1], "# ") + return string(lines[1], "\n\n", metadata, "\n\n", join(lines[2:end], "\n")) + else + return string(metadata, "\n\n", body) + end +end + +function generate_agent_skill_docs() + skills_dir = joinpath(buildrootdoc, "src", "devdocs", "agents", "skills") + pages = String[] + isdir(skills_dir) || return pages + for skill in sort(readdir(skills_dir)) + skill_dir = joinpath(skills_dir, skill) + path = joinpath(skill_dir, "SKILL.md") + isfile(path) || continue + frontmatter, body = split_skill_markdown(path) + metadata = parse_skill_frontmatter(frontmatter, path) + name = skill_metadata_value(metadata, "name", path) + name == skill || error("Agent Skill $path has name '$name' but its directory is '$skill'") + source_rel = "doc/src/devdocs/agents/skills/$skill/SKILL.md" + write( + joinpath(skill_dir, "index.md"), + """ + ```@meta + EditURL = "https://github.com/JuliaLang/julia/blob/master/$source_rel" + ``` + + """ * insert_skill_metadata(body, render_skill_metadata(metadata))) + push!(pages, "devdocs/agents/skills/$skill/index.md") + end + return pages +end +AgentSkillDocs = generate_agent_skill_docs() + +# Shared machinery for the generated tab-completion tables in +# `manual/unicode-input.md` and `manual/emoji-input.md`. The LaTeX and emoji +# tables are each large enough that rendering both on one page takes the +# generated HTML past Documenter's `size_threshold`, so they live on separate +# pages and both call into this module. +module UnicodeTables + +import Markdown + +const NBSP = '\u00A0' + +# Invert the completion tables: map each completed string to every tab +# completion sequence that produces it. +function tab_completions(symbols...) + completions = Dict{String, Vector{String}}() + for each in symbols, (k, v) in each + completions[v] = push!(get!(completions, v, String[]), k) + end + return completions +end + +# `UnicodeData.txt` is downloaded into the build root by `doc/Makefile`, which +# is not the directory this file lives in for out-of-tree builds. +unicode_data_file() = joinpath(Main.buildrootdoc, "UnicodeData.txt") + +# Parsed once and shared by every page that renders a table. +const UNICODE_NAMES = Dict{UInt32, String}() + +function unicode_data() + isempty(UNICODE_NAMES) || return UNICODE_NAMES + open(unicode_data_file()) do unidata + for line in readlines(unidata) + id, name, desc = split(line, ";")[[1, 2, 11]] + codepoint = parse(UInt32, "0x$id") + UNICODE_NAMES[codepoint] = titlecase(lowercase( + name == "" ? desc : desc == "" ? name : "$name / $desc")) + end + end + return UNICODE_NAMES +end + +# Surround combining characters with no-break spaces (i.e '\u00A0'). Follows the same format +# for how unicode is displayed on the unicode.org website: +# https://util.unicode.org/UnicodeJsps/character.jsp?a=0300 +function fix_combining_chars(char) + cat = Base.Unicode.category_code(char) + return cat == 6 || cat == 8 ? "$NBSP$char$NBSP" : "$char" +end + +function table_entries(completions, unicode_dict) + entries = Any[Any[ + ["Code point(s)"], + ["Character(s)"], + ["Tab completion sequence(s)"], + ["Unicode name(s)"], + ]] + for (chars, inputs) in sort!(collect(completions), by = first) + code_points, unicode_names, characters = String[], String[], String[] + for char in chars + push!(code_points, "U+$(uppercase(string(UInt32(char), base = 16, pad = 5)))") + push!(unicode_names, get(unicode_dict, UInt32(char), "(No Unicode name)")) + push!(characters, isempty(characters) ? fix_combining_chars(char) : "$char") + end + inputs_md = [] + for (i, input) in enumerate(inputs) + i > 1 && push!(inputs_md, ", ") + push!(inputs_md, Markdown.Code("", input)) + end + push!(entries, [ + [join(code_points, " + ")], + [join(characters)], + inputs_md, + [join(unicode_names, " + ")], + ]) + end + table = Markdown.Table(entries, [:l, :c, :l, :l]) + # We also need to wrap the Table in a Markdown.MD "document" + return Markdown.MD([table]) +end + +""" + symbol_table(symbols...) + +Render the tab completions in `symbols` (dictionaries mapping a completion +sequence to the string it expands to, such as `REPL.REPLCompletions.latex_symbols`) +as a Markdown table, annotated with the code points and Unicode names of each +completed character. +""" +symbol_table(symbols...) = table_entries(tab_completions(symbols...), unicode_data()) + +end # module UnicodeTables + Manual = [ "manual/getting-started.md", "manual/installation.md", @@ -199,6 +372,7 @@ Manual = [ "manual/faq.md", "manual/noteworthy-differences.md", "manual/unicode-input.md", + "manual/emoji-input.md", "manual/command-line-interface.md", "manual/worldage.md", ] @@ -228,11 +402,21 @@ BaseDocs = [ StdlibDocs = [stdlib.targetfile for stdlib in STDLIB_DOCS] +# HACK: get nicer sorting here, even though we don't have the header +# of the .md files at hand. +sort!(StdlibDocs, by=function(x) + x = replace(x, "stdlib/" => "") + startswith(x, "Libdl") && return lowercase("Dynamic Linker") + startswith(x, "Test") && return lowercase("Unit Testing") + return lowercase(x) +end) + DevDocs = [ "Documentation of Julia's Internals" => [ "devdocs/init.md", "devdocs/ast.md", "devdocs/types.md", + "devdocs/ub.md", "devdocs/object.md", "devdocs/eval.md", "devdocs/callconv.md", @@ -249,6 +433,7 @@ DevDocs = [ "devdocs/stdio.md", "devdocs/boundscheck.md", "devdocs/locks.md", + "devdocs/scheduler-wakeup.md", "devdocs/offset-arrays.md", "devdocs/require.md", "devdocs/inference.md", @@ -261,11 +446,13 @@ DevDocs = [ "devdocs/jit.md", "devdocs/builtins.md", "devdocs/precompile_hang.md", + "devdocs/compiler_changes.md", ], "Developing/debugging Julia's C code" => [ "devdocs/backtraces.md", "devdocs/debuggingtips.md", "devdocs/valgrind.md", + "devdocs/gc-debug.md", "devdocs/external_profilers.md", "devdocs/sanitizers.md", "devdocs/probes.md", @@ -289,6 +476,10 @@ DevDocs = [ "devdocs/contributing/formatting.md", "devdocs/contributing/git-workflow.md", "devdocs/contributing/aiagents.md" + ], + "Agentic Devdocs" => [ + "Index" => "devdocs/agents/README.md", + "Agent Skills" => AgentSkillDocs, ] ] @@ -416,6 +607,7 @@ makedocs( authors = "The Julia Project", pages = PAGES, remotes = documenter_stdlib_remotes, + meta = Dict(:DocTestSyntax => VERSION), ) # Update URLs to external stdlibs (JuliaLang/julia#43199) @@ -486,18 +678,39 @@ function Documenter.Writers.HTMLWriter.expand_versions(dir::String, v::Versions) cd(() -> filter!(!islink, available_folders), dir) filter!(x -> occursin(Base.VERSION_REGEX, x), available_folders) - # Look for docs for an "active" release candidate and insert it - vnums = [VersionNumber(x) for x in available_folders] - master_version = maximum(vnums) - filter!(x -> x.major == 1 && x.minor == master_version.minor-1, vnums) - rc = maximum(vnums) - if !isempty(rc.prerelease) && occursin(r"^rc", rc.prerelease[1]) - src = "v$(rc)" - @assert src ∈ available_folders - push!(v.versions, src => src, pop!(v.versions)) + versions = copy(v.versions) + + # `versions.js` is regenerated by every deployment, including tagged builds and + # builds from `release-*` branches, so the development entry must not be derived + # from the version that happens to be building (that would make e.g. a 1.13.0-rc + # build replace the `v1.14-dev` entry with `v1.13-dev`). Instead, always point it + # at the highest `-dev` folder available, i.e. the docs built from master. + isdev(x) = (p = VersionNumber(x).prerelease; !isempty(p) && p[1] == "dev") + dev_folders = filter(isdev, available_folders) + master_version = if isempty(dev_folders) + maximum(VersionNumber, available_folders) + else + master_folder = argmax(VersionNumber, dev_folders) + versions[end] = master_folder => master_folder + VersionNumber(master_folder) + end + + # Look for docs for an "active" prerelease and insert it. It is enough to look at the + # previous minor release family: if the maximum there still carries prerelease data, + # then the corresponding release has not happened yet. The `-dev` folders are excluded, + # as those hold the development docs of a release branch rather than a prerelease. + prereleases = filter(available_folders) do x + ver = VersionNumber(x) + !isdev(x) && ver.major == master_version.major && ver.minor == master_version.minor-1 + end + if !isempty(prereleases) + src = argmax(VersionNumber, prereleases) + if !isempty(VersionNumber(src).prerelease) + insert!(versions, lastindex(versions), src => src) + end end - return Documenter.Writers.HTMLWriter.expand_versions(dir, v.versions) + return Documenter.Writers.HTMLWriter.expand_versions(dir, versions) end if "deploy" in ARGS diff --git a/doc/man/julia.1 b/doc/man/julia.1 index 9646464e1e63d..b01b4729621b5 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -66,7 +66,7 @@ Print command-line options (this message) Print uncommon options not shown by `-h` .TP ---project[=/@.] +-P, --project[=/@.] Set as the active project/environment. The default @. option will search through parent directories until a Project.toml or JuliaProject.toml file is found. @@ -212,17 +212,25 @@ Enable or disable the polyhedral optimizer Polly (overrides @polly declaration) .TP --code-coverage[={none*|user|all}] -Count executions of source lines (omitting setting is equivalent to `user`) +Record coverage for source lines (omitting setting is equivalent to `user`). Full coverage of +Base itself requires a system image built with coverage counters (`JULIA_COVERAGE_IMAGES=1`). .TP --code-coverage=@ -Count executions of source lines in a file or files under a given directory. A `@` must +Record coverage for source lines in a file or files under a given directory. A `@` must be placed before the path to indicate this option. A `@` with no path will track the current directory. +The path filters reports; newly compiled and interpreted code is instrumented as for `user`. +Compatible instrumented images can also contribute Base and Core counters under the path. .TP --code-coverage=tracefile.info Append coverage information to the LCOV tracefile (filename supports format tokens) +.TP +--code-coverage-mode={hit*|count} +Record whether each line ran (`hit`, the default) or its execution count (`count`, which +may be approximate when code runs on multiple threads) + .TP --track-allocation[={none*|user|all}] Count bytes allocated by each source line (omitting setting is equivalent to `user`) @@ -294,6 +302,13 @@ If --trace-compile is enabled show how long each took to compile in ms --trace-dispatch={stderr|name} Print precompile statements for methods dispatched during execution or save to stderr or a path. +.TP +--trace-eval[={no*|loc|full}] +Show top-level expressions being evaluated. `loc` shows location info only, +`full` shows full expressions (omitting setting is equivalent to `loc`). +Only shows the outermost expression being evaluated, not internal function calls. +Can be controlled programmatically via Base.TRACE_EVAL. + .TP --task-metrics={yes|no*} Enable the collection of per-task metrics. diff --git a/doc/src/base/arrays.md b/doc/src/base/arrays.md index 982c4b49cb2c4..1c7d432473d23 100644 --- a/doc/src/base/arrays.md +++ b/doc/src/base/arrays.md @@ -34,6 +34,7 @@ Base.GenericMemory Base.Memory Base.Memory(::UndefInitializer, ::Int) Base.memoryref +Base.memoryindex Base.Slices Base.RowSlices Base.ColumnSlices @@ -105,6 +106,7 @@ Base.prevind Base.copyto!(::AbstractArray, ::CartesianIndices, ::AbstractArray, ::CartesianIndices) Base.copy! Base.isassigned +Base.isassigned(::Tuple, ::Integer) Base.Colon Base.CartesianIndex Base.CartesianIndices diff --git a/doc/src/base/base.md b/doc/src/base/base.md index ab4bfdb6b105b..d153670864030 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -44,6 +44,7 @@ ans err Base.active_project Base.set_active_project +Base.active_manifest ``` ## [Keywords](@id Keywords) @@ -55,6 +56,9 @@ This is the list of reserved keywords in Julia: `return`, `struct`, `true`, `try`, `using`, `while`. Those keywords are not allowed to be used as variable names. +Names starting and ending with double underscores, or dunders, are reserved for use by Julia. This includes macros. Examples of dunder names currently in use include [`__init__`](@ref), [`__source__`](@ref), [`__module__`](@ref), [`__precompile__`](@ref), [`@__dot__`](@ref), [`@__DIR__`](@ref), [`@__FILE__`](@ref), [`@__LINE__`](@ref), and [`@__MODULE__`](@ref). These reservations are not currently enforced by the parser. + + The following two-word sequences are reserved: `abstract type`, `mutable struct`, `primitive type`. However, you can create variables with names: @@ -108,18 +112,19 @@ where .= . -> +Base.:(:) :: [] ``` -## Standard Modules +## [Standard Modules](@id standard-modules) ```@docs Main Core Base ``` -## Base Submodules +## [Base Submodules](@id base-submodules) ```@docs Base.Broadcast Base.Docs @@ -139,8 +144,6 @@ Core.:(===) Core.isa Base.isequal Base.isless -Base.ispositive -Base.isnegative Base.isunordered Base.ifelse Core.typeassert @@ -291,9 +294,11 @@ Base.:(|>) Base.:(∘) Base.ComposedFunction Base.splat +Base.unsplat Base.Fix Base.Fix1 Base.Fix2 +Returns ``` ## Syntax @@ -379,6 +384,7 @@ Base.@elapsed Base.@allocated Base.@allocations Base.@lock_conflicts +Base.TRACE_EVAL Base.EnvDict Base.ENV Base.Sys.STDLIB @@ -499,6 +505,7 @@ Core.modifyglobal! Core.swapglobal! Core.setglobalonce! Core.replaceglobal! +Core.declare_const ``` ## Documentation @@ -516,7 +523,6 @@ Docs.undocumented_names ```@docs Base.identify_package Base.locate_package -Base.require Base.compilecache Base.isprecompiled Base.get_extension @@ -538,12 +544,14 @@ Meta.parse(::AbstractString) Meta.ParseError Core.QuoteNode Base.macroexpand +Base.macroexpand! Base.@macroexpand Base.@macroexpand1 Base.code_lowered Base.code_typed Base.precompile Base.jit_total_bytes +Base.generating_output ``` ## Meta diff --git a/doc/src/base/c.md b/doc/src/base/c.md index bf7e2577029fe..db4add9fa42b5 100644 --- a/doc/src/base/c.md +++ b/doc/src/base/c.md @@ -3,9 +3,10 @@ ```@docs @ccall ccall -Core.Intrinsics.cglobal +cglobal Base.@cfunction Base.CFunction +Base.@ccallable Base.unsafe_convert Base.cconvert Base.unsafe_load diff --git a/doc/src/base/collections.md b/doc/src/base/collections.md index 55cf1ba5dd30d..850be6c2bfef6 100644 --- a/doc/src/base/collections.md +++ b/doc/src/base/collections.md @@ -49,15 +49,30 @@ Fully implemented by: * [`Pair`](@ref) * [`NamedTuple`](@ref) -## Constructors and Types +## Ranges + +### Constructing Ranges + +```@docs +Base.:(:)(::Any, ::Any, ::Any) +Base.:(:)(::CartesianIndex, ::CartesianIndex, ::CartesianIndex) +Base.range +Base.logrange +``` + +### Range Types ```@docs Base.AbstractRange Base.OrdinalRange Base.AbstractUnitRange Base.StepRange +Base.StepRangeLen Base.UnitRange +Base.AbstractOneTo +Base.OneTo Base.LinRange +Base.LogRange ``` ## General Collections diff --git a/doc/src/base/constants.md b/doc/src/base/constants.md index 14ddbc02698d0..bd7cc1339a21e 100644 --- a/doc/src/base/constants.md +++ b/doc/src/base/constants.md @@ -10,6 +10,7 @@ Base.DEPOT_PATH Base.LOAD_PATH Base.Sys.BINDIR Base.Sys.CPU_THREADS +Base.Sys.EFFECTIVE_CPU_THREADS Base.Sys.WORD_SIZE Base.Sys.KERNEL Base.Sys.ARCH diff --git a/doc/src/base/file.md b/doc/src/base/file.md index 300738a39322d..2e6ee107bc99d 100644 --- a/doc/src/base/file.md +++ b/doc/src/base/file.md @@ -7,6 +7,7 @@ Base.Filesystem.pwd Base.Filesystem.cd(::AbstractString) Base.Filesystem.cd(::Function) Base.Filesystem.readdir +Base.Filesystem.DirEntry Base.Filesystem.walkdir Base.Filesystem.mkdir Base.Filesystem.mkpath diff --git a/doc/src/base/io-network.md b/doc/src/base/io-network.md index cd3bb9fbfa7aa..1a3611faa678c 100644 --- a/doc/src/base/io-network.md +++ b/doc/src/base/io-network.md @@ -3,6 +3,7 @@ ## General I/O ```@docs +IO Base.stdout Base.stderr Base.stdin @@ -12,6 +13,7 @@ Base.open Base.IOStream Base.IOBuffer Base.take!(::Base.GenericIOBuffer) +Base.takestring! Base.Pipe Base.link_pipe! Base.fdio @@ -19,6 +21,7 @@ Base.flush Base.close Base.closewrite Base.write +Base.writepartial Base.read Base.read! Base.readbytes! @@ -115,9 +118,9 @@ PNG images in a window can register this capability with Julia, so that calling types with PNG representations will automatically display the image using the module's window. In order to define a new display backend, one should first create a subtype `D` of the abstract -class [`AbstractDisplay`](@ref). Then, for each MIME type (`mime` string) that can be displayed on `D`, one should +type [`AbstractDisplay`](@ref). Then, for each MIME type (`mime` string) that can be displayed on `D`, one should define a function `display(d::D, ::MIME"mime", x) = ...` that displays `x` as that MIME type, -usually by calling [`show(io, mime, x)`](@ref) or [`repr(io, mime, x)`](@ref). +usually by calling [`show(io, mime, x)`](@ref) or [`repr(mime, x)`](@ref). A [`MethodError`](@ref) should be thrown if `x` cannot be displayed as that MIME type; this is automatic if one calls `show` or `repr`. Finally, one should define a function `display(d::D, x)` that queries [`showable(mime, x)`](@ref) for the `mime` types supported by `D` diff --git a/doc/src/base/iterators.md b/doc/src/base/iterators.md index 1c4831e52bc14..cf2bb3bcac83f 100644 --- a/doc/src/base/iterators.md +++ b/doc/src/base/iterators.md @@ -1,7 +1,6 @@ # Iteration utilities ```@docs -Base.Iterators.Stateful Base.Iterators.zip Base.Iterators.enumerate Base.Iterators.rest @@ -10,6 +9,8 @@ Base.Iterators.take Base.Iterators.takewhile Base.Iterators.drop Base.Iterators.dropwhile +Base.Iterators.findeach +Base.Iterators.nth Base.Iterators.cycle Base.Iterators.repeated Base.Iterators.product @@ -22,4 +23,7 @@ Base.Iterators.accumulate Base.Iterators.reverse Base.Iterators.only Base.Iterators.peel +Base.Iterators.Stateful ``` + +User-provided packages in the Julia ecosystem provide iterator-related functionality not offered in `Iterators`. In particular, have a look at [IterTools.jl](https://github.com/JuliaCollections/IterTools.jl). diff --git a/doc/src/base/math.md b/doc/src/base/math.md index 4f816ce2a6c1d..465e2fedfe245 100644 --- a/doc/src/base/math.md +++ b/doc/src/base/math.md @@ -17,30 +17,24 @@ Base.div Base.div(::Any, ::Any, ::RoundingMode) Base.fld Base.cld -Base.mod Base.rem Base.rem(::Any, ::Any, ::RoundingMode) Base.rem2pi +Base.mod Base.Math.mod2pi +Base.mod1 Base.divrem Base.fldmod -Base.fld1 -Base.mod1 -Base.fldmod1 +Base.cldmod1 Base.:(//) Base.rationalize Base.numerator Base.denominator -Base.:(<<) -Base.:(>>) -Base.:(>>>) -Base.bitrotate -Base.:(:) -Base.range -Base.OneTo -Base.StepRangeLen -Base.logrange -Base.LogRange +``` + +## Comparisons + +```@docs Base.:(==) Base.:(!=) Base.:(!==) @@ -49,80 +43,140 @@ Base.:(<=) Base.:(>) Base.:(>=) Base.cmp -Base.:(~) -Base.:(&) -Base.:(|) -Base.xor -Base.nand -Base.nor +Base.isapprox +``` + +## Logical Operators + +```@docs Base.:(!) && || ``` -## Mathematical Functions +## Trigonometry + +### ... in Radians ```@docs -Base.isapprox Base.sin(::Number) Base.cos(::Number) Base.sincos(::Float64) Base.tan(::Number) +``` + +```@docs +Base.asin(::Number) +Base.acos(::Number) +Base.atan(::Number) +``` + +```@docs +Base.Math.sec(::Number) +Base.Math.csc(::Number) +Base.Math.cot(::Number) +``` + +```@docs +Base.Math.asec(::Number) +Base.Math.acsc(::Number) +Base.Math.acot(::Number) +``` + +### ... in Degrees + +```@docs Base.Math.sind Base.Math.cosd Base.Math.tand Base.Math.sincosd -Base.Math.sinpi -Base.Math.cospi -Base.Math.tanpi -Base.Math.sincospi -Base.sinh(::Number) -Base.cosh(::Number) -Base.tanh(::Number) -Base.asin(::Number) -Base.acos(::Number) -Base.atan(::Number) +``` + +```@docs Base.Math.asind Base.Math.acosd Base.Math.atand -Base.Math.sec(::Number) -Base.Math.csc(::Number) -Base.Math.cot(::Number) +``` + +```@docs Base.Math.secd Base.Math.cscd Base.Math.cotd -Base.Math.asec(::Number) -Base.Math.acsc(::Number) -Base.Math.acot(::Number) +``` + +```@docs Base.Math.asecd Base.Math.acscd Base.Math.acotd -Base.Math.sech(::Number) -Base.Math.csch(::Number) -Base.Math.coth(::Number) +``` + +### ... in Multiples of π + +```@docs +Base.Math.sinpi +Base.Math.cospi +Base.Math.tanpi +Base.Math.sincospi +``` + +### Hyperbolic + +```@docs +Base.sinh(::Number) +Base.cosh(::Number) +Base.tanh(::Number) +``` + +```@docs Base.asinh(::Number) Base.acosh(::Number) Base.atanh(::Number) +``` + +```@docs +Base.Math.sech(::Number) +Base.Math.csch(::Number) +Base.Math.coth(::Number) +``` + +```@docs Base.Math.asech(::Number) Base.Math.acsch(::Number) Base.Math.acoth(::Number) +``` + +### Other Trigonometric Functions + +```@docs Base.Math.sinc Base.Math.cosc Base.Math.deg2rad Base.Math.rad2deg Base.Math.hypot +``` + +## Logarithms + +```@docs Base.log(::Number) Base.log(::Number, ::Number) Base.log2 Base.log10 Base.log1p -Base.Math.frexp +``` + +## Exponentiation + +```@docs Base.exp(::Float64) Base.exp2 Base.exp10 -Base.Math.ldexp -Base.Math.modf Base.expm1 +``` + +## Rounding + +```@docs Base.round Base.Rounding.RoundingMode Base.Rounding.RoundNearest @@ -137,12 +191,27 @@ Base.ceil Base.floor Base.trunc Base.unsafe_trunc +Base.Math.modf +``` + +## Extrema + +```@docs Base.min Base.max Base.minmax +``` + +## Clamping + +```@docs Base.Math.clamp Base.Math.clamp! -Base.abs +``` + +## Checked Operations + +```@docs Base.Checked Base.Checked.checked_abs Base.Checked.checked_neg @@ -158,15 +227,40 @@ Base.Checked.checked_pow Base.Checked.add_with_overflow Base.Checked.sub_with_overflow Base.Checked.mul_with_overflow +``` + +## Signs + +```@docs +Base.abs Base.abs2 +Base.uabs +``` + +```@docs Base.copysign Base.sign Base.signbit Base.flipsign +``` + +```@docs +Base.ispositive +Base.isnegative +``` + +## Roots + +```@docs Base.sqrt(::Number) Base.isqrt Base.Math.cbrt(::AbstractFloat) Base.fourthroot(::Number) +``` + +## Complex Functions + +```@docs Base.real Base.imag Base.reim @@ -174,6 +268,11 @@ Base.conj Base.angle Base.cis Base.cispi +``` + +## Combinatorics and Number Theory + +```@docs Base.binomial Base.factorial Base.gcd @@ -185,7 +284,11 @@ Base.prevpow Base.nextprod Base.invmod Base.powermod -Base.ndigits +``` + +## Other Mathematical Functions + +```@docs Base.add_sum Base.widemul Base.Math.evalpoly @@ -195,12 +298,12 @@ Base.FastMath.@fastmath ## Customizable binary operators -Some unicode characters can be used to define new binary operators +Some Unicode characters can be used to define new binary operators that support infix notation. For example ```⊗(x,y) = kron(x,y)``` defines the `⊗` (otimes) function to be the Kronecker product, -and one can call it as binary operator using infix syntax: +and one can call it as a binary operator using infix syntax: ```C = A ⊗ B``` as well as with the usual prefix syntax ```C = ⊗(A,B)```. diff --git a/doc/src/base/multi-threading.md b/doc/src/base/multi-threading.md index 88dc2b7514a2a..b95363232dccd 100644 --- a/doc/src/base/multi-threading.md +++ b/doc/src/base/multi-threading.md @@ -33,9 +33,27 @@ There are also optional memory ordering parameters for the `unsafe` set of funct select the C/C++-compatible versions of these atomic operations, if that parameter is specified to [`unsafe_load`](@ref), [`unsafe_store!`](@ref), [`unsafe_swap!`](@ref), [`unsafe_replace!`](@ref), and [`unsafe_modify!`](@ref). -!!! warning +### Atomic memory fences + +```@docs +Base.Threads.atomic_fence +Base.Threads.atomic_fence_heavy +Base.Threads.atomic_fence_light +``` - The following APIs are deprecated, though support for them is likely to remain for several releases. +### Atomic values (`Threads.Atomic`) + +!!! note + [`Threads.Atomic`](@ref) is a standalone, [`Ref`](@ref)-like atomic cell. Like `Ref`, it is a + useful building block and is not going to be removed, but an `@atomic` field of a mutable struct + is usually preferable when you have the choice. The `Threads.atomic_*` functions below predate + the [`@atomic`](@ref Base.@atomic) macros and still work, but the macros are the recommended way + to operate on an atomic cell; see + [The `@atomic` reference interface](@ref man-atomic-reference) for how to translate them. + +!!! warning + Storing into a `Threads.Atomic` with the plain `a[] = v` form is deprecated (since uses such as + `a[] += 1` look atomic but are not); use `@atomic a[] = v` instead. ```@docs Base.Threads.Atomic @@ -49,7 +67,6 @@ Base.Threads.atomic_or! Base.Threads.atomic_xor! Base.Threads.atomic_max! Base.Threads.atomic_min! -Base.Threads.atomic_fence ``` ## ccall using a libuv threadpool (Experimental) diff --git a/doc/src/base/numbers.md b/doc/src/base/numbers.md index 0bd9d2d4c57d0..b18b0e61ca629 100644 --- a/doc/src/base/numbers.md +++ b/doc/src/base/numbers.md @@ -33,7 +33,7 @@ Number (Abstract Type) └─ Irrational ``` -### Abstract number types +### Abstract Number Types ```@docs Core.Number @@ -45,7 +45,7 @@ Core.Unsigned Base.AbstractIrrational ``` -### Concrete number types +### Concrete Number Types ```@docs Core.Float16 @@ -76,6 +76,7 @@ Base.Irrational ```@docs Base.digits Base.digits! +Base.ndigits Base.bitstring Base.parse Base.tryparse @@ -85,6 +86,8 @@ Base.unsigned Base.float(::Any) Base.Math.significand Base.Math.exponent +Base.Math.frexp +Base.Math.ldexp Base.complex(::Complex) Base.bswap Base.hex2bytes @@ -131,7 +134,20 @@ Base.Rounding.get_zero_subnormals Base.Rounding.set_zero_subnormals ``` -### Integers +## Bit Operations + +```@docs +Base.:(<<) +Base.:(>>) +Base.:(>>>) +Base.bitrotate +Base.:(~) +Base.:(&) +Base.:(|) +Base.xor +Base.nand +Base.nor +``` ```@docs Base.count_ones @@ -140,6 +156,11 @@ Base.leading_zeros Base.leading_ones Base.trailing_zeros Base.trailing_ones +``` + +### Integers + +```@docs Base.isodd Base.iseven Base.@int128_str diff --git a/doc/src/base/parallel.md b/doc/src/base/parallel.md index e382e8edc56ee..c9bfac74e18dd 100644 --- a/doc/src/base/parallel.md +++ b/doc/src/base/parallel.md @@ -45,6 +45,7 @@ Base.reset(::Base.Threads.Event) Base.Semaphore Base.acquire +Base.@acquire Base.release Base.AbstractLock @@ -57,6 +58,28 @@ Base.@lock Base.Lockable ``` +## Cancellation + +Cooperative cancellation of blocked waits (and, eventually, running +computations) is organized around cancellation token sources and the +tokens they hand out. Blocking operations across `Base`, `Sockets` and +`FileWatching` accept a `cancel` keyword argument governing which token may +interrupt them; the scoped default is [`Base.CANCEL_TOKEN`](@ref). + +```@docs +Base.CancellationTokenSource +Base.CancellationToken +Base.CancellationRequest +Base.cancel! +Base.iscancelled +Base.cancel_severity +Base.CANCEL_TOKEN +Base.@cancel_check +Base.CANCEL_REQUEST_SAFE +Base.CANCEL_REQUEST_ABANDON_EXTERNAL +Base.CANCEL_REQUEST_ABANDON_ALL +``` + ## Channels ```@docs @@ -152,7 +175,7 @@ notifying... done ``` -`OneWayEvent` lets one task to `wait` for another task's `notify`. It is a limited +`OneWayEvent` lets one task `wait` for another task's `notify`. It is a limited communication interface since `wait` can only be used once from a single task (note the non-atomic assignment of `ev.task`) diff --git a/doc/src/base/punctuation.md b/doc/src/base/punctuation.md index 8956cebd53971..5d94c27bba15f 100644 --- a/doc/src/base/punctuation.md +++ b/doc/src/base/punctuation.md @@ -1,6 +1,6 @@ # [Punctuation](@id man-punctuation) -Extended documentation for mathematical symbols & functions is [here](@ref math-ops). +See also the [extended documentation for mathematical symbols & functions](@ref math-ops). | symbol | meaning | |:----------- |:--------------------------------------------------------------------------------------------| diff --git a/doc/src/base/reflection.md b/doc/src/base/reflection.md index 16c0d1fadba4e..11f5e0219cf4c 100644 --- a/doc/src/base/reflection.md +++ b/doc/src/base/reflection.md @@ -48,7 +48,7 @@ of these fields is the `types` field observed in the example above. ## Subtypes The *direct* subtypes of any `DataType` may be listed using [`subtypes`](@ref). For example, -the abstract `DataType` [`AbstractFloat`](@ref) has four (concrete) subtypes: +the abstract `DataType` [`AbstractFloat`](@ref) has five (concrete) subtypes: ```jldoctest; setup = :(using InteractiveUtils) julia> InteractiveUtils.subtypes(AbstractFloat) @@ -85,9 +85,9 @@ the unquoted and interpolated expression ([`Expr`](@ref)) form for a given macro `quote` the expression block itself (otherwise, the macro will be evaluated and the result will be passed instead!). For example: -```jldoctest; setup = :(using InteractiveUtils) -julia> InteractiveUtils.macroexpand(@__MODULE__, :(@edit println("")) ) -:(InteractiveUtils.edit(println, InteractiveUtils.Tuple{(InteractiveUtils.Core).Typeof("")})) +```jldoctest +julia> macroexpand(@__MODULE__, :(@invoke identity(1::Int))) +:(Core.invoke(identity, Base.Tuple{Int}, 1)) ``` The functions `Base.Meta.show_sexpr` and [`dump`](@ref) are used to display S-expr style views @@ -117,8 +117,7 @@ method-specific code-lowering is available using [`code_lowered`](@ref), and the type-inferred form is available using [`code_typed`](@ref). [`code_warntype`](@ref) adds highlighting to the output of [`code_typed`](@ref). -Closer to the machine, the LLVM intermediate representation of a function may be printed using -by [`code_llvm`](@ref), and finally the compiled machine code is available +Closer to the machine, the LLVM intermediate representation of a function may be printed using [`code_llvm`](@ref), and finally the compiled machine code is available using [`code_native`](@ref) (this will trigger JIT compilation/code generation for any function which has not previously been called). @@ -144,15 +143,15 @@ For more information see [`@code_lowered`](@ref), [`@code_typed`](@ref), [`@code The aforementioned functions and macros take the keyword argument `debuginfo` that controls the level of debug information printed. -```jldoctest; setup = :(using InteractiveUtils), filter = r"int.jl:\d+" +```jldoctest; setup = :(using InteractiveUtils), filter = r"\w+\.jl:\d+" julia> InteractiveUtils.@code_typed debuginfo=:source +(1,1) CodeInfo( - @ int.jl:87 within `+` + @ essentials.jl:1190 within `+` 1 ─ %1 = intrinsic Base.add_int(x, y)::Int64 └── return %1 ) => Int64 ``` Possible values for `debuginfo` are: `:none`, `:source`, and `:default`. -Per default debug information is not printed, but that can be changed +By default, debug information is not printed, but that can be changed by setting `Base.IRShow.default_debuginfo[] = :source`. diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 21d075daf9389..2102a1a5125d3 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -312,7 +312,9 @@ end ## API docs ```@docs +Base.ScopedValues.AbstractScopedValue Base.ScopedValues.ScopedValue +Base.ScopedValues.LazyScopedValue Base.ScopedValues.with Base.ScopedValues.@with Base.isassigned(::Base.ScopedValues.ScopedValue) @@ -330,7 +332,7 @@ version of Julia. ## Design inspiration -This design was heavily inspired by [JEPS-429](https://openjdk.org/jeps/429), +This design was heavily inspired by [JEP 429](https://openjdk.org/jeps/429), which in turn was inspired by dynamically scoped free variables in many Lisp dialects. In particular Interlisp-D and its deep binding strategy. -A prior design discussed was context variables ala [PEPS-567](https://peps.python.org/pep-0567/) and implemented in Julia as [ContextVariablesX.jl](https://github.com/tkf/ContextVariablesX.jl). +A prior design discussed was context variables à la [PEP 567](https://peps.python.org/pep-0567/) and implemented in Julia as [ContextVariablesX.jl](https://github.com/tkf/ContextVariablesX.jl). diff --git a/doc/src/base/strings.md b/doc/src/base/strings.md index 15a7c0531de4a..760d05e44abc6 100644 --- a/doc/src/base/strings.md +++ b/doc/src/base/strings.md @@ -1,107 +1,235 @@ # [Strings](@id lib-strings) +This section gives a reference list of string related functions in Julia's `Base` +module. For a general introduction to strings in Julia language, please refer to the +[Strings](@ref man-strings) section of the Julia language manual. + +## Characters + ```@docs -Core.AbstractString Core.AbstractChar Core.Char Base.codepoint +``` + +```@docs +Base.iscntrl +Base.isdigit +Base.isletter +Base.islowercase +Base.isnumeric +Base.isprint +Base.ispunct +Base.isspace +Base.isuppercase +Base.isxdigit +Base.raw_substring +``` + +## String Basics + +```@docs +Core.AbstractString +Core.String +``` + +```@docs +Base.string +Base.repr(::Any) +Core.String(::AbstractString) +``` + +```@docs Base.length(::AbstractString) Base.sizeof(::AbstractString) +Base.textwidth +``` + +```@docs +Base.SubString +Base.LazyString +``` + +## Concatenation + +```@docs Base.:*(::Union{AbstractChar, AbstractString}, ::Union{AbstractChar, AbstractString}...) Base.:^(::Union{AbstractString, AbstractChar}, ::Integer) -Base.string Base.repeat(::AbstractString, ::Integer) Base.repeat(::AbstractChar, ::Integer) -Base.repr(::Any) -Core.String(::AbstractString) -Base.SubString -Base.LazyString -Base.@lazy_str +``` + +## Comparison + +```@docs +Base.isless(::AbstractString, ::AbstractString) +Base.:(==)(::AbstractString, ::AbstractString) +Base.cmp(::AbstractString, ::AbstractString) +``` + +## Encoding + +```@docs Base.transcode -Base.unsafe_string Base.ncodeunits(::AbstractString) Base.codeunit Base.codeunits Base.ascii -Base.Regex -Base.@r_str -Base.SubstitutionString -Base.@s_str +Base.isascii +``` + +```@docs +Base.unsafe_string +``` + +```@docs +Base.thisind +Base.nextind(::AbstractString, ::Integer, ::Integer) +Base.prevind(::AbstractString, ::Integer, ::Integer) +``` + +```@docs +Base.isvalid(::Any) +Base.isvalid(::Any, ::Any) +Base.isvalid(::AbstractString, ::Integer) +Base.ismalformed +Base.isoverlong +Base.show_invalid +``` + +## Non-Standard String Literals + +This section describes several additional string types, and also +[non-standard string literals](@ref non-standard-string-literals). +The list here is not exhaustive, for example there are also +- [version number literals](@ref man-version-number-literals) of the form [`v"..."`](@ref @v_str), +- [markdown string literals](@ref stdlib-markdown-literals) of the form `md"..."`, +- [regular expressions and substitution string literals](@ref base-regex-literals), and +- large integer literals for 128 bit constants of the form + [`int128"..."`](@ref Base.@int128_str) and [`uint128"..."`](@ref Base.@uint128_str). + +```@docs +Base.@lazy_str Base.@raw_str Base.@b_str -Base.takestring! Base.Docs.@html_str Base.Docs.@text_str -Base.isvalid(::Any) -Base.isvalid(::Any, ::Any) -Base.isvalid(::AbstractString, ::Integer) +``` + +## [Regular Expressions](@id base-regex-literals) + +See also the [Julia manual section about this topic](@ref man-regex-literals). + +```@docs +Base.Regex +Base.@r_str +Base.SubstitutionString +Base.@s_str Base.match Base.eachmatch Base.RegexMatch Base.keys(::RegexMatch) -Base.isless(::AbstractString, ::AbstractString) -Base.:(==)(::AbstractString, ::AbstractString) -Base.cmp(::AbstractString, ::AbstractString) +Base.:*(r1::Union{Regex,AbstractString,AbstractChar}, rs::Union{Regex,AbstractString,AbstractChar}...) +Base.:^(::Regex, ::Integer) +``` + +## Padding, Truncating, Chopping and Stripping + +```@docs Base.lpad Base.rpad +``` + +```@docs Base.ltruncate Base.rtruncate Base.ctruncate +``` + +```@docs +Base.first(::AbstractString, ::Integer) +Base.last(::AbstractString, ::Integer) +``` + +```@docs +Base.chop +Base.chopprefix +Base.chopsuffix +Base.chomp +``` + +```@docs +Base.strip +Base.lstrip +Base.rstrip +``` + +## Splitting and Joining + +```@docs +Base.eachsplit +Base.eachrsplit +Base.split +Base.rsplit +``` + +```@docs +Base.join +``` + + +## Find and Replace + +See also the [section about regular expressions](@ref base-regex-literals). + +```@docs Base.findfirst(::AbstractString, ::AbstractString) Base.findnext(::AbstractString, ::AbstractString, ::Integer) Base.findnext(::AbstractChar, ::AbstractString, ::Integer) Base.findlast(::AbstractString, ::AbstractString) Base.findlast(::AbstractChar, ::AbstractString) Base.findprev(::AbstractString, ::AbstractString, ::Integer) +``` + +```@docs Base.occursin -Base.reverse(::Union{String,SubString{String}}) -Base.replace(::IO, s::AbstractString, ::Pair...) -Base.eachsplit -Base.eachrsplit -Base.split -Base.rsplit -Base.strip -Base.lstrip -Base.rstrip +Base.contains +``` + +```@docs Base.startswith Base.endswith -Base.contains -Base.first(::AbstractString, ::Integer) -Base.last(::AbstractString, ::Integer) +``` + +```@docs +Base.replace(::IO, s::AbstractString, ::Pair...) +``` + + +## Transforming Strings + +```@docs +Base.reverse(::Union{String,SubString{String}}) +``` + +```@docs Base.uppercase Base.lowercase Base.titlecase Base.uppercasefirst Base.lowercasefirst -Base.join -Base.chop -Base.chopprefix -Base.chopsuffix -Base.chomp -Base.thisind -Base.nextind(::AbstractString, ::Integer, ::Integer) -Base.prevind(::AbstractString, ::Integer, ::Integer) -Base.textwidth -Base.isascii -Base.iscntrl -Base.isdigit -Base.isletter -Base.islowercase -Base.isnumeric -Base.isprint -Base.ispunct -Base.isspace -Base.isuppercase -Base.isxdigit +``` + +```@docs Base.escape_string Base.escape_raw_string Base.unescape_string ``` -## `AnnotatedString`s +## [Annotated Strings](@id base-annotated-strings) !!! note - The API for AnnotatedStrings is considered experimental and is subject to change between + The API for `AnnotatedStrings` is considered experimental and is subject to change between Julia versions. ```@docs @@ -110,4 +238,5 @@ Base.AnnotatedChar Base.annotatedstring Base.annotations Base.annotate! +Base.unannotate ``` diff --git a/doc/src/devdocs/EscapeAnalysis.md b/doc/src/devdocs/EscapeAnalysis.md index d8efd759fa131..177e7e03c89bb 100644 --- a/doc/src/devdocs/EscapeAnalysis.md +++ b/doc/src/devdocs/EscapeAnalysis.md @@ -22,7 +22,7 @@ defines the convenience entries `code_escapes` and `@code_escapes` for testing a ```@repl EAUtils # InteractiveUtils.@activate Compiler # to use the stdlib version of the Compiler -let JULIA_DIR = normpath(Sys.BINDIR, "..", "share", "julia") +let JULIA_DIR = normpath(Sys.BINDIR, Base.DATAROOTDIR, "julia") include(normpath(JULIA_DIR, "Compiler", "test", "EAUtils.jl")) using .EAUtils end @@ -52,7 +52,7 @@ end The symbols on the side of each call argument and SSA statements represent the following meaning: - `◌` (plain): this value is not analyzed because escape information of it won't be used anyway (when the object is `isbitstype` for example) -- `✓` (green or cyan): this value never escapes (`has_no_escape(result.state[x])` holds), colored blue if it has arg escape also (`has_arg_escape(result.state[x])` holds) +- `✓` (green or cyan): this value never escapes (`has_no_escape(result.state[x])` holds), colored cyan if it has arg escape also (`has_arg_escape(result.state[x])` holds) - `↑` (blue or yellow): this value can escape to the caller via return (`has_return_escape(result.state[x])` holds), colored yellow if it has unhandled thrown escape also (`has_thrown_escape(result.state[x])` holds) - `X` (red): this value can escape to somewhere the escape analysis can't reason about like escapes to a global memory (`has_all_escape(result.state[x])` holds) - `*` (bold): this value's escape state is between the `ReturnEscape` and `AllEscape` in the partial order of [`EscapeInfo`](@ref Base.Compiler.EscapeAnalysis.EscapeInfo), colored yellow if it has unhandled thrown escape also (`has_thrown_escape(result.state[x])` holds) @@ -72,7 +72,7 @@ result.state[Core.SSAValue(3)] # get EscapeInfo of `r3` `EscapeAnalysis` is implemented as a [data-flow analysis](https://en.wikipedia.org/wiki/Data-flow_analysis) that works on a lattice of [`x::EscapeInfo`](@ref Base.Compiler.EscapeAnalysis.EscapeInfo), which is composed of the following properties: -- `x.Analyzed::Bool`: not formally part of the lattice, only indicates `x` has not been analyzed or not +- `x.Analyzed::Bool`: not formally part of the lattice, only indicates whether `x` has been analyzed or not - `x.ReturnEscape::BitSet`: records SSA statements where `x` can escape to the caller via return - `x.ThrownEscape::BitSet`: records SSA statements where `x` can be thrown as exception (used for the [exception handling](@ref EA-Exception-Handling) described below) @@ -346,7 +346,7 @@ Accordingly, `analyze_escapes` is also able to analyze post-inlining IR and coll escape information that is useful for certain memory-related optimizations. However, since certain optimization passes like inlining can change control flows and eliminate dead code, -they can break the inter-procedural validity of escape information. In particularity, +they can break the inter-procedural validity of escape information. In particular, in order to collect inter-procedurally valid escape information, we need to analyze a pre-inlining IR. Because of this reason, `analyze_escapes` can analyze `IRCode` at any Julia-level optimization stage, diff --git a/doc/src/devdocs/agents/README.md b/doc/src/devdocs/agents/README.md new file mode 100644 index 0000000000000..f6bd1dd5e514e --- /dev/null +++ b/doc/src/devdocs/agents/README.md @@ -0,0 +1,21 @@ +# Agent Skills + +Julia keeps canonical project-local Agent Skills under +`doc/src/devdocs/agents/skills/`. These files follow the +[Agent Skills](https://agentskills.io) `SKILL.md` format. + +The `.agents/skills/` and `.claude/skills/` entries are symlinks to the canonical +skills directory for automatic discovery by skills-aware agents. Do not edit +through those discovery paths directly; edit the canonical `SKILL.md` under +`doc/src/devdocs/agents/skills/`. + +The documentation build renders each canonical `SKILL.md` with its Agent Skill metadata: + +- [`doctests`](skills/doctests/index.md) — writing and verifying `jldoctest` code blocks. +- [`test-changes`](skills/test-changes/index.md) — running and updating tests after changing them. +- [`c-static-analysis`](skills/c-static-analysis/index.md) — Clang static analysis and GC-rooting for C/C++ changes under `src/`. +- [`external-deps`](skills/external-deps/index.md) — modifying external dependencies (`deps/`, patches) and JLLs. +- [`buildkite-logs`](skills/buildkite-logs/index.md) — fetching and inspecting Buildkite CI logs without web sign-in. +- [`ci-timing`](skills/ci-timing/index.md) — comparing a PR's Buildkite job durations against recent CI history. +- [`compiler-jl`](skills/compiler-jl/index.md) — developing and testing Compiler.jl. +- [`julia-syntax-lowering`](skills/julia-syntax-lowering/index.md) — developing and testing JuliaSyntax and JuliaLowering. diff --git a/doc/src/devdocs/agents/skills/buildkite-logs/SKILL.md b/doc/src/devdocs/agents/skills/buildkite-logs/SKILL.md new file mode 100644 index 0000000000000..9c7dc65914f6f --- /dev/null +++ b/doc/src/devdocs/agents/skills/buildkite-logs/SKILL.md @@ -0,0 +1,90 @@ +--- +name: buildkite-logs +description: Fetch and inspect Julia Buildkite CI logs and artifacts without web sign-in, including the TTFX benchmark jobs. Use when debugging Julia CI failures, reviewing Buildkite jobs, investigating a TTFX regression, or when the Buildkite MCP is unavailable. +--- + +# Reviewing Buildkite CI logs + +Use this when investigating Julia Buildkite CI failures, especially if the +Buildkite MCP is unavailable. The recipe requires `gh`, `curl`, `python3`, and +network access to GitHub and Buildkite. + +Julia's CI runs on Buildkite. PR builds run in the `julialang/julia-pr` +pipeline, post-merge master builds in `julialang/julia-ci`, and scheduled runs +in `julialang/julia-master-scheduled` (the old `julia-master` pipeline no +longer exists). The public web UI requires sign-in to download `raw_log`, but +two frontend JSON endpoints are anonymously accessible for public pipelines. +Recipe: + +1. Find the build number. `gh pr checks ` gives launcher-job URLs + like `https://buildkite.com/julialang/julia-pr/builds/#`; for a + master commit, use the commit statuses + (`gh api repos/JuliaLang/julia/commits//status`). The `#` + fragments there are only the top-level launcher jobs (Build/Check/Test/…), + not the per-platform jobs. + +2. List all jobs in the build, with names, states, exit statuses, and UUIDs: + + ```sh + curl -sS -H "Accept: application/json" \ + "https://buildkite.com/julialang//builds//data/jobs" \ + -o /tmp/bkjobs.json + python3 -c "import json; [print(j['state'],'|',j.get('exit_status'),'|',j['name'],'|',j['id']) \ + for j in json.load(open('/tmp/bkjobs.json'))['records']]" + ``` + + This `/data/jobs` endpoint is what the build page's frontend uses; it + returns every job in one page (`records`, plus `has_next_page`). Do NOT use + `builds/.json` for job discovery — anonymously it returns build + metadata with an *empty* `jobs` array (the `statistics` field still shows + the true job count). + +3. Fetch a job's log JSON (replace ``, ``, ``): + + ```sh + curl -sS -H "Accept: application/json" \ + "https://buildkite.com/organizations/julialang/pipelines//builds//jobs//log" \ + -o /tmp/bk.json + ``` + + The log text lives under the JSON `output` field, with embedded HTML + (`