From 4afc2615514851af69b7fc330b5561a1ca56f5d8 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 12:05:28 +0100 Subject: [PATCH 01/18] Add local package reference --- docs/Project.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/Project.toml b/docs/Project.toml index bcbe067..b5cb62d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,3 +3,6 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" ResearchSoftwareMetadata = "58378933-4625-47fa-851e-05ee27d397bd" + +[sources] +ResearchSoftwareMetadata = {path = ".."} From 37d4d672ee17bc18a83986cb3df5586cb9851a1b Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 12:23:02 +0100 Subject: [PATCH 02/18] GitHub workflows are now parsed much better to identify operating systems --- src/crosswalk.jl | 8 +- src/utils.jl | 284 +++++++++++++++++++++++++++++++++++++++++------ test/runtests.jl | 204 +++++++++++++++++++++++++++++++--- 3 files changed, 449 insertions(+), 47 deletions(-) diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 5377dec..4050de1 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -114,10 +114,12 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) codemeta["dateCreated"] = init end - platforms = get_os_from_workflows() + platforms = get_os_from_workflows(git_dir) cm_platforms = sort(string.(get(codemeta, "operatingSystem", String[]))) - if length(platforms) ≠ length(cm_platforms) || - any(platforms .≠ cm_platforms) + if isempty(platforms) + isempty(cm_platforms) && + @info "No platform info in codemeta.json and none in workflows" + elseif platforms ≠ cm_platforms if isempty(cm_platforms) @info "No platform info in codemeta.json, so filling from workflows ($platforms)" else diff --git a/src/utils.jl b/src/utils.jl index ea12c07..74eb866 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -77,47 +77,269 @@ function reconcile!(project, codemeta, proj_key, cm_key; end """ - ResearchSoftwareMetadata.get_os_from_workflows() + ResearchSoftwareMetadata.runner_os(label::AbstractString) -Returns the operating systems that the GitHub workflows associated with this package -work on. This is presumed to represent the operating systems that the software runs on. +Return the operating system named by a GitHub runner label ("Linux", +"Windows" or "macOS"), or `nothing` if the label names none, as with +`self-hosted` or an architecture label. """ -function get_os_from_workflows(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - workflow_folder = joinpath(git_dir, ".github", "workflows") - files = filter(isfile, readdir(workflow_folder, join = true)) - oses = Set{String}() - for file in files - jobs = YAML.load_file(file)["jobs"] - for job in keys(jobs) - os = jobs[job]["runs-on"] - if occursin(r"\${{.*}}", os) - k2s = split(replace(os, r"\${{ *([^ ]*) *}}" => s"\1"), ".") - os = jobs[job]["strategy"] - for k2 in k2s - os = os[k2] - end - if os isa String - push!(oses, os) - else - for val in os - push!(oses, val) - end - end +function runner_os(label::AbstractString) + lower = lowercase(label) + (startswith(lower, "ubuntu") || lower == "linux") && return "Linux" + startswith(lower, "windows") && return "Windows" + startswith(lower, "macos") && return "macOS" + return nothing +end + +""" + ResearchSoftwareMetadata.expression_values(value, context::AbstractDict) + +Return the values an entry of a GitHub workflow can take, as strings. A +literal is its own only value; a single `\${{ matrix.x }}` or +`\${{ inputs.x }}` reference takes the values `context` holds under +`"matrix.x"` or `"inputs.x"`; a list takes the values of all of its +elements that can be determined. Returns `nothing` when no value can be +determined: for any other expression, a reference `context` does not +hold, a mapping or an empty entry. +""" +function expression_values(value::AbstractString, context::AbstractDict) + reference = match(r"^\s*\$\{\{\s*((?:matrix|inputs)\.[\w-]+)\s*\}\}\s*$", + value) + if isnothing(reference) + return occursin("\${{", value) ? nothing : [String(value)] + end + name = reference.captures[1] + return haskey(context, name) ? copy(context[name]) : nothing +end + +function expression_values(values::AbstractVector, context::AbstractDict) + found = String[] + for value in values + vals = expression_values(value, context) + isnothing(vals) || append!(found, vals) + end + + return isempty(found) ? nothing : found +end + +expression_values(::AbstractDict, ::AbstractDict) = nothing +expression_values(::Nothing, ::AbstractDict) = nothing +# Numbers and booleans, which YAML reads from unquoted entries +expression_values(value, ::AbstractDict) = [string(value)] + +""" + ResearchSoftwareMetadata.matrix_context(job::AbstractDict, + context::AbstractDict) + +Return `context` extended with the values each `matrix.x` reference can +take in a workflow job: the entries of that axis of the job's +`strategy.matrix`, together with any given for it under `include`. +`exclude` is not applied, and a matrix that is itself an expression adds +nothing. +""" +function matrix_context(job::AbstractDict, context::AbstractDict) + extended = copy(context) + strategy = get(job, "strategy", nothing) + strategy isa AbstractDict || return extended + matrix = get(strategy, "matrix", nothing) + matrix isa AbstractDict || return extended + for (key, value) in matrix + key in ("include", "exclude") && continue + vals = expression_values(value, context) + isnothing(vals) || (extended["matrix.$key"] = vals) + end + included = get(matrix, "include", nothing) + included isa AbstractVector || return extended + for entry in included + entry isa AbstractDict || continue + for (key, value) in entry + vals = expression_values(value, context) + isnothing(vals) || + append!(get!(extended, "matrix.$key", String[]), vals) + end + end + + return extended +end + +""" + ResearchSoftwareMetadata.has_trigger(workflow::AbstractDict, + name::AbstractString) + +Check whether a GitHub workflow is started by the event `name`, such as +`workflow_call`. The workflow's `on` entry may be a single event, a list +of events or a mapping from events to their settings. +""" +function has_trigger(workflow::AbstractDict, name::AbstractString) + return names_trigger(get(workflow, "on", nothing), name) +end + +# Whether the `on` entry of a workflow, in any of its three forms, names an event +function names_trigger(triggers::AbstractDict, name::AbstractString) + return haskey(triggers, name) +end + +function names_trigger(triggers::AbstractVector, name::AbstractString) + return name in triggers +end + +names_trigger(triggers, name::AbstractString) = triggers == name + +""" + ResearchSoftwareMetadata.workflow_inputs(workflow::AbstractDict) + +Return the context a GitHub workflow starts with when nothing is passed +to it: the default of every input it declares under `workflow_dispatch` +or `workflow_call`, held under `"inputs.x"`. +""" +function workflow_inputs(workflow::AbstractDict) + context = Dict{String, Vector{String}}() + triggers = get(workflow, "on", nothing) + triggers isa AbstractDict || return context + for trigger in ("workflow_dispatch", "workflow_call") + settings = get(triggers, trigger, nothing) + settings isa AbstractDict || continue + inputs = get(settings, "inputs", nothing) + inputs isa AbstractDict || continue + for (name, input) in inputs + input isa AbstractDict || continue + vals = expression_values(get(input, "default", nothing), context) + isnothing(vals) || (context["inputs.$name"] = vals) + end + end + + return context +end + +""" + ResearchSoftwareMetadata.runner_labels(runs_on, context::AbstractDict) + +Return the runner labels named by the `runs-on` entry of a workflow job, +or `nothing` if they cannot be determined. The entry may be one label, a +list of labels or a runner group, of which only the `labels` can name an +operating system. +""" +function runner_labels(runs_on::AbstractDict, context::AbstractDict) + return expression_values(get(runs_on, "labels", nothing), context) +end + +function runner_labels(runs_on, context::AbstractDict) + return expression_values(runs_on, context) +end + +# How deeply GitHub lets reusable workflows call one another, which also stops a +# workflow that calls itself from being followed for ever +const MAX_WORKFLOW_DEPTH = 10 + +""" + ResearchSoftwareMetadata.job_runners(job::AbstractDict, + context::AbstractDict, + git_dir::AbstractString, depth::Int) + +Return the runner labels a workflow job can run on, or `nothing` if they +cannot be determined. A job that calls a reusable workflow in the same +repository runs on that workflow's runners, given the inputs the job +passes to it; a reusable workflow in another repository cannot be read. +`depth` counts the reusable workflows followed to reach the job. +""" +function job_runners(job::AbstractDict, context::AbstractDict, + git_dir::AbstractString, depth::Int) + context = matrix_context(job, context) + haskey(job, "runs-on") && return runner_labels(job["runs-on"], context) + uses = get(job, "uses", nothing) + uses isa AbstractString && startswith(uses, "./") || return nothing + depth < MAX_WORKFLOW_DEPTH || return nothing + file = normpath(joinpath(git_dir, uses)) + isfile(file) || return nothing + called = YAML.load_file(file) + called isa AbstractDict || return nothing + inputs = workflow_inputs(called) + passed = get(job, "with", nothing) + if passed isa AbstractDict + for (name, value) in passed + vals = expression_values(value, context) + if isnothing(vals) + delete!(inputs, "inputs.$name") else - push!(oses, os) + inputs["inputs.$name"] = vals end end end + return workflow_runners(called, inputs, git_dir, depth + 1) +end + +""" + ResearchSoftwareMetadata.workflow_runners(workflow::AbstractDict, + context::AbstractDict, + git_dir::AbstractString, + depth::Int) + +Return the runner labels the jobs of a reusable workflow can run on when +it is called with the inputs in `context`, or `nothing` if none can be +determined. +""" +function workflow_runners(workflow::AbstractDict, context::AbstractDict, + git_dir::AbstractString, depth::Int) + jobs = get(workflow, "jobs", nothing) + jobs isa AbstractDict || return nothing + labels = String[] + for job in values(jobs) + job isa AbstractDict || continue + found = job_runners(job, context, git_dir, depth) + isnothing(found) || append!(labels, found) + end + + return isempty(labels) ? nothing : labels +end + +""" + ResearchSoftwareMetadata.get_os_from_workflows(git_dir) + +Return the sorted names of the operating systems ("Linux", "Windows", +"macOS") that the GitHub workflows of the repository at `git_dir` run +on, which are presumed to be the ones the software runs on. The list is +empty if the repository has no workflows. + +Each job's `runs-on` may be a runner label, a list of labels or a runner +group, and may refer to the job's matrix (`\${{ matrix.os }}`, including +values given under `include`) or to a workflow input +(`\${{ inputs.os }}`). A job that calls a reusable workflow in the same +repository counts the runners of that workflow. A job whose operating +system cannot be determined is reported and otherwise ignored: one that +uses any other expression, a self-hosted runner whose labels name no +operating system, or a reusable workflow from another repository. Matrix +`exclude` entries and `if` conditions are not taken into account. +""" +function get_os_from_workflows(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + workflow_folder = joinpath(git_dir, ".github", "workflows") + isdir(workflow_folder) || return String[] + files = filter(readdir(workflow_folder, join = true)) do file + return isfile(file) && endswith(file, r"\.ya?ml") + end platforms = Set{String}() - for os in oses - push!(platforms, - replace(replace(replace(os, "ubuntu" => "Linux"), - "windows" => "Windows"), - r"-.*" => "")) + for file in files + workflow = YAML.load_file(file) + workflow isa AbstractDict || continue + jobs = get(workflow, "jobs", nothing) + jobs isa AbstractDict || continue + context = workflow_inputs(workflow) + for (name, job) in jobs + labels = job isa AbstractDict ? + job_runners(job, context, git_dir, 0) : nothing + oses = isnothing(labels) ? String[] : + filter(!isnothing, runner_os.(labels)) + if !isempty(oses) + union!(platforms, oses) + elseif !has_trigger(workflow, "workflow_call") + # A reusable workflow is counted through the jobs that call it + @info "Cannot determine the operating system for job $name " * + "in $(basename(file)), so ignoring it" + end + end end - return sort(collect(platforms)) + return sort!(collect(platforms)) end """ diff --git a/test/runtests.jl b/test/runtests.jl index 929db6a..230dc12 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -35,7 +35,156 @@ end @test ResearchSoftwareMetadata.split_name("Plato") == (nothing, "Plato") end -function make_fixture(dir; license = "MIT", extra = "", author_details = true) +# Write the named workflow files into a throwaway repository directory and +# return the operating systems read from them +function workflow_os(workflows::Pair{String, String}...) + return mktempdir() do dir + folder = joinpath(dir, ".github", "workflows") + mkpath(folder) + for (name, content) in workflows + write(joinpath(folder, name), content) + end + return ResearchSoftwareMetadata.get_os_from_workflows(dir) + end +end + +# The operating systems read from a workflow of one job, `test`, whose entries +# are given in YAML flow style +function job_os(entries::String) + return workflow_os("a.yaml" => "on: push\njobs: {test: {$entries}}\n") +end + +@testset "Operating systems from workflows" begin + runner_os = ResearchSoftwareMetadata.runner_os + unknown = (:info, r"Cannot determine the operating system for job test") + + @testset "Runner labels" begin + @test runner_os.(["ubuntu-22.04", "ubuntu-24.04-arm", "linux"]) == + fill("Linux", 3) + @test runner_os.(["windows-2022", "Windows"]) == fill("Windows", 2) + @test runner_os.(["macos-latest", "macOS-latest", "macos-14"]) == + fill("macOS", 3) + @test all(isnothing, runner_os.(["self-hosted", "x64", "ARM64"])) + end + + @testset "runs-on" begin + @test job_os("runs-on: ubuntu-latest") == ["Linux"] + @test workflow_os("a.yaml" => """ + on: push + jobs: + a: {runs-on: ubuntu-22.04} + b: {runs-on: windows-2022} + c: {runs-on: macos-14} + d: {runs-on: macOS-latest} + """) == ["Linux", "Windows", "macOS"] + # A list of labels describes one runner + @test job_os("runs-on: [self-hosted, linux, x64]") == ["Linux"] + # Only the labels of a runner group can name an operating system + @test job_os("runs-on: {group: big, labels: [windows-latest]}") == + ["Windows"] + @test_logs unknown @test isempty(job_os("runs-on: {group: big}")) + @test_logs unknown @test isempty(job_os("runs-on: self-hosted")) + end + + @testset "Matrix" begin + on_matrix = "runs-on: '\${{ matrix.os }}', strategy: " + @test job_os(on_matrix * + "{matrix: {os: [ubuntu-latest, windows-latest]}}") == + ["Linux", "Windows"] + @test job_os(on_matrix * "{matrix: {os: macos-latest}}") == ["macOS"] + @test job_os("runs-on: '\${{matrix.os}}', strategy: " * + "{matrix: {os: [macos-latest]}}") == ["macOS"] + @test job_os(on_matrix * "{matrix: {include: " * + "[{os: ubuntu-latest}, {os: windows-latest}]}}") == + ["Linux", "Windows"] + @test job_os(on_matrix * "{matrix: {os: [ubuntu-latest], include: " * + "[{os: macos-latest, julia: 1.11}]}}") == + ["Linux", "macOS"] + # A reference to an axis the matrix does not have + @test_logs unknown @test isempty(job_os(on_matrix * + "{matrix: {julia: [1]}}")) + # Expressions beyond a single reference are not evaluated + fallback = "runs-on: \"\${{ matrix.os || 'ubuntu-latest' }}\", " * + "strategy: {matrix: {os: [ubuntu-latest]}}" + @test_logs unknown @test isempty(job_os(fallback)) + end + + @testset "Reusable workflows" begin + called = """ + on: + workflow_call: + inputs: + os: + required: true + type: string + jobs: + tests: + runs-on: \${{ inputs.os }} + steps: + - run: echo tests + """ + with_matrix = """ + on: push + jobs: + test: + strategy: + matrix: + julia-version: ['1.11', '1'] + os: [ubuntu-latest, macOS-latest, windows-latest] + uses: ./.github/workflows/called.yaml + with: + julia-version: \${{ matrix.julia-version }} + os: \${{ matrix.os }} + """ + with_literal = """ + on: push + jobs: + test: + uses: ./.github/workflows/called.yaml + with: + os: ubuntu-latest + """ + with_default = """ + on: + workflow_call: + inputs: + os: {type: string, default: macos-latest} + jobs: {tests: {runs-on: '\${{ inputs.os }}'}} + """ + # The called workflow is counted through its callers, without a report + @test_logs @test workflow_os("called.yaml" => called, + "matrix.yaml" => with_matrix) == + ["Linux", "Windows", "macOS"] + @test workflow_os("called.yaml" => called, + "literal.yaml" => with_literal) == ["Linux"] + @test_logs @test isempty(workflow_os("called.yaml" => called)) + # An input's default applies when the workflow is not passed one + @test workflow_os("called.yaml" => with_default) == ["macOS"] + # Neither a workflow in another repository nor a missing one can be read + remote = "uses: org/repo/.github/workflows/x.yaml@main" + @test_logs unknown @test isempty(job_os(remote)) + @test_logs unknown @test isempty(workflow_os("matrix.yaml" => + with_matrix)) + # A workflow that calls itself is not followed for ever + @test_logs unknown @test isempty(job_os("uses: ./.github/workflows/a.yaml")) + end + + @testset "Workflow files" begin + linux = "on: push\njobs: {test: {runs-on: ubuntu-latest}}\n" + @test workflow_os("a.yml" => linux) == ["Linux"] + # Only YAML files are workflows, and an empty one has no jobs + @test workflow_os("a.yaml" => linux, + "README.md" => "# Workflows: [unclosed\n", + "empty.yaml" => "") == ["Linux"] + @test isempty(workflow_os()) + mktempdir() do dir + @test isempty(ResearchSoftwareMetadata.get_os_from_workflows(dir)) + end + end +end + +function make_fixture(dir; license = "MIT", extra = "", author_details = true, + workflows = true) project_content = """ name = "RSMDFixture" uuid = "d9a1c9c6-91f3-4f9a-8b4a-9b4c8d3a1e2f" @@ -65,18 +214,20 @@ function make_fixture(dir; license = "MIT", extra = "", author_details = true) open(joinpath(dir, "src", "RSMDFixture.jl"), "w") do io return write(io, src_content) end - mkpath(joinpath(dir, ".github", "workflows")) - open(joinpath(dir, ".github", "workflows", "testing.yaml"), "w") do io - return write(io, - """ - name: CI - on: push - jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - """) + if workflows + mkpath(joinpath(dir, ".github", "workflows")) + open(joinpath(dir, ".github", "workflows", "testing.yaml"), "w") do io + return write(io, + """ + name: CI + on: push + jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + """) + end end run(`$(Git.git()) -C $dir init -q -b main`) run(`$(Git.git()) -C $dir remote add origin @@ -333,6 +484,33 @@ end end end +@testset "Crosswalk without workflows" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + mktempdir() do dir + make_fixture(dir, workflows = false) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test !haskey(codemeta, "operatingSystem") + end + # Operating systems already in codemeta.json are kept + mktempdir() do dir + make_fixture(dir, workflows = false) + open(joinpath(dir, "codemeta.json"), "w") do io + return write(io, + """ + { + "operatingSystem": ["Linux", "macOS"] + } + """) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["operatingSystem"] == ["Linux", "macOS"] + end +end + @testset "Failed crosswalk leaves files unchanged" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir From 560483c7cb344eb51676be8c9795e3e36939d39d Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 12:57:04 +0100 Subject: [PATCH 03/18] Better identification of which operating systems tests run on. --- src/crosswalk.jl | 26 +++--- src/utils.jl | 222 ++++++++++++++++++++++++++++++++++++++--------- test/runtests.jl | 182 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 367 insertions(+), 63 deletions(-) diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 4050de1..9ef6cb7 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -479,20 +479,18 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) else if haskey(codemeta, "codemeta:contIntegration") codemeta["continuousIntegration"] = codemeta["codemeta:contIntegration"]["id"] - elseif isfile(".github/workflows/testing.yaml") - @info "Using .github/workflows/testing.yaml for CI" - codemeta["continuousIntegration"] = urls[repo_index] * - "/actions/workflows/testing.yaml" - codemeta["codemeta:contIntegration"] = Dict("id" => - codemeta["continuousIntegration"]) - elseif isfile(".github/workflows/CI.yaml") - @info "Using .github/workflows/CI.yaml for CI" - codemeta["continuousIntegration"] = urls[repo_index] * - "/actions/workflows/CI.yaml" - codemeta["codemeta:contIntegration"] = Dict("id" => - codemeta["continuousIntegration"]) - elseif isdir(".github/workflows") - @warn "CI not found in codemeta.json, but .github/workflows exists" + else + ci_workflow = get_ci_workflow(git_dir) + if !isnothing(ci_workflow) + @info "Using .github/workflows/$ci_workflow for CI" + codemeta["continuousIntegration"] = urls[repo_index] * + "/actions/workflows/" * + ci_workflow + codemeta["codemeta:contIntegration"] = Dict("id" => + codemeta["continuousIntegration"]) + elseif isdir(joinpath(git_dir, ".github", "workflows")) + @warn "CI not found in codemeta.json, but .github/workflows exists" + end end end diff --git a/src/utils.jl b/src/utils.jl index 74eb866..6eab0a2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -231,28 +231,105 @@ end # workflow that calls itself from being followed for ever const MAX_WORKFLOW_DEPTH = 10 +""" + ResearchSoftwareMetadata.read_workflows(git_dir::AbstractString) + +Return the GitHub workflows of the repository at `git_dir` as +`file => workflow` pairs in file name order: every `.yml` or `.yaml` +file in `.github/workflows` that holds a mapping. The list is empty if +the repository has no workflows. +""" +function read_workflows(git_dir::AbstractString) + workflows = Pair{String, Any}[] + folder = joinpath(git_dir, ".github", "workflows") + isdir(folder) || return workflows + for file in readdir(folder, join = true) + isfile(file) && endswith(file, r"\.ya?ml") || continue + workflow = YAML.load_file(file) + workflow isa AbstractDict && push!(workflows, file => workflow) + end + + return workflows +end + +""" + ResearchSoftwareMetadata.called_workflow(job::AbstractDict, + git_dir::AbstractString, + depth::Int) + +Return the reusable workflow a job calls, or `nothing` if the job calls +none that can be read: only a workflow in the same repository can be, +and only while fewer than `MAX_WORKFLOW_DEPTH` have been followed to +reach the job, which `depth` counts. +""" +function called_workflow(job::AbstractDict, git_dir::AbstractString, depth::Int) + uses = get(job, "uses", nothing) + uses isa AbstractString && startswith(uses, "./") || return nothing + depth < MAX_WORKFLOW_DEPTH || return nothing + file = normpath(joinpath(git_dir, uses)) + isfile(file) || return nothing + called = YAML.load_file(file) + return called isa AbstractDict ? called : nothing +end + +# Whether a step of a workflow job runs the package's tests +function step_runs_tests(step::AbstractDict) + uses = get(step, "uses", nothing) + uses isa AbstractString && + startswith(uses, "julia-actions/julia-runtest") && return true + script = get(step, "run", nothing) + return script isa AbstractString && occursin("Pkg.test(", script) +end + +step_runs_tests(step) = false + +""" + ResearchSoftwareMetadata.runs_tests(job::AbstractDict, + git_dir::AbstractString, depth::Int) + +Check whether a workflow job runs the package's tests: one of its steps +uses the `julia-actions/julia-runtest` action or runs a script that +calls `Pkg.test`, or the job calls a reusable workflow in the same +repository that has such a job. `depth` counts the reusable workflows +followed to reach the job. +""" +function runs_tests(job::AbstractDict, git_dir::AbstractString, depth::Int) + steps = get(job, "steps", nothing) + steps isa AbstractVector && return any(step_runs_tests, steps) + called = called_workflow(job, git_dir, depth) + return !isnothing(called) && workflow_runs_tests(called, git_dir, depth + 1) +end + +# Whether any job of a workflow runs the package's tests +function workflow_runs_tests(workflow::AbstractDict, git_dir::AbstractString, + depth::Int) + jobs = get(workflow, "jobs", nothing) + jobs isa AbstractDict || return false + return any(values(jobs)) do job + return job isa AbstractDict && runs_tests(job, git_dir, depth) + end +end + """ ResearchSoftwareMetadata.job_runners(job::AbstractDict, context::AbstractDict, - git_dir::AbstractString, depth::Int) + git_dir::AbstractString, depth::Int; + tests_only::Bool = false) Return the runner labels a workflow job can run on, or `nothing` if they cannot be determined. A job that calls a reusable workflow in the same repository runs on that workflow's runners, given the inputs the job -passes to it; a reusable workflow in another repository cannot be read. -`depth` counts the reusable workflows followed to reach the job. +passes to it, and on only those of its jobs that run the package's tests +if `tests_only` is set; a reusable workflow in another repository cannot +be read. `depth` counts the reusable workflows followed to reach the job. """ function job_runners(job::AbstractDict, context::AbstractDict, - git_dir::AbstractString, depth::Int) + git_dir::AbstractString, depth::Int; + tests_only::Bool = false) context = matrix_context(job, context) haskey(job, "runs-on") && return runner_labels(job["runs-on"], context) - uses = get(job, "uses", nothing) - uses isa AbstractString && startswith(uses, "./") || return nothing - depth < MAX_WORKFLOW_DEPTH || return nothing - file = normpath(joinpath(git_dir, uses)) - isfile(file) || return nothing - called = YAML.load_file(file) - called isa AbstractDict || return nothing + called = called_workflow(job, git_dir, depth) + isnothing(called) && return nothing inputs = workflow_inputs(called) passed = get(job, "with", nothing) if passed isa AbstractDict @@ -266,27 +343,33 @@ function job_runners(job::AbstractDict, context::AbstractDict, end end - return workflow_runners(called, inputs, git_dir, depth + 1) + return workflow_runners(called, inputs, git_dir, depth + 1, + tests_only = tests_only) end """ ResearchSoftwareMetadata.workflow_runners(workflow::AbstractDict, context::AbstractDict, git_dir::AbstractString, - depth::Int) + depth::Int; + tests_only::Bool = false) Return the runner labels the jobs of a reusable workflow can run on when it is called with the inputs in `context`, or `nothing` if none can be -determined. +determined. If `tests_only` is set, only the jobs that run the package's +tests are counted. """ function workflow_runners(workflow::AbstractDict, context::AbstractDict, - git_dir::AbstractString, depth::Int) + git_dir::AbstractString, depth::Int; + tests_only::Bool = false) jobs = get(workflow, "jobs", nothing) jobs isa AbstractDict || return nothing labels = String[] for job in values(jobs) job isa AbstractDict || continue - found = job_runners(job, context, git_dir, depth) + tests_only && !runs_tests(job, git_dir, depth) && continue + found = job_runners(job, context, git_dir, depth, + tests_only = tests_only) isnothing(found) || append!(labels, found) end @@ -298,48 +381,103 @@ end Return the sorted names of the operating systems ("Linux", "Windows", "macOS") that the GitHub workflows of the repository at `git_dir` run -on, which are presumed to be the ones the software runs on. The list is -empty if the repository has no workflows. - -Each job's `runs-on` may be a runner label, a list of labels or a runner -group, and may refer to the job's matrix (`\${{ matrix.os }}`, including -values given under `include`) or to a workflow input -(`\${{ inputs.os }}`). A job that calls a reusable workflow in the same -repository counts the runners of that workflow. A job whose operating -system cannot be determined is reported and otherwise ignored: one that -uses any other expression, a self-hosted runner whose labels name no -operating system, or a reusable workflow from another repository. Matrix -`exclude` entries and `if` conditions are not taken into account. +the package's tests on, which are presumed to be the ones the software +runs on. If the jobs that run the tests give none, whether because no job +is recognised as running them or because their runners cannot be +determined, the operating systems of every job are returned in their +place. The list is empty if the repository has no workflows. + +A job runs the tests if it uses the `julia-actions/julia-runtest` action +or calls `Pkg.test` in a script. Each job's `runs-on` may be a runner +label, a list of labels or a runner group, and may refer to the job's +matrix (`\${{ matrix.os }}`, including values given under `include`) or +to a workflow input (`\${{ inputs.os }}`). A job that calls a reusable +workflow in the same repository counts the runners of that workflow. A +job whose operating system cannot be determined is reported and +otherwise ignored: one that uses any other expression, a self-hosted +runner whose labels name no operating system, or a reusable workflow +from another repository. Matrix `exclude` entries and `if` conditions +are not taken into account. """ function get_os_from_workflows(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) - workflow_folder = joinpath(git_dir, ".github", "workflows") - isdir(workflow_folder) || return String[] - files = filter(readdir(workflow_folder, join = true)) do file - return isfile(file) && endswith(file, r"\.ya?ml") - end - platforms = Set{String}() - for file in files - workflow = YAML.load_file(file) - workflow isa AbstractDict || continue + tested = Set{String}() + untested = Set{String}() + # Each job whose operating system is unknown, with whether it runs the tests + unknown = Pair{String, Bool}[] + for (file, workflow) in read_workflows(git_dir) jobs = get(workflow, "jobs", nothing) jobs isa AbstractDict || continue context = workflow_inputs(workflow) for (name, job) in jobs + tests = job isa AbstractDict && runs_tests(job, git_dir, 0) labels = job isa AbstractDict ? - job_runners(job, context, git_dir, 0) : nothing + job_runners(job, context, git_dir, 0, tests_only = tests) : + nothing oses = isnothing(labels) ? String[] : filter(!isnothing, runner_os.(labels)) if !isempty(oses) - union!(platforms, oses) + union!(tests ? tested : untested, oses) elseif !has_trigger(workflow, "workflow_call") # A reusable workflow is counted through the jobs that call it - @info "Cannot determine the operating system for job $name " * - "in $(basename(file)), so ignoring it" + push!(unknown, "job $name in $(basename(file))" => tests) end end end - return sort!(collect(platforms)) + # The jobs that do not run the tests only matter when the others give nothing + for (job, tests) in unknown + (tests || isempty(tested)) && + @info "Cannot determine the operating system for $job, " * + "so ignoring it" + end + + return sort!(collect(isempty(tested) ? untested : tested)) +end + +# Whether a workflow is started by any event other than a call from another +# workflow, so that it runs in its own right +function has_own_trigger(workflow::AbstractDict) + return own_trigger(get(workflow, "on", nothing)) +end + +own_trigger(triggers::AbstractDict) = any(!=("workflow_call"), keys(triggers)) +own_trigger(triggers::AbstractVector) = any(!=("workflow_call"), triggers) +own_trigger(triggers::AbstractString) = triggers != "workflow_call" +own_trigger(triggers) = false + +# Rank of a workflow file by how conventional its name is for the workflow that +# runs the tests: `testing` first, then `CI`, in any case and with either extension +function ci_name_rank(file::AbstractString) + stem = lowercase(first(splitext(basename(file)))) + return stem == "testing" ? 0 : stem == "ci" ? 1 : 2 +end + +""" + ResearchSoftwareMetadata.get_ci_workflow(git_dir) + +Return the file name of the GitHub workflow that carries out continuous +integration for the repository at `git_dir`, or `nothing` if there is +none. It is a workflow that runs in its own right, not only when called +by another, and that either has a job running the package's tests (see +[`ResearchSoftwareMetadata.runs_tests`](@ref)) or is named `testing` or +`CI`. Where there are several, a workflow that runs the tests is chosen +over one that does not, then one started by a pull request, then one +started by a push, then one named `testing`, then one named `CI`, then +the first in file name order. +""" +function get_ci_workflow(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) + ranked = Tuple{Bool, Bool, Bool, Int, String}[] + for (file, workflow) in read_workflows(git_dir) + has_own_trigger(workflow) || continue + tests = workflow_runs_tests(workflow, git_dir, 0) + name_rank = ci_name_rank(file) + tests || name_rank < 2 || continue + push!(ranked, + (!tests, !has_trigger(workflow, "pull_request"), + !has_trigger(workflow, "push"), name_rank, basename(file))) + end + + return isempty(ranked) ? nothing : last(minimum(ranked)) end """ diff --git a/test/runtests.jl b/test/runtests.jl index 230dc12..1912a40 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -36,22 +36,72 @@ end end # Write the named workflow files into a throwaway repository directory and -# return the operating systems read from them -function workflow_os(workflows::Pair{String, String}...) +# return what `f` reads from that directory +function from_workflows(f::Function, workflows::Pair{String, String}...) return mktempdir() do dir folder = joinpath(dir, ".github", "workflows") mkpath(folder) for (name, content) in workflows write(joinpath(folder, name), content) end - return ResearchSoftwareMetadata.get_os_from_workflows(dir) + return f(dir) end end -# The operating systems read from a workflow of one job, `test`, whose entries -# are given in YAML flow style -function job_os(entries::String) - return workflow_os("a.yaml" => "on: push\njobs: {test: {$entries}}\n") +# The operating systems read from the named workflow files +function workflow_os(workflows::Pair{String, String}...) + return from_workflows(ResearchSoftwareMetadata.get_os_from_workflows, + workflows...) +end + +# The continuous integration workflow chosen from the named workflow files +function ci_workflow(workflows::Pair{String, String}...) + return from_workflows(ResearchSoftwareMetadata.get_ci_workflow, + workflows...) +end + +# A workflow of one job, `test`, whose entries are given in YAML flow style, +# started by the events in `on` +function one_job(entries::String; on::String = "push") + return "on: $on\njobs: {test: {$entries}}\n" +end + +# The operating systems read from a workflow of one job +job_os(entries::String) = workflow_os("a.yaml" => one_job(entries)) + +# Steps that run the package's tests, through the action and through a script +const RUNTEST = "steps: [{uses: julia-actions/julia-runtest@v1}]" +const PKG_TEST = "steps: [{run: \"julia --project -e 'using Pkg; Pkg.test()'\"}]" + +# A workflow that does not run the tests, on a runner the package may not use +const TAGBOT_JOB = "runs-on: ubuntu-latest, " * + "steps: [{uses: JuliaRegistries/TagBot@v1}]" +const TAGBOT = "TagBot.yaml" => one_job(TAGBOT_JOB, on = "issue_comment") + +# A reusable workflow that lints on Linux and runs the tests on the runner it is +# given +const CALLED = """ + on: + workflow_call: + inputs: + os: + required: true + type: string + jobs: + lint: + runs-on: ubuntu-latest + steps: + - run: echo lint + tests: + runs-on: \${{ inputs.os }} + steps: + - uses: julia-actions/julia-runtest@v1 + """ + +# A workflow of one job that calls `called.yaml` for the runner `os` +function calling(os::String; on::String = "push") + return one_job("uses: ./.github/workflows/called.yaml, with: {os: $os}", + on = on) end @testset "Operating systems from workflows" begin @@ -169,6 +219,35 @@ end @test_logs unknown @test isempty(job_os("uses: ./.github/workflows/a.yaml")) end + @testset "Jobs that run the tests" begin + docs = "steps: [{run: make docs}]" + # Only the runners of the jobs that run the tests are counted + action = one_job("runs-on: windows-latest, $RUNTEST") + script = one_job("runs-on: windows-latest, $PKG_TEST") + @test workflow_os("CI.yaml" => action, TAGBOT) == ["Windows"] + @test workflow_os("CI.yaml" => script, TAGBOT) == ["Windows"] + # ... and so are only the jobs of a called workflow that run them + @test workflow_os("called.yaml" => CALLED, + "CI.yaml" => calling("windows-latest")) == ["Windows"] + # With no job running the tests, every job is counted + no_tests = one_job("runs-on: macos-latest, $docs") + @test workflow_os("docs.yaml" => no_tests, TAGBOT) == ["Linux", "macOS"] + # ... as it is when the runners of those that do cannot be determined + self_hosted = one_job("runs-on: self-hosted, $RUNTEST") + @test_logs unknown @test workflow_os("CI.yaml" => self_hosted, + TAGBOT) == ["Linux"] + # Test jobs whose runner is unknown are reported, others are not + two_tests = """ + on: push + jobs: + test: {runs-on: self-hosted, $RUNTEST} + windows: {runs-on: windows-latest, $RUNTEST} + docs: {runs-on: [self-hosted, x64], $docs} + """ + @test_logs unknown @test workflow_os("CI.yaml" => two_tests, TAGBOT) == + ["Windows"] + end + @testset "Workflow files" begin linux = "on: push\njobs: {test: {runs-on: ubuntu-latest}}\n" @test workflow_os("a.yml" => linux) == ["Linux"] @@ -183,6 +262,62 @@ end end end +@testset "Continuous integration workflow" begin + tests = "runs-on: ubuntu-latest, $RUNTEST" + no_tests = "runs-on: ubuntu-latest, steps: [{run: make docs}]" + + @testset "Workflows that run the tests" begin + @test ci_workflow("whatever.yml" => one_job(tests), TAGBOT) == + "whatever.yml" + @test ci_workflow("whatever.yml" => + one_job("runs-on: ubuntu-latest, $PKG_TEST")) == + "whatever.yml" + # A workflow that runs the tests is chosen over one named for them + @test ci_workflow("CI.yaml" => one_job(no_tests), + "whatever.yaml" => one_job(tests)) == "whatever.yaml" + # The tests may be run by a workflow the job calls + @test ci_workflow("called.yaml" => CALLED, + "caller.yaml" => calling("ubuntu-latest")) == + "caller.yaml" + end + + @testset "Choosing between workflows" begin + on_pr = "[push, pull_request]" + scheduled = "{schedule: [{cron: '0 2 * * 0'}]}" + # Started by a pull request, then by a push, whatever the names + @test ci_workflow("called.yaml" => CALLED, + "CI.yaml" => calling("ubuntu-latest"), + "checks.yaml" => calling("ubuntu-latest", on = on_pr)) == + "checks.yaml" + @test ci_workflow("a.yaml" => one_job(tests, on = scheduled), + "b.yaml" => one_job(tests)) == "b.yaml" + # Then by name, `testing` before `CI`, then in file name order + @test ci_workflow("a.yaml" => one_job(tests), + "CI.yml" => one_job(tests), + "Testing.yml" => one_job(tests)) == "Testing.yml" + @test ci_workflow("a.yaml" => one_job(tests), + "CI.yml" => one_job(tests)) == "CI.yml" + @test ci_workflow("b.yaml" => one_job(tests), + "a.yaml" => one_job(tests)) == "a.yaml" + end + + @testset "No workflow that runs the tests" begin + # A conventionally named workflow is taken to be the one + @test ci_workflow("CI.yml" => one_job(no_tests), TAGBOT) == "CI.yml" + @test ci_workflow("CI.yaml" => one_job(no_tests), + "testing.yaml" => one_job(no_tests)) == "testing.yaml" + @test isnothing(ci_workflow(TAGBOT)) + # A workflow that is only ever called does not run in its own right + @test isnothing(ci_workflow("called.yaml" => CALLED)) + @test isnothing(ci_workflow("CI.yaml" => + one_job(tests, on = "workflow_call"))) + @test isnothing(ci_workflow()) + mktempdir() do dir + @test isnothing(ResearchSoftwareMetadata.get_ci_workflow(dir)) + end + end +end + function make_fixture(dir; license = "MIT", extra = "", author_details = true, workflows = true) project_content = """ @@ -511,6 +646,39 @@ end end end +@testset "Crosswalk finds the CI workflow" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + repo = "https://github.com/example/RSMDFixture.jl" + # The workflow that runs the tests, whatever it is called + mktempdir() do dir + make_fixture(dir, workflows = false) + folder = joinpath(dir, ".github", "workflows") + mkpath(folder) + write(joinpath(folder, "CI.yml"), + one_job("runs-on: windows-latest, $RUNTEST")) + write(joinpath(folder, first(TAGBOT)), last(TAGBOT)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["continuousIntegration"] == + repo * "/actions/workflows/CI.yml" + @test codemeta["codemeta:contIntegration"]["id"] == + codemeta["continuousIntegration"] + @test codemeta["operatingSystem"] == ["Windows"] + end + # Workflows, but none for continuous integration + mktempdir() do dir + make_fixture(dir, workflows = false) + folder = joinpath(dir, ".github", "workflows") + mkpath(folder) + write(joinpath(folder, first(TAGBOT)), last(TAGBOT)) + @test_logs (:warn, r"CI not found") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test !haskey(codemeta, "continuousIntegration") + end +end + @testset "Failed crosswalk leaves files unchanged" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir From fa9e642e08c4c3f0744aca6fb25390e0677812b5 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 14:49:24 +0100 Subject: [PATCH 04/18] Adding license headers to source files can no longer leave a half-finished crosswalk --- src/crosswalk.jl | 26 ++++------------------- src/utils.jl | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 46 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 9ef6cb7..9cc7359 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -571,6 +571,8 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) # Round-trip to ensure correct order if there were elements missing project = order_project(TOML.parse(sprint(TOML.print, project))) + headers = header_changes(git_dir, project["license"]) + # All remote queries have succeeded, so the files can now be written if !isnothing(license_content) file = joinpath(git_dir, "LICENSE") @@ -595,28 +597,8 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) return JSON.print(io, crosswalk_d, 4) end - # Recursively walk through the directory - path = joinpath(git_dir, ".git") - notpath = joinpath(git_dir, ".github") - for (root, _, files) in walkdir(git_dir) - if !startswith(root, path) || startswith(root, notpath) - for file in files - if endswith(file, ".jl") - jl_file = joinpath(root, file) - data = readlines(jl_file) - if startswith(data[1], "# SPDX-License-Identifier:") - data[1] = "# SPDX-License-Identifier: $(project["license"])" - elseif !startswith(data[1], "### A Pluto.jl notebook ###") - pushfirst!(data, "") - pushfirst!(data, - "# SPDX-License-Identifier: $(project["license"])") - end - open(jl_file, "w") do io - return println.(Ref(io), data) - end - end - end - end + for (jl_file, content) in headers + write(jl_file, content) end return nothing diff --git a/src/utils.jl b/src/utils.jl index 6eab0a2..63cf6d2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -581,3 +581,57 @@ function author_details_consistent(details, proj_authors) return true end + +""" + ResearchSoftwareMetadata.source_files(git_dir::AbstractString) + +Return the paths of the julia source files of the repository at +`git_dir`: every `.jl` file that git tracks, or would track because it is +not ignored. Files that git ignores and the contents of submodules are +not included. +""" +function source_files(git_dir::AbstractString) + # NUL-separated, so that git does not quote unusual file names. Precomposed + # unicode has to be off: with it on, as it is by default on macOS, the git + # that Git.jl supplies cannot convert file names, and fails to scan a + # working tree that has any file whose name is not ASCII. The names then + # come back as they are on disk, which is what opening them needs. + listing = read(`$(Git.git()) -c core.precomposeunicode=false -C $git_dir + ls-files -z --cached --others --exclude-standard -- '*.jl'`, + String) + files = joinpath.(git_dir, split(listing, '\0', keepempty = false)) + return unique!(filter!(isfile, files)) +end + +""" + ResearchSoftwareMetadata.header_changes(git_dir::AbstractString, + license::AbstractString) + +Return the julia source files of the repository at `git_dir` whose first +line has to change for it to be the SPDX header of `license`, as +`file => new content` pairs. A file without a header gains one followed by +a blank line, a header naming another license is corrected, and an empty +file becomes the header alone. Files that already carry the header are +left out, as are Pluto notebooks, whose first line Pluto needs. Nothing +is written. +""" +function header_changes(git_dir::AbstractString, license::AbstractString) + header = "# SPDX-License-Identifier: $license" + changes = Pair{String, String}[] + for file in source_files(git_dir) + lines = readlines(file) + if isempty(lines) + lines = [header] + elseif startswith(first(lines), "# SPDX-License-Identifier:") + first(lines) == header && continue + lines[1] = header + elseif startswith(first(lines), "### A Pluto.jl notebook ###") + continue + else + pushfirst!(lines, header, "") + end + push!(changes, file => join(lines, "\n") * "\n") + end + + return changes +end diff --git a/test/runtests.jl b/test/runtests.jl index 1912a40..b1f389c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -679,6 +679,52 @@ end end end +@testset "Source file headers" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + header = "# SPDX-License-Identifier: MIT" + mktempdir() do dir + make_fixture(dir) + # Added after the fixture's commit, so none of these is tracked + before = Dict("src/empty.jl" => "", + "src/bare.jl" => "x = 1\n", + "src/other.jl" => "# SPDX-License-Identifier: ISC\n\nx = 1\n", + "test/deep/with space δ.jl" => "x = 1\n", + "src/done.jl" => "$header\n\nx = 1", + "src/notebook.jl" => "### A Pluto.jl notebook ###\nx = 1", + "scratch/junk.jl" => "x = 1", + ".gitignore" => "scratch/\n") + after = Dict("src/empty.jl" => "$header\n", + "src/bare.jl" => "$header\n\nx = 1\n", + "src/other.jl" => "$header\n\nx = 1\n", + "test/deep/with space δ.jl" => "$header\n\nx = 1\n") + for (name, content) in before + mkpath(dirname(joinpath(dir, name))) + write(joinpath(dir, name), content) + end + contents() = Dict(name => read(joinpath(dir, name), String) + for name in keys(before)) + + # Only the files that need a header are listed, and listing writes nothing + changes = ResearchSoftwareMetadata.header_changes(dir, "MIT") + @test Dict(changes) == + Dict(joinpath(dir, name) => content for (name, content) in after) + @test contents() == before + + # A file with its header, a notebook and an ignored file are not rewritten + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + @test contents() == merge(before, after) + @test isempty(ResearchSoftwareMetadata.header_changes(dir, "MIT")) + end + # A tracked file that has been deleted is passed over + mktempdir() do dir + make_fixture(dir) + rm(joinpath(dir, "src", "RSMDFixture.jl")) + @test isempty(ResearchSoftwareMetadata.source_files(dir)) + @test isempty(ResearchSoftwareMetadata.header_changes(dir, "MIT")) + end +end + @testset "Failed crosswalk leaves files unchanged" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir From 187340b5aaaa22a3df4652d931230f4ac4f2e759 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 15:07:52 +0100 Subject: [PATCH 05/18] Broken (hand-written?) metadata files already in the repo are handled better --- src/crosswalk.jl | 31 +++++++++++++++++++++---------- src/utils.jl | 19 +++++++++++++++++++ test/runtests.jl | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 9cc7359..6280110 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -54,9 +54,21 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) repos = replace.(urls, r"^.*/([^/]+)$" => s"\1") file = joinpath(git_dir, "codemeta.json") - codemeta = isfile(file) ? - JSON.parsefile(file, dicttype = OrderedDict) : - OrderedDict{String, Any}() + codemeta = isfile(file) ? read_json(file) : OrderedDict{String, Any}() + + # .zenodo.json is rewritten whatever it holds, so one that cannot be read + # is no reason to stop, but what it held is lost + file = joinpath(git_dir, ".zenodo.json") + zenodo = OrderedDict{String, Any}() + zenodo_problem = nothing + if isfile(file) + try + zenodo = read_json(file) + catch err + err isa ErrorException || rethrow() + zenodo_problem = first(split(err.msg, '\n')) + end + end codemeta["@context"] = "https://w3id.org/codemeta/3.0" codemeta["type"] = "SoftwareSourceCode" @@ -198,13 +210,9 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) if !isempty(get(codemeta, "author", [])) details = author_details_from_codemeta(codemeta["author"]) source = "codemeta.json" - elseif isfile(joinpath(git_dir, ".zenodo.json")) - zenodo = JSON.parsefile(joinpath(git_dir, ".zenodo.json"), - dicttype = OrderedDict) - if !isempty(get(zenodo, "creators", [])) - details = author_details_from_zenodo(zenodo["creators"]) - source = ".zenodo.json" - end + elseif !isempty(get(zenodo, "creators", [])) + details = author_details_from_zenodo(zenodo["creators"]) + source = ".zenodo.json" end if !isnothing(details) if author_details_consistent(details, project["authors"]) @@ -596,6 +604,9 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) open(file, "w") do io return JSON.print(io, crosswalk_d, 4) end + isnothing(zenodo_problem) || + @warn "$zenodo_problem. It has been overwritten, and any " * + "information in it has been lost" for (jl_file, content) in headers write(jl_file, content) diff --git a/src/utils.jl b/src/utils.jl index 63cf6d2..e7c1b72 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -26,6 +26,25 @@ function parse_author(author::AbstractString) return String(m.captures[1]), String(m.captures[2]) end +""" + ResearchSoftwareMetadata.read_json(file::AbstractString) + +Read a JSON metadata file such as `codemeta.json` into an `OrderedDict`. +Throws an error that names the file if it cannot be parsed or does not +hold a JSON object. +""" +function read_json(file::AbstractString) + contents = try + JSON.parsefile(file, dicttype = OrderedDict) + catch err + error("Unable to read $(basename(file)): " * sprint(showerror, err)) + end + contents isa AbstractDict || + error("Unable to read $(basename(file)): it does not hold a JSON object") + + return contents +end + """ ResearchSoftwareMetadata.reconcile!(project, codemeta, proj_key, cm_key; value = nothing, default = nothing, diff --git a/test/runtests.jl b/test/runtests.jl index b1f389c..5704a10 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -679,6 +679,52 @@ end end end +@testset "Metadata files that cannot be read" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + # A missing comma between two entries of a list + broken = """ + { + "creators": [ + {"name": "Smith, Ann B"} + {"name": "Jones, Bob"} + ] + } + """ + mktempdir() do dir + file = joinpath(dir, "codemeta.json") + write(file, broken) + @test_throws "Unable to read codemeta.json" ResearchSoftwareMetadata.read_json(file) + write(file, "[1, 2]\n") + @test_throws "does not hold a JSON object" ResearchSoftwareMetadata.read_json(file) + write(file, "{\"name\": \"Fixture\"}\n") + @test ResearchSoftwareMetadata.read_json(file)["name"] == "Fixture" + end + # .zenodo.json is rewritten whatever it holds, so the crosswalk carries on, + # whether or not the file would have been used as a source of authors + lost = (:warn, r"Unable to read \.zenodo\.json.*has been overwritten") + for author_details in (false, true) + mktempdir() do dir + make_fixture(dir, author_details = author_details) + write(joinpath(dir, ".zenodo.json"), broken) + @test_logs lost match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) # crosswalk leaves the working directory changed + zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) + @test [c["name"] for c in zenodo["creators"]] == ["Smith, Ann B"] + end + end + # codemeta.json holds metadata kept nowhere else, so the crosswalk stops + mktempdir() do dir + project_content, src_content = make_fixture(dir) + write(joinpath(dir, "codemeta.json"), broken) + @test_throws "Unable to read codemeta.json" ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) # crosswalk leaves the working directory changed + @test read(joinpath(dir, "codemeta.json"), String) == broken + @test read(joinpath(dir, "Project.toml"), String) == project_content + @test !isfile(joinpath(dir, ".zenodo.json")) + @test !isfile(joinpath(dir, "LICENSE")) + end +end + @testset "Source file headers" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) header = "# SPDX-License-Identifier: MIT" From b047b4d372cc27de0f14d6f6d434260238862918 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 15:21:10 +0100 Subject: [PATCH 06/18] Add the license keyword to crosswalk() Also: - no-license error with suggested licenses. - "not an SPDX identifier" error for license argument. - accept the same license written differently in codemeta.json. - Make a real mismatch stop the run. --- README.md | 15 ++++- docs/src/index.md | 15 ++++- src/crosswalk.jl | 149 ++++++++++++++++++++++------------------------ src/utils.jl | 24 ++++++++ test/runtests.jl | 141 +++++++++++++++++++++++++++++++++++++++---- 5 files changed, 253 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 7b64796..f8b8d5f 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ To capture the license you are using and propagate it throughout the metadata f license = "BSD-2-Clause" ``` +A package has to have a license: the crosswalk stops with an error if it finds none, or one that is not an identifier from that list. You can also set the license when you run the crosswalk, as shown below. + All of the other metadata that this package uses lives in a single `[rsmd]` table. To supplement the metadata on the authors of the package, add the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. ```toml @@ -75,18 +77,29 @@ using ResearchSoftwareMetadata ResearchSoftwareMetadata.crosswalk() ``` -If you want to pass in some additional metadata (the `category` of the software, or the `keywords` associated with it, both of which are written back into `[rsmd]` in `Project.toml`), or you want to increase the package version during the crosswalk, you can do that as follows: +If you want to pass in some additional metadata (the `category` of the software or the `keywords` associated with it, both of which are written back into `[rsmd]` in `Project.toml`, or its `license`), or you want to increase the package version during the crosswalk, you can do that as follows: ```julia # Add in additional metadata ResearchSoftwareMetadata.crosswalk(category = "metadata", keywords = ["julia", "metadata", "research software", "RSMD"]) +# Set the license, or change it +ResearchSoftwareMetadata.crosswalk(license = "BSD-2-Clause") + # Increase version number during crosswalk ResearchSoftwareMetadata.increase_patch() # Bump patch version (e.g. 0.4.1 -> 0.4.2) ResearchSoftwareMetadata.increase_minor() # Bump minor version (e.g. 0.4.2 -> 0.5.0) ResearchSoftwareMetadata.increase_major() # Bump major version (e.g. 0.5.0 -> 1.0.0) ``` +If you later change the metadata in `Project.toml` on purpose (a new description or keywords, a removed author, a different license), run the crosswalk with `update = true` so that it knows the change is intended: + +```julia +ResearchSoftwareMetadata.crosswalk(update = true) +``` + +The change is then carried into the other files and reported as information. Without `update = true` the crosswalk treats a disagreement between `Project.toml` and the other files as a possible mistake: it still brings the other files into line, but reports each change as a warning or an error, and it refuses a changed license altogether, rewriting nothing, because a license should never change by accident. The `increase_patch()`, `increase_minor()` and `increase_major()` functions run the crosswalk with `update = true` themselves. + You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. ```julia diff --git a/docs/src/index.md b/docs/src/index.md index f51166b..4f6ca36 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -36,6 +36,8 @@ To capture the license you are using and propagate it throughout the metadata f license = "BSD-2-Clause" ``` +A package has to have a license: the crosswalk stops with an error if it finds none, or one that is not an identifier from that list. You can also set the license when you run the crosswalk, as shown below. + All of the other metadata that this package uses lives in a single `[rsmd]` table. To supplement the metadata on the authors of the package, add the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. ```toml @@ -75,18 +77,29 @@ ResearchSoftwareMetadata.crosswalk() ``` -If you want to pass in some additional metadata (the `category` of the software, or the `keywords` associated with it, both of which are written back into `[rsmd]` in `Project.toml`), or you want to increase the package version during the crosswalk, you can do that as follows: +If you want to pass in some additional metadata (the `category` of the software or the `keywords` associated with it, both of which are written back into `[rsmd]` in `Project.toml`, or its `license`), or you want to increase the package version during the crosswalk, you can do that as follows: ```julia # Add in additional metadata ResearchSoftwareMetadata.crosswalk(category = "metadata", keywords = ["julia", "metadata", "research software", "RSMD"]) +# Set the license, or change it +ResearchSoftwareMetadata.crosswalk(license = "BSD-2-Clause") + # Increase version number during crosswalk ResearchSoftwareMetadata.increase_patch() # Bump patch version (e.g. 0.4.1 -> 0.4.2) ResearchSoftwareMetadata.increase_minor() # Bump minor version (e.g. 0.4.2 -> 0.5.0) ResearchSoftwareMetadata.increase_major() # Bump major version (e.g. 0.5.0 -> 1.0.0) ``` +If you later change the metadata in `Project.toml` on purpose (a new description or keywords, a removed author, a different license), run the crosswalk with `update = true` so that it knows the change is intended: + +```julia +ResearchSoftwareMetadata.crosswalk(update = true) +``` + +The change is then carried into the other files and reported as information. Without `update = true` the crosswalk treats a disagreement between `Project.toml` and the other files as a possible mistake: it still brings the other files into line, but reports each change as a warning or an error, and it refuses a changed license altogether, rewriting nothing, because a license should never change by accident. The `increase_patch()`, `increase_minor()` and `increase_major()` functions run the crosswalk with `update = true` themselves. + You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. ```julia diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 6280110..c08b17a 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -2,7 +2,7 @@ """ crosswalk(git_dir; category = nothing, keywords = nothing, - build = false, update = false) + license = nothing, build = false, update = false) Runs a crosswalk across `Project.toml`, `LICENSE`, `codemeta.json` and `.zenodo.json` as well as the julia source files to enforce consistency between the different metadata formats. @@ -24,13 +24,20 @@ field - `false` leaves the instructions as is, `true` sets it to the same as the and a string sets it to that value. If `update` is true, changes to `Project.toml` (e.g. to the version or the license) are treated as deliberate and propagated to the other metadata files with `@info` messages instead of being reported as warnings or errors. +The license is an [SPDX identifier](https://spdx.org/licenses/) such as `"MIT"`, taken +from `license` in `Project.toml` or, failing that, from `codemeta.json`; the crosswalk +throws an error if there is none or if it is not one SPDX recognises. The `license` argument +sets or changes it, and like `category` is written back into `Project.toml`. A license in +`Project.toml` that differs from the one already in `codemeta.json` is an error, and +nothing is changed, unless `update` is true or the `license` argument is given. If any remote metadata query (orcid.org, ror.org, spdx.org, doi.org or Julia's General registry) cannot be completed, the crosswalk throws an error and all files are left in their original state. """ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`); - category = nothing, keywords = nothing, build = false, - update = false) + category = nothing, keywords = nothing, + license::Union{AbstractString, Nothing} = nothing, + build = false, update = false) project = read_project(git_dir) rsmd = get!(project, "rsmd", OrderedDict{String, Any}()) proj_version = VersionNumber(project["version"]) @@ -321,70 +328,62 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) project["authors"] = authors end - haslicense = false - license = nothing - - if haskey(project, "license") - proj_license = project["license"] - cm_license = "https://spdx.org/licenses/" * proj_license - if haskey(codemeta, "license") - if codemeta["license"] == cm_license - haslicense = true - license = proj_license - elseif update - @info "Updating license in codemeta.json to match Project.toml " * - "($(codemeta["license"]) → $cm_license)" - codemeta["license"] = cm_license - haslicense = true - license = proj_license - else - @error "License mismatch between Project.toml and codemeta.json: " * - "$(codemeta["license"]) ≠ $cm_license" - end - else - codemeta["license"] = cm_license - haslicense = true - license = proj_license - end + # The license every file is written from. Passing `license` or `update` + # makes a change of license deliberate; any other disagreement stops the + # crosswalk here, before anything is written + recorded = get(codemeta, "license", nothing) + recorded isa AbstractString && (recorded = spdx_identifier(recorded)) + if !isnothing(license) + license_id = license + elseif haskey(project, "license") + license_id = project["license"] + isnothing(recorded) || recorded == license_id || update || + error("License mismatch between Project.toml and codemeta.json: " * + "$license_id ≠ $recorded. Nothing has been changed. If " * + "$license_id is the license you want, run " * + "`crosswalk(update = true)`; if not, restore " * + "`license = \"$recorded\"` in Project.toml") + elseif recorded isa AbstractString + license_id = recorded else - if haskey(codemeta, "license") - project["license"] = replace(codemeta["license"], - "https://spdx.org/licenses/" => "") - haslicense = true - license = project["license"] - else - @warn "No license metadata" - end + error("No license found in Project.toml or codemeta.json. Set one " * + "with `crosswalk(license = \"LGPL-3.0-or-later\")`, or add " * + "`license = \"LGPL-3.0-or-later\"` to Project.toml, using " * + LICENSE_ADVICE) end - - open_license = nothing - license_content = nothing - if haslicense - url = "https://spdx.org/licenses/$license.json" - headers = ["Accept" => "application/json"] - response = HTTP.get(url, headers, status_exception = false) - - response.status == 200 || - error("Unable to fetch license text for $license from " * - "spdx.org, HTTP status $(response.status)") - - just_names = replace.(project["authors"], r" *<[^>]+> *" => "") - name_list = join(just_names, ", ", " and ") - json = JSON.parse(String(response.body)) - open_license = json["isOsiApproved"] - content = json["licenseText"] - replaces = [r""i => years, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list] - for r in replaces - content = replace(content, r) - end - license_content = content + for (place, held) in ("codemeta.json" => recorded, + "Project.toml" => get(project, "license", nothing)) + isnothing(held) || held == license_id || + @info "Changing license in $place from $held to $license_id" + end + project["license"] = license_id + codemeta["license"] = "https://spdx.org/licenses/" * license_id + + url = "https://spdx.org/licenses/$license_id.json" + response = HTTP.get(url, ["Accept" => "application/json"], + status_exception = false) + response.status == 404 && + error("`$license_id` is not a recognised SPDX license identifier. " * + "Use " * LICENSE_ADVICE) + response.status == 200 || + error("Unable to fetch license text for $license_id from " * + "spdx.org, HTTP status $(response.status)") + + just_names = replace.(project["authors"], r" *<[^>]+> *" => "") + name_list = join(just_names, ", ", " and ") + json = JSON.parse(String(response.body)) + open_license = json["isOsiApproved"] + license_content = json["licenseText"] + replaces = [r""i => years, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list, + r""i => name_list] + for r in replaces + license_content = replace(license_content, r) end cm_authors = get(codemeta, "author", OrderedDict{String, Any}[]) @@ -556,10 +555,8 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) end push!(crosswalk_d["creators"], dict) end - if !isnothing(open_license) - crosswalk_d["access_right"] = open_license ? "open" : "closed" - end - crosswalk_d["license"] = project["license"] + crosswalk_d["access_right"] = open_license ? "open" : "closed" + crosswalk_d["license"] = license_id dict = OrderedDict{String, String}() dict["scheme"] = "url" dict["identifier"] = codemeta["codeRepository"] @@ -579,16 +576,14 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) # Round-trip to ensure correct order if there were elements missing project = order_project(TOML.parse(sprint(TOML.print, project))) - headers = header_changes(git_dir, project["license"]) + new_headers = header_changes(git_dir, license_id) # All remote queries have succeeded, so the files can now be written - if !isnothing(license_content) - file = joinpath(git_dir, "LICENSE") - open(file, "w") do io - return write(io, license_content) - end - rm(joinpath(git_dir, "LICENSE.md"), force = true) + file = joinpath(git_dir, "LICENSE") + open(file, "w") do io + return write(io, license_content) end + rm(joinpath(git_dir, "LICENSE.md"), force = true) file = joinpath(git_dir, "Project.toml") open(file, "w") do io @@ -608,7 +603,7 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) @warn "$zenodo_problem. It has been overwritten, and any " * "information in it has been lost" - for (jl_file, content) in headers + for (jl_file, content) in new_headers write(jl_file, content) end diff --git a/src/utils.jl b/src/utils.jl index e7c1b72..fc98b7b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -26,6 +26,30 @@ function parse_author(author::AbstractString) return String(m.captures[1]), String(m.captures[2]) end +# Licenses to suggest to a package that has none: approved by both the FSF and +# the OSI, and running from permissive to strong copyleft +const SUGGESTED_LICENSES = ["MIT", "BSD-2-Clause", "Apache-2.0", "MPL-2.0", + "LGPL-3.0-or-later", "GPL-3.0-or-later"] + +# Where to find a license identifier, and which licenses to consider +const LICENSE_ADVICE = "an SPDX identifier from https://spdx.org/licenses/. " * + "We suggest a license approved by both the FSF and " * + "the OSI, such as " * + join(SUGGESTED_LICENSES, ", ", " or ") * "." + +""" + ResearchSoftwareMetadata.spdx_identifier(license::AbstractString) + +Return the bare SPDX identifier of a license that may be written as the +address of its page on spdx.org: `https://spdx.org/licenses/MIT`, the same +with `http` and the same with a trailing `.html` or `.json` all give `MIT`, +as does `MIT` itself. +""" +function spdx_identifier(license::AbstractString) + return replace(strip(license), r"^https?://spdx\.org/licenses/" => "", + r"\.(html|json)$" => "") +end + """ ResearchSoftwareMetadata.read_json(file::AbstractString) diff --git a/test/runtests.jl b/test/runtests.jl index 5704a10..012a709 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -320,11 +320,13 @@ end function make_fixture(dir; license = "MIT", extra = "", author_details = true, workflows = true) + # A fixture with no license has neither the Project.toml entry nor a header + license_entry = isnothing(license) ? "" : "license = \"$license\"\n" + header = isnothing(license) ? "" : "# SPDX-License-Identifier: $license\n\n" project_content = """ name = "RSMDFixture" uuid = "d9a1c9c6-91f3-4f9a-8b4a-9b4c8d3a1e2f" - license = "$license" - authors = ["Ann B Smith "] + $(license_entry)authors = ["Ann B Smith "] version = "0.1.0" $extra """ @@ -339,12 +341,10 @@ function make_fixture(dir; license = "MIT", extra = "", author_details = true, open(joinpath(dir, "Project.toml"), "w") do io return write(io, project_content) end - src_content = """ - # SPDX-License-Identifier: $license - - module RSMDFixture - end - """ + src_content = header * """ + module RSMDFixture + end + """ mkpath(joinpath(dir, "src")) open(joinpath(dir, "src", "RSMDFixture.jl"), "w") do io return write(io, src_content) @@ -374,6 +374,38 @@ function make_fixture(dir; license = "MIT", extra = "", author_details = true, return project_content, src_content end +# The contents of every file of a fixture, to show whether anything has changed +function fixture_files(dir) + files = Dict{String, String}() + for (root, dirs, names) in walkdir(dir) + filter!(!=(".git"), dirs) + for name in names + path = joinpath(root, name) + files[relpath(path, dir)] = read(path, String) + end + end + + return files +end + +# The license a fixture has in each of the places that record one +function fixture_licenses(dir) + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) + return (project = TOML.parsefile(joinpath(dir, "Project.toml"))["license"], + codemeta = codemeta["license"], zenodo = zenodo["license"], + access = zenodo["access_right"], + header = readline(joinpath(dir, "src", "RSMDFixture.jl"))) +end + +# What fixture_licenses gives when every place agrees on an open license +function all_licensed(license) + return (project = license, + codemeta = "https://spdx.org/licenses/" * license, + zenodo = license, access = "open", + header = "# SPDX-License-Identifier: " * license) +end + @testset "Crosswalk without ORCIDs" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir @@ -601,7 +633,7 @@ end src = readlines(joinpath(dir, "src", "RSMDFixture.jl")) @test src[1] == "# SPDX-License-Identifier: BSD-2-Clause" end - # Without update, a license mismatch is an error and is not propagated + # Without update, a license mismatch stops the crosswalk and changes nothing mktempdir() do dir make_fixture(dir, extra = extra) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) @@ -612,10 +644,95 @@ end open(toml, "w") do io return TOML.print(io, project) end - @test_logs (:error, r"License mismatch") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) + before = fixture_files(dir) + @test_throws "License mismatch" ResearchSoftwareMetadata.crosswalk(dir) cd(git_dir) - codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) - @test codemeta["license"] == "https://spdx.org/licenses/MIT" + @test fixture_files(dir) == before + end +end + +@testset "Declaring the license" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + spdx = "https://spdx.org/licenses/MIT" + codemeta_license(license) = "{\n \"license\": \"$license\"\n}\n" + + @test ResearchSoftwareMetadata.spdx_identifier.(["MIT", spdx, + spdx * ".json", + "http://spdx.org/licenses/MIT.html" + ]) == + fill("MIT", 4) + + # The same license written another way in codemeta.json is not a mismatch, + # on the run that puts it right or on the next + for written in ("MIT", "http://spdx.org/licenses/MIT.html", spdx * ".json") + mktempdir() do dir + make_fixture(dir) + write(joinpath(dir, "codemeta.json"), codemeta_license(written)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + @test fixture_licenses(dir) == all_licensed("MIT") + end + end + + # A license only codemeta.json has is taken up, however it is written + mktempdir() do dir + make_fixture(dir, license = nothing) + write(joinpath(dir, "codemeta.json"), codemeta_license("MIT")) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test fixture_licenses(dir) == all_licensed("MIT") + end + + # No license anywhere: an error that says how to choose one, and no change + mktempdir() do dir + make_fixture(dir, license = nothing) + before = fixture_files(dir) + message = try + ResearchSoftwareMetadata.crosswalk(dir) + "no error" + catch err + sprint(showerror, err) + end + cd(git_dir) + @test all(occursin(message), + ["No license found", "crosswalk(license = \"", + "https://spdx.org/licenses/", "FSF", "OSI", + ResearchSoftwareMetadata.SUGGESTED_LICENSES...]) + @test fixture_files(dir) == before + end + + # Not an SPDX identifier: likewise + mktempdir() do dir + make_fixture(dir, license = "mit") + before = fixture_files(dir) + @test_throws "`mit` is not a recognised SPDX license identifier" ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + @test fixture_files(dir) == before + end + + # The license argument gives a package its first license ... + mktempdir() do dir + make_fixture(dir, license = nothing) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, + license = "MIT")) + cd(git_dir) + @test fixture_licenses(dir) == all_licensed("MIT") + @test startswith(read(joinpath(dir, "LICENSE"), String), "MIT License") + # ... and changes one that is already there, without update + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, + license = "BSD-2-Clause")) + cd(git_dir) + @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") + @test occursin("Redistribution", + read(joinpath(dir, "LICENSE"), String)) + # One that SPDX does not have changes nothing + before = fixture_files(dir) + @test_throws "not a recognised SPDX" ResearchSoftwareMetadata.crosswalk(dir, + license = "nonsense") + cd(git_dir) + @test fixture_files(dir) == before end end From ed8d57a9286e7af7feca36ad405fc51328252e8e Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 16:14:47 +0100 Subject: [PATCH 07/18] Figure out license in a new repo correctly - Add LicenseCheck as a dependency and name the license found in the no-license error. - Recognise the whole family (LICENSE, LICENCE, COPYING, with .md/.txt). - Leave alone any file the tool didn't write. --- Project.toml | 2 + src/ResearchSoftwareMetadata.jl | 2 + src/crosswalk.jl | 52 +++++--- src/licensing.jl | 203 ++++++++++++++++++++++++++++++++ src/utils.jl | 24 ---- test/runtests.jl | 122 +++++++++++++++++++ 6 files changed, 363 insertions(+), 42 deletions(-) create mode 100644 src/licensing.jl diff --git a/Project.toml b/Project.toml index c1b0e18..738f3fc 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +LicenseCheck = "726dbf0d-6eb6-41af-b36c-cd770e0f00cc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" @@ -20,6 +21,7 @@ Git = "1" HTTP = "1,2" JSON = "1" JuliaFormatter = "2" +LicenseCheck = "0.2" Logging = "1" TOML = "1" YAML = "0.4" diff --git a/src/ResearchSoftwareMetadata.jl b/src/ResearchSoftwareMetadata.jl index 27d7819..4e339c8 100644 --- a/src/ResearchSoftwareMetadata.jl +++ b/src/ResearchSoftwareMetadata.jl @@ -25,11 +25,13 @@ using TOML using JSON using DataStructures using HTTP +using LicenseCheck using YAML include("project.jl") include("remotequeries.jl") include("utils.jl") +include("licensing.jl") include("crosswalk.jl") include("versioning.jl") diff --git a/src/crosswalk.jl b/src/crosswalk.jl index c08b17a..a5939a1 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -346,7 +346,8 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) elseif recorded isa AbstractString license_id = recorded else - error("No license found in Project.toml or codemeta.json. Set one " * + error("No license found in Project.toml or codemeta.json. " * + describe_license_files(git_dir) * "Set one " * "with `crosswalk(license = \"LGPL-3.0-or-later\")`, or add " * "`license = \"LGPL-3.0-or-later\"` to Project.toml, using " * LICENSE_ADVICE) @@ -369,22 +370,29 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) error("Unable to fetch license text for $license_id from " * "spdx.org, HTTP status $(response.status)") - just_names = replace.(project["authors"], r" *<[^>]+> *" => "") - name_list = join(just_names, ", ", " and ") json = JSON.parse(String(response.body)) open_license = json["isOsiApproved"] - license_content = json["licenseText"] - replaces = [r""i => years, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list, - r""i => name_list] - for r in replaces - license_content = replace(license_content, r) + just_names = replace.(project["authors"], r" *<[^>]+> *" => "") + license_content = license_text(json["licenseText"], years, just_names) + + # A license file left by an earlier run names the authors there were then, + # which are the ones still in codemeta.json + earlier_names = String[] + for author in get(codemeta, "author", []) + author isa AbstractDict || continue + given = get(author, "givenName", nothing) + family = get(author, "familyName", get(author, "name", nothing)) + isnothing(family) || + push!(earlier_names, isnothing(given) ? family : "$given $family") end + earlier_content = license_text(json["licenseText"], years, earlier_names) + license_changes = license_file_changes(git_dir, license_id, + [ + license_content, + earlier_content + ], + replace = update || + !isnothing(license)) cm_authors = get(codemeta, "author", OrderedDict{String, Any}[]) proj_authors = rsmd["author_details"] @@ -579,11 +587,19 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) new_headers = header_changes(git_dir, license_id) # All remote queries have succeeded, so the files can now be written - file = joinpath(git_dir, "LICENSE") - open(file, "w") do io - return write(io, license_content) + if license_changes.write + file = joinpath(git_dir, "LICENSE") + open(file, "w") do io + return write(io, license_content) + end + end + for file in license_changes.remove + rm(file) + @info license_changes.write ? + "Replaced $(basename(file)) with LICENSE" : + "Removed $(basename(file)), which did not hold the " * + "$license_id license" end - rm(joinpath(git_dir, "LICENSE.md"), force = true) file = joinpath(git_dir, "Project.toml") open(file, "w") do io diff --git a/src/licensing.jl b/src/licensing.jl new file mode 100644 index 0000000..e42ca4a --- /dev/null +++ b/src/licensing.jl @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: MIT + +# Licenses to suggest to a package that has none: approved by both the FSF and +# the OSI, and running from permissive to strong copyleft +const SUGGESTED_LICENSES = ["MIT", "BSD-2-Clause", "Apache-2.0", "MPL-2.0", + "LGPL-3.0-or-later", "GPL-3.0-or-later"] + +# Where to find a license identifier, and which licenses to consider +const LICENSE_ADVICE = "an SPDX identifier from https://spdx.org/licenses/. " * + "We suggest a license approved by both the FSF and " * + "the OSI, such as " * + join(SUGGESTED_LICENSES, ", ", " or ") * "." + +""" + ResearchSoftwareMetadata.spdx_identifier(license::AbstractString) + +Return the bare SPDX identifier of a license that may be written as the +address of its page on spdx.org: `https://spdx.org/licenses/MIT`, the same +with `http` and the same with a trailing `.html` or `.json` all give `MIT`, +as does `MIT` itself. +""" +function spdx_identifier(license::AbstractString) + return replace(strip(license), r"^https?://spdx\.org/licenses/" => "", + r"\.(html|json)$" => "") +end + +# The names a license file may have, compared without regard to case +const LICENSE_FILENAMES = [stem * extension + for stem in ("LICENSE", "LICENCE", "COPYING") + for extension in ("", ".MD", ".TXT")] + +""" + ResearchSoftwareMetadata.license_files(git_dir::AbstractString) + +Return the paths of the license files in the top directory of the +repository at `git_dir`: any called `LICENSE`, `LICENCE` or `COPYING`, with +or without a `.md` or `.txt` extension, in upper or lower case. +""" +function license_files(git_dir::AbstractString) + names = filter(readdir(git_dir)) do name + return uppercase(name) in LICENSE_FILENAMES && + isfile(joinpath(git_dir, name)) + end + + return joinpath.(git_dir, names) +end + +""" + ResearchSoftwareMetadata.license_text(template::AbstractString, + years::AbstractString, + names::AbstractVector) + +Return the text of a license from its SPDX `template`, with the +placeholders for the copyright years and holders filled in. +""" +function license_text(template::AbstractString, years::AbstractString, + names::AbstractVector) + holders = join(names, ", ", " and ") + return replace(template, + r""i => years, + r""i => holders, + r""i => holders, + r""i => holders, + r""i => holders, + r""i => holders, + r""i => holders, + r""i => holders) +end + +""" + ResearchSoftwareMetadata.base_identifier(license::AbstractString) + +Return an SPDX identifier without the `-only`, `-or-later` or `+` that says +which versions of a license apply. The text of a license is the same +whichever is meant, so a license file can only be matched this far. +""" +function base_identifier(license::AbstractString) + return replace(license, r"(-only|-or-later|\+)$" => "") +end + +""" + ResearchSoftwareMetadata.licenses_found(file::AbstractString) + +Return the SPDX identifiers of the licenses whose text is found in a file, +which is none if the file is not recognisable as a license. +""" +function licenses_found(file::AbstractString) + return licensecheck(read(file, String)).licenses_found +end + +""" + ResearchSoftwareMetadata.holds_license(file::AbstractString, + license::AbstractString) + +Check whether the text of `license`, an SPDX identifier, is found in a +file, whichever versions of it the identifier allows. +""" +function holds_license(file::AbstractString, license::AbstractString) + return base_identifier(license) in base_identifier.(licenses_found(file)) +end + +# A license text with every year or range of years replaced by a marker, so +# that two texts differing only in their copyright years compare equal +function without_years(text::AbstractString) + return replace(strip(text), r"\b\d{4}(-\d{4})?\b" => "") +end + +""" + ResearchSoftwareMetadata.is_generated(file::AbstractString, + generated::AbstractVector) + +Check whether a license file is one this package wrote: whether its text +is one of the `generated` texts, apart from the copyright years. +""" +function is_generated(file::AbstractString, generated::AbstractVector) + existing = without_years(read(file, String)) + return any(text -> without_years(text) == existing, generated) +end + +""" + ResearchSoftwareMetadata.license_file_changes(git_dir::AbstractString, + license::AbstractString, + generated::AbstractVector; + replace::Bool) + +Decide what a crosswalk does to the license files of the repository at +`git_dir`, whose license is the SPDX identifier `license`, and return it +as `(write, remove)`: whether to write a generated `LICENSE`, and the +files to remove. Nothing is written or removed here. + +A file this package wrote itself (see +[`ResearchSoftwareMetadata.is_generated`](@ref), with the texts it could +have written in `generated`) is written afresh. Any other file that +contains `license` belongs to the user and is left as it is, in which case +no `LICENSE` is written beside it. A file that does not contain `license` +is removed if `replace` is set, and is otherwise an error, since replacing +it would change the licensing of the repository. +""" +function license_file_changes(git_dir::AbstractString, license::AbstractString, + generated::AbstractVector; replace::Bool) + own = String[] + kept = String[] + foreign = String[] + for file in license_files(git_dir) + if is_generated(file, generated) + push!(own, file) + elseif holds_license(file, license) + push!(kept, file) + else + push!(foreign, file) + end + end + if !isempty(foreign) && !replace + file = first(foreign) + found = licenses_found(file) + holds = isempty(found) ? "no license that can be recognised" : + join(found, ", ", " and ") + error("$(basename(file)) does not hold the $license license that " * + "is declared: it holds $holds. Nothing has been changed. To " * + "replace it with the $license license run " * + "`crosswalk(update = true)`; if it is right, correct the " * + "`license` in Project.toml") + end + @debug "License files" generated_here=basename.(own) left_alone=basename.(kept) replaced=basename.(foreign) + write = !isempty(own) || isempty(kept) + target = joinpath(git_dir, "LICENSE") + remove = filter(!=(target), write ? vcat(own, foreign) : foreign) + + return (write = write, remove = remove) +end + +""" + ResearchSoftwareMetadata.describe_license_files(git_dir::AbstractString) + +Return sentences saying which licenses the license files of the repository +at `git_dir` hold, and for a file holding a single license how to declare +it, to help a package that has declared none. The string is empty if +there are no license files or none can be recognised. +""" +function describe_license_files(git_dir::AbstractString) + sentences = String[] + for file in license_files(git_dir) + found = unique(base_identifier.(licenses_found(file))) + isempty(found) && continue + if length(found) > 1 + push!(sentences, + "$(basename(file)) holds the " * join(found, ", ", " and ") * + " licenses. ") + continue + end + # The text of a GNU license does not say which versions are meant + license = only(found) + choices = occursin(r"^[AL]?GPL-", license) ? + [license * "-or-later", license * "-only"] : [license] + calls = join(["`crosswalk(license = \"$choice\")`" + for choice in choices], " or ") + push!(sentences, + "$(basename(file)) holds the $license license, which $calls " * + "would declare. ") + end + + return join(sentences) +end diff --git a/src/utils.jl b/src/utils.jl index fc98b7b..e7c1b72 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -26,30 +26,6 @@ function parse_author(author::AbstractString) return String(m.captures[1]), String(m.captures[2]) end -# Licenses to suggest to a package that has none: approved by both the FSF and -# the OSI, and running from permissive to strong copyleft -const SUGGESTED_LICENSES = ["MIT", "BSD-2-Clause", "Apache-2.0", "MPL-2.0", - "LGPL-3.0-or-later", "GPL-3.0-or-later"] - -# Where to find a license identifier, and which licenses to consider -const LICENSE_ADVICE = "an SPDX identifier from https://spdx.org/licenses/. " * - "We suggest a license approved by both the FSF and " * - "the OSI, such as " * - join(SUGGESTED_LICENSES, ", ", " or ") * "." - -""" - ResearchSoftwareMetadata.spdx_identifier(license::AbstractString) - -Return the bare SPDX identifier of a license that may be written as the -address of its page on spdx.org: `https://spdx.org/licenses/MIT`, the same -with `http` and the same with a trailing `.html` or `.json` all give `MIT`, -as does `MIT` itself. -""" -function spdx_identifier(license::AbstractString) - return replace(strip(license), r"^https?://spdx\.org/licenses/" => "", - r"\.(html|json)$" => "") -end - """ ResearchSoftwareMetadata.read_json(file::AbstractString) diff --git a/test/runtests.jl b/test/runtests.jl index 012a709..3e20236 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -736,6 +736,128 @@ end end end +@testset "License files" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + names(dir) = basename.(ResearchSoftwareMetadata.license_files(dir)) + # The texts this package writes, from which to make other license files + generated = Dict(map(("MIT", "BSD-2-Clause", "GPL-3.0-or-later")) do license + return mktempdir() do dir + make_fixture(dir, license = license) + ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) # crosswalk changes the directory + return license => + read(joinpath(dir, "LICENSE"), String) + end + end) + # An MIT license with someone else's notice, and in the form of older packages + mit = replace(generated["MIT"], "Ann B Smith" => "The Original Holder") + quoted = "The Fixture package is licensed under the MIT \"Expat\" " * + "License:\n\n" * join("> " .* split(mit, '\n'), '\n') + + @testset "Recognising them" begin + @test ResearchSoftwareMetadata.base_identifier.(["GPL-3.0-or-later", + "GPL-3.0-only", + "GPL-3.0+", "MIT"]) == + ["GPL-3.0", "GPL-3.0", "GPL-3.0", "MIT"] + mktempdir() do dir + for name in ("LICENSE", "Licence.md", "COPYING.txt", "LICENSE.rst", + "NOTICE") + write(joinpath(dir, name), "") + end + @test Set(names(dir)) == + Set(["LICENSE", "Licence.md", "COPYING.txt"]) + file = joinpath(dir, "LICENSE") + write(file, mit) + @test ResearchSoftwareMetadata.holds_license(file, "MIT") + @test !ResearchSoftwareMetadata.holds_license(file, "BSD-2-Clause") + # Generated by this package if only the years differ + @test ResearchSoftwareMetadata.is_generated(file, + [replace(mit, + r"\d{4}" => "1999")]) + @test !ResearchSoftwareMetadata.is_generated(file, + [generated["MIT"]]) + write(file, generated["GPL-3.0-or-later"]) + @test ResearchSoftwareMetadata.holds_license(file, "GPL-3.0-only") + write(file, "All rights reserved.\n") + @test isempty(ResearchSoftwareMetadata.licenses_found(file)) + end + end + + # A file of the user's own that holds the declared license is left exactly + # as it is, under its own name, whatever else it says + restricted = mit * "\nThis software may not be used on Sundays.\n" + for (name, text) in ("LICENSE" => mit, "LICENSE.md" => quoted, + "LICENCE" => mit, "COPYING" => mit, + "LICENSE.txt" => restricted) + mktempdir() do dir + make_fixture(dir) + write(joinpath(dir, name), text) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + @test read(joinpath(dir, name), String) == text + @test names(dir) == [name] + end + end + + # One that holds another license, or none, stops the crosswalk ... + for text in (generated["BSD-2-Clause"], "All rights reserved.\n") + mktempdir() do dir + make_fixture(dir) + write(joinpath(dir, "COPYING"), text) + before = fixture_files(dir) + @test_throws "COPYING does not hold the MIT license" ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + @test fixture_files(dir) == before + # ... unless the change is deliberate, when LICENSE takes its place + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, + update = true)) + cd(git_dir) + @test names(dir) == ["LICENSE"] + @test startswith(read(joinpath(dir, "LICENSE"), String), + "MIT License") + end + end + + # The file this package wrote is kept up to date when an author is added + mktempdir() do dir + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + toml = joinpath(dir, "Project.toml") + project = TOML.parsefile(toml) + push!(project["authors"], "Bob Jones ") + open(toml, "w") do io + return TOML.print(io, project) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test occursin("Ann B Smith and Bob Jones", + read(joinpath(dir, "LICENSE"), String)) + end + + # With no license declared, the error says what a license file holds, and + # declaring that license leaves the file alone + mktempdir() do dir + make_fixture(dir, license = nothing) + write(joinpath(dir, "LICENSE.md"), quoted) + @test_throws "LICENSE.md holds the MIT license, which `crosswalk(license = \"MIT\")`" ResearchSoftwareMetadata.crosswalk(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, + license = "MIT")) + cd(git_dir) + @test read(joinpath(dir, "LICENSE.md"), String) == quoted + @test names(dir) == ["LICENSE.md"] + end + # The text of a GNU license does not say which versions apply + mktempdir() do dir + make_fixture(dir, license = nothing) + write(joinpath(dir, "COPYING"), generated["GPL-3.0-or-later"]) + described = ResearchSoftwareMetadata.describe_license_files(dir) + @test occursin("COPYING holds the GPL-3.0 license", described) + @test occursin("\"GPL-3.0-or-later\")` or `crosswalk(license = " * + "\"GPL-3.0-only\")`", described) + end +end + @testset "Crosswalk without workflows" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir From a66344f1b3470ef29c21e0c2a63a9410f2d92dbe Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 16:29:39 +0100 Subject: [PATCH 08/18] Fix mismanaging of source file license headers - A header naming another license is no longer rewritten. - An undeclared header stops the run. The crosswalk stops before writing anything, listing every such file with the license it names and the two remedies. This holds even with update = true or the license keyword. - On a deliberate license change, headers naming the license the package is moving from are rewritten to the new one. That is the only time a header changes. - Each additional_licenses entry is checked against spdx.org; an unrecognised one is an error. - README and docs page: a short passage on the first-line header and additional_licenses. --- README.md | 9 ++++ docs/src/index.md | 9 ++++ src/crosswalk.jl | 48 ++++++++++++-------- src/licensing.jl | 105 +++++++++++++++++++++++++++++++++++++++++++ src/remotequeries.jl | 19 ++++++++ src/utils.jl | 54 ---------------------- test/runtests.jl | 80 ++++++++++++++++++++++++++++++++- 7 files changed, 249 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index f8b8d5f..1042539 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,15 @@ license = "BSD-2-Clause" A package has to have a license: the crosswalk stops with an error if it finds none, or one that is not an identifier from that list. You can also set the license when you run the crosswalk, as shown below. +The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described next, so that the crosswalk knows they are intended and leaves them alone: + +```toml +[rsmd] +additional_licenses = ["MIT"] +``` + +A file marked with a license that is neither the package's nor in that list stops the crosswalk with an error. + All of the other metadata that this package uses lives in a single `[rsmd]` table. To supplement the metadata on the authors of the package, add the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. ```toml diff --git a/docs/src/index.md b/docs/src/index.md index 4f6ca36..604a793 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -38,6 +38,15 @@ license = "BSD-2-Clause" A package has to have a license: the crosswalk stops with an error if it finds none, or one that is not an identifier from that list. You can also set the license when you run the crosswalk, as shown below. +The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described next, so that the crosswalk knows they are intended and leaves them alone: + +```toml +[rsmd] +additional_licenses = ["MIT"] +``` + +A file marked with a license that is neither the package's nor in that list stops the crosswalk with an error. + All of the other metadata that this package uses lives in a single `[rsmd]` table. To supplement the metadata on the authors of the package, add the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. ```toml diff --git a/src/crosswalk.jl b/src/crosswalk.jl index a5939a1..27873f2 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -30,6 +30,10 @@ throws an error if there is none or if it is not one SPDX recognises. The `licen sets or changes it, and like `category` is written back into `Project.toml`. A license in `Project.toml` that differs from the one already in `codemeta.json` is an error, and nothing is changed, unless `update` is true or the `license` argument is given. +Every julia source file is given a first line naming the license. A file that is deliberately +under a different license keeps its own first line, provided that license is listed under +`additional_licenses` in the `[rsmd]` table; a file marked with any other license is an +error, and nothing is changed. If any remote metadata query (orcid.org, ror.org, spdx.org, doi.org or Julia's General registry) cannot be completed, the crosswalk throws an error and all files are left in their original state. @@ -352,28 +356,35 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) "`license = \"LGPL-3.0-or-later\"` to Project.toml, using " * LICENSE_ADVICE) end + # The licenses the package is being moved from, if this run changes it + previous_licenses = String[] for (place, held) in ("codemeta.json" => recorded, "Project.toml" => get(project, "license", nothing)) - isnothing(held) || held == license_id || - @info "Changing license in $place from $held to $license_id" + held isa AbstractString && held ≠ license_id || continue + @info "Changing license in $place from $held to $license_id" + push!(previous_licenses, held) end project["license"] = license_id codemeta["license"] = "https://spdx.org/licenses/" * license_id - url = "https://spdx.org/licenses/$license_id.json" - response = HTTP.get(url, ["Accept" => "application/json"], - status_exception = false) - response.status == 404 && + spdx = get_license_from_spdx(license_id) + isnothing(spdx) && error("`$license_id` is not a recognised SPDX license identifier. " * "Use " * LICENSE_ADVICE) - response.status == 200 || - error("Unable to fetch license text for $license_id from " * - "spdx.org, HTTP status $(response.status)") + open_license = spdx["isOsiApproved"] + + # Licenses the package declares that some of its files are under instead + additional_licenses = String.(vcat(get(rsmd, "additional_licenses", + String[]))) + for additional in additional_licenses + isnothing(get_license_from_spdx(additional)) && + error("`$additional` in additional_licenses is not a " * + "recognised SPDX license identifier. Use an identifier " * + "from https://spdx.org/licenses/") + end - json = JSON.parse(String(response.body)) - open_license = json["isOsiApproved"] just_names = replace.(project["authors"], r" *<[^>]+> *" => "") - license_content = license_text(json["licenseText"], years, just_names) + license_content = license_text(spdx["licenseText"], years, just_names) # A license file left by an earlier run names the authors there were then, # which are the ones still in codemeta.json @@ -385,12 +396,9 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) isnothing(family) || push!(earlier_names, isnothing(given) ? family : "$given $family") end - earlier_content = license_text(json["licenseText"], years, earlier_names) - license_changes = license_file_changes(git_dir, license_id, - [ - license_content, - earlier_content - ], + generated = [license_content, + license_text(spdx["licenseText"], years, earlier_names)] + license_changes = license_file_changes(git_dir, license_id, generated, replace = update || !isnothing(license)) @@ -584,7 +592,9 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) # Round-trip to ensure correct order if there were elements missing project = order_project(TOML.parse(sprint(TOML.print, project))) - new_headers = header_changes(git_dir, license_id) + new_headers = header_changes(git_dir, license_id, + additional = additional_licenses, + previous = previous_licenses) # All remote queries have succeeded, so the files can now be written if license_changes.write diff --git a/src/licensing.jl b/src/licensing.jl index e42ca4a..51c07e5 100644 --- a/src/licensing.jl +++ b/src/licensing.jl @@ -201,3 +201,108 @@ function describe_license_files(git_dir::AbstractString) return join(sentences) end + +""" + ResearchSoftwareMetadata.source_files(git_dir::AbstractString) + +Return the paths of the julia source files of the repository at +`git_dir`: every `.jl` file that git tracks, or would track because it is +not ignored. Files that git ignores and the contents of submodules are +not included. +""" +function source_files(git_dir::AbstractString) + # NUL-separated, so that git does not quote unusual file names. Precomposed + # unicode has to be off: with it on, as it is by default on macOS, the git + # that Git.jl supplies cannot convert file names, and fails to scan a + # working tree that has any file whose name is not ASCII. The names then + # come back as they are on disk, which is what opening them needs. + listing = read(`$(Git.git()) -c core.precomposeunicode=false -C $git_dir + ls-files -z --cached --others --exclude-standard -- '*.jl'`, + String) + files = joinpath.(git_dir, split(listing, '\0', keepempty = false)) + return unique!(filter!(isfile, files)) +end + +""" + ResearchSoftwareMetadata.header_licenses(expression::AbstractString) + +Return the license identifiers named in the SPDX expression of a source +file header: the one identifier of a plain header, and each of those +joined by `AND` or `OR` in a compound one. The exception that follows a +`WITH` is not a license and is left out. +""" +function header_licenses(expression::AbstractString) + licenses = String[] + exception_next = false + for token in split(replace(expression, r"[()]" => " ")) + if exception_next + exception_next = false + elseif uppercase(token) == "WITH" + exception_next = true + elseif uppercase(token) ∉ ("AND", "OR") + push!(licenses, token) + end + end + + return licenses +end + +""" + ResearchSoftwareMetadata.header_changes(git_dir::AbstractString, + license::AbstractString; + additional::AbstractVector = String[], + previous::AbstractVector = String[]) + +Return the julia source files of the repository at `git_dir` whose first +line has to change for the package to be licensed under `license`, as +`file => new content` pairs. Nothing is written. + +A file without an SPDX header gains one for `license`, followed by a blank +line, and an empty file becomes the header alone. A header is left as it +is if every license it names is `license` or one of the `additional` +licenses the package declares. A header naming one of the `previous` +licenses, which the package is being moved from, is changed to `license`. +A header naming any other license is an error, listing every such file, +since changing it would change the licensing of the file. Pluto notebooks, +whose first line Pluto needs, are passed over. +""" +function header_changes(git_dir::AbstractString, license::AbstractString; + additional::AbstractVector = String[], + previous::AbstractVector = String[]) + prefix = "# SPDX-License-Identifier:" + header = "$prefix $license" + allowed = vcat(license, additional) + changes = Pair{String, String}[] + undeclared = String[] + for file in source_files(git_dir) + lines = readlines(file) + if isempty(lines) + lines = [header] + elseif startswith(first(lines), prefix) + named = strip(chopprefix(first(lines), prefix)) + licenses = header_licenses(named) + if !isempty(licenses) && all(in(allowed), licenses) + continue + elseif isempty(licenses) || named in previous + lines[1] = header + else + push!(undeclared, " $(relpath(file, git_dir)): $named") + continue + end + elseif startswith(first(lines), "### A Pluto.jl notebook ###") + continue + else + pushfirst!(lines, header, "") + end + push!(changes, file => join(lines, "\n") * "\n") + end + isempty(undeclared) || + error("These files are marked with a license that is neither " * + "$license nor one of the package's additional licenses:\n" * + join(undeclared, "\n") * "\nNothing has been changed. If a " * + "file is meant to have that license, add the license to " * + "`additional_licenses` in the [rsmd] table of Project.toml; " * + "if not, correct the first line of the file") + + return changes +end diff --git a/src/remotequeries.jl b/src/remotequeries.jl index 4c1bad1..4d6d676 100644 --- a/src/remotequeries.jl +++ b/src/remotequeries.jl @@ -83,6 +83,25 @@ function get_organisation_from_ror(ror::String) end end +""" + ResearchSoftwareMetadata.get_license_from_spdx(license::AbstractString) + +Take an SPDX license identifier and query spdx.org to return a Dict +containing its record, including the text of the license under +`licenseText` and whether the OSI approves it under `isOsiApproved`, or +nothing if spdx.org has no license with that identifier. Throws an error +if spdx.org cannot be reached or returns an unexpected HTTP status. +""" +function get_license_from_spdx(license::AbstractString) + url = "https://spdx.org/licenses/$license.json" + headers = ["Accept" => "application/json"] + response = HTTP.get(url, headers, status_exception = false) + response.status == 200 && return JSON.parse(String(response.body)) + response.status == 404 && return nothing + return error("Unable to fetch license text for $license from " * + "spdx.org, HTTP status $(response.status)") +end + """ ResearchSoftwareMetadata.check_doi(doi::String) diff --git a/src/utils.jl b/src/utils.jl index e7c1b72..82fe378 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -600,57 +600,3 @@ function author_details_consistent(details, proj_authors) return true end - -""" - ResearchSoftwareMetadata.source_files(git_dir::AbstractString) - -Return the paths of the julia source files of the repository at -`git_dir`: every `.jl` file that git tracks, or would track because it is -not ignored. Files that git ignores and the contents of submodules are -not included. -""" -function source_files(git_dir::AbstractString) - # NUL-separated, so that git does not quote unusual file names. Precomposed - # unicode has to be off: with it on, as it is by default on macOS, the git - # that Git.jl supplies cannot convert file names, and fails to scan a - # working tree that has any file whose name is not ASCII. The names then - # come back as they are on disk, which is what opening them needs. - listing = read(`$(Git.git()) -c core.precomposeunicode=false -C $git_dir - ls-files -z --cached --others --exclude-standard -- '*.jl'`, - String) - files = joinpath.(git_dir, split(listing, '\0', keepempty = false)) - return unique!(filter!(isfile, files)) -end - -""" - ResearchSoftwareMetadata.header_changes(git_dir::AbstractString, - license::AbstractString) - -Return the julia source files of the repository at `git_dir` whose first -line has to change for it to be the SPDX header of `license`, as -`file => new content` pairs. A file without a header gains one followed by -a blank line, a header naming another license is corrected, and an empty -file becomes the header alone. Files that already carry the header are -left out, as are Pluto notebooks, whose first line Pluto needs. Nothing -is written. -""" -function header_changes(git_dir::AbstractString, license::AbstractString) - header = "# SPDX-License-Identifier: $license" - changes = Pair{String, String}[] - for file in source_files(git_dir) - lines = readlines(file) - if isempty(lines) - lines = [header] - elseif startswith(first(lines), "# SPDX-License-Identifier:") - first(lines) == header && continue - lines[1] = header - elseif startswith(first(lines), "### A Pluto.jl notebook ###") - continue - else - pushfirst!(lines, header, "") - end - push!(changes, file => join(lines, "\n") * "\n") - end - - return changes -end diff --git a/test/runtests.jl b/test/runtests.jl index 3e20236..0f05f7f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -972,7 +972,7 @@ end # Added after the fixture's commit, so none of these is tracked before = Dict("src/empty.jl" => "", "src/bare.jl" => "x = 1\n", - "src/other.jl" => "# SPDX-License-Identifier: ISC\n\nx = 1\n", + "src/unnamed.jl" => "# SPDX-License-Identifier:\n\nx = 1\n", "test/deep/with space δ.jl" => "x = 1\n", "src/done.jl" => "$header\n\nx = 1", "src/notebook.jl" => "### A Pluto.jl notebook ###\nx = 1", @@ -980,7 +980,7 @@ end ".gitignore" => "scratch/\n") after = Dict("src/empty.jl" => "$header\n", "src/bare.jl" => "$header\n\nx = 1\n", - "src/other.jl" => "$header\n\nx = 1\n", + "src/unnamed.jl" => "$header\n\nx = 1\n", "test/deep/with space δ.jl" => "$header\n\nx = 1\n") for (name, content) in before mkpath(dirname(joinpath(dir, name))) @@ -1010,6 +1010,82 @@ end end end +# Declare the licenses that some files of a fixture are under instead of its own +function declare_additional(dir, licenses) + toml = joinpath(dir, "Project.toml") + project = TOML.parsefile(toml) + get!(project, "rsmd", Dict{String, Any}())["additional_licenses"] = licenses + open(toml, "w") do io + return TOML.print(io, project) + end +end + +@testset "Files under another license" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + changes = ResearchSoftwareMetadata.header_changes + other = "# SPDX-License-Identifier: BSD-3-Clause\n\nx = 1\n" + either = "# SPDX-License-Identifier: MIT OR BSD-3-Clause\n\nx = 1\n" + + @test ResearchSoftwareMetadata.header_licenses.(["MIT", "MIT OR Apache-2.0", + "(GPL-2.0-only WITH " * + "Classpath-exception-2.0) " * + "AND MIT", ""]) == + [["MIT"], ["MIT", "Apache-2.0"], ["GPL-2.0-only", "MIT"], String[]] + + mktempdir() do dir + make_fixture(dir) + file = joinpath(dir, "src", "other.jl") + write(file, other) + # A license the package has not declared is an error naming the file, + # one it has declared is left, and one it is moving from is replaced + @test_throws "src/other.jl: BSD-3-Clause" changes(dir, "MIT") + @test isempty(changes(dir, "MIT", additional = ["BSD-3-Clause"])) + @test changes(dir, "MIT", previous = ["BSD-3-Clause"]) == + [file => "# SPDX-License-Identifier: MIT\n\nx = 1\n"] + # Every license in a choice of licenses has to be one the package has + write(joinpath(dir, "src", "either.jl"), either) + @test_throws "src/either.jl: MIT OR BSD-3-Clause" changes(dir, "MIT", + previous = ["BSD-3-Clause"]) + @test isempty(changes(dir, "MIT", additional = ["BSD-3-Clause"])) + rm(joinpath(dir, "src", "either.jl")) + + # The crosswalk stops, deliberate change of license or not, and + # changes nothing + before = fixture_files(dir) + @test_throws "additional_licenses" ResearchSoftwareMetadata.crosswalk(dir) + @test_throws "src/other.jl: BSD-3-Clause" ResearchSoftwareMetadata.crosswalk(dir, + update = true) + cd(git_dir) # crosswalk leaves the working directory changed + @test fixture_files(dir) == before + + # Once the license is declared the file is left as it is ... + declare_additional(dir, ["BSD-3-Clause"]) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test read(file, String) == other + @test fixture_licenses(dir) == all_licensed("MIT") + project = TOML.parsefile(joinpath(dir, "Project.toml")) + @test project["rsmd"]["additional_licenses"] == ["BSD-3-Clause"] + # ... and stays so when the package changes license, while the files + # that were under the package's license follow it + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, + license = "BSD-2-Clause")) + cd(git_dir) + @test read(file, String) == other + @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") + end + + # An additional license has to be one SPDX recognises + mktempdir() do dir + make_fixture(dir) + declare_additional(dir, ["nonsense"]) + before = fixture_files(dir) + @test_throws "`nonsense` in additional_licenses is not a recognised" ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + @test fixture_files(dir) == before + end +end + @testset "Failed crosswalk leaves files unchanged" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir From d95187de77850a0208860be48e5a3823e5fe3a7b Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 16:38:49 +0100 Subject: [PATCH 09/18] Add relicense!() to force a relicensing of a whole repo --- README.md | 12 ++++ docs/src/index.md | 12 ++++ src/ResearchSoftwareMetadata.jl | 4 +- src/crosswalk.jl | 70 +++++++++++++++++++--- src/licensing.jl | 38 ++++++++---- test/runtests.jl | 102 ++++++++++++++++++++++++++++++++ 6 files changed, 218 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1042539..68a6c2b 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,18 @@ ResearchSoftwareMetadata.crosswalk(update = true) The change is then carried into the other files and reported as information. Without `update = true` the crosswalk treats a disagreement between `Project.toml` and the other files as a possible mistake: it still brings the other files into line, but reports each change as a warning or an error, and it refuses a changed license altogether, rewriting nothing, because a license should never change by accident. The `increase_patch()`, `increase_minor()` and `increase_major()` functions run the crosswalk with `update = true` themselves. +To move a package to a different license, or to start again when its licensing has got into a tangle, use `relicense!`: + +```julia +# Move the package to a new license +ResearchSoftwareMetadata.relicense!("MIT") + +# Put everything under the new license, whatever it was under before +ResearchSoftwareMetadata.relicense!("MIT", overwrite_all = true) +``` + +The first form is the same as `crosswalk(license = "MIT")`: files listed under `additional_licenses` keep their own license, and a file marked with a license you have not declared still stops it. The second form overrides all of that: it removes `additional_licenses`, replaces every license file with a newly written `LICENSE`, and marks every julia file with the new license. It relabels files whoever wrote them, so make sure you are entitled to relicense them first. + You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. ```julia diff --git a/docs/src/index.md b/docs/src/index.md index 604a793..36283f1 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -109,6 +109,18 @@ ResearchSoftwareMetadata.crosswalk(update = true) The change is then carried into the other files and reported as information. Without `update = true` the crosswalk treats a disagreement between `Project.toml` and the other files as a possible mistake: it still brings the other files into line, but reports each change as a warning or an error, and it refuses a changed license altogether, rewriting nothing, because a license should never change by accident. The `increase_patch()`, `increase_minor()` and `increase_major()` functions run the crosswalk with `update = true` themselves. +To move a package to a different license, or to start again when its licensing has got into a tangle, use `relicense!`: + +```julia +# Move the package to a new license +ResearchSoftwareMetadata.relicense!("MIT") + +# Put everything under the new license, whatever it was under before +ResearchSoftwareMetadata.relicense!("MIT", overwrite_all = true) +``` + +The first form is the same as `crosswalk(license = "MIT")`: files listed under `additional_licenses` keep their own license, and a file marked with a license you have not declared still stops it. The second form overrides all of that: it removes `additional_licenses`, replaces every license file with a newly written `LICENSE`, and marks every julia file with the new license. It relabels files whoever wrote them, so make sure you are entitled to relicense them first. + You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. ```julia diff --git a/src/ResearchSoftwareMetadata.jl b/src/ResearchSoftwareMetadata.jl index 4e339c8..5a3ca57 100644 --- a/src/ResearchSoftwareMetadata.jl +++ b/src/ResearchSoftwareMetadata.jl @@ -15,7 +15,9 @@ which enforces consistency across the metadata files and the julia source code, and [`ResearchSoftwareMetadata.increase_patch()`](@ref), [`ResearchSoftwareMetadata.increase_minor()`](@ref) and [`ResearchSoftwareMetadata.increase_major()`](@ref), which bump the package -version and then re-run the crosswalk. +version and then re-run the crosswalk, and +[`ResearchSoftwareMetadata.relicense!()`](@ref), which moves the package to +a new license. """ module ResearchSoftwareMetadata diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 27873f2..84ba2c4 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -42,6 +42,55 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) category = nothing, keywords = nothing, license::Union{AbstractString, Nothing} = nothing, build = false, update = false) + return run_crosswalk(git_dir, category = category, keywords = keywords, + license = license, build = build, update = update, + overwrite_all = false) +end + +""" + relicense!(license, git_dir; overwrite_all = false) + +Relicense the package in the repository at `git_dir` under `license`, an +[SPDX identifier](https://spdx.org/licenses/) such as `"MIT"`, and run a +crosswalk to carry the change into every metadata file. + +As it stands this is `crosswalk(git_dir, license = license)`: `Project.toml`, +`codemeta.json` and `.zenodo.json` take the new license, the `LICENSE` file is +written for it, and the julia source files that were marked with the old +license are marked with the new one. Files marked with one of the +`additional_licenses` of the `[rsmd]` table are left as they are, and the +crosswalk still stops, changing nothing, if a file is marked with a license +that has not been declared. + +If `overwrite_all` is true, nothing about the existing licensing is kept, and +none of it can stop the crosswalk: `additional_licenses` is removed, every +license file (`LICENSE`, `LICENCE` or `COPYING`, with or without a `.md` or +`.txt` extension) is removed and `LICENSE` written in their place, even where +one already held `license` in its own words, and every julia source file is +marked with `license` whatever it was marked with before. This is the way out +when the licensing of a repository has become inconsistent, and the way to +hand over a license file of your own to be maintained by the crosswalk. It +relabels files regardless of who wrote them, so whether they may be +relicensed is for you to establish first; what it replaces can be recovered +from git if it had been committed. + +The crosswalk throws an error, and nothing is changed, if `license` is not an +identifier SPDX recognises or a remote metadata query cannot be completed. +""" +function relicense!(license::AbstractString, + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`); + overwrite_all::Bool = false) + return run_crosswalk(git_dir, category = nothing, keywords = nothing, + license = license, build = false, update = false, + overwrite_all = overwrite_all) +end + +# The crosswalk itself, which `crosswalk` and `relicense!` both run. With +# `overwrite_all` every license file and source file header is put under +# `license`, whatever it held, and the additional licenses are dropped +function run_crosswalk(git_dir::AbstractString; category, keywords, + license::Union{AbstractString, Nothing}, build, + update, overwrite_all::Bool) project = read_project(git_dir) rsmd = get!(project, "rsmd", OrderedDict{String, Any}()) proj_version = VersionNumber(project["version"]) @@ -374,6 +423,7 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) open_license = spdx["isOsiApproved"] # Licenses the package declares that some of its files are under instead + overwrite_all && delete!(rsmd, "additional_licenses") additional_licenses = String.(vcat(get(rsmd, "additional_licenses", String[]))) for additional in additional_licenses @@ -400,7 +450,8 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) license_text(spdx["licenseText"], years, earlier_names)] license_changes = license_file_changes(git_dir, license_id, generated, replace = update || - !isnothing(license)) + !isnothing(license), + overwrite = overwrite_all) cm_authors = get(codemeta, "author", OrderedDict{String, Any}[]) proj_authors = rsmd["author_details"] @@ -594,15 +645,12 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) new_headers = header_changes(git_dir, license_id, additional = additional_licenses, - previous = previous_licenses) + previous = previous_licenses, + overwrite = overwrite_all) # All remote queries have succeeded, so the files can now be written - if license_changes.write - file = joinpath(git_dir, "LICENSE") - open(file, "w") do io - return write(io, license_content) - end - end + # Removed before LICENSE is written: on a file system that ignores case, a + # file called License is the same file as LICENSE for file in license_changes.remove rm(file) @info license_changes.write ? @@ -610,6 +658,12 @@ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) "Removed $(basename(file)), which did not hold the " * "$license_id license" end + if license_changes.write + file = joinpath(git_dir, "LICENSE") + open(file, "w") do io + return write(io, license_content) + end + end file = joinpath(git_dir, "Project.toml") open(file, "w") do io diff --git a/src/licensing.jl b/src/licensing.jl index 51c07e5..dbc993a 100644 --- a/src/licensing.jl +++ b/src/licensing.jl @@ -121,7 +121,8 @@ end ResearchSoftwareMetadata.license_file_changes(git_dir::AbstractString, license::AbstractString, generated::AbstractVector; - replace::Bool) + replace::Bool, + overwrite::Bool = false) Decide what a crosswalk does to the license files of the repository at `git_dir`, whose license is the SPDX identifier `license`, and return it @@ -134,10 +135,16 @@ have written in `generated`) is written afresh. Any other file that contains `license` belongs to the user and is left as it is, in which case no `LICENSE` is written beside it. A file that does not contain `license` is removed if `replace` is set, and is otherwise an error, since replacing -it would change the licensing of the repository. +it would change the licensing of the repository. If `overwrite` is set +none of that applies: every license file is removed and `LICENSE` written. """ function license_file_changes(git_dir::AbstractString, license::AbstractString, - generated::AbstractVector; replace::Bool) + generated::AbstractVector; replace::Bool, + overwrite::Bool = false) + target = joinpath(git_dir, "LICENSE") + overwrite && + return (write = true, + remove = filter(!=(target), license_files(git_dir))) own = String[] kept = String[] foreign = String[] @@ -159,11 +166,12 @@ function license_file_changes(git_dir::AbstractString, license::AbstractString, "is declared: it holds $holds. Nothing has been changed. To " * "replace it with the $license license run " * "`crosswalk(update = true)`; if it is right, correct the " * - "`license` in Project.toml") + "`license` in Project.toml. " * + "`relicense!(\"$license\", overwrite_all = true)` replaces " * + "every license file and header") end @debug "License files" generated_here=basename.(own) left_alone=basename.(kept) replaced=basename.(foreign) write = !isempty(own) || isempty(kept) - target = joinpath(git_dir, "LICENSE") remove = filter(!=(target), write ? vcat(own, foreign) : foreign) return (write = write, remove = remove) @@ -251,7 +259,8 @@ end ResearchSoftwareMetadata.header_changes(git_dir::AbstractString, license::AbstractString; additional::AbstractVector = String[], - previous::AbstractVector = String[]) + previous::AbstractVector = String[], + overwrite::Bool = false) Return the julia source files of the repository at `git_dir` whose first line has to change for the package to be licensed under `license`, as @@ -263,12 +272,14 @@ is if every license it names is `license` or one of the `additional` licenses the package declares. A header naming one of the `previous` licenses, which the package is being moved from, is changed to `license`. A header naming any other license is an error, listing every such file, -since changing it would change the licensing of the file. Pluto notebooks, -whose first line Pluto needs, are passed over. +since changing it would change the licensing of the file. If `overwrite` +is set, every header is changed to `license` whatever it names. Pluto +notebooks, whose first line Pluto needs, are passed over. """ function header_changes(git_dir::AbstractString, license::AbstractString; additional::AbstractVector = String[], - previous::AbstractVector = String[]) + previous::AbstractVector = String[], + overwrite::Bool = false) prefix = "# SPDX-License-Identifier:" header = "$prefix $license" allowed = vcat(license, additional) @@ -281,7 +292,10 @@ function header_changes(git_dir::AbstractString, license::AbstractString; elseif startswith(first(lines), prefix) named = strip(chopprefix(first(lines), prefix)) licenses = header_licenses(named) - if !isempty(licenses) && all(in(allowed), licenses) + if overwrite + first(lines) == header && continue + lines[1] = header + elseif !isempty(licenses) && all(in(allowed), licenses) continue elseif isempty(licenses) || named in previous lines[1] = header @@ -302,7 +316,9 @@ function header_changes(git_dir::AbstractString, license::AbstractString; join(undeclared, "\n") * "\nNothing has been changed. If a " * "file is meant to have that license, add the license to " * "`additional_licenses` in the [rsmd] table of Project.toml; " * - "if not, correct the first line of the file") + "if not, correct the first line of the file. " * + "`relicense!(\"$license\", overwrite_all = true)` puts " * + "every file under $license") return changes end diff --git a/test/runtests.jl b/test/runtests.jl index 0f05f7f..1d80960 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1086,6 +1086,108 @@ end end end +@testset "Relicensing" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + relicense! = ResearchSoftwareMetadata.relicense! + names(dir) = basename.(ResearchSoftwareMetadata.license_files(dir)) + other = "# SPDX-License-Identifier: BSD-3-Clause\n\nx = 1\n" + relicensed = "# SPDX-License-Identifier: BSD-2-Clause\n\nx = 1\n" + # A fixture with a license file of the user's own, in LICENSE.md, and a + # file under another license, which is declared only if asked for + function tangled(dir; declared) + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + own = replace(read(joinpath(dir, "LICENSE"), String), + "Ann B Smith" => "The Original Holder") + mv(joinpath(dir, "LICENSE"), joinpath(dir, "LICENSE.md")) + write(joinpath(dir, "LICENSE.md"), own) + write(joinpath(dir, "src", "other.jl"), other) + declared && declare_additional(dir, ["BSD-3-Clause"]) + return own + end + additional(dir) = get(TOML.parsefile(joinpath(dir, "Project.toml"))["rsmd"], + "additional_licenses", nothing) + + # As it stands, relicensing respects what the package has declared ... + mktempdir() do dir + own = tangled(dir, declared = true) + @test isnothing(relicense!("MIT", dir)) + cd(git_dir) # crosswalk leaves the working directory changed + @test read(joinpath(dir, "LICENSE.md"), String) == own + @test isnothing(relicense!("BSD-2-Clause", dir)) + cd(git_dir) + @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") + @test names(dir) == ["LICENSE"] + @test read(joinpath(dir, "src", "other.jl"), String) == other + @test additional(dir) == ["BSD-3-Clause"] + end + # ... and stops at what it has not + mktempdir() do dir + tangled(dir, declared = false) + before = fixture_files(dir) + @test_throws "src/other.jl: BSD-3-Clause" relicense!("BSD-2-Clause", + dir) + cd(git_dir) + @test fixture_files(dir) == before + end + + # Overwriting everything does neither + for declared in (true, false) + mktempdir() do dir + tangled(dir, declared = declared) + @test isnothing(relicense!("BSD-2-Clause", dir, + overwrite_all = true)) + cd(git_dir) + @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") + @test names(dir) == ["LICENSE"] + @test occursin("Ann B Smith", + read(joinpath(dir, "LICENSE"), String)) + @test read(joinpath(dir, "src", "other.jl"), String) == relicensed + @test isnothing(additional(dir)) + # The package is consistent afterwards + before = fixture_files(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test fixture_files(dir) == before + end + end + # ... even to the license the package has, which hands over a license + # file of the user's own to be kept up to date + mktempdir() do dir + tangled(dir, declared = true) + @test isnothing(relicense!("MIT", dir, overwrite_all = true)) + cd(git_dir) + @test names(dir) == ["LICENSE"] + @test occursin("Ann B Smith", read(joinpath(dir, "LICENSE"), String)) + end + + # A license that SPDX does not have changes nothing, whichever way + for overwrite_all in (false, true) + mktempdir() do dir + tangled(dir, declared = true) + before = fixture_files(dir) + @test_throws "not a recognised SPDX" relicense!("nonsense", dir, + overwrite_all = overwrite_all) + cd(git_dir) + @test fixture_files(dir) == before + end + end + + # A file this package wrote is not lost where the file system ignores case + mktempdir() do dir + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + # In two steps, as the names are one file where case is ignored + mv(joinpath(dir, "LICENSE"), joinpath(dir, "renaming")) + mv(joinpath(dir, "renaming"), joinpath(dir, "License")) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test names(dir) == ["LICENSE"] + @test startswith(read(joinpath(dir, "LICENSE"), String), "MIT License") + end +end + @testset "Failed crosswalk leaves files unchanged" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir From 7bff5a6d8c67e6e41d33ad36ee3276ce6b640496 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 17:12:38 +0100 Subject: [PATCH 10/18] Fix URLs in codemeta.json generated from remote repositories --- codemeta.json | 4 +- src/crosswalk.jl | 44 +++++++++++++------ src/utils.jl | 23 ++++++++++ test/runtests.jl | 111 +++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 163 insertions(+), 19 deletions(-) diff --git a/codemeta.json b/codemeta.json index 4751654..ecb1809 100644 --- a/codemeta.json +++ b/codemeta.json @@ -7,8 +7,8 @@ "codeRepository": "https://github.com/boydorr/ResearchSoftwareMetadata.jl", "name": "ResearchSoftwareMetadata.jl", "issueTracker": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/issues", - "readme": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/main/README.md", - "buildInstructions": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/main/README.md", + "readme": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/HEAD/README.md", + "buildInstructions": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/HEAD/README.md", "dateCreated": "2024-07-17", "operatingSystem": [ "Linux", diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 84ba2c4..4737852 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -102,13 +102,13 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, tag = isempty(tags) ? proj_version : maximum(VersionNumber.(tags)) tag_date = isempty(tags) ? now : readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$tag`) - branch = readchomp(`$(Git.git()) branch --show-current`) - remotes = split(readchomp(`$(Git.git()) remote`), '\n') - urls = String[] - for remote in remotes - push!(urls, - replace(readchomp(`$(Git.git()) remote get-url $remote`), - r"\.git" => "")) + remotes = readlines(`$(Git.git()) remote`) + isempty(remotes) && + error("The repository at $git_dir has no git remote. The metadata " * + "records where the repository is, so add one with " * + "`git remote add origin
`") + urls = map(remotes) do remote + return repository_url(readchomp(`$(Git.git()) remote get-url $remote`)) end repos = replace.(urls, r"^.*/([^/]+)$" => s"\1") @@ -142,15 +142,18 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, update = update)) && @warn "No description metadata, add `description` to [rsmd] in Project.toml" - repo_index = "origin" ∈ remotes ? - repo_index = findfirst(==("origin"), remotes) : 1 + repo_index = something(findfirst(==("origin"), remotes), 1) if haskey(codemeta, "codeRepository") cm_url = codemeta["codeRepository"] - if cm_url ∉ urls - @error "codemeta has wrong repo URL – $cm_url not in $urls – using $(remotes[repo_index])" + found = findfirst(==(repository_url(cm_url)), urls) + if isnothing(found) + @error "codemeta has wrong repo URL - $cm_url not in $urls - using $(remotes[repo_index])" else - repo_index = findfirst(==(cm_url), urls) + repo_index = found + cm_url == urls[found] || + @info "Writing the repository in codemeta.json as " * + "$(urls[found]), not $cm_url" end end codemeta["codeRepository"] = urls[repo_index] @@ -158,14 +161,27 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, if haskey(codemeta, "name") cm_name = codemeta["name"] if cm_name ≠ repos[repo_index] - @error "codemeta has wrong repo repo name – $cm_name not $(repos[repo_index]) – fixing" + @error "codemeta has wrong repo name - $cm_name not $(repos[repo_index]) - fixing" end end codemeta["name"] = repos[repo_index] codemeta["issueTracker"] = urls[repo_index] * "/issues" - readme = urls[repo_index] * "/blob/" * branch * "/README.md" + # The README on the default branch: a hosting site reads `HEAD` as that + # branch, whatever it is called and whichever branch is checked out. A link + # of the same form that names a branch is brought into line, and any other + # link is left + readme = urls[repo_index] * "/blob/HEAD/README.md" + for key in ("readme", "buildInstructions") + link = get(codemeta, key, nothing) + link isa AbstractString && link ≠ readme && + startswith(link, urls[repo_index] * "/blob/") && + endswith(link, "/README.md") || continue + @info "Moving $key in codemeta.json from $link to $readme, which " * + "follows the default branch" + codemeta[key] = readme + end cm_readme = get!(codemeta, "readme", readme) cm_readme == readme || @info "README set to $cm_readme, not $readme" diff --git a/src/utils.jl b/src/utils.jl index 82fe378..79d6b88 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -26,6 +26,29 @@ function parse_author(author::AbstractString) return String(m.captures[1]), String(m.captures[2]) end +""" + ResearchSoftwareMetadata.repository_url(remote::AbstractString) + +Return the web address of a repository from the address of one of its git +remotes, in whichever form git accepts it: `https://github.com/org/Pkg.jl.git`, +`git@github.com:org/Pkg.jl.git` and `ssh://git@github.com/org/Pkg.jl.git` +all give `https://github.com/org/Pkg.jl`. A user name or password in the +address is left out, as is the port of an `ssh` or `git` address. An address +in any other form, such as a path, is returned as it is. +""" +function repository_url(remote::AbstractString) + address = replace(replace(strip(remote), r"/+$" => ""), r"\.git$" => "") + web = match(r"^(https?)://(?:[^@/]*@)?(.+)$", address) + isnothing(web) || return web[1] * "://" * web[2] + other = match(r"^(?:ssh|git)://(?:[^@/]*@)?([^/:]+)(?::\d+)?/(.+)$", + address) + # The form of scp, `user@host:path`, which has no scheme + isnothing(other) && (other = match(r"^[^@/:]+@([^/:]+):/*(.+)$", address)) + isnothing(other) || return "https://" * other[1] * "/" * other[2] + + return String(address) +end + """ ResearchSoftwareMetadata.read_json(file::AbstractString) diff --git a/test/runtests.jl b/test/runtests.jl index 1d80960..95717ba 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -319,7 +319,8 @@ end end function make_fixture(dir; license = "MIT", extra = "", author_details = true, - workflows = true) + workflows = true, + remote = "https://github.com/example/RSMDFixture.jl") # A fixture with no license has neither the Project.toml entry nor a header license_entry = isnothing(license) ? "" : "license = \"$license\"\n" header = isnothing(license) ? "" : "# SPDX-License-Identifier: $license\n\n" @@ -365,8 +366,7 @@ function make_fixture(dir; license = "MIT", extra = "", author_details = true, end end run(`$(Git.git()) -C $dir init -q -b main`) - run(`$(Git.git()) -C $dir remote add origin - https://github.com/example/RSMDFixture.jl`) + isnothing(remote) || run(`$(Git.git()) -C $dir remote add origin $remote`) run(`$(Git.git()) -C $dir add -A`) run(`$(Git.git()) -C $dir -c user.name=Test -c user.email=test@example.com commit -q -m Fixture`) @@ -964,6 +964,111 @@ end end end +@testset "Repository addresses" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + web = "https://github.com/example/RSMDFixture.jl" + + @testset "Remotes" begin + url = ResearchSoftwareMetadata.repository_url + @test url("https://github.com/o/r.git") == "https://github.com/o/r" + @test url("https://github.com/o/r/") == "https://github.com/o/r" + # Only an ending is taken off, not every ".git" in the address + @test url("https://github.com/o/r.github.io.git") == + "https://github.com/o/r.github.io" + @test url("git@github.com:o/r.jl.git") == "https://github.com/o/r.jl" + @test url("ssh://git@github.com/o/r.git") == "https://github.com/o/r" + @test url("ssh://git@host:2222/o/r.git") == "https://host/o/r" + @test url("git://github.com/o/r.git") == "https://github.com/o/r" + # A password or token does not go into the metadata, a port does + @test url("https://user:token@github.com/o/r.git") == + "https://github.com/o/r" + @test url("https://host:8443/o/r.git") == "https://host:8443/o/r" + @test url("/some/local/path") == "/some/local/path" + end + + # A clone made over ssh still gives web addresses throughout + mktempdir() do dir + make_fixture(dir, remote = "git@github.com:example/RSMDFixture.jl.git") + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["codeRepository"] == web + @test codemeta["name"] == "RSMDFixture.jl" + @test codemeta["issueTracker"] == web * "/issues" + @test codemeta["readme"] == web * "/blob/HEAD/README.md" + @test startswith(codemeta["downloadUrl"], web * "/archive/") + zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) + @test zenodo["related_identifiers"][1]["identifier"] == web + end + + # The address of the remote written as git has it is the same repository + mktempdir() do dir + make_fixture(dir) + write(joinpath(dir, "codemeta.json"), + "{\n \"codeRepository\": " * + "\"git@github.com:example/RSMDFixture.jl\"\n}\n") + @test_logs (:info, r"Writing the repository in codemeta.json as") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["codeRepository"] == web + end + + # Without a remote there is no address to record + mktempdir() do dir + make_fixture(dir, remote = nothing) + before = fixture_files(dir) + @test_throws "has no git remote" ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + @test fixture_files(dir) == before + end +end + +@testset "Links to the default branch" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + web = "https://github.com/example/RSMDFixture.jl" + # HEAD is the default branch of the repository, whatever it is called + readme = web * "/blob/HEAD/README.md" + + # A crosswalk on another branch does not link to that branch + mktempdir() do dir + make_fixture(dir) + run(`$(Git.git()) -C $dir checkout -q -b feature`) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) + cd(git_dir) # crosswalk leaves the working directory changed + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["readme"] == readme + @test codemeta["buildInstructions"] == readme + end + + # Links of that form which name a branch are brought into line ... + links(readme, build) = "{\n \"readme\": \"$readme\",\n " * + "\"buildInstructions\": \"$build\"\n}\n" + mktempdir() do dir + make_fixture(dir) + write(joinpath(dir, "codemeta.json"), + links(web * "/blob/rr/feature/README.md", + web * "/blob/main/README.md")) + @test_logs (:info, r"Moving readme in codemeta.json") (:info, + r"Moving buildInstructions") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) + cd(git_dir) + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["readme"] == readme + @test codemeta["buildInstructions"] == readme + end + # ... and any others are kept + mktempdir() do dir + make_fixture(dir) + elsewhere = "https://example.org/RSMDFixture/" + install = web * "/blob/feature/INSTALL.md" + write(joinpath(dir, "codemeta.json"), links(elsewhere, install)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + @test codemeta["readme"] == elsewhere + @test codemeta["buildInstructions"] == install + end +end + @testset "Source file headers" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) header = "# SPDX-License-Identifier: MIT" From 6bf2c74dc7b064a350ae0d8852262e0992f55513 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 17:30:38 +0100 Subject: [PATCH 11/18] Fix tags crashing crosswalk when they don't refer to a release --- src/crosswalk.jl | 10 +++++---- src/remotequeries.jl | 17 +++++++++----- src/utils.jl | 17 ++++++++++++++ test/runtests.jl | 53 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 4737852..bf0dbdd 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -98,10 +98,12 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, now = string(today()) cd(git_dir) init = readchomp(`$(Git.git()) log --max-parents=0 --format=%ad --date=short -n 1`) - tags = readlines(`$(Git.git()) tag -l --sort="version:refname"`) - tag = isempty(tags) ? proj_version : maximum(VersionNumber.(tags)) - tag_date = isempty(tags) ? now : - readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$tag`) + releases = release_tags(git_dir) + latest = isempty(releases) ? nothing : + argmax(release -> release.version, releases) + tag = isnothing(latest) ? proj_version : latest.version + tag_date = isnothing(latest) ? now : + readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$(latest.name)`) remotes = readlines(`$(Git.git()) remote`) isempty(remotes) && error("The repository at $git_dir has no git remote. The metadata " * diff --git a/src/remotequeries.jl b/src/remotequeries.jl index 4d6d676..8740b1a 100644 --- a/src/remotequeries.jl +++ b/src/remotequeries.jl @@ -123,8 +123,9 @@ end Returns the first release date of this package on Julia's `General` Registry, or today's date if the package has not been registered yet. -Throws an error if the registry cannot be reached or returns an -unexpected HTTP status. +The date is that of the repository's tag for the release. Throws an error +if the repository has no such tag, or if the registry cannot be reached or +returns an unexpected HTTP status. """ function get_first_release_date(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) project = read_project(git_dir) @@ -136,9 +137,15 @@ function get_first_release_date(git_dir = readchomp(`$(Git.git()) rev-parse --sh if response.status == 200 data = TOML.parse(String(response.body)) version = minimum(VersionNumber.(keys(data))) - cd(git_dir) - date = readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/v$version`) - return date + releases = release_tags(git_dir) + release = findfirst(release -> release.version == version, releases) + isnothing(release) && + error("The repository has no tag for v$version, the first " * + "release of $package in Julia's General registry, which " * + "is where the date of that release is taken from. Fetch " * + "the tags of the repository: a shallow clone has none") + tag = releases[release].name + return readchomp(`$(Git.git()) -C $git_dir log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$tag`) elseif response.status == 404 @info "No release yet on General, imputing first release will be today" return string(today()) diff --git a/src/utils.jl b/src/utils.jl index 79d6b88..e2a5491 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -49,6 +49,23 @@ function repository_url(remote::AbstractString) return String(address) end +""" + ResearchSoftwareMetadata.release_tags(git_dir::AbstractString) + +Return the release tags of the repository at `git_dir` as `(version, name)` +named tuples: the tags that are a `v` followed by a version number, which +is how TagBot and Registrator tag a release. Any other tag is not a release +of the package and is left out, including a version number without the +`v`, which a repository may use for something else. +""" +function release_tags(git_dir::AbstractString) + names = readlines(`$(Git.git()) -C $git_dir tag -l`) + releases = [(version = tryparse(VersionNumber, name), name = name) + for name in names if startswith(name, "v")] + + return filter!(release -> !isnothing(release.version), releases) +end + """ ResearchSoftwareMetadata.read_json(file::AbstractString) diff --git a/test/runtests.jl b/test/runtests.jl index 95717ba..02a739f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1023,6 +1023,59 @@ end end end +@testset "Release tags" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + tag(dir, name) = run(`$(Git.git()) -C $dir tag $name`) + + # Only a v and a version number is a release of the package + mktempdir() do dir + make_fixture(dir) + @test isempty(ResearchSoftwareMetadata.release_tags(dir)) + foreach(name -> tag(dir, name), + ["v0.1.0", "v0.2", "docs-preview", "2024.1", "vnext", + "Sub-v1.0.0"]) + releases = ResearchSoftwareMetadata.release_tags(dir) + @test Set(releases) == Set([(version = v"0.1.0", name = "v0.1.0"), + (version = v"0.2.0", name = "v0.2")]) + end + + # Other tags do not stop the crosswalk, or pass for a later release than + # the one in Project.toml + mktempdir() do dir + make_fixture(dir) + foreach(name -> tag(dir, name), ["v0.1.0", "docs-preview", "2024.1"]) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) # crosswalk leaves the working directory changed + @test TOML.parsefile(joinpath(dir, "Project.toml"))["version"] == + "0.1.0" + @test JSON.parsefile(joinpath(dir, "codemeta.json"))["version"] == + "v0.1.0" + end + + # A release tag is looked up under the name it has + mktempdir() do dir + make_fixture(dir) + tag(dir, "v0.1") + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test JSON.parsefile(joinpath(dir, "codemeta.json"))["version"] == + "v0.1.0" + end + + # A registered package whose first release has no tag in the repository, + # as in a shallow clone + mktempdir() do dir + make_fixture(dir) + toml = joinpath(dir, "Project.toml") + project = TOML.parsefile(toml) + project["name"] = "Example" + open(toml, "w") do io + return TOML.print(io, project) + end + @test_throws "The repository has no tag for v" ResearchSoftwareMetadata.get_first_release_date(dir) + end +end + @testset "Links to the default branch" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) web = "https://github.com/example/RSMDFixture.jl" From 3344437e206385f0c5c9202eb47e0567e0fc4ea1 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 18:17:16 +0100 Subject: [PATCH 12/18] Make functionality public --- NEWS.md | 2 +- Project.toml | 2 ++ src/ResearchSoftwareMetadata.jl | 5 +++++ test/runtests.jl | 17 +++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index fd277b4..dabd129 100644 --- a/NEWS.md +++ b/NEWS.md @@ -13,7 +13,7 @@ - Handle missing ORCID and ROR records without crashing - Fail on connection errors and unexpected HTTP statuses, writing no files so everything is left in its original state - Add optional top-level Project.toml keys description, keywords, category, development_status and publications as the source for codemeta.json and .zenodo.json, backfilling them from codemeta.json when absent - - Split the source into separate files and export crosswalk, increase_patch, increase_minor, increase_major + - Split the source into separate files - Construct a missing author_details section in Project.toml from the authors field and codemeta.json or .zenodo.json when consistent with authors - v0.1.7 - Bump compat and fix workflows diff --git a/Project.toml b/Project.toml index 738f3fc..15b6dfa 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.3.1" authors = ["Richard Reeve "] [deps] +Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" @@ -15,6 +16,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" [compat] +Compat = "4.10" DataStructures = "0.17,0.18,0.19" Dates = "1.6" Git = "1" diff --git a/src/ResearchSoftwareMetadata.jl b/src/ResearchSoftwareMetadata.jl index 5a3ca57..1676bb0 100644 --- a/src/ResearchSoftwareMetadata.jl +++ b/src/ResearchSoftwareMetadata.jl @@ -29,6 +29,11 @@ using DataStructures using HTTP using LicenseCheck using YAML +using Compat: @compat + +# The supported entry points, which are called by their qualified names +@compat public crosswalk, relicense!, increase_patch, increase_minor, + increase_major include("project.jl") include("remotequeries.jl") diff --git a/test/runtests.jl b/test/runtests.jl index 02a739f..3a2b869 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,6 +24,23 @@ using .GitUtils @test is_repo_clean(git_dir) end +@testset "Public names" begin + entry_points = (:crosswalk, :relicense!, :increase_patch, :increase_minor, + :increase_major) + @test all(name -> isdefined(ResearchSoftwareMetadata, name), entry_points) + # Nothing is exported, so every call is by a qualified name + @test names(ResearchSoftwareMetadata, all = false) ⊆ + [:ResearchSoftwareMetadata, entry_points...] + @test !any(name -> Base.isexported(ResearchSoftwareMetadata, name), + entry_points) + # Names can only be marked as public from Julia 1.11 + if isdefined(Base, :ispublic) + @test all(name -> Base.ispublic(ResearchSoftwareMetadata, name), + entry_points) + @test !Base.ispublic(ResearchSoftwareMetadata, :read_project) + end +end + @testset "Failed metadata lookups" begin @test isnothing(ResearchSoftwareMetadata.get_person_from_orcid("0000-0000-0000-0000")) @test isnothing(ResearchSoftwareMetadata.get_organisation_from_ror("invalid")) From 75b0728c7a660e76ae1f72ff45e109270a1d9360 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 18:35:54 +0100 Subject: [PATCH 13/18] Small fixes to docs, comments and workflows --- .github/workflows/testing.yaml | 1 - README.md | 2 +- docs/make.jl | 16 ++++----------- docs/metadata.jl | 5 +---- docs/src/index.md | 2 +- docs/src/testing.md | 27 +++++++++++++++----------- src/project.jl | 8 ++++---- src/utils.jl | 2 +- test/GitUtils.jl | 8 ++++++-- test/clean_JuliaFormatter.jl | 2 +- test/clean_ResearchSoftwareMetadata.jl | 2 +- test/runtests.jl | 20 +++++++++++++++++++ 12 files changed, 56 insertions(+), 39 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index fb661eb..9ce2a16 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -84,7 +84,6 @@ jobs: shell: julia --project=docs --color=yes {0} run: | using Pkg - Pkg.develop(PackageSpec(path=pwd())) Pkg.instantiate() - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-docdeploy@v1 diff --git a/README.md b/README.md index 68a6c2b..8bda97c 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Then, from the root of your package, you can just run a crosswalk: using Pkg # Create a new project with ResearchSoftwareMetadata in it -Pkg.activate(; temp = true) +Pkg.activate(temp = true) Pkg.add("ResearchSoftwareMetadata") # Carry out a crosswalk between the different metadata formats diff --git a/docs/make.jl b/docs/make.jl index 89554a9..d8d7552 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,27 +1,19 @@ # SPDX-License-Identifier: MIT -using Pkg -"ResearchSoftwareMetadata" ∈ [p.name for p in values(Pkg.dependencies())] && - Pkg.rm("ResearchSoftwareMetadata") -Pkg.develop(url = "https://github.com/boydorr/ResearchSoftwareMetadata.jl.git") - using ResearchSoftwareMetadata using Documenter DocMeta.setdocmeta!(ResearchSoftwareMetadata, :DocTestSetup, - :(using ResearchSoftwareMetadata); recursive = true) + :(using ResearchSoftwareMetadata), recursive = true) -makedocs(; - modules = [ResearchSoftwareMetadata], +makedocs(modules = [ResearchSoftwareMetadata], authors = "Richard Reeve ", sitename = "ResearchSoftwareMetadata.jl", - format = Documenter.HTML(; - canonical = "https://boydorr.github.io/ResearchSoftwareMetadata.jl", + format = Documenter.HTML(canonical = "https://boydorr.github.io/ResearchSoftwareMetadata.jl", edit_link = "main", assets = String[]), pages = ["Home" => "index.md", "Automated package checks" => "testing.md"]) -deploydocs(; - repo = "github.com/boydorr/ResearchSoftwareMetadata.jl", +deploydocs(repo = "github.com/boydorr/ResearchSoftwareMetadata.jl", devbranch = "main") diff --git a/docs/metadata.jl b/docs/metadata.jl index 9277fb5..a66a65c 100644 --- a/docs/metadata.jl +++ b/docs/metadata.jl @@ -2,7 +2,7 @@ using Pkg -# Update Phylo folder packages +# Update the package's own dependencies Pkg.activate(".") Pkg.update() @@ -21,9 +21,6 @@ end # Update docs folder packages Pkg.activate("docs") Pkg.update() -"ResearchSoftwareMetadata" ∈ [p.name for p in values(Pkg.dependencies())] && - Pkg.rm("ResearchSoftwareMetadata") -Pkg.develop(path = ".") # Reformat files in package using JuliaFormatter diff --git a/docs/src/index.md b/docs/src/index.md index 36283f1..593a939 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -77,7 +77,7 @@ Then, from the root of your package, you can just run a crosswalk: using Pkg # Create a new project with ResearchSoftwareMetadata in it -Pkg.activate(; temp = true) +Pkg.activate(temp = true) Pkg.add("ResearchSoftwareMetadata") # Carry out a crosswalk between the different metadata formats diff --git a/docs/src/testing.md b/docs/src/testing.md index df90ab2..e15df29 100644 --- a/docs/src/testing.md +++ b/docs/src/testing.md @@ -19,8 +19,7 @@ into your own package's `test/` directory: - [`GitUtils.jl`](https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/main/test/GitUtils.jl) — provides `is_repo_clean`, which the other two files use to check whether the checks - changed anything. It needs no adaptation (though you may want to change the SPDX header - to your own package's licence identifier). + changed anything. - [`clean_ResearchSoftwareMetadata.jl`](https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/main/test/clean_ResearchSoftwareMetadata.jl) — checks the metadata crosswalk is clean. - [`clean_JuliaFormatter.jl`](https://github.com/boydorr/ResearchSoftwareMetadata.jl/blob/main/test/clean_JuliaFormatter.jl) @@ -29,10 +28,18 @@ into your own package's `test/` directory: You can use either or both of the `clean_*.jl` files; the test-suite code below discovers whichever ones are present. -In each `clean_*.jl` file, replace `ResearchSoftwareMetadata` in the `using` line with -your own package where it stands for the package under test, and set the SPDX header to -your package's licence (otherwise the crosswalk will rewrite it and the check will fail). -For a package called `MyPackage` with an MIT licence, the two files look like this: +None of the three files needs adapting apart from its first line. Each starts with +`# SPDX-License-Identifier: MIT`, which is this package's licence, and the crosswalk stops +with an error if it finds a file marked with a licence that your package has not declared. +So either change that line in each file to your own package's licence, or keep the files +under the MIT licence and say so in your `Project.toml`: + +```toml +[rsmd] +additional_licenses = ["MIT"] +``` + +The two `clean_*.jl` files are: ```julia # SPDX-License-Identifier: MIT @@ -41,7 +48,6 @@ module CleanRSMD using Test using Git using Logging -using MyPackage using ResearchSoftwareMetadata include("GitUtils.jl") @@ -55,7 +61,7 @@ if !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" global_logger(SimpleLogger(stderr, Logging.Warn)) @test_nowarn ResearchSoftwareMetadata.crosswalk() global_logger(SimpleLogger(stderr, Logging.Info)) - @test is_repo_clean(git_dir; strict = haskey(ENV, "RUNNER_OS")) + @test is_repo_clean(git_dir, strict = haskey(ENV, "RUNNER_OS")) end else @test_broken !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" @@ -71,7 +77,6 @@ module CleanJuliaFormatter using Test using Git using JuliaFormatter -using MyPackage include("GitUtils.jl") using .GitUtils @@ -80,8 +85,8 @@ using .GitUtils if !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" @testset "JuliaFormatter" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) - @test_nowarn format(MyPackage) - @test is_repo_clean(git_dir; strict = haskey(ENV, "RUNNER_OS")) + @test_nowarn format(git_dir) + @test is_repo_clean(git_dir, strict = haskey(ENV, "RUNNER_OS")) end else @test_broken !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" diff --git a/src/project.jl b/src/project.jl index e5703b7..fd1f8c3 100644 --- a/src/project.jl +++ b/src/project.jl @@ -22,13 +22,13 @@ end ResearchSoftwareMetadata.order_project(project_d::AbstractDict) Take a parsed `Project.toml` dictionary and return it as an OrderedDict -in canonical order — the order Pkg itself writes, applied recursively to -nested tables — so that files written from it are not reordered when Pkg -next edits them. +in canonical order, which is the order Pkg itself writes, applied +recursively to nested tables, so that files written from it are not +reordered when Pkg next edits them. """ function order_project(project_d::AbstractDict) project = OrderedDict{String, Any}() - for key in sort!(collect(keys(project_d)); by = project_key_order) + for key in sort!(collect(keys(project_d)), by = project_key_order) project[key] = order_value(project_d[key]) end diff --git a/src/utils.jl b/src/utils.jl index e2a5491..404f17f 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -95,7 +95,7 @@ Reconcile a metadata field between `Project.toml` (authoritative) and `codemeta.json`. An explicit `value` (e.g. from a keyword argument to `crosswalk`) takes precedence and is written into both; otherwise the `Project.toml` entry is used, fixing `codemeta.json` with a warning if it -disagrees — or an informational message when `update` is true, meaning +disagrees, or with an informational message when `update` is true, meaning the `Project.toml` change is deliberate and should just propagate. If the field is missing from `Project.toml` but present in `codemeta.json`, it is backfilled into `Project.toml`. If it is absent from both, `default` diff --git a/test/GitUtils.jl b/test/GitUtils.jl index b4e88af..5708989 100644 --- a/test/GitUtils.jl +++ b/test/GitUtils.jl @@ -12,7 +12,7 @@ export is_repo_clean Determine whether the git repository at `repo_path` is clean, reporting on its state. The short-format status (`git status -s`) of the repository is split into three -categories using the two status columns of each entry — the first column records +categories using the two status columns of each entry: the first column records changes staged in the index, the second records changes in the working tree: - **staged**: tracked files with changes recorded in the index; @@ -38,7 +38,11 @@ otherwise. function is_repo_clean(repo_path; strict = false) # Short-format porcelain status: each entry is "XY path", where column X is the # index (staged) status and column Y is the working-tree status. - statuses = readlines(`$(Git.git()) status -s $repo_path`) + # Precomposed unicode has to be off: with it on, as it is by default on macOS, + # the git that Git.jl supplies fails to scan a working tree that has any file + # whose name is not ASCII. + statuses = readlines(`$(Git.git()) -c core.precomposeunicode=false + -C $repo_path status -s`) untracked = filter(s -> startswith(s, "??"), statuses) unstaged = filter(s -> !startswith(s, "??") && s[2] != ' ', statuses) diff --git a/test/clean_JuliaFormatter.jl b/test/clean_JuliaFormatter.jl index dc5334d..ea2771f 100644 --- a/test/clean_JuliaFormatter.jl +++ b/test/clean_JuliaFormatter.jl @@ -13,7 +13,7 @@ if !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" @testset "JuliaFormatter" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) @test_nowarn format(git_dir) - @test is_repo_clean(git_dir; strict = haskey(ENV, "RUNNER_OS")) + @test is_repo_clean(git_dir, strict = haskey(ENV, "RUNNER_OS")) end else @test_broken !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" diff --git a/test/clean_ResearchSoftwareMetadata.jl b/test/clean_ResearchSoftwareMetadata.jl index 550197e..aebade0 100644 --- a/test/clean_ResearchSoftwareMetadata.jl +++ b/test/clean_ResearchSoftwareMetadata.jl @@ -17,7 +17,7 @@ if !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" global_logger(SimpleLogger(stderr, Logging.Warn)) @test_nowarn ResearchSoftwareMetadata.crosswalk() global_logger(SimpleLogger(stderr, Logging.Info)) - @test is_repo_clean(git_dir; strict = haskey(ENV, "RUNNER_OS")) + @test is_repo_clean(git_dir, strict = haskey(ENV, "RUNNER_OS")) end else @test_broken !haskey(ENV, "RUNNER_OS") || ENV["RUNNER_OS"] ≠ "Windows" diff --git a/test/runtests.jl b/test/runtests.jl index 3a2b869..a5dcd20 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -41,6 +41,26 @@ end end end +@testset "Checks for other packages to copy" begin + # The two files are printed in the documentation as they are. This compares + # the listings only, not the instructions around them + page = read(joinpath(@__DIR__, "..", "docs", "src", "testing.md"), String) + for name in ("clean_ResearchSoftwareMetadata.jl", "clean_JuliaFormatter.jl") + file = read(joinpath(@__DIR__, name), String) + @test occursin("```julia\n" * file * "```", page) + end + + # The state of a repository can be read whatever its files are called, and + # from any working directory + mktempdir() do dir + run(`$(Git.git()) -C $dir init -q -b main`) + @test is_repo_clean(dir) + write(joinpath(dir, "data for δ.csv"), "a,b\n") + @test is_repo_clean(dir) + @test !is_repo_clean(dir, strict = true) + end +end + @testset "Failed metadata lookups" begin @test isnothing(ResearchSoftwareMetadata.get_person_from_orcid("0000-0000-0000-0000")) @test isnothing(ResearchSoftwareMetadata.get_organisation_from_ror("invalid")) From 5491089e1b54ff737fbcb8e361f4aea2ab31870f Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 19:21:15 +0100 Subject: [PATCH 14/18] You no longer need to run crosswalk() twice on first use, some added docs and info messages --- README.md | 2 ++ docs/src/index.md | 2 ++ src/crosswalk.jl | 9 +++-- src/utils.jl | 5 ++- test/runtests.jl | 85 +++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 97 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8bda97c..ffbdf95 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ license = "BSD-2-Clause" A package has to have a license: the crosswalk stops with an error if it finds none, or one that is not an identifier from that list. You can also set the license when you run the crosswalk, as shown below. +If the package has no license file, the crosswalk writes a `LICENSE` for that license, naming the people in the `authors` field of `Project.toml` as the copyright holders, so make sure `authors` is right first. A license file you already have is left exactly as it is, provided it holds that license. + The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described next, so that the crosswalk knows they are intended and leaves them alone: ```toml diff --git a/docs/src/index.md b/docs/src/index.md index 593a939..9ce7ee2 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -38,6 +38,8 @@ license = "BSD-2-Clause" A package has to have a license: the crosswalk stops with an error if it finds none, or one that is not an identifier from that list. You can also set the license when you run the crosswalk, as shown below. +If the package has no license file, the crosswalk writes a `LICENSE` for that license, naming the people in the `authors` field of `Project.toml` as the copyright holders, so make sure `authors` is right first. A license file you already have is left exactly as it is, provided it holds that license. + The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described next, so that the crosswalk knows they are intended and leaves them alone: ```toml diff --git a/src/crosswalk.jl b/src/crosswalk.jl index bf0dbdd..8db3d6a 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -342,14 +342,17 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, end end else - authors = project["authors"] - for author in authors + for author in project["authors"] name, email = parse_author(author) detail = OrderedDict{String, Any}("name" => name) isnothing(email) || (detail["email"] = email) push!(author_data, detail) + push!(authors, isnothing(email) ? name : name * " <" * email * ">") end rsmd["author_details"] = author_data + @info "Created author_details in the [rsmd] table of Project.toml " * + "from authors, please add each author's ORCID and ROR " * + "affiliation there if you can" end # Add authors listed in `authors` but missing from author_details @@ -451,7 +454,7 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, "from https://spdx.org/licenses/") end - just_names = replace.(project["authors"], r" *<[^>]+> *" => "") + just_names = first.(parse_author.(project["authors"])) license_content = license_text(spdx["licenseText"], years, just_names) # A license file left by an earlier run names the authors there were then, diff --git a/src/utils.jl b/src/utils.jl index 404f17f..edb7ddd 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -99,7 +99,8 @@ disagrees, or with an informational message when `update` is true, meaning the `Project.toml` change is deliberate and should just propagate. If the field is missing from `Project.toml` but present in `codemeta.json`, it is backfilled into `Project.toml`. If it is absent from both, `default` -is used for `codemeta.json` (when provided) without being backfilled. +(when provided) is written into both, so that the value in use can be +seen and changed in `Project.toml`. `to_cm` and `from_cm` convert values between the `Project.toml` and `codemeta.json` representations. Returns the `Project.toml`-side value, or `nothing` if the field is absent everywhere. @@ -128,6 +129,8 @@ function reconcile!(project, codemeta, proj_key, cm_key; codemeta[cm_key] = to_cm(val) return val elseif !isnothing(default) + @info "Setting $proj_key to its default in Project.toml" + project[proj_key] = default codemeta[cm_key] = to_cm(default) return default end diff --git a/test/runtests.jl b/test/runtests.jl index a5dcd20..6a1a1ee 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -632,13 +632,94 @@ end @test project["rsmd"]["description"] == "A fixture package" @test project["rsmd"]["keywords"] == ["fixture", "metadata"] @test project["rsmd"]["category"] == "metadata" - # Defaults are not backfilled into Project.toml - @test !haskey(project["rsmd"], "development_status") + # A default is written into Project.toml too, where it can be seen + @test project["rsmd"]["development_status"] == "active" codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["developmentStatus"] == "active" end end +@testset "A first crosswalk settles everything" begin + git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) + # Starting from each of these, a second crosswalk has nothing left to do + starts = ["author_details given" => dir -> make_fixture(dir), + "authors alone" => dir -> make_fixture(dir, author_details = false), + "metadata in codemeta.json alone" => function (dir) + make_fixture(dir, author_details = false) + return write(joinpath(dir, "codemeta.json"), + """ + { + "description": "A fixture package", + "keywords": ["zeta", "alpha"], + "applicationCategory": "metadata" + } + """) + end] + @testset "$label" for (label, start) in starts + mktempdir() do dir + start(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, + build = true)) + cd(git_dir) # crosswalk leaves the working directory changed + settled = fixture_files(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test fixture_files(dir) == settled + end + end + + # An entry in authors that is not well formed is put right at once, and + # does not reach the license + mktempdir() do dir + make_fixture(dir, author_details = false) + toml = joinpath(dir, "Project.toml") + project = TOML.parsefile(toml) + project["authors"] = [ + "Ann B Smith " + ] + open(toml, "w") do io + return TOML.print(io, project) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) + cd(git_dir) + settled = fixture_files(dir) + project = TOML.parsefile(toml) + @test project["authors"] == + ["Ann B Smith ", "bob "] + @test occursin("Ann B Smith and bob\n", settled["LICENSE"]) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + @test fixture_files(dir) == settled + end + + # The run that makes author_details from authors asks for what authors + # cannot hold, and no later run does + mktempdir() do dir + make_fixture(dir, author_details = false) + asks(log) = occursin("add each author's ORCID and ROR", + string(log.message)) + for expected in (1, 0) + logger = Test.TestLogger(min_level = Logging.Info) + with_logger(logger) do + return ResearchSoftwareMetadata.crosswalk(dir) + end + cd(git_dir) + @test count(asks, logger.logs) == expected + end + end + + # The defaults in use are written into Project.toml on the first run + mktempdir() do dir + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + cd(git_dir) + rsmd = TOML.parsefile(joinpath(dir, "Project.toml"))["rsmd"] + @test rsmd["keywords"] == ["julia"] + @test rsmd["development_status"] == "active" + end +end + @testset "Propagate Project.toml changes with update" begin git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) extra = """ From 1916c34c3a089bff0d8a7256e10f2ee5e5c3b4ed Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 19:47:23 +0100 Subject: [PATCH 15/18] Stop changing directories --- src/crosswalk.jl | 10 +++--- test/runtests.jl | 92 ++++++------------------------------------------ 2 files changed, 15 insertions(+), 87 deletions(-) diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 8db3d6a..d11f5f6 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -96,21 +96,21 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, proj_version = VersionNumber(project["version"]) now = string(today()) - cd(git_dir) - init = readchomp(`$(Git.git()) log --max-parents=0 --format=%ad --date=short -n 1`) + git = `$(Git.git()) -C $git_dir` + init = readchomp(`$git log --max-parents=0 --format=%ad --date=short -n 1`) releases = release_tags(git_dir) latest = isempty(releases) ? nothing : argmax(release -> release.version, releases) tag = isnothing(latest) ? proj_version : latest.version tag_date = isnothing(latest) ? now : - readchomp(`$(Git.git()) log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$(latest.name)`) - remotes = readlines(`$(Git.git()) remote`) + readchomp(`$git log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$(latest.name)`) + remotes = readlines(`$git remote`) isempty(remotes) && error("The repository at $git_dir has no git remote. The metadata " * "records where the repository is, so add one with " * "`git remote add origin
`") urls = map(remotes) do remote - return repository_url(readchomp(`$(Git.git()) remote get-url $remote`)) + return repository_url(readchomp(`$git remote get-url $remote`)) end repos = replace.(urls, r"^.*/([^/]+)$" => s"\1") diff --git a/test/runtests.jl b/test/runtests.jl index 6a1a1ee..eaf94ce 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -42,13 +42,17 @@ end end @testset "Checks for other packages to copy" begin - # The two files are printed in the documentation as they are. This compares - # the listings only, not the instructions around them + # The two files, and the block at the end of this one that runs them, are + # printed in the documentation as they are. This compares the listings + # only, not the instructions around them page = read(joinpath(@__DIR__, "..", "docs", "src", "testing.md"), String) for name in ("clean_ResearchSoftwareMetadata.jl", "clean_JuliaFormatter.jl") file = read(joinpath(@__DIR__, name), String) @test occursin("```julia\n" * file * "```", page) end + suite = read(@__FILE__, String) + start = findlast("rsmd = get(ENV, \"RSMD_CROSSWALK\", \"FALSE\")", suite) + @test occursin("```julia\n" * suite[first(start):end] * "```", page) # The state of a repository can be read whatever its files are called, and # from any working directory @@ -444,11 +448,12 @@ function all_licensed(license) end @testset "Crosswalk without ORCIDs" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir make_fixture(dir) + # A crosswalk of another repository leaves the working directory alone + here = pwd() @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed + @test pwd() == here codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test length(codemeta["author"]) == 1 author = codemeta["author"][1] @@ -462,7 +467,6 @@ end end @testset "Project.toml as metadata source" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) doi = "10.5281/zenodo.12789179" extra = """ description = "A fixture package" @@ -474,7 +478,6 @@ end mktempdir() do dir make_fixture(dir, extra = extra) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["description"] == "A fixture package" @test codemeta["keywords"] == ["fixture", "metadata"] @@ -492,7 +495,6 @@ end end @testset "Reconstruct author_details" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) # From codemeta.json mktempdir() do dir make_fixture(dir, author_details = false) @@ -518,7 +520,6 @@ end """) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed project = TOML.parsefile(joinpath(dir, "Project.toml")) @test haskey(project["rsmd"], "author_details") detail = project["rsmd"]["author_details"][1] @@ -545,7 +546,6 @@ end """) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed project = TOML.parsefile(joinpath(dir, "Project.toml")) detail = project["rsmd"]["author_details"][1] @test detail["name"] == "Ann B Smith" @@ -570,7 +570,6 @@ end """) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed project = TOML.parsefile(joinpath(dir, "Project.toml")) @test project["rsmd"]["author_details"] == [Dict("name" => "Ann B Smith", "email" => "ann@example.com")] @@ -581,7 +580,6 @@ end end @testset "Add new author from authors" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir make_fixture(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) @@ -594,7 +592,6 @@ end return TOML.print(io, project) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed project = TOML.parsefile(toml) @test project["authors"] == ["Ann B Smith ", "Bob Jones "] @@ -613,7 +610,6 @@ end end @testset "Backfill Project.toml from codemeta.json" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir make_fixture(dir) open(joinpath(dir, "codemeta.json"), "w") do io @@ -627,7 +623,6 @@ end """) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed project = TOML.parsefile(joinpath(dir, "Project.toml")) @test project["rsmd"]["description"] == "A fixture package" @test project["rsmd"]["keywords"] == ["fixture", "metadata"] @@ -640,7 +635,6 @@ end end @testset "A first crosswalk settles everything" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) # Starting from each of these, a second crosswalk has nothing left to do starts = ["author_details given" => dir -> make_fixture(dir), "authors alone" => dir -> make_fixture(dir, author_details = false), @@ -660,10 +654,8 @@ end start(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) - cd(git_dir) # crosswalk leaves the working directory changed settled = fixture_files(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test fixture_files(dir) == settled end end @@ -682,14 +674,12 @@ end return TOML.print(io, project) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) - cd(git_dir) settled = fixture_files(dir) project = TOML.parsefile(toml) @test project["authors"] == ["Ann B Smith ", "bob "] @test occursin("Ann B Smith and bob\n", settled["LICENSE"]) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test fixture_files(dir) == settled end @@ -704,7 +694,6 @@ end with_logger(logger) do return ResearchSoftwareMetadata.crosswalk(dir) end - cd(git_dir) @test count(asks, logger.logs) == expected end end @@ -713,7 +702,6 @@ end mktempdir() do dir make_fixture(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) rsmd = TOML.parsefile(joinpath(dir, "Project.toml"))["rsmd"] @test rsmd["keywords"] == ["julia"] @test rsmd["development_status"] == "active" @@ -721,7 +709,6 @@ end end @testset "Propagate Project.toml changes with update" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) extra = """ description = "A fixture package" category = "metadata" @@ -730,7 +717,6 @@ end mktempdir() do dir make_fixture(dir, extra = extra) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) - cd(git_dir) # crosswalk leaves the working directory changed toml = joinpath(dir, "Project.toml") project = TOML.parsefile(toml) project["license"] = "BSD-2-Clause" @@ -739,7 +725,6 @@ end return TOML.print(io, project) end @test_nowarn ResearchSoftwareMetadata.crosswalk(dir, update = true) - cd(git_dir) codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["license"] == "https://spdx.org/licenses/BSD-2-Clause" @test codemeta["description"] == "An updated fixture package" @@ -755,7 +740,6 @@ end mktempdir() do dir make_fixture(dir, extra = extra) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) - cd(git_dir) # crosswalk leaves the working directory changed toml = joinpath(dir, "Project.toml") project = TOML.parsefile(toml) project["license"] = "BSD-2-Clause" @@ -764,13 +748,11 @@ end end before = fixture_files(dir) @test_throws "License mismatch" ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) @test fixture_files(dir) == before end end @testset "Declaring the license" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) spdx = "https://spdx.org/licenses/MIT" codemeta_license(license) = "{\n \"license\": \"$license\"\n}\n" @@ -788,7 +770,6 @@ end write(joinpath(dir, "codemeta.json"), codemeta_license(written)) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed @test fixture_licenses(dir) == all_licensed("MIT") end end @@ -799,7 +780,6 @@ end write(joinpath(dir, "codemeta.json"), codemeta_license("MIT")) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test fixture_licenses(dir) == all_licensed("MIT") end @@ -813,7 +793,6 @@ end catch err sprint(showerror, err) end - cd(git_dir) @test all(occursin(message), ["No license found", "crosswalk(license = \"", "https://spdx.org/licenses/", "FSF", "OSI", @@ -826,7 +805,6 @@ end make_fixture(dir, license = "mit") before = fixture_files(dir) @test_throws "`mit` is not a recognised SPDX license identifier" ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) @test fixture_files(dir) == before end @@ -835,13 +813,11 @@ end make_fixture(dir, license = nothing) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, license = "MIT")) - cd(git_dir) @test fixture_licenses(dir) == all_licensed("MIT") @test startswith(read(joinpath(dir, "LICENSE"), String), "MIT License") # ... and changes one that is already there, without update @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, license = "BSD-2-Clause")) - cd(git_dir) @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") @test occursin("Redistribution", read(joinpath(dir, "LICENSE"), String)) @@ -849,20 +825,17 @@ end before = fixture_files(dir) @test_throws "not a recognised SPDX" ResearchSoftwareMetadata.crosswalk(dir, license = "nonsense") - cd(git_dir) @test fixture_files(dir) == before end end @testset "License files" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) names(dir) = basename.(ResearchSoftwareMetadata.license_files(dir)) # The texts this package writes, from which to make other license files generated = Dict(map(("MIT", "BSD-2-Clause", "GPL-3.0-or-later")) do license return mktempdir() do dir make_fixture(dir, license = license) ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) # crosswalk changes the directory return license => read(joinpath(dir, "LICENSE"), String) end @@ -912,7 +885,6 @@ end write(joinpath(dir, name), text) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed @test read(joinpath(dir, name), String) == text @test names(dir) == [name] end @@ -925,12 +897,10 @@ end write(joinpath(dir, "COPYING"), text) before = fixture_files(dir) @test_throws "COPYING does not hold the MIT license" ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) @test fixture_files(dir) == before # ... unless the change is deliberate, when LICENSE takes its place @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, update = true)) - cd(git_dir) @test names(dir) == ["LICENSE"] @test startswith(read(joinpath(dir, "LICENSE"), String), "MIT License") @@ -948,7 +918,6 @@ end return TOML.print(io, project) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test occursin("Ann B Smith and Bob Jones", read(joinpath(dir, "LICENSE"), String)) end @@ -961,7 +930,6 @@ end @test_throws "LICENSE.md holds the MIT license, which `crosswalk(license = \"MIT\")`" ResearchSoftwareMetadata.crosswalk(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, license = "MIT")) - cd(git_dir) @test read(joinpath(dir, "LICENSE.md"), String) == quoted @test names(dir) == ["LICENSE.md"] end @@ -977,11 +945,9 @@ end end @testset "Crosswalk without workflows" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir make_fixture(dir, workflows = false) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test !haskey(codemeta, "operatingSystem") end @@ -997,14 +963,12 @@ end """) end @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["operatingSystem"] == ["Linux", "macOS"] end end @testset "Crosswalk finds the CI workflow" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) repo = "https://github.com/example/RSMDFixture.jl" # The workflow that runs the tests, whatever it is called mktempdir() do dir @@ -1015,7 +979,6 @@ end one_job("runs-on: windows-latest, $RUNTEST")) write(joinpath(folder, first(TAGBOT)), last(TAGBOT)) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["continuousIntegration"] == repo * "/actions/workflows/CI.yml" @@ -1030,14 +993,12 @@ end mkpath(folder) write(joinpath(folder, first(TAGBOT)), last(TAGBOT)) @test_logs (:warn, r"CI not found") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test !haskey(codemeta, "continuousIntegration") end end @testset "Metadata files that cannot be read" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) # A missing comma between two entries of a list broken = """ { @@ -1064,7 +1025,6 @@ end make_fixture(dir, author_details = author_details) write(joinpath(dir, ".zenodo.json"), broken) @test_logs lost match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) # crosswalk leaves the working directory changed zenodo = JSON.parsefile(joinpath(dir, ".zenodo.json")) @test [c["name"] for c in zenodo["creators"]] == ["Smith, Ann B"] end @@ -1074,7 +1034,6 @@ end project_content, src_content = make_fixture(dir) write(joinpath(dir, "codemeta.json"), broken) @test_throws "Unable to read codemeta.json" ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) # crosswalk leaves the working directory changed @test read(joinpath(dir, "codemeta.json"), String) == broken @test read(joinpath(dir, "Project.toml"), String) == project_content @test !isfile(joinpath(dir, ".zenodo.json")) @@ -1083,7 +1042,6 @@ end end @testset "Repository addresses" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) web = "https://github.com/example/RSMDFixture.jl" @testset "Remotes" begin @@ -1108,7 +1066,6 @@ end mktempdir() do dir make_fixture(dir, remote = "git@github.com:example/RSMDFixture.jl.git") @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["codeRepository"] == web @test codemeta["name"] == "RSMDFixture.jl" @@ -1126,7 +1083,6 @@ end "{\n \"codeRepository\": " * "\"git@github.com:example/RSMDFixture.jl\"\n}\n") @test_logs (:info, r"Writing the repository in codemeta.json as") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["codeRepository"] == web end @@ -1136,13 +1092,11 @@ end make_fixture(dir, remote = nothing) before = fixture_files(dir) @test_throws "has no git remote" ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) @test fixture_files(dir) == before end end @testset "Release tags" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) tag(dir, name) = run(`$(Git.git()) -C $dir tag $name`) # Only a v and a version number is a release of the package @@ -1163,7 +1117,6 @@ end make_fixture(dir) foreach(name -> tag(dir, name), ["v0.1.0", "docs-preview", "2024.1"]) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed @test TOML.parsefile(joinpath(dir, "Project.toml"))["version"] == "0.1.0" @test JSON.parsefile(joinpath(dir, "codemeta.json"))["version"] == @@ -1175,7 +1128,6 @@ end make_fixture(dir) tag(dir, "v0.1") @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test JSON.parsefile(joinpath(dir, "codemeta.json"))["version"] == "v0.1.0" end @@ -1195,7 +1147,6 @@ end end @testset "Links to the default branch" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) web = "https://github.com/example/RSMDFixture.jl" # HEAD is the default branch of the repository, whatever it is called readme = web * "/blob/HEAD/README.md" @@ -1205,7 +1156,6 @@ end make_fixture(dir) run(`$(Git.git()) -C $dir checkout -q -b feature`) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) - cd(git_dir) # crosswalk leaves the working directory changed codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["readme"] == readme @test codemeta["buildInstructions"] == readme @@ -1221,7 +1171,6 @@ end web * "/blob/main/README.md")) @test_logs (:info, r"Moving readme in codemeta.json") (:info, r"Moving buildInstructions") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["readme"] == readme @test codemeta["buildInstructions"] == readme @@ -1233,7 +1182,6 @@ end install = web * "/blob/feature/INSTALL.md" write(joinpath(dir, "codemeta.json"), links(elsewhere, install)) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) @test codemeta["readme"] == elsewhere @test codemeta["buildInstructions"] == install @@ -1241,7 +1189,6 @@ end end @testset "Source file headers" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) header = "# SPDX-License-Identifier: MIT" mktempdir() do dir make_fixture(dir) @@ -1273,7 +1220,6 @@ end # A file with its header, a notebook and an ignored file are not rewritten @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # crosswalk leaves the working directory changed @test contents() == merge(before, after) @test isempty(ResearchSoftwareMetadata.header_changes(dir, "MIT")) end @@ -1297,7 +1243,6 @@ function declare_additional(dir, licenses) end @testset "Files under another license" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) changes = ResearchSoftwareMetadata.header_changes other = "# SPDX-License-Identifier: BSD-3-Clause\n\nx = 1\n" either = "# SPDX-License-Identifier: MIT OR BSD-3-Clause\n\nx = 1\n" @@ -1331,13 +1276,11 @@ end @test_throws "additional_licenses" ResearchSoftwareMetadata.crosswalk(dir) @test_throws "src/other.jl: BSD-3-Clause" ResearchSoftwareMetadata.crosswalk(dir, update = true) - cd(git_dir) # crosswalk leaves the working directory changed @test fixture_files(dir) == before # Once the license is declared the file is left as it is ... declare_additional(dir, ["BSD-3-Clause"]) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test read(file, String) == other @test fixture_licenses(dir) == all_licensed("MIT") project = TOML.parsefile(joinpath(dir, "Project.toml")) @@ -1346,7 +1289,6 @@ end # that were under the package's license follow it @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, license = "BSD-2-Clause")) - cd(git_dir) @test read(file, String) == other @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") end @@ -1357,13 +1299,11 @@ end declare_additional(dir, ["nonsense"]) before = fixture_files(dir) @test_throws "`nonsense` in additional_licenses is not a recognised" ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) @test fixture_files(dir) == before end end @testset "Relicensing" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) relicense! = ResearchSoftwareMetadata.relicense! names(dir) = basename.(ResearchSoftwareMetadata.license_files(dir)) other = "# SPDX-License-Identifier: BSD-3-Clause\n\nx = 1\n" @@ -1388,10 +1328,8 @@ end mktempdir() do dir own = tangled(dir, declared = true) @test isnothing(relicense!("MIT", dir)) - cd(git_dir) # crosswalk leaves the working directory changed @test read(joinpath(dir, "LICENSE.md"), String) == own @test isnothing(relicense!("BSD-2-Clause", dir)) - cd(git_dir) @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") @test names(dir) == ["LICENSE"] @test read(joinpath(dir, "src", "other.jl"), String) == other @@ -1403,7 +1341,6 @@ end before = fixture_files(dir) @test_throws "src/other.jl: BSD-3-Clause" relicense!("BSD-2-Clause", dir) - cd(git_dir) @test fixture_files(dir) == before end @@ -1413,7 +1350,6 @@ end tangled(dir, declared = declared) @test isnothing(relicense!("BSD-2-Clause", dir, overwrite_all = true)) - cd(git_dir) @test fixture_licenses(dir) == all_licensed("BSD-2-Clause") @test names(dir) == ["LICENSE"] @test occursin("Ann B Smith", @@ -1423,7 +1359,6 @@ end # The package is consistent afterwards before = fixture_files(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test fixture_files(dir) == before end end @@ -1432,7 +1367,6 @@ end mktempdir() do dir tangled(dir, declared = true) @test isnothing(relicense!("MIT", dir, overwrite_all = true)) - cd(git_dir) @test names(dir) == ["LICENSE"] @test occursin("Ann B Smith", read(joinpath(dir, "LICENSE"), String)) end @@ -1444,7 +1378,6 @@ end before = fixture_files(dir) @test_throws "not a recognised SPDX" relicense!("nonsense", dir, overwrite_all = overwrite_all) - cd(git_dir) @test fixture_files(dir) == before end end @@ -1453,25 +1386,21 @@ end mktempdir() do dir make_fixture(dir) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) # In two steps, as the names are one file where case is ignored mv(joinpath(dir, "LICENSE"), joinpath(dir, "renaming")) mv(joinpath(dir, "renaming"), joinpath(dir, "License")) @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) - cd(git_dir) @test names(dir) == ["LICENSE"] @test startswith(read(joinpath(dir, "LICENSE"), String), "MIT License") end end @testset "Failed crosswalk leaves files unchanged" begin - git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`) mktempdir() do dir # An invalid SPDX identifier makes the license lookup fail project_content, src_content = make_fixture(dir, license = "Not-A-License") @test_throws ErrorException ResearchSoftwareMetadata.crosswalk(dir) - cd(git_dir) # crosswalk leaves the working directory changed @test read(joinpath(dir, "Project.toml"), String) == project_content @test read(joinpath(dir, "src", "RSMDFixture.jl"), String) == src_content @@ -1485,8 +1414,7 @@ rsmd = get(ENV, "RSMD_CROSSWALK", "FALSE") if rsmd == "TRUE" || !haskey(ENV, "RUNNER_OS") # Crosswalk runner or local testing # Test RSMD crosswalk and other hygiene issues - # Identify files that are checking package hygiene; use @__DIR__ because - # crosswalk() in earlier testsets leaves the working directory changed + # Identify files that are checking package hygiene cleanbase = map(file -> replace(file, r"clean_(.*).jl$" => s"\1"), filter(str -> occursin(r"^clean_.*\.jl$", str), readdir(@__DIR__))) From c1626b3c80a35c7b00254726333f5c6a67adafd1 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 20:33:20 +0100 Subject: [PATCH 16/18] Don't update dates on branches unless forced --- docs/src/testing.md | 6 +++ src/crosswalk.jl | 36 ++++++++++++-- src/remotequeries.jl | 10 ++-- src/utils.jl | 51 +++++++++++++++++++ test/runtests.jl | 115 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 10 deletions(-) diff --git a/docs/src/testing.md b/docs/src/testing.md index e15df29..7cb7bbb 100644 --- a/docs/src/testing.md +++ b/docs/src/testing.md @@ -158,6 +158,12 @@ Some things to be aware of: repository must be checked out in full (`fetch-depth: 0` on `actions/checkout`). - Windows runners are skipped by both checks (marked as broken) due to file writing issues. +- Until a version has a release tag, its dates in `codemeta.json` are provisional. On + your default branch every crosswalk sets them to today's date (in UTC), so there the + check fails on any later day until the crosswalk is run again; that is what keeps the + dates right in the code you release. On any other branch, and so in a pull request, the + dates are left as they are, and `ResearchSoftwareMetadata.crosswalk(update = true)` or + one of the `increase_` functions brings them up to date. - On CI the repository must be *strictly* clean — no staged, unstaged or untracked changes after the checks run. Locally the criterion is relaxed: only unstaged changes fail the tests, so work you have already staged doesn't stop you running the suite. diff --git a/src/crosswalk.jl b/src/crosswalk.jl index d11f5f6..056ab6f 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -34,6 +34,9 @@ Every julia source file is given a first line naming the license. A file that is under a different license keeps its own first line, provided that license is listed under `additional_licenses` in the `[rsmd]` table; a file marked with any other license is an error, and nothing is changed. +The dates of a version that has no release tag yet are today's date, in UTC, whenever +the crosswalk is run on the repository's default branch or with `update` true; on any +other branch the dates already recorded for that version are kept. If any remote metadata query (orcid.org, ror.org, spdx.org, doi.org or Julia's General registry) cannot be completed, the crosswalk throws an error and all files are left in their original state. @@ -95,14 +98,14 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, rsmd = get!(project, "rsmd", OrderedDict{String, Any}()) proj_version = VersionNumber(project["version"]) - now = string(today()) + now = utc_today() git = `$(Git.git()) -C $git_dir` init = readchomp(`$git log --max-parents=0 --format=%ad --date=short -n 1`) releases = release_tags(git_dir) latest = isempty(releases) ? nothing : argmax(release -> release.version, releases) tag = isnothing(latest) ? proj_version : latest.version - tag_date = isnothing(latest) ? now : + tag_date = isnothing(latest) ? nothing : readchomp(`$git log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$(latest.name)`) remotes = readlines(`$git remote`) isempty(remotes) && @@ -222,6 +225,22 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, cm_version = VersionNumber(get!(codemeta, "version", string(proj_version))) + # A version with no release tag has no date of its own, so the dates given + # to it are provisional. They are brought up to today on the default + # branch, which releases are made from, and by a deliberate run. On any + # other branch the ones already written for this version are kept, so that + # they do not change with the day the crosswalk is run + unreleased = isnothing(latest) || proj_version > tag + keep_dates = unreleased && !update && cm_version == proj_version && + haskey(codemeta, "dateModified") && + !on_default_branch(git_dir, remotes[repo_index]) + keep_dates && + @info "Keeping the dates of this unreleased version, which are " * + "brought up to date on the default branch or by " * + "`crosswalk(update = true)`" + provisional_date = keep_dates ? codemeta["dateModified"] : now + isnothing(tag_date) && (tag_date = provisional_date) + if proj_version == tag @debug "Still on latest release version: $tag" codemeta["dateModified"] = tag_date @@ -242,12 +261,12 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, end elseif proj_version > tag @info "Preparing for new release" - codemeta["dateModified"] = now + codemeta["dateModified"] = provisional_date if cm_version ≠ proj_version @info "Updating codemeta tag version ($cm_version) to " * "new release ($proj_version)" cm_version = proj_version - this_year = string(year(Date(now))) + this_year = string(year(Date(provisional_date))) if this_year ≠ years years = years * "-" * this_year end @@ -274,8 +293,15 @@ function run_crosswalk(git_dir::AbstractString; category, keywords, codemeta["datePublished"] == first_release_date || @warn "codemeta.json publication date inconsistent with Julia's General registry, fixing ($(codemeta["datePublished"]) ≠ $first_release_date)" end - codemeta["datePublished"] = first_release_date + elseif !isempty(releases) + # Not registered, but released: the earliest release tag has the date + first_release = argmin(release -> release.version, releases) + first_release_date = readchomp(`$git log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$(first_release.name)`) + else + first_release_date = keep_dates ? + get(codemeta, "datePublished", now) : now end + codemeta["datePublished"] = first_release_date project["version"] = string(proj_version) codemeta["version"] = "v$cm_version" diff --git a/src/remotequeries.jl b/src/remotequeries.jl index 8740b1a..5b98cf4 100644 --- a/src/remotequeries.jl +++ b/src/remotequeries.jl @@ -122,9 +122,9 @@ end ResearchSoftwareMetadata.get_first_release_date() Returns the first release date of this package on Julia's `General` -Registry, or today's date if the package has not been registered yet. -The date is that of the repository's tag for the release. Throws an error -if the repository has no such tag, or if the registry cannot be reached or +Registry, or `nothing` if the package has not been registered. The date is +that of the repository's tag for the release. Throws an error if the +repository has no such tag, or if the registry cannot be reached or returns an unexpected HTTP status. """ function get_first_release_date(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) @@ -147,8 +147,8 @@ function get_first_release_date(git_dir = readchomp(`$(Git.git()) rev-parse --sh tag = releases[release].name return readchomp(`$(Git.git()) -C $git_dir log -1 --format=%ad --date=format:%Y-%m-%d refs/tags/$tag`) elseif response.status == 404 - @info "No release yet on General, imputing first release will be today" - return string(today()) + @info "No release yet on General" + return nothing end return error("Unable to query Julia's General registry for $package, " * diff --git a/src/utils.jl b/src/utils.jl index edb7ddd..47f9f30 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -66,6 +66,57 @@ function release_tags(git_dir::AbstractString) return filter!(release -> !isnothing(release.version), releases) end +""" + ResearchSoftwareMetadata.utc_today() + +Return today's date in UTC, as text. A date written into the metadata is +then the same wherever the crosswalk is run: on a GitHub runner, whose +clock is in UTC, as on a machine in any other time zone. +""" +utc_today() = string(Date(now(UTC))) + +""" + ResearchSoftwareMetadata.default_branch(git_dir::AbstractString, + remote::AbstractString) + +Return the name of the default branch of a remote of the repository at +`git_dir`, or `nothing` if it cannot be found. The remote itself is asked, +which is the only sure answer. If it cannot be reached, the default branch +that the repository recorded when it was cloned is used, if there is one. +""" +function default_branch(git_dir::AbstractString, remote::AbstractString) + # Asking must fail, not wait for a password or a passphrase to be typed + git = addenv(`$(Git.git()) -C $git_dir`, "GIT_TERMINAL_PROMPT" => "0", + "GIT_SSH_COMMAND" => "ssh -o BatchMode=yes -o ConnectTimeout=10") + answer = read(pipeline(ignorestatus(`$git ls-remote --symref $remote HEAD`), + stderr = devnull), String) + asked = match(r"^ref: refs/heads/(\S+)\s+HEAD$"m, answer) + isnothing(asked) || return String(asked[1]) + recorded = readchomp(pipeline(ignorestatus(`$git symbolic-ref --quiet --short + refs/remotes/$remote/HEAD`), + stderr = devnull)) + + return isempty(recorded) ? nothing : + String(chopprefix(recorded, "$remote/")) +end + +""" + ResearchSoftwareMetadata.on_default_branch(git_dir::AbstractString, + remote::AbstractString) + +Check whether the branch checked out in the repository at `git_dir` is the +default branch of `remote`, which is the branch releases are made from. It +counts as being so if the default branch cannot be found, and as not being +so if no branch is checked out, as when a pull request is being tested. +""" +function on_default_branch(git_dir::AbstractString, remote::AbstractString) + default = default_branch(git_dir, remote) + isnothing(default) && return true + + return readchomp(`$(Git.git()) -C $git_dir branch --show-current`) == + default +end + """ ResearchSoftwareMetadata.read_json(file::AbstractString) diff --git a/test/runtests.jl b/test/runtests.jl index eaf94ce..090af93 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1146,6 +1146,121 @@ end end end +@testset "Dates before a release" begin + today = ResearchSoftwareMetadata.utc_today() + @test occursin(r"^\d{4}-\d{2}-\d{2}$", today) + git(dir, arguments...) = run(`$(Git.git()) -C $dir $arguments`) + stale = "2001-02-03" + # The dates as codemeta.json has them + function dates(dir) + codemeta = JSON.parsefile(joinpath(dir, "codemeta.json")) + return (modified = codemeta["dateModified"], + published = codemeta["datePublished"]) + end + # Make the dates in codemeta.json those of a crosswalk on an earlier day + function age(dir) + file = joinpath(dir, "codemeta.json") + aged = replace(read(file, String), + r"(\"date(Modified|Published)\": \")[^\"]*" => + SubstitutionString("\\g<1>" * stale)) + return write(file, aged) + end + # A fixture whose remote is a repository beside it, with main as its default + function with_remote(test) + return mktempdir() do parent + remote = joinpath(parent, "remote.git") + dir = mkdir(joinpath(parent, "fixture")) + run(`$(Git.git()) init -q --bare -b main $remote`) + make_fixture(dir, remote = remote) + git(dir, "push", "-q", "origin", "main") + return test(dir) + end + end + + @testset "The default branch" begin + with_remote() do dir + @test ResearchSoftwareMetadata.default_branch(dir, "origin") == + "main" + @test ResearchSoftwareMetadata.on_default_branch(dir, "origin") + git(dir, "checkout", "-q", "-b", "feature") + @test !ResearchSoftwareMetadata.on_default_branch(dir, "origin") + # No branch is checked out when a pull request is tested + git(dir, "checkout", "-q", "--detach") + @test !ResearchSoftwareMetadata.on_default_branch(dir, "origin") + end + # A remote that cannot be asked, and nothing recorded for it + mktempdir() do dir + make_fixture(dir) + @test isnothing(ResearchSoftwareMetadata.default_branch(dir, + "origin")) + @test ResearchSoftwareMetadata.on_default_branch(dir, "origin") + # What was recorded when the repository was cloned is used + git(dir, "symbolic-ref", "refs/remotes/origin/HEAD", + "refs/remotes/origin/dev") + @test ResearchSoftwareMetadata.default_branch(dir, "origin") == + "dev" + @test !ResearchSoftwareMetadata.on_default_branch(dir, "origin") + end + end + + # No release tag: on the default branch the dates are today's every time + with_remote() do dir + @test isnothing(ResearchSoftwareMetadata.get_first_release_date(dir)) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) + @test dates(dir) == (modified = today, published = today) + age(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test dates(dir) == (modified = today, published = today) + + # On another branch they are kept, until a deliberate run + git(dir, "checkout", "-q", "-b", "feature") + age(dir) + @test_logs (:info, r"Keeping the dates of this unreleased version") match_mode=:any ResearchSoftwareMetadata.crosswalk(dir) + @test dates(dir) == (modified = stale, published = stale) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, update = true)) + @test dates(dir) == (modified = today, published = today) + + # Dates written for another version are not this version's to keep + age(dir) + toml = joinpath(dir, "Project.toml") + project = TOML.parsefile(toml) + project["version"] = "0.2.0" + open(toml, "w") do io + return TOML.print(io, project) + end + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test dates(dir) == (modified = today, published = today) + end + + # A version bumped since the last release tag is in the same position, and + # a package that is released but not registered was published with its tag + with_remote() do dir + git(dir, "tag", "v0.1.0") + released = readchomp(`$(Git.git()) -C $dir log -1 --format=%ad + --date=short`) + git(dir, "checkout", "-q", "-b", "feature") + @test isnothing(ResearchSoftwareMetadata.increase_minor(dir)) + @test dates(dir) == (modified = today, published = released) + age(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test dates(dir) == (modified = stale, published = released) + git(dir, "branch", "-q", "-M", "main") + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test dates(dir) == (modified = today, published = released) + end + + # A released version has the date of its tag, whatever the branch or the day + with_remote() do dir + git(dir, "tag", "v0.1.0") + released = readchomp(`$(Git.git()) -C $dir log -1 --format=%ad + --date=short`) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir, build = true)) + age(dir) + @test isnothing(ResearchSoftwareMetadata.crosswalk(dir)) + @test dates(dir) == (modified = released, published = released) + end +end + @testset "Links to the default branch" begin web = "https://github.com/example/RSMDFixture.jl" # HEAD is the default branch of the repository, whatever it is called From 4b0ab7d19ea26222d451913a8ecc59da7fe77011 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 20:43:47 +0100 Subject: [PATCH 17/18] Make ready to release --- NEWS.md | 11 +++++++++++ Project.toml | 2 +- codemeta.json | 6 +++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index dabd129..b6b6d62 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,16 @@ # NEWS +- v0.4.0 + - Breaking changes + - Licensing now changes only when asked: a license in Project.toml that disagrees with codemeta.json, a license file holding another license, or a source file marked with an undeclared license stops the crosswalk and changes nothing, where these were logged, partly applied or rewritten. Use `update = true`, the new `license` argument or `relicense!()` to change a license, and `additional_licenses` in [rsmd] for files deliberately under another one + - Existing metadata changes once on the next crosswalk: README links become blob/HEAD so that they follow the default branch, macOS runners are reported as macOS, and default keywords and development status are written into [rsmd] + - `crosswalk()` no longer changes the working directory + - Add `relicense!()` to move a whole package to a new license, and a clear error suggesting licenses when a package has none or one that is not an SPDX identifier + - Leave alone a license file the crosswalk did not write, provided it holds the declared license (uses LicenseCheck.jl) + - Read operating systems and the CI workflow from the jobs that run the tests, including reusable workflows + - Cope with SSH remotes, tags that are not releases, empty source files and unreadable metadata files + - Settle everything in a single crosswalk on first use; use UTC dates, and keep the dates of an unreleased version on branches other than the default + - Mark the entry points as public (not exported) - v0.3.1 - Dependency fix to avoid updating other packages - v0.3.0 diff --git a/Project.toml b/Project.toml index 15b6dfa..6aadef2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ResearchSoftwareMetadata" uuid = "58378933-4625-47fa-851e-05ee27d397bd" license = "MIT" -version = "0.3.1" +version = "0.4.0" authors = ["Richard Reeve "] [deps] diff --git a/codemeta.json b/codemeta.json index ecb1809..9172db3 100644 --- a/codemeta.json +++ b/codemeta.json @@ -14,10 +14,10 @@ "Linux", "macOS" ], - "version": "v0.3.1", - "dateModified": "2026-07-09", + "version": "v0.4.0", + "dateModified": "2026-10-04", "datePublished": "2024-07-18", - "downloadUrl": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/archive/refs/tags/v0.3.1.tar.gz", + "downloadUrl": "https://github.com/boydorr/ResearchSoftwareMetadata.jl/archive/refs/tags/v0.4.0.tar.gz", "license": "https://spdx.org/licenses/MIT", "author": [ { From e2aa4b11ea28be9ec661e715fe69fe35df4b6297 Mon Sep 17 00:00:00 2001 From: Richard Reeve Date: Sun, 4 Oct 2026 20:57:39 +0100 Subject: [PATCH 18/18] Update documentation and docstrings --- README.md | 51 ++++++++++---------- docs/src/index.md | 69 +++++++++++++++++---------- docs/src/testing.md | 12 +++-- src/crosswalk.jl | 110 +++++++++++++++++++++++++++++++------------- src/versioning.jl | 39 ++++++++++++---- 5 files changed, 186 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index ffbdf95..b2fd098 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ built and installed with `add`. For example: (@v1.12) pkg> add ResearchSoftwareMetadata Resolving package versions... Updating `~/.julia/environments/v1.12/Project.toml` - [58378933] + ResearchSoftwareMetadata v0.3.0 + [58378933] + ResearchSoftwareMetadata v0.4.0 Updating `~/.julia/environments/v1.12/Manifest.toml` (@v1.12) pkg> @@ -29,9 +29,7 @@ built and installed with `add`. For example: ## Usage -First you need to add a small amount of additional metadata into your `Project.toml` file. - -To capture the license you are using and propagate it throughout the metadata files and through your julia code, add an [SPDX license identifier](https://spdx.org/licenses/) to the file: +The crosswalk runs on a package that is a git repository with a remote. The one piece of metadata it cannot do without is a license. To capture the license you are using and propagate it throughout the metadata files and through your julia code, add an [SPDX license identifier](https://spdx.org/licenses/) to your `Project.toml` file: ```toml license = "BSD-2-Clause" @@ -41,7 +39,7 @@ A package has to have a license: the crosswalk stops with an error if it finds n If the package has no license file, the crosswalk writes a `LICENSE` for that license, naming the people in the `authors` field of `Project.toml` as the copyright holders, so make sure `authors` is right first. A license file you already have is left exactly as it is, provided it holds that license. -The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described next, so that the crosswalk knows they are intended and leaves them alone: +The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described below, so that the crosswalk knows they are intended and leaves them alone: ```toml [rsmd] @@ -50,7 +48,23 @@ additional_licenses = ["MIT"] A file marked with a license that is neither the package's nor in that list stops the crosswalk with an error. -All of the other metadata that this package uses lives in a single `[rsmd]` table. To supplement the metadata on the authors of the package, add the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. +Then, from the root of your package, you can just run a crosswalk: + +```julia +using Pkg + +# Create a new project with ResearchSoftwareMetadata in it +Pkg.activate(temp = true) +Pkg.add("ResearchSoftwareMetadata") + +# Carry out a crosswalk between the different metadata formats +using ResearchSoftwareMetadata +ResearchSoftwareMetadata.crosswalk() +``` + +The first crosswalk writes `codemeta.json` and `.zenodo.json`, and creates an `[rsmd]` table in `Project.toml`, which is where all of the other metadata that this package uses lives. It warns about whatever it still needs for a complete record: work through those warnings as described below, running the crosswalk again until there are none. + +The crosswalk makes an `[[rsmd.author_details]]` block for each person in the `authors` field. To supplement the metadata on the authors of the package, add to those blocks the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. ```toml [rsmd] @@ -63,7 +77,7 @@ All of the other metadata that this package uses lives in a single `[rsmd]` tabl ror = "00vtgdb53" ``` -You can also optionally add a `description` of the package, `keywords` associated with it, the software `category` it belongs to, its [repostatus.org](https://www.repostatus.org) `development_status`, and the DOIs of any `publications` associated with the package. All of these are propagated into `codemeta.json` and `.zenodo.json`, and any values already in `codemeta.json` but missing from `Project.toml` will be backfilled into it. Any of these keys found at the top level of `Project.toml` (the old layout) are automatically migrated into `[rsmd]`: +The crosswalk warns until the package has a `description` and the software `category` it belongs to. You can also add `keywords` associated with the package, its [repostatus.org](https://www.repostatus.org) `development_status`, and the DOIs of any `publications` associated with it; if you do not, the keywords start as `["julia"]` and the development status as `"active"`. All of these are propagated into `codemeta.json` and `.zenodo.json`, and any values already in `codemeta.json` but missing from `Project.toml` will be backfilled into it. Any of these keys found at the top level of `Project.toml` (the old layout) are automatically migrated into `[rsmd]`: ```toml [rsmd] @@ -74,23 +88,14 @@ development_status = "active" publications = ["10.5281/zenodo.12789179"] ``` -Then, from the root of your package, you can just run a crosswalk: - -```julia -using Pkg - -# Create a new project with ResearchSoftwareMetadata in it -Pkg.activate(temp = true) -Pkg.add("ResearchSoftwareMetadata") - -# Carry out a crosswalk between the different metadata formats -using ResearchSoftwareMetadata -ResearchSoftwareMetadata.crosswalk() -``` +The crosswalk also warns until it has been told where the build instructions for the package are, which you do once, when you run it: `build = true` if they are in the README, or their address if they are somewhere else. If you want to pass in some additional metadata (the `category` of the software or the `keywords` associated with it, both of which are written back into `[rsmd]` in `Project.toml`, or its `license`), or you want to increase the package version during the crosswalk, you can do that as follows: ```julia +# Say that the build instructions are in the README +ResearchSoftwareMetadata.crosswalk(build = true) + # Add in additional metadata ResearchSoftwareMetadata.crosswalk(category = "metadata", keywords = ["julia", "metadata", "research software", "RSMD"]) @@ -123,14 +128,12 @@ ResearchSoftwareMetadata.relicense!("MIT", overwrite_all = true) The first form is the same as `crosswalk(license = "MIT")`: files listed under `additional_licenses` keep their own license, and a file marked with a license you have not declared still stops it. The second form overrides all of that: it removes `additional_licenses`, replaces every license file with a newly written `LICENSE`, and marks every julia file with the new license. It relabels files whoever wrote them, so make sure you are entitled to relicense them first. -You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. +You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. From the root of your package: ```julia Pkg.add("JuliaFormatter") -Pkg.develop("MyPackage") using JuliaFormatter -using MyPackage -format(MyPackage) +format(".") ``` ## Automated checking diff --git a/docs/src/index.md b/docs/src/index.md index 9ce7ee2..562b870 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -20,7 +20,7 @@ built and installed with `add`. For example: (@v1.12) pkg> add ResearchSoftwareMetadata Resolving package versions... Updating `~/.julia/environments/v1.12/Project.toml` - [58378933] + ResearchSoftwareMetadata v0.3.0 + [58378933] + ResearchSoftwareMetadata v0.4.0 Updating `~/.julia/environments/v1.12/Manifest.toml` (@v1.12) pkg> @@ -28,9 +28,7 @@ built and installed with `add`. For example: ## Usage -First you need to add a small amount of additional metadata into your `Project.toml` file. - -To capture the license you are using and propagate it throughout the metadata files and through your julia code, add an [SPDX license identifier](https://spdx.org/licenses/) to the file: +The crosswalk runs on a package that is a git repository with a remote. The one piece of metadata it cannot do without is a license. To capture the license you are using and propagate it throughout the metadata files and through your julia code, add an [SPDX license identifier](https://spdx.org/licenses/) to your `Project.toml` file: ```toml license = "BSD-2-Clause" @@ -40,7 +38,7 @@ A package has to have a license: the crosswalk stops with an error if it finds n If the package has no license file, the crosswalk writes a `LICENSE` for that license, naming the people in the `authors` field of `Project.toml` as the copyright holders, so make sure `authors` is right first. A license file you already have is left exactly as it is, provided it holds that license. -The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described next, so that the crosswalk knows they are intended and leaves them alone: +The crosswalk records the license on the first line of every julia file, as `# SPDX-License-Identifier: BSD-2-Clause`. If some files are deliberately under a different license, give each of them its own first line naming that license, and list those licenses in the `[rsmd]` table described below, so that the crosswalk knows they are intended and leaves them alone: ```toml [rsmd] @@ -49,7 +47,23 @@ additional_licenses = ["MIT"] A file marked with a license that is neither the package's nor in that list stops the crosswalk with an error. -All of the other metadata that this package uses lives in a single `[rsmd]` table. To supplement the metadata on the authors of the package, add the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. +Then, from the root of your package, you can just run a crosswalk: + +```julia +using Pkg + +# Create a new project with ResearchSoftwareMetadata in it +Pkg.activate(temp = true) +Pkg.add("ResearchSoftwareMetadata") + +# Carry out a crosswalk between the different metadata formats +using ResearchSoftwareMetadata +ResearchSoftwareMetadata.crosswalk() +``` + +The first crosswalk writes `codemeta.json` and `.zenodo.json`, and creates an `[rsmd]` table in `Project.toml`, which is where all of the other metadata that this package uses lives. It warns about whatever it still needs for a complete record: work through those warnings as described below, running the crosswalk again until there are none. + +The crosswalk makes an `[[rsmd.author_details]]` block for each person in the `authors` field. To supplement the metadata on the authors of the package, add to those blocks the [ORCID](https://orcid.org) for each author and the [ROR](https://ror.org) for the organisation(s) they are affiliated with. You can add as many authors and as much or as little information as you like about each one by adding additional `[[rsmd.author_details]]` blocks. ```toml [rsmd] @@ -62,7 +76,7 @@ All of the other metadata that this package uses lives in a single `[rsmd]` tabl ror = "00vtgdb53" ``` -You can also optionally add a `description` of the package, `keywords` associated with it, the software `category` it belongs to, its [repostatus.org](https://www.repostatus.org) `development_status`, and the DOIs of any `publications` associated with the package. All of these are propagated into `codemeta.json` and `.zenodo.json`, and any values already in `codemeta.json` but missing from `Project.toml` will be backfilled into it. Any of these keys found at the top level of `Project.toml` (the old layout) are automatically migrated into `[rsmd]`: +The crosswalk warns until the package has a `description` and the software `category` it belongs to. You can also add `keywords` associated with the package, its [repostatus.org](https://www.repostatus.org) `development_status`, and the DOIs of any `publications` associated with it; if you do not, the keywords start as `["julia"]` and the development status as `"active"`. All of these are propagated into `codemeta.json` and `.zenodo.json`, and any values already in `codemeta.json` but missing from `Project.toml` will be backfilled into it. Any of these keys found at the top level of `Project.toml` (the old layout) are automatically migrated into `[rsmd]`: ```toml [rsmd] @@ -73,24 +87,14 @@ development_status = "active" publications = ["10.5281/zenodo.12789179"] ``` -Then, from the root of your package, you can just run a crosswalk: - -```julia -using Pkg - -# Create a new project with ResearchSoftwareMetadata in it -Pkg.activate(temp = true) -Pkg.add("ResearchSoftwareMetadata") - -# Carry out a crosswalk between the different metadata formats -using ResearchSoftwareMetadata -ResearchSoftwareMetadata.crosswalk() -``` - +The crosswalk also warns until it has been told where the build instructions for the package are, which you do once, when you run it: `build = true` if they are in the README, or their address if they are somewhere else. If you want to pass in some additional metadata (the `category` of the software or the `keywords` associated with it, both of which are written back into `[rsmd]` in `Project.toml`, or its `license`), or you want to increase the package version during the crosswalk, you can do that as follows: ```julia +# Say that the build instructions are in the README +ResearchSoftwareMetadata.crosswalk(build = true) + # Add in additional metadata ResearchSoftwareMetadata.crosswalk(category = "metadata", keywords = ["julia", "metadata", "research software", "RSMD"]) @@ -123,14 +127,12 @@ ResearchSoftwareMetadata.relicense!("MIT", overwrite_all = true) The first form is the same as `crosswalk(license = "MIT")`: files listed under `additional_licenses` keep their own license, and a file marked with a license you have not declared still stops it. The second form overrides all of that: it removes `additional_licenses`, replaces every license file with a newly written `LICENSE`, and marks every julia file with the new license. It relabels files whoever wrote them, so make sure you are entitled to relicense them first. -You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. +You might also consider reformatting all of your julia code to a consistent format. A `.JuliaFormatter.toml` file in the package root defines what the formatting standard should be. From the root of your package: ```julia Pkg.add("JuliaFormatter") -Pkg.develop("MyPackage") using JuliaFormatter -using MyPackage -format(MyPackage) +format(".") ``` ## Automated checking @@ -139,9 +141,26 @@ You can also make your package's own test suite check automatically that the met ## Reference guide +These are the functions to call. None of them is exported, so each is called by its full name, such as `ResearchSoftwareMetadata.crosswalk()`. + +```@docs +ResearchSoftwareMetadata +crosswalk +relicense! +increase_patch +increase_minor +increase_major +``` + +### Internals + +Everything else in the package is internal: it is documented here for those working on the package itself, and may change without notice. + ```@index +Pages = ["index.md"] ``` ```@autodocs Modules = [ResearchSoftwareMetadata] +Public = false ``` diff --git a/docs/src/testing.md b/docs/src/testing.md index 7cb7bbb..ed1ae05 100644 --- a/docs/src/testing.md +++ b/docs/src/testing.md @@ -6,7 +6,9 @@ verify that its metadata and formatting stay clean: that running are committed (so the metadata is consistent and up to date), and that running [JuliaFormatter](https://github.com/domluna/JuliaFormatter.jl) leaves the source code unchanged (so the code is well formatted). If either check would change a file, the test -fails until you re-run the crosswalk or formatter and commit the result. +fails until you re-run the crosswalk or formatter and commit the result. The crosswalk +check also fails if the crosswalk logs a warning, so the package's metadata has to be +complete, with a description, a category and build instructions, as well as consistent. This is how ResearchSoftwareMetadata tests itself, and the test files are written so that they can be copied into any other package. @@ -153,9 +155,11 @@ package's, that looks like: Some things to be aware of: -- The crosswalk queries orcid.org, ror.org, spdx.org and the Julia General registry, so - the runner needs network access, and it interrogates the git history and tags, so the - repository must be checked out in full (`fetch-depth: 0` on `actions/checkout`). +- The crosswalk queries orcid.org, ror.org, spdx.org, doi.org and the Julia General + registry, and for a version that has not been released it asks your git remote which + branch is its default, so the runner needs network access. It also interrogates the git + history and tags, so the repository must be checked out in full (`fetch-depth: 0` on + `actions/checkout`). - Windows runners are skipped by both checks (marked as broken) due to file writing issues. - Until a version has a release tag, its dates in `codemeta.json` are provisional. On diff --git a/src/crosswalk.jl b/src/crosswalk.jl index 056ab6f..ab7fc5f 100644 --- a/src/crosswalk.jl +++ b/src/crosswalk.jl @@ -4,42 +4,72 @@ crosswalk(git_dir; category = nothing, keywords = nothing, license = nothing, build = false, update = false) -Runs a crosswalk across `Project.toml`, `LICENSE`, `codemeta.json` and `.zenodo.json` as -well as the julia source files to enforce consistency between the different metadata formats. -It logs warnings and errors if it identifies inconsistencies while it is editing the files. +Run a crosswalk across `Project.toml`, `codemeta.json`, `.zenodo.json`, the license file +and the julia source files of a package, to enforce consistency between the different +metadata formats, and return `nothing`. + `Project.toml` is the authoritative source of metadata: as well as its standard fields, the optional keys `description`, `keywords`, `category`, `development_status` and `publications` (a vector of DOIs of associated papers) in its `[rsmd]` table are propagated into `codemeta.json` and `.zenodo.json`; values found only in `codemeta.json` are backfilled into `Project.toml`. Legacy top-level copies of these keys are migrated into `[rsmd]`. -A missing `author_details` section is likewise constructed from `codemeta.json` or +The crosswalk logs warnings and errors if it identifies inconsistencies while it is editing +the files. + +# Arguments + + - `git_dir`: the repository holding the package. It defaults to the repository that + the working directory is in. + - `category`: the software category. It takes precedence over the one in `Project.toml` + and is written back into it. + - `keywords`: a vector of keyword strings, which likewise takes precedence and is + written back. + - `license`: an [SPDX identifier](https://spdx.org/licenses/) such as `"MIT"`, to set the + license of the package or to change it. It is written back into `Project.toml`. + - `build`: where the build instructions are, for the `buildInstructions` RSMD field. + `false` leaves the instructions as is, `true` sets it to the same as the README, and a + string sets it to that value. + - `update`: if true, changes to `Project.toml` (e.g. to the version or the license) are + treated as deliberate and propagated to the other metadata files with `@info` messages + instead of being reported as warnings or errors. + +# Authors + +A missing `author_details` section in `[rsmd]` is constructed from `codemeta.json` or `.zenodo.json`, provided the information there is consistent with the definitive `authors` -field. New entries in `authors` are propagated into `rsmd.author_details` (with a warning to -add their ORCID and ROR affiliation there), `codemeta.json` and `.zenodo.json`, while authors -in `codemeta.json` that are missing from `authors` are removed with an error. -The software category can be set with the `category` argument, likewise the `keywords` -argument can contain a vector of keyword strings; both take precedence over and are written -back into `Project.toml`. The `build` argument sets the `buildInstructions` RSMD -field - `false` leaves the instructions as is, `true` sets it to the same as the README, -and a string sets it to that value. If `update` is true, changes to `Project.toml` (e.g. to -the version or the license) are treated as deliberate and propagated to the other metadata -files with `@info` messages instead of being reported as warnings or errors. -The license is an [SPDX identifier](https://spdx.org/licenses/) such as `"MIT"`, taken -from `license` in `Project.toml` or, failing that, from `codemeta.json`; the crosswalk -throws an error if there is none or if it is not one SPDX recognises. The `license` argument -sets or changes it, and like `category` is written back into `Project.toml`. A license in -`Project.toml` that differs from the one already in `codemeta.json` is an error, and -nothing is changed, unless `update` is true or the `license` argument is given. -Every julia source file is given a first line naming the license. A file that is deliberately -under a different license keeps its own first line, provided that license is listed under -`additional_licenses` in the `[rsmd]` table; a file marked with any other license is an -error, and nothing is changed. -The dates of a version that has no release tag yet are today's date, in UTC, whenever -the crosswalk is run on the repository's default branch or with `update` true; on any -other branch the dates already recorded for that version are kept. -If any remote metadata query (orcid.org, ror.org, spdx.org, -doi.org or Julia's General registry) cannot be completed, the crosswalk throws an error and -all files are left in their original state. +field, and otherwise from `authors` itself. New entries in `authors` are propagated into +`rsmd.author_details` (with a warning to add their ORCID and ROR affiliation there), +`codemeta.json` and `.zenodo.json`, while authors in `codemeta.json` that are missing from +`authors` are removed with an error. + +# License + +The license is an SPDX identifier, taken from the `license` argument, from `license` in +`Project.toml` or, failing that, from `codemeta.json`; the crosswalk throws an error if +there is none or if it is not one SPDX recognises. A license in `Project.toml` that +differs from the one already in `codemeta.json` is an error, and nothing is changed, unless +`update` is true or the `license` argument is given. + +If the package has no license file, a `LICENSE` is written. A license file that the +crosswalk did not write is left as it is if it holds the license, and is an error if it +does not, unless the license is being changed. + +Every julia source file is given a first line naming the license. A file that is +deliberately under a different license keeps its own first line, provided that license is +listed under `additional_licenses` in the `[rsmd]` table; a file marked with any other +license is an error, and nothing is changed. + +# Dates + +The dates of a version that has no release tag yet are today's date, in UTC, whenever the +crosswalk is run on the repository's default branch or with `update` true; on any other +branch the dates already recorded for that version are kept. + +# Failure + +If any remote metadata query (orcid.org, ror.org, spdx.org, doi.org or Julia's General +registry) cannot be completed, the crosswalk throws an error and all files are left in +their original state. """ function crosswalk(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`); category = nothing, keywords = nothing, @@ -53,9 +83,21 @@ end """ relicense!(license, git_dir; overwrite_all = false) -Relicense the package in the repository at `git_dir` under `license`, an -[SPDX identifier](https://spdx.org/licenses/) such as `"MIT"`, and run a -crosswalk to carry the change into every metadata file. +Relicense a package, and run a crosswalk to carry the change into every +metadata file. + +# Arguments + + - `license`: the [SPDX identifier](https://spdx.org/licenses/) of the new + license, such as `"MIT"`. + - `git_dir`: the repository holding the package. It defaults to the + repository that the working directory is in. + - `overwrite_all`: if true, replace every existing license file and source + file header and drop `additional_licenses`, keeping nothing of the + existing licensing. The default, false, respects what the package has + declared. + +# Keeping or overwriting the existing licensing As it stands this is `crosswalk(git_dir, license = license)`: `Project.toml`, `codemeta.json` and `.zenodo.json` take the new license, the `LICENSE` file is @@ -77,6 +119,8 @@ relabels files regardless of who wrote them, so whether they may be relicensed is for you to establish first; what it replaces can be recovered from git if it had been committed. +# Failure + The crosswalk throws an error, and nothing is changed, if `license` is not an identifier SPDX recognises or a remote metadata query cannot be completed. """ diff --git a/src/versioning.jl b/src/versioning.jl index ef48350..2fbeabc 100644 --- a/src/versioning.jl +++ b/src/versioning.jl @@ -1,10 +1,17 @@ # SPDX-License-Identifier: MIT """ - increase_patch() + increase_patch(git_dir) -Increases the `Project.toml` version number by a patch (e.g. 0.4.1 to 0.4.2), and then -runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. +Increase the version number in the `Project.toml` of a package by a patch (e.g. 0.4.1 to +0.4.2), and then run `ResearchSoftwareMetadata.crosswalk(git_dir, update = true)` to +propagate the new version to the other metadata files. If the crosswalk fails, +`Project.toml` is left as it was. + +# Arguments + + - `git_dir`: the repository holding the package. It defaults to the repository that + the working directory is in. """ function increase_patch(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) project = read_project(git_dir) @@ -29,10 +36,17 @@ function increase_patch(git_dir = readchomp(`$(Git.git()) rev-parse --show-tople end """ - increase_minor() + increase_minor(git_dir) + +Increase the version number in the `Project.toml` of a package by a minor number (e.g. 0.4.1 to +0.5.0), and then run `ResearchSoftwareMetadata.crosswalk(git_dir, update = true)` to +propagate the new version to the other metadata files. If the crosswalk fails, +`Project.toml` is left as it was. -Increases the `Project.toml` version number by a minor number (e.g. 0.4.1 to 0.5.0), and then -runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. +# Arguments + + - `git_dir`: the repository holding the package. It defaults to the repository that + the working directory is in. """ function increase_minor(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) project = read_project(git_dir) @@ -57,10 +71,17 @@ function increase_minor(git_dir = readchomp(`$(Git.git()) rev-parse --show-tople end """ - increase_major() + increase_major(git_dir) + +Increase the version number in the `Project.toml` of a package by a major number (e.g. 0.4.1 to +1.0.0), and then run `ResearchSoftwareMetadata.crosswalk(git_dir, update = true)` to +propagate the new version to the other metadata files. If the crosswalk fails, +`Project.toml` is left as it was. + +# Arguments -Increases the `Project.toml` version number by a major number (e.g. 0.4.1 to 1.0.0), and then -runs `ResearchSoftwareMetadata.crosswalk()` to propagate this information. + - `git_dir`: the repository holding the package. It defaults to the repository that + the working directory is in. """ function increase_major(git_dir = readchomp(`$(Git.git()) rev-parse --show-toplevel`)) project = read_project(git_dir)