From 5c147b35b0e4ef11b615c5126ea011247e3b2527 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 14 Oct 2026 17:23:46 -0500 Subject: [PATCH 01/10] chore: rebase changes --- .github/dependabot.yml | 41 ++ .github/workflows/ci.yaml | 43 +- .github/workflows/pr.yml | 24 +- .github/workflows/promote.yml | 9 +- .github/workflows/promotion-checker.yml | 2 +- .markdownlint-cli2.jsonc | 8 - .pre-commit-config.yaml | 30 +- CODEOWNERS | 6 +- README.md | 2 +- pyproject.toml | 57 +++ requirements.in | 6 - requirements.txt | 123 ----- scripts/generate_mdx.py | 31 +- scripts/promote_collection.py | 10 +- scripts/promote_dataset.py | 10 +- scripts/requirements.txt | 2 - .../Transfer-MCP-veda-data-store.ipynb | 52 ++- .../generate-cmip6-kerchunk-historical.ipynb | 43 +- .../publish-cmip6-kerchunk-stac.ipynb | 10 +- .../collection-and-item-ingest.ipynb | 26 +- .../collection-metadata-reconciliation.ipynb | 44 +- .../create-stac-for-cmr-collections.ipynb | 24 +- .../fwi-stations/FWI_Transformation.ipynb | 8 +- .../generate-rasi-cogs-from-netcdf3.ipynb | 26 +- .../publish-hls-provided-item-metadata.ipynb | 27 +- .../landsat-lakes-discovery/aral-sea.ipynb | 33 +- .../lake-balaton.ipynb | 43 +- .../landsat-lakes-discovery/lake-biwa.ipynb | 45 +- .../mirror-landsat-demos.ipynb | 43 +- .../landsat-lakes-discovery/tonle-sap.ipynb | 45 +- .../landsat-lakes-discovery/vanern.ipynb | 43 +- .../cmip6-generate-organize-cogs.ipynb | 128 +++--- .../cmip6-generate-stac.ipynb | 6 +- .../cmip6_file_organization.py | 7 +- .../soil_moisture_transformation_cog.ipynb | 15 +- transformation-scripts/update-hrefs.ipynb | 10 +- uv.lock | 419 ++++++++++++++++++ 37 files changed, 990 insertions(+), 511 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 .markdownlint-cli2.jsonc create mode 100644 pyproject.toml delete mode 100644 requirements.in delete mode 100644 requirements.txt delete mode 100644 scripts/requirements.txt create mode 100644 uv.lock diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..415eedec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,41 @@ +version: 2 +updates: + - package-ecosystem: uv + directory: / + cooldown: + default-days: 7 + semver-major-days: 30 + schedule: + interval: weekly + groups: + python-dependencies: + patterns: + - "*" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: github-actions + directory: / + cooldown: + default-days: 7 + schedule: + interval: weekly + groups: + github-actions: + patterns: + - "*" + commit-message: + prefix: "ci(deps):" + + - package-ecosystem: pre-commit + directory: / + cooldown: + default-days: 7 + schedule: + interval: weekly + groups: + pre-commit: + patterns: + - "*" + commit-message: + prefix: "ci(deps):" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8522d212..119a9a77 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,31 +7,24 @@ on: pull_request: jobs: - test: + ci: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: test + run: uv run --group dev pytest + - name: lint + run: uv run --group dev pre-commit run --all-files steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Setup Python - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 - with: - python-version: "3.11" - cache: "pip" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Test collections - run: pytest - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Setup Python - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - python-version: "3.11" - cache: "pip" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Run pre-commit hooks - run: pre-commit run --all-files + enable-cache: true + + - name: ${{ matrix.name }} + run: ${{ matrix.run }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a7daedef..d874e308 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -35,7 +35,7 @@ jobs: comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }} has_new_files_to_promote: ${{ steps.changed-files.outputs.all_changed_files_count != 0 }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 @@ -119,7 +119,7 @@ jobs: AIRFLOW_JWT_SECRET: ${{ secrets.AIRFLOW_JWT_SECRET }} AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }} run: | - pip install -r ./scripts/requirements.txt + uv sync --group scripts # Track successful publications all_failed=true @@ -183,14 +183,10 @@ jobs: $status_message" gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY" - - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - python-version: '3.9' - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-pip-${{ hashFiles('requirements.txt') }} + enable-cache: true # If the workflow fails at any point, the PR comment will be updated - name: Update PR comment on overall workflow failure @@ -236,7 +232,7 @@ jobs: run: ls -lah /tmp/upload-artifacts - name: Upload published collections files as directory artifact - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 env: ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }} with: @@ -250,7 +246,7 @@ jobs: echo "${{ github.run_id }}" > run_id.txt - name: Upload Workflow Run ID - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 env: ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }} with: @@ -275,7 +271,7 @@ jobs: skip_mdx_pr_step: ${{ steps.skip_mdx_pr_step_setting.outputs.skip_mdx_pr_step }} steps: - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set SKIP_MDX_PR_STEP condition id: skip_mdx_pr_step_setting @@ -292,7 +288,7 @@ jobs: needs: ['mdx-pr-flag-evaluator', 'publish-new-datasets'] steps: - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Use output from publish-new-datasets run: | @@ -305,7 +301,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES collection_ids="" - pip install -r ./scripts/requirements.txt + uv sync --group scripts for file in ${PUBLISHED_COLLECTION_FILES}; do collection_id=$(python3 ./scripts/generate_mdx.py "$file") collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//') diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 1aab7e5d..9a04bc5a 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -45,7 +45,12 @@ jobs: environment: production steps: - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true - name: Determine artifact name run: echo "ARTIFACT_NAME=collections-to-promote-from-${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV @@ -70,7 +75,7 @@ jobs: AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }} run: | - pip install -r ./scripts/requirements.txt + uv sync --group scripts for file in downloaded-files/*.json; do echo "Processing $file" # Check if collection key exists to determine if it's a dataset or collection diff --git a/.github/workflows/promotion-checker.yml b/.github/workflows/promotion-checker.yml index 1da3c698..bdcf5a89 100644 --- a/.github/workflows/promotion-checker.yml +++ b/.github/workflows/promotion-checker.yml @@ -53,7 +53,7 @@ jobs: needs: check-conditions steps: - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Get Workflow Run ID Artifact run: | diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc deleted file mode 100644 index e5a00e2c..00000000 --- a/.markdownlint-cli2.jsonc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "config": { - "MD013": false, // disable line length checks - "MD033": { - "allowed_elements": [ "a", "b", "details", "summary" ] - } - } -} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f3482e7..284a8e63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,26 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v6.0.0 hooks: - id: check-yaml - - id: file-contents-sorter - files: ^requirements.in$ - id: trailing-whitespace exclude: ^ingestion-data/.*$ + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.1 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format + types_or: [python, pyi, jupyter, pyproject] + - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.10.0 + rev: v0.23.2 hooks: - id: markdownlint-cli2 - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.3.0 - hooks: - - id: black - - id: black-jupyter - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 - hooks: - - id: ruff + args: + - "--fix" + - "--config" + - "pyproject.toml" + - "--configPointer" + - "/tool/markdownlint-cli2" diff --git a/CODEOWNERS b/CODEOWNERS index 296e5839..ebb5e4f4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,5 +1 @@ -* @anayeaye -* @gadomski -* @ividito -* @slesaad -* @smohiudd +* @NASA-IMPACT/veda-data-services diff --git a/README.md b/README.md index 39918351..41cece2d 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ To add data to VEDA you will: ### Step 1: Stage your files -Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request access [here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html) +Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request [access here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html) ### Step 2: Generate STAC metadata in the staging catalog diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9a13a271 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[project] +name = "veda-data" +version = "0.1.0" +description = "Config data used to load datasets into the VEDA catalog" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] + +[dependency-groups] +dev = [ + "pre-commit==4.6.1", + "pystac[validation]==1.8.4", + "pytest==7.4.2", +] +scripts = [ + "pyyaml", + "PyJWT", +] + +[tool.uv] +default-groups = ["dev"] + +[tool.ruff] +line-length = 88 + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # Pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "TID", # flake8-tidy-imports + "TC", # flake8-type-checking + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "BLE", # flake8-blind-except +] +ignore = [ + "BLE001", # Blind Exceptions +] + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = "dynamic" + +[tool.markdownlint-cli2.config] +MD013 = false +MD033 = { allowed_elements = ["a", "b", "details", "summary"] } + +[tool.ruff.lint.isort] +combine-as-imports = true diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 8f9b6370..00000000 --- a/requirements.in +++ /dev/null @@ -1,6 +0,0 @@ -black[jupyter] -pip-tools -pre-commit -pystac[validation] -pytest -ruff diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 64c91392..00000000 --- a/requirements.txt +++ /dev/null @@ -1,123 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile -# -asttokens==2.4.0 - # via stack-data -attrs==23.1.0 - # via - # jsonschema - # referencing -backcall==0.2.0 - # via ipython -black[jupyter]==23.10.0 - # via -r requirements.in -build==1.0.3 - # via pip-tools -cfgv==3.4.0 - # via pre-commit -click==8.1.7 - # via - # black - # pip-tools -decorator==5.1.1 - # via ipython -distlib==0.3.7 - # via virtualenv -executing==2.0.0 - # via stack-data -filelock==3.12.4 - # via virtualenv -identify==2.5.30 - # via pre-commit -iniconfig==2.0.0 - # via pytest -ipython==8.16.1 - # via black -jedi==0.19.1 - # via ipython -jsonschema==4.19.1 - # via pystac -jsonschema-specifications==2023.7.1 - # via jsonschema -matplotlib-inline==0.1.6 - # via ipython -mypy-extensions==1.0.0 - # via black -nodeenv==1.8.0 - # via pre-commit -packaging==23.2 - # via - # black - # build - # pytest -parso==0.8.3 - # via jedi -pathspec==0.11.2 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pip-tools==7.3.0 - # via -r requirements.in -platformdirs==3.11.0 - # via - # black - # virtualenv -pluggy==1.3.0 - # via pytest -pre-commit==3.5.0 - # via -r requirements.in -prompt-toolkit==3.0.39 - # via ipython -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pygments==2.16.1 - # via ipython -pyproject-hooks==1.0.0 - # via build -pystac[validation]==1.8.4 - # via -r requirements.in -pytest==7.4.2 - # via -r requirements.in -python-dateutil==2.8.2 - # via pystac -pyyaml==6.0.1 - # via pre-commit -referencing==0.30.2 - # via - # jsonschema - # jsonschema-specifications -rpds-py==0.10.6 - # via - # jsonschema - # referencing -ruff==0.1.0 - # via -r requirements.in -six==1.16.0 - # via - # asttokens - # python-dateutil -stack-data==0.6.3 - # via ipython -tokenize-rt==5.2.0 - # via black -traitlets==5.11.2 - # via - # ipython - # matplotlib-inline -virtualenv==20.26.6 - # via pre-commit -wcwidth==0.2.8 - # via prompt-toolkit -wheel==0.41.2 - # via pip-tools - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/scripts/generate_mdx.py b/scripts/generate_mdx.py index e8726d53..4db953bf 100644 --- a/scripts/generate_mdx.py +++ b/scripts/generate_mdx.py @@ -5,10 +5,11 @@ Dependency: `dataset.mdx` file """ -import yaml -import os import json import sys +from pathlib import Path + +import yaml def create_frontmatter(input_data): @@ -91,39 +92,37 @@ def create_frontmatter(input_data): json_data["layers"].append(layer) # Convert json to yaml for frontmatter - yaml_data = yaml.dump(json_data, sort_keys=False) - - return yaml_data + return yaml.dump(json_data, sort_keys=False) def safe_open_w(path): """Open "path" for writing, creating any parent directories as needed.""" - os.makedirs(os.path.dirname(path), exist_ok=True) - return open(path, "w") + Path(path).parent.mkdir(parents=True, exist_ok=True) + return Path(path).open("w") if __name__ == "__main__": - input_data = json.load(open(sys.argv[1])) + with Path(sys.argv[1]).open() as f: + input_data = json.load(f) dataset_config = create_frontmatter(input_data) front_matter = f"---\n{dataset_config}---\n" # Path to the existing file - curr_directory = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(curr_directory, "dataset.mdx") + curr_directory = Path(__file__).parent + file_path = curr_directory / "dataset.mdx" # Read the existing content of the file - with open(file_path, "r") as file: - existing_content = file.read() + existing_content = file_path.read_text() # Combine front matter and existing content new_content = front_matter + existing_content # Write the combined content back to the file - output_filepath = os.path.join( - curr_directory, - f"../ingestion-data/dataset-mdx/{input_data['collection']}.data.mdx", + output_filepath = ( + Path(curr_directory) + / f"../ingestion-data/dataset-mdx/{input_data['collection']}.data.mdx" ) - with safe_open_w(output_filepath) as ofile: + with safe_open_w(str(output_filepath)) as ofile: ofile.write(new_content) collection_id = input_data["collection"] diff --git a/scripts/promote_collection.py b/scripts/promote_collection.py index 80d6dd2e..106a7741 100644 --- a/scripts/promote_collection.py +++ b/scripts/promote_collection.py @@ -1,13 +1,13 @@ -from typing import Dict, Any - import json -import sys import os +import sys +from pathlib import Path +from typing import Any from airflow_api import AirflowAPIError, trigger_dag_run -def trigger_collection_dag(payload: Dict[str, Any], stage: str): +def trigger_collection_dag(payload: dict[str, Any], stage: str): """ Triggers the veda_collection_pipeline DAG in either staging or production SM2A. """ @@ -53,7 +53,7 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str): if __name__ == "__main__": try: - with open(sys.argv[1], "r") as file: + with Path(sys.argv[1]).open("r") as file: input_data = json.load(file) stage = sys.argv[2] dag_payload = {"conf": input_data} diff --git a/scripts/promote_dataset.py b/scripts/promote_dataset.py index 1cd3e20d..07a54b88 100644 --- a/scripts/promote_dataset.py +++ b/scripts/promote_dataset.py @@ -1,8 +1,8 @@ -from typing import Dict, Any - import json -import sys import os +import sys +from pathlib import Path +from typing import Any from airflow_api import AirflowAPIError, trigger_dag_run @@ -11,7 +11,7 @@ class MissingFieldError(Exception): pass -def validate_discovery_item_config(item: Dict[str, Any]) -> Dict[str, Any]: +def validate_discovery_item_config(item: dict[str, Any]) -> dict[str, Any]: required_fields = ["bucket", "discovery", "filename_regex", "prefix"] for field in required_fields: if field not in item: @@ -81,7 +81,7 @@ def promote_to_production(payload): if __name__ == "__main__": try: - with open(sys.argv[1], "r") as file: + with Path(sys.argv[1]).open("r") as file: _input = json.load(file) stage = sys.argv[2] discovery_items = _input.get("discovery_items") diff --git a/scripts/requirements.txt b/scripts/requirements.txt deleted file mode 100644 index 42f836aa..00000000 --- a/scripts/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pyyaml -PyJWT \ No newline at end of file diff --git a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb index 0c5cbdbb..34bdf207 100644 --- a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb +++ b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb @@ -2,29 +2,34 @@ "cells": [ { "cell_type": "code", - "execution_count": 105, + "execution_count": null, "id": "0a619c84-a55d-4f93-99e0-c31a42d52932", "metadata": {}, "outputs": [], "source": [ - "from pystac_client import Client\n", + "import base64\n", "import json\n", - "import glob\n", - "import requests\n", + "from pathlib import Path\n", + "\n", "import boto3\n", - "import base64\n", - "import time" + "import requests\n", + "from pystac_client import Client" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "e1a873f2-7b72-4245-a3d0-a1a3d38bde97", "metadata": {}, "outputs": [], "source": [ - "discover_items_file_paths = glob.glob(\"../ingestion-data/discovery-items/*.json\")\n", - "# discover_items_ids = [i.split(\"/\")[-1].split(\".\")[0] for i in discover_items_file_paths]" + "discover_items_file_paths = list(\n", + " Path(\"../ingestion-data/discovery-items\").glob(\"*.json\")\n", + ")\n", + "\n", + "# discover_items_ids = [\n", + "# i.split(\"/\")[-1].split(\".\")[0] for i in discover_items_file_paths\n", + "# ]" ] }, { @@ -49,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": null, "id": "a3457a89-e5e3-4873-a524-51a914488b88", "metadata": {}, "outputs": [ @@ -302,7 +307,7 @@ ], "source": [ "for filename in discover_items_file_paths:\n", - " with open(filename, \"r\") as f:\n", + " with Path(filename).open() as f:\n", " dataset = json.load(f)\n", " if not isinstance(dataset, list):\n", " dataset = [dataset]\n", @@ -313,24 +318,24 @@ " if (\n", " i[\"collection\"] in staging_collections\n", " ): # only create config for collections in staging\n", - " config = dict()\n", + " config = {}\n", " config[\"origin_bucket\"] = i.get(\"bucket\", \"\")\n", " config[\"origin_prefix\"] = i.get(\"prefix\", \"\")\n", " config[\"target_bucket\"] = \"veda-data-store\"\n", " config[\"collection\"] = i.get(\"collection\")\n", "\n", " if \"assets\" in i: # conditional for multi asset discovery items\n", - " for k, v in i[\"assets\"].items():\n", + " for _k, v in i[\"assets\"].items():\n", " config[\"filename_regex\"] = v.get(\"regex\")\n", " transfer_config.append(config.copy())\n", " else:\n", " config[\"filename_regex\"] = i.get(\"filename_regex\", \"\")\n", " transfer_config.append(config)\n", "\n", - " with open(\n", - " f\"../ingestion-data/transfer-config/{id}.json\", \"w\", encoding=\"utf-8\"\n", + " with Path(f\"../ingestion-data/transfer-config/{id}.json\").open(\n", + " \"w\", encoding=\"utf-8\"\n", " ) as file:\n", - " print(f\"writing file: ingestion_data/transfer-config/{id}\")\n", + " print(f\"writing file: ../ingestion-data/transfer-config/{id}\")\n", " json.dump(transfer_config, file, ensure_ascii=False, indent=4)" ] }, @@ -376,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": null, "id": "5a774091-a239-409a-9360-03dec3839ec7", "metadata": {}, "outputs": [ @@ -1448,11 +1453,13 @@ } ], "source": [ - "transfer_items_file_paths = glob.glob(\"../ingestion-data/transfer-config/*.json\")\n", + "transfer_items_file_paths = list(\n", + " Path(\"../ingestion-data/transfer-config\").glob(\"*.json\")\n", + ")\n", "\n", "for transfer_item_file in transfer_items_file_paths:\n", " print(f\"Transferring using config: {transfer_item_file}\")\n", - " with open(transfer_item_file, \"r\") as f:\n", + " with Path(transfer_item_file).open() as f:\n", " transfer_configs = json.load(f)\n", " for transfer_config in transfer_configs:\n", " mwaa_cli_token = get_mwaa_cli_token(\n", @@ -1469,7 +1476,10 @@ " pass\n", " else:\n", " if not dry_run:\n", - " raw_data = f\"dags trigger veda_transfer --conf '{json.dumps(transfer_config)}'\"\n", + " raw_data = (\n", + " \"dags trigger veda_transfer \"\n", + " f\"--conf '{json.dumps(transfer_config)}'\"\n", + " )\n", " mwaa_response = requests.post(\n", " f\"https://{mwaa_cli_token['WebServerHostname']}/aws_mwaa/cli\",\n", " headers={\n", @@ -1488,7 +1498,7 @@ " print(mwaa_response.status_code)\n", " print(f\"stderr: {mwaa_std_err_message}\")\n", " print(f\"stdout: {mwaa_std_out_message}\")\n", - " except:\n", + " except Exception:\n", " print(mwaa_response)" ] }, diff --git a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb index 9efd475b..07d65fe8 100644 --- a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb +++ b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb @@ -16,23 +16,22 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9f7250a6", "metadata": {}, "outputs": [], "source": [ + "from pathlib import Path\n", "from tempfile import TemporaryDirectory\n", + "\n", "import boto3\n", - "from dask_gateway import GatewayCluster, Gateway\n", "import dask.bag as db\n", "import fsspec\n", - "import json\n", - "import os\n", "import ujson\n", "import xarray as xr\n", + "from dask_gateway import Gateway, GatewayCluster\n", "from kerchunk.combine import MultiZarrToZarr\n", "from kerchunk.hdf import SingleHdf5ToZarr\n", - "from typing import Dict\n", "\n", "# Specify the CMIP model and variable to use.\n", "# Here we are using near-surface air temperature from the GISS-E2-1-G GCM\n", @@ -41,7 +40,8 @@ "# If this code were re-used for a protected bucket, anon should be False.\n", "anon = True\n", "# Note: We are only using the historical data in this example.\n", - "# More years of data are available from multiple Shared Socio-Economic Pathways (SSPs) in the s3://nex-gddp-cmip6 bucket.\n", + "# More years of data are available from multiple Shared Socio-Economic Pathways (SSPs)\n", + "# in the s3://nex-gddp-cmip6 bucket.\n", "s3_path = f\"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1*/{variable}/*\"" ] }, @@ -119,15 +119,21 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "b046c17c", "metadata": {}, "outputs": [], "source": [ - "so = dict(mode=\"rb\", anon=anon, default_fill_cache=False, default_cache_type=\"first\")\n", + "so = {\n", + " \"mode\": \"rb\",\n", + " \"anon\": anon,\n", + " \"default_fill_cache\": False,\n", + " \"default_cache_type\": \"first\",\n", + "}\n", "\n", "\n", - "# Use Kerchunk's `SingleHdf5ToZarr` method to create a `Kerchunk` index from a NetCDF file.\n", + "# Use Kerchunk's `SingleHdf5ToZarr` method\n", + "# to create a `Kerchunk` index from a NetCDF file.\n", "def generate_json_reference(u):\n", " with fs_read.open(u, **so) as infile:\n", " fname = u.split(\"/\")[-1].strip(\".nc\")\n", @@ -136,8 +142,8 @@ "\n", "\n", "def write_json(fname, reference_json, temp_dir):\n", - " outf = os.path.join(temp_dir, f\"{fname}.json\")\n", - " with open(outf, \"wb\") as f:\n", + " outf = Path(temp_dir) / f\"{fname}.json\"\n", + " with outf.open(\"wb\") as f:\n", " f.write(reference_json)\n", " return outf" ] @@ -188,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "794be824-0833-4229-a8b4-10b071eced26", "metadata": { "tags": [] @@ -258,7 +264,8 @@ "gateway = Gateway()\n", "clusters = gateway.list_clusters()\n", "\n", - "# connect to an existing cluster - this is useful when the kernel shutdown in the middle of an interactive session\n", + "# connect to an existing cluster\n", + "# this is useful when the kernel shutdown in the middle of an interactive session\n", "if clusters:\n", " cluster = gateway.connect(clusters[0].name)\n", "else:\n", @@ -355,7 +362,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "55ced4a6-0c38-43c8-8c6e-a4fff8a9f7bb", "metadata": {}, "outputs": [ @@ -369,8 +376,8 @@ ], "source": [ "output_fname = f\"combined_CMIP6_daily_{model}_{variable}_kerchunk.json\"\n", - "output_location = os.path.join(temp_dir, output_fname)\n", - "with open(f\"{output_location}\", \"wb\") as f:\n", + "output_location = Path(temp_dir) / output_fname\n", + "with output_location.open(\"wb\") as f:\n", " print(f\"Writing combined kerchunk reference file {output_location}\")\n", " f.write(ujson.dumps(multi_kerchunk).encode())" ] @@ -391,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "ac76c0c0", "metadata": {}, "outputs": [ @@ -841,7 +848,7 @@ ], "source": [ "# Check the data\n", - "ds = xr.open_dataset(m, engine=\"zarr\", backend_kwargs=dict(consolidated=False))\n", + "ds = xr.open_dataset(m, engine=\"zarr\", backend_kwargs={\"consolidated\": False})\n", "display(ds)" ] }, diff --git a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb index 1bd30f3f..4e9e9a0a 100644 --- a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb +++ b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "3c3e536f-f61f-4ad7-a0b4-414387d634d1", "metadata": { "tags": [] @@ -32,11 +32,11 @@ "outputs": [], "source": [ "#!pip install xstac\n", - "import pystac\n", - "import s3fs\n", - "import xstac\n", "import fsspec\n", - "import xarray as xr" + "import pystac\n", + "import requests\n", + "import xarray as xr\n", + "import xstac" ] }, { diff --git a/transformation-scripts/collection-and-item-ingest.ipynb b/transformation-scripts/collection-and-item-ingest.ipynb index 6adfb0ac..1800e13d 100644 --- a/transformation-scripts/collection-and-item-ingest.ipynb +++ b/transformation-scripts/collection-and-item-ingest.ipynb @@ -31,8 +31,9 @@ "metadata": {}, "outputs": [], "source": [ - "import glob\n", "import json\n", + "from pathlib import Path\n", + "\n", "import requests\n", "from cognito_client import CognitoClient" ] @@ -64,7 +65,7 @@ " \"combined_CMIP6_daily_GISS-E2-1-G_tas_kerchunk_DEMO\",\n", "]\n", "\n", - "json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n", + "json_file_paths = Path(\"../ingestion-data/collections\").glob(\"*.json\")\n", "filtered_list = [\n", " item\n", " for item in json_file_paths\n", @@ -74,10 +75,16 @@ " )\n", "]\n", "\n", + "\n", + "def load_json(file_path: str | Path) -> dict:\n", + " with Path(file_path).open() as f:\n", + " return json.load(f)\n", + "\n", + "\n", "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in filtered_list\n", - " if \"id\" in (data := json.load(open(file_path, \"r\")))\n", + " if \"id\" in (data := load_json(file_path))\n", "]" ] }, @@ -182,15 +189,18 @@ " response.raise_for_status()\n", " if response.status_code == 201:\n", " print(\n", - " f\"Request was successful. Find the updated collection at {collection_url}\"\n", + " \"Request was successful. \"\n", + " f\"Find the updated collection at {collection_url}\"\n", " )\n", " else:\n", " print(\n", - " f\"Updating {collection_id} failed. Request failed with status code: {response.status_code}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"Request failed with status code: {response.status_code}\"\n", " )\n", " except requests.RequestException as e:\n", " print(\n", - " f\"Updating {collection_id} failed. An error occurred during the request: {e}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"An error occurred during the request: {e}\"\n", " )\n", " except Exception as e:\n", " print(\n", @@ -241,10 +251,10 @@ " file_path = item[\"filePath\"]\n", "\n", " try:\n", - " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " with Path(file_path).open(encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", - " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", + " # Publish updated collection to the target ingestion `api/collections` endpoint\n", " post_collection(collection, collection_id)\n", "\n", " except requests.RequestException as e:\n", diff --git a/transformation-scripts/collection-metadata-reconciliation.ipynb b/transformation-scripts/collection-metadata-reconciliation.ipynb index 54c5d3d1..d5337495 100644 --- a/transformation-scripts/collection-metadata-reconciliation.ipynb +++ b/transformation-scripts/collection-metadata-reconciliation.ipynb @@ -16,12 +16,13 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import glob\n", "import json\n", + "from pathlib import Path\n", + "\n", "import requests\n", "from cognito_client import CognitoClient" ] @@ -35,16 +36,22 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n", + "json_file_paths = Path(\"../ingestion-data/collections\").glob(\"*.json\")\n", + "\n", + "\n", + "def load_json(file_path: str | Path) -> dict:\n", + " with Path(file_path).open() as f:\n", + " return json.load(f)\n", + "\n", "\n", "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in json_file_paths\n", - " if \"id\" in (data := json.load(open(file_path, \"r\")))\n", + " if \"id\" in (data := load_json(file_path))\n", "]" ] }, @@ -136,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -149,15 +156,18 @@ " response.raise_for_status()\n", " if response.status_code == 201:\n", " print(\n", - " f\"Request was successful. Find the updated collection at {collection_url}\"\n", + " \"Request was successful. \"\n", + " f\"Find the updated collection at {collection_url}\"\n", " )\n", " else:\n", " print(\n", - " f\"Updating {collection_id} failed. Request failed with status code: {response.status_code}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"Request failed with status code: {response.status_code}\"\n", " )\n", " except requests.RequestException as e:\n", " print(\n", - " f\"Updating {collection_id} failed. An error occurred during the request: {e}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"An error occurred during the request: {e}\"\n", " )\n", " except Exception as e:\n", " print(\n", @@ -176,15 +186,14 @@ "\n", "\n", "def retain_external_links(existing_links, retrieved_links):\n", - " unique_hrefs = set(link.get(\"href\") for link in existing_links)\n", + " unique_hrefs = {link.get(\"href\") for link in existing_links}\n", " additional_external_links = [\n", " link\n", " for link in retrieved_links\n", " if link.get(\"rel\") == \"external\" and link.get(\"href\") not in unique_hrefs\n", " ]\n", "\n", - " retained_links = existing_links + additional_external_links\n", - " return retained_links" + " return existing_links + additional_external_links" ] }, { @@ -196,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -361,9 +370,9 @@ " collection_id = item[\"collectionId\"]\n", " file_path = item[\"filePath\"]\n", "\n", - " if VEDA_STAC_API == dev_endpoint:\n", + " if dev_endpoint == VEDA_STAC_API:\n", " url = f\"{VEDA_STAC_API}api/stac/collections/{collection_id}\"\n", - " elif VEDA_STAC_API == staging_endpoint:\n", + " elif staging_endpoint == VEDA_STAC_API:\n", " url = f\"{VEDA_STAC_API}collections/{collection_id}\"\n", "\n", " try:\n", @@ -374,7 +383,7 @@ " retrieved_summaries = json_response.get(\"summaries\", {})\n", " retrieved_links = json_response.get(\"links\", {})\n", "\n", - " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " with Path(file_path).open(encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", " existing_summaries = collection.get(\"summaries\", {})\n", @@ -385,7 +394,8 @@ " )\n", " collection[\"links\"] = retain_external_links(existing_links, retrieved_links)\n", "\n", - " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", + " # Publish the updated collection to the target ingestion\n", + " # `api/collections` endpoint\n", " post_reconciled_collection(collection, collection_id)\n", "\n", " except requests.RequestException as e:\n", diff --git a/transformation-scripts/create-stac-for-cmr-collections.ipynb b/transformation-scripts/create-stac-for-cmr-collections.ipynb index 30d7ff17..7cf71bcf 100644 --- a/transformation-scripts/create-stac-for-cmr-collections.ipynb +++ b/transformation-scripts/create-stac-for-cmr-collections.ipynb @@ -63,22 +63,22 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "67e37cc9-6022-46c2-aa20-4bb87dd502f9", "metadata": { "tags": [] }, "outputs": [], "source": [ - "import earthaccess\n", "import json\n", - "from datetime import datetime\n", + "from pathlib import Path\n", + "\n", + "import earthaccess\n", "import pandas as pd\n", "import pystac\n", - "import requests\n", "import s3fs\n", - "import xstac\n", - "import xarray as xr" + "import xarray as xr\n", + "import xstac" ] }, { @@ -98,7 +98,7 @@ }, "outputs": [ { - "name": "stdin", + "name": "stdout", "output_type": "stream", "text": [ "Enter your Earthdata Login username: aimeeb\n", @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "ae799bbb-06f8-499b-be53-c9ab86762ca7", "metadata": { "tags": [] @@ -458,7 +458,7 @@ "outputs": [], "source": [ "cube_variables = collection.to_dict()[\"cube:variables\"]\n", - "for variable in cube_variables.keys():\n", + "for variable in cube_variables:\n", " cube_variables[variable][\"shape\"][0] = None" ] }, @@ -520,12 +520,14 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "42d3023b-b551-43c1-9984-22553eb41fb1", "metadata": {}, "outputs": [], "source": [ - "with open(f\"../ingestion-data/staging/collections/{collection.id}.json\", \"w+\") as f:\n", + "with Path(f\"../ingestion-data/staging/collections/{collection.id}.json\").open(\n", + " \"w+\"\n", + ") as f:\n", " f.write(json.dumps(collection.to_dict(), indent=2))" ] } diff --git a/transformation-scripts/fwi-stations/FWI_Transformation.ipynb b/transformation-scripts/fwi-stations/FWI_Transformation.ipynb index 0ca3196e..aaf3a1b3 100644 --- a/transformation-scripts/fwi-stations/FWI_Transformation.ipynb +++ b/transformation-scripts/fwi-stations/FWI_Transformation.ipynb @@ -7,12 +7,12 @@ "metadata": {}, "outputs": [], "source": [ + "import re\n", + "from io import StringIO\n", + "\n", "import boto3\n", - "import s3fs\n", "import pandas as pd\n", - "import re\n", - "import os\n", - "from io import StringIO" + "import s3fs" ] }, { diff --git a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb index f2795dcf..bdfe5f8d 100644 --- a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb +++ b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb @@ -42,14 +42,15 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "aa94a74d-58ff-4527-b397-13016133ac71", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import rio_cogeo.cogeo\n", + "from pathlib import Path\n", + "\n", "import pandas as pd\n", + "import rio_cogeo.cogeo\n", "import rioxarray # noqa\n", "import s3fs\n", "import xarray as xr" @@ -682,7 +683,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "6c4e69b1-cb7f-4845-b656-8767a246832b", "metadata": {}, "outputs": [ @@ -699,7 +700,8 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, \"\n", + " f\"last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -721,7 +723,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "08c31207-1bce-4573-9edd-adc441e476a8", "metadata": {}, "outputs": [ @@ -738,7 +740,8 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, \"\n", + " f\"last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -811,7 +814,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "3ad681ca-1617-46d9-a282-2b60c3c749ab", "metadata": {}, "outputs": [ @@ -828,7 +831,8 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, \"\n", + " f\"last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -1545,7 +1549,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "13fa9162-46d8-4466-801b-2e2e49458154", "metadata": {}, "outputs": [], @@ -1558,7 +1562,7 @@ "dates = pd.date_range(\"2014-01-01\", \"2014-03-01\", freq=\"MS\")\n", "\n", "# make sure the dir exists\n", - "os.makedirs(\"COGS\", exist_ok=True)\n", + "Path(\"COGS\").mkdir(parents=True, exist_ok=True)\n", "\n", "for date in dates:\n", " datestr = date.strftime(\"%Y%m\")\n", diff --git a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb index b8389b80..254c2d5c 100644 --- a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb +++ b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb @@ -30,8 +30,9 @@ "metadata": {}, "outputs": [], "source": [ - "import boto3\n", "import json\n", + "\n", + "import boto3\n", "import requests\n", "from pystac import Item\n", "\n", @@ -145,7 +146,8 @@ " )\n", " item_dict[\"links\"] = links\n", "\n", - " # Update the stac version for these items from \"stac_version\": \"1.0.0-beta.2\" and touch up metadata to meet 1.0.0 spec\n", + " # Update the stac version for these items from \"stac_version\": \"1.0.0-beta.2\"\n", + " # and touch up metadata to meet 1.0.0 spec\n", " item_dict[\"stac_version\"] = \"1.0.0\"\n", "\n", " # Add full extension hrefs https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md#stac_extensions\n", @@ -164,7 +166,7 @@ " item_assets = item_dict[\"assets\"]\n", " # Previous location did not have data version number\n", " old_prefix = collection_id.replace(\"-002-\", \"-\")\n", - " for asset_key in item_assets.keys():\n", + " for asset_key in item_assets:\n", " new_href = item_assets[asset_key][\"href\"].replace(old_prefix, collection_id)\n", " item_assets[asset_key][\"href\"] = new_href\n", "\n", @@ -172,7 +174,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception as e:\n", + " except Exception:\n", " print(f\"Invalid {collection_id=} {item.id=}\")\n", "\n", " # Replace the s3 object with the updated metadata for stac version 1.0.0\n", @@ -225,7 +227,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception as e:\n", + " except Exception:\n", " print(f\"invalid {collection_id=} {item.id=}\")\n", "\n", " # Publish to target STAC catalog\n", @@ -236,17 +238,16 @@ " )\n", " if verbose:\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " if not publish_response.reason == \"Created\":\n", + " if publish_response.reason != \"Created\":\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " else:\n", - " if verbose:\n", - " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\"\n", - " )" + " elif verbose:\n", + " print(f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\")" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb index df65234a..ab3ca36c 100644 --- a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb @@ -132,8 +132,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -178,7 +178,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection for \"\n", + " \"the purpose of visualizing RGB images. Note that the full item \"\n", + " \"records are not interchangeable across instruments for purposes \"\n", + " \"other than visualization and that the assets available are not the \"\n", + " \"same for each instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -259,11 +267,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -316,9 +325,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -327,7 +334,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -362,7 +369,9 @@ "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { @@ -372,7 +381,9 @@ "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb index a86da37a..359fd8eb 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb @@ -146,8 +146,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -201,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "id": "dff29fbf-0001-42c0-9058-7fb93ab0396d", "metadata": { "tags": [] @@ -223,7 +223,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection for \"\n", + " \"the purpose of visualizing RGB images. Note that the full item records \"\n", + " \"are not interchangeable across instruments for purposes other than \"\n", + " \"visualization and that the assets available are not the same for each \"\n", + " \"instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -296,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "id": "2bbb4330-787a-4ceb-99a4-a79a12c2f3a2", "metadata": { "tags": [] @@ -304,11 +312,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -347,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": null, "id": "ef5c5f88", "metadata": {}, "outputs": [], @@ -361,9 +370,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -372,7 +379,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -410,22 +417,26 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "id": "bf75d1ca", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 107, + "execution_count": null, "id": "d80b42eb", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb index 5fa41682..af4e83d6 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb @@ -134,8 +134,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -160,14 +160,14 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "d1558909-6bd8-4161-a710-4ce0f0502fc3", "metadata": {}, "outputs": [], "source": [ "collection_id = f\"landsat-c2l2-sr-lakes-{LAKE_NAME_SLUG}\"\n", "collection = {\n", - " \"id\": f\"{collection_id}\",\n", + " \"id\": collection_id,\n", " \"type\": \"Collection\",\n", " \"title\": f\"Selected Landsat 7 through 9 Surface Reflectance Scenes for {LAKE_NAME}\",\n", " \"extent\": {\n", @@ -180,7 +180,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection \"\n", + " \"for the purpose of visualizing RGB images. Note that the full item \"\n", + " \"records are not interchangeable across instruments for purposes \"\n", + " \"other than visualization and that the assets available are not \"\n", + " \"the same for each instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -255,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "c8297cb0-b600-4b96-a3bd-1b2b8b1d05d1", "metadata": { "tags": [] @@ -263,11 +271,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -306,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "34e88a7c", "metadata": {}, "outputs": [], @@ -320,9 +329,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -331,7 +338,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -369,22 +376,26 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "5e419e70", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "23c57e6c", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb index 714b7c73..e96a35ac 100644 --- a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb @@ -35,10 +35,11 @@ ], "source": [ "import json\n", - "import requests\n", "from pathlib import Path\n", - "from pystac_client import Client\n", + "\n", + "import requests\n", "from pystac import Collection, Item\n", + "from pystac_client import Client\n", "\n", "# Source records to correct and publish to target\n", "SRC_STAC_API_URL = \"https://staging-stac.delta-backend.com\"\n", @@ -98,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "af3c723e-df35-4aed-abb1-f9c1dc78909b", "metadata": {}, "outputs": [ @@ -131,12 +132,14 @@ " # Start the new collection we will publish\n", " collection_dict = src_collection.to_dict(include_self_link=False)\n", "\n", - " # Special case for these landsat collections: cog_default assets were mistakenly added to item_assets so fix it\n", + " # Special case for these landsat collections:\n", + " # cog_default assets were mistakenly added to item_assets so fix it\n", " item_assets = collection_dict.get(\"item_assets\")\n", " item_assets.pop(\"cog_default\", None)\n", " collection_dict[\"item_assets\"] = item_assets\n", "\n", - " # Make sure the summaries object from the staged collection is included in the veda-data record (TODO check why this isn't passed through pystac to_dict)\n", + " # Make sure the summaries object from the staged collection is included in\n", + " # the veda-data record (TODO check why this isn't passed through pystac to_dict)\n", " collection_dict[\"summaries\"] = src_collection.summaries._summaries\n", "\n", " # Validate\n", @@ -172,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ac9df7f0-308b-4cc9-9268-55a8a5812047", "metadata": {}, "outputs": [ @@ -358,12 +361,14 @@ " )\n", " item_dict[\"links\"] = links\n", "\n", - " # Special case for these landsat collections: cog_default assets were mistakenly added to item_assets so fix it\n", + " # Special case for these landsat collections:\n", + " # cog_default assets were mistakenly added to item_assets so fix it\n", " item_assets = item_dict.get(\"assets\")\n", " item_assets.pop(\"cog_default\", None)\n", " item_dict[\"assets\"] = item_assets\n", "\n", - " # Remove classification extension (many items do not have valid classification metadata)\n", + " # Remove classification extension\n", + " # (many items do not have valid classification metadata)\n", " extensions = item_dict[\"stac_extensions\"]\n", " item_dict[\"stac_extensions\"] = [\n", " i\n", @@ -376,7 +381,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception as e:\n", + " except Exception:\n", " invalid_items.append(item.id)\n", " print(f\"invalid {item.id=} {len(invalid_items)=}\")\n", "\n", @@ -388,18 +393,17 @@ " )\n", " if verbose:\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " if not publish_response.reason == \"Created\":\n", + " if publish_response.reason != \"Created\":\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", " failed_ingestions.append(item.id)\n", - " else:\n", - " if verbose:\n", - " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\"\n", - " )\n", + " elif verbose:\n", + " print(f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\")\n", "\n", "\n", "if len(invalid_items):\n", @@ -419,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "0d922785-dc2c-4bea-8205-5f69a47b68c7", "metadata": {}, "outputs": [ @@ -460,7 +464,8 @@ " target_item_count = len(target_item_collection)\n", "\n", " print(\n", - " f\"\\n{collection_id} {src_item_count=} {target_item_count=} OK={src_item_count==target_item_count}\"\n", + " f\"\\n{collection_id} {src_item_count=} {target_item_count=}\"\n", + " f\" OK={src_item_count == target_item_count}\"\n", " )\n", "\n", "print(\"\\nfin.\")" diff --git a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb index 6d132129..1956a590 100644 --- a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb @@ -144,8 +144,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -170,14 +170,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "c58bc898-4172-4e59-95dc-19b04273f019", "metadata": {}, "outputs": [], "source": [ "collection_id = f\"landsat-c2l2-sr-lakes-{LAKE_NAME_SLUG}\"\n", "collection = {\n", - " \"id\": f\"{collection_id}\",\n", + " \"id\": collection_id,\n", " \"type\": \"Collection\",\n", " \"title\": f\"Selected Landsat 7 through 9 Surface Reflectance Scenes for {LAKE_NAME}\",\n", " \"extent\": {\n", @@ -190,7 +190,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 \"\n", + " f\"Surface Reflectance scenes covering {LAKE_NAME}. Products from \"\n", + " \"multiple missions and instruments are combined in this single \"\n", + " \"collection for the purpose of visualizing RGB images. Note that \"\n", + " \"the full item records are not interchangeable across instruments \"\n", + " \"for purposes other than visualization and that the assets \"\n", + " \"available are not the same for each instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -263,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "f4dbf0d3-ccef-4651-b279-f881f7f9d84f", "metadata": { "tags": [] @@ -271,11 +279,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -314,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "48e9732a", "metadata": {}, "outputs": [], @@ -328,9 +337,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -339,7 +346,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -377,22 +384,26 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "e6b719d6", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "c080e51e", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb index a974a74b..c5ec97ce 100644 --- a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb @@ -142,8 +142,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -168,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "d1558909-6bd8-4161-a710-4ce0f0502fc3", "metadata": {}, "outputs": [], @@ -188,7 +188,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection for \"\n", + " \"the purpose of visualizing RGB images. Note that the full item records \"\n", + " \"are not interchangeable across instruments for purposes other than \"\n", + " \"visualization and that the assets available are not the same for each \"\n", + " \"instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -263,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "c8297cb0-b600-4b96-a3bd-1b2b8b1d05d1", "metadata": { "tags": [] @@ -271,11 +279,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -314,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "0f66e4e7", "metadata": {}, "outputs": [], @@ -328,9 +337,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -339,7 +346,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -377,22 +384,26 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "90eef7ab", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "f140d9c1", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb index 63d77074..27296971 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb @@ -87,10 +87,8 @@ "outputs": [], "source": [ "import boto3\n", - "import s3fs\n", - "import rioxarray\n", - "import rasterio\n", "import rio_cogeo.cogeo\n", + "import s3fs\n", "import xarray as xr\n", "from rasterio.io import MemoryFile\n", "\n", @@ -139,15 +137,19 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "f5b3ff3f-1060-4223-a127-9780e7bb6c9d", "metadata": {}, "outputs": [], "source": [ - "# COG configuration, deflate compression is slightly smaller than the LZW default for the cog driver\n", - "# and predictor=2 further reduces file size\n", + "# COG configuration, deflate compression is slightly smaller than\n", + "# the LZW default for the cog driver and predictor=2 further reduces file size\n", + "\n", "# https://kokoalberti.com/articles/geotiff-compression-optimization-guide/\n", - "# \"Predictors work especially well when there is some spatial correlation in the data, and pixels have values which are similar to their neighbours. As the name suggests, the floating point predictor can only be used on floating point data.\"\n", + "\n", + "# \"Predictors work especially well when there is some spatial correlation in the data,\n", + "# and pixels have values which are similar to their neighbours. As the name suggests,\n", + "# the floating point predictor can only be used on floating point data.\"\n", "\n", "driver = \"COG\"\n", "compress = \"DEFLATE\"\n", @@ -3951,71 +3953,75 @@ " print(f\"{src_key=}\")\n", "\n", " # Get object and read open as xarray dataset\n", - " with fs.open(f\"{STAGING_BUCKET}/{src_key}\") as fileobj:\n", - " with xr.open_dataset(fileobj, engine=\"h5netcdf\") as ds:\n", - " # Now loop through the individual variables and generate a COG for each of these\n", - " for metric in METRICS:\n", - " # Generate output path\n", - " cog_key = cmip6_file_organization.generate_yearly_fwi_metrics_key(\n", - " src_key, \"v0\", pub_type=\"cog\", metric=metric\n", - " )\n", + " with (\n", + " fs.open(f\"{STAGING_BUCKET}/{src_key}\") as fileobj,\n", + " xr.open_dataset(fileobj, engine=\"h5netcdf\") as ds,\n", + " ):\n", + " # Now loop through the individual variables and generate a COG for each\n", + " for metric in METRICS:\n", + " # Generate output path\n", + " cog_key = cmip6_file_organization.generate_yearly_fwi_metrics_key(\n", + " src_key, \"v0\", pub_type=\"cog\", metric=metric\n", + " )\n", "\n", - " if verbose or extra_dryrun:\n", - " print(f\"{cog_key=}\")\n", - " if extra_dryrun:\n", - " continue\n", + " if verbose or extra_dryrun:\n", + " print(f\"{cog_key=}\")\n", + " if extra_dryrun:\n", + " continue\n", "\n", - " # Read individual metric into data array (only one time in yearly NetCDFs)\n", - " da = ds[metric]\n", + " # Read individual metric into data array\n", + " # (only one time in yearly NetCDFs)\n", + " da = ds[metric]\n", "\n", - " # Realign the x dimension to -180 origin for dataset\n", - " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(\n", - " \"lon\"\n", - " )\n", + " # Realign the x dimension to -180 origin for dataset\n", + " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(\"lon\")\n", "\n", - " # Reverse the DataArray's y dimension. It appears that the source NetCDF's y dimension is inverted.\n", - " da = da.reindex(lat=list(reversed(da.lat)))\n", + " # Reverse the DataArray's y dimension\n", + " # It appears that the source NetCDF's y dimension is inverted\n", + " da = da.reindex(lat=list(reversed(da.lat)))\n", "\n", - " # Convert to COG via rioxarray\n", - " da.rio.set_spatial_dims(\"lon\", \"lat\")\n", - " da.rio.write_crs(\"epsg:4326\", inplace=True)\n", + " # Convert to COG via rioxarray\n", + " da.rio.set_spatial_dims(\"lon\", \"lat\")\n", + " da.rio.write_crs(\"epsg:4326\", inplace=True)\n", "\n", - " # Here we need a memory file to write the output raster to\n", - " with MemoryFile() as memfile:\n", - " da.rio.to_raster(\n", - " memfile.name,\n", - " driver=\"COG\",\n", - " compress=compress,\n", - " predictor=predictor,\n", - " )\n", + " # Here we need a memory file to write the output raster to\n", + " with MemoryFile() as memfile:\n", + " da.rio.to_raster(\n", + " memfile.name,\n", + " driver=\"COG\",\n", + " compress=compress,\n", + " predictor=predictor,\n", + " )\n", "\n", - " # Validate and upload COG\n", - " if rio_cogeo.cogeo.cog_validate(memfile.name)[0]:\n", - " if dryrun:\n", - " print(\n", - " f\"Generation of valid COG {cog_key=} successful. Skipping upload.\"\n", - " )\n", - " continue\n", + " # Validate and upload COG\n", + " if rio_cogeo.cogeo.cog_validate(memfile.name)[0]:\n", + " if dryrun:\n", + " print(\n", + " f\"Generation of valid COG {cog_key=} successful. \"\n", + " \"Skipping upload.\"\n", + " )\n", + " continue\n", "\n", - " if verbose:\n", - " print(f\"START to upload {cog_key=} to {DST_BUCKET=}\")\n", + " if verbose:\n", + " print(f\"START to upload {cog_key=} to {DST_BUCKET=}\")\n", "\n", - " # Upload memory file and confirm success\n", - " if not dry_run:\n", - " r = client.put_object(\n", - " Body=memfile,\n", - " Bucket=DST_BUCKET,\n", - " Key=cog_key,\n", - " )\n", - " if r[\"ResponseMetadata\"][\"HTTPStatusCode\"] != 200:\n", - " error_responses.append(r)\n", - " if verbose:\n", - " print(f\"WARNING unable to upload {cog_key=}\")\n", - " else:\n", - " invalid_cogs.append(cog_key)\n", + " # Upload memory file and confirm success\n", + " if not dryrun:\n", + " r = client.put_object(\n", + " Body=memfile,\n", + " Bucket=DST_BUCKET,\n", + " Key=cog_key,\n", + " )\n", + " if r[\"ResponseMetadata\"][\"HTTPStatusCode\"] != 200:\n", + " error_responses.append(r)\n", + " if verbose:\n", + " print(f\"WARNING unable to upload {cog_key=}\")\n", + " else:\n", + " invalid_cogs.append(cog_key)\n", "\n", "print(\n", - " f\"\\nCOMPLETED with {len(error_responses)} errors and {len(invalid_cogs)} invalid cogs\"\n", + " f\"\\nCOMPLETED with {len(error_responses)} errors \"\n", + " f\"and {len(invalid_cogs)} invalid cogs\"\n", ")\n", "print(error_responses)\n", "print(invalid_cogs)" diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb index c3c91052..13c9656e 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb @@ -31,7 +31,6 @@ "metadata": {}, "outputs": [], "source": [ - "from datetime import datetime, timezone\n", "import pystac" ] }, @@ -53,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "dcf5702f-ed7c-4844-9fcd-a2e693f5ec05", "metadata": { "tags": [] @@ -64,7 +63,8 @@ "TITLE = \"Fire Weather Index Yearly SSP245\"\n", "DESCRIPTION = (\n", " \"FWI is a general index of fire danger and the data are monthly and annual \"\n", - " \"multi-model ensemble’s of the 50th percentile of mean Fire Weather Index from the 33 GCMs\"\n", + " \"multi-model ensemble's of the 50th percentile of mean Fire Weather Index \"\n", + " \"from the 33 GCMs\"\n", ")\n", "DASHBOARD__IS_PERIODIC = True\n", "DASHBOARD__TIME_DENSITY = \"year\"\n", diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py index 38496c83..f84dcde4 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py @@ -1,5 +1,4 @@ -from os.path import basename -from typing import Optional +from pathlib import Path from urllib.parse import urlparse @@ -8,7 +7,7 @@ def generate_yearly_fwi_metrics_key( dst_version: str, ensemble: str = "mme", pub_type: str = "netcdf", - metric: Optional[str] = None, + metric: str | None = None, verbose: bool = False, ): """_summary_ @@ -23,7 +22,7 @@ def generate_yearly_fwi_metrics_key( cadence = "yearly" parsed = urlparse(src_url, allow_fragments=False) - nc_base = basename(parsed.path) + nc_base = Path(parsed.path).name # Assume pattern ensemble-stat_experiment_fwi_metrics_yearly_yyyy # MME50_ssp245_fwi_metrics_yearly_2100 diff --git a/transformation-scripts/soil_moisture_transformation_cog.ipynb b/transformation-scripts/soil_moisture_transformation_cog.ipynb index 1f15bf0a..0698e62e 100644 --- a/transformation-scripts/soil_moisture_transformation_cog.ipynb +++ b/transformation-scripts/soil_moisture_transformation_cog.ipynb @@ -17,17 +17,14 @@ }, "outputs": [], "source": [ - "import s3fs\n", - "import rioxarray\n", - "import rasterio\n", - "import rio_cogeo.cogeo\n", - "import xarray as xr\n", + "import json\n", "import re\n", - "from datetime import datetime\n", - "import pandas as pd\n", - "import boto3\n", "import tempfile\n", - "import json" + "\n", + "import boto3\n", + "import pandas as pd\n", + "import s3fs\n", + "import xarray as xr" ] }, { diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index b3d57b47..c388c7cc 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -13,8 +13,9 @@ "metadata": {}, "outputs": [], "source": [ - "import boto3\n", - "import json" + "import json\n", + "\n", + "import boto3" ] }, { @@ -51,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +92,8 @@ " # Serialize updated JSON\n", " updated_json = json.dumps(data)\n", "\n", - " # Upload updated JSON file back to S3, commented out the line below so it doesn't actually upload\n", + " # Upload updated JSON file back to S3\n", + " # commented out the line below so it doesn't actually upload\n", " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", " print(f\"Updated {key}\")" ] diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..bc082ea3 --- /dev/null +++ b/uv.lock @@ -0,0 +1,419 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, +] + +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.11.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/98/0bf930c4f97d0266b58a89e36c015f56232c52b5d2f207215d48cca9e8f7/platformdirs-4.11.2.tar.gz", hash = "sha256:3a2ae5fca3520a01ab1be8b45613537f52ddf5b5f6f53d88233892dfbf0cd82d", size = 32716, upload-time = "2026-08-10T15:48:06.092Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl", hash = "sha256:7f89089b6ea71bda7962953edcf784b2e2d9d285b40ad88be2bb75c6e9d82ab4", size = 23361, upload-time = "2026-08-10T15:48:04.855Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/3a/ddb78f32a0814e66b18a099377a106a2dcdce92d86a034d69d65df9b256e/pre_commit-4.6.1.tar.gz", hash = "sha256:03e809865c7d178b9979d06c761fcbfe6808fdaded8581a745bb110e52050421", size = 198646, upload-time = "2026-07-21T20:56:58.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, +] + +[[package]] +name = "pystac" +version = "1.8.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/02/4a4bf2aceacd8e8c293434a454a42cab75de07df3f3d15e3dbd09e901500/pystac-1.8.4.tar.gz", hash = "sha256:ab9b93d16c4cca80e3c225cd5815dbaec610ee9320e2119857db17481b9bd2e5", size = 135011, upload-time = "2023-09-25T19:11:45.094Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/528355f091d6522e5f11e02b1309cb4713e21ef8f772dd9b38e8276da011/pystac-1.8.4-py3-none-any.whl", hash = "sha256:0d9917bf3abc71fca3edfb2dd5f66be44b22a63774cda520c81484a126fa780e", size = 176247, upload-time = "2023-09-25T19:11:42.828Z" }, +] + +[package.optional-dependencies] +validation = [ + { name = "jsonschema" }, +] + +[[package]] +name = "pytest" +version = "7.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e5/d0/18209bb95db8ee693a9a04fe056ab0663c6d6b1baf67dd50819dd9cd4bd7/pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069", size = 1354640, upload-time = "2023-09-07T18:45:54.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/d0/e192c4275aecabf74faa1aacd75ef700091913236ec78b1a98f62a2412ee/pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002", size = 324497, upload-time = "2023-09-07T18:45:52.001Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-discovery" +version = "1.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/b7/ac44da2cf0e53ada0e419033c2d058219c95dc1403126f163304c9e814b1/python_discovery-1.5.2.tar.gz", hash = "sha256:45fd4f20a4e3f9b7bf2e0817870bc8e3b320a19658da177af800768c82dbf354", size = 82350, upload-time = "2026-08-12T14:05:26.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/45/689603d04b3bb8d7faa00f25c24acef993aab7813b3dbbfc472a459ab0b5/python_discovery-1.5.2-py3-none-any.whl", hash = "sha256:3e338c2d0f15dfaeea57493f4c2c6caebe0e998ea815c30ae8bf8ee21f1112d3", size = 38350, upload-time = "2026-08-12T14:05:25.113Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/e5/af35f7ea75cf72f2cd079c95ee16797de7cd71f29ea7c68ae5ce7be1eda0/PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", size = 125201, upload-time = "2023-07-18T00:00:23.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/0d/26fb23e8863e0aeaac0c64e03fd27367ad2ae3f3cccf3798ee98ce160368/PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", size = 187867, upload-time = "2023-07-17T23:57:34.35Z" }, + { url = "https://files.pythonhosted.org/packages/28/09/55f715ddbf95a054b764b547f617e22f1d5e45d83905660e9a088078fe67/PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", size = 167530, upload-time = "2023-07-17T23:57:36.975Z" }, + { url = "https://files.pythonhosted.org/packages/5e/94/7d5ee059dfb92ca9e62f4057dcdec9ac08a9e42679644854dc01177f8145/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", size = 732244, upload-time = "2023-07-17T23:57:43.774Z" }, + { url = "https://files.pythonhosted.org/packages/06/92/e0224aa6ebf9dc54a06a4609da37da40bb08d126f5535d81bff6b417b2ae/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", size = 752871, upload-time = "2023-07-17T23:57:51.921Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/efd033ab7199a0b2044dab3b9f7a4f6670e6a52c089de572e928d2873b06/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", size = 757729, upload-time = "2023-07-17T23:57:59.865Z" }, + { url = "https://files.pythonhosted.org/packages/03/5c/c4671451b2f1d76ebe352c0945d4cd13500adb5d05f5a51ee296d80152f7/PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", size = 748528, upload-time = "2023-08-28T18:43:23.207Z" }, + { url = "https://files.pythonhosted.org/packages/73/9c/766e78d1efc0d1fca637a6b62cea1b4510a7fb93617eb805223294fef681/PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", size = 130286, upload-time = "2023-07-17T23:58:02.964Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/65bb4b2d7908044963ebf614fe0fdb080773fc7030d7e39c8d3eddcd4257/PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", size = 144699, upload-time = "2023-07-17T23:58:05.586Z" }, + { url = "https://files.pythonhosted.org/packages/bc/06/1b305bf6aa704343be85444c9d011f626c763abb40c0edc1cad13bfd7f86/PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", size = 178692, upload-time = "2023-08-28T18:43:24.924Z" }, + { url = "https://files.pythonhosted.org/packages/84/02/404de95ced348b73dd84f70e15a41843d817ff8c1744516bf78358f2ffd2/PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", size = 165622, upload-time = "2023-08-28T18:43:26.54Z" }, + { url = "https://files.pythonhosted.org/packages/c7/4c/4a2908632fc980da6d918b9de9c1d9d7d7e70b2672b1ad5166ed27841ef7/PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", size = 696937, upload-time = "2024-01-18T20:40:22.92Z" }, + { url = "https://files.pythonhosted.org/packages/b4/33/720548182ffa8344418126017aa1d4ab4aeec9a2275f04ce3f3573d8ace8/PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", size = 724969, upload-time = "2023-08-28T18:43:28.56Z" }, + { url = "https://files.pythonhosted.org/packages/4f/78/77b40157b6cb5f2d3d31a3d9b2efd1ba3505371f76730d267e8b32cf4b7f/PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", size = 712604, upload-time = "2023-08-28T18:43:30.206Z" }, + { url = "https://files.pythonhosted.org/packages/2e/97/3e0e089ee85e840f4b15bfa00e4e63d84a3691ababbfea92d6f820ea6f21/PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", size = 126098, upload-time = "2023-08-28T18:43:31.835Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/fbade56564ad486809c27b322d0f7e6a89c01f6b4fe208402e90d4443a99/PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", size = 138675, upload-time = "2023-08-28T18:43:33.613Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "veda-data" +version = "0.1.0" +source = { virtual = "." } + +[package.dev-dependencies] +dev = [ + { name = "pre-commit" }, + { name = "pystac", extra = ["validation"] }, + { name = "pytest" }, +] +scripts = [ + { name = "pyyaml" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pre-commit", specifier = "==4.6.1" }, + { name = "pystac", extras = ["validation"], specifier = "==1.8.4" }, + { name = "pytest", specifier = "==7.4.2" }, +] +scripts = [{ name = "pyyaml", specifier = "==6.0.1" }] + +[[package]] +name = "virtualenv" +version = "21.7.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/dc/a6eb1ddfa7f1e390fa599b078453c97edb3f6f846b34fb4eac3e8ea16401/virtualenv-21.7.4.tar.gz", hash = "sha256:c9d960c95fa458171e58222a5ccab7465298e4b6559977865e627c4719f1e825", size = 5345511, upload-time = "2026-08-10T22:54:33.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/4c/eb2f52aeeaf30dbd073d315a251a63ae2b8263171ec4428c135140cb0802/virtualenv-21.7.4-py3-none-any.whl", hash = "sha256:376ec93cd6aab3044fa395d7db226db38043b7b5748948044b2a87168525e843", size = 5324444, upload-time = "2026-08-10T22:54:31.515Z" }, +] From 28a4a824c5ea3233337a8948eec4db1e15cea5b1 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 13 Aug 2026 11:13:44 -0500 Subject: [PATCH 02/10] chore: add locked flag --- .github/dependabot.yml | 2 +- .github/workflows/ci.yaml | 4 ++-- .github/workflows/pr.yml | 4 ++-- .github/workflows/promote.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 415eedec..12515289 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -38,4 +38,4 @@ updates: patterns: - "*" commit-message: - prefix: "ci(deps):" + prefix: "chore(deps):" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 119a9a77..84078d76 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,9 +14,9 @@ jobs: matrix: include: - name: test - run: uv run --group dev pytest + run: uv run --locked --group dev pytest - name: lint - run: uv run --group dev pre-commit run --all-files + run: uv run --locked --group dev pre-commit run --all-files steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index d874e308..0c43b2bf 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -119,7 +119,7 @@ jobs: AIRFLOW_JWT_SECRET: ${{ secrets.AIRFLOW_JWT_SECRET }} AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }} run: | - uv sync --group scripts + uv sync --locked--group scripts # Track successful publications all_failed=true @@ -301,7 +301,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES collection_ids="" - uv sync --group scripts + uv sync --locked --group scripts for file in ${PUBLISHED_COLLECTION_FILES}; do collection_id=$(python3 ./scripts/generate_mdx.py "$file") collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//') diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 9a04bc5a..f237c173 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -75,7 +75,7 @@ jobs: AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }} run: | - uv sync --group scripts + uv sync --locked --group scripts for file in downloaded-files/*.json; do echo "Processing $file" # Check if collection key exists to determine if it's a dataset or collection From 5dc7470a3294fa42e8102ac90e839ea73cb9dbf2 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 13 Aug 2026 13:12:07 -0500 Subject: [PATCH 03/10] chore: remove redundant path --- scripts/generate_mdx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_mdx.py b/scripts/generate_mdx.py index 4db953bf..a5f3ddcf 100644 --- a/scripts/generate_mdx.py +++ b/scripts/generate_mdx.py @@ -119,7 +119,7 @@ def safe_open_w(path): # Write the combined content back to the file output_filepath = ( - Path(curr_directory) + curr_directory / f"../ingestion-data/dataset-mdx/{input_data['collection']}.data.mdx" ) with safe_open_w(str(output_filepath)) as ofile: From 2ed5d35c745bfb996ec786e149c98a70854b16f4 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Sat, 15 Aug 2026 23:39:04 -0500 Subject: [PATCH 04/10] chore: missing space --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0c43b2bf..e8bbdd13 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -119,7 +119,7 @@ jobs: AIRFLOW_JWT_SECRET: ${{ secrets.AIRFLOW_JWT_SECRET }} AIRFLOW_JWT_SUB: ${{ secrets.AIRFLOW_JWT_SUB }} run: | - uv sync --locked--group scripts + uv sync --locked --group scripts # Track successful publications all_failed=true From 5c4d20d0edcda445e52ba310c20129448075caa5 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 19 Aug 2026 12:07:23 -0500 Subject: [PATCH 05/10] chore: update pre-commit --- .pre-commit-config.yaml | 5 +++++ pyproject.toml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 284a8e63..17149243 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,3 +24,8 @@ repos: - "pyproject.toml" - "--configPointer" - "/tool/markdownlint-cli2" + + - repo: https://github.com/astral-sh/uv-pre-commit + rev: 0.12.5 + hooks: + - id: uv-lock diff --git a/pyproject.toml b/pyproject.toml index 9a13a271..7bb2023b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,6 @@ select = [ "TC", # flake8-type-checking "ISC", # flake8-implicit-str-concat "ICN", # flake8-import-conventions - "BLE", # flake8-blind-except ] ignore = [ "BLE001", # Blind Exceptions From c1090ddfb2e4f924c9a356be4d8d7404acd2dedd Mon Sep 17 00:00:00 2001 From: Jamison French Date: Sat, 22 Aug 2026 01:58:37 -0500 Subject: [PATCH 06/10] chore: upgrade pytest --- pyproject.toml | 2 +- uv.lock | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7bb2023b..0761cc55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [] dev = [ "pre-commit==4.6.1", "pystac[validation]==1.8.4", - "pytest==7.4.2", + "pytest==9.0.3", ] scripts = [ "pyyaml", diff --git a/uv.lock b/uv.lock index bc082ea3..1564f9ab 100644 --- a/uv.lock +++ b/uv.lock @@ -144,6 +144,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + [[package]] name = "pystac" version = "1.8.4" @@ -163,17 +172,18 @@ validation = [ [[package]] name = "pytest" -version = "7.4.2" +version = "9.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/d0/18209bb95db8ee693a9a04fe056ab0663c6d6b1baf67dd50819dd9cd4bd7/pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069", size = 1354640, upload-time = "2023-09-07T18:45:54.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/d0/e192c4275aecabf74faa1aacd75ef700091913236ec78b1a98f62a2412ee/pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002", size = 324497, upload-time = "2023-09-07T18:45:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] [[package]] @@ -399,7 +409,7 @@ scripts = [ dev = [ { name = "pre-commit", specifier = "==4.6.1" }, { name = "pystac", extras = ["validation"], specifier = "==1.8.4" }, - { name = "pytest", specifier = "==7.4.2" }, + { name = "pytest", specifier = "==9.0.3" }, ] scripts = [{ name = "pyyaml", specifier = "==6.0.1" }] From 2c60345910cd076183e5090c7fc92cfb76198f79 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 14 Oct 2026 17:26:29 -0500 Subject: [PATCH 07/10] style: pre-commit checks --- scripts/airflow_api.py | 20 ++++++++++---------- uv.lock | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/scripts/airflow_api.py b/scripts/airflow_api.py index 583f840a..ae7f8c7a 100644 --- a/scripts/airflow_api.py +++ b/scripts/airflow_api.py @@ -2,13 +2,12 @@ Supports Airflow 2 and 3 (API v1 / v2) """ +import http.client +import json import os import uuid -import json -import http.client - -from typing import Dict, Any from base64 import b64encode +from typing import Any class AirflowAPIError(Exception): @@ -20,8 +19,8 @@ def __init__(self, msg, status_code=None): def _build_request_body( - conf: Dict[str, Any], dag_id: str, note: str = "", api_version: str = "2" -) -> Dict[str, Any]: + conf: dict[str, Any], dag_id: str, note: str = "", api_version: str = "2" +) -> dict[str, Any]: """Build the request body for the DagRun""" run_id = os.getenv("GITHUB_RUN_ID") if run_id: @@ -45,9 +44,10 @@ def _build_request_body( def _generate_jwt_token(secret: str, sub: str, expiration_time: int = 3600) -> str: """Generate a HS512 JWT Token for Airflow API authentication""" - import jwt import time + import jwt + payload = { "iss": "airflow", "sub": sub, @@ -62,11 +62,11 @@ def _generate_jwt_token(secret: str, sub: str, expiration_time: int = 3600) -> s def trigger_dag_run( base_api_url: str, dag_id: str, - conf: Dict[str, Any], + conf: dict[str, Any], username: str, password: str, api_version: str = None, -) -> Dict[str, Any]: +) -> dict[str, Any]: """ Trigger a DAG run with version-aware API handling @@ -120,4 +120,4 @@ def trigger_dag_run( except AirflowAPIError: raise except Exception as e: - raise AirflowAPIError(f"Failed to trigger the DAG run: {str(e)}") + raise AirflowAPIError(f"Failed to trigger the DAG run: {str(e)}") from e diff --git a/uv.lock b/uv.lock index 1564f9ab..243a8569 100644 --- a/uv.lock +++ b/uv.lock @@ -153,6 +153,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + [[package]] name = "pystac" version = "1.8.4" @@ -400,6 +409,7 @@ dev = [ { name = "pytest" }, ] scripts = [ + { name = "pyjwt" }, { name = "pyyaml" }, ] @@ -411,7 +421,10 @@ dev = [ { name = "pystac", extras = ["validation"], specifier = "==1.8.4" }, { name = "pytest", specifier = "==9.0.3" }, ] -scripts = [{ name = "pyyaml", specifier = "==6.0.1" }] +scripts = [ + { name = "pyjwt" }, + { name = "pyyaml" }, +] [[package]] name = "virtualenv" From 190fae01a28fca0d121d9516b91d91ac37077eef Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 14 Oct 2026 19:18:21 -0500 Subject: [PATCH 08/10] chore: restore ipynb, exclude ipynb from pre-commit checks --- .pre-commit-config.yaml | 25 +++- .../Transfer-MCP-veda-data-store.ipynb | 44 +++---- .../generate-cmip6-kerchunk-historical.ipynb | 38 +++--- .../publish-cmip6-kerchunk-stac.ipynb | 3 +- .../collection-and-item-ingest.ipynb | 25 ++-- .../collection-metadata-reconciliation.ipynb | 39 +++--- .../create-stac-for-cmr-collections.ipynb | 15 +-- .../generate-rasi-cogs-from-netcdf3.ipynb | 23 ++-- .../publish-hls-provided-item-metadata.ipynb | 20 +-- .../landsat-lakes-discovery/aral-sea.ipynb | 29 ++-- .../lake-balaton.ipynb | 39 ++---- .../landsat-lakes-discovery/lake-biwa.ipynb | 41 +++--- .../mirror-landsat-demos.ipynb | 36 +++-- .../landsat-lakes-discovery/tonle-sap.ipynb | 41 +++--- .../landsat-lakes-discovery/vanern.ipynb | 39 ++---- .../cmip6-generate-organize-cogs.ipynb | 124 ++++++++---------- .../cmip6-generate-stac.ipynb | 5 +- transformation-scripts/update-hrefs.ipynb | 5 +- 18 files changed, 250 insertions(+), 341 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17149243..7e805ec5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,16 +3,37 @@ repos: rev: v6.0.0 hooks: - id: check-yaml + exclude: ^.*\.ipynb$ - id: trailing-whitespace - exclude: ^ingestion-data/.*$ + exclude: ^(ingestion-data/.*|.*\.ipynb)$ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.16.1 hooks: - id: ruff-check args: [--fix] + exclude: ^.*\.ipynb$ - id: ruff-format - types_or: [python, pyi, jupyter, pyproject] + types_or: [python, pyi, pyproject] + exclude: ^.*\.ipynb$ + + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.23.2 + hooks: + - id: markdownlint-cli2 + args: + - "--fix" + - "--config" + - "pyproject.toml" + - "--configPointer" + - "/tool/markdownlint-cli2" + exclude: ^.*\.ipynb$ + + - repo: https://github.com/astral-sh/uv-pre-commit + rev: 0.12.5 + hooks: + - id: uv-lock + exclude: ^.*\.ipynb$ - repo: https://github.com/DavidAnson/markdownlint-cli2 rev: v0.23.2 diff --git a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb index 34bdf207..9bc01250 100644 --- a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb +++ b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb @@ -2,14 +2,14 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 105, "id": "0a619c84-a55d-4f93-99e0-c31a42d52932", "metadata": {}, "outputs": [], "source": [ "import base64\n", + "import glob\n", "import json\n", - "from pathlib import Path\n", "\n", "import boto3\n", "import requests\n", @@ -18,18 +18,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "e1a873f2-7b72-4245-a3d0-a1a3d38bde97", "metadata": {}, "outputs": [], "source": [ - "discover_items_file_paths = list(\n", - " Path(\"../ingestion-data/discovery-items\").glob(\"*.json\")\n", - ")\n", - "\n", - "# discover_items_ids = [\n", - "# i.split(\"/\")[-1].split(\".\")[0] for i in discover_items_file_paths\n", - "# ]" + "discover_items_file_paths = glob.glob(\"../ingestion-data/discovery-items/*.json\")\n", + "# discover_items_ids = [i.split(\"/\")[-1].split(\".\")[0] for i in discover_items_file_paths]" ] }, { @@ -54,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 119, "id": "a3457a89-e5e3-4873-a524-51a914488b88", "metadata": {}, "outputs": [ @@ -307,7 +302,7 @@ ], "source": [ "for filename in discover_items_file_paths:\n", - " with Path(filename).open() as f:\n", + " with open(filename) as f:\n", " dataset = json.load(f)\n", " if not isinstance(dataset, list):\n", " dataset = [dataset]\n", @@ -318,24 +313,24 @@ " if (\n", " i[\"collection\"] in staging_collections\n", " ): # only create config for collections in staging\n", - " config = {}\n", + " config = dict()\n", " config[\"origin_bucket\"] = i.get(\"bucket\", \"\")\n", " config[\"origin_prefix\"] = i.get(\"prefix\", \"\")\n", " config[\"target_bucket\"] = \"veda-data-store\"\n", " config[\"collection\"] = i.get(\"collection\")\n", "\n", " if \"assets\" in i: # conditional for multi asset discovery items\n", - " for _k, v in i[\"assets\"].items():\n", + " for k, v in i[\"assets\"].items():\n", " config[\"filename_regex\"] = v.get(\"regex\")\n", " transfer_config.append(config.copy())\n", " else:\n", " config[\"filename_regex\"] = i.get(\"filename_regex\", \"\")\n", " transfer_config.append(config)\n", "\n", - " with Path(f\"../ingestion-data/transfer-config/{id}.json\").open(\n", - " \"w\", encoding=\"utf-8\"\n", + " with open(\n", + " f\"../ingestion-data/transfer-config/{id}.json\", \"w\", encoding=\"utf-8\"\n", " ) as file:\n", - " print(f\"writing file: ../ingestion-data/transfer-config/{id}\")\n", + " print(f\"writing file: ingestion_data/transfer-config/{id}\")\n", " json.dump(transfer_config, file, ensure_ascii=False, indent=4)" ] }, @@ -381,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 118, "id": "5a774091-a239-409a-9360-03dec3839ec7", "metadata": {}, "outputs": [ @@ -1453,13 +1448,11 @@ } ], "source": [ - "transfer_items_file_paths = list(\n", - " Path(\"../ingestion-data/transfer-config\").glob(\"*.json\")\n", - ")\n", + "transfer_items_file_paths = glob.glob(\"../ingestion-data/transfer-config/*.json\")\n", "\n", "for transfer_item_file in transfer_items_file_paths:\n", " print(f\"Transferring using config: {transfer_item_file}\")\n", - " with Path(transfer_item_file).open() as f:\n", + " with open(transfer_item_file) as f:\n", " transfer_configs = json.load(f)\n", " for transfer_config in transfer_configs:\n", " mwaa_cli_token = get_mwaa_cli_token(\n", @@ -1476,10 +1469,7 @@ " pass\n", " else:\n", " if not dry_run:\n", - " raw_data = (\n", - " \"dags trigger veda_transfer \"\n", - " f\"--conf '{json.dumps(transfer_config)}'\"\n", - " )\n", + " raw_data = f\"dags trigger veda_transfer --conf '{json.dumps(transfer_config)}'\"\n", " mwaa_response = requests.post(\n", " f\"https://{mwaa_cli_token['WebServerHostname']}/aws_mwaa/cli\",\n", " headers={\n", @@ -1498,7 +1488,7 @@ " print(mwaa_response.status_code)\n", " print(f\"stderr: {mwaa_std_err_message}\")\n", " print(f\"stdout: {mwaa_std_out_message}\")\n", - " except Exception:\n", + " except:\n", " print(mwaa_response)" ] }, diff --git a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb index 07d65fe8..5010d403 100644 --- a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb +++ b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb @@ -16,12 +16,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "9f7250a6", "metadata": {}, "outputs": [], "source": [ - "from pathlib import Path\n", + "import os\n", "from tempfile import TemporaryDirectory\n", "\n", "import boto3\n", @@ -40,8 +40,7 @@ "# If this code were re-used for a protected bucket, anon should be False.\n", "anon = True\n", "# Note: We are only using the historical data in this example.\n", - "# More years of data are available from multiple Shared Socio-Economic Pathways (SSPs)\n", - "# in the s3://nex-gddp-cmip6 bucket.\n", + "# More years of data are available from multiple Shared Socio-Economic Pathways (SSPs) in the s3://nex-gddp-cmip6 bucket.\n", "s3_path = f\"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1*/{variable}/*\"" ] }, @@ -119,21 +118,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "b046c17c", "metadata": {}, "outputs": [], "source": [ - "so = {\n", - " \"mode\": \"rb\",\n", - " \"anon\": anon,\n", - " \"default_fill_cache\": False,\n", - " \"default_cache_type\": \"first\",\n", - "}\n", + "so = dict(mode=\"rb\", anon=anon, default_fill_cache=False, default_cache_type=\"first\")\n", "\n", "\n", - "# Use Kerchunk's `SingleHdf5ToZarr` method\n", - "# to create a `Kerchunk` index from a NetCDF file.\n", + "# Use Kerchunk's `SingleHdf5ToZarr` method to create a `Kerchunk` index from a NetCDF file.\n", "def generate_json_reference(u):\n", " with fs_read.open(u, **so) as infile:\n", " fname = u.split(\"/\")[-1].strip(\".nc\")\n", @@ -142,8 +135,8 @@ "\n", "\n", "def write_json(fname, reference_json, temp_dir):\n", - " outf = Path(temp_dir) / f\"{fname}.json\"\n", - " with outf.open(\"wb\") as f:\n", + " outf = os.path.join(temp_dir, f\"{fname}.json\")\n", + " with open(outf, \"wb\") as f:\n", " f.write(reference_json)\n", " return outf" ] @@ -194,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "794be824-0833-4229-a8b4-10b071eced26", "metadata": { "tags": [] @@ -264,8 +257,7 @@ "gateway = Gateway()\n", "clusters = gateway.list_clusters()\n", "\n", - "# connect to an existing cluster\n", - "# this is useful when the kernel shutdown in the middle of an interactive session\n", + "# connect to an existing cluster - this is useful when the kernel shutdown in the middle of an interactive session\n", "if clusters:\n", " cluster = gateway.connect(clusters[0].name)\n", "else:\n", @@ -362,7 +354,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "55ced4a6-0c38-43c8-8c6e-a4fff8a9f7bb", "metadata": {}, "outputs": [ @@ -376,8 +368,8 @@ ], "source": [ "output_fname = f\"combined_CMIP6_daily_{model}_{variable}_kerchunk.json\"\n", - "output_location = Path(temp_dir) / output_fname\n", - "with output_location.open(\"wb\") as f:\n", + "output_location = os.path.join(temp_dir, output_fname)\n", + "with open(f\"{output_location}\", \"wb\") as f:\n", " print(f\"Writing combined kerchunk reference file {output_location}\")\n", " f.write(ujson.dumps(multi_kerchunk).encode())" ] @@ -398,7 +390,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "ac76c0c0", "metadata": {}, "outputs": [ @@ -848,7 +840,7 @@ ], "source": [ "# Check the data\n", - "ds = xr.open_dataset(m, engine=\"zarr\", backend_kwargs={\"consolidated\": False})\n", + "ds = xr.open_dataset(m, engine=\"zarr\", backend_kwargs=dict(consolidated=False))\n", "display(ds)" ] }, diff --git a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb index 4e9e9a0a..c98c1547 100644 --- a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb +++ b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "3c3e536f-f61f-4ad7-a0b4-414387d634d1", "metadata": { "tags": [] @@ -34,7 +34,6 @@ "#!pip install xstac\n", "import fsspec\n", "import pystac\n", - "import requests\n", "import xarray as xr\n", "import xstac" ] diff --git a/transformation-scripts/collection-and-item-ingest.ipynb b/transformation-scripts/collection-and-item-ingest.ipynb index 1800e13d..b30fd5e4 100644 --- a/transformation-scripts/collection-and-item-ingest.ipynb +++ b/transformation-scripts/collection-and-item-ingest.ipynb @@ -31,8 +31,8 @@ "metadata": {}, "outputs": [], "source": [ + "import glob\n", "import json\n", - "from pathlib import Path\n", "\n", "import requests\n", "from cognito_client import CognitoClient" @@ -65,7 +65,7 @@ " \"combined_CMIP6_daily_GISS-E2-1-G_tas_kerchunk_DEMO\",\n", "]\n", "\n", - "json_file_paths = Path(\"../ingestion-data/collections\").glob(\"*.json\")\n", + "json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n", "filtered_list = [\n", " item\n", " for item in json_file_paths\n", @@ -75,16 +75,10 @@ " )\n", "]\n", "\n", - "\n", - "def load_json(file_path: str | Path) -> dict:\n", - " with Path(file_path).open() as f:\n", - " return json.load(f)\n", - "\n", - "\n", "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in filtered_list\n", - " if \"id\" in (data := load_json(file_path))\n", + " if \"id\" in (data := json.load(open(file_path)))\n", "]" ] }, @@ -189,18 +183,15 @@ " response.raise_for_status()\n", " if response.status_code == 201:\n", " print(\n", - " \"Request was successful. \"\n", - " f\"Find the updated collection at {collection_url}\"\n", + " f\"Request was successful. Find the updated collection at {collection_url}\"\n", " )\n", " else:\n", " print(\n", - " f\"Updating {collection_id} failed. \"\n", - " f\"Request failed with status code: {response.status_code}\"\n", + " f\"Updating {collection_id} failed. Request failed with status code: {response.status_code}\"\n", " )\n", " except requests.RequestException as e:\n", " print(\n", - " f\"Updating {collection_id} failed. \"\n", - " f\"An error occurred during the request: {e}\"\n", + " f\"Updating {collection_id} failed. An error occurred during the request: {e}\"\n", " )\n", " except Exception as e:\n", " print(\n", @@ -251,10 +242,10 @@ " file_path = item[\"filePath\"]\n", "\n", " try:\n", - " with Path(file_path).open(encoding=\"utf-8\") as file:\n", + " with open(file_path, encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", - " # Publish updated collection to the target ingestion `api/collections` endpoint\n", + " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", " post_collection(collection, collection_id)\n", "\n", " except requests.RequestException as e:\n", diff --git a/transformation-scripts/collection-metadata-reconciliation.ipynb b/transformation-scripts/collection-metadata-reconciliation.ipynb index d5337495..900fc18f 100644 --- a/transformation-scripts/collection-metadata-reconciliation.ipynb +++ b/transformation-scripts/collection-metadata-reconciliation.ipynb @@ -16,12 +16,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ + "import glob\n", "import json\n", - "from pathlib import Path\n", "\n", "import requests\n", "from cognito_client import CognitoClient" @@ -36,22 +36,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "json_file_paths = Path(\"../ingestion-data/collections\").glob(\"*.json\")\n", - "\n", - "\n", - "def load_json(file_path: str | Path) -> dict:\n", - " with Path(file_path).open() as f:\n", - " return json.load(f)\n", - "\n", + "json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n", "\n", "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in json_file_paths\n", - " if \"id\" in (data := load_json(file_path))\n", + " if \"id\" in (data := json.load(open(file_path)))\n", "]" ] }, @@ -143,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -156,18 +150,15 @@ " response.raise_for_status()\n", " if response.status_code == 201:\n", " print(\n", - " \"Request was successful. \"\n", - " f\"Find the updated collection at {collection_url}\"\n", + " f\"Request was successful. Find the updated collection at {collection_url}\"\n", " )\n", " else:\n", " print(\n", - " f\"Updating {collection_id} failed. \"\n", - " f\"Request failed with status code: {response.status_code}\"\n", + " f\"Updating {collection_id} failed. Request failed with status code: {response.status_code}\"\n", " )\n", " except requests.RequestException as e:\n", " print(\n", - " f\"Updating {collection_id} failed. \"\n", - " f\"An error occurred during the request: {e}\"\n", + " f\"Updating {collection_id} failed. An error occurred during the request: {e}\"\n", " )\n", " except Exception as e:\n", " print(\n", @@ -186,14 +177,15 @@ "\n", "\n", "def retain_external_links(existing_links, retrieved_links):\n", - " unique_hrefs = {link.get(\"href\") for link in existing_links}\n", + " unique_hrefs = set(link.get(\"href\") for link in existing_links)\n", " additional_external_links = [\n", " link\n", " for link in retrieved_links\n", " if link.get(\"rel\") == \"external\" and link.get(\"href\") not in unique_hrefs\n", " ]\n", "\n", - " return existing_links + additional_external_links" + " retained_links = existing_links + additional_external_links\n", + " return retained_links" ] }, { @@ -205,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -383,7 +375,7 @@ " retrieved_summaries = json_response.get(\"summaries\", {})\n", " retrieved_links = json_response.get(\"links\", {})\n", "\n", - " with Path(file_path).open(encoding=\"utf-8\") as file:\n", + " with open(file_path, encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", " existing_summaries = collection.get(\"summaries\", {})\n", @@ -394,8 +386,7 @@ " )\n", " collection[\"links\"] = retain_external_links(existing_links, retrieved_links)\n", "\n", - " # Publish the updated collection to the target ingestion\n", - " # `api/collections` endpoint\n", + " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", " post_reconciled_collection(collection, collection_id)\n", "\n", " except requests.RequestException as e:\n", diff --git a/transformation-scripts/create-stac-for-cmr-collections.ipynb b/transformation-scripts/create-stac-for-cmr-collections.ipynb index 7cf71bcf..4a63b35b 100644 --- a/transformation-scripts/create-stac-for-cmr-collections.ipynb +++ b/transformation-scripts/create-stac-for-cmr-collections.ipynb @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "67e37cc9-6022-46c2-aa20-4bb87dd502f9", "metadata": { "tags": [] @@ -71,7 +71,6 @@ "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", "import earthaccess\n", "import pandas as pd\n", @@ -98,7 +97,7 @@ }, "outputs": [ { - "name": "stdout", + "name": "stdin", "output_type": "stream", "text": [ "Enter your Earthdata Login username: aimeeb\n", @@ -450,7 +449,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "ae799bbb-06f8-499b-be53-c9ab86762ca7", "metadata": { "tags": [] @@ -458,7 +457,7 @@ "outputs": [], "source": [ "cube_variables = collection.to_dict()[\"cube:variables\"]\n", - "for variable in cube_variables:\n", + "for variable in cube_variables.keys():\n", " cube_variables[variable][\"shape\"][0] = None" ] }, @@ -520,14 +519,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "42d3023b-b551-43c1-9984-22553eb41fb1", "metadata": {}, "outputs": [], "source": [ - "with Path(f\"../ingestion-data/staging/collections/{collection.id}.json\").open(\n", - " \"w+\"\n", - ") as f:\n", + "with open(f\"../ingestion-data/staging/collections/{collection.id}.json\", \"w+\") as f:\n", " f.write(json.dumps(collection.to_dict(), indent=2))" ] } diff --git a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb index bdfe5f8d..6345c1bd 100644 --- a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb +++ b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb @@ -42,12 +42,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "aa94a74d-58ff-4527-b397-13016133ac71", "metadata": {}, "outputs": [], "source": [ - "from pathlib import Path\n", + "import os\n", "\n", "import pandas as pd\n", "import rio_cogeo.cogeo\n", @@ -683,7 +683,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "6c4e69b1-cb7f-4845-b656-8767a246832b", "metadata": {}, "outputs": [ @@ -700,8 +700,7 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, \"\n", - " f\"last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -723,7 +722,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "08c31207-1bce-4573-9edd-adc441e476a8", "metadata": {}, "outputs": [ @@ -740,8 +739,7 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, \"\n", - " f\"last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -814,7 +812,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "3ad681ca-1617-46d9-a282-2b60c3c749ab", "metadata": {}, "outputs": [ @@ -831,8 +829,7 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, \"\n", - " f\"last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -1549,7 +1546,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "13fa9162-46d8-4466-801b-2e2e49458154", "metadata": {}, "outputs": [], @@ -1562,7 +1559,7 @@ "dates = pd.date_range(\"2014-01-01\", \"2014-03-01\", freq=\"MS\")\n", "\n", "# make sure the dir exists\n", - "Path(\"COGS\").mkdir(parents=True, exist_ok=True)\n", + "os.makedirs(\"COGS\", exist_ok=True)\n", "\n", "for date in dates:\n", " datestr = date.strftime(\"%Y%m\")\n", diff --git a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb index 254c2d5c..8a1fa9bd 100644 --- a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb +++ b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb @@ -146,8 +146,7 @@ " )\n", " item_dict[\"links\"] = links\n", "\n", - " # Update the stac version for these items from \"stac_version\": \"1.0.0-beta.2\"\n", - " # and touch up metadata to meet 1.0.0 spec\n", + " # Update the stac version for these items from \"stac_version\": \"1.0.0-beta.2\" and touch up metadata to meet 1.0.0 spec\n", " item_dict[\"stac_version\"] = \"1.0.0\"\n", "\n", " # Add full extension hrefs https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md#stac_extensions\n", @@ -166,7 +165,7 @@ " item_assets = item_dict[\"assets\"]\n", " # Previous location did not have data version number\n", " old_prefix = collection_id.replace(\"-002-\", \"-\")\n", - " for asset_key in item_assets:\n", + " for asset_key in item_assets.keys():\n", " new_href = item_assets[asset_key][\"href\"].replace(old_prefix, collection_id)\n", " item_assets[asset_key][\"href\"] = new_href\n", "\n", @@ -238,16 +237,17 @@ " )\n", " if verbose:\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n\"\n", - " f\"{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " if publish_response.reason != \"Created\":\n", + " if not publish_response.reason == \"Created\":\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n\"\n", - " f\"{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " elif verbose:\n", - " print(f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\")" + " else:\n", + " if verbose:\n", + " print(\n", + " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\"\n", + " )" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb index ab3ca36c..bbf8592e 100644 --- a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb @@ -178,15 +178,7 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": (\n", - " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", - " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", - " \"missions and instruments are combined in this single collection for \"\n", - " \"the purpose of visualizing RGB images. Note that the full item \"\n", - " \"records are not interchangeable across instruments for purposes \"\n", - " \"other than visualization and that the assets available are not the \"\n", - " \"same for each instrument.\"\n", - " ),\n", + " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -267,12 +259,11 @@ "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -325,7 +316,9 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " return input_string.replace(expected_prefix, replacement_prefix)\n", + " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", + "\n", + " return replaced_string\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -334,7 +327,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", + "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -369,9 +362,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] }, { @@ -381,9 +372,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb index 359fd8eb..3bd060c8 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb @@ -201,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "id": "dff29fbf-0001-42c0-9058-7fb93ab0396d", "metadata": { "tags": [] @@ -223,15 +223,7 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": (\n", - " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", - " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", - " \"missions and instruments are combined in this single collection for \"\n", - " \"the purpose of visualizing RGB images. Note that the full item records \"\n", - " \"are not interchangeable across instruments for purposes other than \"\n", - " \"visualization and that the assets available are not the same for each \"\n", - " \"instrument.\"\n", - " ),\n", + " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -304,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "id": "2bbb4330-787a-4ceb-99a4-a79a12c2f3a2", "metadata": { "tags": [] @@ -312,12 +304,11 @@ "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -356,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 105, "id": "ef5c5f88", "metadata": {}, "outputs": [], @@ -370,7 +361,9 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " return input_string.replace(expected_prefix, replacement_prefix)\n", + " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", + "\n", + " return replaced_string\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -379,7 +372,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", + "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -417,26 +410,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "id": "bf75d1ca", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 107, "id": "d80b42eb", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb index af4e83d6..aed6b28d 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb @@ -160,14 +160,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "d1558909-6bd8-4161-a710-4ce0f0502fc3", "metadata": {}, "outputs": [], "source": [ "collection_id = f\"landsat-c2l2-sr-lakes-{LAKE_NAME_SLUG}\"\n", "collection = {\n", - " \"id\": collection_id,\n", + " \"id\": f\"{collection_id}\",\n", " \"type\": \"Collection\",\n", " \"title\": f\"Selected Landsat 7 through 9 Surface Reflectance Scenes for {LAKE_NAME}\",\n", " \"extent\": {\n", @@ -180,15 +180,7 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": (\n", - " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", - " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", - " \"missions and instruments are combined in this single collection \"\n", - " \"for the purpose of visualizing RGB images. Note that the full item \"\n", - " \"records are not interchangeable across instruments for purposes \"\n", - " \"other than visualization and that the assets available are not \"\n", - " \"the same for each instrument.\"\n", - " ),\n", + " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -263,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "c8297cb0-b600-4b96-a3bd-1b2b8b1d05d1", "metadata": { "tags": [] @@ -271,12 +263,11 @@ "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -315,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "34e88a7c", "metadata": {}, "outputs": [], @@ -329,7 +320,9 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " return input_string.replace(expected_prefix, replacement_prefix)\n", + " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", + "\n", + " return replaced_string\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -338,7 +331,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", + "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -376,26 +369,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "5e419e70", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "23c57e6c", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb index e96a35ac..825b78b2 100644 --- a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb @@ -99,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "af3c723e-df35-4aed-abb1-f9c1dc78909b", "metadata": {}, "outputs": [ @@ -132,14 +132,12 @@ " # Start the new collection we will publish\n", " collection_dict = src_collection.to_dict(include_self_link=False)\n", "\n", - " # Special case for these landsat collections:\n", - " # cog_default assets were mistakenly added to item_assets so fix it\n", + " # Special case for these landsat collections: cog_default assets were mistakenly added to item_assets so fix it\n", " item_assets = collection_dict.get(\"item_assets\")\n", " item_assets.pop(\"cog_default\", None)\n", " collection_dict[\"item_assets\"] = item_assets\n", "\n", - " # Make sure the summaries object from the staged collection is included in\n", - " # the veda-data record (TODO check why this isn't passed through pystac to_dict)\n", + " # Make sure the summaries object from the staged collection is included in the veda-data record (TODO check why this isn't passed through pystac to_dict)\n", " collection_dict[\"summaries\"] = src_collection.summaries._summaries\n", "\n", " # Validate\n", @@ -175,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "ac9df7f0-308b-4cc9-9268-55a8a5812047", "metadata": {}, "outputs": [ @@ -361,14 +359,12 @@ " )\n", " item_dict[\"links\"] = links\n", "\n", - " # Special case for these landsat collections:\n", - " # cog_default assets were mistakenly added to item_assets so fix it\n", + " # Special case for these landsat collections: cog_default assets were mistakenly added to item_assets so fix it\n", " item_assets = item_dict.get(\"assets\")\n", " item_assets.pop(\"cog_default\", None)\n", " item_dict[\"assets\"] = item_assets\n", "\n", - " # Remove classification extension\n", - " # (many items do not have valid classification metadata)\n", + " # Remove classification extension (many items do not have valid classification metadata)\n", " extensions = item_dict[\"stac_extensions\"]\n", " item_dict[\"stac_extensions\"] = [\n", " i\n", @@ -393,17 +389,18 @@ " )\n", " if verbose:\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n\"\n", - " f\"{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " if publish_response.reason != \"Created\":\n", + " if not publish_response.reason == \"Created\":\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n\"\n", - " f\"{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", " failed_ingestions.append(item.id)\n", - " elif verbose:\n", - " print(f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\")\n", + " else:\n", + " if verbose:\n", + " print(\n", + " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\"\n", + " )\n", "\n", "\n", "if len(invalid_items):\n", @@ -423,7 +420,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "0d922785-dc2c-4bea-8205-5f69a47b68c7", "metadata": {}, "outputs": [ @@ -464,8 +461,7 @@ " target_item_count = len(target_item_collection)\n", "\n", " print(\n", - " f\"\\n{collection_id} {src_item_count=} {target_item_count=}\"\n", - " f\" OK={src_item_count == target_item_count}\"\n", + " f\"\\n{collection_id} {src_item_count=} {target_item_count=} OK={src_item_count==target_item_count}\"\n", " )\n", "\n", "print(\"\\nfin.\")" diff --git a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb index 1956a590..0a91948c 100644 --- a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb @@ -170,14 +170,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "c58bc898-4172-4e59-95dc-19b04273f019", "metadata": {}, "outputs": [], "source": [ "collection_id = f\"landsat-c2l2-sr-lakes-{LAKE_NAME_SLUG}\"\n", "collection = {\n", - " \"id\": collection_id,\n", + " \"id\": f\"{collection_id}\",\n", " \"type\": \"Collection\",\n", " \"title\": f\"Selected Landsat 7 through 9 Surface Reflectance Scenes for {LAKE_NAME}\",\n", " \"extent\": {\n", @@ -190,15 +190,7 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": (\n", - " \"Selected low-cloud-cover Landsat Collection-2 Level-2 \"\n", - " f\"Surface Reflectance scenes covering {LAKE_NAME}. Products from \"\n", - " \"multiple missions and instruments are combined in this single \"\n", - " \"collection for the purpose of visualizing RGB images. Note that \"\n", - " \"the full item records are not interchangeable across instruments \"\n", - " \"for purposes other than visualization and that the assets \"\n", - " \"available are not the same for each instrument.\"\n", - " ),\n", + " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -271,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "f4dbf0d3-ccef-4651-b279-f881f7f9d84f", "metadata": { "tags": [] @@ -279,12 +271,11 @@ "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -323,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "48e9732a", "metadata": {}, "outputs": [], @@ -337,7 +328,9 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " return input_string.replace(expected_prefix, replacement_prefix)\n", + " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", + "\n", + " return replaced_string\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -346,7 +339,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", + "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -384,26 +377,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "e6b719d6", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "c080e51e", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb index c5ec97ce..e4a62288 100644 --- a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb @@ -168,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "d1558909-6bd8-4161-a710-4ce0f0502fc3", "metadata": {}, "outputs": [], @@ -188,15 +188,7 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": (\n", - " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", - " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", - " \"missions and instruments are combined in this single collection for \"\n", - " \"the purpose of visualizing RGB images. Note that the full item records \"\n", - " \"are not interchangeable across instruments for purposes other than \"\n", - " \"visualization and that the assets available are not the same for each \"\n", - " \"instrument.\"\n", - " ),\n", + " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -271,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "c8297cb0-b600-4b96-a3bd-1b2b8b1d05d1", "metadata": { "tags": [] @@ -279,12 +271,11 @@ "outputs": [], "source": [ "import json\n", - "from pathlib import Path\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", + "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -323,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "0f66e4e7", "metadata": {}, "outputs": [], @@ -337,7 +328,9 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " return input_string.replace(expected_prefix, replacement_prefix)\n", + " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", + "\n", + " return replaced_string\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -346,7 +339,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", + "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -384,26 +377,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "90eef7ab", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "f140d9c1", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", - " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", - " --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" ] } ], diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb index 27296971..5c242845 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb @@ -137,19 +137,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "f5b3ff3f-1060-4223-a127-9780e7bb6c9d", "metadata": {}, "outputs": [], "source": [ - "# COG configuration, deflate compression is slightly smaller than\n", - "# the LZW default for the cog driver and predictor=2 further reduces file size\n", - "\n", + "# COG configuration, deflate compression is slightly smaller than the LZW default for the cog driver\n", + "# and predictor=2 further reduces file size\n", "# https://kokoalberti.com/articles/geotiff-compression-optimization-guide/\n", - "\n", - "# \"Predictors work especially well when there is some spatial correlation in the data,\n", - "# and pixels have values which are similar to their neighbours. As the name suggests,\n", - "# the floating point predictor can only be used on floating point data.\"\n", + "# \"Predictors work especially well when there is some spatial correlation in the data, and pixels have values which are similar to their neighbours. As the name suggests, the floating point predictor can only be used on floating point data.\"\n", "\n", "driver = \"COG\"\n", "compress = \"DEFLATE\"\n", @@ -3953,75 +3949,71 @@ " print(f\"{src_key=}\")\n", "\n", " # Get object and read open as xarray dataset\n", - " with (\n", - " fs.open(f\"{STAGING_BUCKET}/{src_key}\") as fileobj,\n", - " xr.open_dataset(fileobj, engine=\"h5netcdf\") as ds,\n", - " ):\n", - " # Now loop through the individual variables and generate a COG for each\n", - " for metric in METRICS:\n", - " # Generate output path\n", - " cog_key = cmip6_file_organization.generate_yearly_fwi_metrics_key(\n", - " src_key, \"v0\", pub_type=\"cog\", metric=metric\n", - " )\n", + " with fs.open(f\"{STAGING_BUCKET}/{src_key}\") as fileobj:\n", + " with xr.open_dataset(fileobj, engine=\"h5netcdf\") as ds:\n", + " # Now loop through the individual variables and generate a COG for each of these\n", + " for metric in METRICS:\n", + " # Generate output path\n", + " cog_key = cmip6_file_organization.generate_yearly_fwi_metrics_key(\n", + " src_key, \"v0\", pub_type=\"cog\", metric=metric\n", + " )\n", "\n", - " if verbose or extra_dryrun:\n", - " print(f\"{cog_key=}\")\n", - " if extra_dryrun:\n", - " continue\n", + " if verbose or extra_dryrun:\n", + " print(f\"{cog_key=}\")\n", + " if extra_dryrun:\n", + " continue\n", "\n", - " # Read individual metric into data array\n", - " # (only one time in yearly NetCDFs)\n", - " da = ds[metric]\n", + " # Read individual metric into data array (only one time in yearly NetCDFs)\n", + " da = ds[metric]\n", "\n", - " # Realign the x dimension to -180 origin for dataset\n", - " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(\"lon\")\n", + " # Realign the x dimension to -180 origin for dataset\n", + " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(\n", + " \"lon\"\n", + " )\n", "\n", - " # Reverse the DataArray's y dimension\n", - " # It appears that the source NetCDF's y dimension is inverted\n", - " da = da.reindex(lat=list(reversed(da.lat)))\n", + " # Reverse the DataArray's y dimension. It appears that the source NetCDF's y dimension is inverted.\n", + " da = da.reindex(lat=list(reversed(da.lat)))\n", "\n", - " # Convert to COG via rioxarray\n", - " da.rio.set_spatial_dims(\"lon\", \"lat\")\n", - " da.rio.write_crs(\"epsg:4326\", inplace=True)\n", + " # Convert to COG via rioxarray\n", + " da.rio.set_spatial_dims(\"lon\", \"lat\")\n", + " da.rio.write_crs(\"epsg:4326\", inplace=True)\n", "\n", - " # Here we need a memory file to write the output raster to\n", - " with MemoryFile() as memfile:\n", - " da.rio.to_raster(\n", - " memfile.name,\n", - " driver=\"COG\",\n", - " compress=compress,\n", - " predictor=predictor,\n", - " )\n", + " # Here we need a memory file to write the output raster to\n", + " with MemoryFile() as memfile:\n", + " da.rio.to_raster(\n", + " memfile.name,\n", + " driver=\"COG\",\n", + " compress=compress,\n", + " predictor=predictor,\n", + " )\n", "\n", - " # Validate and upload COG\n", - " if rio_cogeo.cogeo.cog_validate(memfile.name)[0]:\n", - " if dryrun:\n", - " print(\n", - " f\"Generation of valid COG {cog_key=} successful. \"\n", - " \"Skipping upload.\"\n", - " )\n", - " continue\n", + " # Validate and upload COG\n", + " if rio_cogeo.cogeo.cog_validate(memfile.name)[0]:\n", + " if dryrun:\n", + " print(\n", + " f\"Generation of valid COG {cog_key=} successful. Skipping upload.\"\n", + " )\n", + " continue\n", "\n", - " if verbose:\n", - " print(f\"START to upload {cog_key=} to {DST_BUCKET=}\")\n", + " if verbose:\n", + " print(f\"START to upload {cog_key=} to {DST_BUCKET=}\")\n", "\n", - " # Upload memory file and confirm success\n", - " if not dryrun:\n", - " r = client.put_object(\n", - " Body=memfile,\n", - " Bucket=DST_BUCKET,\n", - " Key=cog_key,\n", - " )\n", - " if r[\"ResponseMetadata\"][\"HTTPStatusCode\"] != 200:\n", - " error_responses.append(r)\n", - " if verbose:\n", - " print(f\"WARNING unable to upload {cog_key=}\")\n", - " else:\n", - " invalid_cogs.append(cog_key)\n", + " # Upload memory file and confirm success\n", + " if not dry_run:\n", + " r = client.put_object(\n", + " Body=memfile,\n", + " Bucket=DST_BUCKET,\n", + " Key=cog_key,\n", + " )\n", + " if r[\"ResponseMetadata\"][\"HTTPStatusCode\"] != 200:\n", + " error_responses.append(r)\n", + " if verbose:\n", + " print(f\"WARNING unable to upload {cog_key=}\")\n", + " else:\n", + " invalid_cogs.append(cog_key)\n", "\n", "print(\n", - " f\"\\nCOMPLETED with {len(error_responses)} errors \"\n", - " f\"and {len(invalid_cogs)} invalid cogs\"\n", + " f\"\\nCOMPLETED with {len(error_responses)} errors and {len(invalid_cogs)} invalid cogs\"\n", ")\n", "print(error_responses)\n", "print(invalid_cogs)" diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb index 13c9656e..5f5b011a 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "dcf5702f-ed7c-4844-9fcd-a2e693f5ec05", "metadata": { "tags": [] @@ -63,8 +63,7 @@ "TITLE = \"Fire Weather Index Yearly SSP245\"\n", "DESCRIPTION = (\n", " \"FWI is a general index of fire danger and the data are monthly and annual \"\n", - " \"multi-model ensemble's of the 50th percentile of mean Fire Weather Index \"\n", - " \"from the 33 GCMs\"\n", + " \"multi-model ensemble’s of the 50th percentile of mean Fire Weather Index from the 33 GCMs\"\n", ")\n", "DASHBOARD__IS_PERIODIC = True\n", "DASHBOARD__TIME_DENSITY = \"year\"\n", diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index c388c7cc..4894c5d7 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -92,8 +92,7 @@ " # Serialize updated JSON\n", " updated_json = json.dumps(data)\n", "\n", - " # Upload updated JSON file back to S3\n", - " # commented out the line below so it doesn't actually upload\n", + " # Upload updated JSON file back to S3, commented out the line below so it doesn't actually upload\n", " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", " print(f\"Updated {key}\")" ] From 433be8972c2634d8e93cfa74baaba4fa5c9bfa2c Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 14 Oct 2026 19:26:14 -0500 Subject: [PATCH 09/10] fix: exclude of ipynb --- .pre-commit-config.yaml | 9 +++------ .../Transfer-MCP-veda-data-store.ipynb | 14 +++++++------- .../generate-cmip6-kerchunk-historical.ipynb | 7 ++++--- .../publish-cmip6-kerchunk-stac.ipynb | 7 ++++--- .../collection-and-item-ingest.ipynb | 5 ++--- .../collection-metadata-reconciliation.ipynb | 9 ++++----- .../create-stac-for-cmr-collections.ipynb | 9 +++++---- .../fwi-stations/FWI_Transformation.ipynb | 8 ++++---- .../generate-rasi-cogs-from-netcdf3.ipynb | 3 +-- .../publish-hls-provided-item-metadata.ipynb | 7 +++---- .../landsat-lakes-discovery/aral-sea.ipynb | 4 ++-- .../landsat-lakes-discovery/lake-balaton.ipynb | 4 ++-- .../landsat-lakes-discovery/lake-biwa.ipynb | 4 ++-- .../mirror-landsat-demos.ipynb | 7 +++---- .../landsat-lakes-discovery/tonle-sap.ipynb | 4 ++-- .../landsat-lakes-discovery/vanern.ipynb | 4 ++-- .../cmip6-generate-organize-cogs.ipynb | 4 +++- .../nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb | 1 + .../soil_moisture_transformation_cog.ipynb | 15 +++++++++------ transformation-scripts/update-hrefs.ipynb | 5 ++--- 20 files changed, 65 insertions(+), 65 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7e805ec5..ac146e3d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,20 @@ +exclude: ^.*\.ipynb$ + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: check-yaml - exclude: ^.*\.ipynb$ - id: trailing-whitespace - exclude: ^(ingestion-data/.*|.*\.ipynb)$ + exclude: ^ingestion-data/.*$ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.16.1 hooks: - id: ruff-check args: [--fix] - exclude: ^.*\.ipynb$ - id: ruff-format types_or: [python, pyi, pyproject] - exclude: ^.*\.ipynb$ - repo: https://github.com/DavidAnson/markdownlint-cli2 rev: v0.23.2 @@ -27,13 +26,11 @@ repos: - "pyproject.toml" - "--configPointer" - "/tool/markdownlint-cli2" - exclude: ^.*\.ipynb$ - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.12.5 hooks: - id: uv-lock - exclude: ^.*\.ipynb$ - repo: https://github.com/DavidAnson/markdownlint-cli2 rev: v0.23.2 diff --git a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb index 9bc01250..0c5cbdbb 100644 --- a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb +++ b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb @@ -7,13 +7,13 @@ "metadata": {}, "outputs": [], "source": [ - "import base64\n", - "import glob\n", + "from pystac_client import Client\n", "import json\n", - "\n", - "import boto3\n", + "import glob\n", "import requests\n", - "from pystac_client import Client" + "import boto3\n", + "import base64\n", + "import time" ] }, { @@ -302,7 +302,7 @@ ], "source": [ "for filename in discover_items_file_paths:\n", - " with open(filename) as f:\n", + " with open(filename, \"r\") as f:\n", " dataset = json.load(f)\n", " if not isinstance(dataset, list):\n", " dataset = [dataset]\n", @@ -1452,7 +1452,7 @@ "\n", "for transfer_item_file in transfer_items_file_paths:\n", " print(f\"Transferring using config: {transfer_item_file}\")\n", - " with open(transfer_item_file) as f:\n", + " with open(transfer_item_file, \"r\") as f:\n", " transfer_configs = json.load(f)\n", " for transfer_config in transfer_configs:\n", " mwaa_cli_token = get_mwaa_cli_token(\n", diff --git a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb index 5010d403..9efd475b 100644 --- a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb +++ b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb @@ -21,17 +21,18 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", "from tempfile import TemporaryDirectory\n", - "\n", "import boto3\n", + "from dask_gateway import GatewayCluster, Gateway\n", "import dask.bag as db\n", "import fsspec\n", + "import json\n", + "import os\n", "import ujson\n", "import xarray as xr\n", - "from dask_gateway import Gateway, GatewayCluster\n", "from kerchunk.combine import MultiZarrToZarr\n", "from kerchunk.hdf import SingleHdf5ToZarr\n", + "from typing import Dict\n", "\n", "# Specify the CMIP model and variable to use.\n", "# Here we are using near-surface air temperature from the GISS-E2-1-G GCM\n", diff --git a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb index c98c1547..1bd30f3f 100644 --- a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb +++ b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb @@ -32,10 +32,11 @@ "outputs": [], "source": [ "#!pip install xstac\n", - "import fsspec\n", "import pystac\n", - "import xarray as xr\n", - "import xstac" + "import s3fs\n", + "import xstac\n", + "import fsspec\n", + "import xarray as xr" ] }, { diff --git a/transformation-scripts/collection-and-item-ingest.ipynb b/transformation-scripts/collection-and-item-ingest.ipynb index b30fd5e4..6adfb0ac 100644 --- a/transformation-scripts/collection-and-item-ingest.ipynb +++ b/transformation-scripts/collection-and-item-ingest.ipynb @@ -33,7 +33,6 @@ "source": [ "import glob\n", "import json\n", - "\n", "import requests\n", "from cognito_client import CognitoClient" ] @@ -78,7 +77,7 @@ "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in filtered_list\n", - " if \"id\" in (data := json.load(open(file_path)))\n", + " if \"id\" in (data := json.load(open(file_path, \"r\")))\n", "]" ] }, @@ -242,7 +241,7 @@ " file_path = item[\"filePath\"]\n", "\n", " try:\n", - " with open(file_path, encoding=\"utf-8\") as file:\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", diff --git a/transformation-scripts/collection-metadata-reconciliation.ipynb b/transformation-scripts/collection-metadata-reconciliation.ipynb index 900fc18f..54c5d3d1 100644 --- a/transformation-scripts/collection-metadata-reconciliation.ipynb +++ b/transformation-scripts/collection-metadata-reconciliation.ipynb @@ -22,7 +22,6 @@ "source": [ "import glob\n", "import json\n", - "\n", "import requests\n", "from cognito_client import CognitoClient" ] @@ -45,7 +44,7 @@ "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in json_file_paths\n", - " if \"id\" in (data := json.load(open(file_path)))\n", + " if \"id\" in (data := json.load(open(file_path, \"r\")))\n", "]" ] }, @@ -362,9 +361,9 @@ " collection_id = item[\"collectionId\"]\n", " file_path = item[\"filePath\"]\n", "\n", - " if dev_endpoint == VEDA_STAC_API:\n", + " if VEDA_STAC_API == dev_endpoint:\n", " url = f\"{VEDA_STAC_API}api/stac/collections/{collection_id}\"\n", - " elif staging_endpoint == VEDA_STAC_API:\n", + " elif VEDA_STAC_API == staging_endpoint:\n", " url = f\"{VEDA_STAC_API}collections/{collection_id}\"\n", "\n", " try:\n", @@ -375,7 +374,7 @@ " retrieved_summaries = json_response.get(\"summaries\", {})\n", " retrieved_links = json_response.get(\"links\", {})\n", "\n", - " with open(file_path, encoding=\"utf-8\") as file:\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", " existing_summaries = collection.get(\"summaries\", {})\n", diff --git a/transformation-scripts/create-stac-for-cmr-collections.ipynb b/transformation-scripts/create-stac-for-cmr-collections.ipynb index 4a63b35b..30d7ff17 100644 --- a/transformation-scripts/create-stac-for-cmr-collections.ipynb +++ b/transformation-scripts/create-stac-for-cmr-collections.ipynb @@ -70,14 +70,15 @@ }, "outputs": [], "source": [ - "import json\n", - "\n", "import earthaccess\n", + "import json\n", + "from datetime import datetime\n", "import pandas as pd\n", "import pystac\n", + "import requests\n", "import s3fs\n", - "import xarray as xr\n", - "import xstac" + "import xstac\n", + "import xarray as xr" ] }, { diff --git a/transformation-scripts/fwi-stations/FWI_Transformation.ipynb b/transformation-scripts/fwi-stations/FWI_Transformation.ipynb index aaf3a1b3..0ca3196e 100644 --- a/transformation-scripts/fwi-stations/FWI_Transformation.ipynb +++ b/transformation-scripts/fwi-stations/FWI_Transformation.ipynb @@ -7,12 +7,12 @@ "metadata": {}, "outputs": [], "source": [ - "import re\n", - "from io import StringIO\n", - "\n", "import boto3\n", + "import s3fs\n", "import pandas as pd\n", - "import s3fs" + "import re\n", + "import os\n", + "from io import StringIO" ] }, { diff --git a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb index 6345c1bd..f2795dcf 100644 --- a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb +++ b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb @@ -48,9 +48,8 @@ "outputs": [], "source": [ "import os\n", - "\n", - "import pandas as pd\n", "import rio_cogeo.cogeo\n", + "import pandas as pd\n", "import rioxarray # noqa\n", "import s3fs\n", "import xarray as xr" diff --git a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb index 8a1fa9bd..b8389b80 100644 --- a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb +++ b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb @@ -30,9 +30,8 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", - "\n", "import boto3\n", + "import json\n", "import requests\n", "from pystac import Item\n", "\n", @@ -173,7 +172,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception:\n", + " except Exception as e:\n", " print(f\"Invalid {collection_id=} {item.id=}\")\n", "\n", " # Replace the s3 object with the updated metadata for stac version 1.0.0\n", @@ -226,7 +225,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception:\n", + " except Exception as e:\n", " print(f\"invalid {collection_id=} {item.id=}\")\n", "\n", " # Publish to target STAC catalog\n", diff --git a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb index bbf8592e..df65234a 100644 --- a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb @@ -132,8 +132,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", diff --git a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb index 3bd060c8..a86da37a 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb @@ -146,8 +146,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", diff --git a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb index aed6b28d..5fa41682 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb @@ -134,8 +134,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", diff --git a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb index 825b78b2..714b7c73 100644 --- a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb @@ -35,11 +35,10 @@ ], "source": [ "import json\n", - "from pathlib import Path\n", - "\n", "import requests\n", - "from pystac import Collection, Item\n", + "from pathlib import Path\n", "from pystac_client import Client\n", + "from pystac import Collection, Item\n", "\n", "# Source records to correct and publish to target\n", "SRC_STAC_API_URL = \"https://staging-stac.delta-backend.com\"\n", @@ -377,7 +376,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception:\n", + " except Exception as e:\n", " invalid_items.append(item.id)\n", " print(f\"invalid {item.id=} {len(invalid_items)=}\")\n", "\n", diff --git a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb index 0a91948c..6d132129 100644 --- a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb @@ -144,8 +144,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", diff --git a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb index e4a62288..a974a74b 100644 --- a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb @@ -142,8 +142,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb index 5c242845..63d77074 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb @@ -87,8 +87,10 @@ "outputs": [], "source": [ "import boto3\n", - "import rio_cogeo.cogeo\n", "import s3fs\n", + "import rioxarray\n", + "import rasterio\n", + "import rio_cogeo.cogeo\n", "import xarray as xr\n", "from rasterio.io import MemoryFile\n", "\n", diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb index 5f5b011a..c3c91052 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb @@ -31,6 +31,7 @@ "metadata": {}, "outputs": [], "source": [ + "from datetime import datetime, timezone\n", "import pystac" ] }, diff --git a/transformation-scripts/soil_moisture_transformation_cog.ipynb b/transformation-scripts/soil_moisture_transformation_cog.ipynb index 0698e62e..1f15bf0a 100644 --- a/transformation-scripts/soil_moisture_transformation_cog.ipynb +++ b/transformation-scripts/soil_moisture_transformation_cog.ipynb @@ -17,14 +17,17 @@ }, "outputs": [], "source": [ - "import json\n", + "import s3fs\n", + "import rioxarray\n", + "import rasterio\n", + "import rio_cogeo.cogeo\n", + "import xarray as xr\n", "import re\n", - "import tempfile\n", - "\n", - "import boto3\n", + "from datetime import datetime\n", "import pandas as pd\n", - "import s3fs\n", - "import xarray as xr" + "import boto3\n", + "import tempfile\n", + "import json" ] }, { diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index 4894c5d7..b3d57b47 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -13,9 +13,8 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", - "\n", - "import boto3" + "import boto3\n", + "import json" ] }, { From c9d7ef253e8a4905399ae123aaea01319ce21444 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 14 Oct 2026 19:32:13 -0500 Subject: [PATCH 10/10] revert: cmip6_file_organization.py --- .pre-commit-config.yaml | 2 +- .../nex-gddp-cmip6-fwi/cmip6_file_organization.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac146e3d..44d308f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: ^.*\.ipynb$ +exclude: ^transformation-scripts/.*$ repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py index f84dcde4..38496c83 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py @@ -1,4 +1,5 @@ -from pathlib import Path +from os.path import basename +from typing import Optional from urllib.parse import urlparse @@ -7,7 +8,7 @@ def generate_yearly_fwi_metrics_key( dst_version: str, ensemble: str = "mme", pub_type: str = "netcdf", - metric: str | None = None, + metric: Optional[str] = None, verbose: bool = False, ): """_summary_ @@ -22,7 +23,7 @@ def generate_yearly_fwi_metrics_key( cadence = "yearly" parsed = urlparse(src_url, allow_fragments=False) - nc_base = Path(parsed.path).name + nc_base = basename(parsed.path) # Assume pattern ensemble-stat_experiment_fwi_metrics_yearly_yyyy # MME50_ssp245_fwi_metrics_yearly_2100