OpenRailwayMap-vector integration

.editorconfig

@@ -1,7 +1,10 @@
 root = true
 
-[*.{mss,sql,yml}]
+[*]
 charset = utf-8
 indent_size = 2
 indent_style = space
 insert_final_newline = true
+
+[*.py]
+indent_size = 4

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: hiddewie
+buy_me_a_coffee: hiddewie

.github/workflows/nightly-update.yml

@@ -0,0 +1,170 @@
+name: Nightly update
+
+on:
+  workflow_dispatch: ~
+  schedule:
+    # Run daily at 23:47 or 00:47 Europe/Amsterdam
+    - cron: '47 22 * * *'
+
+jobs:
+  update-data:
+    name: Update data
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull OSM data
+        run: |
+          docker pull ghcr.io/${{ github.repository_owner }}/openrailwaymap-data:latest
+          docker tag ghcr.io/${{ github.repository_owner }}/openrailwaymap-data:latest openrailwaymap-data
+
+      - name: Copy OSM data
+        run: |
+          docker compose create data
+          docker compose cp data:/data/data.osm.pbf data/filtered/data.osm.pbf
+          docker compose cp data:/data/data.osm.pbf.timestamp data/filtered/data.osm.pbf.timestamp
+          docker compose rm data
+
+      - name: Build containers
+        uses: docker/bake-action@v6
+        with:
+          load: true
+          targets: db,import
+          set: |
+            db.cache-from=type=gha,scope=db-${{ runner.arch }}
+            import.cache-from=type=gha,scope=import-${{ runner.arch }}
+
+      - name: Update OSM data
+        run: |
+          docker compose run --no-deps import update
+
+      - name: Push data
+        run: |
+          docker compose build --no-cache data
+          docker tag openrailwaymap-data ghcr.io/${{ github.repository_owner }}/openrailwaymap-data:latest
+          docker push ghcr.io/${{ github.repository_owner }}/openrailwaymap-data:latest
+
+      - name: Start database
+        run: |
+          docker compose up --wait db
+
+      - name: Import data
+        env:
+          OSM2PGSQL_NUMPROC: '8'
+        run: |
+          docker compose run --no-deps import import
+
+      - name: Clear disk space
+        run: |
+          rm -f data/filtered/data.osm.pbf
+          docker system prune -af
+
+      - name: Create database image
+        run: |
+          docker compose stop db
+          DB_CONTAINER_ID="$(docker compose ps --all --format json | jq -r 'select(.Service == "db") | .ID')"
+          docker commit "$DB_CONTAINER_ID" ghcr.io/${{ github.repository_owner }}/openrailwaymap-import-db:latest
+
+      - name: Push database image
+        run: |
+          docker push ghcr.io/${{ github.repository_owner }}/openrailwaymap-import-db:latest
+
+  build-martin:
+    name: Build tiles container
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build containers
+        uses: docker/bake-action@v6
+        with:
+          push: true
+          targets: martin
+          set: |
+            martin.cache-from=type=gha,scope=martin-${{ runner.arch }}
+            martin.tags=ghcr.io/${{ github.repository_owner }}/openrailwaymap-martin:latest
+
+  build-proxy:
+    name: Build proxy container
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get Date
+        id: get-date
+        shell: bash
+        run: |
+          echo "date=$(/bin/date -u "+1.%Y%m%d-%Y-%m-%d")" >> $GITHUB_OUTPUT
+
+      - name: Build containers
+        uses: docker/bake-action@v6
+        with:
+          push: true
+          targets: proxy
+          set: |
+            proxy.cache-from=type=gha,scope=proxy-${{ runner.arch }}
+            proxy.args.PRESET_VERSION=${{ steps.get-date.outputs.date }}
+            proxy.tags=ghcr.io/${{ github.repository_owner }}/openrailwaymap-proxy:latest
+
+  build-api:
+    name: Build API container
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build containers
+        uses: docker/bake-action@v6
+        with:
+          push: true
+          targets: api
+          set: |
+            api.cache-from=type=gha,scope=api-${{ runner.arch }}
+            api.tags=ghcr.io/${{ github.repository_owner }}/openrailwaymap-api:latest

.github/workflows/test.yml

@@ -0,0 +1,164 @@
+name: Test
+
+on:
+  workflow_dispatch: ~
+  push:
+    branches:
+      - master
+  pull_request: ~
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref_name != 'master' }}
+
+jobs:
+  test:
+    name: Build and test
+
+    strategy:
+      matrix:
+        runs-on:
+          - ubuntu-24.04
+          - ubuntu-24.04-arm
+      fail-fast: false
+
+    runs-on: ${{ matrix.runs-on }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get Date
+        id: get-date
+        shell: bash
+        run: |
+          echo "date=$(/bin/date -u "+%Y%m")" >> $GITHUB_OUTPUT
+
+      - name: Cache data
+        uses: actions/cache@v4
+        id: cache-data
+        with:
+          path: |
+            data/berlin.osm.pbf
+            data/filtered/berlin.osm.pbf
+          key: data-${{ steps.get-date.outputs.date }}-berlin
+          enableCrossOsArchive: true
+
+      - name: Download Berlin
+        if: ${{ steps.cache-data.outputs.cache-hit != 'true' }}
+        run: |
+          curl --location --fail --output data/berlin.osm.pbf https://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf
+
+      - name: Build containers
+        uses: docker/bake-action@v6
+        with:
+          load: true
+          targets: db,import,import-test,api,martin,proxy
+          set: |
+            db.cache-from=type=gha,scope=db-${{ runner.arch }}
+            ${{ github.ref_name == 'master' && format('db.cache-to=type=gha,mode=max,scope=db-{0}', runner.arch) || '' }}
+            import.cache-from=type=gha,scope=import-${{ runner.arch }}
+            ${{ github.ref_name == 'master' && format('import.cache-to=type=gha,mode=max,scope=import-{0}', runner.arch) || '' }}
+            import-test.cache-from=type=gha,scope=import-test-${{ runner.arch }}
+            ${{ github.ref_name == 'master' && format('import-test.cache-to=type=gha,mode=max,scope=import-test-{0}', runner.arch) || '' }}
+            api.cache-from=type=gha,scope=api-${{ runner.arch }}
+            ${{ github.ref_name == 'master' && format('api.cache-to=type=gha,mode=max,scope=api-{0}', runner.arch) || '' }}
+            martin.cache-from=type=gha,scope=martin-${{ runner.arch }}
+            ${{ github.ref_name == 'master' && format('martin.cache-to=type=gha,mode=max,scope=martin-{0}', runner.arch) || '' }}
+            proxy.cache-from=type=gha,scope=proxy-${{ runner.arch }}
+            ${{ github.ref_name == 'master' && format('proxy.cache-to=type=gha,mode=max,scope=proxy-{0}', runner.arch) || '' }}
+
+      - name: Start database
+        run: |
+          docker compose up --wait db
+
+      - name: Run import tests
+        run: |
+          docker compose run --rm import-test
+
+      - name: Import data
+        env:
+          OSM2PGSQL_DATAFILE: berlin.osm.pbf
+        run: |
+          docker compose run --rm --no-deps import import
+
+      - name: Start API
+        run: |
+          docker compose up --wait --no-deps api
+
+      - name: Run API tests
+        run: |
+          docker compose run --no-deps api-test
+
+      - name: Start tile server
+        run: |
+          docker compose up --detach --no-deps martin
+
+      - name: Start proxy
+        run: |
+          docker compose up --wait --no-deps proxy
+
+      - name: Run proxy tests
+        run: |
+          docker compose run --no-deps proxy-test
+
+      - name: Download preset
+        if: always()
+        run: |
+          curl -sSf -o preset.zip http://localhost:8000/preset.zip
+
+      - name: Store preset
+        uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: ${{ runner.arch }}-presets
+          path: preset.zip
+          if-no-files-found: warn
+
+      - name: Run UI tests
+        uses: cypress-io/github-action@v7
+        with:
+          working-directory: proxy/test/ui
+          browser: firefox
+
+      - name: Store screenshots
+        uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: ${{ runner.arch }}-screenshots
+          path: proxy/test/ui/cypress/screenshots
+          if-no-files-found: warn
+
+      - name: Print logs
+        if: always()
+        run: |
+          docker compose logs
+
+  feature-test:
+    name: Feature test
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run feature tests
+        run: |
+          docker compose run --rm --build feature-test

.gitignore

@@ -1,5 +1,6 @@
 *.swp
 data/
+db/postgres-data
 standard.xml
 maxspeed.xml
 signals.xml
@@ -11,3 +12,11 @@ tmp
 .kosmtik-config.yml
 .env
 *.osm.pbf
+.idea
+compose.override.yaml
+localhost.pem
+localhost-key.pem
+node_modules
+taginfo.json
+.DS_Store
+/.vs