Sync ega v2.3.2

.dockerignore

@@ -5,4 +5,7 @@ beacon/connections/mongo/data/db/*
 *.json.gz
 adminui/node_modules
 *.vcf.bgz
-ri-tools/files/vcf/files_to_read/*
+ri-tools/files/vcf/files_to_read/*
+*.msgpack
+*.initialized
+ri-tools/refgen-data/*

.env.example

@@ -26,7 +26,7 @@ MONGO_PORT=8085
 MONGO_EXPRESS_PORT=8081
 
 # Container images
-TEMPLATE_UI_IMAGE=ghcr.io/ega-archive/beacon-template-ui:main
+TEMPLATE_UI_IMAGE=ghcr.io/ega-archive/beacon-template-ui:1.1.1
 MONGO_IMAGE=docker.io/library/mongo:8.2.3
 POSTGRES_IMAGE=docker.io/library/postgres:15
 KEYCLOAK_IMAGE=quay.io/keycloak/keycloak:18.0.0

.github/workflows/update_beacon_pi_version.yml

@@ -0,0 +1,75 @@
+name: Update API version on PR
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+    branches:
+      - main
+
+permissions:
+  contents: write
+  pull-requests: read
+
+jobs:
+  update-version:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Extract version from PR body
+        id: version
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          VERSION=$(echo "$PR_BODY" | grep -oE 'Version: v[0-9]+\.[0-9]+\.[0-9]+' | awk '{print $2}')
+
+          if [ -z "$VERSION" ]; then
+            echo "Missing version. Add 'Version: vX.Y.Z' to the PR description."
+            exit 1
+          fi
+
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install PyYAML
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install pyyaml==6.0.1
+
+      - name: Verify PyYAML installation
+        run: |
+          python -c "import sys; print(sys.executable); import yaml; print(yaml.__version__)"
+
+      - name: Update api_version.yml
+        run: |
+          python beacon/conf/api_version.py \
+            "${{ steps.version.outputs.version }}"
+
+      - name: Commit version update
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+
+          git add beacon/conf/api_version.yml
+
+          git commit \
+            -m "Update API version to ${{ steps.version.outputs.version }}" || exit 0
+
+          git push origin HEAD:${{ github.head_ref }}
+
+      - name: Create tag
+        run: |
+          git tag ${{ steps.version.outputs.version }}
+          git push origin ${{ steps.version.outputs.version }}

.gitignore

@@ -14,6 +14,9 @@ adminui/adminclient/migrations/*
 adminui/node_modules
 *.vcf.bgz
 beacon/conf/conf.py
+beacon/models/prova
+*.msgpack
+*.initialized
 # Local certificates and persistent container data
 certs/
 beacon/connections/mongo/data/db/

Dockerfile

@@ -2,7 +2,7 @@
 ## Build env
 ##########################
 
-FROM python:3.10-bullseye AS BUILD
+FROM python:3.10-bookworm AS BUILD
 
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -23,7 +23,7 @@ RUN pip install -r /tmp/requirements.txt
 ##########################
 ## Final image
 ##########################
-FROM python:3.10-bullseye
+FROM python:3.10-bookworm
 
 LABEL maintainer="BU-ISCIII - Bioinformatics Unit ISCIII"
 LABEL maintainer.email="bioinformatica@isciii.es"

README.md

@@ -2,6 +2,8 @@
 
 Welcome to Beacon v2 Production Implementation (B2PI). This is an application that makes an instance of Beacon v2 be production ready.
 
+Note: admin-ui container is a new extra service that will create a UI to manage conf but is still in development, deploy the rest of the services for now, please.
+
 ## Documentation
 
 Please, go to [CRG Beacon docs website](https://b2ri-documentation-demo.ega-archive.org/) to know how to use Beacon v2 Production Implementation.
@@ -756,6 +758,27 @@ Now state checks are available through `/health` endpoint. The implemented check
 
 ![MongoDB vulnerabilities](https://github.com/EGA-archive/beacon-production-prototype/blob/main/ri-tools/files/Machine_State_v3.jpg)
 
+## Alternative Schemas
+
+Now you can also have your own "alternative" schemas included in a plugged model. For that purpose, you will need to have the JSON schema reference and have pydantic's datamodel-code-generator framework installed. To generate the schema, you will need to execute the following command:
+
+```bash
+datamodel-codegen --url <url_to_schema> --input-file-type jsonschema --output <entry_type_name.py> --class-name <Entry_type_name_with_capital_letter> --allow-remote-refs
+```
+Example: for analysis, you need to name the class-name as Analysis.
+
+The name of the python script needs to be separated by underscores. If you include versions, at them at the end, like analysis_schema_v1_0_0.py
+
+After that include it in the models.<model_name>.validators.<entry_type> folder, and then make sure you add it in the supported schemas of models.<model_name>.conf.entry_types.
+
+Note: In case there are fields that are named different as in the original schemas but are the same, you will need to add an alias for the property modifying the generated schemas by datamodel-codegen, in order to be able to map it to the original property name and is compatible with the original spec's schema.
+
+Example:
+
+```bash
+sampleTypeAtTheOrigin: str = Field(alias='sampleOriginType') # output: sampleOriginType
+```
+
 ## Fix for MongoDB exploit (CVE-2025-14847)
 
 Beacon PI repository has been updated so the exploit for MongoDB (CVE-2025-14847) is not an issue anymore. In order to do that, the following points have been implemented:

beacon/auth/tests.py

@@ -13,7 +13,7 @@
 import beacon.conf.conf_override as conf_override
 
 # for keycloak, create aud in mappers, with custom, aud and beacon for audience
-mock_access_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJreS1tUXNxZ0ZYeHdSUVRfRUhuQlJJUGpmbVhfRXZuUTVEbzZWUTJCazdZIn0.eyJleHAiOjE3Nzk3OTg1MzksImlhdCI6MTc3OTc5ODIzOSwianRpIjoiZDQwNGEyNWEtMjk0Yy00M2RjLWE0OTMtNzZiM2M4NjJjYTc4IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2F1dGgvcmVhbG1zL0JlYWNvbiIsImF1ZCI6ImJlYWNvbiIsInN1YiI6IjQ3ZWZmMWIxLTc2MjEtNDU3MC1hMGJiLTAxYTcxOWZiYTBhMiIsInR5cCI6IkJlYXJlciIsImF6cCI6ImJlYWNvbiIsInNlc3Npb25fc3RhdGUiOiI0NDA1OTk3Zi1kNzdkLTQ4YWYtODdmNy1mZTE3ZDJjYTlmZGIiLCJhY3IiOiIxIiwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCBtaWNyb3Byb2ZpbGUtand0Iiwic2lkIjoiNDQwNTk5N2YtZDc3ZC00OGFmLTg3ZjctZmUxN2QyY2E5ZmRiIiwidXBuIjoiamFuZSIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6IkphbmUgU21pdGgiLCJncm91cHMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiamFuZSIsImdpdmVuX25hbWUiOiJKYW5lIiwiZmFtaWx5X25hbWUiOiJTbWl0aCIsImVtYWlsIjoiamFuZS5zbWl0aEBiZWFjb24uZ2E0Z2gifQ.YFtQijg_AxZNytFtcA0EnVUmbnbecpMo2R-fbRCkD6QDY-I2YWV_Glww0yCU_Yp17a5r60uYvotfIcpLKnWMduBLPeP7pyj9UAM_LtidvIKhpQBTQ8azf4eLHEv2Ju0hYDa_24c1u8UKDg6Jqz40gzV4Z7QRIhKhzqBXTdxo5TlIzYZLtzhqx91gVp9SIqaURUBo4wxqisxnFbsOc-J-IQmOw0OwbzB1f7ugG_xXP-wbJ1SbqbeyaB8pwAo7_dkfWezuhBsCdfhcGuAyvn49KPubgGnenKY3jUVzVxE9MbslKuBlLg_A7c5UNnuQ2vLW77jm6zMGRDPjRjoTIHq1bA'
+mock_access_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJreS1tUXNxZ0ZYeHdSUVRfRUhuQlJJUGpmbVhfRXZuUTVEbzZWUTJCazdZIn0.eyJleHAiOjE3ODkyMTY5NjAsImlhdCI6MTc4OTIxNjY2MCwianRpIjoiODhiOGM5YmItNGYxYy00MDBhLWFkYzUtNWUxOGE0ZmNhMTM2IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL2F1dGgvcmVhbG1zL0JlYWNvbiIsImF1ZCI6ImJlYWNvbiIsInN1YiI6IjQ3ZWZmMWIxLTc2MjEtNDU3MC1hMGJiLTAxYTcxOWZiYTBhMiIsInR5cCI6IkJlYXJlciIsImF6cCI6ImJlYWNvbiIsInNlc3Npb25fc3RhdGUiOiI2NjVmYzE4Mi0zNGZkLTRmNDktODIyNi1lYTI3NGJhYmIxYTIiLCJhY3IiOiIxIiwic2NvcGUiOiJvcGVuaWQgcHJvZmlsZSBlbWFpbCBtaWNyb3Byb2ZpbGUtand0Iiwic2lkIjoiNjY1ZmMxODItMzRmZC00ZjQ5LTgyMjYtZWEyNzRiYWJiMWEyIiwidXBuIjoiamFuZSIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6IkphbmUgU21pdGgiLCJncm91cHMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iXSwicHJlZmVycmVkX3VzZXJuYW1lIjoiamFuZSIsImdpdmVuX25hbWUiOiJKYW5lIiwiZmFtaWx5X25hbWUiOiJTbWl0aCIsImVtYWlsIjoiamFuZS5zbWl0aEBiZWFjb24uZ2E0Z2gifQ.ognNPMqZYlQCXmj0ZXu_IodasLHCWJtBHX3ElWEFyakbAkHsOzLxGwrFkRyrmwB1jyg_lXcXt7E9z2mB7FzNfCpzdhIXBqBQf_v9OFjZv0-maVIItjRzmHLF5jnbPwBj6jQVzwkvCHOAqFYx0wBTviDWsVYSgNOuPmZcCj3KKrzDrJzUUSZ82DoyGjc9aTfMJQFFb0Sw-d2-ubN4XlFjwvGreJB3OG0n_eUkbvmFDwxWnR5OKcb7r0IYr15RigaKaqFiurNWC9vsttDZPL2LC5LuesMj6QMA3xkCMVReazj28-bFkOsGIgr9QSGx-OpJ4pKqubP4X7MtIx3TN7xpFw'
 mock_access_token_false = 'public'
 
 #audit --> TODO: get very specific information that we are interested in saving and keeping it (example: what individuals were returned in response)

beacon/budget/__main__.py

@@ -1,8 +1,6 @@
 from beacon.logs.logs import log_with_args_mongo
 from beacon.conf.conf_override import config
 from datetime import datetime, timedelta
-from beacon.request.classes import RequestAttributes
-import aiohttp.web as web
 from beacon.exceptions.exceptions import NumberOfQueriesExceeded, NoPermissionsAvailable
 
 @log_with_args_mongo(config.level)
@@ -22,10 +20,10 @@ def check_budget(self, username):
         else: # Return the time to store in the database
             return time_now
     # Check if there is ip in case the budget is meant to be done by ip and get the remaining budget
-    elif config.query_budget_per_ip == True and RequestAttributes.ip is not None:
+    elif config.query_budget_per_ip == True and self.request_attributes.ip is not None:
         remaining_budget=module.get_remaining_budget_by_ip(self, start_budget_time)
         if len(remaining_budget)>=config.query_budget_amount:
-            raise NumberOfQueriesExceeded("Number of queries exceeded for this ip: {}".format(RequestAttributes.ip))
+            raise NumberOfQueriesExceeded("Number of queries exceeded for this ip: {}".format(self.request_attributes.ip))
         else: # Return the time to store in the database
             return time_now
     # Check if there is username in case the budget is meant to be done only by user and if there was no ip, then throw an exception

beacon/conf/api_version.py

@@ -0,0 +1,14 @@
+import sys
+import yaml
+
+version = sys.argv[1]
+
+with open("beacon/conf/api_version.yml") as f:
+    api_version = yaml.safe_load(f)
+
+api_version["api_version"] = version
+
+with open("beacon/conf/api_version.yml", "w") as f:
+    yaml.safe_dump(api_version, f, sort_keys=False)
+
+print(f"API version updated to {version}")

beacon/conf/api_version.yml

@@ -1 +1 @@
-api_version: v2.2.4
+api_version: v2.3.2

beacon/conf/conf.py

@@ -9,13 +9,14 @@
     raise FileNotFound('There are issues with the api_version.yml file. Check if it can be opened or if has any content')
 
 level=logging.DEBUG
-log_file = '/beacon/logs/beacon.log'
+log_file = '/beacon/logs/log_files/beacon.log'
 beacon_id = 'es.isciii-ciber.beacon'
 beacon_name = 'ISCIII-CIBER Spain AF Beacon'
 api_version = 'v2.2.0' # Version of the Beacon implementation
 uri = 'http://beaconprod:5050'
 uri_subpath = '/api'
 complete_url = uri + uri_subpath
+allowed_uris = ['http://beaconaf-isciiiciber.isciiides.es:8443/api', 'http://apibeacon-isciiiciber.isciiides.es:8443/api'] # public hosts behind Apache (ProxyPreserveHost On)
 environment = 'TEST'
 description = r"This Beacon is based on synthetic data hosted at the <a href='https://ega-archive.org/datasets/EGAD00001003338'>EGA</a>. The dataset contains 2504 samples including genetic data based on 1K Genomes data, and 76 individual attributes and phenotypic data derived from UKBiobank."
 version = api_version_yaml['api_version']
@@ -42,7 +43,7 @@
 org_adress = 'C/ Monforte de Lemos, 5, 28029 Madrid, Spain'
 org_welcome_url = 'https://www.isciii.es/'
 org_contact_url = 'mailto:bioinformatica@isciii.es'
-org_logo_url = 'https://www.isciii.es/images/logo.png'
+org_logo_url = 'https://sirena.isciii.es/wp-content/uploads/2022/10/horizontal.jpg'
 org_info = ''
 
 # Certificates

beacon/conf/conf_default.py

@@ -8,14 +8,20 @@
 except Exception as e:
     raise FileNotFound('There are issues with the api_version.yml file. Check if it can be opened or if has any content')
 
+# logs
 level=logging.DEBUG
-log_file=None
+log_file=None # path needs to be pointing to a file inside a volumed dir, preferably /beacon/logs/log_files, e.g. /beacon/logs/log_files/logs.log
+log_num_of_files_to_keep = 7 # number of files that will be kept for backup, the most recent ones, the oldest will get deleted
+log_rotating_interval = 'midnight' # S, M, H, D, W0, W1, W2, W3, W4, W5, W6, midnight
+
+# Beacon generic conf
 beacon_id = 'org.ega-archive.beacon-ri-demo'  # ID of the Beacon
 beacon_name = 'Beacon Production Implementation demo'  # Name of the Beacon service
 api_version = 'v2.2.0' # Version of the Beacon implementation
 uri = 'http://localhost:5050'
 uri_subpath = '/api'
 complete_url = uri + uri_subpath
+allowed_uris = [] # example: ["http://localhost:5054/api"] or if you want to enable all: ["*"]
 environment = 'test'
 description = r"This Beacon is based on synthetic data hosted at the <a href='https://ega-archive.org/datasets/EGAD00001003338'>EGA</a>. The dataset contains 2504 samples including genetic data based on 1K Genomes data, and 76 individual attributes and phenotypic data derived from UKBiobank."
 version = api_version_yaml['api_version']

beacon/conf/connections/connections_conf.yml

@@ -0,0 +1,5 @@
+mongo:
+  connection_enabled: True
+
+sqlalchemy_postgresql:
+  connection_enabled: False

beacon/conf/datasets/datasets_conf.yml

@@ -6,6 +6,7 @@ test:
 test2:
   isSynthetic: true
   isTest: true
+  isDeprecated: true
 DemoDatasetBreast:
   isSynthetic: true
   isTest: true

beacon/connections/beaconCLI/executor.py

@@ -3,13 +3,12 @@
 from beacon.logs.logs import log_with_args
 from beacon.conf.conf_override import config
 from typing import Optional
-from beacon.request.classes import RequestAttributes
 from beacon.connections.beaconCLI.g_variants import get_variants
 
 @log_with_args(config.level)
 async def execute_function(self, entry_type: str, datasets: list, entry_id: Optional[str]):
-    include = RequestAttributes.qparams.query.includeResultsetResponses
-    limit = RequestAttributes.qparams.query.pagination.limit
+    include = self.request_attributes.qparams.query.includeResultsetResponses
+    limit = self.request_attributes.qparams.query.pagination.limit
     datasets_docs={}
     datasets_count={}
     new_count=0

beacon/connections/mongo/Makefile

@@ -1,4 +1,4 @@
-DATA_PATH = /data/test
+DATA_PATH = /data/test/v2_2_0
 
 build:
 	docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file $(DATA_PATH)/datasets.json --collection datasets

beacon/connections/mongo/budget.py

@@ -1,7 +1,6 @@
 from beacon.connections.mongo.client import get_client
 from beacon.logs.logs import log_with_args_mongo
 from beacon.conf.conf_override import config
-from beacon.request.classes import RequestAttributes
 
 @log_with_args_mongo(config.level)
 def get_remaining_budget_by_user(self, username, start_budget_time):
@@ -25,7 +24,7 @@ def insert_budget(self, username, time_now):
     # Insert in the database one of the uses of the budget for the user/ip performing the query
     budget_query={}
     budget_query["username"]=username
-    budget_query["ip"]=RequestAttributes.ip
+    budget_query["ip"]=self.request_attributes.ip
     budget_query["date"]=time_now
     client[config.query_budget_db_name][config.query_budget_table].insert_one(budget_query)
 
@@ -34,7 +33,7 @@ def get_remaining_budget_by_ip(self, start_budget_time):
     client=get_client()
     # Initiate the dictionary to create the query syntax for getting the remaining budget of the ip that is performing the query
     budget_query={}
-    budget_query["ip"]=RequestAttributes.ip
+    budget_query["ip"]=self.request_attributes.ip
     budget_query["date"]={ "$gt": start_budget_time }
     remaining_budget = client[config.query_budget_db_name][config.query_budget_table].find(budget_query).max_time_ms(100 * 1000)
     remaining_budget=list(remaining_budget)

beacon/connections/mongo/data/CINECA/v2_2_0/cohorts.json

@@ -0,0 +1 @@
+[{"cohortType": "study-defined", "collectionEvents": [{"eventDiseases": {"availability": true, "availabilityCount": 1705}, "eventEthnicities": {"availability": true, "availabilityCount": 2287}, "eventGenders": {"availability": true, "availabilityCount": 1597}, "eventLocations": {"availability": true, "availabilityCount": 1597}}], "id": "CINECA_synthetic_cohort_UK1", "inclusionCriteria": {"ageRange": {"end": {"iso8601duration": "P65Y"}, "start": {"iso8601duration": "P18Y"}}, "genders": [{"id": "NCIT:C16576", "label": "female"}, {"id": "NCIT:C20197", "label": "male"}], "locations": [{"id": "GAZ:00150372", "label": "UK"}]}, "name": "CINECA synthetic cohort UK1", "datasetId": "CINECA_synthetic_cohort_EUROPE_UK1", "_id": "a388c8898dfab6ba5cf82441dcc1359e9a5462dcdc306893a91973abde0e5715"}]