diff --git a/.github/ISSUE_TEMPLATE/language_issues.md b/.github/ISSUE_TEMPLATE/language_issues.md index 8e3a50d4d..703e35819 100644 --- a/.github/ISSUE_TEMPLATE/language_issues.md +++ b/.github/ISSUE_TEMPLATE/language_issues.md @@ -1,5 +1,5 @@ --- -name: Language Issue +name: ThesaurusEntry Issue about: Use this template to report issues with a language --- diff --git a/.github/ISSUE_TEMPLATE/language_request.md b/.github/ISSUE_TEMPLATE/language_request.md index d60895eb8..4a1782603 100644 --- a/.github/ISSUE_TEMPLATE/language_request.md +++ b/.github/ISSUE_TEMPLATE/language_request.md @@ -1,5 +1,5 @@ --- -name: Language Request +name: ThesaurusEntry Request about: Use this template to request a new language --- diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..ba05c5147 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Keep GitHub Actions (including SHA pins) up to date. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 \ No newline at end of file diff --git a/.github/workflows/check-docker-build.yml b/.github/workflows/check-docker-build.yml index 46590c0c7..410d81254 100644 --- a/.github/workflows/check-docker-build.yml +++ b/.github/workflows/check-docker-build.yml @@ -2,12 +2,26 @@ name: Check Docker Build on: pull_request: + paths: + - 'Dockerfile' + - 'docker-compose.yml' + - 'docker-entrypoint.sh' + - 'requirements.txt' + - 'Procfile' + - '.github/workflows/check-docker-build.yml' + +permissions: + contents: read + +concurrency: + group: docker-build-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: name: Check Docker Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Docker Compose Build run: docker compose build diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..a21fa7d1a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,55 @@ +name: "CodeQL" + +on: + push: + branches: [ "main" ] + paths: + - '**.py' + - 'requirements.txt' + - '.github/workflows/codeql.yml' + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + paths: + - '**.py' + - 'requirements.txt' + - '.github/workflows/codeql.yml' + schedule: + - cron: '45 4 * * *' # Daily at 04:45 UTC + +concurrency: + group: codeql-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + with: + languages: ${{ matrix.language }} + queries: security-extended + + - name: Autobuild + uses: github/codeql-action/autobuild@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/issue-auto-unassign.yml b/.github/workflows/issue-auto-unassign.yml index 040cd97cd..47aede255 100644 --- a/.github/workflows/issue-auto-unassign.yml +++ b/.github/workflows/issue-auto-unassign.yml @@ -1,16 +1,21 @@ +name: Auto-Unassign Inactive Issues + on: schedule: # * is a special character in YAML so you have to quote this string - cron: '0 0/12 * * *' workflow_dispatch: # Enable manual runs of the bot +permissions: + issues: write + jobs: unassign_issues: runs-on: ubuntu-latest name: Unassign issues steps: - name: Unassign issues - uses: codethesaurus/unassign-issues@1.3 + uses: codethesaurus/unassign-issues@fec46824b2acd69b012edb5db2101399019be0b6 # 1.3 id: unassign_issues with: token: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/json-validate.yml b/.github/workflows/json-validate.yml index df32f8399..6b7510800 100644 --- a/.github/workflows/json-validate.yml +++ b/.github/workflows/json-validate.yml @@ -5,14 +5,32 @@ on: paths: - '**.json' pull_request: + paths: + - '**.json' + +permissions: + contents: read + +concurrency: + group: json-syntax-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: name: Check JSON Files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: JSON Syntax Check - uses: limitusus/json-syntax-check@v1 - with: - pattern: "\\.json$" \ No newline at end of file + run: | + failed=0 + while IFS= read -r -d '' file; do + if python3 -m json.tool "$file" > /dev/null 2>&1; then + echo "OK: $file" + else + echo "Invalid JSON: $file" + failed=1 + fi + done < <(find . -type f -name '*.json' -not -path './.git/*' -not -path './venv/*' -print0) + exit "$failed" \ No newline at end of file diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 0f686baef..9d4291f2c 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -4,9 +4,28 @@ on: push: branches: - main + paths: + - 'web/**' + - 'codethesaurus/**' + - 'requirements.txt' + - 'manage.py' + - '.github/workflows/run-unit-tests.yml' pull_request: branches: - main + paths: + - 'web/**' + - 'codethesaurus/**' + - 'requirements.txt' + - 'manage.py' + - '.github/workflows/run-unit-tests.yml' + +permissions: + contents: read + +concurrency: + group: unit-tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: run: @@ -14,12 +33,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the branch - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: 3.14 + python-version: '3.14' cache: 'pip' - name: Install dependencies diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 000000000..31387b3ad --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,73 @@ +name: Security Scans + +on: + push: + branches: [ "main" ] + paths: + - '**.py' + - 'requirements.txt' + - '.github/workflows/security.yml' + pull_request: + branches: [ "main" ] + paths: + - '**.py' + - 'requirements.txt' + - '.github/workflows/security.yml' + schedule: + - cron: '30 4 * * *' # Daily at 04:30 UTC + +concurrency: + group: security-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + bandit: + name: Bandit (SAST) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install Bandit + run: pip install bandit==1.9.4 + + - name: Run Bandit + # -r: recursive, -ll: medium severity or higher, -x: exclude directories + run: bandit -r . -ll -x ./venv,./web/tests,./staticfiles + + dependency-check: + name: Dependency Scan (pip-audit) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install pip-audit + run: pip install pip-audit==2.10.1 + + - name: Run pip-audit + run: pip-audit -r requirements.txt + + osv-scanner: + name: Dependency Scan (OSV) + permissions: + contents: read + actions: read + security-events: write + uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc # v2.6.0 \ No newline at end of file diff --git a/.github/workflows/validate-language-info-files.yml b/.github/workflows/validate-language-info-files.yml index e057467bc..c5178f118 100644 --- a/.github/workflows/validate-language-info-files.yml +++ b/.github/workflows/validate-language-info-files.yml @@ -15,12 +15,12 @@ jobs: steps: - name: Checkout the branch - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: 3.14 + python-version: '3.14' cache: 'pip' - name: Install dependencies diff --git a/.github/workflows/validate-meta-info-file.yml b/.github/workflows/validate-meta-info-file.yml index 06cc031f3..e3c3f91e5 100644 --- a/.github/workflows/validate-meta-info-file.yml +++ b/.github/workflows/validate-meta-info-file.yml @@ -15,12 +15,12 @@ jobs: steps: - name: Checkout the branch - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: 3.14 + python-version: '3.14' cache: 'pip' - name: Install dependencies diff --git a/.gitignore b/.gitignore index a4e8fc8b2..cf32682c0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__ public_html/ staticfiles/ *.sqlite3 +/.junie \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..4b87ab358 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,137 @@ +# Code Thesaurus — Agent Guidelines + +These guidelines help coding agents (like Junie, Copilot, OpenCode, etc.) understand the project, its structure, and how to contribute effectively. + +## Critical Rules + +- **Always follow Test-Driven Development (TDD):** When implementing a new feature or fixing a bug, YOU MUST first provide the test case that reproduces the issue or defines the new behavior. Only then provide the implementation. Tests live in `web/tests/`. +- **Respect read-only mode:** Do not modify files unless explicitly asked. +- **Follow existing style:** Match the project's use of PEP 8, pylint, and `isort --profile black`. +- **No interactive commands:** All terminal commands must be non-interactive. +- **Don't make large assumptions:** If something is unclear, ask before making assumptions. +- **Thesaurus data is sacred:** When editing JSON files under `web/thesauruses/`, always validate them with the management commands below before finishing. + +## Project Overview + +Code Thesaurus is a polyglot developer reference tool. It compares a programming-language feature (e.g. "Data Types", "Functions", "Control Structures") side-by-side across one or two languages or databases, or displays a single-language "reference sheet." It's aimed at new and experienced programmers alike, so content must be beginner-friendly and technically correct. + +Most of the site is driven by JSON data files — the Django app mostly reads them and renders comparison/reference pages; the relational database only stores visit/lookup analytics. + +### Key Technologies +- Python 3.14 (per `runtime.txt` and CI; project requires pinned `requirements.txt`) +- Django 6.1 (project package `codethesaurus/`, single app `web/`) +- Bootstrap 5 (CDN) + Font Awesome (CDN) for the frontend +- Pygments (syntax highlighting), django-markdownify (Markdown rendering) +- jsonmerge / jsonschema (thesaurus data handling and validation) +- SQLite for local dev, PostgreSQL in production (via `dj-database-url`) +- Gunicorn (production), WhiteNoise (static files) + +## Repository Structure + +- `AGENTS.md` — This file (project guidance for agents) +- `codethesaurus/` — Django project settings, root URLs, and custom error handlers (400/403/404/500) +- `web/` — The single Django app: + - `models.py` — Non-DB classes that parse the thesaurus JSON (`MetaStructure`, `ThesaurusEntry`, `ThesaurusMetaInfo`) plus analytics DB models (`SiteVisit`, `LookupData`, `MissingLookup`) + - `views.py` — `index`, `about`, `statistics`, `concepts` (comparison/reference pages), `api_reference`, `api_compare`, custom error handlers, and helpers that log visits/lookups/missing items + - `urls.py` — Routes for the above (see **API** below) + - `middleware.py` — `DatabaseDownMiddleware`: catches DB `OperationalError` and renders `error500.html` + - `thesaurus_template_generators.py` — `generate_entry_template()` / `generate_meta_template()` for scaffolding starter JSON + - `templatetags/templatetags.py` — `concept_card` inclusion tag + - `templates/` — Django templates (`base.html`, `index.html`, `concepts.html`, `concept_card.html`, `statistics.html`, `about.html`, error pages, `robots.txt`) + - `static/` — CSS (`app.css`, `pygments_colorful.css`), JS (`checkAvailableStructs.js`, `contributors.js`), images + - `management/commands/` — `generate_template`, `generate_missing_templates`, `validatemetainfofile`, `validatelanginfofiles` + - `tests/` — Django unit/integration tests +- `web/thesauruses/` — **The core data repository** (see **Data Model Summary**) +- `docs/` — A separate, nested MkDocs repository (docs.codethesaur.us); not part of the main git repo +- `.github/workflows/` — CI/CD (see **Pre-merge Checks**) +- Deployment files: `Dockerfile`, `docker-compose.yml`, `docker-entrypoint.sh`, `Procfile` (Heroku), `runtime.txt`, `static.json` + +### API +- `GET /api////` — JSON reference for one language +- `GET /api//////` — JSON comparison of two languages + +## Local Development + +| Task | Command | +|------|---------| +| Install dependencies | `pip install -r requirements.txt` | +| Apply migrations (creates `db.sqlite3` with analytics tables) | `python manage.py migrate` | +| Create superuser (admin only) | `python manage.py createsuperuser` | +| Run dev server | `python manage.py runserver` | +| Run the whole site with Docker | `docker-compose up` (serves at `http://localhost:8000`) | +| Run all tests | `python manage.py test` | +| Validate meta info | `python manage.py validatemetainfofile` | +| Validate language info files | `python manage.py validatelanginfofiles` | +| Generate a starter language structure file | `python manage.py generate_template` | +| Generate missing structure templates | `python manage.py generate_missing_templates` | + +**Note:** The site mostly works without DB data — the database only stores visit/lookup statistics and admin data, so you typically only need `migrate` + `runserver` to develop against thesaurus data. + +## Data Model Summary + +The thesaurus is a file-based (JSON) data model, not a relational one. All entry points understand it through `ThesaurusMetaInfo` → `MetaStructure` → `ThesaurusEntry` in `web/models.py`. + +- **`meta_info.json`** — The root registry: `categories` (currently `langs` and `databases`), `languages` (key → display name for every supported language/database), and `structures` (per-category map of structure key → display name). Editing it requires running `validatemetainfofile`. +- **`_meta/.json`** — One file per concept area (e.g. `data_types.json`). Defines `meta` (`structure`, `structure_name`) and `categories`, where each category maps concept IDs → human-friendly concept names. These concept IDs must match across all language files. +- **`langs///.json`** — One file per language/version/structure (26 languages: ada, bash, c, cpp, csharp, clips, clojure, go, haskell, java, javascript, kotlin, lua, nim, objectivec, perl, php, powershell, python, r, ruby, rust, scala, swift, typescript, vbnet). Each has a `meta` block (`language`, `language_version`, `language_name`, `structure`) and a `concepts` block keyed by the concept IDs from `_meta/`. +- **`databases///.json`** — Same pattern for databases (mongodb, mysql, postgresql) using the database structures (`deletions`, `filtering`, `inserts`, `queries`). + +**Concept field rules** (validated by `validatelanginfofiles`): +- Each concept is one of: `code`, `code` + `comment`, `not-implemented`, or `not-implemented` + `comment`. Other combinations are invalid. +- `code` — the code snippet; a string or an array of strings (arrays are preferred for multiple ways to do something; use the `comment` to differentiate when to pick which). +- `comment` (singular) — explanatory notes. Never `comments`. Backticks (`` `code` ``) may be used to reference code within comments. +- `"not-implemented": true` (hyphenated) — for concepts that don't exist in that language/version. Never `not_implemented`. **Do not** write an algorithm to emulate a missing feature — mark it not-implemented instead. +- Do **not** include a `categories` section in language files; that is handled by the `_meta` files. +- Code blocks must be technically compilable/runnable if copied — no placeholder prose inside code. Keep explanatory text in `comment`. + +**Analytics DB models** (`web/models.py`): +- **SiteVisit** — one row per page view (URL, user agent, referer). +- **LookupData** — one row per comparison/reference lookup, linked to a `SiteVisit` (`entry1`/`entry2` = the pair, or the single language for references). +- **MissingLookup** — records requested languages/structures/concepts that don't exist yet, so maintainers can see gaps. The `/statistics/` page aggregates all three. + +## Coding Standards + +- **PEP 8**: Follow standard Python style. +- **Path handling**: Use `pathlib.Path` for file-system path manipulation, not `os.path` (though `os.path` remains in legacy code — don't extend the pattern). +- **Management commands**: Use `self.stdout.write()` / `self.stderr.write()` with `self.style` (e.g. `self.style.SUCCESS`, `self.style.ERROR`) instead of `print()`. +- **Logic structure**: Break complex or monolithic methods into smaller, focused, descriptive private methods. +- **Models/helpers**: Reuse the helper methods in `web/models.py` (`ThesaurusMetaInfo`, `ThesaurusEntry` — e.g. `is_concept_complete`, `is_category_incomplete`, `has_any_implemented_in_category`) to keep view logic clean, rather than re-implementing JSON traversal. +- **Linting/formatting**: `pylint` and `isort` are pinned in `requirements.txt`. Keep code clean under both. +- **Naming conventions**: Language/entry keys are lowercase (`python`, `javascript`); JSON files are snake_case (`control_structures.json`). +- **Frontend**: UI changes go in `web/templates/` and `web/static/`. Follow existing CSS patterns in `web/static/css/`. Keep pages responsive and accessible; avoid inline styles; use Bootstrap for consistency. +- **Documentation**: Update the external docs (docs.codethesaur.us, built from the nested `docs/` MkDocs repo) for significant changes to core logic or data structures. Use docstrings for complex logic in `views.py` or management commands. + +## Testing Strategy and Contribution + +- **Location**: Tests live in `web/tests/` (`test_urls.py`, `test_views.py`, `test_views_extra.py`, `test_models.py`, `test_models_extra.py`, `test_db_models.py`, `test_categories.py`, `test_generators.py`, `test_templatetags.py`, `test_templates.py`, `test_commands.py`, `test_middleware.py`). +- **TDD Requirement**: When fixing bugs, add a reproducer test before applying the fix. +- **Data changes**: JSON edits under `web/thesauruses/**` require the two validation commands to pass (`validatemetainfofile` after `meta_info.json`/`_meta/` changes, `validatelanginfofiles` after language file changes). Add/update validation coverage in `test_commands.py` when the validators change. +- **Scope**: Include model parsing logic, views, templates, URL routing, and management commands. +- **Keep changes minimal** and avoid large refactors in feature tasks. Make small, targeted changes rather than building for hypothetical future needs. Do not rename files without a valid technical reason. +- **Pull requests**: Follow `.github/PULL_REQUEST_TEMPLATE.md` — note any AI bots used and complete the checklist. The project participates in Hacktoberfest; see `CONTRIBUTING.md` for the issue-claiming workflow. + +### Adding a new language or database +1. Create a directory under `web/thesauruses/` with the language/database key. +2. Create a version subdirectory (e.g. `3` for Python 3). +3. Add JSON files matching the `_meta` structures — use `python manage.py generate_template` to scaffold. +4. Only implement what actually exists in the language; otherwise use `"not-implemented": true`. +5. Register the language in `meta_info.json` (`languages`) and validate with both validation commands. + +## Pre-merge Checks (CI must pass) + +- All tests: `python manage.py test` (after `python manage.py migrate`) +- Thesaurus data validation: `python manage.py validatemetainfofile` + `python manage.py validatelanginfofiles` +- All JSON files must be valid (checked by the `json-validate` workflow using a JSON syntax checker) +- Security scan: `bandit -r . -ll -x ./venv,./web/tests,./staticfiles` +- Dependency scans: `pip-audit -r requirements.txt` and `safety check -r requirements.txt` (run by the `security` workflow) +- CodeQL Python analysis (run by the `codeql` workflow) +- Docker build check: `docker compose build` (run by the `check-docker-build` workflow when Docker/deps change) + +## Agent-Specific Tips (Junie, Copilot, OpenCode, etc.) + +- Search the codebase to infer structure; `web/models.py`, `web/views.py`, and `web/thesaurus_template_generators.py` contain the important reusable code. Reuse those whenever possible. +- Before editing thesaurus data, read the relevant `web/thesauruses/_meta/.json` file to ensure new data matches the expected keys and categories. +- If a request is about adding new language/concept data, it's almost always a JSON edit + validation, not a code change — read the `docs/` project-architecture and thesaurus-editing pages (docs.codethesaur.us) for the current conventions. +- Don't introduce a build step for frontend JS/CSS; the frontend is server-rendered Django templates with hand-rolled static assets. +- The `docs/` directory is a separate git repo (MkDocs); don't commit documentation changes inside the main repo's history. +- Never edit `db.sqlite3` data as a substitute for thesaurus data, and don't rely on it being populated in CI — tests must not depend on it. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d9bfc637b..51c42b61d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,33 @@ FROM python:3.14-slim ENV PYTHONBUFFERED=1 WORKDIR /code + +# Install system dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ libpq-dev \ gcc \ && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies COPY requirements.txt /code/ RUN pip install --no-cache-dir -r requirements.txt -COPY . . + +# Copy the entrypoint script and make it executable +COPY docker-entrypoint.sh /code/ +RUN chmod +x /code/docker-entrypoint.sh + +# Create a non-root user and give them ownership of the code directory +RUN useradd -m django && chown -R django:django /code +USER django + +# Copy the rest of the application code +COPY --chown=django:django . . + +# Expose the port the app runs on EXPOSE 8000 -CMD python manage.py migrate && \ - python manage.py collectstatic --clear --no-input && \ - python manage.py runserver 0.0.0.0:8000 + +# Set the entrypoint script +ENTRYPOINT ["/code/docker-entrypoint.sh"] + +# Default command to run the application +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] diff --git a/README.md b/README.md index d7f700329..91a540eb9 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,16 @@ Check out our [Installation/Running Locally](https://docs.codethesaur.us/install Check out the [Contributing Guide](https://docs.codethesaur.us/contributing/) to learn more about how you can help add more language data, fix bugs, or add features! +## Use of AI tools + +We're happy for you to use AI tools to help you contribute — a lot of our contributors do, and they can be a great way to speed up work or learn something new. But AI-generated changes have to meet the same bar as any other contribution: + +- **Follow the project's conventions and docs** — the structure, naming, and style, including the thesaurus data rules. Code examples must genuinely work in that language; don't let an AI invent syntax or fake a feature that should be marked `"not-implemented": true`. +- **Follow the PR template** — including the "AI bots used" section, telling us what you used and how. (Please actually review the changes before submitting.) +- **Be tested** — run the unit tests and the data-validation commands, and make sure the GitHub Actions checks pass. + +Low-effort AI pull requests — unchecked dumps of generated code that ignore the conventions, skip the template, or aren't tested — will be closed or marked as spam. If we misjudge one of yours, reach out and we'll take another look. + ## Is this project available for Hacktoberfest contributions? Yes! The Code Thesaurus code and documentation projects are both enabled for Hacktoberfest contributions. diff --git a/codethesaurus/settings.py b/codethesaurus/settings.py index fb6397371..111aa035f 100644 --- a/codethesaurus/settings.py +++ b/codethesaurus/settings.py @@ -4,15 +4,15 @@ Generated by 'django-admin startproject' using Django 3.1.1. For more information on this file, see -https://docs.djangoproject.com/en/6.0/topics/settings/ +https://docs.djangoproject.com/en/6.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/6.0/ref/settings/ +https://docs.djangoproject.com/en/6.1/ref/settings/ """ -from pathlib import Path import os +from pathlib import Path + import dj_database_url -import django_on_heroku # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -48,6 +48,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'web.middleware.DatabaseDownMiddleware', 'django.middleware.common.CommonMiddleware', @@ -87,7 +88,7 @@ WSGI_APPLICATION = 'codethesaurus.wsgi.application' # Database -# https://docs.djangoproject.com/en/6.0/ref/settings/#databases +# https://docs.djangoproject.com/en/6.1/ref/settings/#databases DATABASES = { 'default': { @@ -101,7 +102,7 @@ DATABASES['default'].update(db_from_env) # Password validation -# https://docs.djangoproject.com/en/6.0/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/6.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { @@ -119,7 +120,7 @@ ] # Internationalization -# https://docs.djangoproject.com/en/6.0/topics/i18n/ +# https://docs.djangoproject.com/en/6.1/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -130,14 +131,14 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/6.0/howto/static-files/ +# https://docs.djangoproject.com/en/6.1/howto/static-files/ # https://dev.to/fazledyn/deploying-django-3-1-on-heroku-417 -# https://github.com/pkrefta/django-on-heroku/blob/3b2367fec9417230dbfba0235353403865386a41/django_on_heroku/core.py#L106 +# https://whitenoise.readthedocs.io/en/stable/ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') os.makedirs(STATIC_ROOT, exist_ok=True) # Extra places for collectstatic to find static files. @@ -155,30 +156,85 @@ if item == "": break contact = item.split(":") - ADMINS.append((contact[0], contact[1])) -EMAIL_HOST = os.getenv('EMAIL_HOST', '') -EMAIL_HOST_USER = os.getenv('EMAIL_HOST_USER', '') -EMAIL_HOST_PASSWORD = os.getenv('EMAIL_HOST_PASSWORD', '') + ADMINS.append(contact[1]) SERVER_EMAIL = os.getenv('SERVER_EMAIL', '') -EMAIL_USE_SSL = os.getenv('EMAIL_USE_SSL', '') -EMAIL_PORT = os.getenv('EMAIL_PORT', '') -EMAIL_USE_TLS = os.getenv('EMAIL_USE_TLS', '') + +# Django 6.1+ email configuration via the MAILERS framework. +# The deprecated EMAIL_* settings must not be used alongside MAILERS. +# See https://docs.djangoproject.com/en/6.1/topics/email/#email-configuration +MAILERS = { + "default": { + "BACKEND": "django.core.mail.backends.smtp.EmailBackend", + "OPTIONS": { + "host": os.getenv('EMAIL_HOST', ''), + "port": int(os.getenv('EMAIL_PORT', '25') or 25), + "username": os.getenv('EMAIL_HOST_USER', ''), + "password": os.getenv('EMAIL_HOST_PASSWORD', ''), + "use_tls": os.getenv('EMAIL_USE_TLS', '') == 'True', + "use_ssl": os.getenv('EMAIL_USE_SSL', '') == 'True', + }, + }, +} LOGGING = { "version": 1, # the dictConfig format version "disable_existing_loggers": False, # retain the default loggers + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", + }, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", + }, + }, + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", + } + }, "handlers": { + "console": { + "level": "INFO", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + }, + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", + }, "mail_admins": { "level": "ERROR", + "filters": ["require_debug_false"], "class": "django.utils.log.AdminEmailHandler", "include_html": True, }, - } + }, + "loggers": { + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", + }, + "django.server": { + "handlers": ["django.server"], + "level": "INFO", + "propagate": False, + }, + }, } SIMILAR_LEXERS = { "clips": "prolog", } -# Configure Django App for Heroku. -django_on_heroku.settings(locals(), test_runner=False, databases=False, staticfiles=True, logging=True) +# WhiteNoise serves static files from STATIC_ROOT. +STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", + }, +} diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 000000000..06238ce17 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +# Run migrations if needed +echo "Applying database migrations..." +python manage.py migrate --noinput + +# Collect static files if needed (usually for production, but good to have) +if [ "$SYSTEM_ENV" = "PRODUCTION" ] || [ "$SYSTEM_ENV" = "STAGING" ]; then + echo "Collecting static files..." + python manage.py collectstatic --noinput --clear +fi + +exec "$@" diff --git a/requirements.txt b/requirements.txt index 7789ee9ec..332d1461c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,41 +1,31 @@ -asgiref==3.11.1 -astroid==3.0.1 -attrs==25.4.0 -bleach==6.4.0 -colorama==0.4.6 -dill==0.4.1 -dj-database-url==3.1.0 -Django==6.0.6 -django-markdownify==0.9.6 -django-on-heroku==1.1.2 -gunicorn==25.0.1 -importlib_metadata==8.7.1 -isort==5.13.2 -jsonmerge==1.9.2 -jsonschema==4.26.0 -jsonschema-specifications==2025.9.1 -lazy-object-proxy==1.12.0 -Markdown==3.10.1 -mccabe==0.7.0 -packaging==26.0 -platformdirs==4.5.1 -protobuf==6.33.5 -psycopg2-binary==2.9.11 -Pygments==2.19.2 -pylint==3.0.2 -pyparsing==3.3.2 -pytz==2025.2 -referencing==0.37.0 -rpds-py==0.30.0 -six==1.17.0 -sqlparse==0.6.0 -tinycss2==1.4.0 -toml==0.10.2 -tomli==2.4.0 -tomlkit==0.14.0 -typing_extensions==4.15.0 -tzdata==2025.3 -webencodings==0.5.1 -whitenoise==6.11.0 -wrapt==2.1.1 -zipp==3.23.0 +asgiref==3.11.0 +astroid==3.0.1 +attrs==25.4.0 +bleach==6.4.0 +colorama==0.4.6 +dill==0.3.7 +dj-database-url==3.0.1 +Django==6.1.1 +django-markdownify==0.9.7 +gunicorn==26.0.0 +isort==5.12.0 +jsonmerge==1.9.2 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +Markdown==3.8.2 +mccabe==0.7.0 +packaging==26.2 +platformdirs==4.10.0 +psycopg2-binary==2.9.11 +Pygments==2.20.0 +pylint==3.0.2 +referencing==0.37.0 +rpds-py==2026.5.1 +six==1.17.0 +sqlparse==0.6.0 +tinycss2==1.5.1 +tomlkit==0.15.0 +typing_extensions==4.15.0 +tzdata==2026.2 +webencodings==0.5.1 +whitenoise==6.11.0 diff --git a/web/management/commands/generate_missing_templates.py b/web/management/commands/generate_missing_templates.py index 1fc8c0a31..7830b4e4a 100644 --- a/web/management/commands/generate_missing_templates.py +++ b/web/management/commands/generate_missing_templates.py @@ -1,6 +1,7 @@ +from django.core.management import call_command from django.core.management.base import BaseCommand -from web.models import Language, MetaInfo +from web.models import ThesaurusEntry, ThesaurusMetaInfo import os @@ -9,12 +10,12 @@ class Command(BaseCommand): help = 'Generate missing language thesaurus files to be filled out' def handle(self, *args, **options): - meta_info = MetaInfo() + meta_info = ThesaurusMetaInfo() languages = meta_info.languages structures = meta_info.structures for language in languages: - versions = Language(language, languages[language]).versions() + versions = ThesaurusEntry(language, languages[language]).versions() for version in versions: for structure in structures: file_path = os.path.join( @@ -25,4 +26,9 @@ def handle(self, *args, **options): structure + '.json' ) if not os.path.exists(file_path): - os.system(f'python manage.py generate_template "{language}" "{structure}" --language-version="{version}"') \ No newline at end of file + call_command( + 'generate_template', + language, + structure, + language_version=version, + ) \ No newline at end of file diff --git a/web/management/commands/generate_template.py b/web/management/commands/generate_template.py index f60217ef1..6ad989dc0 100644 --- a/web/management/commands/generate_template.py +++ b/web/management/commands/generate_template.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand -from web.thesaurus_template_generators import generate_language_template -from web.models import MetaInfo +from web.thesaurus_template_generators import generate_entry_template +from web.models import ThesaurusMetaInfo from packaging.version import parse as parse_version import os @@ -11,7 +11,7 @@ class Command(BaseCommand): help = 'Generate language thesaurus files to be filled out' def add_arguments(self, parser): - structures = list(MetaInfo().structures.keys()) + structures = list(ThesaurusMetaInfo().structures.keys()) parser.add_argument('language', help="Key of the programming language") parser.add_argument( @@ -28,7 +28,7 @@ def handle(self, *args, **options): def generate_file(self, language, structure, language_version): try: - template = generate_language_template( + template = generate_entry_template( language, structure, language_version diff --git a/web/management/commands/validatelanginfofiles.py b/web/management/commands/validatelanginfofiles.py index e6c35e38f..b3083da99 100644 --- a/web/management/commands/validatelanginfofiles.py +++ b/web/management/commands/validatelanginfofiles.py @@ -3,6 +3,8 @@ from django.core.management.base import BaseCommand, CommandError +from web.models import ThesaurusMetaInfo + class Command(BaseCommand): help = "Reads all language JSON files to ensure they're constructed correctly" @@ -12,18 +14,24 @@ def __init__(self, *args, **kwargs): self.error_count = 0 self.warning_count = 0 self.thesauruses_path = Path("web/thesauruses") + self.metainfo = None def handle(self, *args, **options): - for lang_dir in self.thesauruses_path.iterdir(): - if not lang_dir.is_dir() or lang_dir.name == "_meta": + self.metainfo = ThesaurusMetaInfo() + for category_dir in self.thesauruses_path.iterdir(): + if not category_dir.is_dir() or category_dir.name == "_meta": continue - - for version_dir in lang_dir.iterdir(): - if not version_dir.is_dir(): + + for lang_dir in category_dir.iterdir(): + if not lang_dir.is_dir(): continue - for structure_file in version_dir.glob("*.json"): - self.validate_language_file(structure_file) + for version_dir in lang_dir.iterdir(): + if not version_dir.is_dir(): + continue + + for structure_file in version_dir.glob("*.json"): + self.validate_language_file(structure_file) if self.warning_count > 0: self.stdout.write(self.style.WARNING(f"{self.warning_count} warnings found.")) @@ -55,12 +63,13 @@ def validate_language_file(self, file_path): self.check_meta_section(data, relative_path) self.check_concepts(data, relative_path) + self.check_category_structure_consistency(data, relative_path) def check_meta_section(self, data, relative_path): meta = data.get("meta", {}) - # relative_path is something like "python/3/data_types.json" - # parts[0] is the language directory name - lang_dir = relative_path.parts[0] + # relative_path is something like "langs/python/3/data_types.json" + # parts[1] is the language directory name + lang_dir = relative_path.parts[1] language = meta.get("language") language_version = meta.get("language_version") @@ -80,7 +89,7 @@ def check_meta_section(self, data, relative_path): if not language_name: self.report_error(f"`{relative_path}` has an empty `language_name` attribute and needs to be updated") - elif language_name in ["Human-Friendly Language Name", "Human-Readable Language Name"]: + elif language_name in ["Human-Friendly ThesaurusEntry Name", "Human-Readable ThesaurusEntry Name"]: self.report_error(f"`{relative_path}` has the default `language_name` attribute and needs to be updated") if "categories" in data: @@ -130,3 +139,13 @@ def check_concepts(self, data, relative_path): for key in item_data: if key not in allowed_keys: self.report_warning(f"`{relative_path}`, ID: `{concept_id}` has a line `{key}` that's unknown") + + def check_category_structure_consistency(self, data, relative_path): + """Check if the structure is allowed for the category it is in""" + # relative_path is like "langs/python/3/data_types.json" + category = relative_path.parts[0] + structure_name = relative_path.name.split('.')[0] + + category_structures = self.metainfo.category_structures.get(category, {}) + if structure_name not in category_structures: + self.report_error(f"`{relative_path}` is in category `{category}`, but `{structure_name}` is not a valid structure for this category in `meta_info.json`") diff --git a/web/management/commands/validatemetainfofile.py b/web/management/commands/validatemetainfofile.py index 1fe975b41..9a9b81fb8 100644 --- a/web/management/commands/validatemetainfofile.py +++ b/web/management/commands/validatemetainfofile.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError -from web.models import MetaInfo +from web.models import ThesaurusMetaInfo class Command(BaseCommand): @@ -16,8 +16,9 @@ def __init__(self, *args, **kwargs): self.meta_path = self.thesauruses_path / "_meta" def handle(self, *args, **options): - self.metainfo = MetaInfo() + self.metainfo = ThesaurusMetaInfo() + self.check_category_directories() self.check_thesaurus_directories() self.check_meta_info_consistency() self.check_meta_files_consistency() @@ -31,40 +32,66 @@ def report_error(self, message): self.stderr.write(self.style.ERROR(f"[Error] {message}")) self.error_count += 1 + def check_category_directories(self): + """Check all categories in meta_info.json have corresponding directories and vice versa""" + # Check if all directories in web/thesauruses are accounted for + for category_dir in self.thesauruses_path.iterdir(): + if not category_dir.is_dir() or category_dir.name == "_meta": + continue + if category_dir.name not in self.metainfo.categories: + self.report_error(f"Directory `{category_dir}` exists but `{category_dir.name}` is not listed as a category in `meta_info.json`") + + # Check if all categories in meta_info.json have directories + for category_key in self.metainfo.categories: + path = self.thesauruses_path / category_key + if not path.is_dir(): + self.report_error(f"Category `{category_key}` is listed in `meta_info.json` but directory `{path}` was not found") + def check_thesaurus_directories(self): """Look through thesaurus directories, see if any files don't match""" meta_files = {f.name for f in self.meta_path.iterdir() if f.is_file()} - for lang_dir in self.thesauruses_path.iterdir(): - if not lang_dir.is_dir() or lang_dir.name == "_meta": + for category_dir in self.thesauruses_path.iterdir(): + if not category_dir.is_dir() or category_dir.name == "_meta": continue - lang = lang_dir.name - if lang not in self.metainfo.languages: - self.report_error(f"`{lang_dir}` exists but {lang} is not listed as a language in `meta_info.json`") - - for version_dir in lang_dir.iterdir(): - if version_dir.is_file(): - self.report_error(f"`{version_dir}` is a file but a directory for a version was expected") + for lang_dir in category_dir.iterdir(): + if not lang_dir.is_dir(): continue - for structure_file in version_dir.iterdir(): - if structure_file.name not in meta_files: - self.report_error(f"`{structure_file}` is not a valid concept filename") + lang = lang_dir.name + if lang not in self.metainfo.languages: + self.report_error(f"`{lang_dir}` exists but {lang} is not listed as a language in `meta_info.json`") + + for version_dir in lang_dir.iterdir(): + if version_dir.is_file(): + self.report_error(f"`{version_dir}` is a file but a directory for a version was expected") + continue + + for structure_file in version_dir.iterdir(): + if structure_file.name not in meta_files: + self.report_error(f"`{structure_file}` is not a valid concept filename") def check_meta_info_consistency(self): """Check all language directories exist for languages listed in meta_info.json""" for meta_lang in self.metainfo.languages: - path = self.thesauruses_path / meta_lang - if not path.is_dir(): + found = False + for category_dir in self.thesauruses_path.iterdir(): + if not category_dir.is_dir() or category_dir.name == "_meta": + continue + path = category_dir / meta_lang + if path.is_dir(): + found = True + break + if not found: lang_name = self.metainfo.languages[meta_lang] - self.report_error(f"{lang_name} is listed as a language in `meta_info.json` but the directory `{path}` doesn't exist") + self.report_error(f"{lang_name} is listed as a language in `meta_info.json` but no directory for it was found in any category") def check_meta_files_consistency(self): - """Check structures in _meta match those in MetaInfo""" + """Check structures in _meta match those in ThesaurusMetaInfo""" meta_files = {f.name for f in self.meta_path.iterdir() if f.is_file()} - # Check files in _meta are listed in MetaInfo + # Check files in _meta are listed in ThesaurusMetaInfo for meta_file in meta_files: if not meta_file.endswith(".json"): continue @@ -72,7 +99,7 @@ def check_meta_files_consistency(self): if structure_name not in self.metainfo.structures: self.report_error(f"`{self.meta_path / meta_file}` is not listed as a structure in `meta_info.json`") - # Check structures listed in MetaInfo have corresponding files in _meta + # Check structures listed in ThesaurusMetaInfo have corresponding files in _meta for structure in self.metainfo.structures: path = self.meta_path / f"{structure}.json" if not path.is_file(): diff --git a/web/migrations/0005_rename_language_to_entry_in_lookupdata.py b/web/migrations/0005_rename_language_to_entry_in_lookupdata.py new file mode 100644 index 000000000..767c8e279 --- /dev/null +++ b/web/migrations/0005_rename_language_to_entry_in_lookupdata.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.27 on 2026-01-22 19:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('web', '0004_missinglookup'), + ] + + operations = [ + migrations.RenameField( + model_name='lookupdata', + old_name='language1', + new_name='entry1', + ), + migrations.RenameField( + model_name='lookupdata', + old_name='language2', + new_name='entry2', + ), + ] diff --git a/web/models.py b/web/models.py index 0236e79ca..9b0944724 100644 --- a/web/models.py +++ b/web/models.py @@ -6,6 +6,12 @@ from django.db import models +def _is_safe_path_component(value): + return bool(value) and value not in ( + os.curdir, os.pardir + ) and "/" not in value and "\\" not in value + + # pylint: disable=too-few-public-methods class MetaStructure: """ @@ -25,12 +31,19 @@ def __init__(self, key, name): self.key = key self.name = name + if not _is_safe_path_component(key): + raise FileNotFoundError(f"Structure key is not a safe path component: {key!r}") + if key in MetaStructure._cached_files: self.categories = MetaStructure._cached_files[key] return - meta_structure_file_path = os.path.join( - "web", "thesauruses", "_meta", f"{key}.json") + thesaurus_root = os.path.join("web", "thesauruses") + root_real = os.path.realpath(thesaurus_root) + meta_structure_file_path = os.path.realpath( + os.path.join(thesaurus_root, "_meta", f"{key}.json")) + if not meta_structure_file_path.startswith(root_real + os.sep): + raise FileNotFoundError(f"Structure key escapes the thesaurus dir: {key!r}") with open(meta_structure_file_path, 'r', encoding='UTF-8') as meta_structure_file: meta_structure_file_json = json.load(meta_structure_file) @@ -38,7 +51,7 @@ def __init__(self, key, name): MetaStructure._cached_files[key] = self.categories -class Language: +class ThesaurusEntry: """ Represents a programming language and knows how to fetch concepts for a structure key @@ -46,7 +59,7 @@ class Language: def __init__(self, key, name): """ - Initialize the Language object, which will contain concepts for a given + Initialize the ThesaurusEntry object, which will contain concepts for a given structure :param key: key of the language in the meta_info.json file @@ -58,13 +71,34 @@ def __init__(self, key, name): self.name = name self.concepts = None self.version = None - self.language_dir = os.path.join("web", "thesauruses", self.key) + self.language_dir = None + if not _is_safe_path_component(self.key): + return + + thesaurus_root = os.path.join("web", "thesauruses") + root_real = os.path.realpath(thesaurus_root) + for category in os.listdir(thesaurus_root): + if category == "_meta" or not os.path.isdir(os.path.join(thesaurus_root, category)): + continue + potential_dir = os.path.realpath( + os.path.join(thesaurus_root, category, self.key)) + if potential_dir.startswith(root_real + os.sep) and os.path.isdir(potential_dir): + self.language_dir = potential_dir + break + + if self.language_dir is None: + fallback_dir = os.path.realpath( + os.path.join(thesaurus_root, "langs", self.key)) + if fallback_dir.startswith(root_real + os.sep): + self.language_dir = fallback_dir self.version = None def versions(self): - """Generate all versions and their paths for the Language""" + """Generate all versions and their paths for the ThesaurusEntry""" versions = dict() + if self.language_dir is None: + return versions try: for entry in os.scandir(self.language_dir): if not entry.is_dir(): @@ -85,28 +119,36 @@ def __bool__(self): :rtype: bool """ - return os.path.exists(self.language_dir) + return self.language_dir is not None and os.path.exists(self.language_dir) def load_concepts(self, structure_key, version): """ - Loads the structure file into the Language object + Loads the structure file into the ThesaurusEntry object :param structure_key: the key for the structure to load :param version: the version of the language """ - file_path = os.path.join(self.language_dir, version, f"{structure_key}.json") + if not (_is_safe_path_component(structure_key) and _is_safe_path_component(version)): + raise FileNotFoundError( + f"Unsafe structure/version path components: {structure_key!r} / {version!r}") + root_real = os.path.realpath(os.path.join("web", "thesauruses")) + file_path = os.path.realpath(os.path.join( + self.language_dir, version, f"{structure_key}.json")) + if not file_path.startswith(root_real + os.sep): + raise FileNotFoundError( + f"Structure/version escape the thesaurus dir: {structure_key!r} / {version!r}") with open(file_path, 'r', encoding='UTF-8') as file: file_json = json.load(file) self.concepts = file_json["concepts"] self.version = version def load_filled_concepts(self, structure_key, version): - from web.thesaurus_template_generators import generate_language_template + from web.thesaurus_template_generators import generate_entry_template """ - Loads the concepts from the language's structure file + Loads the concepts from the entry's structure file :param structure_key: the ID for the concept to load - :param version: the version of the language + :param version: the version of the entry :return: a dict containing the code and comment, and possibly the 'not-implemented' flag. They are empty code entries if not specified :rtype: object Filled template @@ -114,7 +156,7 @@ def load_filled_concepts(self, structure_key, version): self.load_concepts(structure_key, version) - template = generate_language_template( + template = generate_entry_template( self.key, structure_key, version @@ -128,24 +170,24 @@ def load_filled_concepts(self, structure_key, version): return response - def load_comparison(self, structure_key, lang, version_lang, version_self): - lang = Language(lang, "") + def load_comparison(self, structure_key, entry_key, version_entry, version_self): + entry_obj = ThesaurusEntry(entry_key, "") self_filled_concept = self.load_filled_concepts(structure_key, version_self) - lang_filled_concept = lang.load_filled_concepts(structure_key, version_lang) + entry_filled_concept = entry_obj.load_filled_concepts(structure_key, version_entry) - if self_filled_concept is False or lang_filled_concept is False: + if self_filled_concept is False or entry_filled_concept is False: return False response = json.dumps({ "meta": { - "language_1": self.key, - "language_version_1": version_self, - "language_2": lang.key, - "language_version_2": version_lang, + "entry_1": self.key, + "entry_version_1": version_self, + "entry_2": entry_obj.key, + "entry_version_2": version_entry, "structure": structure_key }, "concepts1": json.loads(self_filled_concept)['concepts'], - "concepts2": json.loads(lang_filled_concept)['concepts'] + "concepts2": json.loads(entry_filled_concept)['concepts'] }, indent=2) return response @@ -154,7 +196,7 @@ def load_comparison(self, structure_key, lang, version_lang, version_self): def concept(self, concept_key): """ Get the concept (including code and comment) from the concept file for - that Language + that ThesaurusEntry :param concept_key: key for the concept to look up :returns: a dict containing the code and comment, and possibly the @@ -248,8 +290,8 @@ def has_any_implemented_in_category(self, category_concepts_keys): return False -class MissingLanguageError(Exception): - """Error for when a requested language is not defined in `meta.json`""" +class MissingEntryError(Exception): + """Error for when a requested entry is not defined in `meta.json`""" def __init__(self, key): super().__init__() self.key = key @@ -257,87 +299,100 @@ def __init__(self, key): class MissingStructureError(Exception): """ - Error that signals that a specific language & version does not have the structure + Error that signals that a specific entry & version does not have the structure defined """ - def __init__(self, structure, language_key, language_name, language_version): + def __init__(self, structure, entry_key, entry_name, entry_version): super().__init__() self.structure = structure - self.language_key = language_key - self.language_name = language_name - self.language_version = language_version + self.entry_key = entry_key + self.entry_name = entry_name + self.entry_version = entry_version -class MetaInfo: +class ThesaurusMetaInfo: """Holds info about structures and languages""" _cached_structures = None _cached_languages = None def __init__(self): """ - Initializes MetaInfo object with meta language information + Initializes ThesaurusMetaInfo object with meta language information :rtype: None """ - if MetaInfo._cached_structures is not None: - self.structures = MetaInfo._cached_structures - self.languages = MetaInfo._cached_languages + if ThesaurusMetaInfo._cached_structures is not None: + self.structures = ThesaurusMetaInfo._cached_structures + self.languages = ThesaurusMetaInfo._cached_languages + self.categories = getattr(ThesaurusMetaInfo, "_cached_categories", {}) + self.category_structures = getattr(ThesaurusMetaInfo, "_cached_category_structures", {}) return meta_info_file_path = os.path.join( "web", "thesauruses", "meta_info.json") with open(meta_info_file_path, 'r', encoding='UTF-8') as meta_file: meta_info_json = json.load(meta_file) - self.structures = meta_info_json["structures"] + + self.categories = meta_info_json.get("categories", {}) self.languages = meta_info_json["languages"] - MetaInfo._cached_structures = self.structures - MetaInfo._cached_languages = self.languages + + # Flatten structures for backward compatibility where needed, + # but keep track of category-specific ones + self.category_structures = meta_info_json["structures"] + self.structures = {} + for cat_structs in self.category_structures.values(): + self.structures.update(cat_structs) + + ThesaurusMetaInfo._cached_structures = self.structures + ThesaurusMetaInfo._cached_languages = self.languages + ThesaurusMetaInfo._cached_categories = self.categories + ThesaurusMetaInfo._cached_category_structures = self.category_structures - def language_name(self, language_key): + def entry_name(self, entry_key): """ - Given a structure key (from meta_info.json), returns the language's human-friendly name + Given a structure key (from meta_info.json), returns the entry's human-friendly name - :param language_key: key of the language located in the meta_info.json file + :param entry_key: key of the entry located in the meta_info.json file :return: string with the human-friendly name """ - return self.languages[language_key] + return self.languages[entry_key] - def language(self, language_key): + def entry(self, entry_key): """ - Given a language key (from meta_info.json), returns the whole - Language for it + Given a entry key (from meta_info.json), returns the whole + ThesaurusEntry for it - :param language_key: key of the language located in the meta_info.json + :param entry_key: key of the entry located in the meta_info.json file - :return: Language for the requested key - :rtype: Language + :return: ThesaurusEntry for the requested key + :rtype: ThesaurusEntry """ - return Language( - language_key, - self.language_name(language_key), + return ThesaurusEntry( + entry_key, + self.entry_name(entry_key), ) - def load_languages(self, language_keys_versions, meta_structure): - """Tries to load all languages from `language_keys` and the requested `structure`""" - languages = [] - for language_key, version in language_keys_versions: + def load_entries(self, entry_keys_versions, meta_structure): + """Tries to load all entries from `entry_keys` and the requested `structure`""" + entries = [] + for entry_key, version in entry_keys_versions: try: - language = self.language(language_key) - version = version or sorted(language.versions())[-1] - language.load_concepts(meta_structure.key, version) - languages.append(language) + entry = self.entry(entry_key) + version = version or sorted(entry.versions())[-1] + entry.load_concepts(meta_structure.key, version) + entries.append(entry) except FileNotFoundError as file_not_found: raise MissingStructureError( meta_structure, - language_key, - self.language_name(language_key), + entry_key, + self.entry_name(entry_key), version, ) from file_not_found except KeyError as key_error: - raise MissingLanguageError(language_key) from key_error - return languages + raise MissingEntryError(entry_key) from key_error + return entries def structure_name(self, structure_key): @@ -378,9 +433,9 @@ class SiteVisit(models.Model): class LookupData(models.Model): id = models.BigAutoField(primary_key=True) date_time = models.DateTimeField(auto_now_add=True) - language1 = models.CharField(max_length=50) + entry1 = models.CharField(max_length=50) version1 = models.CharField(max_length=20, default='') - language2 = models.CharField(max_length=50) + entry2 = models.CharField(max_length=50) version2 = models.CharField(max_length=20, default='') structure = models.CharField(max_length=50) site_visit = models.ForeignKey(SiteVisit, on_delete=models.CASCADE) diff --git a/web/static/js/checkAvailableStructs.js b/web/static/js/checkAvailableStructs.js index dda475ecb..c3d579db8 100644 --- a/web/static/js/checkAvailableStructs.js +++ b/web/static/js/checkAvailableStructs.js @@ -1,17 +1,28 @@ function main() { - const refConc = document.getElementById('reference-concept'); - const refLang = document.getElementById('lang'); - const compConc = document.getElementById('concept'); - const compLang1 = document.getElementById('lang1'); - const compLang2 = document.getElementById('lang2'); - - // need to make sure the initial concept and language/version - // combinations are ones that are available - setCombination(refConc, refLang); - setCombination(compConc, compLang1, compLang2); + const refConcs = document.querySelectorAll('.reference-concept-select'); + const compConcs = document.querySelectorAll('.concept-select'); - refConc.addEventListener('change', () => setCombination(refConc, refLang)); - compConc.addEventListener('change', () => setCombination(compConc, compLang1, compLang2)); + refConcs.forEach(refConc => { + const category = refConc.dataset.category; + const refEntry = document.getElementById(`entry-${category}`); + if (refEntry) { + setCombination(refConc, refEntry); + refConc.addEventListener('change', () => setCombination(refConc, refEntry)); + refEntry.addEventListener('change', () => setCombination(refConc, refEntry)); + } + }); + + compConcs.forEach(compConc => { + const category = compConc.dataset.category; + const compEntry1 = document.getElementById(`entry1-${category}`); + const compEntry2 = document.getElementById(`entry2-${category}`); + if (compEntry1 && compEntry2) { + setCombination(compConc, compEntry1, compEntry2); + compConc.addEventListener('change', () => setCombination(compConc, compEntry1, compEntry2)); + compEntry1.addEventListener('change', () => setCombination(compConc, compEntry1, compEntry2)); + compEntry2.addEventListener('change', () => setCombination(compConc, compEntry1, compEntry2)); + } + }); } function getSelectText(selectElem) { @@ -22,42 +33,41 @@ function getSelectValue(selectElem) { return selectElem.options[selectElem.selectedIndex].value; } -function setCombination(conc, lang1, lang2=null) { - const lang1Options = lang1.querySelectorAll('option'); - const lang1ClassList = Array.from(lang1.querySelector(`option[value='${getSelectValue(lang1)}']`).classList); - let lang1IsSet = lang1ClassList.includes(getSelectValue(conc)); +function setCombination(conc, entry1, entry2=null) { + const concValue = getSelectValue(conc); + const entry1Options = entry1.querySelectorAll('option'); + const entry1SelectedOption = entry1.querySelector(`option[value='${getSelectValue(entry1).replace(/'/g, "\\'")}']`); + const entry1ClassList = entry1SelectedOption ? Array.from(entry1SelectedOption.classList) : []; + let entry1IsSet = entry1ClassList.includes(concValue); - for (let i of lang1Options) { + for (let i of entry1Options) { const classList = Array.from(i.classList); - if (classList.includes(getSelectValue(conc))) { - if (i.disabled) { - i.disabled = false; + if (classList.includes(concValue)) { + i.disabled = false; + if (!entry1IsSet) { + entry1.value = i.value; + entry1IsSet = true; } - if (!lang1IsSet) { - lang1.value = i.value; - lang1IsSet = true; - } - } else if (!i.disabled) { + } else { i.disabled = true; } } - if (lang2 !== null) { - const lang2Options = lang2.querySelectorAll('option'); - const lang2ClassList = Array.from(lang2.querySelector(`option[value='${getSelectValue(lang2)}']`).classList) - let lang2IsSet = lang2ClassList.includes(getSelectValue(conc)); + if (entry2 !== null) { + const entry2Options = entry2.querySelectorAll('option'); + const entry2SelectedOption = entry2.querySelector(`option[value='${getSelectValue(entry2).replace(/'/g, "\\'")}']`); + const entry2ClassList = entry2SelectedOption ? Array.from(entry2SelectedOption.classList) : []; + let entry2IsSet = entry2ClassList.includes(concValue); - for (let i of lang2Options) { + for (let i of entry2Options) { const classList = Array.from(i.classList); - if (classList.includes(getSelectValue(conc))) { - if (i.disabled) { - i.disabled = false; - } - if (!lang2IsSet && i.value !== getSelectValue(lang1)) { - lang2.value = i.value; - lang2IsSet = true; + if (classList.includes(concValue)) { + i.disabled = false; + if (!entry2IsSet && i.value !== getSelectValue(entry1)) { + entry2.value = i.value; + entry2IsSet = true; } - } else if (!i.disabled) { + } else { i.disabled = true; } } diff --git a/web/templates/error_missing_structure.html b/web/templates/error_missing_structure.html index d6c41373f..81ca3aa39 100644 --- a/web/templates/error_missing_structure.html +++ b/web/templates/error_missing_structure.html @@ -6,11 +6,11 @@

Oops!

- There is no entry about {{name}} for {{lang_name}} version {{version}} yet. + There is no entry about {{name}} for {{entry_name}} version {{version}} yet.

Would you like to add it? Check out our contribution guidelines.
- Then, when you're ready, you can start by adding a thesaurus file on github. + Then, when you're ready, you can start by adding a thesaurus file on github.

Template for `{{key}}.json`

Go to the Home Page » diff --git a/web/templates/index.html b/web/templates/index.html index fd6e415d5..f4cf766f2 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -13,42 +13,48 @@

The Polyglot Developer Reference

-
+ {% for category in languages %} +
+
+

{{ category.label }}

+
+
+

- Learn a Language + Compare {{ category.label }}

- Compare concepts side-by-side with a language you know and one you don't. + Compare concepts side-by-side between different {{ category.label|lower }}.

- - + {% for k, v in category.structures.items %} {% endfor %}
- - + {% for k, lang in category.entries.items %} {% for v in lang %} {% endfor %} {% endfor %} -
@@ -75,19 +81,17 @@

- A quick and easy way to remind yourself how to do something. - - Still under construction. + A quick and easy way to remind yourself how to do something in a specific {{ category.key|slice:":-1" }}.

- - + {% for k, v in category.structures.items %} {% endfor %} - - + {% for k, lang in category.entries.items %} {% for v in lang %} {% endfor %} @@ -104,6 +108,7 @@

+ {% endfor %}