Resolve hardcoded relics and interactive config field preservation issues

.github/workflows/dev_pr_validation.yml

@@ -42,17 +42,65 @@ jobs:
 
       # Run Integration tests
       - name: 'Run Integration tests'
+        shell: bash
         run: |
+          set -o pipefail
           # leverage uv to install other runtime test dependencies in the system site-packages!
 
           uv pip install 'tox<4.0'  # integration tests dependency
 
           # Isolate flaky tests
-          uv run pytest -ra -vvs --run-slow -k via_build_module
-          uv run pytest -ra -vvs --run-slow -k test_build_creates_artifacts
+          uv run pytest -ra -vvs --run-slow -k via_build_module 2>&1 | tee integration_test_via_build_module.log
+          uv run pytest -ra -vvs --run-slow -k test_build_creates_artifacts 2>&1 | tee integration_test_build_creates_artifacts.log
+
+          # Isolate lint tests to prevent race condition with build tests
+          uv run pytest -ra -vvs --run-slow -k test_lint_passes 2>&1 | tee integration_test_lint_passes.log
+
+          # Isolate problematic build_backend_sdist tests for debugging
+          uv run pytest -ra -vvs --run-slow -k build_backend_sdist 2>&1 | tee integration_test_build_backend_sdist.log
+
+          # Run remaining Integration Tests and Unit Tests for sanity
+          uv run pytest -ra -n auto --run-requires_uv --run-slow --run-network_bound -vvs -k 'test_cli' 2>&1 | tee integration_test_remaining.log
 
-          # Run eveything once again for sanity
-          uv run pytest -ra -n auto --run-requires_uv --run-slow --run-network_bound -vvs -k 'test_cli or build_backend_sdist or test_build_creates_artifacts or test_lint_passes'
+      - name: "Combine Integration Test Logs with Headers"
+        if: always()  # Process logs even if tests fail
+        run: |
+          echo "================================================================================" > combined_integration_tests.log
+          echo "PYTEST INVOCATION 1: via_build_module tests" >> combined_integration_tests.log
+          echo "================================================================================" >> combined_integration_tests.log
+          cat integration_test_via_build_module.log >> combined_integration_tests.log
+          echo "" >> combined_integration_tests.log
+
+          echo "================================================================================" >> combined_integration_tests.log
+          echo "PYTEST INVOCATION 2: test_build_creates_artifacts tests" >> combined_integration_tests.log
+          echo "================================================================================" >> combined_integration_tests.log
+          cat integration_test_build_creates_artifacts.log >> combined_integration_tests.log
+          echo "" >> combined_integration_tests.log
+
+          echo "================================================================================" >> combined_integration_tests.log
+          echo "PYTEST INVOCATION 3: test_lint_passes tests (isolated to prevent race condition)" >> combined_integration_tests.log
+          echo "================================================================================" >> combined_integration_tests.log
+          cat integration_test_lint_passes.log >> combined_integration_tests.log
+          echo "" >> combined_integration_tests.log
+
+          echo "================================================================================" >> combined_integration_tests.log
+          echo "PYTEST INVOCATION 4: build_backend_sdist tests (isolated for debugging)" >> combined_integration_tests.log
+          echo "================================================================================" >> combined_integration_tests.log
+          cat integration_test_build_backend_sdist.log >> combined_integration_tests.log
+          echo "" >> combined_integration_tests.log
+
+          echo "================================================================================" >> combined_integration_tests.log
+          echo "PYTEST INVOCATION 5: Remaining test suite (test_cli only)" >> combined_integration_tests.log
+          echo "================================================================================" >> combined_integration_tests.log
+          cat integration_test_remaining.log >> combined_integration_tests.log
+
+      - name: "Upload Combined Integration Test Log as Artifact"
+        if: always()  # Upload logs even if tests fail
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-tests-log
+          path: combined_integration_tests.log
+          if-no-files-found: warn
 
   # CROSS PLATFORM TESTING: 15s on Ubuntu, 25 on mac, 35 on windows
   cross_platform_tests:

pyproject.toml

@@ -41,23 +41,13 @@ include = [
     "LICENSE",
     "README.md",
     "CONTRIBUTING.md",
-    "CHANGELOG.rst",
+    "CHANGELOG.md",
     # "Dockerfile",
 ]
 exclude = [
     "docs/*",
     "requirements/*",
     "scripts/*.py",
-    "tox.ini",
-    ".circleci/config.yml",
-    ".coveragerc",
-    ".DS_Store",
-    ".gitignore",
-    ".prospector.yml",
-    ".pylintrc",
-    ".readthedocs.yml",
-    ".scrutinizer.yml",
-    ".travis.yml"
 ]
 
 [tool.poetry.scripts]

src/cookiecutter_python/backend/helpers.py

@@ -1,3 +1,5 @@
+"""Interactive Dialog wizards to gather User Input for Context variables."""
+
 import json
 import logging
 import typing as t
@@ -69,51 +71,73 @@ def parse_context(config_file: str):
         **{k: v[0] for k, v in choices.items() if k not in user_default_context},
     )
 
+    # Render cookiecutter.json again with context to resolve derived fields
+    # This ensures derived fields like pkg_name get computed properly
+    template = env.get_template('cookiecutter.json')
+    rendered_with_context = template.render({'cookiecutter': context_defaults})
+    resolved_cook_json: t.Mapping[str, t.Any] = json.loads(rendered_with_context)
+
+    # Update context_defaults with resolved derived fields, but preserve user config values
+    for key, value in resolved_cook_json.items():
+        # Only update if the key wasn't explicitly provided by user
+        if key not in user_default_context:
+            context_defaults[key] = value
+
     pipe = InteractiveDialogsPipeline()
 
-    # assert choices['rtd_python_version'] == [], f"DEBUG: {choices['rtd_python_version']}"
-    res = pipe.process(
-        [
-            {
-                "project_name": context_defaults['project_name'],
-                "project_type": {
-                    'default': context_defaults['project_type'],
-                    'choices': choices['project_type'],
-                },
-                "full_name": context_defaults['full_name'],
-                "author_email": context_defaults['author_email'],
-                "github_username": context_defaults['github_username'],
-                "project_short_description": context_defaults['project_short_description'],
-                # "release_date": context_defaults['release_date'],
-                # "year": context_defaults['year'],
-                "version": context_defaults['version'],
-                "initialize_git_repo": {
-                    'default': context_defaults['initialize_git_repo'],
-                    'choices': choices['initialize_git_repo'],
-                },
-                "supported-interpreters": {
-                    # 'default': context_defaults['initialize_git_repo'],
-                    'choices': [
-                        (choice, True)
-                        for choice in user_interpreters.get(
-                            'supported-interpreters',
-                            ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"],
-                        )
-                    ],
-                },
-                "docs_builder": {
-                    'default': context_defaults['docs_builder'],
-                    'choices': choices['docs_builder'],
-                },
-                "rtd_python_version": {
-                    'default': context_defaults['rtd_python_version'],
-                    'choices': choices['rtd_python_version'],
-                },
-                "cicd": {
-                    'default': context_defaults['cicd'],
-                    'choices': choices['cicd'],
-                },
+    # Build the context dynamically to include all fields
+    interactive_context = {}
+
+    # Simple fields (no choices)
+    simple_fields = [
+        'project_name',
+        'project_slug',
+        'pkg_name',
+        'repo_name',
+        'readthedocs_project_slug',
+        'docker_image',
+        'full_name',
+        'author',
+        'author_email',
+        'github_username',
+        'project_short_description',
+        'pypi_subtitle',
+        'version',
+    ]
+
+    for field in simple_fields:
+        if field in context_defaults:
+            interactive_context[field] = context_defaults[field]
+
+    # Choice fields (with options)
+    choice_fields = [
+        'project_type',
+        'initialize_git_repo',
+        'docs_builder',
+        'rtd_python_version',
+        'cicd',
+        # Include Grafana, Loki stack no/yes
+        'include_observability',  # ["no", "yes"]
+    ]
+
+    for field in choice_fields:
+        if field in choices:
+            interactive_context[field] = {
+                'default': context_defaults[field],
+                'choices': choices[field],
             }
-        ]
-    )
+
+    # Special handling for interpreters
+    interactive_context["supported-interpreters"] = {
+        'choices': [
+            (choice, True)
+            for choice in user_interpreters.get(
+                'supported-interpreters',
+                ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"],
+            )
+        ],
+    }
+
+    # assert choices['rtd_python_version'] == [], f"DEBUG: {choices['rtd_python_version']}"
+    res = pipe.process([interactive_context])
     return res

src/cookiecutter_python/cookiecutter.json

@@ -28,5 +28,6 @@
     },
     "docs_builder": ["sphinx", "mkdocs"],
     "rtd_python_version": ["3.8", "3.9", "3.10", "3.11", "3.12"],
-    "cicd": ["stable", "experimental"]
+    "cicd": ["stable", "experimental"],
+    "include_observability": ["no", "yes"]
 }

src/cookiecutter_python/handle/dialogs/lib/project_name.py

@@ -29,33 +29,48 @@ def dialog(self, cookie_vars) -> Mapping[str, str]:
                     'type': 'input',
                     'name': 'project_slug',
                     'message': 'Enter the Project Slug (ie lowercase text, no spaces):',
-                    'default': lambda answers: answers['project_name']
-                    .lower()
-                    .replace(' ', '-'),
+                    'default': cookie_vars.get('project_slug')
+                    or (lambda answers: answers['project_name'].lower().replace(' ', '-')),
                 },
                 {
                     'type': 'input',
                     'name': 'pkg_name',
                     'message': 'Enter the Package Name (ie lowercase text, no spaces):',
-                    'default': lambda answers: answers['project_slug'].replace('-', '_'),
+                    'default': (
+                        cookie_vars['pkg_name']
+                        if cookie_vars.get('pkg_name')
+                        else (lambda answers: answers['project_slug'].replace('-', '_'))
+                    ),
                 },
                 {
                     'type': 'input',
                     'name': 'repo_name',
                     'message': 'Enter the Repository Name (ie lowercase text, no spaces):',
-                    'default': lambda answers: answers['project_slug'],
+                    'default': (
+                        cookie_vars['repo_name']
+                        if cookie_vars.get('repo_name')
+                        else (lambda answers: answers['project_slug'])
+                    ),
                 },
                 {
                     'type': 'input',
                     'name': 'readthedocs_project_slug',
                     'message': 'Enter the ReadTheDocs Project Slug (ie lowercase text, no spaces):',
-                    'default': lambda answers: answers['project_slug'],
+                    'default': (
+                        cookie_vars['readthedocs_project_slug']
+                        if cookie_vars.get('readthedocs_project_slug')
+                        else (lambda answers: answers['project_slug'])
+                    ),
                 },
                 {
                     'type': 'input',
                     'name': 'docker_image',
                     'message': 'Enter the Docker Image Name (ie lowercase text, no spaces):',
-                    'default': lambda answers: answers['project_slug'],
+                    'default': (
+                        cookie_vars['docker_image']
+                        if cookie_vars.get('docker_image')
+                        else (lambda answers: answers['project_slug'])
+                    ),
                 },
                 # full_name
                 {
@@ -69,7 +84,8 @@ def dialog(self, cookie_vars) -> Mapping[str, str]:
                     'type': 'input',
                     'name': 'author',
                     'message': 'Enter author:',
-                    'default': lambda answers: answers['full_name'],
+                    'default': cookie_vars.get('author')
+                    or (lambda answers: answers['full_name']),
                 },
                 # author_email
                 {
@@ -97,7 +113,8 @@ def dialog(self, cookie_vars) -> Mapping[str, str]:
                     'type': 'input',
                     'name': 'pypi_subtitle',
                     'message': 'Enter pypi_subtitle:',
-                    'default': lambda answers: answers['project_short_description'],
+                    'default': cookie_vars.get('pypi_subtitle')
+                    or (lambda answers: answers['project_short_description']),
                 },
                 # release_date
                 {
@@ -158,13 +175,21 @@ def dialog(self, cookie_vars) -> Mapping[str, str]:
                     'choices': cookie_vars['rtd_python_version']['choices'],
                     'default': cookie_vars['rtd_python_version']['default'],
                 },
-                # CICD Pipeline design: stable, experimental
+                # CICD Pipeline design: stable, experimental (the good one)
                 {
                     'type': 'select',
                     'name': 'cicd',
                     'message': 'Select the CI/CD Pipeline Design',
                     'choices': cookie_vars['cicd']['choices'],
                     'default': cookie_vars['cicd']['default'],
                 },
+                # Grafana / Loki compose stack inclusion
+                {
+                    'type': 'select',
+                    'name': 'include_observability',
+                    'message': 'Include Observability Compose Stack (Grafana, Loki)?',
+                    'choices': cookie_vars['include_observability']['choices'],
+                    'default': cookie_vars['include_observability']['default'],
+                },
             ]
         )

src/cookiecutter_python/handle/interactive_cli_pipeline.py

@@ -13,5 +13,6 @@ class InteractiveDialogsPipeline:
     def process(self, request):
         """Process sequence of Interactive User Dialogs, for Context Information."""
         for dialog in self.dialogs:
+            # Call the dialog method (ie ProjectNameDialog.dialog) of each dialog node
             request = NodeFactory.create(dialog).process(request)
         return request

src/cookiecutter_python/hooks/post_gen_project.py

@@ -133,21 +133,28 @@ def post_file_removal(request):
         for path_components in CICD_DELETE[request.vars['cicd']]
     ]
     _delete_files(irrelevant_ci_cd_files)
+
     # Remove generated docs folders, but the input builder (ie mkdocs/sphinx) selected
-    _remove_irrelevant_docs_folders(request.project_dir)
+    # remove Grafana stuff if opted-out of Observability
+    _remove_irrelevant_rendered_folders(request.project_dir)
     # Remove some top-level files depending on input (ie mkdocs.yml)
     _remove_irrelevant_top_level_files(request)
 
 
-def _remove_irrelevant_docs_folders(gen_project_dir: str):
+# find top-level folders and delete the ones with name 'PyGen_TO_DELETE' or 'PyGen_TO_DELETE_OBSERVABILITY'
+TO_DELETE_TEXT: str = 'TO_DELETE'
+"""Signature text to identify folders that must be deleted after generation."""
+
+
+def _remove_irrelevant_rendered_folders(gen_project_dir: str):
     """Remove generated docs folders that are not relevant to the selected docs builder."""
-    # find top-level folders and delte the ones with name 'PyGen_TO_DELETE'
-    for docs_folder_to_delete in (
+
+    for folder_to_delete in (
         folder
         for folder in Path(gen_project_dir).iterdir()
-        if folder.is_dir() and folder.name == 'PyGen_TO_DELETE'
+        if folder.is_dir() and TO_DELETE_TEXT in str(folder.name)
     ):
-        shutil.rmtree(docs_folder_to_delete, ignore_errors=True)
+        shutil.rmtree(folder_to_delete, ignore_errors=True)
 
 
 def _remove_irrelevant_top_level_files(request):

src/cookiecutter_python/{{ cookiecutter.project_slug }}/scripts/gen_api_refs_pages.py

@@ -18,10 +18,10 @@
 # Loop, recursively, over all Python *.py Files, inside 'src' dir and subdirs
 for path in sorted(src.rglob("*.py")):
     ## 1. extract Relative path from Python File and remove suffix (.py)
-    # EG src/biskotaki/cli.py  -->   biskotaki/cli
+    # EG src/{{ cookiecutter.pkg_name }}/cli.py  -->   {{ cookiecutter.pkg_name }}/cli
     _module_path = path.relative_to(src).with_suffix("")
     ## 2. derive corresponding Docs File to write Directives into
-    # EG biskotaki/cli  -->   biskotaki/cli.md
+    # EG {{ cookiecutter.pkg_name }}/cli  -->   {{ cookiecutter.pkg_name }}/cli.md
     doc_path = path.relative_to(src).with_suffix(".md")
     ## 3. derive Full Path to Docs File
     full_doc_path = Path("reference", doc_path)
@@ -71,7 +71,7 @@
 This page provides documentation for our command line tools.
 
 ::: mkdocs-click
-    :module: biskotakigold.cli
+    :module: {{ cookiecutter.pkg_name }}.cli
     :command: main
 """
     # TODO: write better content for the CLI.md file