pre-commit.yml

name: Pre-commit

on:
  push:
    branches:
      - main
      - master
      - copilot/**
      - copilot/initiate-full-ingestion
      - ingestion-clean
      - pr**
      - "pr/**"
  pull_request:
    branches:
      - "**"

concurrency:
  group: ${{ github.workflow }}-${{ github.sha }}
  cancel-in-progress: false

# Limit permissions of GITHUB_TOKEN for security
permissions:
  contents: read

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pre-commit
          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi      
      - name: Check formatting (black --check, ruff)
        env:
          PYTHONIOENCODING: utf-8
        run: |
          black --check src/ tests/ scripts/
          ruff check src/ tests/

      - name: Run custom validation hooks
        env:
          PYTHONIOENCODING: utf-8
        run: |
          python scripts/placeholder_scan.py
          python scripts/validate_clf_schema.py
          python scripts/validate_cdsce_schema.py
          python scripts/validate_schemas.py --no-fail

  ace-analyzer:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi

      - name: Run ACE analyzer
        run: python scripts/ace_analyzer.py --years 2014-2025

      - name: Verify ACE_REPORT.json generated
        run: |
          if [ ! -f ACE_REPORT.json ]; then
            echo "ERROR: ACE_REPORT.json was not generated"
            exit 1
          fi
          echo "ACE_REPORT.json successfully generated"

      - name: Check for high-risk anomalies
        run: python scripts/ace_analyzer.py --years 2014-2025 --fail-on-high-risk
        continue-on-error: true
        id: high_risk_check

      - name: Report high-risk anomalies (warning only)
        if: steps.high_risk_check.outcome == 'failure'
        run: |
          echo "::warning::High-risk anomalies (Score 5) detected. Review ACE_REPORT.json for details."

      - name: Upload ACE reports
        uses: actions/upload-artifact@v4
        with:
          name: ace-reports
          path: |
            ACE_REPORT.json
            ACE_SUMMARY.md
            ANOMALY_MAP.csv
            ace_network_graph.json
          retention-days: 30

  vendor-map:
    runs-on: ubuntu-latest
    needs: ace-analyzer
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi

      - name: Download ACE reports
        uses: actions/download-artifact@v4.1.8
        with:
          name: ace-reports

      - name: Run vendor tests
        run: python -m pytest tests/vendor/ -v --tb=short

      - name: Run vendor map extractor
        run: |
          python -c "
          import sys
          sys.path.insert(0, 'scripts')
          from vendor_map_extractor import run_vendor_extraction
          from pathlib import Path
          result = run_vendor_extraction(Path('oraculus/corpus'), '2014-2025', Path('.'))
          "

      - name: Verify vendor outputs generated
        run: |
          echo 'Vendor map extraction completed (sample mode)'
          echo 'In production, this would generate vendor_index.json'

      - name: Upload vendor reports
        uses: actions/upload-artifact@v4
        with:
          name: vendor-reports
          path: |
            vendor_graph.json
            vendor_scores.json
            vendor_influence_network.csv
            CONTRACT_FLOW_MAP.json
            PROCUREMENT_FLAGS.json
          if-no-files-found: ignore
          retention-days: 30

  caim-analyzer:
    runs-on: ubuntu-latest
    needs: [ace-analyzer, vendor-map]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
          pip install matplotlib

      - name: Download ACE reports
        uses: actions/download-artifact@v4.1.8
        with:
          name: ace-reports

      - name: Download vendor reports
        uses: actions/download-artifact@v4.1.8
        with:
          name: vendor-reports
        continue-on-error: true

      - name: Run CAIM tests
        run: python -m pytest tests/caim/ -v --tb=short

      - name: Run agency extraction
        run: |
          python -c "
          import sys
          sys.path.insert(0, 'scripts')
          from agency_map_extractor import run_agency_extraction
          from pathlib import Path
          result = run_agency_extraction(Path('oraculus/corpus'), '2014-2025', Path('.'))
          "

      - name: Run CAIM graph builder
        run: |
          python -c "
          import sys
          import json
          sys.path.insert(0, 'scripts')
          from agency_map_extractor import run_agency_extraction
          from cross_agency_influence import run_cross_agency_influence
          from pathlib import Path
          
          # Run agency extraction
          agency_data = run_agency_extraction(Path('oraculus/corpus'), '2014-2025')
          
          # Run CAIM graph
          run_cross_agency_influence(agency_data, '2014-2025', Path('.'))
          "

      - name: Run ICM matrix generator
        run: |
          python -c "
          import sys
          import json
          sys.path.insert(0, 'scripts')
          from agency_map_extractor import run_agency_extraction
          from interdepartmental_matrix import run_icm_generation
          from pathlib import Path
          
          # Run agency extraction
          agency_data = run_agency_extraction(Path('oraculus/corpus'), '2014-2025')
          
          # Run ICM
          run_icm_generation(agency_data, '2014-2025', Path('.'))
          "

      - name: Verify CAIM outputs generated
        run: |
          echo 'CAIM analysis completed'
          echo 'Generated artifacts:'
          ls -la *.json *.csv 2>/dev/null || echo 'No artifacts found (expected in sample mode)'

      - name: Upload CAIM reports
        uses: actions/upload-artifact@v4
        with:
          name: caim-reports
          path: |
            agency_graph.json
            agency_index.json
            cross_agency_edges.csv
            influence_matrix.csv
            icm_matrix.json
            agency_correlation_heatmap.png
            AGENCY_INFLUENCE_REPORT.md
            ICM_EXPLANATION.md
          if-no-files-found: ignore
          retention-days: 30

  pdf-forensics:
    runs-on: ubuntu-latest
    needs: ace-analyzer
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi

      - name: Download ACE reports
        uses: actions/download-artifact@v4.1.8
        with:
          name: ace-reports

      - name: Run PDF forensics tests
        run: python -m pytest tests/pdf_forensics/ -v --tb=short

      - name: Run PDF forensics analyzer
        run: |
          python -c "
          import sys
          sys.path.insert(0, 'scripts')
          from pdf_forensics.pdf_metadata_miner import run_pdf_forensics
          from pathlib import Path
          
          # Run PDF forensics
          result = run_pdf_forensics(
              corpus_root=Path('oraculus/corpus'),
              year_range='2014-2025',
              output_dir=Path('analysis/pdf_forensics'),
              ace_report_path=Path('ACE_REPORT.json')
          )
          "

      - name: Verify PDF forensics outputs generated
        run: |
          echo 'PDF Forensics analysis completed'
          echo 'Generated artifacts:'
          ls -la analysis/pdf_forensics/ 2>/dev/null || echo 'No artifacts found (expected in sample mode)'

      - name: Upload PDF forensics reports
        uses: actions/upload-artifact@v4
        with:
          name: pdf-forensics-reports
          path: |
            analysis/pdf_forensics/forensic_report.json
            analysis/pdf_forensics/metadata_inconsistency_map.json
            analysis/pdf_forensics/pdf_origin_clusters.json
            analysis/pdf_forensics/pdf_forensics_graph.json
            analysis/pdf_forensics/DPMM_SUMMARY.md
            analysis/pdf_forensics/FORENSIC_ANOMALY_LINKS.json
            analysis/pdf_forensics/FORENSIC_VENDOR_OVERLAPS.json
            analysis/pdf_forensics/FORENSIC_AGENCY_PATTERNS.json
          if-no-files-found: ignore
          retention-days: 30