From 429a65b9e63d7f834fa16806814a344afbb1338f Mon Sep 17 00:00:00 2001 From: navintkr Date: Thu, 13 Aug 2026 17:27:44 -0500 Subject: [PATCH] Fix azd deployment recovery for restricted environments --- .../infra-scripts.instructions.md | 2 +- docs/DeploymentGuide.md | 9 +-- docs/fabric/DeploymentGuideFabricManual.md | 2 +- infra/main.bicep | 11 +++- infra/main.parameters.json | 3 + infra/scripts/foundry/step_knowledge_base.py | 56 +++++++++++++++++-- 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/.github/instructions/infra-scripts.instructions.md b/.github/instructions/infra-scripts.instructions.md index 6c5b70f9..7e33363f 100644 --- a/.github/instructions/infra-scripts.instructions.md +++ b/.github/instructions/infra-scripts.instructions.md @@ -92,7 +92,7 @@ Optional env vars (user-configurable): ### Deployment flow [`install_microsoft_iq_solution.py`](../../infra/scripts/install_microsoft_iq_solution.py) runs 6 steps unconditionally (matching `ALL_DEPLOYMENT_STEPS`). All required env vars are sourced from `main.bicep` outputs via `azd`; the script aborts on the first step that raises: -1. `setup_knowledge_base` — create AI Search index, upload PDFs, create Foundry IQ knowledge source and knowledge base (via [`foundry/step_knowledge_base.py`](../../infra/scripts/foundry/step_knowledge_base.py)) +1. `setup_knowledge_base` — create AI Search index, upload PDFs, create Foundry IQ knowledge source and knowledge base (via [`foundry/step_knowledge_base.py`](../../infra/scripts/foundry/step_knowledge_base.py)). If Blob upload returns `AuthorizationFailure` because storage policy blocks local data-plane access, use public raw GitHub URLs for citations and continue indexing the local PDF content. 2. `setup_agent` — create AI Foundry agent with Knowledge Base MCP tool (via [`foundry/step_agent_setup.py`](../../infra/scripts/foundry/step_agent_setup.py)) 3. `setup_workspace` — create/find workspace, assign capacity, resume if paused (via [`fabric/step_workspace_setup.py`](../../infra/scripts/fabric/step_workspace_setup.py)) 4. `setup_administrators` — add admins with Graph API resolution + fallback (via [`fabric/step_workspace_admins.py`](../../infra/scripts/fabric/step_workspace_admins.py)) diff --git a/docs/DeploymentGuide.md b/docs/DeploymentGuide.md index abb0cb42..acc2e456 100644 --- a/docs/DeploymentGuide.md +++ b/docs/DeploymentGuide.md @@ -245,6 +245,7 @@ Customize your deployment by setting `azd` environment variables before running | Category | Variable | Description | Default | Example | |----------|----------|-------------|---------|---------| | **Common** | `ENABLE_TELEMETRY` | Enable/disable usage telemetry | `true` | `azd env set ENABLE_TELEMETRY false` | +| | `AZURE_SOLUTION_UNIQUE_TEXT` | Override the 5-character resource-name suffix when a soft-deleted global name is still reserved | Deterministic value | `azd env set AZURE_SOLUTION_UNIQUE_TEXT e13a6` | | **Fabric Capacity** | `FABRIC_CAPACITY_SKU_NAME` | Fabric capacity SKU | `F2` | `azd env set FABRIC_CAPACITY_SKU_NAME F4` | | | `AZURE_EXISTING_FABRIC_CAPACITY_NAME` | Use an existing Fabric capacity (skips creation) | _(empty)_ | `azd env set AZURE_EXISTING_FABRIC_CAPACITY_NAME "my-capacity"` | | | `FABRIC_ADMIN_MEMBERS` | Additional Fabric capacity admins (JSON array of UPNs or object IDs) | `[]` | `azd env set FABRIC_ADMIN_MEMBERS '["user@contoso.com"]'` | @@ -291,7 +292,7 @@ The deployment creates two integrated components in a single Azure Resource Grou #### 2. Microsoft Foundry Resources - **[Microsoft Foundry Hub & Project](https://learn.microsoft.com/azure/ai-studio/concepts/ai-resources)**: Core AI platform for agent management - **[Azure AI Search](https://learn.microsoft.com/azure/search/search-what-is-azure-search)**: Document indexing with [vector search](https://learn.microsoft.com/azure/search/vector-search-overview) and [knowledge base](https://learn.microsoft.com/en-us/azure/search/agentic-retrieval-how-to-create-knowledge-base?tabs=rbac%2C2025-11-01-preview&pivots=csharp) -- **[Azure Storage Account](https://learn.microsoft.com/azure/storage/common/storage-account-overview)**: [Blob storage](https://learn.microsoft.com/azure/storage/blobs/storage-blobs-overview) for documents with direct citations +- **[Azure Storage Account](https://learn.microsoft.com/azure/storage/common/storage-account-overview)**: [Blob storage](https://learn.microsoft.com/azure/storage/blobs/storage-blobs-overview) for documents with direct citations; deployments whose storage policy blocks local data-plane access use public repository URLs for citations - **[Azure OpenAI Models](https://learn.microsoft.com/azure/ai-services/openai/)**: - [`gpt-5-mini`](https://learn.microsoft.com/azure/ai-services/openai/concepts/models) - Chat completion (150K TPM) - [`text-embedding-3-small`](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#embeddings) - Vector embeddings (80K TPM) @@ -307,7 +308,7 @@ The deployment follows a **two-phase automated workflow**, both phases triggered | — | | | [Microsoft Foundry hub](https://learn.microsoft.com/azure/ai-studio/concepts/ai-resources), [project](https://learn.microsoft.com/azure/ai-studio/how-to/create-projects) & [connections](https://learn.microsoft.com/azure/ai-studio/how-to/connections-add) | Create the Foundry hub/project and the AI Search + Storage connections. | | — | | | AI Search service & Storage account | Provision the indexer + blob storage backing the knowledge base. | | — | | | [OpenAI model deployments](https://learn.microsoft.com/azure/ai-services/openai/how-to/create-resource) | Deploy the chat completion and embedding models. | -| 1 | **Phase 2: Solution Bootstrap** | [`install_microsoft_iq_solution.py`](../infra/scripts/install_microsoft_iq_solution.py) (Python, `postprovision` hook) | `setup_knowledge_base` ([`step_knowledge_base.py`](../infra/scripts/foundry/step_knowledge_base.py)) | Create the Azure AI Search index, upload PDFs from [`src/foundry/data/documents/`](../src/foundry/data/documents/), and provision the Foundry IQ knowledge source and knowledge base. | +| 1 | **Phase 2: Solution Bootstrap** | [`install_microsoft_iq_solution.py`](../infra/scripts/install_microsoft_iq_solution.py) (Python, `postprovision` hook) | `setup_knowledge_base` ([`step_knowledge_base.py`](../infra/scripts/foundry/step_knowledge_base.py)) | Create the Azure AI Search index, upload PDFs from [`src/foundry/data/documents/`](../src/foundry/data/documents/), and provision the Foundry IQ knowledge source and knowledge base. If storage policy blocks local Blob access, public repository URLs are used for citations. | | 2 | | | `setup_agent` ([`step_agent_setup.py`](../infra/scripts/foundry/step_agent_setup.py)) | Create the AI Foundry chat agent wired to the Knowledge Base via [MCP](https://modelcontextprotocol.io/introduction). **Best-effort**: transient platform errors are logged as warnings and the deployment continues. | | 3 | | | `setup_workspace` ([`step_workspace_setup.py`](../infra/scripts/fabric/step_workspace_setup.py)) | Create or find the Fabric workspace, assign it to the capacity, and resume the capacity if paused. | | 4 | | | `setup_administrators` ([`step_workspace_admins.py`](../infra/scripts/fabric/step_workspace_admins.py)) | Add [workspace administrators](https://learn.microsoft.com/fabric/get-started/roles-workspaces) using [Graph API](https://learn.microsoft.com/graph/overview) resolution with fallback. | @@ -329,7 +330,7 @@ After successful deployment, you will have a single Azure Resource Group contain | **Microsoft Foundry Hub & Project** | Container for AI agents, model deployments, knowledge bases, and connections. | | **Azure OpenAI deployments** | Two model deployments inside the Foundry project: a chat completion model (default `gpt-5-mini`) and an embedding model (default `text-embedding-3-small`). | | **Azure AI Search** | Vector + keyword search service. Backs the Foundry knowledge base; index name `{solution_suffix}-documents`. | -| **Azure Storage Account** | Blob storage for source documents. Container `{solution_suffix}-documents` is uploaded to by `setup_knowledge_base` and referenced by AI Search citations. | +| **Azure Storage Account** | Blob storage for source documents. `setup_knowledge_base` uses container `{solution_suffix}-documents` when permitted; if storage policy blocks local Blob access, citations point to the public repository copies instead. | | **Log Analytics workspace + Application Insights** | Diagnostic and monitoring sink for the Foundry project, AI Search, and the chat agent. Reused if `AZURE_EXISTING_LOG_ANALYTICS_WORKSPACE_ID` is set. | | **Foundry connections** | Project connections wiring Foundry to AI Search, Blob Storage, and the Knowledge Base MCP endpoint (`{solution_suffix}-kb-mcp-connection`). | @@ -415,7 +416,7 @@ Key outputs: ```bash azd up ``` - This re-executes `setup_knowledge_base` (Step 1), which re-uploads PDFs to blob storage and re-indexes them. + This re-executes `setup_knowledge_base` (Step 1), which uploads PDFs to blob storage when permitted and re-indexes them. If storage policy blocks local Blob access, the indexed citations use public repository URLs. 2. **Explore the Fabric workspace**: open notebooks and run the data pipelines. 3. **Test the agent**: use the test script above, or chat with `ChatAgent` from the Foundry portal. 4. **View dashboards**: access the Power BI reports in the Fabric workspace. diff --git a/docs/fabric/DeploymentGuideFabricManual.md b/docs/fabric/DeploymentGuideFabricManual.md index c0144363..82e931a9 100644 --- a/docs/fabric/DeploymentGuideFabricManual.md +++ b/docs/fabric/DeploymentGuideFabricManual.md @@ -102,6 +102,6 @@ Delete the workspace from the Fabric portal: ## Next steps -- For a fully automated end-to-end deployment that also provisions Foundry, the chat agent, and the knowledge base, switch to the [top-level Deployment Guide](../DeploymentGuide.md). +- For a fully automated end-to-end deployment that also provisions Foundry, the chat agent, and the knowledge base, switch to the [top-level Deployment Guide](../DeploymentGuide.md). That workflow can use public repository citation URLs when organizational storage policy blocks local Blob data-plane access. - For the Work IQ (Copilot Studio) component that orchestrates Fabric IQ + Foundry IQ from a single conversational ingress, see the [Copilot Studio Deployment Guide](../copilot/DeploymentGuide.md). - Microsoft Fabric documentation: [learn.microsoft.com/fabric](https://learn.microsoft.com/fabric/). diff --git a/infra/main.bicep b/infra/main.bicep index 01b8c559..d1fe1937 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -17,6 +17,10 @@ param solutionName string = 'miqsa' @description('A unique text value for the solution. This is used to ensure resource names are unique for global resources. Defaults to a 5-character substring of the unique string generated from the subscription ID, resource group name, and solution name.') param solutionUniqueText string = substring(uniqueString(subscription().id, resourceGroup().name, solutionName), 0, 5) +@maxLength(5) +@description('Optional override for the generated unique text when a globally unique resource name is unavailable.') +param solutionUniqueTextOverride string = '' + @minLength(3) @metadata({ azd: { type: 'location' } }) @description('Azure region for all services. Defaults to the resource group location.') @@ -127,12 +131,13 @@ param embeddingDeploymentCapacity int = 80 param deployingUserPrincipalType string = 'User' // ========== Variables ========== // +var effectiveSolutionUniqueText = empty(solutionUniqueTextOverride) ? solutionUniqueText : solutionUniqueTextOverride var solutionSuffix = toLower(trim(replace( replace( replace( replace( replace( - replace('${solutionName}${solutionUniqueText}', '-', ''), + replace('${solutionName}${effectiveSolutionUniqueText}', '-', ''), '_', ''), '.', ''), '/', ''), @@ -193,7 +198,7 @@ module aifoundry 'deploy_ai_foundry.bicep' = { name: 'deploy_ai_foundry' params: { solutionName: solutionName - solutionUniqueText: solutionUniqueText + solutionUniqueText: effectiveSolutionUniqueText solutionLocation: aiDeploymentsLocation deploymentType: deploymentType gptModelName: gptModelName @@ -227,7 +232,7 @@ output SOLUTION_NAME string = solutionName output SOLUTION_SUFFIX string = solutionSuffix @description('The unique text appended to solution name for global uniqueness') -output SOLUTION_UNIQUE_TEXT string = solutionUniqueText +output SOLUTION_UNIQUE_TEXT string = effectiveSolutionUniqueText // Fabric Outputs @description('The name of the Fabric capacity resource') diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 1ee3a7ab..8bcf2d70 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -5,6 +5,9 @@ "solutionName": { "value": "${AZURE_ENV_NAME}" }, + "solutionUniqueTextOverride": { + "value": "${AZURE_SOLUTION_UNIQUE_TEXT=}" + }, "location": { "value": "${AZURE_LOCATION}" }, diff --git a/infra/scripts/foundry/step_knowledge_base.py b/infra/scripts/foundry/step_knowledge_base.py index 277d98ef..4f2e13f7 100755 --- a/infra/scripts/foundry/step_knowledge_base.py +++ b/infra/scripts/foundry/step_knowledge_base.py @@ -7,9 +7,14 @@ """ import logging +import re +import subprocess from pathlib import Path +from urllib.parse import quote -from common.config import DATA_DIR +from azure.core.exceptions import HttpResponseError + +from common.config import DATA_DIR, REPO_ROOT from common.pdf_utils import process_pdfs_to_documents from foundry.blob_api import create_blob_service_client, upload_pdf_to_blob from foundry.search_api import ( @@ -26,6 +31,40 @@ logger = logging.getLogger(__name__) +def _get_repository_document_urls(pdf_files: list) -> dict: + """Build public raw GitHub URLs for locally checked-out PDF files.""" + try: + remote = subprocess.run( + ["git", "remote", "get-url", "origin"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + remote_head = subprocess.run( + ["git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + raise RuntimeError("Could not determine the GitHub repository for document citations") from exc + + match = re.search(r"github\.com[/:]([^/]+/[^/]+?)(?:\.git)?$", remote) + if not match: + raise RuntimeError(f"Git remote is not a supported GitHub URL: {remote}") + + repository = match.group(1) + branch = remote_head.removeprefix("origin/") or "main" + return { + pdf_path.name: ( + f"https://raw.githubusercontent.com/{repository}/{quote(branch, safe='')}/" + f"{quote(pdf_path.relative_to(REPO_ROOT).as_posix(), safe='/')}" + ) + for pdf_path in pdf_files + } + + def setup_knowledge_base( *, solution_name: str, @@ -77,9 +116,18 @@ def setup_knowledge_base( documents: list = [] if pdf_files: logger.info(f" Uploading {len(pdf_files)} PDF(s) to blob storage…") - for _pdf_path in pdf_files: - _url = upload_pdf_to_blob(_blob_client, blob_endpoint, blob_container_name, _pdf_path) - pdf_blob_urls[_pdf_path.name] = _url + try: + for _pdf_path in pdf_files: + _url = upload_pdf_to_blob(_blob_client, blob_endpoint, blob_container_name, _pdf_path) + pdf_blob_urls[_pdf_path.name] = _url + except HttpResponseError as _exc: + if _exc.error_code != "AuthorizationFailure": + raise + logger.warning( + " Blob access is blocked by storage policy; using public repository URLs " + "for document citations" + ) + pdf_blob_urls = _get_repository_document_urls(pdf_files) logger.info(" Processing and indexing document chunks…") documents = process_pdfs_to_documents(pdf_files, pdf_blob_urls)