From b5bde891e65ef7476e6164be4ce6dbc35f6d8f8a Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:01 +0200 Subject: [PATCH 01/14] RDBC-1099 Sync the client to 7.2.5 Everything the C# 7.2.5 patch brought: the CDC Sink client surface, server-wide connection strings and what uses them, SSO on client certificates, reading an agent conversation back and cancelling the tool calls it awaits, the Azure Service Bus queue ETL broker, S3 checksum validation, and the new server constants. --- ravendb/__init__.py | 45 +- ravendb/documents/ai/ai_conversation.py | 6 + ravendb/documents/ai/ai_operations.py | 36 +- .../operations/ai/agents/__init__.py | 17 + .../get_conversation_messages_operation.py | 284 ++++++++++ .../ai/agents/run_conversation_operation.py | 9 + .../operations/attachments/__init__.py | 4 + .../documents/operations/backups/settings.py | 4 + .../documents/operations/cdc_sink/__init__.py | 127 +++++ .../operations/cdc_sink/configuration.py | 486 ++++++++++++++++++ .../get_connection_string_operation.py | 69 +-- .../operations/connection_strings.py | 70 ++- .../azure_service_bus_connection_settings.py | 85 +++ .../operations/etl/queue/connection.py | 14 + ravendb/documents/operations/ongoing_tasks.py | 111 ++++ ravendb/http/request_executor.py | 2 +- ravendb/primitives/constants.py | 4 + ravendb/serverwide/database_record.py | 4 + ravendb/serverwide/operations/certificates.py | 143 +++++- .../operations/connection_strings.py | 313 +++++++++++ .../test_ai_conversation_messages.py | 217 ++++++++ ...est_azure_service_bus_connection_string.py | 137 +++++ .../tests/operations_tests/test_cdc_sink.py | 284 ++++++++++ .../operations_tests/test_certificates_sso.py | 178 +++++++ .../operations_tests/test_s3_settings.py | 69 +++ .../test_server_wide_connection_strings.py | 250 +++++++++ .../test_certificate_disabled_flag.py | 9 +- 27 files changed, 2897 insertions(+), 80 deletions(-) create mode 100644 ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py create mode 100644 ravendb/documents/operations/cdc_sink/__init__.py create mode 100644 ravendb/documents/operations/cdc_sink/configuration.py create mode 100644 ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py create mode 100644 ravendb/serverwide/operations/connection_strings.py create mode 100644 ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py create mode 100644 ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py create mode 100644 ravendb/tests/operations_tests/test_cdc_sink.py create mode 100644 ravendb/tests/operations_tests/test_certificates_sso.py create mode 100644 ravendb/tests/operations_tests/test_s3_settings.py create mode 100644 ravendb/tests/operations_tests/test_server_wide_connection_strings.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 172d57b8..f3df37c8 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -75,7 +75,11 @@ ) from ravendb.documents.operations.configuration.definitions import StudioConfiguration, StudioEnvironment -from ravendb.documents.operations.connection_strings import ConnectionString +from ravendb.documents.operations.connection_strings import ( + ConnectionString, + ConnectionStringUsage, + ConnectionStringUsageKind, +) # AI Operations from ravendb.documents.ai import ( @@ -115,6 +119,13 @@ GetAiAgentsResponse, AddOrUpdateAiAgentOperation, DeleteAiAgentOperation, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, ) from ravendb.documents.operations.ai import ( ChunkingOptions, @@ -188,6 +199,25 @@ from ravendb.documents.operations.ongoing_tasks import ( OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, + OngoingTaskCdcSink, +) +from ravendb.documents.operations.cdc_sink import ( + AddCdcSinkOperation, + AddCdcSinkOperationResult, + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkProcessState, + CdcSinkRelationType, + CdcSinkTableConfig, + CdcSinkTableLoadState, + CdcSinkTaskState, + UpdateCdcSinkOperation, + UpdateCdcSinkOperationResult, ) from ravendb.documents.operations.revisions import ( RevisionsCollectionConfiguration, @@ -356,6 +386,19 @@ GetCertificatesResponse, PutClientCertificateOperation, SecurityClearance, + CertificateUsage, + SsoIdentifier, + SsoProvider, +) +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + GetServerWideConnectionStringsResult, + PutServerWideConnectionStringOperation, + PutServerWideConnectionStringResult, + RemoveServerWideConnectionStringOperation, + RemoveServerWideConnectionStringResult, + ServerWideConnectionString, + ServerWideConnectionStringUsage, ) from ravendb.serverwide.operations.common import ( BuildNumber, diff --git a/ravendb/documents/ai/ai_conversation.py b/ravendb/documents/ai/ai_conversation.py index 50ea0fef..ea625616 100644 --- a/ravendb/documents/ai/ai_conversation.py +++ b/ravendb/documents/ai/ai_conversation.py @@ -43,6 +43,7 @@ def __init__( conversation_id: str = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): self._store = store self._agent_id = agent_id @@ -50,6 +51,9 @@ def __init__( self._conversation_id = conversation_id self._change_vector = change_vector self._debug = debug + # One-shot: the server drops the tool calls still awaiting a response on the next + # run, and the flag clears itself once that run succeeds. + self._cancel_pending_action_tools = cancel_pending_action_tools self._prompt_parts: List[ContentPart] = [] self._action_responses: Dict[str, AiAgentActionResponse] = {} @@ -194,6 +198,7 @@ def _run_internal( streamed_chunks_callback=streamed_chunks_callback, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, ) try: @@ -203,6 +208,7 @@ def _run_internal( self._change_vector = result.change_vector self._conversation_id = result.conversation_id + self._cancel_pending_action_tools = False self._action_requests = result.action_requests or [] return AiAnswer( diff --git a/ravendb/documents/ai/ai_operations.py b/ravendb/documents/ai/ai_operations.py index 77bf1ad8..93b1d84d 100644 --- a/ravendb/documents/ai/ai_operations.py +++ b/ravendb/documents/ai/ai_operations.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Any, Optional, Type +from typing import TYPE_CHECKING, Dict, Any, Optional, Type, Union import warnings @@ -11,7 +11,9 @@ from ravendb.documents.operations.ai.agents import ( AiAgentConfiguration, AiAgentConfigurationResult, + AiConversationMessagesResult, GetAiAgentsResponse, + GetConversationMessagesOptions, ) @@ -71,6 +73,25 @@ def get_agents(self, agent_id: str = None) -> GetAiAgentsResponse: operation = GetAiAgentOperation(agent_id) return self._store.maintenance.send(operation) + def get_conversation_messages( + self, conversation_id_or_parameters: Union[str, "GetConversationMessagesOptions"] + ) -> "AiConversationMessagesResult": + """ + Reads messages from an AI conversation. Returns the most recent messages by default. + + Args: + conversation_id_or_parameters: The conversation document ID, or a + GetConversationMessagesOptions for full control over paging + (before/after timestamps), page size, and detail level + + Returns: + The conversation's messages, cumulative usage, and paging state + """ + from ravendb.documents.operations.ai.agents import GetConversationMessagesOperation + + operation = GetConversationMessagesOperation(conversation_id_or_parameters) + return self._store.maintenance.send(operation) + def conversation( self, agent_id: str, @@ -78,6 +99,7 @@ def conversation( creation_options: "AiConversationCreationOptions" = None, change_vector: str = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ) -> AiConversation: """ Creates a new conversation with the specified AI agent. @@ -88,12 +110,22 @@ def conversation( creation_options: Optional creation options for the conversation change_vector: Optional change vector for concurrency control debug: Optional flag enabling server-side conversation debugging + cancel_pending_action_tools: Drop the tool calls the conversation is still waiting on + instead of answering them, on the next run. Cleared once that run succeeds. Returns: Conversation operations interface for managing the conversation """ - return AiConversation(self._store, agent_id, creation_options, conversation_id, change_vector, debug) + return AiConversation( + self._store, + agent_id, + creation_options, + conversation_id, + change_vector, + debug, + cancel_pending_action_tools, + ) def conversation_with_id(self, conversation_id: str, change_vector: str = None) -> AiConversation: """ diff --git a/ravendb/documents/operations/ai/agents/__init__.py b/ravendb/documents/operations/ai/agents/__init__.py index d8099d41..5ba05db2 100644 --- a/ravendb/documents/operations/ai/agents/__init__.py +++ b/ravendb/documents/operations/ai/agents/__init__.py @@ -39,6 +39,16 @@ AiConversationParameterOptions, ) +from .get_conversation_messages_operation import ( + GetConversationMessagesOperation, + GetConversationMessagesOptions, + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, +) + __all__ = [ "AiAgentConfiguration", "AiAgentConfigurationResult", @@ -68,4 +78,11 @@ "GetAiAgentsResponse", "AddOrUpdateAiAgentOperation", "DeleteAiAgentOperation", + "GetConversationMessagesOperation", + "GetConversationMessagesOptions", + "AiConversationDetailLevel", + "AiConversationMessage", + "AiConversationMessagesResult", + "AiMessageRole", + "AiToolCallResult", ] diff --git a/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py new file mode 100644 index 00000000..20249dec --- /dev/null +++ b/ravendb/documents/operations/ai/agents/get_conversation_messages_operation.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import enum +import json +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Union + +import requests + +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.ai.agents.run_conversation_operation import AiUsage +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants +from ravendb.tools.utils import Utils + + +class AiConversationDetailLevel(enum.Enum): + """Controls how much of a conversation the server returns.""" + + # User messages and assistant messages that have content. System prompts, tool calls, + # summaries and internal messages are excluded. + SIMPLE = "Simple" + # Adds system messages, tool calls with their results, and per-message usage. + DETAILED = "Detailed" + # No filtering at all, summaries and internal messages included. + FULL = "Full" + + def __str__(self) -> str: + return self.value + + +class AiMessageRole(enum.Enum): + SYSTEM = "System" + USER = "User" + ASSISTANT = "Assistant" + SUMMARY = "Summary" + INTERNAL = "Internal" + + def __str__(self) -> str: + return self.value + + +class AiToolCallResult: + """A tool call the model initiated, with the tool's response inlined.""" + + def __init__( + self, + id_: Optional[str] = None, + name: Optional[str] = None, + arguments: Optional[str] = None, + result: Optional[str] = None, + sub_conversation_id: Optional[str] = None, + ): + self.id_ = id_ + self.name = name + # Arguments the model passed, as a JSON string. + self.arguments = arguments + # None while the call is still pending (ActionRequired). + self.result = result + # Set when the call was a sub-agent invocation - the spawned conversation can be + # read on its own with GetConversationMessagesOperation. + self.sub_conversation_id = sub_conversation_id + + def to_json(self) -> Dict[str, Any]: + return { + "Id": self.id_, + "Name": self.name, + "Arguments": self.arguments, + "Result": self.result, + "SubConversationId": self.sub_conversation_id, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiToolCallResult: + return cls( + id_=json_dict.get("Id"), + name=json_dict.get("Name"), + arguments=json_dict.get("Arguments"), + result=json_dict.get("Result"), + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + +class AiConversationMessage: + """A single message in an AI agent conversation.""" + + def __init__( + self, + role: Optional[AiMessageRole] = None, + content: Optional[str] = None, + attachments: Optional[List[str]] = None, + timestamp: Optional[datetime] = None, + tool_calls: Optional[List[AiToolCallResult]] = None, + usage: Optional[AiUsage] = None, + sub_conversation_id: Optional[str] = None, + ): + self.role = role + # Multiple stored text parts arrive joined with line breaks. None for assistant + # messages that only initiated tool calls. + self.content = content + self.attachments = attachments + # Unique and monotonic within a conversation, so it is safe as a paging cursor. + self.timestamp = timestamp + self.tool_calls = tool_calls + self.usage = usage + # For Internal messages: the sub-conversation this message relates to. + self.sub_conversation_id = sub_conversation_id + + def to_json(self) -> Dict[str, Any]: + return { + "Role": self.role.value if self.role else None, + "Content": self.content, + "Attachments": self.attachments, + "Timestamp": Utils.datetime_to_string(self.timestamp), + "ToolCalls": [tool_call.to_json() for tool_call in self.tool_calls] if self.tool_calls else None, + "Usage": self.usage.to_json() if self.usage else None, + "SubConversationId": self.sub_conversation_id, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessage: + role = json_dict.get("Role") + tool_calls = json_dict.get("ToolCalls") + usage = json_dict.get("Usage") + return cls( + role=AiMessageRole(role) if role else None, + content=json_dict.get("Content"), + attachments=json_dict.get("Attachments"), + timestamp=Utils.string_to_datetime(json_dict.get("Timestamp")), + tool_calls=[AiToolCallResult.from_json(tool_call) for tool_call in tool_calls] if tool_calls else None, + usage=AiUsage.from_json(usage) if usage else None, + sub_conversation_id=json_dict.get("SubConversationId"), + ) + + +class AiConversationMessagesResult: + """The result of reading an AI agent conversation.""" + + def __init__( + self, + conversation_id: Optional[str] = None, + agent: Optional[str] = None, + parameters: Optional[Dict[str, Any]] = None, + total_usage: Optional[AiUsage] = None, + last_message_at: Optional[datetime] = None, + messages: Optional[List[AiConversationMessage]] = None, + has_more_messages: bool = False, + sub_conversation_ids: Optional[List[str]] = None, + attachments: Optional[List[str]] = None, + ): + self.conversation_id = conversation_id + self.agent = agent + # Conversation parameters as a name -> value map. Values are heterogeneous + # (primitives and arrays), so they are handed back as they arrive. + self.parameters = parameters + # Cumulative token usage across every turn of the conversation. + self.total_usage = total_usage + self.last_message_at = last_message_at + # Chronological, oldest first. + self.messages = messages + # Older messages exist for backward/default paging, newer ones for `after` paging. + self.has_more_messages = has_more_messages + self.sub_conversation_ids = sub_conversation_ids + self.attachments = attachments + + def to_json(self) -> Dict[str, Any]: + return { + "ConversationId": self.conversation_id, + "Agent": self.agent, + "Parameters": self.parameters, + "TotalUsage": self.total_usage.to_json() if self.total_usage else None, + "LastMessageAt": Utils.datetime_to_string(self.last_message_at), + "HasMoreMessages": self.has_more_messages, + "SubConversationIds": self.sub_conversation_ids, + "Attachments": self.attachments, + "Messages": [message.to_json() for message in self.messages] if self.messages else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AiConversationMessagesResult: + total_usage = json_dict.get("TotalUsage") + messages = json_dict.get("Messages") + return cls( + conversation_id=json_dict.get("ConversationId"), + agent=json_dict.get("Agent"), + parameters=json_dict.get("Parameters"), + total_usage=AiUsage.from_json(total_usage) if total_usage else None, + last_message_at=Utils.string_to_datetime(json_dict.get("LastMessageAt")), + messages=[AiConversationMessage.from_json(message) for message in messages] if messages else None, + has_more_messages=json_dict.get("HasMoreMessages", False), + sub_conversation_ids=json_dict.get("SubConversationIds"), + attachments=json_dict.get("Attachments"), + ) + + +class GetConversationMessagesOptions: + """Paging and filtering for GetConversationMessagesOperation.""" + + def __init__( + self, + conversation_id: Optional[str] = None, + before: Optional[datetime] = None, + after: Optional[datetime] = None, + page_size: int = constants.int_max, + detail_level: AiConversationDetailLevel = AiConversationDetailLevel.SIMPLE, + ): + self.conversation_id = conversation_id + # Messages older than this timestamp - backward paging, scrolling up in a chat UI. + self.before = before + # Messages newer than this timestamp - catching up after a Changes() notification. + self.after = after + self.page_size = page_size + self.detail_level = detail_level + + def validate(self) -> None: + if not self.conversation_id: + raise ValueError("conversation_id cannot be None or empty") + + if self.before is not None and self.after is not None: + raise ValueError("before and after cannot both be specified.") + + if self.page_size is None or self.page_size <= 0: + raise ValueError("page_size must be greater than 0.") + + +class GetConversationMessagesOperation(MaintenanceOperation[AiConversationMessagesResult]): + """ + Reads messages from an AI agent conversation, with optional timestamp-based paging + and view filtering. Returns the most recent messages by default. + """ + + def __init__(self, conversation_id_or_parameters: Union[str, GetConversationMessagesOptions]): + if conversation_id_or_parameters is None: + raise ValueError("conversation_id_or_parameters cannot be None") + + if isinstance(conversation_id_or_parameters, GetConversationMessagesOptions): + self._parameters = conversation_id_or_parameters + else: + self._parameters = GetConversationMessagesOptions(conversation_id=conversation_id_or_parameters) + + self._parameters.validate() + + def get_command(self, conventions: DocumentConventions) -> RavenCommand[AiConversationMessagesResult]: + return self._GetConversationMessagesCommand(self._parameters) + + class _GetConversationMessagesCommand(RavenCommand[AiConversationMessagesResult]): + def __init__(self, parameters: GetConversationMessagesOptions): + super().__init__(AiConversationMessagesResult) + self._parameters = parameters + + def is_read_request(self) -> bool: + return True + + @staticmethod + def _to_utc_string(value: datetime) -> str: + # Naive datetimes are taken as UTC, aware ones are converted, matching the + # server's EnsureUtc before it formats the cursor. + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return Utils.datetime_to_string(value) + + def create_request(self, node: ServerNode) -> requests.Request: + url = ( + f"{node.url}/databases/{node.database}/ai/agent/conversation/messages" + f"?conversationId={Utils.quote_key(self._parameters.conversation_id)}" + ) + + if self._parameters.before is not None: + url += f"&before={Utils.quote_key(self._to_utc_string(self._parameters.before))}" + if self._parameters.after is not None: + url += f"&after={Utils.quote_key(self._to_utc_string(self._parameters.after))}" + + url += f"&pageSize={self._parameters.page_size}" + url += f"&detailLevel={self._parameters.detail_level.value}" + + return requests.Request("GET", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + return # 404 - conversation not found + + self.result = AiConversationMessagesResult.from_json(json.loads(response)) diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index b830284e..b73add25 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -295,6 +295,7 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): if not agent_id or (isinstance(agent_id, str) and agent_id.isspace()): raise ValueError("agent_id cannot be None or empty") @@ -314,6 +315,7 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._attachments_commands = attachments_commands or [] self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools def get_command(self, conventions: DocumentConventions) -> RavenCommand[ConversationResult[TSchema]]: return RunConversationCommand( @@ -329,6 +331,7 @@ def get_command(self, conventions: DocumentConventions) -> RavenCommand[Conversa conventions=conventions, attachments_commands=self._attachments_commands, debug=self._debug, + cancel_pending_action_tools=self._cancel_pending_action_tools, ) @@ -347,6 +350,7 @@ def __init__( conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, debug: Optional[bool] = None, + cancel_pending_action_tools: bool = False, ): from ravendb.util.util import RaftIdGenerator from ravendb.documents.commands.batches import PutAttachmentCommandData @@ -363,6 +367,7 @@ def __init__( self._streamed_chunks_callback = streamed_chunks_callback self._conventions = conventions self._debug = debug + self._cancel_pending_action_tools = cancel_pending_action_tools self._attachments_commands = attachments_commands or [] # Raft id pinned at construction so retries keep the same id. @@ -409,6 +414,10 @@ def create_request(self, node: ServerNode) -> requests.Request: if self._debug is not None: url += f"&debug={self._debug}" + # Always sent: the server distinguishes "cancel the tool calls still pending" from + # "answer them", and has no default of its own. + url += f"&cancelPendingActionTools={self._cancel_pending_action_tools}" + request_body = ConversationRequestBody( action_responses=self._action_responses, artificial_actions=self._artificial_actions, diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 1dfdcd40..b4924446 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -403,6 +403,7 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, + disable_checksum_validation: bool = None, storage_class: Optional[S3StorageClass] = None, ): self.aws_access_key = aws_access_key @@ -413,6 +414,7 @@ def __init__( self.bucket_name = bucket_name self.custom_server_url = custom_server_url self.force_path_style = force_path_style + self.disable_checksum_validation = disable_checksum_validation self.storage_class = storage_class @classmethod @@ -427,6 +429,7 @@ def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: json_dict.get("BucketName"), json_dict.get("CustomServerUrl"), json_dict.get("ForcePathStyle"), + json_dict.get("DisableChecksumValidation"), S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, ) @@ -440,6 +443,7 @@ def to_json(self) -> dict: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } if self.storage_class is not None: result["StorageClass"] = self.storage_class.value diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 565e3df8..0a242863 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -130,6 +130,7 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, + disable_checksum_validation: bool = None, ): super().__init__( disabled, @@ -143,6 +144,7 @@ def __init__( self.bucket_name = bucket_name self.custom_server_url = custom_server_url self.force_path_style = force_path_style + self.disable_checksum_validation = disable_checksum_validation @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: @@ -157,6 +159,7 @@ def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: json_dict["BucketName"], json_dict["CustomServerUrl"], json_dict["ForcePathStyle"], + json_dict.get("DisableChecksumValidation"), ) def to_json(self) -> Dict[str, Any]: @@ -171,6 +174,7 @@ def to_json(self) -> Dict[str, Any]: "BucketName": self.bucket_name, "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, + "DisableChecksumValidation": self.disable_checksum_validation, } diff --git a/ravendb/documents/operations/cdc_sink/__init__.py b/ravendb/documents/operations/cdc_sink/__init__.py new file mode 100644 index 00000000..e76c8a87 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/__init__.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import requests + +from ravendb.documents.operations.cdc_sink.configuration import ( + AddCdcSinkOperationResult, + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkProcessState, + CdcSinkRelationType, + CdcSinkTableConfig, + CdcSinkTableLoadState, + CdcSinkTaskState, + UpdateCdcSinkOperationResult, +) +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class AddCdcSinkOperation(MaintenanceOperation[AddCdcSinkOperationResult]): + """Adds a new CDC Sink task, streaming an SQL source into RavenDB collections.""" + + def __init__(self, configuration: CdcSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[AddCdcSinkOperationResult]: + return self._AddCdcSinkCommand(self._configuration) + + class _AddCdcSinkCommand(RavenCommand[AddCdcSinkOperationResult], RaftCommand): + def __init__(self, configuration: CdcSinkConfiguration): + super().__init__(AddCdcSinkOperationResult) + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = AddCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class UpdateCdcSinkOperation(MaintenanceOperation[UpdateCdcSinkOperationResult]): + """Updates an existing CDC Sink task.""" + + def __init__(self, task_id: int, configuration: CdcSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._task_id = task_id + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[UpdateCdcSinkOperationResult]: + return self._UpdateCdcSinkCommand(self._task_id, self._configuration) + + class _UpdateCdcSinkCommand(RavenCommand[UpdateCdcSinkOperationResult], RaftCommand): + def __init__(self, task_id: int, configuration: CdcSinkConfiguration): + super().__init__(UpdateCdcSinkOperationResult) + self._task_id = task_id + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink?id={self._task_id}" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = UpdateCdcSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +__all__ = [ + "AddCdcSinkOperation", + "AddCdcSinkOperationResult", + "CdcColumnMapping", + "CdcColumnType", + "CdcSinkConfiguration", + "CdcSinkEmbeddedTableConfig", + "CdcSinkLinkedTableConfig", + "CdcSinkOnDeleteConfig", + "CdcSinkPostgresSettings", + "CdcSinkProcessState", + "CdcSinkRelationType", + "CdcSinkTableConfig", + "CdcSinkTableLoadState", + "CdcSinkTaskState", + "UpdateCdcSinkOperation", + "UpdateCdcSinkOperationResult", +] diff --git a/ravendb/documents/operations/cdc_sink/configuration.py b/ravendb/documents/operations/cdc_sink/configuration.py new file mode 100644 index 00000000..291757f7 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/configuration.py @@ -0,0 +1,486 @@ +from __future__ import annotations + +import enum +from typing import Any, Dict, List, Optional + + +class CdcColumnType(enum.Enum): + """Controls how a CDC Sink column is stored in the target RavenDB document.""" + + # Document property with standard type conversion. JSON/JSONB columns land as plain + # strings unless explicitly marked as JSON. + DEFAULT = "Default" + # Parse the string value as its native JSON type: objects, arrays, strings, numbers, + # booleans and null. + JSON = "Json" + # Store as a RavenDB attachment instead of a document property. + ATTACHMENT = "Attachment" + + def __str__(self) -> str: + return self.value + + +class CdcSinkRelationType(enum.Enum): + """How an embedded table's rows are stored inside the parent document.""" + + # One-to-many, as a JSON array. + ARRAY = "Array" + # One-to-many, as a JSON object keyed by primary key value(s). Composite keys are "pk1,pk2". + MAP = "Map" + # Many-to-one, as a single value/object. + VALUE = "Value" + + def __str__(self) -> str: + return self.value + + +class CdcColumnMapping: + """Maps a single SQL column to a RavenDB document property or attachment.""" + + def __init__( + self, + column: str = None, + name: str = None, + type_: CdcColumnType = CdcColumnType.DEFAULT, + ): + self.column = column + # Document property name for Default and Json, attachment name for Attachment. + self.name = name + self.type_ = type_ + + def to_json(self) -> Dict[str, Any]: + json_dict = { + "Column": self.column, + "Name": self.name, + } + # The server treats an absent Type as Default, so it is only written when it differs. + if self.type_ is not None and self.type_ != CdcColumnType.DEFAULT: + json_dict["Type"] = self.type_.value + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcColumnMapping: + type_ = json_dict.get("Type") + return cls( + column=json_dict.get("Column"), + name=json_dict.get("Name"), + type_=CdcColumnType(type_) if type_ else CdcColumnType.DEFAULT, + ) + + +class CdcSinkOnDeleteConfig: + """ + Controls how DELETE events are handled for a CDC Sink table, root or embedded. + Leave it unset to have deletes processed normally: root documents are deleted and + embedded items are removed from their parent. + """ + + def __init__(self, patch: str = None, ignore_deletes: bool = False): + # JavaScript patch that runs when a DELETE event arrives, before the delete is + # applied. For root tables `this` is the document, for embedded tables the parent; + # `$row` is the raw CDC row of the DELETE event. + self.patch = patch + # When True the DELETE is not applied - the patch (if any) still runs first. + self.ignore_deletes = ignore_deletes + + def to_json(self) -> Dict[str, Any]: + return { + "Patch": self.patch, + "IgnoreDeletes": self.ignore_deletes, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkOnDeleteConfig: + return cls( + patch=json_dict.get("Patch"), + ignore_deletes=json_dict.get("IgnoreDeletes", False), + ) + + +class CdcSinkPostgresSettings: + """ + PostgreSQL-specific settings. Optional on creation - the server auto-fills generated + names when omitted - and immutable once set. + """ + + def __init__(self, publication_name: str = None, slot_name: str = None): + # Publication used for logical replication, auto-filled as rvn_cdc_p_{guid}. + self.publication_name = publication_name + # Logical replication slot, auto-filled as rvn_cdc_s_{guid}. + self.slot_name = slot_name + + def to_json(self) -> Dict[str, Any]: + return { + "PublicationName": self.publication_name, + "SlotName": self.slot_name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkPostgresSettings: + return cls( + publication_name=json_dict.get("PublicationName"), + slot_name=json_dict.get("SlotName"), + ) + + +class CdcSinkLinkedTableConfig: + """A table referenced by document ID link rather than embedded.""" + + def __init__( + self, + source_table_schema: str = None, + source_table_name: str = None, + property_name: str = None, + join_columns: List[str] = None, + linked_collection_name: str = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + # Property name in the document, e.g. "Customer". + self.property_name = property_name + # Foreign key columns used to resolve the link. + self.join_columns = join_columns or [] + # Target collection used for document ID generation, e.g. "Customers" -> "Customers/ALFKI". + self.linked_collection_name = linked_collection_name + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "JoinColumns": self.join_columns, + "LinkedCollectionName": self.linked_collection_name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkLinkedTableConfig: + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + join_columns=json_dict.get("JoinColumns"), + linked_collection_name=json_dict.get("LinkedCollectionName"), + ) + + +class CdcSinkEmbeddedTableConfig: + """A table stored as a nested object, array or map inside its parent's documents.""" + + def __init__( + self, + source_table_schema: str = None, + source_table_name: str = None, + property_name: str = None, + columns: List[CdcColumnMapping] = None, + primary_key_columns: List[str] = None, + join_columns: List[str] = None, + type_: CdcSinkRelationType = CdcSinkRelationType.ARRAY, + patch: str = None, + on_delete: CdcSinkOnDeleteConfig = None, + case_sensitive_keys: bool = False, + embedded_tables: List[CdcSinkEmbeddedTableConfig] = None, + linked_tables: List[CdcSinkLinkedTableConfig] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + # Property name in the parent document, e.g. "Lines". + self.property_name = property_name + self.columns = columns or [] + # Used to match items within arrays and maps on update and delete. + self.primary_key_columns = primary_key_columns or [] + # Foreign key columns joining this table to its parent. + self.join_columns = join_columns or [] + self.type_ = type_ + # JavaScript patch that runs on the parent document after the embedded operation + # has been applied. `this` is the parent, `$row` the raw CDC row, `$old` the item + # as it was before this event (None for inserts). + self.patch = patch + self.on_delete = on_delete + # When False (default) string primary key values and map keys compare + # case-insensitively. + self.case_sensitive_keys = case_sensitive_keys + # Deep nesting. Requires the nested table to carry a denormalized FK to the root. + self.embedded_tables = embedded_tables or [] + self.linked_tables = linked_tables or [] + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "PropertyName": self.property_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "JoinColumns": self.join_columns, + "Type": self.type_.value if self.type_ else None, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete else None, + "CaseSensitiveKeys": self.case_sensitive_keys, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkEmbeddedTableConfig: + type_ = json_dict.get("Type") + on_delete = json_dict.get("OnDelete") + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + property_name=json_dict.get("PropertyName"), + columns=[CdcColumnMapping.from_json(column) for column in json_dict.get("Columns") or []], + primary_key_columns=json_dict.get("PrimaryKeyColumns"), + join_columns=json_dict.get("JoinColumns"), + type_=CdcSinkRelationType(type_) if type_ else CdcSinkRelationType.ARRAY, + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete) if on_delete else None, + case_sensitive_keys=json_dict.get("CaseSensitiveKeys", False), + embedded_tables=[cls.from_json(embedded) for embedded in json_dict.get("EmbeddedTables") or []], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in json_dict.get("LinkedTables") or [] + ], + ) + + +class CdcSinkTableConfig: + """Maps a source SQL table to a RavenDB collection.""" + + def __init__( + self, + collection_name: str = None, + source_table_schema: str = None, + source_table_name: str = None, + columns: List[CdcColumnMapping] = None, + primary_key_columns: List[str] = None, + patch: str = None, + on_delete: CdcSinkOnDeleteConfig = None, + disabled: bool = False, + embedded_tables: List[CdcSinkEmbeddedTableConfig] = None, + linked_tables: List[CdcSinkLinkedTableConfig] = None, + ): + self.collection_name = collection_name + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.columns = columns or [] + # Used for document ID generation. + self.primary_key_columns = primary_key_columns or [] + # JavaScript patch that runs after column mapping and embedded operations. + # `this` is the mapped document, `$row` the raw CDC row, `$old` the document as + # stored before this event (None for inserts). + self.patch = patch + self.on_delete = on_delete + # When True the table is skipped entirely: no initial load and no change capture. + self.disabled = disabled + self.embedded_tables = embedded_tables or [] + self.linked_tables = linked_tables or [] + + def to_json(self) -> Dict[str, Any]: + return { + "CollectionName": self.collection_name, + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "Patch": self.patch, + "OnDelete": self.on_delete.to_json() if self.on_delete else None, + "Disabled": self.disabled, + "EmbeddedTables": [embedded.to_json() for embedded in self.embedded_tables], + "LinkedTables": [linked.to_json() for linked in self.linked_tables], + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkTableConfig: + on_delete = json_dict.get("OnDelete") + return cls( + collection_name=json_dict.get("CollectionName"), + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + columns=[CdcColumnMapping.from_json(column) for column in json_dict.get("Columns") or []], + primary_key_columns=json_dict.get("PrimaryKeyColumns"), + patch=json_dict.get("Patch"), + on_delete=CdcSinkOnDeleteConfig.from_json(on_delete) if on_delete else None, + disabled=json_dict.get("Disabled", False), + embedded_tables=[ + CdcSinkEmbeddedTableConfig.from_json(embedded) for embedded in json_dict.get("EmbeddedTables") or [] + ], + linked_tables=[ + CdcSinkLinkedTableConfig.from_json(linked) for linked in json_dict.get("LinkedTables") or [] + ], + ) + + +class CdcSinkConfiguration: + """A CDC Sink task: an SQL source streamed into RavenDB collections.""" + + def __init__( + self, + name: str = None, + connection_string_name: str = None, + tables: List[CdcSinkTableConfig] = None, + task_id: int = 0, + disabled: bool = False, + mentor_node: str = None, + pin_to_mentor_node: bool = False, + postgres: CdcSinkPostgresSettings = None, + skip_initial_load: bool = False, + ): + self.name = name + self.connection_string_name = connection_string_name + self.tables = tables or [] + self.task_id = task_id + self.disabled = disabled + self.mentor_node = mentor_node + self.pin_to_mentor_node = pin_to_mentor_node + # PostgreSQL only. None for SQL Server, auto-filled on creation if omitted. + self.postgres = postgres + # Skip the initial full-table load and start streaming changes straight away. + # For a target database that is already populated, e.g. from a prior migration. + self.skip_initial_load = skip_initial_load + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "TaskId": self.task_id, + "Disabled": self.disabled, + "ConnectionStringName": self.connection_string_name, + "MentorNode": self.mentor_node, + "PinToMentorNode": self.pin_to_mentor_node, + "Tables": [table.to_json() for table in self.tables], + "Postgres": self.postgres.to_json() if self.postgres else None, + "SkipInitialLoad": self.skip_initial_load, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkConfiguration: + postgres = json_dict.get("Postgres") + return cls( + name=json_dict.get("Name"), + connection_string_name=json_dict.get("ConnectionStringName"), + tables=[CdcSinkTableConfig.from_json(table) for table in json_dict.get("Tables") or []], + task_id=json_dict.get("TaskId", 0), + disabled=json_dict.get("Disabled", False), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode", False), + postgres=CdcSinkPostgresSettings.from_json(postgres) if postgres else None, + skip_initial_load=json_dict.get("SkipInitialLoad", False), + ) + + +class CdcSinkProcessState: + """Which node a CDC Sink task last ran on.""" + + def __init__(self, node_tag: str = None, configuration_name: str = None): + self.node_tag = node_tag + self.configuration_name = configuration_name + + def to_json(self) -> Dict[str, Any]: + return { + "ConfigurationName": self.configuration_name, + "NodeTag": self.node_tag, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkProcessState: + return cls( + node_tag=json_dict.get("NodeTag"), + configuration_name=json_dict.get("ConfigurationName"), + ) + + @staticmethod + def generate_item_name(database_name: str, configuration_name: str) -> str: + return f"values/{database_name}/cdcsink/{configuration_name}" + + +class CdcSinkTableLoadState: + """Per-table initial-load progress inside a CDC Sink task state document.""" + + def __init__( + self, + initial_load_completed: bool = False, + last_key_values: List[str] = None, + key_columns: List[str] = None, + ): + self.initial_load_completed = initial_load_completed + # The last primary key values loaded, in primary key column order, so an + # interrupted initial load can resume. + self.last_key_values = last_key_values + self.key_columns = key_columns + + def to_json(self) -> Dict[str, Any]: + return { + "InitialLoadCompleted": self.initial_load_completed, + "LastKeyValues": self.last_key_values, + "KeyColumns": self.key_columns, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkTableLoadState: + return cls( + initial_load_completed=json_dict.get("InitialLoadCompleted", False), + last_key_values=json_dict.get("LastKeyValues"), + key_columns=json_dict.get("KeyColumns"), + ) + + +class CdcSinkTaskState: + """ + The state document of a CDC Sink task, stored in the @cdc-states collection. + Tracks the last processed LSN and per-table initial load progress. + """ + + COLLECTION_NAME = "@cdc-states" + + def __init__( + self, + configuration_name: str = None, + last_lsn: str = None, + tables: Dict[str, CdcSinkTableLoadState] = None, + ): + self.configuration_name = configuration_name + # The last successfully processed Log Sequence Number, used to resume streaming. + self.last_lsn = last_lsn + # Keyed by "schema.tableName". + self.tables = tables or {} + + def to_json(self) -> Dict[str, Any]: + return { + "ConfigurationName": self.configuration_name, + "LastLsn": self.last_lsn, + "Tables": {name: state.to_json() for name, state in self.tables.items()}, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkTaskState: + return cls( + configuration_name=json_dict.get("ConfigurationName"), + last_lsn=json_dict.get("LastLsn"), + tables={ + name: CdcSinkTableLoadState.from_json(state) for name, state in (json_dict.get("Tables") or {}).items() + }, + ) + + @classmethod + def get_document_id(cls, configuration_name: str) -> str: + # Configuration names compare case-insensitively, but the document ID keeps the + # casing it was created with. + return f"{cls.COLLECTION_NAME}/{configuration_name}" + + +class AddCdcSinkOperationResult: + def __init__(self, raft_command_index: Optional[int] = None, task_id: Optional[int] = None): + self.raft_command_index = raft_command_index + self.task_id = task_id + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index, "TaskId": self.task_id} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AddCdcSinkOperationResult: + return cls( + raft_command_index=json_dict.get("RaftCommandIndex"), + task_id=json_dict.get("TaskId"), + ) + + +class UpdateCdcSinkOperationResult(AddCdcSinkOperationResult): + # Same shape as the add result; kept as its own name so callers read what they got back. + pass diff --git a/ravendb/documents/operations/connection_string/get_connection_string_operation.py b/ravendb/documents/operations/connection_string/get_connection_string_operation.py index 13dbec34..6000e876 100644 --- a/ravendb/documents/operations/connection_string/get_connection_string_operation.py +++ b/ravendb/documents/operations/connection_string/get_connection_string_operation.py @@ -1,10 +1,11 @@ import json -from typing import Dict, Optional +from typing import Dict, Optional, Type, TypeVar import requests from ravendb import RavenCommand, ServerNode from ravendb.documents.operations.ai.ai_connection_string import AiConnectionString +from ravendb.documents.operations.connection_strings import ConnectionString, ConnectionStringUsage from ravendb.documents.operations.definitions import MaintenanceOperation from ravendb.documents.operations.etl.configuration import RavenConnectionString from ravendb.documents.operations.etl.elastic_search.connection import ElasticSearchConnectionString @@ -14,6 +15,8 @@ from ravendb.documents.operations.etl.sql import SqlConnectionString from ravendb.serverwide.server_operation_executor import ConnectionStringType +_T = TypeVar("_T", bound=ConnectionString) + class GetConnectionStringsResult: def __init__( @@ -45,52 +48,32 @@ def to_json(self) -> Dict: "SnowflakeConnectionStrings": [x.to_json() for x in self.snowflake_connection_strings.values()], } + @classmethod + def _parse(cls, json_dict: Optional[Dict[str, Dict]], connection_string_type: Type[_T]) -> Optional[Dict[str, _T]]: + if not json_dict: + return None + + result = {} + for key, value in json_dict.items(): + connection_string = connection_string_type.from_json(value) + # UsedBy is computed server-side and only present on reads. + connection_string.used_by = ConnectionStringUsage.list_from_json(value.get("UsedBy")) + result[key] = connection_string + return result + @classmethod def from_json(cls, json_dict: Dict[str, Dict]) -> "GetConnectionStringsResult": return cls( - raven_connection_strings=( - {key: RavenConnectionString.from_json(rcs) for key, rcs in json_dict["RavenConnectionStrings"].items()} - if json_dict["RavenConnectionStrings"] - else None - ), - sql_connection_strings=( - {key: SqlConnectionString.from_json(sqlcs) for key, sqlcs in json_dict["SqlConnectionStrings"].items()} - if json_dict["SqlConnectionStrings"] - else None - ), - olap_connection_strings=( - { - key: OlapConnectionString.from_json(olapcs) - for key, olapcs in json_dict["OlapConnectionStrings"].items() - } - if json_dict["OlapConnectionStrings"] - else None - ), - ai_connection_strings=( - {key: AiConnectionString.from_json(aics) for key, aics in json_dict["AiConnectionStrings"].items()} - if json_dict["AiConnectionStrings"] - else None - ), - elastic_search_connection_strings=( - { - key: ElasticSearchConnectionString.from_json(escs) - for key, escs in json_dict["ElasticSearchConnectionStrings"].items() - } - if json_dict["ElasticSearchConnectionStrings"] - else None - ), - queue_connection_strings=( - {key: QueueConnectionString.from_json(qcs) for key, qcs in json_dict["QueueConnectionStrings"].items()} - if json_dict["QueueConnectionStrings"] - else None + raven_connection_strings=cls._parse(json_dict.get("RavenConnectionStrings"), RavenConnectionString), + sql_connection_strings=cls._parse(json_dict.get("SqlConnectionStrings"), SqlConnectionString), + olap_connection_strings=cls._parse(json_dict.get("OlapConnectionStrings"), OlapConnectionString), + ai_connection_strings=cls._parse(json_dict.get("AiConnectionStrings"), AiConnectionString), + elastic_search_connection_strings=cls._parse( + json_dict.get("ElasticSearchConnectionStrings"), ElasticSearchConnectionString ), - snowflake_connection_strings=( - { - key: SnowflakeConnectionString.from_json(scs) - for key, scs in json_dict["SnowflakeConnectionStrings"].items() - } - if json_dict["SnowflakeConnectionStrings"] - else None + queue_connection_strings=cls._parse(json_dict.get("QueueConnectionStrings"), QueueConnectionString), + snowflake_connection_strings=cls._parse( + json_dict.get("SnowflakeConnectionStrings"), SnowflakeConnectionString ), ) diff --git a/ravendb/documents/operations/connection_strings.py b/ravendb/documents/operations/connection_strings.py index be5c9a44..c3e2aacd 100644 --- a/ravendb/documents/operations/connection_strings.py +++ b/ravendb/documents/operations/connection_strings.py @@ -1,10 +1,78 @@ +import enum from abc import abstractmethod -from typing import Dict, Any +from typing import Any, Dict, List, Optional + + +class ConnectionStringUsageKind(enum.Enum): + """The kind of task or agent that references a connection string.""" + + RAVEN_ETL = "RavenEtl" + SQL_ETL = "SqlEtl" + OLAP_ETL = "OlapEtl" + ELASTIC_SEARCH_ETL = "ElasticSearchEtl" + QUEUE_ETL = "QueueEtl" + SNOWFLAKE_ETL = "SnowflakeEtl" + QUEUE_SINK = "QueueSink" + EXTERNAL_REPLICATION = "ExternalReplication" + PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" + EMBEDDINGS_GENERATION = "EmbeddingsGeneration" + GEN_AI = "GenAi" + AI_AGENT = "AiAgent" + CDC_SINK = "CdcSink" + + def __str__(self): + return self.value + + +class ConnectionStringUsage: + """ + One task or agent that references a connection string. Computed server-side and + returned when reading connection strings; it is never sent back on a write. + """ + + def __init__( + self, + kind: ConnectionStringUsageKind = None, + id_: Optional[int] = None, + identifier: str = None, + name: str = None, + ): + self.kind = kind + # The numeric task id, for ongoing tasks (ETL, replication, sinks). None for AI agents. + self.id_ = id_ + # The string identifier, for AI agents. None for ongoing tasks. + self.identifier = identifier + self.name = name + + def to_json(self) -> Dict[str, Any]: + return { + "Kind": self.kind.value if self.kind else None, + "Id": self.id_, + "Identifier": self.identifier, + "Name": self.name, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ConnectionStringUsage": + kind = json_dict.get("Kind") + return cls( + kind=ConnectionStringUsageKind(kind) if kind else None, + id_=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + ) + + @classmethod + def list_from_json(cls, json_list: Optional[List[Dict[str, Any]]]) -> List["ConnectionStringUsage"]: + return [cls.from_json(usage) for usage in json_list or []] class ConnectionString: def __init__(self, name: str): self.name = name + # Populated when reading connection strings back from the server. Left empty on + # anything the client builds, and never written by to_json. + self.used_by: List[ConnectionStringUsage] = [] @abstractmethod def get_type(self): diff --git a/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py new file mode 100644 index 00000000..6e8529f2 --- /dev/null +++ b/ravendb/documents/operations/etl/queue/azure_service_bus_connection_settings.py @@ -0,0 +1,85 @@ +from typing import Optional, Dict, Any + + +class AzureServiceBusEntraId: + """Microsoft Entra ID (service principal) credentials for an Azure Service Bus namespace.""" + + def __init__( + self, + namespace: Optional[str] = None, + tenant_id: Optional[str] = None, + client_id: Optional[str] = None, + client_secret: Optional[str] = None, + ): + # Fully qualified namespace, e.g. 'mynamespace.servicebus.windows.net'. + self.namespace = namespace + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + + def to_json(self) -> Dict[str, Any]: + return { + "Namespace": self.namespace, + "TenantId": self.tenant_id, + "ClientId": self.client_id, + "ClientSecret": self.client_secret, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AzureServiceBusEntraId": + return cls( + namespace=json_dict.get("Namespace"), + tenant_id=json_dict.get("TenantId"), + client_id=json_dict.get("ClientId"), + client_secret=json_dict.get("ClientSecret"), + ) + + +class AzureServiceBusPasswordless: + """Machine authentication (Managed Identity) against an Azure Service Bus namespace.""" + + def __init__(self, namespace: Optional[str] = None): + # Fully qualified namespace, e.g. 'mynamespace.servicebus.windows.net'. + self.namespace = namespace + + def to_json(self) -> Dict[str, Any]: + return {"Namespace": self.namespace} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AzureServiceBusPasswordless": + return cls(namespace=json_dict.get("Namespace")) + + +class AzureServiceBusConnectionSettings: + """ + Azure Service Bus connection settings. Exactly one authentication method may be set: + a connection string, Entra ID credentials, or passwordless (Managed Identity). + The server rejects a configuration that sets none or more than one. + """ + + def __init__( + self, + connection_string: Optional[str] = None, + entra_id: Optional[AzureServiceBusEntraId] = None, + passwordless: Optional[AzureServiceBusPasswordless] = None, + ): + self.connection_string = connection_string + self.entra_id = entra_id + self.passwordless = passwordless + + def to_json(self) -> Dict[str, Any]: + return { + "ConnectionString": self.connection_string, + "EntraId": self.entra_id.to_json() if self.entra_id else None, + "Passwordless": self.passwordless.to_json() if self.passwordless else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "AzureServiceBusConnectionSettings": + entra_id_dict = json_dict.get("EntraId") + passwordless_dict = json_dict.get("Passwordless") + return cls( + connection_string=json_dict.get("ConnectionString"), + entra_id=AzureServiceBusEntraId.from_json(entra_id_dict) if entra_id_dict else None, + passwordless=AzureServiceBusPasswordless.from_json(passwordless_dict) if passwordless_dict else None, + ) diff --git a/ravendb/documents/operations/etl/queue/connection.py b/ravendb/documents/operations/etl/queue/connection.py index 985d65d0..02709c9a 100644 --- a/ravendb/documents/operations/etl/queue/connection.py +++ b/ravendb/documents/operations/etl/queue/connection.py @@ -6,6 +6,9 @@ from ravendb.documents.operations.etl.queue.azure_queue_storage_connection_settings import ( AzureQueueStorageConnectionSettings, ) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, +) from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings from ravendb.documents.operations.etl.queue.rabbit_mq_connection_settings import RabbitMqConnectionSettings @@ -16,6 +19,7 @@ class QueueBrokerType(Enum): RABBIT_MQ = "RabbitMq" AZURE_QUEUE_STORAGE = "AzureQueueStorage" AMAZON_SQS = "AmazonSqs" + AZURE_SERVICE_BUS = "AzureServiceBus" class QueueConnectionString(ConnectionString): @@ -27,6 +31,7 @@ def __init__( rabbit_mq_settings: RabbitMqConnectionSettings = None, azure_queue_storage_settings: AzureQueueStorageConnectionSettings = None, amazon_sqs_settings: AmazonSqsConnectionSettings = None, + azure_service_bus_settings: AzureServiceBusConnectionSettings = None, ): super().__init__(name) self.broker_type = broker_type @@ -34,6 +39,7 @@ def __init__( self.rabbit_mq_settings = rabbit_mq_settings self.azure_queue_storage_settings = azure_queue_storage_settings self.amazon_sqs_settings = amazon_sqs_settings + self.azure_service_bus_settings = azure_service_bus_settings @property def get_type(self): @@ -49,6 +55,9 @@ def to_json(self): self.azure_queue_storage_settings.to_json() if self.azure_queue_storage_settings else None ), "AmazonSqsConnectionSettings": self.amazon_sqs_settings.to_json() if self.amazon_sqs_settings else None, + "AzureServiceBusConnectionSettings": ( + self.azure_service_bus_settings.to_json() if self.azure_service_bus_settings else None + ), "Type": ravendb.serverwide.server_operation_executor.ConnectionStringType.QUEUE, } @@ -77,4 +86,9 @@ def from_json(cls, json_dict: dict) -> "QueueConnectionString": if json_dict["AmazonSqsConnectionSettings"] else None ), + azure_service_bus_settings=( + AzureServiceBusConnectionSettings.from_json(json_dict["AzureServiceBusConnectionSettings"]) + if json_dict.get("AzureServiceBusConnectionSettings") + else None + ), ) diff --git a/ravendb/documents/operations/ongoing_tasks.py b/ravendb/documents/operations/ongoing_tasks.py index 62b869c6..cbbb28cf 100644 --- a/ravendb/documents/operations/ongoing_tasks.py +++ b/ravendb/documents/operations/ongoing_tasks.py @@ -1,6 +1,7 @@ from __future__ import annotations import json from enum import Enum +from datetime import datetime from typing import Optional, TYPE_CHECKING, Union import requests @@ -16,6 +17,7 @@ if TYPE_CHECKING: from ravendb.documents.conventions import DocumentConventions + from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration from ravendb.documents.operations.ai.gen_ai_configuration import GenAiConfiguration from ravendb.documents.operations.ai.embeddings_generation_configuration import EmbeddingsGenerationConfiguration @@ -33,6 +35,7 @@ class OngoingTaskType(Enum): PULL_REPLICATION_AS_HUB = "PullReplicationAsHub" PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" QUEUE_SINK = "QueueSink" + CDC_SINK = "CdcSink" EMBEDDINGS_GENERATION = "EmbeddingsGeneration" GEN_AI = "GenAi" @@ -392,6 +395,8 @@ def __init__( access_name: Optional[str] = None, allowed_hub_to_sink_paths: Optional[list] = None, allowed_sink_to_hub_paths: Optional[list] = None, + hub_cursor: Optional[str] = None, + sink_cursor: Optional[str] = None, ): super().__init__( task_id=task_id, @@ -414,6 +419,9 @@ def __init__( self.access_name = access_name self.allowed_hub_to_sink_paths = allowed_hub_to_sink_paths self.allowed_sink_to_hub_paths = allowed_sink_to_hub_paths + # Composite change vectors of the last item each side sent. + self.hub_cursor = hub_cursor + self.sink_cursor = sink_cursor def to_json(self) -> dict: result = super().to_json() @@ -427,6 +435,8 @@ def to_json(self) -> dict: result["AccessName"] = self.access_name result["AllowedHubToSinkPaths"] = self.allowed_hub_to_sink_paths result["AllowedSinkToHubPaths"] = self.allowed_sink_to_hub_paths + result["HubCursor"] = self.hub_cursor + result["SinkCursor"] = self.sink_cursor return result @classmethod @@ -457,6 +467,105 @@ def from_json(cls, json_dict: dict) -> Optional["OngoingTaskPullReplicationAsSin access_name=json_dict.get("AccessName"), allowed_hub_to_sink_paths=json_dict.get("AllowedHubToSinkPaths"), allowed_sink_to_hub_paths=json_dict.get("AllowedSinkToHubPaths"), + hub_cursor=json_dict.get("HubCursor"), + sink_cursor=json_dict.get("SinkCursor"), + ) + + +class OngoingTaskCdcSink(OngoingTask): + """Ongoing task information for a CDC Sink task.""" + + def __init__( + self, + task_id: Optional[int] = None, + responsible_node: Optional[NodeId] = None, + task_state: Optional[OngoingTaskState] = None, + task_connection_status: Optional[OngoingTaskConnectionStatus] = None, + task_name: Optional[str] = None, + error: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: Optional[bool] = None, + configuration: Optional["CdcSinkConfiguration"] = None, + connection_string_name: Optional[str] = None, + factory_name: Optional[str] = None, + last_batch_time: Optional[datetime] = None, + last_checkpoint: Optional[str] = None, + seconds_since_last_batch: Optional[float] = None, + last_activity_time: Optional[datetime] = None, + seconds_since_last_activity: Optional[float] = None, + health_issue: Optional[str] = None, + ): + super().__init__( + task_id=task_id, + task_type=OngoingTaskType.CDC_SINK, + responsible_node=responsible_node, + task_state=task_state, + task_connection_status=task_connection_status, + task_name=task_name, + error=error, + mentor_node=mentor_node, + pin_to_mentor_node=pin_to_mentor_node, + ) + self.configuration = configuration + self.connection_string_name = connection_string_name + self.factory_name = factory_name + # UTC time of the last successfully completed batch, None before the first one. + self.last_batch_time = last_batch_time + # The last persisted checkpoint (LSN/GTID). + self.last_checkpoint = last_checkpoint + self.seconds_since_last_batch = seconds_since_last_batch + # UTC time of the last activity from the source: a poll iteration (SQL Server), + # replication message (PostgreSQL) or binlog event (MySQL). Recent activity with + # a stale last_batch_time means the connection is alive but idle; both stale + # suggests the connection is dead. + self.last_activity_time = last_activity_time + self.seconds_since_last_activity = seconds_since_last_activity + # None when healthy, otherwise a diagnostic from the process itself. + self.health_issue = health_issue + + def to_json(self) -> dict: + result = super().to_json() + result["ConnectionStringName"] = self.connection_string_name + result["FactoryName"] = self.factory_name + result["Configuration"] = self.configuration.to_json() if self.configuration else None + result["LastBatchTime"] = Utils.datetime_to_string(self.last_batch_time) + result["LastCheckpoint"] = self.last_checkpoint + result["SecondsSinceLastBatch"] = self.seconds_since_last_batch + result["LastActivityTime"] = Utils.datetime_to_string(self.last_activity_time) + result["SecondsSinceLastActivity"] = self.seconds_since_last_activity + result["HealthIssue"] = self.health_issue + return result + + @classmethod + def from_json(cls, json_dict: dict) -> Optional["OngoingTaskCdcSink"]: + if json_dict is None: + return None + + from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration + + task_state_str = json_dict.get("TaskState") + task_connection_status_str = json_dict.get("TaskConnectionStatus") + configuration = json_dict.get("Configuration") + return cls( + task_id=json_dict.get("TaskId"), + responsible_node=NodeId.from_json(json_dict.get("ResponsibleNode")), + task_state=OngoingTaskState(task_state_str) if task_state_str else None, + task_connection_status=( + OngoingTaskConnectionStatus(task_connection_status_str) if task_connection_status_str else None + ), + task_name=json_dict.get("TaskName"), + error=json_dict.get("Error"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode"), + configuration=CdcSinkConfiguration.from_json(configuration) if configuration else None, + connection_string_name=json_dict.get("ConnectionStringName"), + factory_name=json_dict.get("FactoryName"), + last_batch_time=Utils.string_to_datetime(json_dict.get("LastBatchTime")), + last_checkpoint=json_dict.get("LastCheckpoint"), + seconds_since_last_batch=json_dict.get("SecondsSinceLastBatch"), + last_activity_time=Utils.string_to_datetime(json_dict.get("LastActivityTime")), + seconds_since_last_activity=json_dict.get("SecondsSinceLastActivity"), + health_issue=json_dict.get("HealthIssue"), ) @@ -637,6 +746,8 @@ def _deserialize_task( return OngoingTaskEmbeddingsGeneration.from_json(json_dict) elif self._task_type == OngoingTaskType.PULL_REPLICATION_AS_SINK: return OngoingTaskPullReplicationAsSink.from_json(json_dict) + elif self._task_type == OngoingTaskType.CDC_SINK: + return OngoingTaskCdcSink.from_json(json_dict) else: # todo: handle more types of tasks return OngoingTask.from_json(json_dict) diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index a9402629..360da4dc 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -56,7 +56,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.2.3" + CLIENT_VERSION = "7.2.5" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates diff --git a/ravendb/primitives/constants.py b/ravendb/primitives/constants.py index a391e8d9..d0bca7bb 100644 --- a/ravendb/primitives/constants.py +++ b/ravendb/primitives/constants.py @@ -27,6 +27,7 @@ class QueryString: class Headers: REQUEST_TIME = "Request-Time" + X_FORWARDED_FOR = "X-Forwarded-For" SERVER_STARTUP_TIME = "Server-Startup-Time" REFRESH_TOPOLOGY = "Refresh-Topology" TOPOLOGY_ETAG = "Topology-Etag" @@ -312,6 +313,9 @@ class Obsolete: class DatabaseRecord: class SupportedFeatures: THROW_REVISION_KEY_TOO_BIG_FIX = "ThrowRevisionKeyTooBigFix" + HASHED_REVISION_PK = "HashedRevisionPk" + PULL_REPLICATION_COMPOSITE_CHANGE_VECTORS = "PullReplicationCompositeChangeVectors" + THROW_CONTROL_CHARACTERS_IN_IDENTIFIER = "ThrowControlCharactersInIdentifier" class VectorSearch: diff --git a/ravendb/serverwide/database_record.py b/ravendb/serverwide/database_record.py index 955b1e84..580a0544 100644 --- a/ravendb/serverwide/database_record.py +++ b/ravendb/serverwide/database_record.py @@ -11,6 +11,7 @@ AutoIndexDefinition, ) from ravendb.documents.operations.backups.settings import PeriodicBackupConfiguration +from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration from ravendb.documents.operations.etl.configuration import RavenConnectionString, RavenEtlConfiguration from ravendb.documents.operations.etl.olap.connection import OlapConnectionString, OlapEtlConfiguration from ravendb.documents.operations.etl.sql import SqlConnectionString, SqlEtlConfiguration @@ -69,6 +70,7 @@ def __init__(self, database_name: Optional[str] = None): self.raven_etls: List[RavenEtlConfiguration] = [] self.sql_etls: List[SqlEtlConfiguration] = [] self.olap_etls: List[OlapEtlConfiguration] = [] + self.cdc_sinks: List[CdcSinkConfiguration] = [] self.embeddings_generations: List = [] self.client: Optional[ClientConfiguration] = None self.studio: Optional[StudioConfiguration] = None @@ -119,6 +121,7 @@ def to_json(self): "RavenEtls": self.raven_etls, "SqlEtls": self.sql_etls, "OlapEtls": self.olap_etls, + "CdcSinks": [cdc_sink.to_json() for cdc_sink in self.cdc_sinks or []], "Client": self.client, "Studio": self.studio, "TruncatedClusterTransactionCommand": self.truncated_cluster_transaction_commands_count, @@ -167,6 +170,7 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.raven_etls = json_dict.get("RavenEtls", None) record.sql_etls = json_dict.get("SqlEtls", None) record.olap_etls = json_dict.get("OlapEtls", None) + record.cdc_sinks = [CdcSinkConfiguration.from_json(cdc_sink) for cdc_sink in json_dict.get("CdcSinks") or []] embeddings_generations_data = json_dict.get("EmbeddingsGenerations", []) if embeddings_generations_data: from ravendb.documents.operations.ai.embeddings_generation_configuration import ( diff --git a/ravendb/serverwide/operations/certificates.py b/ravendb/serverwide/operations/certificates.py index 29a22cba..cc92d62e 100644 --- a/ravendb/serverwide/operations/certificates.py +++ b/ravendb/serverwide/operations/certificates.py @@ -39,6 +39,53 @@ def __str__(self): return self.value +class SsoProvider(enum.Enum): + GITHUB = "Github" + GOOGLE = "Google" + MICROSOFT = "Microsoft" + WINDOWS = "Windows" + + def __str__(self): + return self.value + + +class CertificateUsage(enum.Enum): + RAVEN_SERVER = "RavenServer" + RAVEN_SERVER_FOR_COMMUNICATION = "RavenServerForCommunication" + CLIENT = "Client" + SSO_SERVER = "SsoServer" + SSO_CLIENT = "SsoClient" + WELL_KNOWN_ISSUER = "WellKnownIssuer" + + def __str__(self): + return self.value + + +class SsoIdentifier: + """Identifies an SSO user with the provider that authenticated them.""" + + def __init__(self, provider: SsoProvider = None, identifier: str = None, domain: str = None): + self.provider = provider + self.identifier = identifier + self.domain = domain + + def to_json(self) -> dict: + return { + "Provider": self.provider.value if self.provider else None, + "Domain": self.domain, + "Identifier": self.identifier, + } + + @classmethod + def from_json(cls, json_dict: dict) -> SsoIdentifier: + provider = json_dict.get("Provider") + return cls( + provider=SsoProvider(provider) if provider else None, + identifier=json_dict.get("Identifier"), + domain=json_dict.get("Domain"), + ) + + class CertificateRawData: def __init__(self, raw_data: bytes = None): self.raw_data = raw_data @@ -57,6 +104,10 @@ def __init__( public_key_pinning_hash: str = None, not_before: datetime = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): self.name = name self.security_clearance = security_clearance @@ -68,6 +119,24 @@ def __init__( self.public_key_pinning_hash = public_key_pinning_hash self.not_before = not_before self.disabled = disabled + self.usage = usage + # The SSO servers allowed to authenticate this user, by public-key pinning hash. + self.sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes or [] + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers or [] + + @staticmethod + def _sso_fields_from_json(json_dict: dict) -> dict: + usage = json_dict.get("Usage") + sso_identifiers = json_dict.get("SsoIdentifiers") + return { + "usage": CertificateUsage(usage) if usage else None, + "sso_server_public_key_pinning_hashes": json_dict.get("SsoServerPublicKeyPinningHashes"), + "allow_any_sso_server": json_dict.get("AllowAnySsoServer", False), + "sso_identifiers": ( + [SsoIdentifier.from_json(identifier) for identifier in sso_identifiers] if sso_identifiers else None + ), + } @classmethod def from_json(cls, json_dict: dict) -> CertificateMetadata: @@ -82,6 +151,7 @@ def from_json(cls, json_dict: dict) -> CertificateMetadata: json_dict.get("PublicKeyPinningHash", None), Utils.string_to_datetime(json_dict["NotBefore"]) if "NotBefore" in json_dict else None, json_dict.get("Disabled", False), + **cls._sso_fields_from_json(json_dict), ) @@ -99,6 +169,10 @@ def __init__( collection_primary_key: str = None, public_key_pinning_hash: str = None, disabled: bool = False, + usage: CertificateUsage = None, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = False, + sso_identifiers: List[SsoIdentifier] = None, ): super().__init__( name, @@ -110,6 +184,10 @@ def __init__( collection_primary_key, public_key_pinning_hash, disabled=disabled, + usage=usage, + sso_server_public_key_pinning_hashes=sso_server_public_key_pinning_hashes, + allow_any_sso_server=allow_any_sso_server, + sso_identifiers=sso_identifiers, ) self.certificate = certificate self.password = password @@ -126,6 +204,10 @@ def to_json(self) -> dict: "Certificate": self.certificate, "Password": self.password, "Disabled": self.disabled, + "Usage": self.usage.value if self.usage else None, + "SsoServerPublicKeyPinningHashes": self.sso_server_public_key_pinning_hashes, + "AllowAnySsoServer": self.allow_any_sso_server, + "SsoIdentifiers": [identifier.to_json() for identifier in self.sso_identifiers], } if self.not_after: json_dict.update({"NotAfter": Utils.datetime_to_string(self.not_after)}) @@ -145,6 +227,7 @@ def from_json(cls, json_dict: dict) -> CertificateDefinition: json_dict["CollectionPrimaryKey"], json_dict["PublicKeyPinningHash"], disabled=json_dict.get("Disabled", False), + **cls._sso_fields_from_json(json_dict), ) @@ -456,12 +539,22 @@ def __init__( name: str, clearance: SecurityClearance, disabled: bool = False, + sso_server_public_key_pinning_hashes: List[str] = None, + allow_any_sso_server: bool = None, + sso_identifiers: List[SsoIdentifier] = None, ): self.thumbprint = thumbprint self.permissions = permissions self.name = name self.clearance = clearance self.disabled = disabled + # SSO configuration is opt-in: leave these None to keep the stored settings + # untouched, which is what a plain certificate edit or a disabled-only toggle + # wants. Setting any of them - even to an empty list - replaces the stored value, + # which is how an SSO user's authorizing servers or identifiers get cleared. + self.sso_server_public_key_pinning_hashes = sso_server_public_key_pinning_hashes + self.allow_any_sso_server = allow_any_sso_server + self.sso_identifiers = sso_identifiers def __init__(self, parameters: Parameters): if parameters is None: @@ -476,32 +569,15 @@ def __init__(self, parameters: Parameters): if parameters.permissions is None: raise ValueError("permissions cannot be None") - self.__name = parameters.name - self.__thumbprint = parameters.thumbprint - self.__permissions = parameters.permissions - self.__clearance = parameters.clearance - self.__disabled = parameters.disabled + self.__parameters = parameters def get_command(self, conventions: "DocumentConventions") -> "VoidRavenCommand": - return self.__EditCertificateClientCommand( - self.__thumbprint, self.__name, self.__permissions, self.__clearance, self.__disabled - ) + return self.__EditCertificateClientCommand(self.__parameters) class __EditCertificateClientCommand(VoidRavenCommand, RaftCommand): - def __init__( - self, - thumbprint: str, - name: str, - permissions: Dict[str, DatabaseAccess], - clearance: SecurityClearance, - disabled: bool, - ): + def __init__(self, parameters: "EditClientCertificateOperation.Parameters"): super().__init__() - self.__thumbprint = thumbprint - self.__name = name - self.__permissions = permissions - self.__clearance = clearance - self.__disabled = disabled + self.__parameters = parameters def is_read_request(self) -> bool: return False @@ -509,15 +585,26 @@ def is_read_request(self) -> bool: def create_request(self, node: ServerNode) -> requests.Request: url = f"{node.url}/admin/certificates/edit" - definition = CertificateDefinition() - definition.thumbprint = self.__thumbprint - definition.permissions = self.__permissions - definition.security_clearance = self.__clearance - definition.name = self.__name - definition.disabled = self.__disabled + parameters = self.__parameters + definition = { + "Thumbprint": parameters.thumbprint, + "Name": parameters.name, + "SecurityClearance": parameters.clearance, + "Disabled": parameters.disabled, + "Permissions": parameters.permissions, + } + + # Written only when explicitly provided, so the server leaves the existing SSO + # configuration alone on a partial edit and clears it when an empty list is sent. + if parameters.sso_server_public_key_pinning_hashes is not None: + definition["SsoServerPublicKeyPinningHashes"] = parameters.sso_server_public_key_pinning_hashes + if parameters.allow_any_sso_server is not None: + definition["AllowAnySsoServer"] = parameters.allow_any_sso_server + if parameters.sso_identifiers is not None: + definition["SsoIdentifiers"] = [identifier.to_json() for identifier in parameters.sso_identifiers] request = requests.Request("POST", url) - request.data = definition.to_json() + request.data = definition return request diff --git a/ravendb/serverwide/operations/connection_strings.py b/ravendb/serverwide/operations/connection_strings.py new file mode 100644 index 00000000..e78e53dd --- /dev/null +++ b/ravendb/serverwide/operations/connection_strings.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +import requests + +from ravendb.documents.operations.connection_strings import ( + ConnectionString, + ConnectionStringUsage, + ConnectionStringUsageKind, +) +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.common import ServerOperation +from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.tools.utils import Utils +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class ServerWideConnectionStringUsage(ConnectionStringUsage): + """ + A usage of a server-wide connection string. Adds the database the referencing task + or agent lives in, since server-wide usages are aggregated across every database. + """ + + def __init__( + self, + kind: ConnectionStringUsageKind = None, + id_: Optional[int] = None, + identifier: str = None, + name: str = None, + database_name: str = None, + ): + super().__init__(kind, id_, identifier, name) + self.database_name = database_name + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["DatabaseName"] = self.database_name + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> ServerWideConnectionStringUsage: + kind = json_dict.get("Kind") + return cls( + kind=ConnectionStringUsageKind(kind) if kind else None, + id_=json_dict.get("Id"), + identifier=json_dict.get("Identifier"), + name=json_dict.get("Name"), + database_name=json_dict.get("DatabaseName"), + ) + + +class ServerWideConnectionString: + """ + A connection string that the cluster propagates to every database, except those + listed in excluded_databases. + """ + + NAME_PREFIX = "Server Wide Connection String" + + def __init__( + self, + connection_string: ConnectionString = None, + excluded_databases: List[str] = None, + used_by: List[ServerWideConnectionStringUsage] = None, + ): + self.connection_string = connection_string + # Databases that should not receive this connection string. When None or empty it + # is propagated everywhere. + self.excluded_databases = excluded_databases + # Computed server-side when reading; anything a client sends here is ignored. + self.used_by = used_by or [] + + @property + def name(self) -> Optional[str]: + return self.connection_string.name if self.connection_string else None + + @property + def type(self) -> ConnectionStringType: + if self.connection_string is None: + return ConnectionStringType.NONE + return ConnectionStringType(self.connection_string.get_type) + + @staticmethod + def get_database_record_connection_string_name(name: str) -> str: + """The name a propagated server-wide connection string carries inside a database record.""" + return f"{ServerWideConnectionString.NAME_PREFIX}, {name}" + + def to_json(self) -> Dict[str, Any]: + # The connection string is flattened into the top-level object rather than nested, + # which is what the server-wide endpoint expects. + json_dict = self.connection_string.to_json() if self.connection_string else {} + json_dict["Type"] = self.type.value + json_dict["ExcludedDatabases"] = self.excluded_databases + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> Optional[ServerWideConnectionString]: + if json_dict is None: + return None + + type_ = json_dict.get("Type") + if type_ is None: + return None + + return cls( + connection_string=cls._deserialize_connection_string(json_dict, ConnectionStringType(type_)), + excluded_databases=json_dict.get("ExcludedDatabases"), + used_by=[ServerWideConnectionStringUsage.from_json(usage) for usage in json_dict.get("UsedBy") or []], + ) + + @staticmethod + def _deserialize_connection_string( + json_dict: Dict[str, Any], connection_string_type: ConnectionStringType + ) -> ConnectionString: + from ravendb.documents.operations.ai.ai_connection_string import AiConnectionString + from ravendb.documents.operations.etl.configuration import RavenConnectionString + from ravendb.documents.operations.etl.elastic_search.connection import ElasticSearchConnectionString + from ravendb.documents.operations.etl.olap.connection import OlapConnectionString + from ravendb.documents.operations.etl.queue.connection import QueueConnectionString + from ravendb.documents.operations.etl.snowflake.connection import SnowflakeConnectionString + from ravendb.documents.operations.etl.sql import SqlConnectionString + + types = { + ConnectionStringType.RAVEN: RavenConnectionString, + ConnectionStringType.SQL: SqlConnectionString, + ConnectionStringType.OLAP: OlapConnectionString, + ConnectionStringType.ELASTIC_SEARCH: ElasticSearchConnectionString, + ConnectionStringType.QUEUE: QueueConnectionString, + ConnectionStringType.SNOWFLAKE: SnowflakeConnectionString, + ConnectionStringType.AI: AiConnectionString, + } + + if connection_string_type not in types: + raise NotImplementedError(f"Unknown connection string type: {connection_string_type}") + + return types[connection_string_type].from_json(json_dict) + + +class GetServerWideConnectionStringsResult: + def __init__(self, results: List[ServerWideConnectionString] = None): + self.results = results or [] + + def to_json(self) -> Dict[str, Any]: + return {"Results": [connection_string.to_json() for connection_string in self.results]} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> GetServerWideConnectionStringsResult: + return cls(results=[ServerWideConnectionString.from_json(result) for result in json_dict.get("Results") or []]) + + +class GetServerWideConnectionStringsOperation(ServerOperation[GetServerWideConnectionStringsResult]): + """ + Reads server-wide connection strings from the cluster, either all of them or + filtered by name and type. + """ + + def __init__(self, connection_string_name: str = None, connection_string_type: ConnectionStringType = None): + if connection_string_name is not None and not connection_string_name.strip(): + raise ValueError("Connection string name must not be empty.") + + self._connection_string_name = connection_string_name + self._type = connection_string_type + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[GetServerWideConnectionStringsResult]: + return self._GetServerWideConnectionStringsCommand(self._connection_string_name, self._type) + + class _GetServerWideConnectionStringsCommand(RavenCommand[GetServerWideConnectionStringsResult]): + def __init__(self, connection_string_name: str = None, connection_string_type: ConnectionStringType = None): + super().__init__(GetServerWideConnectionStringsResult) + self._connection_string_name = connection_string_name + self._type = connection_string_type + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + + query_params = [] + if self._connection_string_name is not None: + query_params.append(f"name={Utils.quote_key(self._connection_string_name)}") + if self._type is not None and self._type != ConnectionStringType.NONE: + query_params.append(f"type={self._type.value}") + + if query_params: + url += f"?{'&'.join(query_params)}" + + return requests.Request("GET", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = GetServerWideConnectionStringsResult.from_json(json.loads(response)) + + +class PutServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> PutServerWideConnectionStringResult: + return cls(json_dict.get("RaftCommandIndex")) + + +class PutServerWideConnectionStringOperation(ServerOperation[PutServerWideConnectionStringResult]): + """ + Creates or updates a server-wide connection string. The cluster propagates it to + every database that is not listed in excluded_databases. + """ + + def __init__(self, connection_string: ServerWideConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + + if connection_string.connection_string is None: + raise ValueError("ServerWideConnectionString.connection_string must not be None.") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[PutServerWideConnectionStringResult]: + return self._PutServerWideConnectionStringCommand(self._connection_string) + + class _PutServerWideConnectionStringCommand(RavenCommand[PutServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ServerWideConnectionString): + super().__init__(PutServerWideConnectionStringResult) + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/admin/configuration/server-wide/connection-strings" + + request = requests.Request("PUT", url) + request.data = self._connection_string.to_json() + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = PutServerWideConnectionStringResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class RemoveServerWideConnectionStringResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> RemoveServerWideConnectionStringResult: + return cls(json_dict.get("RaftCommandIndex")) + + +class RemoveServerWideConnectionStringOperation(ServerOperation[RemoveServerWideConnectionStringResult]): + """ + Removes a server-wide connection string from the cluster and from every database + record that received it. Fails when an ongoing task still uses it. + """ + + def __init__(self, connection_string: ConnectionString): + if connection_string is None: + raise ValueError("connection_string cannot be None") + + if not connection_string.name or connection_string.name.isspace(): + raise ValueError("Connection string name must not be empty.") + + self._connection_string = connection_string + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[RemoveServerWideConnectionStringResult]: + return self._RemoveServerWideConnectionStringCommand(self._connection_string) + + class _RemoveServerWideConnectionStringCommand(RavenCommand[RemoveServerWideConnectionStringResult], RaftCommand): + def __init__(self, connection_string: ConnectionString): + super().__init__(RemoveServerWideConnectionStringResult) + self._connection_string = connection_string + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = ( + f"{node.url}/admin/configuration/server-wide/connection-strings" + f"?name={Utils.quote_key(self._connection_string.name)}" + f"&type={self._connection_string.get_type}" + ) + + return requests.Request("DELETE", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = RemoveServerWideConnectionStringResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() diff --git a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py new file mode 100644 index 00000000..ddce3c8c --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py @@ -0,0 +1,217 @@ +""" +Tests for GetConversationMessagesOperation, the 7.2.5 way to read an AI agent +conversation back, and for the cancel-pending-action-tools flag on a run. +""" + +import unittest +from datetime import datetime, timedelta, timezone + +from ravendb.documents.operations.ai.agents import ( + AiConversationDetailLevel, + AiConversationMessage, + AiConversationMessagesResult, + AiMessageRole, + AiToolCallResult, + AiUsage, + GetConversationMessagesOperation, + GetConversationMessagesOptions, + RunConversationOperation, +) +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants + + +class TestGetConversationMessagesOptions(unittest.TestCase): + def test_defaults_to_the_whole_conversation_at_simple_detail(self): + options = GetConversationMessagesOptions("chats/1-A") + + self.assertEqual(constants.int_max, options.page_size) + self.assertEqual(AiConversationDetailLevel.SIMPLE, options.detail_level) + self.assertIsNone(options.before) + self.assertIsNone(options.after) + + def test_a_conversation_id_is_required(self): + with self.assertRaises(ValueError): + GetConversationMessagesOperation("") + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions(None)) + with self.assertRaises(ValueError): + GetConversationMessagesOperation(None) + + def test_paging_backwards_and_forwards_at_once_is_rejected(self): + options = GetConversationMessagesOptions("chats/1-A", before=datetime(2026, 6, 16), after=datetime(2026, 6, 15)) + + with self.assertRaises(ValueError): + GetConversationMessagesOperation(options) + + def test_a_non_positive_page_size_is_rejected(self): + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions("chats/1-A", page_size=0)) + with self.assertRaises(ValueError): + GetConversationMessagesOperation(GetConversationMessagesOptions("chats/1-A", page_size=-1)) + + +class TestGetConversationMessagesCommand(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def _url(self, conversation_id_or_options): + command = GetConversationMessagesOperation(conversation_id_or_options).get_command(None) + return command.create_request(self.node).url + + def test_the_conversation_id_is_escaped_into_the_query_string(self): + url = self._url("chats/1-A") + + self.assertTrue( + url.startswith("http://localhost:8080/databases/db/ai/agent/conversation/messages?conversationId="), + url, + ) + self.assertIn("conversationId=chats%2F1-A", url) + + def test_page_size_and_detail_level_are_always_sent(self): + url = self._url(GetConversationMessagesOptions("chats/1-A", page_size=25)) + + self.assertIn("&pageSize=25", url) + self.assertIn("&detailLevel=Simple", url) + + def test_detail_level_is_sent_by_its_server_side_name(self): + url = self._url(GetConversationMessagesOptions("chats/1-A", detail_level=AiConversationDetailLevel.FULL)) + + self.assertIn("&detailLevel=Full", url) + + def test_a_naive_cursor_is_sent_as_is(self): + # Naive datetimes count as UTC here, the same rule the rest of the client follows. + url = self._url(GetConversationMessagesOptions("chats/1-A", before=datetime(2026, 6, 16, 10, 30, 0))) + + self.assertIn("&before=2026-06-16T10%3A30%3A00.0000000", url) + + def test_an_aware_cursor_is_converted_to_utc(self): + warsaw = datetime(2026, 6, 16, 10, 30, 0, tzinfo=timezone(timedelta(hours=2))) + url = self._url(GetConversationMessagesOptions("chats/1-A", after=warsaw)) + + self.assertIn("&after=2026-06-16T08%3A30%3A00.0000000", url) + + def test_reading_messages_is_a_read_request(self): + command = GetConversationMessagesOperation("chats/1-A").get_command(None) + + self.assertTrue(command.is_read_request()) + + def test_a_missing_conversation_leaves_the_result_unset(self): + # The endpoint answers 404 for an unknown conversation, which reaches us as None. + command = GetConversationMessagesOperation("chats/1-A").get_command(None) + command.set_response(None, False) + + self.assertIsNone(command.result) + + +class TestAiConversationMessagesResult(unittest.TestCase): + RESPONSE = { + "ConversationId": "chats/1-A", + "Agent": "agents/support", + "Parameters": {"customer": "ALFKI", "tags": ["vip", "eu"], "retries": 2}, + "TotalUsage": {"PromptTokens": 10, "CompletionTokens": 5, "TotalTokens": 15, "ReasoningTokens": 3}, + "LastMessageAt": "2026-06-16T10:30:05.0000000", + "HasMoreMessages": True, + "SubConversationIds": ["chats/1-A/sub/1"], + "Attachments": ["invoice.pdf"], + "Messages": [ + {"Role": "User", "Content": "where is my order?", "Timestamp": "2026-06-16T10:30:00.0000000"}, + { + "Role": "Assistant", + "Content": None, + "Timestamp": "2026-06-16T10:30:05.0000000", + "Usage": {"PromptTokens": 10, "CompletionTokens": 5, "TotalTokens": 15}, + "ToolCalls": [ + { + "Id": "call_1", + "Name": "lookup_order", + "Arguments": '{"id":"orders/1-A"}', + "Result": '{"status":"Shipped"}', + "SubConversationId": "chats/1-A/sub/1", + } + ], + }, + ], + } + + def test_result_parses_the_conversation_envelope(self): + result = AiConversationMessagesResult.from_json(self.RESPONSE) + + self.assertEqual("chats/1-A", result.conversation_id) + self.assertEqual("agents/support", result.agent) + self.assertTrue(result.has_more_messages) + self.assertEqual(["chats/1-A/sub/1"], result.sub_conversation_ids) + self.assertEqual(["invoice.pdf"], result.attachments) + self.assertEqual(datetime(2026, 6, 16, 10, 30, 5), result.last_message_at) + + def test_heterogeneous_parameters_come_back_as_they_are(self): + # Parameter values mix primitives and arrays, so they are handed over untouched. + parameters = AiConversationMessagesResult.from_json(self.RESPONSE).parameters + + self.assertEqual({"customer": "ALFKI", "tags": ["vip", "eu"], "retries": 2}, parameters) + + def test_usage_includes_reasoning_tokens(self): + result = AiConversationMessagesResult.from_json(self.RESPONSE) + + self.assertIsInstance(result.total_usage, AiUsage) + self.assertEqual(3, result.total_usage.reasoning_tokens) + self.assertEqual(15, result.total_usage.total_tokens) + + def test_messages_keep_their_role_and_timestamp(self): + messages = AiConversationMessagesResult.from_json(self.RESPONSE).messages + + self.assertEqual(2, len(messages)) + self.assertEqual(AiMessageRole.USER, messages[0].role) + self.assertEqual(AiMessageRole.ASSISTANT, messages[1].role) + self.assertEqual(datetime(2026, 6, 16, 10, 30, 0), messages[0].timestamp) + + def test_tool_calls_carry_their_result_and_sub_conversation(self): + tool_call = AiConversationMessagesResult.from_json(self.RESPONSE).messages[1].tool_calls[0] + + self.assertIsInstance(tool_call, AiToolCallResult) + self.assertEqual("lookup_order", tool_call.name) + self.assertEqual('{"status":"Shipped"}', tool_call.result) + self.assertEqual("chats/1-A/sub/1", tool_call.sub_conversation_id) + + def test_an_assistant_message_that_only_called_tools_has_no_content(self): + messages = AiConversationMessagesResult.from_json(self.RESPONSE).messages + + self.assertIsNone(messages[1].content) + + def test_result_survives_a_json_round_trip(self): + result = AiConversationMessagesResult.from_json(self.RESPONSE) + round_tripped = AiConversationMessagesResult.from_json(result.to_json()) + + self.assertEqual(result.to_json(), round_tripped.to_json()) + + def test_a_pending_tool_call_has_no_result_yet(self): + tool_call = AiToolCallResult.from_json({"Id": "call_1", "Name": "lookup_order", "Arguments": "{}"}) + + self.assertIsNone(tool_call.result) + + def test_an_empty_message_list_stays_empty(self): + result = AiConversationMessagesResult.from_json({"ConversationId": "chats/1-A", "Messages": []}) + + self.assertIsNone(result.messages) + self.assertFalse(result.has_more_messages) + + def test_message_to_json_writes_the_role_by_name(self): + message = AiConversationMessage(role=AiMessageRole.SUMMARY, content="short") + + self.assertEqual("Summary", message.to_json()["Role"]) + + +class TestCancelPendingActionTools(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def _url(self, **kwargs): + operation = RunConversationOperation(agent_id="agents/support", conversation_id="chats/1-A", **kwargs) + return operation.get_command(None).create_request(self.node).url + + def test_the_flag_is_sent_on_every_run(self): + # The server has no default of its own, so the client always states its intent. + self.assertIn("&cancelPendingActionTools=False", self._url()) + + def test_the_flag_is_sent_when_it_is_set(self): + self.assertIn("&cancelPendingActionTools=True", self._url(cancel_pending_action_tools=True)) diff --git a/ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py b/ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py new file mode 100644 index 00000000..78238b20 --- /dev/null +++ b/ravendb/tests/operations_tests/test_azure_service_bus_connection_string.py @@ -0,0 +1,137 @@ +""" +Tests for the Azure Service Bus queue broker added in 7.2.5. +""" + +import unittest + +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsOperation, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.connection_string.remove_connection_string_operation import ( + RemoveConnectionStringOperation, +) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, + AzureServiceBusEntraId, + AzureServiceBusPasswordless, +) +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.tests.test_base import TestBase + +CONNECTION_STRING = "Endpoint=sb://rvn-test.servicebus.windows.net/;SharedAccessKeyName=k;SharedAccessKey=v" + + +class TestAzureServiceBusConnectionSettings(unittest.TestCase): + def test_azure_service_bus_is_a_known_broker(self): + self.assertEqual("AzureServiceBus", QueueBrokerType.AZURE_SERVICE_BUS.value) + + def test_connection_string_authentication_round_trips(self): + settings = AzureServiceBusConnectionSettings(connection_string=CONNECTION_STRING) + serialized = settings.to_json() + + self.assertEqual(CONNECTION_STRING, serialized["ConnectionString"]) + self.assertIsNone(serialized["EntraId"]) + self.assertIsNone(serialized["Passwordless"]) + self.assertEqual(serialized, AzureServiceBusConnectionSettings.from_json(serialized).to_json()) + + def test_entra_id_authentication_round_trips(self): + settings = AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId( + namespace="rvn-test.servicebus.windows.net", + tenant_id="tenant", + client_id="client", + client_secret="secret", + ) + ) + parsed = AzureServiceBusConnectionSettings.from_json(settings.to_json()) + + self.assertEqual("rvn-test.servicebus.windows.net", parsed.entra_id.namespace) + self.assertEqual("tenant", parsed.entra_id.tenant_id) + self.assertEqual("client", parsed.entra_id.client_id) + self.assertEqual("secret", parsed.entra_id.client_secret) + + def test_passwordless_authentication_round_trips(self): + settings = AzureServiceBusConnectionSettings( + passwordless=AzureServiceBusPasswordless(namespace="rvn-test.servicebus.windows.net") + ) + parsed = AzureServiceBusConnectionSettings.from_json(settings.to_json()) + + self.assertEqual("rvn-test.servicebus.windows.net", parsed.passwordless.namespace) + self.assertIsNone(parsed.entra_id) + + def test_the_settings_hang_off_the_queue_connection_string(self): + connection_string = QueueConnectionString( + name="asb1", + broker_type=QueueBrokerType.AZURE_SERVICE_BUS, + azure_service_bus_settings=AzureServiceBusConnectionSettings(connection_string=CONNECTION_STRING), + ) + serialized = connection_string.to_json() + + self.assertEqual("AzureServiceBus", serialized["BrokerType"]) + self.assertEqual(CONNECTION_STRING, serialized["AzureServiceBusConnectionSettings"]["ConnectionString"]) + + def test_a_queue_connection_string_from_an_older_server_has_no_service_bus_settings(self): + parsed = QueueConnectionString.from_json( + { + "Name": "kafka1", + "BrokerType": "Kafka", + "KafkaConnectionSettings": {"BootstrapServers": "localhost:9092"}, + "RabbitMqConnectionSettings": None, + "AzureQueueStorageConnectionSettings": None, + "AmazonSqsConnectionSettings": None, + } + ) + + self.assertIsNone(parsed.azure_service_bus_settings) + + +class TestAzureServiceBusConnectionStringLifecycle(TestBase): + def setUp(self): + super().setUp() + + def test_the_server_stores_and_returns_an_azure_service_bus_connection_string(self): + connection_string = QueueConnectionString( + name="asb1", + broker_type=QueueBrokerType.AZURE_SERVICE_BUS, + azure_service_bus_settings=AzureServiceBusConnectionSettings(connection_string=CONNECTION_STRING), + ) + + put_result = self.store.maintenance.send(PutConnectionStringOperation(connection_string)) + self.assertGreater(put_result.raft_command_index, 0) + + get_result = self.store.maintenance.send(GetConnectionStringsOperation("asb1", ConnectionStringType.QUEUE)) + stored = get_result.queue_connection_strings["asb1"] + self.assertEqual(QueueBrokerType.AZURE_SERVICE_BUS, stored.broker_type) + self.assertEqual(CONNECTION_STRING, stored.azure_service_bus_settings.connection_string) + + remove_result = self.store.maintenance.send(RemoveConnectionStringOperation(connection_string)) + self.assertGreater(remove_result.raft_command_index, 0) + + after_delete = self.store.maintenance.send(GetConnectionStringsOperation("asb1", ConnectionStringType.QUEUE)) + self.assertFalse(after_delete.queue_connection_strings) + + def test_entra_id_credentials_survive_a_server_round_trip(self): + connection_string = QueueConnectionString( + name="asb2", + broker_type=QueueBrokerType.AZURE_SERVICE_BUS, + azure_service_bus_settings=AzureServiceBusConnectionSettings( + entra_id=AzureServiceBusEntraId( + namespace="rvn-test.servicebus.windows.net", + tenant_id="tenant", + client_id="client", + client_secret="secret", + ) + ), + ) + + self.store.maintenance.send(PutConnectionStringOperation(connection_string)) + stored = self.store.maintenance.send( + GetConnectionStringsOperation("asb2", ConnectionStringType.QUEUE) + ).queue_connection_strings["asb2"] + + self.assertEqual("rvn-test.servicebus.windows.net", stored.azure_service_bus_settings.entra_id.namespace) + self.assertEqual("client", stored.azure_service_bus_settings.entra_id.client_id) diff --git a/ravendb/tests/operations_tests/test_cdc_sink.py b/ravendb/tests/operations_tests/test_cdc_sink.py new file mode 100644 index 00000000..9bc02d2e --- /dev/null +++ b/ravendb/tests/operations_tests/test_cdc_sink.py @@ -0,0 +1,284 @@ +""" +Tests for the CDC Sink client surface added in 7.2.5: the configuration tree, +AddCdcSinkOperation / UpdateCdcSinkOperation, and the CdcSink ongoing task. +""" + +import json +import unittest + +from ravendb.documents.operations.cdc_sink import ( + AddCdcSinkOperation, + AddCdcSinkOperationResult, + CdcColumnMapping, + CdcColumnType, + CdcSinkConfiguration, + CdcSinkEmbeddedTableConfig, + CdcSinkLinkedTableConfig, + CdcSinkOnDeleteConfig, + CdcSinkPostgresSettings, + CdcSinkProcessState, + CdcSinkRelationType, + CdcSinkTableConfig, + CdcSinkTaskState, + UpdateCdcSinkOperation, +) +from ravendb.documents.operations.ongoing_tasks import ( + GetOngoingTaskInfoOperation, + OngoingTaskCdcSink, + OngoingTaskType, +) +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.database_record import DatabaseRecord + + +def _configuration() -> CdcSinkConfiguration: + return CdcSinkConfiguration( + name="orders-cdc", + connection_string_name="pg", + postgres=CdcSinkPostgresSettings(publication_name="pub", slot_name="slot"), + skip_initial_load=True, + tables=[ + CdcSinkTableConfig( + collection_name="Orders", + source_table_schema="public", + source_table_name="orders", + columns=[ + CdcColumnMapping("id", "Id"), + CdcColumnMapping("payload", "Payload", CdcColumnType.JSON), + CdcColumnMapping("blob", "file.bin", CdcColumnType.ATTACHMENT), + ], + primary_key_columns=["id"], + patch="this.Total = $row.total;", + on_delete=CdcSinkOnDeleteConfig(patch="this.Archived = true;", ignore_deletes=True), + embedded_tables=[ + CdcSinkEmbeddedTableConfig( + source_table_name="order_lines", + property_name="Lines", + columns=[CdcColumnMapping("line_id", "LineId")], + primary_key_columns=["line_id"], + join_columns=["order_id"], + type_=CdcSinkRelationType.MAP, + case_sensitive_keys=True, + embedded_tables=[ + CdcSinkEmbeddedTableConfig( + source_table_name="line_notes", + property_name="Notes", + columns=[CdcColumnMapping("note_id", "NoteId")], + primary_key_columns=["note_id"], + join_columns=["line_id"], + ) + ], + ) + ], + linked_tables=[ + CdcSinkLinkedTableConfig( + source_table_name="customers", + property_name="Customer", + join_columns=["customer_id"], + linked_collection_name="Customers", + ) + ], + ) + ], + ) + + +class TestCdcSinkConfiguration(unittest.TestCase): + def test_configuration_survives_a_json_round_trip(self): + configuration = _configuration() + serialized = configuration.to_json() + + self.assertEqual(serialized, CdcSinkConfiguration.from_json(serialized).to_json()) + + def test_configuration_serializes_the_task_level_fields(self): + serialized = _configuration().to_json() + + self.assertEqual("orders-cdc", serialized["Name"]) + self.assertEqual("pg", serialized["ConnectionStringName"]) + self.assertTrue(serialized["SkipInitialLoad"]) + self.assertEqual({"PublicationName": "pub", "SlotName": "slot"}, serialized["Postgres"]) + self.assertEqual(0, serialized["TaskId"]) + self.assertFalse(serialized["Disabled"]) + + def test_column_type_is_written_only_when_it_is_not_default(self): + # The server reads a missing Type as Default, and so does the C# client. + self.assertEqual({"Column": "c", "Name": "N"}, CdcColumnMapping("c", "N").to_json()) + self.assertEqual( + {"Column": "c", "Name": "N", "Type": "Json"}, + CdcColumnMapping("c", "N", CdcColumnType.JSON).to_json(), + ) + + def test_column_without_a_type_deserializes_as_default(self): + self.assertEqual(CdcColumnType.DEFAULT, CdcColumnMapping.from_json({"Column": "c", "Name": "N"}).type_) + + def test_embedded_tables_nest_to_any_depth(self): + configuration = CdcSinkConfiguration.from_json(_configuration().to_json()) + lines = configuration.tables[0].embedded_tables[0] + + self.assertEqual(CdcSinkRelationType.MAP, lines.type_) + self.assertTrue(lines.case_sensitive_keys) + self.assertEqual("Notes", lines.embedded_tables[0].property_name) + self.assertEqual(["line_id"], lines.embedded_tables[0].join_columns) + + def test_on_delete_round_trips(self): + on_delete = CdcSinkConfiguration.from_json(_configuration().to_json()).tables[0].on_delete + + self.assertEqual("this.Archived = true;", on_delete.patch) + self.assertTrue(on_delete.ignore_deletes) + + def test_a_table_without_an_on_delete_keeps_it_unset(self): + table = CdcSinkTableConfig.from_json({"CollectionName": "Orders"}) + + self.assertIsNone(table.on_delete) + self.assertEqual([], table.columns) + self.assertEqual([], table.embedded_tables) + + def test_collections_default_to_empty_rather_than_none(self): + configuration = CdcSinkConfiguration() + + self.assertEqual([], configuration.tables) + self.assertEqual([], CdcSinkEmbeddedTableConfig().columns) + self.assertEqual([], CdcSinkLinkedTableConfig().join_columns) + + +class TestCdcSinkTaskState(unittest.TestCase): + def test_task_state_round_trips(self): + payload = { + "ConfigurationName": "orders-cdc", + "LastLsn": "0/16B3748", + "Tables": { + "public.orders": { + "InitialLoadCompleted": True, + "LastKeyValues": ["10"], + "KeyColumns": ["id"], + } + }, + } + state = CdcSinkTaskState.from_json(payload) + + self.assertEqual("0/16B3748", state.last_lsn) + self.assertTrue(state.tables["public.orders"].initial_load_completed) + self.assertEqual(["id"], state.tables["public.orders"].key_columns) + self.assertEqual(payload, state.to_json()) + + def test_document_id_is_built_from_the_state_collection(self): + self.assertEqual("@cdc-states", CdcSinkTaskState.COLLECTION_NAME) + self.assertEqual("@cdc-states/orders-cdc", CdcSinkTaskState.get_document_id("orders-cdc")) + + def test_process_state_item_name(self): + self.assertEqual("values/db/cdcsink/orders-cdc", CdcSinkProcessState.generate_item_name("db", "orders-cdc")) + + +class TestCdcSinkOperations(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def test_add_sends_the_configuration_to_the_admin_endpoint(self): + configuration = _configuration() + command = AddCdcSinkOperation(configuration).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink", request.url) + self.assertEqual(configuration.to_json(), request.data) + self.assertFalse(command.is_read_request()) + + def test_update_addresses_the_task_by_id(self): + command = UpdateCdcSinkOperation(42, _configuration()).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink?id=42", request.url) + + def test_both_operations_are_raft_commands(self): + # The server applies CDC Sink changes through Raft, so retries must reuse one id. + add = AddCdcSinkOperation(_configuration()).get_command(None) + update = UpdateCdcSinkOperation(1, _configuration()).get_command(None) + + self.assertIsInstance(add, RaftCommand) + self.assertIsInstance(update, RaftCommand) + self.assertTrue(add.get_raft_unique_request_id()) + + def test_a_missing_configuration_is_rejected_client_side(self): + with self.assertRaises(ValueError): + AddCdcSinkOperation(None) + with self.assertRaises(ValueError): + UpdateCdcSinkOperation(1, None) + + def test_add_reads_the_task_id_off_the_response(self): + command = AddCdcSinkOperation(_configuration()).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 17, "TaskId": 5}), False) + + self.assertIsInstance(command.result, AddCdcSinkOperationResult) + self.assertEqual(17, command.result.raft_command_index) + self.assertEqual(5, command.result.task_id) + + +class TestCdcSinkOngoingTask(unittest.TestCase): + RESPONSE = { + "TaskId": 7, + "TaskType": "CdcSink", + "TaskName": "orders-cdc", + "TaskState": "Enabled", + "ResponsibleNode": {"NodeTag": "A", "NodeUrl": "http://localhost:8080"}, + "ConnectionStringName": "pg", + "FactoryName": "Npgsql", + "LastCheckpoint": "0/16B3748", + "LastBatchTime": "2026-06-16T10:30:00.0000000", + "SecondsSinceLastBatch": 12.5, + "LastActivityTime": "2026-06-16T10:30:05.0000000", + "SecondsSinceLastActivity": 7.5, + "HealthIssue": None, + "Configuration": {"Name": "orders-cdc", "Tables": [{"CollectionName": "Orders"}]}, + } + + def test_cdc_sink_is_a_known_ongoing_task_type(self): + self.assertEqual("CdcSink", OngoingTaskType.CDC_SINK.value) + + def test_ongoing_task_carries_the_health_and_lag_fields(self): + task = OngoingTaskCdcSink.from_json(self.RESPONSE) + + self.assertEqual(OngoingTaskType.CDC_SINK, task.task_type) + self.assertEqual("Npgsql", task.factory_name) + self.assertEqual("0/16B3748", task.last_checkpoint) + self.assertEqual(12.5, task.seconds_since_last_batch) + self.assertEqual(7.5, task.seconds_since_last_activity) + self.assertIsNone(task.health_issue) + self.assertEqual("2026-06-16T10:30:00.0000000", task.to_json()["LastBatchTime"]) + + def test_ongoing_task_parses_the_nested_configuration(self): + task = OngoingTaskCdcSink.from_json(self.RESPONSE) + + self.assertIsInstance(task.configuration, CdcSinkConfiguration) + self.assertEqual("Orders", task.configuration.tables[0].collection_name) + + def test_get_ongoing_task_info_dispatches_to_the_cdc_sink_result(self): + command = GetOngoingTaskInfoOperation("orders-cdc", OngoingTaskType.CDC_SINK).get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db")) + command.set_response(json.dumps(self.RESPONSE), False) + + self.assertIn("type=CdcSink", request.url) + self.assertIsInstance(command.result, OngoingTaskCdcSink) + self.assertEqual("orders-cdc", command.result.task_name) + + +class TestCdcSinkInDatabaseRecord(unittest.TestCase): + def test_database_record_carries_cdc_sinks(self): + record = DatabaseRecord("db") + record.cdc_sinks = [_configuration()] + + self.assertEqual("orders-cdc", record.to_json()["CdcSinks"][0]["Name"]) + + def test_database_record_parses_cdc_sinks(self): + record = DatabaseRecord.from_json( + {"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}, "CdcSinks": [_configuration().to_json()]} + ) + + self.assertEqual(1, len(record.cdc_sinks)) + self.assertEqual("Orders", record.cdc_sinks[0].tables[0].collection_name) + + def test_a_record_from_a_server_without_cdc_sinks_gets_an_empty_list(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}}) + + self.assertEqual([], record.cdc_sinks) diff --git a/ravendb/tests/operations_tests/test_certificates_sso.py b/ravendb/tests/operations_tests/test_certificates_sso.py new file mode 100644 index 00000000..c3fdd601 --- /dev/null +++ b/ravendb/tests/operations_tests/test_certificates_sso.py @@ -0,0 +1,178 @@ +""" +Tests for the certificate SSO fields added in 7.2.5: the metadata a certificate now +carries, and the opt-in write semantics of EditClientCertificateOperation. +""" + +import unittest + +from ravendb.documents.conventions import DocumentConventions +from ravendb.http.server_node import ServerNode +from ravendb.serverwide.operations.certificates import ( + CertificateDefinition, + CertificateMetadata, + CertificateUsage, + DatabaseAccess, + EditClientCertificateOperation, + SecurityClearance, + SsoIdentifier, + SsoProvider, +) + + +class TestSsoIdentifier(unittest.TestCase): + def test_identifier_round_trips(self): + payload = {"Provider": "Google", "Domain": "ravendb.net", "Identifier": "gracjan@ravendb.net"} + identifier = SsoIdentifier.from_json(payload) + + self.assertEqual(SsoProvider.GOOGLE, identifier.provider) + self.assertEqual("ravendb.net", identifier.domain) + self.assertEqual(payload, identifier.to_json()) + + def test_a_domainless_identifier_is_allowed(self): + identifier = SsoIdentifier.from_json({"Provider": "Github", "Identifier": "gracjan"}) + + self.assertEqual(SsoProvider.GITHUB, identifier.provider) + self.assertIsNone(identifier.domain) + + def test_every_provider_the_server_knows_is_mapped(self): + for name in ("Github", "Google", "Microsoft", "Windows"): + self.assertEqual(name, SsoProvider(name).value) + + +class TestCertificateSsoMetadata(unittest.TestCase): + PAYLOAD = { + "Name": "sso-user", + "SecurityClearance": "ValidUser", + "Thumbprint": "TP", + "Permissions": {}, + "Disabled": False, + "Usage": "SsoClient", + "AllowAnySsoServer": True, + "SsoServerPublicKeyPinningHashes": ["hash-a", "hash-b"], + "SsoIdentifiers": [{"Provider": "Microsoft", "Identifier": "gracjan@ravendb.net", "Domain": "ravendb.net"}], + } + + def test_metadata_parses_the_sso_fields(self): + metadata = CertificateMetadata.from_json(self.PAYLOAD) + + self.assertEqual(CertificateUsage.SSO_CLIENT, metadata.usage) + self.assertTrue(metadata.allow_any_sso_server) + self.assertEqual(["hash-a", "hash-b"], metadata.sso_server_public_key_pinning_hashes) + self.assertEqual(SsoProvider.MICROSOFT, metadata.sso_identifiers[0].provider) + + def test_a_certificate_from_an_older_server_gets_the_neutral_defaults(self): + metadata = CertificateMetadata.from_json({"Name": "n", "SecurityClearance": "Operator", "Permissions": {}}) + + self.assertIsNone(metadata.usage) + self.assertFalse(metadata.allow_any_sso_server) + self.assertEqual([], metadata.sso_server_public_key_pinning_hashes) + self.assertEqual([], metadata.sso_identifiers) + + def test_definition_parses_and_writes_the_sso_fields(self): + definition = CertificateDefinition.from_json( + { + **self.PAYLOAD, + "Certificate": "cert", + "NotAfter": None, + "CollectionSecondaryKeys": [], + "CollectionPrimaryKey": "", + "PublicKeyPinningHash": "h", + } + ) + serialized = definition.to_json() + + self.assertEqual("SsoClient", serialized["Usage"]) + self.assertTrue(serialized["AllowAnySsoServer"]) + self.assertEqual(["hash-a", "hash-b"], serialized["SsoServerPublicKeyPinningHashes"]) + self.assertEqual("gracjan@ravendb.net", serialized["SsoIdentifiers"][0]["Identifier"]) + + def test_a_definition_with_no_usage_writes_none(self): + self.assertIsNone(CertificateDefinition().to_json()["Usage"]) + + def test_every_usage_the_server_knows_is_mapped(self): + for name in ( + "RavenServer", + "RavenServerForCommunication", + "Client", + "SsoServer", + "SsoClient", + "WellKnownIssuer", + ): + self.assertEqual(name, CertificateUsage(name).value) + + +class TestEditClientCertificateSsoPayload(unittest.TestCase): + NODE = ServerNode("http://localhost:8080", "db") + + def _payload(self, **kwargs): + parameters = EditClientCertificateOperation.Parameters( + thumbprint="TP", + permissions={"orders": DatabaseAccess.READ_WRITE}, + name="sso-user", + clearance=SecurityClearance.VALID_USER, + **kwargs, + ) + request = EditClientCertificateOperation(parameters).get_command(None).create_request(self.NODE) + return request.data + + def test_the_base_payload_carries_only_what_the_server_edits(self): + payload = self._payload() + + self.assertEqual({"Thumbprint", "Name", "SecurityClearance", "Disabled", "Permissions"}, set(payload)) + self.assertEqual("TP", payload["Thumbprint"]) + self.assertFalse(payload["Disabled"]) + + def test_permissions_and_clearance_serialize_by_their_server_side_names(self): + payload = self._payload() + encoded = DocumentConventions.json_default(payload["Permissions"]["orders"]) + + self.assertEqual("ReadWrite", encoded) + self.assertEqual("ValidUser", DocumentConventions.json_default(payload["SecurityClearance"])) + + def test_a_plain_edit_leaves_the_stored_sso_configuration_alone(self): + # Omitting the SSO fields is how a regular edit avoids clobbering an SSO user. + payload = self._payload(disabled=True) + + self.assertNotIn("SsoServerPublicKeyPinningHashes", payload) + self.assertNotIn("AllowAnySsoServer", payload) + self.assertNotIn("SsoIdentifiers", payload) + self.assertTrue(payload["Disabled"]) + + def test_an_empty_list_is_sent_so_it_can_clear_the_stored_value(self): + payload = self._payload(sso_server_public_key_pinning_hashes=[], sso_identifiers=[]) + + self.assertEqual([], payload["SsoServerPublicKeyPinningHashes"]) + self.assertEqual([], payload["SsoIdentifiers"]) + + def test_setting_the_sso_fields_writes_them_all(self): + payload = self._payload( + sso_server_public_key_pinning_hashes=["hash-a"], + allow_any_sso_server=True, + sso_identifiers=[SsoIdentifier(SsoProvider.GITHUB, "gracjan")], + ) + + self.assertEqual(["hash-a"], payload["SsoServerPublicKeyPinningHashes"]) + self.assertTrue(payload["AllowAnySsoServer"]) + self.assertEqual({"Provider": "Github", "Domain": None, "Identifier": "gracjan"}, payload["SsoIdentifiers"][0]) + + def test_allow_any_sso_server_can_be_turned_off_explicitly(self): + payload = self._payload(allow_any_sso_server=False) + + self.assertIn("AllowAnySsoServer", payload) + self.assertFalse(payload["AllowAnySsoServer"]) + + def test_the_operation_still_validates_its_required_arguments(self): + with self.assertRaises(ValueError): + EditClientCertificateOperation(None) + with self.assertRaises(ValueError): + EditClientCertificateOperation( + EditClientCertificateOperation.Parameters("TP", {}, None, SecurityClearance.VALID_USER) + ) + with self.assertRaises(ValueError): + EditClientCertificateOperation( + EditClientCertificateOperation.Parameters(None, {}, "n", SecurityClearance.VALID_USER) + ) + with self.assertRaises(ValueError): + EditClientCertificateOperation( + EditClientCertificateOperation.Parameters("TP", None, "n", SecurityClearance.VALID_USER) + ) diff --git a/ravendb/tests/operations_tests/test_s3_settings.py b/ravendb/tests/operations_tests/test_s3_settings.py new file mode 100644 index 00000000..f4a93300 --- /dev/null +++ b/ravendb/tests/operations_tests/test_s3_settings.py @@ -0,0 +1,69 @@ +""" +Tests for DisableChecksumValidation, added in 7.2.5 to both S3 settings classes for +S3-compatible storage that does not support modern object integrity checks. +""" + +import unittest + +from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings +from ravendb.documents.operations.backups.settings import GetBackupConfigurationScript, S3Settings + + +class TestBackupS3Settings(unittest.TestCase): + def _settings(self, **kwargs) -> S3Settings: + return S3Settings( + disabled=False, + get_backup_configuration_script=GetBackupConfigurationScript(), + aws_access_key="key", + aws_secret_key="secret", + aws_session_token=None, + aws_region_name="eu-central-1", + remote_folder_name="backups", + bucket_name="rvn", + custom_server_url="https://minio.local", + force_path_style=True, + **kwargs, + ) + + def test_checksum_validation_is_left_alone_by_default(self): + self.assertIsNone(self._settings().disable_checksum_validation) + + def test_the_flag_is_written(self): + self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + + def test_the_flag_round_trips(self): + serialized = self._settings(disable_checksum_validation=True).to_json() + + self.assertEqual(serialized, S3Settings.from_json(serialized).to_json()) + + def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): + serialized = self._settings().to_json() + del serialized["DisableChecksumValidation"] + + self.assertIsNone(S3Settings.from_json(serialized).disable_checksum_validation) + + +class TestRemoteAttachmentsS3Settings(unittest.TestCase): + def _settings(self, **kwargs) -> RemoteAttachmentsS3Settings: + return RemoteAttachmentsS3Settings( + aws_access_key="key", + aws_secret_key="secret", + aws_region_name="eu-central-1", + bucket_name="rvn", + force_path_style=True, + **kwargs, + ) + + def test_checksum_validation_is_left_alone_by_default(self): + self.assertIsNone(self._settings().disable_checksum_validation) + + def test_the_flag_is_written(self): + self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + + def test_the_flag_round_trips(self): + serialized = self._settings(disable_checksum_validation=True).to_json() + + self.assertEqual(serialized, RemoteAttachmentsS3Settings.from_json(serialized).to_json()) + + def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): + self.assertIsNone(RemoteAttachmentsS3Settings.from_json({"BucketName": "rvn"}).disable_checksum_validation) diff --git a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py new file mode 100644 index 00000000..65314419 --- /dev/null +++ b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py @@ -0,0 +1,250 @@ +""" +Tests for the server-wide connection string operations added in 7.2.5, and for the +UsedBy metadata the server now returns with every connection string. +""" + +import json +import unittest + +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsResult, +) +from ravendb.documents.operations.connection_strings import ConnectionStringUsage, ConnectionStringUsageKind +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType, QueueConnectionString +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.operations.connection_strings import ( + GetServerWideConnectionStringsOperation, + PutServerWideConnectionStringOperation, + RemoveServerWideConnectionStringOperation, + ServerWideConnectionString, + ServerWideConnectionStringUsage, +) +from ravendb.serverwide.server_operation_executor import ConnectionStringType + + +class TestConnectionStringUsage(unittest.TestCase): + def test_ongoing_tasks_are_identified_by_a_numeric_id(self): + usage = ConnectionStringUsage.from_json({"Kind": "RavenEtl", "Id": 3, "Identifier": None, "Name": "etl-to-b"}) + + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, usage.kind) + self.assertEqual(3, usage.id_) + self.assertIsNone(usage.identifier) + self.assertEqual("etl-to-b", usage.name) + + def test_ai_agents_are_identified_by_a_string_identifier(self): + usage = ConnectionStringUsage.from_json( + {"Kind": "AiAgent", "Id": None, "Identifier": "agents/support", "Name": "support"} + ) + + self.assertEqual(ConnectionStringUsageKind.AI_AGENT, usage.kind) + self.assertIsNone(usage.id_) + self.assertEqual("agents/support", usage.identifier) + + def test_cdc_sink_is_a_known_usage_kind(self): + self.assertEqual("CdcSink", ConnectionStringUsageKind.CDC_SINK.value) + + def test_read_connection_strings_carry_their_usages(self): + result = GetConnectionStringsResult.from_json( + { + "RavenConnectionStrings": { + "to-b": { + "Name": "to-b", + "Database": "b", + "TopologyDiscoveryUrls": ["http://localhost:8080"], + "UsedBy": [ + {"Kind": "RavenEtl", "Id": 3, "Name": "etl-1"}, + {"Kind": "AiAgent", "Identifier": "agents/1", "Name": "helper"}, + ], + } + } + } + ) + connection_string = result.raven_connection_strings["to-b"] + + self.assertEqual(2, len(connection_string.used_by)) + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, connection_string.used_by[0].kind) + self.assertEqual("agents/1", connection_string.used_by[1].identifier) + + def test_a_connection_string_nothing_uses_has_an_empty_usage_list(self): + result = GetConnectionStringsResult.from_json( + {"SqlConnectionStrings": {"sql1": {"Name": "sql1", "ConnectionString": "x", "FactoryName": "y"}}} + ) + + self.assertEqual([], result.sql_connection_strings["sql1"].used_by) + + def test_usages_are_never_written_back(self): + # UsedBy is computed server-side; sending it back would be meaningless. + connection_string = RavenConnectionString("to-b", "b", ["http://localhost:8080"]) + connection_string.used_by = [ConnectionStringUsage(ConnectionStringUsageKind.RAVEN_ETL, 3, None, "etl-1")] + + self.assertNotIn("UsedBy", connection_string.to_json()) + + def test_an_absent_bucket_reads_as_none(self): + result = GetConnectionStringsResult.from_json({"RavenConnectionStrings": {}}) + + self.assertIsNone(result.raven_connection_strings) + self.assertIsNone(result.queue_connection_strings) + + +class TestServerWideConnectionString(unittest.TestCase): + def _raven(self) -> ServerWideConnectionString: + return ServerWideConnectionString( + connection_string=RavenConnectionString("shared-raven", "orders", ["http://localhost:8080"]), + excluded_databases=["scratch"], + ) + + def test_name_and_type_are_delegated_to_the_wrapped_connection_string(self): + server_wide = self._raven() + + self.assertEqual("shared-raven", server_wide.name) + self.assertEqual(ConnectionStringType.RAVEN, server_wide.type) + + def test_an_empty_wrapper_reports_no_type(self): + server_wide = ServerWideConnectionString() + + self.assertIsNone(server_wide.name) + self.assertEqual(ConnectionStringType.NONE, server_wide.type) + + def test_the_connection_string_is_flattened_rather_than_nested(self): + serialized = self._raven().to_json() + + self.assertEqual("shared-raven", serialized["Name"]) + self.assertEqual("orders", serialized["Database"]) + self.assertEqual("Raven", serialized["Type"]) + self.assertEqual(["scratch"], serialized["ExcludedDatabases"]) + + def test_it_round_trips_through_the_flat_shape(self): + serialized = self._raven().to_json() + parsed = ServerWideConnectionString.from_json(serialized) + + self.assertIsInstance(parsed.connection_string, RavenConnectionString) + self.assertEqual(serialized, parsed.to_json()) + + def test_every_connection_string_type_is_dispatched_by_its_type_field(self): + queue = ServerWideConnectionString( + connection_string=QueueConnectionString("shared-queue", QueueBrokerType.AZURE_SERVICE_BUS) + ) + parsed = ServerWideConnectionString.from_json(queue.to_json()) + + self.assertIsInstance(parsed.connection_string, QueueConnectionString) + self.assertEqual(QueueBrokerType.AZURE_SERVICE_BUS, parsed.connection_string.broker_type) + + def test_a_payload_without_a_type_cannot_be_parsed(self): + self.assertIsNone(ServerWideConnectionString.from_json({"Name": "x"})) + self.assertIsNone(ServerWideConnectionString.from_json(None)) + + def test_usages_carry_the_database_they_come_from(self): + parsed = ServerWideConnectionString.from_json( + { + "Name": "shared-raven", + "Database": "orders", + "TopologyDiscoveryUrls": [], + "Type": "Raven", + "UsedBy": [{"Kind": "RavenEtl", "Id": 3, "Name": "etl-1", "DatabaseName": "shop"}], + } + ) + + self.assertIsInstance(parsed.used_by[0], ServerWideConnectionStringUsage) + self.assertEqual("shop", parsed.used_by[0].database_name) + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, parsed.used_by[0].kind) + + def test_the_propagated_name_inside_a_database_record_is_prefixed(self): + self.assertEqual( + "Server Wide Connection String, shared-raven", + ServerWideConnectionString.get_database_record_connection_string_name("shared-raven"), + ) + + +class TestServerWideConnectionStringOperations(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + self.server_wide = ServerWideConnectionString( + connection_string=RavenConnectionString("shared-raven", "orders", ["http://localhost:8080"]) + ) + + def test_get_without_a_filter_asks_for_everything(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + request = command.create_request(self.node) + + self.assertEqual("GET", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + self.assertTrue(command.is_read_request()) + + def test_get_filters_by_name_and_type(self): + command = GetServerWideConnectionStringsOperation("shared raven", ConnectionStringType.RAVEN).get_command(None) + url = command.create_request(self.node).url + + self.assertIn("?name=shared%20raven", url) + self.assertIn("&type=Raven", url) + + def test_a_blank_name_is_rejected_client_side(self): + with self.assertRaises(ValueError): + GetServerWideConnectionStringsOperation(" ") + + def test_get_parses_the_results_list(self): + command = GetServerWideConnectionStringsOperation().get_command(None) + command.set_response( + json.dumps( + { + "Results": [ + { + "Name": "shared-raven", + "Database": "orders", + "TopologyDiscoveryUrls": [], + "Type": "Raven", + "ExcludedDatabases": ["scratch"], + } + ] + } + ), + False, + ) + + self.assertEqual(1, len(command.result.results)) + self.assertEqual("shared-raven", command.result.results[0].name) + self.assertEqual(["scratch"], command.result.results[0].excluded_databases) + + def test_put_sends_the_flattened_connection_string(self): + command = PutServerWideConnectionStringOperation(self.server_wide).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/admin/configuration/server-wide/connection-strings", request.url) + self.assertEqual(self.server_wide.to_json(), request.data) + self.assertIsInstance(command, RaftCommand) + + def test_put_reads_back_the_raft_index(self): + command = PutServerWideConnectionStringOperation(self.server_wide).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 11}), False) + + self.assertEqual(11, command.result.raft_command_index) + + def test_put_requires_a_wrapped_connection_string(self): + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + PutServerWideConnectionStringOperation(ServerWideConnectionString()) + + def test_remove_addresses_the_connection_string_by_name_and_type(self): + command = RemoveServerWideConnectionStringOperation(SqlConnectionString("shared sql")).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("DELETE", request.method) + self.assertIn("?name=shared%20sql", request.url) + self.assertIn("&type=Sql", request.url) + self.assertIsInstance(command, RaftCommand) + + def test_remove_requires_a_named_connection_string(self): + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(None) + with self.assertRaises(ValueError): + RemoveServerWideConnectionStringOperation(SqlConnectionString(None)) + + def test_remove_reads_back_the_raft_index(self): + command = RemoveServerWideConnectionStringOperation(SqlConnectionString("sql1")).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 12}), False) + + self.assertEqual(12, command.result.raft_command_index) diff --git a/ravendb/tests/session_tests/test_certificate_disabled_flag.py b/ravendb/tests/session_tests/test_certificate_disabled_flag.py index bfe74d5a..e22d54e9 100644 --- a/ravendb/tests/session_tests/test_certificate_disabled_flag.py +++ b/ravendb/tests/session_tests/test_certificate_disabled_flag.py @@ -5,6 +5,7 @@ import unittest +from ravendb.http.server_node import ServerNode from ravendb.serverwide.operations.certificates import ( CertificateDefinition, CertificateMetadata, @@ -82,10 +83,10 @@ def test_edit_operation_parameters_carries_disabled(self): disabled=True, ) op = EditClientCertificateOperation(params) - # The disabled flag flows through to the command and into the request body - # via the definition's to_json. Verify by reaching through the private - # field on the operation instance. - self.assertTrue(op._EditClientCertificateOperation__disabled) + request = op.get_command(None).create_request(ServerNode("http://localhost:8080", "db")) + + # The disabled flag flows through the command into the request body. + self.assertTrue(request.data["Disabled"]) if __name__ == "__main__": From 661f7153648e3d641d31ad068f524dae0fb45092 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:02 +0200 Subject: [PATCH 02/14] RDBC-1099 Add the Queue Sink client surface --- ravendb/documents/operations/ongoing_tasks.py | 77 +++++ .../operations/queue_sink/__init__.py | 114 ++++++++ .../azure_service_bus_sink_source.py | 42 +++ .../operations/queue_sink/configuration.py | 145 ++++++++++ .../tests/operations_tests/test_queue_sink.py | 272 ++++++++++++++++++ 5 files changed, 650 insertions(+) create mode 100644 ravendb/documents/operations/queue_sink/__init__.py create mode 100644 ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py create mode 100644 ravendb/documents/operations/queue_sink/configuration.py create mode 100644 ravendb/tests/operations_tests/test_queue_sink.py diff --git a/ravendb/documents/operations/ongoing_tasks.py b/ravendb/documents/operations/ongoing_tasks.py index cbbb28cf..e09648ca 100644 --- a/ravendb/documents/operations/ongoing_tasks.py +++ b/ravendb/documents/operations/ongoing_tasks.py @@ -18,6 +18,8 @@ if TYPE_CHECKING: from ravendb.documents.conventions import DocumentConventions from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration + from ravendb.documents.operations.etl.queue.connection import QueueBrokerType + from ravendb.documents.operations.queue_sink.configuration import QueueSinkConfiguration from ravendb.documents.operations.ai.gen_ai_configuration import GenAiConfiguration from ravendb.documents.operations.ai.embeddings_generation_configuration import EmbeddingsGenerationConfiguration @@ -472,6 +474,79 @@ def from_json(cls, json_dict: dict) -> Optional["OngoingTaskPullReplicationAsSin ) +class OngoingTaskQueueSink(OngoingTask): + """Ongoing task information for a queue sink task.""" + + def __init__( + self, + task_id: Optional[int] = None, + responsible_node: Optional[NodeId] = None, + task_state: Optional[OngoingTaskState] = None, + task_connection_status: Optional[OngoingTaskConnectionStatus] = None, + task_name: Optional[str] = None, + error: Optional[str] = None, + mentor_node: Optional[str] = None, + pin_to_mentor_node: Optional[bool] = None, + configuration: Optional["QueueSinkConfiguration"] = None, + broker_type: Optional["QueueBrokerType"] = None, + connection_string_name: Optional[str] = None, + url: Optional[str] = None, + ): + super().__init__( + task_id=task_id, + task_type=OngoingTaskType.QUEUE_SINK, + responsible_node=responsible_node, + task_state=task_state, + task_connection_status=task_connection_status, + task_name=task_name, + error=error, + mentor_node=mentor_node, + pin_to_mentor_node=pin_to_mentor_node, + ) + self.configuration = configuration + self.broker_type = broker_type + self.connection_string_name = connection_string_name + # The broker URL the server resolved from the connection string. + self.url = url + + def to_json(self) -> dict: + result = super().to_json() + result["BrokerType"] = self.broker_type.value if self.broker_type else None + result["ConnectionStringName"] = self.connection_string_name + result["Url"] = self.url + result["Configuration"] = self.configuration.to_json() if self.configuration else None + return result + + @classmethod + def from_json(cls, json_dict: dict) -> Optional["OngoingTaskQueueSink"]: + if json_dict is None: + return None + + from ravendb.documents.operations.etl.queue.connection import QueueBrokerType + from ravendb.documents.operations.queue_sink.configuration import QueueSinkConfiguration + + task_state_str = json_dict.get("TaskState") + task_connection_status_str = json_dict.get("TaskConnectionStatus") + broker_type_str = json_dict.get("BrokerType") + configuration = json_dict.get("Configuration") + return cls( + task_id=json_dict.get("TaskId"), + responsible_node=NodeId.from_json(json_dict.get("ResponsibleNode")), + task_state=OngoingTaskState(task_state_str) if task_state_str else None, + task_connection_status=( + OngoingTaskConnectionStatus(task_connection_status_str) if task_connection_status_str else None + ), + task_name=json_dict.get("TaskName"), + error=json_dict.get("Error"), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode"), + configuration=QueueSinkConfiguration.from_json(configuration) if configuration else None, + broker_type=QueueBrokerType(broker_type_str) if broker_type_str else None, + connection_string_name=json_dict.get("ConnectionStringName"), + url=json_dict.get("Url"), + ) + + class OngoingTaskCdcSink(OngoingTask): """Ongoing task information for a CDC Sink task.""" @@ -748,6 +823,8 @@ def _deserialize_task( return OngoingTaskPullReplicationAsSink.from_json(json_dict) elif self._task_type == OngoingTaskType.CDC_SINK: return OngoingTaskCdcSink.from_json(json_dict) + elif self._task_type == OngoingTaskType.QUEUE_SINK: + return OngoingTaskQueueSink.from_json(json_dict) else: # todo: handle more types of tasks return OngoingTask.from_json(json_dict) diff --git a/ravendb/documents/operations/queue_sink/__init__.py b/ravendb/documents/operations/queue_sink/__init__.py new file mode 100644 index 00000000..8482a68a --- /dev/null +++ b/ravendb/documents/operations/queue_sink/__init__.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import requests + +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.operations.queue_sink.azure_service_bus_sink_source import AzureServiceBusSinkSource +from ravendb.documents.operations.queue_sink.configuration import ( + AddQueueSinkOperationResult, + QueueSinkConfiguration, + QueueSinkProcessState, + QueueSinkScript, + UpdateQueueSinkOperationResult, +) +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class AddQueueSinkOperation(MaintenanceOperation[AddQueueSinkOperationResult]): + """ + Adds a queue sink task, which has RavenDB consume messages from an external broker + such as Kafka or RabbitMQ and store them in the database. + """ + + def __init__(self, configuration: QueueSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[AddQueueSinkOperationResult]: + return self._AddQueueSinkCommand(self._configuration) + + class _AddQueueSinkCommand(RavenCommand[AddQueueSinkOperationResult], RaftCommand): + def __init__(self, configuration: QueueSinkConfiguration): + super().__init__(AddQueueSinkOperationResult) + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/queue-sink" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = AddQueueSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class UpdateQueueSinkOperation(MaintenanceOperation[UpdateQueueSinkOperationResult]): + """Updates an existing queue sink task.""" + + def __init__(self, task_id: int, configuration: QueueSinkConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + + self._task_id = task_id + self._configuration = configuration + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[UpdateQueueSinkOperationResult]: + return self._UpdateQueueSinkCommand(self._task_id, self._configuration) + + class _UpdateQueueSinkCommand(RavenCommand[UpdateQueueSinkOperationResult], RaftCommand): + def __init__(self, task_id: int, configuration: QueueSinkConfiguration): + super().__init__(UpdateQueueSinkOperationResult) + self._task_id = task_id + self._configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/queue-sink?id={self._task_id}" + + request = requests.Request("PUT", url) + request.data = self._configuration.to_json() + return request + + def set_response(self, response: str, from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = UpdateQueueSinkOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +__all__ = [ + "AddQueueSinkOperation", + "AddQueueSinkOperationResult", + "AzureServiceBusSinkSource", + "QueueSinkConfiguration", + "QueueSinkProcessState", + "QueueSinkScript", + "UpdateQueueSinkOperation", + "UpdateQueueSinkOperationResult", +] diff --git a/ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py b/ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py new file mode 100644 index 00000000..fd9844af --- /dev/null +++ b/ravendb/documents/operations/queue_sink/azure_service_bus_sink_source.py @@ -0,0 +1,42 @@ +from __future__ import annotations + + +class AzureServiceBusSinkSource: + """ + Builds the strings that go into QueueSinkScript.queues for an Azure Service Bus sink. + + A queue is its own name; a topic subscription is encoded as "topic;subscription". + Service Bus naming rules forbid ';' in queue, topic and subscription names, so the + separator cannot collide with a real name. + """ + + SEPARATOR = ";" + + @staticmethod + def queue(queue_name: str) -> str: + """The entry for a Service Bus queue. A pass-through, kept for symmetry with subscription().""" + if not queue_name or queue_name.isspace(): + raise ValueError("Queue name must be non-empty.") + + if AzureServiceBusSinkSource.SEPARATOR in queue_name: + raise ValueError(f"Queue name must not contain the '{AzureServiceBusSinkSource.SEPARATOR}' character.") + + return queue_name + + @staticmethod + def subscription(topic_name: str, subscription_name: str) -> str: + """The entry for a topic subscription, in the form 'topic;subscription'.""" + if not topic_name or topic_name.isspace(): + raise ValueError("Topic name must be non-empty.") + + if not subscription_name or subscription_name.isspace(): + raise ValueError("Subscription name must be non-empty.") + + separator = AzureServiceBusSinkSource.SEPARATOR + if separator in topic_name: + raise ValueError(f"Topic name must not contain the '{separator}' character.") + + if separator in subscription_name: + raise ValueError(f"Subscription name must not contain the '{separator}' character.") + + return f"{topic_name}{separator}{subscription_name}" diff --git a/ravendb/documents/operations/queue_sink/configuration.py b/ravendb/documents/operations/queue_sink/configuration.py new file mode 100644 index 00000000..eb261fbe --- /dev/null +++ b/ravendb/documents/operations/queue_sink/configuration.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType + + +class QueueSinkScript: + """ + A user-defined script that consumes messages from one or more queues and decides + how they are stored in RavenDB. + """ + + def __init__( + self, + name: str = None, + queues: List[str] = None, + script: str = None, + disabled: bool = False, + ): + self.name = name + # Broker-specific source entries. For Azure Service Bus these are built with + # AzureServiceBusSinkSource.queue() / .subscription(). + self.queues = queues or [] + self.script = script + self.disabled = disabled + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "Script": self.script, + "Queues": self.queues, + "Disabled": self.disabled, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> QueueSinkScript: + return cls( + name=json_dict.get("Name"), + queues=json_dict.get("Queues"), + script=json_dict.get("Script"), + disabled=json_dict.get("Disabled", False), + ) + + +class QueueSinkConfiguration: + """ + A queue sink task: RavenDB consumes messages from an external broker (Kafka, + RabbitMQ, Azure Queue Storage, Amazon SQS, Azure Service Bus) and stores them. + """ + + def __init__( + self, + name: str = None, + broker_type: QueueBrokerType = None, + connection_string_name: str = None, + scripts: List[QueueSinkScript] = None, + task_id: int = 0, + disabled: bool = False, + mentor_node: str = None, + pin_to_mentor_node: bool = False, + ): + self.name = name + self.broker_type = broker_type + self.connection_string_name = connection_string_name + self.scripts = scripts or [] + self.task_id = task_id + self.disabled = disabled + self.mentor_node = mentor_node + self.pin_to_mentor_node = pin_to_mentor_node + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "TaskId": self.task_id, + "Disabled": self.disabled, + "ConnectionStringName": self.connection_string_name, + "MentorNode": self.mentor_node, + "PinToMentorNode": self.pin_to_mentor_node, + "Scripts": [script.to_json() for script in self.scripts], + "BrokerType": self.broker_type.value if self.broker_type else None, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> QueueSinkConfiguration: + broker_type = json_dict.get("BrokerType") + return cls( + name=json_dict.get("Name"), + broker_type=QueueBrokerType(broker_type) if broker_type else None, + connection_string_name=json_dict.get("ConnectionStringName"), + scripts=[QueueSinkScript.from_json(script) for script in json_dict.get("Scripts") or []], + task_id=json_dict.get("TaskId", 0), + disabled=json_dict.get("Disabled", False), + mentor_node=json_dict.get("MentorNode"), + pin_to_mentor_node=json_dict.get("PinToMentorNode", False), + ) + + +class QueueSinkProcessState: + """Which node a queue sink script last ran on.""" + + def __init__(self, node_tag: str = None, configuration_name: str = None, script_name: str = None): + self.node_tag = node_tag + self.configuration_name = configuration_name + self.script_name = script_name + + def to_json(self) -> Dict[str, Any]: + return { + "ConfigurationName": self.configuration_name, + "ScriptName": self.script_name, + "NodeTag": self.node_tag, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> QueueSinkProcessState: + return cls( + node_tag=json_dict.get("NodeTag"), + configuration_name=json_dict.get("ConfigurationName"), + script_name=json_dict.get("ScriptName"), + ) + + @staticmethod + def generate_item_name(database_name: str, configuration_name: str, transformation_name: str) -> str: + return f"values/{database_name}/queuesink/{configuration_name.lower()}/{transformation_name.lower()}" + + +class AddQueueSinkOperationResult: + def __init__(self, raft_command_index: Optional[int] = None, task_id: Optional[int] = None): + self.raft_command_index = raft_command_index + self.task_id = task_id + + def to_json(self) -> Dict[str, Any]: + return {"RaftCommandIndex": self.raft_command_index, "TaskId": self.task_id} + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> AddQueueSinkOperationResult: + return cls( + raft_command_index=json_dict.get("RaftCommandIndex"), + task_id=json_dict.get("TaskId"), + ) + + +class UpdateQueueSinkOperationResult(AddQueueSinkOperationResult): + # Same shape as the add result; kept as its own name so callers read what they got back. + pass diff --git a/ravendb/tests/operations_tests/test_queue_sink.py b/ravendb/tests/operations_tests/test_queue_sink.py new file mode 100644 index 00000000..71f1d63f --- /dev/null +++ b/ravendb/tests/operations_tests/test_queue_sink.py @@ -0,0 +1,272 @@ +""" +Tests for the Queue Sink client surface: the configuration, the Add/Update operations, +the Azure Service Bus source encoding, and the QueueSink ongoing task. +""" + +import json +import unittest + +from ravendb.documents.operations.etl.queue.connection import QueueBrokerType +from ravendb.documents.operations.ongoing_tasks import ( + GetOngoingTaskInfoOperation, + OngoingTaskQueueSink, + OngoingTaskType, +) +from ravendb.documents.operations.queue_sink import ( + AddQueueSinkOperation, + AddQueueSinkOperationResult, + AzureServiceBusSinkSource, + QueueSinkConfiguration, + QueueSinkProcessState, + QueueSinkScript, + UpdateQueueSinkOperation, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.etl.queue.connection import QueueConnectionString +from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings +from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.serverwide.database_record import DatabaseRecord +from ravendb.tests.test_base import TestBase + + +def _configuration() -> QueueSinkConfiguration: + return QueueSinkConfiguration( + name="orders-sink", + broker_type=QueueBrokerType.KAFKA, + connection_string_name="kafka", + scripts=[ + QueueSinkScript( + name="orders", + queues=["orders-topic"], + script="put('orders/', this);", + ) + ], + ) + + +class TestQueueSinkConfiguration(unittest.TestCase): + def test_configuration_survives_a_json_round_trip(self): + serialized = _configuration().to_json() + + self.assertEqual(serialized, QueueSinkConfiguration.from_json(serialized).to_json()) + + def test_configuration_serializes_the_task_level_fields(self): + serialized = _configuration().to_json() + + self.assertEqual("orders-sink", serialized["Name"]) + self.assertEqual("Kafka", serialized["BrokerType"]) + self.assertEqual("kafka", serialized["ConnectionStringName"]) + self.assertEqual(0, serialized["TaskId"]) + self.assertFalse(serialized["Disabled"]) + + def test_a_script_carries_its_queues_and_disabled_flag(self): + # The wire format comes from reflection over the public properties, so Disabled + # ships even though the server-side ToJson leaves it out. + script = QueueSinkScript("s1", ["q1", "q2"], "put('x/', this);", disabled=True) + serialized = script.to_json() + + self.assertEqual(["q1", "q2"], serialized["Queues"]) + self.assertTrue(serialized["Disabled"]) + self.assertEqual(serialized, QueueSinkScript.from_json(serialized).to_json()) + + def test_collections_default_to_empty_rather_than_none(self): + self.assertEqual([], QueueSinkConfiguration().scripts) + self.assertEqual([], QueueSinkScript().queues) + + def test_a_configuration_from_an_older_server_parses(self): + configuration = QueueSinkConfiguration.from_json({"Name": "sink", "BrokerType": "RabbitMq"}) + + self.assertEqual(QueueBrokerType.RABBIT_MQ, configuration.broker_type) + self.assertEqual([], configuration.scripts) + + def test_process_state_item_name_is_lower_cased(self): + # The server stores the cluster value under a lower-cased key. + self.assertEqual( + "values/DB/queuesink/orders-sink/orders", + QueueSinkProcessState.generate_item_name("DB", "Orders-Sink", "Orders"), + ) + + def test_process_state_round_trips(self): + payload = {"ConfigurationName": "orders-sink", "ScriptName": "orders", "NodeTag": "A"} + + self.assertEqual(payload, QueueSinkProcessState.from_json(payload).to_json()) + + +class TestAzureServiceBusSinkSource(unittest.TestCase): + def test_a_queue_entry_is_the_queue_name(self): + self.assertEqual("orders", AzureServiceBusSinkSource.queue("orders")) + + def test_a_subscription_entry_joins_topic_and_subscription(self): + self.assertEqual("events;audit", AzureServiceBusSinkSource.subscription("events", "audit")) + + def test_an_empty_name_is_rejected(self): + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.queue("") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.queue(" ") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("events", "") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("", "audit") + + def test_the_separator_cannot_appear_inside_a_name(self): + # Service Bus forbids ';' in names, so an entry carrying one could not be decoded. + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.queue("a;b") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("a;b", "audit") + with self.assertRaises(ValueError): + AzureServiceBusSinkSource.subscription("events", "a;b") + + +class TestQueueSinkOperations(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def test_add_sends_the_configuration_to_the_admin_endpoint(self): + configuration = _configuration() + command = AddQueueSinkOperation(configuration).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("PUT", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/queue-sink", request.url) + self.assertEqual(configuration.to_json(), request.data) + self.assertFalse(command.is_read_request()) + + def test_update_addresses_the_task_by_id(self): + command = UpdateQueueSinkOperation(11, _configuration()).get_command(None) + + self.assertEqual( + "http://localhost:8080/databases/db/admin/queue-sink?id=11", + command.create_request(self.node).url, + ) + + def test_both_operations_are_raft_commands(self): + add = AddQueueSinkOperation(_configuration()).get_command(None) + update = UpdateQueueSinkOperation(1, _configuration()).get_command(None) + + self.assertIsInstance(add, RaftCommand) + self.assertIsInstance(update, RaftCommand) + self.assertTrue(add.get_raft_unique_request_id()) + + def test_a_missing_configuration_is_rejected_client_side(self): + with self.assertRaises(ValueError): + AddQueueSinkOperation(None) + with self.assertRaises(ValueError): + UpdateQueueSinkOperation(1, None) + + def test_add_reads_the_task_id_off_the_response(self): + command = AddQueueSinkOperation(_configuration()).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 9, "TaskId": 4}), False) + + self.assertIsInstance(command.result, AddQueueSinkOperationResult) + self.assertEqual(9, command.result.raft_command_index) + self.assertEqual(4, command.result.task_id) + + +class TestQueueSinkOngoingTask(unittest.TestCase): + RESPONSE = { + "TaskId": 4, + "TaskType": "QueueSink", + "TaskName": "orders-sink", + "TaskState": "Enabled", + "ResponsibleNode": {"NodeTag": "A", "NodeUrl": "http://localhost:8080"}, + "BrokerType": "Kafka", + "ConnectionStringName": "kafka", + "Url": "localhost:9092", + "Configuration": {"Name": "orders-sink", "BrokerType": "Kafka", "Scripts": [{"Name": "orders"}]}, + } + + def test_ongoing_task_carries_the_broker_and_url(self): + task = OngoingTaskQueueSink.from_json(self.RESPONSE) + + self.assertEqual(OngoingTaskType.QUEUE_SINK, task.task_type) + self.assertEqual(QueueBrokerType.KAFKA, task.broker_type) + self.assertEqual("localhost:9092", task.url) + self.assertEqual("kafka", task.connection_string_name) + + def test_ongoing_task_parses_the_nested_configuration(self): + task = OngoingTaskQueueSink.from_json(self.RESPONSE) + + self.assertIsInstance(task.configuration, QueueSinkConfiguration) + self.assertEqual("orders", task.configuration.scripts[0].name) + + def test_get_ongoing_task_info_dispatches_to_the_queue_sink_result(self): + command = GetOngoingTaskInfoOperation("orders-sink", OngoingTaskType.QUEUE_SINK).get_command(None) + request = command.create_request(ServerNode("http://localhost:8080", "db")) + command.set_response(json.dumps(self.RESPONSE), False) + + self.assertIn("type=QueueSink", request.url) + self.assertIsInstance(command.result, OngoingTaskQueueSink) + self.assertEqual("orders-sink", command.result.task_name) + + +class TestQueueSinkInDatabaseRecord(unittest.TestCase): + def test_database_record_carries_queue_sinks(self): + record = DatabaseRecord("db") + record.queue_sinks = [_configuration()] + + self.assertEqual("orders-sink", record.to_json()["QueueSinks"][0]["Name"]) + + def test_database_record_parses_queue_sinks(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "QueueSinks": [_configuration().to_json()]}) + + self.assertEqual(1, len(record.queue_sinks)) + self.assertEqual(QueueBrokerType.KAFKA, record.queue_sinks[0].broker_type) + + def test_a_record_from_a_server_without_queue_sinks_gets_an_empty_list(self): + self.assertEqual([], DatabaseRecord.from_json({"DatabaseName": "db"}).queue_sinks) + + +class TestQueueSinkAgainstServer(TestBase): + def setUp(self): + super().setUp() + self.store.maintenance.send( + PutConnectionStringOperation( + QueueConnectionString( + name="kafka", + broker_type=QueueBrokerType.KAFKA, + kafka_settings=KafkaConnectionSettings(bootstrap_servers="localhost:9092"), + ) + ) + ) + + def test_the_server_stores_a_queue_sink_task_and_reads_it_back(self): + result = self.store.maintenance.send(AddQueueSinkOperation(_configuration())) + self.assertGreater(result.task_id, 0) + self.assertGreater(result.raft_command_index, 0) + + task = self.store.maintenance.send(GetOngoingTaskInfoOperation("orders-sink", OngoingTaskType.QUEUE_SINK)) + self.assertIsInstance(task, OngoingTaskQueueSink) + self.assertEqual(QueueBrokerType.KAFKA, task.broker_type) + # The server resolves the broker URL from the connection string. + self.assertEqual("localhost:9092", task.url) + self.assertEqual(["orders-topic"], task.configuration.scripts[0].queues) + self.assertEqual("put('orders/', this);", task.configuration.scripts[0].script) + + def test_the_task_can_be_updated(self): + task_id = self.store.maintenance.send(AddQueueSinkOperation(_configuration())).task_id + + updated = _configuration() + updated.task_id = task_id + updated.scripts[0].queues = ["orders-topic", "returns-topic"] + self.store.maintenance.send(UpdateQueueSinkOperation(task_id, updated)) + + task = self.store.maintenance.send(GetOngoingTaskInfoOperation("orders-sink", OngoingTaskType.QUEUE_SINK)) + self.assertEqual(["orders-topic", "returns-topic"], task.configuration.scripts[0].queues) + + def test_a_configuration_with_no_scripts_is_refused_by_the_server(self): + empty = QueueSinkConfiguration( + name="empty-sink", broker_type=QueueBrokerType.KAFKA, connection_string_name="kafka" + ) + + self.assertRaisesWithMessageContaining( + self.store.maintenance.send, + RavenException, + "'Scripts' list cannot be empty", + AddQueueSinkOperation(empty), + ) From d2fd2d2cd526e8ca9a8f8df87f1bb030a3b358e3 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:02 +0200 Subject: [PATCH 03/14] RDBC-1099 Add the database smuggler, reachable as store.smuggler --- ravendb/documents/operations/operation.py | 5 +- ravendb/documents/smuggler/common.py | 290 ++++++++++++ .../documents/smuggler/database_smuggler.py | 293 ++++++++++++ ravendb/documents/smuggler/result.py | 421 +++++++++++++++++ ravendb/documents/store/definition.py | 14 +- .../tests/documents_tests/test_smuggler.py | 424 ++++++++++++++++++ 6 files changed, 1444 insertions(+), 3 deletions(-) create mode 100644 ravendb/documents/smuggler/database_smuggler.py create mode 100644 ravendb/documents/smuggler/result.py create mode 100644 ravendb/tests/documents_tests/test_smuggler.py diff --git a/ravendb/documents/operations/operation.py b/ravendb/documents/operations/operation.py index 868198d9..18273ceb 100644 --- a/ravendb/documents/operations/operation.py +++ b/ravendb/documents/operations/operation.py @@ -40,13 +40,14 @@ def _get_operation_state_command( ) -> RavenCommand[dict]: return GetOperationStateOperation.GetOperationStateCommand(self.__key, node_tag) - def wait_for_completion(self) -> None: + def wait_for_completion(self) -> Optional[dict]: + """Blocks until the operation finishes, then hands back whatever result it carried.""" while True: status = self.fetch_operations_status() operation_status = status.get("Status") if operation_status == "Completed": - return + return status.get("Result") elif operation_status == "Canceled": raise OperationCancelledException() elif operation_status == "Faulted": diff --git a/ravendb/documents/smuggler/common.py b/ravendb/documents/smuggler/common.py index aa9cf44d..e9189b32 100644 --- a/ravendb/documents/smuggler/common.py +++ b/ravendb/documents/smuggler/common.py @@ -1,7 +1,15 @@ +from __future__ import annotations + import enum +from typing import Any, Dict, Iterable, List, Optional, Set, Type, TypeVar class DatabaseItemType(enum.Enum): + """ + What an export or import operates on. The server takes a combination of these, so + the smuggler options carry them as a set rather than a single value. + """ + NONE = "None" DOCUMENTS = "Documents" REVISION_DOCUMENTS = "RevisionDocuments" @@ -15,6 +23,7 @@ class DatabaseItemType(enum.Enum): LEGACY_ATTACHMENT_DELETIONS = "LegacyAttachmentDeletions" DATABASE_RECORD = "DatabaseRecord" UNKNOWN = "Unknown" + # Kept for callers written against older servers; 7.x replaced it with COUNTER_GROUPS. COUNTERS = "Counters" ATTACHMENTS = "Attachments" COUNTER_GROUPS = "CounterGroups" @@ -22,3 +31,284 @@ class DatabaseItemType(enum.Enum): COMPARE_EXCHANGE_TOMBSTONES = "CompareExchangeTombstones" TIME_SERIES = "TimeSeries" REPLICATION_HUB_CERTIFICATES = "ReplicationHubCertificates" + TIME_SERIES_DELETED_RANGES = "TimeSeriesDeletedRanges" + + def __str__(self) -> str: + return self.value + + +class DatabaseRecordItemType(enum.Enum): + """Which parts of the database record an export or import operates on.""" + + NONE = "None" + CONFLICT_SOLVER_CONFIG = "ConflictSolverConfig" + SETTINGS = "Settings" + REVISIONS = "Revisions" + EXPIRATION = "Expiration" + PERIODIC_BACKUPS = "PeriodicBackups" + EXTERNAL_REPLICATIONS = "ExternalReplications" + RAVEN_CONNECTION_STRINGS = "RavenConnectionStrings" + SQL_CONNECTION_STRINGS = "SqlConnectionStrings" + RAVEN_ETLS = "RavenEtls" + SQL_ETLS = "SqlEtls" + CLIENT = "Client" + SORTERS = "Sorters" + SINK_PULL_REPLICATIONS = "SinkPullReplications" + HUB_PULL_REPLICATIONS = "HubPullReplications" + TIME_SERIES = "TimeSeries" + DOCUMENTS_COMPRESSION = "DocumentsCompression" + ANALYZERS = "Analyzers" + LOCK_MODE = "LockMode" + OLAP_CONNECTION_STRINGS = "OlapConnectionStrings" + OLAP_ETLS = "OlapEtls" + ELASTIC_SEARCH_CONNECTION_STRINGS = "ElasticSearchConnectionStrings" + ELASTIC_SEARCH_ETLS = "ElasticSearchEtls" + POSTGRE_SQL_INTEGRATION = "PostgreSQLIntegration" + QUEUE_CONNECTION_STRINGS = "QueueConnectionStrings" + QUEUE_ETLS = "QueueEtls" + INDEXES_HISTORY = "IndexesHistory" + REFRESH = "Refresh" + QUEUE_SINKS = "QueueSinks" + DATA_ARCHIVAL = "DataArchival" + SNOWFLAKE_CONNECTION_STRINGS = "SnowflakeConnectionStrings" + SNOWFLAKE_ETLS = "SnowflakeEtls" + EMBEDDINGS_GENERATIONS = "EmbeddingsGenerations" + AI_CONNECTION_STRINGS = "AiConnectionStrings" + GEN_AI_ETLS = "GenAiEtls" + AI_AGENTS = "AiAgents" + REMOTE_ATTACHMENTS = "RemoteAttachments" + SCHEMA_VALIDATION = "SchemaValidation" + CDC_SINKS = "CdcSinks" + + def __str__(self) -> str: + return self.value + + +class ExportCompressionAlgorithm(enum.Enum): + ZSTD = "Zstd" + GZIP = "Gzip" + + def __str__(self) -> str: + return self.value + + +_TFlag = TypeVar("_TFlag", DatabaseItemType, DatabaseRecordItemType) + + +def flags_to_string(members: Optional[Iterable[_TFlag]], enum_type: Type[_TFlag]) -> str: + """ + Renders a set of members the way the server reads a [Flags] enum: a comma-separated + list of names, or "None" when nothing is selected. + """ + selected = set(members or ()) + selected.discard(enum_type.NONE) + if not selected: + return enum_type.NONE.value + + # Declaration order keeps the output stable and matches the bit order the server uses. + return ", ".join(member.value for member in enum_type if member in selected) + + +def flags_from_string(value: Optional[str], enum_type: Type[_TFlag]) -> Set[_TFlag]: + """The inverse of flags_to_string.""" + if not value: + return set() + + members = {enum_type(part.strip()) for part in value.split(",") if part.strip()} + members.discard(enum_type.NONE) + return members + + +DEFAULT_OPERATE_ON_TYPES: Set[DatabaseItemType] = { + DatabaseItemType.INDEXES, + DatabaseItemType.DOCUMENTS, + DatabaseItemType.REVISION_DOCUMENTS, + DatabaseItemType.CONFLICTS, + DatabaseItemType.DATABASE_RECORD, + DatabaseItemType.REPLICATION_HUB_CERTIFICATES, + DatabaseItemType.IDENTITIES, + DatabaseItemType.COMPARE_EXCHANGE, + DatabaseItemType.ATTACHMENTS, + DatabaseItemType.COUNTER_GROUPS, + DatabaseItemType.SUBSCRIPTIONS, + DatabaseItemType.TIME_SERIES, + DatabaseItemType.TIME_SERIES_DELETED_RANGES, +} + +DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES: Set[DatabaseRecordItemType] = { + DatabaseRecordItemType.CLIENT, + DatabaseRecordItemType.CONFLICT_SOLVER_CONFIG, + DatabaseRecordItemType.EXPIRATION, + DatabaseRecordItemType.EXTERNAL_REPLICATIONS, + DatabaseRecordItemType.PERIODIC_BACKUPS, + DatabaseRecordItemType.RAVEN_CONNECTION_STRINGS, + DatabaseRecordItemType.RAVEN_ETLS, + DatabaseRecordItemType.REVISIONS, + DatabaseRecordItemType.SETTINGS, + DatabaseRecordItemType.SQL_CONNECTION_STRINGS, + DatabaseRecordItemType.SORTERS, + DatabaseRecordItemType.SQL_ETLS, + DatabaseRecordItemType.HUB_PULL_REPLICATIONS, + DatabaseRecordItemType.SINK_PULL_REPLICATIONS, + DatabaseRecordItemType.TIME_SERIES, + DatabaseRecordItemType.DOCUMENTS_COMPRESSION, + DatabaseRecordItemType.ANALYZERS, + DatabaseRecordItemType.LOCK_MODE, + DatabaseRecordItemType.OLAP_CONNECTION_STRINGS, + DatabaseRecordItemType.OLAP_ETLS, + DatabaseRecordItemType.ELASTIC_SEARCH_CONNECTION_STRINGS, + DatabaseRecordItemType.ELASTIC_SEARCH_ETLS, + DatabaseRecordItemType.POSTGRE_SQL_INTEGRATION, + DatabaseRecordItemType.QUEUE_CONNECTION_STRINGS, + DatabaseRecordItemType.QUEUE_ETLS, + DatabaseRecordItemType.INDEXES_HISTORY, + DatabaseRecordItemType.REFRESH, + DatabaseRecordItemType.DATA_ARCHIVAL, + DatabaseRecordItemType.QUEUE_SINKS, + DatabaseRecordItemType.SNOWFLAKE_ETLS, + DatabaseRecordItemType.SNOWFLAKE_CONNECTION_STRINGS, + DatabaseRecordItemType.EMBEDDINGS_GENERATIONS, + DatabaseRecordItemType.AI_CONNECTION_STRINGS, + DatabaseRecordItemType.GEN_AI_ETLS, + DatabaseRecordItemType.AI_AGENTS, + DatabaseRecordItemType.REMOTE_ATTACHMENTS, + DatabaseRecordItemType.SCHEMA_VALIDATION, + DatabaseRecordItemType.CDC_SINKS, +} + +DEFAULT_MAX_STEPS_FOR_TRANSFORM_SCRIPT = 10 * 1000 + + +class DatabaseSmugglerOptions: + """What an export or import covers, and how it transforms what it moves.""" + + def __init__( + self, + operate_on_types: Set[DatabaseItemType] = None, + operate_on_database_record_types: Set[DatabaseRecordItemType] = None, + include_expired: bool = True, + include_artificial: bool = False, + include_archived: bool = True, + remove_analyzers: bool = False, + transform_script: str = None, + max_steps_for_transform_script: int = DEFAULT_MAX_STEPS_FOR_TRANSFORM_SCRIPT, + encryption_key: str = None, + collections: List[str] = None, + max_read_ops_per_second: Optional[int] = None, + skip_corrupted_data: bool = False, + ): + self.operate_on_types = set(operate_on_types if operate_on_types is not None else DEFAULT_OPERATE_ON_TYPES) + self.operate_on_database_record_types = set( + operate_on_database_record_types + if operate_on_database_record_types is not None + else DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES + ) + self.include_expired = include_expired + self.include_artificial = include_artificial + self.include_archived = include_archived + self.remove_analyzers = remove_analyzers + self.transform_script = transform_script + self.max_steps_for_transform_script = max_steps_for_transform_script + self.encryption_key = encryption_key + # Empty means every collection. + self.collections = collections if collections is not None else [] + self.max_read_ops_per_second = max_read_ops_per_second + # Lets an export continue past corrupted data, reporting it in the result instead + # of stopping. Useful when the database has lost its compression dictionaries. + self.skip_corrupted_data = skip_corrupted_data + + def to_json(self) -> Dict[str, Any]: + return { + "OperateOnTypes": flags_to_string(self.operate_on_types, DatabaseItemType), + "OperateOnDatabaseRecordTypes": flags_to_string( + self.operate_on_database_record_types, DatabaseRecordItemType + ), + "IncludeExpired": self.include_expired, + "IncludeArtificial": self.include_artificial, + "IncludeArchived": self.include_archived, + "RemoveAnalyzers": self.remove_analyzers, + "TransformScript": self.transform_script, + "MaxStepsForTransformScript": self.max_steps_for_transform_script, + "EncryptionKey": self.encryption_key, + "Collections": self.collections, + "MaxReadOpsPerSecond": self.max_read_ops_per_second, + "SkipCorruptedData": self.skip_corrupted_data, + } + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + self.operate_on_types = flags_from_string(json_dict.get("OperateOnTypes"), DatabaseItemType) + self.operate_on_database_record_types = flags_from_string( + json_dict.get("OperateOnDatabaseRecordTypes"), DatabaseRecordItemType + ) + self.include_expired = json_dict.get("IncludeExpired", True) + self.include_artificial = json_dict.get("IncludeArtificial", False) + self.include_archived = json_dict.get("IncludeArchived", True) + self.remove_analyzers = json_dict.get("RemoveAnalyzers", False) + self.transform_script = json_dict.get("TransformScript") + self.max_steps_for_transform_script = json_dict.get( + "MaxStepsForTransformScript", DEFAULT_MAX_STEPS_FOR_TRANSFORM_SCRIPT + ) + self.encryption_key = json_dict.get("EncryptionKey") + self.collections = json_dict.get("Collections") or [] + self.max_read_ops_per_second = json_dict.get("MaxReadOpsPerSecond") + self.skip_corrupted_data = json_dict.get("SkipCorruptedData", False) + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> DatabaseSmugglerOptions: + options = cls() + options._fill_from_json(json_dict) + return options + + +class DatabaseSmugglerExportOptions(DatabaseSmugglerOptions): + def __init__(self, compression_algorithm: ExportCompressionAlgorithm = None, **kwargs): + super().__init__(**kwargs) + # None leaves the choice to the server's own default. + self.compression_algorithm = compression_algorithm + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + if self.compression_algorithm is not None: + json_dict["CompressionAlgorithm"] = self.compression_algorithm.value + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> DatabaseSmugglerExportOptions: + options = cls() + options._fill_from_json(json_dict) + compression_algorithm = json_dict.get("CompressionAlgorithm") + options.compression_algorithm = ( + ExportCompressionAlgorithm(compression_algorithm) if compression_algorithm else None + ) + return options + + +class DatabaseSmugglerImportOptions(DatabaseSmugglerOptions): + def __init__(self, skip_revision_creation: bool = False, **kwargs): + super().__init__(**kwargs) + self.skip_revision_creation = skip_revision_creation + + @classmethod + def from_options(cls, options: DatabaseSmugglerOptions) -> DatabaseSmugglerImportOptions: + """Carries the shared options over from an export, the way export-to-database does.""" + return cls( + operate_on_types=set(options.operate_on_types), + include_expired=options.include_expired, + include_artificial=options.include_artificial, + include_archived=options.include_archived, + max_steps_for_transform_script=options.max_steps_for_transform_script, + remove_analyzers=options.remove_analyzers, + transform_script=options.transform_script, + ) + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["SkipRevisionCreation"] = self.skip_revision_creation + return json_dict + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> DatabaseSmugglerImportOptions: + options = cls() + options._fill_from_json(json_dict) + options.skip_revision_creation = json_dict.get("SkipRevisionCreation", False) + return options diff --git a/ravendb/documents/smuggler/database_smuggler.py b/ravendb/documents/smuggler/database_smuggler.py new file mode 100644 index 00000000..7e37f061 --- /dev/null +++ b/ravendb/documents/smuggler/database_smuggler.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import json +import os +import tempfile +from typing import IO, TYPE_CHECKING, List, Optional, Union + +import requests + +from ravendb.documents.commands.bulkinsert import GetNextOperationIdCommand +from ravendb.documents.operations.operation import Operation +from ravendb.documents.smuggler.common import ( + DatabaseItemType, + DatabaseSmugglerExportOptions, + DatabaseSmugglerImportOptions, + DatabaseSmugglerOptions, +) +from ravendb.documents.smuggler.result import SmugglerResult +from ravendb.http.misc import ResponseDisposeHandling +from ravendb.http.raven_command import VoidRavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.primitives import constants +from ravendb.util.request_utils import RequestUtils + +if TYPE_CHECKING: + from ravendb.documents.store.definition import DocumentStore + +_IMPORT_OPTIONS_PART = "importOptions" +_COPY_BUFFER_SIZE = 8192 + +_PERIODIC_BACKUP = constants.Documents.PeriodicBackup +_LEGACY_INCREMENTAL_BACKUP_EXTENSION = ".ravendb-incremental-dump" +_LEGACY_FULL_BACKUP_EXTENSION = ".ravendb-full-dump" + +# A full backup or snapshot is restored before the incremental files that build on it. +_FULL_BACKUP_EXTENSIONS = ( + _PERIODIC_BACKUP.FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.SNAPSHOT_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_SNAPSHOT_EXTENSTION, +) + +# Snapshots are deliberately absent: they are restored, not imported. +_BACKUP_EXTENSIONS = ( + _PERIODIC_BACKUP.FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_FULL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.INCREMENTAL_BACKUP_EXTENSTION, + _PERIODIC_BACKUP.ENCRYPTED_INCREMENTAL_BACKUP_EXTENSTION, + _LEGACY_INCREMENTAL_BACKUP_EXTENSION, + _LEGACY_FULL_BACKUP_EXTENSION, +) + + +class SmugglerOperation(Operation): + """An export or import operation, whose result is a typed SmugglerResult.""" + + def wait_for_completion(self) -> SmugglerResult: + return SmugglerResult.from_json(super().wait_for_completion()) + + +def _is_backup_file(file_path: str) -> bool: + return os.path.splitext(file_path)[1].lower() in _BACKUP_EXTENSIONS + + +def _backup_sort_key(file_path: str): + name, extension = os.path.splitext(os.path.basename(file_path)) + # A full backup is restored before the incremental files that build on it. + return name, 0 if extension.lower() in _FULL_BACKUP_EXTENSIONS else 1, os.path.getmtime(file_path) + + +class DatabaseSmuggler: + """ + Exports a database to a file or stream and imports it back, through the server's + smuggler endpoints. Reached as ``store.smuggler``. + """ + + def __init__(self, store: "DocumentStore", database_name: str = None): + self._store = store + self._database_name = database_name if database_name is not None else store.database + self._request_executor = None + + @property + def _executor(self): + if self._request_executor is None and self._database_name is not None: + self._request_executor = self._store.get_request_executor(self._database_name) + return self._request_executor + + def for_database(self, database_name: str) -> DatabaseSmuggler: + if self._database_name is not None and database_name is not None: + if database_name.lower() == self._database_name.lower(): + return self + + return DatabaseSmuggler(self._store, database_name) + + def _assert_database_set(self) -> None: + if self._executor is None: + raise RuntimeError("Cannot use Smuggler without a database defined, did you forget to call for_database?") + + def _next_operation_id(self): + command = GetNextOperationIdCommand() + self._executor.execute_command(command) + return command.result, command.node_tag + + def _operation_for(self, operation_id: int, node_tag: str) -> SmugglerOperation: + return SmugglerOperation( + self._executor, + lambda: None, + self._executor.conventions, + operation_id, + node_tag, + ) + + def export( + self, + options: DatabaseSmugglerExportOptions, + to_file_or_stream: Union[str, IO[bytes]], + ) -> SmugglerOperation: + """ + Exports the database. ``to_file_or_stream`` is either a path, in which case the + file (and any missing parent directory) is created, or a writable binary stream, + which is left open. + + The download runs to completion before this returns, so the operation handed back + is only there to read the server-side result and progress. + """ + if options is None: + raise ValueError("options cannot be None") + if to_file_or_stream is None: + raise ValueError("to_file_or_stream cannot be None") + + self._assert_database_set() + + if isinstance(to_file_or_stream, str): + directory = os.path.dirname(os.path.abspath(to_file_or_stream)) + if directory and not os.path.isdir(directory): + os.makedirs(directory, exist_ok=True) + + with open(to_file_or_stream, "wb") as destination: + return self._export_to_stream(options, destination) + + return self._export_to_stream(options, to_file_or_stream) + + def _export_to_stream(self, options: DatabaseSmugglerExportOptions, destination: IO[bytes]) -> SmugglerOperation: + operation_id, node_tag = self._next_operation_id() + self._executor.execute_command(self._ExportCommand(options, destination, operation_id, node_tag)) + return self._operation_for(operation_id, node_tag) + + def export_to_database( + self, + options: DatabaseSmugglerExportOptions, + to_smuggler: DatabaseSmuggler, + ) -> SmugglerOperation: + """ + Streams an export straight into another database. Returns the import operation on + the receiving side, which is the one worth waiting on. + """ + if options is None: + raise ValueError("options cannot be None") + if to_smuggler is None: + raise ValueError("to_smuggler cannot be None") + + import_options = DatabaseSmugglerImportOptions.from_options(options) + + # The sync client cannot hand one side's response stream to the other side's + # request body, so the export is staged on disk first. + with tempfile.TemporaryDirectory() as directory: + staged = os.path.join(directory, "export.ravendbdump") + self.export(options, staged).wait_for_completion() + return to_smuggler.import_data(import_options, staged) + + def import_data( + self, + options: DatabaseSmugglerImportOptions, + from_file_or_stream: Union[str, IO[bytes]], + ) -> SmugglerOperation: + """ + Imports a dump. ``from_file_or_stream`` is either a path, which is opened and + closed here, or a readable binary stream, which is left open. + """ + if options is None: + raise ValueError("options cannot be None") + if from_file_or_stream is None: + raise ValueError("from_file_or_stream cannot be None") + + self._assert_database_set() + + if isinstance(from_file_or_stream, str): + with open(from_file_or_stream, "rb") as source: + return self._import_from_stream(options, source) + + return self._import_from_stream(options, from_file_or_stream) + + def _import_from_stream(self, options: DatabaseSmugglerImportOptions, source: IO[bytes]) -> SmugglerOperation: + operation_id, node_tag = self._next_operation_id() + self._executor.execute_command(self._ImportCommand(options, source, operation_id, node_tag)) + return self._operation_for(operation_id, node_tag) + + def import_incremental(self, options: DatabaseSmugglerImportOptions, from_directory: str) -> None: + """ + Imports a backup directory in order, waiting for each file. Indexes and + subscriptions come from the last file only, so a later incremental file cannot + resurrect an index the backup dropped. + """ + if options is None: + raise ValueError("options cannot be None") + + files = sorted( + ( + os.path.join(from_directory, name) + for name in os.listdir(from_directory) + if _is_backup_file(os.path.join(from_directory, name)) + ), + key=_backup_sort_key, + ) + + if not files: + return + + original_operate_on_types = self._configure_options_for_incremental_import(options) + for file_path in files[:-1]: + self.import_data(options, file_path).wait_for_completion() + + options.operate_on_types = original_operate_on_types + self.import_data(options, files[-1]).wait_for_completion() + + @staticmethod + def _configure_options_for_incremental_import(options: DatabaseSmugglerOptions) -> set: + options.operate_on_types.add(DatabaseItemType.TOMBSTONES) + options.operate_on_types.add(DatabaseItemType.COMPARE_EXCHANGE_TOMBSTONES) + + original = set(options.operate_on_types) + options.operate_on_types.discard(DatabaseItemType.INDEXES) + options.operate_on_types.discard(DatabaseItemType.SUBSCRIPTIONS) + return original + + class _ExportCommand(VoidRavenCommand): + def __init__( + self, + options: DatabaseSmugglerExportOptions, + destination: IO[bytes], + operation_id: int, + node_tag: str = None, + ): + super().__init__() + self._options = options + self._destination = destination + self._operation_id = operation_id + self._selected_node_tag = node_tag + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/smuggler/export?operationId={self._operation_id}" + + request = requests.Request("POST", url) + request.data = self._options.to_json() + return request + + def send(self, session: requests.Session, request: requests.Request) -> requests.Response: + prepared_request = session.prepare_request(request) + RequestUtils.remove_zstd_encoding(prepared_request) + return session.send(prepared_request, cert=session.cert, stream=True) + + def process_response(self, cache, response: requests.Response, url) -> ResponseDisposeHandling: + for chunk in response.iter_content(chunk_size=_COPY_BUFFER_SIZE): + if chunk: + self._destination.write(chunk) + + return ResponseDisposeHandling.AUTOMATIC + + class _ImportCommand(VoidRavenCommand): + def __init__( + self, + options: DatabaseSmugglerImportOptions, + source: IO[bytes], + operation_id: int, + node_tag: str = None, + ): + super().__init__() + self._options = options + self._source = source + self._operation_id = operation_id + self._selected_node_tag = node_tag + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/smuggler/import?operationId={self._operation_id}" + + request = requests.Request("POST", url) + # The server reads the options section first and the dump second, so the order + # of these parts matters. + request.files = { + _IMPORT_OPTIONS_PART: (None, json.dumps(self._options.to_json()), "application/json"), + "file": ("name", self._source, "application/octet-stream"), + } + return request diff --git a/ravendb/documents/smuggler/result.py b/ravendb/documents/smuggler/result.py new file mode 100644 index 00000000..359efe4e --- /dev/null +++ b/ravendb/documents/smuggler/result.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from ravendb.tools.utils import Utils + + +class Counts: + """How much of one item type an export or import got through.""" + + def __init__( + self, + start_time: Optional[datetime] = None, + processed: bool = False, + read_count: int = 0, + skipped: bool = False, + errored_count: int = 0, + size_in_bytes: int = 0, + ): + self.start_time = start_time + self.processed = processed + self.read_count = read_count + self.skipped = skipped + self.errored_count = errored_count + self.size_in_bytes = size_in_bytes + + def to_json(self) -> Dict[str, Any]: + return { + "StartTime": Utils.datetime_to_string(self.start_time) if self.start_time else None, + "Processed": self.processed, + "ReadCount": self.read_count, + "Skipped": self.skipped, + "ErroredCount": self.errored_count, + "SizeInBytes": self.size_in_bytes, + } + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + start_time = json_dict.get("StartTime") + self.start_time = Utils.string_to_datetime(start_time) if start_time else None + self.processed = json_dict.get("Processed", False) + self.read_count = json_dict.get("ReadCount", 0) + self.skipped = json_dict.get("Skipped", False) + self.errored_count = json_dict.get("ErroredCount", 0) + self.size_in_bytes = json_dict.get("SizeInBytes", 0) + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]): + counts = cls() + if json_dict: + counts._fill_from_json(json_dict) + return counts + + def __repr__(self) -> str: + text = f"Read: {self.read_count:,}." + if self.errored_count: + text += f" Errored: {self.errored_count:,}." + if self.size_in_bytes: + text += f" Size: {self.size_in_bytes:,} bytes." + return text + + +class CountsWithLastEtag(Counts): + def __init__(self, last_etag: int = 0, **kwargs): + super().__init__(**kwargs) + self.last_etag = last_etag + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["LastEtag"] = self.last_etag + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.last_etag = json_dict.get("LastEtag", 0) + + +class CountsWithLastEtagAndAttachments(CountsWithLastEtag): + def __init__(self, attachments: Optional[Counts] = None, **kwargs): + super().__init__(**kwargs) + self.attachments = attachments if attachments is not None else Counts() + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["Attachments"] = self.attachments.to_json() + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.attachments = Counts.from_json(json_dict.get("Attachments")) + + def __repr__(self) -> str: + if self.attachments and self.attachments.read_count: + return f"{super().__repr__()} Attachments: {self.attachments!r}" + return super().__repr__() + + +class CountsWithSkippedCountAndLastEtag(CountsWithLastEtag): + def __init__(self, skipped_count: int = 0, **kwargs): + super().__init__(**kwargs) + self.skipped_count = skipped_count + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["SkippedCount"] = self.skipped_count + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.skipped_count = json_dict.get("SkippedCount", 0) + + def __repr__(self) -> str: + return f"Skipped: {self.skipped_count:,}. {super().__repr__()}" + + +class CountsWithSkippedCountAndLastEtagAndAttachments(CountsWithLastEtagAndAttachments): + def __init__(self, skipped_count: int = 0, **kwargs): + super().__init__(**kwargs) + self.skipped_count = skipped_count + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["SkippedCount"] = self.skipped_count + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.skipped_count = json_dict.get("SkippedCount", 0) + + def __repr__(self) -> str: + if self.skipped_count: + return f"Skipped: {self.skipped_count:,}. {super().__repr__()}" + return super().__repr__() + + +class DatabaseRecordProgress(Counts): + """ + Which parts of the database record an import wrote. The server sends only the flags it + actually set, so anything it leaves out reads as False. + """ + + def __init__( + self, + sorters_updated: bool = False, + analyzers_updated: bool = False, + sink_pull_replications_updated: bool = False, + hub_pull_replications_updated: bool = False, + raven_etls_updated: bool = False, + sql_etls_updated: bool = False, + snowflake_etls_updated: bool = False, + embeddings_generations_updated: bool = False, + gen_ai_tasks_updated: bool = False, + external_replications_updated: bool = False, + periodic_backups_updated: bool = False, + conflict_solver_config_updated: bool = False, + schema_validation_config_updated: bool = False, + time_series_configuration_updated: bool = False, + documents_compression_configuration_updated: bool = False, + revisions_configuration_updated: bool = False, + expiration_configuration_updated: bool = False, + refresh_configuration_updated: bool = False, + data_archival_configuration_updated: bool = False, + remote_attachments_configuration_updated: bool = False, + raven_connection_strings_updated: bool = False, + sql_connection_strings_updated: bool = False, + snowflake_connection_strings_updated: bool = False, + ai_connection_strings_updated: bool = False, + ai_agents_updated: bool = False, + client_configuration_updated: bool = False, + unused_database_ids_updated: bool = False, + lock_mode_updated: bool = False, + olap_etls_updated: bool = False, + olap_connection_strings_updated: bool = False, + elastic_search_etls_updated: bool = False, + elastic_search_connection_strings_updated: bool = False, + postre_s_q_l_configuration_updated: bool = False, + queue_etls_updated: bool = False, + queue_connection_strings_updated: bool = False, + queue_sinks_updated: bool = False, + cdc_sinks_updated: bool = False, + indexes_history_updated: bool = False, + **kwargs, + ): + super().__init__(**kwargs) + self.sorters_updated = sorters_updated + self.analyzers_updated = analyzers_updated + self.sink_pull_replications_updated = sink_pull_replications_updated + self.hub_pull_replications_updated = hub_pull_replications_updated + self.raven_etls_updated = raven_etls_updated + self.sql_etls_updated = sql_etls_updated + self.snowflake_etls_updated = snowflake_etls_updated + self.embeddings_generations_updated = embeddings_generations_updated + self.gen_ai_tasks_updated = gen_ai_tasks_updated + self.external_replications_updated = external_replications_updated + self.periodic_backups_updated = periodic_backups_updated + self.conflict_solver_config_updated = conflict_solver_config_updated + self.schema_validation_config_updated = schema_validation_config_updated + self.time_series_configuration_updated = time_series_configuration_updated + self.documents_compression_configuration_updated = documents_compression_configuration_updated + self.revisions_configuration_updated = revisions_configuration_updated + self.expiration_configuration_updated = expiration_configuration_updated + self.refresh_configuration_updated = refresh_configuration_updated + self.data_archival_configuration_updated = data_archival_configuration_updated + self.remote_attachments_configuration_updated = remote_attachments_configuration_updated + self.raven_connection_strings_updated = raven_connection_strings_updated + self.sql_connection_strings_updated = sql_connection_strings_updated + self.snowflake_connection_strings_updated = snowflake_connection_strings_updated + self.ai_connection_strings_updated = ai_connection_strings_updated + self.ai_agents_updated = ai_agents_updated + self.client_configuration_updated = client_configuration_updated + self.unused_database_ids_updated = unused_database_ids_updated + self.lock_mode_updated = lock_mode_updated + self.olap_etls_updated = olap_etls_updated + self.olap_connection_strings_updated = olap_connection_strings_updated + self.elastic_search_etls_updated = elastic_search_etls_updated + self.elastic_search_connection_strings_updated = elastic_search_connection_strings_updated + self.postre_s_q_l_configuration_updated = postre_s_q_l_configuration_updated + self.queue_etls_updated = queue_etls_updated + self.queue_connection_strings_updated = queue_connection_strings_updated + self.queue_sinks_updated = queue_sinks_updated + self.cdc_sinks_updated = cdc_sinks_updated + self.indexes_history_updated = indexes_history_updated + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + if self.sorters_updated: + json_dict["SortersUpdated"] = True + if self.analyzers_updated: + json_dict["AnalyzersUpdated"] = True + if self.sink_pull_replications_updated: + json_dict["SinkPullReplicationsUpdated"] = True + if self.hub_pull_replications_updated: + json_dict["HubPullReplicationsUpdated"] = True + if self.raven_etls_updated: + json_dict["RavenEtlsUpdated"] = True + if self.sql_etls_updated: + json_dict["SqlEtlsUpdated"] = True + if self.snowflake_etls_updated: + json_dict["SnowflakeEtlsUpdated"] = True + if self.embeddings_generations_updated: + json_dict["EmbeddingsGenerationsUpdated"] = True + if self.gen_ai_tasks_updated: + json_dict["GenAiTasksUpdated"] = True + if self.external_replications_updated: + json_dict["ExternalReplicationsUpdated"] = True + if self.periodic_backups_updated: + json_dict["PeriodicBackupsUpdated"] = True + if self.conflict_solver_config_updated: + json_dict["ConflictSolverConfigUpdated"] = True + if self.schema_validation_config_updated: + json_dict["SchemaValidationConfigUpdated"] = True + if self.time_series_configuration_updated: + json_dict["TimeSeriesConfigurationUpdated"] = True + if self.documents_compression_configuration_updated: + json_dict["DocumentsCompressionConfigurationUpdated"] = True + if self.revisions_configuration_updated: + json_dict["RevisionsConfigurationUpdated"] = True + if self.expiration_configuration_updated: + json_dict["ExpirationConfigurationUpdated"] = True + if self.refresh_configuration_updated: + json_dict["RefreshConfigurationUpdated"] = True + if self.data_archival_configuration_updated: + json_dict["DataArchivalConfigurationUpdated"] = True + if self.remote_attachments_configuration_updated: + json_dict["RemoteAttachmentsConfigurationUpdated"] = True + if self.raven_connection_strings_updated: + json_dict["RavenConnectionStringsUpdated"] = True + if self.sql_connection_strings_updated: + json_dict["SqlConnectionStringsUpdated"] = True + if self.snowflake_connection_strings_updated: + json_dict["SnowflakeConnectionStringsUpdated"] = True + if self.ai_connection_strings_updated: + json_dict["AiConnectionStringsUpdated"] = True + if self.ai_agents_updated: + json_dict["AiAgentsUpdated"] = True + if self.client_configuration_updated: + json_dict["ClientConfigurationUpdated"] = True + if self.unused_database_ids_updated: + json_dict["UnusedDatabaseIdsUpdated"] = True + if self.lock_mode_updated: + json_dict["LockModeUpdated"] = True + if self.olap_etls_updated: + json_dict["OlapEtlsUpdated"] = True + if self.olap_connection_strings_updated: + json_dict["OlapConnectionStringsUpdated"] = True + if self.elastic_search_etls_updated: + json_dict["ElasticSearchEtlsUpdated"] = True + if self.elastic_search_connection_strings_updated: + json_dict["ElasticSearchConnectionStringsUpdated"] = True + if self.postre_s_q_l_configuration_updated: + json_dict["PostreSQLConfigurationUpdated"] = True + if self.queue_etls_updated: + json_dict["QueueEtlsUpdated"] = True + if self.queue_connection_strings_updated: + json_dict["QueueConnectionStringsUpdated"] = True + if self.queue_sinks_updated: + json_dict["QueueSinksUpdated"] = True + if self.cdc_sinks_updated: + json_dict["CdcSinksUpdated"] = True + if self.indexes_history_updated: + json_dict["IndexesHistoryUpdated"] = True + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.sorters_updated = json_dict.get("SortersUpdated", False) + self.analyzers_updated = json_dict.get("AnalyzersUpdated", False) + self.sink_pull_replications_updated = json_dict.get("SinkPullReplicationsUpdated", False) + self.hub_pull_replications_updated = json_dict.get("HubPullReplicationsUpdated", False) + self.raven_etls_updated = json_dict.get("RavenEtlsUpdated", False) + self.sql_etls_updated = json_dict.get("SqlEtlsUpdated", False) + self.snowflake_etls_updated = json_dict.get("SnowflakeEtlsUpdated", False) + self.embeddings_generations_updated = json_dict.get("EmbeddingsGenerationsUpdated", False) + self.gen_ai_tasks_updated = json_dict.get("GenAiTasksUpdated", False) + self.external_replications_updated = json_dict.get("ExternalReplicationsUpdated", False) + self.periodic_backups_updated = json_dict.get("PeriodicBackupsUpdated", False) + self.conflict_solver_config_updated = json_dict.get("ConflictSolverConfigUpdated", False) + self.schema_validation_config_updated = json_dict.get("SchemaValidationConfigUpdated", False) + self.time_series_configuration_updated = json_dict.get("TimeSeriesConfigurationUpdated", False) + self.documents_compression_configuration_updated = json_dict.get( + "DocumentsCompressionConfigurationUpdated", False + ) + self.revisions_configuration_updated = json_dict.get("RevisionsConfigurationUpdated", False) + self.expiration_configuration_updated = json_dict.get("ExpirationConfigurationUpdated", False) + self.refresh_configuration_updated = json_dict.get("RefreshConfigurationUpdated", False) + self.data_archival_configuration_updated = json_dict.get("DataArchivalConfigurationUpdated", False) + self.remote_attachments_configuration_updated = json_dict.get("RemoteAttachmentsConfigurationUpdated", False) + self.raven_connection_strings_updated = json_dict.get("RavenConnectionStringsUpdated", False) + self.sql_connection_strings_updated = json_dict.get("SqlConnectionStringsUpdated", False) + self.snowflake_connection_strings_updated = json_dict.get("SnowflakeConnectionStringsUpdated", False) + self.ai_connection_strings_updated = json_dict.get("AiConnectionStringsUpdated", False) + self.ai_agents_updated = json_dict.get("AiAgentsUpdated", False) + self.client_configuration_updated = json_dict.get("ClientConfigurationUpdated", False) + self.unused_database_ids_updated = json_dict.get("UnusedDatabaseIdsUpdated", False) + self.lock_mode_updated = json_dict.get("LockModeUpdated", False) + self.olap_etls_updated = json_dict.get("OlapEtlsUpdated", False) + self.olap_connection_strings_updated = json_dict.get("OlapConnectionStringsUpdated", False) + self.elastic_search_etls_updated = json_dict.get("ElasticSearchEtlsUpdated", False) + self.elastic_search_connection_strings_updated = json_dict.get("ElasticSearchConnectionStringsUpdated", False) + self.postre_s_q_l_configuration_updated = json_dict.get("PostreSQLConfigurationUpdated", False) + self.queue_etls_updated = json_dict.get("QueueEtlsUpdated", False) + self.queue_connection_strings_updated = json_dict.get("QueueConnectionStringsUpdated", False) + self.queue_sinks_updated = json_dict.get("QueueSinksUpdated", False) + self.cdc_sinks_updated = json_dict.get("CdcSinksUpdated", False) + self.indexes_history_updated = json_dict.get("IndexesHistoryUpdated", False) + + @property + def updated(self) -> List[str]: + """The parts that were written, for when you just want to see the list.""" + return sorted(name for name, value in vars(self).items() if name.endswith("_updated") and value is True) + + +class SmugglerProgressBase: + """Per-item-type counts, shared by a smuggler operation's progress and its result.""" + + _SECTIONS = ( + ("database_record", "DatabaseRecord", DatabaseRecordProgress), + ("documents", "Documents", CountsWithSkippedCountAndLastEtagAndAttachments), + ("revision_documents", "RevisionDocuments", CountsWithSkippedCountAndLastEtagAndAttachments), + ("tombstones", "Tombstones", CountsWithLastEtag), + ("conflicts", "Conflicts", CountsWithLastEtag), + ("identities", "Identities", CountsWithLastEtag), + ("indexes", "Indexes", Counts), + ("compare_exchange", "CompareExchange", CountsWithLastEtag), + ("subscriptions", "Subscriptions", Counts), + ("counters", "Counters", CountsWithSkippedCountAndLastEtag), + ("compare_exchange_tombstones", "CompareExchangeTombstones", Counts), + ("time_series", "TimeSeries", CountsWithSkippedCountAndLastEtag), + ("replication_hub_certificates", "ReplicationHubCertificates", Counts), + ("time_series_deleted_ranges", "TimeSeriesDeletedRanges", CountsWithSkippedCountAndLastEtag), + ) + + def __init__(self): + for attribute, _, counts_type in self._SECTIONS: + setattr(self, attribute, counts_type()) + + def to_json(self) -> Dict[str, Any]: + return {key: getattr(self, attribute).to_json() for attribute, key, _ in self._SECTIONS} + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + for attribute, key, counts_type in self._SECTIONS: + setattr(self, attribute, counts_type.from_json(json_dict.get(key))) + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]): + progress = cls() + if json_dict: + progress._fill_from_json(json_dict) + return progress + + +class SmugglerResult(SmugglerProgressBase): + """ + What a smuggler export or import actually moved. Read it off a finished operation: + + operation = store.smuggler.import_data(options, "dump.ravendbdump") + result = operation.wait_for_completion() + print(result.documents.read_count) + """ + + def __init__(self): + super().__init__() + self.messages: List[str] = [] + self.elapsed: Optional[str] = None + self.message: Optional[str] = None + + def to_json(self) -> Dict[str, Any]: + json_dict = super().to_json() + json_dict["Messages"] = self.messages + json_dict["Elapsed"] = self.elapsed + json_dict["Message"] = self.message + return json_dict + + def _fill_from_json(self, json_dict: Dict[str, Any]) -> None: + super()._fill_from_json(json_dict) + self.messages = json_dict.get("Messages") or [] + self.elapsed = json_dict.get("Elapsed") + self.message = json_dict.get("Message") + + def __repr__(self) -> str: + return f"Documents: {self.documents!r} Revisions: {self.revision_documents!r} Indexes: {self.indexes!r}" diff --git a/ravendb/documents/store/definition.py b/ravendb/documents/store/definition.py index 0f3ed0be..e7fcab26 100644 --- a/ravendb/documents/store/definition.py +++ b/ravendb/documents/store/definition.py @@ -44,6 +44,7 @@ if TYPE_CHECKING: from ravendb.documents.indexes.abstract_index_creation_tasks import AbstractIndexCreationTask + from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler from ravendb.http.misc import AggressiveCacheMode, AggressiveCacheOptions @@ -341,7 +342,7 @@ def __init__(self, urls: Union[str, List[str]] = None, database: Optional[str] = self.__aggressive_cache_changes: Dict[str, "DocumentStore._AggressiveCacheInvalidator"] = {} self.__maintenance_operation_executor: Optional[MaintenanceOperationExecutor] = None self.__operation_executor: Optional[OperationExecutor] = None - # todo: database smuggler + self.__smuggler: Optional["DatabaseSmuggler"] = None self.__multi_db_hilo: Optional[MultiDatabaseHiLoGenerator] = None self.__identifier: Optional[str] = None self.__add_change_lock = threading.Lock() @@ -714,6 +715,17 @@ def operations(self) -> OperationExecutor: return self.__operation_executor + @property + def smuggler(self) -> "DatabaseSmuggler": + self.assert_initialized() + + if self.__smuggler is None: + from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler + + self.__smuggler = DatabaseSmuggler(self) + + return self.__smuggler + @property def time_series(self) -> TimeSeriesOperations: if self.__time_series_operation is None: diff --git a/ravendb/tests/documents_tests/test_smuggler.py b/ravendb/tests/documents_tests/test_smuggler.py new file mode 100644 index 00000000..9748ddba --- /dev/null +++ b/ravendb/tests/documents_tests/test_smuggler.py @@ -0,0 +1,424 @@ +""" +Tests for the database smuggler: the options that say what an export or import covers, +the request shapes, backup-file ordering for incremental imports, and a full export / +import round trip against a real server. +""" + +import io +import json +import os +import tempfile +import unittest + +from ravendb.documents.smuggler.common import ( + DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES, + DEFAULT_OPERATE_ON_TYPES, + DatabaseItemType, + DatabaseRecordItemType, + DatabaseSmugglerExportOptions, + DatabaseSmugglerImportOptions, + DatabaseSmugglerOptions, + ExportCompressionAlgorithm, + flags_from_string, + flags_to_string, +) +from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler, _backup_sort_key, _is_backup_file +from ravendb.documents.smuggler.result import SmugglerResult +from ravendb.http.server_node import ServerNode +from ravendb.tests.test_base import TestBase, User + + +class TestItemTypeFlags(unittest.TestCase): + def test_nothing_selected_renders_as_none(self): + self.assertEqual("None", flags_to_string(set(), DatabaseItemType)) + self.assertEqual("None", flags_to_string(None, DatabaseItemType)) + self.assertEqual("None", flags_to_string({DatabaseItemType.NONE}, DatabaseItemType)) + + def test_members_render_as_a_comma_separated_list(self): + # This is the shape the server's Enum.Parse reads a [Flags] value back from. + self.assertEqual( + "Documents, Indexes", + flags_to_string({DatabaseItemType.INDEXES, DatabaseItemType.DOCUMENTS}, DatabaseItemType), + ) + + def test_rendering_is_stable_regardless_of_set_order(self): + one = flags_to_string({DatabaseItemType.DOCUMENTS, DatabaseItemType.ATTACHMENTS}, DatabaseItemType) + other = flags_to_string({DatabaseItemType.ATTACHMENTS, DatabaseItemType.DOCUMENTS}, DatabaseItemType) + + self.assertEqual(one, other) + + def test_flags_round_trip(self): + members = {DatabaseItemType.DOCUMENTS, DatabaseItemType.TIME_SERIES_DELETED_RANGES} + + self.assertEqual(members, flags_from_string(flags_to_string(members, DatabaseItemType), DatabaseItemType)) + + def test_parsing_tolerates_spacing_and_drops_none(self): + self.assertEqual( + {DatabaseItemType.DOCUMENTS, DatabaseItemType.INDEXES}, + flags_from_string("None,Documents , Indexes", DatabaseItemType), + ) + self.assertEqual(set(), flags_from_string("", DatabaseItemType)) + + def test_the_database_record_types_the_7_2_5_server_knows(self): + for name in ("QueueSinks", "CdcSinks", "SchemaValidation", "RemoteAttachments", "AiAgents"): + self.assertEqual(name, DatabaseRecordItemType(name).value) + + +class TestSmugglerOptions(unittest.TestCase): + def test_defaults_match_the_server_side_defaults(self): + options = DatabaseSmugglerOptions() + + self.assertEqual(DEFAULT_OPERATE_ON_TYPES, options.operate_on_types) + self.assertEqual(DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES, options.operate_on_database_record_types) + self.assertTrue(options.include_expired) + self.assertFalse(options.include_artificial) + self.assertTrue(options.include_archived) + self.assertEqual(10000, options.max_steps_for_transform_script) + self.assertEqual([], options.collections) + + def test_each_instance_gets_its_own_collections(self): + # A shared default would leak one export's narrowing into the next. + first = DatabaseSmugglerOptions() + first.operate_on_types.add(DatabaseItemType.TOMBSTONES) + first.collections.append("Orders") + + second = DatabaseSmugglerOptions() + self.assertNotIn(DatabaseItemType.TOMBSTONES, second.operate_on_types) + self.assertEqual([], second.collections) + + def test_body_carries_the_selections_as_flag_strings(self): + options = DatabaseSmugglerOptions( + operate_on_types={DatabaseItemType.DOCUMENTS}, + operate_on_database_record_types={DatabaseRecordItemType.SETTINGS}, + ) + body = options.to_json() + + self.assertEqual("Documents", body["OperateOnTypes"]) + self.assertEqual("Settings", body["OperateOnDatabaseRecordTypes"]) + + def test_options_round_trip(self): + options = DatabaseSmugglerOptions( + operate_on_types={DatabaseItemType.DOCUMENTS, DatabaseItemType.ATTACHMENTS}, + collections=["Orders"], + transform_script="this.Foo = 1;", + max_read_ops_per_second=200, + skip_corrupted_data=True, + ) + + self.assertEqual(options.to_json(), DatabaseSmugglerOptions.from_json(options.to_json()).to_json()) + + def test_export_options_leave_the_compression_choice_to_the_server(self): + self.assertNotIn("CompressionAlgorithm", DatabaseSmugglerExportOptions().to_json()) + + def test_export_options_send_a_chosen_compression_algorithm(self): + options = DatabaseSmugglerExportOptions(compression_algorithm=ExportCompressionAlgorithm.GZIP) + + self.assertEqual("Gzip", options.to_json()["CompressionAlgorithm"]) + self.assertEqual( + ExportCompressionAlgorithm.GZIP, + DatabaseSmugglerExportOptions.from_json(options.to_json()).compression_algorithm, + ) + + def test_import_options_always_state_the_revision_choice(self): + self.assertFalse(DatabaseSmugglerImportOptions().to_json()["SkipRevisionCreation"]) + self.assertTrue(DatabaseSmugglerImportOptions(skip_revision_creation=True).to_json()["SkipRevisionCreation"]) + + def test_import_options_copy_only_the_shared_settings_of_an_export(self): + # Collections and the database-record selection stay at the import defaults, the + # same subset the C# copy constructor carries over. + source = DatabaseSmugglerOptions( + operate_on_types={DatabaseItemType.DOCUMENTS}, + operate_on_database_record_types={DatabaseRecordItemType.SETTINGS}, + collections=["Orders"], + transform_script="this.Foo = 1;", + include_expired=False, + ) + copied = DatabaseSmugglerImportOptions.from_options(source) + + self.assertEqual({DatabaseItemType.DOCUMENTS}, copied.operate_on_types) + self.assertEqual("this.Foo = 1;", copied.transform_script) + self.assertFalse(copied.include_expired) + self.assertEqual([], copied.collections) + self.assertEqual(DEFAULT_OPERATE_ON_DATABASE_RECORD_TYPES, copied.operate_on_database_record_types) + + +class TestSmugglerRequests(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + + def test_export_posts_the_options_with_the_operation_id(self): + options = DatabaseSmugglerExportOptions() + command = DatabaseSmuggler._ExportCommand(options, io.BytesIO(), 42, "A") + request = command.create_request(self.node) + + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/smuggler/export?operationId=42", request.url) + self.assertEqual(options.to_json(), request.data) + self.assertEqual("A", command.selected_node_tag) + + def test_import_posts_the_options_and_the_dump_as_two_parts(self): + options = DatabaseSmugglerImportOptions(skip_revision_creation=True) + command = DatabaseSmuggler._ImportCommand(options, io.BytesIO(b"dump"), 43, "B") + request = command.create_request(self.node) + + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/smuggler/import?operationId=43", request.url) + # The server reads the options section before the dump, so the order matters. + self.assertEqual(["importOptions", "file"], list(request.files)) + self.assertTrue(json.loads(request.files["importOptions"][1])["SkipRevisionCreation"]) + self.assertEqual("B", command.selected_node_tag) + + def test_a_smuggler_without_a_database_refuses_to_run(self): + class _StoreWithoutDatabase: + database = None + + def get_request_executor(self, database=None): + return None + + smuggler = DatabaseSmuggler(_StoreWithoutDatabase()) + with self.assertRaises(RuntimeError): + smuggler.export(DatabaseSmugglerExportOptions(), io.BytesIO()) + with self.assertRaises(RuntimeError): + smuggler.import_data(DatabaseSmugglerImportOptions(), io.BytesIO()) + + def test_missing_arguments_are_rejected_client_side(self): + smuggler = DatabaseSmuggler(type("_Store", (), {"database": "db"})()) + + with self.assertRaises(ValueError): + smuggler.export(None, io.BytesIO()) + with self.assertRaises(ValueError): + smuggler.export(DatabaseSmugglerExportOptions(), None) + with self.assertRaises(ValueError): + smuggler.import_data(None, io.BytesIO()) + with self.assertRaises(ValueError): + smuggler.import_data(DatabaseSmugglerImportOptions(), None) + + +class TestBackupFileOrdering(unittest.TestCase): + def test_backup_extensions_are_recognized(self): + for name in ( + "2026-06-16.ravendb-full-backup", + "2026-06-16.ravendb-incremental-backup", + "2026-06-16.ravendb-encrypted-full-backup", + "2026-06-16.ravendb-encrypted-incremental-backup", + "old.ravendb-full-dump", + "old.ravendb-incremental-dump", + ): + self.assertTrue(_is_backup_file(name), name) + + def test_other_files_are_ignored(self): + for name in ("notes.txt", "export.ravendbdump", "2026-06-16.ravendb-snapshot"): + self.assertFalse(_is_backup_file(name), name) + + def test_a_full_backup_sorts_before_its_incrementals(self): + with tempfile.TemporaryDirectory() as directory: + names = [ + "2026-06-16-10-00.ravendb-incremental-backup", + "2026-06-16-10-00.ravendb-full-backup", + "2026-06-15-10-00.ravendb-full-backup", + ] + for name in names: + open(os.path.join(directory, name), "wb").close() + + ordered = sorted((os.path.join(directory, name) for name in names), key=_backup_sort_key) + + self.assertEqual( + [ + "2026-06-15-10-00.ravendb-full-backup", + "2026-06-16-10-00.ravendb-full-backup", + "2026-06-16-10-00.ravendb-incremental-backup", + ], + [os.path.basename(path) for path in ordered], + ) + + def test_incremental_import_narrows_and_restores_the_selection(self): + # Indexes and subscriptions come from the last file only, so an earlier + # incremental cannot bring back an index a later backup dropped. + options = DatabaseSmugglerImportOptions() + original = DatabaseSmuggler._configure_options_for_incremental_import(options) + + self.assertIn(DatabaseItemType.TOMBSTONES, options.operate_on_types) + self.assertIn(DatabaseItemType.COMPARE_EXCHANGE_TOMBSTONES, options.operate_on_types) + self.assertNotIn(DatabaseItemType.INDEXES, options.operate_on_types) + self.assertNotIn(DatabaseItemType.SUBSCRIPTIONS, options.operate_on_types) + self.assertIn(DatabaseItemType.INDEXES, original) + self.assertIn(DatabaseItemType.SUBSCRIPTIONS, original) + + +class TestSmugglerAgainstServer(TestBase): + def setUp(self): + super().setUp() + + def _store_users(self, store, count: int): + with store.open_session() as session: + for i in range(count): + session.store(User(name=f"user-{i}"), f"users/{i}") + session.save_changes() + + def test_exports_a_database_to_a_file_and_imports_it_into_another(self): + self._store_users(self.store, 5) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "export.ravendbdump") + self.store.smuggler.export(DatabaseSmugglerExportOptions(), dump).wait_for_completion() + + self.assertTrue(os.path.isfile(dump)) + self.assertGreater(os.path.getsize(dump), 0) + + with self.get_document_store() as target: + imported = target.smuggler.import_data(DatabaseSmugglerImportOptions(), dump).wait_for_completion() + self.assertIsInstance(imported, SmugglerResult) + self.assertEqual(5, imported.documents.read_count) + self.assertTrue(imported.messages) + + with target.open_session() as session: + self.assertEqual("user-3", session.load("users/3", User).name) + self.assertEqual(5, len(session.load(["users/0", "users/1", "users/2", "users/3", "users/4"]))) + + def test_exports_into_a_stream(self): + self._store_users(self.store, 2) + + destination = io.BytesIO() + exported = self.store.smuggler.export(DatabaseSmugglerExportOptions(), destination).wait_for_completion() + + self.assertGreater(len(destination.getvalue()), 0) + self.assertEqual(2, exported.documents.read_count) + self.assertIsNotNone(exported.elapsed) + + def test_an_export_narrowed_to_one_collection_leaves_the_rest_behind(self): + self._store_users(self.store, 3) + with self.store.open_session() as session: + session.store({"Name": "an order", "@metadata": {"@collection": "Orders"}}, "orders/1") + session.save_changes() + + options = DatabaseSmugglerExportOptions(operate_on_types={DatabaseItemType.DOCUMENTS}, collections=["Users"]) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "users-only.ravendbdump") + self.store.smuggler.export(options, dump).wait_for_completion() + + with self.get_document_store() as target: + target.smuggler.import_data(DatabaseSmugglerImportOptions(), dump).wait_for_completion() + + with target.open_session() as session: + self.assertIsNotNone(session.load("users/0", User)) + self.assertIsNone(session.load("orders/1")) + + def test_export_creates_the_target_directory(self): + self._store_users(self.store, 1) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "nested", "deeper", "export.ravendbdump") + self.store.smuggler.export(DatabaseSmugglerExportOptions(), dump).wait_for_completion() + + self.assertTrue(os.path.isfile(dump)) + + def test_for_database_targets_another_database_on_the_same_store(self): + self._store_users(self.store, 2) + + with self.get_document_store() as target: + smuggler = self.store.smuggler + self.assertIs(smuggler, smuggler.for_database(self.store.database)) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "export.ravendbdump") + smuggler.export(DatabaseSmugglerExportOptions(), dump).wait_for_completion() + + self.store.smuggler.for_database(target.database).import_data( + DatabaseSmugglerImportOptions(), dump + ).wait_for_completion() + + with target.open_session() as session: + self.assertEqual("user-1", session.load("users/1", User).name) + + +class TestSmugglerResult(unittest.TestCase): + RESPONSE = { + "Documents": { + "ReadCount": 5, + "SkippedCount": 1, + "ErroredCount": 0, + "SizeInBytes": 120, + "LastEtag": 9, + "Attachments": {"ReadCount": 2, "SizeInBytes": 40}, + }, + "RevisionDocuments": {"ReadCount": 3}, + "Indexes": {"ReadCount": 1}, + "TimeSeriesDeletedRanges": {"ReadCount": 4}, + "DatabaseRecord": {"ReadCount": 1, "QueueSinksUpdated": True, "CdcSinksUpdated": True}, + "Messages": ["Processed 5 documents."], + "Elapsed": "00:00:01.2340000", + } + + def test_counts_are_parsed_per_item_type(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(5, result.documents.read_count) + self.assertEqual(1, result.documents.skipped_count) + self.assertEqual(9, result.documents.last_etag) + self.assertEqual(3, result.revision_documents.read_count) + self.assertEqual(1, result.indexes.read_count) + self.assertEqual(4, result.time_series_deleted_ranges.read_count) + + def test_attachments_hang_off_the_documents_count(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(2, result.documents.attachments.read_count) + self.assertEqual(40, result.documents.attachments.size_in_bytes) + + def test_messages_and_elapsed_are_carried(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(["Processed 5 documents."], result.messages) + self.assertEqual("00:00:01.2340000", result.elapsed) + + def test_the_database_record_reports_only_what_was_written(self): + record = SmugglerResult.from_json(self.RESPONSE).database_record + + self.assertTrue(record.queue_sinks_updated) + self.assertTrue(record.cdc_sinks_updated) + self.assertFalse(record.sorters_updated) + self.assertEqual(["cdc_sinks_updated", "queue_sinks_updated"], record.updated) + + def test_the_database_record_writes_back_only_the_true_flags(self): + # This is how the server sends them, and how C# writes them out. + serialized = SmugglerResult.from_json(self.RESPONSE).database_record.to_json() + + self.assertEqual(["QueueSinksUpdated", "CdcSinksUpdated"], [k for k in serialized if k.endswith("Updated")]) + + def test_a_missing_section_reads_as_zero_rather_than_raising(self): + result = SmugglerResult.from_json({}) + + self.assertEqual(0, result.documents.read_count) + self.assertEqual(0, result.identities.read_count) + self.assertEqual([], result.messages) + self.assertEqual([], result.database_record.updated) + + def test_a_null_result_reads_as_an_empty_one(self): + self.assertEqual(0, SmugglerResult.from_json(None).documents.read_count) + + def test_result_round_trips(self): + result = SmugglerResult.from_json(self.RESPONSE) + + self.assertEqual(result.to_json(), SmugglerResult.from_json(result.to_json()).to_json()) + + def test_every_section_the_server_sends_has_a_home(self): + keys = set(SmugglerResult().to_json()) + + for name in ( + "DatabaseRecord", + "Documents", + "RevisionDocuments", + "Tombstones", + "Conflicts", + "Identities", + "Indexes", + "CompareExchange", + "Subscriptions", + "Counters", + "CompareExchangeTombstones", + "TimeSeries", + "ReplicationHubCertificates", + "TimeSeriesDeletedRanges", + ): + self.assertIn(name, keys) From c251cc3c4cbfee93136561ae71426c94badd3418 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:02 +0200 Subject: [PATCH 04/14] RDBC-1099 Harden DatabaseRecord and AutoIndexDefinition deserialization --- ravendb/documents/indexes/definitions.py | 31 +++-- ravendb/serverwide/database_record.py | 24 +++- .../operations_tests/test_database_record.py | 109 ++++++++++++++++++ 3 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 ravendb/tests/operations_tests/test_database_record.py diff --git a/ravendb/documents/indexes/definitions.py b/ravendb/documents/indexes/definitions.py index 27d04376..b6b05000 100644 --- a/ravendb/documents/indexes/definitions.py +++ b/ravendb/documents/indexes/definitions.py @@ -393,25 +393,38 @@ def __init__( @classmethod def from_json(cls, json_dict: Dict) -> AutoIndexDefinition: + index_type = json_dict.get("Type") + priority = json_dict.get("Priority") + state = json_dict.get("State") + # The C# constructor starts both field maps empty, and the rest are nullable, so a + # payload that leaves any of them out is well-formed. return cls( - IndexType(json_dict.get("Type")), + IndexType(index_type) if index_type else None, json_dict.get("Name"), - IndexPriority(json_dict.get("Priority")), - IndexState(json_dict.get("State")) if json_dict.get("State", None) else None, + IndexPriority(priority) if priority else None, + IndexState(state) if state else None, json_dict.get("Collection"), - {name: AutoIndexFieldOptions.from_json(value) for name, value in json_dict.get("MapFields").items()}, - {name: AutoIndexFieldOptions.from_json(value) for name, value in json_dict.get("GroupByFields").items()}, + { + name: AutoIndexFieldOptions.from_json(value) + for name, value in (json_dict.get("MapFields") or {}).items() + }, + { + name: AutoIndexFieldOptions.from_json(value) + for name, value in (json_dict.get("GroupByFields") or {}).items() + }, ) def to_json(self) -> Dict: return { - "Type": self.index_type.value, + "Type": self.index_type.value if self.index_type is not None else None, "Name": self.name, - "Priority": self.priority.value, + "Priority": self.priority.value if self.priority is not None else None, "State": self.state.value if self.state is not None else None, "Collection": self.collection, - "MapFields": {key: map_field.to_json() for key, map_field in self.map_fields.items()}, - "GroupByFields": {key: group_by_field.to_json() for key, group_by_field in self.group_by_fields.items()}, + "MapFields": {key: map_field.to_json() for key, map_field in (self.map_fields or {}).items()}, + "GroupByFields": { + key: group_by_field.to_json() for key, group_by_field in (self.group_by_fields or {}).items() + }, } diff --git a/ravendb/serverwide/database_record.py b/ravendb/serverwide/database_record.py index 580a0544..a449e599 100644 --- a/ravendb/serverwide/database_record.py +++ b/ravendb/serverwide/database_record.py @@ -12,6 +12,7 @@ ) from ravendb.documents.operations.backups.settings import PeriodicBackupConfiguration from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration +from ravendb.documents.operations.queue_sink.configuration import QueueSinkConfiguration from ravendb.documents.operations.etl.configuration import RavenConnectionString, RavenEtlConfiguration from ravendb.documents.operations.etl.olap.connection import OlapConnectionString, OlapEtlConfiguration from ravendb.documents.operations.etl.sql import SqlConnectionString, SqlEtlConfiguration @@ -71,6 +72,7 @@ def __init__(self, database_name: Optional[str] = None): self.sql_etls: List[SqlEtlConfiguration] = [] self.olap_etls: List[OlapEtlConfiguration] = [] self.cdc_sinks: List[CdcSinkConfiguration] = [] + self.queue_sinks: List[QueueSinkConfiguration] = [] self.embeddings_generations: List = [] self.client: Optional[ClientConfiguration] = None self.studio: Optional[StudioConfiguration] = None @@ -101,7 +103,7 @@ def to_json(self): "Indexes": self.indexes, "IndexesHistory": self.indexes_history_story, "AutoIndexes": ( - {key: AutoIndexDefinition.to_json(auto_index) for key, auto_index in self.auto_indexes} + {key: auto_index.to_json() for key, auto_index in self.auto_indexes.items()} if self.auto_indexes else None ), @@ -122,6 +124,7 @@ def to_json(self): "SqlEtls": self.sql_etls, "OlapEtls": self.olap_etls, "CdcSinks": [cdc_sink.to_json() for cdc_sink in self.cdc_sinks or []], + "QueueSinks": [queue_sink.to_json() for queue_sink in self.queue_sinks or []], "Client": self.client, "Studio": self.studio, "TruncatedClusterTransactionCommand": self.truncated_cluster_transaction_commands_count, @@ -136,7 +139,12 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.deletion_in_progress = json_dict.get("DeletionInProgress", None) record.rolling_indexes = json_dict.get("RollingIndexes", None) record.database_state = json_dict.get("DatabaseState", None) - record.lock_mode = DatabaseRecord.DatabaseLockMode(json_dict.get("LockMode", None)) + lock_mode = json_dict.get("LockMode") + # A record without a lock mode is unlocked - the C# field has no initializer, so it + # lands on the enum's zero value. + record.lock_mode = ( + DatabaseRecord.DatabaseLockMode(lock_mode) if lock_mode else DatabaseRecord.DatabaseLockMode.UNLOCK + ) record.topology = json_dict.get("Topology", None) record.conflict_solver_config = json_dict.get("ConflictSolverConfig", None) record.documents_compression = ( @@ -148,9 +156,12 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.analyzers = json_dict.get("Analyzers", None) record.indexes = json_dict.get("Indexes", None) record.indexes_history_story = json_dict.get("IndexesHistory", None) - record.auto_indexes = { - key: AutoIndexDefinition.from_json(auto_index) for key, auto_index in json_dict.get("AutoIndexes").items() - } + auto_indexes = json_dict.get("AutoIndexes") + record.auto_indexes = ( + {key: AutoIndexDefinition.from_json(auto_index) for key, auto_index in auto_indexes.items()} + if auto_indexes + else None + ) record.settings = json_dict.get("Settings", None) record.revisions = json_dict.get("Revisions", None) if json_dict.get("TimeSeries", None): @@ -171,6 +182,9 @@ def from_json(cls, json_dict: dict) -> DatabaseRecord: record.sql_etls = json_dict.get("SqlEtls", None) record.olap_etls = json_dict.get("OlapEtls", None) record.cdc_sinks = [CdcSinkConfiguration.from_json(cdc_sink) for cdc_sink in json_dict.get("CdcSinks") or []] + record.queue_sinks = [ + QueueSinkConfiguration.from_json(queue_sink) for queue_sink in json_dict.get("QueueSinks") or [] + ] embeddings_generations_data = json_dict.get("EmbeddingsGenerations", []) if embeddings_generations_data: from ravendb.documents.operations.ai.embeddings_generation_configuration import ( diff --git a/ravendb/tests/operations_tests/test_database_record.py b/ravendb/tests/operations_tests/test_database_record.py new file mode 100644 index 00000000..24f76e76 --- /dev/null +++ b/ravendb/tests/operations_tests/test_database_record.py @@ -0,0 +1,109 @@ +""" +Tests for DatabaseRecord serialization, in particular the fields whose C# counterparts +have no initializer and so are simply absent from a server payload. +""" + +import unittest + +from ravendb.documents.indexes.definitions import AutoIndexDefinition, IndexPriority, IndexType +from ravendb.serverwide.database_record import DatabaseRecord + +AUTO_INDEX = { + "Type": "AutoMap", + "Name": "Auto/Orders/ByCompany", + "Priority": "Normal", + "State": "Normal", + "Collection": "Orders", + "MapFields": {}, + "GroupByFields": {}, +} + + +class TestDatabaseRecordFromJson(unittest.TestCase): + def test_a_payload_without_optional_sections_parses(self): + # The C# fields for these have no initializer, so a server is free to omit them. + record = DatabaseRecord.from_json({"DatabaseName": "db"}) + + self.assertEqual("db", record.database_name) + self.assertIsNone(record.auto_indexes) + self.assertIsNone(record.indexes) + + def test_a_record_without_a_lock_mode_is_unlocked(self): + record = DatabaseRecord.from_json({"DatabaseName": "db"}) + + self.assertEqual(DatabaseRecord.DatabaseLockMode.UNLOCK, record.lock_mode) + + def test_a_lock_mode_is_read_when_the_server_sends_one(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "LockMode": "PreventDeletesError"}) + + self.assertEqual(DatabaseRecord.DatabaseLockMode.PREVENT_DELETES_ERROR, record.lock_mode) + + def test_auto_indexes_are_parsed_into_definitions(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {"Auto/Orders/ByCompany": AUTO_INDEX}}) + + self.assertIsInstance(record.auto_indexes["Auto/Orders/ByCompany"], AutoIndexDefinition) + self.assertEqual("Orders", record.auto_indexes["Auto/Orders/ByCompany"].collection) + + def test_an_empty_auto_index_map_reads_as_no_auto_indexes(self): + self.assertIsNone(DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {}}).auto_indexes) + + +class TestDatabaseRecordToJson(unittest.TestCase): + def test_auto_indexes_are_serialized_by_name(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {"Auto/Orders/ByCompany": AUTO_INDEX}}) + serialized = record.to_json()["AutoIndexes"] + + self.assertEqual(["Auto/Orders/ByCompany"], list(serialized)) + self.assertEqual("Orders", serialized["Auto/Orders/ByCompany"]["Collection"]) + self.assertEqual("AutoMap", serialized["Auto/Orders/ByCompany"]["Type"]) + + def test_a_record_with_no_auto_indexes_writes_none(self): + self.assertIsNone(DatabaseRecord("db").to_json()["AutoIndexes"]) + + def test_a_parsed_record_survives_a_serialization_round_trip(self): + record = DatabaseRecord.from_json({"DatabaseName": "db", "AutoIndexes": {"Auto/Orders/ByCompany": AUTO_INDEX}}) + round_tripped = DatabaseRecord.from_json(record.to_json()) + + self.assertEqual( + record.auto_indexes["Auto/Orders/ByCompany"].collection, + round_tripped.auto_indexes["Auto/Orders/ByCompany"].collection, + ) + + +class TestAutoIndexDefinitionFromJson(unittest.TestCase): + """ + The C# constructor starts MapFields and GroupByFields empty and leaves the rest + nullable, so a payload that omits any of them is well-formed. + """ + + def test_a_minimal_payload_parses(self): + definition = AutoIndexDefinition.from_json({"Name": "Auto/Orders", "Collection": "Orders"}) + + self.assertEqual("Auto/Orders", definition.name) + self.assertEqual("Orders", definition.collection) + self.assertEqual({}, definition.map_fields) + self.assertEqual({}, definition.group_by_fields) + self.assertIsNone(definition.index_type) + self.assertIsNone(definition.priority) + + def test_an_empty_payload_parses(self): + self.assertIsNone(AutoIndexDefinition.from_json({}).name) + + def test_a_full_payload_still_parses(self): + definition = AutoIndexDefinition.from_json(AUTO_INDEX) + + self.assertEqual(IndexType.AUTO_MAP, definition.index_type) + self.assertEqual(IndexPriority.NORMAL, definition.priority) + self.assertEqual("Orders", definition.collection) + + def test_to_json_survives_the_optional_fields_being_unset(self): + serialized = AutoIndexDefinition.from_json({"Name": "Auto/Orders"}).to_json() + + self.assertIsNone(serialized["Type"]) + self.assertIsNone(serialized["Priority"]) + self.assertEqual({}, serialized["MapFields"]) + + def test_definition_round_trips(self): + definition = AutoIndexDefinition.from_json(AUTO_INDEX) + + self.assertEqual(definition.to_json(), AutoIndexDefinition.from_json(definition.to_json()).to_json()) From 6a796b2c270d966eea560c77947373f8c50e945b Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:02 +0200 Subject: [PATCH 05/14] RDBC-1099 Export the queue sink and smuggler types and gate their licensed tests - Export the queue sink and smuggler types from the package root - Gate the licensed queue sink tests behind RAVENDB_LICENSE --- ravendb/__init__.py | 31 +++++++++++++------ .../tests/operations_tests/test_queue_sink.py | 7 +++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ravendb/__init__.py b/ravendb/__init__.py index f3df37c8..1732cc2c 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -200,7 +200,27 @@ OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, OngoingTaskCdcSink, -) + OngoingTaskQueueSink, +) +from ravendb.documents.operations.queue_sink import ( + AddQueueSinkOperation, + AddQueueSinkOperationResult, + AzureServiceBusSinkSource, + QueueSinkConfiguration, + QueueSinkProcessState, + QueueSinkScript, + UpdateQueueSinkOperation, + UpdateQueueSinkOperationResult, +) +from ravendb.documents.smuggler.common import ( + DatabaseItemType, + DatabaseRecordItemType, + DatabaseSmugglerExportOptions, + DatabaseSmugglerImportOptions, + DatabaseSmugglerOptions, + ExportCompressionAlgorithm, +) +from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, AddCdcSinkOperationResult, @@ -732,15 +752,6 @@ # IChangesConnectionState # todo: Smuggler -# DatabaseItemType -# DatabaseRecordItemType -# DatabaseSmuggler -# DatabaseSmugglerExportOptions -# IDatabaseSmugglerExportOptions -# DatabaseSmugglerImportOptions -# IDatabaseSmugglerImportOptions -# DatabaseSmugglerOptions -# IDatabaseSmugglerOptions # todo: Certificates # AddDatabaseNodeOperation diff --git a/ravendb/tests/operations_tests/test_queue_sink.py b/ravendb/tests/operations_tests/test_queue_sink.py index 71f1d63f..e9a25e7e 100644 --- a/ravendb/tests/operations_tests/test_queue_sink.py +++ b/ravendb/tests/operations_tests/test_queue_sink.py @@ -4,6 +4,7 @@ """ import json +import os import unittest from ravendb.documents.operations.etl.queue.connection import QueueBrokerType @@ -223,6 +224,10 @@ def test_a_record_from_a_server_without_queue_sinks_gets_an_empty_list(self): class TestQueueSinkAgainstServer(TestBase): + # Creating a queue sink task is licensed: an unlicensed server accepts it on a fresh + # start but rejects it once the suite has built up databases and tasks, so the two + # tests that create one are gated the way the rest of this repo gates licensed tests. + def setUp(self): super().setUp() self.store.maintenance.send( @@ -235,6 +240,7 @@ def setUp(self): ) ) + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") def test_the_server_stores_a_queue_sink_task_and_reads_it_back(self): result = self.store.maintenance.send(AddQueueSinkOperation(_configuration())) self.assertGreater(result.task_id, 0) @@ -248,6 +254,7 @@ def test_the_server_stores_a_queue_sink_task_and_reads_it_back(self): self.assertEqual(["orders-topic"], task.configuration.scripts[0].queues) self.assertEqual("put('orders/', this);", task.configuration.scripts[0].script) + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") def test_the_task_can_be_updated(self): task_id = self.store.maintenance.send(AddQueueSinkOperation(_configuration())).task_id From 2c27d08b3ad78406ce39395ded2a2bf53c214542 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:02 +0200 Subject: [PATCH 06/14] RDBC-1099 Type license limits with LimitType and LicenseLimitException --- ravendb/exceptions/commercial.py | 89 +++++++++++++++++++ ravendb/exceptions/exception_dispatcher.py | 3 + .../test_commercial_limits.py | 61 +++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 ravendb/exceptions/commercial.py create mode 100644 ravendb/tests/operations_tests/test_commercial_limits.py diff --git a/ravendb/exceptions/commercial.py b/ravendb/exceptions/commercial.py new file mode 100644 index 00000000..8e7cfcc8 --- /dev/null +++ b/ravendb/exceptions/commercial.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import enum +from typing import Optional + +from ravendb.exceptions.raven_exceptions import RavenException + + +class LimitType(enum.Enum): + """ + Which licensed capability a server refused. Carried by LicenseLimitException when the + server names one. + """ + + INVALID_LICENSE = "InvalidLicense" # Invalid License + FORBIDDEN_HOST = "ForbiddenHost" # Forbidden Host + DYNAMIC_NODE_DISTRIBUTION = "DynamicNodeDistribution" # Dynamic Nodes Distribution + CLUSTER_SIZE = "ClusterSize" # Cluster Size + SNAPSHOT_BACKUP = "SnapshotBackup" # Snapshot Backup + CLOUD_BACKUP = "CloudBackup" # Cloud Backup + ENCRYPTION = "Encryption" + DOCUMENTS_COMPRESSION = "DocumentsCompression" # Documents Compression + ROLLING_INDEXES = "RollingIndexes" # Rolling Indexes + EXTERNAL_REPLICATION = "ExternalReplication" # External Replication + RAVEN_ETL = "RavenEtl" # Raven ETL + SQL_ETL = "SqlEtl" # SQL ETL + OLAP_ETL = "OlapEtl" # OLAP ETL + ELASTIC_SEARCH_ETL = "ElasticSearchEtl" # ElasticSearch ETL + QUEUE_ETL = "QueueEtl" # Queue ETL + SNOWFLAKE_ETL = "SnowflakeEtl" # Snowflake ETL + EMBEDDINGS_GENERATION = "EmbeddingsGeneration" # Embeddings Generation + GEN_AI = "GenAi" # Gen AI + AI_AGENT = "AiAgent" # AI Agent + AI_ASSISTANT = "AiAssistant" # AI Assistant + QUILL = "Quill" + CORES = "Cores" # Cores Limit + SNMP = "Snmp" # SNMP + POSTGRE_SQL_INTEGRATION = "PostgreSqlIntegration" # PostgreSql Integration + POWER_B_I = "PowerBI" # Power BI + DELAYED_EXTERNAL_REPLICATION = "DelayedExternalReplication" # Delayed External Replication + HIGHLY_AVAILABLE_TASKS = "HighlyAvailableTasks" # Highly Available Tasks + PULL_REPLICATION_AS_HUB = "PullReplicationAsHub" # Pull Replication As Hub + PULL_REPLICATION_AS_SINK = "PullReplicationAsSink" # Pull Replication As Sink + TIME_SERIES_ROLLUPS_AND_RETENTION = "TimeSeriesRollupsAndRetention" # Time Series Rollups and Retention + ENCRYPTED_BACKUP = "EncryptedBackup" # Encrypted Backup + ADDITIONAL_ASSEMBLIES_FROM_NU_GET = "AdditionalAssembliesFromNuGet" # Additional Assemblies from NuGet + MONITORING_ENDPOINTS = "MonitoringEndpoints" # Monitoring Endpoints + READ_ONLY_CERTIFICATES = "ReadOnlyCertificates" # Read-only Certificates + CONCURRENT_SUBSCRIPTIONS = "ConcurrentSubscriptions" # Concurrent Subscriptions + TCP_DATA_COMPRESSION = "TcpDataCompression" # TCP Data Compression + SERVER_WIDE_BACKUPS = "ServerWideBackups" # Server Wide Backups + SERVER_WIDE_EXTERNAL_REPLICATIONS = "ServerWideExternalReplications" # Server Wide External Replications + SERVER_WIDE_CUSTOM_SORTERS = "ServerWideCustomSorters" # Server Wide Custom Sorters + SERVER_WIDE_ANALYZERS = "ServerWideAnalyzers" # Server Wide Analyzers + SERVER_WIDE_CONNECTION_STRINGS = "ServerWideConnectionStrings" # Server Wide Connection Strings + INDEX_CLEANUP = "IndexCleanup" # Index Cleanup + PERIODIC_BACKUP = "PeriodicBackup" # Periodic Backup + CLIENT_CONFIGURATION = "ClientConfiguration" # Client Configuration + STUDIO_CONFIGURATION = "StudioConfiguration" # Studio Configuration + QUEUE_SINK = "QueueSink" # Queue Sink + CDC_SINK = "CdcSink" # CDC Sink + DATA_ARCHIVAL = "DataArchival" # Data Archival + SHARDING = "Sharding" + SUBSCRIPTIONS = "Subscriptions" + REVISIONS_CONFIGURATION = "RevisionsConfiguration" # Revisions Configuration + EXPIRATION = "Expiration" + REFRESH = "Refresh" + INDEXES = "Indexes" + CUSTOM_SORTERS = "CustomSorters" # Custom Sorters + CUSTOM_ANALYZERS = "CustomAnalyzers" # Custom Analyzers + REMOTE_ATTACHMENTS = "RemoteAttachments" # Remote Attachments + SCHEMA_VALIDATION = "SchemaValidation" # Schema Validation + SSO = "Sso" # SSO + + def __str__(self) -> str: + return self.value + + +class LicenseLimitException(RavenException): + """ + Raised when the server refuses an operation because the license does not cover it. + + The server answers HTTP 402 and names the feature in the message; when it also sends a + machine-readable limit, it lands in :attr:`limit_type`. + """ + + def __init__(self, message: str = None, limit_type: Optional[LimitType] = None): + super().__init__(message) + self.limit_type = limit_type diff --git a/ravendb/exceptions/exception_dispatcher.py b/ravendb/exceptions/exception_dispatcher.py index b06ca2a2..2ac3e600 100644 --- a/ravendb/exceptions/exception_dispatcher.py +++ b/ravendb/exceptions/exception_dispatcher.py @@ -5,6 +5,7 @@ from datetime import timedelta from ravendb.exceptions.cluster import NodeIsPassiveException, NoLoaderException +from ravendb.exceptions.commercial import LicenseLimitException from ravendb.exceptions.documents import DocumentConflictException, DocumentDoesNotExistException from ravendb.exceptions.documents.bulkinsert import BulkInsertAbortedException, BulkInsertProtocolViolationException from ravendb.exceptions.documents.indexes import IndexDoesNotExistException @@ -60,6 +61,8 @@ # cluster "NodeIsPassiveException": NodeIsPassiveException, "NoLoaderException": NoLoaderException, + # commercial + "LicenseLimitException": LicenseLimitException, } diff --git a/ravendb/tests/operations_tests/test_commercial_limits.py b/ravendb/tests/operations_tests/test_commercial_limits.py new file mode 100644 index 00000000..ea5279f8 --- /dev/null +++ b/ravendb/tests/operations_tests/test_commercial_limits.py @@ -0,0 +1,61 @@ +""" +Tests for the commercial limit surface: the LimitType enum the 7.2.5 patch extended, and +LicenseLimitException coming back typed from a 402 instead of a bare RavenException. +""" + +import unittest + +from ravendb.exceptions.commercial import LicenseLimitException, LimitType +from ravendb.exceptions.exception_dispatcher import ExceptionDispatcher +from ravendb.exceptions.raven_exceptions import RavenException + + +class TestLimitType(unittest.TestCase): + def test_the_limits_7_2_5_added(self): + self.assertEqual("ServerWideConnectionStrings", LimitType.SERVER_WIDE_CONNECTION_STRINGS.value) + self.assertEqual("CdcSink", LimitType.CDC_SINK.value) + self.assertEqual("Sso", LimitType.SSO.value) + + def test_a_limit_parses_from_the_name_the_server_uses(self): + self.assertEqual(LimitType.QUEUE_SINK, LimitType("QueueSink")) + self.assertEqual(LimitType.SCHEMA_VALIDATION, LimitType("SchemaValidation")) + + def test_every_member_stringifies_to_its_wire_name(self): + for member in LimitType: + self.assertEqual(member.value, str(member)) + + +class TestLicenseLimitException(unittest.TestCase): + def test_it_is_a_raven_exception(self): + self.assertIsInstance(LicenseLimitException("nope"), RavenException) + + def test_it_can_carry_the_limit_it_hit(self): + exception = LicenseLimitException("nope", LimitType.QUEUE_SINK) + + self.assertEqual(LimitType.QUEUE_SINK, exception.limit_type) + + def test_the_dispatcher_returns_it_for_a_402(self): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", + object_type="Raven.Client.Exceptions.Commercial.LicenseLimitException", + message="no", + error="Your license doesn't support using the queue sink feature.", + ) + + exception = ExceptionDispatcher.get(schema, 402) + + self.assertIsInstance(exception, LicenseLimitException) + self.assertIn("queue sink", str(exception)) + # The server does not put the limit on the wire, so it stays unset here, exactly + # as it does in the C# client. + self.assertIsNone(exception.limit_type) + + def test_a_402_from_another_exception_type_is_left_alone(self): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", + object_type="Raven.Client.Exceptions.RavenException", + message="no", + error="something else", + ) + + self.assertNotIsInstance(ExceptionDispatcher.get(schema, 402), LicenseLimitException) From 7f56b0e169ce5bb50b2edc68987df8761d8aca32 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:03 +0200 Subject: [PATCH 07/14] RDBC-1099 Convert between backup and remote-attachment cloud settings Ports RemoteAttachmentExtensions. Backup S3Settings gains the StorageClass it was missing, and the S3 and Azure from_json / to_json paths stop requiring keys that C# treats as optional, since a converted instance carries no backup script. --- .../operations/attachments/__init__.py | 43 ++++ .../documents/operations/backups/settings.py | 93 ++++++-- .../operations_tests/test_s3_settings.py | 207 ++++++++++++++---- 3 files changed, 275 insertions(+), 68 deletions(-) diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index b4924446..53c0efa8 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -449,6 +449,30 @@ def to_json(self) -> dict: result["StorageClass"] = self.storage_class.value return result + def to_s3_settings(self) -> Optional["S3Settings"]: + """ + The same bucket as periodic-backup settings, enabled for direct upload. + Returns None when the bucket is not set, which is the minimum the server needs. + """ + from ravendb.documents.operations.backups.settings import S3Settings + + if not self.bucket_name or self.bucket_name.isspace(): + return None + + return S3Settings( + disabled=False, + aws_access_key=self.aws_access_key, + aws_secret_key=self.aws_secret_key, + aws_session_token=self.aws_session_token, + aws_region_name=self.aws_region_name, + remote_folder_name=self.remote_folder_name, + bucket_name=self.bucket_name, + custom_server_url=self.custom_server_url, + force_path_style=self.force_path_style, + disable_checksum_validation=self.disable_checksum_validation, + storage_class=self.storage_class, + ) + class RemoteAttachmentsAzureSettings: def __init__( @@ -484,6 +508,25 @@ def to_json(self) -> dict: "SasToken": self.sas_token, } + def to_azure_settings(self) -> Optional["AzureSettings"]: + """ + The same container as periodic-backup settings, enabled for direct upload. + Returns None when the container is not set, which is the minimum the server needs. + """ + from ravendb.documents.operations.backups.settings import AzureSettings + + if not self.storage_container or self.storage_container.isspace(): + return None + + return AzureSettings( + disabled=False, + storage_container=self.storage_container, + remote_folder_name=self.remote_folder_name, + account_name=self.account_name, + account_key=self.account_key, + sas_token=self.sas_token, + ) + class RemoteAttachmentsDestinationConfiguration: def __init__( diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 0a242863..99f01014 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -62,7 +62,9 @@ def __init__(self, disabled: bool = None, get_backup_configuration_script: GetBa def to_json(self) -> Dict[str, Any]: return { "Disabled": self.disabled, - "GetBackupConfigurationScript": self.get_backup_configuration_script.to_json(), + "GetBackupConfigurationScript": ( + self.get_backup_configuration_script.to_json() if self.get_backup_configuration_script else None + ), } @@ -131,6 +133,7 @@ def __init__( custom_server_url: str = None, force_path_style: bool = None, disable_checksum_validation: bool = None, + storage_class: "S3StorageClass" = None, ): super().__init__( disabled, @@ -145,27 +148,37 @@ def __init__( self.custom_server_url = custom_server_url self.force_path_style = force_path_style self.disable_checksum_validation = disable_checksum_validation + self.storage_class = storage_class @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> S3Settings: return cls( - json_dict["Disabled"], - GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]), - json_dict["AwsAccessKey"], - json_dict["AwsSecretKey"], - json_dict["AwsSessionToken"], - json_dict["AwsRegionName"], - json_dict["RemoteFolderName"], - json_dict["BucketName"], - json_dict["CustomServerUrl"], - json_dict["ForcePathStyle"], + json_dict.get("Disabled"), + ( + GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]) + if json_dict.get("GetBackupConfigurationScript") + else None + ), + # C# deserializes by reflection, so a key the server left out just keeps the + # field's default rather than failing. + json_dict.get("AwsAccessKey"), + json_dict.get("AwsSecretKey"), + json_dict.get("AwsSessionToken"), + json_dict.get("AwsRegionName"), + json_dict.get("RemoteFolderName"), + json_dict.get("BucketName"), + json_dict.get("CustomServerUrl"), + json_dict.get("ForcePathStyle"), json_dict.get("DisableChecksumValidation"), + S3StorageClass(json_dict["StorageClass"]) if json_dict.get("StorageClass") else None, ) def to_json(self) -> Dict[str, Any]: return { "Disabled": self.disabled, - "GetBackupConfigurationScript": self.get_backup_configuration_script.to_json(), + "GetBackupConfigurationScript": ( + self.get_backup_configuration_script.to_json() if self.get_backup_configuration_script else None + ), "AwsAccessKey": self.aws_access_key, "AwsSecretKey": self.aws_secret_key, "AwsSessionToken": self.aws_session_token, @@ -175,8 +188,28 @@ def to_json(self) -> Dict[str, Any]: "CustomServerUrl": self.custom_server_url, "ForcePathStyle": self.force_path_style, "DisableChecksumValidation": self.disable_checksum_validation, + # The server treats an absent storage class as its own default, so only send one + # when the caller picked it. + **({"StorageClass": self.storage_class.value} if self.storage_class is not None else {}), } + def to_remote_attachments_s3_settings(self) -> "RemoteAttachmentsS3Settings": + """The same bucket as remote-attachment settings, dropping the backup-only fields.""" + from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings + + return RemoteAttachmentsS3Settings( + aws_access_key=self.aws_access_key, + aws_secret_key=self.aws_secret_key, + aws_session_token=self.aws_session_token, + aws_region_name=self.aws_region_name, + remote_folder_name=self.remote_folder_name, + bucket_name=self.bucket_name, + custom_server_url=self.custom_server_url, + force_path_style=self.force_path_style, + disable_checksum_validation=self.disable_checksum_validation, + storage_class=self.storage_class, + ) + class GlacierSettings(AmazonSettings): def __init__( @@ -205,7 +238,11 @@ def __init__( def from_json(cls, json_dict: Dict[str, Any]) -> GlacierSettings: return cls( json_dict["Disabled"], - GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]), + ( + GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]) + if json_dict.get("GetBackupConfigurationScript") + else None + ), json_dict["AwsAccessKey"], json_dict["AwsSecretKey"], json_dict["AwsSessionToken"], @@ -217,7 +254,9 @@ def from_json(cls, json_dict: Dict[str, Any]) -> GlacierSettings: def to_json(self) -> Dict[str, Any]: return { "Disabled": self.disabled, - "GetBackupConfigurationScript": self.get_backup_configuration_script.to_json(), + "GetBackupConfigurationScript": ( + self.get_backup_configuration_script.to_json() if self.get_backup_configuration_script else None + ), "AwsAccessKey": self.aws_access_key, "AwsSecretKey": self.aws_secret_key, "AwsSessionToken": self.aws_session_token, @@ -248,17 +287,17 @@ def __init__( @classmethod def from_json(cls, json_dict: Dict[str, Any]) -> AzureSettings: return cls( - json_dict["Disabled"], + json_dict.get("Disabled"), ( GetBackupConfigurationScript.from_json(json_dict["GetBackupConfigurationScript"]) - if json_dict["GetBackupConfigurationScript"] + if json_dict.get("GetBackupConfigurationScript") else None ), - json_dict["StorageContainer"], - json_dict["RemoteFolderName"], - json_dict["AccountName"], - json_dict["AccountKey"], - json_dict["SasToken"], + json_dict.get("StorageContainer"), + json_dict.get("RemoteFolderName"), + json_dict.get("AccountName"), + json_dict.get("AccountKey"), + json_dict.get("SasToken"), ) def to_json(self) -> Dict[str, Any]: @@ -274,6 +313,18 @@ def to_json(self) -> Dict[str, Any]: "SasToken": self.sas_token, } + def to_remote_attachments_azure_settings(self) -> "RemoteAttachmentsAzureSettings": + """The same container as remote-attachment settings, dropping the backup-only fields.""" + from ravendb.documents.operations.attachments import RemoteAttachmentsAzureSettings + + return RemoteAttachmentsAzureSettings( + storage_container=self.storage_container, + remote_folder_name=self.remote_folder_name, + account_name=self.account_name, + account_key=self.account_key, + sas_token=self.sas_token, + ) + class FtpSettings(BackupSettings): def __init__( diff --git a/ravendb/tests/operations_tests/test_s3_settings.py b/ravendb/tests/operations_tests/test_s3_settings.py index f4a93300..7d04c2f1 100644 --- a/ravendb/tests/operations_tests/test_s3_settings.py +++ b/ravendb/tests/operations_tests/test_s3_settings.py @@ -1,69 +1,182 @@ """ -Tests for DisableChecksumValidation, added in 7.2.5 to both S3 settings classes for -S3-compatible storage that does not support modern object integrity checks. +Tests for the S3 settings the 7.2.5 patch touched: the DisableChecksumValidation flag on +both settings classes, and the conversions between periodic-backup and remote-attachment +settings that the patch kept in step. """ import unittest -from ravendb.documents.operations.attachments import RemoteAttachmentsS3Settings -from ravendb.documents.operations.backups.settings import GetBackupConfigurationScript, S3Settings +from ravendb.documents.operations.attachments import RemoteAttachmentsAzureSettings, RemoteAttachmentsS3Settings +from ravendb.documents.operations.backups.settings import ( + AzureSettings, + GetBackupConfigurationScript, + S3Settings, + S3StorageClass, +) -class TestBackupS3Settings(unittest.TestCase): - def _settings(self, **kwargs) -> S3Settings: - return S3Settings( - disabled=False, - get_backup_configuration_script=GetBackupConfigurationScript(), - aws_access_key="key", - aws_secret_key="secret", - aws_session_token=None, - aws_region_name="eu-central-1", - remote_folder_name="backups", - bucket_name="rvn", - custom_server_url="https://minio.local", - force_path_style=True, - **kwargs, - ) +def _remote_settings() -> RemoteAttachmentsS3Settings: + return RemoteAttachmentsS3Settings( + aws_access_key="key", + aws_secret_key="secret", + aws_session_token="token", + aws_region_name="eu-central-1", + remote_folder_name="attachments", + bucket_name="bucket", + custom_server_url="https://minio.local", + force_path_style=True, + disable_checksum_validation=True, + storage_class=S3StorageClass.GLACIER, + ) + + +class TestDisableChecksumValidation(unittest.TestCase): + def test_remote_attachment_settings_carry_the_flag(self): + serialized = _remote_settings().to_json() + + self.assertTrue(serialized["DisableChecksumValidation"]) + self.assertTrue(RemoteAttachmentsS3Settings.from_json(serialized).disable_checksum_validation) + + def test_backup_settings_carry_the_flag(self): + settings = S3Settings(bucket_name="bucket", disable_checksum_validation=True) + + self.assertTrue(settings.to_json()["DisableChecksumValidation"]) + self.assertTrue(S3Settings.from_json(settings.to_json()).disable_checksum_validation) + + def test_the_flag_is_absent_rather_than_true_by_default(self): + # Checksum validation protects data integrity, so opting out has to be deliberate. + self.assertIsNone(S3Settings(bucket_name="bucket").to_json()["DisableChecksumValidation"]) + self.assertIsNone(RemoteAttachmentsS3Settings(bucket_name="bucket").to_json()["DisableChecksumValidation"]) + + +class TestBackupSettingsStorageClass(unittest.TestCase): + def test_a_chosen_storage_class_is_sent(self): + settings = S3Settings(bucket_name="bucket", storage_class=S3StorageClass.GLACIER) + + self.assertEqual("Glacier", settings.to_json()["StorageClass"]) + + def test_an_unset_storage_class_is_left_to_the_server(self): + self.assertNotIn("StorageClass", S3Settings(bucket_name="bucket").to_json()) + + def test_storage_class_round_trips(self): + settings = S3Settings(bucket_name="bucket", storage_class=S3StorageClass.STANDARD) + + self.assertEqual(S3StorageClass.STANDARD, S3Settings.from_json(settings.to_json()).storage_class) + + +class TestBackupSettingsWithoutAScript(unittest.TestCase): + def test_settings_without_a_configuration_script_serialize(self): + # C# writes GetBackupConfigurationScript?.ToJson(), so an absent script is normal. + self.assertIsNone(S3Settings(bucket_name="bucket").to_json()["GetBackupConfigurationScript"]) + + def test_settings_without_a_configuration_script_parse(self): + settings = S3Settings.from_json({"Disabled": False, "BucketName": "bucket"}) - def test_checksum_validation_is_left_alone_by_default(self): - self.assertIsNone(self._settings().disable_checksum_validation) + self.assertEqual("bucket", settings.bucket_name) + self.assertIsNone(settings.get_backup_configuration_script) - def test_the_flag_is_written(self): - self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + def test_a_script_still_round_trips(self): + settings = S3Settings(bucket_name="bucket", get_backup_configuration_script=GetBackupConfigurationScript("run")) - def test_the_flag_round_trips(self): - serialized = self._settings(disable_checksum_validation=True).to_json() + self.assertEqual("run", S3Settings.from_json(settings.to_json()).get_backup_configuration_script.exec) - self.assertEqual(serialized, S3Settings.from_json(serialized).to_json()) - def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): - serialized = self._settings().to_json() - del serialized["DisableChecksumValidation"] +class TestS3SettingsConversions(unittest.TestCase): + def test_remote_settings_become_backup_settings(self): + backup = _remote_settings().to_s3_settings() - self.assertIsNone(S3Settings.from_json(serialized).disable_checksum_validation) + self.assertIsInstance(backup, S3Settings) + self.assertEqual("bucket", backup.bucket_name) + self.assertEqual("eu-central-1", backup.aws_region_name) + self.assertTrue(backup.force_path_style) + self.assertTrue(backup.disable_checksum_validation) + self.assertEqual(S3StorageClass.GLACIER, backup.storage_class) + def test_the_conversion_enables_the_settings_for_direct_upload(self): + self.assertFalse(_remote_settings().to_s3_settings().disabled) -class TestRemoteAttachmentsS3Settings(unittest.TestCase): - def _settings(self, **kwargs) -> RemoteAttachmentsS3Settings: - return RemoteAttachmentsS3Settings( + def test_a_bucketless_remote_setting_converts_to_nothing(self): + # The bucket is the minimum the server needs, so there is nothing to convert. + self.assertIsNone(RemoteAttachmentsS3Settings(aws_access_key="key").to_s3_settings()) + self.assertIsNone(RemoteAttachmentsS3Settings(bucket_name=" ").to_s3_settings()) + + def test_backup_settings_become_remote_settings(self): + remote = S3Settings( + bucket_name="bucket", aws_access_key="key", - aws_secret_key="secret", - aws_region_name="eu-central-1", - bucket_name="rvn", - force_path_style=True, - **kwargs, + disable_checksum_validation=True, + storage_class=S3StorageClass.GLACIER, + ).to_remote_attachments_s3_settings() + + self.assertIsInstance(remote, RemoteAttachmentsS3Settings) + self.assertEqual("bucket", remote.bucket_name) + self.assertTrue(remote.disable_checksum_validation) + self.assertEqual(S3StorageClass.GLACIER, remote.storage_class) + + def test_a_conversion_round_trip_keeps_every_shared_field(self): + original = _remote_settings() + + self.assertEqual(original.to_json(), original.to_s3_settings().to_remote_attachments_s3_settings().to_json()) + + def test_backup_only_fields_are_dropped_on_the_way_out(self): + backup = S3Settings( + bucket_name="bucket", + disabled=True, + get_backup_configuration_script=GetBackupConfigurationScript("run"), + ) + remote = backup.to_remote_attachments_s3_settings() + + self.assertNotIn("Disabled", remote.to_json()) + self.assertNotIn("GetBackupConfigurationScript", remote.to_json()) + + +class TestAzureSettingsConversions(unittest.TestCase): + def _remote(self) -> RemoteAttachmentsAzureSettings: + return RemoteAttachmentsAzureSettings( + storage_container="container", + remote_folder_name="attachments", + account_name="account", + account_key="key", + sas_token="token", ) - def test_checksum_validation_is_left_alone_by_default(self): - self.assertIsNone(self._settings().disable_checksum_validation) + def test_remote_settings_become_backup_settings(self): + backup = self._remote().to_azure_settings() + + self.assertIsInstance(backup, AzureSettings) + self.assertEqual("container", backup.storage_container) + self.assertEqual("account", backup.account_name) + self.assertFalse(backup.disabled) + + def test_a_containerless_remote_setting_converts_to_nothing(self): + self.assertIsNone(RemoteAttachmentsAzureSettings(account_name="account").to_azure_settings()) + self.assertIsNone(RemoteAttachmentsAzureSettings(storage_container=" ").to_azure_settings()) + + def test_backup_settings_become_remote_settings(self): + remote = AzureSettings( + storage_container="container", account_name="account" + ).to_remote_attachments_azure_settings() - def test_the_flag_is_written(self): - self.assertTrue(self._settings(disable_checksum_validation=True).to_json()["DisableChecksumValidation"]) + self.assertIsInstance(remote, RemoteAttachmentsAzureSettings) + self.assertEqual("container", remote.storage_container) - def test_the_flag_round_trips(self): - serialized = self._settings(disable_checksum_validation=True).to_json() + def test_a_conversion_round_trip_keeps_every_shared_field(self): + original = self._remote() + + self.assertEqual( + original.to_json(), original.to_azure_settings().to_remote_attachments_azure_settings().to_json() + ) + + def test_backup_only_fields_are_dropped_on_the_way_out(self): + backup = AzureSettings( + storage_container="container", + disabled=True, + get_backup_configuration_script=GetBackupConfigurationScript("run"), + ) + remote = backup.to_remote_attachments_azure_settings() - self.assertEqual(serialized, RemoteAttachmentsS3Settings.from_json(serialized).to_json()) + self.assertNotIn("Disabled", remote.to_json()) + self.assertNotIn("GetBackupConfigurationScript", remote.to_json()) - def test_settings_from_a_pre_7_2_5_server_parse_without_the_flag(self): - self.assertIsNone(RemoteAttachmentsS3Settings.from_json({"BucketName": "rvn"}).disable_checksum_validation) + def test_partial_backup_settings_parse(self): + self.assertEqual("container", AzureSettings.from_json({"StorageContainer": "container"}).storage_container) From 9874829e4206df5343602f860288121f6361fd45 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:03 +0200 Subject: [PATCH 08/14] RDBC-1099 Add CDC Sink schema discovery and mapping preview --- .../documents/operations/cdc_sink/__init__.py | 16 ++ .../documents/operations/cdc_sink/schema.py | 271 ++++++++++++++++++ .../documents/operations/cdc_sink/testing.py | 213 ++++++++++++++ .../tests/operations_tests/test_cdc_sink.py | 268 +++++++++++++++++ 4 files changed, 768 insertions(+) create mode 100644 ravendb/documents/operations/cdc_sink/schema.py create mode 100644 ravendb/documents/operations/cdc_sink/testing.py diff --git a/ravendb/documents/operations/cdc_sink/__init__.py b/ravendb/documents/operations/cdc_sink/__init__.py index e76c8a87..4eb5100f 100644 --- a/ravendb/documents/operations/cdc_sink/__init__.py +++ b/ravendb/documents/operations/cdc_sink/__init__.py @@ -21,6 +21,22 @@ CdcSinkTaskState, UpdateCdcSinkOperationResult, ) +from ravendb.documents.operations.cdc_sink.schema import ( + CdcSinkSchemaRequest, + CdcSinkSourceColumn, + CdcSinkSourceForeignKey, + CdcSinkSourceSchema, + CdcSinkSourceTable, + GetCdcSinkSchemaOperation, +) +from ravendb.documents.operations.cdc_sink.testing import ( + TestCdcSinkMappingOperation, + TestCdcSinkMappingRequest, + TestCdcSinkMappingResult, + TestCdcSinkOperation, + TestCdcSinkRowResult, + TestCdcSinkRowSelector, +) from ravendb.documents.operations.definitions import MaintenanceOperation from ravendb.http.raven_command import RavenCommand from ravendb.http.server_node import ServerNode diff --git a/ravendb/documents/operations/cdc_sink/schema.py b/ravendb/documents/operations/cdc_sink/schema.py new file mode 100644 index 00000000..798a47a8 --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/schema.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import requests + +from ravendb.documents.operations.cdc_sink.configuration import CdcColumnType +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class CdcSinkSourceColumn: + """ + One source column as CDC schema discovery sees it. The field names mirror + CdcColumnMapping so a discovered column can be turned into a mapping directly. + """ + + def __init__( + self, + name: str = None, + native_type: str = None, + suggested_type: CdcColumnType = CdcColumnType.DEFAULT, + is_primary_key: bool = False, + is_cdc_capturable: bool = False, + unsupported_reason: str = None, + ): + self.name = name + # The source-side type as the database reports it, e.g. "varchar", "jsonb". + self.native_type = native_type + self.suggested_type = suggested_type + self.is_primary_key = is_primary_key + # False when the source type has no CDC mapping, or the column is not in + # SQL Server's capture list. The reason is then in unsupported_reason. + self.is_cdc_capturable = is_cdc_capturable + self.unsupported_reason = unsupported_reason + + def to_json(self) -> Dict[str, Any]: + return { + "Name": self.name, + "NativeType": self.native_type, + "SuggestedType": self.suggested_type.value if self.suggested_type else None, + "IsPrimaryKey": self.is_primary_key, + "IsCdcCapturable": self.is_cdc_capturable, + "UnsupportedReason": self.unsupported_reason, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceColumn: + suggested_type = json_dict.get("SuggestedType") + return cls( + name=json_dict.get("Name"), + native_type=json_dict.get("NativeType"), + suggested_type=CdcColumnType(suggested_type) if suggested_type else CdcColumnType.DEFAULT, + is_primary_key=json_dict.get("IsPrimaryKey", False), + is_cdc_capturable=json_dict.get("IsCdcCapturable", False), + unsupported_reason=json_dict.get("UnsupportedReason"), + ) + + +class CdcSinkSourceForeignKey: + """A foreign key leaving a source table, which is what a linked table is built from.""" + + def __init__( + self, + columns: List[str] = None, + referenced_schema: str = None, + referenced_table: str = None, + referenced_columns: List[str] = None, + ): + self.columns = columns or [] + self.referenced_schema = referenced_schema + self.referenced_table = referenced_table + self.referenced_columns = referenced_columns or [] + + def to_json(self) -> Dict[str, Any]: + return { + "Columns": self.columns, + "ReferencedSchema": self.referenced_schema, + "ReferencedTable": self.referenced_table, + "ReferencedColumns": self.referenced_columns, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceForeignKey: + return cls( + columns=json_dict.get("Columns") or [], + referenced_schema=json_dict.get("ReferencedSchema"), + referenced_table=json_dict.get("ReferencedTable"), + referenced_columns=json_dict.get("ReferencedColumns") or [], + ) + + +class CdcSinkSourceTable: + """One source table as CDC schema discovery sees it, annotated with capturability.""" + + def __init__( + self, + source_table_schema: str = None, + source_table_name: str = None, + columns: List[CdcSinkSourceColumn] = None, + primary_key_columns: List[str] = None, + foreign_keys: List[CdcSinkSourceForeignKey] = None, + is_cdc_enabled: bool = False, + unsupported_reason: str = None, + warnings: List[str] = None, + ): + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.columns = columns or [] + self.primary_key_columns = primary_key_columns or [] + self.foreign_keys = foreign_keys or [] + # Whether CDC tracking is already active at the source. Always true for + # PostgreSQL and MySQL, where membership is a database-level concern. + self.is_cdc_enabled = is_cdc_enabled + # Set when the whole table cannot be captured. + self.unsupported_reason = unsupported_reason + # Table-scoped findings that do not make it unusable, such as a REPLICA + # IDENTITY that will not carry row-identifying columns on DELETE. + self.warnings = warnings or [] + + def to_json(self) -> Dict[str, Any]: + return { + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "Columns": [column.to_json() for column in self.columns], + "PrimaryKeyColumns": self.primary_key_columns, + "ForeignKeys": [foreign_key.to_json() for foreign_key in self.foreign_keys], + "IsCdcEnabled": self.is_cdc_enabled, + "UnsupportedReason": self.unsupported_reason, + "Warnings": self.warnings, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceTable: + return cls( + source_table_schema=json_dict.get("SourceTableSchema"), + source_table_name=json_dict.get("SourceTableName"), + columns=[CdcSinkSourceColumn.from_json(column) for column in json_dict.get("Columns") or []], + primary_key_columns=json_dict.get("PrimaryKeyColumns") or [], + foreign_keys=[ + CdcSinkSourceForeignKey.from_json(foreign_key) for foreign_key in json_dict.get("ForeignKeys") or [] + ], + is_cdc_enabled=json_dict.get("IsCdcEnabled", False), + unsupported_reason=json_dict.get("UnsupportedReason"), + warnings=json_dict.get("Warnings") or [], + ) + + +class CdcSinkSourceSchema: + """The source database's tables, columns, keys and CDC readiness.""" + + def __init__( + self, + catalog_name: str = None, + tables: List[CdcSinkSourceTable] = None, + errors: List[str] = None, + has_permission_to_setup: bool = False, + warnings: List[str] = None, + ): + self.catalog_name = catalog_name + self.tables = tables or [] + # Whole-request failures: validation, an unreachable source, or a connection-level + # blocker such as PostgreSQL wal_level not being logical. + self.errors = errors or [] + # Whether the connecting user can provision CDC itself. Distinct from a table's + # is_cdc_enabled, which says whether CDC is already running. + self.has_permission_to_setup = has_permission_to_setup + self.warnings = warnings or [] + + @property + def success(self) -> bool: + """True when nothing blocks setting CDC up. Warnings do not count against it.""" + return len(self.errors) == 0 + + def to_json(self) -> Dict[str, Any]: + return { + "CatalogName": self.catalog_name, + "Tables": [table.to_json() for table in self.tables], + "Errors": self.errors, + "HasPermissionToSetup": self.has_permission_to_setup, + "Warnings": self.warnings, + "Success": self.success, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> CdcSinkSourceSchema: + return cls( + catalog_name=json_dict.get("CatalogName"), + tables=[CdcSinkSourceTable.from_json(table) for table in json_dict.get("Tables") or []], + errors=json_dict.get("Errors") or [], + has_permission_to_setup=json_dict.get("HasPermissionToSetup", False), + warnings=json_dict.get("Warnings") or [], + ) + + +class CdcSinkSchemaRequest: + """The body of a schema-discovery call. Give it either a connection or a name.""" + + def __init__( + self, + connection: SqlConnectionString = None, + connection_string_name: str = None, + schemas: List[str] = None, + ): + # Inline credentials, for when the connection has not been saved to the database + # record yet. When set, connection_string_name is ignored. + self.connection = connection + self.connection_string_name = connection_string_name + # PostgreSQL only, defaults to ["public"] server-side. Each entry is validated + # against ^[A-Za-z_][A-Za-z0-9_]*$, so quoted names with hyphens are rejected. + self.schemas = schemas + + def to_json(self) -> Dict[str, Any]: + return { + "Connection": self.connection.to_json() if self.connection else None, + "ConnectionStringName": self.connection_string_name, + "Schemas": self.schemas, + } + + +class GetCdcSinkSchemaOperation(MaintenanceOperation[CdcSinkSourceSchema]): + """ + Browses the source database a CDC Sink would read: tables, columns, primary and + foreign keys, each annotated with what CDC can capture. Requires DatabaseAdmin. + """ + + def __init__( + self, + connection_or_name: Any = None, + schemas: List[str] = None, + request: CdcSinkSchemaRequest = None, + ): + if request is not None: + self._request = request + elif isinstance(connection_or_name, SqlConnectionString): + self._request = CdcSinkSchemaRequest(connection=connection_or_name, schemas=schemas) + elif isinstance(connection_or_name, str): + self._request = CdcSinkSchemaRequest(connection_string_name=connection_or_name, schemas=schemas) + else: + raise ValueError("Pass either a SqlConnectionString, a connection string name, or a request") + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[CdcSinkSourceSchema]: + return self._GetCdcSinkSchemaCommand(self._request) + + class _GetCdcSinkSchemaCommand(RavenCommand[CdcSinkSourceSchema]): + def __init__(self, request: CdcSinkSchemaRequest): + super().__init__(CdcSinkSourceSchema) + self._request = request + + def is_read_request(self) -> bool: + # A POST that changes nothing server-side, so failover to the fastest node is fine. + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink/schema" + + request = requests.Request("POST", url) + request.data = self._request.to_json() + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = CdcSinkSourceSchema.from_json(json.loads(response)) diff --git a/ravendb/documents/operations/cdc_sink/testing.py b/ravendb/documents/operations/cdc_sink/testing.py new file mode 100644 index 00000000..40ddce5c --- /dev/null +++ b/ravendb/documents/operations/cdc_sink/testing.py @@ -0,0 +1,213 @@ +""" +The CDC Sink mapping preview: run rows from the source table through the configured +mapping and patches without saving anything, to see what documents would come out. + +Named ``testing`` rather than ``test`` so ``unittest discover`` does not pick the module +up as a test file. +""" + +from __future__ import annotations + +import enum +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import requests + +from ravendb.documents.operations.cdc_sink.configuration import CdcSinkConfiguration +from ravendb.documents.operations.definitions import MaintenanceOperation +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + + +class TestCdcSinkRowSelector(enum.Enum): + """Sample the first N rows by primary key, or fetch one row by its key values.""" + + FIRST = "First" + BY_PRIMARY_KEY = "ByPrimaryKey" + + def __str__(self) -> str: + return self.value + + +class TestCdcSinkOperation(enum.Enum): + """Whether to drive each row through the table's Patch or its OnDelete.Patch.""" + + UPSERT = "Upsert" + DELETE = "Delete" + + def __str__(self) -> str: + return self.value + + +class TestCdcSinkRowResult: + """ + One row's worth of preview output. ``document`` and ``source_row`` come back as JSON + text rather than parsed objects, so parse them yourself when you need the values. + """ + + def __init__( + self, + document_id: str = None, + document: str = None, + source_row: str = None, + would_delete: bool = False, + ignore_deletes: bool = False, + debug_output: List[str] = None, + error: str = None, + ): + self.document_id = document_id + # The mapped and patched document, as JSON text. Stays at the pre-patch mapping + # if the script called del() or put(). + self.document = document + self.source_row = source_row + # True for a Delete run when OnDelete.IgnoreDeletes is false. + self.would_delete = would_delete + self.ignore_deletes = ignore_deletes + # Whatever the patch script passed to output(). + self.debug_output = debug_output + # Set when this row failed on its own, for example the patch threw. + self.error = error + + def to_json(self) -> Dict[str, Any]: + return { + "DocumentId": self.document_id, + "Document": self.document, + "SourceRow": self.source_row, + "WouldDelete": self.would_delete, + "IgnoreDeletes": self.ignore_deletes, + "DebugOutput": self.debug_output, + "Error": self.error, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> TestCdcSinkRowResult: + return cls( + document_id=json_dict.get("DocumentId"), + document=json_dict.get("Document"), + source_row=json_dict.get("SourceRow"), + would_delete=json_dict.get("WouldDelete", False), + ignore_deletes=json_dict.get("IgnoreDeletes", False), + debug_output=json_dict.get("DebugOutput"), + error=json_dict.get("Error"), + ) + + +class TestCdcSinkMappingResult: + """ + Always an array shape, so a single-row test and a multi-row sample read the same. + A row that failed on its own carries its error; a failure that stopped the whole + request lands in ``errors`` and leaves ``results`` empty. + """ + + def __init__( + self, + results: List[TestCdcSinkRowResult] = None, + errors: List[str] = None, + warnings: List[str] = None, + ): + self.results = results or [] + self.errors = errors or [] + # Advisory notes that do not invalidate the results, for example that linked and + # embedded tables are not exercised in test mode. + self.warnings = warnings or [] + + def to_json(self) -> Dict[str, Any]: + return { + "Results": [result.to_json() for result in self.results], + "Errors": self.errors, + "Warnings": self.warnings, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> TestCdcSinkMappingResult: + return cls( + results=[TestCdcSinkRowResult.from_json(result) for result in json_dict.get("Results") or []], + errors=json_dict.get("Errors") or [], + warnings=json_dict.get("Warnings") or [], + ) + + +class TestCdcSinkMappingRequest: + """The body of a mapping preview: the task configuration plus which rows to run.""" + + def __init__( + self, + configuration: CdcSinkConfiguration = None, + connection: SqlConnectionString = None, + source_table_schema: str = None, + source_table_name: str = None, + row_selector: TestCdcSinkRowSelector = TestCdcSinkRowSelector.FIRST, + primary_key_values: List[str] = None, + operation: TestCdcSinkOperation = TestCdcSinkOperation.UPSERT, + max_rows: int = 1, + ): + # The same configuration used to create the task: driver, table lookup, column + # mapping and patch scripts all come from here. + self.configuration = configuration + # Inline credentials, for when the connection has not been saved yet. When null, + # the configuration's connection_string_name is used. + self.connection = connection + self.source_table_schema = source_table_schema + self.source_table_name = source_table_name + self.row_selector = row_selector + # In the target table's primary key order. Required with BY_PRIMARY_KEY. + self.primary_key_values = primary_key_values + self.operation = operation + # Only meaningful with FIRST, and must be 1 with BY_PRIMARY_KEY. Capped at 5,000 + # server-side: this endpoint is a preview, not a bulk fetch. + self.max_rows = max_rows + + def to_json(self) -> Dict[str, Any]: + return { + "Configuration": self.configuration.to_json() if self.configuration else None, + "Connection": self.connection.to_json() if self.connection else None, + "SourceTableSchema": self.source_table_schema, + "SourceTableName": self.source_table_name, + "RowSelector": self.row_selector.value if self.row_selector else None, + "PrimaryKeyValues": self.primary_key_values, + "Operation": self.operation.value if self.operation else None, + "MaxRows": self.max_rows, + } + + +class TestCdcSinkMappingOperation(MaintenanceOperation[TestCdcSinkMappingResult]): + """ + Previews how source rows would become documents, before saving a CDC Sink task. + Requires DatabaseAdmin. + """ + + def __init__(self, request: TestCdcSinkMappingRequest): + if request is None: + raise ValueError("request cannot be None") + + self._request = request + + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[TestCdcSinkMappingResult]: + return self._TestCdcSinkMappingCommand(self._request) + + class _TestCdcSinkMappingCommand(RavenCommand[TestCdcSinkMappingResult]): + def __init__(self, request: TestCdcSinkMappingRequest): + super().__init__(TestCdcSinkMappingResult) + self._request = request + + def is_read_request(self) -> bool: + # A POST that changes nothing server-side, so failover to the fastest node is fine. + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/cdc-sink/test" + + request = requests.Request("POST", url) + request.data = self._request.to_json() + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + + self.result = TestCdcSinkMappingResult.from_json(json.loads(response)) diff --git a/ravendb/tests/operations_tests/test_cdc_sink.py b/ravendb/tests/operations_tests/test_cdc_sink.py index 9bc02d2e..8dcb0a15 100644 --- a/ravendb/tests/operations_tests/test_cdc_sink.py +++ b/ravendb/tests/operations_tests/test_cdc_sink.py @@ -4,6 +4,7 @@ """ import json +import os import unittest from ravendb.documents.operations.cdc_sink import ( @@ -27,7 +28,23 @@ OngoingTaskCdcSink, OngoingTaskType, ) +from ravendb.documents.operations.cdc_sink.schema import ( + CdcSinkSourceSchema, + GetCdcSinkSchemaOperation, +) +from ravendb.documents.operations.cdc_sink.testing import ( + TestCdcSinkMappingOperation, + TestCdcSinkMappingRequest, + TestCdcSinkMappingResult, + TestCdcSinkOperation, + TestCdcSinkRowSelector, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, +) +from ravendb.documents.operations.etl.sql import SqlConnectionString from ravendb.http.server_node import ServerNode +from ravendb.tests.test_base import TestBase from ravendb.http.topology import RaftCommand from ravendb.serverwide.database_record import DatabaseRecord @@ -282,3 +299,254 @@ def test_a_record_from_a_server_without_cdc_sinks_gets_an_empty_list(self): record = DatabaseRecord.from_json({"DatabaseName": "db", "LockMode": "Unlock", "AutoIndexes": {}}) self.assertEqual([], record.cdc_sinks) + + +class TestCdcSinkSchemaDiscovery(unittest.TestCase): + RESPONSE = { + "CatalogName": "northwind", + "HasPermissionToSetup": True, + "Warnings": ["SQL Server Agent is not running."], + "Errors": [], + "Tables": [ + { + "SourceTableSchema": "public", + "SourceTableName": "orders", + "IsCdcEnabled": True, + "PrimaryKeyColumns": ["id"], + "Warnings": ["REPLICA IDENTITY will not carry row-identifying columns on DELETE."], + "Columns": [ + { + "Name": "id", + "NativeType": "bigint", + "SuggestedType": "Default", + "IsPrimaryKey": True, + "IsCdcCapturable": True, + }, + {"Name": "payload", "NativeType": "jsonb", "SuggestedType": "Json", "IsCdcCapturable": True}, + {"Name": "blob", "NativeType": "bytea", "SuggestedType": "Attachment", "IsCdcCapturable": True}, + { + "Name": "weird", + "NativeType": "cube", + "IsCdcCapturable": False, + "UnsupportedReason": "No CDC mapping for this type.", + }, + ], + "ForeignKeys": [ + { + "Columns": ["customer_id"], + "ReferencedSchema": "public", + "ReferencedTable": "customers", + "ReferencedColumns": ["id"], + } + ], + } + ], + } + + def test_schema_response_is_parsed(self): + schema = CdcSinkSourceSchema.from_json(self.RESPONSE) + + self.assertEqual("northwind", schema.catalog_name) + self.assertTrue(schema.has_permission_to_setup) + self.assertEqual(1, len(schema.tables)) + self.assertEqual(["SQL Server Agent is not running."], schema.warnings) + + def test_success_follows_errors_not_warnings(self): + self.assertTrue(CdcSinkSourceSchema.from_json(self.RESPONSE).success) + self.assertFalse(CdcSinkSourceSchema.from_json({"Errors": ["boom"]}).success) + # A warning is advisory, so it must not flip success. + self.assertTrue(CdcSinkSourceSchema.from_json({"Warnings": ["heads up"]}).success) + + def test_columns_carry_their_suggested_mapping(self): + columns = CdcSinkSourceSchema.from_json(self.RESPONSE).tables[0].columns + + self.assertEqual(CdcColumnType.DEFAULT, columns[0].suggested_type) + self.assertTrue(columns[0].is_primary_key) + self.assertEqual(CdcColumnType.JSON, columns[1].suggested_type) + self.assertEqual(CdcColumnType.ATTACHMENT, columns[2].suggested_type) + + def test_an_uncapturable_column_says_why(self): + column = CdcSinkSourceSchema.from_json(self.RESPONSE).tables[0].columns[3] + + self.assertFalse(column.is_cdc_capturable) + self.assertEqual("No CDC mapping for this type.", column.unsupported_reason) + + def test_foreign_keys_are_parsed(self): + foreign_key = CdcSinkSourceSchema.from_json(self.RESPONSE).tables[0].foreign_keys[0] + + self.assertEqual(["customer_id"], foreign_key.columns) + self.assertEqual("customers", foreign_key.referenced_table) + self.assertEqual(["id"], foreign_key.referenced_columns) + + def test_schema_round_trips(self): + schema = CdcSinkSourceSchema.from_json(self.RESPONSE) + + self.assertEqual(schema.to_json(), CdcSinkSourceSchema.from_json(schema.to_json()).to_json()) + + def test_an_empty_response_parses(self): + schema = CdcSinkSourceSchema.from_json({}) + + self.assertEqual([], schema.tables) + self.assertTrue(schema.success) + + def test_the_operation_takes_a_connection_or_a_name(self): + node = ServerNode("http://localhost:8080", "db") + connection = SqlConnectionString("pg", "Host=localhost", "Npgsql") + + by_connection = GetCdcSinkSchemaOperation(connection, ["public"]).get_command(None) + request = by_connection.create_request(node) + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink/schema", request.url) + self.assertEqual("pg", request.data["Connection"]["Name"]) + self.assertEqual(["public"], request.data["Schemas"]) + self.assertIsNone(request.data["ConnectionStringName"]) + + by_name = GetCdcSinkSchemaOperation("pg").get_command(None) + self.assertEqual("pg", by_name.create_request(node).data["ConnectionStringName"]) + + def test_the_operation_allows_fastest_node_failover(self): + # A POST, but nothing changes server-side. + self.assertTrue(GetCdcSinkSchemaOperation("pg").get_command(None).is_read_request()) + + def test_the_operation_needs_a_connection_or_a_name(self): + with self.assertRaises(ValueError): + GetCdcSinkSchemaOperation() + with self.assertRaises(ValueError): + GetCdcSinkSchemaOperation(42) + + +class TestCdcSinkMappingPreview(unittest.TestCase): + def setUp(self): + self.node = ServerNode("http://localhost:8080", "db") + self.request = TestCdcSinkMappingRequest( + configuration=_configuration(), + connection=SqlConnectionString("pg", "Host=localhost", "Npgsql"), + source_table_schema="public", + source_table_name="orders", + max_rows=3, + ) + + def test_the_request_carries_the_configuration_and_the_row_choice(self): + body = self.request.to_json() + + self.assertEqual("orders", body["SourceTableName"]) + self.assertEqual("First", body["RowSelector"]) + self.assertEqual("Upsert", body["Operation"]) + self.assertEqual(3, body["MaxRows"]) + self.assertEqual("orders-cdc", body["Configuration"]["Name"]) + + def test_a_by_primary_key_request_names_the_key_values(self): + request = TestCdcSinkMappingRequest( + configuration=_configuration(), + row_selector=TestCdcSinkRowSelector.BY_PRIMARY_KEY, + primary_key_values=["42"], + operation=TestCdcSinkOperation.DELETE, + ) + body = request.to_json() + + self.assertEqual("ByPrimaryKey", body["RowSelector"]) + self.assertEqual(["42"], body["PrimaryKeyValues"]) + self.assertEqual("Delete", body["Operation"]) + + def test_the_operation_posts_to_the_test_endpoint(self): + command = TestCdcSinkMappingOperation(self.request).get_command(None) + request = command.create_request(self.node) + + self.assertEqual("POST", request.method) + self.assertEqual("http://localhost:8080/databases/db/admin/cdc-sink/test", request.url) + self.assertTrue(command.is_read_request()) + + def test_a_missing_request_is_rejected_client_side(self): + with self.assertRaises(ValueError): + TestCdcSinkMappingOperation(None) + + def test_row_results_are_parsed(self): + command = TestCdcSinkMappingOperation(self.request).get_command(None) + command.set_response( + json.dumps( + { + "Results": [ + { + "DocumentId": "orders/42", + "Document": '{"Id":42}', + "SourceRow": '{"id":42}', + "WouldDelete": False, + "IgnoreDeletes": False, + "DebugOutput": ["mapped"], + }, + {"DocumentId": "orders/43", "Error": "the patch threw"}, + ], + "Errors": [], + "Warnings": ["Linked tables are not exercised in test mode."], + } + ), + False, + ) + result = command.result + + self.assertIsInstance(result, TestCdcSinkMappingResult) + self.assertEqual(2, len(result.results)) + self.assertEqual("orders/42", result.results[0].document_id) + self.assertEqual(["mapped"], result.results[0].debug_output) + # A row that failed on its own carries the error; the request still succeeded. + self.assertEqual("the patch threw", result.results[1].error) + self.assertEqual([], result.errors) + self.assertEqual(1, len(result.warnings)) + + def test_a_whole_request_failure_leaves_no_rows(self): + result = TestCdcSinkMappingResult.from_json({"Errors": ["Cannot open connection"]}) + + self.assertEqual([], result.results) + self.assertEqual(["Cannot open connection"], result.errors) + + def test_result_round_trips(self): + result = TestCdcSinkMappingResult.from_json( + {"Results": [{"DocumentId": "orders/1", "WouldDelete": True}], "Errors": [], "Warnings": []} + ) + + self.assertEqual(result.to_json(), TestCdcSinkMappingResult.from_json(result.to_json()).to_json()) + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestCdcSinkAgainstServer(TestBase): + # CDC Sink is licensed, and each of these needs the server to accept the payload + # before it ever reaches a source database. + + def test_the_server_answers_schema_discovery_with_a_structured_result(self): + # There is no PostgreSQL to reach, so what matters is that the server parsed the + # request and answered in the shape the client expects instead of failing. + schema = self.store.maintenance.send(GetCdcSinkSchemaOperation("no-such-connection-string")) + + self.assertIsInstance(schema, CdcSinkSourceSchema) + self.assertFalse(schema.success) + self.assertTrue(any("no-such-connection-string" in error for error in schema.errors)) + + def test_the_server_reads_a_full_configuration_out_of_a_mapping_preview(self): + request = TestCdcSinkMappingRequest( + configuration=_configuration(), + connection=SqlConnectionString("pg", "Host=localhost;Database=nope", "Npgsql"), + source_table_schema="public", + source_table_name="orders", + ) + + result = self.store.maintenance.send(TestCdcSinkMappingOperation(request)) + + self.assertIsInstance(result, TestCdcSinkMappingResult) + # It reached the driver, which means the whole configuration tree deserialized. + self.assertTrue(any("source database" in error for error in result.errors)) + + def test_a_cdc_sink_task_is_stored_and_read_back(self): + self.store.maintenance.send( + PutConnectionStringOperation(SqlConnectionString("pg", "Host=localhost;Database=nope", "Npgsql")) + ) + + result = self.store.maintenance.send(AddCdcSinkOperation(_configuration())) + self.assertGreater(result.task_id, 0) + + task = self.store.maintenance.send(GetOngoingTaskInfoOperation("orders-cdc", OngoingTaskType.CDC_SINK)) + self.assertIsInstance(task, OngoingTaskCdcSink) + self.assertEqual("pg", task.connection_string_name) + self.assertEqual("Orders", task.configuration.tables[0].collection_name) + self.assertEqual( + ["Lines"], [embedded.property_name for embedded in task.configuration.tables[0].embedded_tables] + ) From 02ec7b638408c014ba04b1b4d889fed00daab0ce Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:03 +0200 Subject: [PATCH 09/14] RDBC-1099 Cover the follow-up surface with licensed tests and export it - Exercise server-wide connection strings against a licensed server - Export the follow-up types from the package root --- ravendb/__init__.py | 28 +++++++++ .../test_server_wide_connection_strings.py | 62 +++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 1732cc2c..0199060a 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -221,6 +221,34 @@ ExportCompressionAlgorithm, ) from ravendb.documents.smuggler.database_smuggler import DatabaseSmuggler +from ravendb.documents.operations.cdc_sink.schema import ( + CdcSinkSchemaRequest, + CdcSinkSourceColumn, + CdcSinkSourceForeignKey, + CdcSinkSourceSchema, + CdcSinkSourceTable, + GetCdcSinkSchemaOperation, +) +from ravendb.documents.operations.cdc_sink.testing import ( + TestCdcSinkMappingOperation, + TestCdcSinkMappingRequest, + TestCdcSinkMappingResult, + TestCdcSinkOperation, + TestCdcSinkRowResult, + TestCdcSinkRowSelector, +) +from ravendb.documents.smuggler.result import ( + Counts, + CountsWithLastEtag, + CountsWithLastEtagAndAttachments, + CountsWithSkippedCountAndLastEtag, + CountsWithSkippedCountAndLastEtagAndAttachments, + DatabaseRecordProgress, + SmugglerProgressBase, + SmugglerResult, +) +from ravendb.documents.smuggler.database_smuggler import SmugglerOperation +from ravendb.exceptions.commercial import LicenseLimitException, LimitType from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, AddCdcSinkOperationResult, diff --git a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py index 65314419..871832b2 100644 --- a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py +++ b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py @@ -4,6 +4,7 @@ """ import json +import os import unittest from ravendb.documents.operations.connection_string.get_connection_string_operation import ( @@ -23,6 +24,7 @@ ServerWideConnectionStringUsage, ) from ravendb.serverwide.server_operation_executor import ConnectionStringType +from ravendb.tests.test_base import TestBase class TestConnectionStringUsage(unittest.TestCase): @@ -248,3 +250,63 @@ def test_remove_reads_back_the_raft_index(self): command.set_response(json.dumps({"RaftCommandIndex": 12}), False) self.assertEqual(12, command.result.raft_command_index) + + +@unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") +class TestServerWideConnectionStringsAgainstServer(TestBase): + # Server-wide connection strings are licensed: writing one hits the license gate on an + # unlicensed server, so these only run when a license is configured. + + def _connection_string(self, name: str) -> ServerWideConnectionString: + return ServerWideConnectionString( + RavenConnectionString(name, database="db1", topology_discovery_urls=[self.store.urls[0]]) + ) + + def test_a_connection_string_is_stored_listed_and_removed(self): + put = self.store.maintenance.server.send( + PutServerWideConnectionStringOperation(self._connection_string("sw-raven")) + ) + self.assertGreater(put.raft_command_index, 0) + + listed = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + self.assertIn("sw-raven", [result.name for result in listed.results]) + + removed = self.store.maintenance.server.send( + RemoveServerWideConnectionStringOperation(RavenConnectionString("sw-raven")) + ) + self.assertGreater(removed.raft_command_index, 0) + + listed = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + self.assertNotIn("sw-raven", [result.name for result in listed.results]) + + def test_a_stored_connection_string_reads_back_with_its_type_and_urls(self): + self.store.maintenance.server.send(PutServerWideConnectionStringOperation(self._connection_string("sw-typed"))) + try: + listed = self.store.maintenance.server.send(GetServerWideConnectionStringsOperation()) + stored = next(result for result in listed.results if result.name == "sw-typed") + + self.assertEqual(ConnectionStringType.RAVEN, stored.type) + self.assertIsInstance(stored.connection_string, RavenConnectionString) + self.assertEqual("db1", stored.connection_string.database) + self.assertEqual([self.store.urls[0]], stored.connection_string.topology_discovery_urls) + # Nothing references it yet. + self.assertEqual([], stored.used_by) + finally: + self.store.maintenance.server.send( + RemoveServerWideConnectionStringOperation(RavenConnectionString("sw-typed")) + ) + + def test_filtering_by_name_and_type_narrows_the_listing(self): + for name in ("sw-one", "sw-two"): + self.store.maintenance.server.send(PutServerWideConnectionStringOperation(self._connection_string(name))) + try: + listed = self.store.maintenance.server.send( + GetServerWideConnectionStringsOperation("sw-one", ConnectionStringType.RAVEN) + ) + + self.assertEqual(["sw-one"], [result.name for result in listed.results]) + finally: + for name in ("sw-one", "sw-two"): + self.store.maintenance.server.send( + RemoveServerWideConnectionStringOperation(RavenConnectionString(name)) + ) From 2677bf4a6b4fb715b8b81069c549fe748894719b Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:03 +0200 Subject: [PATCH 10/14] RDBC-1099 Assert what a completed operation reports, not that it reports nothing This test pinned wait_for_completion returning None, which is the behaviour the previous commit deliberately changed. It now checks the result the server sent. --- .../tests/raven_commands_tests/test_by_index_actions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ravendb/tests/raven_commands_tests/test_by_index_actions.py b/ravendb/tests/raven_commands_tests/test_by_index_actions.py index a0457c3c..e5b5e3a6 100644 --- a/ravendb/tests/raven_commands_tests/test_by_index_actions.py +++ b/ravendb/tests/raven_commands_tests/test_by_index_actions.py @@ -112,8 +112,12 @@ def test_delete_by_index_success(self): response.operation_id, response.operation_node_tag, ) - # wait_for_completion doesnt return anything (None) when operation state is 'Completed' - self.assertIsNone(x.wait_for_completion()) + # wait_for_completion hands back the result the server reported for the operation. + # 45, not 50: the range is quoted, so the server compares DocNumber as a string + # and "5".."9" fall outside "0".."49" lexicographically. + result = x.wait_for_completion() + self.assertEqual(45, result["Total"]) + self.assertEqual("Processed 45 items.", result["Message"]) if __name__ == "__main__": From ccf7b48de9cd30410281c987c95e95a9f50e718b Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:04 +0200 Subject: [PATCH 11/14] RDBC-1112 Sync the client to 7.2.6 Everything the C# 7.2.6 patch brought: session JsonPatch behind the new SessionPatchBehavior convention with the RFC 6902 operation and batch command under it, a per-turn output schema for an agent conversation, chunk text on embeddings generation, QueryToolFailedException, the vector-search alias fix, and the client version itself. --- ravendb/__init__.py | 7 + ravendb/documents/ai/__init__.py | 1 + ravendb/documents/ai/ai_conversation.py | 42 +++ ravendb/documents/ai/ai_output_options.py | 72 +++++ ravendb/documents/commands/batches.py | 43 +++ ravendb/documents/conventions.py | 32 +++ .../ai/agents/run_conversation_operation.py | 17 +- .../ai/embeddings_generation_configuration.py | 6 + ravendb/documents/operations/batch.py | 60 ++-- .../operations/cdc_sink/configuration.py | 4 +- ravendb/documents/operations/json_patch.py | 147 ++++++++++ ravendb/documents/session/document_session.py | 151 +++++++++- ravendb/documents/session/misc.py | 11 +- ravendb/documents/session/query.py | 6 +- .../tokens/query_tokens/definitions.py | 21 ++ ravendb/exceptions/exception_dispatcher.py | 2 + ravendb/exceptions/raven_exceptions.py | 6 + ravendb/http/request_executor.py | 2 +- .../ai_agent_tests/test_ai_output_options.py | 139 +++++++++ .../test_chunking_options.py | 40 +++ .../test_commercial_limits.py | 25 +- .../tests/operations_tests/test_json_patch.py | 133 +++++++++ .../tests/session_tests/test_json_patch.py | 270 ++++++++++++++++++ .../test_query_token_aliasing.py | 94 ++++++ 24 files changed, 1299 insertions(+), 32 deletions(-) create mode 100644 ravendb/documents/ai/ai_output_options.py create mode 100644 ravendb/documents/operations/json_patch.py create mode 100644 ravendb/tests/ai_agent_tests/test_ai_output_options.py create mode 100644 ravendb/tests/operations_tests/test_json_patch.py create mode 100644 ravendb/tests/session_tests/test_json_patch.py create mode 100644 ravendb/tests/session_tests/test_query_token_aliasing.py diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 0199060a..21654a7e 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -2,6 +2,7 @@ BatchOptions, DeleteAttachmentCommandData, PatchCommandData, + JsonPatchCommandData, PutAttachmentCommandData, CommandData, CopyAttachmentCommandData, @@ -166,6 +167,11 @@ ) from ravendb.documents.operations.lazy.definition import LazyOperation from ravendb.documents.operations.misc import DeleteByQueryOperation, GetOperationStateOperation, QueryOperationOptions +from ravendb.documents.operations.json_patch import ( + JsonPatchDocument, + JsonPatchOperation, + JsonPatchResult, +) from ravendb.documents.operations.patch import ( PatchOperation, PatchByQueryOperation, @@ -249,6 +255,7 @@ ) from ravendb.documents.smuggler.database_smuggler import SmugglerOperation from ravendb.exceptions.commercial import LicenseLimitException, LimitType +from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, AddCdcSinkOperationResult, diff --git a/ravendb/documents/ai/__init__.py b/ravendb/documents/ai/__init__.py index 9c2c750a..6860b6b2 100644 --- a/ravendb/documents/ai/__init__.py +++ b/ravendb/documents/ai/__init__.py @@ -15,3 +15,4 @@ "AiMessagePromptFields", "AiMessagePromptTypes", ] +from ravendb.documents.ai.ai_output_options import AiOutputOptions diff --git a/ravendb/documents/ai/ai_conversation.py b/ravendb/documents/ai/ai_conversation.py index ea625616..50216309 100644 --- a/ravendb/documents/ai/ai_conversation.py +++ b/ravendb/documents/ai/ai_conversation.py @@ -6,6 +6,7 @@ from datetime import timedelta from ravendb.documents.ai.ai_answer import AiAnswer, AiConversationStatus +from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.ai.content_part import ContentPart, TextPart from ravendb.documents.operations.ai.agents import ( AiAgentActionRequest, @@ -149,16 +150,56 @@ def run(self) -> AiAnswer: if self._handle_server_reply(r): return r + def run_with_schema(self, output_options: "AiOutputOptions") -> AiAnswer: + """ + Runs one turn with the output format overridden for that turn only, leaving the + agent's own schema in place for later turns. Pass + ``AiOutputOptions(no_schema=True)`` to get free-form text back instead of JSON. + """ + if output_options is None: + raise ValueError("output_options cannot be None") + + self._dispatched_tool_ids.clear() + + while True: + r = self._run_internal(output_options=output_options) + if self._handle_server_reply(r): + return r + def stream(self, stream_property_path: str = None, on_chunk: Optional[Callable[[str], None]] = None) -> AiAnswer: while True: r = self._run_internal(stream_property_path=stream_property_path, streamed_chunks_callback=on_chunk) if self._handle_server_reply(r): return r + def stream_with_schema( + self, + stream_property_path: str = None, + on_chunk: Optional[Callable[[str], None]] = None, + output_options: "AiOutputOptions" = None, + ) -> AiAnswer: + """ + Streams one turn with the output format overridden for that turn only. + ``stream_property_path`` is ignored when the options ask for no schema, since + free-form text has no property to stream from. + """ + if output_options is None: + raise ValueError("output_options cannot be None") + + while True: + r = self._run_internal( + stream_property_path=stream_property_path, + streamed_chunks_callback=on_chunk, + output_options=output_options, + ) + if self._handle_server_reply(r): + return r + def _run_internal( self, stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, + output_options: Optional["AiOutputOptions"] = None, ) -> AiAnswer: from ravendb.documents.operations.ai.agents import RunConversationOperation import time @@ -197,6 +238,7 @@ def _run_internal( stream_property_path=stream_property_path, streamed_chunks_callback=streamed_chunks_callback, attachments_commands=self._attachments_commands, + output_options=output_options, debug=self._debug, cancel_pending_action_tools=self._cancel_pending_action_tools, ) diff --git a/ravendb/documents/ai/ai_output_options.py b/ravendb/documents/ai/ai_output_options.py new file mode 100644 index 00000000..d97a5f9d --- /dev/null +++ b/ravendb/documents/ai/ai_output_options.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, Optional + + +class AiOutputOptions: + """ + Overrides the output format for a single conversation turn. + + The agent still needs a default output schema when it is created; these options take + precedence over it, and only for the turn they are passed to. + + Give it exactly one of: + + - ``sample_object`` - an object the server turns into a JSON schema at request time + - ``output_schema`` - an explicit JSON schema string + - ``no_schema=True`` - no structured output at all, so the model answers in free text + """ + + def __init__( + self, + sample_object: Any = None, + output_schema: str = None, + no_schema: bool = False, + ): + if no_schema and (sample_object is not None or output_schema is not None): + raise ValueError( + "no_schema asks the model for free-form text, so it cannot be combined with " + "an output schema or a sample object. Drop one of them." + ) + + if output_schema is not None and (not output_schema or output_schema.isspace()): + raise ValueError("output_schema cannot be empty or whitespace") + + self.sample_object = sample_object + # Takes precedence over sample_object when both are set. + self.output_schema = output_schema + self.no_schema = no_schema + + def to_json(self) -> Dict[str, Any]: + json_dict = {} + + if self.sample_object is not None: + sample = self.sample_object + if callable(getattr(sample, "to_json", None)): + sample = sample.to_json() + # The server reads this as a JSON string, not as a nested object. + json_dict["SampleObject"] = json.dumps(sample) + + if self.output_schema is not None: + json_dict["OutputSchema"] = self.output_schema + + if self.no_schema: + json_dict["NoSchema"] = True + + return json_dict + + @classmethod + def from_json(cls, json_dict: Optional[Dict[str, Any]]) -> Optional[AiOutputOptions]: + if not json_dict: + return None + + options = cls.__new__(cls) + sample = json_dict.get("SampleObject") + if sample is not None: + options.sample_object = json.loads(sample) if isinstance(sample, str) else sample + else: + options.sample_object = None + options.output_schema = json_dict.get("OutputSchema") + options.no_schema = json_dict.get("NoSchema", False) + return options diff --git a/ravendb/documents/commands/batches.py b/ravendb/documents/commands/batches.py index 700a9e61..4b1eb2e5 100644 --- a/ravendb/documents/commands/batches.py +++ b/ravendb/documents/commands/batches.py @@ -22,6 +22,7 @@ from ravendb.util.util import RaftIdGenerator if TYPE_CHECKING: + from ravendb.documents.operations.json_patch import JsonPatchDocument from ravendb.documents.conventions import DocumentConventions from ravendb.documents.operations.attachments import RemoteAttachmentParameters from ravendb.documents.operations.patch import PatchRequest @@ -48,6 +49,7 @@ class CommandType(Enum): TIME_SERIES = "TimeSeries" TIME_SERIES_BULK_INSERT = "TIME_SERIES_BULK_INSERT" TIME_SERIES_COPY = "TIME_SERIES_COPY" + JSON_PATCH = "JsonPatch" BATCH_PATCH = "BatchPATCH" BATCH_TRACK_CHANGES = "BatchTrackChanges" CLIENT_ANY_COMMAND = "CLIENT_ANY_COMMAND" @@ -80,6 +82,8 @@ def from_csharp_value_str(cls, value: str) -> CommandType: return cls.COMPARE_EXCHANGE_DELETE elif value == "Counters": return cls.COUNTERS + elif value == "JsonPatch": + return cls.JSON_PATCH elif value == "BatchPATCH": return cls.BATCH_PATCH elif value == "BatchTrackChanges": @@ -421,6 +425,45 @@ def serialize(self, conventions: DocumentConventions) -> dict: } +class JsonPatchCommandData(CommandData): + """ + A batch command carrying RFC 6902 operations instead of a JavaScript script. + The server applies them structurally, without running a patch script. + """ + + def __init__(self, key: str, patch: "JsonPatchDocument"): + super().__init__(key, None, None, CommandType.JSON_PATCH) + if not key: + raise ValueError("Key cannot be None") + if patch is None: + raise ValueError("Patch cannot be None") + + self.__patch = patch + self.return_document: Union[None, bool] = None + + def __consumer(session: InMemoryDocumentSessionOperations) -> None: + self.return_document = session.advanced.is_loaded(key) + + self.__on_before_save_changes = __consumer + + @property + def json_patch(self) -> "JsonPatchDocument": + return self.__patch + + @property + def on_before_save_changes(self): + return self.__on_before_save_changes + + def serialize(self, conventions: DocumentConventions) -> dict: + return { + "Id": self.key, + "ChangeVector": None, + "JsonPatch": {"Operations": self.__patch.to_json()}, + "ReturnDocument": bool(self.return_document), + "Type": CommandType.JSON_PATCH.value, + } + + class PatchCommandData(CommandData): def __init__( self, diff --git a/ravendb/documents/conventions.py b/ravendb/documents/conventions.py index b4d9005d..86edb476 100644 --- a/ravendb/documents/conventions.py +++ b/ravendb/documents/conventions.py @@ -33,6 +33,22 @@ ) +class SessionPatchBehavior(Enum): + """ + Which command the session's patch methods emit. + + JSON_PATCH generates RFC 6902 operations where it can, and still falls back to a + JavaScript patch for anything JsonPatch cannot express. JAVA_SCRIPT always emits the + JavaScript patch, which is what this client did before JsonPatch support existed. + """ + + JSON_PATCH = "JsonPatch" + JAVA_SCRIPT = "JavaScript" + + def __str__(self) -> str: + return self.value + + class DocumentConventions(object): @classmethod def default_conventions(cls): @@ -64,6 +80,7 @@ def __init__(self): self.throw_if_query_page_size_is_not_set = False self._send_application_identifier = True self._save_enums_as_integers: Optional[bool] = None + self._session_patch_behavior = SessionPatchBehavior.JSON_PATCH self._disable_atomic_document_writes_in_cluster_wide_transaction: Optional[bool] = None # Configuration @@ -189,6 +206,20 @@ def save_enums_as_integers(self) -> bool: def save_enums_as_integers(self, value: bool): self._save_enums_as_integers = value + @property + def session_patch_behavior(self) -> "SessionPatchBehavior": + """ + Whether the session's patch methods emit RFC 6902 JsonPatch commands (the default) + or the JavaScript patch commands this client used before. Set it to + SessionPatchBehavior.JAVA_SCRIPT to stay on the old code path entirely. + """ + return self._session_patch_behavior + + @session_patch_behavior.setter + def session_patch_behavior(self, value: "SessionPatchBehavior") -> None: + self._assert_not_frozen() + self._session_patch_behavior = value + @property def find_python_class_name(self) -> Callable[[type], str]: return self._find_python_class_name @@ -437,6 +468,7 @@ def clone(self) -> DocumentConventions: cloned._should_ignore_entity_changes = self._should_ignore_entity_changes cloned._original_configuration = self._original_configuration cloned._save_enums_as_integers = self._save_enums_as_integers + cloned._session_patch_behavior = self._session_patch_behavior cloned.identity_parts_separator = self.identity_parts_separator cloned.disable_topology_updates = self.disable_topology_updates cloned._find_identity_property_name = self._find_identity_property_name diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index b73add25..49a6d7ee 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -10,6 +10,7 @@ from ravendb.http.server_node import ServerNode import requests from ravendb.http.misc import ResponseDisposeHandling +from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.ai.content_part import ContentPart TSchema = TypeVar("TSchema") @@ -256,15 +257,17 @@ def __init__( user_prompt: Optional[List[ContentPart]] = None, creation_options: Optional[AiConversationCreationOptions] = None, attachment_commands: Optional[List[Any]] = None, + output_options: Optional[AiOutputOptions] = None, ): self.action_responses: Optional[List[AiAgentActionResponse]] = action_responses self.artificial_actions: Optional[List[AiAgentArtificialActionResponse]] = artificial_actions self.user_prompt: Optional[List[ContentPart]] = user_prompt self.creation_options: Optional[AiConversationCreationOptions] = creation_options self.attachment_commands: Optional[List[Any]] = attachment_commands + self.output_options: Optional[AiOutputOptions] = output_options def to_json(self) -> Dict[str, Any]: - return { + body = { "ActionResponses": ( [resp.to_json() for resp in self.action_responses] if self.action_responses is not None else None ), @@ -280,6 +283,12 @@ def to_json(self) -> Dict[str, Any]: ), } + # Only sent when the caller overrides the agent's own schema for this turn. + if self.output_options is not None: + body["OutputOptions"] = self.output_options.to_json() + + return body + class RunConversationOperation(MaintenanceOperation[ConversationResult[TSchema]]): def __init__( @@ -294,6 +303,7 @@ def __init__( stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, + output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, ): @@ -314,6 +324,7 @@ def __init__( self._stream_property_path = stream_property_path self._streamed_chunks_callback = streamed_chunks_callback self._attachments_commands = attachments_commands or [] + self._output_options = output_options self._debug = debug self._cancel_pending_action_tools = cancel_pending_action_tools @@ -330,6 +341,7 @@ def get_command(self, conventions: DocumentConventions) -> RavenCommand[Conversa streamed_chunks_callback=self._streamed_chunks_callback, conventions=conventions, attachments_commands=self._attachments_commands, + output_options=self._output_options, debug=self._debug, cancel_pending_action_tools=self._cancel_pending_action_tools, ) @@ -349,6 +361,7 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, + output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, ): @@ -366,6 +379,7 @@ def __init__( self._stream_property_path = stream_property_path self._streamed_chunks_callback = streamed_chunks_callback self._conventions = conventions + self._output_options = output_options self._debug = debug self._cancel_pending_action_tools = cancel_pending_action_tools self._attachments_commands = attachments_commands or [] @@ -424,6 +438,7 @@ def create_request(self, node: ServerNode) -> requests.Request: user_prompt=self._prompt_parts, creation_options=self._options, attachment_commands=self._attachments_commands if self._attachments_commands else None, + output_options=self._output_options, ) body = json.dumps(request_body.to_json()) request = requests.Request("POST", url) diff --git a/ravendb/documents/operations/ai/embeddings_generation_configuration.py b/ravendb/documents/operations/ai/embeddings_generation_configuration.py index b20ef25b..9022cd9d 100644 --- a/ravendb/documents/operations/ai/embeddings_generation_configuration.py +++ b/ravendb/documents/operations/ai/embeddings_generation_configuration.py @@ -37,6 +37,7 @@ def __init__( chunking_options_for_querying: ChunkingOptions = None, embeddings_cache_expiration: timedelta = None, embeddings_cache_for_querying_expiration: timedelta = None, + store_chunk_text: bool = False, disabled: bool = False, mentor_node: str = None, pin_to_mentor_node: bool = False, @@ -59,6 +60,9 @@ def __init__( self.embeddings_transformation = embeddings_transformation self.quantization = quantization self.chunking_options_for_querying = chunking_options_for_querying + # Keeps each chunk's text next to its embedding, which helps when debugging or + # highlighting but costs storage. Off by default. + self.store_chunk_text = store_chunk_text self.embeddings_cache_expiration = ( embeddings_cache_expiration if embeddings_cache_expiration is not None @@ -220,6 +224,7 @@ def to_json(self) -> Dict[str, Any]: if self.embeddings_cache_for_querying_expiration else None ), + "StoreChunkText": self.store_chunk_text, } ) return result @@ -255,4 +260,5 @@ def from_json(cls, json_dict: Dict[str, Any]) -> "EmbeddingsGenerationConfigurat mentor_node=json_dict.get("MentorNode", None), pin_to_mentor_node=json_dict.get("PinToMentorNode", False), allow_etl_on_non_encrypted_channel=json_dict.get("AllowEtlOnNonEncryptedChannel", False), + store_chunk_text=json_dict.get("StoreChunkText", False), ) diff --git a/ravendb/documents/operations/batch.py b/ravendb/documents/operations/batch.py index d4e619ec..d9b61e64 100644 --- a/ravendb/documents/operations/batch.py +++ b/ravendb/documents/operations/batch.py @@ -115,6 +115,8 @@ def get_command_type(obj_node: dict) -> CommandType: self._handle_delete(batch_result) elif command_type == CommandType.PATCH: self._handle_patch(batch_result) + elif command_type == CommandType.JSON_PATCH: + self._handle_json_patch(batch_result) elif command_type == CommandType.ATTACHMENT_PUT: self._handle_attachment_put(batch_result) elif command_type == CommandType.ATTACHMENT_DELETE: @@ -198,33 +200,49 @@ def _handle_patch(self, batch_result: dict) -> None: status = PatchStatus(patch_status) if status in (PatchStatus.CREATED, PatchStatus.PATCHED): - document = batch_result.get("ModifiedDocument") - if not document: - return + self._refresh_tracked_document(batch_result, CommandType.PATCH) - key = self._get_string_field(batch_result, CommandType.PUT, "Id") - session_document_info = self._session.documents_by_id.get(key) - if session_document_info is None: - return + def _refresh_tracked_document(self, batch_result: dict, command_type: CommandType) -> None: + """Brings a tracked entity back in step with the document the server returned.""" + document = batch_result.get("ModifiedDocument") + if not document: + return - document_info = self._get_or_add_modifications(key, session_document_info, True) + key = self._get_string_field(batch_result, CommandType.PUT, "Id") + session_document_info = self._session.documents_by_id.get(key) + if session_document_info is None: + return - change_vector = self._get_string_field(batch_result, CommandType.PATCH, "ChangeVector") - last_modified = self._get_string_field(batch_result, CommandType.PATCH, "LastModified") + document_info = self._get_or_add_modifications(key, session_document_info, True) - document_info.change_vector = change_vector - document_info.metadata[constants.Documents.Metadata.KEY] = key - document_info.metadata[constants.Documents.Metadata.CHANGE_VECTOR] = change_vector - document_info.metadata[constants.Documents.Metadata.LAST_MODIFIED] = last_modified + change_vector = self._get_string_field(batch_result, command_type, "ChangeVector") + last_modified = self._get_string_field(batch_result, command_type, "LastModified") - document_info.document = document - self._apply_metadata_modifications(key, document_info) + document_info.change_vector = change_vector + document_info.metadata[constants.Documents.Metadata.KEY] = key + document_info.metadata[constants.Documents.Metadata.CHANGE_VECTOR] = change_vector + document_info.metadata[constants.Documents.Metadata.LAST_MODIFIED] = last_modified - if document_info.entity is not None: - self._session.entity_to_json.populate_entity(document_info.entity, key, document_info.document) - self._session.after_save_changes_invoke( - AfterSaveChangesEventArgs(self._session, document_info.key, document_info.entity) - ) + document_info.document = document + self._apply_metadata_modifications(key, document_info) + + if document_info.entity is not None: + self._session.entity_to_json.populate_entity(document_info.entity, key, document_info.document) + self._session.after_save_changes_invoke( + AfterSaveChangesEventArgs(self._session, document_info.key, document_info.entity) + ) + + def _handle_json_patch(self, batch_result: dict) -> None: + # Same bookkeeping as a JavaScript patch, except the server only reports Patched + # for a JsonPatch: it never creates a missing document. + patch_status = batch_result.get("PatchStatus") + if not patch_status: + self._throw_missing_field(CommandType.JSON_PATCH, "PatchStatus") + + if PatchStatus(patch_status) != PatchStatus.PATCHED: + return + + self._refresh_tracked_document(batch_result, CommandType.JSON_PATCH) def _handle_delete(self, batch_result: dict) -> None: self._handle_delete_internal(batch_result, CommandType.DELETE) diff --git a/ravendb/documents/operations/cdc_sink/configuration.py b/ravendb/documents/operations/cdc_sink/configuration.py index 291757f7..752dd0c9 100644 --- a/ravendb/documents/operations/cdc_sink/configuration.py +++ b/ravendb/documents/operations/cdc_sink/configuration.py @@ -104,9 +104,9 @@ class CdcSinkPostgresSettings: """ def __init__(self, publication_name: str = None, slot_name: str = None): - # Publication used for logical replication, auto-filled as rvn_cdc_p_{guid}. + # Publication used for logical replication, auto-filled as rvn_cdc_p_{taskId}. self.publication_name = publication_name - # Logical replication slot, auto-filled as rvn_cdc_s_{guid}. + # Logical replication slot, auto-filled as rvn_cdc_s_{taskId}. self.slot_name = slot_name def to_json(self) -> Dict[str, Any]: diff --git a/ravendb/documents/operations/json_patch.py b/ravendb/documents/operations/json_patch.py new file mode 100644 index 00000000..6f3f5062 --- /dev/null +++ b/ravendb/documents/operations/json_patch.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import requests + +from ravendb.documents.operations.definitions import IOperation +from ravendb.documents.operations.patch import PatchStatus +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode + +if TYPE_CHECKING: + from ravendb.documents.conventions import DocumentConventions + from ravendb.documents.store.definition import DocumentStore + from ravendb.http.http_cache import HttpCache + + +def escape_json_pointer_segment(segment: str) -> str: + """Escapes one path segment per RFC 6901: '~' becomes '~0' and '/' becomes '~1'.""" + return segment.replace("~", "~0").replace("/", "~1") + + +class JsonPatchDocument: + """ + A list of RFC 6902 operations to apply to one document, in order. + + Paths are JSON pointers: ``/Name``, ``/Address/City``, ``/Items/0``. ``/Items/-`` + means "append to the array". Build one with the methods below and hand it to + :class:`JsonPatchOperation`. + """ + + def __init__(self, operations: List[Dict[str, Any]] = None): + self.operations: List[Dict[str, Any]] = operations or [] + + def add(self, path: str, value: Any) -> JsonPatchDocument: + """Creates a member, or replaces it when it already exists. Use '/array/-' to append.""" + self.operations.append({"op": "add", "path": path, "value": value}) + return self + + def remove(self, path: str) -> JsonPatchDocument: + self.operations.append({"op": "remove", "path": path}) + return self + + def replace(self, path: str, value: Any) -> JsonPatchDocument: + """Overwrites what is already at the path. Fails when nothing is there.""" + self.operations.append({"op": "replace", "path": path, "value": value}) + return self + + def move(self, from_path: str, path: str) -> JsonPatchDocument: + self.operations.append({"op": "move", "from": from_path, "path": path}) + return self + + def copy(self, from_path: str, path: str) -> JsonPatchDocument: + self.operations.append({"op": "copy", "from": from_path, "path": path}) + return self + + def test(self, path: str, value: Any) -> JsonPatchDocument: + """Fails the whole patch unless the path already holds this value.""" + self.operations.append({"op": "test", "path": path, "value": value}) + return self + + def to_json(self) -> List[Dict[str, Any]]: + return list(self.operations) + + @classmethod + def from_json(cls, json_list: Optional[List[Dict[str, Any]]]) -> JsonPatchDocument: + return cls(list(json_list or [])) + + def __len__(self) -> int: + return len(self.operations) + + def __repr__(self) -> str: + return f"JsonPatchDocument({self.operations!r})" + + +class JsonPatchResult: + def __init__( + self, + status: Optional[PatchStatus] = None, + modified_document: Optional[dict] = None, + original_document: Optional[dict] = None, + debug: Optional[dict] = None, + ): + self.status = status + self.modified_document = modified_document + self.original_document = original_document + self.debug = debug + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> JsonPatchResult: + status = json_dict.get("Status") + return cls( + status=PatchStatus(status) if status else None, + modified_document=json_dict.get("ModifiedDocument"), + original_document=json_dict.get("OriginalDocument"), + debug=json_dict.get("Debug"), + ) + + +class JsonPatchOperation(IOperation[JsonPatchResult]): + """ + Applies a set of RFC 6902 operations to one document, in the order they were added. + + See https://ravendb.net/docs/article-page/latest/csharp/client-api/operations/patching/json-patch-syntax + """ + + def __init__(self, key: str, json_patch_document: JsonPatchDocument): + if not key: + raise ValueError("key cannot be None or empty") + if json_patch_document is None: + raise ValueError("json_patch_document cannot be None") + + self._key = key + self._json_patch_document = json_patch_document + + def get_command( + self, + store: "DocumentStore", + conventions: "DocumentConventions", + cache: "HttpCache" = None, + ) -> RavenCommand[JsonPatchResult]: + return self._JsonPatchCommand(self._key, self._json_patch_document) + + class _JsonPatchCommand(RavenCommand[JsonPatchResult]): + def __init__(self, key: str, json_patch_document: JsonPatchDocument): + super().__init__(JsonPatchResult) + self._key = key + self._json_patch_document = json_patch_document + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + from ravendb.tools.utils import Utils + + url = f"{node.url}/databases/{node.database}/json-patch?id={Utils.quote_key(self._key)}" + + request = requests.Request("PATCH", url) + request.data = {"Operations": self._json_patch_document.to_json()} + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + return + + self.result = JsonPatchResult.from_json(json.loads(response)) diff --git a/ravendb/documents/session/document_session.py b/ravendb/documents/session/document_session.py index bb9d1fc0..808453ba 100644 --- a/ravendb/documents/session/document_session.py +++ b/ravendb/documents/session/document_session.py @@ -2,7 +2,8 @@ import abc import copy -from datetime import datetime +from datetime import datetime, timedelta +from enum import Enum import http import json import os @@ -79,6 +80,8 @@ from ravendb.documents.session.operations.lazy import LazyLoadOperation, LazySessionOperations from ravendb.documents.session.operations.operations import MultiGetOperation, LoadStartingWithOperation from ravendb.documents.session.operations.query import QueryOperation +from ravendb.documents.conventions import SessionPatchBehavior +from ravendb.documents.operations.json_patch import JsonPatchDocument, escape_json_pointer_segment from ravendb.documents.session.misc import ( SessionOptions, ResponseTimeInformation, @@ -94,6 +97,7 @@ from ravendb.tools.time_series import TSRangeHelper from ravendb.tools.utils import Utils, Stopwatch, CaseInsensitiveDict from ravendb.documents.commands.batches import ( + JsonPatchCommandData, PatchCommandData, CommandType, DeleteCommandData, @@ -962,6 +966,76 @@ def __try_merge_patches(self, document_key: str, patch_request: PatchRequest) -> self.defer(PatchCommandData(document_key, None, new_patch_request, None)) return True + # --- JsonPatch support ------------------------------------------------- + + @staticmethod + def _to_json_pointer(path: str) -> Optional[str]: + """ + Turns a client path such as "Address.City" or "Items[0]" into the RFC 6901 + pointer "/Address/City" or "/Items/0". Returns None when the path is not + something JsonPatch can address, which sends the caller back to JavaScript. + """ + if not path or path.isspace(): + return None + + segments: List[str] = [] + for part in path.split("."): + if not part: + return None + + name, _, rest = part.partition("[") + if not name or name.isspace(): + return None + segments.append(name) + + # trailing [0][1]... indexers + while rest: + index, closed, rest = rest.partition("]") + if not closed or not index.isdigit(): + return None + segments.append(index) + if rest.startswith("["): + rest = rest[1:] + elif rest: + return None + + return "/" + "/".join(escape_json_pointer_segment(segment) for segment in segments) + + @staticmethod + def _can_json_patch_value(value: object) -> bool: + """ + JsonPatch carries values the conventions render as a single JSON scalar, which + is exactly what the JavaScript path would have sent for them. Composites stay + on the JavaScript path, which already knows how to serialize them. C# draws the + same line by taking value types and strings and rejecting the rest. + """ + return value is None or isinstance(value, (str, bool, int, float, Enum, datetime, timedelta)) + + @staticmethod + def _addresses_an_index(json_pointer: str) -> bool: + """A pointer ending in a number addresses an existing element, not a member.""" + return json_pointer.rsplit("/", 1)[-1].isdigit() + + def _has_javascript_patch(self, document_key: str) -> bool: + return IdTypeAndName.create(document_key, CommandType.PATCH, None) in self._session._deferred_commands_map + + def _use_json_patch(self) -> bool: + conventions = self._session._request_executor.conventions + return conventions.session_patch_behavior == SessionPatchBehavior.JSON_PATCH + + def _defer_json_patch(self, document_key: str, patch: JsonPatchDocument) -> None: + """Merges into a JsonPatch already deferred for this document, or defers a new one.""" + command = self._session._deferred_commands_map.get( + IdTypeAndName.create(document_key, CommandType.JSON_PATCH, None) + ) + if command is not None: + self._session._deferred_commands.remove(command) + merged = JsonPatchDocument(list(command.json_patch.operations) + list(patch.operations)) + self.defer(JsonPatchCommandData(document_key, merged)) + return + + self.defer(JsonPatchCommandData(document_key, patch)) + def increment(self, key_or_entity: Union[object, str], path: str, value_to_add: object) -> None: if not isinstance(key_or_entity, str): metadata = self.get_metadata_for(key_or_entity) @@ -980,6 +1054,25 @@ def patch(self, key_or_entity: Union[object, str], path: str, value: object) -> metadata = self.get_metadata_for(key_or_entity) key_or_entity = metadata[constants.Documents.Metadata.ID] + if ( + self._use_json_patch() + and self._can_json_patch_value(value) + and not self._has_javascript_patch(key_or_entity) + ): + json_pointer = self._to_json_pointer(path) + if json_pointer is not None: + patch = JsonPatchDocument() + # "add" creates a member or replaces it, which is what assigning to a + # named property did. An existing element is overwritten with "replace", + # since "add" would insert before the index and shift the rest. + if self._addresses_an_index(json_pointer): + patch.replace(json_pointer, value) + else: + patch.add(json_pointer, value) + + self._defer_json_patch(key_or_entity, patch) + return + patch_request = PatchRequest() patch_request.script = f"this.{path} = args.val_{self.__values_count};" patch_request.values = {f"val_{self.__values_count}": value} @@ -1001,6 +1094,12 @@ def patch_array( array_adder(script_array) + if self._use_json_patch() and not self._has_javascript_patch(key_or_entity): + patch = self._array_json_patch(path_to_array, script_array) + if patch is not None: + self._defer_json_patch(key_or_entity, patch) + return + patch_request = PatchRequest() patch_request.script = script_array.script patch_request.values = script_array.parameters @@ -1023,6 +1122,12 @@ def patch_object( dictionary_adder(script_map) + if self._use_json_patch() and not self._has_javascript_patch(key_or_entity): + patch = self._map_json_patch(path_to_object, script_map) + if patch is not None: + self._defer_json_patch(key_or_entity, patch) + return + patch_request = PatchRequest() patch_request.script = script_map.script patch_request.values = script_map.parameters @@ -1030,6 +1135,50 @@ def patch_object( if not self.__try_merge_patches(key_or_entity, patch_request): self.defer(PatchCommandData(key_or_entity, None, patch_request, None)) + def _array_json_patch(self, path_to_array: str, script_array: JavaScriptArray) -> Optional[JsonPatchDocument]: + json_pointer = self._to_json_pointer(path_to_array) + if json_pointer is None or not script_array.recorded_operations: + return None + + patch = JsonPatchDocument() + for name, argument in script_array.recorded_operations: + if name == "add": + if not self._can_json_patch_value(argument): + return None + # "/-" appends, which is what push did. + patch.add(f"{json_pointer}/-", argument) + elif name == "remove_at": + patch.remove(f"{json_pointer}/{argument}") + else: + # remove_all filters with a predicate; JsonPatch has no equivalent. + return None + + return patch + + def _map_json_patch(self, path_to_map: str, script_map: JavaScriptMap) -> Optional[JsonPatchDocument]: + json_pointer = self._to_json_pointer(path_to_map) + if json_pointer is None or not script_map.recorded_operations: + return None + + patch = JsonPatchDocument() + for name, key, value in script_map.recorded_operations: + key_text = str(key) + # The server rejects a whitespace-only path segment. + if not key_text or key_text.isspace(): + return None + + escaped = escape_json_pointer_segment(key_text) + if name == "put": + if not self._can_json_patch_value(value): + return None + patch.add(f"{json_pointer}/{escaped}", value) + elif name == "remove": + patch.remove(f"{json_pointer}/{escaped}") + else: + return None + + return patch + def add_or_patch(self, key: str, entity: object, path_to_object: str, value: object) -> None: patch_request = PatchRequest() patch_request.script = f"this.{path_to_object} = args.val_{self.__values_count}" diff --git a/ravendb/documents/session/misc.py b/ravendb/documents/session/misc.py index 84d201e1..8f6ab376 100644 --- a/ravendb/documents/session/misc.py +++ b/ravendb/documents/session/misc.py @@ -5,7 +5,7 @@ import threading from abc import ABC from enum import Enum -from typing import Union, Optional, TYPE_CHECKING, List, Dict, Generic, TypeVar +from typing import Union, Optional, TYPE_CHECKING, List, Dict, Generic, Tuple, TypeVar from ravendb.http.misc import LoadBalanceBehavior, ReadBalanceBehavior @@ -282,6 +282,8 @@ def __init__(self, suffix: int, path_to_array: str): self.__arg_counter = 0 self.__script_lines = [] self.__parameters: Dict[str, object] = {} + # What the caller asked for, kept structurally so a JsonPatch can be built from it. + self.recorded_operations: List[Tuple[str, object]] = [] @property def script(self) -> str: @@ -303,6 +305,7 @@ def __func(value) -> str: args = ",".join(list(map(__func, u))) self.__script_lines.append(f"this.{self.__path_to_array}.push({args});") + self.recorded_operations.extend(("add", value) for value in u) return self def remove_at(self, index: int) -> JavaScriptArray: @@ -310,12 +313,14 @@ def remove_at(self, index: int) -> JavaScriptArray: self.__script_lines.append(f"this.{self.__path_to_array}.splice(args.{argument_name}, 1);") self.__parameters[argument_name] = index + self.recorded_operations.append(("remove_at", index)) return self def remove_all(self, predicate_js: str) -> "JavaScriptArray": path = self.__path_to_array self.__script_lines.append(f"this.{path} = this.{path}.filter(function(item){{ return !({predicate_js}); }});") + self.recorded_operations.append(("remove_all", predicate_js)) return self @@ -326,6 +331,8 @@ def __init__(self, suffix: int, path_to_map: str): self._arg_counter = 0 self._script_lines = [] self._parameters: Dict[str, object] = {} + # What the caller asked for, kept structurally so a JsonPatch can be built from it. + self.recorded_operations: List[Tuple[str, object, object]] = [] @property def script(self) -> str: @@ -353,11 +360,13 @@ def put(self, key: _T_Key, value: _T_Value) -> JavaScriptMap[_T_Key, _T_Value]: formatted_key = self._format_key_for_javascript(key) self._script_lines.append(f"this.{self._path_to_map}[{formatted_key}] = args.{argument_name};") self.parameters[argument_name] = value + self.recorded_operations.append(("put", key, value)) return self def remove(self, key: _T_Key) -> JavaScriptMap[_T_Key, _T_Value]: formatted_key = self._format_key_for_javascript(key) self._script_lines.append(f"delete this.{self._path_to_map}[{formatted_key}];") + self.recorded_operations.append(("remove", key, None)) return self diff --git a/ravendb/documents/session/query.py b/ravendb/documents/session/query.py index 5d3542bd..9d0b1084 100644 --- a/ravendb/documents/session/query.py +++ b/ravendb/documents/session/query.py @@ -1317,9 +1317,11 @@ def add_from_alias_to_where_tokens(self, from_alias: str) -> None: raise RuntimeError("Alias cannot be None or empty") tokens = self.__get_current_where_tokens() - for token in tokens: + for index, token in enumerate(tokens): if isinstance(token, WhereToken): - token.add_alias(from_alias) + # add_alias returns a new token rather than mutating this one, so the + # qualified field name only takes effect once it replaces the original. + tokens[index] = token.add_alias(from_alias) def add_alias_to_includes_tokens(self, from_alias: str) -> str: if self._includes_alias is None: diff --git a/ravendb/documents/session/tokens/query_tokens/definitions.py b/ravendb/documents/session/tokens/query_tokens/definitions.py index 23f261fb..8d8b63a4 100644 --- a/ravendb/documents/session/tokens/query_tokens/definitions.py +++ b/ravendb/documents/session/tokens/query_tokens/definitions.py @@ -1076,6 +1076,27 @@ def __init__( self._task_name = task_name self._document_id = document_id + def add_alias(self, alias: str) -> WhereToken: + # The base implementation builds a plain WhereToken, which would drop every + # vector-search setting. Rebuild this token instead, with the field qualified. + if self.field_name == "id()": + return self + + token = VectorSearchToken( + wrapped_field_name=f"{alias}.{self.field_name}", + parameter_name=self.parameter_name, + source_quantization_type=self._source_quantization_type, + target_quantization_type=self._target_quantization_type, + similarity_threshold=self._similarity_threshold, + number_of_candidates_for_querying=self._number_of_candidates_for_querying, + is_exact=self._is_exact, + task_name=self._task_name, + document_id=self._document_id, + ) + token.options = self.options + token.where_operator = self.where_operator + return token + def write_to(self, writer: List[str]) -> None: """ Builds the vector search query string components and appends them to the writer list. diff --git a/ravendb/exceptions/exception_dispatcher.py b/ravendb/exceptions/exception_dispatcher.py index 2ac3e600..5b5dbf74 100644 --- a/ravendb/exceptions/exception_dispatcher.py +++ b/ravendb/exceptions/exception_dispatcher.py @@ -17,6 +17,7 @@ IndexCompactionInProgressException, InsufficientQuotaException, MissingAiAgentParameterException, + QueryToolFailedException, PortInUseException, RateLimitException, RavenException, @@ -48,6 +49,7 @@ "InsufficientQuotaException": InsufficientQuotaException, "TooManyTokensException": TooManyTokensException, "MissingAiAgentParameterException": MissingAiAgentParameterException, + "QueryToolFailedException": QueryToolFailedException, # documents "DocumentConflictException": DocumentConflictException, "DocumentDoesNotExistException": DocumentDoesNotExistException, diff --git a/ravendb/exceptions/raven_exceptions.py b/ravendb/exceptions/raven_exceptions.py index 6b6efee9..12f48cad 100644 --- a/ravendb/exceptions/raven_exceptions.py +++ b/ravendb/exceptions/raven_exceptions.py @@ -82,6 +82,12 @@ class TooManyTokensException(TooManyRequestsException): pass +class QueryToolFailedException(AiException): + """Raised when an agent's query tool could not run the query it was asked for.""" + + pass + + class MissingAiAgentParameterException(RavenException): pass diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index 360da4dc..242e1c79 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -56,7 +56,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.2.5" + CLIENT_VERSION = "7.2.6" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates diff --git a/ravendb/tests/ai_agent_tests/test_ai_output_options.py b/ravendb/tests/ai_agent_tests/test_ai_output_options.py new file mode 100644 index 00000000..6add1585 --- /dev/null +++ b/ravendb/tests/ai_agent_tests/test_ai_output_options.py @@ -0,0 +1,139 @@ +""" +Tests for the per-turn output schema override added in 7.2.6: AiOutputOptions, the +run_with_schema / stream_with_schema entry points, and how the options reach the wire. +""" + +import json +import unittest + +from ravendb.documents.ai.ai_conversation import AiConversation +from ravendb.documents.ai.ai_output_options import AiOutputOptions +from ravendb.documents.operations.ai.agents.run_conversation_operation import ( + ConversationRequestBody, + ConversationResult, + RunConversationOperation, +) +from ravendb.http.server_node import ServerNode + + +def _request_body(operation: RunConversationOperation) -> dict: + request = operation.get_command(None).create_request(ServerNode("http://localhost:8080", "db")) + return json.loads(request.data) if isinstance(request.data, str) else request.data + + +class TestAiOutputOptions(unittest.TestCase): + def test_a_sample_object_is_sent_as_json_text(self): + # The server reads SampleObject as a string, not as a nested object. + options = AiOutputOptions(sample_object={"Name": "sample", "Age": 1}) + + self.assertEqual({"Name": "sample", "Age": 1}, json.loads(options.to_json()["SampleObject"])) + + def test_a_sample_object_may_be_an_entity_with_to_json(self): + class _Answer: + def to_json(self): + return {"Name": "sample"} + + self.assertEqual( + {"Name": "sample"}, json.loads(AiOutputOptions(sample_object=_Answer()).to_json()["SampleObject"]) + ) + + def test_an_explicit_schema_is_sent_verbatim(self): + schema = '{"type":"object","properties":{"Name":{"type":"string"}}}' + + self.assertEqual(schema, AiOutputOptions(output_schema=schema).to_json()["OutputSchema"]) + + def test_no_schema_asks_for_free_form_text(self): + self.assertEqual({"NoSchema": True}, AiOutputOptions(no_schema=True).to_json()) + + def test_empty_options_send_nothing(self): + # Nothing set means the agent's own schema stays in charge. + self.assertEqual({}, AiOutputOptions().to_json()) + + def test_no_schema_cannot_be_combined_with_a_schema(self): + with self.assertRaises(ValueError): + AiOutputOptions(no_schema=True, output_schema="{}") + with self.assertRaises(ValueError): + AiOutputOptions(no_schema=True, sample_object={"Name": "x"}) + + def test_an_empty_schema_string_is_rejected(self): + with self.assertRaises(ValueError): + AiOutputOptions(output_schema="") + with self.assertRaises(ValueError): + AiOutputOptions(output_schema=" ") + + def test_options_round_trip(self): + for options in ( + AiOutputOptions(sample_object={"Name": "x"}), + AiOutputOptions(output_schema='{"type":"object"}'), + AiOutputOptions(no_schema=True), + ): + self.assertEqual(options.to_json(), AiOutputOptions.from_json(options.to_json()).to_json()) + + def test_nothing_parses_to_nothing(self): + self.assertIsNone(AiOutputOptions.from_json(None)) + self.assertIsNone(AiOutputOptions.from_json({})) + + +class TestOutputOptionsOnTheWire(unittest.TestCase): + def test_the_request_body_carries_the_options(self): + body = _request_body( + RunConversationOperation( + agent_id="agent", + conversation_id="chats/1", + prompt_parts=[], + output_options=AiOutputOptions(no_schema=True), + ) + ) + + self.assertEqual({"NoSchema": True}, body["OutputOptions"]) + + def test_a_turn_without_options_does_not_mention_them(self): + # Leaving the key out is what keeps the agent's configured schema in charge. + body = _request_body(RunConversationOperation(agent_id="agent", conversation_id="chats/1", prompt_parts=[])) + + self.assertNotIn("OutputOptions", body) + + def test_the_request_body_serializes_a_schema_override(self): + body = ConversationRequestBody(output_options=AiOutputOptions(output_schema='{"type":"string"}')).to_json() + + self.assertEqual('{"type":"string"}', body["OutputOptions"]["OutputSchema"]) + + def test_the_other_body_fields_are_untouched(self): + body = _request_body( + RunConversationOperation( + agent_id="agent", + conversation_id="chats/1", + prompt_parts=[], + output_options=AiOutputOptions(no_schema=True), + ) + ) + + for key in ("ActionResponses", "ArtificialActions", "CreationOptions", "UserPrompt"): + self.assertIn(key, body) + + +class TestConversationRunEntryPoints(unittest.TestCase): + def test_the_schema_overriding_entry_points_exist(self): + for name in ("run", "run_with_schema", "stream", "stream_with_schema"): + self.assertTrue(hasattr(AiConversation, name), name) + + def test_they_refuse_to_run_without_options(self): + conversation = AiConversation.__new__(AiConversation) + + with self.assertRaises(ValueError): + conversation.run_with_schema(None) + with self.assertRaises(ValueError): + conversation.stream_with_schema("Answer", lambda chunk: None, None) + + +class TestFreeFormAnswers(unittest.TestCase): + def test_a_raw_text_answer_is_read_as_a_string(self): + # With no schema the server answers with a bare string rather than an object. + result = ConversationResult.from_json({"ConversationId": "chats/1", "Response": "just some prose"}) + + self.assertEqual("just some prose", result.response) + + def test_a_structured_answer_is_still_read_as_an_object(self): + result = ConversationResult.from_json({"ConversationId": "chats/1", "Response": {"Name": "sample"}}) + + self.assertEqual({"Name": "sample"}, result.response) diff --git a/ravendb/tests/embeddings_generation_tests/test_chunking_options.py b/ravendb/tests/embeddings_generation_tests/test_chunking_options.py index 4c930a65..4a957035 100644 --- a/ravendb/tests/embeddings_generation_tests/test_chunking_options.py +++ b/ravendb/tests/embeddings_generation_tests/test_chunking_options.py @@ -1,6 +1,9 @@ import unittest from ravendb.documents.operations.ai.chunking_options import ChunkingOptions, ChunkingMethod +from ravendb.documents.operations.ai.embeddings_generation_configuration import ( + EmbeddingsGenerationConfiguration, +) class TestChunkingOptionsContextPrefix(unittest.TestCase): @@ -78,3 +81,40 @@ def test_from_json_missing_int_keys_use_csharp_defaults(self): if __name__ == "__main__": unittest.main() + + +class TestStoreChunkText(unittest.TestCase): + """StoreChunkText, added in 7.2.6, keeps each chunk's text next to its embedding.""" + + @staticmethod + def _configuration(**kwargs) -> EmbeddingsGenerationConfiguration: + return EmbeddingsGenerationConfiguration( + name="embeddings", + identifier="embeddings", + collection="Orders", + connection_string_name="ai", + chunking_options_for_querying=ChunkingOptions( + chunking_method=ChunkingMethod.PLAIN_TEXT_SPLIT, max_tokens_per_chunk=512 + ), + **kwargs, + ) + + def test_it_is_off_unless_asked_for(self): + # It costs storage, so opting in has to be deliberate. + self.assertFalse(self._configuration().store_chunk_text) + self.assertFalse(self._configuration().to_json()["StoreChunkText"]) + + def test_it_reaches_the_wire_when_set(self): + self.assertTrue(self._configuration(store_chunk_text=True).to_json()["StoreChunkText"]) + + def test_it_round_trips(self): + serialized = self._configuration(store_chunk_text=True).to_json() + + self.assertTrue(EmbeddingsGenerationConfiguration.from_json(serialized).store_chunk_text) + self.assertEqual(serialized, EmbeddingsGenerationConfiguration.from_json(serialized).to_json()) + + def test_a_configuration_from_an_older_server_reads_as_off(self): + serialized = self._configuration().to_json() + del serialized["StoreChunkText"] + + self.assertFalse(EmbeddingsGenerationConfiguration.from_json(serialized).store_chunk_text) diff --git a/ravendb/tests/operations_tests/test_commercial_limits.py b/ravendb/tests/operations_tests/test_commercial_limits.py index ea5279f8..92edff90 100644 --- a/ravendb/tests/operations_tests/test_commercial_limits.py +++ b/ravendb/tests/operations_tests/test_commercial_limits.py @@ -1,13 +1,14 @@ """ -Tests for the commercial limit surface: the LimitType enum the 7.2.5 patch extended, and -LicenseLimitException coming back typed from a 402 instead of a bare RavenException. +Tests for the typed exceptions the 7.2.5 and 7.2.6 patches added: the LimitType enum, +LicenseLimitException coming back from a 402 instead of a bare RavenException, and +QueryToolFailedException for a failed agent query tool. """ import unittest from ravendb.exceptions.commercial import LicenseLimitException, LimitType from ravendb.exceptions.exception_dispatcher import ExceptionDispatcher -from ravendb.exceptions.raven_exceptions import RavenException +from ravendb.exceptions.raven_exceptions import AiException, QueryToolFailedException, RavenException class TestLimitType(unittest.TestCase): @@ -59,3 +60,21 @@ def test_a_402_from_another_exception_type_is_left_alone(self): ) self.assertNotIsInstance(ExceptionDispatcher.get(schema, 402), LicenseLimitException) + + +class TestQueryToolFailedException(unittest.TestCase): + def test_it_is_an_ai_exception(self): + self.assertIsInstance(QueryToolFailedException("nope"), AiException) + + def test_the_dispatcher_returns_it(self): + schema = ExceptionDispatcher.ExceptionSchema( + url="http://localhost:8080", + object_type="Raven.Client.Exceptions.QueryToolFailedException", + message="no", + error="The agent's query tool could not run the query.", + ) + + exception = ExceptionDispatcher.get(schema, 500) + + self.assertIsInstance(exception, QueryToolFailedException) + self.assertIn("query tool", str(exception)) diff --git a/ravendb/tests/operations_tests/test_json_patch.py b/ravendb/tests/operations_tests/test_json_patch.py new file mode 100644 index 00000000..d27375ce --- /dev/null +++ b/ravendb/tests/operations_tests/test_json_patch.py @@ -0,0 +1,133 @@ +""" +Tests for the RFC 6902 patch document, the standalone JsonPatchOperation, and the batch +command that carries the same operations inside save_changes. +""" + +import json +import unittest + +from ravendb.documents.commands.batches import CommandType, JsonPatchCommandData +from ravendb.documents.operations.json_patch import ( + JsonPatchDocument, + JsonPatchOperation, + JsonPatchResult, + escape_json_pointer_segment, +) +from ravendb.documents.operations.patch import PatchStatus +from ravendb.http.server_node import ServerNode +from ravendb.tests.test_base import TestBase + +DOCUMENT = { + "Name": "original", + "Tags": ["a", "b"], + "@metadata": {"@collection": "Tests"}, +} + + +class TestJsonPatchDocument(unittest.TestCase): + def test_operations_keep_the_order_they_were_added_in(self): + patch = JsonPatchDocument().add("/a", 1).remove("/b").replace("/c", 2) + + self.assertEqual(["add", "remove", "replace"], [op["op"] for op in patch.to_json()]) + + def test_each_operation_carries_its_path_and_value(self): + self.assertEqual( + {"op": "add", "path": "/Name", "value": "x"}, JsonPatchDocument().add("/Name", "x").to_json()[0] + ) + self.assertEqual({"op": "remove", "path": "/Name"}, JsonPatchDocument().remove("/Name").to_json()[0]) + + def test_move_and_copy_carry_a_from(self): + self.assertEqual({"op": "move", "from": "/a", "path": "/b"}, JsonPatchDocument().move("/a", "/b").to_json()[0]) + self.assertEqual({"op": "copy", "from": "/a", "path": "/b"}, JsonPatchDocument().copy("/a", "/b").to_json()[0]) + + def test_builders_chain(self): + self.assertEqual(3, len(JsonPatchDocument().add("/a", 1).test("/a", 1).remove("/a"))) + + def test_it_round_trips(self): + patch = JsonPatchDocument().add("/a", 1).remove("/b") + + self.assertEqual(patch.to_json(), JsonPatchDocument.from_json(patch.to_json()).to_json()) + + def test_pointer_segments_are_escaped(self): + # RFC 6901: '~' becomes '~0' and '/' becomes '~1', in that order. + self.assertEqual("~0", escape_json_pointer_segment("~")) + self.assertEqual("~1", escape_json_pointer_segment("/")) + self.assertEqual("a~1b~0c", escape_json_pointer_segment("a/b~c")) + + +class TestJsonPatchCommandData(unittest.TestCase): + def test_it_serializes_as_a_json_patch_command(self): + command = JsonPatchCommandData("docs/1", JsonPatchDocument().add("/Name", "x")) + serialized = command.serialize(None) + + self.assertEqual("docs/1", serialized["Id"]) + self.assertEqual("JsonPatch", serialized["Type"]) + self.assertIsNone(serialized["ChangeVector"]) + self.assertEqual([{"op": "add", "path": "/Name", "value": "x"}], serialized["JsonPatch"]["Operations"]) + self.assertFalse(serialized["ReturnDocument"]) + + def test_its_command_type(self): + self.assertEqual(CommandType.JSON_PATCH, JsonPatchCommandData("docs/1", JsonPatchDocument()).command_type) + + def test_the_server_type_string_parses_back(self): + self.assertEqual(CommandType.JSON_PATCH, CommandType.from_csharp_value_str("JsonPatch")) + + def test_missing_arguments_are_rejected(self): + with self.assertRaises(ValueError): + JsonPatchCommandData("", JsonPatchDocument()) + with self.assertRaises(ValueError): + JsonPatchCommandData("docs/1", None) + + +class TestJsonPatchOperationRequest(unittest.TestCase): + def test_it_patches_the_json_patch_endpoint(self): + operation = JsonPatchOperation("docs/1", JsonPatchDocument().add("/Name", "x")) + request = operation.get_command(None, None, None).create_request(ServerNode("http://localhost:8080", "db")) + + self.assertEqual("PATCH", request.method) + self.assertEqual("http://localhost:8080/databases/db/json-patch?id=docs%2F1", request.url) + self.assertEqual({"Operations": [{"op": "add", "path": "/Name", "value": "x"}]}, request.data) + + def test_it_is_not_a_read_request(self): + self.assertFalse( + JsonPatchOperation("docs/1", JsonPatchDocument()).get_command(None, None, None).is_read_request() + ) + + def test_it_reads_the_status_back(self): + command = JsonPatchOperation("docs/1", JsonPatchDocument()).get_command(None, None, None) + command.set_response(json.dumps({"Status": "Patched", "ModifiedDocument": {"Name": "x"}}), False) + + self.assertIsInstance(command.result, JsonPatchResult) + self.assertEqual(PatchStatus.PATCHED, command.result.status) + self.assertEqual({"Name": "x"}, command.result.modified_document) + + def test_missing_arguments_are_rejected(self): + with self.assertRaises(ValueError): + JsonPatchOperation("", JsonPatchDocument()) + with self.assertRaises(ValueError): + JsonPatchOperation("docs/1", None) + + +class TestJsonPatchOperationAgainstServer(TestBase): + def test_it_applies_operations_in_order(self): + with self.store.open_session() as session: + session.store(dict(DOCUMENT), "docs/1") + session.save_changes() + + patch = JsonPatchDocument().add("/Name", "patched").add("/Tags/-", "c").replace("/Tags/0", "z") + result = self.store.operations.send(JsonPatchOperation("docs/1", patch)) + + self.assertEqual(PatchStatus.PATCHED, result.status) + + with self.store.open_session() as session: + document = session.load("docs/1", dict) + self.assertEqual("patched", document["Name"]) + self.assertEqual(["z", "b", "c"], document["Tags"]) + + def test_patching_a_document_that_is_not_there(self): + self.assertRaisesWithMessageContaining( + self.store.operations.send, + Exception, + "Cannot apply json patch", + JsonPatchOperation("docs/nope", JsonPatchDocument().add("/Name", "x")), + ) diff --git a/ravendb/tests/session_tests/test_json_patch.py b/ravendb/tests/session_tests/test_json_patch.py new file mode 100644 index 00000000..e2de20b0 --- /dev/null +++ b/ravendb/tests/session_tests/test_json_patch.py @@ -0,0 +1,270 @@ +""" +Tests for the session patch methods emitting JsonPatch commands in place of a JavaScript +patch, and for the cases that keep them on the JavaScript path. +""" + +import unittest +from datetime import datetime, timedelta + +from ravendb import DocumentStore +from ravendb.documents.conventions import DocumentConventions, SessionPatchBehavior +from ravendb.documents.operations.patch import PatchStatus +from ravendb.documents.session.document_session import DocumentSession +from ravendb.tests.test_base import TestBase + +_to_pointer = DocumentSession._Advanced._to_json_pointer +_can_patch = DocumentSession._Advanced._can_json_patch_value + +DOCUMENT = { + "Name": "original", + "Tags": ["a", "b"], + "Counts": {"x": 1}, + "Address": {"City": "Warsaw"}, + "@metadata": {"@collection": "Tests"}, +} + + +class Thing: + def __init__(self, Name: str = None, Tags: list = None): + self.Name = Name + self.Tags = Tags + + +def _deferred(session): + return [command.command_type.value for command in session._deferred_commands] + + +class TestPathToJsonPointer(unittest.TestCase): + def test_a_plain_member(self): + self.assertEqual("/Name", _to_pointer("Name")) + + def test_a_nested_member(self): + self.assertEqual("/Address/City", _to_pointer("Address.City")) + + def test_an_array_index(self): + self.assertEqual("/Tags/0", _to_pointer("Tags[0]")) + + def test_an_index_in_the_middle(self): + self.assertEqual("/Orders/2/Lines/0/Price", _to_pointer("Orders[2].Lines[0].Price")) + + def test_consecutive_indexers(self): + self.assertEqual("/Grid/1/2", _to_pointer("Grid[1][2]")) + + def test_a_member_needing_escaping(self): + self.assertEqual("/a~1b", _to_pointer("a/b")) + + def test_paths_json_patch_cannot_address(self): + # Each of these sends the caller back to the JavaScript patch. + for path in ("", " ", "a..b", "Tags[]", "Tags[x]", "Tags[0", "[0]", "Tags[0]junk"): + self.assertIsNone(_to_pointer(path), path) + + +class TestValuesJsonPatchCanCarry(unittest.TestCase): + def test_json_scalars_are_fine(self): + for value in (None, "text", True, 1, 1.5): + self.assertTrue(_can_patch(value), repr(value)) + + def test_so_is_anything_the_conventions_render_as_one_scalar(self): + for value in (datetime.now(), timedelta(minutes=5), PatchStatus.PATCHED): + self.assertTrue(_can_patch(value), repr(value)) + + def test_anything_richer_stays_on_javascript(self): + # The JavaScript path already knows how to serialize these, so it keeps them. + for value in ({"a": 1}, ["a"], object(), (1, 2)): + self.assertFalse(_can_patch(value), repr(value)) + + +class TestSessionPatchBehaviorConvention(unittest.TestCase): + def test_json_patch_is_the_default(self): + self.assertEqual(SessionPatchBehavior.JSON_PATCH, DocumentConventions().session_patch_behavior) + + def test_it_can_be_turned_off(self): + conventions = DocumentConventions() + conventions.session_patch_behavior = SessionPatchBehavior.JAVA_SCRIPT + + self.assertEqual(SessionPatchBehavior.JAVA_SCRIPT, conventions.session_patch_behavior) + + def test_a_clone_keeps_it(self): + # The request executor works off a clone, which is what the session reads. + conventions = DocumentConventions() + conventions.session_patch_behavior = SessionPatchBehavior.JAVA_SCRIPT + + self.assertEqual(SessionPatchBehavior.JAVA_SCRIPT, conventions.clone().session_patch_behavior) + + +class TestSessionEmitsJsonPatch(TestBase): + def setUp(self): + super().setUp() + with self.store.open_session() as session: + session.store(dict(DOCUMENT), "docs/1") + session.save_changes() + + def test_a_scalar_patch_becomes_a_json_patch(self): + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Name", "patched") + self.assertEqual(["JsonPatch"], _deferred(session)) + session.save_changes() + + with self.store.open_session() as session: + self.assertEqual("patched", session.load("docs/1", dict)["Name"]) + + def test_a_nested_path_and_an_array_index(self): + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Address.City", "Krakow") + session.advanced.patch("docs/1", "Tags[0]", "z") + session.save_changes() + + with self.store.open_session() as session: + document = session.load("docs/1", dict) + self.assertEqual("Krakow", document["Address"]["City"]) + self.assertEqual(["z", "b"], document["Tags"]) + + def test_patches_for_one_document_merge_into_a_single_command(self): + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Name", "one") + session.advanced.patch("docs/1", "Address.City", "two") + + self.assertEqual(["JsonPatch"], _deferred(session)) + self.assertEqual(2, len(session._deferred_commands[0].json_patch)) + + def test_an_array_adder_appends(self): + with self.store.open_session() as session: + session.advanced.patch_array("docs/1", "Tags", lambda array: array.add("c").add("d")) + self.assertEqual(["JsonPatch"], _deferred(session)) + session.save_changes() + + with self.store.open_session() as session: + self.assertEqual(["a", "b", "c", "d"], session.load("docs/1", dict)["Tags"]) + + def test_an_array_adder_removes_by_index(self): + with self.store.open_session() as session: + session.advanced.patch_array("docs/1", "Tags", lambda array: array.remove_at(0)) + session.save_changes() + + with self.store.open_session() as session: + self.assertEqual(["b"], session.load("docs/1", dict)["Tags"]) + + def test_a_map_adder_puts_and_removes(self): + with self.store.open_session() as session: + session.advanced.patch_object("docs/1", "Counts", lambda m: m.put("y", 2)) + self.assertEqual(["JsonPatch"], _deferred(session)) + session.save_changes() + + with self.store.open_session() as session: + self.assertEqual({"x": 1, "y": 2}, session.load("docs/1", dict)["Counts"]) + + with self.store.open_session() as session: + session.advanced.patch_object("docs/1", "Counts", lambda m: m.remove("x")) + session.save_changes() + + with self.store.open_session() as session: + self.assertEqual({"y": 2}, session.load("docs/1", dict)["Counts"]) + + def test_a_datetime_lands_exactly_as_the_javascript_path_wrote_it(self): + # A value the conventions stringify has to serialize the same on both paths, + # otherwise switching the default would rewrite what already-stored documents hold. + stamp = datetime(2026, 9, 15, 10, 30, 45, 123400) + + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Stamp", stamp) + self.assertEqual(["JsonPatch"], _deferred(session)) + session.save_changes() + + with self.store.open_session() as session: + via_json_patch = session.load("docs/1", dict)["Stamp"] + + with DocumentStore(self.store.urls, self.store.database) as store: + store.conventions.session_patch_behavior = SessionPatchBehavior.JAVA_SCRIPT + store.initialize() + + with store.open_session() as session: + session.store(dict(DOCUMENT), "docs/2") + session.save_changes() + + with store.open_session() as session: + session.advanced.patch("docs/2", "Stamp", stamp) + self.assertEqual(["PATCH"], _deferred(session)) + session.save_changes() + + with store.open_session() as session: + self.assertEqual(via_json_patch, session.load("docs/2", dict)["Stamp"]) + + def test_a_tracked_entity_is_refreshed_from_the_result(self): + with self.store.open_session() as session: + session.store(Thing("original", ["a", "b"]), "things/1") + session.save_changes() + + with self.store.open_session() as session: + loaded = session.load("things/1", Thing) + session.advanced.patch("things/1", "Name", "refreshed") + session.save_changes() + + self.assertEqual("refreshed", loaded.Name) + + +class TestSessionFallsBackToJavaScript(TestBase): + def setUp(self): + super().setUp() + with self.store.open_session() as session: + session.store(dict(DOCUMENT), "docs/1") + session.save_changes() + + def test_a_value_json_patch_cannot_carry(self): + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Address", {"City": "Gdansk"}) + + self.assertEqual(["PATCH"], _deferred(session)) + session.save_changes() + + with self.store.open_session() as session: + self.assertEqual("Gdansk", session.load("docs/1", dict)["Address"]["City"]) + + def test_an_array_predicate_json_patch_cannot_express(self): + with self.store.open_session() as session: + session.advanced.patch_array("docs/1", "Tags", lambda array: array.remove_all("item == 'a'")) + + self.assertEqual(["PATCH"], _deferred(session)) + + def test_a_path_json_patch_cannot_address(self): + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Tags[x]", "z") + + self.assertEqual(["PATCH"], _deferred(session)) + + def test_a_javascript_patch_already_deferred_keeps_the_rest_on_javascript(self): + # Mixing both command types for one document would apply them out of order. + with self.store.open_session() as session: + session.advanced.patch("docs/1", "Address", {"City": "Gdansk"}) + session.advanced.patch("docs/1", "Name", "also js") + + self.assertEqual(["PATCH"], _deferred(session)) + + def test_increment_stays_on_javascript(self): + # JsonPatch has no read-modify-write operation. + with self.store.open_session() as session: + session.advanced.increment("docs/1", "Counts.x", 5) + + self.assertEqual(["PATCH"], _deferred(session)) + + +class TestSessionPatchOptedOut(TestBase): + def test_the_convention_turns_json_patch_off_entirely(self): + with DocumentStore(self.store.urls, self.store.database) as store: + store.conventions.session_patch_behavior = SessionPatchBehavior.JAVA_SCRIPT + store.initialize() + + with store.open_session() as session: + session.store(dict(DOCUMENT), "opted/1") + session.save_changes() + + with store.open_session() as session: + session.advanced.patch("opted/1", "Name", "via javascript") + session.advanced.patch_array("opted/1", "Tags", lambda array: array.add("c")) + + self.assertEqual(["PATCH"], _deferred(session)) + session.save_changes() + + with store.open_session() as session: + document = session.load("opted/1", dict) + self.assertEqual("via javascript", document["Name"]) + self.assertEqual(["a", "b", "c"], document["Tags"]) diff --git a/ravendb/tests/session_tests/test_query_token_aliasing.py b/ravendb/tests/session_tests/test_query_token_aliasing.py new file mode 100644 index 00000000..8fd73165 --- /dev/null +++ b/ravendb/tests/session_tests/test_query_token_aliasing.py @@ -0,0 +1,94 @@ +""" +Tests for qualifying where-token field names with a query alias, which 7.2.6 made +overridable so a vector search does not lose its settings on the way through. +""" + +import unittest + +from ravendb.documents.indexes.vector.options import VectorEmbeddingType +from ravendb.documents.session.tokens.query_tokens.definitions import ( + MoreLikeThisToken, + VectorSearchToken, + WhereOperator, + WhereToken, +) + + +def _vector_token(field_name: str = "Vector") -> VectorSearchToken: + return VectorSearchToken( + wrapped_field_name=field_name, + parameter_name="p0", + source_quantization_type=VectorEmbeddingType.SINGLE, + target_quantization_type=VectorEmbeddingType.SINGLE, + similarity_threshold=0.8, + number_of_candidates_for_querying=32, + is_exact=True, + ) + + +def _render(token) -> str: + writer = [] + token.write_to(writer) + return "".join(writer) + + +class TestWhereTokenAliasing(unittest.TestCase): + def test_a_plain_where_token_gains_the_alias(self): + token = WhereToken.create(WhereOperator.EQUALS, "Name", "p0") + + self.assertEqual("x.Name", token.add_alias("x").field_name) + + def test_the_document_id_field_is_left_alone(self): + token = WhereToken.create(WhereOperator.EQUALS, "id()", "p0") + + self.assertIs(token, token.add_alias("x")) + + def test_add_alias_does_not_mutate_the_original(self): + # It returns a new token, which is why callers have to use the return value. + token = WhereToken.create(WhereOperator.EQUALS, "Name", "p0") + aliased = token.add_alias("x") + + self.assertIsNot(token, aliased) + self.assertEqual("Name", token.field_name) + + +class TestVectorSearchTokenAliasing(unittest.TestCase): + def test_aliasing_keeps_the_token_type(self): + # The base implementation would hand back a plain WhereToken. + self.assertIsInstance(_vector_token().add_alias("x"), VectorSearchToken) + + def test_aliasing_qualifies_the_field(self): + self.assertEqual("x.Vector", _vector_token().add_alias("x").field_name) + + def test_aliasing_keeps_every_vector_setting(self): + aliased = _vector_token().add_alias("x") + + self.assertEqual(0.8, aliased._similarity_threshold) + self.assertEqual(32, aliased._number_of_candidates_for_querying) + self.assertTrue(aliased._is_exact) + + def test_the_aliased_token_still_renders_as_a_vector_search(self): + self.assertEqual("exact(vector.search(x.Vector, $p0, 0.8, 32))", _render(_vector_token().add_alias("x"))) + + def test_the_document_id_field_is_left_alone(self): + token = _vector_token("id()") + + self.assertIs(token, token.add_alias("x")) + + def test_a_task_backed_token_keeps_its_task(self): + token = VectorSearchToken( + wrapped_field_name="Vector", + parameter_name="p0", + source_quantization_type=VectorEmbeddingType.SINGLE, + target_quantization_type=VectorEmbeddingType.SINGLE, + task_name="embeddings-task", + ) + + self.assertEqual("embeddings-task", token.add_alias("x")._task_name) + + +class TestMoreLikeThisToken(unittest.TestCase): + def test_it_is_not_a_where_token_here(self): + # C# had to override AddAlias on it because MoreLikeThisToken derives from + # WhereToken there. In this client it does not, so alias rewriting skips it. + self.assertFalse(issubclass(MoreLikeThisToken, WhereToken)) From 9d935d20a48da5e9415bab148a1a8c97cbf7c87b Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:04 +0200 Subject: [PATCH 12/14] RDBC-1112 Keep every new parameter additive and leave wait_for_completion alone Four constructors took a new parameter in the middle of their existing signature, which breaks anyone passing positionally: output_options on RunConversationOperation and its command, store_chunk_text on EmbeddingsGenerationConfiguration, and disable_checksum_validation on RemoteAttachmentsS3Settings. All four move to the end. Operation.wait_for_completion also started returning the server's result instead of None, which is a change every caller can see. It returns None again; the result now comes from _wait_for_completion_result, and the smuggler is its only caller. Also covers the six symbols the coverage audit found untested: import_incremental, export_to_database, get_conversation_messages, CdcSinkTableLoadState, ConnectionStringUsage.list_from_json and UpdateQueueSinkOperationResult. --- .../ai/agents/run_conversation_operation.py | 4 +- .../ai/embeddings_generation_configuration.py | 2 +- .../operations/attachments/__init__.py | 22 ++-- ravendb/documents/operations/operation.py | 11 +- .../documents/smuggler/database_smuggler.py | 2 +- .../test_ai_conversation_messages.py | 40 +++++++ .../tests/documents_tests/test_smuggler.py | 102 ++++++++++++++++++ .../tests/operations_tests/test_cdc_sink.py | 23 ++++ .../tests/operations_tests/test_queue_sink.py | 9 ++ .../test_server_wide_connection_strings.py | 21 ++++ .../test_by_index_actions.py | 10 +- 11 files changed, 223 insertions(+), 23 deletions(-) diff --git a/ravendb/documents/operations/ai/agents/run_conversation_operation.py b/ravendb/documents/operations/ai/agents/run_conversation_operation.py index 49a6d7ee..5dae0647 100644 --- a/ravendb/documents/operations/ai/agents/run_conversation_operation.py +++ b/ravendb/documents/operations/ai/agents/run_conversation_operation.py @@ -303,9 +303,9 @@ def __init__( stream_property_path: Optional[str] = None, streamed_chunks_callback: Optional[Callable[[str], None]] = None, attachments_commands: Optional[List[Any]] = None, - output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, + output_options: Optional[AiOutputOptions] = None, ): if not agent_id or (isinstance(agent_id, str) and agent_id.isspace()): raise ValueError("agent_id cannot be None or empty") @@ -361,9 +361,9 @@ def __init__( streamed_chunks_callback: Optional[Callable[[str], None]] = None, conventions: Optional[DocumentConventions] = None, attachments_commands: Optional[List[Any]] = None, - output_options: Optional[AiOutputOptions] = None, debug: Optional[bool] = None, cancel_pending_action_tools: bool = False, + output_options: Optional[AiOutputOptions] = None, ): from ravendb.util.util import RaftIdGenerator from ravendb.documents.commands.batches import PutAttachmentCommandData diff --git a/ravendb/documents/operations/ai/embeddings_generation_configuration.py b/ravendb/documents/operations/ai/embeddings_generation_configuration.py index 9022cd9d..90b2937d 100644 --- a/ravendb/documents/operations/ai/embeddings_generation_configuration.py +++ b/ravendb/documents/operations/ai/embeddings_generation_configuration.py @@ -37,12 +37,12 @@ def __init__( chunking_options_for_querying: ChunkingOptions = None, embeddings_cache_expiration: timedelta = None, embeddings_cache_for_querying_expiration: timedelta = None, - store_chunk_text: bool = False, disabled: bool = False, mentor_node: str = None, pin_to_mentor_node: bool = False, task_id: int = 0, allow_etl_on_non_encrypted_channel: bool = False, + store_chunk_text: bool = False, ): super().__init__( name=name, diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 53c0efa8..fdc8718e 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -403,8 +403,8 @@ def __init__( bucket_name: str = None, custom_server_url: str = None, force_path_style: bool = None, - disable_checksum_validation: bool = None, storage_class: Optional[S3StorageClass] = None, + disable_checksum_validation: bool = None, ): self.aws_access_key = aws_access_key self.aws_secret_key = aws_secret_key @@ -421,16 +421,16 @@ def __init__( def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: storage_class_raw = json_dict.get("StorageClass") return cls( - json_dict.get("AwsAccessKey"), - json_dict.get("AwsSecretKey"), - json_dict.get("AwsSessionToken"), - json_dict.get("AwsRegionName"), - json_dict.get("RemoteFolderName"), - json_dict.get("BucketName"), - json_dict.get("CustomServerUrl"), - json_dict.get("ForcePathStyle"), - json_dict.get("DisableChecksumValidation"), - S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + aws_access_key=json_dict.get("AwsAccessKey"), + aws_secret_key=json_dict.get("AwsSecretKey"), + aws_session_token=json_dict.get("AwsSessionToken"), + aws_region_name=json_dict.get("AwsRegionName"), + remote_folder_name=json_dict.get("RemoteFolderName"), + bucket_name=json_dict.get("BucketName"), + custom_server_url=json_dict.get("CustomServerUrl"), + force_path_style=json_dict.get("ForcePathStyle"), + storage_class=S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + disable_checksum_validation=json_dict.get("DisableChecksumValidation"), ) def to_json(self) -> dict: diff --git a/ravendb/documents/operations/operation.py b/ravendb/documents/operations/operation.py index 18273ceb..0ea5b667 100644 --- a/ravendb/documents/operations/operation.py +++ b/ravendb/documents/operations/operation.py @@ -40,8 +40,15 @@ def _get_operation_state_command( ) -> RavenCommand[dict]: return GetOperationStateOperation.GetOperationStateCommand(self.__key, node_tag) - def wait_for_completion(self) -> Optional[dict]: - """Blocks until the operation finishes, then hands back whatever result it carried.""" + def wait_for_completion(self) -> None: + """Blocks until the operation finishes.""" + self._wait_for_completion_result() + + def _wait_for_completion_result(self) -> Optional[dict]: + """ + The same wait, handing back the result the server reported. Kept separate so + wait_for_completion goes on returning nothing, which is what callers expect. + """ while True: status = self.fetch_operations_status() operation_status = status.get("Status") diff --git a/ravendb/documents/smuggler/database_smuggler.py b/ravendb/documents/smuggler/database_smuggler.py index 7e37f061..7fa9ad40 100644 --- a/ravendb/documents/smuggler/database_smuggler.py +++ b/ravendb/documents/smuggler/database_smuggler.py @@ -55,7 +55,7 @@ class SmugglerOperation(Operation): """An export or import operation, whose result is a typed SmugglerResult.""" def wait_for_completion(self) -> SmugglerResult: - return SmugglerResult.from_json(super().wait_for_completion()) + return SmugglerResult.from_json(self._wait_for_completion_result()) def _is_backup_file(file_path: str) -> bool: diff --git a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py index ddce3c8c..9284c1d3 100644 --- a/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py +++ b/ravendb/tests/ai_agent_tests/test_ai_conversation_messages.py @@ -17,6 +17,7 @@ GetConversationMessagesOptions, RunConversationOperation, ) +from ravendb.documents.ai.ai_operations import AiOperations from ravendb.http.server_node import ServerNode from ravendb.primitives import constants @@ -215,3 +216,42 @@ def test_the_flag_is_sent_on_every_run(self): def test_the_flag_is_sent_when_it_is_set(self): self.assertIn("&cancelPendingActionTools=True", self._url(cancel_pending_action_tools=True)) + + +class TestGetConversationMessagesEntryPoint(unittest.TestCase): + """AiOperations.get_conversation_messages takes either an id or full options.""" + + class _RecordingMaintenance: + def __init__(self): + self.sent = None + + def send(self, operation): + self.sent = operation + return "result" + + class _Store: + def __init__(self, maintenance): + self.maintenance = maintenance + + def _operations(self): + maintenance = self._RecordingMaintenance() + return AiOperations(self._Store(maintenance)), maintenance + + def test_a_bare_id_becomes_an_operation(self): + operations, maintenance = self._operations() + + self.assertEqual("result", operations.get_conversation_messages("chats/1")) + self.assertIsInstance(maintenance.sent, GetConversationMessagesOperation) + + def test_options_are_passed_through(self): + operations, maintenance = self._operations() + options = GetConversationMessagesOptions( + conversation_id="chats/1", page_size=10, detail_level=AiConversationDetailLevel.FULL + ) + + operations.get_conversation_messages(options) + url = maintenance.sent.get_command(None).create_request(ServerNode("http://localhost:8080", "db")).url + + self.assertIn("conversationId=chats%2F1", url) + self.assertIn("pageSize=10", url) + self.assertIn("detailLevel=Full", url) diff --git a/ravendb/tests/documents_tests/test_smuggler.py b/ravendb/tests/documents_tests/test_smuggler.py index 9748ddba..97e0ca85 100644 --- a/ravendb/tests/documents_tests/test_smuggler.py +++ b/ravendb/tests/documents_tests/test_smuggler.py @@ -422,3 +422,105 @@ def test_every_section_the_server_sends_has_a_home(self): "TimeSeriesDeletedRanges", ): self.assertIn(name, keys) + + +class TestSmugglerIncrementalImport(TestBase): + """ + import_incremental walks a backup directory in order. Indexes and subscriptions come + from the last file only, so an earlier incremental cannot resurrect what a later + backup dropped. + """ + + def _dump_of(self, store, options: DatabaseSmugglerExportOptions, path: str) -> None: + store.smuggler.export(options, path).wait_for_completion() + + def test_it_imports_every_file_in_order(self): + with self.store.open_session() as session: + session.store(User(name="first"), "users/1") + session.save_changes() + + with tempfile.TemporaryDirectory() as directory: + full = os.path.join(directory, "2026-06-16-10-00.ravendb-full-backup") + self._dump_of(self.store, DatabaseSmugglerExportOptions(), full) + + with self.store.open_session() as session: + session.store(User(name="second"), "users/2") + session.save_changes() + + incremental = os.path.join(directory, "2026-06-16-11-00.ravendb-incremental-backup") + self._dump_of(self.store, DatabaseSmugglerExportOptions(), incremental) + + with self.get_document_store() as target: + target.smuggler.import_incremental(DatabaseSmugglerImportOptions(), directory) + + with target.open_session() as session: + self.assertEqual("first", session.load("users/1", User).name) + self.assertEqual("second", session.load("users/2", User).name) + + def test_an_empty_directory_is_a_no_op(self): + with tempfile.TemporaryDirectory() as directory: + # Nothing that looks like a backup file, so there is nothing to import. + open(os.path.join(directory, "notes.txt"), "wb").close() + + self.assertIsNone(self.store.smuggler.import_incremental(DatabaseSmugglerImportOptions(), directory)) + + def test_it_restores_the_selection_it_narrowed(self): + options = DatabaseSmugglerImportOptions() + before = set(options.operate_on_types) + + with tempfile.TemporaryDirectory() as directory: + dump = os.path.join(directory, "2026-06-16-10-00.ravendb-full-backup") + self._dump_of(self.store, DatabaseSmugglerExportOptions(), dump) + self.store.smuggler.import_incremental(options, directory) + + # Tombstones are added on the way in and kept; indexes and subscriptions come back. + self.assertIn(DatabaseItemType.INDEXES, options.operate_on_types) + self.assertIn(DatabaseItemType.SUBSCRIPTIONS, options.operate_on_types) + self.assertTrue(before.issubset(options.operate_on_types)) + + def test_missing_options_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + self.store.smuggler.import_incremental(None, directory) + + +class TestSmugglerExportToDatabase(TestBase): + def test_it_moves_documents_into_another_database(self): + with self.store.open_session() as session: + for i in range(4): + session.store(User(name=f"user-{i}"), f"users/{i}") + session.save_changes() + + with self.get_document_store() as target: + result = self.store.smuggler.export_to_database( + DatabaseSmugglerExportOptions(), target.smuggler + ).wait_for_completion() + + # The import side is the one worth reporting, so that is what comes back. + self.assertIsInstance(result, SmugglerResult) + self.assertEqual(4, result.documents.read_count) + + with target.open_session() as session: + self.assertEqual("user-2", session.load("users/2", User).name) + + def test_it_carries_the_export_selection_over(self): + with self.store.open_session() as session: + session.store(User(name="a user"), "users/1") + session.store({"Name": "an order", "@metadata": {"@collection": "Orders"}}, "orders/1") + session.save_changes() + + with self.get_document_store() as target: + self.store.smuggler.export_to_database( + DatabaseSmugglerExportOptions(operate_on_types={DatabaseItemType.DOCUMENTS}, collections=["Users"]), + target.smuggler, + ).wait_for_completion() + + with target.open_session() as session: + self.assertIsNotNone(session.load("users/1", User)) + self.assertIsNone(session.load("orders/1")) + + def test_missing_arguments_are_rejected(self): + with self.assertRaises(ValueError): + self.store.smuggler.export_to_database(None, self.store.smuggler) + with self.assertRaises(ValueError): + self.store.smuggler.export_to_database(DatabaseSmugglerExportOptions(), None) diff --git a/ravendb/tests/operations_tests/test_cdc_sink.py b/ravendb/tests/operations_tests/test_cdc_sink.py index 8dcb0a15..97de1bab 100644 --- a/ravendb/tests/operations_tests/test_cdc_sink.py +++ b/ravendb/tests/operations_tests/test_cdc_sink.py @@ -20,6 +20,7 @@ CdcSinkProcessState, CdcSinkRelationType, CdcSinkTableConfig, + CdcSinkTableLoadState, CdcSinkTaskState, UpdateCdcSinkOperation, ) @@ -550,3 +551,25 @@ def test_a_cdc_sink_task_is_stored_and_read_back(self): self.assertEqual( ["Lines"], [embedded.property_name for embedded in task.configuration.tables[0].embedded_tables] ) + + +class TestCdcSinkTableLoadState(unittest.TestCase): + """Per-table initial-load progress, so an interrupted load can pick up where it stopped.""" + + def test_it_round_trips(self): + payload = {"InitialLoadCompleted": True, "LastKeyValues": ["42"], "KeyColumns": ["id"]} + + self.assertEqual(payload, CdcSinkTableLoadState.from_json(payload).to_json()) + + def test_it_carries_the_resume_position(self): + state = CdcSinkTableLoadState.from_json({"LastKeyValues": ["42", "b"], "KeyColumns": ["id", "code"]}) + + self.assertEqual(["42", "b"], state.last_key_values) + self.assertEqual(["id", "code"], state.key_columns) + self.assertFalse(state.initial_load_completed) + + def test_an_empty_payload_reads_as_not_started(self): + state = CdcSinkTableLoadState.from_json({}) + + self.assertFalse(state.initial_load_completed) + self.assertIsNone(state.last_key_values) diff --git a/ravendb/tests/operations_tests/test_queue_sink.py b/ravendb/tests/operations_tests/test_queue_sink.py index e9a25e7e..e618b42b 100644 --- a/ravendb/tests/operations_tests/test_queue_sink.py +++ b/ravendb/tests/operations_tests/test_queue_sink.py @@ -21,6 +21,7 @@ QueueSinkProcessState, QueueSinkScript, UpdateQueueSinkOperation, + UpdateQueueSinkOperationResult, ) from ravendb.documents.operations.connection_string.put_connection_string_operation import ( PutConnectionStringOperation, @@ -160,6 +161,14 @@ def test_a_missing_configuration_is_rejected_client_side(self): with self.assertRaises(ValueError): UpdateQueueSinkOperation(1, None) + def test_update_reads_the_task_id_off_the_response(self): + command = UpdateQueueSinkOperation(4, _configuration()).get_command(None) + command.set_response(json.dumps({"RaftCommandIndex": 11, "TaskId": 4}), False) + + self.assertIsInstance(command.result, UpdateQueueSinkOperationResult) + self.assertEqual(11, command.result.raft_command_index) + self.assertEqual(4, command.result.task_id) + def test_add_reads_the_task_id_off_the_response(self): command = AddQueueSinkOperation(_configuration()).get_command(None) command.set_response(json.dumps({"RaftCommandIndex": 9, "TaskId": 4}), False) diff --git a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py index 871832b2..d2478aec 100644 --- a/ravendb/tests/operations_tests/test_server_wide_connection_strings.py +++ b/ravendb/tests/operations_tests/test_server_wide_connection_strings.py @@ -310,3 +310,24 @@ def test_filtering_by_name_and_type_narrows_the_listing(self): self.store.maintenance.server.send( RemoveServerWideConnectionStringOperation(RavenConnectionString(name)) ) + + +class TestConnectionStringUsageList(unittest.TestCase): + def test_a_list_of_usages_is_parsed(self): + usages = ConnectionStringUsage.list_from_json( + [ + {"Kind": "RavenEtl", "Id": 1, "Name": "etl"}, + {"Kind": "AiAgent", "Identifier": "agents/1", "Name": "agent"}, + ] + ) + + self.assertEqual(2, len(usages)) + self.assertEqual(ConnectionStringUsageKind.RAVEN_ETL, usages[0].kind) + self.assertEqual(1, usages[0].id_) + # An AI agent is identified by a string rather than a numeric task id. + self.assertEqual("agents/1", usages[1].identifier) + self.assertIsNone(usages[1].id_) + + def test_nothing_parses_to_an_empty_list(self): + self.assertEqual([], ConnectionStringUsage.list_from_json(None)) + self.assertEqual([], ConnectionStringUsage.list_from_json([])) diff --git a/ravendb/tests/raven_commands_tests/test_by_index_actions.py b/ravendb/tests/raven_commands_tests/test_by_index_actions.py index e5b5e3a6..a51fbe3d 100644 --- a/ravendb/tests/raven_commands_tests/test_by_index_actions.py +++ b/ravendb/tests/raven_commands_tests/test_by_index_actions.py @@ -112,12 +112,10 @@ def test_delete_by_index_success(self): response.operation_id, response.operation_node_tag, ) - # wait_for_completion hands back the result the server reported for the operation. - # 45, not 50: the range is quoted, so the server compares DocNumber as a string - # and "5".."9" fall outside "0".."49" lexicographically. - result = x.wait_for_completion() - self.assertEqual(45, result["Total"]) - self.assertEqual("Processed 45 items.", result["Message"]) + # wait_for_completion returns nothing; _wait_for_completion_result is what + # carries the server's report, and the smuggler is its only caller. + self.assertIsNone(x.wait_for_completion()) + self.assertEqual(45, x._wait_for_completion_result()["Total"]) if __name__ == "__main__": From 24828f28aff533d338c57d3352e19e1e86940559 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:04 +0200 Subject: [PATCH 13/14] RDBC-1112 Reach the whole public surface from the package root - Export the two new types callers reach for by hand - Export the types this branch's own API hands back - Reach the rest of the public surface from the package root - Leave only real gaps in the commented backlog - Guard every export with the import test --- ravendb/__init__.py | 658 +++++++++++++++++++------------- ravendb/tests/test_imports.py | 697 +++++++++++++++++++++------------- 2 files changed, 816 insertions(+), 539 deletions(-) diff --git a/ravendb/__init__.py b/ravendb/__init__.py index 21654a7e..b4e6c79b 100644 --- a/ravendb/__init__.py +++ b/ravendb/__init__.py @@ -37,6 +37,13 @@ PutAttachmentOperation, GetAttachmentOperation, AttachmentRequest, + ConfigureRemoteAttachmentsOperation, + ConfigureRemoteAttachmentsOperationResult, + GetRemoteAttachmentsConfigurationOperation, + RemoteAttachmentsAzureSettings, + RemoteAttachmentsConfiguration, + RemoteAttachmentsDestinationConfiguration, + RemoteAttachmentsS3Settings, ) from ravendb.documents.operations.backups.settings import ( BackupConfiguration, @@ -167,6 +174,7 @@ ) from ravendb.documents.operations.lazy.definition import LazyOperation from ravendb.documents.operations.misc import DeleteByQueryOperation, GetOperationStateOperation, QueryOperationOptions +from ravendb.documents.conventions import SessionPatchBehavior from ravendb.documents.operations.json_patch import ( JsonPatchDocument, JsonPatchOperation, @@ -203,10 +211,12 @@ GetPullReplicationTasksInfoOperation, ) from ravendb.documents.operations.ongoing_tasks import ( + GetOngoingTaskInfoOperation, OngoingTaskPullReplicationAsSink, OngoingTaskPullReplicationAsHub, OngoingTaskCdcSink, OngoingTaskQueueSink, + OngoingTaskType, ) from ravendb.documents.operations.queue_sink import ( AddQueueSinkOperation, @@ -255,6 +265,7 @@ ) from ravendb.documents.smuggler.database_smuggler import SmugglerOperation from ravendb.exceptions.commercial import LicenseLimitException, LimitType +from ravendb.exceptions.raven_exceptions import QueryToolFailedException from ravendb.documents.ai.ai_output_options import AiOutputOptions from ravendb.documents.operations.cdc_sink import ( AddCdcSinkOperation, @@ -319,7 +330,7 @@ AggregationQueryBase, ) from ravendb.documents.queries.group_by import GroupBy, GroupByMethod -from ravendb.documents.queries.highlighting import HighlightingOptions, QueryHighlightings +from ravendb.documents.queries.highlighting import HighlightingOptions, Highlightings, QueryHighlightings from ravendb.documents.queries.index_query import IndexQuery from ravendb.documents.queries.misc import SearchOperator from ravendb.documents.queries.raven_document_query import RavenDocumentQuery @@ -417,14 +428,11 @@ # StatusCode # UriUtility -# ServerWide -# CompactSettings from ravendb.json.metadata_as_dictionary import MetadataAsDictionary from ravendb.json.result import BatchCommandResult from ravendb.serverwide.commands import GetDatabaseTopologyCommand, GetClusterTopologyCommand from ravendb.serverwide.misc import DocumentsCompressionConfiguration, DeletionInProgressStatus -# IDatabaseTaskStatus from ravendb.serverwide.operations.certificates import ( CertificateMetadata, EditClientCertificateOperation, @@ -472,103 +480,406 @@ GenerateEntityIdOnTheClient, ) +# The rest of the public surface, reachable straight from `ravendb` like everything above. +from ravendb.documents.bulk_insert_operation import ( + BulkInsertOperation, + BulkInsertOptions, +) +from ravendb.documents.commands.batches import ( + CommandType, + IndexBatchOptions, + ReplicationBatchOptions, +) +from ravendb.documents.commands.crud import ( + ConditionalGetResult, + PutResult, +) +from ravendb.documents.indexes.definitions import ( + AggregationOperation, + AutoFieldIndexing, + FieldIndexing, + FieldStorage, + FieldTermVector, + GroupByArrayBehavior, + IndexDefinitionBase, + IndexDefinitionCompareDifferences, + IndexErrors, + IndexLockMode, + IndexPriority, + IndexRunningStatus, + IndexState, + IndexType, + IndexingError, + SearchEngineType, + SortOptions, +) +from ravendb.documents.indexes.spatial.configuration import ( + AutoSpatialMethodType, + SpatialFieldType, + SpatialOptions, + SpatialOptionsFactory, + SpatialRelation, + SpatialSearchStrategy, + SpatialUnits, +) +from ravendb.documents.ai.ai_conversation import AiHandleErrorStrategy +from ravendb.documents.operations.ai.add_gen_ai_operation import AddGenAiOperation +from ravendb.documents.operations.ai.update_gen_ai_operation import UpdateGenAiOperation +from ravendb.documents.operations.ai.agents.ai_agent_configuration import AiAgentToolQueryOptions +from ravendb.documents.operations.backups.settings import ( + BackupEncryptionSettings, + BackupType, + GetBackupConfigurationScript, + RetentionPolicy, + CompressionLevel, + EncryptionMode, + S3StorageClass, + SnapshotSettings, +) +from ravendb.documents.operations.compact import CompactDatabaseOperation +from ravendb.documents.operations.connection_string.get_connection_string_operation import ( + GetConnectionStringsOperation, + GetConnectionStringsResult, +) +from ravendb.documents.operations.connection_string.put_connection_string_operation import ( + PutConnectionStringOperation, + PutConnectionStringResult, +) +from ravendb.documents.operations.connection_string.remove_connection_string_operation import ( + RemoveConnectionStringOperation, + RemoveConnectionStringResult, +) +from ravendb.documents.operations.counters import ( + CounterBatch, + CounterBatchOperation, + CounterDetail, + CounterOperation, + CounterOperationType, + CountersDetail, + DocumentCountersOperation, + GetCountersOperation, +) +from ravendb.documents.operations.definitions import ( + IOperation, + MaintenanceOperation, + OperationExceptionResult, + OperationIdResult, + VoidMaintenanceOperation, + VoidOperation, +) +from ravendb.documents.operations.etl.configuration import RavenConnectionString +from ravendb.documents.operations.etl.etl_operation_results import ( + AddEtlOperationResult, + UpdateEtlOperationResult, +) +from ravendb.documents.operations.etl.olap.connection import OlapConnectionString +from ravendb.documents.operations.etl.queue.connection import ( + QueueBrokerType, + QueueConnectionString, +) +from ravendb.documents.operations.etl.queue.amazon_sqs_connection_settings import ( + AmazonSqsConnectionSettings, + AmazonSqsCredentials, +) +from ravendb.documents.operations.etl.queue.azure_queue_storage_connection_settings import ( + AzureQueueStorageConnectionSettings, + EntraId, + Passwordless, +) +from ravendb.documents.operations.etl.queue.azure_service_bus_connection_settings import ( + AzureServiceBusConnectionSettings, + AzureServiceBusEntraId, + AzureServiceBusPasswordless, +) +from ravendb.documents.operations.etl.queue.kafka_connection_settings import KafkaConnectionSettings +from ravendb.documents.operations.etl.queue.rabbit_mq_connection_settings import RabbitMqConnectionSettings +from ravendb.documents.operations.etl.sql import SqlConnectionString +from ravendb.documents.operations.expiration.operations import ( + ConfigureExpirationOperation, + ConfigureExpirationOperationResult, +) +from ravendb.documents.operations.identities import ( + GetIdentitiesOperation, + NextIdentityForOperation, + SeedIdentityForOperation, +) +from ravendb.documents.operations.indexes import ( + DeleteIndexErrorsOperation, + IndexStatus, + ResetIndexOperation, +) +from ravendb.documents.operations.ongoing_tasks import ( + DeleteOngoingTaskOperation, + NodeId, + OngoingTask, + OngoingTaskConnectionStatus, + OngoingTaskEmbeddingsGeneration, + OngoingTaskGenAi, + OngoingTaskState, + ToggleOngoingTaskStateOperation, +) +from ravendb.documents.operations.operation import Operation +from ravendb.documents.operations.refresh.configuration import ( + ConfigureRefreshOperation, + ConfigureRefreshOperationResult, +) +from ravendb.documents.operations.replication.definitions import ReplicationType +from ravendb.documents.operations.revisions import RevisionIncludeResult +from ravendb.documents.operations.schema_validation import ( + ConfigureSchemaValidationOperation, + ConfigureSchemaValidationOperationResult, + GetSchemaValidationConfiguration, + SchemaDefinition, + SchemaValidationConfiguration, + StartSchemaValidationOperation, + ValidateSchemaProgress, + ValidateSchemaResult, +) +from ravendb.documents.operations.server_misc import ( + DisableDatabaseToggleResult, + ToggleDatabasesStateOperation, +) +from ravendb.documents.operations.sorters import ( + DeleteSorterOperation, + PutSortersOperation, +) +from ravendb.documents.operations.statistics import ( + CollectionDetails, + DetailedCollectionStatistics, + GetDetailedCollectionStatisticsOperation, +) +from ravendb.documents.operations.time_series import ( + ConfigureRawTimeSeriesPolicyOperation, + ConfigureTimeSeriesOperation, + ConfigureTimeSeriesOperationResult, + ConfigureTimeSeriesPolicyOperation, + ConfigureTimeSeriesValueNamesOperation, + GetMultipleTimeSeriesOperation, + GetTimeSeriesOperation, + GetTimeSeriesStatisticsOperation, + RawTimeSeriesPolicy, + RemoveTimeSeriesPolicyOperation, + TimeSeriesBatchOperation, + TimeSeriesCollectionConfiguration, + TimeSeriesConfiguration, + TimeSeriesDetails, + TimeSeriesItemDetail, + TimeSeriesOperation, + TimeSeriesPolicy, + TimeSeriesRangeResult, + TimeSeriesStatistics, +) +from ravendb.exceptions.raven_exceptions import ( + AiException, + BadResponseException, + ClientVersionMismatchException, + ConcurrencyException, + ConflictException, + IndexCompactionInProgressException, + InsufficientQuotaException, + MissingAiAgentParameterException, + PortInUseException, + RateLimitException, + RavenException, + RefusedToAnswerException, + ReplicationHubNotFoundException, + SchemaValidationException, + TooManyRequestsException, + TooManyTokensException, + UnsuccessfulAiRequestException, +) +from ravendb.http.misc import ( + AggressiveCacheMode, + ResponseDisposeHandling, +) +from ravendb.serverwide.misc import CompactSettings +from ravendb.serverwide.operations.analyzers import ( + DeleteServerWideAnalyzerOperation, + PutServerWideAnalyzersOperation, +) +from ravendb.serverwide.operations.common import ( + AddDatabaseNodeOperation, + DatabasePromotionStatus, + DatabasePutResult, + DatabaseSettings, + DeleteDatabaseOperation, + DeleteDatabaseResult, + ModifyOngoingTaskResult, + PromoteDatabaseNodeOperation, + ReorderDatabaseMembersOperation, + ServerOperation, + ServerWideOperation, + VoidServerOperation, +) +from ravendb.serverwide.operations.configuration import ( + DeleteServerWideTaskOperation, + GetDatabaseSettingsOperation, + GetServerWideBackupConfigurationOperation, + GetServerWideBackupConfigurationsOperation, + PutDatabaseSettingsOperation, + PutServerWideBackupConfigurationOperation, + ServerWideBackupConfiguration, +) +from ravendb.serverwide.operations.documents_compression import ( + DocumentCompressionConfigurationResult, + UpdateDocumentsCompressionConfigurationOperation, +) +from ravendb.serverwide.operations.logs import ( + AdminLogsConfiguration, + AuditLogsConfiguration, + GetLogsConfigurationOperation, + GetLogsConfigurationResult, + LogFilter, + LogFilterAction, + LogLevel, + LogsConfiguration, + MicrosoftLogsConfiguration, + SetLogsConfigurationOperation, +) +from ravendb.serverwide.operations.ongoing_tasks import ( + IServerWideTask, + ServerWideTaskResponse, + SetDatabasesLockOperation, +) +from ravendb.serverwide.operations.sorters import ( + DeleteServerWideSorterOperation, + PutServerWideSortersOperation, +) +from ravendb.changes.database_changes import DatabaseChanges +from ravendb.changes.observers import ( + ActionObserver, + Observable, +) +from ravendb.changes.types import ( + CounterChange, + CounterChangeTypes, + DatabaseChange, + DocumentChange, + DocumentChangeType, + IndexChange, + IndexChangeTypes, + OperationStatusChange, + TimeSeriesChange, + TimeSeriesChangeTypes, + TopologyChange, +) +from ravendb.documents.commands.stream import ( + StreamResult, + StreamResultResponse, +) +from ravendb.documents.commands.subscriptions import UpdateSubscriptionResult +from ravendb.documents.indexes.abstract_index_creation_tasks import AbstractJavaScriptIndexCreationTask +from ravendb.documents.indexes.counters import ( + AbstractCountersIndexCreationTask, + AbstractGenericCountersIndexCreationTask, + CountersIndexDefinition, + CountersIndexDefinitionBuilder, +) +from ravendb.documents.indexes.stats import IndexStats +from ravendb.documents.indexes.time_series import ( + AbstractGenericTimeSeriesIndexCreationTask, + AbstractMultiMapTimeSeriesIndexCreationTask, + AbstractTimeSeriesIndexCreationTask, + TimeSeriesIndexDefinition, + TimeSeriesIndexDefinitionBuilder, +) +from ravendb.documents.operations.etl.transformation import Transformation +from ravendb.documents.operations.lazy.revisions import ( + LazyRevisionOperation, + LazyRevisionOperations, +) +from ravendb.documents.queries.facets.definitions import FacetSetup +from ravendb.documents.queries.more_like_this import MoreLikeThisStopWords +from ravendb.documents.queries.spatial import WktField +from ravendb.documents.queries.time_series import ( + TimeSeriesAggregationResult, + TimeSeriesQueryBuilder, + TimeSeriesQueryResult, + TimeSeriesRangeAggregation, + TimeSeriesRawResult, + TypedTimeSeriesAggregationResult, + TypedTimeSeriesRangeAggregation, + TypedTimeSeriesRawResult, +) +from ravendb.documents.session.cluster_transaction_operation import LazyClusterTransactionOperations +from ravendb.documents.session.document_session import ( + SessionDocumentCounters, + SessionDocumentRollupTypedTimeSeries, + SessionDocumentTimeSeries, + SessionDocumentTypedTimeSeries, + SessionTimeSeriesBase, +) +from ravendb.documents.session.document_session_revisions import ( + DocumentSessionRevisions, + DocumentSessionRevisionsBase, +) +from ravendb.documents.session.loaders.include import ( + SubscriptionIncludeBuilder, + TimeSeriesIncludeBuilder, +) +from ravendb.documents.session.operations.lazy import ( + LazyGetCompareExchangeValueOperation, + LazyGetCompareExchangeValuesOperation, +) +from ravendb.documents.session.operations.operations import ( + GetRevisionOperation, + GetRevisionsCountOperation, +) +from ravendb.documents.session.operations.stream import StreamOperation +from ravendb.documents.session.stream_statistics import StreamQueryStatistics +from ravendb.documents.session.time_series import ( + AbstractTimeSeriesRange, + TimeSeriesCountRange, + TimeSeriesEntry, + TimeSeriesRange, + TimeSeriesRangeType, + TimeSeriesTimeRange, + TypedTimeSeriesEntry, + TypedTimeSeriesRollupEntry, +) +from ravendb.documents.subscriptions.document_subscriptions import DocumentSubscriptions +from ravendb.documents.subscriptions.options import ( + SubscriptionCreationOptions, + SubscriptionOpeningStrategy, + SubscriptionUpdateOptions, + SubscriptionWorkerOptions, +) +from ravendb.documents.subscriptions.revision import Revision +from ravendb.documents.subscriptions.state import SubscriptionState +from ravendb.documents.subscriptions.worker import ( + SubscriptionBatch, + SubscriptionWorker, +) +from ravendb.documents.time_series import TimeSeriesOperations + # todo: Serverwide -# ReorderDatabaseMembersOperation # UpdateDatabaseOperation -# GetServerWideBackupConfigurationOperation # SetDatabaseDynamicDistributionOperation # UpdateUnusedDatabasesOperation # todo: Serverwide Operations -# Operations # DeleteDatabasesOperation -# ServerWideOperationCompletionAwaiter -# GetLogsConfigurationResult -# GetLogsConfigurationOperation -# LogMode -# SetLogsConfigurationOperation # DeleteServerWideBackupConfigurationOperation -# GetServerWideBackupConfigurationsOperation -# PutServerWideBackupConfigurationOperation -# ServerWideBackupConfiguration -# DatabaseSettings -# GetDatabaseSettingsOperation -# PutDatabaseSettingsOperation -# GetTcpInfoCommand # AddClusterNodeCommand -# ServerWide # ModifyConflictSolverOperation # todo: Operations and Commands -# BulkInsertOperation -# CollectionDetails # BackupTaskType # DatabaseHealthCheckOperation -# DetailedCollectionStatistics -# GetDetailedCollectionStatisticsOperation -# OperationAbstractions -# CompactDatabaseOperation -# PutConnectionStringOperation -# DeleteSorterOperation -# PutSortersOperation -# CompareExchangeValueJsonConverter -# ICompareExchangeValue # GetServerWideExternalReplicationsResponse -# GetNextOperationIdCommand -# KillOperationCommand -# NextIdentityForCommand -# SeedIdentityForCommand -# ExplainQueryCommand -# GetIdentitiesOperation -# OperationCompletionAwaiter -# DeleteIndexErrorsOperation -# ResetIndexOperation -# GetServerWideBackupConfigurationsResponse -# NextIdentityForOperation -# SeedIdentityForOperation -# IOperationProgress -# IOperationResult -# ReplicationHubAccessResponse # GetConflictsCommand -# PutAttachmentCommandHelper -# SetupDocumentBase -# StreamResultResponse -# StreamResult -# GetRevisionOperation -# GetRevisionsCountOperation -# IEagerSessionOperations -# LazyClusterTransactionOperations -# LazyGetCompareExchangeValueOperation -# LazyGetCompareExchangeValuesOperation -# LazyRevisionOperation -# LazyRevisionOperations -# StreamOperation -# GetConnectionStringsOperation -# RemoveConnectionStringOperation # SqlEtlTable # OlapEtlFileFormat # OlapEtlTable -# Transformation # AddEtlOperation # UpdateEtlOperation # ResetEtlOperation -# DisableDatabaseToggleResult -# ConfigureExpirationOperation -# DeleteOngoingTaskOperation -# OngoingTaskType # RunningBackup # NextBackup -# GetOngoingTaskInfoOperation -# ToggleOngoingTaskStateOperation -# ConfigureRefreshOperation -# ConfigureRefreshOperationResult -# ToggleDatabasesStateOperation # StartTransactionsRecordingOperation # StopTransactionsRecordingOperation # todo: backup -# BackupEncryptionSettings -# BackupEncryptionSettings # GetPeriodicBackupStatusOperation # GetPeriodicBackupStatusOperationResult # LastRaftIndex @@ -581,227 +892,28 @@ # UpdatePeriodicBackupOperationResult # UploadProgress # UploadState -# CompressionLevel -# GetBackupConfigurationScript # RestoreBackupConfigurationBase # RestoreFromAzureConfiguration # RestoreFromGoogleCloudConfiguration # RestoreFromS3Configuration # RestoreType -# RetentionPolicy # todo: Indexes -# Enums -# IndexDefinitionHelper -# IndexStats -# Indexes -# IndexDefinitionBase -# AbstractCsharpIndexCreationTask -# AbstractCsharpMultiMapIndexCreationTask -# AbstractJavaScriptIndexCreationTask # AbstractJavaScriptMultiMapIndexCreationTask # AbstractRawJavaScriptIndexCreationTask -# AbstractCountersIndexCreationTask -# AbstractGenericCountersIndexCreationTask -# AbstractCsharpCountersIndexCreationTask # AbstractMultiMapCountersIndexCreationTask # AbstractRawJavaScriptCountersIndexCreationTask -# CountersIndexDefinition -# CountersIndexDefinitionBuilder -# AbstractGenericTimeSeriesIndexCreationTask -# AbstractMultiMapTimeSeriesIndexCreationTask -# AbstractCsharpTimeSeriesIndexCreationTask # AbstractRawJavaScriptTimeSeriesIndexCreationTask -# AbstractTimeSeriesIndexCreationTask -# TimeSeriesIndexDefinition -# TimeSeriesIndexDefinitionBuilder - -# todo: Store -# DocumentAbstractions - -# todo: Subscriptions -# SubscriptionBatch -# DocumentSubscriptions -# SubscriptionWorker -# SubscriptionWorkerOptions -# SubscriptionCreationOptions -# Revision -# SubscriptionState -# SubscriptionCreationOptions -# UpdateSubscriptionResult -# SubscriptionOpeningStrategy -# SubscriptionUpdateOptions - -# todo: Session -# IAbstractDocumentQueryImpl -# ILazyRevisionsOperations -# IAdvancedSessionOperations -# IDocumentQueryBuilder -# IDocumentQueryBaseSingle -# IEnumerableQuery -# IFilterDocumentQueryBase -# IGraphDocumentQuery -# IGroupByDocumentQuery -# IQueryBase -# QueryEvents -# QueryOptions -# StreamQueryStatistics -# SessionEvents -# ILazyClusterTransactionOperations -# ISessionDocumentAppendTimeSeriesBase -# ISessionDocumentDeleteTimeSeriesBase -# ISessionDocumentRollupTypedAppendTimeSeriesBase -# ISessionDocumentRollupTypedTimeSeries -# ISessionDocumentTimeSeries -# ISessionDocumentTypedAppendTimeSeriesBase -# ISessionDocumentTypedTimeSeries -# DocumentResultStream -# SessionDocumentRollupTypedTimeSeries -# SessionDocumentTimeSeries -# SessionDocumentTypedTimeSeries -# SessionTimeSeriesBase -# ICounterIncludeBuilder -# IAbstractTimeSeriesIncludeBuilder -# ICompareExchangeValueIncludeBuilder -# IDocumentIncludeBuilder -# IGenericIncludeBuilder -# IGenericRevisionIncludeBuilder -# IGenericTimeSeriesIncludeBuilder -# ISubscriptionIncludeBuilder -# ISubscriptionTimeSeriesIncludeBuilder -# TimeSeriesIncludeBuilder -# SubscriptionIncludeBuilder: -# ILazyLoaderWithInclude -# ITimeSeriesIncludeBuilder -# DocumentSessionAttachments -# DocumentSessionAttachmentsBase -# DocumentSessionRevisions -# DocumentSessionRevisionsBase -# IAttachmentsSessionOperations -# IRevisionsSessionOperations -# MetadataObject -# ISessionDocumentCounters -# CounterInternalTypes -# SessionDocumentCounters -# TimeSeriesEntry -# TimeSeriesValue -# TimeSeriesValuesHelper -# TypedTimeSeriesEntry -# TypedTimeSeriesRollupEntry -# TimeSeriesOperations - -# todo: Batch -# StreamResult - -# todo: Counters -# CounterBatch -# GetCountersOperation -# CounterBatchOperation -# CounterOperationType -# CounterOperation -# DocumentCountersOperation -# CounterDetail -# CountersDetail # todo: TimeSeries # AggregationType -# RawTimeSeriesTypes -# ConfigureRawTimeSeriesPolicyOperation -# ConfigureTimeSeriesOperation -# ConfigureTimeSeriesOperationResult -# ConfigureTimeSeriesPolicyOperation -# ConfigureTimeSeriesValueNamesOperation -# GetMultipleTimeSeriesOperation -# GetTimeSeriesOperation -# GetTimeSeriesStatisticsOperation -# RawTimeSeriesPolicy -# RemoveTimeSeriesPolicyOperation -# TimeSeriesBatchOperation -# TimeSeriesCollectionConfiguration -# TimeSeriesConfiguration -# TimeSeriesDetails -# TimeSeriesItemDetail -# TimeSeriesOperation -# TimeSeriesPolicy -# TimeSeriesRange -# TimeSeriesCountRange -# TimeSeriesRangeType -# TimeSeriesTimeRange -# TimeSeriesRangeResult -# TimeSeriesStatistics -# AbstractTimeSeriesRange - -# todo: Auth -# AuthOptions - -# todo: Types -# Callbacks -# Contracts -# Types - -# todo: Queries -# WktField -# FacetSetup -# Facets -# HighlightingParameters -# Hightlightings -# ITimeSeriesQueryBuilder -# TimeSeriesAggregationResult -# TimeSeriesQueryBuilder -# TimeSeriesQueryResult -# TimeSeriesRangeAggregation -# TimeSeriesRawResult -# TypedTimeSeriesAggregationResult -# TypedTimeSeriesRangeAggregation -# TypedTimeSeriesRawResult - -# todo: More Like This -# IMoreLikeThisBuilderBase -# MoreLikeThisStopWords - -# todo: Suggestions -# ISuggestionOperations - -# todo: Attachments -# Attachments # todo: Analyzers # DeleteAnalyzerOperation # PutAnalyzersOperation -# todo: Changes -# IndexChange -# DatabaseChangesOptions -# DocumentChange -# TimeSeriesChange -# CounterChange -# IDatabaseChanges -# DatabaseChange -# OperationStatusChange -# IDatabaseChanges -# DatabaseChanges -# IConnectableChanges -# IChangesObservable -# ChangesObservable -# DatabaseConnectionState -# IChangesConnectionState - -# todo: Smuggler - -# todo: Certificates -# AddDatabaseNodeOperation -# PromoteDatabaseNodeOperation -# DeleteServerWideAnalyzerOperation -# PutServerWideAnalyzersOperation -# DocumentCompressionConfigurationResult -# UpdateDocumentsCompressionConfigurationOperation -# IServerWideTask -# DeleteServerWideTaskOperation -# SetDatabasesLockOperation +# todo: Server-wide tasks # ToggleServerWideTaskStateOperation # GetServerWideExternalReplicationOperation # PutServerWideExternalReplicationOperation -# ServerWideTaskResponse # ServerWideExternalReplication -# DeleteServerWideSorterOperation -# PutServerWideSortersOperation diff --git a/ravendb/tests/test_imports.py b/ravendb/tests/test_imports.py index 0f00b9fd..32b3e7a2 100644 --- a/ravendb/tests/test_imports.py +++ b/ravendb/tests/test_imports.py @@ -28,20 +28,19 @@ def test_imports_at_top_level(self): from ravendb import BuildNumber from ravendb import GetBuildNumberOperation - # from ravendb import ReorderDatabaseMembersOperation + from ravendb import ReorderDatabaseMembersOperation from ravendb import ConfigureRevisionsForConflictsOperation from ravendb import ConfigureRevisionsForConflictsResult # from ravendb import UpdateDatabaseOperation - # from ravendb import GetServerWideBackupConfigurationOperation + from ravendb import GetServerWideBackupConfigurationOperation + # from ravendb import SetDatabaseDynamicDistributionOperation # from ravendb import UpdateUnusedDatabasesOperation - # from ravendb import Operations # from ravendb import DeleteDatabasesOperation from ravendb import GetDatabaseNamesOperation from ravendb import GetServerWideOperationStateOperation - # from ravendb import ServerWideOperationCompletionAwaiter from ravendb import CertificateMetadata from ravendb import EditClientCertificateOperation from ravendb import ReplaceClusterCertificateOperation @@ -50,45 +49,42 @@ def test_imports_at_top_level(self): from ravendb import GetServerWideClientConfigurationOperation from ravendb import PutServerWideClientConfigurationOperation - # from ravendb import GetLogsConfigurationResult - # from ravendb import GetLogsConfigurationOperation - # from ravendb import LogMode - # from ravendb import SetLogsConfigurationOperation + from ravendb import GetLogsConfigurationResult + from ravendb import GetLogsConfigurationOperation + from ravendb import SetLogsConfigurationOperation + # from ravendb import DeleteServerWideBackupConfigurationOperation from ravendb import GetServerWideClientConfigurationOperation - # from ravendb import GetServerWideBackupConfigurationsOperation - # from ravendb import PutServerWideBackupConfigurationOperation - # from ravendb import ServerWideBackupConfiguration - # from ravendb import DatabaseSettings - # from ravendb import GetDatabaseSettingsOperation - # from ravendb import PutDatabaseSettingsOperation + from ravendb import GetServerWideBackupConfigurationsOperation + from ravendb import PutServerWideBackupConfigurationOperation + from ravendb import ServerWideBackupConfiguration + from ravendb import DatabaseSettings + from ravendb import GetDatabaseSettingsOperation + from ravendb import PutDatabaseSettingsOperation from ravendb import GetDatabaseTopologyCommand from ravendb import GetClusterTopologyCommand - # from ravendb import GetTcpInfoCommand # from ravendb import AddClusterNodeCommand from ravendb import CreateDatabaseOperation - # from ravendb import ServerWide # from ravendb import ModifyConflictSolverOperation from ravendb import ConnectionString - # from ravendb import BulkInsertOperation - # from ravendb import CollectionDetails + from ravendb import BulkInsertOperation + from ravendb import CollectionDetails from ravendb import BackupConfiguration # from ravendb import BackupTaskType # from ravendb import DatabaseHealthCheckOperation - # from ravendb import DetailedCollectionStatistics - # from ravendb import GetDetailedCollectionStatisticsOperation - # from ravendb import OperationAbstractions - # from ravendb import CompactDatabaseOperation - # from ravendb import PutConnectionStringOperation + from ravendb import DetailedCollectionStatistics + from ravendb import GetDetailedCollectionStatisticsOperation + from ravendb import CompactDatabaseOperation + from ravendb import PutConnectionStringOperation from ravendb import PatchOperation - # from ravendb import DeleteSorterOperation - # from ravendb import PutSortersOperation + from ravendb import DeleteSorterOperation + from ravendb import PutSortersOperation from ravendb import PatchByQueryOperation from ravendb import PutCompareExchangeValueOperation from ravendb import GetCompareExchangeValueOperation @@ -99,35 +95,28 @@ def test_imports_at_top_level(self): from ravendb import DeleteCompareExchangeValueOperation from ravendb import CompareExchangeSessionValue - # from ravendb import CompareExchangeValueJsonConverter from ravendb import CompareExchangeValueState from ravendb import DeleteByQueryOperation from ravendb import GetCollectionStatisticsOperation from ravendb import CollectionStatistics # from ravendb import GetServerWideExternalReplicationsResponse - # from ravendb import GetNextOperationIdCommand - # from ravendb import KillOperationCommand from ravendb import DeleteDocumentCommand - # from ravendb import NextIdentityForCommand - # from ravendb import SeedIdentityForCommand - # from ravendb import ExplainQueryCommand - # from ravendb import GetIdentitiesOperation + from ravendb import GetIdentitiesOperation from ravendb import GetStatisticsOperation from ravendb import DatabaseStatistics from ravendb import GetOperationStateOperation from ravendb import IndexInformation from ravendb import MaintenanceOperationExecutor - # from ravendb import OperationCompletionAwaiter from ravendb import ClientConfiguration from ravendb import GetClientConfigurationOperation from ravendb import PutClientConfigurationOperation from ravendb import PutDocumentCommand from ravendb import GetIndexNamesOperation - # from ravendb import DeleteIndexErrorsOperation + from ravendb import DeleteIndexErrorsOperation from ravendb import DisableIndexOperation from ravendb import EnableIndexOperation from ravendb import GetIndexingStatusOperation @@ -142,14 +131,11 @@ def test_imports_at_top_level(self): from ravendb import StopIndexOperation from ravendb import StartIndexOperation - # from ravendb import ResetIndexOperation + from ravendb import ResetIndexOperation from ravendb import DeleteIndexOperation - # from ravendb import GetServerWideBackupConfigurationsResponse - # from ravendb import NextIdentityForOperation - # from ravendb import SeedIdentityForOperation - # from ravendb import IOperationProgress - # from ravendb import IOperationResult + from ravendb import NextIdentityForOperation + from ravendb import SeedIdentityForOperation from ravendb import UpdateExternalReplicationOperation from ravendb import PullReplicationDefinitionAndCurrentConnections from ravendb import PutPullReplicationAsHubOperation @@ -164,8 +150,6 @@ def test_imports_at_top_level(self): from ravendb import UpdatePullReplicationAsSinkOperation from ravendb import GetPullReplicationTasksInfoOperation - # from ravendb import IExternalReplication - # from ravendb import ReplicationHubAccessResponse # from ravendb import GetConflictsCommand from ravendb import SetIndexesLockOperation from ravendb import SetIndexesPriorityOperation @@ -176,22 +160,17 @@ def test_imports_at_top_level(self): from ravendb import PatchCommandData from ravendb import PutAttachmentCommandData - # from ravendb import PutAttachmentCommandHelper from ravendb import CommandData from ravendb import GetDatabaseRecordOperation - # from ravendb import SetupDocumentBase - # from ravendb import StreamResultResponse - # from ravendb import StreamResult + from ravendb import StreamResultResponse + from ravendb import StreamResult from ravendb import BatchOperation - # from ravendb import GetRevisionOperation - # from ravendb import GetRevisionsCountOperation + from ravendb import GetRevisionOperation + from ravendb import GetRevisionsCountOperation from ravendb import Lazy - # from ravendb import IEagerSessionOperations - # from ravendb import ILazyOperation - # from ravendb import ILazySessionOperations from ravendb import LazyAggregationQueryOperation from ravendb import LazyLoadOperation from ravendb import LazyQueryOperation @@ -199,19 +178,19 @@ def test_imports_at_top_level(self): from ravendb import LazyStartsWithOperation from ravendb import LazySuggestionQueryOperation - # from ravendb import LazyClusterTransactionOperations - # from ravendb import LazyGetCompareExchangeValueOperation - # from ravendb import LazyGetCompareExchangeValuesOperation + from ravendb import LazyClusterTransactionOperations + from ravendb import LazyGetCompareExchangeValueOperation + from ravendb import LazyGetCompareExchangeValuesOperation from ravendb import LazyConditionalLoadOperation - # from ravendb import LazyRevisionOperation - # from ravendb import LazyRevisionOperations + from ravendb import LazyRevisionOperation + from ravendb import LazyRevisionOperations from ravendb import LoadOperation from ravendb import LoadStartingWithOperation from ravendb import MultiGetOperation from ravendb import QueryOperation - # from ravendb import StreamOperation + from ravendb import StreamOperation from ravendb import DeleteAttachmentOperation from ravendb import PutAttachmentOperation from ravendb import PatchResult @@ -237,8 +216,8 @@ def test_imports_at_top_level(self): from ravendb import StudioConfiguration from ravendb import StudioEnvironment - # from ravendb import GetConnectionStringsOperation - # from ravendb import RemoveConnectionStringOperation + from ravendb import GetConnectionStringsOperation + from ravendb import RemoveConnectionStringOperation from ravendb import EtlConfiguration from ravendb import RavenEtlConfiguration from ravendb import SqlEtlConfiguration @@ -248,7 +227,7 @@ def test_imports_at_top_level(self): # from ravendb import OlapEtlFileFormat # from ravendb import OlapEtlTable - # from ravendb import Transformation + from ravendb import Transformation from ravendb import ExpirationConfiguration from ravendb import PullReplicationAsSink from ravendb import PullReplicationDefinition @@ -256,30 +235,31 @@ def test_imports_at_top_level(self): # from ravendb import AddEtlOperation # from ravendb import UpdateEtlOperation # from ravendb import ResetEtlOperation - # from ravendb import DisableDatabaseToggleResult - # from ravendb import ConfigureExpirationOperation - # from ravendb import DeleteOngoingTaskOperation + from ravendb import DisableDatabaseToggleResult + from ravendb import ConfigureExpirationOperation + from ravendb import DeleteOngoingTaskOperation from ravendb import OngoingTaskPullReplicationAsSink from ravendb import OngoingTaskPullReplicationAsHub - # from ravendb import OngoingTaskType + from ravendb import OngoingTaskType + # from ravendb import RunningBackup # from ravendb import NextBackup - # from ravendb import GetOngoingTaskInfoOperation - # from ravendb import ToggleOngoingTaskStateOperation - # from ravendb import ConfigureRefreshOperation + from ravendb import GetOngoingTaskInfoOperation + from ravendb import ToggleOngoingTaskStateOperation + from ravendb import ConfigureRefreshOperation from ravendb import RefreshConfiguration - # from ravendb import ConfigureRefreshOperationResult - # from ravendb import ToggleDatabasesStateOperation + from ravendb import ConfigureRefreshOperationResult + from ravendb import ToggleDatabasesStateOperation + # from ravendb import StartTransactionsRecordingOperation # from ravendb import StopTransactionsRecordingOperation from ravendb import AmazonSettings from ravendb import AzureSettings - # from ravendb import BackupEncryptionSettings - # from ravendb import BackupEncryptionSettings - # from ravendb import Enums + from ravendb import BackupEncryptionSettings + from ravendb import BackupEncryptionSettings from ravendb import FtpSettings from ravendb import GlacierSettings from ravendb import LocalSettings @@ -300,8 +280,8 @@ def test_imports_at_top_level(self): # from ravendb import UpdatePeriodicBackupOperationResult # from ravendb import UploadProgress # from ravendb import UploadState - # from ravendb import CompressionLevel - # from ravendb import GetBackupConfigurationScript + from ravendb import CompressionLevel + from ravendb import GetBackupConfigurationScript from ravendb import GoogleCloudSettings # from ravendb import RestoreBackupConfigurationBase @@ -309,86 +289,77 @@ def test_imports_at_top_level(self): # from ravendb import RestoreFromGoogleCloudConfiguration # from ravendb import RestoreFromS3Configuration # from ravendb import RestoreType - # from ravendb import RetentionPolicy + from ravendb import RetentionPolicy from ravendb import GetIndexOperation from ravendb import GetIndexErrorsOperation - # from ravendb import Enums from ravendb import IndexDeploymentMode from ravendb import IndexDefinition from ravendb import AbstractCommonApiForIndexes from ravendb import AbstractIndexDefinitionBuilder - # from ravendb import Errors from ravendb import AdditionalAssembly - # from ravendb import IndexDefinitionHelper from ravendb import IndexFieldOptions - # from ravendb import Spatial from ravendb import IndexingStatus from ravendb import RollingIndex from ravendb import RollingIndexDeployment from ravendb import RollingIndexState - # from ravendb import IndexStats + from ravendb import IndexStats from ravendb import IndexSourceType - # from ravendb import Indexes - # from ravendb import StronglyTyped - # from ravendb import IndexDefinitionBase + from ravendb import IndexDefinitionBase from ravendb import AnalyzerDefinition - # from ravendb import AbstractCsharpIndexCreationTask - # from ravendb import AbstractCsharpMultiMapIndexCreationTask - # from ravendb import AbstractJavaScriptIndexCreationTask + from ravendb import AbstractJavaScriptIndexCreationTask + # from ravendb import AbstractJavaScriptMultiMapIndexCreationTask # from ravendb import AbstractRawJavaScriptIndexCreationTask from ravendb import AutoIndexDefinition from ravendb import AutoIndexFieldOptions from ravendb import AutoSpatialOptions - # from ravendb import AbstractCountersIndexCreationTask - # from ravendb import AbstractGenericCountersIndexCreationTask - # from ravendb import AbstractCsharpCountersIndexCreationTask + from ravendb import AbstractCountersIndexCreationTask + from ravendb import AbstractGenericCountersIndexCreationTask + # from ravendb import AbstractMultiMapCountersIndexCreationTask # from ravendb import AbstractRawJavaScriptCountersIndexCreationTask - # from ravendb import CountersIndexDefinition - # from ravendb import CountersIndexDefinitionBuilder - # from ravendb import AbstractGenericTimeSeriesIndexCreationTask - # from ravendb import AbstractMultiMapTimeSeriesIndexCreationTask - # from ravendb import AbstractCsharpTimeSeriesIndexCreationTask + from ravendb import CountersIndexDefinition + from ravendb import CountersIndexDefinitionBuilder + from ravendb import AbstractGenericTimeSeriesIndexCreationTask + from ravendb import AbstractMultiMapTimeSeriesIndexCreationTask + # from ravendb import AbstractRawJavaScriptTimeSeriesIndexCreationTask - # from ravendb import AbstractTimeSeriesIndexCreationTask - # from ravendb import TimeSeriesIndexDefinition - # from ravendb import TimeSeriesIndexDefinitionBuilder + from ravendb import AbstractTimeSeriesIndexCreationTask + from ravendb import TimeSeriesIndexDefinition + from ravendb import TimeSeriesIndexDefinitionBuilder from ravendb import ExternalReplication from ravendb import ReplicationNode from ravendb import ExternalReplicationBase - # from ravendb import DocumentAbstractions from ravendb import DocumentStore from ravendb import DocumentStoreBase from ravendb import IdTypeAndName - # from ravendb import SubscriptionBatch - # from ravendb import DocumentSubscriptions - # from ravendb import SubscriptionWorker - # from ravendb import SubscriptionWorkerOptions - # from ravendb import SubscriptionCreationOptions - # from ravendb import Revision - # from ravendb import SubscriptionState - # from ravendb import SubscriptionCreationOptions - # from ravendb import UpdateSubscriptionResult - # from ravendb import SubscriptionOpeningStrategy - # from ravendb import SubscriptionUpdateOptions + from ravendb import SubscriptionBatch + from ravendb import DocumentSubscriptions + from ravendb import SubscriptionWorker + from ravendb import SubscriptionWorkerOptions + from ravendb import SubscriptionCreationOptions + from ravendb import Revision + from ravendb import SubscriptionState + from ravendb import SubscriptionCreationOptions + from ravendb import UpdateSubscriptionResult + from ravendb import SubscriptionOpeningStrategy + from ravendb import SubscriptionUpdateOptions from ravendb import AbstractDocumentQuery from ravendb import CmpXchg from ravendb import DocumentInfo from ravendb import DocumentQuery from ravendb import DocumentQueryHelper - # from ravendb import DocumentsById from ravendb import DocumentsChanges from ravendb import DocumentSession from ravendb import EntityToJson @@ -396,97 +367,55 @@ def test_imports_at_top_level(self): from ravendb import GroupByDocumentQuery from ravendb import GroupByField - # from ravendb import ILazyRevisionsOperations - # from ravendb import IAdvancedSessionOperations - # from ravendb import IDocumentQueryBaseSingle - # from ravendb import IEnumerableQuery - # from ravendb import IFilterDocumentQueryBase - # from ravendb import IGroupByDocumentQuery from ravendb import IncludesUtil from ravendb import InMemoryDocumentSessionOperations - # from ravendb import QueryBase from ravendb import MethodCall from ravendb import OrderingType - # from ravendb import QueryEvents - # from ravendb import QueryOptions from ravendb import QueryStatistics - # from ravendb import StreamQueryStatistics + from ravendb import StreamQueryStatistics from ravendb import RawDocumentQuery - # from ravendb import SessionEvents from ravendb import WhereParams - # from ravendb import ILazyClusterTransactionOperations - # from ravendb import ISessionDocumentAppendTimeSeriesBase - # from ravendb import ISessionDocumentDeleteTimeSeriesBase - # from ravendb import ISessionDocumentRollupTypedAppendTimeSeriesBase - # from ravendb import ISessionDocumentRollupTypedTimeSeries - # from ravendb import ISessionDocumentTimeSeries - # from ravendb import ISessionDocumentTypedAppendTimeSeriesBase - # from ravendb import ISessionDocumentTypedTimeSeries from ravendb import MetadataAsDictionary - # from ravendb import DocumentResultStream - # from ravendb import SessionDocumentRollupTypedTimeSeries - # from ravendb import SessionDocumentTimeSeries - # from ravendb import SessionDocumentTypedTimeSeries - # from ravendb import SessionTimeSeriesBase - # from ravendb import ICounterIncludeBuilder - # from ravendb import IAbstractTimeSeriesIncludeBuilder - # from ravendb import ICompareExchangeValueIncludeBuilder - # from ravendb import IDocumentIncludeBuilder - # from ravendb import IGenericIncludeBuilder - # from ravendb import IGenericRevisionIncludeBuilder - # from ravendb import IGenericTimeSeriesIncludeBuilder - # from ravendb import ISubscriptionIncludeBuilder - # from ravendb import ISubscriptionTimeSeriesIncludeBuilder - # from ravendb import TimeSeriesIncludeBuilder - # from ravendb import SubscriptionIncludeBuilder - # from ravendb import ILazyLoaderWithInclude + from ravendb import SessionDocumentRollupTypedTimeSeries + from ravendb import SessionDocumentTimeSeries + from ravendb import SessionDocumentTypedTimeSeries + from ravendb import SessionTimeSeriesBase + from ravendb import TimeSeriesIncludeBuilder + from ravendb import SubscriptionIncludeBuilder from ravendb import LoaderWithInclude - # from ravendb import ITimeSeriesIncludeBuilder from ravendb import LazyMultiLoaderWithInclude from ravendb import MultiLoaderWithInclude from ravendb import DocumentQueryCustomization - # from ravendb import DocumentSessionAttachments - # from ravendb import DocumentSessionAttachmentsBase - # from ravendb import DocumentSessionRevisions - # from ravendb import DocumentSessionRevisionsBase - # from ravendb import IAttachmentsSessionOperations - # from ravendb import IDocumentQueryCustomization - # from ravendb import IRevisionsSessionOperations + from ravendb import DocumentSessionRevisions + from ravendb import DocumentSessionRevisionsBase from ravendb import ResponseTimeInformation - # from ravendb import MetadataObject from ravendb import TransactionMode from ravendb import ConditionalLoadResult - # from ravendb import ISessionDocumentCounters from ravendb import ClusterTransactionOperations - # from ravendb import CounterInternalTypes - # from ravendb import IIncludeBuilder from ravendb import IncludeBuilder from ravendb import IncludeBuilderBase - # from ravendb import IQueryIncludeBuilder from ravendb import QueryIncludeBuilder from ravendb import QueryIncludeBuilder from ravendb import BatchCommandResult - # from ravendb import SessionDocumentCounters - # from ravendb import TimeSeriesEntry - # from ravendb import TimeSeriesValue - # from ravendb import TimeSeriesValuesHelper - # from ravendb import TypedTimeSeriesEntry - # from ravendb import TypedTimeSeriesRollupEntry - # from ravendb import TimeSeriesOperations - # from ravendb import StreamResult + from ravendb import SessionDocumentCounters + from ravendb import TimeSeriesEntry + from ravendb import TypedTimeSeriesEntry + from ravendb import TypedTimeSeriesRollupEntry + from ravendb import TimeSeriesOperations + from ravendb import StreamResult from ravendb import SessionOptions from ravendb import CommandData from ravendb import CopyAttachmentCommandData @@ -500,50 +429,45 @@ def test_imports_at_top_level(self): from ravendb import DeleteCompareExchangeCommandData from ravendb import Lazy - # from ravendb import CounterBatch - # from ravendb import GetCountersOperation - # from ravendb import CounterBatchOperation - # from ravendb import CounterOperationType - # from ravendb import CounterOperation - # from ravendb import DocumentCountersOperation - # from ravendb import CounterDetail - # from ravendb import CountersDetail + from ravendb import CounterBatch + from ravendb import GetCountersOperation + from ravendb import CounterBatchOperation + from ravendb import CounterOperationType + from ravendb import CounterOperation + from ravendb import DocumentCountersOperation + from ravendb import CounterDetail + from ravendb import CountersDetail + # from ravendb import AggregationType - # from ravendb import RawTimeSeriesTypes - # from ravendb import ConfigureRawTimeSeriesPolicyOperation - # from ravendb import ConfigureTimeSeriesOperation - # from ravendb import ConfigureTimeSeriesOperationResult - # from ravendb import ConfigureTimeSeriesPolicyOperation - # from ravendb import ConfigureTimeSeriesValueNamesOperation - # from ravendb import GetMultipleTimeSeriesOperation - # from ravendb import GetTimeSeriesOperation - # from ravendb import GetTimeSeriesStatisticsOperation - # from ravendb import RawTimeSeriesPolicy - # from ravendb import RemoveTimeSeriesPolicyOperation - # from ravendb import TimeSeriesBatchOperation - # from ravendb import TimeSeriesCollectionConfiguration - # from ravendb import TimeSeriesConfiguration - # from ravendb import TimeSeriesDetails - # from ravendb import TimeSeriesItemDetail - # from ravendb import TimeSeriesOperation - # from ravendb import TimeSeriesPolicy - # from ravendb import TimeSeriesRange - # from ravendb import TimeSeriesCountRange - # from ravendb import TimeSeriesRangeType - # from ravendb import TimeSeriesTimeRange - # from ravendb import TimeSeriesRangeResult - # from ravendb import TimeSeriesStatistics - # from ravendb import AbstractTimeSeriesRange - # from ravendb import AuthOptions - # from ravendb import Callbacks - # from ravendb import Contracts - # from ravendb import Types + from ravendb import ConfigureRawTimeSeriesPolicyOperation + from ravendb import ConfigureTimeSeriesOperation + from ravendb import ConfigureTimeSeriesOperationResult + from ravendb import ConfigureTimeSeriesPolicyOperation + from ravendb import ConfigureTimeSeriesValueNamesOperation + from ravendb import GetMultipleTimeSeriesOperation + from ravendb import GetTimeSeriesOperation + from ravendb import GetTimeSeriesStatisticsOperation + from ravendb import RawTimeSeriesPolicy + from ravendb import RemoveTimeSeriesPolicyOperation + from ravendb import TimeSeriesBatchOperation + from ravendb import TimeSeriesCollectionConfiguration + from ravendb import TimeSeriesConfiguration + from ravendb import TimeSeriesDetails + from ravendb import TimeSeriesItemDetail + from ravendb import TimeSeriesOperation + from ravendb import TimeSeriesPolicy + from ravendb import TimeSeriesRange + from ravendb import TimeSeriesCountRange + from ravendb import TimeSeriesRangeType + from ravendb import TimeSeriesTimeRange + from ravendb import TimeSeriesRangeResult + from ravendb import TimeSeriesStatistics + from ravendb import AbstractTimeSeriesRange from ravendb import IndexQuery from ravendb import GroupBy from ravendb import QueryOperator from ravendb import SearchOperator - # from ravendb import IIndexQuery from ravendb import GroupByMethod from ravendb import ProjectionBehavior from ravendb import SpatialCriteriaFactory @@ -553,7 +477,7 @@ def test_imports_at_top_level(self): from ravendb import WktCriteria from ravendb import PointField - # from ravendb import WktField + from ravendb import WktField from ravendb import RangeBuilder from ravendb import FacetBuilder from ravendb import FacetAggregationField @@ -561,16 +485,13 @@ def test_imports_at_top_level(self): from ravendb import RangeFacet from ravendb import FacetBase - # from ravendb import FacetSetup - # from ravendb import Facets + from ravendb import FacetSetup from ravendb import AggregationRawDocumentQuery from ravendb import QueryData from ravendb import QueryOperationOptions from ravendb import QueryResult from ravendb import HighlightingOptions - # from ravendb import HighlightingParameters - # from ravendb import Hightlightings from ravendb import QueryTimings from ravendb import AggregationDocumentQuery from ravendb import AggregationQueryBase @@ -582,26 +503,23 @@ def test_imports_at_top_level(self): from ravendb import QueryHighlightings from ravendb import SorterDefinition - # from ravendb import ITimeSeriesQueryBuilder - # from ravendb import TimeSeriesAggregationResult - # from ravendb import TimeSeriesQueryBuilder - # from ravendb import TimeSeriesQueryResult - # from ravendb import TimeSeriesRangeAggregation - # from ravendb import TimeSeriesRawResult - # from ravendb import TypedTimeSeriesAggregationResult - # from ravendb import TypedTimeSeriesRangeAggregation - # from ravendb import TypedTimeSeriesRawResult - # from ravendb import MoreLikeThisBuilderBase + from ravendb import TimeSeriesAggregationResult + from ravendb import TimeSeriesQueryBuilder + from ravendb import TimeSeriesQueryResult + from ravendb import TimeSeriesRangeAggregation + from ravendb import TimeSeriesRawResult + from ravendb import TypedTimeSeriesAggregationResult + from ravendb import TypedTimeSeriesRangeAggregation + from ravendb import TypedTimeSeriesRawResult from ravendb import MoreLikeThisOperations from ravendb import MoreLikeThisBase from ravendb import MoreLikeThisBuilder from ravendb import MoreLikeThisOptions - # from ravendb import MoreLikeThisStopWords + from ravendb import MoreLikeThisStopWords from ravendb import SuggestionBuilder from ravendb import SuggestionDocumentQuery - # from ravendb import SuggestionOperations from ravendb import StringDistanceTypes from ravendb import SuggestionBuilder from ravendb import SuggestionDocumentQuery @@ -610,41 +528,24 @@ def test_imports_at_top_level(self): from ravendb import SuggestionResult from ravendb import SuggestionSortMode - # from ravendb import Attachments from ravendb import GetAttachmentOperation from ravendb import AttachmentRequest # from ravendb import DeleteAnalyzerOperation # from ravendb import PutAnalyzersOperation - # from ravendb import IndexChange - # from ravendb import DatabaseChangesOptions - # from ravendb import DocumentChange - # from ravendb import TimeSeriesChange - # from ravendb import CounterChange - # from ravendb import IDatabaseChanges - # from ravendb import DatabaseChange - # from ravendb import OperationStatusChange - # from ravendb import IDatabaseChanges - # from ravendb import DatabaseChanges - # from ravendb import IConnectableChanges - # from ravendb import IChangesObservable - # from ravendb import ChangesObservable - # from ravendb import DatabaseConnectionState - # from ravendb import IChangesConnectionState - # from ravendb import HiloIdGenerator - # from ravendb import MultiDatabaseHiLoIdGenerator - # from ravendb import MultiTypeHiLoIdGenerator - # from ravendb import HiloRangeValue - # from ravendb import MultiDatabaseHiLoIdGenerator - # from ravendb import DatabaseItemType - # from ravendb import DatabaseRecordItemType - # from ravendb import DatabaseSmuggler - # from ravendb import DatabaseSmugglerExportOptions - # from ravendb import IDatabaseSmugglerExportOptions - # from ravendb import DatabaseSmugglerImportOptions - # from ravendb import IDatabaseSmugglerImportOptions - # from ravendb import DatabaseSmugglerOptions - # from ravendb import IDatabaseSmugglerOptions + from ravendb import IndexChange + from ravendb import DocumentChange + from ravendb import TimeSeriesChange + from ravendb import CounterChange + from ravendb import DatabaseChange + from ravendb import OperationStatusChange + from ravendb import DatabaseChanges + from ravendb import DatabaseItemType + from ravendb import DatabaseRecordItemType + from ravendb import DatabaseSmuggler + from ravendb import DatabaseSmugglerExportOptions + from ravendb import DatabaseSmugglerImportOptions + from ravendb import DatabaseSmugglerOptions from ravendb import CertificateDefinition from ravendb import CertificateRawData from ravendb import CreateClientCertificateOperation @@ -656,24 +557,288 @@ def test_imports_at_top_level(self): from ravendb import PutClientCertificateOperation from ravendb import SecurityClearance - # from ravendb import AddDatabaseNodeOperation - # from ravendb import PromoteDatabaseNodeOperation - # from ravendb import DeleteServerWideAnalyzerOperation - # from ravendb import PutServerWideAnalyzersOperation - # from ravendb import DocumentCompressionConfigurationResult - # from ravendb import UpdateDocumentsCompressionConfigurationOperation - # from ravendb import IServerWideTask - # from ravendb import DeleteServerWideTaskOperation - # from ravendb import SetDatabasesLockOperation + from ravendb import AddDatabaseNodeOperation + from ravendb import PromoteDatabaseNodeOperation + from ravendb import DeleteServerWideAnalyzerOperation + from ravendb import PutServerWideAnalyzersOperation + from ravendb import DocumentCompressionConfigurationResult + from ravendb import UpdateDocumentsCompressionConfigurationOperation + from ravendb import IServerWideTask + from ravendb import DeleteServerWideTaskOperation + from ravendb import SetDatabasesLockOperation + # from ravendb import ToggleServerWideTaskStateOperation # from ravendb import GetServerWideExternalReplicationOperation # from ravendb import PutServerWideExternalReplicationOperation - # from ravendb import ServerWideTaskResponse + from ravendb import ServerWideTaskResponse + # from ravendb import ServerWideExternalReplication - # from ravendb import DeleteServerWideSorterOperation - # from ravendb import PutServerWideSortersOperation - # from ravendb import ObjectMapper - # from ravendb import Mapping + from ravendb import DeleteServerWideSorterOperation + from ravendb import PutServerWideSortersOperation from ravendb import DocumentStore return + + def test_the_rest_of_the_public_surface_imports(self): + # Everything else ravendb/__init__.py binds. Dropping an export fails here. + from ravendb import AbstractIndexCreationTask + from ravendb import ActionObserver + from ravendb import AddCdcSinkOperation + from ravendb import AddCdcSinkOperationResult + from ravendb import AddEmbeddingsGenerationOperation + from ravendb import AddEtlOperationResult + from ravendb import AddGenAiOperation + from ravendb import AddOrUpdateAiAgentOperation + from ravendb import AddQueueSinkOperation + from ravendb import AddQueueSinkOperationResult + from ravendb import AdminLogsConfiguration + from ravendb import AggregationOperation + from ravendb import AggressiveCacheMode + from ravendb import AiAgentActionRequest + from ravendb import AiAgentActionRequestType + from ravendb import AiAgentActionResponse + from ravendb import AiAgentArtificialActionResponse + from ravendb import AiAgentChatTrimmingConfiguration + from ravendb import AiAgentConfiguration + from ravendb import AiAgentConfigurationResult + from ravendb import AiAgentHistoryConfiguration + from ravendb import AiAgentParameter + from ravendb import AiAgentParameterPolicy + from ravendb import AiAgentParameterValueType + from ravendb import AiAgentPersistenceConfiguration + from ravendb import AiAgentSummarizationByTokens + from ravendb import AiAgentToolAction + from ravendb import AiAgentToolQuery + from ravendb import AiAgentToolQueryOptions + from ravendb import AiAgentToolSubAgent + from ravendb import AiAgentTruncateChat + from ravendb import AiConversation + from ravendb import AiConversationCreationOptions + from ravendb import AiConversationDetailLevel + from ravendb import AiConversationMessage + from ravendb import AiConversationMessagesResult + from ravendb import AiConversationParameter + from ravendb import AiConversationParameterOptions + from ravendb import AiConversationResult + from ravendb import AiException + from ravendb import AiHandleErrorStrategy + from ravendb import AiMessagePromptFields + from ravendb import AiMessagePromptTypes + from ravendb import AiMessageRole + from ravendb import AiOperations + from ravendb import AiOutputOptions + from ravendb import AiToolCallResult + from ravendb import AiUsage + from ravendb import AmazonSqsConnectionSettings + from ravendb import AmazonSqsCredentials + from ravendb import AuditLogsConfiguration + from ravendb import AutoFieldIndexing + from ravendb import AutoSpatialMethodType + from ravendb import AzureQueueStorageConnectionSettings + from ravendb import AzureServiceBusConnectionSettings + from ravendb import AzureServiceBusEntraId + from ravendb import AzureServiceBusPasswordless + from ravendb import AzureServiceBusSinkSource + from ravendb import BackupType + from ravendb import BadResponseException + from ravendb import BulkInsertOptions + from ravendb import CdcColumnMapping + from ravendb import CdcColumnType + from ravendb import CdcSinkConfiguration + from ravendb import CdcSinkEmbeddedTableConfig + from ravendb import CdcSinkLinkedTableConfig + from ravendb import CdcSinkOnDeleteConfig + from ravendb import CdcSinkPostgresSettings + from ravendb import CdcSinkProcessState + from ravendb import CdcSinkRelationType + from ravendb import CdcSinkSchemaRequest + from ravendb import CdcSinkSourceColumn + from ravendb import CdcSinkSourceForeignKey + from ravendb import CdcSinkSourceSchema + from ravendb import CdcSinkSourceTable + from ravendb import CdcSinkTableConfig + from ravendb import CdcSinkTableLoadState + from ravendb import CdcSinkTaskState + from ravendb import CertificateUsage + from ravendb import ChunkingMethod + from ravendb import ChunkingOptions + from ravendb import ClientVersionMismatchException + from ravendb import CommandType + from ravendb import CompactSettings + from ravendb import ConcurrencyException + from ravendb import ConditionalGetResult + from ravendb import ConfigureExpirationOperationResult + from ravendb import ConfigureRemoteAttachmentsOperation + from ravendb import ConfigureRemoteAttachmentsOperationResult + from ravendb import ConfigureSchemaValidationOperation + from ravendb import ConfigureSchemaValidationOperationResult + from ravendb import ConflictException + from ravendb import ConnectionStringUsage + from ravendb import ConnectionStringUsageKind + from ravendb import ContentPart + from ravendb import ConversationResult + from ravendb import CounterChangeTypes + from ravendb import Counts + from ravendb import CountsWithLastEtag + from ravendb import CountsWithLastEtagAndAttachments + from ravendb import CountsWithSkippedCountAndLastEtag + from ravendb import CountsWithSkippedCountAndLastEtagAndAttachments + from ravendb import DatabasePromotionStatus + from ravendb import DatabasePutResult + from ravendb import DatabaseRecordProgress + from ravendb import DeleteAiAgentOperation + from ravendb import DeleteDatabaseOperation + from ravendb import DeleteDatabaseResult + from ravendb import DocumentChangeType + from ravendb import EmbeddingPathConfiguration + from ravendb import EmbeddingsGenerationConfiguration + from ravendb import EmbeddingsTransformation + from ravendb import EncryptionMode + from ravendb import EntraId + from ravendb import ExportCompressionAlgorithm + from ravendb import FieldIndexing + from ravendb import FieldStorage + from ravendb import FieldTermVector + from ravendb import GenerateEntityIdOnTheClient + from ravendb import GetAiAgentOperation + from ravendb import GetAiAgentsResponse + from ravendb import GetCdcSinkSchemaOperation + from ravendb import GetConnectionStringsResult + from ravendb import GetConversationMessagesOperation + from ravendb import GetConversationMessagesOptions + from ravendb import GetRemoteAttachmentsConfigurationOperation + from ravendb import GetSchemaValidationConfiguration + from ravendb import GetServerWideConnectionStringsOperation + from ravendb import GetServerWideConnectionStringsResult + from ravendb import GroupByArrayBehavior + from ravendb import HiLoIdGenerator + from ravendb import HiLoResult + from ravendb import Highlightings + from ravendb import IOperation + from ravendb import IndexBatchOptions + from ravendb import IndexChangeTypes + from ravendb import IndexCompactionInProgressException + from ravendb import IndexDefinitionCompareDifferences + from ravendb import IndexErrors + from ravendb import IndexLockMode + from ravendb import IndexPriority + from ravendb import IndexRunningStatus + from ravendb import IndexState + from ravendb import IndexStatus + from ravendb import IndexType + from ravendb import IndexingError + from ravendb import InsufficientQuotaException + from ravendb import JavaScriptMap + from ravendb import JsonPatchCommandData + from ravendb import JsonPatchDocument + from ravendb import JsonPatchOperation + from ravendb import JsonPatchResult + from ravendb import KafkaConnectionSettings + from ravendb import LazyOperation + from ravendb import LicenseLimitException + from ravendb import LimitType + from ravendb import LogFilter + from ravendb import LogFilterAction + from ravendb import LogLevel + from ravendb import LogsConfiguration + from ravendb import MaintenanceOperation + from ravendb import MicrosoftLogsConfiguration + from ravendb import MissingAiAgentParameterException + from ravendb import ModifyOngoingTaskResult + from ravendb import MultiDatabaseHiLoGenerator + from ravendb import MultiTypeHiLoGenerator + from ravendb import NodeId + from ravendb import NullsOrdering + from ravendb import Observable + from ravendb import OlapConnectionString + from ravendb import OngoingTask + from ravendb import OngoingTaskCdcSink + from ravendb import OngoingTaskConnectionStatus + from ravendb import OngoingTaskEmbeddingsGeneration + from ravendb import OngoingTaskGenAi + from ravendb import OngoingTaskQueueSink + from ravendb import OngoingTaskState + from ravendb import Operation + from ravendb import OperationExceptionResult + from ravendb import OperationIdResult + from ravendb import OptimisticConcurrencyMode + from ravendb import Passwordless + from ravendb import PortInUseException + from ravendb import PutConnectionStringResult + from ravendb import PutResult + from ravendb import PutServerWideConnectionStringOperation + from ravendb import PutServerWideConnectionStringResult + from ravendb import QueryToolFailedException + from ravendb import QueueBrokerType + from ravendb import QueueConnectionString + from ravendb import QueueSinkConfiguration + from ravendb import QueueSinkProcessState + from ravendb import QueueSinkScript + from ravendb import RabbitMqConnectionSettings + from ravendb import RateLimitException + from ravendb import RavenConnectionString + from ravendb import RavenDocumentQuery + from ravendb import RavenException + from ravendb import RefusedToAnswerException + from ravendb import RemoteAttachmentsAzureSettings + from ravendb import RemoteAttachmentsConfiguration + from ravendb import RemoteAttachmentsDestinationConfiguration + from ravendb import RemoteAttachmentsS3Settings + from ravendb import RemoveConnectionStringResult + from ravendb import RemoveServerWideConnectionStringOperation + from ravendb import RemoveServerWideConnectionStringResult + from ravendb import ReplicationBatchOptions + from ravendb import ReplicationHubNotFoundException + from ravendb import ReplicationType + from ravendb import ResponseDisposeHandling + from ravendb import RevisionIncludeResult + from ravendb import RunConversationOperation + from ravendb import S3StorageClass + from ravendb import SchemaDefinition + from ravendb import SchemaValidationConfiguration + from ravendb import SchemaValidationException + from ravendb import SearchEngineType + from ravendb import ServerOperation + from ravendb import ServerWideConnectionString + from ravendb import ServerWideConnectionStringUsage + from ravendb import ServerWideOperation + from ravendb import SessionPatchBehavior + from ravendb import SmugglerOperation + from ravendb import SmugglerProgressBase + from ravendb import SmugglerResult + from ravendb import SnapshotSettings + from ravendb import SortOptions + from ravendb import SpatialFieldType + from ravendb import SpatialOptions + from ravendb import SpatialOptionsFactory + from ravendb import SpatialRelation + from ravendb import SpatialSearchStrategy + from ravendb import SpatialUnits + from ravendb import SqlConnectionString + from ravendb import SsoIdentifier + from ravendb import SsoProvider + from ravendb import StartSchemaValidationOperation + from ravendb import TestCdcSinkMappingOperation + from ravendb import TestCdcSinkMappingRequest + from ravendb import TestCdcSinkMappingResult + from ravendb import TestCdcSinkOperation + from ravendb import TestCdcSinkRowResult + from ravendb import TestCdcSinkRowSelector + from ravendb import TextPart + from ravendb import TimeSeriesChangeTypes + from ravendb import TooManyRequestsException + from ravendb import TooManyTokensException + from ravendb import TopologyChange + from ravendb import UnsuccessfulAiRequestException + from ravendb import UpdateCdcSinkOperation + from ravendb import UpdateCdcSinkOperationResult + from ravendb import UpdateEmbeddingsGenerationOperation + from ravendb import UpdateEtlOperationResult + from ravendb import UpdateGenAiOperation + from ravendb import UpdateQueueSinkOperation + from ravendb import UpdateQueueSinkOperationResult + from ravendb import ValidateSchemaProgress + from ravendb import ValidateSchemaResult + from ravendb import VoidMaintenanceOperation + from ravendb import VoidOperation + from ravendb import VoidServerOperation From db2921df1d30e1230fcbdb6fc34283f67dd0ca29 Mon Sep 17 00:00:00 2001 From: Gracjan Sadowicz Date: Wed, 16 Sep 2026 15:20:04 +0200 Subject: [PATCH 14/14] RDBC-1112 Bump the packaged version to 7.2.6 CLIENT_VERSION moved to 7.2.6 with the rest of the sync, but setup.py still said 7.2.3.post2, so a release cut straight off this branch would have announced 7.2.6 on the wire while publishing a version PyPI already holds. The two always move together. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bb935aa5..7443e92f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ravendb", packages=find_packages(exclude=["*.tests.*", "tests", "*.tests", "tests.*"]), - version="7.2.3.post2", + version="7.2.6", long_description_content_type="text/markdown", long_description=open("README_pypi.md").read(), description="Python client for RavenDB NoSQL Database",