Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,713 changes: 870 additions & 843 deletions go/gen/sift/data_imports/v2/data_imports.pb.go

Large diffs are not rendered by default.

134 changes: 134 additions & 0 deletions go/gen/sift/data_imports/v2/data_imports_vtproto.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions protos/sift/data_imports/v2/data_imports.proto
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@ message CreateDataImportFromUrlRequest {
BatchConfig batch_config = 7 [(google.api.field_behavior) = OPTIONAL];
UlogConfig ulog_config = 8 [(google.api.field_behavior) = OPTIONAL];
McapConfig mcap_config = 9 [(google.api.field_behavior) = OPTIONAL];
// If true, the import does not mark its run stopped, so run-stop automations
// do not fire. Set this when more data is coming to the run, then call
// StopRun after the last import. Ignored when the import has no run.
bool leave_run_open = 10 [(google.api.field_behavior) = OPTIONAL];
}

message CreateDataImportFromUrlResponse {
Expand All @@ -111,6 +115,10 @@ message CreateDataImportFromUploadRequest {
BatchConfig batch_config = 7 [(google.api.field_behavior) = OPTIONAL];
UlogConfig ulog_config = 8 [(google.api.field_behavior) = OPTIONAL];
McapConfig mcap_config = 9 [(google.api.field_behavior) = OPTIONAL];
// If true, the import does not mark its run stopped, so run-stop automations
// do not fire. Set this when more data is coming to the run, then call
// StopRun after the last import. Ignored when the import has no run.
bool leave_run_open = 10 [(google.api.field_behavior) = OPTIONAL];
}

message CreateDataImportFromUploadResponse {
Expand Down
1 change: 1 addition & 0 deletions python/changelog.d/+leave-run-open.feat.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`client.data_import.import_from_path` and `Run.import_data` take `leave_run_open` (default false). A finished import stops its run and fires run-stop automations unless you set it, then call `run.stop()` after the last import.
196 changes: 100 additions & 96 deletions python/lib/sift/data_imports/v2/data_imports_pb2.py

Large diffs are not rendered by default.

18 changes: 16 additions & 2 deletions python/lib/sift/data_imports/v2/data_imports_pb2.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,16 @@ class CreateDataImportFromUrlRequest(google.protobuf.message.Message):
BATCH_CONFIG_FIELD_NUMBER: builtins.int
ULOG_CONFIG_FIELD_NUMBER: builtins.int
MCAP_CONFIG_FIELD_NUMBER: builtins.int
LEAVE_RUN_OPEN_FIELD_NUMBER: builtins.int
url: builtins.str
"""The url to import. HTTP and S3 urls are supported.
If you need to import non-public S3 objects, please contact Sift to set that up.
"""
leave_run_open: builtins.bool
"""If true, the import does not mark its run stopped, so run-stop automations
do not fire. Set this when more data is coming to the run, then call
StopRun after the last import. Ignored when the import has no run.
"""
@property
def csv_config(self) -> global___CsvConfig: ...
@property
Expand Down Expand Up @@ -302,9 +308,10 @@ class CreateDataImportFromUrlRequest(google.protobuf.message.Message):
batch_config: global___BatchConfig | None = ...,
ulog_config: global___UlogConfig | None = ...,
mcap_config: global___McapConfig | None = ...,
leave_run_open: builtins.bool = ...,
) -> None: ...
def HasField(self, field_name: typing.Literal["batch_config", b"batch_config", "ch10_config", b"ch10_config", "csv_config", b"csv_config", "hdf5_config", b"hdf5_config", "mcap_config", b"mcap_config", "parquet_config", b"parquet_config", "tdms_config", b"tdms_config", "ulog_config", b"ulog_config"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["batch_config", b"batch_config", "ch10_config", b"ch10_config", "csv_config", b"csv_config", "hdf5_config", b"hdf5_config", "mcap_config", b"mcap_config", "parquet_config", b"parquet_config", "tdms_config", b"tdms_config", "ulog_config", b"ulog_config", "url", b"url"]) -> None: ...
def ClearField(self, field_name: typing.Literal["batch_config", b"batch_config", "ch10_config", b"ch10_config", "csv_config", b"csv_config", "hdf5_config", b"hdf5_config", "leave_run_open", b"leave_run_open", "mcap_config", b"mcap_config", "parquet_config", b"parquet_config", "tdms_config", b"tdms_config", "ulog_config", b"ulog_config", "url", b"url"]) -> None: ...

global___CreateDataImportFromUrlRequest = CreateDataImportFromUrlRequest

Expand Down Expand Up @@ -367,6 +374,12 @@ class CreateDataImportFromUploadRequest(google.protobuf.message.Message):
BATCH_CONFIG_FIELD_NUMBER: builtins.int
ULOG_CONFIG_FIELD_NUMBER: builtins.int
MCAP_CONFIG_FIELD_NUMBER: builtins.int
LEAVE_RUN_OPEN_FIELD_NUMBER: builtins.int
leave_run_open: builtins.bool
"""If true, the import does not mark its run stopped, so run-stop automations
do not fire. Set this when more data is coming to the run, then call
StopRun after the last import. Ignored when the import has no run.
"""
@property
def csv_config(self) -> global___CsvConfig: ...
@property
Expand Down Expand Up @@ -394,9 +407,10 @@ class CreateDataImportFromUploadRequest(google.protobuf.message.Message):
batch_config: global___BatchConfig | None = ...,
ulog_config: global___UlogConfig | None = ...,
mcap_config: global___McapConfig | None = ...,
leave_run_open: builtins.bool = ...,
) -> None: ...
def HasField(self, field_name: typing.Literal["batch_config", b"batch_config", "ch10_config", b"ch10_config", "csv_config", b"csv_config", "hdf5_config", b"hdf5_config", "mcap_config", b"mcap_config", "parquet_config", b"parquet_config", "tdms_config", b"tdms_config", "ulog_config", b"ulog_config"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["batch_config", b"batch_config", "ch10_config", b"ch10_config", "csv_config", b"csv_config", "hdf5_config", b"hdf5_config", "mcap_config", b"mcap_config", "parquet_config", b"parquet_config", "tdms_config", b"tdms_config", "ulog_config", b"ulog_config"]) -> None: ...
def ClearField(self, field_name: typing.Literal["batch_config", b"batch_config", "ch10_config", b"ch10_config", "csv_config", b"csv_config", "hdf5_config", b"hdf5_config", "leave_run_open", b"leave_run_open", "mcap_config", b"mcap_config", "parquet_config", b"parquet_config", "tdms_config", b"tdms_config", "ulog_config", b"ulog_config"]) -> None: ...

global___CreateDataImportFromUploadRequest = CreateDataImportFromUploadRequest

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,24 @@ class DataImportsLowLevelClient(LowLevelClientBase, WithGrpcClient):
def __init__(self, grpc_client: GrpcClient):
WithGrpcClient.__init__(self, grpc_client=grpc_client)

async def create_from_upload(self, config: ImportConfig) -> tuple[str, str]:
async def create_from_upload(
self, config: ImportConfig, *, leave_run_open: bool = False
) -> tuple[str, str]:
"""Create a data import and get back a presigned upload URL.

Args:
config: The import configuration.
leave_run_open: If true, the import does not mark its run stopped.
Run-stop automations do not fire. Set this when more data is
coming to the run, then stop the run after the last import.
Ignored when the import has no run.

Returns:
A tuple of (data_import_id, upload_url).
"""
request = CreateDataImportFromUploadRequest()
_set_config_on_request(request, config)
request.leave_run_open = leave_run_open
response = await self._grpc_client.get_stub(
DataImportServiceStub
).CreateDataImportFromUpload(request)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,18 @@

from __future__ import annotations

import pytest
from sift.data_imports.v2.data_imports_pb2 import CreateDataImportFromUploadRequest
from unittest.mock import MagicMock

from sift_client._internal.low_level_wrappers.data_imports import _set_config_on_request
import pytest
from sift.data_imports.v2.data_imports_pb2 import (
CreateDataImportFromUploadRequest,
CreateDataImportFromUploadResponse,
)

from sift_client._internal.low_level_wrappers.data_imports import (
DataImportsLowLevelClient,
_set_config_on_request,
)
from sift_client.sift_types.channel import ChannelDataType
from sift_client.sift_types.data_import import McapDataColumn, McapImportConfig

Expand All @@ -32,3 +40,22 @@ def test_unknown_config_type_raises(self):
request = CreateDataImportFromUploadRequest()
with pytest.raises(TypeError, match="Unsupported import config type"):
_set_config_on_request(request, object()) # type: ignore[arg-type]


@pytest.mark.asyncio
async def test_create_from_upload_sets_leave_run_open():
"""leave_run_open is copied onto the upload request."""
captured: dict[str, CreateDataImportFromUploadRequest] = {}

class _Stub:
async def CreateDataImportFromUpload(self, request): # noqa: N802
captured["request"] = request
return CreateDataImportFromUploadResponse(data_import_id="id", upload_url="url")

client = DataImportsLowLevelClient(grpc_client=MagicMock())
client._grpc_client.get_stub = MagicMock(return_value=_Stub())
config = McapImportConfig(asset_name="a", data=[])

await client.create_from_upload(config, leave_run_open=True)

assert captured["request"].leave_run_open is True
9 changes: 8 additions & 1 deletion python/lib/sift_client/_tests/resources/test_data_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,14 @@ async def _import(self, tmp_path, filename, detected):
job = MagicMock()
api.client.async_.jobs.get = AsyncMock(return_value=job)

assert await api.import_from_path(path, asset="my_asset", show_progress=False) is job
assert (
await api.import_from_path(
path, asset="my_asset", show_progress=False, leave_run_open=True
)
is job
)
api._low_level_client.create_from_upload.assert_awaited()
assert api._low_level_client.create_from_upload.await_args.kwargs["leave_run_open"] is True

@pytest.mark.asyncio
async def test_mcap_data_cleared(self, tmp_path, monkeypatch):
Expand Down
Loading
Loading