Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions scripts/promote_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,19 @@ class MissingFieldError(Exception):
def validate_discovery_item_config(item: Dict[str, Any]) -> Dict[str, Any]:
if "bucket" not in item:
raise MissingFieldError(
"Missing required field 'bucket' in discovery item: {item}"
f"Missing required field 'bucket' in discovery item: {item}"
)
if "discovery" not in item:
raise MissingFieldError(
"Missing required field 'discovery' in discovery item: {item}"
f"Missing required field 'discovery' in discovery item: {item}"
)
if "filename_regex" not in item:
raise MissingFieldError(
"Missing required field 'filename_regex' in discovery item: {item}"
f"Missing required field 'filename_regex' in discovery item: {item}"
)
if "prefix" not in item:
raise MissingFieldError(
"Missing required field 'prefix' in discovery item: {item}"
f"Missing required field 'prefix' in discovery item: {item}"
)
return item

Expand Down Expand Up @@ -114,13 +114,18 @@ def promote_to_production(payload):
if __name__ == "__main__":
try:
with open(sys.argv[1], "r") as file:
input = json.load(file)
_input = json.load(file)
stage = sys.argv[2]
discovery_items = input.get("discovery_items")
discovery_items = _input.get("discovery_items")
validated_discovery_items = [
validate_discovery_item_config(item) for item in discovery_items
]
dag_payload = {"conf": input}
transfer = _input.get("transfer")
if transfer is not None and not isinstance(transfer, bool):
raise ValueError(
f"Invalid value for 'transfer': {transfer!r}. Must be a boolean."

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ooh I didn't know about !r, thanks for introducing me to it!

)
dag_payload = {"conf": _input}
if stage == "production":
promote_to_production(dag_payload)
elif stage == "staging":
Expand Down
Loading