diff --git a/.gitignore b/.gitignore index 51a41e7..589bdf7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ readme.txt output/ __pycache__ .env +bounding-boxes/outdir/ \ No newline at end of file diff --git a/bounding-boxes/readme.md b/bounding-boxes/readme.md new file mode 100644 index 0000000..39eae15 --- /dev/null +++ b/bounding-boxes/readme.md @@ -0,0 +1,155 @@ +# Bounding Boxes Formats: COCO + +This folder contains Python scripts for converting annotation results from the COCO format to Datasaur Schema format and vice versa. + +- [Formats](#formats) + - [COCO](#coco) + - [Datasaur Schema](#datasaur-schema) +- [Prerequisite](#prerequisite) +- [Usage](#usage) + - [`coco_to_datasaur_schemas`](#coco_to_datasaur_schemas) + - [`datasaur_schemas_to_coco`](#datasaur_schemas_to_coco) + + +## Formats + +### COCO + +[COCO (Common Objects in Context)](https://cocodataset.org/#home) has formats for several annotation tasks. +The script here will only support cases where the `segmentation` mask are in a quadrilateral shape. +In other words, it will only work for cases where there are 8 values per list in the `segmentation` key. + +```json +{ + "annotations": [ + { + "segmentation": [["x1", "y1", "x2", "y2", "x3", "y3", "x4", "y4", "x5", "y5", "x6", "y6", "x7", "y7", "x8", "y8"]] + } + ] +} +``` + +### Datasaur Schema + +[Datasaur Schema](https://docs.datasaur.ai/compatibility-and-updates/supported-formats#datasaur-schema-format) is a customized format Datasaur uses for both creating and exporting a Datasaur project. + + +## Prerequisite + +The script was developed and tested using Python 3.10.13 + +## Usage + +### `coco_to_datasaur_schemas` + +This function transforms a dictionary representation of a COCO file into a list of `DatasaurSchema` dicts. + +Parameters: +- `coco_json`: A dictionary representation of a COCO file. +- `custom_labelset`: Optional. A dictionary representation of a Datasaur bounding box label set to provide additional information. Useful to set default value, custom color, or setting a specific attribute as `DROPDOWN` type. See [here](./samples/custom-label-set.json) for an example JSON file. + > [!IMPORTANT] + > You can obtain a custom labelset JSON by copy-pasting from Datasaur's BBox Label Set editor. Please keep in mind that there are some attributes that won't be parsed, such as label's `id` and question's `internalId`, as they will be generated upon creation. + > Other than that, the script will also ignore the following TEXT config: `minLength`, `maxLength` and `pattern` + +Running `python src/coco_to_datasaur_schemas.py` should run the function against a sample `COCO.json` file. The function expect a `dict` representation of a COCO file and will return an array of `DatasaurSchema` also in a `dict` format. + +Note: The function will ignore the following fields as they are not used in Datasaur Schema: +- `licenses` +- `info` +- `annotation.area`, `annotation.bbox` + +Example usage: +- as a function + ```python + coco_json = json.load(file) + datasaur_schemas = coco_to_datasaur_schemas(coco_json) + ``` +- as a script + ``` + $ usage: coco_to_datasaur_schemas [-h] [--custom-labelset CUSTOM_LABELSET] [--outdir OUTDIR] [--log-level LOG_LEVEL] [--ignored-attributes IGNORED_ATTRIBUTES] coco_filepath + + positional arguments: + coco_filepath Path to COCO JSON file + + options: + -h, --help show this help message and exit + --custom-labelset CUSTOM_LABELSET + Path to custom labelset JSON file (useful for specifying DROPDOWN attributes) + --outdir OUTDIR Output directory for Datasaur schemas + --log-level LOG_LEVEL + --ignored-attributes IGNORED_ATTRIBUTES + Comma separated strings of attribute keys to ignore. Default: occluded + + $ python src/coco_to_datasaur_schemas.py samples/COCO.json + ``` + +### `datasaur_schemas_to_coco` + +This function transforms a list of Datasaur Schema object, represented as a list of dictionaries, into a single COCO object. + +Parameters: +- `schema_objects` (list[Any]): A list of dictionaries representing the result of `json.load`-ing the Datasaur Schema JSON. +- `licenses` (list[COCOLicense] | None, optional): A list of COCOLicense objects. Defaults to None. +- `info` (COCOInfo | None, optional): A COCOInfo object. Defaults to None. + +Running `python src/datasaur_schemas_to_coco.py` will read the sample zipfile `samples/bbox-export.zip` and transform the JSON file under the `REVIEW` directory into a single `COCO` object, which will be written to `outdir/out-coco.json`. + +As `licenses` and `info` parameters are optional, when not provided it will use the following values: + +```json +{ + "licenses": [ + { + "name": "dummy-datasaur-license", + "id": 0, + "URL": "" + } + ], + "info": { + "description": "Exported from Datasaur", + "url": "https://datasaur.ai", + "version": "v0.1", + "year": 2024, + "contributor": "Datasaur", + "date_created": "2024-04-23" + } +} +``` + + +Example usage: +- as a function + ```python + schemas = [...] + coco_obj = datasaur_schemas_to_coco(schemas) + + # or provide your own `licenses` and `info` + coco_obj = datasaur_schemas_to_coco( + schemas, + licenses=[{"name": "dummy-license", "id": 0, "url": ""}], + info={ + "contributor": "contributor", + "date_created": "2024-01-01", + "description": "dataset-description", + "url": "http://example.com", + "version": "v0.1", + "year": 2024, + }, + ) + ``` +- as a script + ``` + $ python src/datasaur_schemas_to_coco.py -h + usage: datasaur_schemas_to_coco [-h] [--outfile OUTFILE] [--license-and-info-json LICENSE_AND_INFO_JSON] zip_filepath + + positional arguments: + zip_filepath Path to Datasaur export ZIP file + + options: + -h, --help show this help message and exit + --outfile OUTFILE Output directory for COCO JSON file + --license-and-info-json LICENSE_AND_INFO_JSON + Path to JSON file containing licenses and info data + + $ python src/datasaur_schemas_to_coco.py samples/bbox-export.zip + ``` diff --git a/bounding-boxes/samples/COCO.json b/bounding-boxes/samples/COCO.json new file mode 100644 index 0000000..b0f0e18 --- /dev/null +++ b/bounding-boxes/samples/COCO.json @@ -0,0 +1,70 @@ +{ + "licenses": [ + { + "name": "DATASAUR", + "id": 0, + "URL": "datasaur.ai" + } + ], + "info": { + "contributor": "datasaur.ai", + "date_created": "2024-04-23", + "description": "Description of the dataset", + "url": "", + "version": "v0.1", + "year": 2024 + }, + "categories": [ + { + "id": 1, + "name": "sample_category", + "supercategory": "" + }, + { + "id": 2, + "name": "merchant_name", + "supercategory": "" + } + ], + "images": [ + { + "id": 1, + "width": 239.0, + "height": 991.0, + "file_name": "sample.png", + "license": 0, + "flickr_url": "", + "coco_url": "", + "date_captured": 0 + } + ], + "annotations": [ + { + "id": 1, + "image_id": 1, + "category_id": 1, + "segmentation": [[73, 153, 113, 153, 113, 169, 74, 169]], + "bbox": [73, 153, 40, 16], + "area": 7938, + "iscrowd": 0, + "attributes": { + "text": "sample text" + } + }, + { + "id": 2, + "image_id": 1, + "category_id": 2, + "segmentation": [[78, 158, 118, 158, 118, 174, 79, 174]], + "bbox": [78, 158, 40, 16], + "area": 7938, + "iscrowd": 0, + "attributes": { + "text": "Hokkaido-Ya", + "bio_tag": "B", + "order_idx": "1", + "occluded": false + } + } + ] +} diff --git a/bounding-boxes/samples/bbox-NDU4OWNmZTA.zip b/bounding-boxes/samples/bbox-NDU4OWNmZTA.zip new file mode 100644 index 0000000..c560784 Binary files /dev/null and b/bounding-boxes/samples/bbox-NDU4OWNmZTA.zip differ diff --git a/bounding-boxes/samples/bbox-export.zip b/bounding-boxes/samples/bbox-export.zip new file mode 100644 index 0000000..ae234b9 Binary files /dev/null and b/bounding-boxes/samples/bbox-export.zip differ diff --git a/bounding-boxes/samples/custom-label-set.json b/bounding-boxes/samples/custom-label-set.json new file mode 100644 index 0000000..47e9428 --- /dev/null +++ b/bounding-boxes/samples/custom-label-set.json @@ -0,0 +1,44 @@ +{ + "classes": [ + { + "name": "merchant_name", + "color": "#219896", + "questions": [ + { + "id": 0, + "internalId": 123, + "label": "bio_tag", + "required": false, + "type": "DROPDOWN", + "config": { + "options": [ + { + "id": "B", + "label": "B" + }, + { + "id": "I", + "label": "I" + } + ], + "multiple": false, + "defaultValue": "B" + } + }, + { + "id": 0, + "internalId": 123, + "label": "label-2", + "type": "TEXT", + "required": false, + "config": { + "multiple": false + } + } + ] + }, + { + "name": "plain_label" + } + ] +} \ No newline at end of file diff --git a/bounding-boxes/samples/license-and-info.json b/bounding-boxes/samples/license-and-info.json new file mode 100644 index 0000000..7279c85 --- /dev/null +++ b/bounding-boxes/samples/license-and-info.json @@ -0,0 +1,17 @@ +{ + "licenses": [ + { + "name": "datasaur", + "id": 0, + "url": "datasaur.ai" + } + ], + "info": { + "contributor": "datasaur.ai", + "date_created": "2024-04-23", + "description": "Description of the dataset", + "url": "", + "version": "v0.1", + "year": 2024 + } +} diff --git a/bounding-boxes/src/__init__.py b/bounding-boxes/src/__init__.py new file mode 100644 index 0000000..6e02a98 --- /dev/null +++ b/bounding-boxes/src/__init__.py @@ -0,0 +1,2 @@ +from coco_to_datasaur_schemas import coco_to_datasaur_schemas +from datasaur_schemas_to_coco import datasaur_schemas_to_coco diff --git a/bounding-boxes/src/coco_to_datasaur_schemas.py b/bounding-boxes/src/coco_to_datasaur_schemas.py new file mode 100644 index 0000000..db50604 --- /dev/null +++ b/bounding-boxes/src/coco_to_datasaur_schemas.py @@ -0,0 +1,300 @@ +import json +import logging +import os +from argparse import ArgumentParser +from dataclasses import asdict +from math import floor +from typing import Any, List, Set + +from common.defaults import defaults +from common.logger import log as _log +from common.random_color import random_color +from common.scrub import scrub +from formats.coco import validate_annotation, validate_segmentation +from formats.datasaur_schema import ( + DatasaurSchema, + DSBBoxLabel, + DSBBoxLabelClass, + DSBBoxLabelClassQuestions, + DSBboxLabelSet, + DSBBoxProjectData, + DSPage, + DSPoint, + DSShape, + GenericIdAndName, + QuestionConfig, +) +from formats.bbox_labelset import validate_bbox_labelset + + +def log(message, level=logging.DEBUG, **kwargs): + logger = logging.getLogger( + __name__ if __name__ != "__main__" else "coco_to_datasaur_schemas" + ) + return _log(message=message, logger=logger, level=level, **kwargs) + + +def coco_to_datasaur_schemas( + coco_json: Any, custom_labelset: Any | None, ignored_attributes: list[str] | None +) -> List[dict]: + """ + Raises: + Exception: If the segmentation of an annotation does not have 8 elements. + """ + + # validate segmentation first + for annot in coco_json["annotations"]: + validate_annotation(annot) + + log(message="creating BBoxLabelSet from COCO categories", level=logging.DEBUG) + bbox_label_set = DSBboxLabelSet( + id=None, + name="BBox Label Set", + classes=bbox_label_classes_from_coco(coco_json["categories"], custom_labelset), + ) + images = coco_json["images"] + + retval: List[dict] = [] + for image in images: + annotations_by_images = [ + annot + for annot in coco_json["annotations"] + if annot["image_id"] == image["id"] + ] + + bbox_labels = [ + bbox_label_from_coco_annotation( + annot, + labelset=bbox_label_set, + ignored_attributes=ignored_attributes, + ) + for annot in annotations_by_images + ] + + schema = DatasaurSchema( + version="1", + data=DSBBoxProjectData( + kinds=["BBOX_BASED"], + bboxLabels=bbox_labels, + bboxLabelSets=[bbox_label_set], + document=GenericIdAndName(name=image["file_name"], id=None), + project=None, + pages=[ + DSPage( + pageIndex=0, + pageHeight=floor(image["height"]), + pageWidth=floor(image["width"]), + ) + ], + ), + ) + + retval.append(asdict(schema)) + + return retval + + +def main() -> None: + parser = ArgumentParser(prog="coco_to_datasaur_schemas") + parser.add_argument("coco_filepath", type=str, help="Path to COCO JSON file") + parser.add_argument( + "--custom-labelset", + type=str, + help="Path to custom labelset JSON file (useful for specifying DROPDOWN attributes)", + ) + parser.add_argument( + "--outdir", + type=str, + help="Output directory for Datasaur schemas", + default="./outdir/", + ) + parser.add_argument("--log-level", type=str, default="INFO") + parser.add_argument( + "--ignored-attributes", + type=str, + default="occluded", + help="Comma separated strings of attribute keys to ignore. Default: occluded", + ) + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format="%(message)s") + + coco_filepath = os.path.abspath(args.coco_filepath) + log("reading COCO JSON file", filepath=coco_filepath) + with open(coco_filepath) as f: + json_data = json.load(f) + + custom_labelset = None + if args.custom_labelset: + custom_labelset_filepath = os.path.abspath(args.custom_labelset) + with open(custom_labelset_filepath) as f: + custom_labelset = json.load(f) + validate_bbox_labelset(custom_labelset) + + ignored_attributes = None + if args.ignored_attributes: + ignored_attributes = args.ignored_attributes.split(",") + + log("converting COCO to Datasaur Schema") + schemas = coco_to_datasaur_schemas( + json_data, + custom_labelset=custom_labelset, + ignored_attributes=ignored_attributes, + ) + + outdir = os.path.abspath(args.outdir) + os.makedirs(outdir, exist_ok=True) + log("writing to file", directory=outdir) + for schema in schemas: + image_filepath = schema["data"]["document"]["name"] + image_filename = os.path.basename(image_filepath) + filename, _extension = os.path.splitext(image_filename) + answer_filename = filename + ".json" + + with open(os.path.join(outdir, answer_filename), "w") as f: + json.dump(scrub(schema), f, indent=2) + + +def bbox_label_classes_from_coco( + coco_categories: List[dict], + custom_labelset: Any | None, +) -> List[DSBBoxLabelClass]: + custom_classes = custom_labelset["classes"] if custom_labelset else [] + + retval = [] + for category in coco_categories: + # check and use questions from custom class + custom_class = next( + (item for item in custom_classes if item["name"] == category["name"]), None + ) + + questions = [ + DSBBoxLabelClassQuestions( + id=index, + label=q.get("label", f"Question {index}"), + config=QuestionConfig( + multiline=q.get("config", {}).get("multiline", None), + multiple=q.get("config", {}).get("multiple", None), + options=q.get("config", {}).get("options", None), + defaultValue=q.get("config", {}).get("defaultValue", None), + ), + required=q.get("required", False), + type=q.get("type", "TEXT"), + ) + for index, q in enumerate(defaults(custom_class, "questions", [])) + ] + + retval.append( + DSBBoxLabelClass( + id=str(category["id"]), + name=category["name"], + captionAllowed=defaults(custom_class, "captionAllowed", True), + captionRequired=defaults(custom_class, "captionRequired", False), + color=defaults(custom_class, "color", random_color(category["name"])), + questions=questions, + ) + ) + + return retval + + +def shape_from_coco_segmentation(segmentation: List[float]) -> DSShape: + points: List[DSPoint] = [] + + for i in range(0, 8, 2): + points.append(DSPoint(x=segmentation[i], y=segmentation[i + 1])) + + return DSShape(pageIndex=0, points=points) + + +def shape_from_coco_annotation(annotation: dict) -> list[DSShape]: + segmentations_valid = all( + validate_segmentation(segment) for segment in annotation["segmentation"] + ) and (len(annotation["segmentation"]) > 0) + + if segmentations_valid: + return [ + shape_from_coco_segmentation(segment) + for segment in annotation["segmentation"] + ] + + log("found some invalid segmentations, using bbox instead", annotation=annotation) + x, y, width, height = annotation["bbox"] + return [ + DSShape( + pageIndex=0, + points=[ + DSPoint(x=x, y=y), + DSPoint(x=x + width, y=y), + DSPoint(x=x + width, y=y + height), + DSPoint(x=x, y=y + height), + ], + ) + ] + + +def bbox_label_from_coco_annotation( + annotation: dict, labelset: DSBboxLabelSet, ignored_attributes: list[str] | None +) -> DSBBoxLabel: + if ignored_attributes is None: + ignored_attributes = [] + + attributes = annotation["attributes"] + + stringified_id = str(annotation["category_id"]) + bbox_label_class = next( + item for item in labelset.classes if item.id == stringified_id + ) + + bbox_shapes = shape_from_coco_annotation(annotation) + + # check if attributes have any other key + # if found, add to labelset.classes + answers = None + set_ignored_attributes = set([*ignored_attributes, "text"]) + if attributes.keys() - set_ignored_attributes: + answers = {} + keys: Set = attributes.keys() - set_ignored_attributes + + for key in keys: + questions = bbox_label_class.questions + if questions is None: + questions = [] + + if key not in [q.label for q in questions]: + questions.append( + DSBBoxLabelClassQuestions( + id=len(questions), + label=key, + required=False, + type="TEXT", + config=QuestionConfig( + multiline=False, + multiple=False, + options=None, + defaultValue=None, + ), + ) + ) + + question_id = next(q.id for q in questions if q.label == key) + answers[str(question_id)] = str(attributes[key]) + + bbox_label_class.questions = questions + + return DSBBoxLabel( + id=str(annotation["id"]), + caption=str(attributes.get("text", "")), + bboxLabelClassId=bbox_label_class.id, + bboxLabelClassName=bbox_label_class.name, + shapes=bbox_shapes, + labeledBy=None, + acceptedByUserId=None, + labeledByUserId=None, + rejectedByUserId=None, + status=None, + answers=answers, + ) + + +if __name__ == "__main__": + main() diff --git a/bounding-boxes/src/common/defaults.py b/bounding-boxes/src/common/defaults.py new file mode 100644 index 0000000..eacf29f --- /dev/null +++ b/bounding-boxes/src/common/defaults.py @@ -0,0 +1,2 @@ +def defaults(obj: dict | None, key: str, default): + return obj[key] if obj and key in obj else default diff --git a/bounding-boxes/src/common/logger.py b/bounding-boxes/src/common/logger.py new file mode 100644 index 0000000..183bf09 --- /dev/null +++ b/bounding-boxes/src/common/logger.py @@ -0,0 +1,23 @@ +# https://docs.python.org/3/howto/logging-cookbook.html#implementing-structured-logging +import json +import logging + + +class StructuredMessage: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def __str__(self) -> str: + return json.dumps(self.kwargs) + + +def log(message, logger: logging.Logger, level=logging.INFO, **kwargs): + logger.log( + level=level, + msg=StructuredMessage( + level=logging._levelToName[level], + message=message, + **kwargs, + logger=logger.name, + ), + ) diff --git a/bounding-boxes/src/common/random_color.py b/bounding-boxes/src/common/random_color.py new file mode 100644 index 0000000..77aa557 --- /dev/null +++ b/bounding-boxes/src/common/random_color.py @@ -0,0 +1,7 @@ +import random + + +def random_color(seed: str) -> str: + random.seed(seed) + r = lambda: random.randint(0, 255) + return "#%02X%02X%02X" % (r(), r(), r()) diff --git a/bounding-boxes/src/common/scrub.py b/bounding-boxes/src/common/scrub.py new file mode 100644 index 0000000..540a012 --- /dev/null +++ b/bounding-boxes/src/common/scrub.py @@ -0,0 +1,24 @@ +import copy + + +def scrub(x): + """ + Adapted from https://stackoverflow.com/a/11410935 + """ + ret = copy.deepcopy(x) + if isinstance(x, dict): + for key, value in x.items(): + if isinstance(value, dict): + ret[key] = scrub(value) + continue + + if value is None: + del ret[key] + continue + + ret[key] = scrub(value) + elif isinstance(x, list): + for index, item in enumerate(x): + ret[index] = scrub(item) + + return ret diff --git a/bounding-boxes/src/datasaur_schemas_to_coco.py b/bounding-boxes/src/datasaur_schemas_to_coco.py new file mode 100644 index 0000000..397ed03 --- /dev/null +++ b/bounding-boxes/src/datasaur_schemas_to_coco.py @@ -0,0 +1,288 @@ +import json +import os +from argparse import ArgumentParser +from dataclasses import asdict +from shutil import rmtree +from typing import Any, Dict +from zipfile import Path as ZipPath +from zipfile import ZipFile +import logging +import tempfile + + +from common.logger import log as _log +from formats.coco import ( + COCO, + COCOAnnotation, + COCOCategory, + COCOImage, + COCOLicense, + COCOInfo, +) + + +def log(message, level=logging.DEBUG, **kwargs): + logger = logging.getLogger( + __name__ if __name__ != "__main__" else "datasaur_schemas_to_coco" + ) + return _log(message=message, logger=logger, level=level, **kwargs) + + +def datasaur_schemas_to_coco( + schema_objects: list[Any], + licenses: list[dict] | None = None, + info: dict | None = None, +) -> dict: + """ + Convert Datasaur schema objects to COCO format. + + Args: + schema_objects (list[Any]): A list of dictionaries representing the result of `json.load`-ing the Datasaur Schema JSON. + licenses (list[COCOLicense] | None, optional): A list of COCOLicense objects. Defaults to None. + info (COCOInfo | None, optional): A COCOInfo object. Defaults to None. + + Returns: + dict: A dictionary representing the COCO format. + + """ + if licenses is None: + licenses = [{"name": "dummy-datasaur-license", "id": 0, "url": ""}] + + if info is None: + info = { + "contributor": "Datasaur", + "date_created": "2024-04-23", + "description": "Exported from Datasaur", + "url": "https://datasaur.ai", + "version": "v0.1", + "year": 2024, + } + + coco_info = COCOInfo(**info) + coco_licenses = [COCOLicense(**l) for l in licenses] + + schemas = [s for s in schema_objects] + + # assuming all DatasaurSchema are from the same project, + # there will be the same bboxLabelSet + categories: list[COCOCategory] = coco_categories_from_datasaur_schema(schemas[0]) + + # images from datasaur schemas -- name, dimension info + images: list[COCOImage] = [ + coco_images_from_datasaur_schema(id, schema) + for id, schema in enumerate(schemas, start=1) + ] + + # annotations from bboxLabels + annotations: list[COCOAnnotation] = coco_annots_from_datasaur_schemas( + schemas, categories + ) + + return asdict( + COCO( + info=coco_info, + licenses=coco_licenses, + categories=categories, + images=images, + annotations=annotations, + ) + ) + + +def main() -> None: + parser = ArgumentParser(prog="datasaur_schemas_to_coco") + parser.add_argument( + "zip_filepath", type=str, help="Path to Datasaur export ZIP file" + ) + parser.add_argument( + "--outfile", + type=str, + help="Output directory for COCO JSON file", + default="./outdir/coco.json", + ) + parser.add_argument( + "--license-and-info-json", + type=str, + help="Path to JSON file containing licenses and info data", + default="samples/license-and-info.json", + ) + parser.add_argument("--log-level", type=str, default="INFO") + args = parser.parse_args() + logging.basicConfig(level=args.log_level, format="%(message)s") + + export_zip = os.path.abspath(args.zip_filepath) + with tempfile.TemporaryDirectory() as temp_destination: + log("using temp directory", directory=temp_destination) + outfile = os.path.abspath(args.outfile) + outdir = os.path.dirname(outfile) + os.makedirs(outdir, exist_ok=True) + + extracted_files: list[str] = unzip_export_result( + export_zip=export_zip, dest=temp_destination + ) + + schemas = [load_datasaur_schema_file(f) for f in extracted_files] + + log("reading licenses and info file", filepath=args.license_and_info_json) + license_and_info = json.load(open(os.path.abspath(args.license_and_info_json))) + + log("converting datasaur schemas to COCO format", count=len(schemas)) + coco = datasaur_schemas_to_coco( + schemas, + licenses=license_and_info.get("licenses", None), + info=license_and_info.get("info", None), + ) + + log("writing COCO JSON file", outfile=outfile) + with open(outfile, "w") as wf: + json.dump(coco, wf, indent=2) + + log("cleaning up temp directory", directory=temp_destination) + + +def coco_annots_from_datasaur_schemas( + schemas: list[dict], categories: list[COCOCategory] +) -> list[COCOAnnotation]: + + name_to_id: Dict[str, int] = {x.name: x.id for x in categories} + annots: list[COCOAnnotation] = [] + for image_id, schema in enumerate(schemas, start=1): + if ( + schema["data"]["bboxLabelSets"] is None + or len(schema["data"]["bboxLabels"]) < 1 + ): + continue + + labelset = schema["data"]["bboxLabelSets"][0] + label_id_to_question_info = { + label["id"]: {str(q.get("id")): q for q in label.get("questions", {})} + for label in labelset["classes"] + } + + if schema["data"]["bboxLabels"] is None: + continue + + for bbox_label in schema["data"]["bboxLabels"]: + answers = bbox_label.get("answers", None) + attributes = {} + if answers: + questions = label_id_to_question_info.get( + bbox_label["bboxLabelClassId"], None + ) + + if questions: + for key, value in answers.items(): + question_label = questions[key]["label"] + attributes[question_label] = value + + annots.append( + COCOAnnotation( + id=len(annots) + 1, + image_id=image_id, + category_id=name_to_id[bbox_label["bboxLabelClassName"]], + segmentation=shapes_to_segmentation(bbox_label["shapes"]), + bbox=shapes_to_bbox(bbox_label["shapes"]), + attributes={"text": bbox_label.get("caption", None), **attributes}, + area=0, + iscrowd=0, + ) + ) + + return annots + + +def coco_categories_from_datasaur_schema(schema: dict) -> list[COCOCategory]: + if ( + schema["data"]["bboxLabelSets"] is None + or len(schema["data"]["bboxLabelSets"]) < 1 + ): + return [] + + retval: list[COCOCategory] = [] + + # Currently, there can only be 1 bboxLabelSets + bbox_label_set = schema["data"]["bboxLabelSets"][0] + for i, label_class in enumerate(bbox_label_set["classes"], 1): + retval.append(COCOCategory(id=i, name=label_class["name"], supercategory="")) + + return retval + + +def coco_images_from_datasaur_schema(id: int, schema: dict) -> COCOImage: + width, height = 0, 0 + try: + if schema["data"]["pages"] is not None and len(schema["data"]["pages"]) >= 1: + width = schema["data"]["pages"][0]["pageWidth"] + height = schema["data"]["pages"][0]["pageHeight"] + except KeyError: + # some older Bounding Box projects may not have pageWidth / pageHeight populated + pass + + return COCOImage( + id=id, + file_name=schema["data"]["document"]["name"], + width=width, + height=height, + license=0, + flickr_url="", + coco_url="", + ) + + +def shapes_to_bbox(shapes: list[dict]) -> list[float]: + x_coords = [point["x"] for shape in shapes for point in shape["points"]] + y_coords = [point["y"] for shape in shapes for point in shape["points"]] + + x_min = min(*x_coords) + x_max = max(*x_coords) + y_min, y_max = min(*y_coords), max(*y_coords) + + return [min(*x_coords), min(*y_coords), x_max - x_min, y_max - y_min] + + +def shapes_to_segmentation(shapes: list[dict]) -> list[list[float]]: + retval: list[list[float]] = [] + for shape in shapes: + segmentation: list[float] = [ + dot for point in shape["points"] for dot in [point["x"], point["y"]] + ] + retval.append(segmentation) + return retval + + +def unzip_export_result(export_zip: str, dest: str) -> list[str]: + retval: list[str] = [] + log("unzipping export result to temp directory", export_zip=export_zip) + with ZipFile(export_zip, "r") as zf: + project_dirs: list[str] = [] + + for zippath in ZipPath(zf).iterdir(): + if zippath.is_dir(): + project_dirs.append(zippath.name) + + if len(project_dirs) == 0: + log("no project dir found in export result", level=logging.ERROR) + raise Exception("no project dir found") + + prefixes = [f"{project_dir}/REVIEW" for project_dir in project_dirs] + for zip_content in zf.infolist(): + if not any(zip_content.filename.startswith(prefix) for prefix in prefixes): + continue + + if zip_content.is_dir(): + zf.extract(zip_content.filename, dest) + continue + retval.append(zf.extract(zip_content, dest)) + + return retval + + +def load_datasaur_schema_file(filepath: str): + with open(filepath) as f: + data = json.load(f) + + return data + + +if __name__ == "__main__": + main() diff --git a/bounding-boxes/src/formats/bbox_labelset.py b/bounding-boxes/src/formats/bbox_labelset.py new file mode 100644 index 0000000..5d8b00f --- /dev/null +++ b/bounding-boxes/src/formats/bbox_labelset.py @@ -0,0 +1,81 @@ +LABELSET_KEYS = {"classes", "name", "autoLabelProvider"} +CLASS_KEYS = {"id", "name", "captionAllowed", "captionRequired", "color", "questions"} +QUESTION_KEYS = {"id", "label", "required", "type", "config", "internalId"} +DROPDOWN_KEYS = {"defaultValue", "multiple", "options"} +UNUSED_TEXT_KEYS = {"minLength", "pattern", "maxLength"} +TEXT_KEYS = {"defaultValue", "multiline", "multiple", *UNUSED_TEXT_KEYS} +OPTION_KEYS = {"id", "label", "parentId"} + + +def validate_bbox_labelset(bbox_labelset: dict): + assertKeys(bbox_labelset, LABELSET_KEYS, "labelset object") + if "classes" not in bbox_labelset: + raise AssertionError("expected classes in labelset") + + for index, labelclass in enumerate(bbox_labelset["classes"]): + assertKeys(labelclass, CLASS_KEYS, f"labelclass: {index}") + if "name" not in labelclass: + raise AssertionError("expected name in labelclass") + + for question in labelclass.get("questions", []): + assertExact( + question, + QUESTION_KEYS, + f"question of label: {labelclass['name']}", + ) + + if question["type"] == "DROPDOWN": + assertDropdownQuestion(question) + + if question["type"] == "TEXT": + assertKeys( + question["config"], + TEXT_KEYS, + f"config of question: {question['label']}", + ) + + +def assertDropdownQuestion(question: dict): + assertKeys( + question["config"], + DROPDOWN_KEYS, + f"config of question {question['label']}", + ) + + if not isinstance(question["config"]["options"], list): + raise AssertionError( + f"expects options as a list, received {type(question['config']['options'])}" + ) + + for opt in question["config"]["options"]: + assertKeys( + opt, + OPTION_KEYS, + f"options of question {question['label']}", + ) + + +def assertKeys(dict: dict, valid_keys: set, identifier: str | None = None): + """ + Ensure that dict keys are subset of valid_keys + """ + keys = set(dict.keys()) + if not keys.issubset(valid_keys): + raise AssertionError( + f"Invalid keys detected in {identifier}: {keys.difference(valid_keys)}" + ) + + +def assertExact(dict: dict, valid_keys: set, identifier: str | None = None): + """ + Ensure that dict keys are exactly equal to valid_keys + """ + keys = set(dict.keys()) + if not keys == valid_keys: + if keys.issubset(valid_keys): + raise AssertionError( + f"Missing keys detected in {identifier}: {valid_keys.difference(keys)}" + ) + raise AssertionError( + f"Excess keys detected in {identifier}: {keys.difference(valid_keys)}" + ) diff --git a/bounding-boxes/src/formats/coco.py b/bounding-boxes/src/formats/coco.py new file mode 100644 index 0000000..0591e5c --- /dev/null +++ b/bounding-boxes/src/formats/coco.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass +from typing import Dict, List + +SUPPORTED_SEGMENTATION_LENGTH = 8 + + +@dataclass +class COCOLicense: + name: str + id: int + url: str + + +@dataclass +class COCOCategory: + id: int + name: str + supercategory: str + + +@dataclass +class COCOImage: + id: int + width: float + height: float + file_name: str + license: int + flickr_url: str + coco_url: str + + +@dataclass +class COCOAnnotation: + id: int + image_id: int + category_id: int + """ + Expected a list of 8 numbers, representing four vertices starting from top-left and going clockwise + """ + segmentation: List[List[float]] + + """ + [x, y, width, height] + """ + bbox: List[float] + area: float + """ + int, but mainly 0/1 + """ + iscrowd: int + attributes: Dict[str, str | int | bool] + + +@dataclass +class COCOInfo: + contributor: str + date_created: str + description: str + url: str + version: str + year: int | str + + +@dataclass +class COCOForInput: + categories: List[COCOCategory] + images: List[COCOImage] + annotations: List[COCOAnnotation] + + +@dataclass +class COCO(COCOForInput): + licenses: List[COCOLicense] + info: COCOInfo + + +def validate_annotation(annotation): + """ + Ensure that the segmentation of an annotation has 8 elements, + or failing that, ensure it has a valid bounding boxes + """ + bbox = annotation["bbox"] + + shoud_throw = len(bbox) != 4 + + if shoud_throw: + for segmentation in annotation["segmentation"]: + if not validate_segmentation(segmentation): + raise Exception( + "expect segmentation to be a list-of-list of 8 elements" + ) + + +def validate_segmentation(segmentation): + if len(segmentation) != SUPPORTED_SEGMENTATION_LENGTH: + return False + + return True diff --git a/bounding-boxes/src/formats/datasaur_schema.py b/bounding-boxes/src/formats/datasaur_schema.py new file mode 100644 index 0000000..b84deaa --- /dev/null +++ b/bounding-boxes/src/formats/datasaur_schema.py @@ -0,0 +1,98 @@ +from dataclasses import dataclass +from typing import List, Optional, Dict + + +@dataclass +class GenericIdAndName: + id: Optional[str] + name: str + + +@dataclass +class DSPoint: + x: float + y: float + + +@dataclass +class DSShape: + pageIndex: int + points: List[DSPoint] + + +@dataclass +class DSBBoxLabel: + id: str + bboxLabelClassId: str + bboxLabelClassName: str + caption: Optional[str] + shapes: List[DSShape] + """ + tba + """ + answers: Optional[Dict[str, str | int | bool]] + + # For Datasaur Schema used in input, we can ignore these + status: Optional[str] + labeledBy: Optional[str] + labeledByUserId: Optional[int] + acceptedByUserId: Optional[int] + rejectedByUserId: Optional[int] + + +@dataclass +class QuestionConfig: + multiline: Optional[bool] + multiple: Optional[bool] + options: Optional[List[dict[str, str]]] + defaultValue: Optional[str] + + +@dataclass +class DSBBoxLabelClassQuestions: + id: int + label: str + required: bool + type: str + config: QuestionConfig + pass + + +@dataclass +class DSBBoxLabelClass(GenericIdAndName): + id: str + color: Optional[str] + captionAllowed: bool + captionRequired: bool + """ + tba + """ + questions: Optional[List[DSBBoxLabelClassQuestions]] + + +@dataclass +class DSBboxLabelSet(GenericIdAndName): + classes: List[DSBBoxLabelClass] + + +@dataclass +class DSPage: + pageIndex: int + pageHeight: int + pageWidth: int + + +@dataclass +class DSBBoxProjectData: + kinds: List[str] + pages: Optional[List[DSPage]] + bboxLabelSets: Optional[List[DSBboxLabelSet]] + bboxLabels: Optional[List[DSBBoxLabel]] + document: GenericIdAndName + project: Optional[GenericIdAndName] + + +@dataclass +class DatasaurSchema: + data: DSBBoxProjectData + version: str diff --git a/create-project-async/api_client.py b/create-project-async/api_client.py index b541cc1..bfbebf5 100644 --- a/create-project-async/api_client.py +++ b/create-project-async/api_client.py @@ -1,15 +1,52 @@ -from os import environ - +import logging +import traceback import fire + +from os import environ +from src.batched_project import DEFAULT_BATCH_SIZE, BatchedProject +from src.helper import parse_multiple_config from src.job import Job +from src.logger import log as log from src.project import Project +DEFAULT_OPERATIONS_PATH = "create_project.json" +DEFAULT_DOCUMENTS_PATH = "documents" + + +def log_error(message, level=logging.ERROR, **kwargs): + logger = logging.getLogger("api_client") + return log(logger=logger, level=level, message=message, **kwargs) + + +def create_project( + base_url, + client_id, + client_secret, + team_id, + documents_path=DEFAULT_DOCUMENTS_PATH, + operations_path=DEFAULT_OPERATIONS_PATH, +): + try: + Project(base_url=base_url, id=client_id, secret=client_secret, documents_path=documents_path).create( + team_id=team_id, + operations_path=operations_path, + ) + except Exception as e: + raise SystemExit(e) + -def create_project(base_url, client_id, client_secret, team_id, documents_path="documents", operations_path="create_project.json"): +def create_batched_projects( + base_url, + client_id, + client_secret, + team_id, + documents_path=DEFAULT_DOCUMENTS_PATH, + operations_path=DEFAULT_OPERATIONS_PATH, + document_batch_size=DEFAULT_BATCH_SIZE, +): try: - Project(base_url=base_url, id=client_id, secret=client_secret).create( + BatchedProject(base_url=base_url, id=client_id, secret=client_secret, documents_path=documents_path, document_batch_size=document_batch_size).create( team_id=team_id, - documents_path=documents_path, operations_path=operations_path, ) except Exception as e: @@ -29,6 +66,31 @@ def get_job_status(base_url, client_id, client_secret, job_id): raise SystemExit(e) +def create_multiple_projects( + base_url, + client_id, + client_secret, + team_id, + operations_path=DEFAULT_OPERATIONS_PATH, + config="config.csv", +): + project_configs = parse_multiple_config(config) + for name, documents_path in project_configs: + try: + Project(base_url=base_url, id=client_id, secret=client_secret, documents_path=documents_path).create( + team_id=team_id, + operations_path=operations_path, + name=name, + ) + except Exception as e: + log_error( + message=f"Error creating project: {name}", + exception=traceback.format_exception(e), + ) + + if __name__ == "__main__": environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + logging.basicConfig(level=logging.INFO, format="%(message)s") + fire.Fire() diff --git a/create-project-async/config.csv b/create-project-async/config.csv new file mode 100644 index 0000000..c3f2457 --- /dev/null +++ b/create-project-async/config.csv @@ -0,0 +1,2 @@ +"project 1","./documents" +"project 2","./documents" diff --git a/create-project-async/readme.md b/create-project-async/readme.md index f88d50f..788ccd0 100644 --- a/create-project-async/readme.md +++ b/create-project-async/readme.md @@ -2,10 +2,12 @@ ## Pre-requisite -``` -# install dependencies -python -m pip install -r src/requirements.txt -``` +- Python 3.10 or higher +- Install dependencies + ``` + # install dependencies + python -m pip install -r src/requirements.txt + ``` ## Create Project (v2) @@ -17,7 +19,7 @@ In this new mutation, we no longer support uploading files directly to the Graph ### With Local Files -Local files are located under `document/` folder. +Local files are located under `documents/` folder. Every file inside the directory will be uploaded to Datasaur as part of the project creation process. **Note**: If you want to use pairing files, such as inputfile.jpg with inputfile.txt, ensure that they share the same filename. These paired files are commonly used for OCR / Audio projects with transcription. @@ -30,6 +32,40 @@ python api_client.py create_project \ --team_id TEAM_ID ``` +### Batched Project Creation + +A wrapper around the logic for `create_project`. + +Requires an additional `--document_batch_size` param. The value of this param determines the max amount of documents in each project. The batch size value must be between 1 and 100. + +``` +python api_client.py create_batched_projects \ + --base_url BASE_URL \ + --client_id CLIENT_ID \ + --client_secret CLIENT_SECRET \ + --team_id TEAM_ID \ + --document_batch_size 100 +``` + +## Create Multiple Projects + +A thin wrapper around the logic for `create_project`. +`CONFIG_FILE` should be a CSV file without header, where the first column contains project names, while the second column contains path to the folders. + +```csv +"project1","./documents" +"project2","./documents" +``` + +``` +python api_client.py create_multiple_projects \ + --base_url BASE_URL \ + --client_id CLIENT_ID \ + --client_secret CLIENT_SECRET \ + --team_id TEAM_ID + --config CONFIG_FILE +``` + ## Get Job Status ``` diff --git a/create-project-async/src/batched_project.py b/create-project-async/src/batched_project.py new file mode 100644 index 0000000..8dc36d4 --- /dev/null +++ b/create-project-async/src/batched_project.py @@ -0,0 +1,64 @@ +import json +from src.graphql_document_creator import GraphQLDocumentCreator +from src.graphql_utils import GraphQLUtils +from src.project import Project + +DEFAULT_BATCH_SIZE = 100 + + +class BatchedProject(Project): + def __init__(self, base_url: str, id: str, secret: str, documents_path: str, document_batch_size=DEFAULT_BATCH_SIZE): + if not 1 <= document_batch_size <= 100: + raise ValueError("document_batch_size must be between 1 and 100") + + super().__init__(base_url, id, secret, documents_path) + self.document_batch_size = document_batch_size + self.graphql_utils = GraphQLUtils(base_url=self.base_url, headers=self.headers, + client_id=self.client_id, client_secret=self.client_secret) + + def create(self, team_id: str, operations_path: str, name: str | None = None): + chunked_gql_documents = self.__get_chunked_gql_documents() + print(f"creating {len(chunked_gql_documents)} projects...") + for index, gql_documents in enumerate(chunked_gql_documents): + operations = self._get_operations( + team_id, operations_path, gql_documents, name) + + name_with_batch_number = self.__get_name_with_batch_number( + operations["variables"]["input"]["name"], index) + operations["variables"]["input"]["name"] = name_with_batch_number + + self.__create_project_from_chunk(operations) + + def __get_chunked_gql_documents(self): + gql_documents = GraphQLDocumentCreator( + proxy_url=self.proxy_url, headers=self.headers, documents_path=self.documents_path).create() + + return [gql_documents[i:i + self.document_batch_size] for i in range(0, len(gql_documents), self.document_batch_size)] + + def __create_project_from_chunk(self, operations): + self.__report_before_request(operations) + + graphql_response = self.graphql_utils.call_graphql( + data={ + "query": operations["query"], + "variables": json.dumps(operations["variables"]), + "operationName": operations.get( + "operationName", "Datasaur API client - createProject" + ), + } + ) + + self.graphql_utils.process_graphql_response(graphql_response) + + def __report_before_request(self, operations): + print("\n" + "=" * 50 + "\n") + print("Project creation request sent.") + print(f"Project name: {operations['variables']['input']['name']}") + print( + f"Number of documents: {len(operations['variables']['input']['documents'])}") + print( + f"Document names: {', '.join(doc['document']['name'] for doc in operations['variables']['input']['documents'])}") + print("") + + def __get_name_with_batch_number(self, name: str, index: int): + return f"{name} - batch {index + 1}" if name else None diff --git a/create-project-async/src/graphql_document_creator.py b/create-project-async/src/graphql_document_creator.py new file mode 100644 index 0000000..bedadd1 --- /dev/null +++ b/create-project-async/src/graphql_document_creator.py @@ -0,0 +1,103 @@ +import glob +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import TypedDict + +from requests import post + + +class GraphQLDocument(TypedDict): + document: dict + extras: list[dict] | None + + +class GraphQLDocumentCreator: + MAX_WORKERS = 8 + + def __init__(self, proxy_url: str, headers: dict[str, str], documents_path: str): + self.proxy_url = proxy_url + self.headers = headers + self.documents_path = documents_path + + def create(self): + mapped_documents = self.__get_mapped_documents() + return self.__get_graphql_documents(mapped_documents) + + def __get_mapped_documents(self): + filepaths = list(glob.iglob(f"{self.documents_path}/*")) + sorted_filepaths = self.__sort_possible_extra_files_last(filepaths) + mapped_documents = self.__map_documents(sorted_filepaths) + return mapped_documents + + def __get_graphql_documents(self, mapped_documents: dict[str, dict]): + print(f"uploading files using {self.MAX_WORKERS} workers") + graphql_documents: list[GraphQLDocument] = [] + with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor: + futures = [ + executor.submit( + self.__upload_and_create_document, + key=key, + mapped_documents=mapped_documents, + ) + for key in mapped_documents.keys() + ] + + uploaded = 0 + for future in as_completed(futures): + gql_document = future.result() + uploaded += 1 + print(f"uploaded {uploaded}/{len(futures)} files", end="\r") + graphql_documents.append(gql_document) + + print() + + return graphql_documents + + def __upload_and_create_document(self, key: str, mapped_documents: dict[str, dict]): + upload_document_response = self.__upload_file(mapped_documents[key]["document"]) + document: GraphQLDocument = { + "document": { + "name": os.path.basename(mapped_documents[key]["document"]), + "objectKey": upload_document_response["objectKey"], + }, + "extras": None, + } + + if "extra" in mapped_documents[key]: + upload_extra_response = self.__upload_file( + filepath=mapped_documents[key]["extra"] + ) + document["extras"] = [ + { + "name": os.path.basename(mapped_documents[key]["extra"]), + "objectKey": upload_extra_response["objectKey"], + } + ] + return document + + def __upload_file(self, filepath: str): + with post( + url=self.proxy_url, + headers=self.headers, + files=[("file", open(filepath, "rb"))], + ) as response: + response.raise_for_status() + return response.json() + + def __sort_possible_extra_files_last(self, filepaths: list[str]): + # Sort file paths ending with .json, .txt, or .xml to be at the end + EXTRA_FILES_EXTENSIONS = (".json", ".txt", ".xml") + return sorted( + filepaths, + key=lambda x: (os.path.splitext(x)[-1] in EXTRA_FILES_EXTENSIONS, x), + ) + + def __map_documents(self, filepaths: list[str]): + mapped_documents: dict[str, dict] = {} + for filepath in filepaths: + filename = os.path.basename(filepath).split(".")[0] + if filename in mapped_documents: + mapped_documents[filename]["extra"] = filepath + else: + mapped_documents[filename] = {"document": filepath} + return mapped_documents diff --git a/create-project-async/src/graphql_utils.py b/create-project-async/src/graphql_utils.py new file mode 100644 index 0000000..3e1d408 --- /dev/null +++ b/create-project-async/src/graphql_utils.py @@ -0,0 +1,30 @@ +import json + +from requests import post + + +class GraphQLUtils: + def __init__(self, base_url, client_id, client_secret, headers): + self.base_url = base_url + self.graphql_url = f"{base_url}/graphql" + self.client_id = client_id + self.client_secret = client_secret + self.headers = headers + + def call_graphql(self, data): + return post(url=self.graphql_url, headers=self.headers, data=data) + + def process_graphql_response(self, response): + if "json" in response.headers["content-type"]: + json_response: dict = json.loads(response.text.encode("utf8")) + if "errors" in json_response: + print(json.dumps(json_response["errors"], indent=1)) + else: + job = json_response["data"]["result"]["job"] + print(json.dumps(job, indent=1)) + print("Check job status using the command below") + get_job_status_command = f"python api_client.py get_job_status --base_url {self.base_url} --client_id {self.client_id} --client_secret {self.client_secret} --job_id {job['id']}" + print(get_job_status_command) + else: + print(response.text.encode("utf8")) + print(response) diff --git a/create-project-async/src/helper.py b/create-project-async/src/helper.py index 1fcee73..815ca01 100644 --- a/create-project-async/src/helper.py +++ b/create-project-async/src/helper.py @@ -1,16 +1,28 @@ from oauthlib.oauth2 import BackendApplicationClient from requests_oauthlib import OAuth2Session import json +from csv import reader as csvreader def get_access_token(base_url, client_id, client_secret): + print("Getting access token...") client = BackendApplicationClient(client_id=client_id) oauth = OAuth2Session(client=client) - token = oauth.fetch_token(token_url=base_url + '/api/oauth/token', - client_id=client_id, client_secret=client_secret) - return token['access_token'] + token = oauth.fetch_token( + token_url=base_url + "/api/oauth/token", + client_id=client_id, + client_secret=client_secret, + ) + print("Access token received.") + return token["access_token"] def get_operations(file_name): - with open(file_name, 'r') as file: + with open(file_name, "r") as file: return json.loads(file.read()) + + +def parse_multiple_config(config_path: str): + with open(config_path, "r") as file: + config_reader = csvreader(file) + yield from config_reader diff --git a/create-project-async/src/logger.py b/create-project-async/src/logger.py new file mode 100644 index 0000000..183bf09 --- /dev/null +++ b/create-project-async/src/logger.py @@ -0,0 +1,23 @@ +# https://docs.python.org/3/howto/logging-cookbook.html#implementing-structured-logging +import json +import logging + + +class StructuredMessage: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def __str__(self) -> str: + return json.dumps(self.kwargs) + + +def log(message, logger: logging.Logger, level=logging.INFO, **kwargs): + logger.log( + level=level, + msg=StructuredMessage( + level=logging._levelToName[level], + message=message, + **kwargs, + logger=logger.name, + ), + ) diff --git a/create-project-async/src/project.py b/create-project-async/src/project.py index 11d7d3e..77d4054 100644 --- a/create-project-async/src/project.py +++ b/create-project-async/src/project.py @@ -1,110 +1,60 @@ -import glob import json import os -from requests import post +from src.graphql_document_creator import GraphQLDocument, GraphQLDocumentCreator +from src.graphql_utils import GraphQLUtils from src.helper import get_access_token, get_operations class Project: - def __init__(self, base_url: str, id:str, secret:str): + def __init__(self, base_url: str, id: str, secret: str, documents_path: str): + if os.path.isfile(documents_path): + raise NotImplementedError( + "createProject with a list of documents is not yet implemented" + ) + self.base_url = base_url self.graphql_url = f"{base_url}/graphql" self.proxy_url = f"{base_url}/api/static/proxy/upload" self.client_id = id self.client_secret = secret - self.headers = None - - def create(self, team_id, operations_path, documents_path): - if (os.path.isfile(documents_path)): - raise NotImplementedError("createProject with a list of documents is not yet implemented") - - access_token = get_access_token(self.base_url, self.client_id, self.client_secret) - self.headers = self.__add_headers(key='Authorization', value=f"Bearer {access_token}") - operations = get_operations(operations_path) - - operations["variables"]["input"]["documents"] = [] - operations["variables"]["input"]["teamId"] = team_id + self.documents_path = documents_path - filepaths = list(glob.iglob(f"{documents_path}/*")) - sorted_filepaths = self.__sort_possible_extra_files_last(filepaths) - mapped_documents = self.__map_documents(sorted_filepaths) + access_token = get_access_token( + self.base_url, self.client_id, self.client_secret + ) + self.headers: dict[str, str] = { + "Authorization": f"Bearer {access_token}", + } - for key in mapped_documents: - upload_document_response = self.__upload_file(filepath=mapped_documents[key]["document"]) - documents = { - "document": { - "name": os.path.basename(mapped_documents[key]["document"]), - "objectKey": upload_document_response["objectKey"] - } - } + def create(self, team_id: str, operations_path: str, name: str | None = None): + gql_documents = GraphQLDocumentCreator( + proxy_url=self.proxy_url, headers=self.headers, documents_path=self.documents_path).create() - if "extra" in mapped_documents[key]: - upload_extra_response = self.__upload_file(filepath=mapped_documents[key]["extra"]) - documents["extras"] = [ - { - "name": os.path.basename(mapped_documents[key]["extra"]), - "objectKey": upload_extra_response["objectKey"] - } - ] + operations = self._get_operations( + team_id, operations_path, gql_documents, name) - operations["variables"]["input"]["documents"].append(documents) + gql = GraphQLUtils(base_url=self.base_url, headers=self.headers, + client_id=self.client_id, client_secret=self.client_secret) - graphql_response = self.__call_graphql( + graphql_response = gql.call_graphql( data={ "query": operations["query"], "variables": json.dumps(operations["variables"]), - "operationName": operations.get("operationName", "Datasaur API client - createProject") + "operationName": operations.get( + "operationName", "Datasaur API client - createProject" + ), } ) - self.__process_graphql_response(graphql_response) - - def __upload_file(self, filepath): - with post( - url=self.proxy_url, - headers=self.headers, - files=[('file', open(filepath, 'rb'))] - ) as response: - response.raise_for_status() - return response.json() - - def __call_graphql(self, data): - return post(url=self.graphql_url, headers=self.headers, data=data) - def __process_graphql_response(self, response): - if "json" in response.headers["content-type"]: - json_response: dict = json.loads(response.text.encode("utf8")) - if "errors" in json_response: - print(json.dumps(json_response["errors"], indent=1)) - else: - job = json_response["data"]["result"]["job"] - print(json.dumps(job, indent=1)) - print("Check job status using command bellow") - get_job_status_command = f"python api_client.py get_job_status --base_url {self.base_url} --client_id {self.client_id} --client_secret {self.client_secret} --job_id {job['id']}" - print(get_job_status_command) - else: - print(response.text.encode("utf8")) - print(response) - - def __add_headers(self, key, value): - if self.headers is None: - self.headers = {key: value} - else: - self.headers[key] = value + gql.process_graphql_response(graphql_response) - return self.headers + def _get_operations(self, team_id: str, operations_path: str, documents: list[GraphQLDocument], name: str | None): + operations = get_operations(operations_path) + operations["variables"]["input"]["teamId"] = team_id + if name is not None: + operations["variables"]["input"]["name"] = name - def __sort_possible_extra_files_last(self, filepaths): - # Sort file paths ending with .json or .txt to be at the end - filepaths.sort(key=lambda x: (x.endswith('.json') or x.endswith('.txt'), x)) - return filepaths + operations["variables"]["input"]["documents"] = documents - def __map_documents(self, filepaths): - mapped_documents = {} - for filepath in filepaths: - filename = os.path.basename(filepath).split('.')[0] - if filename in mapped_documents: - mapped_documents[filename]["extra"] = filepath - else: - mapped_documents[filename] = { "document": filepath } - return mapped_documents + return operations diff --git a/create_project_via_eos.json b/create_project_via_eos.json new file mode 100644 index 0000000..db31150 --- /dev/null +++ b/create_project_via_eos.json @@ -0,0 +1,88 @@ +{ + "operationName": "CreateProjectMutation", + "variables": { + "input": { + "teamId": "", + "externalObjectStorageId": "", + "name": "Demo via API", + "documents": [ + { + "document": { + "name": "file-1.mp3", + "objectKey": "path/to/file/in/eos/file-1.mp3" + }, + "extras": [ + { + "name": "file-1.json", + "objectKey": "path/to/file/in/eos/file-1.json" + } + ] + } + ], + "documentAssignments": [ + { + "email": "demo@example.com", + "documents": [ + { + "fileName": "file-1.mp3", + "part": 0 + } + ], + "role": "LABELER_AND_REVIEWER" + }, + { + "email": "demo.labeler-1@example.com", + "documents": [ + { + "fileName": "file-1.mp3", + "part": 0 + } + ], + "role": "LABELER" + } + ], + "kinds": [ + "TOKEN_BASED" + ], + "purpose": "LABELING", + "creationSettings": { + "anonymizationConfig": null, + "customHeaderColumns": [ + { + "name": "Column 1", + "displayed": true, + "labelerRestricted": false + } + ], + "enableTabularMarkdownParsing": false, + "fileTransformerId": null, + "firstRowAsHeader": false, + "sentenceSeparator": "\n", + "splitDocumentConfig": null, + "tokenizer": "WINK", + "transcriptConfig": { + "method": "TRANSCRIPTION" + }, + "viewer": { + "mode": "TOKEN" + } + }, + "tokenLabelSets": [], + "rowQuestions": null, + "documentQuestions": null, + "bboxLabelSets": null, + "kindsDocumentSettings": { + "tokenBasedSettings": { + "allTokensMustBeLabeled": false, + "allowArcDrawing": false, + "allowCharacterBasedLabeling": false, + "allowMultiLabels": true, + "editSentenceTokenizer": "WINK", + "textLabelMaxTokenLength": 999999, + "autoScrollWhenLabeling": true + } + } + } + }, + "query": "mutation CreateProjectMutation($input: LaunchProjectInput!) { result: createProject(input: $input) { name job { id status progress resultId errors { id stack args } } } }" +} \ No newline at end of file diff --git a/create_project_via_eos.py b/create_project_via_eos.py new file mode 100644 index 0000000..abd97c4 --- /dev/null +++ b/create_project_via_eos.py @@ -0,0 +1,23 @@ +import fire +import json +import os + +from toolbox.get_access_token import get_access_token +from toolbox.get_operations import get_operations +from toolbox.post_request import post_request + +def create_project_via_eos(base_url, client_id, client_secret): + url = base_url + "/graphql" + access_token = get_access_token(base_url, client_id, client_secret) + operations = get_operations("create_project_via_eos.json") + + response = post_request(url, access_token, operations) + if "json" in response.headers["content-type"]: + json_response = json.loads(response.text.encode("utf8")) + return json_response + else: + return response.text + +if __name__ == "__main__": + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + fire.Fire(create_project_via_eos) diff --git a/export-eos.json b/export-eos.json new file mode 100644 index 0000000..8eecb36 --- /dev/null +++ b/export-eos.json @@ -0,0 +1,17 @@ +{ + "operationName": "ExportTextProjectQuery", + "variables": { + "input": { + "projectIds": [""], + "role": "REVIEWER", + "format": "DATASAUR_SCHEMA", + "fileName": "", + "method": "EXTERNAL_OBJECT_STORAGE", + "externalObjectStorageParameter": { + "externalObjectStorageId": "", + "prefix": "" + } + } + }, + "query": "query ExportTextProjectQuery($input: ExportTextProjectInput!) {\n result: exportTextProject(input: $input) {\n ...ExportRequestRedirectResultFragment\n __typename\n }\n}\n\nfragment ExportRequestRedirectResultFragment on ExportRequestResult {\n exportId\n fileUrl\n queued\n redirect\n key\n __typename\n}\n" +} diff --git a/export.py b/export.py index 605a263..62df076 100644 --- a/export.py +++ b/export.py @@ -11,21 +11,30 @@ POOLING_INVERVAL = 0.5 # 0.5s -def export_project(base_url, client_id, client_secret, project_id, export_file_name, export_format, output_dir): +def export_project( + base_url, + client_id, + client_secret, + project_id, + export_file_name, + export_format, + output_dir, + operation_path="export.json", +): url = base_url + "/graphql" access_token = get_access_token(base_url, client_id, client_secret) - operations = get_operations('export.json') + operations = get_operations(operation_path) operations["variables"]["input"]["fileName"] = export_file_name operations["variables"]["input"]["projectIds"] = [project_id] operations["variables"]["input"]["format"] = export_format response = post_request(url, access_token, operations) - if 'json' in response.headers['content-type']: - json_response = json.loads(response.text.encode('utf8')) + if "json" in response.headers["content-type"]: + json_response = json.loads(response.text.encode("utf8")) print(json.dumps(json_response, indent=1)) - if len(json_response["data"]["result"]["fileUrl"]) > 0: + if json_response["data"]["result"]["fileUrl"]: export_id = json_response["data"]["result"]["exportId"] poll_export_delivery_status(url, access_token, export_id) @@ -34,28 +43,36 @@ def export_project(base_url, client_id, client_secret, project_id, export_file_n os.makedirs(output_dir, exist_ok=True) file_response_url = urlparse(file_url) file_name = os.path.basename(file_response_url.path) - output_file = output_dir + '/' + file_name - open(output_file, 'wb').write(file_response.content) + output_file = output_dir + "/" + file_name + open(output_file, "wb").write(file_response.content) return "Success downloading the file. Output file:" + output_file + else: + export_id = json_response["data"]["result"]["exportId"] + poll_export_delivery_status(url, access_token, export_id) + return ( + "Success exporting the project. Check your storage bucket for the file." + ) else: return response def poll_export_delivery_status(url, access_token, export_id): - operations = get_operations('get_export_delivery_status.json') + operations = get_operations("get_export_delivery_status.json") operations["variables"]["exportId"] = export_id while True: time.sleep(POOLING_INVERVAL) response = post_request(url, access_token, operations) - if 'json' in response.headers['content-type']: - json_response = json.loads(response.text.encode('utf8')) - delivery_status = json_response["data"]["exportDeliveryStatus"]["deliveryStatus"] - if (delivery_status == "QUEUED"): + if "json" in response.headers["content-type"]: + json_response = json.loads(response.text.encode("utf8")) + delivery_status = json_response["data"]["exportDeliveryStatus"][ + "deliveryStatus" + ] + if delivery_status == "QUEUED": print("Waiting for exported file to be ready...") - elif (delivery_status == "DELIVERED"): + elif delivery_status == "DELIVERED": print("Exported file is ready") break - elif (delivery_status == "FAILED"): + elif delivery_status == "FAILED": print("Failed to export file") break else: @@ -66,21 +83,27 @@ def poll_export_delivery_status(url, access_token, export_id): def get_access_token(base_url, client_id, client_secret): client = BackendApplicationClient(client_id=client_id) oauth = OAuth2Session(client=client) - token = oauth.fetch_token(token_url=base_url + '/api/oauth/token', client_id=client_id, - client_secret=client_secret) - return token['access_token'] + token = oauth.fetch_token( + token_url=base_url + "/api/oauth/token", + client_id=client_id, + client_secret=client_secret, + ) + return token["access_token"] def get_operations(file_name): - with open(file_name, 'r') as file: + with open(file_name, "r") as file: return json.loads(file.read()) def post_request(url, access_token, operations): - headers = {'Authorization': 'Bearer ' + access_token, 'Content-Type': 'application/json'} + headers = { + "Authorization": "Bearer " + access_token, + "Content-Type": "application/json", + } return requests.request("POST", url, headers=headers, data=json.dumps(operations)) -if __name__ == '__main__': - os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' +if __name__ == "__main__": + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" fire.Fire(export_project)