diff --git a/src/prep/api/dataio.py b/src/prep/api/dataio.py
index a3c4793..6957876 100644
--- a/src/prep/api/dataio.py
+++ b/src/prep/api/dataio.py
@@ -7,7 +7,7 @@
from datasets import get_dataset_config_names, get_dataset_split_names
from rich import print as rprint
-from ..constants import HF_PREFIX, PREVIEW_PREFIX, SAVE_PREFIX, SKIP, WARN_PREFIX
+from ..constants import HF_PREFIX, SAVE_PREFIX, SKIP
from .formatutils import iter_images, write_to_file
from .log import get_logger
from .pathio import PathIO
@@ -140,16 +140,18 @@ def do_preview(
):
try:
row = d[i]
- rprint(f"[bold]Sample {i}[/bold]:", row)
-
- for j, img in enumerate(iter_images(row)):
- path = write_to_file(
- img,
- save_root / f"{i:06d}_{j:02d}",
- )
- rprint(f"{PREVIEW_PREFIX}Preview image saved: {path}")
+ image_paths = [
+ str(write_to_file(img, save_root / f"{i:06d}_{j:02d}"))
+ for j, img in enumerate(iter_images(row))
+ ]
+
+ rprint(
+ f"[bold]Sample {i}[/bold]:"
+ + (f" (images {image_paths})" if image_paths else ""),
+ row,
+ )
except Exception as exc:
- logger.warning(f"{WARN_PREFIX}Failed to preview {i}th sample: {exc}")
+ logger.warning(f"Failed to preview {i}th sample: {exc}")
continue
def do_save(
diff --git a/src/prep/api/formatter.py b/src/prep/api/formatter.py
index d45e48d..8f7ddfc 100644
--- a/src/prep/api/formatter.py
+++ b/src/prep/api/formatter.py
@@ -4,7 +4,7 @@
from datasets import ClassLabel, Features, Image, List, Value
-from ..constants import ERROR_PREFIX, ID_PATTERN, IMAGE_TAG, WARN_PREFIX
+from ..constants import ID_PATTERN, IMAGE_TAG
from .log import get_logger
from .types import (
DataFormat,
@@ -91,7 +91,7 @@ def get(
return _FORMATTER_REGISTRY[("_", "show", split)]
k = (id_, target_format, split)
if k not in _FORMATTER_REGISTRY:
- logger.warning(f"{WARN_PREFIX}Formatter pipeline not registered for {k}. ")
+ logger.warning(f"Formatter pipeline not registered for {k}. ")
logger.warning(
"Fallback to generic pipeline 'auto',"
" which may cause unexpected formatting issues.",
@@ -213,7 +213,11 @@ def cast_dataset(
case "eval":
return FormatterPipeline.cast_eval(d, args)
case "cls":
- return FormatterPipeline.cast_cls(d, args)
+ d = FormatterPipeline.cast_cls(d, args)
+ from collections import Counter
+
+ print("label", Counter(d["label"]))
+ return d
case "show":
return d
case _:
@@ -230,20 +234,20 @@ def load(self, override_src: str | None, args: ProcArgs) -> "Dataset":
try:
d = self.cast_dataset(d, self.target_format, args)
except Warning as e:
- logger.warning(f"{WARN_PREFIX}Casting warning {str(self)}")
- logger.warning(f"{WARN_PREFIX}{e}")
+ logger.warning(f"Casting warning {str(self)}")
+ logger.warning(f"{e}")
except Exception as e:
- logger.error(f"{ERROR_PREFIX}Casting failed {str(self)}")
- logger.error(f"{ERROR_PREFIX}{e}")
+ logger.error(f"Casting failed {str(self)}")
+ logger.error(f"{e}")
try:
for i in range(10):
self.check_sample(d[i])
except Warning as e:
- logger.warning(f"{WARN_PREFIX}Validation warning {str(self)}")
- logger.warning(f"{WARN_PREFIX}{e}")
+ logger.warning(f"Validation warning {str(self)}")
+ logger.warning(f"{e}")
except Exception as e:
- logger.error(f"{ERROR_PREFIX}Validation failed {str(self)}")
- logger.error(f"{ERROR_PREFIX}{e}")
+ logger.error(f"Validation failed {str(self)}")
+ logger.error(f"{e}")
return d
diff --git a/src/prep/constants.py b/src/prep/constants.py
index cd63a17..c75838f 100644
--- a/src/prep/constants.py
+++ b/src/prep/constants.py
@@ -23,6 +23,11 @@ def is_env_enabled(env_var: str, default: str = "0") -> bool:
FORMATTING_PATTERN = re.compile(r"ANSWER:|||\\boxed\{")
ID_PATTERN = re.compile(r"^[a-zA-Z0-9\._-]+$")
DEFAULT_UI = None if is_env_enabled("UI", "0") else False
+# interactive mode off -> disable progress bars
+if DEFAULT_UI is not None:
+ from datasets import disable_progress_bars
+
+ disable_progress_bars()
SAVE_PREFIX = "💾\t"
HF_PREFIX = "☁️\t"
diff --git a/tests/cli/test_clip.py b/tests/cli/test_clip.py
index b8abbb0..04fa6bd 100644
--- a/tests/cli/test_clip.py
+++ b/tests/cli/test_clip.py
@@ -7,7 +7,7 @@
from prep.cli import app
EXPECTED = """\
-WARNING ⚠️ Formatter pipeline not registered for ('auto', 'clip', 'train').
+WARNING Formatter pipeline not registered for ('auto', 'clip', 'train').
WARNING Fallback to generic pipeline 'auto', which may cause unexpected formatting issues."""
diff --git a/tests/cli/test_cls.py b/tests/cli/test_cls.py
index 1ea2122..820d398 100644
--- a/tests/cli/test_cls.py
+++ b/tests/cli/test_cls.py
@@ -4,15 +4,20 @@
from prep.cli import app
EXPECTED = """\
-WARNING ⚠️ Formatter pipeline not registered for ('cifar10', 'cls', 'train').
+WARNING Formatter pipeline not registered for ('cifar10', 'cls', 'train').
WARNING Fallback to generic pipeline 'auto', which may cause unexpected formatting issues.
+label Counter({0: 5000, 6: 5000, 2: 5000, 7: 5000, 1: 5000, 4: 5000, 5: 5000, 3: 5000, 8: 5000, 9: 5000})
Dataset({
features: ['id', 'image', 'label', 'extra_info'],
num_rows: 50000
})
-Sample 0:
-{'id': 'cifar10/train00000000', 'image': , 'label': 0, 'extra_info': ''}
+Sample 0: (images ['out/cls/cifar10/train/000000_00.png'])
+{
+ 'id': 'cifar10/train00000000',
+ 'image': ,
+ 'label': 0,
+ 'extra_info': ''
+}
💾\tAbout to save to disk -> PosixPath('out/cls/cifar10/train')...
💾\tNot saved (pass `--save` to override or `export UI=1` to enable interactive prompts)
☁️\tAbout to upload to HF -> 'cifar10' (subset='default', split='train', private=True)...