feat(data): add dpdata format conversion

deepmd/dpmodel/utils/lmdb_data.py

@@ -869,10 +869,6 @@ def compute_block_targets(
         Each element is ``(system_indices_in_block, target_frame_count)``.
         Returns empty list if no expansion is needed (all targets == actual).
     """
-    from deepmd.utils.data_system import (
-        prob_sys_size_ext,
-    )
-
     # Parse block definitions from the auto_prob string
     # Format: "prob_sys_size;stt:end:weight;stt:end:weight;..."
     block_str = auto_prob_style.split(";")[1:]
@@ -918,8 +914,18 @@ def compute_block_targets(
         )
         blocks = nonempty
 
-    # Compute per-system probabilities using the standard function
-    sys_probs = prob_sys_size_ext(auto_prob_style, nsystems, system_nframes)
+    # Compute the same per-system probabilities as prob_sys_size_ext locally.
+    # Keeping this framework-agnostic LMDB module independent of data_system
+    # avoids an import cycle when the legacy adapter imports the LMDB reader.
+    block_weights = np.asarray([weight for _, _, weight in blocks], dtype=float)
+    assert np.all(block_weights >= 0), "the weight of a block should be no less than 0"
+    block_probs = block_weights / np.sum(block_weights)
+    sys_probs = np.zeros(nsystems, dtype=np.float64)
+    for block_idx, (stt, end, _weight) in enumerate(blocks):
+        block_frames = np.asarray(system_nframes[stt:end], dtype=float)
+        sys_probs[stt:end] = (
+            block_frames / np.sum(block_frames) * block_probs[block_idx]
+        )
 
     # Group systems by block, compute block-level frames and prob
     block_info: list[tuple[list[int], int, float]] = []  # (sys_ids, frames, prob)

deepmd/pd/entrypoints/main.py

@@ -69,6 +69,7 @@
 from deepmd.utils.data_system import (
     get_data,
     process_systems,
+    validate_lmdb_systems,
 )
 from deepmd.utils.path import (
     DPPath,
@@ -109,10 +110,28 @@ def prepare_trainer_input_single(
         )
         training_systems = training_dataset_params["systems"]
         trn_patterns = training_dataset_params.get("rglob_patterns", None)
-        training_systems = process_systems(training_systems, patterns=trn_patterns)
+        training_systems = process_systems(
+            training_systems,
+            patterns=trn_patterns,
+            fmt=training_dataset_params.get("format", None),
+            out_fmt=training_dataset_params.get(
+                "out_format", training_dataset_params.get("output_format", None)
+            ),
+        )
+        validate_lmdb_systems(training_systems, backend_name="Paddle", supported=False)
         if validation_systems is not None:
             val_patterns = validation_dataset_params.get("rglob_patterns", None)
-            validation_systems = process_systems(validation_systems, val_patterns)
+            validation_systems = process_systems(
+                validation_systems,
+                val_patterns,
+                fmt=validation_dataset_params.get("format", None),
+                out_fmt=validation_dataset_params.get(
+                    "out_format", validation_dataset_params.get("output_format", None)
+                ),
+            )
+            validate_lmdb_systems(
+                validation_systems, backend_name="Paddle", supported=False
+            )
 
         # stat files
         stat_file_path_single = data_dict_single.get("stat_file")

deepmd/pt/entrypoints/main.py

@@ -91,6 +91,7 @@
 from deepmd.utils.data_system import (
     get_data,
     process_systems,
+    validate_lmdb_systems,
 )
 from deepmd.utils.path import (
     DPPath,
@@ -183,10 +184,25 @@ def prepare_trainer_input_single(
         def _make_dp_loader_set(
             systems: str | list[str],
             dataset_params: dict[str, Any],
-        ) -> DpLoaderSet:
-            """Create a DpLoaderSet from systems with pattern expansion."""
+        ) -> DpLoaderSet | LmdbDataset:
+            """Create a dataset from systems with pattern expansion/conversion."""
             patterns = dataset_params.get("rglob_patterns")
-            systems = process_systems(systems, patterns=patterns)
+            systems = process_systems(
+                systems,
+                patterns=patterns,
+                fmt=dataset_params.get("format"),
+                out_fmt=dataset_params.get(
+                    "out_format", dataset_params.get("output_format")
+                ),
+            )
+            lmdb_path = validate_lmdb_systems(systems, backend_name="PyTorch")
+            if lmdb_path is not None:
+                return LmdbDataset(
+                    lmdb_path,
+                    model_params_single["type_map"],
+                    dataset_params["batch_size"],
+                    auto_prob_style=dataset_params.get("auto_prob"),
+                )
             return DpLoaderSet(
                 systems,
                 dataset_params["batch_size"],
@@ -196,7 +212,11 @@ def _make_dp_loader_set(
             )
 
         # LMDB path: single string → LmdbDataset
-        if isinstance(training_systems, str) and is_lmdb(training_systems):
+        if (
+            training_dataset_params.get("format", None) is None
+            and isinstance(training_systems, str)
+            and is_lmdb(training_systems)
+        ):
             auto_prob = training_dataset_params.get("auto_prob", None)
             train_data_single = LmdbDataset(
                 training_systems,
@@ -206,6 +226,7 @@ def _make_dp_loader_set(
             )
             if (
                 validation_systems is not None
+                and validation_dataset_params.get("format", None) is None
                 and isinstance(validation_systems, str)
                 and is_lmdb(validation_systems)
             ):
@@ -397,50 +418,52 @@ def train(
             "Calculate neighbor statistics... (add --skip-neighbor-stat to skip this step)"
         )
 
-        if not multi_task:
-            type_map = config["model"].get("type_map")
-            training_systems = config["training"]["training_data"].get("systems")
+        def _get_neighbor_stat_data_from_params(
+            dataset_params: dict[str, Any],
+            type_map: list[str] | None,
+        ) -> Any:
+            training_systems = dataset_params.get("systems")
             if (
-                training_systems is not None
+                dataset_params.get("format") is None
+                and training_systems is not None
                 and isinstance(training_systems, str)
                 and is_lmdb(training_systems)
             ):
+                systems = [training_systems]
+            else:
+                systems = process_systems(
+                    training_systems,
+                    patterns=dataset_params.get("rglob_patterns"),
+                    fmt=dataset_params.get("format"),
+                    out_fmt=dataset_params.get(
+                        "out_format", dataset_params.get("output_format")
+                    ),
+                )
+            lmdb_path = validate_lmdb_systems(systems, backend_name="PyTorch")
+            if lmdb_path is not None:
                 from deepmd.dpmodel.utils.lmdb_data import (
                     make_neighbor_stat_data,
                 )
 
-                train_data = make_neighbor_stat_data(training_systems, type_map)
-            else:
-                train_data = get_data(
-                    config["training"]["training_data"], 0, type_map, None
-                )
+                return make_neighbor_stat_data(lmdb_path, type_map)
+            return get_data(dataset_params, 0, type_map, None)
+
+        if not multi_task:
+            type_map = config["model"].get("type_map")
+            train_data = _get_neighbor_stat_data_from_params(
+                config["training"]["training_data"], type_map
+            )
             config["model"], min_nbor_dist = BaseModel.update_sel(
                 train_data, type_map, config["model"]
             )
         else:
             min_nbor_dist = {}
             for model_item in config["model"]["model_dict"]:
                 type_map = config["model"]["model_dict"][model_item].get("type_map")
-                training_systems = config["training"]["data_dict"][model_item][
-                    "training_data"
-                ].get("systems")
-                if (
-                    training_systems is not None
-                    and isinstance(training_systems, str)
-                    and is_lmdb(training_systems)
-                ):
-                    from deepmd.dpmodel.utils.lmdb_data import (
-                        make_neighbor_stat_data,
-                    )
-
-                    train_data = make_neighbor_stat_data(training_systems, type_map)
-                else:
-                    train_data = get_data(
-                        config["training"]["data_dict"][model_item]["training_data"],
-                        0,
-                        type_map,
-                        None,
-                    )
+                train_data = _get_neighbor_stat_data_from_params(
+                    config["training"]["data_dict"][model_item]["training_data"],
+                    type_map,
+                )
                 config["model"]["model_dict"][model_item], min_nbor_dist[model_item] = (
                     BaseModel.update_sel(
                         train_data, type_map, config["model"]["model_dict"][model_item]

deepmd/pt_expt/entrypoints/main.py

@@ -37,6 +37,7 @@
     DeepmdDataSystem,
     get_data,
     process_systems,
+    validate_lmdb_systems,
 )
 from deepmd.utils.path import (
     DPPath,
@@ -118,13 +119,32 @@ def _get_neighbor_stat_data(
     ``make_neighbor_stat_data``; falls back to the legacy ``get_data`` for
     npy/HDF5 directories.
     """
-    lmdb_path = _detect_lmdb_path(dataset_params.get("systems"))
+    lmdb_path = (
+        None
+        if dataset_params.get("format") is not None
+        else _detect_lmdb_path(dataset_params.get("systems"))
+    )
     if lmdb_path is not None:
         from deepmd.dpmodel.utils.lmdb_data import (
             make_neighbor_stat_data,
         )
 
         return make_neighbor_stat_data(lmdb_path, type_map)
+    systems = process_systems(
+        dataset_params["systems"],
+        patterns=dataset_params.get("rglob_patterns"),
+        fmt=dataset_params.get("format"),
+        out_fmt=dataset_params.get("out_format", dataset_params.get("output_format")),
+    )
+    converted_lmdb_path = validate_lmdb_systems(
+        systems, backend_name="PyTorch exportable"
+    )
+    if converted_lmdb_path is not None:
+        from deepmd.dpmodel.utils.lmdb_data import (
+            make_neighbor_stat_data,
+        )
+
+        return make_neighbor_stat_data(converted_lmdb_path, type_map)
     return get_data(dataset_params, 0, type_map, None)
 
 
@@ -140,7 +160,11 @@ def _build_data_system(
     :class:`DeepmdDataSystem` path with system expansion.
     """
     systems_raw = dataset_params["systems"]
-    lmdb_path = _detect_lmdb_path(systems_raw)
+    lmdb_path = (
+        None
+        if dataset_params.get("format") is not None
+        else _detect_lmdb_path(systems_raw)
+    )
     if lmdb_path is not None:
         return LmdbDataSystem(
             lmdb_path=lmdb_path,
@@ -152,7 +176,20 @@ def _build_data_system(
     systems = process_systems(
         systems_raw,
         patterns=dataset_params.get("rglob_patterns"),
+        fmt=dataset_params.get("format"),
+        out_fmt=dataset_params.get("out_format", dataset_params.get("output_format")),
+    )
+    converted_lmdb_path = validate_lmdb_systems(
+        systems, backend_name="PyTorch exportable"
     )
+    if converted_lmdb_path is not None:
+        return LmdbDataSystem(
+            lmdb_path=converted_lmdb_path,
+            type_map=type_map,
+            batch_size=dataset_params["batch_size"],
+            auto_prob_style=dataset_params.get("auto_prob"),
+            seed=seed,
+        )
     return DeepmdDataSystem(
         systems=systems,
         batch_size=dataset_params["batch_size"],

deepmd/utils/argcheck.py

@@ -5004,6 +5004,19 @@ def training_data_args() -> list[
     doc_patterns = (
         "The customized patterns used in `rglob` to collect all training systems. "
     )
+    doc_format = (
+        "The input data format passed to dpdata for automatic conversion. "
+        "If this key is not set, `systems` must already point to DeePMD data. "
+        "If this key is set to a non-DeePMD format, each selected input path is "
+        "loaded by dpdata and converted before training. Use dpdata format names "
+        "such as `extxyz`, `ase/structure`, `ase/traj`, or `auto`."
+    )
+    doc_out_format = (
+        "The output data format passed to dpdata for automatic conversion. "
+        "When `format` requests conversion from a non-DeePMD format, this key "
+        "defaults to `lmdb`. Use a DeePMD format supported by dpdata, such as "
+        "`lmdb`, `deepmd/hdf5`, or `deepmd/npy`."
+    )
     doc_batch_size = f'This key can be \n\n\
 - list: the length of which is the same as the {link_sys}. The batch size of each system is given by the elements of the list.\n\n\
 - int: all {link_sys} use the same batch size.\n\n\
@@ -5042,6 +5055,20 @@ def training_data_args() -> list[
             default=None,
             doc=doc_patterns + doc_only_pt_supported,
         ),
+        Argument(
+            "format",
+            [str, None],
+            optional=True,
+            doc=doc_format,
+        ),
+        Argument(
+            "out_format",
+            [str, None],
+            optional=True,
+            default="lmdb",
+            doc=doc_out_format,
+            alias=["output_format"],
+        ),
         Argument(
             "batch_size",
             [list[int], int, str],
@@ -5101,6 +5128,19 @@ def validation_data_args() -> list[
     doc_patterns = (
         "The customized patterns used in `rglob` to collect all validation systems. "
     )
+    doc_format = (
+        "The input data format passed to dpdata for automatic conversion. "
+        "If this key is not set, `systems` must already point to DeePMD data. "
+        "If this key is set to a non-DeePMD format, each selected input path is "
+        "loaded by dpdata and converted before validation. Use dpdata format names "
+        "such as `extxyz`, `ase/structure`, `ase/traj`, or `auto`."
+    )
+    doc_out_format = (
+        "The output data format passed to dpdata for automatic conversion. "
+        "When `format` requests conversion from a non-DeePMD format, this key "
+        "defaults to `lmdb`. Use a DeePMD format supported by dpdata, such as "
+        "`lmdb`, `deepmd/hdf5`, or `deepmd/npy`."
+    )
     doc_batch_size = f'This key can be \n\n\
 - list: the length of which is the same as the {link_sys}. The batch size of each system is given by the elements of the list.\n\n\
 - int: all {link_sys} use the same batch size.\n\n\
@@ -5126,6 +5166,20 @@ def validation_data_args() -> list[
             default=None,
             doc=doc_patterns + doc_only_pt_supported,
         ),
+        Argument(
+            "format",
+            [str, None],
+            optional=True,
+            doc=doc_format,
+        ),
+        Argument(
+            "out_format",
+            [str, None],
+            optional=True,
+            default="lmdb",
+            doc=doc_out_format,
+            alias=["output_format"],
+        ),
         Argument(
             "batch_size",
             [list[int], int, str],