From 4e358fbd02fce600c26386eef6dfa2fc8c053a5a Mon Sep 17 00:00:00 2001 From: CarloLucibello Date: Tue, 20 Oct 2026 06:43:44 +0200 Subject: [PATCH] Name a from_generator cache by a given fingerprint alone, as in Python Forward a user-given `fingerprint` to `datasets`' own keyword instead of hashing it with `features` and `split`, so Julia and Python code calling `from_generator` with the same `fingerprint`, `cache_dir` and `split` share one cache. Also date the 0.4.4 changelog section. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +++++++++++-- docs/src/guide.md | 3 ++- src/toplevel.jl | 15 ++++++++++----- test/dataset.jl | 5 +++++ test/iterabledataset.jl | 4 ++++ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e174192..b24acbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,15 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.4.4] +## [0.4.5] + +### Changed +- A `fingerprint` given to `Dataset.from_generator` or `Dataset(stream)` names the cache alone, + as in Python, instead of being hashed with `features` and `split`. Julia and Python code calling + `from_generator` with the same `fingerprint`, `cache_dir` and `split` share one cache instead of + building identical copies. A dataset cached under a `fingerprint` by 0.4.4 is built again once. + +## [0.4.4] - 2026-10-20 ### Changed - The Python dependencies require `datasets` 5.1 (`>=5.1, <6`, previously `>=4.0`); an existing @@ -365,7 +373,8 @@ Baseline. Changes up to and including this release are recorded in the [git history](https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/commits/main) and the [GitHub releases](https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/releases). -[0.4.4]: https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/compare/v0.4.3...HEAD +[0.4.5]: https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/compare/v0.4.4...HEAD +[0.4.4]: https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/compare/v0.4.3...v0.4.4 [0.4.3]: https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/compare/v0.4.2...v0.4.3 [0.4.2]: https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/compare/v0.4.1...v0.4.2 [0.4.1]: https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/compare/v0.4.0...v0.4.1 diff --git a/docs/src/guide.md b/docs/src/guide.md index aef33e1..598954e 100644 --- a/docs/src/guide.md +++ b/docs/src/guide.md @@ -117,7 +117,8 @@ julia> ds[1:32]["ids"]; # a 513×32 Matrix{UInt16} A stream can't be fingerprinted, so without `fingerprint` it is rebuilt on every call, in a temporary directory. With one, a later call, also in a new session, loads the cached dataset -without streaming. A Julia iterable of examples (`Dict`s or `NamedTuple`s), or a function +without streaming, as does Python's `Dataset.from_generator` with the same `fingerprint`, +`cache_dir` and `split`. A Julia iterable of examples (`Dict`s or `NamedTuple`s), or a function returning one, is cached under a fingerprint of its content, as the function of `map` is. With `batched = true` it yields batches of columns, which convert faster: diff --git a/src/toplevel.jl b/src/toplevel.jl index 29210c7..f2bb460 100644 --- a/src/toplevel.jl +++ b/src/toplevel.jl @@ -212,7 +212,10 @@ The result is returned in the default `"julia"` format. An `IterableDataset`, or a source with something that can't be hashed, has no fingerprint: its dataset is rebuilt on every call, in a temporary directory deleted at the end of the session (or in `cache_dir`, if given). Pass `fingerprint`, a string naming the examples, to - cache them under it instead, and change it when the examples change. + cache them under it instead, and change it when the examples change. As in Python, the cache + is then named by `fingerprint` alone, so `datasets.Dataset.from_generator` with the same + `fingerprint` and `cache_dir` shares it, as long as it has the same `split` (Python's default + is `"train"`, while a stream's carries over): a call with another `split` throws. !!! warning "No multiprocessing (`num_proc`)" The examples of a Julia source or of an `IterableDataset` are generated in this process, so @@ -283,9 +286,10 @@ end Dataset(stream::IterableDataset; kws...) = from_generator(stream; kws...) -# `datasets` names the cache of `from_generator` by hashing the generator function, which pickles -# as `fingerprint` in a `JuliaFunction`, along with `features` and `split`. (Its own `fingerprint` -# keyword would name the cache by the fingerprint alone.) +# A given `fingerprint` goes to `datasets`' own keyword, which names the cache by it alone, so that +# Python's `from_generator` with the same `fingerprint` and `cache_dir` shares the cache. Otherwise +# `datasets` names it by hashing the generator function, which pickles as our content hash in a +# `JuliaFunction`, along with `features` and `split`. function _from_generator(generator::Py, source, hashed_kws; fingerprint = nothing, cache_dir = nothing, num_proc = nothing, kws...) if num_proc isa Integer && num_proc > 1 @@ -296,7 +300,8 @@ function _from_generator(generator::Py, source, hashed_kws; fingerprint = nothin "Julia function, have it work on batches with several threads, e.g. with " * "`Threads.@threads`.")) end - fp = fingerprint === nothing ? _content_hash(source, hashed_kws) : fingerprint + fingerprint === nothing || return _generate(generator; fingerprint, cache_dir, kws...) + fp = _content_hash(source, hashed_kws) if !(fp isa AbstractString) @info "`from_generator` can't fingerprint its source ($(fp.msg)), so the dataset is \ rebuilt on every call, in a temporary directory unless `cache_dir` is given. Pass \ diff --git a/test/dataset.jl b/test/dataset.jl index cd6e00c..e0f78fa 100644 --- a/test/dataset.jl +++ b/test/dataset.jl @@ -237,6 +237,11 @@ end f = Dataset.from_generator(rows; fingerprint = "rows-v1", cache_dir = dir) @test Dataset.from_generator(() -> error("not iterated"); fingerprint = "rows-v1", cache_dir = dir)[:] == f[:] + # as does one built by Python's `from_generator` under that fingerprint + datasets.Dataset.from_generator(pyeval("lambda: iter([{'x': 7}])", @__MODULE__); + fingerprint = "py-v1", cache_dir = dir) + @test Dataset.from_generator(() -> error("not iterated"); fingerprint = "py-v1", + cache_dir = dir)[:] == Dict("x" => [7]) # a function returning batches of columns, whose arrays' last axis indexes the rows b = Dataset.from_generator(; batched = true, cache_dir = dir) do diff --git a/test/iterabledataset.jl b/test/iterabledataset.jl index bb67b91..2f7836a 100644 --- a/test/iterabledataset.jl +++ b/test/iterabledataset.jl @@ -198,6 +198,10 @@ end @test string(ds.py.split) == "validation" @test Dataset(blocks; fingerprint = "blocks-v1", cache_dir = dir, split = "validation").cache_files == ds.cache_files + # Python's `from_generator` shares the cache of the same `fingerprint` + boom = pyeval("lambda: (_ for _ in ()).throw(RuntimeError('iterated'))", @__MODULE__) + py = datasets.Dataset.from_generator(boom; fingerprint = "blocks-v1", cache_dir = dir, split = "validation") + @test pyconvert(String, py.cache_files[0]["filename"]) == ds.cache_files[1]["filename"] @test_throws ArgumentError Dataset(blocks; num_proc = 2) gen = pyeval("lambda: iter([{'x': 1}, {'x': 2}])", @__MODULE__)