From 995566594e01f9eed813f3e1f68c96e8f9b5a00e Mon Sep 17 00:00:00 2001 From: bin <178128333+xlb3055@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:11:51 +0800 Subject: [PATCH 1/4] docs: point task guides at paths that exist The quicktour linked tasks/extended and community_tasks, and the custom-task guide linked default_prompts.py. None of those paths exist on main. Signed-off-by: bin <178128333+xlb3055@users.noreply.github.com> --- docs/source/adding-a-custom-task.mdx | 4 ++-- docs/source/quicktour.mdx | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx index c68149a1e..fe68674f1 100644 --- a/docs/source/adding-a-custom-task.mdx +++ b/docs/source/adding-a-custom-task.mdx @@ -25,7 +25,7 @@ from lighteval.tasks.requests import Doc # Define as many as you need for your different tasks def prompt_fn(line: dict, task_name: str): """Defines how to go from a dataset line to a doc object. - Follow examples in src/lighteval/tasks/default_prompts.py, or get more info + Follow examples in src/lighteval/tasks/tasks/, or get more info about what this function should do in the README. """ return Doc( @@ -158,5 +158,5 @@ lighteval accelerate \ lighteval accelerate \ "model_name=openai-community/gpt2" \ "myothertask|3" \ - --custom-tasks community_tasks/my_custom_task.py + --custom-tasks examples/nanotron/custom_task.py ``` diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx index 919c8a1a1..0e454b7b2 100644 --- a/docs/source/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -72,9 +72,9 @@ For example ``` All officially supported tasks can be found at the [tasks_list](available-tasks) and in the -[extended folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/extended). -Moreover, community-provided tasks can be found in the -[community](https://github.com/huggingface/lighteval/tree/main/community_tasks) folder. +[tasks folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/tasks). +Multilingual tasks are in +[multilingual/tasks](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/multilingual/tasks). For more details on the implementation of the tasks, such as how prompts are constructed or which metrics are used, you can examine the [implementation file](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/default_tasks.py). From 067745f87c39246bce3f2f3b668a652b2dce50d8 Mon Sep 17 00:00:00 2001 From: bin <178128333+xlb3055@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:22:27 +0800 Subject: [PATCH 2/4] docs: keep the custom-task example path a placeholder examples/nanotron/custom_task.py does not define the myothertask example from this guide, so copying the command would fail. Use the same placeholder as the command template above. Co-Authored-By: Claude --- docs/source/adding-a-custom-task.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx index fe68674f1..f2127bb9a 100644 --- a/docs/source/adding-a-custom-task.mdx +++ b/docs/source/adding-a-custom-task.mdx @@ -158,5 +158,5 @@ lighteval accelerate \ lighteval accelerate \ "model_name=openai-community/gpt2" \ "myothertask|3" \ - --custom-tasks examples/nanotron/custom_task.py + --custom-tasks {path_to_your_custom_task_file} ``` From 5e481d037874420a235b6a4116efbdc5ee172906 Mon Sep 17 00:00:00 2001 From: bin <178128333+xlb3055@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:23:25 +0800 Subject: [PATCH 3/4] docs: use the same existing task paths as #1258 Signed-off-by: bin <178128333+xlb3055@users.noreply.github.com> --- docs/source/adding-a-custom-task.mdx | 2 +- docs/source/quicktour.mdx | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx index f2127bb9a..e59d1cab6 100644 --- a/docs/source/adding-a-custom-task.mdx +++ b/docs/source/adding-a-custom-task.mdx @@ -25,7 +25,7 @@ from lighteval.tasks.requests import Doc # Define as many as you need for your different tasks def prompt_fn(line: dict, task_name: str): """Defines how to go from a dataset line to a doc object. - Follow examples in src/lighteval/tasks/tasks/, or get more info + Follow examples in src/lighteval/tasks/tasks, or get more info about what this function should do in the README. """ return Doc( diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx index 0e454b7b2..80cab686b 100644 --- a/docs/source/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -72,9 +72,9 @@ For example ``` All officially supported tasks can be found at the [tasks_list](available-tasks) and in the -[tasks folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/tasks). -Multilingual tasks are in -[multilingual/tasks](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/multilingual/tasks). +[extended folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/tasks). +Moreover, community-provided tasks can be found in the +[community](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/multilingual/tasks) folder. For more details on the implementation of the tasks, such as how prompts are constructed or which metrics are used, you can examine the [implementation file](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/default_tasks.py). From f6c66f1e33f2448c65250db98e6a2e6b63ad7537 Mon Sep 17 00:00:00 2001 From: bin <178128333+xlb3055@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:27:37 +0800 Subject: [PATCH 4/4] docs: describe the merged task folders without the old categories #1016 folded the extended and community suites into src/lighteval/tasks/tasks, so labelling that folder "extended" and the multilingual folder "community" would be misleading. Co-Authored-By: Claude --- docs/source/quicktour.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx index 80cab686b..0e454b7b2 100644 --- a/docs/source/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -72,9 +72,9 @@ For example ``` All officially supported tasks can be found at the [tasks_list](available-tasks) and in the -[extended folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/tasks). -Moreover, community-provided tasks can be found in the -[community](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/multilingual/tasks) folder. +[tasks folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/tasks). +Multilingual tasks are in +[multilingual/tasks](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/multilingual/tasks). For more details on the implementation of the tasks, such as how prompts are constructed or which metrics are used, you can examine the [implementation file](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/default_tasks.py).