From 6c5cccb0f44ef255fbf18783dd8a9021d703cafe Mon Sep 17 00:00:00 2001 From: Codada Date: Sun, 11 Oct 2026 04:10:44 +0000 Subject: [PATCH] fix(knowledge): account for reranking model availability --- api/fields/dataset_fields.py | 7 + api/openapi/markdown/console-openapi.md | 3 + api/openapi/markdown/service-openapi.md | 2 + .../knowledge/dataset_read_service.py | 1 + api/services/knowledge/datasets/adapters.py | 25 +++- .../knowledge/datasets/application.py | 45 +++++- api/services/knowledge/entities/datasets.py | 2 + .../service_api/dataset/test_dataset_apis.py | 1 + .../unit_tests/fields/test_dataset_fields.py | 8 ++ .../knowledge/test_dataset_adapters.py | 57 ++++++++ .../test_dataset_application_service.py | 133 +++++++++++++++++- .../api/console/datasets/types.gen.ts | 3 + .../generated/api/console/datasets/zod.gen.ts | 3 + .../generated/api/console/rag/types.gen.ts | 1 + .../generated/api/console/rag/zod.gen.ts | 1 + .../generated/api/service/types.gen.ts | 2 + .../generated/api/service/zod.gen.ts | 2 + .../list/dataset-card/availability.ts | 5 + .../__tests__/corner-labels.spec.tsx | 11 ++ .../operations-dropdown.browser.spec.tsx | 5 +- .../dataset-card/components/corner-labels.tsx | 3 +- .../components/dataset-card-footer.tsx | 3 +- .../components/dataset-card-header.tsx | 5 +- .../dataset-card/components/description.tsx | 3 +- .../components/operations-dropdown.tsx | 3 +- .../datasets/list/dataset-card/index.tsx | 3 +- .../__tests__/tag-selector.spec.tsx | 6 +- .../components/dataset-card-tags.tsx | 6 +- web/models/datasets.ts | 2 + 29 files changed, 328 insertions(+), 23 deletions(-) create mode 100644 web/app/components/datasets/list/dataset-card/availability.ts diff --git a/api/fields/dataset_fields.py b/api/fields/dataset_fields.py index e6902f1cab25a5..129f07d1eda331 100644 --- a/api/fields/dataset_fields.py +++ b/api/fields/dataset_fields.py @@ -158,6 +158,13 @@ class DatasetDetailResponse(ResponseModel): embedding_model: str | None embedding_model_provider: str | None embedding_available: bool | None = None + reranking_available: bool | None = Field( + default=None, + description=( + "Whether the configured reranking model is available. True when retrieval does not require a " + "reranking model; null when availability has not been evaluated." + ), + ) retrieval_model_dict: DatasetRetrievalModelResponse = Field( description="Retrieval configuration for the knowledge base." ) diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 50264af47c25c2..1ea58af4fca33d 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -18171,6 +18171,7 @@ Site token customization strategy | permission_keys | [ string ] | | No | | pipeline_id | string | | Yes | | provider | string | | Yes | +| reranking_available | boolean | Whether the configured reranking model is available. True when retrieval does not require a reranking model; null when availability has not been evaluated. | No | | retrieval_model_dict | [DatasetRetrievalModelResponse](#datasetretrievalmodelresponse) | Retrieval configuration for the knowledge base. | Yes | | runtime_mode | string | | Yes | | summary_index_setting | [DatasetSummaryIndexSettingResponse](#datasetsummaryindexsettingresponse) | Summary index configuration. | No | @@ -18214,6 +18215,7 @@ Site token customization strategy | permission_keys | [ string ] | | No | | pipeline_id | string | | Yes | | provider | string | | Yes | +| reranking_available | boolean | Whether the configured reranking model is available. True when retrieval does not require a reranking model; null when availability has not been evaluated. | No | | retrieval_model_dict | [DatasetRetrievalModelResponse](#datasetretrievalmodelresponse) | Retrieval configuration for the knowledge base. | Yes | | runtime_mode | string | | Yes | | summary_index_setting | [DatasetSummaryIndexSettingResponse](#datasetsummaryindexsettingresponse) | Summary index configuration. | No | @@ -18297,6 +18299,7 @@ Site token customization strategy | permission_keys | [ string ] | | No | | pipeline_id | string | | Yes | | provider | string | | Yes | +| reranking_available | boolean | Whether the configured reranking model is available. True when retrieval does not require a reranking model; null when availability has not been evaluated. | No | | retrieval_model_dict | [DatasetRetrievalModelResponse](#datasetretrievalmodelresponse) | Retrieval configuration for the knowledge base. | Yes | | runtime_mode | string | | Yes | | summary_index_setting | [DatasetSummaryIndexSettingResponse](#datasetsummaryindexsettingresponse) | Summary index configuration. | No | diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index c2c93229f62678..935afbb825b388 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -2829,6 +2829,7 @@ Enum class for custom configuration status. | permission | string | | Yes | | pipeline_id | string | | Yes | | provider | string | | Yes | +| reranking_available | boolean | Whether the configured reranking model is available. True when retrieval does not require a reranking model; null when availability has not been evaluated. | No | | retrieval_model_dict | [DatasetRetrievalModelResponse](#datasetretrievalmodelresponse) | Retrieval configuration for the knowledge base. | Yes | | runtime_mode | string | | Yes | | summary_index_setting | [DatasetSummaryIndexSettingResponse](#datasetsummaryindexsettingresponse) | Summary index configuration. | No | @@ -2871,6 +2872,7 @@ Enum class for custom configuration status. | permission | string | | Yes | | pipeline_id | string | | Yes | | provider | string | | Yes | +| reranking_available | boolean | Whether the configured reranking model is available. True when retrieval does not require a reranking model; null when availability has not been evaluated. | No | | retrieval_model_dict | [DatasetRetrievalModelResponse](#datasetretrievalmodelresponse) | Retrieval configuration for the knowledge base. | Yes | | runtime_mode | string | | Yes | | summary_index_setting | [DatasetSummaryIndexSettingResponse](#datasetsummaryindexsettingresponse) | Summary index configuration. | No | diff --git a/api/services/knowledge/dataset_read_service.py b/api/services/knowledge/dataset_read_service.py index 90b076d4400b03..f4af9652bfd07b 100644 --- a/api/services/knowledge/dataset_read_service.py +++ b/api/services/knowledge/dataset_read_service.py @@ -63,6 +63,7 @@ def _dataset_detail_values(dataset: Dataset, prefetch: DatasetDetailPrefetch) -> "embedding_model": dataset.embedding_model, "embedding_model_provider": dataset.embedding_model_provider, "embedding_available": vars(dataset).get("embedding_available"), + "reranking_available": None, "retrieval_model_dict": dataset.retrieval_model_dict, "summary_index_setting": dataset.summary_index_setting, "tags": [{"id": tag.id, "name": tag.name, "type": tag.type} for tag in prefetch.tags.get(owner, [])], diff --git a/api/services/knowledge/datasets/adapters.py b/api/services/knowledge/datasets/adapters.py index 985373cbba351e..72edb394429655 100644 --- a/api/services/knowledge/datasets/adapters.py +++ b/api/services/knowledge/datasets/adapters.py @@ -59,6 +59,20 @@ def load_actor(session: Session, context: RequestContext) -> Account: def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord: if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]: item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"])) + retrieval = item["retrieval_model_dict"] + reranking_model = retrieval.get("reranking_model") + if isinstance(reranking_model, dict): + provider = reranking_model.get("reranking_provider_name") + if isinstance(provider, str) and provider: + try: + normalized_provider = str(ModelProviderID(provider)) + except ValueError: + # Leave invalid stored IDs unmatched so the list can report them as unavailable. + return item + item["retrieval_model_dict"] = { + **retrieval, + "reranking_model": {**reranking_model, "reranking_provider_name": normalized_provider}, + } return item @@ -110,10 +124,15 @@ def visibility(self, context: RequestContext) -> DatasetVisibility: ) def embedding_models(self, workspace_id: str) -> set[str]: + return self._active_models(workspace_id, ModelType.TEXT_EMBEDDING) + + def reranking_models(self, workspace_id: str) -> set[str]: + return self._active_models(workspace_id, ModelType.RERANK) + + @staticmethod + def _active_models(workspace_id: str, model_type: ModelType) -> set[str]: manager = create_plugin_provider_manager(tenant_id=workspace_id) - models = manager.get_configurations(tenant_id=workspace_id).get_models( - model_type=ModelType.TEXT_EMBEDDING, only_active=True - ) + models = manager.get_configurations(tenant_id=workspace_id).get_models(model_type=model_type, only_active=True) return {f"{model.model}:{model.provider.provider}" for model in models} def list_datasets( diff --git a/api/services/knowledge/datasets/application.py b/api/services/knowledge/datasets/application.py index 279bac2438d831..b5e8117d304c81 100644 --- a/api/services/knowledge/datasets/application.py +++ b/api/services/knowledge/datasets/application.py @@ -57,6 +57,7 @@ def list_datasets( self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None, include_own: bool ) -> DatasetPage: ... def embedding_models(self, workspace_id: str) -> set[str]: ... + def reranking_models(self, workspace_id: str) -> set[str]: ... def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ... def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ... def update_dataset( @@ -104,19 +105,57 @@ def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models ) + @staticmethod + def _requires_reranking_model(item: DatasetDetailRecord) -> bool: + retrieval = item["retrieval_model_dict"] + return ( + item["provider"] != "external" + and item["indexing_technique"] == "high_quality" + and bool(retrieval.get("reranking_enable")) + and not ( + retrieval.get("search_method") == "hybrid_search" + and retrieval.get("reranking_mode") == "weighted_score" + ) + ) + + @classmethod + def _reranking_available(cls, item: DatasetDetailRecord, models: set[str]) -> bool: + if not cls._requires_reranking_model(item): + return True + model = item["retrieval_model_dict"].get("reranking_model") + if not isinstance(model, dict): + return False + provider = model.get("reranking_provider_name") + name = model.get("reranking_model_name") + if not isinstance(provider, str) or not provider or not isinstance(name, str) or not name: + return False + return f"{name}:{provider}" in models + + def _model_availability( + self, context: RequestContext, items: Sequence[DatasetDetailRecord], *, listing: bool + ) -> None: + embedding_models = self._operations.embedding_models(context.active_workspace_id) + reranking_models = ( + self._operations.reranking_models(context.active_workspace_id) + if any(self._requires_reranking_model(item) for item in items) + else set() + ) + for item in items: + self._embedding_available(item, embedding_models, listing=listing) + item["reranking_available"] = self._reranking_available(item, reranking_models) + def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage: visibility = self._operations.visibility(context) ids, include_own = visibility.list_scope(rbac_enabled=self._rbac_enabled) result = self._operations.list_datasets(context, query, ids, include_own) - models = self._operations.embedding_models(context.active_workspace_id) for item in result["data"]: item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions)) - self._embedding_available(item, models, listing=True) + self._model_availability(context, result["data"], listing=True) return result def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord: result = self._operations.get_dataset(context, self._dataset(context, dataset_id)) - self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False) + self._model_availability(context, [result], listing=False) return result def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord: diff --git a/api/services/knowledge/entities/datasets.py b/api/services/knowledge/entities/datasets.py index 21d5d659b99fe2..29670237149a3a 100644 --- a/api/services/knowledge/entities/datasets.py +++ b/api/services/knowledge/entities/datasets.py @@ -37,6 +37,8 @@ class DatasetDetailRecord(TypedDict): embedding_model_provider: str | None # None means availability has not been evaluated by the application service. embedding_available: bool | None + # None means the retrieval dependency has not been evaluated. + reranking_available: bool | None retrieval_model_dict: Mapping[str, JsonValue] summary_index_setting: Mapping[str, JsonValue] | None tags: list[DatasetTagRecord] diff --git a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py index 03aebf9a47b086..4137d5f9f849bd 100644 --- a/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py +++ b/api/tests/unit_tests/controllers/service_api/dataset/test_dataset_apis.py @@ -132,6 +132,7 @@ def dataset(controller_session: Session, tenant: Tenant, account: Account) -> Da "embedding_model", "embedding_model_provider", "embedding_available", + "reranking_available", "retrieval_model_dict", "summary_index_setting", "tags", diff --git a/api/tests/unit_tests/fields/test_dataset_fields.py b/api/tests/unit_tests/fields/test_dataset_fields.py index 02f0692079b547..2c82b5bb8fc9e6 100644 --- a/api/tests/unit_tests/fields/test_dataset_fields.py +++ b/api/tests/unit_tests/fields/test_dataset_fields.py @@ -113,6 +113,14 @@ def test_dataset_detail_preserves_permission_keys(): assert response["permission_keys"] == ["dataset.acl.readonly", "dataset.acl.edit"] +def test_dataset_detail_preserves_reranking_availability() -> None: + response = _dump_dataset_detail(_dataset_detail_payload(reranking_available=False)) + + assert response["embedding_available"] is True + assert response["reranking_available"] is False + assert _dump_dataset_detail(_dataset_detail_payload())["reranking_available"] is None + + def test_dataset_detail_expands_legacy_null_nested_fields(): response = _dump_dataset_detail( _dataset_detail_payload( diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py index dce7c1d5242d00..f108dcf21a591c 100644 --- a/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py +++ b/api/tests/unit_tests/services/knowledge/test_dataset_adapters.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from types import SimpleNamespace from unittest.mock import patch import pytest @@ -11,7 +12,9 @@ DatasetQueryListResponse, RelatedAppListResponse, ) +from core.rag.index_processor.constant.index_type import IndexTechniqueType from extensions.application_services.app import AppServices +from graphon.model_runtime.entities.model_entities import ModelType from machinery.context import RequestContext from models import Account, App, Dataset, Document from models.account import Tenant, TenantAccountJoin, TenantAccountRole @@ -40,6 +43,60 @@ REF = DatasetRef("tenant", "dataset") +@pytest.mark.parametrize( + ("method", "model_type"), + [ + (SQLAlchemyDatasetOperations.embedding_models, ModelType.TEXT_EMBEDDING), + (SQLAlchemyDatasetOperations.reranking_models, ModelType.RERANK), + ], +) +def test_model_availability_uses_active_workspace_models( + operations: SQLAlchemyDatasetOperations, + method: Callable[[SQLAlchemyDatasetOperations, str], set[str]], + model_type: ModelType, +) -> None: + with patch("services.knowledge.datasets.adapters.create_plugin_provider_manager") as factory: + configurations = factory.return_value.get_configurations.return_value + configurations.get_models.return_value = [ + SimpleNamespace(model="model", provider=SimpleNamespace(provider="langgenius/jina/jina")), + ] + + assert method(operations, CONTEXT.active_workspace_id) == {"model:langgenius/jina/jina"} + + factory.assert_called_once_with(tenant_id=CONTEXT.active_workspace_id) + factory.return_value.get_configurations.assert_called_once_with(tenant_id=CONTEXT.active_workspace_id) + configurations.get_models.assert_called_once_with(model_type=model_type, only_active=True) + + +@pytest.mark.parametrize(("provider", "expected"), [("jina", "langgenius/jina/jina"), ("bad/provider", "bad/provider")]) +def test_dataset_reads_normalize_reranking_provider_without_changing_storage( + operations: SQLAlchemyDatasetOperations, + sqlite_session_factory: sessionmaker[Session], + provider: str, + expected: str, +) -> None: + retrieval = { + "search_method": "hybrid_search", + "reranking_enable": True, + "reranking_model": {"reranking_provider_name": provider, "reranking_model_name": "reranker"}, + } + with sqlite_session_factory.begin() as session: + stored = session.get(Dataset, REF.dataset_id) + assert stored is not None + stored.indexing_technique = IndexTechniqueType.HIGH_QUALITY + stored.retrieval_model = retrieval + + result = operations.get_dataset(CONTEXT, REF) + assert result["retrieval_model_dict"]["reranking_model"] == { + "reranking_provider_name": expected, + "reranking_model_name": "reranker", + } + with sqlite_session_factory() as session: + stored = session.get(Dataset, REF.dataset_id) + assert stored is not None + assert stored.retrieval_model == retrieval + + def dataset(**values: object) -> Dataset: return Dataset( **{ diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_application_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_application_service.py index 391c12abde7454..f3ecdbf838e6f1 100644 --- a/api/tests/unit_tests/services/knowledge/test_dataset_application_service.py +++ b/api/tests/unit_tests/services/knowledge/test_dataset_application_service.py @@ -83,11 +83,20 @@ def test_list_applies_visibility_and_reports_model_availability( "data": [ { "id": "a", + "provider": "vendor", "indexing_technique": "high_quality", "embedding_model": "missing", "embedding_model_provider": "provider", + "retrieval_model_dict": {"search_method": "semantic_search", "reranking_enable": False}, + }, + { + "id": "b", + "provider": "vendor", + "indexing_technique": "economy", + "embedding_model": None, + "embedding_model_provider": None, + "retrieval_model_dict": {"search_method": "semantic_search", "reranking_enable": False}, }, - {"id": "b", "indexing_technique": "economy", "embedding_model": None, "embedding_model_provider": None}, ] } operations.embedding_models.return_value = set() @@ -98,6 +107,128 @@ def test_list_applies_visibility_and_reports_model_availability( assert result["data"][1]["embedding_available"] is True +def test_list_reports_unavailable_reranking_dependency( + service: DatasetApplicationService, operations: MagicMock +) -> None: + operations.visibility.return_value = DatasetVisibility() + operations.embedding_models.return_value = {"embedding:langgenius/jina/jina"} + operations.reranking_models.return_value = set() + operations.list_datasets.return_value = { + "data": [ + { + "id": "dataset", + "provider": "vendor", + "indexing_technique": "high_quality", + "embedding_model": "embedding", + "embedding_model_provider": "langgenius/jina/jina", + "retrieval_model_dict": { + "search_method": "hybrid_search", + "reranking_enable": True, + "reranking_mode": "reranking_model", + "reranking_model": { + "reranking_provider_name": "langgenius/jina/jina", + "reranking_model_name": "jina-reranker-v1-base-en", + }, + }, + } + ] + } + + result = service.list_datasets(CONTEXT, DatasetListFilter()) + + assert result["data"][0]["embedding_available"] is True + assert result["data"][0]["reranking_available"] is False + + operations.reranking_models.return_value = {"jina-reranker-v1-base-en:langgenius/jina/jina"} + result = service.list_datasets(CONTEXT, DatasetListFilter()) + assert result["data"][0]["reranking_available"] is True + + +@pytest.mark.parametrize("listing", [True, False], ids=["list", "detail"]) +@pytest.mark.parametrize( + ("overrides", "retrieval_overrides", "active_models", "available", "requires_reranking"), + [ + ({}, {}, set(), False, True), + ({}, {}, {"reranker:langgenius/jina/jina"}, True, True), + ({}, {}, {"reranker:other/jina/jina"}, False, True), + ({}, {"reranking_mode": "weighted_score"}, set(), True, False), + ({}, {"reranking_enable": False}, set(), True, False), + ({"indexing_technique": "economy"}, {}, set(), True, False), + ({"provider": "external"}, {}, set(), True, False), + ({}, {"reranking_model": {}}, set(), False, True), + ( + {}, + {"reranking_model": {"reranking_provider_name": "bad/provider", "reranking_model_name": "reranker"}}, + set(), + False, + True, + ), + ({}, {"search_method": "semantic_search", "reranking_mode": "weighted_score"}, set(), False, True), + ({}, {"search_method": "full_text_search", "reranking_mode": "weighted_score"}, set(), False, True), + ({}, {"reranking_mode": None}, set(), False, True), + ], + ids=[ + "missing-model", + "active-model", + "different-provider", + "weighted-score", + "disabled-reranking", + "economy", + "external", + "unconfigured-model", + "invalid-provider-id", + "semantic-reranking", + "full-text-reranking", + "legacy-mode", + ], +) +def test_dataset_reranking_availability( + service: DatasetApplicationService, + operations: MagicMock, + listing: bool, + overrides: dict[str, object], + retrieval_overrides: dict[str, object], + active_models: set[str], + available: bool, + requires_reranking: bool, +) -> None: + item = { + "id": "dataset", + "provider": "vendor", + "indexing_technique": "high_quality", + "embedding_model": "embedding", + "embedding_model_provider": "langgenius/jina/jina", + "retrieval_model_dict": { + "search_method": "hybrid_search", + "reranking_enable": True, + "reranking_mode": "reranking_model", + "reranking_model": { + "reranking_provider_name": "langgenius/jina/jina", + "reranking_model_name": "reranker", + }, + **retrieval_overrides, + }, + **overrides, + } + operations.visibility.return_value = DatasetVisibility() + operations.list_datasets.return_value = {"data": [item]} + operations.get_dataset.return_value = item + operations.embedding_models.return_value = {"embedding:langgenius/jina/jina"} + operations.reranking_models.return_value = active_models + + if listing: + result = service.list_datasets(CONTEXT, DatasetListFilter())["data"][0] + else: + result = service.get_dataset(CONTEXT, dataset_id="dataset") + + assert result["embedding_available"] is True + assert result["reranking_available"] is available + if requires_reranking: + operations.reranking_models.assert_called_once_with(CONTEXT.active_workspace_id) + else: + operations.reranking_models.assert_not_called() + + @pytest.mark.parametrize( ("method", "extra"), [ diff --git a/packages/contracts/generated/api/console/datasets/types.gen.ts b/packages/contracts/generated/api/console/datasets/types.gen.ts index 6c84febc239ed7..85b5b082b53898 100644 --- a/packages/contracts/generated/api/console/datasets/types.gen.ts +++ b/packages/contracts/generated/api/console/datasets/types.gen.ts @@ -52,6 +52,7 @@ export type DatasetDetailWithPartialMembersResponse = { permission_keys?: Array pipeline_id: string | null provider: string + reranking_available?: boolean | null retrieval_model_dict: DatasetRetrievalModelResponse runtime_mode: string | null summary_index_setting?: DatasetSummaryIndexSettingResponse @@ -128,6 +129,7 @@ export type DatasetDetailResponse = { permission_keys?: Array pipeline_id: string | null provider: string + reranking_available?: boolean | null retrieval_model_dict: DatasetRetrievalModelResponse runtime_mode: string | null summary_index_setting?: DatasetSummaryIndexSettingResponse @@ -595,6 +597,7 @@ export type DatasetListItemResponse = { permission_keys?: Array pipeline_id: string | null provider: string + reranking_available?: boolean | null retrieval_model_dict: DatasetRetrievalModelResponse runtime_mode: string | null summary_index_setting?: DatasetSummaryIndexSettingResponse diff --git a/packages/contracts/generated/api/console/datasets/zod.gen.ts b/packages/contracts/generated/api/console/datasets/zod.gen.ts index 0d7a89c3f856fb..c75d75ec60d381 100644 --- a/packages/contracts/generated/api/console/datasets/zod.gen.ts +++ b/packages/contracts/generated/api/console/datasets/zod.gen.ts @@ -1012,6 +1012,7 @@ export const zDatasetDetailWithPartialMembersResponse = z.object({ permission_keys: z.array(z.string()).optional(), pipeline_id: z.string().nullable(), provider: z.string(), + reranking_available: z.boolean().nullish(), retrieval_model_dict: zDatasetRetrievalModelResponse, runtime_mode: z.string().nullable(), summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(), @@ -1055,6 +1056,7 @@ export const zDatasetDetailResponse = z.object({ permission_keys: z.array(z.string()).optional(), pipeline_id: z.string().nullable(), provider: z.string(), + reranking_available: z.boolean().nullish(), retrieval_model_dict: zDatasetRetrievalModelResponse, runtime_mode: z.string().nullable(), summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(), @@ -1099,6 +1101,7 @@ export const zDatasetListItemResponse = z.object({ permission_keys: z.array(z.string()).optional(), pipeline_id: z.string().nullable(), provider: z.string(), + reranking_available: z.boolean().nullish(), retrieval_model_dict: zDatasetRetrievalModelResponse, runtime_mode: z.string().nullable(), summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(), diff --git a/packages/contracts/generated/api/console/rag/types.gen.ts b/packages/contracts/generated/api/console/rag/types.gen.ts index 70953beb7726cf..8641635bfeac46 100644 --- a/packages/contracts/generated/api/console/rag/types.gen.ts +++ b/packages/contracts/generated/api/console/rag/types.gen.ts @@ -59,6 +59,7 @@ export type DatasetDetailResponse = { permission_keys?: Array pipeline_id: string | null provider: string + reranking_available?: boolean | null retrieval_model_dict: DatasetRetrievalModelResponse runtime_mode: string | null summary_index_setting?: DatasetSummaryIndexSettingResponse diff --git a/packages/contracts/generated/api/console/rag/zod.gen.ts b/packages/contracts/generated/api/console/rag/zod.gen.ts index a5afe6f494881d..1924c0222bade4 100644 --- a/packages/contracts/generated/api/console/rag/zod.gen.ts +++ b/packages/contracts/generated/api/console/rag/zod.gen.ts @@ -655,6 +655,7 @@ export const zDatasetDetailResponse = z.object({ permission_keys: z.array(z.string()).optional(), pipeline_id: z.string().nullable(), provider: z.string(), + reranking_available: z.boolean().nullish(), retrieval_model_dict: zDatasetRetrievalModelResponse, runtime_mode: z.string().nullable(), summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(), diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.ts index b6224965fd0b9a..b583a8bc6d48f7 100644 --- a/packages/contracts/generated/api/service/types.gen.ts +++ b/packages/contracts/generated/api/service/types.gen.ts @@ -587,6 +587,7 @@ export type DatasetDetailResponse = { permission: string pipeline_id: string | null provider: string + reranking_available?: boolean | null retrieval_model_dict: DatasetRetrievalModelResponse runtime_mode: string | null summary_index_setting?: DatasetSummaryIndexSettingResponse @@ -627,6 +628,7 @@ export type DatasetDetailWithPartialMembersResponse = { permission: string pipeline_id: string | null provider: string + reranking_available?: boolean | null retrieval_model_dict: DatasetRetrievalModelResponse runtime_mode: string | null summary_index_setting?: DatasetSummaryIndexSettingResponse diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.ts index fcf453513caa48..9adfc1be443f5d 100644 --- a/packages/contracts/generated/api/service/zod.gen.ts +++ b/packages/contracts/generated/api/service/zod.gen.ts @@ -786,6 +786,7 @@ export const zDatasetDetailResponse = z.object({ permission: z.string(), pipeline_id: z.string().nullable(), provider: z.string(), + reranking_available: z.boolean().nullish(), retrieval_model_dict: zDatasetRetrievalModelResponse, runtime_mode: z.string().nullable(), summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(), @@ -829,6 +830,7 @@ export const zDatasetDetailWithPartialMembersResponse = z.object({ permission: z.string(), pipeline_id: z.string().nullable(), provider: z.string(), + reranking_available: z.boolean().nullish(), retrieval_model_dict: zDatasetRetrievalModelResponse, runtime_mode: z.string().nullable(), summary_index_setting: zDatasetSummaryIndexSettingResponse.optional(), diff --git a/web/app/components/datasets/list/dataset-card/availability.ts b/web/app/components/datasets/list/dataset-card/availability.ts new file mode 100644 index 00000000000000..7ff57cd2f940b8 --- /dev/null +++ b/web/app/components/datasets/list/dataset-card/availability.ts @@ -0,0 +1,5 @@ +import type { DataSet } from '@/models/datasets' + +export const isDatasetAvailable = ( + dataset: Pick, +) => dataset.embedding_available && dataset.reranking_available !== false diff --git a/web/app/components/datasets/list/dataset-card/components/__tests__/corner-labels.spec.tsx b/web/app/components/datasets/list/dataset-card/components/__tests__/corner-labels.spec.tsx index bcf10ecde5ab33..aea29cc1dfa2a8 100644 --- a/web/app/components/datasets/list/dataset-card/components/__tests__/corner-labels.spec.tsx +++ b/web/app/components/datasets/list/dataset-card/components/__tests__/corner-labels.spec.tsx @@ -38,4 +38,15 @@ describe('CornerLabels', () => { expect(container).toBeEmptyDOMElement() }) + + it('labels an unavailable reranking dependency and clears the label when it recovers', () => { + const dataset = { ...createDataset(true), reranking_available: false } + const { rerender } = render() + + expect(screen.getByText('dataset.cornerLabel.unavailable')).toBeInTheDocument() + + rerender() + + expect(screen.queryByText('dataset.cornerLabel.unavailable')).not.toBeInTheDocument() + }) }) diff --git a/web/app/components/datasets/list/dataset-card/components/__tests__/operations-dropdown.browser.spec.tsx b/web/app/components/datasets/list/dataset-card/components/__tests__/operations-dropdown.browser.spec.tsx index af9a324adeaa8c..59ecc7c57ec277 100644 --- a/web/app/components/datasets/list/dataset-card/components/__tests__/operations-dropdown.browser.spec.tsx +++ b/web/app/components/datasets/list/dataset-card/components/__tests__/operations-dropdown.browser.spec.tsx @@ -28,7 +28,7 @@ vi.mock('@/features/system-features/client', () => ({ }), })) -it('keeps operations visible on keyboard focus without covering the unavailable label', async () => { +it('keeps operations visible on keyboard focus without covering the reranking unavailable label', async () => { // Real CSS and native Tab navigation catch visibility-hidden triggers that happy-dom misses. const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity } }, @@ -53,7 +53,8 @@ it('keeps operations visible on keyboard focus without covering the unavailable id: 'dataset-1', name: 'Test Dataset', runtime_mode: 'general', - embedding_available: false, + embedding_available: true, + reranking_available: false, permission_keys: [DatasetACLPermission.Edit], } as DataSet const screen = await render( diff --git a/web/app/components/datasets/list/dataset-card/components/corner-labels.tsx b/web/app/components/datasets/list/dataset-card/components/corner-labels.tsx index c64dda9968b58f..75e4dfb4d898db 100644 --- a/web/app/components/datasets/list/dataset-card/components/corner-labels.tsx +++ b/web/app/components/datasets/list/dataset-card/components/corner-labels.tsx @@ -2,6 +2,7 @@ import type { DataSet } from '@/models/datasets' import * as React from 'react' import { useTranslation } from 'react-i18next' import CornerLabel from '@/app/components/base/corner-label' +import { isDatasetAvailable } from '../availability' type CornerLabelsProps = { dataset: DataSet @@ -10,7 +11,7 @@ type CornerLabelsProps = { const CornerLabels = ({ dataset }: CornerLabelsProps) => { const { t } = useTranslation(['dataset']) - if (!dataset.embedding_available) { + if (!isDatasetAvailable(dataset)) { return ( $['cornerLabel.unavailable'], { ns: 'dataset' })} diff --git a/web/app/components/datasets/list/dataset-card/components/dataset-card-footer.tsx b/web/app/components/datasets/list/dataset-card/components/dataset-card-footer.tsx index 9fb84e13fa60ff..53de0e6fe569fe 100644 --- a/web/app/components/datasets/list/dataset-card/components/dataset-card-footer.tsx +++ b/web/app/components/datasets/list/dataset-card/components/dataset-card-footer.tsx @@ -12,6 +12,7 @@ import * as React from 'react' import { useMemo } from 'react' import { useTranslation } from 'react-i18next' import { useFormatTimeFromNow } from '@/hooks/use-format-time-from-now' +import { isDatasetAvailable } from '../availability' const EXTERNAL_PROVIDER = 'external' @@ -46,7 +47,7 @@ const DatasetCardFooter = ({ dataset }: DatasetCardFooterProps) => {
diff --git a/web/app/components/datasets/list/dataset-card/components/dataset-card-header.tsx b/web/app/components/datasets/list/dataset-card/components/dataset-card-header.tsx index 9f712094b701f8..6aea721d77de5b 100644 --- a/web/app/components/datasets/list/dataset-card/components/dataset-card-header.tsx +++ b/web/app/components/datasets/list/dataset-card/components/dataset-card-header.tsx @@ -6,6 +6,7 @@ import { useTranslation } from 'react-i18next' import AppIcon from '@/app/components/base/app-icon' import { useKnowledge } from '@/hooks/use-knowledge' import { DOC_FORM_ICON_CLASS_WITH_BG, DOC_FORM_TEXT } from '@/models/datasets' +import { isDatasetAvailable } from '../availability' const EXTERNAL_PROVIDER = 'external' const docModeInfoClassName = @@ -26,7 +27,7 @@ type DocModeInfoProps = { const DocModeInfo = ({ dataset, isExternalProvider, isShowDocModeInfo }: DocModeInfoProps) => { const { t } = useTranslation(['dataset']) const { formatIndexingTechniqueAndMethod } = useKnowledge() - const isPipeline = dataset.embedding_available && dataset.runtime_mode === 'rag_pipeline' + const isPipeline = isDatasetAvailable(dataset) && dataset.runtime_mode === 'rag_pipeline' if (isExternalProvider) { return ( @@ -113,7 +114,7 @@ const DatasetCardHeader = ({ dataset, nameId }: DatasetCardHeaderProps) => {
diff --git a/web/app/components/datasets/list/dataset-card/components/description.tsx b/web/app/components/datasets/list/dataset-card/components/description.tsx index c278c346429822..f7143befddcda7 100644 --- a/web/app/components/datasets/list/dataset-card/components/description.tsx +++ b/web/app/components/datasets/list/dataset-card/components/description.tsx @@ -1,6 +1,7 @@ import type { DataSet } from '@/models/datasets' import { cn } from '@langgenius/dify-ui/cn' import * as React from 'react' +import { isDatasetAvailable } from '../availability' type DescriptionProps = { dataset: DataSet @@ -10,7 +11,7 @@ const Description = ({ dataset }: DescriptionProps) => (
diff --git a/web/app/components/datasets/list/dataset-card/components/operations-dropdown.tsx b/web/app/components/datasets/list/dataset-card/components/operations-dropdown.tsx index dd38777f850129..11f32c7c076567 100644 --- a/web/app/components/datasets/list/dataset-card/components/operations-dropdown.tsx +++ b/web/app/components/datasets/list/dataset-card/components/operations-dropdown.tsx @@ -19,6 +19,7 @@ import { workspacePermissionKeysAtom } from '@/context/permission-state' import { userProfileQueryOptions } from '@/features/account-profile/client' import { systemFeaturesQueryOptions } from '@/features/system-features/client' import { getDatasetACLCapabilities } from '@/utils/permission' +import { isDatasetAvailable } from '../availability' type OperationsDropdownProps = { dataset: DataSet @@ -83,7 +84,7 @@ const OperationsDropdown = ({
{ document.addEventListener('click', onOuterClick) try { - render() + render() await user.click(screen.getByRole('combobox', { name: i18n.noTag })) @@ -182,9 +182,7 @@ describe('TagSelector', () => { document.addEventListener('click', onOuterClick) try { - render( - , - ) + render() await user.click(screen.getByRole('combobox', { name: i18n.addTag })) await user.click(await screen.findByRole('option', { name: 'Knowledge' })) diff --git a/web/features/tag-management/components/dataset-card-tags.tsx b/web/features/tag-management/components/dataset-card-tags.tsx index 62659d55269793..c96b9c6ab9411d 100644 --- a/web/features/tag-management/components/dataset-card-tags.tsx +++ b/web/features/tag-management/components/dataset-card-tags.tsx @@ -4,7 +4,7 @@ import { TagSelector } from '@/features/tag-management/components/tag-selector' type DatasetCardTagsProps = { datasetId: string - embeddingAvailable: boolean + available: boolean tags: Tag[] onOpenTagManagement?: () => void onTagsChange?: () => void @@ -13,7 +13,7 @@ type DatasetCardTagsProps = { export const DatasetCardTags = ({ datasetId, - embeddingAvailable, + available, tags, onOpenTagManagement = () => {}, onTagsChange, @@ -26,7 +26,7 @@ export const DatasetCardTags = ({ onOpenTagManagement={onOpenTagManagement} onTagsChange={onTagsChange} canBindOrUnbindTags={canBindOrUnbindTags} - className={cn('mx-3 w-auto', !embeddingAvailable && 'opacity-30')} + className={cn('mx-3 w-auto', !available && 'opacity-30')} onClick={(event) => event.stopPropagation()} /> ) diff --git a/web/models/datasets.ts b/web/models/datasets.ts index e4c9f044536a96..a8617d11fdaee2 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -1,3 +1,4 @@ +import type { DatasetDetailResponse } from '@dify/contracts/api/console/datasets/types.gen' import type { TagResponse as Tag } from '@dify/contracts/api/console/tags/types.gen' import type { DataSourceNotionPage, DataSourceProvider } from './common' import type { DatasourceType } from './pipeline' @@ -76,6 +77,7 @@ export type DataSet = { embedding_model: string embedding_model_provider: string embedding_available: boolean + reranking_available?: DatasetDetailResponse['reranking_available'] retrieval_model_dict: RetrievalConfig retrieval_model: RetrievalConfig tags: Tag[]