Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions api/fields/dataset_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,13 @@ class DatasetDetailResponse(ResponseModel):
embedding_model: str | None
embedding_model_provider: str | None
embedding_available: bool | None = None
reranking_available: bool | None = Field(
default=None,
description=(
"Whether the configured reranking model is available. True when retrieval does not require a "
"reranking model; null when availability has not been evaluated."
),
)
retrieval_model_dict: DatasetRetrievalModelResponse = Field(
description="Retrieval configuration for the knowledge base."
)
Expand Down
3 changes: 3 additions & 0 deletions api/openapi/markdown/console-openapi.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions api/openapi/markdown/service-openapi.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions api/services/knowledge/dataset_read_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def _dataset_detail_values(dataset: Dataset, prefetch: DatasetDetailPrefetch) ->
"embedding_model": dataset.embedding_model,
"embedding_model_provider": dataset.embedding_model_provider,
"embedding_available": vars(dataset).get("embedding_available"),
"reranking_available": None,
"retrieval_model_dict": dataset.retrieval_model_dict,
"summary_index_setting": dataset.summary_index_setting,
"tags": [{"id": tag.id, "name": tag.name, "type": tag.type} for tag in prefetch.tags.get(owner, [])],
Expand Down
25 changes: 22 additions & 3 deletions api/services/knowledge/datasets/adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,20 @@ def load_actor(session: Session, context: RequestContext) -> Account:
def _normalize_provider(item: DatasetDetailRecord) -> DatasetDetailRecord:
if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:
item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
retrieval = item["retrieval_model_dict"]
reranking_model = retrieval.get("reranking_model")
if isinstance(reranking_model, dict):
provider = reranking_model.get("reranking_provider_name")
if isinstance(provider, str) and provider:
try:
normalized_provider = str(ModelProviderID(provider))
except ValueError:
# Leave invalid stored IDs unmatched so the list can report them as unavailable.
return item
item["retrieval_model_dict"] = {
**retrieval,
"reranking_model": {**reranking_model, "reranking_provider_name": normalized_provider},
}
return item


Expand Down Expand Up @@ -112,10 +126,15 @@ def visibility(self, context: RequestContext) -> DatasetVisibility:
)

def embedding_models(self, workspace_id: str) -> set[str]:
return self._active_models(workspace_id, ModelType.TEXT_EMBEDDING)

def reranking_models(self, workspace_id: str) -> set[str]:
return self._active_models(workspace_id, ModelType.RERANK)

@staticmethod
def _active_models(workspace_id: str, model_type: ModelType) -> set[str]:
manager = create_plugin_provider_manager(tenant_id=workspace_id)
models = manager.get_configurations(tenant_id=workspace_id).get_models(
model_type=ModelType.TEXT_EMBEDDING, only_active=True
)
models = manager.get_configurations(tenant_id=workspace_id).get_models(model_type=model_type, only_active=True)
return {f"{model.model}:{model.provider.provider}" for model in models}

def list_datasets(
Expand Down
45 changes: 42 additions & 3 deletions api/services/knowledge/datasets/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ def list_datasets(
self, context: RequestContext, query: DatasetListFilter, accessible_ids: list[str] | None
) -> DatasetPage: ...
def embedding_models(self, workspace_id: str) -> set[str]: ...
def reranking_models(self, workspace_id: str) -> set[str]: ...
def get_dataset(self, context: RequestContext, ref: DatasetRef) -> DatasetDetailRecord: ...
def create_dataset(self, context: RequestContext, values: Mapping[str, Any]) -> DatasetDetailRecord: ...
def update_dataset(
Expand Down Expand Up @@ -105,19 +106,57 @@ def _embedding_available(item: DatasetDetailRecord, models: set[str], *, listing
or f"{item['embedding_model']}:{item['embedding_model_provider']}" in models
)

@staticmethod
def _requires_reranking_model(item: DatasetDetailRecord) -> bool:
retrieval = item["retrieval_model_dict"]
return (
item["provider"] != "external"
and item["indexing_technique"] == "high_quality"
and bool(retrieval.get("reranking_enable"))
and not (
retrieval.get("search_method") == "hybrid_search"
and retrieval.get("reranking_mode") == "weighted_score"
)
)

@classmethod
def _reranking_available(cls, item: DatasetDetailRecord, models: set[str]) -> bool:
if not cls._requires_reranking_model(item):
return True
model = item["retrieval_model_dict"].get("reranking_model")
if not isinstance(model, dict):
return False
provider = model.get("reranking_provider_name")
name = model.get("reranking_model_name")
if not isinstance(provider, str) or not provider or not isinstance(name, str) or not name:
return False
return f"{name}:{provider}" in models

def _model_availability(
self, context: RequestContext, items: Sequence[DatasetDetailRecord], *, listing: bool
) -> None:
embedding_models = self._operations.embedding_models(context.active_workspace_id)
reranking_models = (
self._operations.reranking_models(context.active_workspace_id)
if any(self._requires_reranking_model(item) for item in items)
else set()
)
for item in items:
self._embedding_available(item, embedding_models, listing=listing)
item["reranking_available"] = self._reranking_available(item, reranking_models)

def list_datasets(self, context: RequestContext, query: DatasetListFilter) -> DatasetPage:
visibility = self._operations.visibility(context)
ids = visibility.list_scope(rbac_enabled=self._rbac_enabled)
result = self._operations.list_datasets(context, query, ids)
models = self._operations.embedding_models(context.active_workspace_id)
for item in result["data"]:
item["permission_keys"] = list(visibility.overrides.get(item["id"], visibility.default_permissions))
self._embedding_available(item, models, listing=True)
self._model_availability(context, result["data"], listing=True)
return result

def get_dataset(self, context: RequestContext, *, dataset_id: str) -> DatasetDetailRecord:
result = self._operations.get_dataset(context, self._dataset(context, dataset_id))
self._embedding_available(result, self._operations.embedding_models(context.active_workspace_id), listing=False)
self._model_availability(context, [result], listing=False)
return result

def create_dataset(self, context: RequestContext, *, values: Mapping[str, Any]) -> DatasetDetailRecord:
Expand Down
2 changes: 2 additions & 0 deletions api/services/knowledge/entities/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ class DatasetDetailRecord(TypedDict):
embedding_model_provider: str | None
# None means availability has not been evaluated by the application service.
embedding_available: bool | None
# None means the retrieval dependency has not been evaluated.
reranking_available: bool | None
retrieval_model_dict: Mapping[str, JsonValue]
summary_index_setting: Mapping[str, JsonValue] | None
tags: list[DatasetTagRecord]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def make_dataset_service_grant(tenant_id: str, dataset_ids: frozenset[str] | Non
"embedding_model",
"embedding_model_provider",
"embedding_available",
"reranking_available",
"retrieval_model_dict",
"summary_index_setting",
"tags",
Expand Down
8 changes: 8 additions & 0 deletions api/tests/unit_tests/fields/test_dataset_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,14 @@ def test_dataset_detail_preserves_permission_keys():
assert response["permission_keys"] == ["dataset.acl.readonly", "dataset.acl.edit"]


def test_dataset_detail_preserves_reranking_availability() -> None:
response = _dump_dataset_detail(_dataset_detail_payload(reranking_available=False))

assert response["embedding_available"] is True
assert response["reranking_available"] is False
assert _dump_dataset_detail(_dataset_detail_payload())["reranking_available"] is None


def test_dataset_detail_expands_legacy_null_nested_fields():
response = _dump_dataset_detail(
_dataset_detail_payload(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@
DatasetQueryListResponse,
RelatedAppListResponse,
)
from core.entities.model_entities import ModelStatus, ModelWithProviderEntity, SimpleModelProviderEntity
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from enums import DeploymentEdition
from graphon.model_runtime.entities.common_entities import I18nObject
from graphon.model_runtime.entities.model_entities import FetchFrom, ModelType
from graphon.model_runtime.entities.provider_entities import ConfigurateMethod, ProviderEntity
from machinery.context import RequestContext
from models import Account, App, Dataset, Document
from models.account import Tenant, TenantAccountJoin, TenantAccountRole
Expand Down Expand Up @@ -49,6 +54,78 @@
REF = DatasetRef("tenant", "dataset")


@pytest.mark.parametrize(
("method", "model_type"),
[
(SQLAlchemyDatasetOperations.embedding_models, ModelType.TEXT_EMBEDDING),
(SQLAlchemyDatasetOperations.reranking_models, ModelType.RERANK),
],
)
def test_model_availability_uses_active_workspace_models(
operations: SQLAlchemyDatasetOperations,
method: Callable[[SQLAlchemyDatasetOperations, str], set[str]],
model_type: ModelType,
) -> None:
with patch("services.knowledge.datasets.adapters.create_plugin_provider_manager") as factory:
configurations = factory.return_value.get_configurations.return_value
configurations.get_models.return_value = [
ModelWithProviderEntity(
model="model",
label=I18nObject(en_US="Model"),
model_type=model_type,
fetch_from=FetchFrom.PREDEFINED_MODEL,
model_properties={},
status=ModelStatus.ACTIVE,
provider=SimpleModelProviderEntity(
ProviderEntity(
provider="langgenius/jina/jina",
label=I18nObject(en_US="Jina"),
supported_model_types=[model_type],
configurate_methods=[ConfigurateMethod.PREDEFINED_MODEL],
)
),
),
]

assert method(operations, CONTEXT.active_workspace_id) == {"model:langgenius/jina/jina"}

factory.assert_called_once_with(tenant_id=CONTEXT.active_workspace_id)
factory.return_value.get_configurations.assert_called_once_with(tenant_id=CONTEXT.active_workspace_id)
configurations.get_models.assert_called_once_with(model_type=model_type, only_active=True)


@pytest.mark.parametrize(
("provider", "expected"),
[("jina", "langgenius/jina/jina"), ("bad/provider", "bad/provider"), ("", "")],
)
def test_dataset_reads_normalize_reranking_provider_without_changing_storage(
operations: SQLAlchemyDatasetOperations,
sqlite_session_factory: sessionmaker[Session],
provider: str,
expected: str,
) -> None:
retrieval = {
"search_method": "hybrid_search",
"reranking_enable": True,
"reranking_model": {"reranking_provider_name": provider, "reranking_model_name": "reranker"},
}
with sqlite_session_factory.begin() as session:
stored = session.get(Dataset, REF.dataset_id)
assert stored is not None
stored.indexing_technique = IndexTechniqueType.HIGH_QUALITY
stored.retrieval_model = retrieval

result = operations.get_dataset(CONTEXT, REF)
assert result["retrieval_model_dict"]["reranking_model"] == {
"reranking_provider_name": expected,
"reranking_model_name": "reranker",
}
with sqlite_session_factory() as session:
stored = session.get(Dataset, REF.dataset_id)
assert stored is not None
assert stored.retrieval_model == retrieval


def dataset(**values: object) -> Dataset:
row = Dataset(
id="dataset",
Expand Down
Loading
Loading