diff --git a/api/serializers.py b/api/serializers.py index 114415c7..1eecbe9f 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -402,6 +402,15 @@ def _rbac_allowed_pks(self, request, obj, action, filt) -> set: cache = request._rbac_allowed_pks_cache = {} parent = getattr(self, "parent", None) rows = getattr(parent, "instance", None) if parent is not None else None + # Look the page up before building it: this runs once per row per + # action, and listing + collecting the whole page's pks on every call + # made a list of N rows cost O(N^2) before the cache was consulted. + key = ( + (model._meta.label_lower, action, None, obj.pk) if rows is None + else (model._meta.label_lower, action, id(parent), id(rows)) + ) + if key in cache: + return cache[key] if rows is None: page = [obj] else: @@ -410,7 +419,6 @@ def _rbac_allowed_pks(self, request, obj, action, filt) -> set: except TypeError: # a single instance, not a list page = [rows] pks = [r.pk for r in page if isinstance(r, model)] or [obj.pk] - key = (model._meta.label_lower, action, id(parent), len(pks)) if key not in cache: cache[key] = set( model._default_manager.filter(pk__in=pks) @@ -2968,9 +2976,35 @@ def far_end(term): ) -def _point_cable(point): - t = point.terminations.all().first() # ≤1 due to the per-port unique rule - return CableMiniSerializer(t.cable).data if t is not None else None +_CHOICE_LABELS: dict = {} + + +def _choice_label(obj, field: str): + """``obj.get__display()`` without Django rebuilding the whole choices + dict on every call (216 interface types, once per row of a list).""" + key = (type(obj), field) + labels = _CHOICE_LABELS.get(key) + if labels is None: + f = type(obj)._meta.get_field(field) + labels = _CHOICE_LABELS[key] = {k: str(v) for k, v in f.flatchoices} + value = getattr(obj, field) + return labels.get(value, value) + + +def _point_cable(point, context=None): + # ≤1 due to the per-port unique rule. Read the prefetch directly: .first() + # clones and re-slices the queryset on every row of a component list. + t = next(iter(point.terminations.all()), None) + if t is None: + return None + if context is None: + return CableMiniSerializer(t.cable).data + # One serializer per request: building a ModelSerializer's fields per + # cabled row was most of a large component list's time. + ser = context.get("_cable_mini_serializer") + if ser is None: + ser = context["_cable_mini_serializer"] = CableMiniSerializer() + return ser.to_representation(t.cable) def _point_far_end(point): @@ -2978,7 +3012,7 @@ def _point_far_end(point): def _point_reservation(point): - r = point.reservations.all().first() # ≤1 due to the per-port unique rule + r = next(iter(point.reservations.all()), None) # ≤1 (per-port unique rule) if r is None: return None return { @@ -3080,7 +3114,7 @@ def _mini(rel): @extend_schema_field(OpenApiTypes.OBJECT) def get_cable(self, obj): - return _point_cable(obj) + return _point_cable(obj, self.context) # What the cable reaches: the far end's device and port, for the port # labels a faceplate can print ("far-end device"), the same names the @@ -3126,7 +3160,7 @@ def get_lag_member_count(self, obj) -> int: return len(obj.lag_members.all()) def get_type_display(self, obj) -> str: - return obj.get_type_display() + return _choice_label(obj, "type") @extend_schema_field(serializers.ListField()) def get_ip_addresses(self, obj): @@ -3376,7 +3410,7 @@ def get_reservation(self, obj): @extend_schema_field(OpenApiTypes.OBJECT) def get_cable(self, obj): - return _point_cable(obj) + return _point_cable(obj, self.context) def get_front_port_count(self, obj) -> int: return obj.front_ports.count() @@ -3450,7 +3484,7 @@ def get_reservation(self, obj): @extend_schema_field(OpenApiTypes.OBJECT) def get_cable(self, obj): - return _point_cable(obj) + return _point_cable(obj, self.context) def validate(self, attrs): attrs = super().validate(attrs) @@ -3508,14 +3542,14 @@ class _DevicePortSerializer(TaggableSerializerMixin, NumIdModelSerializer): @extend_schema_field(OpenApiTypes.OBJECT) def get_cable(self, obj): - return _point_cable(obj) + return _point_cable(obj, self.context) @extend_schema_field(OpenApiTypes.OBJECT) def get_reservation(self, obj): return _point_reservation(obj) def get_type_display(self, obj) -> str: - return obj.get_type_display() if obj.type else "" + return _choice_label(obj, "type") if obj.type else "" class ConsolePortSerializer(_DevicePortSerializer): @@ -6445,7 +6479,7 @@ class CircuitTerminationSerializer(serializers.ModelSerializer): @extend_schema_field(CableMiniSerializer(allow_null=True)) def get_cable(self, obj): - return _point_cable(obj) + return _point_cable(obj, self.context) @extend_schema_field(OpenApiTypes.OBJECT) def get_connected_to(self, obj): diff --git a/api/viewsets.py b/api/viewsets.py index 1f27351e..b8e91aef 100644 --- a/api/viewsets.py +++ b/api/viewsets.py @@ -1605,6 +1605,7 @@ class IPAddressViewSet(FieldWriteAllowList, CloneableMixin, TenantScopedViewSet) # assigned_vm: is_primary_for_vm reads the VM's primary_ip_id, so # without the join every VM-assigned row lazy-loads its VM (#122). "assigned_vm", "prefix__vrf", "prefix__site", "site", + "assigned_interface__device", ) .prefetch_related("tags") .all() @@ -2529,6 +2530,26 @@ def _after_save(self, obj): pass +def _status_usage_expr(): + """Sum of every relation StatusSerializer counts, as correlated subqueries, + so a list of N statuses is one statement instead of 14 COUNT(*) per row.""" + from django.db.models import Count, IntegerField, OuterRef, Subquery, Value + from django.db.models.functions import Coalesce + + from .serializers import StatusSerializer + + expr = Value(0) + for rn in StatusSerializer._USAGE_RELS: + rel = Status._meta.get_field(rn) + fk = rel.field.name + sub = ( + rel.related_model._default_manager.filter(**{fk: OuterRef("pk")}) + .order_by().values(fk).annotate(c=Count("pk")).values("c") + ) + expr = expr + Coalesce(Subquery(sub, output_field=IntegerField()), Value(0)) + return expr + + class StatusViewSet(_IpCatalogViewSet): queryset = Status.objects.all().order_by("weight", "name") serializer_class = StatusSerializer @@ -2543,7 +2564,7 @@ def get_queryset(self): avail = self.request.query_params.get("available_to") if avail: qs = qs.filter(available_to__contains=[avail]) - return qs + return qs.annotate(usage_count_annotated=_status_usage_expr()) def _after_save(self, obj): # At most one default status per (tenant, object-type): strip each slug @@ -4218,7 +4239,7 @@ def normalize_bulk_updates(self, updates): "parent__device", "lag__device", "bridge__device", ) .prefetch_related( - "tags", "terminations__cable", "reservations", "ip_addresses", "children", + "tags", "terminations__cable__status", "reservations", "ip_addresses", "children", "lag_members", "tagged_vlans", "mac_addresses", "tunnel_terminations__tunnel", *FAR_END_PREFETCH, @@ -4545,7 +4566,7 @@ def perform_update(self, serializer): class CableViewSet(TenantScopedViewSet): queryset = ( - Cable.objects.prefetch_related( + Cable.objects.select_related("status").prefetch_related( "terminations__interface__device", "terminations__front_port__device", "terminations__rear_port__device", @@ -4938,7 +4959,7 @@ def create(self, request): class RearPortViewSet(_DevicePortViewSet): queryset = ( RearPort.objects.select_related("device") - .prefetch_related("tags", "terminations__cable", "reservations", "front_ports") + .prefetch_related("tags", "terminations__cable__status", "reservations", "front_ports") .order_by("device__name", NATURAL_NAME) ) serializer_class = RearPortSerializer @@ -4949,7 +4970,7 @@ class RearPortViewSet(_DevicePortViewSet): class FrontPortViewSet(_DevicePortViewSet): queryset = ( FrontPort.objects.select_related("device", "rear_port") - .prefetch_related("tags", "terminations__cable", "reservations") + .prefetch_related("tags", "terminations__cable__status", "reservations") .order_by("device__name", NATURAL_NAME) ) serializer_class = FrontPortSerializer @@ -4959,7 +4980,7 @@ class FrontPortViewSet(_DevicePortViewSet): class ConsolePortViewSet(_DevicePortViewSet): queryset = ( ConsolePort.objects.select_related("device") - .prefetch_related("tags", "terminations__cable", "reservations") + .prefetch_related("tags", "terminations__cable__status", "reservations") .order_by("device__name", NATURAL_NAME) ) serializer_class = ConsolePortSerializer @@ -4991,7 +5012,7 @@ class AntennaViewSet(_DevicePortViewSet): class ConsoleServerPortViewSet(_DevicePortViewSet): queryset = ( ConsoleServerPort.objects.select_related("device") - .prefetch_related("tags", "terminations__cable", "reservations") + .prefetch_related("tags", "terminations__cable__status", "reservations") .order_by("device__name", NATURAL_NAME) ) serializer_class = ConsoleServerPortSerializer @@ -5001,7 +5022,7 @@ class ConsoleServerPortViewSet(_DevicePortViewSet): class PowerPortViewSet(_DevicePortViewSet): queryset = ( PowerPort.objects.select_related("device") - .prefetch_related("tags", "terminations__cable", "reservations", "outlets") + .prefetch_related("tags", "terminations__cable__status", "reservations", "outlets") .order_by("device__name", NATURAL_NAME) ) serializer_class = PowerPortSerializer @@ -5010,7 +5031,7 @@ class PowerPortViewSet(_DevicePortViewSet): class PowerOutletViewSet(_DevicePortViewSet): queryset = ( PowerOutlet.objects.select_related("device", "power_port") - .prefetch_related("tags", "terminations__cable", "reservations") + .prefetch_related("tags", "terminations__cable__status", "reservations") .order_by("device__name", NATURAL_NAME) ) serializer_class = PowerOutletSerializer @@ -5535,7 +5556,7 @@ def get_serializer_class(self): class VirtualMachineViewSet(CloneableMixin, TenantScopedViewSet): - queryset = VirtualMachine.objects.all().order_by(NATURAL_NAME) + queryset = VirtualMachine.objects.select_related("status").order_by(NATURAL_NAME) serializer_class = VirtualMachineSerializer def perform_create(self, serializer): @@ -5686,8 +5707,8 @@ def get_queryset(self): qs = ( super() .get_queryset() - .select_related("vm") - .prefetch_related("tags", "ip_addresses") + .select_related("vm__status", "vlan", "vrf") + .prefetch_related("tags", "ip_addresses", "tagged_vlans") ) if self.request: vm = self.request.query_params.get("vm") diff --git a/auth_api/api.py b/auth_api/api.py index 9b2a4584..e610d33d 100644 --- a/auth_api/api.py +++ b/auth_api/api.py @@ -492,7 +492,7 @@ class Meta: class ObjectPermissionViewSet(viewsets.ModelViewSet): queryset = ( ObjectPermission.objects.all() - .prefetch_related("groups", "users", "tenants") + .prefetch_related("groups", "users", "tenants", "sites") .order_by("name") ) serializer_class = ObjectPermissionSerializer diff --git a/auth_api/apps.py b/auth_api/apps.py index 8fb7aa79..13d3a0c9 100644 --- a/auth_api/apps.py +++ b/auth_api/apps.py @@ -8,3 +8,20 @@ class AuthApiConfig(AppConfig): def ready(self): # Register the drf-spectacular auth extension for our token scheme. from . import schema # noqa: F401 + + # A grant, its scope or a user's groups changing mid-request must not + # be answered from the per-request RBAC memo. + from django.contrib.auth import get_user_model + from django.db.models.signals import m2m_changed, post_delete, post_save + + from .models import ObjectPermission + from .rbac import invalidate_request_cache + + for sig in (post_save, post_delete): + sig.connect(invalidate_request_cache, sender=ObjectPermission, + dispatch_uid=f"rbac-memo-{sig}") + for through in (ObjectPermission.users.through, ObjectPermission.groups.through, + ObjectPermission.tenants.through, ObjectPermission.sites.through, + get_user_model().groups.through): + m2m_changed.connect(invalidate_request_cache, sender=through, + dispatch_uid=f"rbac-memo-m2m-{through.__name__}") diff --git a/auth_api/rbac.py b/auth_api/rbac.py index 8e080c55..07635ce3 100644 --- a/auth_api/rbac.py +++ b/auth_api/rbac.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +import contextvars import logging from django.core.exceptions import FieldError @@ -20,6 +21,36 @@ def _is_super(user) -> bool: return bool(getattr(user, "is_authenticated", False) and user.is_superuser) +# Per-request memo for applicable_permissions(). Set by RequestCacheMiddleware +# for the duration of one HTTP request only: background jobs, management +# commands and WebSocket consumers (which keep one user object for the life of +# the socket) never see a cache, so a revoked grant can't linger there. +_request_cache: contextvars.ContextVar = contextvars.ContextVar( + "rbac_request_cache", default=None +) + + +class RequestCacheMiddleware: + """Scope the RBAC permission memo to a single request.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + token = _request_cache.set({}) + try: + return self.get_response(request) + finally: + _request_cache.reset(token) + + +def invalidate_request_cache(*args, **kwargs): + """Drop the memo when a grant or group membership changes mid-request.""" + cache = _request_cache.get() + if cache is not None: + cache.clear() + + def applicable_permissions(user, tenant): """Enabled ObjectPermissions that apply to ``user`` in ``tenant``. @@ -30,6 +61,13 @@ def applicable_permissions(user, tenant): if not getattr(user, "is_authenticated", False): return ObjectPermission.objects.none() + # has_action/row_filter/effective_actions each call this; without the memo + # one /api/me/ of a site-scoped user ran this query and its two prefetches + # 53 times. + cache = _request_cache.get() + key = (user.pk, tenant.pk if tenant is not None else None) + if cache is not None and key in cache: + return list(cache[key]) qs = ( ObjectPermission.objects.filter(enabled=True) .filter(Q(users=user) | Q(groups__in=user.groups.all())) @@ -42,6 +80,8 @@ def applicable_permissions(user, tenant): if scoped and (tenant is None or tenant.pk not in {t.pk for t in scoped}): continue out.append(perm) + if cache is not None: + cache[key] = tuple(out) return out diff --git a/danbyte/settings.py b/danbyte/settings.py index 0151e62f..4f7db65f 100644 --- a/danbyte/settings.py +++ b/danbyte/settings.py @@ -155,6 +155,7 @@ "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", + "auth_api.rbac.RequestCacheMiddleware", # Rolling idle-session timeout (admin-configurable; no-op when unset). After # auth so request.user is resolved. "core.middleware.SessionIdleTimeoutMiddleware",