Performance tweaks

api/serializers.py

@@ -402,6 +402,15 @@ def _rbac_allowed_pks(self, request, obj, action, filt) -> set:
             cache = request._rbac_allowed_pks_cache = {}
         parent = getattr(self, "parent", None)
         rows = getattr(parent, "instance", None) if parent is not None else None
+        # Look the page up before building it: this runs once per row per
+        # action, and listing + collecting the whole page's pks on every call
+        # made a list of N rows cost O(N^2) before the cache was consulted.
+        key = (
+            (model._meta.label_lower, action, None, obj.pk) if rows is None
+            else (model._meta.label_lower, action, id(parent), id(rows))
+        )
+        if key in cache:
+            return cache[key]
         if rows is None:
             page = [obj]
         else:
@@ -410,7 +419,6 @@ def _rbac_allowed_pks(self, request, obj, action, filt) -> set:
             except TypeError:  # a single instance, not a list
                 page = [rows]
         pks = [r.pk for r in page if isinstance(r, model)] or [obj.pk]
-        key = (model._meta.label_lower, action, id(parent), len(pks))
         if key not in cache:
             cache[key] = set(
                 model._default_manager.filter(pk__in=pks)
@@ -2968,17 +2976,43 @@ def far_end(term):
 )
 
 
-def _point_cable(point):
-    t = point.terminations.all().first()  # ≤1 due to the per-port unique rule
-    return CableMiniSerializer(t.cable).data if t is not None else None
+_CHOICE_LABELS: dict = {}
+
+
+def _choice_label(obj, field: str):
+    """``obj.get_<field>_display()`` without Django rebuilding the whole choices
+    dict on every call (216 interface types, once per row of a list)."""
+    key = (type(obj), field)
+    labels = _CHOICE_LABELS.get(key)
+    if labels is None:
+        f = type(obj)._meta.get_field(field)
+        labels = _CHOICE_LABELS[key] = {k: str(v) for k, v in f.flatchoices}
+    value = getattr(obj, field)
+    return labels.get(value, value)
+
+
+def _point_cable(point, context=None):
+    # ≤1 due to the per-port unique rule. Read the prefetch directly: .first()
+    # clones and re-slices the queryset on every row of a component list.
+    t = next(iter(point.terminations.all()), None)
+    if t is None:
+        return None
+    if context is None:
+        return CableMiniSerializer(t.cable).data
+    # One serializer per request: building a ModelSerializer's fields per
+    # cabled row was most of a large component list's time.
+    ser = context.get("_cable_mini_serializer")
+    if ser is None:
+        ser = context["_cable_mini_serializer"] = CableMiniSerializer()
+    return ser.to_representation(t.cable)
 
 
 def _point_far_end(point):
     return far_end(next(iter(point.terminations.all()), None))
 
 
 def _point_reservation(point):
-    r = point.reservations.all().first()  # ≤1 due to the per-port unique rule
+    r = next(iter(point.reservations.all()), None)  # ≤1 (per-port unique rule)
     if r is None:
         return None
     return {
@@ -3080,7 +3114,7 @@ def _mini(rel):
 
     @extend_schema_field(OpenApiTypes.OBJECT)
     def get_cable(self, obj):
-        return _point_cable(obj)
+        return _point_cable(obj, self.context)
 
     # What the cable reaches: the far end's device and port, for the port
     # labels a faceplate can print ("far-end device"), the same names the
@@ -3126,7 +3160,7 @@ def get_lag_member_count(self, obj) -> int:
         return len(obj.lag_members.all())
 
     def get_type_display(self, obj) -> str:
-        return obj.get_type_display()
+        return _choice_label(obj, "type")
 
     @extend_schema_field(serializers.ListField())
     def get_ip_addresses(self, obj):
@@ -3376,7 +3410,7 @@ def get_reservation(self, obj):
 
     @extend_schema_field(OpenApiTypes.OBJECT)
     def get_cable(self, obj):
-        return _point_cable(obj)
+        return _point_cable(obj, self.context)
 
     def get_front_port_count(self, obj) -> int:
         return obj.front_ports.count()
@@ -3450,7 +3484,7 @@ def get_reservation(self, obj):
 
     @extend_schema_field(OpenApiTypes.OBJECT)
     def get_cable(self, obj):
-        return _point_cable(obj)
+        return _point_cable(obj, self.context)
 
     def validate(self, attrs):
         attrs = super().validate(attrs)
@@ -3508,14 +3542,14 @@ class _DevicePortSerializer(TaggableSerializerMixin, NumIdModelSerializer):
 
     @extend_schema_field(OpenApiTypes.OBJECT)
     def get_cable(self, obj):
-        return _point_cable(obj)
+        return _point_cable(obj, self.context)
 
     @extend_schema_field(OpenApiTypes.OBJECT)
     def get_reservation(self, obj):
         return _point_reservation(obj)
 
     def get_type_display(self, obj) -> str:
-        return obj.get_type_display() if obj.type else ""
+        return _choice_label(obj, "type") if obj.type else ""
 
 
 class ConsolePortSerializer(_DevicePortSerializer):
@@ -6445,7 +6479,7 @@ class CircuitTerminationSerializer(serializers.ModelSerializer):
 
     @extend_schema_field(CableMiniSerializer(allow_null=True))
     def get_cable(self, obj):
-        return _point_cable(obj)
+        return _point_cable(obj, self.context)
 
     @extend_schema_field(OpenApiTypes.OBJECT)
     def get_connected_to(self, obj):

api/viewsets.py

@@ -1605,6 +1605,7 @@ class IPAddressViewSet(FieldWriteAllowList, CloneableMixin, TenantScopedViewSet)
             # assigned_vm: is_primary_for_vm reads the VM's primary_ip_id, so
             # without the join every VM-assigned row lazy-loads its VM (#122).
             "assigned_vm", "prefix__vrf", "prefix__site", "site",
+            "assigned_interface__device",
         )
         .prefetch_related("tags")
         .all()
@@ -2529,6 +2530,26 @@ def _after_save(self, obj):
         pass
 
 
+def _status_usage_expr():
+    """Sum of every relation StatusSerializer counts, as correlated subqueries,
+    so a list of N statuses is one statement instead of 14 COUNT(*) per row."""
+    from django.db.models import Count, IntegerField, OuterRef, Subquery, Value
+    from django.db.models.functions import Coalesce
+
+    from .serializers import StatusSerializer
+
+    expr = Value(0)
+    for rn in StatusSerializer._USAGE_RELS:
+        rel = Status._meta.get_field(rn)
+        fk = rel.field.name
+        sub = (
+            rel.related_model._default_manager.filter(**{fk: OuterRef("pk")})
+            .order_by().values(fk).annotate(c=Count("pk")).values("c")
+        )
+        expr = expr + Coalesce(Subquery(sub, output_field=IntegerField()), Value(0))
+    return expr
+
+
 class StatusViewSet(_IpCatalogViewSet):
     queryset = Status.objects.all().order_by("weight", "name")
     serializer_class = StatusSerializer
@@ -2543,7 +2564,7 @@ def get_queryset(self):
             avail = self.request.query_params.get("available_to")
             if avail:
                 qs = qs.filter(available_to__contains=[avail])
-        return qs
+        return qs.annotate(usage_count_annotated=_status_usage_expr())
 
     def _after_save(self, obj):
         # At most one default status per (tenant, object-type): strip each slug
@@ -4218,7 +4239,7 @@ def normalize_bulk_updates(self, updates):
             "parent__device", "lag__device", "bridge__device",
         )
         .prefetch_related(
-            "tags", "terminations__cable", "reservations", "ip_addresses", "children",
+            "tags", "terminations__cable__status", "reservations", "ip_addresses", "children",
             "lag_members", "tagged_vlans", "mac_addresses",
             "tunnel_terminations__tunnel",
             *FAR_END_PREFETCH,
@@ -4545,7 +4566,7 @@ def perform_update(self, serializer):
 
 class CableViewSet(TenantScopedViewSet):
     queryset = (
-        Cable.objects.prefetch_related(
+        Cable.objects.select_related("status").prefetch_related(
             "terminations__interface__device",
             "terminations__front_port__device",
             "terminations__rear_port__device",
@@ -4938,7 +4959,7 @@ def create(self, request):
 class RearPortViewSet(_DevicePortViewSet):
     queryset = (
         RearPort.objects.select_related("device")
-        .prefetch_related("tags", "terminations__cable", "reservations", "front_ports")
+        .prefetch_related("tags", "terminations__cable__status", "reservations", "front_ports")
         .order_by("device__name", NATURAL_NAME)
     )
     serializer_class = RearPortSerializer
@@ -4949,7 +4970,7 @@ class RearPortViewSet(_DevicePortViewSet):
 class FrontPortViewSet(_DevicePortViewSet):
     queryset = (
         FrontPort.objects.select_related("device", "rear_port")
-        .prefetch_related("tags", "terminations__cable", "reservations")
+        .prefetch_related("tags", "terminations__cable__status", "reservations")
         .order_by("device__name", NATURAL_NAME)
     )
     serializer_class = FrontPortSerializer
@@ -4959,7 +4980,7 @@ class FrontPortViewSet(_DevicePortViewSet):
 class ConsolePortViewSet(_DevicePortViewSet):
     queryset = (
         ConsolePort.objects.select_related("device")
-        .prefetch_related("tags", "terminations__cable", "reservations")
+        .prefetch_related("tags", "terminations__cable__status", "reservations")
         .order_by("device__name", NATURAL_NAME)
     )
     serializer_class = ConsolePortSerializer
@@ -4991,7 +5012,7 @@ class AntennaViewSet(_DevicePortViewSet):
 class ConsoleServerPortViewSet(_DevicePortViewSet):
     queryset = (
         ConsoleServerPort.objects.select_related("device")
-        .prefetch_related("tags", "terminations__cable", "reservations")
+        .prefetch_related("tags", "terminations__cable__status", "reservations")
         .order_by("device__name", NATURAL_NAME)
     )
     serializer_class = ConsoleServerPortSerializer
@@ -5001,7 +5022,7 @@ class ConsoleServerPortViewSet(_DevicePortViewSet):
 class PowerPortViewSet(_DevicePortViewSet):
     queryset = (
         PowerPort.objects.select_related("device")
-        .prefetch_related("tags", "terminations__cable", "reservations", "outlets")
+        .prefetch_related("tags", "terminations__cable__status", "reservations", "outlets")
         .order_by("device__name", NATURAL_NAME)
     )
     serializer_class = PowerPortSerializer
@@ -5010,7 +5031,7 @@ class PowerPortViewSet(_DevicePortViewSet):
 class PowerOutletViewSet(_DevicePortViewSet):
     queryset = (
         PowerOutlet.objects.select_related("device", "power_port")
-        .prefetch_related("tags", "terminations__cable", "reservations")
+        .prefetch_related("tags", "terminations__cable__status", "reservations")
         .order_by("device__name", NATURAL_NAME)
     )
     serializer_class = PowerOutletSerializer
@@ -5535,7 +5556,7 @@ def get_serializer_class(self):
 
 
 class VirtualMachineViewSet(CloneableMixin, TenantScopedViewSet):
-    queryset = VirtualMachine.objects.all().order_by(NATURAL_NAME)
+    queryset = VirtualMachine.objects.select_related("status").order_by(NATURAL_NAME)
     serializer_class = VirtualMachineSerializer
 
     def perform_create(self, serializer):
@@ -5686,8 +5707,8 @@ def get_queryset(self):
         qs = (
             super()
             .get_queryset()
-            .select_related("vm")
-            .prefetch_related("tags", "ip_addresses")
+            .select_related("vm__status", "vlan", "vrf")
+            .prefetch_related("tags", "ip_addresses", "tagged_vlans")
         )
         if self.request:
             vm = self.request.query_params.get("vm")

auth_api/api.py

@@ -492,7 +492,7 @@ class Meta:
 class ObjectPermissionViewSet(viewsets.ModelViewSet):
     queryset = (
         ObjectPermission.objects.all()
-        .prefetch_related("groups", "users", "tenants")
+        .prefetch_related("groups", "users", "tenants", "sites")
         .order_by("name")
     )
     serializer_class = ObjectPermissionSerializer

auth_api/apps.py

@@ -8,3 +8,20 @@ class AuthApiConfig(AppConfig):
     def ready(self):
         # Register the drf-spectacular auth extension for our token scheme.
         from . import schema  # noqa: F401
+
+        # A grant, its scope or a user's groups changing mid-request must not
+        # be answered from the per-request RBAC memo.
+        from django.contrib.auth import get_user_model
+        from django.db.models.signals import m2m_changed, post_delete, post_save
+
+        from .models import ObjectPermission
+        from .rbac import invalidate_request_cache
+
+        for sig in (post_save, post_delete):
+            sig.connect(invalidate_request_cache, sender=ObjectPermission,
+                        dispatch_uid=f"rbac-memo-{sig}")
+        for through in (ObjectPermission.users.through, ObjectPermission.groups.through,
+                        ObjectPermission.tenants.through, ObjectPermission.sites.through,
+                        get_user_model().groups.through):
+            m2m_changed.connect(invalidate_request_cache, sender=through,
+                                dispatch_uid=f"rbac-memo-m2m-{through.__name__}")

auth_api/rbac.py

@@ -8,6 +8,7 @@
 """
 from __future__ import annotations
 
+import contextvars
 import logging
 
 from django.core.exceptions import FieldError
@@ -20,6 +21,36 @@ def _is_super(user) -> bool:
     return bool(getattr(user, "is_authenticated", False) and user.is_superuser)
 
 
+# Per-request memo for applicable_permissions(). Set by RequestCacheMiddleware
+# for the duration of one HTTP request only: background jobs, management
+# commands and WebSocket consumers (which keep one user object for the life of
+# the socket) never see a cache, so a revoked grant can't linger there.
+_request_cache: contextvars.ContextVar = contextvars.ContextVar(
+    "rbac_request_cache", default=None
+)
+
+
+class RequestCacheMiddleware:
+    """Scope the RBAC permission memo to a single request."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        token = _request_cache.set({})
+        try:
+            return self.get_response(request)
+        finally:
+            _request_cache.reset(token)
+
+
+def invalidate_request_cache(*args, **kwargs):
+    """Drop the memo when a grant or group membership changes mid-request."""
+    cache = _request_cache.get()
+    if cache is not None:
+        cache.clear()
+
+
 def applicable_permissions(user, tenant):
     """Enabled ObjectPermissions that apply to ``user`` in ``tenant``.
 
@@ -30,6 +61,13 @@ def applicable_permissions(user, tenant):
 
     if not getattr(user, "is_authenticated", False):
         return ObjectPermission.objects.none()
+    # has_action/row_filter/effective_actions each call this; without the memo
+    # one /api/me/ of a site-scoped user ran this query and its two prefetches
+    # 53 times.
+    cache = _request_cache.get()
+    key = (user.pk, tenant.pk if tenant is not None else None)
+    if cache is not None and key in cache:
+        return list(cache[key])
     qs = (
         ObjectPermission.objects.filter(enabled=True)
         .filter(Q(users=user) | Q(groups__in=user.groups.all()))
@@ -42,6 +80,8 @@ def applicable_permissions(user, tenant):
         if scoped and (tenant is None or tenant.pk not in {t.pk for t in scoped}):
             continue
         out.append(perm)
+    if cache is not None:
+        cache[key] = tuple(out)
     return out
 
 

danbyte/settings.py

@@ -155,6 +155,7 @@
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "auth_api.rbac.RequestCacheMiddleware",
     # Rolling idle-session timeout (admin-configurable; no-op when unset). After
     # auth so request.user is resolved.
     "core.middleware.SessionIdleTimeoutMiddleware",