From ca02ab2563da5a867c4618d1fda53fb4864da52a Mon Sep 17 00:00:00 2001
From: iska <iska@iskamag.com>
Date: Fri, 8 May 2026 02:36:44 +0500
Subject: [PATCH 01/11] x86-64: cpu-local allocation

---
 runtime/allocate.lisp       |  83 ++++++++++++++++-------
 runtime/runtime-x86-64.lisp | 132 ++++++++++++++++++++++++++++++------
 supervisor/x86-64/cpu.lisp  |   4 +-
 system/gc.lisp              |   8 ++-
 4 files changed, 181 insertions(+), 46 deletions(-)

diff --git a/runtime/allocate.lisp b/runtime/allocate.lisp
index d9a8b8994..2d96ddd1f 100644
--- a/runtime/allocate.lisp
+++ b/runtime/allocate.lisp
@@ -55,6 +55,10 @@
 
 (sys.int::defglobal *bytes-consed*)
 
+
+(defconstant sys.int::+tlab-size+ (* 256 1024))
+(defconstant sys.int::+tlab-object-size-limit+ (* 64 1024))
+
 (defvar *maximum-allocation-attempts* 5
   "GC this many times before giving up on an allocation.")
 
@@ -437,7 +441,31 @@
              (mezzano.supervisor:debug-print-line "A-M-R failed."))
            nil))))
 
+(defun %do-get-new-tlab ()
+  (let* ((size (* sys.int::+tlab-size+ 8))
+         (old-bump (sys.int::%atomic-fixnum-add-symbol 'sys.int::*general-area-young-gen-bump*
+                                                        size))
+         (new-bump (+ old-bump size)))
+    (and (<= new-bump sys.int::*general-area-young-gen-limit*)
+         old-bump)))
+
+(defun %get-new-tlab (words)
+  (when (or (zerop sys.int::*general-area-young-gen-bump*)
+            (> words sys.int::+tlab-object-size-limit+))
+    (return-from %get-new-tlab nil))
+  (let ((result (%do-get-new-tlab)))
+    (when result
+      (let ((cpu (mezzano.supervisor::local-cpu))
+            (limit (+ result (* sys.int::+tlab-size+ 8))))
+        (setf (mezzano.supervisor::cpu-tlab-bump cpu) result
+              (mezzano.supervisor::cpu-tlab-limit cpu) limit)
+        t))))
+
 (defun %slow-allocate-from-general-area (tag data words)
+    ;; TLAB ran out. Try getting a new one.
+  (when (%get-new-tlab words)
+    (return-from %slow-allocate-from-general-area
+      (%allocate-from-general-area tag data words)))
   (let ((gc-count 0)
         (start-time (mezzano.supervisor:get-high-precision-timer)))
     (tagbody
@@ -446,30 +474,33 @@
          (mezzano.supervisor:inhibit-thread-pool-blocking-hijack
            (mezzano.supervisor:with-mutex (*allocator-lock*)
              (mezzano.supervisor:with-pseudo-atomic
-               (tagbody
+	       (tagbody
                 INNER-LOOP
                   (multiple-value-bind (result ignore1 ignore2 failurep)
-                      (%do-allocate-from-general-area tag data words)
+		      (%do-slow-allocate-from-general-area tag data words)
                     (declare (ignore ignore1 ignore2))
                     (when (not failurep)
-                      (update-allocation-time start-time)
-                      (return-from %slow-allocate-from-general-area
+		      (update-allocation-time start-time)
+		      ;; (mezzano.supervisor::debug-print-line "what")
+		      (return-from %slow-allocate-from-general-area
                         result)))
-                  ;; No memory. If there's memory available, then expand the area, otherwise run the GC.
-                  ;; Running the GC cannot be done when pseudo-atomic.
-                  (cond ((expand-allocation-area :general
-                                                 (* words 8)
-                                                 '*general-area-expansion-granularity*
-                                                 'sys.int::*general-area-young-gen-limit*
-                                                 sys.int::+address-tag-general+)
-                         ;; Successfully expanded the area. Retry the allocation.
-                         (go INNER-LOOP))
-                        (t
-                         ;; No memory do expand, bail out and run the GC.
-                         ;; This cannot be done when pseudo-atomic.
-                         (when sys.int::*gc-enable-logging*
-                           (mezzano.supervisor:debug-print-line "General area expansion failed, performing GC."))
-                         (go DO-GC))))))))
+                  (cond 
+		    ;; No memory. If there's memory available, then expand the area, otherwise run the GC.
+		    ;; Running the GC cannot be done when pseudo-atomic.
+		    ((expand-allocation-area :general
+                                             (* sys.int::+tlab-size+ 8)
+					     ;; (* words 8)
+                                             '*general-area-expansion-granularity*
+                                             'sys.int::*general-area-young-gen-limit*
+                                             sys.int::+address-tag-general+)
+                     ;; Successfully expanded the area. Retry the allocation.
+                     (go INNER-LOOP))
+                    (t
+                     ;; No memory do expand, bail out and run the GC.
+                     ;; This cannot be done when pseudo-atomic.
+                     (when sys.int::*gc-enable-logging*
+		       (mezzano.supervisor:debug-print-line "General area expansion failed, performing GC."))
+                     (go DO-GC))))))))
      DO-GC
        ;; Must occur outside the locks.
        (when (> gc-count *maximum-allocation-attempts*)
@@ -486,12 +517,13 @@
     (when (oddp words)
       (incf words))
     (let ((bytes (* words 8)))
-      (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* bytes)
+      ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* bytes)
       ;; ### This won't accurately track if the thread gets footholded
       ;; partway through the add...
       (incf (mezzano.supervisor:thread-bytes-consed
              (mezzano.supervisor:current-thread))
             bytes))
+    ;; (mezzano.supervisor:debug-print-line "b " (mezzano.supervisor::cpu-tlab-bump (mezzano.supervisor::local-cpu)))
     (ecase area
       ((nil)
        (%allocate-from-general-area tag data words))
@@ -508,11 +540,11 @@
      (cons car cdr))
     (:pinned
      (log-allocation-profile-entry 2)
-     (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32)
+     ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32)
      (%cons-in-pinned-area car cdr))
     (:wired
      (log-allocation-profile-entry 2)
-     (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32)
+     ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32)
      (%cons-in-wired-area car cdr))))
 
 #-(or x86-64 arm64)
@@ -691,7 +723,7 @@
      for i from 0 do
        (let ((result (%allocate-function-1 tag data words wiredp)))
          (when result
-           (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* (* words 8))
+           ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* (* words 8))
            (update-allocation-time start-time)
            (return result)))
        (when (not (eql i 0))
@@ -886,7 +918,8 @@ This area exists below the stack and is never allocated or mapped.")
      :finalizer (lambda ()
                   (when stack-address
                     (release-memory-range stack-address size)
-                    (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* (- size))))
+                    ;; (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* (- size))
+		    ))
      :area :wired)
     (tagbody
      RETRY
@@ -920,7 +953,7 @@ This area exists below the stack and is never allocated or mapped.")
                     (if wired
                         (setf sys.int::*wired-stack-area-bump* (align-up (+ bump size) +stack-region-alignment+))
                         (setf sys.int::*stack-area-bump* (align-up (+ bump size) +stack-region-alignment+)))
-                    (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* size)
+                    ;; (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* size)
                     (setf (stack-base stack) addr
                           ;; Notify the finalizer that the stack has been allocated & should be freed.
                           stack-address addr)
diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp
index cb9346b70..d82902810 100644
--- a/runtime/runtime-x86-64.lisp
+++ b/runtime/runtime-x86-64.lisp
@@ -502,9 +502,8 @@ the GC must be deferred during FILL-WORDS."
   ;; Build bignum.
   (sys.lap-x86:jmp (:named-call sys.int::%%make-bignum-64-rax)))
 
-;; This relies on memory being initialized to zero, so it looks like
-;; many simple vectors of length 0.
-(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words))
+
+(sys.int::define-lap-function %do-slow-allocate-from-general-area ((tag data words))
   (:gc :no-frame :layout #*0)
   ;; Attempt to quickly allocate from the general area.
   ;; Returns (values tag data words t) on failure, just the object on success.
@@ -557,6 +556,64 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:mov32 :ecx #.(ash 4 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret))
 
+
+
+;; This relies on memory being initialized to zero, so it looks like
+;; many simple vectors of length 0.
+(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words))
+  (:gc :no-frame :layout #*0)
+  ;; Attempt to quickly allocate from the general area.
+  ;; Returns (values tag data words t) on failure, just the object on success.
+  ;; R8 = tag; R9 = data; R10 = words.
+  ;; Fetch symbol value cells.
+  (sys.lap-x86:mov64 :r12 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
+  (sys.lap-x86:fs) (sys.lap-x86:mov64 :r11 (:object-location nil #.mezzano.supervisor::+cpu-tlab-limit+)) ; Allocation limit for the CPU
+
+  ;; R12 = newspace-bit. R11 = limit.
+  ;; Assemble the final header value in RDI.
+  (sys.lap-x86:mov64 :rdi :r9)
+  (sys.lap-x86:shl64 :rdi #.(- sys.int::+object-data-shift+ sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:lea64 :rdi (:rdi (:r8 #.(ash 1 (- sys.int::+object-type-shift+ sys.int::+n-fixnum-bits+)))))
+  ;; If a garbage collection occurs, it must rewind IP back here.
+  (:gc :no-frame :layout #*0 :restart t)
+  ;; Fetch and increment the current bump pointer.
+  (sys.lap-x86:lea64 :rbx ((:r10 8))) ; words * 8
+  (sys.lap-x86:fs) (sys.lap-x86:xadd64 (:object-location nil #.mezzano.supervisor::+cpu-tlab-bump+) :rbx) ; We're incrementing the slot in local-cpu's struct, hence the fs
+  ;; RBX is old bump pointer, the address of the cons.
+  ;; Find the new bump pointer.
+  (sys.lap-x86:lea64 :rsi (:rbx (:r10 8)))
+    ;; Test against limit.
+  (sys.lap-x86:cmp64 :rsi :r11)
+  (sys.lap-x86:ja SLOW-PATH)
+  (sys.lap-x86:shr64 :rsi #.sys.int::+n-fixnum-bits+)
+  ;; Generate the object.
+  ;; Unfixnumize address. This still looks like a fixnum due to alignment.
+  (sys.lap-x86:shr64 :rbx #.sys.int::+n-fixnum-bits+)
+  ;; Set address bits and the tag bits.
+  ;; Set address bits, tag bits, and the mark bit.
+  (sys.lap-x86:mov64 :rax #.(logior (ash sys.int::+address-tag-general+ sys.int::+address-tag-shift+)
+                                    sys.int::+tag-object+))
+  (sys.lap-x86:or64 :rax (:object :r12 #.sys.int::+symbol-value-cell-value+))
+  (sys.lap-x86:or64 :rbx :rax)
+  ;; RBX now points to a 0-element simple-vector, followed by however much empty space is required.
+  ;; The gc metadata at this point has :restart t, so if a GC occurs before
+  ;; writing the final header, this process will be restarted from the beginning.
+  ;; This is required as the GC will only copy 2 words, leaving the rest of the memory in an invalid state.
+  ;; Write back the header.
+  ;; This must be done in a single write so the GC always sees a correct header.
+  (sys.lap-x86:mov64 (:object :rbx -1) :rdi)
+  (sys.lap-x86:prefetcht0 (:rsi))
+  ;; Leave restart region.
+  (:gc :no-frame :layout #*0)
+  ;; Done. Return everything.
+  (sys.lap-x86:mov64 :r8 :rbx)
+  (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:ret)
+  SLOW-PATH
+  (sys.lap-x86:mov64 :r11 t)
+  (sys.lap-x86:mov32 :ecx #.(ash 4 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:ret))
+
 (sys.int::define-lap-function %allocate-from-general-area ((tag data words))
   (:gc :no-frame :layout #*0 :incoming-arguments :rcx)
   ;; Attempt to quickly allocate from the general area. Will call
@@ -566,19 +623,56 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:cmp64 :rcx #.(ash 3 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:jne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*0)
+  ;; Check if object is too large.
+  (sys.lap-x86:mov64 :r11 (:symbol-global-cell sys.int::+tlab-object-size-limit+))
+  (sys.lap-x86:mov64 :r11 (:object :r11 #.sys.int::+symbol-value-cell-value+))
+  (sys.lap-x86:shr64 :r11 #.sys.int::+n-fixnum-bits+)
+  (sys.lap-x86:mov64 :rax :r10)
+  (sys.lap-x86:shr64 :rax #.sys.int::+n-fixnum-bits+)
+  (sys.lap-x86:cmp64 :rax :r11)
+  (sys.lap-x86:ja TOO-LARGE)
   ;; Update allocation meter.
   ;; *BYTES-CONSED* is updated elsewhere.
-  (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-allocation-count*))
-  (sys.lap-x86:lock)
-  (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-allocation-count*))
+  ;; (sys.lap-x86:lock)
+  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
   ;; Try the real fast allocator.
   (sys.lap-x86:call (:named-call %do-allocate-from-general-area))
   (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:jne SLOW-PATH)
   ;; Done. Return everything.
-  (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-fast-path-hits*))
-  (sys.lap-x86:lock)
-  (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-fast-path-hits*))
+  ;; (sys.lap-x86:lock)
+  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:ret)
+  TOO-LARGE
+  (sys.lap-x86:call (:named-call %do-slow-allocate-from-general-area))
+  (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:jne SLOW-PATH)
+  (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:ret)
+  SLOW-PATH
+  ;; Tail call into %SLOW-ALLOCATE-FROM-GENERAL-AREA.
+  (sys.lap-x86:mov32 :ecx #.(ash 3 #.sys.int::+n-fixnum-bits+))
+  SLOW-PATH-BAD-ARGS
+  (:gc :no-frame :layout #*0 :incoming-arguments :rcx)
+  (sys.lap-x86:jmp (:named-call %slow-allocate-from-general-area)))
+
+(sys.int::define-lap-function %old-allocate-from-general-area ((tag data words))
+  (:gc :no-frame :layout #*0 :incoming-arguments :rcx)
+  ;; Attempt to quickly allocate from the general area. Will call
+  ;; %SLOW-ALLOCATE-FROM-GENERAL-AREA if things get too hairy.
+  ;; R8 = tag; R9 = data; R10 = words
+  ;; Check argument count.
+  (sys.lap-x86:cmp64 :rcx #.(ash 3 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:jne SLOW-PATH-BAD-ARGS)
+  (:gc :no-frame :layout #*0)
+  ;; Try the real fast allocator.
+  (sys.lap-x86:call (:named-call %do-slow-allocate-from-general-area))
+  (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:jne SLOW-PATH)
+  ;; Done. Return everything.
   (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret)
   SLOW-PATH
@@ -641,13 +735,13 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:jne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*0)
   ;; Update allocation meter.
-  (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-allocation-count*))
-  (sys.lap-x86:lock)
-  (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
-  (sys.lap-x86:mov64 :rbx (:symbol-global-cell *bytes-consed*))
-  (sys.lap-x86:lock)
-  (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+)
-                     #.(ash 16 sys.int::+n-fixnum-bits+))
+  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-allocation-count*))
+  ;; (sys.lap-x86:lock)
+  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *bytes-consed*))
+  ;; (sys.lap-x86:lock)
+  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+)
+  ;;                    #.(ash 16 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:gs)
   (sys.lap-x86:add64 (:object nil #.mezzano.supervisor::+thread-bytes-consed+)
                      #.(ash 16 sys.int::+n-fixnum-bits+))
@@ -664,9 +758,9 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:jne SLOW-PATH)
   ;; Done. Return everything.
-  (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-fast-path-hits*))
-  (sys.lap-x86:lock)
-  (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-fast-path-hits*))
+  ;; (sys.lap-x86:lock)
+  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret)
   SLOW-PATH
diff --git a/supervisor/x86-64/cpu.lisp b/supervisor/x86-64/cpu.lisp
index b00519b66..b998706ec 100644
--- a/supervisor/x86-64/cpu.lisp
+++ b/supervisor/x86-64/cpu.lisp
@@ -112,7 +112,9 @@
   irq-stack
   page-fault-stack
   lapic-timer-active
-  page-fault-hook)
+  page-fault-hook
+  (tlab-bump 0)
+  (tlab-limit 0))
 
 (defconstant +ap-trampoline-physical-address+ #x7000
   "Where the AP trampoline should be copied to in physical memory.
diff --git a/system/gc.lisp b/system/gc.lisp
index 6694ea9e5..85c596945 100644
--- a/system/gc.lisp
+++ b/system/gc.lisp
@@ -165,7 +165,8 @@ If FULL is true, then a major collection will be forced."
                        (total-time (- gc-end gc-start))
                        (total-seconds (/ total-time (float internal-time-units-per-second))))
                   (gc-log "GC took " (truncate (* total-seconds 1000)) "ms")
-                  (incf *gc-time* total-seconds)))
+                  (incf *gc-time* total-seconds))
+		(fixup-tlabs))
            (setf *gc-in-progress* nil)))
        ;; TODO: catch & report errors.
        (run-finalizers))))
@@ -2473,3 +2474,8 @@ No type information will be provided."
          ;; Leave the weak pointer completely empty.
          ;; No references to any other object.
          (setf (%object-ref-t finalizer +weak-pointer-finalizer+) nil))))
+
+(defun fixup-tlabs ()
+  (dolist (cpu mezzano.supervisor::*cpus*)
+    (setf (mezzano.supervisor::cpu-tlab-limit cpu) 0
+	  (mezzano.supervisor::cpu-tlab-bump cpu) 0)))

From 557513afe548da0de50b69a9244428bf33e3d406 Mon Sep 17 00:00:00 2001
From: iska <iska@iskamag.com>
Date: Fri, 8 May 2026 03:14:04 +0500
Subject: [PATCH 02/11] remove dead code

---
 runtime/runtime-x86-64.lisp | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp
index d82902810..50166db34 100644
--- a/runtime/runtime-x86-64.lisp
+++ b/runtime/runtime-x86-64.lisp
@@ -659,29 +659,6 @@ the GC must be deferred during FILL-WORDS."
   (:gc :no-frame :layout #*0 :incoming-arguments :rcx)
   (sys.lap-x86:jmp (:named-call %slow-allocate-from-general-area)))
 
-(sys.int::define-lap-function %old-allocate-from-general-area ((tag data words))
-  (:gc :no-frame :layout #*0 :incoming-arguments :rcx)
-  ;; Attempt to quickly allocate from the general area. Will call
-  ;; %SLOW-ALLOCATE-FROM-GENERAL-AREA if things get too hairy.
-  ;; R8 = tag; R9 = data; R10 = words
-  ;; Check argument count.
-  (sys.lap-x86:cmp64 :rcx #.(ash 3 #.sys.int::+n-fixnum-bits+))
-  (sys.lap-x86:jne SLOW-PATH-BAD-ARGS)
-  (:gc :no-frame :layout #*0)
-  ;; Try the real fast allocator.
-  (sys.lap-x86:call (:named-call %do-slow-allocate-from-general-area))
-  (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
-  (sys.lap-x86:jne SLOW-PATH)
-  ;; Done. Return everything.
-  (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
-  (sys.lap-x86:ret)
-  SLOW-PATH
-  ;; Tail call into %SLOW-ALLOCATE-FROM-GENERAL-AREA.
-  (sys.lap-x86:mov32 :ecx #.(ash 3 #.sys.int::+n-fixnum-bits+))
-  SLOW-PATH-BAD-ARGS
-  (:gc :no-frame :layout #*0 :incoming-arguments :rcx)
-  (sys.lap-x86:jmp (:named-call %slow-allocate-from-general-area)))
-
 (sys.int::define-lap-function do-cons ((car cdr))
   (:gc :no-frame :layout #*0)
   ;; Attempt to quickly allocate a cons.

From 72ec55da75f61ef6801d28c41202980437f24cc5 Mon Sep 17 00:00:00 2001
From: iska <iska@iskamag.com>
Date: Fri, 8 May 2026 12:40:13 +0500
Subject: [PATCH 03/11] allocate: move cons/general counters from global
 atomics to per-cpu fields

---
 runtime/allocate.lisp       | 64 ++++++++++++++++++++++---------------
 runtime/runtime-arm64.lisp  | 32 ++++++++-----------
 runtime/runtime-x86-64.lisp | 28 +++++++---------
 supervisor/arm64/cpu.lisp   |  7 +++-
 supervisor/x86-64/cpu.lisp  |  7 +++-
 5 files changed, 74 insertions(+), 64 deletions(-)

diff --git a/runtime/allocate.lisp b/runtime/allocate.lisp
index 2d96ddd1f..3f50fe042 100644
--- a/runtime/allocate.lisp
+++ b/runtime/allocate.lisp
@@ -48,12 +48,25 @@
 (sys.int::defglobal *general-area-expansion-granularity*)
 (sys.int::defglobal *cons-area-expansion-granularity*)
 
-(sys.int::defglobal *general-fast-path-hits*)
-(sys.int::defglobal *general-allocation-count*)
-(sys.int::defglobal *cons-fast-path-hits*)
-(sys.int::defglobal *cons-allocation-count*)
+(defun bytes-consed ()
+  (loop for cpu in mezzano.supervisor::*cpus*
+        sum (mezzano.supervisor::cpu-bytes-consed cpu)))
 
-(sys.int::defglobal *bytes-consed*)
+(defun general-allocation-count ()
+  (loop for cpu in mezzano.supervisor::*cpus*
+        sum (mezzano.supervisor::cpu-general-allocation-count cpu)))
+
+(defun general-fast-path-hits ()
+  (loop for cpu in mezzano.supervisor::*cpus*
+        sum (mezzano.supervisor::cpu-general-fast-path-hits cpu)))
+
+(defun cons-allocation-count ()
+  (loop for cpu in mezzano.supervisor::*cpus*
+        sum (mezzano.supervisor::cpu-cons-allocation-count cpu)))
+
+(defun cons-fast-path-hits ()
+  (loop for cpu in mezzano.supervisor::*cpus*
+        sum (mezzano.supervisor::cpu-cons-fast-path-hits cpu)))
 
 
 (defconstant sys.int::+tlab-size+ (* 256 1024))
@@ -94,11 +107,6 @@
         *enable-allocation-profiling* nil
         *general-area-expansion-granularity* sys.int::+allocation-minimum-alignment+
         *cons-area-expansion-granularity* sys.int::+allocation-minimum-alignment+
-        *general-fast-path-hits* 0
-        *general-allocation-count* 0
-        *cons-fast-path-hits* 0
-        *cons-allocation-count* 0
-        *bytes-consed* 0
         *allocator-lock* (mezzano.supervisor:make-mutex "Allocator")
         *allocation-fudge* (* 8 1024 1024)
         sys.int::*generation-size-ratio* 2)
@@ -336,7 +344,7 @@
 
 #-(or x86-64 arm64)
 (defun %allocate-from-general-area (tag data words)
-  (sys.int::%atomic-fixnum-add-symbol '*general-allocation-count* 1)
+  (incf (mezzano.supervisor::cpu-general-allocation-count (mezzano.supervisor::local-cpu)) 1)
   (%slow-allocate-from-general-area tag data words))
 
 #-(or x86-64 arm64)
@@ -517,9 +525,8 @@
     (when (oddp words)
       (incf words))
     (let ((bytes (* words 8)))
-      ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* bytes)
-      ;; ### This won't accurately track if the thread gets footholded
-      ;; partway through the add...
+      (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu))
+            bytes)
       (incf (mezzano.supervisor:thread-bytes-consed
              (mezzano.supervisor:current-thread))
             bytes))
@@ -539,18 +546,21 @@
     ((nil)
      (cons car cdr))
     (:pinned
-     (log-allocation-profile-entry 2)
-     ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32)
-     (%cons-in-pinned-area car cdr))
+      (log-allocation-profile-entry 2)
+      (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) 32)
+      (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) 32)
+      (%cons-in-pinned-area car cdr))
     (:wired
-     (log-allocation-profile-entry 2)
-     ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32)
-     (%cons-in-wired-area car cdr))))
+      (log-allocation-profile-entry 2)
+      (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) 32)
+      (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) 32)
+      (%cons-in-wired-area car cdr))))
 
 #-(or x86-64 arm64)
 (defun cons (car cdr)
-  (sys.int::%atomic-fixnum-add-symbol '*cons-allocation-count* 1)
-  (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 16)
+  (incf (mezzano.supervisor::cpu-cons-allocation-count (mezzano.supervisor::local-cpu)) 1)
+  (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) 16)
+  (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) 16)
   (slow-cons car cdr))
 
 #-(or x86-64 arm64)
@@ -722,10 +732,12 @@
      with inhibit-gc = nil
      for i from 0 do
        (let ((result (%allocate-function-1 tag data words wiredp)))
-         (when result
-           ;; (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* (* words 8))
-           (update-allocation-time start-time)
-           (return result)))
+           (when result
+             (let ((cbytes (* words 8)))
+               (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) cbytes)
+               (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) cbytes))
+             (update-allocation-time start-time)
+            (return result)))
        (when (not (eql i 0))
          ;; The GC has been run at least once.
          (when (expand-function-area words wiredp)
diff --git a/runtime/runtime-arm64.lisp b/runtime/runtime-arm64.lisp
index dace691a3..ab74cc27d 100644
--- a/runtime/runtime-arm64.lisp
+++ b/runtime/runtime-arm64.lisp
@@ -406,12 +406,9 @@
   (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*00)
   ;; Update allocation meter.
-  ;; *BYTES-CONSED* is updated elsewhere.
-  (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *general-allocation-count*))
-  ;; FIXME: Should be atomic add.
-  (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
   ;; Check *ENABLE-ALLOCATION-PROFILING*
   ;; FIXME: This only tests the global value.
   (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *enable-allocation-profiling*))
@@ -428,11 +425,9 @@
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:b.ne SLOW-PATH)
   ;; Done. Return everything.
-  (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *general-fast-path-hits*))
-  ;; FIXME: Should be atomic add.
-  (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)
@@ -569,15 +564,15 @@
   (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*00)
   ;; Update allocation meter.
-  ;; FIXME: Should be atomic add.
-  (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *cons-allocation-count*))
-  (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
-  (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *bytes-consed*))
-  (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
+  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
+  (mezzano.lap.arm64:ldr :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+))
+  (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+))
+  (mezzano.lap.arm64:str :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+))
   ;; Check *ENABLE-ALLOCATION-PROFILING*
   ;; FIXME: This only tests the global value.
   #| Logging every cons tends to explode the profile buffer & exhaust memory.
@@ -596,10 +591,9 @@
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:b.ne SLOW-PATH)
   ;; Done. Return everything.
-  (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *cons-fast-path-hits*))
-  (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)
diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp
index 50166db34..02533d33e 100644
--- a/runtime/runtime-x86-64.lisp
+++ b/runtime/runtime-x86-64.lisp
@@ -632,18 +632,15 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:cmp64 :rax :r11)
   (sys.lap-x86:ja TOO-LARGE)
   ;; Update allocation meter.
-  ;; *BYTES-CONSED* is updated elsewhere.
-  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-allocation-count*))
-  ;; (sys.lap-x86:lock)
-  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:fs)
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-general-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+))
   ;; Try the real fast allocator.
   (sys.lap-x86:call (:named-call %do-allocate-from-general-area))
   (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:jne SLOW-PATH)
   ;; Done. Return everything.
-  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-fast-path-hits*))
-  ;; (sys.lap-x86:lock)
-  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:fs)
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-general-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret)
   TOO-LARGE
@@ -712,13 +709,11 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:jne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*0)
   ;; Update allocation meter.
-  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-allocation-count*))
-  ;; (sys.lap-x86:lock)
-  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
-  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *bytes-consed*))
-  ;; (sys.lap-x86:lock)
-  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+)
-  ;;                    #.(ash 16 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:fs)
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-cons-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:fs)
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-bytes-consed+)
+                     #.(ash 16 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:gs)
   (sys.lap-x86:add64 (:object nil #.mezzano.supervisor::+thread-bytes-consed+)
                      #.(ash 16 sys.int::+n-fixnum-bits+))
@@ -735,9 +730,8 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:jne SLOW-PATH)
   ;; Done. Return everything.
-  ;; (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-fast-path-hits*))
-  ;; (sys.lap-x86:lock)
-  ;; (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:fs)
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-cons-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret)
   SLOW-PATH
diff --git a/supervisor/arm64/cpu.lisp b/supervisor/arm64/cpu.lisp
index 8320487d2..bf3e8d0f8 100644
--- a/supervisor/arm64/cpu.lisp
+++ b/supervisor/arm64/cpu.lisp
@@ -18,7 +18,12 @@
   idle-thread
   wired-stack
   (sp-el1 0)
-  page-fault-hook)
+  page-fault-hook
+  (bytes-consed 0)
+  (cons-allocation-count 0)
+  (cons-fast-path-hits 0)
+  (general-allocation-count 0)
+  (general-fast-path-hits 0))
 
 (defun initialize-boot-cpu ()
   (setf (arm64-cpu-self *bsp-cpu*) *bsp-cpu*)
diff --git a/supervisor/x86-64/cpu.lisp b/supervisor/x86-64/cpu.lisp
index b998706ec..8b901f9e3 100644
--- a/supervisor/x86-64/cpu.lisp
+++ b/supervisor/x86-64/cpu.lisp
@@ -114,7 +114,12 @@
   lapic-timer-active
   page-fault-hook
   (tlab-bump 0)
-  (tlab-limit 0))
+  (tlab-limit 0)
+  (bytes-consed 0)
+  (cons-allocation-count 0)
+  (cons-fast-path-hits 0)
+  (general-allocation-count 0)
+  (general-fast-path-hits 0))
 
 (defconstant +ap-trampoline-physical-address+ #x7000
   "Where the AP trampoline should be copied to in physical memory.

From 337d92b6138a664e2e161890d5769c48ecfff2e3 Mon Sep 17 00:00:00 2001
From: iska <iska@iskamag.com>
Date: Fri, 8 May 2026 14:04:55 +0500
Subject: [PATCH 04/11] arm64: add TLAB support

---
 runtime/runtime-arm64.lisp | 78 ++++++++++++++++++++++++++++++++++++--
 supervisor/arm64/cpu.lisp  | 14 +++++++
 system/gc.lisp             |  2 +-
 3 files changed, 89 insertions(+), 5 deletions(-)

diff --git a/runtime/runtime-arm64.lisp b/runtime/runtime-arm64.lisp
index ab74cc27d..962706fcb 100644
--- a/runtime/runtime-arm64.lisp
+++ b/runtime/runtime-arm64.lisp
@@ -420,6 +420,13 @@
   (mezzano.lap.arm64:ldr :x4 (:object :x6 #.sys.int::+symbol-value-cell-value+))
   (mezzano.lap.arm64:subs :xzr :x4 :x26)
   (mezzano.lap.arm64:b.ne SLOW-PATH)
+  ;; Check if object exceeds the TLAB size limit.
+  (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell sys.int::+tlab-object-size-limit+))
+  (mezzano.lap.arm64:ldr :x4 (:object :x6 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:add :x4 :xzr :x4 :lsr #.sys.int::+n-fixnum-bits+)
+  (mezzano.lap.arm64:add :x6 :xzr :x2 :lsr #.sys.int::+n-fixnum-bits+)
+  (mezzano.lap.arm64:subs :xzr :x6 :x4)
+  (mezzano.lap.arm64:b.hi TOO-LARGE)
   ;; Try the real fast allocator.
   (mezzano.lap.arm64:named-call %do-allocate-from-general-area)
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
@@ -432,6 +439,16 @@
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)
   (mezzano.lap.arm64:ret)
+  TOO-LARGE
+  (:gc :no-frame :layout #*00)
+  ;; Object is too large for the TLAB. Use the global allocator directly.
+  (mezzano.lap.arm64:named-call %do-slow-allocate-from-general-area)
+  (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (mezzano.lap.arm64:b.ne SLOW-PATH)
+  (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
+  (:gc :no-frame :layout #*)
+  (mezzano.lap.arm64:ret)
   SLOW-PATH
   (:gc :no-frame :layout #*00)
   ;; Tail call into %SLOW-ALLOCATE-FROM-GENERAL-AREA.
@@ -441,9 +458,9 @@
   (:gc :no-frame :layout #* :incoming-arguments :rcx)
   (mezzano.lap.arm64:named-tail-call %slow-allocate-from-general-area))
 
-(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words))
+(sys.int::define-lap-function %do-slow-allocate-from-general-area ((tag data words))
   (:gc :no-frame :layout #*)
-  ;; Attempt to quickly allocate from the general area.
+  ;; Attempt to allocate from the global general area using atomic operations.
   ;; Returns (values tag data words t) on failure, just the object on success.
   ;; X0 = tag; X1 = data; X2 = words.
   ;; Fetch symbol value cells.
@@ -464,7 +481,7 @@
   ;; Release atomic add to increment the bump pointer
   (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9))
   (mezzano.lap.arm64:add :x11 :x6 :x10)
-  ;; X6 is old bump pointer, the address of the cons.
+  ;; X6 is old bump pointer, the address of the allocation.
   ;; X11 is the new bump pointer.
   ;; Test against limit.
   (mezzano.lap.arm64:ldr :x10 (:object :x3 #.sys.int::+symbol-value-cell-value+))
@@ -475,11 +492,64 @@
   (mezzano.lap.arm64:add :x6 :xzr :x6 :lsr #.sys.int::+n-fixnum-bits+)
   ;; Set address bits and the tag bits.
   ;; Set address bits, tag bits, and the mark bit.
+  (mezzano.lap.arm64:ldr :x9 (:pc global-address-object-tag))
+  (mezzano.lap.arm64:orr :x6 :x6 :x9)
+  (mezzano.lap.arm64:ldr :x9 (:object :x4 #.sys.int::+symbol-value-cell-value+))
+  (mezzano.lap.arm64:orr :x6 :x6 :x9)
+  ;; Write back the header.
+  (mezzano.lap.arm64:str :x12 (:object :x6 -1))
+  ;; Leave restart region.
+  (:gc :no-frame :layout #*)
+  ;; Done. Return everything.
+  (mezzano.lap.arm64:orr :x0 :xzr :x6)
+  (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
+  (mezzano.lap.arm64:ret)
+  SLOW-PATH
+  (mezzano.lap.arm64:ldr :x3 (:constant t))
+  (mezzano.lap.arm64:movz :x5 #.(ash 4 #.sys.int::+n-fixnum-bits+))
+  (mezzano.lap.arm64:ret)
+  (:align 16)
+  global-address-object-tag
+  (:d64/le #.(logior (ash sys.int::+address-tag-general+ sys.int::+address-tag-shift+)
+                     sys.int::+tag-object+)))
+
+(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words))
+  (:gc :no-frame :layout #*)
+  ;; Attempt to quickly allocate from the general area using the per-CPU TLAB.
+  ;; Returns (values tag data words t) on failure, just the object on success.
+  ;; X0 = tag; X1 = data; X2 = words.
+  ;; Load per-CPU TLAB limit and newspace bit.
+  (mezzano.lap.arm64:ldr :x3 (:object :x27 #.mezzano.supervisor::+arm64-cpu-tlab-limit+))
+  (mezzano.lap.arm64:ldr :x4 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
+  ;; X3 = tlab limit. X4 = newspace-bit.
+  ;; Assemble the final header value in X12.
+  (mezzano.lap.arm64:add :x12 :xzr :x0 :lsl #.(- sys.int::+object-type-shift+ sys.int::+n-fixnum-bits+))
+  (mezzano.lap.arm64:add :x12 :x12 :x1 :lsl #.(- sys.int::+object-data-shift+ sys.int::+n-fixnum-bits+))
+  ;; If a garbage collection occurs, it must rewind IP back here.
+  (:gc :no-frame :layout #* :restart t)
+  ;; Fetch and increment the per-CPU TLAB bump pointer.
+  (mezzano.lap.arm64:add :x10 :xzr :x2 :lsl 3) ; words * 8
+  ;; Address generation for the per-CPU tlab-bump slot.
+  ;; Linked GC mode is not needed as this will be repeated due to the restart region.
+  (mezzano.lap.arm64:add :x9 :x27 #.(+ (- sys.int::+tag-object+) 8 (* mezzano.supervisor::+arm64-cpu-tlab-bump+ 8)))
+  ;; Release atomic add to increment the bump pointer
+  (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9))
+  (mezzano.lap.arm64:add :x11 :x6 :x10)
+  ;; X6 is old bump pointer, the address of the allocation.
+  ;; X11 is the new bump pointer.
+  ;; Test against per-CPU TLAB limit.
+  (mezzano.lap.arm64:subs :xzr :x11 :x3)
+  (mezzano.lap.arm64:b.hi SLOW-PATH)
+  ;; Generate the object.
+  ;; Unfixnumize address. This still looks like a fixnum due to alignment.
+  (mezzano.lap.arm64:add :x6 :xzr :x6 :lsr #.sys.int::+n-fixnum-bits+)
+  ;; Set address bits and the tag bits.
+  ;; Set address bits, tag bits, and the mark bit.
   (mezzano.lap.arm64:ldr :x9 (:pc general-address-object-tag))
   (mezzano.lap.arm64:orr :x6 :x6 :x9)
   (mezzano.lap.arm64:ldr :x9 (:object :x4 #.sys.int::+symbol-value-cell-value+))
   (mezzano.lap.arm64:orr :x6 :x6 :x9)
-  ;; RBX now points to a 0-element simple-vector, followed by however much empty space is required.
+  ;; X6 now points to a 0-element simple-vector, followed by however much empty space is required.
   ;; The gc metadata at this point has :restart t, so if a GC occurs before
   ;; writing the final header, this process will be restarted from the beginning.
   ;; This is required as the GC will only copy 2 words, leaving the rest of the memory in an invalid state.
diff --git a/supervisor/arm64/cpu.lisp b/supervisor/arm64/cpu.lisp
index bf3e8d0f8..26db55799 100644
--- a/supervisor/arm64/cpu.lisp
+++ b/supervisor/arm64/cpu.lisp
@@ -19,6 +19,8 @@
   wired-stack
   (sp-el1 0)
   page-fault-hook
+  (tlab-bump 0)
+  (tlab-limit 0)
   (bytes-consed 0)
   (cons-allocation-count 0)
   (cons-fast-path-hits 0)
@@ -82,6 +84,18 @@
 (defun local-cpu ()
   (local-cpu-info))
 
+(defun cpu-tlab-bump (cpu)
+  (arm64-cpu-tlab-bump cpu))
+
+(defun (setf cpu-tlab-bump) (value cpu)
+  (setf (arm64-cpu-tlab-bump cpu) value))
+
+(defun cpu-tlab-limit (cpu)
+  (arm64-cpu-tlab-limit cpu))
+
+(defun (setf cpu-tlab-limit) (value cpu)
+  (setf (arm64-cpu-tlab-limit cpu) value))
+
 (defun initialize-cpu ()
   (setf (arm64-cpu-cpu-id *bsp-cpu*) (fdt-boot-cpuid))
   (push-wired *bsp-cpu* *cpus*))
diff --git a/system/gc.lisp b/system/gc.lisp
index 85c596945..f0acea042 100644
--- a/system/gc.lisp
+++ b/system/gc.lisp
@@ -2478,4 +2478,4 @@ No type information will be provided."
 (defun fixup-tlabs ()
   (dolist (cpu mezzano.supervisor::*cpus*)
     (setf (mezzano.supervisor::cpu-tlab-limit cpu) 0
-	  (mezzano.supervisor::cpu-tlab-bump cpu) 0)))
+          (mezzano.supervisor::cpu-tlab-bump cpu) 0)))

From 9fdd67333d1cec0d88cc3f03553f2afc1b23e184 Mon Sep 17 00:00:00 2001
From: iska <iska@iskamag.com>
Date: Fri, 15 May 2026 03:11:33 +0500
Subject: [PATCH 05/11] x86-64: prefetchnta is faster

---
 runtime/runtime-x86-64.lisp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp
index 02533d33e..3fa327dbf 100644
--- a/runtime/runtime-x86-64.lisp
+++ b/runtime/runtime-x86-64.lisp
@@ -602,7 +602,7 @@ the GC must be deferred during FILL-WORDS."
   ;; Write back the header.
   ;; This must be done in a single write so the GC always sees a correct header.
   (sys.lap-x86:mov64 (:object :rbx -1) :rdi)
-  (sys.lap-x86:prefetcht0 (:rsi))
+  (sys.lap-x86:prefetchnta (:rsi))
   ;; Leave restart region.
   (:gc :no-frame :layout #*0)
   ;; Done. Return everything.

From 23d71657dffb6aa31a4fe74841ed4c1cfa77a60c Mon Sep 17 00:00:00 2001
From: iska <iska@iskamag.com>
Date: Sat, 16 May 2026 18:16:12 +0500
Subject: [PATCH 06/11] allocate: indent properly

---
 runtime/allocate.lisp | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/runtime/allocate.lisp b/runtime/allocate.lisp
index 3f50fe042..366355c4c 100644
--- a/runtime/allocate.lisp
+++ b/runtime/allocate.lisp
@@ -482,32 +482,31 @@
          (mezzano.supervisor:inhibit-thread-pool-blocking-hijack
            (mezzano.supervisor:with-mutex (*allocator-lock*)
              (mezzano.supervisor:with-pseudo-atomic
-	       (tagbody
+               (tagbody
                 INNER-LOOP
                   (multiple-value-bind (result ignore1 ignore2 failurep)
-		      (%do-slow-allocate-from-general-area tag data words)
+                      (%do-slow-allocate-from-general-area tag data words)
                     (declare (ignore ignore1 ignore2))
                     (when (not failurep)
-		      (update-allocation-time start-time)
+                      (update-allocation-time start-time)
 		      ;; (mezzano.supervisor::debug-print-line "what")
-		      (return-from %slow-allocate-from-general-area
+                      (return-from %slow-allocate-from-general-area
                         result)))
-                  (cond 
-		    ;; No memory. If there's memory available, then expand the area, otherwise run the GC.
-		    ;; Running the GC cannot be done when pseudo-atomic.
-		    ((expand-allocation-area :general
-                                             (* sys.int::+tlab-size+ 8)
-					     ;; (* words 8)
-                                             '*general-area-expansion-granularity*
-                                             'sys.int::*general-area-young-gen-limit*
-                                             sys.int::+address-tag-general+)
+                  ;; No memory. If there's memory available, then expand the area, otherwise run the GC.
+                  ;; Running the GC cannot be done when pseudo-atomic.
+                  (cond ((expand-allocation-area :general
+                                                 (* sys.int::+tlab-size+ 8)
+                                                 ;; (* words 8)
+                                                 '*general-area-expansion-granularity*
+                                                 'sys.int::*general-area-young-gen-limit*
+                                                 sys.int::+address-tag-general+)
                      ;; Successfully expanded the area. Retry the allocation.
                      (go INNER-LOOP))
                     (t
                      ;; No memory do expand, bail out and run the GC.
                      ;; This cannot be done when pseudo-atomic.
                      (when sys.int::*gc-enable-logging*
-		       (mezzano.supervisor:debug-print-line "General area expansion failed, performing GC."))
+                       (mezzano.supervisor:debug-print-line "General area expansion failed, performing GC."))
                      (go DO-GC))))))))
      DO-GC
        ;; Must occur outside the locks.

From 243be79559ccc9133c1b7ab05e92913c0d073f9e Mon Sep 17 00:00:00 2001
From: Sylvia Harrington <sylvia.harrington28@gmail.com>
Date: Sun, 17 May 2026 14:07:21 +0100
Subject: [PATCH 07/11] arm64: Use :object-location for accessing cpu locations

---
 runtime/runtime-arm64.lisp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/runtime/runtime-arm64.lisp b/runtime/runtime-arm64.lisp
index 962706fcb..d36366dc5 100644
--- a/runtime/runtime-arm64.lisp
+++ b/runtime/runtime-arm64.lisp
@@ -406,9 +406,9 @@
   (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*00)
   ;; Update allocation meter.
-  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
   ;; Check *ENABLE-ALLOCATION-PROFILING*
   ;; FIXME: This only tests the global value.
   (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *enable-allocation-profiling*))
@@ -432,9 +432,9 @@
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:b.ne SLOW-PATH)
   ;; Done. Return everything.
-  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)
@@ -519,7 +519,7 @@
   ;; Returns (values tag data words t) on failure, just the object on success.
   ;; X0 = tag; X1 = data; X2 = words.
   ;; Load per-CPU TLAB limit and newspace bit.
-  (mezzano.lap.arm64:ldr :x3 (:object :x27 #.mezzano.supervisor::+arm64-cpu-tlab-limit+))
+  (mezzano.lap.arm64:ldr :x3 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-tlab-limit+))
   (mezzano.lap.arm64:ldr :x4 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
   ;; X3 = tlab limit. X4 = newspace-bit.
   ;; Assemble the final header value in X12.
@@ -531,7 +531,7 @@
   (mezzano.lap.arm64:add :x10 :xzr :x2 :lsl 3) ; words * 8
   ;; Address generation for the per-CPU tlab-bump slot.
   ;; Linked GC mode is not needed as this will be repeated due to the restart region.
-  (mezzano.lap.arm64:add :x9 :x27 #.(+ (- sys.int::+tag-object+) 8 (* mezzano.supervisor::+arm64-cpu-tlab-bump+ 8)))
+  (mezzano.lap.arm64:add :x9 :x27 #.(+ (- sys.int::+tag-object+) 8 (mezzano.runtime::location-offset mezzano.supervisor::+arm64-cpu-tlab-bump+)))
   ;; Release atomic add to increment the bump pointer
   (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9))
   (mezzano.lap.arm64:add :x11 :x6 :x10)
@@ -634,12 +634,12 @@
   (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*00)
   ;; Update allocation meter.
-  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
-  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
   (mezzano.lap.arm64:ldr :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:str :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+))
@@ -661,9 +661,9 @@
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:b.ne SLOW-PATH)
   ;; Done. Return everything.
-  (mezzano.lap.arm64:ldr :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)

From a0c1fb57c6caf02ac2c335c7cdd1799011066531 Mon Sep 17 00:00:00 2001
From: Sylvia Harrington <sylvia.harrington28@gmail.com>
Date: Sun, 17 May 2026 14:08:21 +0100
Subject: [PATCH 08/11] supervisor: Share common parts of the x86-64/arm64 cpu
 struct

---
 runtime/runtime-x86-64.lisp | 14 +++++++-------
 supervisor/arm64/cpu.lisp   |  9 +--------
 supervisor/x86-64/cpu.lisp  |  9 +--------
 3 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp
index 3fa327dbf..3092f02b3 100644
--- a/runtime/runtime-x86-64.lisp
+++ b/runtime/runtime-x86-64.lisp
@@ -567,7 +567,7 @@ the GC must be deferred during FILL-WORDS."
   ;; R8 = tag; R9 = data; R10 = words.
   ;; Fetch symbol value cells.
   (sys.lap-x86:mov64 :r12 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
-  (sys.lap-x86:fs) (sys.lap-x86:mov64 :r11 (:object-location nil #.mezzano.supervisor::+cpu-tlab-limit+)) ; Allocation limit for the CPU
+  (sys.lap-x86:fs) (sys.lap-x86:mov64 :r11 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-tlab-limit+)) ; Allocation limit for the CPU
 
   ;; R12 = newspace-bit. R11 = limit.
   ;; Assemble the final header value in RDI.
@@ -578,7 +578,7 @@ the GC must be deferred during FILL-WORDS."
   (:gc :no-frame :layout #*0 :restart t)
   ;; Fetch and increment the current bump pointer.
   (sys.lap-x86:lea64 :rbx ((:r10 8))) ; words * 8
-  (sys.lap-x86:fs) (sys.lap-x86:xadd64 (:object-location nil #.mezzano.supervisor::+cpu-tlab-bump+) :rbx) ; We're incrementing the slot in local-cpu's struct, hence the fs
+  (sys.lap-x86:fs) (sys.lap-x86:xadd64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-tlab-bump+) :rbx) ; We're incrementing the slot in local-cpu's struct, hence the fs
   ;; RBX is old bump pointer, the address of the cons.
   ;; Find the new bump pointer.
   (sys.lap-x86:lea64 :rsi (:rbx (:r10 8)))
@@ -633,14 +633,14 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:ja TOO-LARGE)
   ;; Update allocation meter.
   (sys.lap-x86:fs)
-  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-general-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-general-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+))
   ;; Try the real fast allocator.
   (sys.lap-x86:call (:named-call %do-allocate-from-general-area))
   (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:jne SLOW-PATH)
   ;; Done. Return everything.
   (sys.lap-x86:fs)
-  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-general-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-general-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret)
   TOO-LARGE
@@ -710,9 +710,9 @@ the GC must be deferred during FILL-WORDS."
   (:gc :no-frame :layout #*0)
   ;; Update allocation meter.
   (sys.lap-x86:fs)
-  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-cons-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-cons-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:fs)
-  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-bytes-consed+)
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-bytes-consed+)
                      #.(ash 16 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:gs)
   (sys.lap-x86:add64 (:object nil #.mezzano.supervisor::+thread-bytes-consed+)
@@ -731,7 +731,7 @@ the GC must be deferred during FILL-WORDS."
   (sys.lap-x86:jne SLOW-PATH)
   ;; Done. Return everything.
   (sys.lap-x86:fs)
-  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+cpu-cons-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+))
+  (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-cons-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+))
   (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (sys.lap-x86:ret)
   SLOW-PATH
diff --git a/supervisor/arm64/cpu.lisp b/supervisor/arm64/cpu.lisp
index 26db55799..8f90d85f7 100644
--- a/supervisor/arm64/cpu.lisp
+++ b/supervisor/arm64/cpu.lisp
@@ -18,14 +18,7 @@
   idle-thread
   wired-stack
   (sp-el1 0)
-  page-fault-hook
-  (tlab-bump 0)
-  (tlab-limit 0)
-  (bytes-consed 0)
-  (cons-allocation-count 0)
-  (cons-fast-path-hits 0)
-  (general-allocation-count 0)
-  (general-fast-path-hits 0))
+  page-fault-hook)
 
 (defun initialize-boot-cpu ()
   (setf (arm64-cpu-self *bsp-cpu*) *bsp-cpu*)
diff --git a/supervisor/x86-64/cpu.lisp b/supervisor/x86-64/cpu.lisp
index 8b901f9e3..b00519b66 100644
--- a/supervisor/x86-64/cpu.lisp
+++ b/supervisor/x86-64/cpu.lisp
@@ -112,14 +112,7 @@
   irq-stack
   page-fault-stack
   lapic-timer-active
-  page-fault-hook
-  (tlab-bump 0)
-  (tlab-limit 0)
-  (bytes-consed 0)
-  (cons-allocation-count 0)
-  (cons-fast-path-hits 0)
-  (general-allocation-count 0)
-  (general-fast-path-hits 0))
+  page-fault-hook)
 
 (defconstant +ap-trampoline-physical-address+ #x7000
   "Where the AP trampoline should be copied to in physical memory.

From 05bf1f37f08d7996db06495a30e94d752415218f Mon Sep 17 00:00:00 2001
From: Sylvia Harrington <sylvia.harrington28@gmail.com>
Date: Sun, 17 May 2026 16:30:17 +0100
Subject: [PATCH 09/11] arm64: Fix add/sub instruction with negative imm
 swapping

---
 compiler/lap-arm64.lisp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/lap-arm64.lisp b/compiler/lap-arm64.lisp
index 6b51abd40..c1d0fe156 100644
--- a/compiler/lap-arm64.lisp
+++ b/compiler/lap-arm64.lisp
@@ -834,7 +834,7 @@
              ;; switch add/sub around for negative values
              (when (minusp imm-value)
                (setf imm-value (- imm-value))
-               (setf opc (logxor opc (ash opcode 30))))
+               (setf opc (logxor opc (ash 1 30))))
              (emit-instruction (logior #x11000000
                                        opc
                                        (if (eql amount 12)

From 615c63d64575e45199efe9dc754e2dec431a6f12 Mon Sep 17 00:00:00 2001
From: Sylvia Harrington <sylvia.harrington28@gmail.com>
Date: Sun, 17 May 2026 16:34:22 +0100
Subject: [PATCH 10/11] arm64: Fix assembly accesses to the current cpu via x27

---
 runtime/runtime-arm64.lisp | 30 ++++++++++++++++++------------
 supervisor/arm64/cpu.lisp  |  3 +--
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/runtime/runtime-arm64.lisp b/runtime/runtime-arm64.lisp
index d36366dc5..da8f11ef2 100644
--- a/runtime/runtime-arm64.lisp
+++ b/runtime/runtime-arm64.lisp
@@ -406,9 +406,10 @@
   (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*00)
   ;; Update allocation meter.
-  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
+  (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+))
   ;; Check *ENABLE-ALLOCATION-PROFILING*
   ;; FIXME: This only tests the global value.
   (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *enable-allocation-profiling*))
@@ -432,9 +433,10 @@
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:b.ne SLOW-PATH)
   ;; Done. Return everything.
-  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
+  (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)
@@ -519,7 +521,8 @@
   ;; Returns (values tag data words t) on failure, just the object on success.
   ;; X0 = tag; X1 = data; X2 = words.
   ;; Load per-CPU TLAB limit and newspace bit.
-  (mezzano.lap.arm64:ldr :x3 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-tlab-limit+))
+  (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object
+  (mezzano.lap.arm64:ldr :x3 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-tlab-limit+))
   (mezzano.lap.arm64:ldr :x4 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
   ;; X3 = tlab limit. X4 = newspace-bit.
   ;; Assemble the final header value in X12.
@@ -531,7 +534,8 @@
   (mezzano.lap.arm64:add :x10 :xzr :x2 :lsl 3) ; words * 8
   ;; Address generation for the per-CPU tlab-bump slot.
   ;; Linked GC mode is not needed as this will be repeated due to the restart region.
-  (mezzano.lap.arm64:add :x9 :x27 #.(+ (- sys.int::+tag-object+) 8 (mezzano.runtime::location-offset mezzano.supervisor::+arm64-cpu-tlab-bump+)))
+  (mezzano.lap.arm64:ldr :x9 (:x27)) ; load cpu object
+  (mezzano.lap.arm64:add :x9 :x9 #.(+ (- sys.int::+tag-object+) 8 (mezzano.runtime::location-offset mezzano.supervisor::+arm64-cpu-tlab-bump+)))
   ;; Release atomic add to increment the bump pointer
   (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9))
   (mezzano.lap.arm64:add :x11 :x6 :x10)
@@ -634,12 +638,13 @@
   (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS)
   (:gc :no-frame :layout #*00)
   ;; Update allocation meter.
-  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
+  (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
-  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+))
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-bytes-consed+))
   (mezzano.lap.arm64:ldr :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:str :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+))
@@ -661,9 +666,10 @@
   (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:b.ne SLOW-PATH)
   ;; Done. Return everything.
-  (mezzano.lap.arm64:ldr :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
+  (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object
+  (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
   (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+))
-  (mezzano.lap.arm64:str :x9 (:object-location :x27 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
+  (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16))
   (:gc :no-frame :layout #*)
diff --git a/supervisor/arm64/cpu.lisp b/supervisor/arm64/cpu.lisp
index 8f90d85f7..e994f7fd8 100644
--- a/supervisor/arm64/cpu.lisp
+++ b/supervisor/arm64/cpu.lisp
@@ -69,8 +69,7 @@
 
 (sys.int::define-lap-function local-cpu-info (())
   (:gc :no-frame :layout #*)
-  (mezzano.lap.arm64:orr :x9 :xzr :x27)
-  (mezzano.lap.arm64:ldr :x0 (:x9))
+  (mezzano.lap.arm64:ldr :x0 (:x27))
   (mezzano.lap.arm64:movz :x5 #.(ash 1 sys.int::+n-fixnum-bits+))
   (mezzano.lap.arm64:ret))
 

From 19d7995a0082f11ce58282316b19f8cd58d75003 Mon Sep 17 00:00:00 2001
From: Iska Mag <iska@iskamag.com>
Date: Mon, 15 Jun 2026 00:26:16 +0500
Subject: [PATCH 11/11] runtime: switch TLABS from per-CPU to per-thread

---
 runtime/allocate.lisp       |  8 ++++----
 runtime/runtime-arm64.lisp  | 16 +++++++---------
 runtime/runtime-x86-64.lisp |  4 ++--
 supervisor/arm64/cpu.lisp   | 12 ------------
 supervisor/thread.lisp      |  5 +++--
 system/gc.lisp              |  8 +++++---
 6 files changed, 21 insertions(+), 32 deletions(-)

diff --git a/runtime/allocate.lisp b/runtime/allocate.lisp
index 366355c4c..e0874a02d 100644
--- a/runtime/allocate.lisp
+++ b/runtime/allocate.lisp
@@ -463,10 +463,10 @@
     (return-from %get-new-tlab nil))
   (let ((result (%do-get-new-tlab)))
     (when result
-      (let ((cpu (mezzano.supervisor::local-cpu))
+      (let ((thread (mezzano.supervisor::current-thread))
             (limit (+ result (* sys.int::+tlab-size+ 8))))
-        (setf (mezzano.supervisor::cpu-tlab-bump cpu) result
-              (mezzano.supervisor::cpu-tlab-limit cpu) limit)
+        (setf (mezzano.supervisor::thread-tlab-bump thread) result
+              (mezzano.supervisor::thread-tlab-limit thread) limit)
         t))))
 
 (defun %slow-allocate-from-general-area (tag data words)
@@ -529,7 +529,7 @@
       (incf (mezzano.supervisor:thread-bytes-consed
              (mezzano.supervisor:current-thread))
             bytes))
-    ;; (mezzano.supervisor:debug-print-line "b " (mezzano.supervisor::cpu-tlab-bump (mezzano.supervisor::local-cpu)))
+    ;; (mezzano.supervisor:debug-print-line "b " (mezzano.supervisor::thread-tlab-bump (mezzano.supervisor::current-thread)))
     (ecase area
       ((nil)
        (%allocate-from-general-area tag data words))
diff --git a/runtime/runtime-arm64.lisp b/runtime/runtime-arm64.lisp
index da8f11ef2..a21c7e3c1 100644
--- a/runtime/runtime-arm64.lisp
+++ b/runtime/runtime-arm64.lisp
@@ -517,12 +517,11 @@
 
 (sys.int::define-lap-function %do-allocate-from-general-area ((tag data words))
   (:gc :no-frame :layout #*)
-  ;; Attempt to quickly allocate from the general area using the per-CPU TLAB.
+  ;; Attempt to quickly allocate from the general area using the per-thread TLAB.
   ;; Returns (values tag data words t) on failure, just the object on success.
   ;; X0 = tag; X1 = data; X2 = words.
-  ;; Load per-CPU TLAB limit and newspace bit.
-  (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object
-  (mezzano.lap.arm64:ldr :x3 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-tlab-limit+))
+  ;; Load per-thread TLAB limit and newspace bit.
+  (mezzano.lap.arm64:ldr :x3 (:object :x28 #.mezzano.supervisor::+thread-tlab-limit+))
   (mezzano.lap.arm64:ldr :x4 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
   ;; X3 = tlab limit. X4 = newspace-bit.
   ;; Assemble the final header value in X12.
@@ -530,18 +529,17 @@
   (mezzano.lap.arm64:add :x12 :x12 :x1 :lsl #.(- sys.int::+object-data-shift+ sys.int::+n-fixnum-bits+))
   ;; If a garbage collection occurs, it must rewind IP back here.
   (:gc :no-frame :layout #* :restart t)
-  ;; Fetch and increment the per-CPU TLAB bump pointer.
+  ;; Fetch and increment the per-thread TLAB bump pointer.
   (mezzano.lap.arm64:add :x10 :xzr :x2 :lsl 3) ; words * 8
-  ;; Address generation for the per-CPU tlab-bump slot.
+  ;; Address generation for the per-thread tlab-bump slot.
   ;; Linked GC mode is not needed as this will be repeated due to the restart region.
-  (mezzano.lap.arm64:ldr :x9 (:x27)) ; load cpu object
-  (mezzano.lap.arm64:add :x9 :x9 #.(+ (- sys.int::+tag-object+) 8 (mezzano.runtime::location-offset mezzano.supervisor::+arm64-cpu-tlab-bump+)))
+  (mezzano.lap.arm64:add :x9 :x28 #.(+ (- sys.int::+tag-object+) 8 (* mezzano.supervisor::+thread-tlab-bump+ 8)))
   ;; Release atomic add to increment the bump pointer
   (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9))
   (mezzano.lap.arm64:add :x11 :x6 :x10)
   ;; X6 is old bump pointer, the address of the allocation.
   ;; X11 is the new bump pointer.
-  ;; Test against per-CPU TLAB limit.
+  ;; Test against per-thread TLAB limit.
   (mezzano.lap.arm64:subs :xzr :x11 :x3)
   (mezzano.lap.arm64:b.hi SLOW-PATH)
   ;; Generate the object.
diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp
index 3092f02b3..95dde9ce7 100644
--- a/runtime/runtime-x86-64.lisp
+++ b/runtime/runtime-x86-64.lisp
@@ -567,7 +567,7 @@ the GC must be deferred during FILL-WORDS."
   ;; R8 = tag; R9 = data; R10 = words.
   ;; Fetch symbol value cells.
   (sys.lap-x86:mov64 :r12 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*))
-  (sys.lap-x86:fs) (sys.lap-x86:mov64 :r11 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-tlab-limit+)) ; Allocation limit for the CPU
+  (sys.lap-x86:gs) (sys.lap-x86:mov64 :r11 (:object nil #.mezzano.supervisor::+thread-tlab-limit+)) ; Allocation limit for the thread
 
   ;; R12 = newspace-bit. R11 = limit.
   ;; Assemble the final header value in RDI.
@@ -578,7 +578,7 @@ the GC must be deferred during FILL-WORDS."
   (:gc :no-frame :layout #*0 :restart t)
   ;; Fetch and increment the current bump pointer.
   (sys.lap-x86:lea64 :rbx ((:r10 8))) ; words * 8
-  (sys.lap-x86:fs) (sys.lap-x86:xadd64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-tlab-bump+) :rbx) ; We're incrementing the slot in local-cpu's struct, hence the fs
+  (sys.lap-x86:gs) (sys.lap-x86:xadd64 (:object nil #.mezzano.supervisor::+thread-tlab-bump+) :rbx)
   ;; RBX is old bump pointer, the address of the cons.
   ;; Find the new bump pointer.
   (sys.lap-x86:lea64 :rsi (:rbx (:r10 8)))
diff --git a/supervisor/arm64/cpu.lisp b/supervisor/arm64/cpu.lisp
index e994f7fd8..36d32dd1d 100644
--- a/supervisor/arm64/cpu.lisp
+++ b/supervisor/arm64/cpu.lisp
@@ -76,18 +76,6 @@
 (defun local-cpu ()
   (local-cpu-info))
 
-(defun cpu-tlab-bump (cpu)
-  (arm64-cpu-tlab-bump cpu))
-
-(defun (setf cpu-tlab-bump) (value cpu)
-  (setf (arm64-cpu-tlab-bump cpu) value))
-
-(defun cpu-tlab-limit (cpu)
-  (arm64-cpu-tlab-limit cpu))
-
-(defun (setf cpu-tlab-limit) (value cpu)
-  (setf (arm64-cpu-tlab-limit cpu) value))
-
 (defun initialize-cpu ()
   (setf (arm64-cpu-cpu-id *bsp-cpu*) (fdt-boot-cpuid))
   (push-wired *bsp-cpu* *cpus*))
diff --git a/supervisor/thread.lisp b/supervisor/thread.lisp
index 0603a60a4..0bc97e79d 100644
--- a/supervisor/thread.lisp
+++ b/supervisor/thread.lisp
@@ -55,8 +55,6 @@ can be reprotected.")
 
 (defstruct (cpu
             (:area :wired))
-  (tlab-bump 0)
-  (tlab-limit 0)
   (bytes-consed 0)
   (cons-allocation-count 0)
   (cons-fast-path-hits 0)
@@ -136,6 +134,9 @@ can be reprotected.")
   (switch-time-start 0) ; set when the thread is switched to, used to update run-time
   ;; Time spent in the slow allocation path (including GC time).
   (allocation-time 0)
+  ;; TLAB (Thread-Local Allocation Buffer) bump pointer and limit.
+  (tlab-bump 0)
+  (tlab-limit 0)
   ;; Per-thread allocation meter.
   (bytes-consed 0)
   ;; Slots used as part of the multiple-value return convention.
diff --git a/system/gc.lisp b/system/gc.lisp
index f0acea042..a2b4cf80c 100644
--- a/system/gc.lisp
+++ b/system/gc.lisp
@@ -2476,6 +2476,8 @@ No type information will be provided."
          (setf (%object-ref-t finalizer +weak-pointer-finalizer+) nil))))
 
 (defun fixup-tlabs ()
-  (dolist (cpu mezzano.supervisor::*cpus*)
-    (setf (mezzano.supervisor::cpu-tlab-limit cpu) 0
-          (mezzano.supervisor::cpu-tlab-bump cpu) 0)))
+  (do ((thread mezzano.supervisor::*all-threads*
+               (mezzano.supervisor::thread-global-next thread)))
+      ((null thread))
+    (setf (mezzano.supervisor::thread-tlab-limit thread) 0
+          (mezzano.supervisor::thread-tlab-bump thread) 0)))