diff --git a/compiler/lap-arm64.lisp b/compiler/lap-arm64.lisp index 6b51abd40..c1d0fe156 100644 --- a/compiler/lap-arm64.lisp +++ b/compiler/lap-arm64.lisp @@ -834,7 +834,7 @@ ;; switch add/sub around for negative values (when (minusp imm-value) (setf imm-value (- imm-value)) - (setf opc (logxor opc (ash opcode 30)))) + (setf opc (logxor opc (ash 1 30)))) (emit-instruction (logior #x11000000 opc (if (eql amount 12) diff --git a/runtime/allocate.lisp b/runtime/allocate.lisp index d9a8b8994..e0874a02d 100644 --- a/runtime/allocate.lisp +++ b/runtime/allocate.lisp @@ -48,12 +48,29 @@ (sys.int::defglobal *general-area-expansion-granularity*) (sys.int::defglobal *cons-area-expansion-granularity*) -(sys.int::defglobal *general-fast-path-hits*) -(sys.int::defglobal *general-allocation-count*) -(sys.int::defglobal *cons-fast-path-hits*) -(sys.int::defglobal *cons-allocation-count*) +(defun bytes-consed () + (loop for cpu in mezzano.supervisor::*cpus* + sum (mezzano.supervisor::cpu-bytes-consed cpu))) -(sys.int::defglobal *bytes-consed*) +(defun general-allocation-count () + (loop for cpu in mezzano.supervisor::*cpus* + sum (mezzano.supervisor::cpu-general-allocation-count cpu))) + +(defun general-fast-path-hits () + (loop for cpu in mezzano.supervisor::*cpus* + sum (mezzano.supervisor::cpu-general-fast-path-hits cpu))) + +(defun cons-allocation-count () + (loop for cpu in mezzano.supervisor::*cpus* + sum (mezzano.supervisor::cpu-cons-allocation-count cpu))) + +(defun cons-fast-path-hits () + (loop for cpu in mezzano.supervisor::*cpus* + sum (mezzano.supervisor::cpu-cons-fast-path-hits cpu))) + + +(defconstant sys.int::+tlab-size+ (* 256 1024)) +(defconstant sys.int::+tlab-object-size-limit+ (* 64 1024)) (defvar *maximum-allocation-attempts* 5 "GC this many times before giving up on an allocation.") @@ -90,11 +107,6 @@ *enable-allocation-profiling* nil *general-area-expansion-granularity* sys.int::+allocation-minimum-alignment+ *cons-area-expansion-granularity* sys.int::+allocation-minimum-alignment+ - *general-fast-path-hits* 0 - *general-allocation-count* 0 - *cons-fast-path-hits* 0 - *cons-allocation-count* 0 - *bytes-consed* 0 *allocator-lock* (mezzano.supervisor:make-mutex "Allocator") *allocation-fudge* (* 8 1024 1024) sys.int::*generation-size-ratio* 2) @@ -332,7 +344,7 @@ #-(or x86-64 arm64) (defun %allocate-from-general-area (tag data words) - (sys.int::%atomic-fixnum-add-symbol '*general-allocation-count* 1) + (incf (mezzano.supervisor::cpu-general-allocation-count (mezzano.supervisor::local-cpu)) 1) (%slow-allocate-from-general-area tag data words)) #-(or x86-64 arm64) @@ -437,7 +449,31 @@ (mezzano.supervisor:debug-print-line "A-M-R failed.")) nil)))) +(defun %do-get-new-tlab () + (let* ((size (* sys.int::+tlab-size+ 8)) + (old-bump (sys.int::%atomic-fixnum-add-symbol 'sys.int::*general-area-young-gen-bump* + size)) + (new-bump (+ old-bump size))) + (and (<= new-bump sys.int::*general-area-young-gen-limit*) + old-bump))) + +(defun %get-new-tlab (words) + (when (or (zerop sys.int::*general-area-young-gen-bump*) + (> words sys.int::+tlab-object-size-limit+)) + (return-from %get-new-tlab nil)) + (let ((result (%do-get-new-tlab))) + (when result + (let ((thread (mezzano.supervisor::current-thread)) + (limit (+ result (* sys.int::+tlab-size+ 8)))) + (setf (mezzano.supervisor::thread-tlab-bump thread) result + (mezzano.supervisor::thread-tlab-limit thread) limit) + t)))) + (defun %slow-allocate-from-general-area (tag data words) + ;; TLAB ran out. Try getting a new one. + (when (%get-new-tlab words) + (return-from %slow-allocate-from-general-area + (%allocate-from-general-area tag data words))) (let ((gc-count 0) (start-time (mezzano.supervisor:get-high-precision-timer))) (tagbody @@ -449,27 +485,29 @@ (tagbody INNER-LOOP (multiple-value-bind (result ignore1 ignore2 failurep) - (%do-allocate-from-general-area tag data words) + (%do-slow-allocate-from-general-area tag data words) (declare (ignore ignore1 ignore2)) (when (not failurep) (update-allocation-time start-time) + ;; (mezzano.supervisor::debug-print-line "what") (return-from %slow-allocate-from-general-area result))) ;; No memory. If there's memory available, then expand the area, otherwise run the GC. ;; Running the GC cannot be done when pseudo-atomic. (cond ((expand-allocation-area :general - (* words 8) + (* sys.int::+tlab-size+ 8) + ;; (* words 8) '*general-area-expansion-granularity* 'sys.int::*general-area-young-gen-limit* sys.int::+address-tag-general+) - ;; Successfully expanded the area. Retry the allocation. - (go INNER-LOOP)) - (t - ;; No memory do expand, bail out and run the GC. - ;; This cannot be done when pseudo-atomic. - (when sys.int::*gc-enable-logging* - (mezzano.supervisor:debug-print-line "General area expansion failed, performing GC.")) - (go DO-GC)))))))) + ;; Successfully expanded the area. Retry the allocation. + (go INNER-LOOP)) + (t + ;; No memory do expand, bail out and run the GC. + ;; This cannot be done when pseudo-atomic. + (when sys.int::*gc-enable-logging* + (mezzano.supervisor:debug-print-line "General area expansion failed, performing GC.")) + (go DO-GC)))))))) DO-GC ;; Must occur outside the locks. (when (> gc-count *maximum-allocation-attempts*) @@ -486,12 +524,12 @@ (when (oddp words) (incf words)) (let ((bytes (* words 8))) - (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* bytes) - ;; ### This won't accurately track if the thread gets footholded - ;; partway through the add... + (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) + bytes) (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) bytes)) + ;; (mezzano.supervisor:debug-print-line "b " (mezzano.supervisor::thread-tlab-bump (mezzano.supervisor::current-thread))) (ecase area ((nil) (%allocate-from-general-area tag data words)) @@ -507,18 +545,21 @@ ((nil) (cons car cdr)) (:pinned - (log-allocation-profile-entry 2) - (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32) - (%cons-in-pinned-area car cdr)) + (log-allocation-profile-entry 2) + (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) 32) + (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) 32) + (%cons-in-pinned-area car cdr)) (:wired - (log-allocation-profile-entry 2) - (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 32) - (%cons-in-wired-area car cdr)))) + (log-allocation-profile-entry 2) + (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) 32) + (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) 32) + (%cons-in-wired-area car cdr)))) #-(or x86-64 arm64) (defun cons (car cdr) - (sys.int::%atomic-fixnum-add-symbol '*cons-allocation-count* 1) - (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* 16) + (incf (mezzano.supervisor::cpu-cons-allocation-count (mezzano.supervisor::local-cpu)) 1) + (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) 16) + (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) 16) (slow-cons car cdr)) #-(or x86-64 arm64) @@ -690,10 +731,12 @@ with inhibit-gc = nil for i from 0 do (let ((result (%allocate-function-1 tag data words wiredp))) - (when result - (sys.int::%atomic-fixnum-add-symbol '*bytes-consed* (* words 8)) - (update-allocation-time start-time) - (return result))) + (when result + (let ((cbytes (* words 8))) + (incf (mezzano.supervisor::cpu-bytes-consed (mezzano.supervisor::local-cpu)) cbytes) + (incf (mezzano.supervisor:thread-bytes-consed (mezzano.supervisor:current-thread)) cbytes)) + (update-allocation-time start-time) + (return result))) (when (not (eql i 0)) ;; The GC has been run at least once. (when (expand-function-area words wiredp) @@ -886,7 +929,8 @@ This area exists below the stack and is never allocated or mapped.") :finalizer (lambda () (when stack-address (release-memory-range stack-address size) - (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* (- size)))) + ;; (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* (- size)) + )) :area :wired) (tagbody RETRY @@ -920,7 +964,7 @@ This area exists below the stack and is never allocated or mapped.") (if wired (setf sys.int::*wired-stack-area-bump* (align-up (+ bump size) +stack-region-alignment+)) (setf sys.int::*stack-area-bump* (align-up (+ bump size) +stack-region-alignment+))) - (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* size) + ;; (sys.int::%atomic-fixnum-add-symbol 'sys.int::*bytes-allocated-to-stacks* size) (setf (stack-base stack) addr ;; Notify the finalizer that the stack has been allocated & should be freed. stack-address addr) diff --git a/runtime/runtime-arm64.lisp b/runtime/runtime-arm64.lisp index dace691a3..a21c7e3c1 100644 --- a/runtime/runtime-arm64.lisp +++ b/runtime/runtime-arm64.lisp @@ -406,12 +406,10 @@ (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS) (:gc :no-frame :layout #*00) ;; Update allocation meter. - ;; *BYTES-CONSED* is updated elsewhere. - (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *general-allocation-count*)) - ;; FIXME: Should be atomic add. - (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object + (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+)) (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+)) - (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-allocation-count+)) ;; Check *ENABLE-ALLOCATION-PROFILING* ;; FIXME: This only tests the global value. (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *enable-allocation-profiling*)) @@ -423,16 +421,32 @@ (mezzano.lap.arm64:ldr :x4 (:object :x6 #.sys.int::+symbol-value-cell-value+)) (mezzano.lap.arm64:subs :xzr :x4 :x26) (mezzano.lap.arm64:b.ne SLOW-PATH) + ;; Check if object exceeds the TLAB size limit. + (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell sys.int::+tlab-object-size-limit+)) + (mezzano.lap.arm64:ldr :x4 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:add :x4 :xzr :x4 :lsr #.sys.int::+n-fixnum-bits+) + (mezzano.lap.arm64:add :x6 :xzr :x2 :lsr #.sys.int::+n-fixnum-bits+) + (mezzano.lap.arm64:subs :xzr :x6 :x4) + (mezzano.lap.arm64:b.hi TOO-LARGE) ;; Try the real fast allocator. (mezzano.lap.arm64:named-call %do-allocate-from-general-area) (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) (mezzano.lap.arm64:b.ne SLOW-PATH) ;; Done. Return everything. - (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *general-fast-path-hits*)) - ;; FIXME: Should be atomic add. - (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object + (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+)) (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+)) - (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-general-fast-path-hits+)) + (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) + (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16)) + (:gc :no-frame :layout #*) + (mezzano.lap.arm64:ret) + TOO-LARGE + (:gc :no-frame :layout #*00) + ;; Object is too large for the TLAB. Use the global allocator directly. + (mezzano.lap.arm64:named-call %do-slow-allocate-from-general-area) + (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) + (mezzano.lap.arm64:b.ne SLOW-PATH) (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16)) (:gc :no-frame :layout #*) @@ -446,9 +460,9 @@ (:gc :no-frame :layout #* :incoming-arguments :rcx) (mezzano.lap.arm64:named-tail-call %slow-allocate-from-general-area)) -(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words)) +(sys.int::define-lap-function %do-slow-allocate-from-general-area ((tag data words)) (:gc :no-frame :layout #*) - ;; Attempt to quickly allocate from the general area. + ;; Attempt to allocate from the global general area using atomic operations. ;; Returns (values tag data words t) on failure, just the object on success. ;; X0 = tag; X1 = data; X2 = words. ;; Fetch symbol value cells. @@ -469,7 +483,7 @@ ;; Release atomic add to increment the bump pointer (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9)) (mezzano.lap.arm64:add :x11 :x6 :x10) - ;; X6 is old bump pointer, the address of the cons. + ;; X6 is old bump pointer, the address of the allocation. ;; X11 is the new bump pointer. ;; Test against limit. (mezzano.lap.arm64:ldr :x10 (:object :x3 #.sys.int::+symbol-value-cell-value+)) @@ -480,11 +494,64 @@ (mezzano.lap.arm64:add :x6 :xzr :x6 :lsr #.sys.int::+n-fixnum-bits+) ;; Set address bits and the tag bits. ;; Set address bits, tag bits, and the mark bit. + (mezzano.lap.arm64:ldr :x9 (:pc global-address-object-tag)) + (mezzano.lap.arm64:orr :x6 :x6 :x9) + (mezzano.lap.arm64:ldr :x9 (:object :x4 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:orr :x6 :x6 :x9) + ;; Write back the header. + (mezzano.lap.arm64:str :x12 (:object :x6 -1)) + ;; Leave restart region. + (:gc :no-frame :layout #*) + ;; Done. Return everything. + (mezzano.lap.arm64:orr :x0 :xzr :x6) + (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) + (mezzano.lap.arm64:ret) + SLOW-PATH + (mezzano.lap.arm64:ldr :x3 (:constant t)) + (mezzano.lap.arm64:movz :x5 #.(ash 4 #.sys.int::+n-fixnum-bits+)) + (mezzano.lap.arm64:ret) + (:align 16) + global-address-object-tag + (:d64/le #.(logior (ash sys.int::+address-tag-general+ sys.int::+address-tag-shift+) + sys.int::+tag-object+))) + +(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words)) + (:gc :no-frame :layout #*) + ;; Attempt to quickly allocate from the general area using the per-thread TLAB. + ;; Returns (values tag data words t) on failure, just the object on success. + ;; X0 = tag; X1 = data; X2 = words. + ;; Load per-thread TLAB limit and newspace bit. + (mezzano.lap.arm64:ldr :x3 (:object :x28 #.mezzano.supervisor::+thread-tlab-limit+)) + (mezzano.lap.arm64:ldr :x4 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*)) + ;; X3 = tlab limit. X4 = newspace-bit. + ;; Assemble the final header value in X12. + (mezzano.lap.arm64:add :x12 :xzr :x0 :lsl #.(- sys.int::+object-type-shift+ sys.int::+n-fixnum-bits+)) + (mezzano.lap.arm64:add :x12 :x12 :x1 :lsl #.(- sys.int::+object-data-shift+ sys.int::+n-fixnum-bits+)) + ;; If a garbage collection occurs, it must rewind IP back here. + (:gc :no-frame :layout #* :restart t) + ;; Fetch and increment the per-thread TLAB bump pointer. + (mezzano.lap.arm64:add :x10 :xzr :x2 :lsl 3) ; words * 8 + ;; Address generation for the per-thread tlab-bump slot. + ;; Linked GC mode is not needed as this will be repeated due to the restart region. + (mezzano.lap.arm64:add :x9 :x28 #.(+ (- sys.int::+tag-object+) 8 (* mezzano.supervisor::+thread-tlab-bump+ 8))) + ;; Release atomic add to increment the bump pointer + (mezzano.lap.arm64:ldaddl :x10 :x6 (:x9)) + (mezzano.lap.arm64:add :x11 :x6 :x10) + ;; X6 is old bump pointer, the address of the allocation. + ;; X11 is the new bump pointer. + ;; Test against per-thread TLAB limit. + (mezzano.lap.arm64:subs :xzr :x11 :x3) + (mezzano.lap.arm64:b.hi SLOW-PATH) + ;; Generate the object. + ;; Unfixnumize address. This still looks like a fixnum due to alignment. + (mezzano.lap.arm64:add :x6 :xzr :x6 :lsr #.sys.int::+n-fixnum-bits+) + ;; Set address bits and the tag bits. + ;; Set address bits, tag bits, and the mark bit. (mezzano.lap.arm64:ldr :x9 (:pc general-address-object-tag)) (mezzano.lap.arm64:orr :x6 :x6 :x9) (mezzano.lap.arm64:ldr :x9 (:object :x4 #.sys.int::+symbol-value-cell-value+)) (mezzano.lap.arm64:orr :x6 :x6 :x9) - ;; RBX now points to a 0-element simple-vector, followed by however much empty space is required. + ;; X6 now points to a 0-element simple-vector, followed by however much empty space is required. ;; The gc metadata at this point has :restart t, so if a GC occurs before ;; writing the final header, this process will be restarted from the beginning. ;; This is required as the GC will only copy 2 words, leaving the rest of the memory in an invalid state. @@ -569,15 +636,16 @@ (mezzano.lap.arm64:b.ne SLOW-PATH-BAD-ARGS) (:gc :no-frame :layout #*00) ;; Update allocation meter. - ;; FIXME: Should be atomic add. - (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *cons-allocation-count*)) - (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object + (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+)) (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+)) - (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) - (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *bytes-consed*)) - (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-allocation-count+)) + (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-bytes-consed+)) + (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+)) + (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-bytes-consed+)) + (mezzano.lap.arm64:ldr :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+)) (mezzano.lap.arm64:add :x9 :x9 #.(ash 16 sys.int::+n-fixnum-bits+)) - (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:str :x9 (:object :x28 #.mezzano.supervisor::+thread-bytes-consed+)) ;; Check *ENABLE-ALLOCATION-PROFILING* ;; FIXME: This only tests the global value. #| Logging every cons tends to explode the profile buffer & exhaust memory. @@ -596,10 +664,10 @@ (mezzano.lap.arm64:subs :xzr :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) (mezzano.lap.arm64:b.ne SLOW-PATH) ;; Done. Return everything. - (mezzano.lap.arm64:ldr :x6 (:symbol-global-cell *cons-fast-path-hits*)) - (mezzano.lap.arm64:ldr :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:ldr :x10 (:x27)) ; load cpu object + (mezzano.lap.arm64:ldr :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+)) (mezzano.lap.arm64:add :x9 :x9 #.(ash 1 sys.int::+n-fixnum-bits+)) - (mezzano.lap.arm64:str :x9 (:object :x6 #.sys.int::+symbol-value-cell-value+)) + (mezzano.lap.arm64:str :x9 (:object-location :x10 #.mezzano.supervisor::+arm64-cpu-cons-fast-path-hits+)) (mezzano.lap.arm64:movz :x5 #.(ash 1 #.sys.int::+n-fixnum-bits+)) (mezzano.lap.arm64:ldp :x29 :x30 (:post :sp 16)) (:gc :no-frame :layout #*) diff --git a/runtime/runtime-x86-64.lisp b/runtime/runtime-x86-64.lisp index cb9346b70..95dde9ce7 100644 --- a/runtime/runtime-x86-64.lisp +++ b/runtime/runtime-x86-64.lisp @@ -502,9 +502,8 @@ the GC must be deferred during FILL-WORDS." ;; Build bignum. (sys.lap-x86:jmp (:named-call sys.int::%%make-bignum-64-rax))) -;; This relies on memory being initialized to zero, so it looks like -;; many simple vectors of length 0. -(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words)) + +(sys.int::define-lap-function %do-slow-allocate-from-general-area ((tag data words)) (:gc :no-frame :layout #*0) ;; Attempt to quickly allocate from the general area. ;; Returns (values tag data words t) on failure, just the object on success. @@ -557,6 +556,64 @@ the GC must be deferred during FILL-WORDS." (sys.lap-x86:mov32 :ecx #.(ash 4 #.sys.int::+n-fixnum-bits+)) (sys.lap-x86:ret)) + + +;; This relies on memory being initialized to zero, so it looks like +;; many simple vectors of length 0. +(sys.int::define-lap-function %do-allocate-from-general-area ((tag data words)) + (:gc :no-frame :layout #*0) + ;; Attempt to quickly allocate from the general area. + ;; Returns (values tag data words t) on failure, just the object on success. + ;; R8 = tag; R9 = data; R10 = words. + ;; Fetch symbol value cells. + (sys.lap-x86:mov64 :r12 (:symbol-global-cell sys.int::*young-gen-newspace-bit-raw*)) + (sys.lap-x86:gs) (sys.lap-x86:mov64 :r11 (:object nil #.mezzano.supervisor::+thread-tlab-limit+)) ; Allocation limit for the thread + + ;; R12 = newspace-bit. R11 = limit. + ;; Assemble the final header value in RDI. + (sys.lap-x86:mov64 :rdi :r9) + (sys.lap-x86:shl64 :rdi #.(- sys.int::+object-data-shift+ sys.int::+n-fixnum-bits+)) + (sys.lap-x86:lea64 :rdi (:rdi (:r8 #.(ash 1 (- sys.int::+object-type-shift+ sys.int::+n-fixnum-bits+))))) + ;; If a garbage collection occurs, it must rewind IP back here. + (:gc :no-frame :layout #*0 :restart t) + ;; Fetch and increment the current bump pointer. + (sys.lap-x86:lea64 :rbx ((:r10 8))) ; words * 8 + (sys.lap-x86:gs) (sys.lap-x86:xadd64 (:object nil #.mezzano.supervisor::+thread-tlab-bump+) :rbx) + ;; RBX is old bump pointer, the address of the cons. + ;; Find the new bump pointer. + (sys.lap-x86:lea64 :rsi (:rbx (:r10 8))) + ;; Test against limit. + (sys.lap-x86:cmp64 :rsi :r11) + (sys.lap-x86:ja SLOW-PATH) + (sys.lap-x86:shr64 :rsi #.sys.int::+n-fixnum-bits+) + ;; Generate the object. + ;; Unfixnumize address. This still looks like a fixnum due to alignment. + (sys.lap-x86:shr64 :rbx #.sys.int::+n-fixnum-bits+) + ;; Set address bits and the tag bits. + ;; Set address bits, tag bits, and the mark bit. + (sys.lap-x86:mov64 :rax #.(logior (ash sys.int::+address-tag-general+ sys.int::+address-tag-shift+) + sys.int::+tag-object+)) + (sys.lap-x86:or64 :rax (:object :r12 #.sys.int::+symbol-value-cell-value+)) + (sys.lap-x86:or64 :rbx :rax) + ;; RBX now points to a 0-element simple-vector, followed by however much empty space is required. + ;; The gc metadata at this point has :restart t, so if a GC occurs before + ;; writing the final header, this process will be restarted from the beginning. + ;; This is required as the GC will only copy 2 words, leaving the rest of the memory in an invalid state. + ;; Write back the header. + ;; This must be done in a single write so the GC always sees a correct header. + (sys.lap-x86:mov64 (:object :rbx -1) :rdi) + (sys.lap-x86:prefetchnta (:rsi)) + ;; Leave restart region. + (:gc :no-frame :layout #*0) + ;; Done. Return everything. + (sys.lap-x86:mov64 :r8 :rbx) + (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+)) + (sys.lap-x86:ret) + SLOW-PATH + (sys.lap-x86:mov64 :r11 t) + (sys.lap-x86:mov32 :ecx #.(ash 4 #.sys.int::+n-fixnum-bits+)) + (sys.lap-x86:ret)) + (sys.int::define-lap-function %allocate-from-general-area ((tag data words)) (:gc :no-frame :layout #*0 :incoming-arguments :rcx) ;; Attempt to quickly allocate from the general area. Will call @@ -566,19 +623,30 @@ the GC must be deferred during FILL-WORDS." (sys.lap-x86:cmp64 :rcx #.(ash 3 #.sys.int::+n-fixnum-bits+)) (sys.lap-x86:jne SLOW-PATH-BAD-ARGS) (:gc :no-frame :layout #*0) + ;; Check if object is too large. + (sys.lap-x86:mov64 :r11 (:symbol-global-cell sys.int::+tlab-object-size-limit+)) + (sys.lap-x86:mov64 :r11 (:object :r11 #.sys.int::+symbol-value-cell-value+)) + (sys.lap-x86:shr64 :r11 #.sys.int::+n-fixnum-bits+) + (sys.lap-x86:mov64 :rax :r10) + (sys.lap-x86:shr64 :rax #.sys.int::+n-fixnum-bits+) + (sys.lap-x86:cmp64 :rax :r11) + (sys.lap-x86:ja TOO-LARGE) ;; Update allocation meter. - ;; *BYTES-CONSED* is updated elsewhere. - (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-allocation-count*)) - (sys.lap-x86:lock) - (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+)) + (sys.lap-x86:fs) + (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-general-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+)) ;; Try the real fast allocator. (sys.lap-x86:call (:named-call %do-allocate-from-general-area)) (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+)) (sys.lap-x86:jne SLOW-PATH) ;; Done. Return everything. - (sys.lap-x86:mov64 :rbx (:symbol-global-cell *general-fast-path-hits*)) - (sys.lap-x86:lock) - (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+)) + (sys.lap-x86:fs) + (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-general-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+)) + (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+)) + (sys.lap-x86:ret) + TOO-LARGE + (sys.lap-x86:call (:named-call %do-slow-allocate-from-general-area)) + (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+)) + (sys.lap-x86:jne SLOW-PATH) (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+)) (sys.lap-x86:ret) SLOW-PATH @@ -641,12 +709,10 @@ the GC must be deferred during FILL-WORDS." (sys.lap-x86:jne SLOW-PATH-BAD-ARGS) (:gc :no-frame :layout #*0) ;; Update allocation meter. - (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-allocation-count*)) - (sys.lap-x86:lock) - (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+)) - (sys.lap-x86:mov64 :rbx (:symbol-global-cell *bytes-consed*)) - (sys.lap-x86:lock) - (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) + (sys.lap-x86:fs) + (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-cons-allocation-count+) #.(ash 1 sys.int::+n-fixnum-bits+)) + (sys.lap-x86:fs) + (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-bytes-consed+) #.(ash 16 sys.int::+n-fixnum-bits+)) (sys.lap-x86:gs) (sys.lap-x86:add64 (:object nil #.mezzano.supervisor::+thread-bytes-consed+) @@ -664,9 +730,8 @@ the GC must be deferred during FILL-WORDS." (sys.lap-x86:cmp64 :rcx #.(ash 1 #.sys.int::+n-fixnum-bits+)) (sys.lap-x86:jne SLOW-PATH) ;; Done. Return everything. - (sys.lap-x86:mov64 :rbx (:symbol-global-cell *cons-fast-path-hits*)) - (sys.lap-x86:lock) - (sys.lap-x86:add64 (:object :rbx #.sys.int::+symbol-value-cell-value+) #.(ash 1 sys.int::+n-fixnum-bits+)) + (sys.lap-x86:fs) + (sys.lap-x86:add64 (:object-location nil #.mezzano.supervisor::+x86-64-cpu-cons-fast-path-hits+) #.(ash 1 sys.int::+n-fixnum-bits+)) (sys.lap-x86:mov32 :ecx #.(ash 1 #.sys.int::+n-fixnum-bits+)) (sys.lap-x86:ret) SLOW-PATH diff --git a/supervisor/arm64/cpu.lisp b/supervisor/arm64/cpu.lisp index 8320487d2..36d32dd1d 100644 --- a/supervisor/arm64/cpu.lisp +++ b/supervisor/arm64/cpu.lisp @@ -69,8 +69,7 @@ (sys.int::define-lap-function local-cpu-info (()) (:gc :no-frame :layout #*) - (mezzano.lap.arm64:orr :x9 :xzr :x27) - (mezzano.lap.arm64:ldr :x0 (:x9)) + (mezzano.lap.arm64:ldr :x0 (:x27)) (mezzano.lap.arm64:movz :x5 #.(ash 1 sys.int::+n-fixnum-bits+)) (mezzano.lap.arm64:ret)) diff --git a/supervisor/thread.lisp b/supervisor/thread.lisp index 0603a60a4..0bc97e79d 100644 --- a/supervisor/thread.lisp +++ b/supervisor/thread.lisp @@ -55,8 +55,6 @@ can be reprotected.") (defstruct (cpu (:area :wired)) - (tlab-bump 0) - (tlab-limit 0) (bytes-consed 0) (cons-allocation-count 0) (cons-fast-path-hits 0) @@ -136,6 +134,9 @@ can be reprotected.") (switch-time-start 0) ; set when the thread is switched to, used to update run-time ;; Time spent in the slow allocation path (including GC time). (allocation-time 0) + ;; TLAB (Thread-Local Allocation Buffer) bump pointer and limit. + (tlab-bump 0) + (tlab-limit 0) ;; Per-thread allocation meter. (bytes-consed 0) ;; Slots used as part of the multiple-value return convention. diff --git a/system/gc.lisp b/system/gc.lisp index 6694ea9e5..a2b4cf80c 100644 --- a/system/gc.lisp +++ b/system/gc.lisp @@ -165,7 +165,8 @@ If FULL is true, then a major collection will be forced." (total-time (- gc-end gc-start)) (total-seconds (/ total-time (float internal-time-units-per-second)))) (gc-log "GC took " (truncate (* total-seconds 1000)) "ms") - (incf *gc-time* total-seconds))) + (incf *gc-time* total-seconds)) + (fixup-tlabs)) (setf *gc-in-progress* nil))) ;; TODO: catch & report errors. (run-finalizers)))) @@ -2473,3 +2474,10 @@ No type information will be provided." ;; Leave the weak pointer completely empty. ;; No references to any other object. (setf (%object-ref-t finalizer +weak-pointer-finalizer+) nil)))) + +(defun fixup-tlabs () + (do ((thread mezzano.supervisor::*all-threads* + (mezzano.supervisor::thread-global-next thread))) + ((null thread)) + (setf (mezzano.supervisor::thread-tlab-limit thread) 0 + (mezzano.supervisor::thread-tlab-bump thread) 0)))