diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index 2c1cdad..0000000
--- a/.dockerignore
+++ /dev/null
@@ -1,6 +0,0 @@
-node_modules
-_scriptsRepo
-planning
-.git
-*.md
-.env*
diff --git a/.env.local.enc b/.env.local.enc
deleted file mode 100644
index 975352d..0000000
--- a/.env.local.enc
+++ /dev/null
@@ -1,39 +0,0 @@
-#ENC[AES256_GCM,data:49vPC3ejk6KW,iv:/8DJO87b9DmxyGGE3YEdtkuSBgamtfrj78UUwR+rIYg=,tag:oPs5/UBTY2t4i0LXr9Awzg==,type:comment]
-NODE_ENV=ENC[AES256_GCM,data:5cCNfFJSNlddkdQ=,iv:bGnAmGCA3G8gDhHY19am0DKWoU6KdSybsNqfT2sfTbE=,tag:0yBDaJ/wE4wpIZoSCPgPDg==,type:str]
-PORT=ENC[AES256_GCM,data:5dgzWw==,iv:U4YpZa4CqLgUOjcJWG7kshc4KFz1UfYOPIWfVb6TskM=,tag:vA5nfYr5ilf4GvzR20vC5A==,type:str]
-DATABASE_URL=ENC[AES256_GCM,data:AA9Bw7d+JuiXhRUgeglhffwtJrHYfj1VCv8dAiIAAiRcZ5l4H9IseZCdwpkK1HAVSXmQCmQ=,iv:yssOT+JBz5jOtSxmzQw1aIrVHCvsI+/j1dN8041TPcw=,tag:GfLz9+oD23z5gTIWLv0dmw==,type:str]
-SESSION_SECRET=ENC[AES256_GCM,data:lr56mYmFH1TtEtqHATUzYLWzmqRPylM3TrpqXBjO4v7gRcobA3p3Asjv47TkvfpChURnCeTrB0c552Ue4TeqtQ==,iv:XdE23plTLUqvFwuSYRgf1XKBcKTRi3iWZrMqUSsWz0g=,tag:6FX+Eh9E8sU4f9Dc27jEHw==,type:str]
-#ENC[AES256_GCM,data:IEu179YO951HSIYzjlc6TeStpw==,iv:i/5zqvMv2uLlNNKJqsWnP/iEthC5Ec5Xry4/NboK7Kw=,tag:ypkVagiqkQ8KU3+amaRKgA==,type:comment]
-S3_BUCKET=ENC[AES256_GCM,data:Ue3XV054QBAUD+tlNzCF,iv:87tQ2skRacvhPtz27kMn2OKhnEOeD5Xhjnr1hm3c9v0=,tag:nGuWJQscbFzEkD5h0RpeUg==,type:str]
-S3_PUBLIC_BUCKET=ENC[AES256_GCM,data:EM8riKv9mrTHQzwxf6IV,iv:7rodyVXj6tDY8q2hqDWO7IHCM2oezNczXKAYd4wtr3A=,tag:e09QO2uhX3DQeZqdbX3jOw==,type:str]
-S3_ENDPOINT=ENC[AES256_GCM,data:sU2BhYR9d9f66hkf/W0wQo/00v6KEmZ1rHNSGh+Eb+xTQo4J1ts2hNm0+P+lOqex/XgTMnw1/5Ik992Vx5pfWYU=,iv:RAV3b3nRRj4BQIyZKcCI86mi7etuW4s9ekydKBgMHZA=,tag:gowf9g1eikXZ5vj97rFW0g==,type:str]
-S3_ACCESS_KEY=ENC[AES256_GCM,data:TLLepLD+gvgOcfGtmK+3nAe5+2bOSs0nePlb2kumHdA=,iv:0L+LZIeAzzYfKS4kG5KWrAQqgBoC/FLeic3m8lx5kls=,tag:JXk+2nAJT0o5exiyKg6OCw==,type:str]
-S3_SECRET_KEY=ENC[AES256_GCM,data:8GR4v7rLL+D30JkbazuxrDq0J+DzU1lutFGwn5VJmsMSSIgnB13QtNSh2yXNHIx6P2vVzUK7+4y3hMX8hCRoVw==,iv:miUA9Cakcs4n9o/Nq8I5msO4bVFBiPVw+abONUNS50U=,tag:5dZMMAkeMTWS3/m8d3bUzA==,type:str]
-#ENC[AES256_GCM,data:IKY1FUjvrQ==,iv:o3wmybvpWNy4ck3n308cf65Ymgsl4NZVcjC0BIPl774=,tag:Z5geKTBEDtwTsMgblbxYHw==,type:comment]
-BACKUP_S3_PREFIX=ENC[AES256_GCM,data:BRtyCGLVhRD1,iv:vSLjzAwYdwTsrj+dmcZNUCKt4bzrsJAr/y6YgJQFfyM=,tag:o0YrfPie6RmapWoFOEpmzw==,type:str]
-CF_ACCOUNT_ID=ENC[AES256_GCM,data:oUwbof7bTJF9R6hXVaepVZwoRcM04jsuifiuTHXYZcA=,iv:zS/HpnIa03NYsAKMNFKTIfmhLwqVctioF8u0lxePbXo=,tag:7RvBUgfKfiQs/ghdiQlwOg==,type:str]
-CF_API_TOKEN=ENC[AES256_GCM,data:yyn1DorN2jB1ZzQNooVFuMbAmlgpAeHVOD5CileKoJfGe5J76bGaxNcJJFLoJYHvztRiLV0=,iv:d1UYqCoiN0ficU848qNSHuCtCc+zg5IQcJg33Miq5GE=,tag:YVrJDhPoEDvD5Em5b7yobA==,type:str]
-#ENC[AES256_GCM,data:HazdGoFon9cRJlRpBhNshZxXb/o=,iv:lnrde4ZwuoU9TALA7sqZ4PG8fF7nwmfuy5IGR73YkaM=,tag:Qis8L+tI68YB+bzwVxwVAg==,type:comment]
-OIDC_ISSUER_URL=ENC[AES256_GCM,data:y7BOYQeMzgc3hrpQaRR2SbrUMBHY,iv:mqxpxl7uxhENYdtAq9oEZweF8hVXwHOe4CIPaxGrjEA=,tag:Fl6fUouzdPsa+A7P/mNbww==,type:str]
-OIDC_ISSUER_INTERNAL_URL=ENC[AES256_GCM,data:ewz0UA8h+Qm8ZX/fm6zRQ35L8M+a4vxbsn21Qm/hxHY=,iv:2yaS8I97rNO1IzBY1TbVew0k66/2IaxC77OojbzRGPo=,tag:e9IRiOAZTkafwzYGnU9F/w==,type:str]
-OIDC_CLIENT_ID=ENC[AES256_GCM,data:e1JtQRqun/OB5qo=,iv:u8ngNpfwPuoBUmEP7LDD1X37Zefceesh1sZhvDHGjyk=,tag:4MxrWpUNtXWC27DYJf/p5A==,type:str]
-OIDC_CLIENT_SECRET=ENC[AES256_GCM,data:Ndmt6Z7Pyb+lJP32wCGxsB/xhLNr992btFZCNC3NmOk6GgjiRozN0ToMUAN2kyMwWarpvF3qytHZgmjUmWxECA==,iv:PSbDpOTRjCQ6JZhLog0twdZkQFPzS3hUYZlsO0AoqRU=,tag:sE2KiiWrBMJqgGpJUK9llw==,type:str]
-AUTH_INTERNAL_API_KEY=ENC[AES256_GCM,data:rb5uBSRQm+36OyDhtxeCRk1k+MZ8oDmLzWjtJ2h8Q5I=,iv:LsAlcjQL6bo6WPnOmXU79dj13dW+KUDHi1r9nz76yoo=,tag:/psnnv5wtV3uSOd9rtDxdw==,type:str]
-OIDC_ACCOUNT_URL=ENC[AES256_GCM,data:/Ql8BIA4N7LlesbiFvAS0XARhwZV,iv:XXA0C6gZ9l17StxsU0VSmGoi7d06T9K7PW88EJp/tnA=,tag:2+fWnnAHhi9vvRMHFCgKPQ==,type:str]
-APP_URL=ENC[AES256_GCM,data:W8YCjhg2qwH6iR0Gao66tK9wXZp+,iv:6teKauGRn8tSB4yGF3dmQj0wwf/8h2gI6XPQY69z83c=,tag:BJg0GDPP3NXVixLnzTKYhQ==,type:str]
-sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0K25jb0h1VlBCZnV6eWJk\nSWpSbWFhVjF5M0xHTm1FODdualhiMDB2OFZRCmhseUR2ZkxxL0dkdUtUeXBTVE02\naWFqY3pKblVFNGN6S09GcWFnSnZxa2MKLS0tIFpISjlqdFN4R2EwWitOMHI0ZnVw\nZS9MSEZKL2Fvd2xKTlQza3hWMFYwYzQKyYv3gzjWQLjvh83DSWecC6X7YIYSJa/B\nkyuw6KhRAwPJNtRkDML6SWh6JMnG15cJO9uP65gi+PO6XjWmWbUvvg==\n-----END AGE ENCRYPTED FILE-----\n
-sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr
-sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrYjdqMUlyZkVGL1dnTWEy\nSEtZZVFRRjlnc2F6UGlKMERKTHdOVTI0Z2xnCmpRQnR1MjZvS0s0WkppQkpyV3dj\nbkJGUmYxU1NBRjVrUjFCclNnS3IwbVUKLS0tIGxqd0wvRVEvQ1JlcjJOWDlZd3RN\nUkFJdFFOTmVOcnBEcUFXcGNDT2NBN1EK15cvctDmsGLnUPElLqNt5t4Fd/ygBzJh\niCazrSUpIA8RDiBxUFn3qc//dXhLVACyasPaD/cAsMI1YnKCBLIbjg==\n-----END AGE ENCRYPTED FILE-----\n
-sops_age__list_1__map_recipient=age1ysddqggsx3h8zkv7xn3z26sjak5pqms6pyqhnky9ukrvpk7es5jsayz8w7
-sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzaGIvR0lQTUhGejdDUDdO\nazJpRlZ6Z2IvWXFTa3I5TVNDVm0zSkNTVmlNCmFGUk5DWFVqYVpEMkpBWEFScnVa\ncXR2YUJJZE5xa1A0RU1pYVRGT1lNVlEKLS0tIFdYRkRUVnAzSXVERko5bHVJVFVN\nMkFGbElWeXlXQWxpRTlUcEY3SFd2ZDQKUnfXknBPhlVYJgS30nge4kag3yJniuSB\nWUO4HvMDxkj8mXKsKGed2ny8cYkCZsqXOdxaWWGtk5GsUe/v9dUw/g==\n-----END AGE ENCRYPTED FILE-----\n
-sops_age__list_2__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy
-sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQm9VZENUZVg1bkpHQ200\nVjUvOWg3N0M0aTBFSTdrdG9EMjJwTTFvank4Ci9qaHB6OVphYksxR04xTWJiekJM\nYWRNWkpnTEpmR0NzczRHVk0wR1dkMDgKLS0tIFlZVFhuZ2U0MUlWOTYvbndFeHZF\nWTM1RUFoNzdaVmlXck9YWjhrV0tybjAKzVao/TJ3ZzAtxNfptKP0myEKRE9Tf8r5\n2HtfWp7su0l6/kHrUdlKf9PKe8k9ebuxEvSjreB/tj+BdwMZkWrg4w==\n-----END AGE ENCRYPTED FILE-----\n
-sops_age__list_3__map_recipient=age1qn0x93jhqjpqwvx5tgxnrwq5e3vuzur9whrkdnrvapd58esm45rqfkuxqh
-sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSZWRxM0lnekxWcEFJRGM4\nT2lkWlpTcVNJa1ZHRVJsbHl4QjVjWWxXQW1VCnNGZ3I2YUwxMnlYMXAzeEw3K1FN\namREblh2Mjc4SkIwTFRZVGsvSEhmV1EKLS0tIGN0UTdwdy9xR0I5MDFPa1dPeFoz\nYkhLeUkzQjJYNDNnTzdKNFpjM09LQzgKKRE/yB21pgXUe+kv0pw7UfjEs/RHDsAb\nhsPcj09HpAhauFzQxkIqFyOagshZ7c/OhAbfPwmZ2ycm5mLW8WOSRQ==\n-----END AGE ENCRYPTED FILE-----\n
-sops_age__list_4__map_recipient=age1h86dek80u5t677tsparz395uk3zvz4yuj9m5t2v2nsdfsvyjmafsra5yt7
-sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOdktDT21TWXBWdmZaNEdn\nOVFiNFQrTmIxVlBFMlJIVlZqRHo3NC9FSENnCjBqQWI3alpwbFFHYmJkZW1JaWoz\ndXdGSFo4R0RoK0g5Q2Z0QjNrYXBsVmMKLS0tIHdxNEJHTFZ5dk8zYUJRQWJVWFRJ\nWHllTnQ0MHhmdzZLNFVCMnRMS2Q4OG8KSR+VznPpiTPuM1PCgQqnHyJcgxwf69We\nrmcyR3jNbIgOv/re1ZhvtfQg2vJhF4fz7QvTyQWFZbIniFgCLKEcJw==\n-----END AGE ENCRYPTED FILE-----\n
-sops_age__list_5__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h
-sops_lastmodified=2026-08-06T18:36:16Z
-sops_mac=ENC[AES256_GCM,data:AaWybTpzzvC1IMHF7eM/kczSMCY+O66Tj0gxuSOQRho93zcrEl0CCNfsq7Ptk/g8Bl+TIVZywgAIiIHKXzKKliQQaKcn2ONtFI2/tFU9HCCjbod3EtD6rP5Uw5BtOhaozUVyQlYNmr/AquZb3KoWsE6Ci0NRmJ7ldh5pSiEfNbA=,iv:sZbxN6kiHyPaqi2SlzZqZewEYuEd8egCSFtuQO2RP/0=,tag:e8saJo/8cdvk5bdlTFLg9Q==,type:str]
-sops_unencrypted_suffix=_unencrypted
-sops_version=3.11.0
diff --git a/.env.prod-v2.enc b/.env.prod-v2.enc
new file mode 100644
index 0000000..16a6600
--- /dev/null
+++ b/.env.prod-v2.enc
@@ -0,0 +1,25 @@
+#ENC[AES256_GCM,data:uyDJVNkWT0y9PTSPX15CsqPxpqCLEutWb4qjKfhtC0BN17PKeVnXeH7A+1eaAQIPjwc8Q8PhGFayFb9VGWMqAh7p9lOeabLL2NfKG8xg432tCyyAazDZFn7hpT/GNAy20SUw/nJiah+i0w+X,iv:254kJ0MBxAkYsasc+asoK5mGMofzqqw2xahqXjIuP+w=,tag:7fUACA6CPX0mbGMlREOeUA==,type:comment]
+SIGNING_KEY=ENC[AES256_GCM,data:A4PAGMONiXri0uQNshY+ur2H1/DSTTkmNcEOh8EVA64PPjhqP7Kt0yuG0g==,iv:Zr0zSimpuW/+c66YO70oHgbpobdiqpbktbb4TGwfiJU=,tag:+tptSTSTlcGiHJIFf0+JcA==,type:str]
+LOCATION_KEY=ENC[AES256_GCM,data:iATgADyTGtQPQznCKg65km+7r0Kw8Pg7vDFp3Bbk3CWA57GZ4M9+4N1ZeQ==,iv:906m8zdHUf5R9VGZsW8HbDaZR4lxS9Cq/1tVr+fuGhM=,tag:SBJCCdodedt2CM/AI7VnQg==,type:str]
+SESSION_SECRET=ENC[AES256_GCM,data:tuILJNlEID8jYVMgQZJF8nrq4RDUfBfD1czDQu/uKVJclq+SrGwK2DYuZg==,iv:sHxO1eu0MvB1rcf1MhpoeX7r1D7jCiW82nmxNP3ovAE=,tag:wFhn34oC7wHlUtnHSfcq8w==,type:str]
+OIDC_CLIENT_SECRET=ENC[AES256_GCM,data:pnIuM3zHuGaUK2eOMYiX5stV5rrd+sICNWxplZYizaLWhiMQ08LR4ftv4SbxVvBVdCb+dwsQZsfQ/tG8NJfUPg==,iv:P6xz2mR1epra8PnO0Lm88GXQsF7lxSsHOP+daa+qwvo=,tag:xghe0n8nqAM4Fjo5Nbz32w==,type:str]
+R2_API_TOKEN=ENC[AES256_GCM,data:o1pmULq8mwcvSTOBWLkBVe7zKWV2CewsP85zexG3CVV1znCkHjhgSW6guqmNWapf8GtnJAg=,iv:DqhfFRBQaF6o4Sz7yRzdzbONEv/IIWPNYKrFYrbh+SQ=,tag:oep7noE6DjCoGkKuQYARvQ==,type:str]
+R2_ACCESS_KEY_ID=ENC[AES256_GCM,data:3ZtWCOe9S3MPjs4cEvRwV732FmuEH5izGGgS+ImBXLg=,iv:CqSvo/Mj76NV6XEB7iqocxYD4Af2zlRJVvaN+FAyO3A=,tag:sjuYfiqngGGFh++GQbRS2Q==,type:str]
+R2_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:PXkrTwKFyFRQgPzJ7QFPThsPqcTmsrYwJMdwQBmDPbyW2Jjf+u8m4xYe7KFT6t0iEM4AAvWBf+DzoYsRcyRPeg==,iv:cl/LUndMCodY904gmyIr0ohoJPmGH3yZHfxj+fFYm2A=,tag:t9u+7vEzWmlmSEtaHNpX7Q==,type:str]
+AUTH_INTERNAL_API_KEY=ENC[AES256_GCM,data:gLtDfJwLbEzzq/ojnn+c9ClXl6AP5oNie+XnMNfWijfzCVXbaVNxDZu35NHhkHVp2T6gRubR+jbd2b0AQC76ow==,iv:joRi9MGLWXbrkNEYpOh2x8ztsmcTxs/9+jw4R8JpFEo=,tag:k29GhDHkc6rp4P3mcpJuvg==,type:str]
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCSGpETTJkaFFBRTlWWGdE\nMlhGVU5SaUh4L3hQa3dDYWZleHFwTTZRdjJvClhMOCtVam5CMXJKT3BYNEpGQzVn\nUVFVZmNNbXJOODNqUXdCdzhGUHFQVmcKLS0tIDVvdDZ0MlUzaW9SNWJzcnZ5d1NX\nR1Awdk1TeGNPL0JsZjRIUk1lblBHblkKtBQfM03Ny/oKza4+p77SPE+B9H2HeHbI\njIcOsOXg3K7ZrN8Zqm146kU5A+zhPuJEuRy64g8xC35LYH+dUwbLtw==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr
+sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQZXRwY3lHM2F5bCtDMFpR\ndGpDdUpCWVNwb0NWUHgwdGNSbHp4ZzgwK1c0ClBVTzJka3JzK2tsM0F0MkpObTJF\nelhTdWhLZG1tV3RRUGgxT2VNNnNEaDgKLS0tIDArM1ZDdCttSmNyVWlKdk9qTDZF\nNTF3NzdVaXBDZit4RDlXNnhUcHdMaEkKZUK3FHDGsoKJHDai75Uz2sP9ZkvB3WBU\nupiRVOcBx6qcI+8A6LMehtx3Ei41NXN0Cs/WOv7ClzbQzgy27sK+Ug==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_1__map_recipient=age1ysddqggsx3h8zkv7xn3z26sjak5pqms6pyqhnky9ukrvpk7es5jsayz8w7
+sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHczBES0ErM0RCbmxSTFkv\nZWdvcDJqVlphSWxjaEI1cHhNUjI3ak1sK1FBCi84azdkYWZRcVR3ZEViR1B2cWdj\nMDBCeWhtN21wVXU2dURtaFAxS21rY0EKLS0tIHU5MzJGTTQ2RjA4WnBCSHJNZjk2\naWRONHBaUWJ2bk9sWnljQ01aYjdiUlUKIbhUc1duZ/Q38RZw3Ct1MLHz73HWjUx4\npCZXlpQc4ajzqzwd71FLhOuIAfRuD/kxAgOCI1rrBAuPcjJ9GWUlPQ==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_2__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy
+sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1b1E1bEtvWXZHOCtneGl2\nUHpaRGVkTUYzR05lL1g4enRZdHI3VS9PR3dFCmV2UzhwRURTMnhwb29ZdTQrMmpv\nYnREYStvZ1pVQlQ4UFp5eThGSVhMZTgKLS0tIGc0UjV3REQzWWtJYUhIUm5TWW5R\nZGNtOUk0amdLbkcySFUvTXFMWGE1YXMKM7/x6cmsD8v3GGIeer/OnIuix9OoJTzO\n4Dn8LQhpwhiQzMGOcpb6wKH3hlVtzvaScNnP684ExgeqJVDsFPeuPg==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_3__map_recipient=age1qn0x93jhqjpqwvx5tgxnrwq5e3vuzur9whrkdnrvapd58esm45rqfkuxqh
+sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlL0xhcWd5aGVpYmlSb095\nOGUzRStvdG5seURQMkg2L2l2Wi85dUNqb2tnCmJSTkRIRHQvZnB4c2YyTGFuOU5u\nWHBBU1VUZlpkRzRteXhNaWZuSE40aG8KLS0tIE8xdjN6cHp3N1BxM0hGVEwwVnhw\nakFmM281ZFRQT2I2dm1Zc1F3b29wWlEK6wp6sV/YqcQOvBDYw4/rGSXr+0qsNm06\nBVEKknx/zYMiYDKeQcIsLu9yIEHlURJ0atfop48BUkVsR6PbgmpgNQ==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_4__map_recipient=age1h86dek80u5t677tsparz395uk3zvz4yuj9m5t2v2nsdfsvyjmafsra5yt7
+sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNdHpWaU0ycGgwaUcxM0JK\nWmZxWlRvZnlPOEI2c3pJMyszK1gyWENZb0ZNCnpzTUNIZGpsZ2VpUWd5V2tPQnF2\nYjZnVG84Y3B2R01uYWFFbXdsTjYxdlEKLS0tIHBlYmFpd29xT3NLV1VjUCtUVmhR\nVW1YSDg0ZzBtQTd0R0dUQ29LZVdZR3MKeSP91xyZHIt6QXr/Lz7ZhWCn0JvX8gfX\nJg9cDsHdmsVRFdn2ipWP5Acx2qp/C5MosWENB3ITrgV9TEkJ7E9Jcw==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_5__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h
+sops_lastmodified=2026-10-05T18:44:52Z
+sops_mac=ENC[AES256_GCM,data:Cq6n0j7gJ5itoCtqhh4tRgvquH0TcgddlxgEB70wKghvNS8WPmoA55OSvOwTLbwvdGYcxjuIy/VukdJlSqZ2gu3wwaxbnlaJgSGBecIeWlgXk+1BnQu3Au+laECE+spEMSlmxh3rTBKtw26baIGivV6ojfUxOyDeByBtikt2lC4=,iv:zUbxFgvlqYHCRm2Ihp/DnBv5Fm5wV+6n1VSA8YG82tI=,tag:/bU46QR0pMaBKbCAwxuWYQ==,type:str]
+sops_unencrypted_suffix=_unencrypted
+sops_version=3.11.0
diff --git a/.env.staging.enc b/.env.staging.enc
new file mode 100644
index 0000000..7737a68
--- /dev/null
+++ b/.env.staging.enc
@@ -0,0 +1,25 @@
+#ENC[AES256_GCM,data:L18HvVhVLrA+ki80CVjZV9xs+DsOsEvszm+F9g2yTGZttt84xfWMIqJ3Zlo3b+3A4F/jEhZ8KPLfBqfp7ylI5QsOmfk8PR4zvrtFMHYE/AZB+Keiuin/tK+rag==,iv:x3yKH8w3vyOFcMVhvZ8/MTn0sa27vStt+x67t/YzN6E=,tag:Balm2ehHO8woo2OFrwDvHg==,type:comment]
+SIGNING_KEY=ENC[AES256_GCM,data:Uf9wdQf+EhbqfbufFyyYvwyn2XbKVneF3tU23OTEmfV4e/jvnsi9a0GVEA==,iv:wvaJeETb8ryvRLRQxPILmO23JcFEgPrbVqlZt+TxB1k=,tag:dHC2GV+JhSIKxV9VetbO9A==,type:str]
+LOCATION_KEY=ENC[AES256_GCM,data:srIB/K3yBVjTRd/2VivT6qaaGg83Fxa4VCd+Zk7H2O1L7xxrV1qLX1ofXQ==,iv:GqYlEr0SVb+X+EFL6Q+wszRhcUkakon3Z0TA95PKbNE=,tag:jrk9EZkixCmI8sHzSaDaxg==,type:str]
+SESSION_SECRET=ENC[AES256_GCM,data:M2/TE/R4+NDKyeWBCYN0lzkadg+2MaESsOPh9EoRQYxBGAhZHP4Lt3o+OQ==,iv:ClvpJl/soXCA8fPRmPo+io8b167q7WiODLlpAN27jfY=,tag:Rjhe41njbZ6LNIEQ7J143w==,type:str]
+OIDC_CLIENT_SECRET=ENC[AES256_GCM,data:qKP3W3LSShbJ6pQK0VjNWYYIab6OLha6kZoJM8sUkWXf2JHOL4ZWAtisDsVIkxjzcJWJJ55ZC/bK4sRlVDq8rQ==,iv:AWXZHqNR+jUL3+U7bLepAR6KbD9mwnNXAKuV2GiwIhU=,tag:UUhOqRg3WBhH5m7jenMYkw==,type:str]
+R2_API_TOKEN=ENC[AES256_GCM,data:6xecHO7T6D7npUiqIIHo9CectjDBNwm4VlS/ZGHNwLNqxIMyoHbN5XeKVgo7Taa4B701MNE=,iv:+iNkvek2zJNloaMPZpWMkryREm0naFnQHokbqhFjemA=,tag:i4S8brDoEiBa3gKq7ZqHZA==,type:str]
+R2_ACCESS_KEY_ID=ENC[AES256_GCM,data:ups38t8iyZuDi0k4X76sOh0Njd0F2SJn6ozIiTJOv3g=,iv:jxBqZBUPz81BRYCC34ZWR2heOh8N/ca0biFQab4sX/M=,tag:VlGZQ3M61bLKLhkS6isaYw==,type:str]
+R2_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:deaqHLFsI1dMX+daJCMhQ5FH6GkKc6Jec+ooo3tZz8nlc3HtxMYooHP0g8GbkIX7q2VBjQeOLwUraZzYWdBs2w==,iv:u/5e3cjkdeQyj7QWp7J2WBvUKE8klRYgBfxNmoQpRrs=,tag:cl3ztXz1aJjPTEPIyK7CLQ==,type:str]
+AUTH_INTERNAL_API_KEY=ENC[AES256_GCM,data:tUYoRLvtDCseoEJsVfaWOWPJqS37/2Mefvme7vbIOHEhPI6nTEimCmDZbV/SsYv/1SuSIW34toqcxH1Fnl7MIg==,iv:Urp+ZTkZLvTpybFAt4MMH6o9CQGICZjSIzjrb+AA1S4=,tag:I9lCrmiIuwBwIZww26/OfQ==,type:str]
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaWFJTTEZFRElBckZsSEE3\na1pGQzRac2FGOE1ZSDBmZDBodGZjRjFYNjNjCk9RNVVoWVNCNU5JV2FvVnM2NzJW\nSmhPU1dCN0REdFQrUjFCaGppVlAxS3cKLS0tIGt6OGdNNFhxWGRZT0pNbHY5QWtp\nOWJMUzliSkVqZE8rTStGWFF2eUpNRDAKJMxcuM9S+RNJA6M8cOoDKu21Cd/VLm5A\niHHFWPVgcaHSHVTQnzwbo0zHkE2ue/xhdctm1QD9nRe+AtFm0ogRzA==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr
+sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXek54eDFkVnBCNXkrdjMv\nalptTUVUTXAydDBJdjMwcWlKT2Izdm9GckNNCmw2L1o0dmZyWWVHTjhNd3JmVXM1\nVlVDSHk3ZXgzSmt5S0hYTytuaVEyaEkKLS0tIHpiQmU5K1BjSklYMHVLaW1yM2V6\neW4zeFJFeFEyYXBIMkZHSXFPc1JDUkEKXXzkYVfW0z5dhpRIfQYcKnNY3Z46+HDl\nSIL8E6qzYGB+Wmny0OHkhdy9/yGErCxrwJWH5RGjFt6NzjSvMwOUiw==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_1__map_recipient=age1ysddqggsx3h8zkv7xn3z26sjak5pqms6pyqhnky9ukrvpk7es5jsayz8w7
+sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQZzRiL3YwU2pybmFVL1k3\nczJ4L3ZkQ0ZsOGlWVlhqbWRMWVN2RUNnTlZzCjJldkdFTnBTcHA1YkkxREpac2xO\nMUVwNkxGVWNydlRFNEN5cGFQck5na28KLS0tIHZEaHhNSTIxcnh2TFkxcjRDczZl\nOUY3cjE1MFBLbFhlQ3hqVXZyWmQ3K2sKyLWvRb79PjFk683XW2SPRbHMX7eZerAi\n1hhjnE8n1V+EJvu6coB+/IG+WQPgT/t8JY/332RQswn/rw+AMwGx9w==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_2__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy
+sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTRTZDa2ZvaDcxZ2tpUTJj\nOGUrbkdkdjNJZmJQcVRkK21OTnUwbUl0d0VrCmNzQ2VOQjk4SDR1UWRrVDRHb3pa\nK0JBeHN5akd2QXhBOW9jVG5rRVRHNDgKLS0tIEFkSlRwNjZzZTdWd01mcnVQYU5h\nK0owb3V4VFZOeldmdVBndUo2NTdlQ28K3v4iS1YCiFPRsSo5fyw99l003KokQqpN\ntEMwdbtD61YPRRgFcrMmN71SkyjoNLUrX/5atBHnEaDkivRw3tCeqg==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_3__map_recipient=age1qn0x93jhqjpqwvx5tgxnrwq5e3vuzur9whrkdnrvapd58esm45rqfkuxqh
+sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1aHJMR01jbnBpdkU3TTdn\nV2Z1dk1aRW5oY1VQdC9KUEQ3NDN6alF0aTBJCnhBcEc0dlJGTWRKZ0dDQ3loWUwz\nUDJiZG1tZzltN0wvbUUyRFpYbDJKM3MKLS0tIDZMK1BySllJeGkxNTRscysrdHJE\nRHVIc1RvMHJFc1dRd3F2SUh1Z1hKZUkKA6vg2GmwBBmGk7UHcb1sbhZIf42Ds3k+\nmg0YaSE7k4iBqm44Lh3arFrmfaosCyPeYB1wyqXlxR+k0nvvijpzvQ==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_4__map_recipient=age1h86dek80u5t677tsparz395uk3zvz4yuj9m5t2v2nsdfsvyjmafsra5yt7
+sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMYzN2cTFMM1hDd3hyYmFp\nOWpoTEM2K3UwNnhjeEVScEVCTmlDSEJlMDBvCmsrZUEzYVhSTHBmSGZ0Mzg0QWtW\nb1YxM3N3VllYZ1VMcEJpc05KZG4xcWcKLS0tIElDT1ZGOWJxRVorNjRpRGZ5akJY\nTlloR0d1QnBMV0pxckRUVzc2bHJVa28KZYl/NOcr3pLFAn+4yaQDejqKJr5A2e0X\neDQuVacIicznbRcryp+IpGfdW8+H+BgXif6insXqjCz0lX2q/xopnw==\n-----END AGE ENCRYPTED FILE-----\n
+sops_age__list_5__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h
+sops_lastmodified=2026-10-04T16:46:10Z
+sops_mac=ENC[AES256_GCM,data:v/OHKSnYStOf//dP8FAnRNfUsn5ixX9GjKBNfJP6ByL9av5MyMUS7Pd7q8lDv/RhQHFn3v+KISOk7f9Ezp5/vLW+7+Rip/FRbViMuzEqwRjCkKqZNO79hzPIzYT+mvknirtDk6BO3NqaYAan/1vnRCwTq43TTSsMoW0swCUirZI=,iv:sizhCYXhA1u0TBlg2joo2W/CpJCXw0SdGqyYycDA4KE=,tag:oGulAwmz5BCr5Zfjbx2vdQ==,type:str]
+sops_unencrypted_suffix=_unencrypted
+sops_version=3.11.0
diff --git a/.env.test b/.env.test
deleted file mode 100644
index c93772b..0000000
--- a/.env.test
+++ /dev/null
@@ -1,14 +0,0 @@
-# Underlay
-NODE_ENV=development
-DATABASE_URL=postgresql://underlay:underlay@localhost:5432/underlay
-SESSION_SECRET=dev-secret-change-me
-
-# ARK
-ARK_DEFAULT_NAAN=12345
-
-# S3 (MinIO in dev)
-S3_BUCKET=underlay
-S3_REGION=us-east-1
-S3_ENDPOINT=http://localhost:9000
-S3_ACCESS_KEY=minioadmin
-S3_SECRET_KEY=minioadmin
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fef46aa..43eb47f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,9 +9,9 @@ run-name: >-
on:
push:
- branches: [main]
+ branches: [main, v2-edge]
pull_request:
- branches: [main]
+ branches: [main, v2-edge]
concurrency:
group: ci-${{ github.ref }}
@@ -21,7 +21,7 @@ jobs:
check:
name: Lint, typecheck, test
runs-on: ubuntu-latest
- timeout-minutes: 10
+ timeout-minutes: 20
steps:
- name: Checkout
@@ -52,3 +52,12 @@ jobs:
- name: Test
run: pnpm test
+
+ - name: Browser bundle
+ run: pnpm --filter @underlay/protocol check-browser
+
+ - name: CLI build
+ run: pnpm --filter @underlay/cli build && node packages/cli/dist/cli.js --help
+
+ - name: Web build and SSR smoke
+ run: pnpm --filter @underlay/web build && pnpm --filter @underlay/web smoke
diff --git a/.github/workflows/deploy-mirror.yml b/.github/workflows/deploy-mirror.yml
deleted file mode 100644
index b65ae20..0000000
--- a/.github/workflows/deploy-mirror.yml
+++ /dev/null
@@ -1,177 +0,0 @@
-name: Deploy Mirror
-
-run-name: 'Deploy mirror: ${{ github.sha }}'
-
-concurrency:
- group: deploy-mirror
- cancel-in-progress: true
-
-on:
- workflow_dispatch:
-
-env:
- REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
-
-permissions:
- contents: read
- packages: write
-
-jobs:
- deploy:
- name: Deploy Mirror
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout
- uses: actions/checkout@v4
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
-
- - name: Log in to GHCR
- uses: docker/login-action@v3
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract metadata
- id: meta
- uses: docker/metadata-action@v5
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
- - name: Set up Node.js
- uses: actions/setup-node@v4
- with:
- node-version: 24
-
- - name: Install pnpm
- run: corepack enable && corepack prepare pnpm@latest --activate
-
- - name: Install and typecheck
- run: |
- pnpm install --frozen-lockfile
- pnpm typecheck
-
- - name: Build and push
- uses: docker/build-push-action@v6
- with:
- context: .
- target: production
- push: true
- provenance: false
- sbom: false
- cache-from: type=gha
- cache-to: type=gha,mode=max
- tags: |
- ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:mirror-${{ github.sha }}
- ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:mirror-latest
- labels: ${{ steps.meta.outputs.labels }}
-
- - name: Start SSH agent
- uses: webfactory/ssh-agent@v0.9.0
- with:
- ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
- - name: Install sops
- run: |
- curl -LO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
- sudo mv sops-v3.9.4.linux.amd64 /usr/local/bin/sops
- sudo chmod +x /usr/local/bin/sops
-
- - name: Extract DEPLOY_HOST from env file
- id: host
- env:
- SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_SECRET_KEY }}
- run: |
- set -euo pipefail
- DEPLOY_HOST=$(sops -d --input-type dotenv --output-type dotenv .env.dev.enc | grep '^DEPLOY_HOST=' | cut -d= -f2)
- if [[ -z "$DEPLOY_HOST" ]]; then
- echo "::error::DEPLOY_HOST not found in .env.dev.enc"
- exit 1
- fi
- echo "deploy_host=${DEPLOY_HOST}" >> $GITHUB_OUTPUT
-
- - name: Add known hosts
- run: |
- mkdir -p ~/.ssh
- ssh-keyscan -H "${{ steps.host.outputs.deploy_host }}" >> ~/.ssh/known_hosts 2>/dev/null
-
- - name: Deploy over SSH
- env:
- SSH_USER: ${{ secrets.SSH_USER }}
- DEPLOY_HOST: ${{ steps.host.outputs.deploy_host }}
- REPO: ${{ github.repository }}
- BRANCH: ${{ github.ref_name }}
- GHCR_USER: ${{ secrets.GHCR_USER }}
- GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
- IMAGE_TAG: mirror-${{ github.sha }}
- run: |
- ssh "${SSH_USER}@${DEPLOY_HOST}" \
- "env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS'
- set -euo pipefail
-
- REPO="${1:?missing repo}"
- BRANCH="${2:-main}"
-
- : "${IMAGE_TAG:?missing IMAGE_TAG}"
- : "${GHCR_USER:?missing GHCR_USER}"
- : "${GHCR_TOKEN:?missing GHCR_TOKEN}"
-
- STACK_NAME="underlay-mirror"
- REPO_NAME="${REPO##*/}"
- APP_DIR="/srv/${REPO_NAME}-mirror"
- REPO_SSH="git@github.com:${REPO}.git"
-
- ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null
- chmod 600 ~/.ssh/known_hosts
-
- if [[ ! -d "${APP_DIR}/.git" ]]; then
- sudo mkdir -p "${APP_DIR}"
- sudo chown -R "$USER:$USER" "${APP_DIR}"
- git clone --branch "${BRANCH}" "${REPO_SSH}" "${APP_DIR}"
- fi
-
- cd "${APP_DIR}"
- git fetch --prune --tags origin
- git checkout "${BRANCH}"
- git pull origin "${BRANCH}"
-
- # Decrypt prod env to source S3 creds and API keys
- umask 077
- sops -d --input-type dotenv --output-type dotenv .env.prod.enc > .env
- set -a
- source <(grep -v '^#' .env | grep -v '^$')
- set +a
-
- # Init swarm if not already active
- if ! sudo docker info --format '{{.Swarm.LocalNodeState}}' | grep -qx active; then
- sudo docker swarm init
- fi
-
- echo "$GHCR_TOKEN" | sudo docker login ghcr.io -u "$GHCR_USER" --password-stdin
-
- sudo docker pull "ghcr.io/${REPO}:${IMAGE_TAG}"
-
- # Deploy using docker-compose.yml with mirror config.
- # S3 creds and API key are sourced from prod .env above.
- sudo env \
- IMAGE="ghcr.io/${REPO}" IMAGE_TAG="$IMAGE_TAG" \
- PORT=3002 APP_REPLICAS=1 \
- UNDERLAY_MODE=mirror \
- UNDERLAY_UPSTREAM="${UNDERLAY_UPSTREAM:-https://www.underlay.org}" \
- UNDERLAY_NODE_NAME="${UNDERLAY_NODE_NAME:-}" \
- S3_ENDPOINT="${S3_ENDPOINT:-}" S3_REGION="${S3_REGION:-auto}" \
- S3_ACCESS_KEY="${S3_ACCESS_KEY:-}" S3_SECRET_KEY="${S3_SECRET_KEY:-}" \
- S3_BUCKET="${S3_BUCKET:-underlay}" \
- S3_PUBLIC_BUCKET="${S3_PUBLIC_BUCKET:-underlaypublic}" \
- UNDERLAY_UPSTREAM_API_KEY="${UNDERLAY_UPSTREAM_API_KEY:-}" \
- SESSION_SECRET="$(openssl rand -hex 32)" \
- docker stack deploy -c docker-compose.yml \
- --with-registry-auth --resolve-image always --prune "${STACK_NAME}"
-
- sudo docker stack services "${STACK_NAME}"
- echo "Mirror deployed as ${STACK_NAME} (image: ${IMAGE_TAG})"
- EOS
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
deleted file mode 100644
index a071d74..0000000
--- a/.github/workflows/deploy.yml
+++ /dev/null
@@ -1,239 +0,0 @@
-name: Deploy
-
-run-name: >-
- ${{
- github.event_name == 'workflow_run' && format('Deploy dev: {0}', github.event.workflow_run.head_commit.message) ||
- github.event_name == 'release' && format('Deploy prod: {0}', github.event.release.tag_name) ||
- format('Deploy: {0}', github.sha)
- }}
-
-concurrency:
- group: >-
- deploy-${{
- github.event_name == 'release' && format('prod-{0}', github.event.release.tag_name) ||
- github.event_name == 'workflow_run' && format('ci-{0}', github.event.workflow_run.head_branch) ||
- github.ref
- }}
- cancel-in-progress: true
-
-on:
- workflow_run:
- workflows: [CI]
- types: [completed]
- branches: [main]
- release:
- types: [published]
- workflow_dispatch:
- inputs:
- no_cache:
- description: Build without Docker cache
- required: false
- default: false
- type: boolean
-
-env:
- REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
-
-permissions:
- contents: read
- packages: write
-
-jobs:
- deploy:
- name: Deploy
- runs-on: ubuntu-latest
- if: >-
- github.event_name == 'workflow_dispatch' ||
- github.event_name == 'release' ||
- (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
-
- steps:
- - name: Checkout
- uses: actions/checkout@v4
- with:
- ref: ${{ github.event.workflow_run.head_sha || github.sha }}
-
- - name: Set deployment vars
- id: vars
- run: |
- if [[ "${{ github.event_name }}" == "release" ]]; then
- echo "image_tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
- echo "env_file=.env.prod.enc" >> $GITHUB_OUTPUT
- echo "stack_name=underlay-prod" >> $GITHUB_OUTPUT
- else
- echo "image_tag=${{ github.sha }}" >> $GITHUB_OUTPUT
- echo "env_file=.env.dev.enc" >> $GITHUB_OUTPUT
- echo "stack_name=underlay-dev" >> $GITHUB_OUTPUT
- fi
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
-
- - name: Log in to GHCR
- uses: docker/login-action@v3
- with:
- registry: ${{ env.REGISTRY }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Extract metadata
- id: meta
- uses: docker/metadata-action@v5
- with:
- images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
- - name: Build and push
- uses: docker/build-push-action@v6
- with:
- context: .
- target: production
- push: true
- provenance: false
- sbom: false
- no-cache: ${{ inputs.no_cache || false }}
- cache-from: type=gha
- cache-to: type=gha,mode=max
- tags: |
- ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.vars.outputs.image_tag }}
- ${{ steps.meta.outputs.tags }}
- labels: ${{ steps.meta.outputs.labels }}
-
- - name: Start SSH agent
- uses: webfactory/ssh-agent@v0.9.0
- with:
- ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
- - name: Install sops
- run: |
- curl -LO https://github.com/getsops/sops/releases/download/v3.9.4/sops-v3.9.4.linux.amd64
- sudo mv sops-v3.9.4.linux.amd64 /usr/local/bin/sops
- sudo chmod +x /usr/local/bin/sops
-
- - name: Extract DEPLOY_HOST from env file
- id: host
- env:
- SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_SECRET_KEY }}
- run: |
- set -euo pipefail
- DEPLOY_HOST=$(sops -d --input-type dotenv --output-type dotenv "${{ steps.vars.outputs.env_file }}" | grep '^DEPLOY_HOST=' | cut -d= -f2)
- if [[ -z "$DEPLOY_HOST" ]]; then
- echo "::error::DEPLOY_HOST not found in ${{ steps.vars.outputs.env_file }}"
- exit 1
- fi
- echo "deploy_host=${DEPLOY_HOST}" >> $GITHUB_OUTPUT
-
- - name: Add known hosts
- run: |
- mkdir -p ~/.ssh
- ssh-keyscan -H "${{ steps.host.outputs.deploy_host }}" >> ~/.ssh/known_hosts 2>/dev/null
-
- - name: Deploy over SSH
- env:
- SSH_USER: ${{ secrets.SSH_USER }}
- DEPLOY_HOST: ${{ steps.host.outputs.deploy_host }}
- REPO: ${{ github.repository }}
- BRANCH: ${{ github.ref_name }}
- GHCR_USER: ${{ secrets.GHCR_USER }}
- GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
- IMAGE_TAG: ${{ steps.vars.outputs.image_tag }}
- ENV_FILE: ${{ steps.vars.outputs.env_file }}
- STACK_NAME: ${{ steps.vars.outputs.stack_name }}
- run: |
- ssh "${SSH_USER}@${DEPLOY_HOST}" \
- "env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' ENV_FILE='${ENV_FILE}' STACK_NAME='${STACK_NAME}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS'
- set -euo pipefail
-
- REPO="${1:?missing repo}"
- BRANCH="${2:-main}"
-
- : "${IMAGE_TAG:?missing IMAGE_TAG}"
- : "${GHCR_USER:?missing GHCR_USER}"
- : "${GHCR_TOKEN:?missing GHCR_TOKEN}"
- : "${STACK_NAME:?missing STACK_NAME}"
-
- REPO_NAME="${REPO##*/}"
- APP_DIR="/srv/${STACK_NAME}"
- REPO_SSH="git@github.com:${REPO}.git"
-
- ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null
- chmod 600 ~/.ssh/known_hosts
-
- if [[ ! -d "${APP_DIR}/.git" ]]; then
- sudo mkdir -p "${APP_DIR}"
- sudo chown -R "$USER:$USER" "${APP_DIR}"
- git clone --branch "${BRANCH}" "${REPO_SSH}" "${APP_DIR}"
- fi
-
- cd "${APP_DIR}"
- git fetch --prune --tags origin
- git checkout --detach "${IMAGE_TAG}"
-
- : "${ENV_FILE:?missing ENV_FILE}"
- umask 077
- sops -d --input-type dotenv --output-type dotenv "$ENV_FILE" > .env
-
- # Init swarm if not already active
- if ! sudo docker info --format '{{.Swarm.LocalNodeState}}' | grep -qx active; then
- sudo docker swarm init
- fi
-
- echo "$GHCR_TOKEN" | sudo docker login ghcr.io -u "$GHCR_USER" --password-stdin
-
- sudo docker pull "ghcr.io/${REPO}:${IMAGE_TAG}"
-
- # Deploy/update stack — export .env vars for compose interpolation
- # (docker stack deploy does NOT read .env files automatically)
- # IMAGE and IMAGE_TAG are set last to prevent .env from overriding them
- sudo env \
- $(grep -v '^#' .env | grep -v '^$' | xargs) \
- IMAGE="ghcr.io/${REPO}" IMAGE_TAG="${IMAGE_TAG}" \
- docker stack deploy -c docker-compose.yml \
- --with-registry-auth --resolve-image always --prune "${STACK_NAME}"
-
- # Verify the deployed image matches what was built
- DEPLOYED_IMAGE=$(sudo docker service inspect "${STACK_NAME}_app" --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}')
- echo "Deployed image: ${DEPLOYED_IMAGE}"
- if [[ "$DEPLOYED_IMAGE" != *"${IMAGE_TAG}"* ]]; then
- echo "::warning::Deployed image tag doesn't match expected ${IMAGE_TAG}"
- fi
-
- sudo docker stack services "${STACK_NAME}"
-
- # Wait for rollout
- wait_rollout() {
- local svc="$1" timeout="${2:-300}"
- local end=$((SECONDS + timeout))
- while (( SECONDS < end )); do
- local state
- state="$(sudo docker service inspect "$svc" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{end}}' 2>/dev/null || echo "")"
- echo " $svc: update_state=$state"
- if [[ "$state" == "rollback_started" || "$state" == "rollback_completed" ]]; then
- echo " ERROR: $svc rolled back!"
- sudo docker service ps "$svc" --no-trunc --format '{{.Name}} {{.CurrentState}} {{.Error}}' | head -10
- return 1
- fi
- if [[ "$state" == "completed" ]] || [[ -z "$state" ]]; then
- # Verify all running tasks are healthy (not just started)
- local unhealthy
- unhealthy="$(sudo docker service ps "$svc" --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -cv '^Running' || true)"
- if [[ "$unhealthy" == "0" ]]; then
- echo " $svc rollout complete"
- return 0
- fi
- fi
- sleep 10
- done
- echo "Rollout timeout for $svc"
- sudo docker service ps "$svc" --no-trunc --format '{{.Name}} {{.CurrentState}} {{.Error}}' | head -10
- return 1
- }
-
- wait_rollout "${STACK_NAME}_app" 300
- wait_rollout "${STACK_NAME}_cron" 120
-
- # Cleanup old images
- sudo docker image prune -a --filter "until=72h" -f
-
- echo "Deployed ${STACK_NAME} @ ${IMAGE_TAG} to $(hostname)"
- EOS
diff --git a/.gitignore b/.gitignore
index a3d73bf..accafc1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,9 +8,6 @@ package-lock.json
.DS_Store
coverage
data/
-*.db
-*.db-journal
-*.db-wal
planning/
.claude/
@@ -27,5 +24,7 @@ Thumbs.db
*.log
npm-debug.log*
-# pnpm store cache (created by in-container installs)
-.pnpm-store/
+
+# Cloudflare local state and dev secrets (v2)
+.wrangler/
+.dev.vars*
diff --git a/.oxlintrc.json b/.oxlintrc.json
index 4d256f8..6a78d54 100644
--- a/.oxlintrc.json
+++ b/.oxlintrc.json
@@ -3,7 +3,131 @@
"rules": {
"no-unused-vars": "error",
"no-console": "off",
- "eqeqeq": ["error", "always", { "null": "ignore" }]
+ "eqeqeq": [
+ "error",
+ "always",
+ {
+ "null": "ignore"
+ }
+ ]
},
- "ignorePatterns": ["node_modules", "dist"]
+ "ignorePatterns": ["node_modules", "dist"],
+ "overrides": [
+ {
+ "files": ["packages/protocol/src/**/*.ts"],
+ "rules": {
+ "no-restricted-imports": [
+ "error",
+ {
+ "patterns": [
+ {
+ "group": [
+ "drizzle-orm",
+ "drizzle-orm/*",
+ "@libsql/*",
+ "libsql",
+ "hono",
+ "hono/*",
+ "better-auth",
+ "better-auth/*",
+ "@better-auth/*",
+ "react",
+ "react-dom",
+ "react-router",
+ "@underlay/server",
+ "@underlay/server/*",
+ "@underlay/web",
+ "@underlay/web/*",
+ "@underlay/cli",
+ "@underlay/cli/*"
+ ],
+ "message": "@underlay/protocol loads in Node, Workers and browsers with no app around it: no database, server, auth or UI imports (plan decision 18)."
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "files": [
+ "packages/protocol/src/repo/**/*.ts",
+ "packages/protocol/src/stores/{memory,r2,s3}.ts"
+ ],
+ "rules": {
+ "no-restricted-imports": [
+ "error",
+ {
+ "patterns": [
+ {
+ "group": [
+ "drizzle-orm",
+ "drizzle-orm/*",
+ "@libsql/*",
+ "libsql",
+ "hono",
+ "hono/*",
+ "better-auth",
+ "better-auth/*",
+ "@better-auth/*",
+ "react",
+ "react-dom",
+ "react-router",
+ "@underlay/server",
+ "@underlay/server/*",
+ "@underlay/web",
+ "@underlay/web/*",
+ "@underlay/cli",
+ "@underlay/cli/*",
+ "**/stores/**",
+ "node:*"
+ ],
+ "message": "Repositories work over any Store, and stores other than fileStore run on every platform: no node: imports, and repo/ doesn't import a store (plan decision 18)."
+ }
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "files": [
+ "packages/protocol/src/{constants,file-refs,format,hash,input-rules,jcs,root,semver,sha256,utf8,validate}.ts",
+ "packages/protocol/src/tree/**/*.ts"
+ ],
+ "rules": {
+ "no-restricted-imports": [
+ "error",
+ {
+ "patterns": [
+ {
+ "group": [
+ "drizzle-orm",
+ "drizzle-orm/*",
+ "@libsql/*",
+ "libsql",
+ "hono",
+ "hono/*",
+ "better-auth",
+ "better-auth/*",
+ "@better-auth/*",
+ "react",
+ "react-dom",
+ "react-router",
+ "@underlay/server",
+ "@underlay/server/*",
+ "@underlay/web",
+ "@underlay/web/*",
+ "@underlay/cli",
+ "@underlay/cli/*",
+ "**/repo/**",
+ "**/stores/**",
+ "node:*"
+ ],
+ "message": "The format half of @underlay/protocol is pure: no stores, repositories, I/O or platform (plan decision 18)."
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
}
diff --git a/.sops.yaml b/.sops.yaml
index 47614a9..66c16f3 100644
--- a/.sops.yaml
+++ b/.sops.yaml
@@ -1,9 +1,9 @@
# SOPS configuration — specifies which age public keys can decrypt.
-# Matches .env.local, .env.prod, and .env.dev files (both plain and .enc encrypted).
+# Matches .env.local, .env.prod, .env.prod-v2, .env.dev and .env.staging files (both plain and .enc encrypted).
# Generate a keypair: age-keygen -o key.txt
creation_rules:
- - path_regex: \.env\.(local|prod|dev)(\.enc)?$
+ - path_regex: \.env\.(local|prod|prod-v2|dev|staging)(\.enc)?$
age: >-
age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr,
age1ysddqggsx3h8zkv7xn3z26sjak5pqms6pyqhnky9ukrvpk7es5jsayz8w7,
diff --git a/Caddyfile b/Caddyfile
deleted file mode 100644
index a2e9ca4..0000000
--- a/Caddyfile
+++ /dev/null
@@ -1,21 +0,0 @@
-# Host-level Caddy — manages TLS for all domains.
-# Deploy to /etc/caddy/Caddyfile on the server.
-# Reload with: systemctl reload caddy
-# NOTE: Use 127.0.0.1 (not localhost) — Docker Swarm publishes on IPv4 only.
-
-# Single Hono server handles both SSR and API on one port.
-
-dev.underlay.org {
- tls internal
- reverse_proxy 127.0.0.1:3000
-}
-
-www.underlay.org {
- tls internal
- reverse_proxy 127.0.0.1:3001
-}
-
-mirror.underlay.org {
- tls internal
- reverse_proxy 127.0.0.1:3002
-}
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 374a1ca..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,40 +0,0 @@
-# --- Dev stage ---
-FROM node:24-slim AS dev
-WORKDIR /app
-RUN corepack enable && corepack prepare pnpm@latest --activate
-RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
-COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml ./
-RUN pnpm install
-COPY . .
-CMD ["pnpm", "dev:app"]
-
-# --- Build stage ---
-FROM node:24-slim AS build
-WORKDIR /app
-RUN corepack enable && corepack prepare pnpm@latest --activate
-RUN apt-get update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
-COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml ./
-RUN pnpm install --frozen-lockfile
-COPY . .
-RUN pnpm build
-
-# --- Production stage ---
-FROM node:24-slim AS production
-WORKDIR /app
-RUN corepack enable && corepack prepare pnpm@latest --activate
-RUN apt-get update && apt-get install -y python3 make g++ curl && rm -rf /var/lib/apt/lists/*
-COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml ./
-RUN pnpm install --frozen-lockfile --prod && apt-get purge -y python3 make g++ && apt-get autoremove -y
-COPY --from=build /app/dist ./dist
-COPY --from=build /app/server.ts ./server.ts
-COPY --from=build /app/src/db ./src/db
-COPY --from=build /app/src/lib ./src/lib
-COPY --from=build /app/src/api ./src/api
-COPY --from=build /app/tools ./tools
-COPY --from=build /app/tsconfig.json ./tsconfig.json
-COPY --from=build /app/drizzle.config.ts ./drizzle.config.ts
-COPY --from=build /app/public ./public
-
-ENV NODE_ENV=production
-EXPOSE ${PORT:-3000}
-CMD ["node", "--import", "tsx/esm", "server.ts"]
diff --git a/README.md b/README.md
index 8597718..ec8a8e7 100644
--- a/README.md
+++ b/README.md
@@ -1,520 +1,75 @@
-
Underlay
+
Underlay
Underlay is a protocol for giving structured data a permanent address. You push JSON records and a JSON Schema. You get back a versioned, content-addressed snapshot you can point to forever.
-Every piece of content — records, schemas, and files — is identified by its SHA-256 hash. Versions are manifests that reference these hashes. Storage is deduplicated globally, transfers only move data the other side doesn't have, and provenance is built in: any record can be traced back to every collection and version that includes it.
+Every piece of content — records, schemas, and files — is identified by its SHA-256 hash. A version is a tree of those hashes with a single root hash, so versions share the data they have in common, transfers only move what the other side doesn't have, and provenance is built in: a record's hash finds the collections and versions that include it.
-Schemas are first-class objects: inspectable, comparable, and alignable across independently authored datasets. Two collections that independently define the same Author type produce the same schema hash — alignment falls out of the data model automatically. The infrastructure doesn't need to solve interoperability. It provides enough structure that interoperability can be solved dynamically by the tools and models that consume the data.
+Schemas are first-class objects: inspectable, comparable, and alignable across independently authored datasets. Two collections that independently publish an identical Author schema share its schema hash — alignment falls out of the data model automatically. The infrastructure doesn't need to solve interoperability. It provides enough structure that interoperability can be solved dynamically by the tools and models that consume the data.
-The protocol is simple: push records in, pull records out, trust the versions. The intelligence lives in the actors, not the store. The reference implementation runs at [underlay.org](https://underlay.org).
+The protocol is simple: push records in, pull records out, trust the versions. The intelligence lives in the actors, not the store. The reference implementation runs at [www.underlay.org](https://www.underlay.org).
Built by [Knowledge Futures](https://www.knowledgefutures.org), a 501(c)(3) public charity.
-## Quick Start
+## Repository layout
-### Prerequisites
+This branch is Underlay v2: one codebase that runs on Cloudflare Workers (D1, R2, Queues) and,
+for development and tests, on Node (SQLite, S3 or the filesystem). It is a pnpm workspace:
-- [Node.js](https://nodejs.org/) 24+ (LTS)
-- [Docker](https://www.docker.com/) and Docker Compose
+| Package | What it is |
+| ------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `packages/protocol` (`@underlay/protocol`) | The format: canonical JSON, hashing, validation, trees, version roots, the repository layout, the signed version log, tree sync and `fsck`. Stores for memory, the filesystem, S3 and R2. Runs in Node, Workers and browsers. |
+| `packages/server` (`@underlay/server`) | The Hono app: API, push and commit, files, jobs, billing counters, storage locations. Worker entry `src/worker.ts`, Node entry `src/node/main.ts`. Migrations in `drizzle/`. |
+| `packages/web` (`@underlay/web`) | The UI: React Router pages rendered on the server by both entries, plus the client bundle. |
+| `packages/cli` (`@underlay/cli`) | The command line: a local repository, pull by tree sync, push by delta push. |
+| `packages/migrate` (`@underlay/migrate`) | Converts a v1 (Postgres) instance into v2. |
-### Development
+`docs/protocol-v2.md` is the specification of Underlay protocol v2, with test vectors in
+`packages/protocol/test/vectors/`. `docs/v1-read-api.md` is the inventory of v1's read API that
+v2 was built against, and what v2 changes.
-```bash
-git clone https://github.com/knowledgefutures/underlay.git
-cd underlay
-./dev.sh
-```
-
-This starts:
+## Development
-- **PostgreSQL 17** on port 5433 (host) → 5432 (container)
-- **Underlay** on port 4100
-
-For team members with SOPS keys, the dev script auto-decrypts `.env.local` from `.env.local.enc`. External contributors should run `cp .env.test .env.local` first.
-
-### Without Docker
+Node 24 and pnpm 10.
```bash
pnpm install
-cp .env.test .env.local
-# Edit .env.local with your Postgres and S3 connection strings
-pnpm db:migrate
-pnpm db:seed
-pnpm dev:app
+pnpm typecheck # every package
+pnpm test # every package's tests
+pnpm lint && pnpm fmt:check
```
-### Default Seed User
-
-The seed script creates a "Knowledge Futures" org with sample collections.
-In production, user accounts are created automatically on first sign-in via [KF Auth](https://auth.knowledgefutures.org) (OIDC SSO).
-
-## Content-Addressed Storage
-
-Everything in Underlay is content-addressed by SHA-256:
-
-- **Records** are stored as objects in a global `record_objects` table, keyed by the hash of their canonical JSON (`{"id":...,"type":...,"data":...}`). The same record in ten collections is stored once.
-- **Schemas** are stored in a global `schemas` table, keyed by content hash. Two collections that define the same type share the same schema row.
-- **Files** are stored in S3, keyed by SHA-256 of their bytes.
-- **Versions** are manifests — join tables (`version_records`, `version_schemas`, `version_files`) that reference content by hash. Creating a new version that shares 99% of its records with the previous version adds only the new records to storage. `version_file_refs` indexes the `$file` references in a version's records at commit, so file access checks never scan record bodies.
-
-This architecture enables hash negotiation for push and pull (only transfer what the other side doesn't have), provenance (which collections contain this exact record), and forking (copy the manifest, not the data).
-
-## CLI
-
-The CLI wraps the same versioning logic as the server: hashing, diffing, semver derivation. Versions exist locally in a `.underlay/` directory. You can commit multiple times before pushing, inspect history offline, and push when ready.
+Run the app on Node with SQLite and blobs on disk (port 4200); migrations apply on start:
```bash
-pnpm cli init my-collection
-pnpm cli schema-set schema.json
-pnpm cli add records.jsonl
-pnpm cli status
-pnpm cli commit -m "initial load"
-pnpm cli remote add origin https://underlay.org -t ul_mykey -c my-org/my-collection
-pnpm cli push
-```
-
-The CLI source lives in `src/cli/`. For npm distribution, `packages/cli/` is a thin publish wrapper that uses esbuild to bundle into a standalone `@underlay/cli` package.
-
-### Local store format
-
-```
-.underlay/
- config.json # remotes (url, token, collection)
- HEAD # current version semver (e.g. v1.2.0)
- objects/ab/cd/abcd1234... # record content, keyed by hash
- schemas/ef/01/ef012345... # schema JSON, keyed by hash
- versions/v1.0.0.json # version manifest (schemas, records, files, semver)
- staging/records.jsonl # staged records before commit
- staging/schema.json # staged schema before commit
-```
-
-## Architecture
-
-| Layer | Technology |
-| ------------ | ------------------------------------------------------------- |
-| Server | Hono 4 + @hono/node-server |
-| Frontend | React 19 + React Router v7 (SSR + client hydration) |
-| Styling | Tailwind CSS 4 (@tailwindcss/vite) |
-| Build | Vite 6 (client + SSR bundles) |
-| Database | PostgreSQL 17 + Drizzle ORM |
-| File Storage | S3-compatible (Cloudflare R2 in production) |
-| Auth | KF Auth SSO (OIDC) for web sessions + API keys (programmatic) |
-| Deployment | Docker Swarm on Hetzner, Caddy reverse proxy, Cloudflare DNS |
-| CI/CD | GitHub Actions → GHCR → SSH → `docker stack deploy` |
-| Secrets | SOPS + age encryption |
-
-The app runs as a single Hono server on one port (default 3000). In dev, Vite runs in middleware mode for HMR. In production, Vite builds client and SSR bundles that Hono serves directly.
-
-## Project Structure
-
+pnpm --filter @underlay/web build
+DB_URL=file:/tmp/ul.sqlite BLOB_DIR=/tmp/ul-blobs npx tsx packages/server/src/node/main.ts
```
-server.ts # Hono entry point (API routes + SSR)
-vite.config.ts # Vite config (React, Tailwind, SSR)
-src/
-├── entry-client.tsx # Client hydration entry
-├── entry-server.tsx # SSR rendering (renderToPipeableStream)
-├── App.tsx # React Router routes (filesystem-based)
-├── route-gen.ts # Filesystem → route pattern conversion (wires *.data.ts loaders)
-├── global.css # Tailwind theme
-├── api/ # API route handlers
-│ ├── auth.server.ts # API auth middleware (API keys, internal tokens)
-│ ├── rate-limit.server.ts # Global API rate limiting (60/min anon, 5k/min authed)
-│ ├── accounts.ts # Account/org profiles, members, avatars
-│ ├── agent.ts # Agent share page (token-authenticated HTML instructions)
-│ ├── collections.ts # Collection CRUD + export, transfer, fork
-│ ├── discussion.ts # Page-anchored discussion threads
-│ ├── organizations.ts # Organization creation
-│ ├── webhooks.ts # Collection webhooks + delivery log
-│ ├── versions.ts # Version read APIs (manifest, records, diff) + privacy filtering
-│ ├── negotiate.ts # Push protocol: hash negotiation, record upload, commit
-│ ├── records.ts # Provenance + batch record fetch
-│ ├── files.ts # Content-addressed file storage
-│ ├── schemas.ts # Schema discovery, search, labeling
-│ ├── query.ts # SQL query tool (SQLite export + LLM SQL generation)
-│ ├── ark.ts # ARK identifier management
-│ ├── ark-middleware.server.ts # ARK resolution middleware
-│ ├── kf-summary.ts # Internal summary endpoint for KF dashboards
-│ ├── admin.ts # Admin endpoints (mirror mode)
-│ └── health.ts # Health check
-├── db/
-│ ├── schema.ts # Drizzle table definitions
-│ ├── client.server.ts # Database client
-│ ├── migrate.ts # Migration runner
-│ ├── seed.ts # Seed data (destructive with --force)
-│ ├── seedKfCollections.ts # Non-destructive seed of the KF sample collections
-│ ├── seed-helpers.ts # Record/schema insertion shared by both seeds
-│ └── migrations/ # Generated SQL migrations
-├── lib/
-│ ├── core/ # Pure functions shared by server and CLI (each with *.test.ts)
-│ │ ├── hash.ts # hashRecord, hashSchema (SHA-256)
-│ │ ├── semver.ts # deriveSemver
-│ │ ├── version-hash.ts # computeVersionHash, computePublicHash
-│ │ ├── privacy.ts # getPrivateTypes, getPrivateFields, filterRecordData
-│ │ ├── validate.ts # AJV schema validation
-│ │ ├── types.ts # Shared type definitions
-│ │ └── index.ts # Re-exports
-│ ├── version-helpers.server.ts # Re-exports core + DB-dependent helpers (collection access checks)
-│ ├── collection-access.ts # Access decisions behind those checks (write role, key scope, negotiate sessions)
-│ ├── auth.ts # better-auth config (KF Auth OIDC, API keys, orgs)
-│ ├── auth.server.ts # Session helpers
-│ ├── auth-client.ts # better-auth React client
-│ ├── auth-middleware.ts # React Router requireAuth middleware
-│ ├── auth-internal.server.ts # KF Auth internal API client (optional)
-│ ├── mirror-config.ts # Mirror mode config (UNDERLAY_* env vars)
-│ ├── mirror-sync.ts # Server-to-server mirroring
-│ ├── sqlite-gen.ts # Version → SQLite database generation
-│ ├── s3.ts # S3 client
-│ ├── webhooks.server.ts # Webhook delivery, retries, SSRF guard
-│ ├── slug.ts # Organization slug rules (reserved names)
-│ ├── query-params.ts # ?limit / ?offset parsing
-│ └── ark.ts # ARK identifier utilities
-├── cli/ # CLI source (local versioning + push/pull)
-│ ├── cli.ts # Commander entry point
-│ ├── commands/ # init, schema-set, add, status, commit, log, diff, remote, push, pull
-│ └── lib/ # Local store, config, staging helpers
-├── routes/ # React pages (filesystem routing; sibling *.data.ts = server loaders)
-│ ├── index.tsx # Landing page
-│ ├── explore.tsx # Browse public collections
-│ ├── dashboard.tsx # User's collections
-│ ├── protocol.tsx # Protocol specification
-│ ├── query.tsx # SQL query explorer
-│ ├── records/[hash].tsx # Record detail + provenance
-│ ├── schemas/ # Schema browser
-│ ├── settings/ # Account settings + API keys
-│ ├── docs/ # Documentation
-│ └── [owner]/ # Dynamic owner routes
-│ ├── index.tsx
-│ ├── settings/ # Org settings
-│ │ ├── index.tsx
-│ │ ├── members.tsx
-│ │ └── keys.tsx
-│ └── [collection]/
-│ ├── index.tsx
-│ ├── versions.tsx
-│ ├── schemas.tsx
-│ ├── v/[n].tsx
-│ ├── diff.tsx
-│ └── settings.tsx
-├── components/ # Shared React components
-packages/
-└── cli/ # npm publish wrapper (@underlay/cli)
- └── package.json # esbuild bundles src/cli → dist/cli.js
-public/
-├── llms.txt # Machine-readable API docs for LLMs
-tools/
-├── backupDb.ts # Postgres backup → S3
-├── restore.ts # Restore database from an S3 backup
-├── pruneBackups.ts # Retention pruning of old backups
-├── cleanupSessions.ts # Prune expired negotiate sessions; fail stranded finalizes
-├── pruneWebhookLogs.ts # Prune old webhook delivery logs
-├── verifyRecordSharing.ts # Check metadata-patch record sharing (needs a scratch DB)
-├── verifyFileRefs.ts # Check file access via version_file_refs matches the old scans (scratch DB)
-├── seedMirror.ts # Minimal seed for mirror instances
-└── cron.ts # Scheduled tasks (backup, prune backups, session cleanup, webhook-log pruning, mirror sync)
-```
-
-## Protocol and Documentation
-
-The protocol and the platform are documented together:
-
-| Resource | URL | Purpose |
-| ------------- | ------------------------------------------ | ------------------------------------------------------------------- |
-| Protocol spec | [/protocol](https://underlay.org/protocol) | Full protocol: data model, hashing, push, pull, provenance, privacy |
-| User docs | [/docs](https://underlay.org/docs) | Concepts, integration guide, API reference, quickstart |
-| llms.txt | [/llms.txt](https://underlay.org/llms.txt) | Machine-readable API docs for LLMs and bots |
-
-### Key API endpoints
-
-All pushes use the negotiate protocol — a three-step flow similar to git's pack negotiation. Two of
-those steps have a chunked form for collections that don't fit in a single request: the manifest can
-upload in pieces, and the commit can run in the background. A chunked, asynchronous push produces
-the same version hash as the simple one.
-
-A session tracks its progress in two counters on `negotiate_sessions` — `manifest_received` (distinct
-hashes in the manifest) and `manifest_needed` (of those, still awaiting a record). Each manifest
-chunk and records batch moves them by the rows it actually added or flipped, so the status poll and
-the commit checks never recount the manifest, however large the push.
-
-| Endpoint | Purpose |
-| ------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
-| `POST .../versions/negotiate` | Start a push session (server returns which hashes it needs) |
-| `POST .../versions/negotiate/:sessionId/manifest` | Upload the manifest in NDJSON chunks, when it is too large to send in one body |
-| `POST .../versions/negotiate/:sessionId/records` | Send only the needed records (NDJSON) |
-| `POST .../versions/negotiate/:sessionId/commit` | Validate, hash, and create the immutable version. `?async=true` returns 202 and finalizes in the background |
-| `GET .../versions/negotiate/:sessionId` | Session status, and the result or error of an async commit |
-| `GET .../versions/:semver/manifest` | Version manifest (add `?since=` for delta; both keyset-paginated) |
-| `GET .../versions/:semver/records` | Paginated records |
-| `GET .../versions/:semver/records.ndjson` | Every record streamed as NDJSON in one request — the bulk read path, resumable via `?after=` |
-| `GET .../export` | Version archive (`.tar.gz`), streamed as it's read; byte-identical per version. `?version=` picks one |
-| `GET .../versions/:semver/diff?from=...` | Diff between two versions |
-| `POST /api/records/batch` | Fetch records by hash (JSONL stream) |
-| `GET /api/records/:hash/provenance` | Find all collections containing a record |
-| `POST .../fork` | Fork a collection (copies manifest, not data; 403 for a non-member if the source holds private content) |
-| `GET /api/schemas` | Search schemas across all collections |
-
-## Privacy
-
-Privacy is part of the protocol, not just a hosted-instance feature. Four layers compose — a reader sees content only if it passes all four:
-
-- **Private collections**: `collections.public = false` — the whole collection 404s for non-members, and nothing below is evaluated. ARK identifiers into a private collection refuse to resolve.
-- **Private types**: `"private": true` on a schema root hides all records of that type from public readers.
-- **Private fields**: `"private": true` on a schema property strips that field from public responses.
-- **Private records**: `"private": true` on a **manifest entry** when pushing (not on the record body — a `private` key there is ignored) hides that specific record.
-
-Record-level privacy is stored **per version**, on that version's reference to the record (`version_records.private`), never on the globally deduplicated record object — so two collections holding byte-identical content can disagree about privacy. Two consequences follow: **privacy is re-declared on every push (omitting the flag means public, not "unchanged")**, and **redaction is forward-only** — older versions are immutable and keep serving the record, and a file stays downloadable while any older version references it publicly.
-
-The `private` flag is not part of the record hash — a record's content identity doesn't change when you change who can see it. Each version has two hashes: a **private hash** (all content, used by owners for integrity) and a **public hash** (excludes private types, fields, and records, verifiable by anyone). A version is identified by **both**: a push is a duplicate only when both match, which is what makes a privacy-only re-push a legitimate new version rather than a "no changes" conflict.
-
-Privacy filtering is implemented in `src/lib/core/privacy.ts` (pure functions) and enforced at the API layer in every module that returns record bodies, hashes, schemas, file lists, diffs, or counts: `versions.ts`, `collections.ts` (export, browse, fork), `records.ts`, `query.ts`, `files.ts`, `schemas.ts`, and `ark.ts`.
-
-### Access control
-
-- **API keys** carry `metadata.scope`. Because that metadata is client-supplied, self-service creation is clamped at `write` — requesting `admin` yields `write`, and admin keys are minted server-side only.
-- **Collection-scoped keys** (share links, agent links) carry `metadata.collectionIds` and are confined to those collections; account- and org-level endpoints refuse them outright.
-- **`?token=` capability URLs** authenticate `GET`/`HEAD` only, so a link prefetch can never drive a mutation. Share-link clients calling a POST send the token as a `Bearer` header.
-- **Files** are never served directly from storage: every read is access-checked and answered with a short-lived presigned URL.
-
-## Schema System
-
-Underlay uses **globally deduplicated, content-addressed schemas** for record validation and interoperability.
-
-- Each record type in a collection has its own JSON Schema, stored as an immutable, content-addressed row in the global `schemas` table.
-- A version declares its full set of type-to-schema bindings via the `version_schemas` join table.
-- If two collections define the same fields and types for a record type, they produce the same schema hash. Alignment is automatic.
-- Schemas are never modified. Evolving a type produces a new hash and a new row.
-
-### Push payload (negotiate)
-
-```json
-{
- "base_version": null,
- "schemas": {
- "Author": { "type": "object", "properties": { "name": { "type": "string" } } },
- "Pub": {
- "type": "object",
- "properties": {
- "title": { "type": "string" },
- "authorId": { "type": "string", "x-ref-type": "Author" }
- }
- }
- },
- "manifest": [{ "id": "auth-1", "type": "Author", "hash": "abc123..." }]
-}
-```
-
-The server replies with the record hashes it doesn't have; the client streams just those records (NDJSON) and commits.
-### Relationship annotations
+`pnpm --filter @underlay/server dev:node` runs the same entry under `tsx watch`.
+`packages/server/src/node/main.ts` lists the Node environment variables (S3, KF Auth, signing
+and location keys); without them it uses development secrets and a throwaway signing key, and
+sign-in points at a local KF Auth. The Node entry is for development only.
-Fields that hold record IDs of another type use `"x-ref-type": "TypeName"` to document the relationship. This enables linked-record navigation in the UI and helps LLMs understand the relational graph.
-
-### Schema labeling
-
-Schemas can be labeled post-hoc with human-readable names or URIs (e.g. `schema.org/Person`, `dc.author.v1`). Labels enable discovery across collections without upfront coordination.
-
-- `POST /api/schemas/:id/labels` - Add a label
-- `DELETE /api/schemas/:id/labels/:label` - Remove a label
-- `GET /api/schemas?label=...` - Search by label
-- Labels are injected as `x-underlay-labels` in schema exports (opt-out via `?raw=true`)
-
-### Versioning semantics
-
-- **Major bump**: Schema set changed (type added, removed, or schema modified)
-- **Minor bump**: Records changed, schema set identical
-- **Patch bump**: Only metadata changed (readme, message)
+Other checks CI runs: `pnpm --filter @underlay/protocol check-browser` (the browser bundle),
+`pnpm --filter @underlay/cli build`, and `pnpm --filter @underlay/web build && pnpm --filter
+@underlay/web smoke` (server-rendered pages).
## Deployment
-### Infrastructure
-
-- **Hetzner** - Single box (8 vCPU, 16GB RAM) running Docker Swarm
-- **Caddy** - Host-level reverse proxy, TLS via `tls internal` (Cloudflare Full mode)
-- **Cloudflare** - DNS + CDN + DDoS protection
-- **R2** - Object storage (zero egress fees), two buckets:
- - `S3_BUCKET` (**private** — public access disabled, no custom domain): `files/` content-addressed immutable uploads, `_backups/` compressed Postgres dumps. Every read is a short-lived presigned URL minted after an access check; there is no stable public file URL.
- - `S3_PUBLIC_BUCKET` (world-readable, fronted by `ASSETS_BASE_URL`): avatars and other inherently-public static assets.
-
-### Stacks
-
-Two Docker Swarm stacks run on the same box:
-
-| Stack | Domain | Host Port | Purpose |
-| --------------- | ---------------- | --------- | ---------- |
-| `underlay-prod` | www.underlay.org | 3001 | Production |
-| `underlay-dev` | dev.underlay.org | 3000 | Staging |
-
-Container-internal port is always 3000. Host port is configured via `PORT` in .env files.
-
-### CI/CD Flow
-
-1. Push to `main` → deploys to `dev.underlay.org`
-2. Create a release/tag → deploys to `www.underlay.org`
-3. Manual dispatch → choose environment
-
-The workflow: build Docker image → push to GHCR → decrypt env file for `DEPLOY_HOST` → SSH to server → `docker stack deploy` → wait for healthy rollout.
-
-Required GitHub secrets: `SSH_PRIVATE_KEY`, `SSH_USER`, `GHCR_USER`, `GHCR_TOKEN`, `SOPS_AGE_SECRET_KEY`.
-
-### Docker Compose Files
-
-| File | Purpose |
-| ----------------------------- | ---------------------------------------------- |
-| `docker-compose.yml` | Deployed stacks (prod & dev via Swarm) |
-| `docker-compose.local.yml` | Local development (source-mounted, hot reload) |
-| `docker-compose.withauth.yml` | Self-hosted: app + KF Auth + MinIO + Caddy |
-
-### Self-Hosting
-
-Run the Underlay with a bundled auth server (no external auth provider needed):
-
-```bash
-DOMAIN=https://my-instance.com docker compose -f docker-compose.withauth.yml up -d
-```
-
-This starts Postgres, KF Auth (auth + account), MinIO (S3-compatible storage), the Underlay app, and Caddy with automatic TLS. On first boot, an init container auto-generates all secrets (session keys, OAuth client credentials, S3 credentials).
-
-Optional configuration (via environment variables or `.env` file):
-
-- `SMTP_*` vars for email delivery (password resets, invitations)
-- `GITHUB_CLIENT_ID`/`GITHUB_CLIENT_SECRET` for GitHub login
-- `GOOGLE_CLIENT_ID`/`GOOGLE_CLIENT_SECRET` for Google login
-- `ORCID_CLIENT_ID`/`ORCID_CLIENT_SECRET` for ORCID login
-
-To use external S3 (AWS, Cloudflare R2, etc.) instead of bundled MinIO, remove the `minio` and `minio-init` services and set `S3_BUCKET`, `S3_REGION`, `S3_ENDPOINT`, `S3_ACCESS_KEY`, `S3_SECRET_KEY` in the app environment.
-
-Supporting files live in `selfhost/` (Caddyfile, Postgres init script). See [/docs/self-host](https://underlay.org/docs/self-host) for full details.
-
-## Environment Variables
-
-### Core
-
-| Variable | Description |
-| ---------------- | ------------------------------------------------------------------------------------------------------ |
-| `DATABASE_URL` | PostgreSQL connection string |
-| `SESSION_SECRET` | Secret for signing session cookies (**required in production** — the app throws at startup without it) |
-| `PORT` | Server port (default: 3000) |
-| `APP_URL` | Public base URL of this instance (default: `http://localhost:4100`) |
-
-### S3 storage
-
-| Variable | Description |
-| ------------------------ | --------------------------------------------------------------------------------------------------- |
-| `S3_BUCKET` | Private bucket for collection files and backups. Public access must be OFF |
-| `S3_PUBLIC_BUCKET` | World-readable bucket for avatars and similar assets (default: `underlaypublic`) |
-| `S3_PRESIGN_TTL_SECONDS` | Lifetime of presigned file-download URLs, in seconds (default: `300`) |
-| `S3_REGION` | S3 region (`auto` for R2) |
-| `S3_ENDPOINT` | S3 endpoint URL |
-| `S3_ACCESS_KEY` | S3 access key |
-| `S3_SECRET_KEY` | S3 secret key |
-| `ASSETS_BASE_URL` | Public base URL for uploaded assets like avatars (optional, default: `https://assets.underlay.org`) |
-
-### Auth (KF Auth OIDC)
-
-| Variable | Description |
-| -------------------------- | -------------------------------------------------------------------------------- |
-| `OIDC_ISSUER_URL` | KF Auth issuer URL |
-| `OIDC_ISSUER_INTERNAL_URL` | Issuer URL for server-to-server calls (optional, defaults to `OIDC_ISSUER_URL`) |
-| `OIDC_CLIENT_ID` | OAuth client ID (default: `kf_underlay`) |
-| `OIDC_CLIENT_SECRET` | OAuth client secret |
-| `OIDC_ACCOUNT_URL` | KF Account UI URL (account management links) |
-| `AUTH_INTERNAL_API_KEY` | Key for KF Auth's internal API (optional; also authenticates `/api/kf/summary`) |
-| `AUTH_INTERNAL_API_URL` | KF Auth internal API base URL (optional, defaults to `OIDC_ISSUER_INTERNAL_URL`) |
-| `INTERNAL_API_TOKEN` | Legacy `x-internal-token` for internal service calls (optional) |
-
-### Optional features
-
-| Variable | Description |
-| --------------------------- | ------------------------------------------------------------------------- |
-| `ARK_DEFAULT_NAAN` | Default NAAN for ARK identifiers (placeholder `12345` if unset) |
-| `CF_ACCOUNT_ID` | Cloudflare account ID for LLM-powered natural-language SQL (optional) |
-| `CF_API_TOKEN` | Cloudflare API token for LLM-powered natural-language SQL (optional) |
-| `UNDERLAY_MODE` | `origin` (default) or `mirror` — read-only mirror of an upstream instance |
-| `UNDERLAY_NODE_NAME` | Display name for this mirror node |
-| `UNDERLAY_UPSTREAM` | Upstream Underlay URL to mirror from |
-| `UNDERLAY_UPSTREAM_API_KEY` | API key for the upstream instance |
-| `UNDERLAY_SYNC_SCHEDULE` | Cron schedule for mirror sync (default: `0 0 * * 0`) |
-| `MIRROR_ADMIN_EMAILS` | Comma-separated emails allowed to use the mirror admin UI/API |
-| `CORS_ORIGINS` | Extra allowed CORS origins, comma-separated (APP_URL is always allowed) |
-| `MAX_FILE_UPLOAD_BYTES` | Max file upload size in bytes (default: 100 MB) |
-| `MAX_RECORDS_BATCH_BYTES` | Max body of one negotiate records batch in bytes (default: 128 MB) |
-
-`NODE_ENV` is set in `docker-compose.yml` `environment:` block (not in .env files).
-
-## Scripts
+Each deployment is a wrangler env in `packages/server/wrangler.jsonc` with its own Worker, D1
+database, bucket and queues: `staging` (staging.underlay.org) and `prod` (www.underlay.org):
```bash
-# Development
-pnpm dev # Start full local stack (Docker)
-pnpm dev:app # Start server without Docker
-pnpm build # Build for production (client + SSR)
-pnpm start # Start production server
-pnpm cli # Run CLI locally (e.g. pnpm cli init, pnpm cli add)
-
-# Code quality
-pnpm typecheck # TypeScript type checking
-pnpm lint # Lint with oxlint
-pnpm fmt # Format with oxfmt
-pnpm fmt:check # Check formatting
-pnpm test # Run tests (Vitest)
-pnpm test:watch # Run tests in watch mode
-
-# Database
-pnpm db:generate # Generate Drizzle migrations from schema changes
-pnpm db:migrate # Run pending migrations
-pnpm db:seed # Seed database
-pnpm db:seed-kf # Add the KF sample collections (non-destructive)
-
-# Tools
-pnpm tool:backup # Manual database backup to S3
-pnpm tool:restore # List S3 backups; restore one with `-- --yes`
-pnpm tool:pruneBackups # Prune old backups (supports `-- --dry-run`)
-pnpm tool:cleanupSessions # Prune expired negotiate sessions
-pnpm tool:pruneWebhookLogs # Prune old webhook delivery logs
-pnpm tool:verifyRecordSharing # Verify metadata-patch record sharing (scratch DB)
-pnpm tool:verifyFileRefs # Verify file access/listing via version_file_refs (scratch DB)
-pnpm tool:seed-mirror # Seed a mirror instance (admin org only)
-
-# Secrets (SOPS + age)
-pnpm secrets:encrypt:local # Encrypt .env.local → .env.local.enc
-pnpm secrets:encrypt:prod # Encrypt .env.prod → .env.prod.enc
-pnpm secrets:encrypt:dev # Encrypt .env.dev → .env.dev.enc
-pnpm secrets:decrypt:local # Decrypt .env.local.enc → .env.local
-pnpm secrets:decrypt:prod # Decrypt .env.prod.enc → .env.prod
-pnpm secrets:decrypt:dev # Decrypt .env.dev.enc → .env.dev
+pnpm --filter @underlay/web build
+cd packages/server
+npx wrangler d1 migrations apply underlay-staging --env staging --remote # new migrations only
+npx wrangler deploy --env staging
```
-## Maintenance Checklist
-
-When adding or changing features, update these locations:
-
-| What | Where | Purpose |
-| ----------------- | --------------------------------------- | ------------------------------------------ |
-| Protocol spec | `src/routes/protocol.tsx` | Protocol documentation page |
-| API documentation | `public/llms.txt` | Machine-readable docs for LLMs and bots |
-| Concepts | `src/routes/docs/concepts.tsx` | Core concepts explanation |
-| API reference | `src/routes/docs/api/*.tsx` | Endpoint-level docs with examples |
-| Integration guide | `src/routes/docs/integration.tsx` | Developer onboarding guide |
-| Quick start | `src/routes/docs/quickstart.tsx` | Getting started tutorial |
-| Self-hosting | `src/routes/docs/self-host.tsx` | Deployment instructions |
-| DB schema | `src/db/schema.ts` → `pnpm db:generate` | Schema changes need a migration |
-| Core library | `src/lib/core/` | Hashing, semver, privacy, validation |
-| CLI commands | `src/cli/commands/` | Local versioning and sync |
-| Schema discovery | `src/api/schemas.ts` | Schema search, labeling, cross-referencing |
-| Encrypted secrets | `.env.{local,dev,prod}.enc` | Re-encrypt after changing .env files |
-
-### Privacy features
-
-Privacy is part of the protocol. The system supports three levels (type-level, field-level, record-level) via `"private": true` annotations. When changing how privacy works, update:
-
-- `src/lib/core/privacy.ts` - pure filtering functions (shared by server and CLI)
-- `src/api/versions.ts` - API-level filtering
-- `src/api/files.ts` - file access checks
-- `src/api/schemas.ts` - public schema filtering
-- `src/routes/protocol.tsx` - protocol spec
-- `public/llms.txt` - Privacy section
-- `src/routes/docs/concepts.tsx` - Privacy section
-- `src/routes/docs/integration.tsx` - Privacy section
+Secrets are SOPS-encrypted per deployment (`.env.staging.enc`); decrypt them into the shell, never
+to a file. Add migrations with `cd packages/server && npx drizzle-kit generate --name `;
+never edit one that a deployment has applied.
## License
-MIT
+MIT. See [LICENSE](LICENSE).
diff --git a/dev.sh b/dev.sh
deleted file mode 100755
index d094d6b..0000000
--- a/dev.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cd "$(dirname "$0")"
-
-# Decrypt local env if needed
-if [[ -f .env.local.enc ]] && [[ ! -f .env.local ]]; then
- sops -d --input-type dotenv --output-type dotenv --output .env.local .env.local.enc
-fi
-
-# Load env vars
-set -a
-[[ -f .env.local ]] && source .env.local
-set +a
-
-# Find an available port, incrementing from PORT (default 4100)
-BASE_PORT="${PORT:-4100}"
-PORT="$BASE_PORT"
-while lsof -iTCP:"$PORT" -sTCP:LISTEN -t &>/dev/null; do
- ((PORT++))
-done
-if [[ "$PORT" -ne "$BASE_PORT" ]]; then
- echo "Port $BASE_PORT in use, using $PORT"
-fi
-export PORT
-
-trap "docker compose -f docker-compose.local.yml down" EXIT
-
-docker compose --env-file .env.local -f docker-compose.local.yml up --build --attach app
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
deleted file mode 100644
index 42caeee..0000000
--- a/docker-compose.local.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-# Local development compose — source-mounted for fast reload.
-# Start with: ./dev.sh
-# Access at localhost:${PORT:-4100}
-
-name: underlay-local
-
-services:
- db:
- image: postgres:17-alpine
- environment:
- POSTGRES_USER: underlay
- POSTGRES_PASSWORD: underlay
- POSTGRES_DB: underlay
- # Sized for bulk ingest as well as everyday dev. A multi-million-record push
- # sorts the whole record set twice to fold the version digests, and builds
- # indexes over it — at 8MB work_mem that spills to disk in many small merge
- # passes. max_connections is low, so the worst case here is bounded.
- command: >
- postgres
- -c shared_buffers=1GB
- -c effective_cache_size=3GB
- -c work_mem=64MB
- -c maintenance_work_mem=512MB
- -c max_wal_size=4GB
- -c checkpoint_completion_target=0.9
- -c max_connections=50
- # Matches the production stack: Docker's 64 MB /dev/shm default is too small
- # for Postgres parallel query workers once tables get large.
- shm_size: 1gb
- ports:
- - '5433:5432'
- volumes:
- - pgdata:/var/lib/postgresql/data
- healthcheck:
- test: ['CMD-SHELL', 'pg_isready -U underlay']
- interval: 5s
- timeout: 3s
- retries: 5
-
- app:
- build:
- context: .
- dockerfile: Dockerfile
- target: dev
- ports:
- - '${PORT:-4100}:${PORT:-4100}'
- - '24688:24688'
- volumes:
- - .:/app
- - app_node_modules:/app/node_modules
- env_file:
- - .env.local
- environment:
- PORT: ${PORT:-4100}
- DATABASE_URL: postgresql://underlay:underlay@db:5432/underlay
- command: sh -c "pnpm db:migrate && pnpm db:seed && pnpm dev:app"
- depends_on:
- db:
- condition: service_healthy
-
-volumes:
- pgdata:
- app_node_modules:
diff --git a/docker-compose.withauth.yml b/docker-compose.withauth.yml
deleted file mode 100644
index 22cd682..0000000
--- a/docker-compose.withauth.yml
+++ /dev/null
@@ -1,240 +0,0 @@
-# docker-compose.withauth.yml — Self-hosted Underlay with bundled auth.
-#
-# Runs: Underlay app + KF Auth (auth + account) + Postgres + MinIO (S3) + Caddy
-# One command: docker compose -f docker-compose.withauth.yml up
-#
-# First run generates secrets automatically via the init container.
-# Set DOMAIN=https://your-domain.com in your shell or .env file.
-#
-# To use external S3 instead of bundled MinIO, remove the minio service and set
-# S3_BUCKET, S3_REGION, S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY in the app
-# environment block (or pass them as env vars that the init container writes
-# into .env.app).
-
-name: underlay-withauth
-
-services:
- # --- Init container: generates secrets + config ---
- withauth-init:
- image: alpine:3.20
- entrypoint: /bin/sh
- command:
- - -c
- - |
- if [ -f /config/.env.withauth ]; then
- echo "Config already exists, skipping init."
- exit 0
- fi
- apk add --no-cache openssl
- AUTH_SECRET=$$(openssl rand -hex 32)
- SESSION_SECRET=$$(openssl rand -hex 32)
- CLIENT_SECRET=$$(openssl rand -hex 32)
- INTERNAL_KEY=$$(openssl rand -hex 32)
- MINIO_ACCESS=$$(openssl rand -hex 16)
- MINIO_SECRET=$$(openssl rand -hex 32)
- printf '%s\n' \
- "BETTER_AUTH_SECRET=$$AUTH_SECRET" \
- "BETTER_AUTH_URL=$${DOMAIN:-http://localhost}/auth" \
- "ACCOUNT_URL=$${DOMAIN:-http://localhost}/account" \
- "DATABASE_URL=postgres://kfauth:kfauth@postgres:5432/kfauth" \
- "SMTP_HOST=$${SMTP_HOST:-localhost}" \
- "SMTP_PORT=$${SMTP_PORT:-25}" \
- "SMTP_FROM=$${SMTP_FROM:-noreply@localhost}" \
- "SMTP_USER=$${SMTP_USER:-}" \
- "SMTP_PASS=$${SMTP_PASS:-}" \
- "KF_INTERNAL_API_KEY=$$INTERNAL_KEY" \
- "BASE_PATH=/auth" \
- "APPS_REGISTRY_FILE=/config/apps.withauth.yaml" \
- "GITHUB_CLIENT_ID=$${GITHUB_CLIENT_ID:-}" \
- "GITHUB_CLIENT_SECRET=$${GITHUB_CLIENT_SECRET:-}" \
- "GOOGLE_CLIENT_ID=$${GOOGLE_CLIENT_ID:-}" \
- "GOOGLE_CLIENT_SECRET=$${GOOGLE_CLIENT_SECRET:-}" \
- "ORCID_CLIENT_ID=$${ORCID_CLIENT_ID:-}" \
- "ORCID_CLIENT_SECRET=$${ORCID_CLIENT_SECRET:-}" \
- "AUTH_SERVICE_NAME=$${AUTH_SERVICE_NAME:-Underlay Auth}" \
- > /config/.env.withauth
- printf '%s\n' \
- "- client_id: underlay" \
- " client_secret: $$CLIENT_SECRET" \
- " redirect_uris:" \
- " - $${DOMAIN:-http://localhost}/api/auth/oauth2/callback/kf-auth" \
- " skip_consent: true" \
- " display_name: \"Underlay\"" \
- " allow_sign_up: true" \
- > /config/apps.withauth.yaml
- printf '%s\n' \
- "SESSION_SECRET=$$SESSION_SECRET" \
- "OIDC_ISSUER_URL=$${DOMAIN:-http://localhost}/auth" \
- "OIDC_ISSUER_INTERNAL_URL=http://auth:3000" \
- "OIDC_ACCOUNT_URL=$${DOMAIN:-http://localhost}/account" \
- "OIDC_CLIENT_ID=underlay" \
- "OIDC_CLIENT_SECRET=$$CLIENT_SECRET" \
- "AUTH_INTERNAL_API_KEY=$$INTERNAL_KEY" \
- "DATABASE_URL=postgres://kfauth:kfauth@postgres:5432/app" \
- "S3_BUCKET=underlay" \
- "S3_PUBLIC_BUCKET=underlaypublic" \
- "S3_REGION=us-east-1" \
- "S3_ENDPOINT=http://minio:9000" \
- "S3_ACCESS_KEY=$$MINIO_ACCESS" \
- "S3_SECRET_KEY=$$MINIO_SECRET" \
- > /config/.env.app
- printf '%s\n' \
- "MINIO_ROOT_USER=$$MINIO_ACCESS" \
- "MINIO_ROOT_PASSWORD=$$MINIO_SECRET" \
- > /config/.env.minio
- echo "Init complete."
- volumes:
- - withauth-config:/config
- environment:
- - DOMAIN
- - SMTP_HOST
- - SMTP_PORT
- - SMTP_FROM
- - SMTP_USER
- - SMTP_PASS
- - GITHUB_CLIENT_ID
- - GITHUB_CLIENT_SECRET
- - GOOGLE_CLIENT_ID
- - GOOGLE_CLIENT_SECRET
- - ORCID_CLIENT_ID
- - ORCID_CLIENT_SECRET
- - AUTH_SERVICE_NAME
-
- # --- Shared Postgres (two databases: kfauth + app) ---
- postgres:
- image: postgres:16-alpine
- environment:
- POSTGRES_USER: kfauth
- POSTGRES_PASSWORD: kfauth
- POSTGRES_DB: kfauth
- volumes:
- - pgdata:/var/lib/postgresql/data
- - ./selfhost/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
- healthcheck:
- test: ['CMD-SHELL', 'pg_isready -U kfauth']
- interval: 5s
- timeout: 5s
- retries: 10
-
- # --- MinIO (S3-compatible object storage) ---
- # Remove this service if using external S3/R2 — set S3_* vars in app environment instead.
- minio:
- image: minio/minio:latest
- depends_on:
- withauth-init:
- condition: service_completed_successfully
- volumes:
- - minio-data:/data
- - withauth-config:/config:ro
- entrypoint: /bin/sh
- command:
- - -c
- - |
- set -a && . /config/.env.minio && set +a
- exec minio server /data --console-address ":9001"
- healthcheck:
- test: ['CMD', 'mc', 'ready', 'local']
- interval: 5s
- timeout: 5s
- retries: 10
-
- # Create the default bucket on first run
- minio-init:
- image: minio/mc:latest
- depends_on:
- minio:
- condition: service_healthy
- withauth-init:
- condition: service_completed_successfully
- volumes:
- - withauth-config:/config:ro
- entrypoint: /bin/sh
- command:
- - -c
- - |
- set -a && . /config/.env.minio && set +a
- mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
- # Two buckets by design:
- # underlay — PRIVATE. Collection files; every read goes through the
- # API, which access-checks and then hands out a
- # short-lived presigned URL. Never make this public.
- # underlaypublic — world-readable assets (org avatars) served directly.
- mc mb --ignore-existing local/underlay
- mc mb --ignore-existing local/underlaypublic
- mc anonymous set download local/underlaypublic
- echo "Buckets ready."
-
- # --- Auth server (kf-auth) ---
- auth:
- image: ghcr.io/knowledgefutures/kf-auth:latest
- depends_on:
- postgres:
- condition: service_healthy
- withauth-init:
- condition: service_completed_successfully
- environment:
- NODE_ENV: production
- PORT: 3000
- volumes:
- - withauth-config:/config:ro
- command: sh -c "set -a && . /config/.env.withauth && set +a && node dist/server.js"
-
- # --- Account server (kf-auth account UI) ---
- account:
- image: ghcr.io/knowledgefutures/kf-auth:latest
- depends_on:
- postgres:
- condition: service_healthy
- withauth-init:
- condition: service_completed_successfully
- environment:
- NODE_ENV: production
- PORT: 3001
- volumes:
- - withauth-config:/config:ro
- command: sh -c "set -a && . /config/.env.withauth && set +a && node dist/server-account.js"
-
- # --- Underlay app ---
- app:
- image: ghcr.io/knowledgefutures/underlay:latest
- depends_on:
- postgres:
- condition: service_healthy
- withauth-init:
- condition: service_completed_successfully
- auth:
- condition: service_started
- minio-init:
- condition: service_completed_successfully
- environment:
- NODE_ENV: production
- PORT: 4100
- APP_URL: ${DOMAIN:-http://localhost}
- volumes:
- - withauth-config:/config:ro
- command: sh -c "set -a && . /config/.env.app && set +a && node --import tsx/esm server.ts"
-
- # --- Caddy reverse proxy ---
- caddy:
- image: caddy:2-alpine
- ports:
- - '80:80'
- - '443:443'
- volumes:
- - ./selfhost/Caddyfile:/etc/caddy/Caddyfile:ro
- - caddy-data:/data
- - caddy-config:/config
- environment:
- DOMAIN: ${DOMAIN:-localhost}
- APP_PORT: '4100'
- depends_on:
- - auth
- - account
- - app
-
-volumes:
- pgdata:
- minio-data:
- withauth-config:
- caddy-data:
- caddy-config:
diff --git a/docker-compose.yml b/docker-compose.yml
deleted file mode 100644
index d64f78b..0000000
--- a/docker-compose.yml
+++ /dev/null
@@ -1,187 +0,0 @@
-# Stack file — deploy with: docker stack deploy -c docker-compose.yml
-# SOPS decrypts .env.{prod,dev}.enc → .env before this runs (see deploy workflow).
-
-services:
- app:
- image: ${IMAGE:-ghcr.io/knowledgefutures/underlay}:${IMAGE_TAG:-latest}
- env_file:
- - .env
- environment:
- NODE_ENV: production
- # Push commits accumulate per-record structures in memory, so the heap has
- # to scale with the largest collection pushed, not with steady-state
- # traffic. 448 MB could not commit much past ~100k records.
- #
- # Measured: a 500k-record push costs ~900 MB of heap above baseline
- # (~1.8 KB/record), so 2 GB carries a ~1M-record push with headroom.
- # Raise APP_HEAP_MB (and APP_MEMORY_LIMIT with it) for a one-off larger
- # ingest rather than making it the standing default — dev and prod share
- # one 16 GB box, so the defaults here are paid four times over.
- #
- # Do not read this as "3.11M records will fit if we go high enough": at
- # that size the push path needs the streaming commit, not a bigger number.
- # See planning/local/demos/arxiv-ingest-measurements-v1.md.
- NODE_OPTIONS: '--max-old-space-size=${APP_HEAP_MB:-2048}'
- PORT: ${PORT:-3000}
- UNDERLAY_MODE: ${UNDERLAY_MODE:-origin}
- UNDERLAY_UPSTREAM: ${UNDERLAY_UPSTREAM:-}
- UNDERLAY_NODE_NAME: ${UNDERLAY_NODE_NAME:-}
- UNDERLAY_SYNC_SCHEDULE: ${UNDERLAY_SYNC_SCHEDULE:-0 0 * * 0}
- ports:
- - '${PORT:-3000}:${PORT:-3000}'
- deploy:
- replicas: ${APP_REPLICAS:-2}
- resources:
- limits:
- # Must exceed APP_HEAP_MB — the V8 heap is only part of RSS.
- #
- # Sizing on the shared 16 GB box: dev + prod × 2 replicas = 4 app
- # containers. They idle at a few hundred MB each and only one climbs
- # at a time (a push is one request on one replica of one stack), so
- # the realistic peak is 3× idle + 1× limit, not 4× limit. Reservations
- # stay low deliberately — they are what Swarm schedules against, and
- # over-reserving would strand memory the other stack needs.
- memory: ${APP_MEMORY_LIMIT:-2560m}
- cpus: '1.0'
- reservations:
- memory: 384m
- cpus: '0.25'
- update_config:
- parallelism: 1
- delay: 10s
- order: start-first
- failure_action: rollback
- rollback_config:
- parallelism: 1
- order: start-first
- restart_policy:
- condition: on-failure
- tmpfs:
- - /tmp:size=64m
- healthcheck:
- test:
- [
- 'CMD-SHELL',
- 'node -e "Promise.all([fetch(\"http://127.0.0.1:${PORT:-3000}/api/health\"),fetch(\"http://127.0.0.1:${PORT:-3000}/\")]).then(rs=>{for(const r of rs)if(!r.ok)throw r.status}).catch(()=>process.exit(1))"',
- ]
- interval: 30s
- timeout: 10s
- retries: 3
- start_period: 45s
- logging:
- driver: json-file
- options:
- max-size: '10m'
- max-file: '3'
- networks:
- - appnet
-
- cron:
- image: ${IMAGE:-ghcr.io/knowledgefutures/underlay}:${IMAGE_TAG:-latest}
- env_file:
- - .env
- environment:
- NODE_ENV: production
- NODE_OPTIONS: '--max-old-space-size=128'
- UNDERLAY_MODE: ${UNDERLAY_MODE:-origin}
- UNDERLAY_UPSTREAM: ${UNDERLAY_UPSTREAM:-}
- UNDERLAY_NODE_NAME: ${UNDERLAY_NODE_NAME:-}
- UNDERLAY_SYNC_SCHEDULE: ${UNDERLAY_SYNC_SCHEDULE:-0 0 * * 0}
- command: ['node', '--import', 'tsx/esm', 'tools/cron.ts']
- deploy:
- replicas: 1
- resources:
- limits:
- memory: 192m
- cpus: '0.5'
- reservations:
- memory: 64m
- cpus: '0.1'
- restart_policy:
- condition: any
- tmpfs:
- - /tmp:size=16m
- logging:
- driver: json-file
- options:
- max-size: '5m'
- max-file: '3'
- networks:
- - appnet
-
- postgres:
- image: postgres:17-alpine
- env_file:
- - .env
- environment:
- POSTGRES_USER: ${POSTGRES_USER:-underlay}
- POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-underlay}
- POSTGRES_DB: ${POSTGRES_DB:-underlay}
- command: >
- -c shared_buffers=1GB
- -c effective_cache_size=3GB
- -c work_mem=32MB
- -c maintenance_work_mem=256MB
- -c max_connections=200
- -c max_wal_size=2GB
- -c min_wal_size=256MB
- -c wal_buffers=32MB
- -c checkpoint_completion_target=0.9
- -c random_page_cost=1.1
- volumes:
- - pgdata:/var/lib/postgresql/data
- # Postgres allocates dynamic shared memory in /dev/shm for parallel query
- # workers. Docker's 64 MB default is enough for small scans and not enough
- # for a parallel aggregate over a few hundred thousand rows, which fails
- # with "could not resize shared memory segment ... No space left on device".
- #
- # This must be the long-form tmpfs *volume*. `shm_size` is ignored by
- # `docker stack deploy`, and the short `tmpfs:` list syntax documents only
- # mode/uid/gid — not size. An earlier `tmpfs: - /dev/shm:size=1g` here was
- # therefore silently a no-op, the 64 MB default stayed in place on dev, and
- # a 300k-record commit died on it.
- #
- # 1 GiB, in bytes: the field takes a raw integer or a unit string.
- - type: tmpfs
- target: /dev/shm
- tmpfs:
- size: 1073741824
- ports:
- - '${DB_PORT:-5432}:5432'
- networks:
- - appnet
- - dbaccess
- stop_grace_period: 60s
- deploy:
- replicas: 1
- resources:
- limits:
- memory: 4g
- cpus: '2.0'
- reservations:
- memory: 1g
- cpus: '0.5'
- placement:
- constraints: [node.role == manager]
- restart_policy:
- condition: any
- healthcheck:
- test: ['CMD-SHELL', 'pg_isready -U ${POSTGRES_USER:-underlay}']
- interval: 10s
- timeout: 5s
- retries: 5
- logging:
- driver: json-file
- options:
- max-size: '10m'
- max-file: '3'
-
-networks:
- appnet:
- driver: overlay
- dbaccess:
- driver: overlay
- attachable: true
-
-volumes:
- pgdata:
diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md
new file mode 100644
index 0000000..2aa2dc8
--- /dev/null
+++ b/docs/protocol-v2.md
@@ -0,0 +1,1021 @@
+# Underlay protocol, version 2
+
+**Status:** Stable. Frozen 2026-10-03.
+
+**Abstract.** This document specifies version 2 of the Underlay protocol: the canonical encoding
+and hashing of records, schemas and files; the input rules a publisher's data must satisfy; the
+construction of record and file trees; the version root and its hash; the repository layout in
+which collections are stored; the signed version log; the pack format by which versions are
+copied; and the HTTP interface by which servers serve and accept versions.
+
+**Change control.** A change to any rule or value that alters which inputs are accepted, how a
+tree is built, or what is hashed requires a new protocol version (the `underlay` member of every
+root, Section 10). Clarifications and additions that alter none of these are recorded in
+[Appendix B](#appendix-b-revision-history).
+
+**Reference implementation.** `@underlay/protocol` (`packages/protocol` in the Underlay
+repository). Test vectors: `packages/protocol/test/vectors/v2.json`
+([Appendix A](#appendix-a-test-vectors)). Where this document and the reference implementation
+disagree, this document is authoritative.
+
+## 1. Conventions and terminology
+
+### 1.1 Requirements language
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT",
+"RECOMMENDED", "NOT RECOMMENDED", "MAY" and "OPTIONAL" in this document are to be interpreted as
+described in BCP 14 ([RFC 2119](https://www.rfc-editor.org/rfc/rfc2119),
+[RFC 8174](https://www.rfc-editor.org/rfc/rfc8174)) when, and only when, they appear in all
+capitals.
+
+Sections and paragraphs marked _informative_, and all examples and notes, are not normative.
+
+### 1.2 Terminology
+
+- **Collection**: a named sequence of versions, identified by a collection id.
+- **Record**: a triple of an id, a type and a JSON value `data` (Section 4).
+- **Type**: a named class of records, identified by a type slug and described by a schema.
+- **Schema**: a JSON Schema document that the records of one type MUST satisfy (Section 5).
+- **File**: a byte string, identified by its hash (Section 6).
+- **Access set**: one of the two partitions of a version's content, `public` and `private`
+ (Section 9).
+- **Tree**: a sorted set of entries partitioned into tree nodes by the rules of Section 8.
+- **Tree node**: a leaf or interior node of a tree.
+- **Version**: an immutable state of a collection, described by a root document (Section 10).
+- **Repository**: the objects that represent one or more collections in a storage location
+ (Section 11).
+- **Server**: an HTTP service that implements Section 11.3 and, if it accepts publications,
+ Section 11.4. Also called an _Underlay node_.
+- **Client**: any party that reads from or publishes to a server.
+- **Organization**: the party a collection belongs to, named by `owner` in its `collection.json`
+ (Section 11.1).
+- **Owner**: a party permitted to read a collection's private set. Who is an owner is determined
+ by the server.
+
+### 1.3 Notation
+
+- **hash(x)** is the SHA-256 digest of the octet string `x`, written as 64 lowercase hexadecimal
+ characters. A string is hashed as its UTF-8 encoding.
+- **JCS(v)** is the canonical JSON serialization of the value `v` (Section 2).
+- **Key order** is the order defined in Section 7.
+- `+` between strings denotes concatenation.
+- Sizes are in octets (bytes). KiB and MiB are 2¹⁰ and 2²⁰ octets.
+- Counts are non-negative integers not greater than 2⁵³ − 1.
+
+## 2. Canonical JSON
+
+Every JSON document that is hashed MUST be serialized as specified by
+[RFC 8785](https://www.rfc-editor.org/rfc/rfc8785) (JSON Canonicalization Scheme, JCS):
+
+- no insignificant whitespace;
+- strings escaped as by ECMAScript `JSON.stringify`;
+- numbers serialized as by ECMAScript `Number.prototype.toString`, with `-0` serialized as `0`;
+- object members ordered by the UTF-16 code units of their keys.
+
+Note (informative): ECMAScript objects enumerate integer-like keys (`"9"`, `"10"`) first, in
+numeric order, regardless of insertion order. Sorting keys into a new object and serializing it
+with `JSON.stringify` therefore does not produce JCS. An implementation in ECMAScript must emit
+object members as strings itself (reference: `packages/protocol/src/jcs.ts`).
+
+## 3. Input rules
+
+A client MUST apply these rules to every record line it publishes, and a server MUST apply them to
+every record line it receives in a publication (Section 11.4). The rules apply to the whole line,
+including members that are otherwise ignored. They are evaluated on the source text, because a
+parsed value no longer carries the information they require. A server MUST also apply the syntax,
+duplicate key, unsafe integer and lone surrogate rules to the schemas it receives; the depth rule
+applies to record lines only.
+
+| Rule | A line or document is rejected if | Code |
+| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ |
+| Syntax | it is not JSON as defined by RFC 8259, including an unterminated string or an invalid escape | `syntax` |
+| Duplicate keys | an object has two members whose keys are equal after unescaping | `duplicate_key` |
+| Unsafe integers | it contains an integer literal (no fraction, no exponent) whose magnitude exceeds 2⁵³ − 1 (`9007199254740991`). Literals with a fraction or exponent (`1e20`, `9007199254740993.0`) are accepted and take their IEEE 754 binary64 value | `unsafe_integer` |
+| Lone surrogates | a string or key contains a UTF-16 surrogate code unit, literal or `\u`-escaped, that is not part of a surrogate pair | `lone_surrogate` |
+| Depth | the line nests more than `MAX_JSON_DEPTH` + 1 (65) levels. Each object or array is one level and the envelope is level 1, so `data` and every other member may nest 64 levels | `too_deep` |
+| Envelope | the line is not an object, has no `data` member, or has a `private` member that is not a boolean | `bad_envelope` |
+| Record id | `id` is absent, not a string, empty, or longer than `MAX_ID_BYTES` (1,024) UTF-8 bytes | `bad_id` |
+| Type slug | `type` is absent, not a string, empty, longer than `MAX_TYPE_BYTES` (128) UTF-8 bytes, begins with `.`, or contains `/`, `\`, U+0000–U+001F or U+007F | `bad_type` |
+| Record size | the canonical form (Section 4) is longer than `MAX_RECORD_BYTES` (8,388,608) bytes | `record_too_large` |
+
+A record line that breaks more than one rule MUST be reported with the code of the first rule it
+breaks in this order:
+
+1. the text is scanned from the start; the first of an unterminated string or invalid `\u`
+ escape (`syntax`), a duplicate key, an unsafe integer, a lone surrogate or an excess of depth
+ that occurs in the text determines the code, whether or not the text is otherwise JSON;
+ otherwise, a text that is not JSON is `syntax`;
+2. `bad_envelope`, if the line is not an object;
+3. `bad_id`;
+4. `bad_type`;
+5. `bad_envelope`, if `data` is absent or `private` is not a boolean;
+6. `record_too_large`.
+
+The codes are part of the protocol. A server reports them (Section 11.4).
+
+Members of a record line other than `id`, `type`, `data` and `private` MUST be ignored. They are
+not part of the record, its canonical form or its hash.
+
+Strings are not Unicode-normalized. `é` as U+00E9 and as U+0065 U+0301 are distinct ids with
+distinct hashes.
+
+## 4. Records
+
+A record consists of an `id` (a string), a `type` (a type slug) and `data` (any JSON value). The
+**canonical form** of a record is the string
+
+```
+'{"id":' + JCS(id) + ',"type":' + JCS(type) + ',"data":' + JCS(data) + '}'
+```
+
+The envelope members appear in the fixed order `id`, `type`, `data`; only `data` is canonicalized
+under Section 2.
+
+- **Record hash** = hash(canonical form).
+- **Record size** = the length of the canonical form in bytes.
+- Within a version, a (type, id) pair MUST identify at most one record, across both access sets.
+
+**File references.** A record references a file through any object, at any depth of `data`,
+whose `$file` member is a string consisting of `sha256:` followed by 64 lowercase hexadecimal
+characters. The referenced file hash is the hexadecimal part. An object that is a file reference
+is not searched for further references. An object whose `$file` member has any other value is not
+a reference, and its members are searched as usual.
+
+## 5. Schemas
+
+A type's schema is a JSON object that is a JSON Schema draft-07 document (Section 5.1).
+**Schema hash** = hash(JCS(schema)).
+
+A schema MUST be rejected if:
+
+- its root `private` member is present and is not a boolean;
+- a schema that is the value of a member of a `properties` object, at any depth, has
+ `"private": true` (field-level privacy is not supported);
+- its canonical form is longer than `MAX_SCHEMA_BYTES` (262,144) bytes;
+- any `pattern` value or `patternProperties` key is longer than `MAX_PATTERN_LENGTH` (256) UTF-16
+ code units. A `pattern` member inside `const`, `enum`, `default` or `examples` is data, not a
+ regular expression, and is not limited;
+- the slug it is given under is not a valid type slug (Section 3).
+
+A root `"private": true` makes the type private (Section 9).
+
+### 5.1 Validation dialect
+
+**Acceptance of schemas.** A schema MUST be rejected unless:
+
+- its root `$schema` member, if present, is `http://json-schema.org/draft-07/schema` or
+ `http://json-schema.org/draft-07/schema#`;
+- it is valid against the draft-07 meta-schema;
+- every `pattern` value and `patternProperties` key compiles as an ECMAScript regular expression
+ with the `u` flag;
+- every `$ref` resolves within the schema or to the draft-07 meta-schema. References are resolved
+ against the base URI `https://schema.underlay.invalid/` unless a `$id` establishes another.
+
+**Validation of records.** A record's `data` MUST be validated against its type's schema under
+draft-07, with these refinements:
+
+- keywords adjacent to `$ref` are applied, as in draft 2019-09;
+- keywords not defined by draft-07 are ignored, including later drafts' keywords
+ (`unevaluatedProperties`, `dependentRequired`, `prefixItems`, …) and draft-04's `id`. `$defs`
+ is honoured as a container of subschemas, and `$anchor` is honoured;
+- `pattern` is an ECMAScript regular expression with the `u` flag;
+- `multipleOf` m accepts x when the floating-point remainder r = x mod m satisfies
+ |r| < 1.1920929 × 10⁻⁷ or |m − r| < 1.1920929 × 10⁻⁷;
+- string length is measured in Unicode code points;
+- object members are the parsed document's own keys; names such as `__proto__` and `toString`
+ have no special meaning.
+
+**Formats.** `format` constrains strings only, and only for the names `date`, `time`,
+`date-time`, `iso-time`, `iso-date-time`, `duration`, `uri`, `uri-reference`, `uri-template`,
+`url`, `email`, `hostname`, `ipv4`, `ipv6`, `regex`, `uuid`, `json-pointer`,
+`json-pointer-uri-fragment`, `relative-json-pointer` and `byte`. Each is defined as in
+ajv-formats 3.0 in "full" mode. In particular, `date-time` and `time` require a time zone,
+`date-time` accepts `T`, `t` or whitespace as the separator, and `email` requires a dot in the
+domain. Other format names are ignored.
+
+Only the verdict (valid or invalid) is normative. Error messages, and the number of errors
+reported, are not.
+
+Note (informative): the reference validator is `@cfworker/json-schema` with these refinements
+(`packages/protocol/src/validate.ts`). Over all public production data it gives the same verdicts
+as the AJV configuration of Underlay v1 (139 schemas, 330,300 records, 85,773 mutated records).
+
+## 6. Files
+
+**File hash** = hash(the file's bytes). A file's **size** is its length in bytes.
+
+## 7. Key order and boundary hash
+
+- **Key order.** Keys are compared lexicographically by their UTF-8 encodings, octet by octet; a
+ proper prefix precedes the longer key. This is Unicode code point order. Implementations MUST
+ NOT compare UTF-16 code units (ECMAScript's default `<` and `sort()`), which order differently
+ when one key has a character at or above U+10000 where the other has one in U+E000–U+FFFF.
+- **Boundary hash.** u(k) is the first 8 bytes of hash(k), read as a big-endian unsigned 64-bit
+ integer.
+- **tz(k)** is the number of trailing zero bits of u(k), from 0 to 64.
+
+## 8. Trees
+
+A tree is a set of entries with unique keys, in key order, partitioned into tree nodes. There are
+two kinds:
+
+| Kind | Key | Entry | Entry size |
+| ----------- | --------------- | ------------------------------ | ------------ |
+| Record tree | record id | `[id, recordHash, recordSize]` | `recordSize` |
+| File tree | file hash (hex) | `[fileHash, fileSize]` | `fileSize` |
+
+### 8.1 Shape
+
+| Parameter | Value |
+| ----------------------------- | ----- |
+| `LEAF_BOUNDARY_BITS` | 10 |
+| `INTERIOR_BOUNDARY_BITS_STEP` | 6 |
+| `LEAF_MAX_ENTRIES` | 8,192 |
+| `INTERIOR_MAX_CHILDREN` | 1,024 |
+
+- **Leaves (level 0).** The entries are taken in key order. The current leaf ends after entry
+ `k` if tz(k) ≥ 10, if the leaf holds `LEAF_MAX_ENTRIES` entries, or if `k` is the last entry.
+- **Interior level i, i ≥ 1.** The nodes of level i − 1 are taken in order. The current level-i
+ node ends after child `c` if 10 + 6i ≤ 64 and tz(last key of `c`) ≥ 10 + 6i, if the node has
+ `INTERIOR_MAX_CHILDREN` children, or if `c` is the last node of level i − 1.
+- **Root.** Levels are built in ascending order. The root is the single node of the lowest level
+ that has exactly one node. If the tree has one leaf, that leaf is the root. An empty tree has no
+ nodes, and its root is `null`.
+
+Consequences (informative): the same entry set yields the same tree regardless of construction
+order; a natural boundary (tz(k) ≥ 10) depends only on its key, and a forced split only on the
+position since the preceding boundary; any range between two natural boundaries can therefore be
+rebuilt independently. The mean leaf holds 1,024 entries and the mean interior fan-out is 64.
+
+### 8.2 Node encoding
+
+```
+leaf: {"e":[entry, ...],"t":"leaf"}
+interior: {"e":[[lastKey, childHash, count, bytes], ...],"l":level,"t":"node"}
+```
+
+- A node is encoded as JCS, which places members in the order shown.
+- `lastKey` is the last key under the child, `childHash` the child's node hash, `count` the number
+ of entries under the child, and `bytes` the sum of their entry sizes. `level` is the node's
+ level.
+- **Node hash** = hash(encoded node).
+
+### 8.3 Validity
+
+A tree is valid if and only if building its entries under Section 8.1 yields the same root hash.
+
+A receiver of tree nodes from another party (Section 11.2) MUST reject a node unless:
+
+- its bytes hash to the expected node hash and are its canonical encoding;
+- its keys are strictly increasing, within the node and across the tree;
+- each interior entry's `lastKey`, `count` and `bytes` equal those of the child it names, and
+ `count` ≥ 1;
+- each child is exactly one level below its parent;
+- in a record tree, each key is a valid record id (Section 3).
+
+A receiver MUST reject a record leaf unless each of its entries matches the corresponding line of
+the leaf's body (Section 11), with an out-of-line pointer resolved to its record: the line hashes
+to the entry's record hash, is exactly the entry's size in bytes, and is the canonical form of a
+record whose `id` is the entry's key and whose `type` is the type of the tree.
+
+A node that hashes correctly but violates a structural rule is invalid; accepting it would admit
+two roots for one entry set.
+
+Record documents (members `id`, `type`, `data`) and node documents (members `e`, `l`, `t`) have
+disjoint member names, so neither can be interpreted as the other.
+
+## 9. Access sets
+
+Each version has two access sets, `public` and `private`.
+
+- **Records.** A record published with `"private": true`, and every record of a private type,
+ belongs to the private set. Every other record belongs to the public set.
+- **Types.** Each set lists, for each type it contains, the type's schema hash and the root of the
+ tree of that set's records of the type.
+ - A private type appears in the private set only.
+ - A public type appears in the public set, including when it has no public records (with a
+ `null` root). It also appears in the private set if it has private records.
+- **Files.** A file belongs to each set that contains a record referencing it. A file declared in
+ a publication (`files.add`, Section 11.4) also belongs to the private set, whether or not records
+ reference it, and remains declared in later versions until a publication removes the declaration
+ (`files.remove`).
+- **Readers.** Owners MAY read both sets. Any other reader MAY read the public set only, and only
+ of a collection whose visibility is `"public"`. Visibility is collection state outside the
+ version, recorded in `collection.json` (Section 11.1); changing it changes no version.
+
+## 10. Versions
+
+```
+SetObject = {
+ "types": { slug: { "schema": schemaHash, "root": treeHash | null, "count": n, "bytes": b }, ... },
+ "files": { "root": treeHash | null, "count": n, "bytes": b }
+}
+PrivateSetObject = SetObject + { "salt": 64 hex characters }
+root = { "underlay": 2, "metadata": object | null, "public": SetObject, "private": commitment | null }
+```
+
+- `count` and `bytes` are the tree root's totals, or 0 for a `null` root.
+- **Commitment** = hash(JCS(PrivateSetObject)).
+- The salt is 32 random bytes, encoded as hexadecimal. A writer MUST choose it once per collection
+ and MUST reuse it for every version of that collection, so that an unchanged private set keeps
+ its commitment.
+- `private` is `null` if and only if the private set is empty: it lists no types and no files.
+- **Version hash** = `"ulv2:"` + hash(JCS(root)).
+
+A version hash commits to content only. A root has no parent pointer; identical content yields the
+same version hash in any collection, except where a private set is present, since salts differ
+between collections. Lineage, semver, messages and authorship are recorded in the version log
+(Section 11.1).
+
+A reader of the public set can verify the version hash and every object of the public set, and
+learns of the private set only whether it exists. An owner additionally obtains the
+PrivateSetObject, including its salt, and can verify it against the commitment.
+
+### 10.1 Semver
+
+Each version of a collection has a semver of the form `v..`. The server that
+commits a version MUST assign it from the differences between the version and its base
+(Section 11.4), with M, m and p the base's major, minor and patch:
+
+1. The first version of a collection is `v1.0.0`.
+2. If a type was added or removed, or a type's schema hash changed, the semver is `v(M+1).0.0`.
+ Making a type private or public changes its schema and is therefore covered by this rule.
+ When a type's schema changes, every record of the type carried over from the base MUST be
+ validated against the new schema, and the publication MUST be refused if any fails.
+3. Otherwise, if any record was added, removed or changed in either set, the semver is
+ `vM.(m+1).0`. A record moving between sets is a change.
+4. Otherwise (only the metadata or the file sets changed), the semver is `vM.m.(p+1)`.
+
+A publication whose version hash equals its base's version hash MUST NOT create a version.
+
+Semvers are unique within a collection and strictly increasing. A version converted from Underlay
+v1 retains the semver it had in v1.
+
+## 11. Repository layout
+
+A repository is the representation of collections in a storage location (an object store such
+as an S3-compatible bucket). The layout is identical in a server's own storage and in a mirror.
+Keys are relative to the location's prefix.
+
+```
+nodes/ encoded node (Section 8.2), gzip
+bodies/.ndjson.gz the body of a record leaf
+records/.json.gz an out-of-line record, gzip
+schemas/.json JCS(schema)
+roots/.json JCS(root); is the version hash without "ulv2:"
+private/.json JCS(PrivateSetObject); only where private sets are held
+files/ file bytes
+collections//collection.json collection description (Section 11.1)
+collections//log/.json version log entry
+collections//head.json log head
+```
+
+- **Bodies.** The body of a record leaf contains one line per entry of the leaf, in entry order,
+ each terminated by `\n`. A line is either the canonical form of the entry's record or an
+ out-of-line pointer `{"$ref":""}`, in which case the record is stored at
+ `records/.json.gz`. A body is one or more concatenated gzip members (RFC 1952
+ §2.2); readers MUST accept any number of members. Which records are stored out of line, and
+ where members divide, are the writer's choice.
+- **Immutability.** Every object outside `collections/` is content-addressed and MUST NOT change
+ once written. A reader that does not trust a location MUST verify each object before use: nodes
+ against their hash, body lines against the leaf's entries, roots against the version hash, and
+ schemas and PrivateSetObjects against their hashes.
+- **Write order.** A writer MUST write every object a version reaches (leaves and their bodies,
+ then interior nodes, then the PrivateSetObject and root) before the version's log entry, and the
+ log entry before `head.json`. A reader that finds `head.json` can therefore read every object
+ it reaches.
+- **Self-containment.** No object in a location refers to another location.
+- Readers MUST ignore keys outside this layout.
+
+Note (informative): with no out-of-line pointers, the concatenation of a type's bodies in tree
+order is that type's records as gzip-compressed NDJSON. Data internal to a server (push sessions,
+staged uploads, indexes) is not part of a repository. The reference server's location check
+writes `.underlay/check.json` under the prefix.
+
+### 11.1 Version log
+
+Each collection has one log entry per version:
+
+```
+entry = {"actorId","appId","baseSemver","collectionId","createdAt","keyId","message","prev","semver","seq","sig","versionHash"}
+```
+
+- `collectionId` is the id of the collection whose log contains the entry. It is signed, so that
+ an entry or a log cannot be presented as another collection's.
+- `seq` is the version's position in the log, starting at 1. `semver` and `versionHash` identify
+ the version; `baseSemver` is its base's semver.
+- `createdAt` is an ISO 8601 timestamp in UTC.
+- `appId`, `actorId`, `baseSemver` and `message` MAY be `null`. Writers SHOULD write `actorId` as
+ `null`, since a log is as public as its collection; the member is retained so that existing
+ entries verify.
+- `keyId` is the first 16 hexadecimal characters of hash(raw public key) of the signing key.
+- `sig` is the Ed25519 signature over the UTF-8 encoding of JCS(entry without `sig`), encoded as
+ base64url without padding.
+- **Entry hash** = hash(JCS(entry)), with `sig` included.
+- `prev` is the entry hash of entry `seq − 1`, or `null` when `seq` is 1.
+- `head.json` is JCS(`{"entryHash","seq","versionHash"}`) of the latest entry. It is overwritten
+ after each new entry is written.
+- `collection.json` describes the collection (below).
+
+**Collection description.** `collection.json` is a JSON object:
+
+```
+{"id", "owner": {"id", "did", "handle", "name"}, "slug", "name", "description", "visibility", "ark", "keys"}
+```
+
+| Member | Value |
+| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `id` | The collection id |
+| `owner` | The organization that owns the collection. `id` is its identifier at the server that wrote the file; `did` its DID, or `null`; `handle` its handle (Section 11.3.1), or `null`; `name` its display name |
+| `slug` | The collection's slug, unique among its owner's collections |
+| `name` | The collection's display name |
+| `description` | A string, or `null` |
+| `visibility` | `"public"` or `"private"`. A private collection's versions, public sets included, are readable by owners only (Section 9) |
+| `ark` | The collection's ARK (`ark:/`), or `null` |
+| `keys` | An array of `{"id", "alg": "Ed25519", "publicKey"}`, where `publicKey` is the raw public key in base64url: every key that has signed an entry of the log |
+
+- It is neither hashed nor signed. Readers MUST parse it as JSON, MUST NOT depend on its
+ serialization, and MUST ignore members they do not recognize.
+- A writer MUST write it before the collection's first log entry, and MUST rewrite it whenever a
+ member changes, whether or not a version is published. A copy of a repository (a mirror)
+ SHOULD carry each rewrite. The file is how a repository records, without a server, who owns a
+ collection and whether it is public.
+
+A verifier MUST use a key only under the id derived from it: a key listed under any other id MUST
+be ignored.
+
+A log is valid if and only if every entry from 1 to `head.seq` is present; every entry's
+`collectionId` names the collection being read; every `prev` equals the entry hash of the
+preceding entry; every signature verifies under a trusted key; and `head.entryHash` and
+`head.versionHash` equal the last entry's hash and `versionHash`.
+
+Which keys a verifier trusts is not specified by this version of the protocol (Section 13).
+
+### 11.2 Packs
+
+A version is transferred between repositories as a **pack**: the repository objects the version
+reaches that the receiver's base version does not, each under its repository key. A pack is an
+uncompressed POSIX tar archive, with PAX extended headers for names longer than 100 bytes. Objects
+are carried as stored. File bytes and `collections/` objects are not carried in packs.
+
+A pack contains, in this order:
+
+1. the schemas the base does not have;
+2. for each set sent, public first: for each of the set's record trees, the tree nodes not present
+ at the same position in the base's tree of that type (in the same set, otherwise in the other
+ set), parents before children, and after each new leaf the out-of-line records its body points
+ to, then its body; then the new tree nodes of the set's file tree, in the same way;
+3. the PrivateSetObject, if the private set is sent;
+4. the root.
+
+A receiver MUST NOT depend on any order except that a leaf precedes its body, out-of-line records
+precede the body that points to them, and the root is last.
+
+A receiver MUST, before writing an object, verify it against its key: tree nodes and out-of-line
+records by hash; bodies line by line against their leaf's entries (Section 8.3), the leaf having
+arrived first; and schemas, the PrivateSetObject and the root by hash and canonical form. It MUST
+reject a root that does not have exactly the members of Section 10, with `underlay` equal to 2,
+`metadata` an object or `null`, valid type slugs, and set totals of 0 for `null` roots, and a
+PrivateSetObject that is empty or lacks a 64-hex `salt`. It MUST then, for every tree of every set received, merge the entry changes from its base tree into that
+base tree under Section 8.1 and obtain exactly the received root, count and bytes, and MUST hold a
+body for every new record leaf. It MUST write the root only after all checks pass, and MUST refuse
+a pack that fails any check.
+
+### 11.3 Serving over HTTP
+
+A server serves each collection under a **collection URL**: an absolute URL without a trailing
+slash, relative to which the paths below are resolved. The form of collection URLs is the
+server's choice; Section 11.3.1 gives the RECOMMENDED form.
+
+| Route | Serves | Section | Required |
+| --------------------------------------------------- | ----------------------------------- | ------- | -------- |
+| `GET ` | the collection | 11.3.3 | MUST |
+| `GET /versions` | its versions, newest first | 11.3.3 | MUST |
+| `GET /versions/` | one version | 11.3.3 | MUST |
+| `GET /versions//records` | a page of records | 11.3.4 | MUST |
+| `GET /versions//records//` | one record | 11.3.4 | MUST |
+| `GET /versions//records.ndjson` | every record, streamed | 11.3.4 | MUST |
+| `GET /versions//records.ndjson.gz` | the public set's records, as stored | 11.3.4 | SHOULD |
+| `GET /records///history` | one record across versions | 11.3.4 | SHOULD |
+| `GET /schemas` | a version's type set | 11.3.5 | MUST |
+| `GET /versions//diff` | the records two versions differ by | 11.3.5 | MUST |
+| `GET /versions//files` | a version's files | 11.3.5 | MUST |
+| `GET`, `HEAD /files/` | one file's bytes | 11.3.5 | MUST |
+| `GET /export` | a version as a tar archive | 11.3.5 | SHOULD |
+| `GET /log` | the version log | 11.3.6 | MUST |
+| `GET /versions//pack` | a version's objects | 11.3.6 | MUST |
+| `GET /versions//manifest` | a version's record ids and hashes | 11.3.6 | MUST |
+
+Publishing (Section 11.4) adds routes under the same collection URL.
+
+#### 11.3.1 Collection URLs
+
+The RECOMMENDED form of a collection URL is `//`, where `` is the
+server's API base URL, `` the collection's slug and `` one of:
+
+- a **hosted handle**: an organization's slug at the server, matching
+ `^[a-z0-9]+(-[a-z0-9]+)*$`;
+- a **domain handle**: a domain name the organization has shown it controls, such as
+ `press.mit.edu`;
+- a **DID**: the organization's DID, beginning `did:`.
+
+A slug contains neither `.` nor `:`, so the three forms are distinguished by their text. The
+**collection id form** `/_/` names a collection by its id; `_` is not a
+handle.
+
+- A server that offers this form MUST resolve each owner form it supports, and the id form, to
+ the same collection, and MUST serve it at each URL rather than redirect. An owner form the
+ server does not support is a 404.
+- Every response from a route of this section or Section 11.4 for a collection the caller may
+ read MUST carry the header `x-underlay-collection` with the collection id.
+- A handle can change, and a handle given up can later name another organization. A client that
+ keeps a reference to a collection SHOULD keep the id form, or the DID form where the owner has
+ a DID, and not a handle.
+
+Informative: underlay.org's `` is `https://underlay.org/api/collections`, and its hosted
+handles are its organization slugs. It does not yet resolve DIDs or domain handles.
+
+#### 11.3.2 Conventions
+
+- **Version selectors.** `` is a semver (the leading `v` is optional), a version hash
+ (`ulv2:`), or `latest`. A version hash that names several versions of the collection (a
+ reverted change repeats a hash) selects the latest of them. Wherever a query parameter names a
+ version (`base`, `from`, `since`, `version`), it takes the same forms and MUST name a version of
+ the same collection.
+- **Responses** are JSON (`application/json`) unless stated otherwise. Timestamps are ISO 8601
+ strings in UTC. A server MAY add members to any object in a response; a client MUST ignore
+ members it does not recognize. The members listed in this section are the standard ones.
+- **Records** in responses are Record objects:
+
+ ```
+ Record = {"id", "type", "data", "hash", "private"?}
+ ```
+
+ `hash` is the record hash (Section 4). `"private": true` marks a record of the private set, and
+ appears only in responses to owners.
+
+- **Pagination.** A paged response carries `pagination: {"limit", "hasMore", "nextCursor"}`. While
+ `hasMore` is true, a client obtains the next page by repeating the request with `cursor` set to
+ `nextCursor`. Cursors are opaque. A server MAY cap `limit` and MAY return fewer items than
+ requested.
+- **Access.** A caller who is an owner reads both sets of every version. Any other caller reads
+ the public sets of a collection whose visibility is `"public"`, and nothing of a private one.
+ Counts and totals in a response are of what the caller may read.
+- **Withheld content.** A server MAY withhold a record or file for legal reasons. Counts and
+ totals MAY still include withheld records.
+
+**Errors.** Error responses carry a JSON body `{"error": }`.
+
+- 404: the collection, version, base, record or file does not exist or the caller may not read
+ it. A server MUST NOT distinguish these cases.
+- 403: `sets=all` was requested by a caller who may read the public set but not the private set.
+- 400: a malformed parameter, such as a `sets` that is neither `public` nor `all`.
+- 451: the server may not serve a file the caller could otherwise read, for legal reasons. A
+ server MUST answer 404 rather than 451 for a file the caller may not read.
+- Other statuses carry their HTTP meanings, including 401 for invalid credentials and 429, with
+ `Retry-After`, for rate limiting.
+
+**Authentication** is the server's choice. A server without access control MUST serve public sets
+only and MUST answer `sets=all` with 403.
+
+#### 11.3.3 Collections and versions
+
+**Collection.** `GET ` returns:
+
+```
+{"id", "owner", "slug", "name", "description", "visibility", "ark", "createdAt", "updatedAt",
+ "versionCount", "head"}
+```
+
+- `id`, `owner`, `slug`, `name`, `description` and `visibility` are as in `collection.json`
+ (Section 11.1).
+- `ark` is the collection's ARK as a URL that resolves it, ending in `ark:/`, or
+ `null`.
+- `head` is `{"semver", "hash"}` of the latest version, or `null` before the first.
+
+**Version summary.**
+
+```
+VersionSummary = {"semver", "hash", "baseSemver", "message", "appId", "createdAt",
+ "recordCount", "fileCount", "totalBytes", "typeCounts", "ark"}
+```
+
+- `semver`, `hash`, `baseSemver`, `message` and `appId` are as in the version's log entry
+ (Section 11.1); `createdAt` is when it was published.
+- `recordCount`, `fileCount` and `totalBytes` total the sets the caller may read; `totalBytes`
+ is the sum of their record and file sizes. `typeCounts` maps each type slug to its number of
+ records in those sets.
+- `ark` is the version's ARK as a URL that resolves it, or `null`.
+
+**Versions.** `GET /versions?limit=&offset=` returns a JSON array of
+VersionSummary, newest first, at most `limit` of them after skipping `offset`.
+
+**Version.** `GET /versions/` returns the VersionSummary with two more members:
+`metadata`, the root's metadata (Section 10); and `schemas`, mapping each type slug the caller
+may read to its schema.
+
+#### 11.3.4 Records
+
+Records are ordered by type slug, then by id, each in key order (Section 7).
+
+**Records.** `GET /versions//records?type=&limit=&cursor=` returns a
+page:
+
+```
+{"records": [Record, ...], "pagination": {"limit", "hasMore", "nextCursor", "total"}}
+```
+
+- With `type`, the page holds that type's records; without it, every type's.
+- `total` is the number of records the request selects, across all pages.
+- `offset=`, without a cursor, skips the first n records.
+
+**Record.** `GET /versions//records//` returns the Record with
+`semver`, the version's semver. `` and `` are percent-encoded path segments.
+
+**Records as NDJSON.** `GET /versions//records.ndjson` streams every record the
+caller may read, one Record per line, with content type `application/x-ndjson`.
+
+- `type=` limits the stream to one type.
+- `after_type=&after=` resumes after that record, through the types that follow;
+ `type=&after=` resumes within the type. `after` without `after_type` or `type` is a 400.
+- The header `x-underlay-record-count` gives the number of lines.
+
+**Public records as stored.** `GET /versions//records.ndjson.gz?type=` returns
+the public set's records, or one type's, as gzip (`application/gzip`). Each line is the
+canonical form of a record (Section 4), without `hash`. The body MAY consist of several gzip
+members (RFC 1952 §2.2), so that a server can send stored leaf bodies unchanged. With `type`, a
+type with no public records is a 404.
+
+**History.** `GET /records///history` returns the versions in which a record
+was added, changed or removed, oldest first, as the caller may read them:
+
+```
+{"type", "id", "changes": [{"seq", "semver", "createdAt", "change", "hash"}, ...], "truncated"}
+```
+
+- `change` is `"added"`, `"updated"` or `"removed"`; `hash` is the record hash after the change,
+ or `null` for a removal.
+- A server MAY consider only its latest versions; `truncated` is then true.
+- A record never present is a 404.
+
+#### 11.3.5 Schemas, differences and files
+
+**Schemas.** `GET /schemas?version=&raw=` returns the type set of a version
+(default `latest`):
+
+```
+{"semver", "schemas": [{"slug", "schemaHash", "schema"}, ...]}
+```
+
+A server MAY annotate `schema` with members whose names begin with `x-`. With `raw=true` it MUST
+NOT: `schema` is then the schema as published, and JCS(`schema`) hashes to `schemaHash`.
+
+**Differences.** `GET /versions//diff?from=&limit=&cursor=` returns the
+records that differ between version `from` and version ``, one page at a time:
+
+```
+{"from", "to", "added": [{"id", "type", "data"}, ...], "updated": [{"id", "type", "data"}, ...],
+ "removed": [{"id", "type"}, ...], "pagination": {"limit", "hasMore", "nextCursor"},
+ "meta": {"schemaChanged", "metadataChanged", "filesAdded", "filesRemoved"}}
+```
+
+- `from` defaults to the version before ``. For the first version it is `null`, and every
+ record is added.
+- `added` and `updated` carry the records' data in ``.
+- To an owner, a record that moved between sets with an unchanged hash has not changed. To a
+ reader of the public set, a record that left it is removed, and one that joined it is added.
+- `meta` compares the versions' type sets and metadata. `filesAdded` and `filesRemoved` count the
+ files the caller may read that were added and removed; they are given on the first page and are
+ 0 on later pages.
+
+**File list.** `GET /versions//files` returns a JSON array of the version's files
+that the caller may read, in key order:
+
+```
+[{"hash", "size", "mimeType", "referenceCount"}, ...]
+```
+
+`referenceCount` is the number of records the caller may read that reference the file. A server
+MAY cap the length of the list; the manifest (Section 11.3.6) pages through every file.
+
+**Files.** `GET /files/` returns the file's bytes, or a redirect to them. A
+`HEAD` response carries `content-length`. `` is 64 hexadecimal characters, optionally
+prefixed with `sha256:`. A server MUST serve a file only to a caller who may read a set that holds
+it (Section 9).
+
+**Export.** `GET /export?version=&format=tar|tar.gz` returns a version (default
+`latest`) as a tar archive, gzipped by default, of what the caller may read:
+
+```
+manifest.json {"collection", "version", "schemas", "files_missing", "files_withheld"}
+README.md the version's metadata.readme, when it has one
+records/.ndjson one Record per line
+files/ file bytes
+```
+
+`files_missing` lists files the server does not hold, and `files_withheld` files it may not serve.
+
+#### 11.3.6 Replication
+
+These routes let a client or another server copy a collection and verify it (Section 11.3.7).
+
+**Log.** `GET /log?after=&limit=` returns `{"collection", "head", "entries"}`:
+
+- `collection` is the collection's `collection.json` (Section 11.1) and `head` its `head.json`;
+ either MAY be `null` before the first version;
+- `entries` are the log entries with `seq` greater than `after` (default 0), in ascending `seq`
+ order, at most `limit` of them. A server MAY cap `limit` and MAY return fewer entries than
+ requested. A client obtains the remainder by repeating the request with `after` set to the last
+ `seq` received, until it reaches `head.seq`.
+
+**Pack.** `GET /versions//pack?base=&sets=public|all` returns the pack (Section
+11.2) of version `` against `base`, with content type `application/x-tar`. Without `base`, the
+pack holds every object the version reaches. `sets` defaults to `public`; `all` includes the
+private set. The response carries the headers `x-underlay-version` (the version hash),
+`x-underlay-base` (the base's version hash, or empty) and `x-underlay-sets` (the sets sent).
+
+**Manifest.** `GET /versions//manifest?cursor=&limit=` returns the version's
+records as the caller may read them, without bodies:
+
+```
+{"semver", "hash", "schemas": {slug: schemaHash}, "records": [{"id", "type", "hash", "private"?}, ...],
+ "files": [fileHash, ...], "pagination": {"limit", "hasMore", "nextCursor"}}
+```
+
+- Records are in order of type, then id. A record of the private set carries `"private": true`.
+ A caller who cannot read the private set receives the public set only.
+- `files` lists the hashes of the files the caller may read, on the first page only (an empty
+ array on later pages). A server MAY cap the list; `filesTruncated: true` then says so.
+- `since=` returns the changes since that version instead of `records`:
+ `delta: {"added", "updated", "removed"}`, each an array of `{"id", "type", "hash"}`, an update
+ with `previousHash`, and `since`, that version's semver. To an owner, each carries `"private":
+true` for the private set, and a move between sets is an update, with `previousPrivate`.
+
+#### 11.3.7 Verification
+
+A client MUST NOT trust a server's responses: it verifies the log under Section 11.1, receives
+packs under Section 11.2, and verifies file bytes against their hash. A copy obtained from any
+server, including a mirror operated by a third party, then carries the same guarantees as one
+obtained from the origin. The other routes are not verifiable on their own; a client that requires
+proof of content reads packs, or checks each Record's `hash` against a verified tree.
+
+### 11.4 Publishing
+
+A client publishes a version by **delta push**: it opens a session against a base version,
+uploads the records it adds or changes and the ids it deletes, and commits. The server builds the
+trees, writes and signs the log entry, and assigns the semver (Section 10.1). Delta push is the
+only publication mechanism; a server MUST NOT accept tree nodes or packs from clients.
+
+```
+POST /push open a session
+POST /push//records upload records (NDJSON)
+POST /push//deletes upload deletes (NDJSON)
+PUT /files/ upload a file
+POST /push//commit commit
+GET /push/ session status
+DELETE /push/ abandon the session
+```
+
+**Opening a session.** The request body is a JSON object. Every member is OPTIONAL.
+
+| Member | Meaning |
+| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `base` | The semver of the version the changes are against. If it is not the collection's head, the server MUST answer 409 with `currentVersion`. `null` or absent: the head at the time of opening |
+| `schemas` | The complete type set, slug → schema (Section 5). It replaces the base's type set; a type omitted is removed with its records. Absent: the base's type set |
+| `metadata` | Replaces the base's metadata. An object or `null` |
+| `metadata_patch` | An object whose top-level members are merged into the base's metadata. Ignored if `metadata` is present. With neither, the base's metadata is kept |
+| `files` | `{"add": [fileHash, …], "remove": [fileHash, …]}`: files to declare or remove in addition to those records reference (Section 9) |
+| `message` | A string recorded in the log entry |
+| `app_id`, `actor_id` | Strings identifying the publishing application and actor |
+| `strip_unknown_fields` | A boolean; see **Records** |
+
+The response is 200 with `{"session_id", "base", "needed_files", "expires_at", "limits"}`:
+
+- `base` is the semver the session is against, or `null` if the collection has no versions;
+- `needed_files` lists the declared files the server does not hold for the collection; the client
+ MUST upload them before committing;
+- `expires_at` is when the session expires; it is extended by each records or deletes request;
+- `limits` are the server's limits, by which a client MUST size its requests:
+
+| Limit | Meaning |
+| ---------------------- | ----------------------------------------------------------------- |
+| `open_bytes` | Maximum body of the open request, in bytes |
+| `batch_bytes` | Maximum body of a records or deletes request, in bytes |
+| `batch_lines` | Maximum lines in a records or deletes request |
+| `session_idle_seconds` | Time without an upload after which an open session expires |
+| `open_sessions` | Maximum sessions one user may have open or committing at once |
+| `file_bytes` | Maximum file size accepted by `PUT /files/` |
+
+**Records.** `POST …/records` takes NDJSON record lines `{"id", "type", "data", "private"?}` and
+answers 200 with `{"received": n}`. Each line MUST satisfy the input rules (Section 3) and its
+type's schema (Section 5.1), and its type MUST be in the session's type set.
+
+- If `data` is an object with top-level members not listed in its schema's root `properties`, the
+ record MUST be refused, unless the session set `strip_unknown_fields`, in which case those
+ members are removed before the record is hashed. A schema without root `properties` admits any
+ members.
+- `"private": true` places the record in the private set (Section 9).
+- If any line fails, the server MUST answer 422 with `validationErrors`, one per failing line
+ identified by its 1-based `line` number among the request's non-empty lines, and MUST store
+ nothing from the request. A server MAY list only the first failures, with `totalErrors` giving
+ their number.
+
+**Deletes.** `POST …/deletes` takes NDJSON lines `{"type", "id"}` and answers 200 with
+`{"received": n}`. Each line MUST satisfy the syntax, duplicate key, unsafe integer and lone surrogate rules, and its `id` and `type` the record id and type slug rules (Section 3); the type MUST be in the session's type set. Deleting a (type, id) the base does
+not hold is not an error.
+
+A records or deletes request with no lines is answered with 400.
+
+Within a session, the later upload of a (type, id) supersedes an earlier one, whether either is a
+record or a delete. Uploads MAY be repeated and divided across any number of requests.
+
+**Files.** `PUT /files/` with the file's bytes stores the file for the
+collection. The server answers 201, or 400 if the bytes do not hash to ``. A server MAY
+offer other upload mechanisms for larger files. Every file referenced by a new record, and every
+declared file, MUST be held for the collection when the session is committed. A file is held for a
+collection if it is in a file tree of the base version, if it was in the public set of any earlier
+version of the collection, or if its bytes were uploaded to the collection. A file held only for
+another collection is not held. A server MUST hash the bytes of every upload, including when it
+already stores the file.
+
+**Commit.** `POST …/commit` builds the version.
+
+- 201 with `{"semver", "hash", "recordCount", "fileCount", "changes": {"added", "removed",
+"updated"}}` when the commit completes within the request.
+- 202 with `{"session_id", "status": "committing"}` when the commit runs in the background. A
+ client MAY request this with `?async=true`; a server MAY choose it for any commit. The client
+ polls `GET …/push/` until `status` is `committed`, when `result` holds the 201 body, or
+ `failed`, when `error` holds the failure body.
+- A commit refused because files are not held (422 with `filesNeeded`) leaves the session open:
+ the client uploads the files and commits the same session again. A polling client sees `status`
+ `open`, with the refusal in `error`.
+- Committing a session that has already committed answers 201 with the same body.
+- A client that holds the base SHOULD compute the new version's hash itself and compare it with
+ `hash`.
+
+**Clients without a copy** (informative). A client that keeps no copy of the collection reads the
+base's manifest (Section 11.3.6), compares each of its records with the manifest by (type, id),
+hash and set, uploads the records that are new, changed or moved between sets, and deletes the
+(type, id) pairs it no longer has. The upload is then proportional to the changes.
+
+**Errors.** Errors carry a JSON body `{"error": }`. Authentication and 404 follow
+Section 11.3.2.
+
+- 403: the caller may read the collection but not publish to it, or the session belongs to
+ another user.
+- 400: a malformed body, a `metadata` that is neither an object nor `null`, a `metadata_patch` that is not an object, or a request with no
+ lines.
+- 409: `base` is not the head (with `currentVersion`); the head changed before the commit; the
+ session is not open; or the publication changes nothing (with the head's `hash`).
+- 413: a body exceeds `open_bytes` or `batch_bytes`, a request exceeds `batch_lines`, or a file
+ exceeds `file_bytes`.
+- 422: records or deletes fail; a schema is refused; records carried over from the base fail a
+ changed schema (`validationErrors`); or the commit lacks files (`filesNeeded`).
+- 429: the user has `open_sessions` sessions open or committing, or a rate limit applies (with
+ `Retry-After`).
+
+## 12. Limits and constants
+
+| Constant | Value | Section |
+| ----------------------------- | ------------------------------- | ------- |
+| `PROTOCOL_VERSION` | 2 | 10 |
+| `VERSION_HASH_PREFIX` | `ulv2:` | 10 |
+| `LEAF_BOUNDARY_BITS` | 10 | 8.1 |
+| `INTERIOR_BOUNDARY_BITS_STEP` | 6 | 8.1 |
+| `LEAF_MAX_ENTRIES` | 8,192 | 8.1 |
+| `INTERIOR_MAX_CHILDREN` | 1,024 | 8.1 |
+| `MAX_SAFE_INTEGER_LITERAL` | 9,007,199,254,740,991 (2⁵³ − 1) | 3 |
+| `MAX_JSON_DEPTH` | 64 | 3 |
+| `MAX_RECORD_BYTES` | 8,388,608 (8 MiB) | 3 |
+| `MAX_ID_BYTES` | 1,024 | 3 |
+| `MAX_TYPE_BYTES` | 128 | 3 |
+| `MAX_SCHEMA_BYTES` | 262,144 (256 KiB) | 5 |
+| `MAX_PATTERN_LENGTH` | 256 (UTF-16 code units) | 5 |
+
+The reference implementation defines these in `packages/protocol/src/constants.ts`; the
+test vectors repeat them.
+
+Server limits (Section 11.4) are not protocol constants. Informative: underlay.org advertises
+`open_bytes` 8 MiB, `batch_bytes` 16 MiB, `batch_lines` 10,000, `session_idle_seconds` 3,600,
+`open_sessions` 20 and `file_bytes` 32 MiB; caps log pages at 1,000 entries and manifest pages at
+25,000 records; stores records over 64 KiB out of line; and commits in the background when a
+session uploaded more than 100,000 records or a schema change revalidates more than 100,000.
+
+## 13. Security considerations
+
+- **Existence.** A server answers 404, not 403, for content a caller may not read (Section
+ 11.3.2), so that a response does not confirm the content exists.
+- **Content by hash.** A server MUST NOT serve a tree node or body by hash alone, and MUST serve a
+ record, schema or file located by hash only where it occurs in a set the caller may read.
+ Otherwise a small private object with guessable content could be confirmed by computing its
+ hash.
+- **Presence during publication.** A server MUST NOT treat content it holds for other collections
+ as present in a session. Records are always uploaded in full, and a file counts only if it is
+ held for the collection (Section 11.4). This prevents a publisher from confirming, or binding
+ into its own collection, content it does not possess.
+- **Private set commitment.** The salt prevents confirmation of guessed private content from the
+ commitment. Because `private` is `null` exactly when the private set is empty, the root reveals
+ whether a private set exists.
+- **Split views.** A version hash proves a version's content, not that a server shows every reader
+ the same versions. The hash-chained, signed version log (Section 11.1) makes omission and
+ reordering detectable by readers who compare log heads.
+- **Signing key trust (open issue).** This version of the protocol does not specify how a verifier
+ obtains trusted signing keys. A verifier that trusts the keys listed in a `collection.json`
+ served by an untrusted server can be presented with a forged history on first contact. The
+ reference client trusts the keys of the `collection.json` it is served and anchors on the last
+ entry it has verified.
+- **Visibility in copies.** `collection.json` records whether a collection is public, but a
+ repository enforces nothing: whoever can read a storage location can read everything in it. A
+ server that serves a repository MUST apply `visibility` and the sets (Section 9) itself, and a
+ location that holds a private collection or a private set MUST NOT be publicly readable.
+- **Handles.** A handle names an organization only while the organization holds it
+ (Section 11.3.1). References meant to last name a collection by its id, or its owner by DID.
+- **File serving.** File bytes SHOULD be served from an origin separate from the server's own
+ pages, and with `Content-Disposition: attachment`, so that an uploaded HTML or SVG file cannot
+ run in the server's origin.
+
+## Appendix A. Test vectors
+
+`packages/protocol/test/vectors/v2.json` contains:
+
+- the constants (Section 12);
+- JCS input and output pairs;
+- input-rule verdicts per record line (`inputRules`), covering every code, the depth limit at
+ its boundary and the order of codes; and, for lines too long to list, a recipe and verdict
+ (`inputRuleRecipes`), covering the record size limit at its boundary;
+- schema acceptance verdicts (`schemaRules`): schemas, with the slug each is given under,
+ accepted or rejected under Sections 5 and 5.1;
+- canonical forms and hashes of records and schemas;
+- boundary hashes;
+- a list in key order;
+- one leaf and one interior node, encoded;
+- tree roots for several entry sets: empty, one entry, 1,000 and 100,000 entries, Unicode keys,
+ and a key set with no natural boundaries, so that every leaf split is forced;
+- a file tree;
+- two version roots, one with a private set and its commitment;
+- file-reference extraction cases;
+- one signed log entry, with the key seed that signed it, its signed bytes, its entry hash and the
+ corresponding `head.json`.
+
+Tree vectors give a recipe for generating their entries rather than listing them. Ed25519
+signatures are deterministic, so the log entry vector is reproducible.
+`packages/protocol/scripts/gen-vectors.ts --check` regenerates the vectors and fails on any
+difference.
+
+## Appendix B. Revision history
+
+Changes made while implementing the design (rationale in `edge-redesign-build.md`):
+
+1. Interior entries carry each child's last key, not its first. The boundary rule is defined on
+ last keys, and a merge needs them to reuse unchanged subtrees without reading them.
+2. Record-tree entries carry the record's size, so `bytes` can be verified from nodes alone.
+3. Record ids are limited to 1,024 UTF-8 bytes, which bounds node size.
+4. The unsafe-integer rule applies to integer literals in the source text.
+5. File references have one definition (Section 4); v1 had two.
+6. `LEAF_MAX_ENTRIES` is 8,192 (the design had 16,384). Chunking remains fixed-probability rather
+ than size-aware, since size-aware boundaries depend on position and preclude independent
+ rebuilding of ranges.
+7. Roots are stored as `roots/.json`, without the `ulv2:` prefix.
+8. A record leaf's body is one object of one or more gzip members (Section 11), not several
+ objects.
+9. Packs are pull-only (Section 11.2). Clients publish by delta push (Section 11.4); a server
+ accepts no tree nodes from outside.
+10. Log entries carry `collectionId` (Section 11.1). Added 2026-10-03, before any log held real
+ data.
+
+Clarifications that change no hash, tree or accepted input:
+
+11. 2026-10-04: "format 2" renamed "protocol v2". In the reference implementation
+ `FORMAT_VERSION` became `PROTOCOL_VERSION`, and the vectors file's top-level `format` member
+ became `protocolVersion`.
+12. 2026-10-04: `actorId` written as `null` (Section 11.1).
+13. 2026-10-04: `head.versionHash` checked against the last entry (Section 11.1);
+ `MAX_PATTERN_LENGTH` declared and counted in UTF-16 code units, and `pattern` members inside
+ `const`, `enum`, `default` and `examples` exempted (Section 5); `bad_id` and `bad_type` for
+ absent members, and the order of input-rule codes, stated (Section 3); extra members of a
+ record line ignored (Section 3); `collection.json` serialization declared non-normative
+ (Section 11.1).
+14. 2026-10-05: rewritten in normative form, with requirements language, terminology, a constants
+ table and security considerations, and aligned with the reference implementation: the input
+ rules apply to the whole record line, depth included, and their codes follow text order
+ (Section 3); field-level privacy is defined on `properties` members (Section 5); a declared
+ file belongs to the private set whether or not records reference it, and stays declared until
+ removed, where the earlier "unless the push marks it public" described no mechanism
+ (Section 9); revalidation on a schema change (Section 10.1); `collection.json` has no
+ `description` (Section 11.1); packs are ordered set by set (Section 11.2); which files are
+ held for a collection, and the 400, 451 and repeated-commit responses (Sections 11.3, 11.4).
+15. 2026-10-05: the reference implementation brought into line with this document, changing no
+ hash or tree: a key with an invalid escape is `syntax`, not an exception, and a `\u` escape
+ needs exactly four hex digits (Section 3); field-level `private` and the pattern limit apply
+ to subschemas whose names are also data keywords (Section 5); a receiver checks the shape of
+ the root and PrivateSetObject (Section 11.2). The test vectors add `syntax`, `bad_type`,
+ depth-boundary and code-order cases, `inputRuleRecipes` (`record_too_large`) and
+ `schemaRules` (Sections 5, 5.1); existing vectors are unchanged. Delete lines are subject to the input rules, and a non-object `metadata_patch` is a 400 (Section 11.4).
+16. 2026-10-05: `collection.json` gains `description` (which item 14 had noted as absent),
+ `visibility` and `ark`; `owner` becomes an object carrying the organization's id, DID, handle
+ and name; and a writer rewrites the file whenever a member changes (Section 11.1). Section 11.3
+ specifies the read API servers already served (the collection, versions, records, record
+ history, schemas, differences, file list and export) with the recommended collection URL
+ form, owner handles and DIDs, the collection id form `_/`, and the
+ `x-underlay-collection` header; the manifest's `files` and `since` are documented. No hash,
+ tree or accepted input changes.
diff --git a/docs/v1-read-api.md b/docs/v1-read-api.md
new file mode 100644
index 0000000..308e1e4
--- /dev/null
+++ b/docs/v1-read-api.md
@@ -0,0 +1,242 @@
+# v1 read API: what v2 must match
+
+An inventory of v1's read endpoints (repo root `src/api/*`) and what the existing React UI
+(`src/routes/**/*.data.ts`, `src/components/*`) actually reads, for the v2 read path. It was
+taken from `main` at `7f6e1c6` on 2026-10-03.
+
+Within each section, "Response" is the shape and "UI reads" is what the UI uses. Where they
+differ, only the UI fields are essential.
+
+## Cross-cutting
+
+- **Errors**: `{ error, statusCode }`. Uncaught errors give 500 `Internal server error`.
+- **Missing vs hidden**: an invisible collection returns 404 `Collection not found`, the same as
+ a nonexistent one.
+- **Auth, in order**:
+ - `Authorization: Bearer `: an invalid key is 401;
+ - `?token=` on GET/HEAD: invalid falls back to anonymous;
+ - the session cookie;
+ - anonymous for GETs.
+- **Scope**: `permissions.collections` (admin > write > read). `metadata.collectionIds` limits a
+ key to those collections.
+- **Owner access** means org membership, and a key's collection list (if any) covering the
+ collection. **Visible** means public or owner.
+- **Rate limit**: a 60 s window. 60 requests per IP when anonymous, 5,000 per user. Sends the
+ `X-RateLimit-*` headers; over the limit, 429 with `Retry-After`.
+- **Semver params** accept `1.0.0`, `v1.0.0` or `1`. They're stored as `v1.2.3`.
+- **Share links**:
+ - A view link is a read key with `{scope:'read', collectionIds:[id], linkShare:true}`, valid 30
+ days, used as `//?token=ul_…`.
+ - An agent link is a 1-hour write key at `/agent/`, which serves an HTML page.
+ - The UI builds API URLs with `apiUrlBuilder`, which forwards `?token=`.
+
+## SSR context
+
+`GET /api/context` uses the session cookie only and never returns 401.
+
+```ts
+{ currentUser: null | { id, slug /*default org*/, displayName, avatarUrl, kfRole,
+ defaultOrg: {slug, displayName} | null,
+ orgs: [{ organizationId, slug, displayName, role, isDefault }] },
+ mirrorConfig: { enabled, upstream, nodeName, syncSchedule },
+ kfAccountUrl, kfAuthUrl }
+```
+
+- **UI reads**:
+ - `currentUser.id/slug/displayName/avatarUrl/kfRole/orgs[].slug/isDefault`;
+ - `isOwner` = `kfRole==='admin' || slug===owner || orgs.some(o => o.slug===owner)`.
+- **SSR fetches** go to `http://127.0.0.1:${PORT}` (`fetchBase`). On Workers they must go
+ in-process (build doc finding 9).
+
+## Collections
+
+**`GET /api/collections`**
+
+- **Query**: `q`, `owner`, `tag`, `sort` (`name`, `records`, `featured`, or default updatedAt),
+ `mine=true` (needs a session), `limit` (≤100, default 50), `offset`.
+- **Response**:
+ ```
+ { collections: [{ id, slug, name, public, ownerSlug, ownerName, createdAt, updatedAt,
+ description, tags, latestVersion /*semver*/, recordCount, fileCount, totalBytes, lastPushAt }],
+ facets: { owners: [{slug,name,count}], tags: [{name,count}] },
+ featuredTags: string[], featuredCollections: [same item] }
+ ```
+- **UI reads** (explore and dashboard): `ownerSlug, slug, name, public, description, tags,
+latestVersion, recordCount, totalBytes, lastPushAt, updatedAt`, plus the facets and featured
+ fields.
+- **v1 bugs**: sorting, tag filters and facets run in memory over a window. The home page reads
+ `semver`, which doesn't exist.
+
+**`GET /api/collections/:owner/:slug`**
+
+- **Response**:
+ ```
+ { id, slug, name, public, ownerSlug, ownerName, createdAt, updatedAt, description, ark,
+ versionCount, latestVersion: null | { semver, hash, message, metadata, appId, pushedBy,
+ baseSemver, recordCount, fileCount, totalBytes, createdAt, typeCounts: [{type,count}] } }
+ ```
+- **UI reads**:
+ - collection fields: `public, id, ownerSlug, ownerName, description, versionCount, ark, name,
+slug`;
+ - version fields: `latestVersion.semver`, `.metadata.readme/description/tags`, and the
+ overview's `semver, recordCount, fileCount, totalBytes, createdAt, typeCounts` (array or
+ object), `message, baseSemver, appId, pushedBy, hash`.
+- Non-owners lose `pushedBy/actorId/signature` and private types in `typeCounts`.
+
+**`GET /api/accounts/:owner/collections`**
+
+- **Response**: `[{ id, slug, name, public, createdAt, updatedAt }]`. Members also see private
+ collections; an unknown org gives `[]`.
+
+**`GET /api/collections/:owner/:slug/export?version=v1.2.3`**
+
+- Returns `--.tar.gz` containing:
+ - `manifest.json`: `{collection:{owner,slug,name,description}, version:{semver,hash,message,recordCount,fileCount,totalBytes,createdAt}, schemas, files_missing}`;
+ - `records/.ndjson`;
+ - `files/`.
+
+## Versions
+
+**`GET .../versions?limit&offset`**
+
+- **Response**: a bare array, newest first:
+ `[{ semver, hash, message, appId, actorId? (owner), recordCount, fileCount, totalBytes, createdAt, ark }]`.
+- **UI reads**: `semver, message, recordCount, fileCount, totalBytes, createdAt, hash, ark`. The
+ version picker uses `?limit=20` and accepts an array or `{versions}`.
+
+**`GET .../versions/latest`, `GET .../versions/:n`**
+
+- **Response**:
+ ```
+ { semver, major, minor, patch, hash, baseSemver, message, metadata, pushedBy, appId, actorId,
+ recordCount, fileCount, typeCounts: {type: n}, totalBytes, createdAt,
+ schemas: { slug: JSONSchema }, ark }
+ ```
+- **UI reads**: `semver, schemas` (its keys, and `[t].properties` for table columns),
+ `recordCount, fileCount, totalBytes, createdAt, appId, hash, ark, message, metadata,
+typeCounts, baseSemver, pushedBy`.
+
+**`GET .../versions/:n/records?type&limit(≤2000, default 100)&offset(≤10000)&after|cursor`**
+
+- **Response**:
+ `{ records: [{ id, type, data, hash, ark? }], pagination: { limit, hasMore, nextCursor, total } }`.
+- **Cursor**: `base64url(JSON {r:[recordId, recordHash]})`, or a bare id.
+- **UI reads**: `records[].id/.data/.hash/.ark` and `pagination.total`. The UI pages by offset;
+ past offset 10k, v1 answers 400.
+
+**`GET .../versions/:n/records.ndjson?type&after`**
+
+- Lines are `{id,type,data,hash}`. `X-Underlay-Record-Count` gives the total. Gzip is applied
+ when accepted.
+
+**`GET .../versions/:n/files`**
+
+- **Response**: a bare array, `[{ hash, size, mimeType, createdAt, references: [{recordId,type,field}] }]`.
+- **UI reads**: `hash, mimeType, size, references[].type/.recordId`.
+
+**`GET .../versions/:n/manifest?since&limit(≤100k)&cursor`**
+
+- **Full**:
+ `{ semver, hash, schemas: {slug: schemaHash}, records: [{id,type,hash,private?}], files: string[], pagination }`.
+- **Delta**:
+ `{ semver, hash, since, schemas, delta: {added, updated (+previousHash), removed}, files, pagination, truncated }`.
+- Used by the CLI pull and mirror sync, not the UI.
+
+**`GET .../versions/:n/diff?from&limit(≤5000)&cursor`**
+
+- **Response**:
+ `{ from, to, added: [{id,type,data}], updated: [{id,type,data}], removed: string[], pagination, meta: {schemaChanged, metadataChanged, filesAdded, filesRemoved} }`.
+- **UI reads**: `added, updated, removed, meta.*`. It also reads `meta.readmeChanged`, which v1
+ never returned.
+
+## Schemas
+
+- **`GET /api/schemas?q|label|slug|schema_hash&limit&offset`**
+ - Returns `[{ id, schema, schemaHash, createdAt, labels: string[] }]`. With `schema_hash` it
+ returns one object plus `usageCount`.
+ - A schema is visible when it's non-private in a public collection, or in one of the caller's
+ orgs.
+- **`GET /api/schemas/:id`**
+ - Returns
+ `{ id, schema, schemaHash, createdAt, labels: [{label, createdAt}], usage: [{slug, semver, collection: "owner/slug"}] }`.
+- **`GET /api/collections/:owner/:slug/schemas?version&raw`**
+ - Returns
+ `{ version, semver, schemas: [{ slug, schemaId, schemaHash, schema (+ 'x-underlay-labels') }] }`.
+
+## Records, files, accounts
+
+- **`GET /api/records/:hash/provenance`**
+ - Returns
+ `{ hash, recordId, type, data, size, createdAt, firstSeen, references: [{owner, collection, collectionName, semver, versionCreatedAt}] }`.
+ - References are public collections only.
+- **`POST /api/records/batch`**: `{hashes}` in, NDJSON out.
+- **Files**:
+ - `HEAD|GET /api/collections/:owner/:slug/files/:hash`: GET is a 302 to a presigned URL with
+ `attachment`.
+ - `GET /api/collections/files/:hash`: the same, for any collection the caller can read.
+ - `POST .../files/presign {hashes}` returns `{hash: url|null}`.
+- **Accounts**:
+ - `GET /api/accounts/:slug` returns the org row, plus `displayName` and `arkShoulder`.
+ - `GET /api/accounts/:slug/members` returns `[{role, slug, displayName}]`.
+ - `GET /api/accounts/me` returns
+ `{id, name, email, image, slug, displayName, createdAt, orgs:[{organizationId, role, slug, name, isDefault}]}`.
+- **ARK**:
+ - `GET /api/ark/resolve?path=ark:…` returns `{type:'redirect', url, metadata}`, or 404
+ `{type:'not_found'}`.
+ - `GET .../ark` returns `{enabled, customUrl, arkUrl, shoulder, arkId}`.
+ - `GET .../ark/record-types` returns `[{recordType, redirectUrlField}]`.
+- **Webhooks**:
+ - `GET .../webhooks` returns
+ `{webhooks:[{id,url,bumpFilter,enabled,createdAt,lastDeliveryAt}]}`.
+ - `GET .../webhooks/:id/deliveries` returns `{deliveries:[…]}`.
+- **Health**: `GET /api/health` returns `{status:'ok', timestamp}`.
+- **KF summary**: `GET /api/kf/summary?kf_org_id`, with the KF internal key, returns per-org
+ collection stats.
+
+## v1 behaviour v2 deliberately changes or fixes
+
+- **Records order**:
+ - With `?type=`, records are in id order.
+ - Without it, they're in (type, id) order, and the cursor becomes (type, id) (edge-redesign.md
+ Read path).
+ - Offsets have no 10k cap: they cost O(height) in v2.
+- **Non-owner hashes and counts**:
+ - Non-owners get the public set's real contents, and its counts are exact, not upper bounds.
+ - A version's `hash` is the v2 version hash for everyone. There's no separate public hash.
+- **Field-level privacy is gone**, so nothing is stripped from records.
+- **Manifest `files` and version `/files`** come from the set's file tree and are
+ privacy-correct. The manifest lists files on its first page only, at most 25,000, then
+ `filesTruncated: true`; `/files` returns at most 10,000, with `references` always `[]` and a
+ `referenceCount`.
+- **Inconsistent v1 fields** (`typeCounts` array vs object, `labels` list vs objects) are kept
+ where the UI depends on them.
+- **Auth**: an invalid `?token=` is a 401, as a Bearer key is; it no longer falls back to
+ anonymous. Org-owned keys spend their org's rate budget.
+- **Rate limits**: no `X-RateLimit-*` headers; a 429 has `Retry-After: 60`. Anonymous pages
+ (ARK resolution and `/api/auth/*` included) have their own budget of 600 a minute per IP, and
+ expensive reads cost more than one unit (`packages/server/src/lib/limits.ts`).
+- **Version params** also accept a `ulv2:` version hash.
+- **`/api/context`**: no `mirrorConfig`; adds `siteHost`. Server-rendered pages call the API
+ in-process on both runtimes.
+- **Records**: no per-record `ark`; members' private records carry `private: true`.
+- **records.ndjson**: no server-side gzip. `?after_type=T&after=id` resumes after (T, id)
+ through the later types; `?type=T&after=id` stays within T; `after` with neither, or
+ `after_type` without `after`, is a 400. `X-Underlay-Record-Count` counts what the request
+ returns, after the resume point. Members' private lines end `"private":true`.
+ `records.ndjson.gz?type=` is new (one type's public records as stored).
+- **Manifest**: the default limit is 10,000 and the maximum 25,000; no `truncated` field; delta
+ `removed` entries are `{id, type, hash}`. For members, delta entries in the private set carry
+ `private: true`, and a move between sets is under `updated` with `previousPrivate` (with
+ `previousHash` equal to `hash` when the record didn't change).
+- **Diff**: without `from`, the diff is against the version before (the first version's is
+ against nothing); `removed` entries are `{id, type}`, not bare ids.
+- **Provenance**: covers the caller's own organizations' collections as well as public ones,
+ adds `recordHash`, and is capped at 300 presences and 100 versions per collection.
+- **`POST /api/records/batch`**: 1 to 100 hashes.
+- **Export**: `?format=tar|tar.gz`; `manifest.json` adds `files_withheld`, and `README.md` is
+ included when the metadata has a readme. Tar entries carry the version's creation time, so a
+ version exports to the same bytes every time.
+- **Health**: `{ok: true, version: 2, deployment, time}`.
+- **Agent links**: `GET /agent/` is ported. It serves the HTML instructions page for the
+ share panel's 1-hour, one-collection write key, now describing delta push; a key that isn't
+ such a live key gets a 404 page.
diff --git a/drizzle.config.ts b/drizzle.config.ts
deleted file mode 100644
index d81dc9d..0000000
--- a/drizzle.config.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import type { Config } from 'drizzle-kit'
-
-export default {
- schema: './src/db/schema.ts',
- out: './src/db/migrations',
- dialect: 'postgresql',
- dbCredentials: {
- url: process.env.DATABASE_URL ?? 'postgresql://underlay:underlay@localhost:5432/underlay',
- },
-} satisfies Config
diff --git a/package.json b/package.json
index cf45a5d..6eefbab 100644
--- a/package.json
+++ b/package.json
@@ -4,90 +4,18 @@
"private": true,
"type": "module",
"scripts": {
- "dev": "./dev.sh",
- "dev:app": "tsx watch --clear-screen=false --env-file=.env.local server.ts",
- "build": "vite build --outDir dist/client && vite build --ssr src/entry-server.tsx --outDir dist/server",
- "start": "NODE_ENV=production node --import tsx/esm server.ts",
- "typecheck": "tsc --noEmit",
"lint": "oxlint .",
"fmt": "oxfmt",
"fmt:check": "oxfmt --check",
- "db:generate": "drizzle-kit generate",
- "db:migrate": "tsx src/db/migrate.ts",
- "db:seed": "tsx src/db/seed.ts",
- "db:seed-kf": "tsx src/db/seedKfCollections.ts",
- "tool:backup": "tsx tools/backupDb.ts",
- "tool:restore": "tsx tools/restore.ts",
- "tool:pruneBackups": "tsx tools/pruneBackups.ts",
- "tool:cleanupSessions": "tsx tools/cleanupSessions.ts",
- "tool:pruneWebhookLogs": "tsx tools/pruneWebhookLogs.ts",
- "tool:seed-mirror": "tsx tools/seedMirror.ts",
- "tool:verifyRecordSharing": "tsx tools/verifyRecordSharing.ts",
- "tool:verifyFileRefs": "tsx tools/verifyFileRefs.ts",
- "cli": "tsx src/cli/cli.ts",
- "test": "vitest run",
- "test:watch": "vitest",
- "secrets:encrypt:local": "sops -e --input-type dotenv --output-type dotenv --output .env.local.enc .env.local",
- "secrets:encrypt:prod": "sops -e --input-type dotenv --output-type dotenv --output .env.prod.enc .env.prod",
- "secrets:encrypt:dev": "sops -e --input-type dotenv --output-type dotenv --output .env.dev.enc .env.dev",
- "secrets:decrypt:local": "sops -d --input-type dotenv --output-type dotenv --output .env.local .env.local.enc",
- "secrets:decrypt:prod": "sops -d --input-type dotenv --output-type dotenv --output .env.prod .env.prod.enc",
- "secrets:decrypt:dev": "sops -d --input-type dotenv --output-type dotenv --output .env.dev .env.dev.enc"
- },
- "dependencies": {
- "@aws-sdk/client-s3": "^3.750.0",
- "@aws-sdk/s3-request-presigner": "^3.1104.0",
- "@better-auth/api-key": "^1.6.11",
- "@codemirror/autocomplete": "^6.20.1",
- "@codemirror/commands": "^6.10.3",
- "@codemirror/lang-sql": "^6.10.0",
- "@codemirror/state": "^6.6.0",
- "@codemirror/view": "^6.41.1",
- "@hono/node-server": "^1",
- "@scalar/hono-api-reference": "^0.11.0",
- "@tanstack/react-query": "^5.101.0",
- "ajv": "^8.17.0",
- "ajv-formats": "^3.0.0",
- "better-auth": "^1.6.11",
- "better-sqlite3": "^12.9.0",
- "commander": "^14.0.0",
- "drizzle-orm": "^0.45.0",
- "hono": "^4",
- "hono-zod-openapi": "^1.1.1",
- "isomorphic-dompurify": "^3.16.0",
- "lucide-react": "^1.11.0",
- "marked": "^18.0.0",
- "node-cron": "^4.0.0",
- "postgres": "^3.4.0",
- "react": "^19.1.0",
- "react-dom": "^19.1.0",
- "react-router": "^7",
- "sql.js": "^1.14.1",
- "tar-stream": "^3.1.8",
- "tsx": "^4.19.0",
- "undici": "7.27.2",
- "uuid": "^14.0.0",
- "zod": "^4.4.3"
+ "typecheck": "pnpm --filter \"./packages/*\" typecheck",
+ "test": "pnpm --filter \"./packages/*\" --workspace-concurrency=1 test"
},
"devDependencies": {
- "@tailwindcss/vite": "^4.1.0",
- "@types/better-sqlite3": "^7.6.13",
- "@types/node": "^25.0.0",
- "@types/react": "^19.1.0",
- "@types/react-dom": "^19.1.0",
- "@types/tar-stream": "^3.1.4",
- "@vitejs/plugin-react": "^5.2.0",
- "babel-plugin-react-compiler": "^1.0.0",
- "drizzle-kit": "^0.31.0",
- "happy-dom": "^20.9.0",
"lint-staged": "^17.0.4",
"oxfmt": "latest",
"oxlint": "latest",
"simple-git-hooks": "^2.13.1",
- "tailwindcss": "^4.1.0",
- "typescript": "^6.0.0",
- "vite": "^6",
- "vitest": "^4.1.6"
+ "tsx": "^4.19.0"
},
"simple-git-hooks": {
"pre-commit": "pnpm lint-staged"
diff --git a/packages/cli/README.md b/packages/cli/README.md
index aa0e02b..5af9c6d 100644
--- a/packages/cli/README.md
+++ b/packages/cli/README.md
@@ -1,36 +1,40 @@
# @underlay/cli
-Source lives in `src/cli`; this package just bundles it (`pnpm --filter @underlay/cli build`).
-
-Currently **unpublished and not built** — runnable only from the repo via `pnpm cli`.
-
-## ⚠️ Review record privacy before publishing this for the first time
-
-The CLI predates per-version record privacy (`version_records.private`, added 2026-08) and
-**cannot express it**. Publishing as-is would put a privacy-blind client in users' hands.
-
-Two concrete gaps, both must be closed first:
-
-1. **`src/cli/commands/push.ts` — manifest entries omit `private`.** The manifest is built as
- `{ id, type, hash }`. The server takes each push's manifest as the authoritative statement of
- which records are private, so a push that omits the flag marks every record public. Re-pushing
- a collection that has private records would **silently publish them** in the new version.
-2. **`src/cli/commands/add.ts` — `private` is dropped at ingest.** It parses only
- `{ id, type, data }` and stores the canonical hashed object, which by design excludes privacy
- (privacy is contextual, not part of the content hash). So the flag is lost before `push` could
- ever send it. The local store needs somewhere to carry it — e.g. a per-record privacy set in
- the version manifest (`src/cli/lib/store.ts`, `VersionManifest`), kept outside the hash.
-
-Also settle the server-side semantics this depends on before shipping: a manifest entry's
-`private` is `z.boolean().optional()`, but ingest currently collapses it (`r.private ?? false`),
-so **omitted means public**. If that becomes "omitted means inherit from the base version", the
-CLI's obligations change. See `planning/reference-privacy-model.md`.
-
-## Also check before publishing
-
-- **The npm name `@underlay/cli` is already taken** by a 2023 package from the earlier Underlay
- project ("CLI utility for downloading datasets specified in underlay.yaml", maintainers
- `octref`, `joelg@mit.edu`, author "Knowledge Futures Group"). Publishing needs that account, a
- version above `0.0.1`, or a different name.
-- The CLI is several releases stale against the API — verify it against the current negotiate
- protocol (chunked manifest, async commit) before shipping.
+The Underlay v2 command line. Source is in `src/`; `pnpm --filter @underlay/cli build` bundles it
+to `dist/cli.js`. Unpublished.
+
+A working directory holds a local repository in `.underlay/`: a repository in the protocol layout
+(`repo/`), local versions, staging and remotes (`src/local.ts`). Versions are built with
+`buildVersion` from `@underlay/protocol`, the registry's own commit engine, so pushing a version
+gives the registry the same trees.
+
+```
+underlay init [dir] | clone [dir] [--token]
+underlay schema-set stage the type set ({type: schema})
+underlay add [--strip-unknown-fields]
+ stage records (NDJSON {id, type, data, private?})
+underlay rm stage deletes
+underlay meta-set | --clear
+underlay file add store files records reference ({"$file":"sha256:…"})
+underlay commit -m
+underlay status | log | diff
+underlay fsck [--files] verify the local repository
+underlay remote add -c [-t ] | remove | list
+underlay pull [remote] [--force] remote defaults to origin
+underlay push [remote]
+```
+
+- **pull** verifies the registry's signed log after what was last seen, then receives the newest
+ version as a pack against the last synced one; every tree is re-derived before it's accepted.
+ With a token it asks for the private sets too (`sets=all`), and falls back to the public sets
+ if the token can't read them.
+- **push** sends the local changes since the last sync as a delta push (upserts with their set,
+ and deletes), uploads the files they need, and then requires the registry's new version to be
+ the local one: the same hash, or, when the registry's private salt differs from a new local
+ repository's, the same records, files and metadata, checked by pulling it back. It refuses
+ while the registry has versions not yet pulled.
+- **Privacy** is carried: `private: true` on a record puts it in the private set locally and on
+ the registry, and private types (`"private": true` in the schema) are private throughout.
+
+Before publishing: the npm name `@underlay/cli` belongs to a 2023 package from the earlier
+Underlay project, so publishing needs that account or another name.
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 0a60401..2831863 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,7 @@
{
"name": "@underlay/cli",
- "version": "0.1.0",
+ "version": "0.2.0",
+ "description": "The Underlay command line: a local repository in .underlay/, pulled and pushed with a registry.",
"bin": {
"underlay": "./dist/cli.js"
},
@@ -9,9 +10,18 @@
],
"type": "module",
"scripts": {
- "build": "esbuild ../../src/cli/cli.ts --bundle --platform=node --format=esm --outfile=dist/cli.js --banner:js='#!/usr/bin/env node'"
+ "build": "esbuild src/cli.ts --bundle --platform=node --format=esm --outfile=dist/cli.js --banner:js=\"#!/usr/bin/env node\nimport { createRequire } from 'node:module'; const require = createRequire(import.meta.url);\"",
+ "test": "vitest run",
+ "typecheck": "tsc --noEmit"
+ },
+ "dependencies": {
+ "@underlay/protocol": "workspace:*",
+ "commander": "^14.0.3"
},
"devDependencies": {
- "esbuild": "^0.25.0"
+ "@types/node": "^25.0.0",
+ "esbuild": "^0.27.7",
+ "typescript": "^6.0.0",
+ "vitest": "^4.1.6"
}
}
diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
new file mode 100644
index 0000000..87f17d3
--- /dev/null
+++ b/packages/cli/src/cli.ts
@@ -0,0 +1,147 @@
+/**
+ * underlay: the command line for Underlay v2. A local repository in .underlay/
+ * (local.ts), versions built by the protocol's commit engine, and tree sync with
+ * a registry.
+ */
+import { Command } from 'commander'
+
+import { commit } from './commands/commit.js'
+import { diff, fsck, log, remoteAdd, remoteList, remoteRemove, status } from './commands/info.js'
+import { add, fileAdd, metaSet, rm, schemaSet } from './commands/stage.js'
+import { clone, pull, push } from './commands/sync.js'
+import { CliError, Local } from './local.js'
+
+const say = (line: string) => console.log(line)
+const here = () => Local.require()
+/** Commander wants actions that resolve to nothing. */
+const run = async (p: Promise) => {
+ await p
+}
+
+const program = new Command()
+ .name('underlay')
+ .description('Underlay: versioned, content-addressed collections of records')
+ .version('0.2.0')
+
+program
+ .command('init')
+ .description('Create a local repository')
+ .argument('[dir]', 'directory', '.')
+ .action((dir: string) => {
+ Local.init(dir)
+ say(`Initialized an Underlay repository in ${dir}`)
+ })
+
+program
+ .command('clone')
+ .description('Create a local repository from a registry collection')
+ .argument('', 'registry URL, e.g. https://www.underlay.org')
+ .argument('', 'owner/slug')
+ .argument('[dir]', 'directory (default: the slug)')
+ .option('-t, --token ', 'API key (also fetches the private sets you can read)')
+ .action((url: string, collection: string, dir: string | undefined, o: { token?: string }) =>
+ run(clone(url, collection, dir ?? collection.split('/')[1]!, o, say)),
+ )
+
+program
+ .command('schema-set')
+ .description('Stage the type set from a JSON file of {type: schema}')
+ .argument('')
+ .action((file: string) => schemaSet(here(), file, say))
+
+program
+ .command('add')
+ .description('Stage records from an NDJSON file of {id, type, data, private?}')
+ .argument('')
+ .option('--strip-unknown-fields', 'drop fields the schema does not define')
+ .action((file: string, o: { stripUnknownFields?: boolean }) => run(add(here(), file, o, say)))
+
+program
+ .command('rm')
+ .description('Stage deletes')
+ .argument('')
+ .argument('')
+ .action((type: string, ids: string[]) => rm(here(), type, ids, say))
+
+program
+ .command('meta-set')
+ .description('Stage the version metadata from a JSON file')
+ .argument('[file]')
+ .option('--clear', 'stage no metadata')
+ .action((file: string | undefined, o: { clear?: boolean }) => {
+ if (!file && !o.clear) throw new CliError('Give a metadata file, or --clear')
+ metaSet(here(), o.clear ? null : file!, say)
+ })
+
+const files = program.command('file').description('Files records can reference')
+files
+ .command('add')
+ .description('Store files locally and print their $file references')
+ .argument('')
+ .action((paths: string[]) => run(fileAdd(here(), paths, say)))
+
+program
+ .command('status')
+ .description('Show the head, remotes and staged changes')
+ .action(() => status(here(), say))
+
+program
+ .command('commit')
+ .description('Make a local version from the staged changes')
+ .requiredOption('-m, --message ')
+ .action((o: { message: string }) => run(commit(here(), o.message, say)))
+
+program
+ .command('log')
+ .description('List local versions')
+ .action(() => log(here(), say))
+
+program
+ .command('diff')
+ .description('Compare two local versions')
+ .argument('')
+ .argument('')
+ .action((from: string, to: string) => diff(here(), from, to, say))
+
+program
+ .command('fsck')
+ .description('Check the local repository: versions, trees, bodies, files and logs')
+ .option('--files', 'hash every file, not just check it is there at its size')
+ .action((o: { files?: boolean }) => run(fsck(here(), o, say)))
+
+const remote = program.command('remote').description('Manage registries')
+remote
+ .command('add')
+ .argument('')
+ .argument('')
+ .requiredOption('-c, --collection ')
+ .option('-t, --token ', 'API key')
+ .action((name: string, url: string, o: { collection: string; token?: string }) =>
+ remoteAdd(here(), name, url, o, say),
+ )
+remote
+ .command('remove')
+ .argument('')
+ .action((name: string) => remoteRemove(here(), name, say))
+remote.command('list').action(() => remoteList(here(), say))
+
+program
+ .command('pull')
+ .description("Fetch a remote's newest version and make it the head")
+ .argument('[remote]', 'remote name', 'origin')
+ .option('--force', 'replace local versions that were not pushed')
+ .action((name: string, o: { force?: boolean }) => run(pull(here(), name, o, say)))
+
+program
+ .command('push')
+ .description('Publish the head to a remote')
+ .argument('[remote]', 'remote name', 'origin')
+ .action((name: string) => run(push(here(), name, {}, say)))
+
+program.parseAsync().catch((err: unknown) => {
+ if (err instanceof CliError) {
+ console.error(err.message)
+ process.exit(1)
+ }
+ throw err
+})
diff --git a/packages/cli/src/client.ts b/packages/cli/src/client.ts
new file mode 100644
index 0000000..7c1b497
--- /dev/null
+++ b/packages/cli/src/client.ts
@@ -0,0 +1,94 @@
+/** HTTP access to a registry collection (the v2 API). */
+import {
+ type CollectionInfo,
+ type LogEntry,
+ type PackObject,
+ type SyncSets,
+ untar,
+} from '@underlay/protocol'
+
+import { CliError, type RemoteConfig } from './local.js'
+
+export type Fetch = (url: string, init?: RequestInit) => Promise
+
+export interface LogPage {
+ collection: CollectionInfo | null
+ head: { seq: number; entryHash: string; versionHash: string } | null
+ entries: LogEntry[]
+}
+
+export class Client {
+ readonly base: string
+
+ constructor(
+ readonly remote: RemoteConfig,
+ readonly fetchImpl: Fetch = (url, init) => fetch(url, init),
+ ) {
+ this.base = `${remote.url.replace(/\/+$/, '')}/api/collections/${remote.collection}`
+ }
+
+ async request(path: string, init: RequestInit = {}): Promise {
+ const headers = new Headers(init.headers)
+ if (this.remote.token) headers.set('authorization', `Bearer ${this.remote.token}`)
+ return this.fetchImpl(`${this.base}${path}`, { ...init, headers })
+ }
+
+ async json(path: string, init: RequestInit = {}): Promise {
+ const res = await this.request(path, init)
+ const body = (await res.json().catch(() => null)) as (T & { error?: string }) | null
+ if (!res.ok) {
+ throw new CliError(
+ `${init.method ?? 'GET'} ${path}: ${res.status} ${body?.error ?? res.statusText}`,
+ )
+ }
+ return body as T
+ }
+
+ post(path: string, body: unknown): Promise {
+ return this.json(path, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: JSON.stringify(body),
+ })
+ }
+
+ postNdjson(path: string, lines: string[]): Promise {
+ return this.json(path, {
+ method: 'POST',
+ headers: { 'content-type': 'application/x-ndjson' },
+ body: lines.join('\n') + '\n',
+ })
+ }
+
+ /** Log entries after `after` (every page). */
+ async log(after: number): Promise {
+ const first = await this.json(`/log?after=${after}`)
+ const entries = [...first.entries]
+ while (first.head && entries.length > 0 && entries[entries.length - 1]!.seq < first.head.seq) {
+ const page = await this.json(`/log?after=${entries[entries.length - 1]!.seq}`)
+ if (page.entries.length === 0) break
+ entries.push(...page.entries)
+ }
+ return { ...first, entries }
+ }
+
+ /** A version's pack against a base; with `all`, falls back to public when refused. */
+ async pack(
+ version: string,
+ base: string | null,
+ sets: SyncSets,
+ ): Promise<{ objects: AsyncIterable; sets: SyncSets }> {
+ const query = new URLSearchParams({ sets })
+ if (base) query.set('base', base)
+ const res = await this.request(`/versions/${encodeURIComponent(version)}/pack?${query}`)
+ if (res.status === 403 && sets === 'all') return this.pack(version, base, 'public')
+ if (!res.ok || !res.body) throw new CliError(`Fetching ${version}: ${res.status}`)
+ const body = res.body
+ return {
+ sets,
+ objects: (async function* () {
+ for await (const f of untar(body)) yield { key: f.name, bytes: f.bytes }
+ })(),
+ }
+ }
+}
diff --git a/packages/cli/src/commands/commit.ts b/packages/cli/src/commands/commit.ts
new file mode 100644
index 0000000..6360c1c
--- /dev/null
+++ b/packages/cli/src/commands/commit.ts
@@ -0,0 +1,208 @@
+/**
+ * `underlay commit`: staged changes become a local version, built by the same
+ * engine the registry commits with (`buildVersion`), so the registry builds the
+ * same trees when the version is pushed.
+ */
+import {
+ buildVersion,
+ type BuildTypeInput,
+ type Change,
+ compareUtf8,
+ compileSchema,
+ deriveSemver,
+ fileTree,
+ hashSchema,
+ isPrivateSchema,
+ iterate,
+ keys,
+ MissingFilesError,
+ rebuildFileRefs,
+ OUT_OF_LINE_BYTES,
+ type RecordEntry,
+ RepoSource,
+ type SetObject,
+ sha256Hex,
+ utf8ByteLength,
+} from '@underlay/protocol'
+
+import { CliError, type Local, type LocalVersion, type StagedOp } from '../local.js'
+import { versionState, type VersionState } from '../state.js'
+import type { Say } from './stage.js'
+
+/**
+ * Sizes of files entering a set: from the local file store (`underlay file
+ * add`), else from the file trees of the versions this repository already has.
+ */
+function fileSizer(local: Local, known: SetObject[]) {
+ return async (hashes: string[]) => {
+ const out = new Map()
+ for (const h of hashes) {
+ const head = await local.repo.blobs.head(keys.file(h))
+ if (head) out.set(h, head.size)
+ }
+ const missing = new Set(hashes.filter((h) => !out.has(h)))
+ for (const set of known) {
+ if (missing.size === 0) break
+ for await (const e of iterate(new RepoSource(fileTree, local.repo), set.files.root)) {
+ if (missing.delete(e.key)) out.set(e.key, e.size)
+ }
+ }
+ return out
+ }
+}
+
+/**
+ * The file reference count trees of a version. A pulled version doesn't carry
+ * them (they're writer bookkeeping), so they're rebuilt once from its records:
+ * every reference counts, and a private-set file no record references was
+ * declared.
+ */
+export async function ensureRefs(
+ local: Local,
+ state: VersionState,
+): Promise<{ public: string | null; private: string | null }> {
+ if (state.version.refs) return state.version.refs
+ let refs
+ try {
+ refs = await rebuildFileRefs(local.repo, { public: state.public, private: state.private })
+ } catch (err) {
+ throw new CliError(`${state.version.semver}: ${(err as Error).message}`)
+ }
+ local.writeVersion({ ...state.version, refs })
+ return refs
+}
+
+/** The staged operations, last one per (type, id) winning, sorted per type. */
+function opsByType(ops: StagedOp[]): Map {
+ const last = new Map()
+ for (const op of ops) last.set(`${op.type}\u0000${op.id}`, op)
+ const byType = new Map()
+ for (const op of last.values()) byType.set(op.type, [...(byType.get(op.type) ?? []), op])
+ for (const list of byType.values()) list.sort((a, b) => compareUtf8(a.id, b.id))
+ return byType
+}
+
+export async function commit(local: Local, message: string, say: Say): Promise {
+ const head = local.headVersion()
+ const base = head ? await versionState(local, head) : null
+ const stagedSchemas = local.stagedSchemas()
+ const stagedMetadata = local.stagedMetadata()
+ const ops = local.stagedOps()
+ if (!stagedSchemas && stagedMetadata === undefined && ops.length === 0) {
+ throw new CliError('Nothing staged.')
+ }
+ const schemas = stagedSchemas ?? base?.schemas ?? {}
+ if (Object.keys(schemas).length === 0)
+ throw new CliError('No schemas. Run `underlay schema-set`.')
+ const metadata = stagedMetadata !== undefined ? stagedMetadata : (base?.root.metadata ?? null)
+ const repo = local.repo
+ const byType = opsByType(ops)
+ for (const type of byType.keys()) {
+ if (!schemas[type]) throw new CliError(`Staged records of type "${type}", which has no schema`)
+ }
+
+ const types: BuildTypeInput[] = []
+ for (const [slug, schema] of Object.entries(schemas)) {
+ const privateType = isPrivateSchema(schema)
+ const inPub = !!base?.public.types[slug]?.root
+ const inPriv = !!base?.private.types[slug]?.root
+ const validate = compileSchema(schema)
+ const typeOps = byType.get(slug) ?? []
+ // Revalidate staged records against the schema they will be committed under.
+ for (const op of typeOps) {
+ if (op.op !== 'put') continue
+ const errs = validate((JSON.parse(op.canonical) as { data: unknown }).data)
+ if (errs.length > 0) throw new CliError(`${slug} ${op.id}: ${errs.join('; ')}`)
+ }
+ const entries = new Map()
+ for (const op of typeOps) {
+ if (op.op !== 'put') continue
+ const hash = sha256Hex(op.canonical)
+ const size = utf8ByteLength(op.canonical)
+ const body =
+ size > OUT_OF_LINE_BYTES ? await repo.putOutOfLine(hash, op.canonical) : op.canonical
+ entries.set(op, { key: op.id, hash, size, body })
+ }
+ // As the registry splits a delta push: an upsert goes to its set and is a
+ // delete in the other; a delete goes to both.
+ const stream = (set: 'public' | 'private'): Change[] => {
+ const inBase = set === 'public' ? inPub : inPriv
+ const out: Change[] = []
+ for (const op of typeOps) {
+ if (op.op === 'del') {
+ if (inBase) out.push({ key: op.id, entry: null })
+ continue
+ }
+ const target = privateType || op.private ? 'private' : 'public'
+ if (target === set) out.push({ key: op.id, entry: { ...entries.get(op)! } })
+ else if (inBase) out.push({ key: op.id, entry: null })
+ }
+ return out
+ }
+ types.push({
+ slug,
+ schema,
+ schemaHash: hashSchema(schema),
+ public: privateType ? null : stream('public'),
+ private: stream('private'),
+ })
+ }
+
+ const refs = base ? await ensureRefs(local, base) : null
+ const known = base ? [base.public, base.private] : []
+ let built
+ try {
+ built = await buildVersion(repo, {
+ base: base
+ ? { hash: base.version.hash, publicRefsRoot: refs!.public, privateRefsRoot: refs!.private }
+ : null,
+ types,
+ metadata,
+ salt: local.salt(),
+ fileSizes: fileSizer(local, known),
+ validate: (schema, data) => {
+ const errs = compileSchema(schema)(data)
+ return errs.length > 0 ? errs : null
+ },
+ })
+ } catch (err) {
+ if (err instanceof MissingFilesError) {
+ throw new CliError(
+ `Records reference files this repository doesn't have; add them with \`underlay file add\`:\n ${err.hashes.join('\n ')}`,
+ )
+ }
+ throw err
+ }
+ if (built.status === 'invalid') {
+ throw new CliError(
+ `The new schemas reject ${built.total} existing record(s):\n ${built.errors
+ .slice(0, 20)
+ .map((e) => `${e.type} ${e.recordId}: ${e.errors.join('; ')}`)
+ .join('\n ')}`,
+ )
+ }
+ if (base && built.versionHash === base.version.hash) {
+ local.clearStaging()
+ throw new CliError('No changes: the staged changes leave the version as it is.')
+ }
+ const sv = deriveSemver(head?.semver ?? null, built.schemaChanged, built.recordsChanged)
+ if (local.version(sv.semver)) {
+ throw new CliError(`Version ${sv.semver} already exists here`)
+ }
+ const version: LocalVersion = {
+ semver: sv.semver,
+ hash: built.versionHash,
+ baseSemver: head?.semver ?? null,
+ message,
+ createdAt: new Date().toISOString(),
+ refs: { public: built.publicRefsRoot, private: built.privateRefsRoot },
+ sets: 'all',
+ }
+ local.writeVersion(version)
+ local.setHead(version.semver)
+ local.clearStaging()
+ const { added, removed, updated } = built.stats
+ say(`${sv.semver} ${built.versionHash.slice(0, 17)}… ${message}`)
+ say(` +${added} ~${updated} -${removed} record(s)`)
+ return version
+}
diff --git a/packages/cli/src/commands/info.ts b/packages/cli/src/commands/info.ts
new file mode 100644
index 0000000..6e2e30c
--- /dev/null
+++ b/packages/cli/src/commands/info.ts
@@ -0,0 +1,141 @@
+/** Read-only commands: status, log, diff, and managing remotes. */
+import { diffTrees, fsck as checkRepo, listAll, recordTree, RepoSource } from '@underlay/protocol'
+
+import { CliError, type Local } from '../local.js'
+import { versionState } from '../state.js'
+import type { Say } from './stage.js'
+
+export function status(local: Local, say: Say): void {
+ const head = local.headVersion()
+ say(head ? `On ${head.semver} (${head.hash.slice(0, 17)}…)` : 'No versions yet.')
+ for (const name of Object.keys(local.remotes())) {
+ const t = local.tracking(name)
+ if (!t) say(` ${name}: never synced`)
+ else if (head && t.hash === head.hash) say(` ${name}: up to date at ${t.semver}`)
+ else say(` ${name}: last synced at ${t.semver}; ${head?.semver ?? 'nothing'} not pushed`)
+ }
+ const schemas = local.stagedSchemas()
+ const metadata = local.stagedMetadata()
+ const ops = local.stagedOps()
+ if (!schemas && metadata === undefined && ops.length === 0) {
+ say('Nothing staged.')
+ return
+ }
+ say('Staged:')
+ if (schemas) say(` schemas: ${Object.keys(schemas).join(', ')}`)
+ if (metadata !== undefined) say(metadata === null ? ' metadata: cleared' : ' metadata')
+ const puts = ops.filter((o) => o.op === 'put').length
+ if (puts > 0) say(` ${puts} upsert(s)`)
+ if (ops.length > puts) say(` ${ops.length - puts} delete(s)`)
+}
+
+export function log(local: Local, say: Say): void {
+ const head = local.head()
+ const versions = local.versions()
+ if (versions.length === 0) say('No versions yet.')
+ for (const v of versions.reverse()) {
+ const mark = v.semver === head ? '*' : ' '
+ const from = v.remote ? ` [${v.remote}]` : ''
+ say(
+ `${mark} ${v.semver} ${v.createdAt.slice(0, 19)} ${v.hash.slice(0, 17)}…${from} ${v.message ?? ''}`,
+ )
+ }
+}
+
+/** Records added, changed and removed between two local versions, per type. */
+export async function diff(local: Local, from: string, to: string, say: Say): Promise {
+ const a = local.version(from)
+ const b = local.version(to)
+ if (!a || !b) throw new CliError(`No local version ${!a ? from : to}`)
+ const sa = await versionState(local, a)
+ const sb = await versionState(local, b)
+ const source = new RepoSource(recordTree, local.repo)
+ const slugs = new Set([...Object.keys(sa.schemas), ...Object.keys(sb.schemas)])
+ for (const slug of [...slugs].sort()) {
+ const n = { added: 0, updated: 0, removed: 0 }
+ const sample: string[] = []
+ for (const set of ['public', 'private'] as const) {
+ for await (const d of diffTrees(
+ source,
+ sa[set].types[slug]?.root ?? null,
+ sb[set].types[slug]?.root ?? null,
+ )) {
+ const kind = !d.before ? 'added' : !d.after ? 'removed' : 'updated'
+ n[kind]++
+ if (sample.length < 5)
+ sample.push(`${kind === 'added' ? '+' : kind === 'removed' ? '-' : '~'}${d.key}`)
+ }
+ }
+ if (!sa.schemas[slug]) say(`${slug}: new type`)
+ else if (!sb.schemas[slug]) say(`${slug}: removed`)
+ if (n.added + n.updated + n.removed > 0) {
+ say(`${slug}: +${n.added} ~${n.updated} -${n.removed} ${sample.join(' ')}`)
+ }
+ }
+ if (JSON.stringify(sa.root.metadata) !== JSON.stringify(sb.root.metadata)) say('metadata changed')
+}
+
+export function remoteAdd(
+ local: Local,
+ name: string,
+ url: string,
+ opts: { collection?: string; token?: string },
+ say: Say,
+): void {
+ if (!opts.collection || !/^[^/]+\/[^/]+$/.test(opts.collection)) {
+ throw new CliError('Give the collection as --collection owner/slug')
+ }
+ const remotes = local.remotes()
+ if (remotes[name]) throw new CliError(`Remote "${name}" already exists`)
+ remotes[name] = { url, collection: opts.collection, ...(opts.token ? { token: opts.token } : {}) }
+ local.setRemotes(remotes)
+ say(`Added ${name}: ${url} ${opts.collection}`)
+}
+
+export function remoteRemove(local: Local, name: string, say: Say): void {
+ const remotes = local.remotes()
+ if (!remotes[name]) throw new CliError(`No remote named "${name}"`)
+ delete remotes[name]
+ local.setRemotes(remotes)
+ local.setTracking(name, null)
+ say(`Removed ${name}`)
+}
+
+export function remoteList(local: Local, say: Say): void {
+ for (const [name, r] of Object.entries(local.remotes())) {
+ say(`${name} ${r.url} ${r.collection}${r.token ? ' (token)' : ''}`)
+ }
+}
+
+/**
+ * Check the local repository: every local version's root, trees, bodies and
+ * files, and any collection log stored in it (under the keys its
+ * collection.json declares, which shows integrity, not who signed). A clone
+ * verifies the remote's log over the API and doesn't keep it.
+ */
+export async function fsck(local: Local, opts: { files?: boolean }, say: Say): Promise {
+ const repo = local.repo
+ const ids = new Set()
+ for await (const k of listAll(repo.blobs, 'collections/')) {
+ const id = k.split('/')[1]
+ if (id) ids.add(id)
+ }
+ let ok = true
+ const show = (what: string, r: Awaited>) => {
+ say(
+ `${what}: ${r.ok ? 'ok' : 'PROBLEMS'} (${r.versions} versions, ${r.trees} trees, ${r.leaves} leaves, ${r.records} records, ${r.files} files)`,
+ )
+ for (const e of r.errors) say(` ${e}`)
+ if (r.moreErrors) say(` …and ${r.moreErrors} more`)
+ ok &&= r.ok
+ }
+ show(
+ 'local versions',
+ await checkRepo(repo, {
+ versions: local.versions().map((v) => v.hash),
+ fileBytes: !!opts.files,
+ }),
+ )
+ for (const id of ids) show(`log of ${id}`, await checkRepo(repo, { collectionId: id }))
+ if (!ok) throw new CliError('fsck found problems')
+}
diff --git a/packages/cli/src/commands/stage.ts b/packages/cli/src/commands/stage.ts
new file mode 100644
index 0000000..1ef3699
--- /dev/null
+++ b/packages/cli/src/commands/stage.ts
@@ -0,0 +1,151 @@
+/**
+ * Staging: what the next `underlay commit` will contain. Records go through the
+ * same input rules and schema validation as the registry's ingest, so what
+ * stages here is what a push will accept.
+ */
+import { createReadStream, readFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { createInterface } from 'node:readline'
+
+import {
+ checkSchema,
+ compileSchema,
+ InputRuleError,
+ isPrivateSchema,
+ keys,
+ parseRecordLine,
+ recordCanonical,
+ sha256Hex,
+ stripToSchema,
+} from '@underlay/protocol'
+
+import { CliError, type Local, type StagedOp } from '../local.js'
+import { versionState } from '../state.js'
+
+export type Say = (line: string) => void
+
+/** The schemas the next commit will have: staged, else the head's. */
+export async function currentSchemas(
+ local: Local,
+): Promise>> {
+ const staged = local.stagedSchemas()
+ if (staged) return staged
+ const head = local.headVersion()
+ return head ? (await versionState(local, head)).schemas : {}
+}
+
+/** Stage the full type set from a JSON file `{slug: schema, …}`. */
+export async function schemaSet(local: Local, file: string, say: Say): Promise {
+ const schemas = JSON.parse(readFileSync(resolve(file), 'utf8')) as unknown
+ if (!schemas || typeof schemas !== 'object' || Array.isArray(schemas)) {
+ throw new CliError('A schema file is an object of type → schema')
+ }
+ const out: Record> = {}
+ for (const [slug, body] of Object.entries(schemas)) {
+ if (!body || typeof body !== 'object' || Array.isArray(body)) {
+ throw new CliError(`Schema "${slug}" must be an object`)
+ }
+ const err = checkSchema(slug, body)
+ if (err) throw new CliError(err)
+ compileSchema(body)
+ await local.repo.putSchema(body)
+ out[slug] = body as Record
+ }
+ if (Object.keys(out).length === 0) throw new CliError('The schema file defines no types')
+ local.stageSchemas(out)
+ say(`Staged ${Object.keys(out).length} type(s): ${Object.keys(out).join(', ')}`)
+}
+
+export interface AddOptions {
+ /** Drop fields the schema doesn't define instead of refusing the record. */
+ stripUnknownFields?: boolean
+}
+
+/** Stage upserts from an NDJSON file of `{id, type, data, private?}`. */
+export async function add(local: Local, file: string, opts: AddOptions, say: Say): Promise {
+ const schemas = await currentSchemas(local)
+ if (Object.keys(schemas).length === 0) {
+ throw new CliError('No schemas yet. Stage them first with `underlay schema-set`.')
+ }
+ const errors: string[] = []
+ const ops: StagedOp[] = []
+ let line = 0
+ const rl = createInterface({ input: createReadStream(resolve(file)), crlfDelay: Infinity })
+ for await (const text of rl) {
+ line++
+ if (text.trim() === '') continue
+ try {
+ const rec = parseRecordLine(text)
+ const schema = schemas[rec.type]
+ if (!schema) throw new CliError(`No schema for type "${rec.type}"`)
+ let { data, canonical } = rec
+ const props = schema.properties as Record | undefined
+ if (props && data !== null && typeof data === 'object' && !Array.isArray(data)) {
+ const extra = Object.keys(data).filter((k) => !(k in props))
+ if (extra.length > 0) {
+ if (!opts.stripUnknownFields) {
+ throw new CliError(
+ `Fields not in the schema: ${extra.join(', ')} (use --strip-unknown-fields to drop them)`,
+ )
+ }
+ data = stripToSchema(data as Record, props)
+ canonical = recordCanonical(rec.id, rec.type, data)
+ }
+ }
+ const errs = compileSchema(schema)(data)
+ if (errs.length > 0) throw new CliError(errs.join('; '))
+ const isPrivate = rec.private === true && !isPrivateSchema(schema)
+ ops.push({
+ op: 'put',
+ type: rec.type,
+ id: rec.id,
+ canonical,
+ ...(isPrivate ? { private: true as const } : {}),
+ })
+ } catch (err) {
+ if (!(err instanceof CliError || err instanceof InputRuleError)) throw err
+ if (errors.length < 20) errors.push(`line ${line}: ${err.message}`)
+ else if (errors.length === 20) errors.push('…')
+ }
+ }
+ if (errors.length > 0)
+ throw new CliError(`Nothing staged; invalid records:\n ${errors.join('\n ')}`)
+ local.stageOps(ops)
+ say(`Staged ${ops.length} record(s)`)
+ return ops.length
+}
+
+/** Stage deletes of records by type and id. */
+export function rm(local: Local, type: string, ids: string[], say: Say): void {
+ if (ids.length === 0) throw new CliError('Name at least one record id')
+ local.stageOps(ids.map((id) => ({ op: 'del' as const, type, id })))
+ say(`Staged ${ids.length} delete(s)`)
+}
+
+/** Stage the version metadata from a JSON object file, or clear it. */
+export function metaSet(local: Local, file: string | null, say: Say): void {
+ if (file === null) {
+ local.stageMetadata(null)
+ say('Staged: no metadata')
+ return
+ }
+ const value = JSON.parse(readFileSync(resolve(file), 'utf8')) as unknown
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
+ throw new CliError('Metadata must be a JSON object')
+ }
+ local.stageMetadata(value as Record)
+ say('Staged metadata')
+}
+
+/** Store files in the local repository, so records can reference them. */
+export async function fileAdd(local: Local, paths: string[], say: Say): Promise {
+ const out: string[] = []
+ for (const p of paths) {
+ const bytes = new Uint8Array(readFileSync(resolve(p)))
+ const hash = sha256Hex(bytes)
+ await local.repo.blobs.put(keys.file(hash), bytes, { ifAbsent: true })
+ say(`${p}: {"$file":"sha256:${hash}"}`)
+ out.push(hash)
+ }
+ return out
+}
diff --git a/packages/cli/src/commands/sync.ts b/packages/cli/src/commands/sync.ts
new file mode 100644
index 0000000..27240a6
--- /dev/null
+++ b/packages/cli/src/commands/sync.ts
@@ -0,0 +1,419 @@
+/**
+ * Moving versions between the local repository and a registry
+ * (edge-redesign-build.md, "Tree sync").
+ *
+ * pull verify the remote's signed log after what was last seen, then receive
+ * the newest version as a pack against the last one pulled or pushed
+ * (receiveVersion re-derives every tree before anything is accepted).
+ * push the local changes since the last sync, sent as a delta push (upserts
+ * and deletes) with the files they need; then the registry's new version
+ * must be the local one. Same hash, or (when the private salt differs)
+ * the same trees, checked by pulling it back.
+ */
+import {
+ compareUtf8,
+ type DiffEntry,
+ diffTrees,
+ fileTree,
+ isPrivateSchema,
+ keys,
+ type PrivateSetObject,
+ receiveVersion,
+ type RecordEntry,
+ recordTree,
+ RepoSource,
+ type SetObject,
+ type SyncSets,
+ verifyLogEntries,
+} from '@underlay/protocol'
+
+import { Client, type Fetch } from '../client.js'
+import { CliError, Local, type LocalVersion, type Tracking } from '../local.js'
+import { recordById, versionState, type VersionState } from '../state.js'
+import type { Say } from './stage.js'
+
+const BATCH_LINES = 5000
+const BATCH_BYTES = 8 * 1024 * 1024
+const SMALL_UPLOAD_BYTES = 32 * 1024 * 1024
+const POLL_MS = 1000
+
+export interface SyncOptions {
+ fetch?: Fetch
+ /** pull: replace local commits that weren't pushed. */
+ force?: boolean
+}
+
+// --- pull ------------------------------------------------------------------------
+
+export async function pull(
+ local: Local,
+ name: string,
+ opts: SyncOptions,
+ say: Say,
+): Promise {
+ const cfg = local.remote(name)
+ const client = new Client(cfg, opts.fetch)
+ const t = local.tracking(name)
+ const page = await client.log(t?.seq ?? 0)
+ if (!page.collection || page.entries.length === 0) {
+ say(t ? 'Already up to date.' : 'The remote has no versions yet.')
+ return null
+ }
+ const keys = page.collection.keys
+ const verified = await verifyLogEntries(
+ page.entries,
+ keys,
+ t && { seq: t.seq, entryHash: t.entryHash },
+ page.collection.id,
+ )
+ const latest = page.entries[page.entries.length - 1]!
+ const head = local.headVersion()
+ if (head && head.hash !== t?.hash && !opts.force) {
+ throw new CliError(
+ t
+ ? `${head.semver} isn't on ${name}; push it first, or pull --force to drop it.`
+ : `This repository has versions that didn't come from ${name}; pull --force to replace them.`,
+ )
+ }
+ const want: SyncSets = cfg.token ? 'all' : 'public'
+ // The last version synced anchors the pack, if this repository holds the sets asked for.
+ const base = t && (t.sets === 'all' || want === 'public') ? t.hash : null
+ const pack = await client.pack(latest.versionHash, base, want)
+ const got = await receiveVersion(local.repo, pack.objects, {
+ target: latest.versionHash,
+ base,
+ sets: pack.sets,
+ })
+ if (pack.sets === 'all' && got.root.private) {
+ local.setSalt((await local.repo.privateSet(got.root.private)).salt)
+ }
+ const version: LocalVersion = {
+ semver: latest.semver,
+ hash: latest.versionHash,
+ baseSemver: latest.baseSemver,
+ message: latest.message,
+ createdAt: latest.createdAt,
+ refs: null,
+ remote: name,
+ sets: pack.sets,
+ }
+ local.writeVersion(version)
+ local.setHead(version.semver)
+ local.setTracking(name, {
+ seq: latest.seq,
+ entryHash: verified!.entryHash,
+ semver: latest.semver,
+ hash: latest.versionHash,
+ sets: pack.sets,
+ keys,
+ })
+ const c = got.changes
+ say(`Pulled ${latest.semver} from ${name}: +${c.added} ~${c.updated} -${c.removed} record(s)`)
+ return version
+}
+
+/** `underlay clone [dir]`: init, add the remote as origin, pull. */
+export async function clone(
+ url: string,
+ collection: string,
+ dir: string,
+ opts: SyncOptions & { token?: string },
+ say: Say,
+): Promise {
+ const local = Local.init(dir)
+ local.setRemotes({
+ origin: { url, collection, ...(opts.token ? { token: opts.token } : {}) },
+ })
+ await pull(local, 'origin', opts, say)
+ return local
+}
+
+// --- push ------------------------------------------------------------------------
+
+type Op = { put: string } | { del: string }
+
+/** A record's push line: its canonical form, plus the private flag where it matters. */
+function putLine(r: RecordEntry, isPrivate: boolean): string {
+ return isPrivate ? `${r.body!.slice(0, -1)},"private":true}` : r.body!
+}
+
+/**
+ * The upserts and deletes that turn `base` into `head`, per type in key order:
+ * a record present in head is sent with the set it's in; one gone from both
+ * sets is deleted.
+ */
+async function* changesToPush(
+ local: Local,
+ base: VersionState | null,
+ head: VersionState,
+): AsyncGenerator {
+ const repo = local.repo
+ const source = new RepoSource(recordTree, repo)
+ for (const [slug, schema] of Object.entries(head.schemas)) {
+ const privateType = isPrivateSchema(schema)
+ const pubDiff = diffTrees(
+ source,
+ base?.public.types[slug]?.root ?? null,
+ head.public.types[slug]?.root ?? null,
+ )[Symbol.asyncIterator]()
+ const privDiff = diffTrees(
+ source,
+ base?.private.types[slug]?.root ?? null,
+ head.private.types[slug]?.root ?? null,
+ )[Symbol.asyncIterator]()
+ let a = await pubDiff.next()
+ let b = await privDiff.next()
+ while (!a.done || !b.done) {
+ const ka = a.done ? null : a.value.key
+ const kb = b.done ? null : b.value.key
+ const key = ka === null ? kb! : kb === null ? ka : compareUtf8(ka, kb) <= 0 ? ka : kb
+ const pd: DiffEntry | null = ka === key ? a.value! : null
+ const vd: DiffEntry | null = kb === key ? b.value! : null
+ if (pd) a = await pubDiff.next()
+ if (vd) b = await privDiff.next()
+ if (pd?.after) {
+ const r = await recordById(repo, head.public.types[slug]!.root, key)
+ yield { put: putLine(r!, false) }
+ } else if (vd?.after) {
+ const r = await recordById(repo, head.private.types[slug]!.root, key)
+ yield { put: putLine(r!, !privateType) }
+ } else {
+ yield { del: JSON.stringify({ type: slug, id: key }) }
+ }
+ }
+ }
+}
+
+/** Files the head's sets have that the base's don't, which the registry may lack. */
+async function newFiles(local: Local, base: VersionState | null, head: VersionState) {
+ const out = new Map()
+ const source = new RepoSource(fileTree, local.repo)
+ const pairs: [SetObject | null, SetObject][] = [
+ [base?.public ?? null, head.public],
+ [base?.private ?? null, head.private],
+ ]
+ for (const [b, h] of pairs) {
+ for await (const d of diffTrees(source, b?.files.root ?? null, h.files.root)) {
+ if (d.after) out.set(d.key, d.after.size)
+ }
+ }
+ return out
+}
+
+async function uploadFiles(local: Local, client: Client, files: Map, say: Say) {
+ for (const [hash, size] of files) {
+ const obj = await local.repo.blobs.get(keys.file(hash))
+ if (!obj)
+ throw new CliError(`File ${hash} isn't in this repository; add it with \`underlay file add\``)
+ if (size <= SMALL_UPLOAD_BYTES) {
+ const res = await client.request(`/files/${hash}`, {
+ method: 'PUT',
+ headers: { 'content-type': 'application/octet-stream' },
+ body: (await obj.bytes()) as BodyInit,
+ })
+ if (!res.ok) throw new CliError(`Uploading file ${hash}: ${res.status}`)
+ continue
+ }
+ // Large: a direct upload to storage, then wait for the registry to verify it.
+ type Part = { partNumber: number; url: string }
+ const ticket = await client.post<{
+ id: string
+ url?: string
+ partBytes?: number
+ partCount?: number
+ parts?: Part[]
+ }>('/files/uploads', { hash, size })
+ const fetchRaw = client.fetchImpl
+ const parts: { partNumber: number; etag: string }[] = []
+ if (ticket.url) {
+ const res = await fetchRaw(ticket.url, {
+ method: 'PUT',
+ body: (await obj.bytes()) as BodyInit,
+ })
+ if (!res.ok) throw new CliError(`Uploading file ${hash}: ${res.status}`)
+ } else {
+ const partBytes = ticket.partBytes ?? Math.ceil(size / ticket.parts!.length)
+ const partCount = ticket.partCount ?? ticket.parts!.length
+ // The ticket presigns the first page of parts; the rest come a page at a time.
+ let page = ticket.parts!
+ for (let n = 1; n <= partCount; n++) {
+ if (!page.some((x) => x.partNumber === n)) {
+ page = (
+ await client.json<{ parts: Part[] }>(`/files/uploads/${ticket.id}/parts?from=${n}`)
+ ).parts
+ }
+ const p = page.find((x) => x.partNumber === n)
+ if (!p) throw new CliError(`No upload URL for part ${n} of ${hash}`)
+ const offset = (p.partNumber - 1) * partBytes
+ const chunk = await local.repo.blobs.get(keys.file(hash), { offset, length: partBytes })
+ const res = await fetchRaw(p.url, {
+ method: 'PUT',
+ body: (await chunk!.bytes()) as BodyInit,
+ })
+ if (!res.ok) throw new CliError(`Uploading part ${p.partNumber} of ${hash}: ${res.status}`)
+ parts.push({ partNumber: p.partNumber, etag: res.headers.get('etag') ?? '' })
+ }
+ }
+ await client.post(`/files/uploads/${ticket.id}/complete`, parts.length > 0 ? { parts } : {})
+ for (;;) {
+ const s = await client.json<{ status: string; error?: string }>(`/files/uploads/${ticket.id}`)
+ if (s.status === 'verified') break
+ if (s.status === 'failed')
+ throw new CliError(`File ${hash} failed verification: ${s.error ?? ''}`)
+ await new Promise((r) => setTimeout(r, POLL_MS))
+ }
+ }
+ if (files.size > 0) say(`Uploaded ${files.size} file(s)`)
+}
+
+/** Send ops in batches under the registry's request limits. */
+async function sendOps(client: Client, sid: string, ops: AsyncIterable) {
+ const pending = { put: [] as string[], del: [] as string[] }
+ const bytes = { put: 0, del: 0 }
+ const counts = { put: 0, del: 0 }
+ const flush = async (kind: 'put' | 'del') => {
+ if (pending[kind].length === 0) return
+ await client.postNdjson(`/push/${sid}/${kind === 'put' ? 'records' : 'deletes'}`, pending[kind])
+ counts[kind] += pending[kind].length
+ pending[kind] = []
+ bytes[kind] = 0
+ }
+ for await (const op of ops) {
+ const kind = 'put' in op ? 'put' : 'del'
+ const line = 'put' in op ? op.put : op.del
+ pending[kind].push(line)
+ bytes[kind] += line.length + 1
+ if (pending[kind].length >= BATCH_LINES || bytes[kind] >= BATCH_BYTES) await flush(kind)
+ }
+ await flush('put')
+ await flush('del')
+ return counts
+}
+
+/** Same records, files and metadata: what a push must reproduce. */
+function sameContent(a: VersionState, b: VersionState, sets: SyncSets): boolean {
+ const pick = (s: VersionState) => ({
+ metadata: s.root.metadata,
+ public: s.public,
+ private: sets === 'all' ? { types: s.private.types, files: s.private.files } : null,
+ })
+ return JSON.stringify(pick(a)) === JSON.stringify(pick(b))
+}
+
+export async function push(
+ local: Local,
+ name: string,
+ opts: SyncOptions,
+ say: Say,
+): Promise {
+ const cfg = local.remote(name)
+ const client = new Client(cfg, opts.fetch)
+ const head = local.headVersion()
+ if (!head) throw new CliError('Nothing to push: commit something first.')
+ const t: Tracking | null = local.tracking(name)
+ if (t && head.hash === t.hash) {
+ say('Everything up to date.')
+ return head
+ }
+ const remote = await client.log(t?.seq ?? 0)
+ if (remote.entries.length > 0) {
+ throw new CliError(`${name} has versions this repository hasn't pulled. Pull first.`)
+ }
+ const tracked = t ? local.version(t.semver) : null
+ const baseState = tracked ? await versionState(local, tracked) : null
+ const headState = await versionState(local, head)
+ const sets: SyncSets = t?.sets ?? 'all'
+
+ await uploadFiles(local, client, await newFiles(local, baseState, headState), say)
+ const session = await client.post<{ session_id: string }>('/push', {
+ base: t?.semver ?? null,
+ schemas: headState.schemas,
+ metadata: headState.root.metadata,
+ message: head.message,
+ })
+ const sid = session.session_id
+ const sent = await sendOps(client, sid, changesToPush(local, baseState, headState))
+
+ const res = await client.request(`/push/${sid}/commit`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: '{}',
+ })
+ let result = (await res.json()) as { semver?: string; hash?: string; error?: string }
+ if (res.status === 202) {
+ for (;;) {
+ await new Promise((r) => setTimeout(r, POLL_MS))
+ const s = await client.json<{
+ status: string
+ result: { semver: string; hash: string } | null
+ error: { error?: string } | null
+ }>(`/push/${sid}`)
+ if (s.status === 'committed') {
+ result = s.result!
+ break
+ }
+ if (s.status !== 'committing')
+ throw new CliError(`Push failed: ${s.error?.error ?? s.status}`)
+ }
+ } else if (res.status !== 201) {
+ throw new CliError(`Push failed: ${res.status} ${result.error ?? ''}`)
+ }
+
+ // The registry's log now ends with the new version.
+ const after = await client.log(t?.seq ?? 0)
+ const verified = await verifyLogEntries(
+ after.entries,
+ after.collection!.keys,
+ t && { seq: t.seq, entryHash: t.entryHash },
+ after.collection!.id,
+ )
+ const entry = after.entries[after.entries.length - 1]
+ if (!entry || entry.versionHash !== result.hash) {
+ throw new CliError('The registry committed, but its log does not end with the new version')
+ }
+ if (result.hash !== head.hash) {
+ // Typically the private salt: the registry's differs from a local one. Pull
+ // its version back and compare content, not hashes.
+ const pack = await client.pack(result.hash!, t?.hash ?? null, sets)
+ const got = await receiveVersion(local.repo, pack.objects, {
+ target: result.hash!,
+ base: t?.hash ?? null,
+ sets: pack.sets,
+ })
+ const theirs = await versionState(local, {
+ ...head,
+ hash: result.hash!,
+ sets: pack.sets,
+ })
+ if (!sameContent(theirs, headState, pack.sets)) {
+ throw new CliError(
+ `${name} built ${result.semver} (${result.hash}) with different content from ${head.semver}`,
+ )
+ }
+ if (got.root.private && pack.sets === 'all') {
+ local.setSalt(((await local.repo.privateSet(got.root.private)) as PrivateSetObject).salt)
+ }
+ }
+ const version: LocalVersion = {
+ semver: result.semver!,
+ hash: result.hash!,
+ baseSemver: t?.semver ?? null,
+ message: head.message,
+ createdAt: entry.createdAt,
+ refs: result.hash === head.hash ? head.refs : null,
+ remote: name,
+ sets,
+ }
+ local.writeVersion(version)
+ local.setHead(version.semver)
+ local.setTracking(name, {
+ seq: entry.seq,
+ entryHash: verified!.entryHash,
+ semver: version.semver,
+ hash: version.hash,
+ sets,
+ keys: after.collection!.keys,
+ })
+ say(`Pushed ${version.semver} to ${name}: ${sent.put} upsert(s), ${sent.del} delete(s)`)
+ return version
+}
diff --git a/packages/cli/src/local.ts b/packages/cli/src/local.ts
new file mode 100644
index 0000000..7f135a7
--- /dev/null
+++ b/packages/cli/src/local.ts
@@ -0,0 +1,243 @@
+/**
+ * The local repository: `.underlay/` in a working directory.
+ *
+ * repo/ a repository in the protocol layout (fileStore): nodes,
+ * bodies, roots, schemas, private set objects, files
+ * HEAD the current local version's semver (empty before the first)
+ * versions/.json local versions (LocalVersion)
+ * salt the private-set salt (the registry's, once pulled)
+ * staging/schemas.json staged type schemas (the full new type set)
+ * staging/metadata.json staged metadata
+ * staging/ops.ndjson staged upserts and deletes, in order (the last per record wins)
+ * config.json remotes
+ * remotes/.json what was last pulled from or pushed to a remote (Tracking)
+ */
+import {
+ appendFileSync,
+ existsSync,
+ mkdirSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs'
+import { dirname, join, resolve } from 'node:path'
+
+import {
+ compareSemver,
+ fileStore,
+ newSalt,
+ openRepo,
+ type PublicKeyInfo,
+ type Repo,
+} from '@underlay/protocol'
+
+export const DIR = '.underlay'
+
+export class CliError extends Error {
+ constructor(message: string) {
+ super(message)
+ this.name = 'CliError'
+ }
+}
+
+export interface LocalVersion {
+ semver: string
+ hash: string
+ baseSemver: string | null
+ message: string | null
+ createdAt: string
+ /**
+ * The file reference count trees (writer bookkeeping, not in the version
+ * itself). Null when unknown, as for a pulled version; rebuilt on demand.
+ */
+ refs: { public: string | null; private: string | null } | null
+ /** Set when this version came from, or was confirmed by, a remote. */
+ remote?: string
+ /** The sets this repository holds of the version. */
+ sets: 'public' | 'all'
+}
+
+export interface RemoteConfig {
+ url: string
+ /** owner/slug */
+ collection: string
+ token?: string
+}
+
+export interface Tracking {
+ seq: number
+ entryHash: string
+ semver: string
+ hash: string
+ sets: 'public' | 'all'
+ /** The keys the remote's log was verified with. */
+ keys: PublicKeyInfo[]
+}
+
+/** One staged operation: an upsert (canonical record) or a delete. */
+export type StagedOp =
+ | { op: 'put'; type: string; id: string; canonical: string; private?: true }
+ | { op: 'del'; type: string; id: string }
+
+const readJson = (path: string): T | null =>
+ existsSync(path) ? (JSON.parse(readFileSync(path, 'utf8')) as T) : null
+const writeJson = (path: string, value: unknown) => {
+ mkdirSync(dirname(path), { recursive: true })
+ writeFileSync(path, JSON.stringify(value, null, 2) + '\n')
+}
+
+export class Local {
+ readonly dir: string
+ #repo: Repo | undefined
+
+ constructor(readonly root: string) {
+ this.dir = join(root, DIR)
+ }
+
+ /** The repository containing `start`, or null. */
+ static find(start = process.cwd()): Local | null {
+ let dir = resolve(start)
+ for (;;) {
+ if (existsSync(join(dir, DIR))) return new Local(dir)
+ const parent = dirname(dir)
+ if (parent === dir) return null
+ dir = parent
+ }
+ }
+
+ static require(start = process.cwd()): Local {
+ const local = Local.find(start)
+ if (!local) throw new CliError('Not an Underlay repository. Run `underlay init` first.')
+ return local
+ }
+
+ static init(dir: string): Local {
+ const local = new Local(resolve(dir))
+ if (existsSync(local.dir)) throw new CliError(`${local.dir} already exists`)
+ mkdirSync(join(local.dir, 'repo'), { recursive: true })
+ mkdirSync(join(local.dir, 'versions'), { recursive: true })
+ mkdirSync(join(local.dir, 'staging'), { recursive: true })
+ writeFileSync(join(local.dir, 'HEAD'), '')
+ writeFileSync(join(local.dir, 'salt'), newSalt())
+ writeJson(join(local.dir, 'config.json'), { remotes: {} })
+ return local
+ }
+
+ /** The local objects. Trusted: everything in it was verified on the way in. */
+ get repo(): Repo {
+ return (this.#repo ??= openRepo(fileStore(join(this.dir, 'repo')), { trusted: true }))
+ }
+
+ // --- Versions ---
+
+ head(): string | null {
+ const s = readFileSync(join(this.dir, 'HEAD'), 'utf8').trim()
+ return s || null
+ }
+
+ setHead(semver: string): void {
+ writeFileSync(join(this.dir, 'HEAD'), semver)
+ }
+
+ headVersion(): LocalVersion | null {
+ const h = this.head()
+ return h ? this.version(h) : null
+ }
+
+ version(semver: string): LocalVersion | null {
+ return readJson(join(this.dir, 'versions', `${semver}.json`))
+ }
+
+ writeVersion(v: LocalVersion): void {
+ writeJson(join(this.dir, 'versions', `${v.semver}.json`), v)
+ }
+
+ versions(): LocalVersion[] {
+ const dir = join(this.dir, 'versions')
+ return readdirSync(dir)
+ .filter((f) => f.endsWith('.json'))
+ .map((f) => readJson(join(dir, f))!)
+ .sort((a, b) => compareSemver(a.semver, b.semver))
+ }
+
+ salt(): string {
+ return readFileSync(join(this.dir, 'salt'), 'utf8').trim()
+ }
+
+ setSalt(salt: string): void {
+ writeFileSync(join(this.dir, 'salt'), salt)
+ }
+
+ // --- Staging ---
+
+ stagedSchemas(): Record> | null {
+ return readJson(join(this.dir, 'staging', 'schemas.json'))
+ }
+
+ stageSchemas(schemas: Record>): void {
+ writeJson(join(this.dir, 'staging', 'schemas.json'), schemas)
+ }
+
+ /** Staged metadata: `undefined` when none is staged (null is a staged clear). */
+ stagedMetadata(): Record | null | undefined {
+ const v = readJson<{ metadata: Record | null }>(
+ join(this.dir, 'staging', 'metadata.json'),
+ )
+ return v ? v.metadata : undefined
+ }
+
+ stageMetadata(metadata: Record | null): void {
+ writeJson(join(this.dir, 'staging', 'metadata.json'), { metadata })
+ }
+
+ stagedOps(): StagedOp[] {
+ const path = join(this.dir, 'staging', 'ops.ndjson')
+ if (!existsSync(path)) return []
+ return readFileSync(path, 'utf8')
+ .split('\n')
+ .filter((l) => l.length > 0)
+ .map((l) => JSON.parse(l) as StagedOp)
+ }
+
+ stageOps(ops: StagedOp[]): void {
+ if (ops.length === 0) return
+ mkdirSync(join(this.dir, 'staging'), { recursive: true })
+ appendFileSync(
+ join(this.dir, 'staging', 'ops.ndjson'),
+ ops.map((o) => JSON.stringify(o)).join('\n') + '\n',
+ )
+ }
+
+ clearStaging(): void {
+ rmSync(join(this.dir, 'staging'), { recursive: true, force: true })
+ mkdirSync(join(this.dir, 'staging'), { recursive: true })
+ }
+
+ // --- Remotes ---
+
+ remotes(): Record {
+ return readJson<{ remotes: Record }>(join(this.dir, 'config.json'))!
+ .remotes
+ }
+
+ setRemotes(remotes: Record): void {
+ writeJson(join(this.dir, 'config.json'), { remotes })
+ }
+
+ remote(name: string): RemoteConfig {
+ const r = this.remotes()[name]
+ if (!r) throw new CliError(`No remote named "${name}". Add one with \`underlay remote add\`.`)
+ return r
+ }
+
+ tracking(name: string): Tracking | null {
+ return readJson(join(this.dir, 'remotes', `${name}.json`))
+ }
+
+ setTracking(name: string, t: Tracking | null): void {
+ const path = join(this.dir, 'remotes', `${name}.json`)
+ if (t) writeJson(path, t)
+ else rmSync(path, { force: true })
+ }
+}
diff --git a/packages/cli/src/state.ts b/packages/cli/src/state.ts
new file mode 100644
index 0000000..627029e
--- /dev/null
+++ b/packages/cli/src/state.ts
@@ -0,0 +1,57 @@
+/** Reading a local version back: its root, its sets, its schemas and its records. */
+import {
+ compareUtf8,
+ emptySet,
+ type PrivateSetObject,
+ type RecordEntry,
+ recordTree,
+ type Repo,
+ RepoSource,
+ type SetObject,
+ type VersionRoot,
+} from '@underlay/protocol'
+
+import type { Local, LocalVersion } from './local.js'
+
+export interface VersionState {
+ version: LocalVersion
+ root: VersionRoot
+ public: SetObject
+ /** Empty when the version has no private set or this repository doesn't hold it. */
+ private: PrivateSetObject | SetObject
+ /** Every type's schema, from whichever set holds the type. */
+ schemas: Record>
+}
+
+export async function versionState(local: Local, v: LocalVersion): Promise {
+ const repo = local.repo
+ const root = await repo.root(v.hash)
+ const priv = v.sets === 'all' && root.private ? await repo.privateSet(root.private) : emptySet()
+ const schemas: Record