From b9641f705778d3109171c6d78b0fb5efedec4803 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 14:54:35 +0100 Subject: [PATCH 01/31] add mappings to the checkpoint block An optional list of core mappings on block-1, persisted next to the core table. Old readers ignore the flag, old blocks decode with mappings null. Co-Authored-By: Claude Fable 5.1 --- build.js | 22 +++++++++++++++++++++ lib/encoding.js | 3 ++- spec/hyperschema/index.js | 37 +++++++++++++++++++++++++++++++++--- spec/hyperschema/schema.json | 26 +++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/build.js b/build.js index 9eca427..6a4ad37 100644 --- a/build.js +++ b/build.js @@ -228,6 +228,23 @@ bee.register({ ] }) +bee.register({ + name: 'mapping', + compact: true, + fields: [ + { + name: 'from', + type: 'fixed32', + required: true + }, + { + name: 'to', + type: 'fixed32', + required: true + } + ] +}) + bee.register({ name: 'block-0', fields: [ @@ -343,6 +360,11 @@ bee.register({ { name: 'invalidation', // TODO: in future block-2 should be required type: 'uint' + }, + { + name: 'mappings', + type: '@bee/mapping', + array: true } ] }) diff --git a/lib/encoding.js b/lib/encoding.js index 6b90043..3da1e74 100644 --- a/lib/encoding.js +++ b/lib/encoding.js @@ -91,7 +91,8 @@ function decodeBlock0(state, seq) { values: null, cohorts, t: T, - invalidation: 0 + invalidation: 0, + mappings: null } } diff --git a/spec/hyperschema/index.js b/spec/hyperschema/index.js index 3093d62..caf8254 100644 --- a/spec/hyperschema/index.js +++ b/spec/hyperschema/index.js @@ -458,7 +458,8 @@ const encoding14 = { (m.values ? 16 : 0) | (m.cohorts ? 32 : 0) | (m.t ? 64 : 0) | - (m.invalidation ? 128 : 0) + (m.invalidation ? 128 : 0) | + (m.mappings ? 256 : 0) c.uint.preencode(state, m.type) c.uint.preencode(state, m.checkpoint) @@ -473,6 +474,7 @@ const encoding14 = { if (m.cohorts) encoding14_9.preencode(state, m.cohorts) if (m.t) c.uint.preencode(state, m.t) if (m.invalidation) c.uint.preencode(state, m.invalidation) + if (m.mappings) encoding14_12.preencode(state, m.mappings) }, encode(state, m) { const flags = @@ -483,7 +485,8 @@ const encoding14 = { (m.values ? 16 : 0) | (m.cohorts ? 32 : 0) | (m.t ? 64 : 0) | - (m.invalidation ? 128 : 0) + (m.invalidation ? 128 : 0) | + (m.mappings ? 256 : 0) c.uint.encode(state, m.type) c.uint.encode(state, m.checkpoint) @@ -498,6 +501,7 @@ const encoding14 = { if (m.cohorts) encoding14_9.encode(state, m.cohorts) if (m.t) c.uint.encode(state, m.t) if (m.invalidation) c.uint.encode(state, m.invalidation) + if (m.mappings) encoding14_12.encode(state, m.mappings) }, decode(state) { const r0 = c.uint.decode(state) @@ -517,11 +521,36 @@ const encoding14 = { values: (flags & 16) !== 0 ? encoding14_8.decode(state) : null, cohorts: (flags & 32) !== 0 ? encoding14_9.decode(state) : null, t: (flags & 64) !== 0 ? c.uint.decode(state) : 0, - invalidation: (flags & 128) !== 0 ? c.uint.decode(state) : 0 + invalidation: (flags & 128) !== 0 ? c.uint.decode(state) : 0, + mappings: (flags & 256) !== 0 ? encoding14_12.decode(state) : null } } } +// @bee/mapping +const encoding15 = { + preencode(state, m) { + c.fixed32.preencode(state, m.from) + c.fixed32.preencode(state, m.to) + }, + encode(state, m) { + c.fixed32.encode(state, m.from) + c.fixed32.encode(state, m.to) + }, + decode(state) { + const r0 = c.fixed32.decode(state) + const r1 = c.fixed32.decode(state) + + return { + from: r0, + to: r1 + } + } +} + +// @bee/block-1.mappings, deferred due to recusive use +const encoding14_12 = c.array(encoding15) + function setVersion(v) { version = v } @@ -575,6 +604,8 @@ function getEncoding(name) { return encoding13 case '@bee/block-1': return encoding14 + case '@bee/mapping': + return encoding15 default: throw new Error('Encoder not found ' + name) } diff --git a/spec/hyperschema/schema.json b/spec/hyperschema/schema.json index 309401c..bb752dc 100644 --- a/spec/hyperschema/schema.json +++ b/spec/hyperschema/schema.json @@ -409,6 +409,32 @@ "name": "invalidation", "type": "uint", "version": 1 + }, + { + "name": "mappings", + "array": true, + "type": "@bee/mapping", + "version": 1 + } + ] + }, + { + "name": "mapping", + "namespace": "bee", + "compact": true, + "flagsPosition": -1, + "fields": [ + { + "name": "from", + "required": true, + "type": "fixed32", + "version": 1 + }, + { + "name": "to", + "required": true, + "type": "fixed32", + "version": 1 } ] } From ec11aab11179ac5679a5f00fb14d8080063d8cd2 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 14:54:35 +0100 Subject: [PATCH 02/31] add mapCore hook to remap cores on open A mapping redirects every read of a core to another core holding the same blocks. The hook is asked once per referenced core when it is opened, so getCore, getCoreKey, getContextByKey and getContext are async. The core table keeps keys as referenced, mappings are learnt from checkpoints and written back on the next one, and the node cache and diff key on the opened core so both spellings share an entry. move and checkout stay sync, remap runs before the next operation and re-roots the tree if the hook moves it. Co-Authored-By: Claude Fable 5.1 --- index.js | 58 ++++++++++++++++++++++++++++++++++----- lib/cache.js | 6 ++++- lib/changes.js | 2 +- lib/context.js | 72 ++++++++++++++++++++++++++++++------------------- lib/copy.js | 8 +++--- lib/diff.js | 16 ++++++++--- lib/mappings.js | 70 +++++++++++++++++++++++++++++++++++++++++++++++ lib/write.js | 8 +++--- test/basic.js | 4 +-- 9 files changed, 197 insertions(+), 47 deletions(-) create mode 100644 lib/mappings.js diff --git a/index.js b/index.js index 8915cee..b2184ef 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,7 @@ const NodeCache = require('./lib/cache.js') const WriteBatch = require('./lib/write.js') const copyChanges = require('./lib/copy.js') const CoreContext = require('./lib/context.js') +const Mappings = require('./lib/mappings.js') const SessionConfig = require('./lib/session-config.js') const Downloader = require('./lib/download.js') const { inflate, inflateValue } = require('./lib/inflate.js') @@ -28,13 +29,15 @@ class Hyperbee extends EventEmitter { timeout = config.timeout, wait = config.wait, trace = config.trace, + mapCore = null, core = getRootCore(store, key, getEncryptionProvider), context = new CoreContext( store, core, new NodeCache(maxCacheSize), core, - getEncryptionProvider + getEncryptionProvider, + new Mappings(mapCore) ), root = null, view = false, @@ -55,6 +58,7 @@ class Hyperbee extends EventEmitter { this.autoUpdate = autoUpdate this.preload = preload + this._remapping = null this.ready().catch(noop) } @@ -128,16 +132,17 @@ class Hyperbee extends EventEmitter { timeout = this.config.timeout, wait = this.config.wait } = {}) { - const context = key ? this.context.getContextByKey(key) : this.context + const context = key ? this.context.getContextUnmapped(key) : this.context const root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null) return this._makeView(context, root, writable, 0, timeout, wait) } move({ length = this.core.length, key = null, writable = this.writable } = {}) { - this.context = key ? this.context.getContextByKey(key) : this.context + this.context = key ? this.context.getContextUnmapped(key) : this.context this.writable = writable this._setRoot(this._nodeAtSeq(length - 1), true) + if (key !== null) this.remap().catch(noop) } snapshot({ timeout = this.config.timeout, wait = this.config.wait } = {}) { @@ -172,11 +177,47 @@ class Hyperbee extends EventEmitter { return seq < 0 ? EMPTY : this.context.createTreeNode(0, seq, 0, false, null) } + // moves the bee onto the core its own core is mapped to. The mappings the + // table carries are picked up from the checkpoint as it is inflated, and the + // tip is only consulted when it is local so this never waits on a peer + remap() { + if (this._remapping === null) { + this._remapping = this._remap().finally(() => { + this._remapping = null + }) + } + return this._remapping + } + + async _remap() { + const previous = this.context + const mapped = await previous.mappings.map(previous.core.key) + + if (!b4a.equals(mapped, previous.core.key)) { + this.context = previous.getContextUnmapped(mapped) + await this.context.core.ready() + + const root = this.root + if (root !== null && root !== EMPTY && root.context === previous && root.core === 0) { + this.root = this.context.createTreeNode(0, root.seq, 0, false, null) + } + } + + try { + await this.context.update(this.config.options({ wait: false })) + } catch (err) { + if (err.code !== 'BLOCK_NOT_AVAILABLE') throw err + } + } + async ready() { if (!this.core.opened) await this.core.ready() - if (this.root) return + if (this.root) return this.remap() if (this.preload) await this.preload() + if (this.root) return this.remap() + + await this.remap() if (this.root) return this._setRoot(this._lastNodeInCore(), false) @@ -202,10 +243,14 @@ class Hyperbee extends EventEmitter { await this.context.update(config) const cores = [] + const seen = new Set() // index 0 is the core this bee is viewing, the rest are referenced cores for (let i = 0; i <= this.context.cores.length; i++) { - const key = this.context.getCoreKey(i) + const key = await this.context.getCoreKey(i) + const hex = b4a.toString(key, 'hex') + if (seen.has(hex)) continue + seen.add(hex) if (!local && b4a.equals(key, this.context.local.key)) continue cores.push(key) } @@ -297,7 +342,7 @@ class Hyperbee extends EventEmitter { const value = key.value || (await inflateValue(key, config)) return { - core: key.context.getCore(key.core), + core: await key.context.getCore(key.core), offset: key.offset, seq: key.seq, key: key.key, @@ -307,6 +352,7 @@ class Hyperbee extends EventEmitter { async bootstrap(config) { if (!this.root) await this.ready() + if (this._remapping !== null) await this._remapping if (this.unbatch) await this._rollback(config) return this.root === EMPTY ? null : this.root } diff --git a/lib/cache.js b/lib/cache.js index 8c30044..00e9b9b 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -96,7 +96,11 @@ module.exports = class NodeCache { } } +// nodes are cached under the core they were read from, so a block reached +// through a mapped core and through the core it maps to share an entry. A +// pointer whose core is not open yet can only miss function cacheId(node) { - const id = node.context.getCore(node.core).id + const core = node.context.openedCore(node.core) + const id = core !== null ? core.id : node.context.core.id + ':' + node.core return id + '@' + node.seq + '.' + node.offset } diff --git a/lib/changes.js b/lib/changes.js index 109617a..590b97f 100644 --- a/lib/changes.js +++ b/lib/changes.js @@ -21,7 +21,7 @@ class ChangesStream extends Readable { async _openp() { await this.tree.bootstrap(this.config) if (this.head === null) this.head = this.tree.head() - if (this.head !== null) this.context = this.tree.context.getContextByKey(this.head.key) + if (this.head !== null) this.context = await this.tree.context.getContextByKey(this.head.key) } async _readp() { diff --git a/lib/context.js b/lib/context.js index 483513c..cd2504e 100644 --- a/lib/context.js +++ b/lib/context.js @@ -4,6 +4,7 @@ const c = require('compact-encoding') const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors') const { TreeNodePointer } = require('./tree.js') const { decodeBlock, TYPE_LATEST } = require('./encoding.js') +const Mappings = require('./mappings.js') const CHECKPOINT_KEY = 'hyperbee/checkpoint' @@ -14,6 +15,7 @@ class CoreContext { cache, core, getEncryptionProvider, + mappings = new Mappings(), lock = new ScopeLock(), other = new Map() ) { @@ -24,6 +26,7 @@ class CoreContext { this.getEncryptionProvider = getEncryptionProvider this.lock = lock this.other = other + this.mappings = mappings this.length = 0 this.checkpoint = 0 this.invalidation = 0 @@ -100,6 +103,7 @@ class CoreContext { const block = decodeBlock(buffer, seq) cores = block.metadata ? block.metadata.cores : [] + if (block.mappings !== null) this.mappings.add(block.mappings) } if (length <= this.length) return @@ -134,7 +138,7 @@ class CoreContext { } async getBlockType(core, config) { - const hc = this.getCore(core) + const hc = await this.getCore(core) if (hc.length === 0) return TYPE_LATEST const block = await this.getBlock(hc.length - 1, core, config, null) return block.type @@ -192,27 +196,37 @@ class CoreContext { return this.cores.length } - getCore(index) { + // the table keeps keys as they were referenced, a core is mapped when opened + async getCore(index) { if (index === 0) return this.core if (index > this.cores.length) throw new Error('Bad core index: ' + index) - if (this.opened[index - 1] === null) { - const key = this.cores[index - 1].key - this.opened[index - 1] = this.store.get({ - key, - encryption: this.getEncryptionProvider(key), - inflightRange: [256, 512] - }) - } + + const key = await this.mappings.map(this.cores[index - 1].key) + const opened = this.opened[index - 1] + if (opened !== null && b4a.equals(opened.key, key)) return opened + + this.opened[index - 1] = this._openCore(key) return this.opened[index - 1] } getCoreKey(index) { - if (index === 0) return this.core.key + if (index === 0) return Promise.resolve(this.core.key) if (index > this.cores.length) throw new Error('Bad core index: ' + index) - if (this.opened[index - 1] !== null && this.opened[index - 1].key) { - return this.opened[index - 1].key - } - return this.cores[index - 1].key + return this.mappings.map(this.cores[index - 1].key) + } + + // the core a pointer reads from once opened, null before that + openedCore(index) { + if (index === 0) return this.core + return index > this.opened.length ? null : this.opened[index - 1] + } + + _openCore(key) { + return this.store.get({ + key, + encryption: this.getEncryptionProvider(key), + inflightRange: [256, 512] + }) } async getBlock(seq, core, config, origin) { @@ -232,7 +246,7 @@ class CoreContext { await this.updateMaybe(config, core, origin) } - const hc = this.getCore(core) + const hc = await this.getCore(core) const buffer = await hc.get(seq, config) if (buffer === null) throw BLOCK_NOT_AVAILABLE() @@ -242,22 +256,22 @@ class CoreContext { } getLocalContext() { - return this.getContextByKey(this.local.key) + return this.getContextUnmapped(this.local.key) + } + + async getContextByKey(key) { + return this.getContextUnmapped(await this.mappings.map(key)) } - getContextByKey(key) { + // the context of a key as given, only for keys known not to be mapped or + // for a tree that remaps itself before use + getContextUnmapped(key) { if (b4a.equals(key, this.core.key)) return this const hex = b4a.toString(key, 'hex') if (this.other.has(hex)) return this.other.get(hex) - const ctx = this._createContext( - this.store.get({ - key, - encryption: this.getEncryptionProvider(key), - inflightRange: [256, 512] - }) - ) + const ctx = this._createContext(this._openCore(key)) this.other.set(hex, ctx) return ctx } @@ -273,10 +287,11 @@ class CoreContext { if (core === 0) return this if (core > this.cores.length) throw new Error('Bad core index: ' + core) - const hex = b4a.toString(this.cores[core - 1].key, 'hex') + const hc = await this.getCore(core) + const hex = b4a.toString(hc.key, 'hex') if (this.other.has(hex)) return this.other.get(hex) - const ctx = this._createContext(this.getCore(core)) + const ctx = this._createContext(hc) this.other.set(hex, ctx) return ctx } @@ -293,9 +308,10 @@ class CoreContext { const local = this.local const cache = this.cache const getEncryptionProvider = this.getEncryptionProvider + const mappings = this.mappings const lock = this.lock const other = this.other - return new CoreContext(store, local, cache, core, getEncryptionProvider, lock, other) + return new CoreContext(store, local, cache, core, getEncryptionProvider, mappings, lock, other) } } diff --git a/lib/copy.js b/lib/copy.js index 395af72..1cede58 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -15,7 +15,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { for (let i = changes.length - 1; i >= 0; i--) { const change = changes[i] - const src = tree.context.getContextByKey(change.head.key) + const src = await tree.context.getContextByKey(change.head.key) const srcHex = b4a.toString(src.core.key, 'hex') const batch = change.batch const start = base + blocks.length @@ -48,6 +48,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { } blk.metadata = null + blk.mappings = null blk.previous = null blocks.push(blk) @@ -62,9 +63,10 @@ module.exports = async function copyChanges(tree, changes, config, map) { } } - if (local.changed) { + if (local.changed || local.mappings.dirty) { local.checkpoint = base + blocks.length blocks[blocks.length - 1].metadata = local.flush() + blocks[blocks.length - 1].mappings = local.mappings.flush() } const buffers = new Array(blocks.length) @@ -83,7 +85,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { if (p.core !== 0 && !src.hasCore(p.core)) await src.updateMaybe(config, p.core, origin) - const key = src.getCoreKey(p.core) + const key = await src.getCoreKey(p.core) const seq = copied.get(b4a.toString(key, 'hex') + ':' + p.seq) if (seq !== undefined) { diff --git a/lib/diff.js b/lib/diff.js index 4184550..fd8f467 100644 --- a/lib/diff.js +++ b/lib/diff.js @@ -178,8 +178,18 @@ function inflateTop(it, top) { function isSame(a, b) { if (a.seq !== b.seq || a.offset !== b.offset) return false - const k1 = a.context.getCoreKey(a.core) - const k2 = b.context.getCoreKey(b.core) + const c1 = a.context.openedCore(a.core) + const c2 = b.context.openedCore(b.core) + if (c1 !== null && c2 !== null) return c1 === c2 || b4a.equals(c1.key, c2.key) - return k1 === k2 || b4a.equals(k1, k2) ? true : false + const k1 = tableKey(a) + const k2 = tableKey(b) + return k1 !== null && k2 !== null && b4a.equals(k1, k2) +} + +// the key as referenced, before the core was opened and mapped +function tableKey(p) { + if (p.core === 0) return p.context.core.key + const entry = p.context.cores[p.core - 1] + return entry === undefined ? null : entry.key } diff --git a/lib/mappings.js b/lib/mappings.js new file mode 100644 index 0000000..29e3227 --- /dev/null +++ b/lib/mappings.js @@ -0,0 +1,70 @@ +const b4a = require('b4a') + +// a mapping redirects every read of a core to another core holding the same +// blocks. Mappings are immutable facts, shared by every context of a tree and +// persisted next to the core table on the next checkpoint +class Mappings { + constructor(hook = null) { + this.hook = hook + this.known = new Map() + this.pending = new Map() + this.dirty = false + } + + map(key) { + const hex = b4a.toString(key, 'hex') + + const known = this.known.get(hex) + if (known !== undefined) return Promise.resolve(known) + if (this.hook === null) return Promise.resolve(key) + + const pending = this.pending.get(hex) + if (pending !== undefined) return pending + + const promise = this._ask(key, hex) + this.pending.set(hex, promise) + return promise + } + + async _ask(key, hex) { + let to = null + try { + to = await this.hook(key) + } finally { + this.pending.delete(hex) + } + + if (to === null || to === undefined) to = key + + // a record met while the hook was running wins over no mapping + const known = this.known.get(hex) + if (known !== undefined && b4a.equals(to, key)) return known + + if (!b4a.equals(to, key)) this.dirty = true + this.known.set(hex, to) + return to + } + + add(mappings) { + for (const { from, to } of mappings) { + const hex = b4a.toString(from, 'hex') + const known = this.known.get(hex) + if (known !== undefined && !b4a.equals(known, from)) continue + this.known.set(hex, to) + } + } + + flush() { + this.dirty = false + + const mappings = [] + for (const [hex, to] of this.known) { + const from = b4a.from(hex, 'hex') + if (!b4a.equals(from, to)) mappings.push({ from, to }) + } + + return mappings.length === 0 ? null : mappings + } +} + +module.exports = Mappings diff --git a/lib/write.js b/lib/write.js index bfd2832..86d9855 100644 --- a/lib/write.js +++ b/lib/write.js @@ -94,7 +94,7 @@ module.exports = class WriteBatch { const length = this._getLength(root) - context = this._getContext(root) + context = await this._getContext(root) const changed = length === 0 const seq = length === 0 ? 0 : length - 1 @@ -463,7 +463,8 @@ module.exports = class WriteBatch { values: null, cohorts: null, t, - invalidation: context.invalidation + invalidation: context.invalidation, + mappings: null } for (const k of update.values) { @@ -525,9 +526,10 @@ module.exports = class WriteBatch { } // TODO: make this transaction safe - if (context.changed) { + if (context.changed || context.mappings.dirty) { context.checkpoint = context.core.length + blocks.length blocks[blocks.length - 1].metadata = context.flush() + blocks[blocks.length - 1].mappings = context.mappings.flush() } for (let i = 0; i < blocks.length; i++) { diff --git a/test/basic.js b/test/basic.js index c243efa..bc915e7 100644 --- a/test/basic.js +++ b/test/basic.js @@ -1335,7 +1335,7 @@ test('inflight range - secondary core opened via context.getCore()', async funct const value = await db2.get(b4a.from('a')) t.alike(value.value, b4a.from('1')) - const hc = db2.context.getCore(1) + const hc = await db2.context.getCore(1) t.alike(hc.replicator.inflightRange, INFLIGHT_RANGE) }) @@ -1356,7 +1356,7 @@ test('inflight range - secondary core opened via context.getContextByKey()', asy w.tryPut(b4a.from('b'), b4a.from('2')) await w.flush() - const ctx = db2.context.getContextByKey(db1.core.key) + const ctx = await db2.context.getContextByKey(db1.core.key) t.alike(ctx.core.replicator.inflightRange, INFLIGHT_RANGE) }) From 20775e0bb52423c20952b318ce39a0150f2e1eb8 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 14:54:35 +0100 Subject: [PATCH 03/31] test mapCore Co-Authored-By: Claude Fable 5.1 --- test/all.js | 1 + test/map-core.js | 448 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 449 insertions(+) create mode 100644 test/map-core.js diff --git a/test/all.js b/test/all.js index 57f28cd..06f26fd 100644 --- a/test/all.js +++ b/test/all.js @@ -17,6 +17,7 @@ async function runTests() { await import('./fuzz-diff.js') await import('./fuzz-multiwriter.js') await import('./perf.js') + await import('./map-core.js') await import('./prefetch.js') await import('./tree.js') await import('./undo.js') diff --git a/test/map-core.js b/test/map-core.js new file mode 100644 index 0000000..6e3db3b --- /dev/null +++ b/test/map-core.js @@ -0,0 +1,448 @@ +const test = require('brittle') +const b4a = require('b4a') +const Corestore = require('corestore') +const Hypercore = require('hypercore') +const Bee = require('../') + +const hex = (k) => b4a.toString(k, 'hex') +const key = (i) => b4a.from('key/' + String(i).padStart(4, '0')) +const value = (i) => b4a.from('value-' + i) + +async function source(store, n) { + const a = new Bee(store.namespace('a')) + await a.ready() + const w = a.write() + for (let i = 0; i < n; i++) w.tryPut(key(i), value(i)) + await w.flush() + return a +} + +// a core holding the same blocks, the target of a mapping +async function copyCore(store, src, name) { + const dst = store.get({ name }) + await dst.ready() + for (let i = 0; i < src.length; i++) await dst.append(await src.get(i)) + return dst +} + +async function readAll(bee, n, t, msg) { + for (let i = 0; i < n; i++) { + const node = await bee.get(key(i)) + if (!node || !b4a.equals(node.value, value(i))) return t.fail(msg + ': key ' + i) + } + t.pass(msg) +} + +test('mapCore - the viewed core is read through its mapping', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + + const asked = [] + const mapCore = (k) => { + asked.push(hex(k)) + return b4a.equals(k, a.core.key) ? copy.key : null + } + + const b = new Bee(store.namespace('b'), { key: a.core.key, mapCore, writable: false }) + await b.ready() + + t.alike(b.core.key, copy.key, 'the view sits on the mapped core') + t.ok(asked.includes(hex(a.core.key)), 'the hook was asked about the original') + await readAll(b, 50, t, 'every entry reads through the mapping') + t.alike((await b.cores()).map(hex), [hex(copy.key)], 'cores() lists the mapped core only') + + await b.close() +}) + +test('mapCore - the references of a writer resolve to the mapped core', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + + const w = new Bee(store.namespace('w'), { mapCore }) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + await w.remap() + t.alike(w.core.key, copy.key, 'moving onto the original lands on the mapping') + + const batch = w.write() + for (let i = 50; i < 60; i++) batch.tryPut(key(i), value(i)) + await batch.flush() + + // the local core references the copy, never the original + const referenced = (await w.cores({ local: false })).map(hex) + t.absent(referenced.includes(hex(a.core.key)), 'the original key is not in the table') + + await w.close() + + const again = new Bee(store.namespace('w')) + await again.ready() + t.alike( + (await again.cores({ local: false })).map(hex), + referenced, + 'the persisted table holds the mapped keys' + ) + await readAll(again, 60, t, 'a reader without the hook reads everything via the persisted table') + await again.close() +}) + +test('mapCore - an existing table is remapped on load and persisted on the next flush', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + + // a writer that referenced the original before any mapping existed + const w = new Bee(store.namespace('w')) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + { + const batch = w.write() + for (let i = 50; i < 60; i++) batch.tryPut(key(i), value(i)) + await batch.flush() + } + t.ok( + (await w.cores({ local: false })).some((k) => b4a.equals(k, a.core.key)), + 'table names the original' + ) + await w.close() + + const copy = await copyCore(store, a.core, 'copy') + const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + + const mapped = new Bee(store.namespace('w'), { mapCore }) + await mapped.ready() + const cores = (await mapped.cores({ local: false })).map(hex) + t.ok(cores.includes(hex(copy.key)), 'the loaded table is remapped') + t.absent(cores.includes(hex(a.core.key)), 'the original is gone from the loaded table') + await readAll(mapped, 60, t, 'reads go through the remapped table') + + { + const batch = mapped.write() + batch.tryPut(key(60), value(60)) + await batch.flush() + } + await mapped.close() + + const plain = new Bee(store.namespace('w')) + await plain.ready() + const persisted = (await plain.cores({ local: false })).map(hex) + t.ok(persisted.includes(hex(copy.key)), 'the remapped table was persisted by the flush') + t.absent(persisted.includes(hex(a.core.key)), 'without the hook, the original is still gone') + await readAll(plain, 61, t, 'a reader without the hook reads through the persisted mapping') + await plain.close() +}) + +test('mapCore - the persisted mapping survives an undo below it', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + + const w = new Bee(store.namespace('w')) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + for (let i = 50; i < 55; i++) { + const batch = w.write() + batch.tryPut(key(i), value(i)) + await batch.flush() + } + await w.close() + + const copy = await copyCore(store, a.core, 'copy') + const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + + const mapped = new Bee(store.namespace('w'), { mapCore }) + await mapped.ready() + { + const batch = mapped.write() + batch.tryPut(key(55), value(55)) + await batch.flush() + } + await mapped.close() + + // a reader without the hook, viewing a root written before the mapping was + // recorded, still resolves the table from its latest checkpoint + const plain = new Bee(store.namespace('w')) + await plain.ready() + const old = plain.undo(3) + await readAll(old, 53, t, 'an undone root reads through the persisted mapping') + t.is(await old.get(key(53)), null, 'the undone writes are gone') + t.alike( + (await old.cores({ local: false })).map(hex), + [hex(copy.key)], + 'the undone view resolves the original to the copy' + ) + await old.close() + await plain.close() +}) + +test('mapCore - a mapping met in another writer is recorded locally', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + + // b writes on top of a with the hook, recording the mapping in its checkpoint + const b = new Bee(store.namespace('b'), { mapCore }) + await b.ready() + b.move({ key: a.core.key, length: a.core.length }) + { + const batch = b.write() + batch.tryPut(key(50), value(50)) + await batch.flush() + } + + // c never has the hook and only ever sees b's head + const c = new Bee(store.namespace('c')) + await c.ready() + c.move({ key: b.context.local.key, length: b.context.local.length }) + { + const batch = c.write() + batch.tryPut(key(51), value(51)) + await batch.flush() + } + await readAll(c, 52, t, 'c reads a through the mapping b recorded') + await c.close() + await b.close() + + const again = new Bee(store.namespace('c')) + await again.ready() + await readAll(again, 52, t, "a reopen of c reads a through c's own checkpoint") + t.absent( + (await again.cores()).some((k) => b4a.equals(k, a.core.key)), + 'the original is not opened anywhere' + ) + await again.close() +}) + +test('mapCore - a remote reads the whole tree after the original data is gone', async function (t) { + const origin = new Corestore(await t.tmp()) + t.teardown(() => origin.close()) + + const a = await source(origin, 200) + + const w = new Bee(origin.namespace('w')) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + for (let i = 200; i < 220; i++) { + const batch = w.write() + batch.tryPut(key(i), value(i)) + await batch.flush() + } + const wKey = w.context.local.key + await w.close() + + const copy = await copyCore(origin, a.core, 'copy') + const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + + const mapped = new Bee(origin.namespace('w'), { mapCore }) + await mapped.ready() + { + const batch = mapped.write() + batch.tryPut(key(220), value(220)) + await batch.flush() + } + await mapped.close() + + await a.core.clear(0, a.core.length) + t.is(await a.core.has(0), false, 'the original holds no data any more') + t.is(copy.contiguousLength, copy.length, 'the copy still holds every block') + + // a remote that only ever sees the origin + const remote = new Corestore(await t.tmp()) + t.teardown(() => remote.close()) + const stop = replicateStores(origin, remote) + + const reader = new Bee(remote.session(), { key: wKey, mapCore, writable: false }) + await reader.ready() + await reader.core.update({ wait: true }) + t.is(reader.core.length, 21, 'the remote learnt the head') + + let count = 0 + for await (const { key: k, value: v } of reader.createReadStream()) { + const expected = b4a.equals(k, key(count)) && b4a.equals(v, value(count)) + if (!expected) return t.fail('bad entry ' + count) + count++ + } + t.is(count, 221, 'the whole tree was read through the mapping') + await readAll(reader, 221, t, 'every point read resolves too') + t.alike( + (await reader.cores()).map(hex).sort(), + [hex(wKey), hex(copy.key)].sort(), + 'only the writer and the copy were opened' + ) + t.absent( + await remote.storage.hasCore(Hypercore.discoveryKey(a.core.key)), + 'the remote never asked for the original' + ) + await reader.close() + + await stop() + + // a second remote without the hook: a head written before the mapping was + // recorded runs into the cleared original, the head that carries it does not + const other = new Corestore(await t.tmp()) + t.teardown(() => other.close()) + const stopOther = replicateStores(origin, other) + + { + const core = other.get({ key: wKey }) + await core.update({ wait: true }) + await core.download({ start: 0, end: core.length }).done() + await core.close() + } + + const before = new Bee(other.session(), { key: wKey, writable: false }) + await before.ready() + before.move({ length: 20 }) + await t.exception( + before.get(key(0), { wait: false }), + /BLOCK_NOT_AVAILABLE/, + 'before the record, a reader without the hook hits missing data' + ) + await before.close() + + const after = new Bee(other.session(), { key: wKey, writable: false }) + await after.ready() + await readAll(after, 221, t, 'at the recorded head, the same reader resolves everything') + await after.close() + + await stopOther() +}) + +test('mapCore - move and checkout onto an original resolve before the first operation', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + await a.core.clear(0, a.core.length) + + let asked = 0 + const mapCore = (k) => { + asked++ + return b4a.equals(k, a.core.key) ? copy.key : null + } + + const w = new Bee(store.namespace('w'), { mapCore }) + await w.ready() + + w.move({ key: a.core.key, length: a.core.length }) + await readAll(w, 50, t, 'reads after an un-awaited move go through the mapping') + t.alike(w.head().key, copy.key, 'the head is on the mapped core') + t.alike(w.core.key, copy.key, 'the bee sits on the mapped core') + + { + const batch = w.write() + batch.tryPut(key(50), value(50)) + await batch.flush() + } + t.absent( + (await w.cores()).some((k) => b4a.equals(k, a.core.key)), + 'the write referenced the mapped core' + ) + + const view = w.checkout({ key: a.core.key, length: a.core.length }) + await readAll(view, 50, t, 'a checkout onto the original reads through the mapping') + t.alike(view.head().key, copy.key, 'the view head is on the mapped core') + await view.close() + + const asked1 = asked + const again = w.checkout({ key: a.core.key, length: a.core.length }) + await again.ready() + t.alike(again.head().key, copy.key, 'a second checkout lands on the mapped core') + t.is(asked, asked1, 'without asking the hook again') + await again.close() + + await w.close() +}) + +test('mapCore - a read in flight survives its core being remapped', async function (t) { + const origin = new Corestore(await t.tmp()) + t.teardown(() => origin.close()) + + const a = await source(origin, 50) + const copy = await copyCore(origin, a.core, 'copy') + + const w = new Bee(origin.namespace('w')) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + { + const batch = w.write() + batch.tryPut(key(50), value(50)) + await batch.flush() + } + const wKey = w.context.local.key + await w.close() + + // the remote holds the writer but nothing of the core it references + const remote = new Corestore(await t.tmp()) + t.teardown(() => remote.close()) + { + const stop = replicateStores(origin, remote) + const core = remote.get({ key: wKey }) + await core.update({ wait: true }) + await core.download({ start: 0, end: core.length }).done() + await core.close() + await stop() + } + + const reader = new Bee(remote.session(), { key: wKey, writable: false }) + await reader.ready() + + // a read that has to wait for a peer to serve the referenced core + const read = reader.get(key(0)) + for (let i = 0; i < 100 && reader.context.opened[0] === null; i++) await immediate() + const stale = reader.context.opened[0] + t.alike(stale.key, a.core.key, 'the read opened the core as referenced') + + reader.context.mappings.add([{ from: a.core.key, to: copy.key }]) + const hc = await reader.context.getCore(1) + t.alike(hc.key, copy.key, 'the next open resolves to the mapped core') + t.absent(stale.closing, 'the superseded session is left to the reads holding it') + + const stop = replicateStores(origin, remote) + const node = await read + t.alike(node.value, value(0), 'the read issued before the remap still resolved') + + await readAll(reader, 51, t, 'later reads go through the mapping') + t.alike( + (await reader.cores()).map(hex).sort(), + [hex(wKey), hex(copy.key)].sort(), + 'only the writer and the mapped core are listed' + ) + await reader.close() + await stop() +}) + +function immediate() { + return new Promise((resolve) => setImmediate(resolve)) +} + +function replicateStores(a, b) { + const s1 = a.replicate(true) + const s2 = b.replicate(false) + s1.pipe(s2).pipe(s1) + s1.on('error', noop) + s2.on('error', noop) + return async () => { + const closed = Promise.all([ + new Promise((resolve) => s1.once('close', resolve)), + new Promise((resolve) => s2.once('close', resolve)) + ]) + s1.destroy() + s2.destroy() + await closed + } +} + +function noop() {} From 40b5a73f6bef20c8c174a971e2fc09432497de74 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 14:54:35 +0100 Subject: [PATCH 04/31] document mapCore and remap Co-Authored-By: Claude Fable 5.1 --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index c570bed..646a6da 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,15 @@ Options include: autoUpdate: false, // Reload root node when underlying Hypercore is appended to? preload: null, // A function called by ready() after the Hypercore is ready. Can be async. wait: true, // Wait for Hypercore to download blocks + mapCore: null, // async (key) => mappedKey | null. Redirect reads of a core to another core holding the same blocks } ``` +`mapCore` is asked once per core the tree references (and the core the tree +is opened on). A non-null answer redirects every read of that core to the +mapped core. Mappings are persisted next to the core table on the next write, +so readers that open the tree later resolve them without the hook. + If you pass your own `core`, consider increasing the inflight range on it (`256` minimum recommended, e.g. `{ inflightRange: [256, 512] }` when creating the core via Corestore) to match `hyperbee2` constructed cores. #### `await db.ready()` @@ -159,6 +165,15 @@ Options: } ``` +If `key` is mapped by `mapCore` the tree is moved onto the mapped core before +the next operation, see `db.remap()`. + +#### `await db.remap()` + +Resolves the mapping of the core the tree is open on and moves the tree onto +it. Called by `ready()`, `move()` and `checkout()`, and awaited by every +operation, so it only needs calling to observe `db.core` right after a move. + #### `db.snapshot([options])` Returns a new Hyperbee that is a read only view of the current tree. From 5775fd82469577a32bf62b5bd4ba46165c83ff24 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 16:15:27 +0100 Subject: [PATCH 05/31] restart remap for a move made while remapping remap deduped on the in-flight promise, so a second move before the first resolved was overwritten by the first mapping and never resolved itself. The remap now loops until the context it handled is still the current one. Co-Authored-By: Claude Fable 5.1 --- index.js | 37 ++++++++++++++++++++++++------------- test/map-core.js | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/index.js b/index.js index b2184ef..351ae8b 100644 --- a/index.js +++ b/index.js @@ -189,24 +189,35 @@ class Hyperbee extends EventEmitter { return this._remapping } + // a move during the remap starts it over for the new context async _remap() { - const previous = this.context - const mapped = await previous.mappings.map(previous.core.key) + let handled = null - if (!b4a.equals(mapped, previous.core.key)) { - this.context = previous.getContextUnmapped(mapped) - await this.context.core.ready() + while (this.context !== handled) { + const context = this.context + const mapped = await context.mappings.map(context.core.key) + if (this.context !== context) continue - const root = this.root - if (root !== null && root !== EMPTY && root.context === previous && root.core === 0) { - this.root = this.context.createTreeNode(0, root.seq, 0, false, null) + if (!b4a.equals(mapped, context.core.key)) { + const next = context.getContextUnmapped(mapped) + await next.core.ready() + if (this.context !== context) continue + + this.context = next + + const root = this.root + if (root !== null && root !== EMPTY && root.context === context && root.core === 0) { + this.root = next.createTreeNode(0, root.seq, 0, false, null) + } } - } - try { - await this.context.update(this.config.options({ wait: false })) - } catch (err) { - if (err.code !== 'BLOCK_NOT_AVAILABLE') throw err + handled = this.context + + try { + await handled.update(this.config.options({ wait: false })) + } catch (err) { + if (err.code !== 'BLOCK_NOT_AVAILABLE') throw err + } } } diff --git a/test/map-core.js b/test/map-core.js index 6e3db3b..794170b 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -428,6 +428,50 @@ function immediate() { return new Promise((resolve) => setImmediate(resolve)) } +test('mapCore - a second move while the first is still being remapped wins', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copyA = await copyCore(store, a.core, 'copy-a') + + const b = new Bee(store.namespace('b')) + await b.ready() + { + const batch = b.write() + for (let i = 0; i < 10; i++) batch.tryPut(key(100 + i), value(100 + i)) + await batch.flush() + } + const copyB = await copyCore(store, b.core, 'copy-b') + + let releaseA = null + const mapCore = (k) => { + if (b4a.equals(k, a.core.key)) { + return new Promise((resolve) => { + releaseA = () => resolve(copyA.key) + }) + } + if (b4a.equals(k, b.core.key)) return copyB.key + return null + } + + const w = new Bee(store.namespace('w'), { mapCore }) + await w.ready() + + w.move({ key: a.core.key, length: a.core.length }) + w.move({ key: b.core.key, length: b.core.length }) + for (let i = 0; i < 10 && releaseA === null; i++) await immediate() + releaseA() + + await w.remap() + t.alike(w.core.key, copyB.key, 'the bee sits on the mapping of the last move') + t.alike(w.head(), { length: b.core.length, key: copyB.key }, 'with the head of the last move') + t.alike((await w.get(key(100))).value, value(100), 'and reads its data') + t.is(await w.get(key(0)), null, 'not the data of the first move') + + await w.close() +}) + function replicateStores(a, b) { const s1 = a.replicate(true) const s2 = b.replicate(false) From 55be721db9a25a57f351f3db3348b45090c08226 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 17:19:46 +0100 Subject: [PATCH 06/31] resolve mappings synchronously Every core of a table is asked for when the checkpoint is inflated and the tree remaps the core it is opened on, so the mapping of any key met afterwards is known and getCore, getCoreKey, getContextByKey and getContext are sync again. A key handed in by the caller of write or createChangesStream is asked for before it is used. Co-Authored-By: Claude Fable 5.1 --- index.js | 10 +++++----- lib/changes.js | 5 ++++- lib/context.js | 26 +++++++++++--------------- lib/copy.js | 4 ++-- lib/diff.js | 16 +++------------- lib/mappings.js | 13 +++++++++++++ lib/write.js | 3 ++- test/basic.js | 4 ++-- test/map-core.js | 27 ++++++++++++++++++++++++++- 9 files changed, 68 insertions(+), 40 deletions(-) diff --git a/index.js b/index.js index 351ae8b..085ceb2 100644 --- a/index.js +++ b/index.js @@ -132,13 +132,13 @@ class Hyperbee extends EventEmitter { timeout = this.config.timeout, wait = this.config.wait } = {}) { - const context = key ? this.context.getContextUnmapped(key) : this.context + const context = key ? this.context.getContextByKey(key) : this.context const root = length === 0 ? EMPTY : context.createTreeNode(0, length - 1, 0, false, null) return this._makeView(context, root, writable, 0, timeout, wait) } move({ length = this.core.length, key = null, writable = this.writable } = {}) { - this.context = key ? this.context.getContextUnmapped(key) : this.context + this.context = key ? this.context.getContextByKey(key) : this.context this.writable = writable this._setRoot(this._nodeAtSeq(length - 1), true) @@ -199,7 +199,7 @@ class Hyperbee extends EventEmitter { if (this.context !== context) continue if (!b4a.equals(mapped, context.core.key)) { - const next = context.getContextUnmapped(mapped) + const next = context.getContextByKey(mapped) await next.core.ready() if (this.context !== context) continue @@ -258,7 +258,7 @@ class Hyperbee extends EventEmitter { // index 0 is the core this bee is viewing, the rest are referenced cores for (let i = 0; i <= this.context.cores.length; i++) { - const key = await this.context.getCoreKey(i) + const key = this.context.getCoreKey(i) const hex = b4a.toString(key, 'hex') if (seen.has(hex)) continue seen.add(hex) @@ -353,7 +353,7 @@ class Hyperbee extends EventEmitter { const value = key.value || (await inflateValue(key, config)) return { - core: await key.context.getCore(key.core), + core: key.context.getCore(key.core), offset: key.offset, seq: key.seq, key: key.key, diff --git a/lib/changes.js b/lib/changes.js index 590b97f..2f649f8 100644 --- a/lib/changes.js +++ b/lib/changes.js @@ -21,7 +21,10 @@ class ChangesStream extends Readable { async _openp() { await this.tree.bootstrap(this.config) if (this.head === null) this.head = this.tree.head() - if (this.head !== null) this.context = await this.tree.context.getContextByKey(this.head.key) + if (this.head !== null) { + await this.tree.context.mappings.map(this.head.key) + this.context = this.tree.context.getContextByKey(this.head.key) + } } async _readp() { diff --git a/lib/context.js b/lib/context.js index cd2504e..c3da9a9 100644 --- a/lib/context.js +++ b/lib/context.js @@ -104,6 +104,7 @@ class CoreContext { cores = block.metadata ? block.metadata.cores : [] if (block.mappings !== null) this.mappings.add(block.mappings) + await this.mappings.discover(cores) } if (length <= this.length) return @@ -138,7 +139,7 @@ class CoreContext { } async getBlockType(core, config) { - const hc = await this.getCore(core) + const hc = this.getCore(core) if (hc.length === 0) return TYPE_LATEST const block = await this.getBlock(hc.length - 1, core, config, null) return block.type @@ -197,11 +198,11 @@ class CoreContext { } // the table keeps keys as they were referenced, a core is mapped when opened - async getCore(index) { + getCore(index) { if (index === 0) return this.core if (index > this.cores.length) throw new Error('Bad core index: ' + index) - const key = await this.mappings.map(this.cores[index - 1].key) + const key = this.mappings.resolve(this.cores[index - 1].key) const opened = this.opened[index - 1] if (opened !== null && b4a.equals(opened.key, key)) return opened @@ -210,9 +211,9 @@ class CoreContext { } getCoreKey(index) { - if (index === 0) return Promise.resolve(this.core.key) + if (index === 0) return this.core.key if (index > this.cores.length) throw new Error('Bad core index: ' + index) - return this.mappings.map(this.cores[index - 1].key) + return this.mappings.resolve(this.cores[index - 1].key) } // the core a pointer reads from once opened, null before that @@ -246,7 +247,7 @@ class CoreContext { await this.updateMaybe(config, core, origin) } - const hc = await this.getCore(core) + const hc = this.getCore(core) const buffer = await hc.get(seq, config) if (buffer === null) throw BLOCK_NOT_AVAILABLE() @@ -256,16 +257,11 @@ class CoreContext { } getLocalContext() { - return this.getContextUnmapped(this.local.key) + return this.getContextByKey(this.local.key) } - async getContextByKey(key) { - return this.getContextUnmapped(await this.mappings.map(key)) - } - - // the context of a key as given, only for keys known not to be mapped or - // for a tree that remaps itself before use - getContextUnmapped(key) { + getContextByKey(key) { + key = this.mappings.resolve(key) if (b4a.equals(key, this.core.key)) return this const hex = b4a.toString(key, 'hex') @@ -287,7 +283,7 @@ class CoreContext { if (core === 0) return this if (core > this.cores.length) throw new Error('Bad core index: ' + core) - const hc = await this.getCore(core) + const hc = this.getCore(core) const hex = b4a.toString(hc.key, 'hex') if (this.other.has(hex)) return this.other.get(hex) diff --git a/lib/copy.js b/lib/copy.js index 1cede58..cf84eca 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -15,7 +15,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { for (let i = changes.length - 1; i >= 0; i--) { const change = changes[i] - const src = await tree.context.getContextByKey(change.head.key) + const src = tree.context.getContextByKey(change.head.key) const srcHex = b4a.toString(src.core.key, 'hex') const batch = change.batch const start = base + blocks.length @@ -85,7 +85,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { if (p.core !== 0 && !src.hasCore(p.core)) await src.updateMaybe(config, p.core, origin) - const key = await src.getCoreKey(p.core) + const key = src.getCoreKey(p.core) const seq = copied.get(b4a.toString(key, 'hex') + ':' + p.seq) if (seq !== undefined) { diff --git a/lib/diff.js b/lib/diff.js index fd8f467..4184550 100644 --- a/lib/diff.js +++ b/lib/diff.js @@ -178,18 +178,8 @@ function inflateTop(it, top) { function isSame(a, b) { if (a.seq !== b.seq || a.offset !== b.offset) return false - const c1 = a.context.openedCore(a.core) - const c2 = b.context.openedCore(b.core) - if (c1 !== null && c2 !== null) return c1 === c2 || b4a.equals(c1.key, c2.key) + const k1 = a.context.getCoreKey(a.core) + const k2 = b.context.getCoreKey(b.core) - const k1 = tableKey(a) - const k2 = tableKey(b) - return k1 !== null && k2 !== null && b4a.equals(k1, k2) -} - -// the key as referenced, before the core was opened and mapped -function tableKey(p) { - if (p.core === 0) return p.context.core.key - const entry = p.context.cores[p.core - 1] - return entry === undefined ? null : entry.key + return k1 === k2 || b4a.equals(k1, k2) ? true : false } diff --git a/lib/mappings.js b/lib/mappings.js index 29e3227..eef0afc 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -11,6 +11,19 @@ class Mappings { this.dirty = false } + // the mapping of a key as far as it is known, the key itself otherwise + resolve(key) { + const known = this.known.get(b4a.toString(key, 'hex')) + return known === undefined ? key : known + } + + // every core of a table is asked for before the table is used + discover(cores) { + const pending = [] + for (const { key } of cores) pending.push(this.map(key)) + return Promise.all(pending) + } + map(key) { const hex = b4a.toString(key, 'hex') diff --git a/lib/write.js b/lib/write.js index 86d9855..dee2f5e 100644 --- a/lib/write.js +++ b/lib/write.js @@ -94,7 +94,8 @@ module.exports = class WriteBatch { const length = this._getLength(root) - context = await this._getContext(root) + if (this.key) await this.tree.context.mappings.map(this.key) + context = this._getContext(root) const changed = length === 0 const seq = length === 0 ? 0 : length - 1 diff --git a/test/basic.js b/test/basic.js index bc915e7..c243efa 100644 --- a/test/basic.js +++ b/test/basic.js @@ -1335,7 +1335,7 @@ test('inflight range - secondary core opened via context.getCore()', async funct const value = await db2.get(b4a.from('a')) t.alike(value.value, b4a.from('1')) - const hc = await db2.context.getCore(1) + const hc = db2.context.getCore(1) t.alike(hc.replicator.inflightRange, INFLIGHT_RANGE) }) @@ -1356,7 +1356,7 @@ test('inflight range - secondary core opened via context.getContextByKey()', asy w.tryPut(b4a.from('b'), b4a.from('2')) await w.flush() - const ctx = await db2.context.getContextByKey(db1.core.key) + const ctx = db2.context.getContextByKey(db1.core.key) t.alike(ctx.core.replicator.inflightRange, INFLIGHT_RANGE) }) diff --git a/test/map-core.js b/test/map-core.js index 794170b..d713462 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -406,7 +406,7 @@ test('mapCore - a read in flight survives its core being remapped', async functi t.alike(stale.key, a.core.key, 'the read opened the core as referenced') reader.context.mappings.add([{ from: a.core.key, to: copy.key }]) - const hc = await reader.context.getCore(1) + const hc = reader.context.getCore(1) t.alike(hc.key, copy.key, 'the next open resolves to the mapped core') t.absent(stale.closing, 'the superseded session is left to the reads holding it') @@ -472,6 +472,31 @@ test('mapCore - a second move while the first is still being remapped wins', asy await w.close() }) +test('mapCore - a write onto a key given by the caller resolves it', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + await a.core.clear(0, a.core.length) + + const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + + const w = new Bee(store.namespace('w'), { mapCore }) + await w.ready() + + const batch = w.write({ key: a.core.key, length: a.core.length }) + batch.tryPut(key(50), value(50)) + await batch.flush() + + await readAll(w, 51, t, 'the batch built on the mapped core') + t.absent( + (await w.cores()).some((k) => b4a.equals(k, a.core.key)), + 'the key as given is not among the cores read' + ) + await w.close() +}) + function replicateStores(a, b) { const s1 = a.replicate(true) const s2 = b.replicate(false) From bd90e18d7aea00a93a76a0d1d8babec20a0e3527 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 18:09:02 +0100 Subject: [PATCH 07/31] move mappings onto the checkpoint metadata The mappings sit next to the core table in the framed metadata struct instead of on the block itself. The schema is versioned from here on so the field, added in version 2, is only read when the metadata carries the flags byte and older checkpoints decode as before. Co-Authored-By: Claude Fable 5.1 --- build.js | 12 ++++++------ lib/context.js | 7 +++++-- lib/copy.js | 2 -- lib/encoding.js | 5 ++--- lib/write.js | 4 +--- spec/hyperschema/index.js | 31 ++++++++++++++++++------------- spec/hyperschema/schema.json | 20 ++++++++++---------- 7 files changed, 42 insertions(+), 39 deletions(-) diff --git a/build.js b/build.js index 6a4ad37..92a7378 100644 --- a/build.js +++ b/build.js @@ -1,6 +1,6 @@ const Hyperschema = require('hyperschema') -const schema = Hyperschema.from('./spec/hyperschema', { versioned: false }) +const schema = Hyperschema.from('./spec/hyperschema') const bee = schema.namespace('bee') bee.register({ @@ -298,6 +298,11 @@ bee.register({ type: '@bee/core', array: true, required: true + }, + { + name: 'mappings', + type: '@bee/mapping', + array: true } ] }) @@ -360,11 +365,6 @@ bee.register({ { name: 'invalidation', // TODO: in future block-2 should be required type: 'uint' - }, - { - name: 'mappings', - type: '@bee/mapping', - array: true } ] }) diff --git a/lib/context.js b/lib/context.js index c3da9a9..8bf6fc6 100644 --- a/lib/context.js +++ b/lib/context.js @@ -103,7 +103,9 @@ class CoreContext { const block = decodeBlock(buffer, seq) cores = block.metadata ? block.metadata.cores : [] - if (block.mappings !== null) this.mappings.add(block.mappings) + if (cores.length > 0 && block.metadata.mappings !== null) { + this.mappings.add(block.metadata.mappings) + } await this.mappings.discover(cores) } @@ -295,7 +297,8 @@ class CoreContext { flush() { this.changed = false return { - cores: this.cores + cores: this.cores, + mappings: this.mappings.flush() } } diff --git a/lib/copy.js b/lib/copy.js index cf84eca..ca08b28 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -48,7 +48,6 @@ module.exports = async function copyChanges(tree, changes, config, map) { } blk.metadata = null - blk.mappings = null blk.previous = null blocks.push(blk) @@ -66,7 +65,6 @@ module.exports = async function copyChanges(tree, changes, config, map) { if (local.changed || local.mappings.dirty) { local.checkpoint = base + blocks.length blocks[blocks.length - 1].metadata = local.flush() - blocks[blocks.length - 1].mappings = local.mappings.flush() } const buffers = new Array(blocks.length) diff --git a/lib/encoding.js b/lib/encoding.js index 3da1e74..e380dd6 100644 --- a/lib/encoding.js +++ b/lib/encoding.js @@ -85,14 +85,13 @@ function decodeBlock0(state, seq) { checkpoint: blk.checkpoint, batch: blk.batch, previous: blk.previous, - metadata: { cores: blk.cores }, + metadata: { cores: blk.cores, mappings: null }, tree, keys: blk.data.map(dataToKey), values: null, cohorts, t: T, - invalidation: 0, - mappings: null + invalidation: 0 } } diff --git a/lib/write.js b/lib/write.js index dee2f5e..0c761a0 100644 --- a/lib/write.js +++ b/lib/write.js @@ -464,8 +464,7 @@ module.exports = class WriteBatch { values: null, cohorts: null, t, - invalidation: context.invalidation, - mappings: null + invalidation: context.invalidation } for (const k of update.values) { @@ -530,7 +529,6 @@ module.exports = class WriteBatch { if (context.changed || context.mappings.dirty) { context.checkpoint = context.core.length + blocks.length blocks[blocks.length - 1].metadata = context.flush() - blocks[blocks.length - 1].mappings = context.mappings.flush() } for (let i = 0; i < blocks.length; i++) { diff --git a/spec/hyperschema/index.js b/spec/hyperschema/index.js index caf8254..9cfbce3 100644 --- a/spec/hyperschema/index.js +++ b/spec/hyperschema/index.js @@ -1,12 +1,12 @@ // This file is autogenerated by the hyperschema compiler -// Schema Version: 1 +// Schema Version: 2 /* eslint-disable camelcase */ /* eslint-disable quotes */ /* eslint-disable space-before-function-paren */ const { c } = require('hyperschema/runtime') -const VERSION = 1 +const VERSION = 2 // eslint-disable-next-line no-unused-vars let version = VERSION @@ -423,15 +423,25 @@ const encoding13_0 = encoding12_7 const encoding13 = { preencode(state, m) { encoding13_0.preencode(state, m.cores) + state.end++ // max flag is 1 so always one byte + + if (version >= 2 && m.mappings) encoding13_1.preencode(state, m.mappings) }, encode(state, m) { + const flags = (version >= 2 && m.mappings) ? 1 : 0 + encoding13_0.encode(state, m.cores) + c.uint.encode(state, flags) + + if (version >= 2 && m.mappings) encoding13_1.encode(state, m.mappings) }, decode(state) { const r0 = encoding13_0.decode(state) + const flags = state.start < state.end ? c.uint.decode(state) : 0 return { - cores: r0 + cores: r0, + mappings: (version >= 2 && (flags & 1) !== 0) ? encoding13_1.decode(state) : null } } } @@ -458,8 +468,7 @@ const encoding14 = { (m.values ? 16 : 0) | (m.cohorts ? 32 : 0) | (m.t ? 64 : 0) | - (m.invalidation ? 128 : 0) | - (m.mappings ? 256 : 0) + (m.invalidation ? 128 : 0) c.uint.preencode(state, m.type) c.uint.preencode(state, m.checkpoint) @@ -474,7 +483,6 @@ const encoding14 = { if (m.cohorts) encoding14_9.preencode(state, m.cohorts) if (m.t) c.uint.preencode(state, m.t) if (m.invalidation) c.uint.preencode(state, m.invalidation) - if (m.mappings) encoding14_12.preencode(state, m.mappings) }, encode(state, m) { const flags = @@ -485,8 +493,7 @@ const encoding14 = { (m.values ? 16 : 0) | (m.cohorts ? 32 : 0) | (m.t ? 64 : 0) | - (m.invalidation ? 128 : 0) | - (m.mappings ? 256 : 0) + (m.invalidation ? 128 : 0) c.uint.encode(state, m.type) c.uint.encode(state, m.checkpoint) @@ -501,7 +508,6 @@ const encoding14 = { if (m.cohorts) encoding14_9.encode(state, m.cohorts) if (m.t) c.uint.encode(state, m.t) if (m.invalidation) c.uint.encode(state, m.invalidation) - if (m.mappings) encoding14_12.encode(state, m.mappings) }, decode(state) { const r0 = c.uint.decode(state) @@ -521,8 +527,7 @@ const encoding14 = { values: (flags & 16) !== 0 ? encoding14_8.decode(state) : null, cohorts: (flags & 32) !== 0 ? encoding14_9.decode(state) : null, t: (flags & 64) !== 0 ? c.uint.decode(state) : 0, - invalidation: (flags & 128) !== 0 ? c.uint.decode(state) : 0, - mappings: (flags & 256) !== 0 ? encoding14_12.decode(state) : null + invalidation: (flags & 128) !== 0 ? c.uint.decode(state) : 0 } } } @@ -548,8 +553,8 @@ const encoding15 = { } } -// @bee/block-1.mappings, deferred due to recusive use -const encoding14_12 = c.array(encoding15) +// @bee/metadata.mappings, deferred due to recusive use +const encoding13_1 = c.array(encoding15) function setVersion(v) { version = v diff --git a/spec/hyperschema/schema.json b/spec/hyperschema/schema.json index bb752dc..1fe05a6 100644 --- a/spec/hyperschema/schema.json +++ b/spec/hyperschema/schema.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "schema": [ { "name": "tree-pointer-0", @@ -325,7 +325,7 @@ "name": "metadata", "namespace": "bee", "compact": false, - "flagsPosition": -1, + "flagsPosition": 1, "fields": [ { "name": "cores", @@ -333,6 +333,12 @@ "array": true, "type": "@bee/core", "version": 1 + }, + { + "name": "mappings", + "array": true, + "type": "@bee/mapping", + "version": 2 } ] }, @@ -409,12 +415,6 @@ "name": "invalidation", "type": "uint", "version": 1 - }, - { - "name": "mappings", - "array": true, - "type": "@bee/mapping", - "version": 1 } ] }, @@ -428,13 +428,13 @@ "name": "from", "required": true, "type": "fixed32", - "version": 1 + "version": 2 }, { "name": "to", "required": true, "type": "fixed32", - "version": 1 + "version": 2 } ] } From 0663659d99bec9fcd782ca8a4f6c49bedadff2f9 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Mon, 19 Oct 2026 18:09:36 +0100 Subject: [PATCH 08/31] open mapped cores through the session preload A core whose mapping is not known yet is opened with a corestore preload that asks the hook, so the session only ever opens on the core it maps to and nothing in the tree has to resolve keys itself: getCore and the context helpers are sync, the table is left as referenced, and the tree no longer remaps itself on ready, move or checkout. Contexts carry the key they were opened for, as a preloaded session has none until open. Co-Authored-By: Claude Fable 5.1 --- README.md | 17 +++------ index.js | 88 ++++++++++++++++------------------------------- lib/cache.js | 2 +- lib/changes.js | 5 +-- lib/context.js | 89 ++++++++++++++++++++++++++++++++++++++---------- lib/copy.js | 2 +- lib/mappings.js | 15 ++++---- lib/write.js | 1 - test/map-core.js | 4 +-- 9 files changed, 118 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 646a6da..9c62921 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,10 @@ Options include: } ``` -`mapCore` is asked once per core the tree references (and the core the tree -is opened on). A non-null answer redirects every read of that core to the -mapped core. Mappings are persisted next to the core table on the next write, -so readers that open the tree later resolve them without the hook. +`mapCore` is asked once per core the tree opens, the one it is opened on +included. A non-null answer opens the mapped core in its place. Mappings are +persisted next to the core table on the next write, so readers that open the +tree later resolve them without the hook. If you pass your own `core`, consider increasing the inflight range on it (`256` minimum recommended, e.g. `{ inflightRange: [256, 512] }` when creating the core via Corestore) to match `hyperbee2` constructed cores. @@ -165,14 +165,7 @@ Options: } ``` -If `key` is mapped by `mapCore` the tree is moved onto the mapped core before -the next operation, see `db.remap()`. - -#### `await db.remap()` - -Resolves the mapping of the core the tree is open on and moves the tree onto -it. Called by `ready()`, `move()` and `checkout()`, and awaited by every -operation, so it only needs calling to observe `db.core` right after a move. +If `key` is mapped by `mapCore` the core is opened as the core it maps to. #### `db.snapshot([options])` diff --git a/index.js b/index.js index 085ceb2..87628a2 100644 --- a/index.js +++ b/index.js @@ -30,14 +30,18 @@ class Hyperbee extends EventEmitter { wait = config.wait, trace = config.trace, mapCore = null, - core = getRootCore(store, key, getEncryptionProvider), + mappings = new Mappings(mapCore), + core = getRootCore(store, key, getEncryptionProvider, mappings), context = new CoreContext( store, core, new NodeCache(maxCacheSize), core, getEncryptionProvider, - new Mappings(mapCore) + mappings, + undefined, + undefined, + key ), root = null, view = false, @@ -58,7 +62,6 @@ class Hyperbee extends EventEmitter { this.autoUpdate = autoUpdate this.preload = preload - this._remapping = null this.ready().catch(noop) } @@ -75,8 +78,8 @@ class Hyperbee extends EventEmitter { head() { if (!this.root) return null - if (this.root === EMPTY) return { length: 0, key: this.context.core.key } - return { length: this.root.seq + 1, key: this.root.context.core.key } + if (this.root === EMPTY) return { length: 0, key: this.context.coreKey() } + return { length: this.root.seq + 1, key: this.root.context.coreKey() } } get cache() { @@ -142,7 +145,6 @@ class Hyperbee extends EventEmitter { this.writable = writable this._setRoot(this._nodeAtSeq(length - 1), true) - if (key !== null) this.remap().catch(noop) } snapshot({ timeout = this.config.timeout, wait = this.config.wait } = {}) { @@ -177,58 +179,24 @@ class Hyperbee extends EventEmitter { return seq < 0 ? EMPTY : this.context.createTreeNode(0, seq, 0, false, null) } - // moves the bee onto the core its own core is mapped to. The mappings the - // table carries are picked up from the checkpoint as it is inflated, and the - // tip is only consulted when it is local so this never waits on a peer - remap() { - if (this._remapping === null) { - this._remapping = this._remap().finally(() => { - this._remapping = null - }) - } - return this._remapping - } - - // a move during the remap starts it over for the new context - async _remap() { - let handled = null - - while (this.context !== handled) { - const context = this.context - const mapped = await context.mappings.map(context.core.key) - if (this.context !== context) continue - - if (!b4a.equals(mapped, context.core.key)) { - const next = context.getContextByKey(mapped) - await next.core.ready() - if (this.context !== context) continue - - this.context = next - - const root = this.root - if (root !== null && root !== EMPTY && root.context === context && root.core === 0) { - this.root = next.createTreeNode(0, root.seq, 0, false, null) - } - } - - handled = this.context - - try { - await handled.update(this.config.options({ wait: false })) - } catch (err) { - if (err.code !== 'BLOCK_NOT_AVAILABLE') throw err - } + // the latest table and mappings of a core are picked up when its tip is + // local, never waiting on a peer for them + async _updateLocal() { + try { + await this.context.update(this.config.options({ wait: false })) + } catch (err) { + if (err.code !== 'BLOCK_NOT_AVAILABLE') throw err } } async ready() { if (!this.core.opened) await this.core.ready() - if (this.root) return this.remap() + if (this.root) return if (this.preload) await this.preload() - if (this.root) return this.remap() + if (this.root) return - await this.remap() + await this._updateLocal() if (this.root) return this._setRoot(this._lastNodeInCore(), false) @@ -258,7 +226,10 @@ class Hyperbee extends EventEmitter { // index 0 is the core this bee is viewing, the rest are referenced cores for (let i = 0; i <= this.context.cores.length; i++) { - const key = this.context.getCoreKey(i) + const key = + i === 0 + ? this.context.coreKey() + : await this.context.mappings.map(this.context.cores[i - 1].key) const hex = b4a.toString(key, 'hex') if (seen.has(hex)) continue seen.add(hex) @@ -363,7 +334,6 @@ class Hyperbee extends EventEmitter { async bootstrap(config) { if (!this.root) await this.ready() - if (this._remapping !== null) await this._remapping if (this.unbatch) await this._rollback(config) return this.root === EMPTY ? null : this.root } @@ -445,10 +415,12 @@ function toEncryptionProvider(encryption) { return () => null } -function getRootCore(store, key, getEncryptionProvider) { - const encryption = getEncryptionProvider(key) - - return key - ? store.get({ key, encryption, inflightRange: [256, 512] }) - : store.get({ key, name: 'bee', encryption, inflightRange: [256, 512] }) +function getRootCore(store, key, getEncryptionProvider, mappings) { + if (key) return CoreContext.openCore(store, key, getEncryptionProvider, mappings) + return store.get({ + key, + name: 'bee', + encryption: getEncryptionProvider(key), + inflightRange: [256, 512] + }) } diff --git a/lib/cache.js b/lib/cache.js index 00e9b9b..3ee54f3 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -101,6 +101,6 @@ module.exports = class NodeCache { // pointer whose core is not open yet can only miss function cacheId(node) { const core = node.context.openedCore(node.core) - const id = core !== null ? core.id : node.context.core.id + ':' + node.core + const id = core !== null && core.key !== null ? core.id : node.context.id + ':' + node.core return id + '@' + node.seq + '.' + node.offset } diff --git a/lib/changes.js b/lib/changes.js index 2f649f8..109617a 100644 --- a/lib/changes.js +++ b/lib/changes.js @@ -21,10 +21,7 @@ class ChangesStream extends Readable { async _openp() { await this.tree.bootstrap(this.config) if (this.head === null) this.head = this.tree.head() - if (this.head !== null) { - await this.tree.context.mappings.map(this.head.key) - this.context = this.tree.context.getContextByKey(this.head.key) - } + if (this.head !== null) this.context = this.tree.context.getContextByKey(this.head.key) } async _readp() { diff --git a/lib/context.js b/lib/context.js index 8bf6fc6..21c7d0a 100644 --- a/lib/context.js +++ b/lib/context.js @@ -7,6 +7,9 @@ const { decodeBlock, TYPE_LATEST } = require('./encoding.js') const Mappings = require('./mappings.js') const CHECKPOINT_KEY = 'hyperbee/checkpoint' +const INFLIGHT_RANGE = [256, 512] + +let ids = 0 class CoreContext { constructor( @@ -17,12 +20,15 @@ class CoreContext { getEncryptionProvider, mappings = new Mappings(), lock = new ScopeLock(), - other = new Map() + other = new Map(), + key = core.key ) { + this.id = ids++ this.store = store this.local = local this.cache = cache this.core = core + this.key = key this.getEncryptionProvider = getEncryptionProvider this.lock = lock this.other = other @@ -106,7 +112,6 @@ class CoreContext { if (cores.length > 0 && block.metadata.mappings !== null) { this.mappings.add(block.metadata.mappings) } - await this.mappings.discover(cores) } if (length <= this.length) return @@ -147,8 +152,13 @@ class CoreContext { return block.type } + // the key of the core this context reads, as opened once it is + coreKey() { + return this.core.key || this.key + } + getCoreOffsetLocalByKey(key) { - if (b4a.equals(key, this.core.key)) return 0 + if (b4a.equals(key, this.coreKey())) return 0 for (let i = 0; i < this.cores.length; i++) { const k = this.cores[i].key @@ -163,7 +173,7 @@ class CoreContext { getCoreOffsetLocal(context, core) { if (context === this) return core - const key = core === 0 ? context.core.key : context.cores[core - 1].key + const key = core === 0 ? context.coreKey() : context.cores[core - 1].key const offset = this.getCoreOffsetLocalByKey(key) if (offset > -1) return offset @@ -204,17 +214,24 @@ class CoreContext { if (index === 0) return this.core if (index > this.cores.length) throw new Error('Bad core index: ' + index) - const key = this.mappings.resolve(this.cores[index - 1].key) + const key = this.cores[index - 1].key const opened = this.opened[index - 1] - if (opened !== null && b4a.equals(opened.key, key)) return opened + if ( + opened !== null && + (opened.key === null || b4a.equals(opened.key, this.mappings.resolve(key))) + ) { + return opened + } this.opened[index - 1] = this._openCore(key) return this.opened[index - 1] } getCoreKey(index) { - if (index === 0) return this.core.key + if (index === 0) return this.coreKey() if (index > this.cores.length) throw new Error('Bad core index: ' + index) + const opened = this.opened[index - 1] + if (opened !== null && opened.key !== null) return opened.key return this.mappings.resolve(this.cores[index - 1].key) } @@ -225,11 +242,7 @@ class CoreContext { } _openCore(key) { - return this.store.get({ - key, - encryption: this.getEncryptionProvider(key), - inflightRange: [256, 512] - }) + return openCore(this.store, key, this.getEncryptionProvider, this.mappings) } async getBlock(seq, core, config, origin) { @@ -264,12 +277,13 @@ class CoreContext { getContextByKey(key) { key = this.mappings.resolve(key) - if (b4a.equals(key, this.core.key)) return this + const mine = this.coreKey() + if (mine !== null && b4a.equals(key, mine)) return this const hex = b4a.toString(key, 'hex') if (this.other.has(hex)) return this.other.get(hex) - const ctx = this._createContext(this._openCore(key)) + const ctx = this._createContext(this._openCore(key), key) this.other.set(hex, ctx) return ctx } @@ -286,10 +300,11 @@ class CoreContext { if (core > this.cores.length) throw new Error('Bad core index: ' + core) const hc = this.getCore(core) - const hex = b4a.toString(hc.key, 'hex') + const key = this.getCoreKey(core) + const hex = b4a.toString(key, 'hex') if (this.other.has(hex)) return this.other.get(hex) - const ctx = this._createContext(hc) + const ctx = this._createContext(hc, key) this.other.set(hex, ctx) return ctx } @@ -302,7 +317,7 @@ class CoreContext { } } - _createContext(core) { + _createContext(core, key) { const store = this.store const local = this.local const cache = this.cache @@ -310,8 +325,46 @@ class CoreContext { const mappings = this.mappings const lock = this.lock const other = this.other - return new CoreContext(store, local, cache, core, getEncryptionProvider, mappings, lock, other) + return new CoreContext( + store, + local, + cache, + core, + getEncryptionProvider, + mappings, + lock, + other, + key + ) } } +// a core whose mapping is not known yet is opened through a preload that asks +// for it, so the session only ever opens on the core it maps to +function openCore(store, key, getEncryptionProvider, mappings) { + const known = mappings.get(key) + if (known !== null || mappings.hook === null) { + const mapped = known === null ? key : known + return store.get({ + key: mapped, + encryption: getEncryptionProvider(mapped), + inflightRange: INFLIGHT_RANGE + }) + } + + return store.get({ + key, + encryption: getEncryptionProvider(key), + inflightRange: INFLIGHT_RANGE, + preload: preloadCore(key, getEncryptionProvider, mappings) + }) +} + +async function preloadCore(key, getEncryptionProvider, mappings) { + const mapped = await mappings.map(key) + return { key: mapped, encryption: getEncryptionProvider(mapped) } +} + +CoreContext.openCore = openCore + module.exports = CoreContext diff --git a/lib/copy.js b/lib/copy.js index ca08b28..c779878 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -16,7 +16,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { for (let i = changes.length - 1; i >= 0; i--) { const change = changes[i] const src = tree.context.getContextByKey(change.head.key) - const srcHex = b4a.toString(src.core.key, 'hex') + const srcHex = b4a.toString(src.coreKey(), 'hex') const batch = change.batch const start = base + blocks.length const first = change.head.length - batch.length diff --git a/lib/mappings.js b/lib/mappings.js index eef0afc..4e8fa73 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -11,17 +11,16 @@ class Mappings { this.dirty = false } - // the mapping of a key as far as it is known, the key itself otherwise - resolve(key) { + // the mapping of a key once known, null until then + get(key) { const known = this.known.get(b4a.toString(key, 'hex')) - return known === undefined ? key : known + return known === undefined ? null : known } - // every core of a table is asked for before the table is used - discover(cores) { - const pending = [] - for (const { key } of cores) pending.push(this.map(key)) - return Promise.all(pending) + // the mapping of a key as far as it is known, the key itself otherwise + resolve(key) { + const known = this.get(key) + return known === null ? key : known } map(key) { diff --git a/lib/write.js b/lib/write.js index 0c761a0..e455446 100644 --- a/lib/write.js +++ b/lib/write.js @@ -94,7 +94,6 @@ module.exports = class WriteBatch { const length = this._getLength(root) - if (this.key) await this.tree.context.mappings.map(this.key) context = this._getContext(root) const changed = length === 0 diff --git a/test/map-core.js b/test/map-core.js index d713462..07c9c54 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -68,7 +68,7 @@ test('mapCore - the references of a writer resolve to the mapped core', async fu const w = new Bee(store.namespace('w'), { mapCore }) await w.ready() w.move({ key: a.core.key, length: a.core.length }) - await w.remap() + await w.core.ready() t.alike(w.core.key, copy.key, 'moving onto the original lands on the mapping') const batch = w.write() @@ -463,7 +463,7 @@ test('mapCore - a second move while the first is still being remapped wins', asy for (let i = 0; i < 10 && releaseA === null; i++) await immediate() releaseA() - await w.remap() + await w.core.ready() t.alike(w.core.key, copyB.key, 'the bee sits on the mapping of the last move') t.alike(w.head(), { length: b.core.length, key: copyB.key }, 'with the head of the last move') t.alike((await w.get(key(100))).value, value(100), 'and reads its data') From 084b969ce012f087807c43cc00d8fd2afe7a2cf5 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Tue, 20 Oct 2026 09:56:06 +0100 Subject: [PATCH 09/31] load mappings statically Mappings are a table the tree is given, at open or through addMappings, and learns from every checkpoint it inflates. The hook and the session preload go, core resolution is a sync lookup, and cores() gains a recursive walk over the core tables of every core reached so a caller can find what to map before reading. cores() now describes the root's core rather than the context the tree was last moved onto. Co-Authored-By: Claude Fable 5.1 --- README.md | 35 ++++++-- index.js | 66 +++++++------- lib/cache.js | 6 +- lib/context.js | 94 ++++---------------- lib/copy.js | 2 +- lib/mappings.js | 49 ++--------- test/map-core.js | 224 +++++++++++++++++++++-------------------------- 7 files changed, 189 insertions(+), 287 deletions(-) diff --git a/README.md b/README.md index 9c62921..583b71c 100644 --- a/README.md +++ b/README.md @@ -62,16 +62,12 @@ Options include: autoUpdate: false, // Reload root node when underlying Hypercore is appended to? preload: null, // A function called by ready() after the Hypercore is ready. Can be async. wait: true, // Wait for Hypercore to download blocks - mapCore: null, // async (key) => mappedKey | null. Redirect reads of a core to another core holding the same blocks + mappings: null, // [{ from, to }]. Read core `from` as core `to`, which holds the same blocks } ``` -`mapCore` is asked once per core the tree opens, the one it is opened on -included. A non-null answer opens the mapped core in its place. Mappings are -persisted next to the core table on the next write, so readers that open the -tree later resolve them without the hook. - -If you pass your own `core`, consider increasing the inflight range on it (`256` minimum recommended, e.g. `{ inflightRange: [256, 512] }` when creating the core via Corestore) to match `hyperbee2` constructed cores. +Mappings are persisted next to the core table on the next write, so readers +that open the tree later resolve them without being given them. #### `await db.ready()` @@ -134,6 +130,29 @@ Read only. Boolean indicating whether the [Corestore][corestore] has closed. Calls `replicate` on underlying [Corestore][corestore]. +#### `await db.cores([options])` + +The keys of the cores the tree reads from, mapped: the core the root lives on +first, then the cores it references. Options: + +```js +{ + local: true, // Boolean. Include the local core + recursive: false, // Boolean. Also walk the core tables of every core reached + timeout: ..., // Number. Timeout for the checkpoint reads + wait: ... // Boolean. Wait for checkpoint blocks to download +} +``` + +With `recursive` the result covers everything the tree can reach through core +tables, one checkpoint read per core. It is a superset of what the current +root references, as tables only ever grow. + +#### `db.addMappings(mappings)` + +Add `[{ from, to }]` mappings after open. Cores already opened as `from` are +reopened as `to` on their next use. + #### `db.checkout([options])` Returns a new Hyperbee as a view on the underlying [Hypercore][hypercore]. @@ -165,7 +184,7 @@ Options: } ``` -If `key` is mapped by `mapCore` the core is opened as the core it maps to. +If `key` is mapped the core it maps to is opened instead. #### `db.snapshot([options])` diff --git a/index.js b/index.js index 87628a2..3e85df9 100644 --- a/index.js +++ b/index.js @@ -29,19 +29,16 @@ class Hyperbee extends EventEmitter { timeout = config.timeout, wait = config.wait, trace = config.trace, - mapCore = null, - mappings = new Mappings(mapCore), - core = getRootCore(store, key, getEncryptionProvider, mappings), + mappings = null, + mappingTable = new Mappings(mappings), + core = getRootCore(store, key, getEncryptionProvider, mappingTable), context = new CoreContext( store, core, new NodeCache(maxCacheSize), core, getEncryptionProvider, - mappings, - undefined, - undefined, - key + mappingTable ), root = null, view = false, @@ -78,8 +75,8 @@ class Hyperbee extends EventEmitter { head() { if (!this.root) return null - if (this.root === EMPTY) return { length: 0, key: this.context.coreKey() } - return { length: this.root.seq + 1, key: this.root.context.coreKey() } + if (this.root === EMPTY) return { length: 0, key: this.context.core.key } + return { length: this.root.seq + 1, key: this.root.context.core.key } } get cache() { @@ -216,30 +213,40 @@ class Hyperbee extends EventEmitter { this.emit('close') } - async cores({ local = true, ...opts } = {}) { + // the cores the tree reads from, mapped. Recursive walks the core tables of + // every core reached, so it covers what the tree can reach through them + async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) await this.ready() - await this.context.update(config) const cores = [] const seen = new Set() - - // index 0 is the core this bee is viewing, the rest are referenced cores - for (let i = 0; i <= this.context.cores.length; i++) { - const key = - i === 0 - ? this.context.coreKey() - : await this.context.mappings.map(this.context.cores[i - 1].key) - const hex = b4a.toString(key, 'hex') - if (seen.has(hex)) continue - seen.add(hex) - if (!local && b4a.equals(key, this.context.local.key)) continue - cores.push(key) + const queue = [this.root && this.root !== EMPTY ? this.root.context : this.context] + + while (queue.length > 0) { + const context = queue.shift() + await context.update(config) + + // index 0 is the core this context is viewing, the rest are referenced cores + for (let i = 0; i <= context.cores.length; i++) { + const key = context.getCoreKey(i) + const hex = b4a.toString(key, 'hex') + if (seen.has(hex)) continue + seen.add(hex) + + if (recursive && i > 0) queue.push(await context.getContext(i, config, null)) + if (!local && b4a.equals(key, this.context.local.key)) continue + cores.push(key) + } } return cores } + addMappings(mappings) { + this.context.mappings.add(mappings) + } + createReadStream(options) { return new RangeStream(this, options) } @@ -416,11 +423,10 @@ function toEncryptionProvider(encryption) { } function getRootCore(store, key, getEncryptionProvider, mappings) { - if (key) return CoreContext.openCore(store, key, getEncryptionProvider, mappings) - return store.get({ - key, - name: 'bee', - encryption: getEncryptionProvider(key), - inflightRange: [256, 512] - }) + if (key) key = mappings.resolve(key) + const encryption = getEncryptionProvider(key) + + return key + ? store.get({ key, encryption, inflightRange: [256, 512] }) + : store.get({ key, name: 'bee', encryption, inflightRange: [256, 512] }) } diff --git a/lib/cache.js b/lib/cache.js index 3ee54f3..8c30044 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -96,11 +96,7 @@ module.exports = class NodeCache { } } -// nodes are cached under the core they were read from, so a block reached -// through a mapped core and through the core it maps to share an entry. A -// pointer whose core is not open yet can only miss function cacheId(node) { - const core = node.context.openedCore(node.core) - const id = core !== null && core.key !== null ? core.id : node.context.id + ':' + node.core + const id = node.context.getCore(node.core).id return id + '@' + node.seq + '.' + node.offset } diff --git a/lib/context.js b/lib/context.js index 21c7d0a..aea7dc6 100644 --- a/lib/context.js +++ b/lib/context.js @@ -7,9 +7,6 @@ const { decodeBlock, TYPE_LATEST } = require('./encoding.js') const Mappings = require('./mappings.js') const CHECKPOINT_KEY = 'hyperbee/checkpoint' -const INFLIGHT_RANGE = [256, 512] - -let ids = 0 class CoreContext { constructor( @@ -20,15 +17,12 @@ class CoreContext { getEncryptionProvider, mappings = new Mappings(), lock = new ScopeLock(), - other = new Map(), - key = core.key + other = new Map() ) { - this.id = ids++ this.store = store this.local = local this.cache = cache this.core = core - this.key = key this.getEncryptionProvider = getEncryptionProvider this.lock = lock this.other = other @@ -152,13 +146,8 @@ class CoreContext { return block.type } - // the key of the core this context reads, as opened once it is - coreKey() { - return this.core.key || this.key - } - getCoreOffsetLocalByKey(key) { - if (b4a.equals(key, this.coreKey())) return 0 + if (b4a.equals(key, this.core.key)) return 0 for (let i = 0; i < this.cores.length; i++) { const k = this.cores[i].key @@ -173,7 +162,7 @@ class CoreContext { getCoreOffsetLocal(context, core) { if (context === this) return core - const key = core === 0 ? context.coreKey() : context.cores[core - 1].key + const key = core === 0 ? context.core.key : context.cores[core - 1].key const offset = this.getCoreOffsetLocalByKey(key) if (offset > -1) return offset @@ -214,35 +203,26 @@ class CoreContext { if (index === 0) return this.core if (index > this.cores.length) throw new Error('Bad core index: ' + index) - const key = this.cores[index - 1].key + const key = this.mappings.resolve(this.cores[index - 1].key) const opened = this.opened[index - 1] - if ( - opened !== null && - (opened.key === null || b4a.equals(opened.key, this.mappings.resolve(key))) - ) { - return opened - } + if (opened !== null && b4a.equals(opened.key, key)) return opened this.opened[index - 1] = this._openCore(key) return this.opened[index - 1] } getCoreKey(index) { - if (index === 0) return this.coreKey() + if (index === 0) return this.core.key if (index > this.cores.length) throw new Error('Bad core index: ' + index) - const opened = this.opened[index - 1] - if (opened !== null && opened.key !== null) return opened.key return this.mappings.resolve(this.cores[index - 1].key) } - // the core a pointer reads from once opened, null before that - openedCore(index) { - if (index === 0) return this.core - return index > this.opened.length ? null : this.opened[index - 1] - } - _openCore(key) { - return openCore(this.store, key, this.getEncryptionProvider, this.mappings) + return this.store.get({ + key, + encryption: this.getEncryptionProvider(key), + inflightRange: [256, 512] + }) } async getBlock(seq, core, config, origin) { @@ -277,13 +257,12 @@ class CoreContext { getContextByKey(key) { key = this.mappings.resolve(key) - const mine = this.coreKey() - if (mine !== null && b4a.equals(key, mine)) return this + if (b4a.equals(key, this.core.key)) return this const hex = b4a.toString(key, 'hex') if (this.other.has(hex)) return this.other.get(hex) - const ctx = this._createContext(this._openCore(key), key) + const ctx = this._createContext(this._openCore(key)) this.other.set(hex, ctx) return ctx } @@ -300,11 +279,10 @@ class CoreContext { if (core > this.cores.length) throw new Error('Bad core index: ' + core) const hc = this.getCore(core) - const key = this.getCoreKey(core) - const hex = b4a.toString(key, 'hex') + const hex = b4a.toString(hc.key, 'hex') if (this.other.has(hex)) return this.other.get(hex) - const ctx = this._createContext(hc, key) + const ctx = this._createContext(hc) this.other.set(hex, ctx) return ctx } @@ -317,7 +295,7 @@ class CoreContext { } } - _createContext(core, key) { + _createContext(core) { const store = this.store const local = this.local const cache = this.cache @@ -325,46 +303,8 @@ class CoreContext { const mappings = this.mappings const lock = this.lock const other = this.other - return new CoreContext( - store, - local, - cache, - core, - getEncryptionProvider, - mappings, - lock, - other, - key - ) + return new CoreContext(store, local, cache, core, getEncryptionProvider, mappings, lock, other) } } -// a core whose mapping is not known yet is opened through a preload that asks -// for it, so the session only ever opens on the core it maps to -function openCore(store, key, getEncryptionProvider, mappings) { - const known = mappings.get(key) - if (known !== null || mappings.hook === null) { - const mapped = known === null ? key : known - return store.get({ - key: mapped, - encryption: getEncryptionProvider(mapped), - inflightRange: INFLIGHT_RANGE - }) - } - - return store.get({ - key, - encryption: getEncryptionProvider(key), - inflightRange: INFLIGHT_RANGE, - preload: preloadCore(key, getEncryptionProvider, mappings) - }) -} - -async function preloadCore(key, getEncryptionProvider, mappings) { - const mapped = await mappings.map(key) - return { key: mapped, encryption: getEncryptionProvider(mapped) } -} - -CoreContext.openCore = openCore - module.exports = CoreContext diff --git a/lib/copy.js b/lib/copy.js index c779878..ca08b28 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -16,7 +16,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { for (let i = changes.length - 1; i >= 0; i--) { const change = changes[i] const src = tree.context.getContextByKey(change.head.key) - const srcHex = b4a.toString(src.coreKey(), 'hex') + const srcHex = b4a.toString(src.core.key, 'hex') const batch = change.batch const start = base + blocks.length const first = change.head.length - batch.length diff --git a/lib/mappings.js b/lib/mappings.js index 4e8fa73..0848b73 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -4,11 +4,10 @@ const b4a = require('b4a') // blocks. Mappings are immutable facts, shared by every context of a tree and // persisted next to the core table on the next checkpoint class Mappings { - constructor(hook = null) { - this.hook = hook + constructor(mappings = null) { this.known = new Map() - this.pending = new Map() this.dirty = false + if (mappings !== null) this.add(mappings) } // the mapping of a key once known, null until then @@ -23,46 +22,13 @@ class Mappings { return known === null ? key : known } - map(key) { - const hex = b4a.toString(key, 'hex') - - const known = this.known.get(hex) - if (known !== undefined) return Promise.resolve(known) - if (this.hook === null) return Promise.resolve(key) - - const pending = this.pending.get(hex) - if (pending !== undefined) return pending - - const promise = this._ask(key, hex) - this.pending.set(hex, promise) - return promise - } - - async _ask(key, hex) { - let to = null - try { - to = await this.hook(key) - } finally { - this.pending.delete(hex) - } - - if (to === null || to === undefined) to = key - - // a record met while the hook was running wins over no mapping - const known = this.known.get(hex) - if (known !== undefined && b4a.equals(to, key)) return known - - if (!b4a.equals(to, key)) this.dirty = true - this.known.set(hex, to) - return to - } - add(mappings) { for (const { from, to } of mappings) { + if (b4a.equals(from, to)) continue const hex = b4a.toString(from, 'hex') - const known = this.known.get(hex) - if (known !== undefined && !b4a.equals(known, from)) continue + if (this.known.has(hex)) continue this.known.set(hex, to) + this.dirty = true } } @@ -70,10 +36,7 @@ class Mappings { this.dirty = false const mappings = [] - for (const [hex, to] of this.known) { - const from = b4a.from(hex, 'hex') - if (!b4a.equals(from, to)) mappings.push({ from, to }) - } + for (const [hex, to] of this.known) mappings.push({ from: b4a.from(hex, 'hex'), to }) return mappings.length === 0 ? null : mappings } diff --git a/test/map-core.js b/test/map-core.js index 07c9c54..8548d31 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -33,66 +33,60 @@ async function readAll(bee, n, t, msg) { t.pass(msg) } -test('mapCore - the viewed core is read through its mapping', async function (t) { +test('mappings - the viewed core is read through its mapping', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) const a = await source(store, 50) const copy = await copyCore(store, a.core, 'copy') + await a.core.clear(0, a.core.length) - const asked = [] - const mapCore = (k) => { - asked.push(hex(k)) - return b4a.equals(k, a.core.key) ? copy.key : null - } - - const b = new Bee(store.namespace('b'), { key: a.core.key, mapCore, writable: false }) + const mappings = [{ from: a.core.key, to: copy.key }] + const b = new Bee(store.namespace('b'), { key: a.core.key, mappings, writable: false }) await b.ready() t.alike(b.core.key, copy.key, 'the view sits on the mapped core') - t.ok(asked.includes(hex(a.core.key)), 'the hook was asked about the original') await readAll(b, 50, t, 'every entry reads through the mapping') t.alike((await b.cores()).map(hex), [hex(copy.key)], 'cores() lists the mapped core only') await b.close() }) -test('mapCore - the references of a writer resolve to the mapped core', async function (t) { +test('mappings - the references of a writer resolve to the mapped core', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) const a = await source(store, 50) const copy = await copyCore(store, a.core, 'copy') - const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + const mappings = [{ from: a.core.key, to: copy.key }] - const w = new Bee(store.namespace('w'), { mapCore }) + const w = new Bee(store.namespace('w'), { mappings }) await w.ready() w.move({ key: a.core.key, length: a.core.length }) - await w.core.ready() t.alike(w.core.key, copy.key, 'moving onto the original lands on the mapping') const batch = w.write() for (let i = 50; i < 60; i++) batch.tryPut(key(i), value(i)) await batch.flush() - // the local core references the copy, never the original const referenced = (await w.cores({ local: false })).map(hex) - t.absent(referenced.includes(hex(a.core.key)), 'the original key is not in the table') - + t.absent(referenced.includes(hex(a.core.key)), 'the original is not among the cores read') await w.close() + await a.core.clear(0, a.core.length) + const again = new Bee(store.namespace('w')) await again.ready() t.alike( (await again.cores({ local: false })).map(hex), referenced, - 'the persisted table holds the mapped keys' + 'the persisted mappings resolve the table' ) - await readAll(again, 60, t, 'a reader without the hook reads everything via the persisted table') + await readAll(again, 60, t, 'a reader without the mappings reads everything via the checkpoint') await again.close() }) -test('mapCore - an existing table is remapped on load and persisted on the next flush', async function (t) { +test('mappings - an existing table is remapped on load and persisted on the next flush', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) @@ -114,9 +108,9 @@ test('mapCore - an existing table is remapped on load and persisted on the next await w.close() const copy = await copyCore(store, a.core, 'copy') - const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + const mappings = [{ from: a.core.key, to: copy.key }] - const mapped = new Bee(store.namespace('w'), { mapCore }) + const mapped = new Bee(store.namespace('w'), { mappings }) await mapped.ready() const cores = (await mapped.cores({ local: false })).map(hex) t.ok(cores.includes(hex(copy.key)), 'the loaded table is remapped') @@ -130,16 +124,18 @@ test('mapCore - an existing table is remapped on load and persisted on the next } await mapped.close() + await a.core.clear(0, a.core.length) + const plain = new Bee(store.namespace('w')) await plain.ready() const persisted = (await plain.cores({ local: false })).map(hex) - t.ok(persisted.includes(hex(copy.key)), 'the remapped table was persisted by the flush') - t.absent(persisted.includes(hex(a.core.key)), 'without the hook, the original is still gone') - await readAll(plain, 61, t, 'a reader without the hook reads through the persisted mapping') + t.ok(persisted.includes(hex(copy.key)), 'the mappings were persisted by the flush') + t.absent(persisted.includes(hex(a.core.key)), 'without them given, the original is still gone') + await readAll(plain, 61, t, 'a reader without the mappings reads through the persisted ones') await plain.close() }) -test('mapCore - the persisted mapping survives an undo below it', async function (t) { +test('mappings - the persisted mapping survives an undo below it', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) @@ -156,9 +152,9 @@ test('mapCore - the persisted mapping survives an undo below it', async function await w.close() const copy = await copyCore(store, a.core, 'copy') - const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + const mappings = [{ from: a.core.key, to: copy.key }] - const mapped = new Bee(store.namespace('w'), { mapCore }) + const mapped = new Bee(store.namespace('w'), { mappings }) await mapped.ready() { const batch = mapped.write() @@ -167,7 +163,9 @@ test('mapCore - the persisted mapping survives an undo below it', async function } await mapped.close() - // a reader without the hook, viewing a root written before the mapping was + await a.core.clear(0, a.core.length) + + // a reader without the mappings, viewing a root written before they were // recorded, still resolves the table from its latest checkpoint const plain = new Bee(store.namespace('w')) await plain.ready() @@ -183,16 +181,16 @@ test('mapCore - the persisted mapping survives an undo below it', async function await plain.close() }) -test('mapCore - a mapping met in another writer is recorded locally', async function (t) { +test('mappings - a mapping met in another writer is recorded locally', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) const a = await source(store, 50) const copy = await copyCore(store, a.core, 'copy') - const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + const mappings = [{ from: a.core.key, to: copy.key }] - // b writes on top of a with the hook, recording the mapping in its checkpoint - const b = new Bee(store.namespace('b'), { mapCore }) + // b writes on top of a with the mapping, recording it in its checkpoint + const b = new Bee(store.namespace('b'), { mappings }) await b.ready() b.move({ key: a.core.key, length: a.core.length }) { @@ -201,7 +199,9 @@ test('mapCore - a mapping met in another writer is recorded locally', async func await batch.flush() } - // c never has the hook and only ever sees b's head + await a.core.clear(0, a.core.length) + + // c is never given the mapping and only ever sees b's head const c = new Bee(store.namespace('c')) await c.ready() c.move({ key: b.context.local.key, length: b.context.local.length }) @@ -224,7 +224,7 @@ test('mapCore - a mapping met in another writer is recorded locally', async func await again.close() }) -test('mapCore - a remote reads the whole tree after the original data is gone', async function (t) { +test('mappings - a remote reads the whole tree after the original data is gone', async function (t) { const origin = new Corestore(await t.tmp()) t.teardown(() => origin.close()) @@ -242,9 +242,9 @@ test('mapCore - a remote reads the whole tree after the original data is gone', await w.close() const copy = await copyCore(origin, a.core, 'copy') - const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) + const mappings = [{ from: a.core.key, to: copy.key }] - const mapped = new Bee(origin.namespace('w'), { mapCore }) + const mapped = new Bee(origin.namespace('w'), { mappings }) await mapped.ready() { const batch = mapped.write() @@ -262,7 +262,7 @@ test('mapCore - a remote reads the whole tree after the original data is gone', t.teardown(() => remote.close()) const stop = replicateStores(origin, remote) - const reader = new Bee(remote.session(), { key: wKey, mapCore, writable: false }) + const reader = new Bee(remote.session(), { key: wKey, mappings, writable: false }) await reader.ready() await reader.core.update({ wait: true }) t.is(reader.core.length, 21, 'the remote learnt the head') @@ -288,8 +288,8 @@ test('mapCore - a remote reads the whole tree after the original data is gone', await stop() - // a second remote without the hook: a head written before the mapping was - // recorded runs into the cleared original, the head that carries it does not + // a second remote never given the mappings: a head written before they were + // recorded runs into the cleared original, the head that carries them does not const other = new Corestore(await t.tmp()) t.teardown(() => other.close()) const stopOther = replicateStores(origin, other) @@ -307,7 +307,7 @@ test('mapCore - a remote reads the whole tree after the original data is gone', await t.exception( before.get(key(0), { wait: false }), /BLOCK_NOT_AVAILABLE/, - 'before the record, a reader without the hook hits missing data' + 'before the record, a reader without the mappings hits missing data' ) await before.close() @@ -319,7 +319,7 @@ test('mapCore - a remote reads the whole tree after the original data is gone', await stopOther() }) -test('mapCore - move and checkout onto an original resolve before the first operation', async function (t) { +test('mappings - move, checkout and write onto an original resolve synchronously', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) @@ -327,46 +327,32 @@ test('mapCore - move and checkout onto an original resolve before the first oper const copy = await copyCore(store, a.core, 'copy') await a.core.clear(0, a.core.length) - let asked = 0 - const mapCore = (k) => { - asked++ - return b4a.equals(k, a.core.key) ? copy.key : null - } - - const w = new Bee(store.namespace('w'), { mapCore }) + const mappings = [{ from: a.core.key, to: copy.key }] + const w = new Bee(store.namespace('w'), { mappings }) await w.ready() w.move({ key: a.core.key, length: a.core.length }) - await readAll(w, 50, t, 'reads after an un-awaited move go through the mapping') + t.alike(w.core.key, copy.key, 'the bee sits on the mapped core right after the move') t.alike(w.head().key, copy.key, 'the head is on the mapped core') - t.alike(w.core.key, copy.key, 'the bee sits on the mapped core') - - { - const batch = w.write() - batch.tryPut(key(50), value(50)) - await batch.flush() - } - t.absent( - (await w.cores()).some((k) => b4a.equals(k, a.core.key)), - 'the write referenced the mapped core' - ) + await readAll(w, 50, t, 'reads go through the mapping') const view = w.checkout({ key: a.core.key, length: a.core.length }) - await readAll(view, 50, t, 'a checkout onto the original reads through the mapping') - t.alike(view.head().key, copy.key, 'the view head is on the mapped core') + t.alike(view.head().key, copy.key, 'a checkout onto the original sits on the mapped core') + await readAll(view, 50, t, 'and reads through it') await view.close() - const asked1 = asked - const again = w.checkout({ key: a.core.key, length: a.core.length }) - await again.ready() - t.alike(again.head().key, copy.key, 'a second checkout lands on the mapped core') - t.is(asked, asked1, 'without asking the hook again') - await again.close() - + const batch = w.write({ key: a.core.key, length: a.core.length }) + batch.tryPut(key(50), value(50)) + await batch.flush() + await readAll(w, 51, t, 'a batch keyed by the original built on the mapped core') + t.absent( + (await w.cores()).some((k) => b4a.equals(k, a.core.key)), + 'the original is not among the cores read' + ) await w.close() }) -test('mapCore - a read in flight survives its core being remapped', async function (t) { +test('mappings - added after open, a read in flight survives its core being remapped', async function (t) { const origin = new Corestore(await t.tmp()) t.teardown(() => origin.close()) @@ -405,7 +391,7 @@ test('mapCore - a read in flight survives its core being remapped', async functi const stale = reader.context.opened[0] t.alike(stale.key, a.core.key, 'the read opened the core as referenced') - reader.context.mappings.add([{ from: a.core.key, to: copy.key }]) + reader.addMappings([{ from: a.core.key, to: copy.key }]) const hc = reader.context.getCore(1) t.alike(hc.key, copy.key, 'the next open resolves to the mapped core') t.absent(stale.closing, 'the superseded session is left to the reads holding it') @@ -424,77 +410,65 @@ test('mapCore - a read in flight survives its core being remapped', async functi await stop() }) -function immediate() { - return new Promise((resolve) => setImmediate(resolve)) -} - -test('mapCore - a second move while the first is still being remapped wins', async function (t) { +test('mappings - cores({ recursive: true }) walks the tables of every core reached', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) + // b links a, then clears and rewrites the tree: nothing in b's tree points + // into a any more but b's table still names it, and c builds on b const a = await source(store, 50) - const copyA = await copyCore(store, a.core, 'copy-a') const b = new Bee(store.namespace('b')) await b.ready() + b.move({ key: a.core.key, length: a.core.length }) { const batch = b.write() - for (let i = 0; i < 10; i++) batch.tryPut(key(100 + i), value(100 + i)) + batch.tryPut(key(50), value(50)) await batch.flush() } - const copyB = await copyCore(store, b.core, 'copy-b') - - let releaseA = null - const mapCore = (k) => { - if (b4a.equals(k, a.core.key)) { - return new Promise((resolve) => { - releaseA = () => resolve(copyA.key) - }) - } - if (b4a.equals(k, b.core.key)) return copyB.key - return null + { + const batch = b.write() + batch.tryClear() + for (let i = 0; i < 50; i++) batch.tryPut(key(i), value(100 + i)) + await batch.flush() } - const w = new Bee(store.namespace('w'), { mapCore }) - await w.ready() - - w.move({ key: a.core.key, length: a.core.length }) - w.move({ key: b.core.key, length: b.core.length }) - for (let i = 0; i < 10 && releaseA === null; i++) await immediate() - releaseA() - - await w.core.ready() - t.alike(w.core.key, copyB.key, 'the bee sits on the mapping of the last move') - t.alike(w.head(), { length: b.core.length, key: copyB.key }, 'with the head of the last move') - t.alike((await w.get(key(100))).value, value(100), 'and reads its data') - t.is(await w.get(key(0)), null, 'not the data of the first move') - - await w.close() -}) + const c = new Bee(store.namespace('c')) + await c.ready() + c.move({ key: b.context.local.key, length: b.context.local.length }) + { + const batch = c.write() + batch.tryPut(key(50), value(50)) + await batch.flush() + } -test('mapCore - a write onto a key given by the caller resolves it', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) + const direct = (await c.cores({ local: false })).map(hex).sort() + const reachable = (await c.cores({ local: false, recursive: true })).map(hex).sort() + t.alike(direct, [hex(b.context.local.key)], "c's table names b only") + t.alike( + reachable, + [hex(a.core.key), hex(b.context.local.key)].sort(), + 'recursively a is reached through b' + ) - const a = await source(store, 50) + // the flow of a migration: find what is reachable, copy it, map it in one go const copy = await copyCore(store, a.core, 'copy') await a.core.clear(0, a.core.length) + c.addMappings([{ from: a.core.key, to: copy.key }]) - const mapCore = (k) => (b4a.equals(k, a.core.key) ? copy.key : null) - - const w = new Bee(store.namespace('w'), { mapCore }) - await w.ready() - - const batch = w.write({ key: a.core.key, length: a.core.length }) - batch.tryPut(key(50), value(50)) - await batch.flush() - - await readAll(w, 51, t, 'the batch built on the mapped core') - t.absent( - (await w.cores()).some((k) => b4a.equals(k, a.core.key)), - 'the key as given is not among the cores read' + t.alike( + (await c.cores({ local: false, recursive: true })).map(hex).sort(), + [hex(copy.key), hex(b.context.local.key)].sort(), + 'the walk reports the mapped core' ) - await w.close() + for (let i = 0; i < 50; i++) { + const node = await c.get(key(i)) + if (!node || !b4a.equals(node.value, value(100 + i))) return t.fail('bad value for key ' + i) + } + t.alike((await c.get(key(50))).value, value(50), 'every entry still reads') + + await c.close() + await b.close() }) function replicateStores(a, b) { @@ -514,4 +488,8 @@ function replicateStores(a, b) { } } +function immediate() { + return new Promise((resolve) => setImmediate(resolve)) +} + function noop() {} From a8bd0d97984670d55b1aadbc850545664015d2b4 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Tue, 20 Oct 2026 12:30:35 +0100 Subject: [PATCH 10/31] move a tree onto the copy when its own core gets mapped Co-Authored-By: Claude Fable 5.1 --- index.js | 12 ++++++++++++ test/map-core.js | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/index.js b/index.js index 3e85df9..ac1e46f 100644 --- a/index.js +++ b/index.js @@ -243,8 +243,20 @@ class Hyperbee extends EventEmitter { return cores } + // a tree sitting on a core that just got mapped moves onto the mapped core addMappings(mappings) { this.context.mappings.add(mappings) + + const previous = this.context + const mapped = previous.mappings.resolve(previous.core.key) + if (b4a.equals(mapped, previous.core.key)) return + + this.context = previous.getContextByKey(mapped) + + const root = this.root + if (root !== null && root !== EMPTY && root.context === previous && root.core === 0) { + this.root = this.context.createTreeNode(0, root.seq, 0, false, null) + } } createReadStream(options) { diff --git a/test/map-core.js b/test/map-core.js index 8548d31..32e156c 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -471,6 +471,28 @@ test('mappings - cores({ recursive: true }) walks the tables of every core reach await b.close() }) +test('mappings - added after open, a tree sitting on the mapped core moves onto the copy', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + + const b = new Bee(store.namespace('b'), { key: a.core.key, writable: false }) + await b.ready() + t.alike(b.core.key, a.core.key, 'the tree sits on the original') + await readAll(b, 50, t, 'and reads it') + + await a.core.clear(0, a.core.length) + b.addMappings([{ from: a.core.key, to: copy.key }]) + + t.alike(b.core.key, copy.key, 'the tree moved onto the copy') + t.alike(b.head().key, copy.key, 'with its head') + t.alike((await b.cores()).map(hex), [hex(copy.key)], 'and lists it') + await readAll(b, 50, t, 'reads go to the copy') + await b.close() +}) + function replicateStores(a, b) { const s1 = a.replicate(true) const s2 = b.replicate(false) From ead40e25f20ae287c97ff818eacc0f502badc088 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Tue, 20 Oct 2026 12:37:50 +0100 Subject: [PATCH 11/31] walk past a referenced core that is not a tree Co-Authored-By: Claude Fable 5.1 --- index.js | 20 ++++++++++++++++++-- test/map-core.js | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index ac1e46f..86c81fd 100644 --- a/index.js +++ b/index.js @@ -221,11 +221,19 @@ class Hyperbee extends EventEmitter { const cores = [] const seen = new Set() - const queue = [this.root && this.root !== EMPTY ? this.root.context : this.context] + const root = this.root && this.root !== EMPTY ? this.root.context : this.context + const queue = [root] while (queue.length > 0) { const context = queue.shift() - await context.update(config) + + // a referenced core whose checkpoint cannot be read as a tree is a leaf + try { + await context.update(config) + } catch (err) { + if (context === root || isNotAvailable(err)) throw err + continue + } // index 0 is the core this context is viewing, the rest are referenced cores for (let i = 0; i <= context.cores.length; i++) { @@ -427,6 +435,14 @@ class Hyperbee extends EventEmitter { module.exports = Hyperbee +function isNotAvailable(err) { + return ( + err.code === 'BLOCK_NOT_AVAILABLE' || + err.code === 'REQUEST_TIMEOUT' || + err.code === 'REQUEST_CANCELLED' + ) +} + function noop() {} function toEncryptionProvider(encryption) { diff --git a/test/map-core.js b/test/map-core.js index 32e156c..227de5d 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -493,6 +493,31 @@ test('mappings - added after open, a tree sitting on the mapped core moves onto await b.close() }) +test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + + const other = store.get({ name: 'other' }) + await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) + + // a table entry for the foreign core, as a reference into it would leave + const b = new Bee(store.namespace('b')) + await b.ready() + await b.context.getLocalContext().getCoreOffsetByKey(other.key, b.config) + const batch = b.write() + batch.tryPut(key(0), value(0)) + await batch.flush() + + const direct = (await b.cores({ local: false })).map(hex) + t.alike(direct, [hex(other.key)], 'the foreign core is listed') + t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') + + await a.close() + await b.close() +}) + function replicateStores(a, b) { const s1 = a.replicate(true) const s2 = b.replicate(false) From 0099dd28bd11c9773c1881f65db121487014d39e Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Tue, 20 Oct 2026 12:39:34 +0100 Subject: [PATCH 12/31] resolve the table of the root from its block in cores() Co-Authored-By: Claude Fable 5.1 --- index.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 86c81fd..cf92991 100644 --- a/index.js +++ b/index.js @@ -224,14 +224,21 @@ class Hyperbee extends EventEmitter { const root = this.root && this.root !== EMPTY ? this.root.context : this.context const queue = [root] + // the table of the root resolves from the block the root lives in, as a + // read would, never from the tip of its core + if (root === this.root.context) { + const block = await root.getBlock(this.root.seq, 0, config, null) + await root._inflateCheckpoint(this.root.seq + 1, block, config, false) + } + while (queue.length > 0) { const context = queue.shift() // a referenced core whose checkpoint cannot be read as a tree is a leaf try { - await context.update(config) + if (context !== root) await context.update(config) } catch (err) { - if (context === root || isNotAvailable(err)) throw err + if (isNotAvailable(err)) throw err continue } From 40cae8213402241c55deb450952816a12e74dfb3 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Tue, 20 Oct 2026 14:21:17 +0100 Subject: [PATCH 13/31] share a Mappings between trees Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- index.js | 6 +++++- test/map-core.js | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 583b71c..1e8d42c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Options include: autoUpdate: false, // Reload root node when underlying Hypercore is appended to? preload: null, // A function called by ready() after the Hypercore is ready. Can be async. wait: true, // Wait for Hypercore to download blocks - mappings: null, // [{ from, to }]. Read core `from` as core `to`, which holds the same blocks + mappings: null, // [{ from, to }] or a Hyperbee.Mappings to share between trees. Read core `from` as core `to`, which holds the same blocks } ``` diff --git a/index.js b/index.js index cf92991..ae1b994 100644 --- a/index.js +++ b/index.js @@ -30,7 +30,7 @@ class Hyperbee extends EventEmitter { wait = config.wait, trace = config.trace, mappings = null, - mappingTable = new Mappings(mappings), + mappingTable = mappings instanceof Mappings ? mappings : new Mappings(mappings), core = getRootCore(store, key, getEncryptionProvider, mappingTable), context = new CoreContext( store, @@ -67,6 +67,10 @@ class Hyperbee extends EventEmitter { return bee instanceof Hyperbee } + static get Mappings() { + return Mappings + } + isGenesis() { const head = this.head() if (!head) return false diff --git a/test/map-core.js b/test/map-core.js index 227de5d..3567abc 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -518,6 +518,32 @@ test('mappings - cores({ recursive: true }) does not descend into a core that is await b.close() }) +test('mappings - trees given the same Mappings share what each of them learns', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + const copy = await copyCore(store, a.core, 'copy') + await a.core.clear(0, a.core.length) + + const mappings = new Bee.Mappings() + const b = new Bee(store.namespace('b'), { key: a.core.key, mappings, writable: false }) + const c = new Bee(store.namespace('c'), { key: a.core.key, mappings, writable: false }) + await b.ready() + await c.ready() + t.alike(b.core.key, a.core.key, 'nothing is mapped yet') + + b.addMappings([{ from: a.core.key, to: copy.key }]) + t.alike(b.core.key, copy.key, 'the tree told moved') + t.ok(c.context.mappings.get(a.core.key) !== null, 'the other tree knows the mapping') + c.addMappings([]) + t.alike(c.core.key, copy.key, 'and moves once asked to look') + await readAll(c, 50, t, 'reading through it') + + await b.close() + await c.close() +}) + function replicateStores(a, b) { const s1 = a.replicate(true) const s2 = b.replicate(false) From 0ee8ae2b897ff5f66f79c79005b4c2b2f47b1205 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 10:08:08 +0100 Subject: [PATCH 14/31] mappings imported as array and flushed immediately --- README.md | 11 +++++-- index.js | 55 +++++++++++++++---------------- lib/context.js | 3 +- lib/copy.js | 1 + lib/mappings.js | 5 +-- lib/write.js | 12 ++++++- test/map-core.js | 86 +++++++++++++++++++++++++++++++++++------------- 7 files changed, 117 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 1e8d42c..0a41b0e 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Options include: autoUpdate: false, // Reload root node when underlying Hypercore is appended to? preload: null, // A function called by ready() after the Hypercore is ready. Can be async. wait: true, // Wait for Hypercore to download blocks - mappings: null, // [{ from, to }] or a Hyperbee.Mappings to share between trees. Read core `from` as core `to`, which holds the same blocks + mappings: null, // [{ from, to }]. Read core `from` as core `to`, which holds the same blocks } ``` @@ -151,7 +151,14 @@ root references, as tables only ever grow. #### `db.addMappings(mappings)` Add `[{ from, to }]` mappings after open. Cores already opened as `from` are -reopened as `to` on their next use. +reopened as `to` on their next use. They are persisted with the core table on +the next write. + +#### `await db.flushMappings(mappings)` + +Add the mappings and, on a writable tree, write them into a checkpoint right +away, so a reader of the head gets them with the table. Writes one block +holding the root as it is; writes nothing if every mapping was known. #### `db.checkout([options])` diff --git a/index.js b/index.js index ae1b994..ea8cfec 100644 --- a/index.js +++ b/index.js @@ -30,7 +30,7 @@ class Hyperbee extends EventEmitter { wait = config.wait, trace = config.trace, mappings = null, - mappingTable = mappings instanceof Mappings ? mappings : new Mappings(mappings), + mappingTable = new Mappings(mappings), core = getRootCore(store, key, getEncryptionProvider, mappingTable), context = new CoreContext( store, @@ -67,10 +67,6 @@ class Hyperbee extends EventEmitter { return bee instanceof Hyperbee } - static get Mappings() { - return Mappings - } - isGenesis() { const head = this.head() if (!head) return false @@ -180,16 +176,6 @@ class Hyperbee extends EventEmitter { return seq < 0 ? EMPTY : this.context.createTreeNode(0, seq, 0, false, null) } - // the latest table and mappings of a core are picked up when its tip is - // local, never waiting on a peer for them - async _updateLocal() { - try { - await this.context.update(this.config.options({ wait: false })) - } catch (err) { - if (err.code !== 'BLOCK_NOT_AVAILABLE') throw err - } - } - async ready() { if (!this.core.opened) await this.core.ready() @@ -197,9 +183,6 @@ class Hyperbee extends EventEmitter { if (this.preload) await this.preload() if (this.root) return - await this._updateLocal() - if (this.root) return - this._setRoot(this._lastNodeInCore(), false) if (this.autoUpdate) { this.core.on('append', this.update.bind(this, null)) @@ -221,20 +204,13 @@ class Hyperbee extends EventEmitter { // every core reached, so it covers what the tree can reach through them async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) - await this.ready() + const node = await this.bootstrap(config) const cores = [] const seen = new Set() - const root = this.root && this.root !== EMPTY ? this.root.context : this.context + const root = node === null ? this.context : node.context const queue = [root] - // the table of the root resolves from the block the root lives in, as a - // read would, never from the tip of its core - if (root === this.root.context) { - const block = await root.getBlock(this.root.seq, 0, config, null) - await root._inflateCheckpoint(this.root.seq + 1, block, config, false) - } - while (queue.length > 0) { const context = queue.shift() @@ -262,6 +238,18 @@ class Hyperbee extends EventEmitter { return cores } + // the mappings are written into a checkpoint right away, so a reader of + // this tree gets them with the table even before anything else is written + async flushMappings(mappings) { + await this.bootstrap(this.config) + this.addMappings(mappings) + if (!this.writable || !this.context.mappings.dirty) return + + const batch = this.write() + batch.touch = true + await batch.flush() + } + // a tree sitting on a core that just got mapped moves onto the mapped core addMappings(mappings) { this.context.mappings.add(mappings) @@ -370,10 +358,21 @@ class Hyperbee extends EventEmitter { } } + // the table and mappings of a root come from the checkpoint its own block + // names, so nothing recorded up to the root is missed by a read that meets + // an older block first async bootstrap(config) { if (!this.root) await this.ready() if (this.unbatch) await this._rollback(config) - return this.root === EMPTY ? null : this.root + if (this.root === EMPTY) return null + + const root = this.root + if (root.context.length < root.seq + 1) { + const block = await root.context.getBlock(root.seq, 0, config, null) + await root.context._inflateCheckpoint(root.seq + 1, block, config, false) + } + + return root } async _rollback(config) { diff --git a/lib/context.js b/lib/context.js index aea7dc6..f5a3fa6 100644 --- a/lib/context.js +++ b/lib/context.js @@ -104,7 +104,8 @@ class CoreContext { cores = block.metadata ? block.metadata.cores : [] if (cores.length > 0 && block.metadata.mappings !== null) { - this.mappings.add(block.metadata.mappings) + const persisted = b4a.equals(this.core.key, this.local.key) + this.mappings.add(block.metadata.mappings, { persisted }) } } diff --git a/lib/copy.js b/lib/copy.js index ca08b28..df43935 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -75,6 +75,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { } await local.core.append(buffers) + local.length = local.core.length return changes.length diff --git a/lib/mappings.js b/lib/mappings.js index 0848b73..bb467e4 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -22,13 +22,14 @@ class Mappings { return known === null ? key : known } - add(mappings) { + // what is already in a checkpoint of our own core needs no writing + add(mappings, { persisted = false } = {}) { for (const { from, to } of mappings) { if (b4a.equals(from, to)) continue const hex = b4a.toString(from, 'hex') if (this.known.has(hex)) continue this.known.set(hex, to) - this.dirty = true + if (!persisted) this.dirty = true } } diff --git a/lib/write.js b/lib/write.js index e455446..fd20838 100644 --- a/lib/write.js +++ b/lib/write.js @@ -39,6 +39,7 @@ module.exports = class WriteBatch { this.applied = 0 this.root = null this.ops = [] + this.touch = false } tryPut(key, value) { @@ -110,6 +111,12 @@ module.exports = class WriteBatch { if (op.applied) this.applied++ } + // a touch rewrites the root as it is, so a checkpoint is written + if (this.touch && !this.root.changed) { + await retainAndInflate(this.root, this.snapshot, this.config) + this.root.mark() + } + await this._flush() await this.close() @@ -424,7 +431,7 @@ module.exports = class WriteBatch { const length = context.core.length // if noop and not genesis, bail early - if (this.applied === 0 && this.length > 0) return + if (this.applied === 0 && this.length > 0 && !this.touch) return if (minValue > -1 && minValue + update.size < this.preferredBlockSize) { // TODO: repack the value into the block @@ -539,6 +546,9 @@ module.exports = class WriteBatch { await context.core.append(buffers) + // the table written is the table held + context.length = context.core.length + for (let i = 0; i < batch.length; i++) { const update = batch[i] diff --git a/test/map-core.js b/test/map-core.js index 3567abc..d903a50 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -135,7 +135,7 @@ test('mappings - an existing table is remapped on load and persisted on the next await plain.close() }) -test('mappings - the persisted mapping survives an undo below it', async function (t) { +test('mappings - an undo below the record reads through the mappings it is given', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) @@ -165,12 +165,12 @@ test('mappings - the persisted mapping survives an undo below it', async functio await a.core.clear(0, a.core.length) - // a reader without the mappings, viewing a root written before they were - // recorded, still resolves the table from its latest checkpoint - const plain = new Bee(store.namespace('w')) - await plain.ready() - const old = plain.undo(3) - await readAll(old, 53, t, 'an undone root reads through the persisted mapping') + // the record lives in the last checkpoint, a root below it has a table + // without it: the mappings have to be given, as they are on reopen + const again = new Bee(store.namespace('w'), { mappings }) + await again.ready() + const old = again.undo(3) + await readAll(old, 53, t, 'an undone root reads through the mappings') t.is(await old.get(key(53)), null, 'the undone writes are gone') t.alike( (await old.cores({ local: false })).map(hex), @@ -178,7 +178,7 @@ test('mappings - the persisted mapping survives an undo below it', async functio 'the undone view resolves the original to the copy' ) await old.close() - await plain.close() + await again.close() }) test('mappings - a mapping met in another writer is recorded locally', async function (t) { @@ -387,7 +387,7 @@ test('mappings - added after open, a read in flight survives its core being rema // a read that has to wait for a peer to serve the referenced core const read = reader.get(key(0)) - for (let i = 0; i < 100 && reader.context.opened[0] === null; i++) await immediate() + for (let i = 0; i < 100 && !reader.context.opened[0]; i++) await immediate() const stale = reader.context.opened[0] t.alike(stale.key, a.core.key, 'the read opened the core as referenced') @@ -518,30 +518,72 @@ test('mappings - cores({ recursive: true }) does not descend into a core that is await b.close() }) -test('mappings - trees given the same Mappings share what each of them learns', async function (t) { +test('mappings - flushMappings writes them into a checkpoint right away', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) const a = await source(store, 50) + + const w = new Bee(store.namespace('w')) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + { + const batch = w.write() + batch.tryPut(key(50), value(50)) + await batch.flush() + } + const local = w.context.local + const length = local.length + const copy = await copyCore(store, a.core, 'copy') + await w.flushMappings([{ from: a.core.key, to: copy.key }]) + + t.is(local.length, length + 1, 'one block was written') + t.alike(w.head(), { key: local.key, length: local.length }, 'the head moved onto it') + await readAll(w, 51, t, 'the tree reads as before') + await w.close() + await a.core.clear(0, a.core.length) - const mappings = new Bee.Mappings() - const b = new Bee(store.namespace('b'), { key: a.core.key, mappings, writable: false }) - const c = new Bee(store.namespace('c'), { key: a.core.key, mappings, writable: false }) + // a reader of the head gets the mappings with the table, nothing else needed + const again = new Bee(store.namespace('w')) + await again.ready() + t.alike(again.head().length, length + 1) + await readAll(again, 51, t, 'a reader without the mappings reads through the checkpoint') + t.alike((await again.cores({ local: false })).map(hex), [hex(copy.key)]) + await again.close() + + // nothing new to record, nothing written + const idle = new Bee(store.namespace('w')) + await idle.ready() + await idle.flushMappings([{ from: a.core.key, to: copy.key }]) + t.is(idle.context.local.length, length + 1, 'known mappings write nothing') + await idle.close() +}) + +test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + + const other = store.get({ name: 'other' }) + await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) + + // a table entry for the foreign core, as a reference into it would leave + const b = new Bee(store.namespace('b')) await b.ready() - await c.ready() - t.alike(b.core.key, a.core.key, 'nothing is mapped yet') + await b.context.getLocalContext().getCoreOffsetByKey(other.key, b.config) + const batch = b.write() + batch.tryPut(key(0), value(0)) + await batch.flush() - b.addMappings([{ from: a.core.key, to: copy.key }]) - t.alike(b.core.key, copy.key, 'the tree told moved') - t.ok(c.context.mappings.get(a.core.key) !== null, 'the other tree knows the mapping') - c.addMappings([]) - t.alike(c.core.key, copy.key, 'and moves once asked to look') - await readAll(c, 50, t, 'reading through it') + const direct = (await b.cores({ local: false })).map(hex) + t.alike(direct, [hex(other.key)], 'the foreign core is listed') + t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') + await a.close() await b.close() - await c.close() }) function replicateStores(a, b) { From f141ed6b7d622e14bf4c1819ff77329a314efaff Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 10:58:39 +0100 Subject: [PATCH 15/31] load mappings with flushMappings, write them with a touch flushMappings only loads: it reads what the tree holds in its checkpoint first, so only what is new is dirty, and moves a tree sitting on a mapped core. A batch written with touch rewrites the root as it is so the checkpoint carries the mappings, and writes nothing if they are in one already. addMappings and Mappings.get go. Co-Authored-By: Claude Fable 5.1 --- README.md | 15 ++++++-------- index.js | 15 +++----------- lib/mappings.js | 10 ++-------- lib/write.js | 14 ++++++++++--- test/map-core.js | 51 ++++++++++++++++++++++++++++++++++++------------ 5 files changed, 61 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 0a41b0e..d4dcc24 100644 --- a/README.md +++ b/README.md @@ -148,17 +148,12 @@ With `recursive` the result covers everything the tree can reach through core tables, one checkpoint read per core. It is a superset of what the current root references, as tables only ever grow. -#### `db.addMappings(mappings)` - -Add `[{ from, to }]` mappings after open. Cores already opened as `from` are -reopened as `to` on their next use. They are persisted with the core table on -the next write. - #### `await db.flushMappings(mappings)` -Add the mappings and, on a writable tree, write them into a checkpoint right -away, so a reader of the head gets them with the table. Writes one block -holding the root as it is; writes nothing if every mapping was known. +Load `[{ from, to }]` mappings, first reading those the tree already holds in +its checkpoint, so a following batch only writes what is new. Cores already +opened as `from` are reopened as `to` on their next use, and a tree sitting on +a mapped core moves onto the mapped one. #### `db.checkout([options])` @@ -236,6 +231,8 @@ Options: key: null, // Buffer or null. Key of the Hypercore. autoUpdate: true, // Boolean .Should Hyperbee automatically reflect updates // after each flush()? + touch: false, // Boolean. Write a checkpoint holding the mappings even if + // nothing changed, unless they are in one already compat: false, // Boolean. Write blocks compatible with Hyperbee 1? type: encoding.TYPE_LATEST, // Number. Block format to use. inlineValueSize: 1024, // Values smaller than this byte length are diff --git a/index.js b/index.js index ea8cfec..cc1acb7 100644 --- a/index.js +++ b/index.js @@ -238,20 +238,11 @@ class Hyperbee extends EventEmitter { return cores } - // the mappings are written into a checkpoint right away, so a reader of - // this tree gets them with the table even before anything else is written + // loads the mappings, once those the tree already holds in its checkpoint + // are known, so only what is new is written by the next batch. A tree + // sitting on a core that just got mapped moves onto the mapped core async flushMappings(mappings) { await this.bootstrap(this.config) - this.addMappings(mappings) - if (!this.writable || !this.context.mappings.dirty) return - - const batch = this.write() - batch.touch = true - await batch.flush() - } - - // a tree sitting on a core that just got mapped moves onto the mapped core - addMappings(mappings) { this.context.mappings.add(mappings) const previous = this.context diff --git a/lib/mappings.js b/lib/mappings.js index bb467e4..4b4dcd3 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -10,16 +10,10 @@ class Mappings { if (mappings !== null) this.add(mappings) } - // the mapping of a key once known, null until then - get(key) { - const known = this.known.get(b4a.toString(key, 'hex')) - return known === undefined ? null : known - } - // the mapping of a key as far as it is known, the key itself otherwise resolve(key) { - const known = this.get(key) - return known === null ? key : known + const known = this.known.get(b4a.toString(key, 'hex')) + return known === undefined ? key : known } // what is already in a checkpoint of our own core needs no writing diff --git a/lib/write.js b/lib/write.js index fd20838..33be6a2 100644 --- a/lib/write.js +++ b/lib/write.js @@ -14,6 +14,7 @@ module.exports = class WriteBatch { const { length = -1, key = null, + touch = false, autoUpdate = true, compat = false, type = compat ? TYPE_COMPAT : TYPE_LATEST, @@ -39,7 +40,7 @@ module.exports = class WriteBatch { this.applied = 0 this.root = null this.ops = [] - this.touch = false + this.touch = touch } tryPut(key, value) { @@ -91,6 +92,12 @@ module.exports = class WriteBatch { const ops = this.ops + // a touch with nothing to write reads nothing for it either + if (this.touch && ops.length === 0 && !this.tree.context.mappings.dirty) { + await this.close() + return + } + const root = await this.tree.bootstrap(this.config) const length = this._getLength(root) @@ -111,8 +118,9 @@ module.exports = class WriteBatch { if (op.applied) this.applied++ } - // a touch rewrites the root as it is, so a checkpoint is written - if (this.touch && !this.root.changed) { + // a touch writes the mappings the tree holds into a checkpoint by + // rewriting the root as it is, unless they are in one already + if (this.touch && !this.root.changed && this.tree.context.mappings.dirty) { await retainAndInflate(this.root, this.snapshot, this.config) this.root.mark() } diff --git a/test/map-core.js b/test/map-core.js index d903a50..b8e34b7 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -391,7 +391,7 @@ test('mappings - added after open, a read in flight survives its core being rema const stale = reader.context.opened[0] t.alike(stale.key, a.core.key, 'the read opened the core as referenced') - reader.addMappings([{ from: a.core.key, to: copy.key }]) + await reader.flushMappings([{ from: a.core.key, to: copy.key }]) const hc = reader.context.getCore(1) t.alike(hc.key, copy.key, 'the next open resolves to the mapped core') t.absent(stale.closing, 'the superseded session is left to the reads holding it') @@ -454,7 +454,7 @@ test('mappings - cores({ recursive: true }) walks the tables of every core reach // the flow of a migration: find what is reachable, copy it, map it in one go const copy = await copyCore(store, a.core, 'copy') await a.core.clear(0, a.core.length) - c.addMappings([{ from: a.core.key, to: copy.key }]) + await c.flushMappings([{ from: a.core.key, to: copy.key }]) t.alike( (await c.cores({ local: false, recursive: true })).map(hex).sort(), @@ -484,7 +484,7 @@ test('mappings - added after open, a tree sitting on the mapped core moves onto await readAll(b, 50, t, 'and reads it') await a.core.clear(0, a.core.length) - b.addMappings([{ from: a.core.key, to: copy.key }]) + await b.flushMappings([{ from: a.core.key, to: copy.key }]) t.alike(b.core.key, copy.key, 'the tree moved onto the copy') t.alike(b.head().key, copy.key, 'with its head') @@ -518,7 +518,7 @@ test('mappings - cores({ recursive: true }) does not descend into a core that is await b.close() }) -test('mappings - flushMappings writes them into a checkpoint right away', async function (t) { +test('mappings - a touch writes the mappings loaded into a checkpoint', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) @@ -537,10 +537,15 @@ test('mappings - flushMappings writes them into a checkpoint right away', async const copy = await copyCore(store, a.core, 'copy') await w.flushMappings([{ from: a.core.key, to: copy.key }]) + t.is(local.length, length, 'loading writes nothing') - t.is(local.length, length + 1, 'one block was written') + await w.write({ touch: true }).flush() + t.is(local.length, length + 1, 'a touch writes one block') t.alike(w.head(), { key: local.key, length: local.length }, 'the head moved onto it') await readAll(w, 51, t, 'the tree reads as before') + + await w.write({ touch: true }).flush() + t.is(local.length, length + 1, 'a touch with nothing new writes nothing') await w.close() await a.core.clear(0, a.core.length) @@ -548,17 +553,39 @@ test('mappings - flushMappings writes them into a checkpoint right away', async // a reader of the head gets the mappings with the table, nothing else needed const again = new Bee(store.namespace('w')) await again.ready() - t.alike(again.head().length, length + 1) await readAll(again, 51, t, 'a reader without the mappings reads through the checkpoint') t.alike((await again.cores({ local: false })).map(hex), [hex(copy.key)]) + + // nor does loading what the checkpoint already holds make a touch write + await again.flushMappings([{ from: a.core.key, to: copy.key }]) + await again.write({ touch: true }).flush() + t.is(again.context.local.length, length + 1, 'known mappings write nothing') await again.close() +}) + +test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = await source(store, 50) + + const other = store.get({ name: 'other' }) + await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) + + // a table entry for the foreign core, as a reference into it would leave + const b = new Bee(store.namespace('b')) + await b.ready() + await b.context.getLocalContext().getCoreOffsetByKey(other.key, b.config) + const batch = b.write() + batch.tryPut(key(0), value(0)) + await batch.flush() - // nothing new to record, nothing written - const idle = new Bee(store.namespace('w')) - await idle.ready() - await idle.flushMappings([{ from: a.core.key, to: copy.key }]) - t.is(idle.context.local.length, length + 1, 'known mappings write nothing') - await idle.close() + const direct = (await b.cores({ local: false })).map(hex) + t.alike(direct, [hex(other.key)], 'the foreign core is listed') + t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') + + await a.close() + await b.close() }) test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { From 711ebd61863ae9f0c83b4b981bdc0d33669f1029 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 12:36:00 +0100 Subject: [PATCH 16/31] cores checks tree node before walking --- index.js | 18 ++++-------------- lib/context.js | 12 +++++++++++- lib/encoding.js | 7 +++++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/index.js b/index.js index cc1acb7..62215f5 100644 --- a/index.js +++ b/index.js @@ -214,12 +214,10 @@ class Hyperbee extends EventEmitter { while (queue.length > 0) { const context = queue.shift() - // a referenced core whose checkpoint cannot be read as a tree is a leaf - try { - if (context !== root) await context.update(config) - } catch (err) { - if (isNotAvailable(err)) throw err - continue + // a referenced core that is not a tree is a leaf + if (context !== root) { + if (!(await context.isTree(config))) continue + await context.update(config) } // index 0 is the core this context is viewing, the rest are referenced cores @@ -436,14 +434,6 @@ class Hyperbee extends EventEmitter { module.exports = Hyperbee -function isNotAvailable(err) { - return ( - err.code === 'BLOCK_NOT_AVAILABLE' || - err.code === 'REQUEST_TIMEOUT' || - err.code === 'REQUEST_CANCELLED' - ) -} - function noop() {} function toEncryptionProvider(encryption) { diff --git a/lib/context.js b/lib/context.js index f5a3fa6..4234800 100644 --- a/lib/context.js +++ b/lib/context.js @@ -3,7 +3,7 @@ const ScopeLock = require('scope-lock') const c = require('compact-encoding') const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors') const { TreeNodePointer } = require('./tree.js') -const { decodeBlock, TYPE_LATEST } = require('./encoding.js') +const { decodeBlock, isBlock, TYPE_LATEST } = require('./encoding.js') const Mappings = require('./mappings.js') const CHECKPOINT_KEY = 'hyperbee/checkpoint' @@ -41,6 +41,16 @@ class CoreContext { // return '[CoreContext]' // } + async isTree(config) { + await this.core.ready() + if (this.core.length === 0) return true + + const buffer = await this.core.get(this.core.length - 1, config) + if (buffer === null) throw BLOCK_NOT_AVAILABLE() + + return isBlock(buffer) + } + async update(config) { await this.core.ready() diff --git a/lib/encoding.js b/lib/encoding.js index e380dd6..b3d1c82 100644 --- a/lib/encoding.js +++ b/lib/encoding.js @@ -15,6 +15,7 @@ exports.TYPE_LATEST = 1 exports.Block = Block1 exports.encodeBlock = encodeBlock exports.decodeBlock = decodeBlock +exports.isBlock = isBlock function encodeBlock(block) { if (block.type === 1) { @@ -33,6 +34,12 @@ function encodeBlock(block) { throw new Error('Unknown block type: ' + block.type) } +function isBlock(buffer) { + const state = { start: 0, end: buffer.byteLength, buffer } + const type = state.end > 0 ? c.uint.decode(state) : 0 + return type === 0 || type === 1 || type === TYPE_COMPAT +} + function decodeBlock(buffer, seq) { const state = { start: 0, end: buffer.byteLength, buffer } const type = state.end > 0 ? c.uint.decode(state) : 0 From a118350cc207989be6c55387a932eab96cfaf3a9 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 13:10:26 +0100 Subject: [PATCH 17/31] pass mappings to a write, load them only from checkpoints A batch written with mappings records them in its checkpoint, writing one block of its own if there is nothing else to write and something is new. Mappings are only ever loaded when a checkpoint is inflated, so flushMappings, addMappings, touch and the mappings option go. Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +- index.js | 35 +--- lib/context.js | 1 - lib/mappings.js | 8 +- lib/write.js | 27 ++- test/map-core.js | 444 ++++++++++++----------------------------------- 6 files changed, 126 insertions(+), 402 deletions(-) diff --git a/README.md b/README.md index d4dcc24..f0719bf 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,6 @@ Options include: autoUpdate: false, // Reload root node when underlying Hypercore is appended to? preload: null, // A function called by ready() after the Hypercore is ready. Can be async. wait: true, // Wait for Hypercore to download blocks - mappings: null, // [{ from, to }]. Read core `from` as core `to`, which holds the same blocks } ``` @@ -148,13 +147,6 @@ With `recursive` the result covers everything the tree can reach through core tables, one checkpoint read per core. It is a superset of what the current root references, as tables only ever grow. -#### `await db.flushMappings(mappings)` - -Load `[{ from, to }]` mappings, first reading those the tree already holds in -its checkpoint, so a following batch only writes what is new. Cores already -opened as `from` are reopened as `to` on their next use, and a tree sitting on -a mapped core moves onto the mapped one. - #### `db.checkout([options])` Returns a new Hyperbee as a view on the underlying [Hypercore][hypercore]. @@ -231,8 +223,9 @@ Options: key: null, // Buffer or null. Key of the Hypercore. autoUpdate: true, // Boolean .Should Hyperbee automatically reflect updates // after each flush()? - touch: false, // Boolean. Write a checkpoint holding the mappings even if - // nothing changed, unless they are in one already + mappings: null, // [{ from, to }]. Read core `from` as core `to`, which holds the + // same blocks, and write them into this checkpoint, even with + // nothing else to write. Readers of the tree learn them from it compat: false, // Boolean. Write blocks compatible with Hyperbee 1? type: encoding.TYPE_LATEST, // Number. Block format to use. inlineValueSize: 1024, // Values smaller than this byte length are diff --git a/index.js b/index.js index 62215f5..9a4ae7c 100644 --- a/index.js +++ b/index.js @@ -8,7 +8,6 @@ const NodeCache = require('./lib/cache.js') const WriteBatch = require('./lib/write.js') const copyChanges = require('./lib/copy.js') const CoreContext = require('./lib/context.js') -const Mappings = require('./lib/mappings.js') const SessionConfig = require('./lib/session-config.js') const Downloader = require('./lib/download.js') const { inflate, inflateValue } = require('./lib/inflate.js') @@ -29,16 +28,13 @@ class Hyperbee extends EventEmitter { timeout = config.timeout, wait = config.wait, trace = config.trace, - mappings = null, - mappingTable = new Mappings(mappings), - core = getRootCore(store, key, getEncryptionProvider, mappingTable), + core = getRootCore(store, key, getEncryptionProvider), context = new CoreContext( store, core, new NodeCache(maxCacheSize), core, - getEncryptionProvider, - mappingTable + getEncryptionProvider ), root = null, view = false, @@ -200,8 +196,6 @@ class Hyperbee extends EventEmitter { this.emit('close') } - // the cores the tree reads from, mapped. Recursive walks the core tables of - // every core reached, so it covers what the tree can reach through them async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) const node = await this.bootstrap(config) @@ -236,25 +230,6 @@ class Hyperbee extends EventEmitter { return cores } - // loads the mappings, once those the tree already holds in its checkpoint - // are known, so only what is new is written by the next batch. A tree - // sitting on a core that just got mapped moves onto the mapped core - async flushMappings(mappings) { - await this.bootstrap(this.config) - this.context.mappings.add(mappings) - - const previous = this.context - const mapped = previous.mappings.resolve(previous.core.key) - if (b4a.equals(mapped, previous.core.key)) return - - this.context = previous.getContextByKey(mapped) - - const root = this.root - if (root !== null && root !== EMPTY && root.context === previous && root.core === 0) { - this.root = this.context.createTreeNode(0, root.seq, 0, false, null) - } - } - createReadStream(options) { return new RangeStream(this, options) } @@ -347,9 +322,6 @@ class Hyperbee extends EventEmitter { } } - // the table and mappings of a root come from the checkpoint its own block - // names, so nothing recorded up to the root is missed by a read that meets - // an older block first async bootstrap(config) { if (!this.root) await this.ready() if (this.unbatch) await this._rollback(config) @@ -441,8 +413,7 @@ function toEncryptionProvider(encryption) { return () => null } -function getRootCore(store, key, getEncryptionProvider, mappings) { - if (key) key = mappings.resolve(key) +function getRootCore(store, key, getEncryptionProvider) { const encryption = getEncryptionProvider(key) return key diff --git a/lib/context.js b/lib/context.js index 4234800..03691df 100644 --- a/lib/context.js +++ b/lib/context.js @@ -209,7 +209,6 @@ class CoreContext { return this.cores.length } - // the table keeps keys as they were referenced, a core is mapped when opened getCore(index) { if (index === 0) return this.core if (index > this.cores.length) throw new Error('Bad core index: ' + index) diff --git a/lib/mappings.js b/lib/mappings.js index 4b4dcd3..a063786 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -1,22 +1,16 @@ const b4a = require('b4a') -// a mapping redirects every read of a core to another core holding the same -// blocks. Mappings are immutable facts, shared by every context of a tree and -// persisted next to the core table on the next checkpoint class Mappings { - constructor(mappings = null) { + constructor() { this.known = new Map() this.dirty = false - if (mappings !== null) this.add(mappings) } - // the mapping of a key as far as it is known, the key itself otherwise resolve(key) { const known = this.known.get(b4a.toString(key, 'hex')) return known === undefined ? key : known } - // what is already in a checkpoint of our own core needs no writing add(mappings, { persisted = false } = {}) { for (const { from, to } of mappings) { if (b4a.equals(from, to)) continue diff --git a/lib/write.js b/lib/write.js index 33be6a2..e5804c1 100644 --- a/lib/write.js +++ b/lib/write.js @@ -14,7 +14,7 @@ module.exports = class WriteBatch { const { length = -1, key = null, - touch = false, + mappings = null, autoUpdate = true, compat = false, type = compat ? TYPE_COMPAT : TYPE_LATEST, @@ -40,7 +40,8 @@ module.exports = class WriteBatch { this.applied = 0 this.root = null this.ops = [] - this.touch = touch + this.mappings = mappings + this.touched = false } tryPut(key, value) { @@ -92,12 +93,6 @@ module.exports = class WriteBatch { const ops = this.ops - // a touch with nothing to write reads nothing for it either - if (this.touch && ops.length === 0 && !this.tree.context.mappings.dirty) { - await this.close() - return - } - const root = await this.tree.bootstrap(this.config) const length = this._getLength(root) @@ -118,11 +113,14 @@ module.exports = class WriteBatch { if (op.applied) this.applied++ } - // a touch writes the mappings the tree holds into a checkpoint by - // rewriting the root as it is, unless they are in one already - if (this.touch && !this.root.changed && this.tree.context.mappings.dirty) { - await retainAndInflate(this.root, this.snapshot, this.config) - this.root.mark() + if (this.mappings !== null) { + this.tree.context.mappings.add(this.mappings) + + if (!this.root.changed && this.tree.context.mappings.dirty) { + await retainAndInflate(this.root, this.snapshot, this.config) + this.root.mark() + this.touched = true + } } await this._flush() @@ -439,7 +437,7 @@ module.exports = class WriteBatch { const length = context.core.length // if noop and not genesis, bail early - if (this.applied === 0 && this.length > 0 && !this.touch) return + if (this.applied === 0 && this.length > 0 && !this.touched) return if (minValue > -1 && minValue + update.size < this.preferredBlockSize) { // TODO: repack the value into the block @@ -554,7 +552,6 @@ module.exports = class WriteBatch { await context.core.append(buffers) - // the table written is the table held context.length = context.core.length for (let i = 0; i < batch.length; i++) { diff --git a/test/map-core.js b/test/map-core.js index b8e34b7..dc55e03 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -33,188 +33,132 @@ async function readAll(bee, n, t, msg) { t.pass(msg) } -test('mappings - the viewed core is read through its mapping', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) +// a writer whose tree reaches a, and a copy of a to map it to +async function setup(store, n = 50) { + const a = await source(store, n) - const a = await source(store, 50) - const copy = await copyCore(store, a.core, 'copy') - await a.core.clear(0, a.core.length) + const w = new Bee(store.namespace('w')) + await w.ready() + w.move({ key: a.core.key, length: a.core.length }) + { + const batch = w.write() + batch.tryPut(key(n), value(n)) + await batch.flush() + } + const copy = await copyCore(store, a.core, 'copy') const mappings = [{ from: a.core.key, to: copy.key }] - const b = new Bee(store.namespace('b'), { key: a.core.key, mappings, writable: false }) - await b.ready() - - t.alike(b.core.key, copy.key, 'the view sits on the mapped core') - await readAll(b, 50, t, 'every entry reads through the mapping') - t.alike((await b.cores()).map(hex), [hex(copy.key)], 'cores() lists the mapped core only') - await b.close() -}) + return { a, w, copy, mappings } +} -test('mappings - the references of a writer resolve to the mapped core', async function (t) { +test('mappings - a write with mappings records them, readers read through them', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) - const a = await source(store, 50) - const copy = await copyCore(store, a.core, 'copy') - const mappings = [{ from: a.core.key, to: copy.key }] - - const w = new Bee(store.namespace('w'), { mappings }) - await w.ready() - w.move({ key: a.core.key, length: a.core.length }) - t.alike(w.core.key, copy.key, 'moving onto the original lands on the mapping') + const { a, w, copy, mappings } = await setup(store) + const local = w.context.local + const length = local.length - const batch = w.write() - for (let i = 50; i < 60; i++) batch.tryPut(key(i), value(i)) - await batch.flush() + await w.write({ mappings }).flush() + t.is(local.length, length + 1, 'with nothing else to write, one block') + t.alike(w.head(), { key: local.key, length: local.length }, 'the head moved onto it') - const referenced = (await w.cores({ local: false })).map(hex) - t.absent(referenced.includes(hex(a.core.key)), 'the original is not among the cores read') + await w.write({ mappings }).flush() + t.is(local.length, length + 1, 'mappings already written write nothing') await w.close() await a.core.clear(0, a.core.length) + const r = new Bee(store.namespace('w'), { writable: false }) + await r.ready() + await readAll(r, 51, t, 'a reader reads through the mappings of the checkpoint') + t.alike((await r.cores({ local: false })).map(hex), [hex(copy.key)], 'and lists the copy only') + await r.close() + const again = new Bee(store.namespace('w')) await again.ready() - t.alike( - (await again.cores({ local: false })).map(hex), - referenced, - 'the persisted mappings resolve the table' - ) - await readAll(again, 60, t, 'a reader without the mappings reads everything via the checkpoint') + await again.write({ mappings }).flush() + t.is(again.context.local.length, length + 1, 'nor after a reopen') await again.close() }) -test('mappings - an existing table is remapped on load and persisted on the next flush', async function (t) { +test('mappings - a write with ops carries the mappings in its checkpoint', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) - const a = await source(store, 50) + const { a, w, copy, mappings } = await setup(store) + const length = w.context.local.length - // a writer that referenced the original before any mapping existed - const w = new Bee(store.namespace('w')) - await w.ready() - w.move({ key: a.core.key, length: a.core.length }) - { - const batch = w.write() - for (let i = 50; i < 60; i++) batch.tryPut(key(i), value(i)) - await batch.flush() - } - t.ok( + const batch = w.write({ mappings }) + batch.tryPut(key(51), value(51)) + await batch.flush() + t.is(w.context.local.length, length + 1, 'one block for both') + t.absent( (await w.cores({ local: false })).some((k) => b4a.equals(k, a.core.key)), - 'table names the original' + 'the original is not among the cores read' ) await w.close() - const copy = await copyCore(store, a.core, 'copy') - const mappings = [{ from: a.core.key, to: copy.key }] - - const mapped = new Bee(store.namespace('w'), { mappings }) - await mapped.ready() - const cores = (await mapped.cores({ local: false })).map(hex) - t.ok(cores.includes(hex(copy.key)), 'the loaded table is remapped') - t.absent(cores.includes(hex(a.core.key)), 'the original is gone from the loaded table') - await readAll(mapped, 60, t, 'reads go through the remapped table') - - { - const batch = mapped.write() - batch.tryPut(key(60), value(60)) - await batch.flush() - } - await mapped.close() - await a.core.clear(0, a.core.length) - const plain = new Bee(store.namespace('w')) - await plain.ready() - const persisted = (await plain.cores({ local: false })).map(hex) - t.ok(persisted.includes(hex(copy.key)), 'the mappings were persisted by the flush') - t.absent(persisted.includes(hex(a.core.key)), 'without them given, the original is still gone') - await readAll(plain, 61, t, 'a reader without the mappings reads through the persisted ones') - await plain.close() + const r = new Bee(store.namespace('w'), { writable: false }) + await r.ready() + await readAll(r, 52, t, 'a reader reads everything through the copy') + t.ok((await r.cores()).some((k) => b4a.equals(k, copy.key))) + await r.close() }) -test('mappings - an undo below the record reads through the mappings it is given', async function (t) { +test('mappings - an undo below the record reads through the mappings of the head', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) - const a = await source(store, 50) - - const w = new Bee(store.namespace('w')) - await w.ready() - w.move({ key: a.core.key, length: a.core.length }) - for (let i = 50; i < 55; i++) { + const { a, w, copy, mappings } = await setup(store) + for (let i = 51; i < 55; i++) { const batch = w.write() batch.tryPut(key(i), value(i)) await batch.flush() } + await w.write({ mappings }).flush() await w.close() - const copy = await copyCore(store, a.core, 'copy') - const mappings = [{ from: a.core.key, to: copy.key }] - - const mapped = new Bee(store.namespace('w'), { mappings }) - await mapped.ready() - { - const batch = mapped.write() - batch.tryPut(key(55), value(55)) - await batch.flush() - } - await mapped.close() - await a.core.clear(0, a.core.length) - // the record lives in the last checkpoint, a root below it has a table - // without it: the mappings have to be given, as they are on reopen - const again = new Bee(store.namespace('w'), { mappings }) - await again.ready() - const old = again.undo(3) - await readAll(old, 53, t, 'an undone root reads through the mappings') - t.is(await old.get(key(53)), null, 'the undone writes are gone') - t.alike( - (await old.cores({ local: false })).map(hex), - [hex(copy.key)], - 'the undone view resolves the original to the copy' - ) + const r = new Bee(store.namespace('w'), { writable: false }) + await r.ready() + await readAll(r, 55, t, 'the head reads through the copy') + + const old = r.undo(4) + await readAll(old, 52, t, 'an undone root reads through the mappings the head carried') + t.is(await old.get(key(54)), null, 'the undone writes are gone') + t.alike((await old.cores({ local: false })).map(hex), [hex(copy.key)]) await old.close() - await again.close() + await r.close() }) test('mappings - a mapping met in another writer is recorded locally', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) - const a = await source(store, 50) - const copy = await copyCore(store, a.core, 'copy') - const mappings = [{ from: a.core.key, to: copy.key }] - - // b writes on top of a with the mapping, recording it in its checkpoint - const b = new Bee(store.namespace('b'), { mappings }) - await b.ready() - b.move({ key: a.core.key, length: a.core.length }) - { - const batch = b.write() - batch.tryPut(key(50), value(50)) - await batch.flush() - } + const { a, w, mappings } = await setup(store) + await w.write({ mappings }).flush() await a.core.clear(0, a.core.length) - // c is never given the mapping and only ever sees b's head + // c is never given the mapping and only ever sees w's head const c = new Bee(store.namespace('c')) await c.ready() - c.move({ key: b.context.local.key, length: b.context.local.length }) + c.move({ key: w.context.local.key, length: w.context.local.length }) { const batch = c.write() batch.tryPut(key(51), value(51)) await batch.flush() } - await readAll(c, 52, t, 'c reads a through the mapping b recorded') + await readAll(c, 52, t, 'c reads a through the mapping w recorded') await c.close() - await b.close() + await w.close() - const again = new Bee(store.namespace('c')) + const again = new Bee(store.namespace('c'), { writable: false }) await again.ready() await readAll(again, 52, t, "a reopen of c reads a through c's own checkpoint") t.absent( @@ -228,44 +172,21 @@ test('mappings - a remote reads the whole tree after the original data is gone', const origin = new Corestore(await t.tmp()) t.teardown(() => origin.close()) - const a = await source(origin, 200) - - const w = new Bee(origin.namespace('w')) - await w.ready() - w.move({ key: a.core.key, length: a.core.length }) - for (let i = 200; i < 220; i++) { - const batch = w.write() - batch.tryPut(key(i), value(i)) - await batch.flush() - } + const { a, w, copy, mappings } = await setup(origin, 200) const wKey = w.context.local.key + const before = w.context.local.length + await w.write({ mappings }).flush() await w.close() - const copy = await copyCore(origin, a.core, 'copy') - const mappings = [{ from: a.core.key, to: copy.key }] - - const mapped = new Bee(origin.namespace('w'), { mappings }) - await mapped.ready() - { - const batch = mapped.write() - batch.tryPut(key(220), value(220)) - await batch.flush() - } - await mapped.close() - await a.core.clear(0, a.core.length) - t.is(await a.core.has(0), false, 'the original holds no data any more') - t.is(copy.contiguousLength, copy.length, 'the copy still holds every block') - // a remote that only ever sees the origin const remote = new Corestore(await t.tmp()) t.teardown(() => remote.close()) const stop = replicateStores(origin, remote) - const reader = new Bee(remote.session(), { key: wKey, mappings, writable: false }) + const reader = new Bee(remote.session(), { key: wKey, writable: false }) await reader.ready() await reader.core.update({ wait: true }) - t.is(reader.core.length, 21, 'the remote learnt the head') let count = 0 for await (const { key: k, value: v } of reader.createReadStream()) { @@ -273,8 +194,7 @@ test('mappings - a remote reads the whole tree after the original data is gone', if (!expected) return t.fail('bad entry ' + count) count++ } - t.is(count, 221, 'the whole tree was read through the mapping') - await readAll(reader, 221, t, 'every point read resolves too') + t.is(count, 201, 'the whole tree was read through the mapping') t.alike( (await reader.cores()).map(hex).sort(), [hex(wKey), hex(copy.key)].sort(), @@ -285,15 +205,12 @@ test('mappings - a remote reads the whole tree after the original data is gone', 'the remote never asked for the original' ) await reader.close() - await stop() - // a second remote never given the mappings: a head written before they were - // recorded runs into the cleared original, the head that carries them does not + // a head from before the record has a table without the mappings const other = new Corestore(await t.tmp()) t.teardown(() => other.close()) const stopOther = replicateStores(origin, other) - { const core = other.get({ key: wKey }) await core.update({ wait: true }) @@ -301,74 +218,47 @@ test('mappings - a remote reads the whole tree after the original data is gone', await core.close() } - const before = new Bee(other.session(), { key: wKey, writable: false }) - await before.ready() - before.move({ length: 20 }) + const old = new Bee(other.session(), { key: wKey, writable: false }) + await old.ready() + old.move({ length: before }) await t.exception( - before.get(key(0), { wait: false }), + old.get(key(0), { wait: false }), /BLOCK_NOT_AVAILABLE/, - 'before the record, a reader without the mappings hits missing data' + 'before the record the original is read' ) - await before.close() - - const after = new Bee(other.session(), { key: wKey, writable: false }) - await after.ready() - await readAll(after, 221, t, 'at the recorded head, the same reader resolves everything') - await after.close() - + await old.close() await stopOther() }) -test('mappings - move, checkout and write onto an original resolve synchronously', async function (t) { +test('mappings - move, checkout and write onto an original resolve once known', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) - const a = await source(store, 50) - const copy = await copyCore(store, a.core, 'copy') + const { a, w, copy, mappings } = await setup(store) + await w.write({ mappings }).flush() await a.core.clear(0, a.core.length) - const mappings = [{ from: a.core.key, to: copy.key }] - const w = new Bee(store.namespace('w'), { mappings }) - await w.ready() - - w.move({ key: a.core.key, length: a.core.length }) - t.alike(w.core.key, copy.key, 'the bee sits on the mapped core right after the move') - t.alike(w.head().key, copy.key, 'the head is on the mapped core') - await readAll(w, 50, t, 'reads go through the mapping') - const view = w.checkout({ key: a.core.key, length: a.core.length }) - t.alike(view.head().key, copy.key, 'a checkout onto the original sits on the mapped core') + t.alike(view.head().key, copy.key, 'a checkout onto the original sits on the copy') await readAll(view, 50, t, 'and reads through it') await view.close() const batch = w.write({ key: a.core.key, length: a.core.length }) batch.tryPut(key(50), value(50)) await batch.flush() - await readAll(w, 51, t, 'a batch keyed by the original built on the mapped core') - t.absent( - (await w.cores()).some((k) => b4a.equals(k, a.core.key)), - 'the original is not among the cores read' - ) + await readAll(w, 51, t, 'a batch keyed by the original builds on the copy') + + w.move({ key: a.core.key, length: a.core.length }) + t.alike(w.core.key, copy.key, 'a move onto the original lands on the copy') await w.close() }) -test('mappings - added after open, a read in flight survives its core being remapped', async function (t) { +test('mappings - a read in flight survives its core being remapped', async function (t) { const origin = new Corestore(await t.tmp()) t.teardown(() => origin.close()) - const a = await source(origin, 50) - const copy = await copyCore(origin, a.core, 'copy') - - const w = new Bee(origin.namespace('w')) - await w.ready() - w.move({ key: a.core.key, length: a.core.length }) - { - const batch = w.write() - batch.tryPut(key(50), value(50)) - await batch.flush() - } + const { a, w, copy, mappings } = await setup(origin) const wKey = w.context.local.key - await w.close() // the remote holds the writer but nothing of the core it references const remote = new Corestore(await t.tmp()) @@ -385,28 +275,26 @@ test('mappings - added after open, a read in flight survives its core being rema const reader = new Bee(remote.session(), { key: wKey, writable: false }) await reader.ready() - // a read that has to wait for a peer to serve the referenced core const read = reader.get(key(0)) for (let i = 0; i < 100 && !reader.context.opened[0]; i++) await immediate() const stale = reader.context.opened[0] t.alike(stale.key, a.core.key, 'the read opened the core as referenced') - await reader.flushMappings([{ from: a.core.key, to: copy.key }]) - const hc = reader.context.getCore(1) - t.alike(hc.key, copy.key, 'the next open resolves to the mapped core') - t.absent(stale.closing, 'the superseded session is left to the reads holding it') - + // the writer records the mapping, the reader moves onto that head + await w.write({ mappings }).flush() const stop = replicateStores(origin, remote) - const node = await read - t.alike(node.value, value(0), 'the read issued before the remap still resolved') + await reader.core.update({ wait: true }) + reader.move({ length: reader.core.length }) + await reader.get(key(50)) + + t.alike(reader.context.getCore(1).key, copy.key, 'the next open resolves to the copy') + t.absent(stale.closing, 'the superseded session is left to the reads holding it') + t.alike((await read).value, value(0), 'the read issued before the remap still resolved') await readAll(reader, 51, t, 'later reads go through the mapping') - t.alike( - (await reader.cores()).map(hex).sort(), - [hex(wKey), hex(copy.key)].sort(), - 'only the writer and the mapped core are listed' - ) + await reader.close() + await w.close() await stop() }) @@ -442,24 +330,26 @@ test('mappings - cores({ recursive: true }) walks the tables of every core reach await batch.flush() } - const direct = (await c.cores({ local: false })).map(hex).sort() - const reachable = (await c.cores({ local: false, recursive: true })).map(hex).sort() - t.alike(direct, [hex(b.context.local.key)], "c's table names b only") t.alike( - reachable, + (await c.cores({ local: false })).map(hex), + [hex(b.context.local.key)], + "c's table names b only" + ) + t.alike( + (await c.cores({ local: false, recursive: true })).map(hex).sort(), [hex(a.core.key), hex(b.context.local.key)].sort(), 'recursively a is reached through b' ) - // the flow of a migration: find what is reachable, copy it, map it in one go + // find what is reachable, copy it, record it const copy = await copyCore(store, a.core, 'copy') await a.core.clear(0, a.core.length) - await c.flushMappings([{ from: a.core.key, to: copy.key }]) + await c.write({ mappings: [{ from: a.core.key, to: copy.key }] }).flush() t.alike( (await c.cores({ local: false, recursive: true })).map(hex).sort(), [hex(copy.key), hex(b.context.local.key)].sort(), - 'the walk reports the mapped core' + 'the walk reports the copy' ) for (let i = 0; i < 50; i++) { const node = await c.get(key(i)) @@ -471,129 +361,10 @@ test('mappings - cores({ recursive: true }) walks the tables of every core reach await b.close() }) -test('mappings - added after open, a tree sitting on the mapped core moves onto the copy', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) - - const a = await source(store, 50) - const copy = await copyCore(store, a.core, 'copy') - - const b = new Bee(store.namespace('b'), { key: a.core.key, writable: false }) - await b.ready() - t.alike(b.core.key, a.core.key, 'the tree sits on the original') - await readAll(b, 50, t, 'and reads it') - - await a.core.clear(0, a.core.length) - await b.flushMappings([{ from: a.core.key, to: copy.key }]) - - t.alike(b.core.key, copy.key, 'the tree moved onto the copy') - t.alike(b.head().key, copy.key, 'with its head') - t.alike((await b.cores()).map(hex), [hex(copy.key)], 'and lists it') - await readAll(b, 50, t, 'reads go to the copy') - await b.close() -}) - -test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) - - const a = await source(store, 50) - - const other = store.get({ name: 'other' }) - await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) - - // a table entry for the foreign core, as a reference into it would leave - const b = new Bee(store.namespace('b')) - await b.ready() - await b.context.getLocalContext().getCoreOffsetByKey(other.key, b.config) - const batch = b.write() - batch.tryPut(key(0), value(0)) - await batch.flush() - - const direct = (await b.cores({ local: false })).map(hex) - t.alike(direct, [hex(other.key)], 'the foreign core is listed') - t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') - - await a.close() - await b.close() -}) - -test('mappings - a touch writes the mappings loaded into a checkpoint', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) - - const a = await source(store, 50) - - const w = new Bee(store.namespace('w')) - await w.ready() - w.move({ key: a.core.key, length: a.core.length }) - { - const batch = w.write() - batch.tryPut(key(50), value(50)) - await batch.flush() - } - const local = w.context.local - const length = local.length - - const copy = await copyCore(store, a.core, 'copy') - await w.flushMappings([{ from: a.core.key, to: copy.key }]) - t.is(local.length, length, 'loading writes nothing') - - await w.write({ touch: true }).flush() - t.is(local.length, length + 1, 'a touch writes one block') - t.alike(w.head(), { key: local.key, length: local.length }, 'the head moved onto it') - await readAll(w, 51, t, 'the tree reads as before') - - await w.write({ touch: true }).flush() - t.is(local.length, length + 1, 'a touch with nothing new writes nothing') - await w.close() - - await a.core.clear(0, a.core.length) - - // a reader of the head gets the mappings with the table, nothing else needed - const again = new Bee(store.namespace('w')) - await again.ready() - await readAll(again, 51, t, 'a reader without the mappings reads through the checkpoint') - t.alike((await again.cores({ local: false })).map(hex), [hex(copy.key)]) - - // nor does loading what the checkpoint already holds make a touch write - await again.flushMappings([{ from: a.core.key, to: copy.key }]) - await again.write({ touch: true }).flush() - t.is(again.context.local.length, length + 1, 'known mappings write nothing') - await again.close() -}) - -test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) - - const a = await source(store, 50) - - const other = store.get({ name: 'other' }) - await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) - - // a table entry for the foreign core, as a reference into it would leave - const b = new Bee(store.namespace('b')) - await b.ready() - await b.context.getLocalContext().getCoreOffsetByKey(other.key, b.config) - const batch = b.write() - batch.tryPut(key(0), value(0)) - await batch.flush() - - const direct = (await b.cores({ local: false })).map(hex) - t.alike(direct, [hex(other.key)], 'the foreign core is listed') - t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') - - await a.close() - await b.close() -}) - test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { const store = new Corestore(await t.tmp()) t.teardown(() => store.close()) - const a = await source(store, 50) - const other = store.get({ name: 'other' }) await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) @@ -609,7 +380,6 @@ test('mappings - cores({ recursive: true }) does not descend into a core that is t.alike(direct, [hex(other.key)], 'the foreign core is listed') t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') - await a.close() await b.close() }) From ad0e4635d4b47a2e725200888522699d5ae6ca2b Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 13:50:18 +0100 Subject: [PATCH 18/31] remove touched flag --- lib/write.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/write.js b/lib/write.js index e5804c1..5c46e45 100644 --- a/lib/write.js +++ b/lib/write.js @@ -41,7 +41,6 @@ module.exports = class WriteBatch { this.root = null this.ops = [] this.mappings = mappings - this.touched = false } tryPut(key, value) { @@ -119,7 +118,6 @@ module.exports = class WriteBatch { if (!this.root.changed && this.tree.context.mappings.dirty) { await retainAndInflate(this.root, this.snapshot, this.config) this.root.mark() - this.touched = true } } @@ -437,7 +435,8 @@ module.exports = class WriteBatch { const length = context.core.length // if noop and not genesis, bail early - if (this.applied === 0 && this.length > 0 && !this.touched) return + const mapping = this.mappings !== null && context.mappings.dirty + if (this.applied === 0 && this.length > 0 && !mapping) return if (minValue > -1 && minValue + update.size < this.preferredBlockSize) { // TODO: repack the value into the block From ab7e811971d9b9277af9dc56ed7a94a47fd4e2f2 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 15:04:10 +0100 Subject: [PATCH 19/31] only load checkpoint when metadata changed --- index.js | 11 ++--------- lib/context.js | 2 +- lib/write.js | 7 ++----- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/index.js b/index.js index 9a4ae7c..429e3bd 100644 --- a/index.js +++ b/index.js @@ -199,6 +199,7 @@ class Hyperbee extends EventEmitter { async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) const node = await this.bootstrap(config) + if (node !== null) await node.context.getBlock(node.seq, 0, config, null) const cores = [] const seen = new Set() @@ -325,15 +326,7 @@ class Hyperbee extends EventEmitter { async bootstrap(config) { if (!this.root) await this.ready() if (this.unbatch) await this._rollback(config) - if (this.root === EMPTY) return null - - const root = this.root - if (root.context.length < root.seq + 1) { - const block = await root.context.getBlock(root.seq, 0, config, null) - await root.context._inflateCheckpoint(root.seq + 1, block, config, false) - } - - return root + return this.root === EMPTY ? null : this.root } async _rollback(config) { diff --git a/lib/context.js b/lib/context.js index 03691df..d6700f3 100644 --- a/lib/context.js +++ b/lib/context.js @@ -106,7 +106,7 @@ class CoreContext { const checkpoint = block.checkpoint let cores = null - if (checkpoint > 0) { + if (checkpoint > 0 && checkpoint !== this.checkpoint) { const seq = checkpoint - 1 const buffer = await this.core.get(seq, config) if (buffer === null) throw BLOCK_NOT_AVAILABLE() diff --git a/lib/write.js b/lib/write.js index 5c46e45..070c032 100644 --- a/lib/write.js +++ b/lib/write.js @@ -113,12 +113,9 @@ module.exports = class WriteBatch { } if (this.mappings !== null) { + if (!this.root.changed) await retainAndInflate(this.root, this.snapshot, this.config) this.tree.context.mappings.add(this.mappings) - - if (!this.root.changed && this.tree.context.mappings.dirty) { - await retainAndInflate(this.root, this.snapshot, this.config) - this.root.mark() - } + if (!this.root.changed && this.tree.context.mappings.dirty) this.root.mark() } await this._flush() From e90d698589d911fc11bb5523c5412e584c1a57f7 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 16:14:59 +0100 Subject: [PATCH 20/31] bump invalidation pointer on remap --- index.js | 5 ++++- lib/copy.js | 2 ++ lib/write.js | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 429e3bd..ff0674c 100644 --- a/index.js +++ b/index.js @@ -199,7 +199,10 @@ class Hyperbee extends EventEmitter { async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) const node = await this.bootstrap(config) - if (node !== null) await node.context.getBlock(node.seq, 0, config, null) + if (node !== null) { + const block = await node.context.getBlock(node.seq, 0, config, null) + await node.context._inflateCheckpoint(node.seq + 1, block, config, false) + } const cores = [] const seen = new Set() diff --git a/lib/copy.js b/lib/copy.js index df43935..a7589ac 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -64,6 +64,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { if (local.changed || local.mappings.dirty) { local.checkpoint = base + blocks.length + if (local.mappings.dirty) local.invalidation++ blocks[blocks.length - 1].metadata = local.flush() } @@ -71,6 +72,7 @@ module.exports = async function copyChanges(tree, changes, config, map) { for (let i = 0; i < blocks.length; i++) { blocks[i].checkpoint = local.checkpoint + blocks[i].invalidation = local.invalidation buffers[i] = encodeBlock(blocks[i]) } diff --git a/lib/write.js b/lib/write.js index 070c032..3076f75 100644 --- a/lib/write.js +++ b/lib/write.js @@ -536,11 +536,13 @@ module.exports = class WriteBatch { // TODO: make this transaction safe if (context.changed || context.mappings.dirty) { context.checkpoint = context.core.length + blocks.length + if (context.mappings.dirty) context.invalidation++ blocks[blocks.length - 1].metadata = context.flush() } for (let i = 0; i < blocks.length; i++) { blocks[i].checkpoint = context.checkpoint + blocks[i].invalidation = context.invalidation buffers[i] = encodeBlock(blocks[i]) } From f5ee4d8121f7493e8adc34ed3a7eacdb88ecdd73 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 17:02:03 +0100 Subject: [PATCH 21/31] remove isTree logic --- index.js | 6 +----- lib/context.js | 12 +----------- lib/encoding.js | 7 ------- test/map-core.js | 22 ---------------------- 4 files changed, 2 insertions(+), 45 deletions(-) diff --git a/index.js b/index.js index ff0674c..4b62e31 100644 --- a/index.js +++ b/index.js @@ -212,11 +212,7 @@ class Hyperbee extends EventEmitter { while (queue.length > 0) { const context = queue.shift() - // a referenced core that is not a tree is a leaf - if (context !== root) { - if (!(await context.isTree(config))) continue - await context.update(config) - } + if (context !== root) await context.update(config) // index 0 is the core this context is viewing, the rest are referenced cores for (let i = 0; i <= context.cores.length; i++) { diff --git a/lib/context.js b/lib/context.js index d6700f3..946270f 100644 --- a/lib/context.js +++ b/lib/context.js @@ -3,7 +3,7 @@ const ScopeLock = require('scope-lock') const c = require('compact-encoding') const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors') const { TreeNodePointer } = require('./tree.js') -const { decodeBlock, isBlock, TYPE_LATEST } = require('./encoding.js') +const { decodeBlock, TYPE_LATEST } = require('./encoding.js') const Mappings = require('./mappings.js') const CHECKPOINT_KEY = 'hyperbee/checkpoint' @@ -41,16 +41,6 @@ class CoreContext { // return '[CoreContext]' // } - async isTree(config) { - await this.core.ready() - if (this.core.length === 0) return true - - const buffer = await this.core.get(this.core.length - 1, config) - if (buffer === null) throw BLOCK_NOT_AVAILABLE() - - return isBlock(buffer) - } - async update(config) { await this.core.ready() diff --git a/lib/encoding.js b/lib/encoding.js index b3d1c82..e380dd6 100644 --- a/lib/encoding.js +++ b/lib/encoding.js @@ -15,7 +15,6 @@ exports.TYPE_LATEST = 1 exports.Block = Block1 exports.encodeBlock = encodeBlock exports.decodeBlock = decodeBlock -exports.isBlock = isBlock function encodeBlock(block) { if (block.type === 1) { @@ -34,12 +33,6 @@ function encodeBlock(block) { throw new Error('Unknown block type: ' + block.type) } -function isBlock(buffer) { - const state = { start: 0, end: buffer.byteLength, buffer } - const type = state.end > 0 ? c.uint.decode(state) : 0 - return type === 0 || type === 1 || type === TYPE_COMPAT -} - function decodeBlock(buffer, seq) { const state = { start: 0, end: buffer.byteLength, buffer } const type = state.end > 0 ? c.uint.decode(state) : 0 diff --git a/test/map-core.js b/test/map-core.js index dc55e03..9831182 100644 --- a/test/map-core.js +++ b/test/map-core.js @@ -361,28 +361,6 @@ test('mappings - cores({ recursive: true }) walks the tables of every core reach await b.close() }) -test('mappings - cores({ recursive: true }) does not descend into a core that is not a tree', async function (t) { - const store = new Corestore(await t.tmp()) - t.teardown(() => store.close()) - - const other = store.get({ name: 'other' }) - await other.append([b4a.from('not a tree block'), b4a.from('nor this')]) - - // a table entry for the foreign core, as a reference into it would leave - const b = new Bee(store.namespace('b')) - await b.ready() - await b.context.getLocalContext().getCoreOffsetByKey(other.key, b.config) - const batch = b.write() - batch.tryPut(key(0), value(0)) - await batch.flush() - - const direct = (await b.cores({ local: false })).map(hex) - t.alike(direct, [hex(other.key)], 'the foreign core is listed') - t.alike((await b.cores({ local: false, recursive: true })).map(hex), direct, 'and walked past') - - await b.close() -}) - function replicateStores(a, b) { const s1 = a.replicate(true) const s2 = b.replicate(false) From 5a16349c8027cee3f99a27217e80151ec2bf25e5 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 17:19:28 +0100 Subject: [PATCH 22/31] review by @mafintsoh --- lib/context.js | 2 +- lib/mappings.js | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/context.js b/lib/context.js index 946270f..4fabf88 100644 --- a/lib/context.js +++ b/lib/context.js @@ -205,7 +205,7 @@ class CoreContext { const key = this.mappings.resolve(this.cores[index - 1].key) const opened = this.opened[index - 1] - if (opened !== null && b4a.equals(opened.key, key)) return opened + if (opened !== null && (key === opened.key || b4a.equals(opened.key, key))) return opened this.opened[index - 1] = this._openCore(key) return this.opened[index - 1] diff --git a/lib/mappings.js b/lib/mappings.js index a063786..2552176 100644 --- a/lib/mappings.js +++ b/lib/mappings.js @@ -4,9 +4,11 @@ class Mappings { constructor() { this.known = new Map() this.dirty = false + this._flushed = null } resolve(key) { + if (this.known.size === 0) return key const known = this.known.get(b4a.toString(key, 'hex')) return known === undefined ? key : known } @@ -17,17 +19,21 @@ class Mappings { const hex = b4a.toString(from, 'hex') if (this.known.has(hex)) continue this.known.set(hex, to) + this._flushed = null if (!persisted) this.dirty = true } } flush() { this.dirty = false + if (this.known.size === 0) return null - const mappings = [] - for (const [hex, to] of this.known) mappings.push({ from: b4a.from(hex, 'hex'), to }) + if (this._flushed === null) { + this._flushed = [] + for (const [hex, to] of this.known) this._flushed.push({ from: b4a.from(hex, 'hex'), to }) + } - return mappings.length === 0 ? null : mappings + return this._flushed } } From 590e49878153c29f5d2c1efb123dc57162e337c6 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 17:58:48 +0100 Subject: [PATCH 23/31] more review --- lib/context.js | 7 ++++++- lib/copy.js | 3 +-- lib/write.js | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/context.js b/lib/context.js index 4fabf88..d36a98b 100644 --- a/lib/context.js +++ b/lib/context.js @@ -103,7 +103,7 @@ class CoreContext { const block = decodeBlock(buffer, seq) cores = block.metadata ? block.metadata.cores : [] - if (cores.length > 0 && block.metadata.mappings !== null) { + if (block.metadata && block.metadata.mappings !== null) { const persisted = b4a.equals(this.core.key, this.local.key) this.mappings.add(block.metadata.mappings, { persisted }) } @@ -287,7 +287,12 @@ class CoreContext { return ctx } + needsFlush() { + return this.changed || this.mappings.dirty + } + flush() { + if (this.mappings.dirty) this.invalidation++ this.changed = false return { cores: this.cores, diff --git a/lib/copy.js b/lib/copy.js index a7589ac..02b8e27 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -62,9 +62,8 @@ module.exports = async function copyChanges(tree, changes, config, map) { } } - if (local.changed || local.mappings.dirty) { + if (local.needsFlush()) { local.checkpoint = base + blocks.length - if (local.mappings.dirty) local.invalidation++ blocks[blocks.length - 1].metadata = local.flush() } diff --git a/lib/write.js b/lib/write.js index 3076f75..4a8cb9e 100644 --- a/lib/write.js +++ b/lib/write.js @@ -534,9 +534,8 @@ module.exports = class WriteBatch { } // TODO: make this transaction safe - if (context.changed || context.mappings.dirty) { + if (context.needsFlush()) { context.checkpoint = context.core.length + blocks.length - if (context.mappings.dirty) context.invalidation++ blocks[blocks.length - 1].metadata = context.flush() } From 7ebcbf101d59aad9cba5191b070d8abd56a8299e Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 18:04:08 +0100 Subject: [PATCH 24/31] remove context length change --- lib/write.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/write.js b/lib/write.js index 4a8cb9e..5fd7e0c 100644 --- a/lib/write.js +++ b/lib/write.js @@ -549,8 +549,6 @@ module.exports = class WriteBatch { await context.core.append(buffers) - context.length = context.core.length - for (let i = 0; i < batch.length; i++) { const update = batch[i] From 4c1eac7b2e458f00d1a01028303479d1daaf072d Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 18:33:34 +0100 Subject: [PATCH 25/31] simplify and remove --- lib/copy.js | 1 - lib/write.js | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/copy.js b/lib/copy.js index 02b8e27..a2d55bb 100644 --- a/lib/copy.js +++ b/lib/copy.js @@ -76,7 +76,6 @@ module.exports = async function copyChanges(tree, changes, config, map) { } await local.core.append(buffers) - local.length = local.core.length return changes.length diff --git a/lib/write.js b/lib/write.js index 5fd7e0c..5912c91 100644 --- a/lib/write.js +++ b/lib/write.js @@ -113,9 +113,10 @@ module.exports = class WriteBatch { } if (this.mappings !== null) { - if (!this.root.changed) await retainAndInflate(this.root, this.snapshot, this.config) - this.tree.context.mappings.add(this.mappings) - if (!this.root.changed && this.tree.context.mappings.dirty) this.root.mark() + const local = this.tree.context.getLocalContext() + await local.update(this.config) + context.mappings.add(this.mappings) + if (!this.root.changed && context.mappings.dirty) this.root.mark() } await this._flush() From ad13e6d3c34879e70b233dda3dde67f56ef81f90 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 19:22:28 +0100 Subject: [PATCH 26/31] only set mappings on write flush --- lib/write.js | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/write.js b/lib/write.js index 5912c91..5d0d39e 100644 --- a/lib/write.js +++ b/lib/write.js @@ -112,13 +112,6 @@ module.exports = class WriteBatch { if (op.applied) this.applied++ } - if (this.mappings !== null) { - const local = this.tree.context.getLocalContext() - await local.update(this.config) - context.mappings.add(this.mappings) - if (!this.root.changed && context.mappings.dirty) this.root.mark() - } - await this._flush() await this.close() @@ -365,6 +358,16 @@ module.exports = class WriteBatch { } async _flush() { + const context = this.tree.context.getLocalContext() + const config = this.tree.config + + await context.update(config) + + if (this.mappings !== null) { + context.mappings.add(this.mappings) + if (this.root && !this.root.changed && context.mappings.dirty) this.root.mark() + } + if (!this.root || !this.root.changed) { return } @@ -383,11 +386,6 @@ module.exports = class WriteBatch { const stack = [this.root] const values = [] - const context = this.tree.context.getLocalContext() - const config = this.tree.config - - await context.update(config) - while (stack.length > 0) { const node = stack.pop() From 342315a4709540d01c4b39155260f103c50bf8a8 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 19:22:48 +0100 Subject: [PATCH 27/31] add getAllContexts to simplify cores() --- index.js | 33 ++++++++------------------------- lib/context.js | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/index.js b/index.js index 4b62e31..3ba47a4 100644 --- a/index.js +++ b/index.js @@ -199,35 +199,18 @@ class Hyperbee extends EventEmitter { async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) const node = await this.bootstrap(config) - if (node !== null) { - const block = await node.context.getBlock(node.seq, 0, config, null) - await node.context._inflateCheckpoint(node.seq + 1, block, config, false) - } - - const cores = [] - const seen = new Set() const root = node === null ? this.context : node.context - const queue = [root] - - while (queue.length > 0) { - const context = queue.shift() - - if (context !== root) await context.update(config) - // index 0 is the core this context is viewing, the rest are referenced cores - for (let i = 0; i <= context.cores.length; i++) { - const key = context.getCoreKey(i) - const hex = b4a.toString(key, 'hex') - if (seen.has(hex)) continue - seen.add(hex) - - if (recursive && i > 0) queue.push(await context.getContext(i, config, null)) - if (!local && b4a.equals(key, this.context.local.key)) continue - cores.push(key) - } + if (node !== null) { + const block = await root.getBlock(node.seq, 0, config, null) + await root._inflateCheckpoint(node.seq + 1, block, config, false) } - return cores + const contexts = recursive ? await root.getAllContexts(config) : null + const keys = recursive ? contexts.map((c) => c.core.key) : root.getCoreKeys() + + if (local) return keys + return keys.filter((k) => !b4a.equals(k, this.context.local.key)) } createReadStream(options) { diff --git a/lib/context.js b/lib/context.js index d36a98b..99a6c87 100644 --- a/lib/context.js +++ b/lib/context.js @@ -217,6 +217,41 @@ class CoreContext { return this.mappings.resolve(this.cores[index - 1].key) } + // index 0 is the core this context is viewing, the rest are referenced cores + getCoreKeys() { + const keys = [] + const seen = new Set() + + for (let i = 0; i <= this.cores.length; i++) { + const key = this.getCoreKey(i) + const hex = b4a.toString(key, 'hex') + if (seen.has(hex)) continue + seen.add(hex) + keys.push(key) + } + + return keys + } + + async getAllContexts(config) { + const all = [this] + const seen = new Set([b4a.toString(this.core.key, 'hex')]) + + for (let i = 0; i < all.length; i++) { + const context = all[i] + if (i > 0) await context.update(config) + + for (let j = 1; j <= context.cores.length; j++) { + const hex = b4a.toString(context.getCoreKey(j), 'hex') + if (seen.has(hex)) continue + seen.add(hex) + all.push(await context.getContext(j, config, null)) + } + } + + return all + } + _openCore(key) { return this.store.get({ key, From 61bc297ad0f0867d879171d1d20949f9e64bf998 Mon Sep 17 00:00:00 2001 From: Mathias Buus Date: Wed, 21 Oct 2026 20:32:09 +0200 Subject: [PATCH 28/31] Refactor core key retrieval and context update logic Removed duplicate checks for core keys and optimized context retrieval. --- lib/context.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/context.js b/lib/context.js index 99a6c87..91dac7e 100644 --- a/lib/context.js +++ b/lib/context.js @@ -220,13 +220,9 @@ class CoreContext { // index 0 is the core this context is viewing, the rest are referenced cores getCoreKeys() { const keys = [] - const seen = new Set() for (let i = 0; i <= this.cores.length; i++) { const key = this.getCoreKey(i) - const hex = b4a.toString(key, 'hex') - if (seen.has(hex)) continue - seen.add(hex) keys.push(key) } @@ -239,14 +235,16 @@ class CoreContext { for (let i = 0; i < all.length; i++) { const context = all[i] - if (i > 0) await context.update(config) + const pending = [] for (let j = 1; j <= context.cores.length; j++) { const hex = b4a.toString(context.getCoreKey(j), 'hex') if (seen.has(hex)) continue seen.add(hex) - all.push(await context.getContext(j, config, null)) + pending.push(context.getContext(j, config, null)) } + + for (const c of Promise.all(pending)) all.push(c) } return all From 147f99b54b70a702e8839defea3484be050a6429 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 20:05:31 +0100 Subject: [PATCH 29/31] always set root after write flush --- index.js | 2 ++ test/basic.js | 3 ++- test/undo.js | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 3ba47a4..0e8281d 100644 --- a/index.js +++ b/index.js @@ -339,6 +339,8 @@ class Hyperbee extends EventEmitter { } _setRoot(root, emit) { + if (root !== null && root !== EMPTY) this.context = root.context + if (root === null || !root.equivalentTo(this.root)) { this.root = root if (emit) this.emit('update') diff --git a/test/basic.js b/test/basic.js index c243efa..0e349c1 100644 --- a/test/basic.js +++ b/test/basic.js @@ -698,7 +698,8 @@ test('reindex preserves batches across splits, value blocks and cores', async fu t.is(remote.length, 5) const source = b.head() - t.absent(b4a.equals(source.key, b.core.key), 'b is viewing c, its writes live in its local core') + t.alike(source.key, b.context.local.key, 'b built on c, its writes live in its local core') + t.absent(b4a.equals(source.key, genesis.key)) // stop at c's change, so the copies keep pointing into c's core for untouched nodes a.move(source) diff --git a/test/undo.js b/test/undo.js index 7bfc503..28e9a69 100644 --- a/test/undo.js +++ b/test/undo.js @@ -1,5 +1,7 @@ const test = require('brittle') const b4a = require('b4a') +const Corestore = require('corestore') +const Bee = require('../') const { create } = require('./helpers') test('undo', async function (t) { @@ -164,3 +166,39 @@ test('move', async function (t) { ]) } }) + +test('undo after writing on top of another core', async function (t) { + const store = new Corestore(await t.tmp()) + t.teardown(() => store.close()) + + const a = new Bee(store.namespace('a')) + await a.ready() + for (const k of ['a', 'b']) { + const w = a.write() + w.tryPut(b4a.from(k), b4a.from(k)) + await w.flush() + } + + const b = new Bee(store.namespace('b')) + await b.ready() + b.move({ key: a.core.key, length: a.core.length }) + for (const k of ['c', 'd']) { + const w = b.write() + w.tryPut(b4a.from(k), b4a.from(k)) + await w.flush() + } + + t.alike(b.core.key, b.context.local.key, 'the tree views the core it wrote to') + + const head = b.checkout() + t.alike((await head.get(b4a.from('d'))).value, b4a.from('d'), 'a checkout sees the writes') + await head.close() + + const undone = b.undo(1) + t.alike((await undone.get(b4a.from('c'))).value, b4a.from('c'), 'undo keeps the earlier write') + t.is(await undone.get(b4a.from('d')), null, 'and drops the last one') + await undone.close() + + await a.close() + await b.close() +}) From 83a4758cced09f80a02f97b9d48b44a2f4126536 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 20:12:59 +0100 Subject: [PATCH 30/31] simplify cores --- index.js | 13 ++++--------- lib/context.js | 29 ++++++++++++++++------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 0e8281d..8fb6e63 100644 --- a/index.js +++ b/index.js @@ -198,16 +198,11 @@ class Hyperbee extends EventEmitter { async cores({ local = true, recursive = false, ...opts } = {}) { const config = this.config.options(opts) - const node = await this.bootstrap(config) - const root = node === null ? this.context : node.context - - if (node !== null) { - const block = await root.getBlock(node.seq, 0, config, null) - await root._inflateCheckpoint(node.seq + 1, block, config, false) - } + await this.ready() - const contexts = recursive ? await root.getAllContexts(config) : null - const keys = recursive ? contexts.map((c) => c.core.key) : root.getCoreKeys() + const keys = recursive + ? (await this.context.getAllContexts(config)).map((c) => c.core.key) + : await this.context.getCoreKeys(config) if (local) return keys return keys.filter((k) => !b4a.equals(k, this.context.local.key)) diff --git a/lib/context.js b/lib/context.js index 91dac7e..98cf7be 100644 --- a/lib/context.js +++ b/lib/context.js @@ -217,34 +217,31 @@ class CoreContext { return this.mappings.resolve(this.cores[index - 1].key) } - // index 0 is the core this context is viewing, the rest are referenced cores - getCoreKeys() { - const keys = [] - - for (let i = 0; i <= this.cores.length; i++) { - const key = this.getCoreKey(i) - keys.push(key) - } + async getCoreKeys(config) { + await this.update(config) + const keys = [] + for (let i = 0; i <= this.cores.length; i++) keys.push(this.getCoreKey(i)) return keys } async getAllContexts(config) { + await this.update(config) + const all = [this] const seen = new Set([b4a.toString(this.core.key, 'hex')]) for (let i = 0; i < all.length; i++) { - const context = all[i] const pending = [] - for (let j = 1; j <= context.cores.length; j++) { - const hex = b4a.toString(context.getCoreKey(j), 'hex') + for (let j = 1; j <= all[i].cores.length; j++) { + const hex = b4a.toString(all[i].getCoreKey(j), 'hex') if (seen.has(hex)) continue seen.add(hex) - pending.push(context.getContext(j, config, null)) + pending.push(loadContext(all[i], j, config)) } - for (const c of Promise.all(pending)) all.push(c) + for (const c of await Promise.all(pending)) all.push(c) } return all @@ -345,4 +342,10 @@ class CoreContext { } } +async function loadContext(context, index, config) { + const ctx = await context.getContext(index, config, null) + await ctx.update(config) + return ctx +} + module.exports = CoreContext From 594eb6ae11acce01ede579500d37e088988df252 Mon Sep 17 00:00:00 2001 From: Christophe Diederichs Date: Wed, 21 Oct 2026 20:14:56 +0100 Subject: [PATCH 31/31] lint --- spec/hyperschema/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/hyperschema/index.js b/spec/hyperschema/index.js index 9cfbce3..3a9e555 100644 --- a/spec/hyperschema/index.js +++ b/spec/hyperschema/index.js @@ -428,7 +428,7 @@ const encoding13 = { if (version >= 2 && m.mappings) encoding13_1.preencode(state, m.mappings) }, encode(state, m) { - const flags = (version >= 2 && m.mappings) ? 1 : 0 + const flags = version >= 2 && m.mappings ? 1 : 0 encoding13_0.encode(state, m.cores) c.uint.encode(state, flags) @@ -441,7 +441,7 @@ const encoding13 = { return { cores: r0, - mappings: (version >= 2 && (flags & 1) !== 0) ? encoding13_1.decode(state) : null + mappings: version >= 2 && (flags & 1) !== 0 ? encoding13_1.decode(state) : null } } }