Support remapping cores

README.md

@@ -65,7 +65,8 @@ Options include:
 }
 ```
 
-If you pass your own `core`, consider increasing the inflight range on it (`256` minimum recommended, e.g. `{ inflightRange: [256, 512] }` when creating the core via Corestore) to match `hyperbee2` constructed cores.
+Mappings are persisted next to the core table on the next write, so readers
+that open the tree later resolve them without being given them.
 
 #### `await db.ready()`
 
@@ -128,6 +129,24 @@ Read only. Boolean indicating whether the [Corestore][corestore] has closed.
 
 Calls `replicate` on underlying [Corestore][corestore].
 
+#### `await db.cores([options])`
+
+The keys of the cores the tree reads from, mapped: the core the root lives on
+first, then the cores it references. Options:
+
+```js
+{
+  local: true,       // Boolean. Include the local core
+  recursive: false,  // Boolean. Also walk the core tables of every core reached
+  timeout: ...,      // Number. Timeout for the checkpoint reads
+  wait: ...          // Boolean. Wait for checkpoint blocks to download
+}
+```
+
+With `recursive` the result covers everything the tree can reach through core
+tables, one checkpoint read per core. It is a superset of what the current
+root references, as tables only ever grow.
+
 #### `db.checkout([options])`
 
 Returns a new Hyperbee as a view on the underlying [Hypercore][hypercore].
@@ -159,6 +178,8 @@ Options:
 }
 ```
 
+If `key` is mapped the core it maps to is opened instead.
+
 #### `db.snapshot([options])`
 
 Returns a new Hyperbee that is a read only view of the current tree.
@@ -202,6 +223,9 @@ Options:
   key: null,                      // Buffer or null. Key of the Hypercore.
   autoUpdate: true,               // Boolean .Should Hyperbee automatically reflect updates
                                   // after each flush()?
+  mappings: null,                // [{ from, to }]. Read core `from` as core `to`, which holds the
+                                  // same blocks, and write them into this checkpoint, even with
+                                  // nothing else to write. Readers of the tree learn them from it
   compat: false,                  // Boolean. Write blocks compatible with Hyperbee 1?
   type: encoding.TYPE_LATEST,     // Number. Block format to use.
   inlineValueSize: 1024,          // Values smaller than this byte length are

build.js

@@ -1,6 +1,6 @@
 const Hyperschema = require('hyperschema')
 
-const schema = Hyperschema.from('./spec/hyperschema', { versioned: false })
+const schema = Hyperschema.from('./spec/hyperschema')
 const bee = schema.namespace('bee')
 
 bee.register({
@@ -228,6 +228,23 @@ bee.register({
   ]
 })
 
+bee.register({
+  name: 'mapping',
+  compact: true,
+  fields: [
+    {
+      name: 'from',
+      type: 'fixed32',
+      required: true
+    },
+    {
+      name: 'to',
+      type: 'fixed32',
+      required: true
+    }
+  ]
+})
+
 bee.register({
   name: 'block-0',
   fields: [
@@ -281,6 +298,11 @@ bee.register({
       type: '@bee/core',
       array: true,
       required: true
+    },
+    {
+      name: 'mappings',
+      type: '@bee/mapping',
+      array: true
     }
   ]
 })

index.js

@@ -196,21 +196,16 @@ class Hyperbee extends EventEmitter {
     this.emit('close')
   }
 
-  async cores({ local = true, ...opts } = {}) {
+  async cores({ local = true, recursive = false, ...opts } = {}) {
     const config = this.config.options(opts)
     await this.ready()
-    await this.context.update(config)
 
-    const cores = []
+    const keys = recursive
+      ? (await this.context.getAllContexts(config)).map((c) => c.core.key)
+      : await this.context.getCoreKeys(config)
 
-    // index 0 is the core this bee is viewing, the rest are referenced cores
-    for (let i = 0; i <= this.context.cores.length; i++) {
-      const key = this.context.getCoreKey(i)
-      if (!local && b4a.equals(key, this.context.local.key)) continue
-      cores.push(key)
-    }
-
-    return cores
+    if (local) return keys
+    return keys.filter((k) => !b4a.equals(k, this.context.local.key))
   }
 
   createReadStream(options) {
@@ -339,6 +334,8 @@ class Hyperbee extends EventEmitter {
   }
 
   _setRoot(root, emit) {
+    if (root !== null && root !== EMPTY) this.context = root.context
+
     if (root === null || !root.equivalentTo(this.root)) {
       this.root = root
       if (emit) this.emit('update')

lib/context.js

@@ -4,6 +4,7 @@ const c = require('compact-encoding')
 const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors')
 const { TreeNodePointer } = require('./tree.js')
 const { decodeBlock, TYPE_LATEST } = require('./encoding.js')
+const Mappings = require('./mappings.js')
 
 const CHECKPOINT_KEY = 'hyperbee/checkpoint'
 
@@ -14,6 +15,7 @@ class CoreContext {
     cache,
     core,
     getEncryptionProvider,
+    mappings = new Mappings(),
     lock = new ScopeLock(),
     other = new Map()
   ) {
@@ -24,6 +26,7 @@ class CoreContext {
     this.getEncryptionProvider = getEncryptionProvider
     this.lock = lock
     this.other = other
+    this.mappings = mappings
     this.length = 0
     this.checkpoint = 0
     this.invalidation = 0
@@ -93,13 +96,17 @@ class CoreContext {
     const checkpoint = block.checkpoint
     let cores = null
 
-    if (checkpoint > 0) {
+    if (checkpoint > 0 && checkpoint !== this.checkpoint) {
       const seq = checkpoint - 1
       const buffer = await this.core.get(seq, config)
       if (buffer === null) throw BLOCK_NOT_AVAILABLE()
       const block = decodeBlock(buffer, seq)
 
       cores = block.metadata ? block.metadata.cores : []
+      if (block.metadata && block.metadata.mappings !== null) {
+        const persisted = b4a.equals(this.core.key, this.local.key)
+        this.mappings.add(block.metadata.mappings, { persisted })
+      }
     }
 
     if (length <= this.length) return
@@ -195,24 +202,57 @@ class CoreContext {
   getCore(index) {
     if (index === 0) return this.core
     if (index > this.cores.length) throw new Error('Bad core index: ' + index)
-    if (this.opened[index - 1] === null) {
-      const key = this.cores[index - 1].key
-      this.opened[index - 1] = this.store.get({
-        key,
-        encryption: this.getEncryptionProvider(key),
-        inflightRange: [256, 512]
-      })
-    }
+
+    const key = this.mappings.resolve(this.cores[index - 1].key)
+    const opened = this.opened[index - 1]
+    if (opened !== null && (key === opened.key || b4a.equals(opened.key, key))) return opened
+
+    this.opened[index - 1] = this._openCore(key)
     return this.opened[index - 1]
   }
 
   getCoreKey(index) {
     if (index === 0) return this.core.key
     if (index > this.cores.length) throw new Error('Bad core index: ' + index)
-    if (this.opened[index - 1] !== null && this.opened[index - 1].key) {
-      return this.opened[index - 1].key
+    return this.mappings.resolve(this.cores[index - 1].key)
+  }
+
+  async getCoreKeys(config) {
+    await this.update(config)
+
+    const keys = []
+    for (let i = 0; i <= this.cores.length; i++) keys.push(this.getCoreKey(i))
+    return keys
+  }
+
+  async getAllContexts(config) {
+    await this.update(config)
+
+    const all = [this]
+    const seen = new Set([b4a.toString(this.core.key, 'hex')])
+
+    for (let i = 0; i < all.length; i++) {
+      const pending = []
+
+      for (let j = 1; j <= all[i].cores.length; j++) {
+        const hex = b4a.toString(all[i].getCoreKey(j), 'hex')
+        if (seen.has(hex)) continue
+        seen.add(hex)
+        pending.push(loadContext(all[i], j, config))
+      }
+
+      for (const c of await Promise.all(pending)) all.push(c)
     }
-    return this.cores[index - 1].key
+
+    return all
+  }
+
+  _openCore(key) {
+    return this.store.get({
+      key,
+      encryption: this.getEncryptionProvider(key),
+      inflightRange: [256, 512]
+    })
   }
 
   async getBlock(seq, core, config, origin) {
@@ -246,18 +286,13 @@ class CoreContext {
   }
 
   getContextByKey(key) {
+    key = this.mappings.resolve(key)
     if (b4a.equals(key, this.core.key)) return this
 
     const hex = b4a.toString(key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const ctx = this._createContext(
-      this.store.get({
-        key,
-        encryption: this.getEncryptionProvider(key),
-        inflightRange: [256, 512]
-      })
-    )
+    const ctx = this._createContext(this._openCore(key))
     this.other.set(hex, ctx)
     return ctx
   }
@@ -273,18 +308,25 @@ class CoreContext {
     if (core === 0) return this
     if (core > this.cores.length) throw new Error('Bad core index: ' + core)
 
-    const hex = b4a.toString(this.cores[core - 1].key, 'hex')
+    const hc = this.getCore(core)
+    const hex = b4a.toString(hc.key, 'hex')
     if (this.other.has(hex)) return this.other.get(hex)
 
-    const ctx = this._createContext(this.getCore(core))
+    const ctx = this._createContext(hc)
     this.other.set(hex, ctx)
     return ctx
   }
 
+  needsFlush() {
+    return this.changed || this.mappings.dirty
+  }
+
   flush() {
+    if (this.mappings.dirty) this.invalidation++
     this.changed = false
     return {
-      cores: this.cores
+      cores: this.cores,
+      mappings: this.mappings.flush()
     }
   }
 
@@ -293,10 +335,17 @@ class CoreContext {
     const local = this.local
     const cache = this.cache
     const getEncryptionProvider = this.getEncryptionProvider
+    const mappings = this.mappings
     const lock = this.lock
     const other = this.other
-    return new CoreContext(store, local, cache, core, getEncryptionProvider, lock, other)
+    return new CoreContext(store, local, cache, core, getEncryptionProvider, mappings, lock, other)
   }
 }
 
+async function loadContext(context, index, config) {
+  const ctx = await context.getContext(index, config, null)
+  await ctx.update(config)
+  return ctx
+}
+
 module.exports = CoreContext

lib/copy.js

@@ -62,7 +62,7 @@ module.exports = async function copyChanges(tree, changes, config, map) {
     }
   }
 
-  if (local.changed) {
+  if (local.needsFlush()) {
     local.checkpoint = base + blocks.length
     blocks[blocks.length - 1].metadata = local.flush()
   }
@@ -71,6 +71,7 @@ module.exports = async function copyChanges(tree, changes, config, map) {
 
   for (let i = 0; i < blocks.length; i++) {
     blocks[i].checkpoint = local.checkpoint
+    blocks[i].invalidation = local.invalidation
     buffers[i] = encodeBlock(blocks[i])
   }
 

lib/encoding.js

@@ -85,7 +85,7 @@ function decodeBlock0(state, seq) {
     checkpoint: blk.checkpoint,
     batch: blk.batch,
     previous: blk.previous,
-    metadata: { cores: blk.cores },
+    metadata: { cores: blk.cores, mappings: null },
     tree,
     keys: blk.data.map(dataToKey),
     values: null,

lib/mappings.js

@@ -0,0 +1,40 @@
+const b4a = require('b4a')
+
+class Mappings {
+  constructor() {
+    this.known = new Map()
+    this.dirty = false
+    this._flushed = null
+  }
+
+  resolve(key) {
+    if (this.known.size === 0) return key
+    const known = this.known.get(b4a.toString(key, 'hex'))
+    return known === undefined ? key : known
+  }
+
+  add(mappings, { persisted = false } = {}) {
+    for (const { from, to } of mappings) {
+      if (b4a.equals(from, to)) continue
+      const hex = b4a.toString(from, 'hex')
+      if (this.known.has(hex)) continue
+      this.known.set(hex, to)
+      this._flushed = null
+      if (!persisted) this.dirty = true
+    }
+  }
+
+  flush() {
+    this.dirty = false
+    if (this.known.size === 0) return null
+
+    if (this._flushed === null) {
+      this._flushed = []
+      for (const [hex, to] of this.known) this._flushed.push({ from: b4a.from(hex, 'hex'), to })
+    }
+
+    return this._flushed
+  }
+}
+
+module.exports = Mappings