Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
7b5af5e
Add WIRE-385 schedule recovery scenario
huangminghuang Sep 11, 2026
82c56a6
Address WIRE-385 recovery scenario feedback
huangminghuang Sep 11, 2026
c8d83dd
Verify WIRE-385 explicit schedule publication and activation
huangminghuang Sep 11, 2026
4b76086
Fund replacement operators before recovery delivery
huangminghuang Sep 12, 2026
d63e0e7
Observe later replacement duties after quorum races
huangminghuang Sep 12, 2026
8d53c83
Require replacement signers in recovery flow quorum
huangminghuang Sep 12, 2026
dac51ad
Address PR review feedback
huangminghuang Sep 15, 2026
1f53681
Classify revoked operator rejections in flow heartbeat
huangminghuang Sep 15, 2026
f0f18d5
Merge current Tools master into WIRE-385 branch
huangminghuang Sep 21, 2026
e678da4
Fix clean-install schedule recovery build
huangminghuang Sep 21, 2026
e741d8c
Seed Ethereum bootstrap from depot schedule
huangminghuang Sep 21, 2026
20d8b27
Merge remote-tracking branch 'origin/master' into fix/wire-385-schedu…
huangminghuang Sep 22, 2026
52136d1
Merge remote-tracking branch 'origin/master' into fix/wire-385-schedu…
huangminghuang Sep 22, 2026
17ae20e
Use generated epoch state in the recovery flow
huangminghuang Sep 22, 2026
f3237b0
Keep WIRE-385 bootstrap coverage and defer recovery scenarios
huangminghuang Sep 22, 2026
f3c5b71
Fix collateral flow withdrawal eligibility
huangminghuang Sep 23, 2026
59e9cf6
Address review with post-termination rotation coverage
huangminghuang Sep 24, 2026
082d21a
Fix termination flow default group-size assertion
huangminghuang Sep 24, 2026
f836463
Merge master and integrate canonical slug writers
huangminghuang Sep 25, 2026
d50fc78
Clarify read-only Solana epoch cursor projection
huangminghuang Sep 25, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 162 additions & 2 deletions packages/flow-batch-operator-termination/src/TerminationScenario.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,16 @@ import Assert from "node:assert"
import { PublicKey } from "@solana/web3.js"
import type { BN } from "@coral-xyz/anchor"
import { SysioContracts } from "@wireio/sdk-core"
import { OperatorType } from "@wireio/opp-typescript-models"
import {
AttestationType,
BatchOperatorGroups,
Envelope,
OperatorType
} from "@wireio/opp-typescript-models"
import {
BatchOperatorSchedule,
ClusterBuildPhase,
ClusterConfigProvider,
EthereumCollateralTool,
FlowScenario,
Report,
Expand All @@ -14,6 +21,7 @@ import {
Steps,
WireOperatorProvisioningTool,
getLogger,
loadOutpostContract,
matchesProtoEnum,
outputKey,
packedSlugValue,
Expand Down Expand Up @@ -137,6 +145,16 @@ interface SolanaAccountClient {
fetch(address: PublicKey): Promise<unknown>
}

/** The outpost cursors advance only after an inbound envelope is accepted. */
interface EthereumInboundView {
nextEpochIndex(): Promise<bigint>
}

/** Read-only projection of the Solana outpost configuration's inbound epoch cursor. */
interface SolanaOutpostConfigAccount {
nextEpochIndex: number
}

/** The SOL outpost's on-chain collateral ledger from the `OperatorRegistry` PDA (a read). */
async function readSolanaCollateralLedger(
ctx: ClusterBuildContext
Expand Down Expand Up @@ -188,11 +206,14 @@ async function readSolanaCollateralLedger(
* outpost's escrow ledger returns to 0, and each wallet is credited the
* exact bond amount (wei/lamport-exact — any drift means the outpost decoded
* a different amount than the depot encoded).
* 9. **ContinuedRotation** — standing operators fill the vacated seat, every
* observed published window excludes the terminated operator, and both
* outposts accept another complete rotation after the remits land.
*/
export class TerminationScenario extends FlowScenario {
readonly name = "flow-batch-operator-termination"
readonly description =
"Non-bootstrapped batch operator bonds ETH + SOL, misses its scheduled deliveries, is terminated, and both bonds are remitted back"
"Batch operator termination remits both bonds and standing operators keep both outposts advancing"

override readonly defaults: ClusterBuildOptions = {
epochDurationSec: Constants.EpochDurationSec,
Expand Down Expand Up @@ -709,5 +730,144 @@ export class TerminationScenario extends FlowScenario {
quickStepOptions
)
)

// ── 9. Remittance is not enough: the following duty groups must deliver ──
ClusterBuildPhase.create(
cluster,
"ContinuedRotation",
"Standing operators absorb the termination and both outposts complete another rotation"
).push(
verifyStep(
Actor.Sysio,
"post-remit-rotation",
"published groups exclude the terminated operator and both outposts advance through a full window",
async ctx => {
const operator = ctx.keyStore.assertOperator(
Constants.DoomedOperatorLabel
),
addresses = EthereumCollateralTool.loadOutpostAddresses(
ClusterConfigProvider.ethereumDeploymentsPath(ctx.config)
),
ethereum = loadOutpostContract<EthereumInboundView>(
ctx.config.ethereumPath,
addresses,
"OPPInbound",
["outpost"],
ctx.ethereum.wallet.signer
),
program = SolanaCollateralTool.loadOppOutpostProgram(
ctx,
solanaKeypair(operator.solana)
),
configAddress = SolanaOutpostProgramTool.derivePda(
program.programId,
Buffer.from(SolanaOutpostBootstrapper.PdaSeed.OutpostConfig)
),
accounts: Record<string, SolanaAccountClient> = program.account,
readCursors = async () => {
const [ethNext, solConfig] = await Promise.all([
ethereum.nextEpochIndex(),
accounts.outpostConfig.fetch(configAddress)
])
return [
Number(ethNext),
Number((solConfig as SolanaOutpostConfigAccount).nextEpochIndex)
]
},
baseline = await readCursors(),
start = await Steps.contracts.sysio.epoch.readEpochState(ctx),
standing = new Set(
ctx.keyStore.operators
.filter(
entry =>
entry.type === OperatorType.BATCH &&
entry.account !== operator.account
)
.map(entry => entry.account)
),
groupCount = start.batch_op_groups.length,
groupSize = BatchOperatorSchedule.resolve(
ctx.config
).operatorsPerEpoch,
targetEpoch =
Math.max(Number(start.current_epoch_index), ...baseline) +
groupCount,
chains = [Constants.EthereumChainCode, Constants.SolanaChainCode]

Assert.ok(
groupCount > 1,
"termination regression requires multiple duty groups"
)
await pollUntil(
`both outposts accept post-remit epochs through ${targetEpoch}`,
async () => {
const { rows } = await ctx.wire.getOutboundEnvelopes()
for (const chain of chains) {
const row = rows.find(
entry => packedSlugValue(entry.chain_code) === chain
)
Assert.ok(
row != null,
`missing outbound envelope for chain ${chain}`
)
const envelope = Envelope.fromBinary(
Buffer.from(row.raw_envelope, "hex")
),
announcements = envelope.messages.flatMap(message =>
(message.payload?.attestations ?? [])
.filter(
entry =>
entry.type === AttestationType.BATCH_OPERATOR_GROUPS
)
.map(entry => BatchOperatorGroups.fromBinary(entry.data))
)
Assert.ok(
announcements.length > 0,
`no published group window at epoch ${row.epoch_index}`
)
for (const announcement of announcements) {
const groups = announcement.groups.map(group =>
group.operators.map(member =>
Buffer.from(member.address).toString("utf8")
)
),
members = groups.flat()
Assert.equal(
groups.length,
groupCount,
"published window lost a group"
)
Assert.ok(
groups.every(group => group.length === groupSize),
"standing operators did not fill every seat"
)
Assert.ok(
!members.includes(operator.account),
"terminated operator re-entered a published group"
)
Assert.equal(
new Set(members).size,
groupCount * groupSize,
"published groups repeat an operator"
)
Assert.ok(
members.every(account => standing.has(account)),
"replacement is not a standing operator"
)
}
}
const cursors = await readCursors()
log.info(
`[${this.name}] post-remit ETH/SOL next epochs=${cursors.join("/")}; target>${targetEpoch}`
)
return cursors.every(epoch => epoch > targetEpoch)
},
Constants.remitDeadlineMs(),
Constants.PollIntervalMs
)
},
remitStepOptions
)
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ async function readWithdrawQueueRows(
* schedule prefers non-bootstrapped operators, and its group must relay).
* 3. **DepositEthereum** — bond on the ETH outpost → depot credits the balance row.
* 4. **DepositSolana** — bond on the SOL outpost → all-chain rule met → ACTIVE.
* 5. **WithdrawRequest** — release half the ETH bond → depot queues it.
* 5. **WithdrawRequest** — release half the ETH bond; verify reserved collateral leaves the operator ACTIVE.
* 6. **WaitAndFlush** — the wait window elapses; `flushwtdw` drains the queue.
* 7. **ProcessRemit** — WITHDRAW_REMIT lands on the ETH outpost; escrow decrements.
*/
Expand All @@ -78,12 +78,12 @@ export class CollateralLifecycleScenario extends FlowScenario {
{
chainCode: Constants.EthereumChainCode,
tokenCode: Constants.EthereumTokenCode,
minimumBond: Number(Constants.BondAmount)
minimumBond: Number(Constants.MinimumBond)
},
{
chainCode: Constants.SolanaChainCode,
tokenCode: Constants.SolanaTokenCode,
minimumBond: Number(Constants.BondAmount)
minimumBond: Number(Constants.MinimumBond)
}
]
}
Expand Down Expand Up @@ -210,11 +210,11 @@ export class CollateralLifecycleScenario extends FlowScenario {
)
)

// ── 5. Withdraw half the ETH bond → depot queues it ──
// ── 5. Withdraw excess ETH collateral while retaining relay eligibility ──
ClusterBuildPhase.create(
cluster,
"WithdrawRequest",
"Release half the ETH bond; depot enqueues wtdwqueue"
"Release half the ETH bond; depot enqueues wtdwqueue and retains eligibility"
).push(
EthereumCollateralTool.planWithdrawal(
Actor.User,
Expand Down Expand Up @@ -246,6 +246,41 @@ export class CollateralLifecycleScenario extends FlowScenario {
)
},
stepOptions
),
verifyStep(
Actor.Sysio,
"depot-status-active-after-withdraw",
"reserved withdrawal retains the minimum ETH collateral and ACTIVE status",
async ctx => {
const operator = await readDepositorRow(ctx),
requests = await readWithdrawQueueRows(ctx),
ethBalance = operator?.balances.find(
balance =>
slugValue(balance.chain_code) === Constants.EthereumChainCode &&
slugValue(balance.token_code) === Constants.EthereumTokenCode
),
reservedAmount = requests
.filter(
request =>
slugValue(request.chain_code) === Constants.EthereumChainCode &&
slugValue(request.token_code) === Constants.EthereumTokenCode
)
.reduce((sum, request) => sum + BigInt(request.amount), 0n)
if (
ethBalance == null ||
BigInt(ethBalance.balance) - reservedAmount < Constants.MinimumBond ||
!matchesProtoEnum(
operator.status,
SysioOpregOperatorstatus,
SysioOpregOperatorstatus.OPERATOR_STATUS_ACTIVE
)
) {
throw new Error(
`Withdrawing excess ETH collateral must retain the minimum bond and ACTIVE status; balance=${ethBalance?.balance}, reserved=${reservedAmount}, status=${operator?.status}`
)
}
},
stepOptions
)
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@ import { SlugName } from "@wireio/sdk-core"
import { ProtocolTiming } from "@wireio/cluster-tool"

/**
* Constants for the collateral-lifecycle flow. Amounts + epoch budgets carry
* over from the previously-validated flow run (2026-06): the bond is deposited
* Constants for the collateral-lifecycle flow. Twice the minimum bond is deposited
* on BOTH outpost chains (all-chain collateral invariant), half the ETH bond is
* withdrawn mid-flow, and every poll deadline derives from extension-inclusive
* withdrawn while retaining the minimum. Every poll deadline uses extension-inclusive
* epochs ({@link ProtocolTiming.effectiveEpochSec}) so the flow scales with the
* epoch duration and survives extended epochs.
*/
Expand All @@ -29,9 +28,11 @@ export namespace CollateralLifecycleScenarioConstants {
*/
export const AdHocDaemonCount = 1

/** Minimum collateral retained per chain to keep the depositor eligible to relay. */
export const MinimumBond = 1_000_000n
/** Collateral bonded per chain (raw outpost units — wei / lamports). */
export const BondAmount = 2_000_000n
/** ETH bond released mid-flow (half — stays above the minimum on the rest). */
/** ETH bond released mid-flow (half — leaves exactly the required minimum). */
export const WithdrawAmount = 1_000_000n
/** Escrow expected on the ETH outpost after the withdraw remit. */
export const ExpectedRemainingBalance = BondAmount - WithdrawAmount
Expand Down
Loading