From c238b19811fe036e0ac69d3e93758343bb4bd068 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:23:49 +0530 Subject: [PATCH 01/22] =?UTF-8?q?feat!:=20=E2=AC=86=EF=B8=8F=20?= =?UTF-8?q?=F0=9F=8E=B8=20upgrade=20the=20postgres=20image=20to=2018?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves the database from postgres 12 to 18. Postgres 13 and later let a boolean column be part of the range index the indexer builds when historical tracking is on, which postgres 12 rejected. The data volume is now mounted at /var/lib/postgresql because postgres 18 stores its files in a version folder underneath. BREAKING CHANGE: the database is a new major version and its volume path changed, so any existing data volume has to be recreated and the chain reindexed from the start. --- docker-compose.yml | 4 +++- docker/pg-Dockerfile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9f4f4688..5286aadc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,9 @@ services: ports: - 5999:5432 volumes: - - db-data:/var/lib/postgresql/data + # PostgreSQL 18+ images store data in a major-version subdirectory, so the mount is the + # parent `/var/lib/postgresql` (not `…/data`). See docker-library/postgres#1259. + - db-data:/var/lib/postgresql environment: POSTGRES_PASSWORD: postgres healthcheck: diff --git a/docker/pg-Dockerfile b/docker/pg-Dockerfile index cb141e39..1d2a1aa8 100644 --- a/docker/pg-Dockerfile +++ b/docker/pg-Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:12-alpine +FROM postgres:18-alpine # Variables needed at runtime to configure postgres and run the initdb scripts ENV POSTGRES_DB='postgres' From ed7ec00deeb916022fcb233c8cd76ce48873579b Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:23:51 +0530 Subject: [PATCH 02/22] =?UTF-8?q?chore:=20=F0=9F=94=A5=20delete=20the=20db?= =?UTF-8?q?=20migration=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The redesign reindexes the whole chain from block zero, so the incremental schema-migration files no longer apply and would only run stale changes on restart. --- ...ema_changes_for_portfolio_kind_support.sql | 14 -- .../11_handle_classic_ticker_claimed.sql | 176 ------------------ db/migrations/12_add_new_8_chain_events.sql | 50 ----- .../13_add_identity_to_asset_transactions.sql | 33 ---- .../14_confidential_assets_events_8_chain.sql | 65 ------- db/migrations/15_add_missing_transfer_nft.sql | 1 - db/migrations/16_rename_submit_unsigned.sql | 12 -- .../17_add_identity_to_portfolio_movement.sql | 19 -- ...d_reward_destination_to_staking_events.sql | 4 - ...19_add_instruction_unlocked_event_type.sql | 1 - .../1_fix_multi_sig_proposals_data.sql | 48 ----- db/migrations/20_evm_transactions.sql | 29 --- .../2_update_instruction_affirmation_id.sql | 3 - .../3_update_failed_instruction_status.sql | 3 - db/migrations/4_add_new_events_extrinsics.sql | 8 - db/migrations/5_add_enum_text_columns.sql | 31 --- .../6_new_events_for_7300000_spec.sql | 25 --- ...ticker_null_constraints_on_investments.sql | 2 - db/migrations/8_events_for_8000000_spec.sql | 54 ------ db/migrations/9_events_for_7400000_spec.sql | 5 - 20 files changed, 583 deletions(-) delete mode 100644 db/migrations/10_schema_changes_for_portfolio_kind_support.sql delete mode 100644 db/migrations/11_handle_classic_ticker_claimed.sql delete mode 100644 db/migrations/12_add_new_8_chain_events.sql delete mode 100644 db/migrations/13_add_identity_to_asset_transactions.sql delete mode 100644 db/migrations/14_confidential_assets_events_8_chain.sql delete mode 100644 db/migrations/15_add_missing_transfer_nft.sql delete mode 100644 db/migrations/16_rename_submit_unsigned.sql delete mode 100644 db/migrations/17_add_identity_to_portfolio_movement.sql delete mode 100644 db/migrations/18_add_reward_destination_to_staking_events.sql delete mode 100644 db/migrations/19_add_instruction_unlocked_event_type.sql delete mode 100644 db/migrations/1_fix_multi_sig_proposals_data.sql delete mode 100644 db/migrations/20_evm_transactions.sql delete mode 100644 db/migrations/2_update_instruction_affirmation_id.sql delete mode 100644 db/migrations/3_update_failed_instruction_status.sql delete mode 100644 db/migrations/4_add_new_events_extrinsics.sql delete mode 100644 db/migrations/5_add_enum_text_columns.sql delete mode 100644 db/migrations/6_new_events_for_7300000_spec.sql delete mode 100644 db/migrations/7_drop_ticker_null_constraints_on_investments.sql delete mode 100644 db/migrations/8_events_for_8000000_spec.sql delete mode 100644 db/migrations/9_events_for_7400000_spec.sql diff --git a/db/migrations/10_schema_changes_for_portfolio_kind_support.sql b/db/migrations/10_schema_changes_for_portfolio_kind_support.sql deleted file mode 100644 index 6acda9f0..00000000 --- a/db/migrations/10_schema_changes_for_portfolio_kind_support.sql +++ /dev/null @@ -1,14 +0,0 @@ -alter table "portfolio_movements" add column if not exists "to_account" text; -alter table "portfolio_movements" add column if not exists "from_account" text; - -alter table "portfolio_movements" alter column "from_id" drop not null; -alter table "portfolio_movements" alter column "to_id" drop not null; - -alter table "legs" add column if not exists "to_account" text; -alter table "legs" add column if not exists "from_account" text; - -alter table "instruction_affirmations" add column if not exists "account" text; -alter table "instruction_events" add column if not exists "account" text; - -alter table "asset_transactions" add column if not exists "from_account" text; -alter table "asset_transactions" add column if not exists "to_account" text; \ No newline at end of file diff --git a/db/migrations/11_handle_classic_ticker_claimed.sql b/db/migrations/11_handle_classic_ticker_claimed.sql deleted file mode 100644 index b44d53b8..00000000 --- a/db/migrations/11_handle_classic_ticker_claimed.sql +++ /dev/null @@ -1,176 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - -INSERT INTO ticker_reservations -(id, ticker, identity_id, expiry, created_block_id, updated_block_id, _id, _block_range) -SELECT DISTINCT ON(event_arg_1) - event_arg_1, - event_arg_1, - event_arg_0, - case when event_arg_2 != 'null' then to_timestamp(event_arg_2::bigint / 1000.0) else NULL end, - block_id, - block_id, - uuid_generate_v4(), - int8range(block_id::bigint, NULL::bigint) -FROM events -WHERE module_id = 'asset' - AND event_id_text = 'TickerRegistered' - AND NOT EXISTS ( - SELECT 1 - FROM ticker_reservations - WHERE ticker_reservations.id = events.event_arg_1 - ) -ORDER BY event_arg_1, block_id DESC -ON CONFLICT DO NOTHING; - -DO $$ -DECLARE - transfer_event RECORD; - closed_expiry timestamp; - registered_ticker text; - closed_created_block_id text; -BEGIN - FOR transfer_event IN - SELECT event_arg_1 AS ticker, event_arg_0 AS new_identity, block_id - FROM events - WHERE module_id = 'asset' - AND event_id_text = 'TickerTransferred' - ORDER BY block_id ASC - LOOP - IF EXISTS ( - SELECT 1 - FROM ticker_reservations - WHERE id = transfer_event.ticker - AND identity_id = transfer_event.new_identity - AND created_block_id = transfer_event.block_id - ) THEN - CONTINUE; - END IF; - - SELECT id, expiry, created_block_id INTO registered_ticker, closed_expiry, closed_created_block_id - FROM ticker_reservations - WHERE id = transfer_event.ticker - AND upper(_block_range) IS NULL - LIMIT 1; - - IF registered_ticker IS NOT NULL THEN - IF closed_created_block_id::bigint <= transfer_event.block_id::bigint THEN - UPDATE ticker_reservations tr - SET _block_range = int8range(closed_created_block_id::bigint, transfer_event.block_id::bigint), - updated_block_id = transfer_event.block_id - WHERE id = registered_ticker - AND upper(tr._block_range) IS NULL; - - INSERT INTO ticker_reservations - (id, ticker, identity_id, expiry, created_block_id, updated_block_id, _id, _block_range) - VALUES ( - transfer_event.ticker, - transfer_event.ticker, - transfer_event.new_identity, - closed_expiry, - transfer_event.block_id, - transfer_event.block_id, - uuid_generate_v4(), - int8range(transfer_event.block_id::bigint, NULL::bigint) - ); - ELSE - UPDATE ticker_reservations tr - SET identity_id = transfer_event.new_identity, - updated_block_id = transfer_event.block_id - WHERE id = registered_ticker - AND upper(tr._block_range) IS NULL; - END IF; - ELSE - -- Handle the case where the ticker wasn't registered yet - INSERT INTO ticker_reservations - (id, ticker, identity_id, expiry, created_block_id, updated_block_id, _id, _block_range) - VALUES ( - transfer_event.ticker, - transfer_event.ticker, - transfer_event.new_identity, - closed_expiry, - transfer_event.block_id, - transfer_event.block_id, - uuid_generate_v4(), - int8range(transfer_event.block_id::bigint, NULL::bigint) - ); - END IF; - END LOOP; -END $$; - -DO $$ -DECLARE - asset_event RECORD; - closed_identity_id text; - closed_expiry timestamp; - closed_created_block_id bigint; - registered_ticker text; - target_asset_id text; -BEGIN - FOR asset_event IN - SELECT event_arg_1 AS ticker, event_arg_2 AS asset_id, block_id, event_id_text - FROM events - WHERE module_id = 'asset' - AND event_id_text IN ('TickerLinkedToAsset', 'TickerUnlinkedFromAsset') - ORDER BY block_id ASC - LOOP - -- Determine the target asset_id - IF asset_event.event_id_text = 'TickerUnlinkedFromAsset' THEN - target_asset_id := NULL; - ELSE - target_asset_id := asset_event.asset_id; - END IF; - - -- Idempotency check: skip if this specific link/unlink has already been processed - IF EXISTS ( - SELECT 1 - FROM ticker_reservations - WHERE id = asset_event.ticker - AND ( - (asset_id = target_asset_id) OR - (asset_id IS NULL AND target_asset_id IS NULL) - ) - AND created_block_id = asset_event.block_id - ) THEN - CONTINUE; - END IF; - - -- Retrieve details from the currently active reservation before closing it - SELECT id, identity_id, expiry, created_block_id INTO registered_ticker, closed_identity_id, closed_expiry, closed_created_block_id - FROM ticker_reservations - WHERE id = asset_event.ticker - AND upper(_block_range) IS NULL - LIMIT 1; - - IF registered_ticker IS NOT NULL THEN - IF closed_created_block_id::bigint < asset_event.block_id::bigint THEN - UPDATE ticker_reservations tr - SET _block_range = int8range(tr.created_block_id::bigint, asset_event.block_id::bigint), - updated_block_id = asset_event.block_id - WHERE id = asset_event.ticker - AND upper(tr._block_range) IS NULL; - - INSERT INTO ticker_reservations - (id, ticker, identity_id, asset_id, expiry, created_block_id, updated_block_id, _id, _block_range) - VALUES ( - asset_event.ticker, - asset_event.ticker, - closed_identity_id, - target_asset_id, - closed_expiry, - asset_event.block_id, - asset_event.block_id, - uuid_generate_v4(), - int8range(asset_event.block_id::bigint, NULL::bigint) - ); - ELSE - UPDATE ticker_reservations tr - SET asset_id = target_asset_id, - updated_block_id = asset_event.block_id - WHERE id = asset_event.ticker - AND upper(tr._block_range) IS NULL; - END IF; - END IF; - END LOOP; -END $$; - - diff --git a/db/migrations/12_add_new_8_chain_events.sql b/db/migrations/12_add_new_8_chain_events.sql deleted file mode 100644 index 9a410897..00000000 --- a/db/migrations/12_add_new_8_chain_events.sql +++ /dev/null @@ -1,50 +0,0 @@ -alter type "7a0b4cc03e" add value if not exists 'beefy' after 'validators'; -alter type "7a0b4cc03e" add value if not exists 'revive' after 'beefy'; - -alter type "0bf3c7d4ef" add value if not exists 'self_register_did' after 'unlink_child_identity'; -alter type "0bf3c7d4ef" add value if not exists 'remove_key' after 'sudo_as'; -alter type "0bf3c7d4ef" add value if not exists 'set_mandatory_receiver_affirmation' after 'lock_instruction'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_funds' after 'set_mandatory_receiver_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'unlock_instruction' after 'transfer_funds'; -alter type "0bf3c7d4ef" add value if not exists 'approve_subsidy' after 'decrease_polyx_limit'; -alter type "0bf3c7d4ef" add value if not exists 'revoke_subsidy' after 'approve_subsidy'; -alter type "0bf3c7d4ef" add value if not exists 'accept_subsidy' after 'revoke_subsidy'; -alter type "0bf3c7d4ef" add value if not exists 'remove_subsidy' after 'accept_subsidy'; -alter type "0bf3c7d4ef" add value if not exists 'report_double_voting' after 'sumbit_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'report_double_voting_unsigned' after 'report_double_voting'; -alter type "0bf3c7d4ef" add value if not exists 'set_new_genesis' after 'report_double_voting_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'report_fork_voting' after 'set_new_genesis'; -alter type "0bf3c7d4ef" add value if not exists 'report_fork_voting_unsigned' after 'report_fork_voting'; -alter type "0bf3c7d4ef" add value if not exists 'report_future_block_voting' after 'report_fork_voting_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'report_future_block_voting_unsigned' after 'report_future_block_voting'; -alter type "0bf3c7d4ef" add value if not exists 'eth_transact' after 'report_future_block_voting_unsigned'; -alter type "0bf3c7d4ef" add value if not exists 'eth_instantiate_with_code' after 'eth_transact'; -alter type "0bf3c7d4ef" add value if not exists 'eth_call' after 'eth_instantiate_with_code'; -alter type "0bf3c7d4ef" add value if not exists 'eth_substrate_call' after 'eth_call'; -alter type "0bf3c7d4ef" add value if not exists 'map_account' after 'eth_substrate_call'; -alter type "0bf3c7d4ef" add value if not exists 'unmap_account' after 'map_account'; -alter type "0bf3c7d4ef" add value if not exists 'dispatch_as_fallback_account' after 'unmap_account'; - - -alter type "8f5a39c8ee" add value if not exists 'BurnedDebt' after 'Withdraw'; -alter type "8f5a39c8ee" add value if not exists 'BurnedHeld' after 'BurnedDebt'; -alter type "8f5a39c8ee" add value if not exists 'Held' after 'BurnedHeld'; -alter type "8f5a39c8ee" add value if not exists 'MintedCredit' after 'Held'; -alter type "8f5a39c8ee" add value if not exists 'Released' after 'MintedCredit'; -alter type "8f5a39c8ee" add value if not exists 'TransferAndHold' after 'Released'; -alter type "8f5a39c8ee" add value if not exists 'TransferOnHold' after 'TransferAndHold'; -alter type "8f5a39c8ee" add value if not exists 'Unexpected' after 'TransferOnHold'; -alter type "8f5a39c8ee" add value if not exists 'RootsPruned' after 'Slashed'; -alter type "8f5a39c8ee" add value if not exists 'RootStored' after 'RootsPruned'; -alter type "8f5a39c8ee" add value if not exists 'NFTHoldingsUpdated' after 'NFTPortfolioUpdated'; -alter type "8f5a39c8ee" add value if not exists 'NewQueued' after 'ValidatorReenabled'; -alter type "8f5a39c8ee" add value if not exists 'KeyRemoved' after 'KeyChanged'; -alter type "8f5a39c8ee" add value if not exists 'InstructionUnlocked' after 'VenueSignersUpdated'; -alter type "8f5a39c8ee" add value if not exists 'MandatoryReceiverAffirmationSet' after 'InstructionUnlocked'; -alter type "8f5a39c8ee" add value if not exists 'AcceptedSubsidy' after 'UpdatedPolyxLimit'; -alter type "8f5a39c8ee" add value if not exists 'ApprovedSubsidy' after 'AcceptedSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'RemovedPendingSubsidy' after 'ApprovedSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'RemovedSubsidy' after 'RemovedPendingSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'SubsidyDebited' after 'RemovedSubsidy'; -alter type "8f5a39c8ee" add value if not exists 'EthExtrinsicRevert' after 'RootStored'; - diff --git a/db/migrations/13_add_identity_to_asset_transactions.sql b/db/migrations/13_add_identity_to_asset_transactions.sql deleted file mode 100644 index f3f17704..00000000 --- a/db/migrations/13_add_identity_to_asset_transactions.sql +++ /dev/null @@ -1,33 +0,0 @@ -alter table "asset_transactions" add column if not exists "from_identity_id" text; -alter table "asset_transactions" add column if not exists "to_identity_id" text; - --- Backfill from portfolio IDs (format: did/portfolioNumber) -update "asset_transactions" -set "from_identity_id" = split_part("from_portfolio_id", '/', 1) -where "from_portfolio_id" is not null - and "from_identity_id" is null; - -update "asset_transactions" -set "to_identity_id" = split_part("to_portfolio_id", '/', 1) -where "to_portfolio_id" is not null - and "to_identity_id" is null; - --- Backfill account-based transactions from the accounts table. --- Note: this reflects the current identity association for the account, --- not necessarily the identity at the time of the transaction. -update "asset_transactions" at -set "from_identity_id" = a."identity_id" -from "accounts" a -where at."from_account" = a."id" - and at."from_identity_id" is null - and a."identity_id" is not null; - -update "asset_transactions" at -set "to_identity_id" = a."identity_id" -from "accounts" a -where at."to_account" = a."id" - and at."to_identity_id" is null - and a."identity_id" is not null; - -create index if not exists "asset_transactions_from_identity_id" on "asset_transactions" ("from_identity_id"); -create index if not exists "asset_transactions_to_identity_id" on "asset_transactions" ("to_identity_id"); diff --git a/db/migrations/14_confidential_assets_events_8_chain.sql b/db/migrations/14_confidential_assets_events_8_chain.sql deleted file mode 100644 index c40aa5b2..00000000 --- a/db/migrations/14_confidential_assets_events_8_chain.sql +++ /dev/null @@ -1,65 +0,0 @@ -alter type "7a0b4cc03e" add value if not exists 'didregistrars' after 'revive'; -alter type "7a0b4cc03e" add value if not exists 'polymeshtransactionpayment' after 'didregistrars'; -alter type "7a0b4cc03e" add value if not exists 'multiblockmigrations' after 'polymeshtransactionpayment'; -alter type "7a0b4cc03e" add value if not exists 'confidentialassets' after 'multiblockmigrations'; - -alter type "0bf3c7d4ef" add value if not exists 'force_set_cursor' after 'dispatch_as_fallback_account'; -alter type "0bf3c7d4ef" add value if not exists 'force_set_active_cursor' after 'force_set_cursor'; -alter type "0bf3c7d4ef" add value if not exists 'force_onboard_mbms' after 'force_set_active_cursor'; -alter type "0bf3c7d4ef" add value if not exists 'clear_historic' after 'force_onboard_mbms'; -alter type "0bf3c7d4ef" add value if not exists 'register_accounts' after 'clear_historic'; -alter type "0bf3c7d4ef" add value if not exists 'register_encryption_keys' after 'register_accounts'; -alter type "0bf3c7d4ef" add value if not exists 'register_account_assets' after 'register_encryption_keys'; -alter type "0bf3c7d4ef" add value if not exists 'mint_asset' after 'register_account_assets'; -alter type "0bf3c7d4ef" add value if not exists 'create_settlement' after 'mint_asset'; -alter type "0bf3c7d4ef" add value if not exists 'sender_affirmation' after 'create_settlement'; -alter type "0bf3c7d4ef" add value if not exists 'receiver_affirmation' after 'sender_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'mediator_affirmation' after 'receiver_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'sender_update_counter' after 'mediator_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'sender_revert_affirmation' after 'sender_update_counter'; -alter type "0bf3c7d4ef" add value if not exists 'receiver_revert_affirmation' after 'sender_revert_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'receiver_claim' after 'receiver_revert_affirmation'; -alter type "0bf3c7d4ef" add value if not exists 'batched_settlement' after 'receiver_claim'; -alter type "0bf3c7d4ef" add value if not exists 'register_fee_accounts' after 'batched_settlement'; -alter type "0bf3c7d4ef" add value if not exists 'topup_fee_accounts' after 'register_fee_accounts'; -alter type "0bf3c7d4ef" add value if not exists 'submit_batched_proofs' after 'topup_fee_accounts'; -alter type "0bf3c7d4ef" add value if not exists 'relayer_submit_batched_proofs' after 'submit_batched_proofs'; -alter type "0bf3c7d4ef" add value if not exists 'execute_instant_settlement' after 'relayer_submit_batched_proofs'; -alter type "0bf3c7d4ef" add value if not exists 'instant_sender_affirmation' after 'execute_instant_settlement'; -alter type "0bf3c7d4ef" add value if not exists 'instant_receiver_affirmation' after 'instant_sender_affirmation'; - -alter type "8f5a39c8ee" add value if not exists 'AllowanceSpent' after 'CreatedAssetTransfer'; -alter type "8f5a39c8ee" add value if not exists 'FundsTransferred' after 'TickerLinkedToAsset'; -alter type "8f5a39c8ee" add value if not exists 'AccountAssetRegistered' after 'EthExtrinsicRevert'; -alter type "8f5a39c8ee" add value if not exists 'AccountCurveTreeRootUpdated' after 'AccountAssetRegistered'; -alter type "8f5a39c8ee" add value if not exists 'AccountRegistered' after 'AccountCurveTreeRootUpdated'; -alter type "8f5a39c8ee" add value if not exists 'AccountStateLeafInserted' after 'AccountRegistered'; -alter type "8f5a39c8ee" add value if not exists 'AssetCurveTreeRootUpdated' after 'AccountStateLeafInserted'; -alter type "8f5a39c8ee" add value if not exists 'AssetMinted' after 'AssetCurveTreeRootUpdated'; -alter type "8f5a39c8ee" add value if not exists 'AssetStateLeafUpdated' after 'AssetMinted'; -alter type "8f5a39c8ee" add value if not exists 'AssetUpdated' after 'AssetStateLeafUpdated'; -alter type "8f5a39c8ee" add value if not exists 'EncryptionKeyRegistered' after 'AssetUpdated'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountCurveTreeRootUpdated' after 'EncryptionKeyRegistered'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountDeposited' after 'FeeAccountCurveTreeRootUpdated'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountStateLeafInserted' after 'FeeAccountDeposited'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountUpdated' after 'FeeAccountStateLeafInserted'; -alter type "8f5a39c8ee" add value if not exists 'FeeAccountWithdrawn' after 'FeeAccountUpdated'; -alter type "8f5a39c8ee" add value if not exists 'MediatorAffirmed' after 'FeeAccountWithdrawn'; -alter type "8f5a39c8ee" add value if not exists 'MediatorRejected' after 'MediatorAffirmed'; -alter type "8f5a39c8ee" add value if not exists 'ReceiverAffirmationReverted' after 'MediatorRejected'; -alter type "8f5a39c8ee" add value if not exists 'ReceiverAffirmed' after 'ReceiverAffirmationReverted'; -alter type "8f5a39c8ee" add value if not exists 'ReceiverClaimed' after 'ReceiverAffirmed'; -alter type "8f5a39c8ee" add value if not exists 'RelayerBatchedProofs' after 'ReceiverClaimed'; -alter type "8f5a39c8ee" add value if not exists 'SenderAffirmationReverted' after 'RelayerBatchedProofs'; -alter type "8f5a39c8ee" add value if not exists 'SenderAffirmed' after 'SenderAffirmationReverted'; -alter type "8f5a39c8ee" add value if not exists 'SenderCounterUpdated' after 'SenderAffirmed'; -alter type "8f5a39c8ee" add value if not exists 'SettlementCreated' after 'SenderCounterUpdated'; -alter type "8f5a39c8ee" add value if not exists 'SettlementStatusUpdated' after 'SettlementCreated'; -alter type "8f5a39c8ee" add value if not exists 'HistoricCleared' after 'SettlementStatusUpdated'; -alter type "8f5a39c8ee" add value if not exists 'MigrationAdvanced' after 'HistoricCleared'; -alter type "8f5a39c8ee" add value if not exists 'MigrationCompleted' after 'MigrationAdvanced'; -alter type "8f5a39c8ee" add value if not exists 'MigrationFailed' after 'MigrationCompleted'; -alter type "8f5a39c8ee" add value if not exists 'MigrationSkipped' after 'MigrationFailed'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeCompleted' after 'MigrationSkipped'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeFailed' after 'UpgradeCompleted'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeStarted' after 'UpgradeFailed'; diff --git a/db/migrations/15_add_missing_transfer_nft.sql b/db/migrations/15_add_missing_transfer_nft.sql deleted file mode 100644 index 386c8282..00000000 --- a/db/migrations/15_add_missing_transfer_nft.sql +++ /dev/null @@ -1 +0,0 @@ -alter type "0bf3c7d4ef" add value if not exists 'transfer_nft' after 'redeem_nft'; diff --git a/db/migrations/16_rename_submit_unsigned.sql b/db/migrations/16_rename_submit_unsigned.sql deleted file mode 100644 index 99d5a7b4..00000000 --- a/db/migrations/16_rename_submit_unsigned.sql +++ /dev/null @@ -1,12 +0,0 @@ -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM pg_enum e - JOIN pg_type t ON e.enumtypid = t.oid - WHERE t.typname = '0bf3c7d4ef' - AND e.enumlabel = 'sumbit_unsigned' - ) THEN - ALTER TYPE "0bf3c7d4ef" RENAME VALUE 'sumbit_unsigned' TO 'submit_unsigned'; - END IF; -END $$; \ No newline at end of file diff --git a/db/migrations/17_add_identity_to_portfolio_movement.sql b/db/migrations/17_add_identity_to_portfolio_movement.sql deleted file mode 100644 index 22b4a2f0..00000000 --- a/db/migrations/17_add_identity_to_portfolio_movement.sql +++ /dev/null @@ -1,19 +0,0 @@ -alter table "portfolio_movements" add column if not exists "identity_id" text; - --- Backfill from portfolio IDs (format: did/portfolioNumber) -update "portfolio_movements" -set "identity_id" = split_part("from_id", '/', 1) -where "from_id" is not null - and "identity_id" is null; - --- Backfill account-based transactions from the accounts table. --- Note: this reflects the current identity association for the account, --- not necessarily the identity at the time of the transaction. -update "portfolio_movements" at -set "identity_id" = a."identity_id" -from "accounts" a -where at."from_account" = a."id" - and at."identity_id" is null - and a."identity_id" is not null; - -create index if not exists "portfolio_movements_identity_id" on "portfolio_movements" ("identity_id"); diff --git a/db/migrations/18_add_reward_destination_to_staking_events.sql b/db/migrations/18_add_reward_destination_to_staking_events.sql deleted file mode 100644 index 4d6d8a80..00000000 --- a/db/migrations/18_add_reward_destination_to_staking_events.sql +++ /dev/null @@ -1,4 +0,0 @@ -alter table "staking_events" add column if not exists "reward_destination" text; -alter table "staking_events" add column if not exists "reward_destination_account" text; - -update "staking_events" set reward_destination = 'LegacyUnknown' where reward_destination is null and event_id in ('Reward', 'Rewarded'); diff --git a/db/migrations/19_add_instruction_unlocked_event_type.sql b/db/migrations/19_add_instruction_unlocked_event_type.sql deleted file mode 100644 index f972d578..00000000 --- a/db/migrations/19_add_instruction_unlocked_event_type.sql +++ /dev/null @@ -1 +0,0 @@ -alter type "3e29b3f361" add value if not exists 'InstructionUnlocked' after 'InstructionLocked'; diff --git a/db/migrations/1_fix_multi_sig_proposals_data.sql b/db/migrations/1_fix_multi_sig_proposals_data.sql deleted file mode 100644 index c38ab3e7..00000000 --- a/db/migrations/1_fix_multi_sig_proposals_data.sql +++ /dev/null @@ -1,48 +0,0 @@ --- convert all module names to lowercase -UPDATE multi_sig_proposals -SET params = jsonb_set( - params, - '{proposals}', - ( - SELECT jsonb_agg( - jsonb_set( - proposal, - '{module}', - to_jsonb(lower(proposal->>'module')) - ) - ) - FROM jsonb_array_elements(params->'proposals') AS proposal - ) -) -WHERE params->'proposals' IS NOT NULL; - --- correct the data format of 'params' when proposal with batch_all or batch_atomic is made -UPDATE multi_sig_proposals -SET params = jsonb_set( - jsonb_set(params, '{isBatch}', 'true'), - '{proposals}', - ( - SELECT jsonb_agg( - ( - elem - 'method' - 'section' -- Remove old keys - ) || jsonb_build_object( - 'args', elem->>'args', -- stringify args - 'call', ltrim( - lower( - regexp_replace( - elem->>'method', - '([A-Z])', - '_\1', - 'g' - ) - ), - '_' - ), -- Convert method to snake_case and set as 'call' - 'module', elem->>'section' -- Map section to module - ) - ) - FROM jsonb_array_elements(params->'proposals') AS proposal, - jsonb_array_elements((proposal->>'args')::jsonb->'calls') AS elem - ) -) -WHERE params->'proposals'->0->>'call' in ('batch_all', 'batch_atomic'); diff --git a/db/migrations/20_evm_transactions.sql b/db/migrations/20_evm_transactions.sql deleted file mode 100644 index df653097..00000000 --- a/db/migrations/20_evm_transactions.sql +++ /dev/null @@ -1,29 +0,0 @@ --- Support for indexing Ethereum transactions submitted through `revive.ethTransact`. --- --- The `evm_transactions` and `evm_account_mappings` tables, along with the `EvmCallKindEnum` type, --- are created by the node from `schema.graphql`. Only the two pre-existing tables need altering. - -alter table "extrinsics" add column if not exists "eth_address" text; -alter table "extrinsics" add column if not exists "eth_tx_hash" text; - -create index if not exists data_extrinsic_eth_address on extrinsics (eth_address); -create index if not exists data_extrinsic_eth_tx_hash on extrinsics (eth_tx_hash); - --- `key_type` cannot be derived here: telling an Ethereum key from a substrate one means base58 --- decoding the address and checking for the `0xEE` padding, which postgres has no built in for. --- Existing rows get a provisional 'substrate' so the column can be NOT NULL. --- --- That default is wrong for any Ethereum key attributed before this change - registering a DID or --- joining an identity emits the `0xEE` padded SS58 like any other key, and those accounts were --- indexed all along. `scripts/backfill/eth-transact-senders.ts` decodes every account offline and --- reclassifies them. -alter table "accounts" add column if not exists "key_type" text; -update "accounts" set "key_type" = 'substrate' where "key_type" is null; -alter table "accounts" alter column "key_type" set not null; - --- `evm_address` needs the same decode (plus keccak256 for substrate keys), so existing rows are --- left null. Every account has one - Ethereum keys drop their `0xEE` padding, substrate keys are --- hashed and truncated - and the same backfill fills them all, matching the forward path. -alter table "accounts" add column if not exists "evm_address" text; - -create index if not exists data_account_evm_address on accounts (evm_address); diff --git a/db/migrations/2_update_instruction_affirmation_id.sql b/db/migrations/2_update_instruction_affirmation_id.sql deleted file mode 100644 index 22e65a5f..00000000 --- a/db/migrations/2_update_instruction_affirmation_id.sql +++ /dev/null @@ -1,3 +0,0 @@ -update instruction_affirmations -set id = party_id || '/' || off_chain_receipt_id -where off_chain_receipt_id is not null; \ No newline at end of file diff --git a/db/migrations/3_update_failed_instruction_status.sql b/db/migrations/3_update_failed_instruction_status.sql deleted file mode 100644 index 8a74f07e..00000000 --- a/db/migrations/3_update_failed_instruction_status.sql +++ /dev/null @@ -1,3 +0,0 @@ -update instructions -set status = 'Failed' -where failure_reason is not null and status != 'Failed'; \ No newline at end of file diff --git a/db/migrations/4_add_new_events_extrinsics.sql b/db/migrations/4_add_new_events_extrinsics.sql deleted file mode 100644 index e0e4dae1..00000000 --- a/db/migrations/4_add_new_events_extrinsics.sql +++ /dev/null @@ -1,8 +0,0 @@ - -alter type "0bf3c7d4ef" add value if not exists 'set_disable_fees' after 'burn_account_balance'; -alter type "0bf3c7d4ef" add value if not exists 'initiate_corporate_action_and_ballot' after 'initiate_corporate_action_and_distribute'; -alter type "0bf3c7d4ef" add value if not exists 'update_global_metadata_spec' after 'unlink_ticker_from_asset_id'; - -alter type "8f5a39c8ee" add value if not exists 'GlobalMetadataSpecUpdated' after 'TickerUnlinkedFromAsset'; -alter type "8f5a39c8ee" add value if not exists 'AllowIdentityToCreatePortfolios' after 'FundsMovedBetweenPortfolios'; -alter type "8f5a39c8ee" add value if not exists 'RevokeCreatePortfoliosPermission' after 'AllowIdentityToCreatePortfolios'; diff --git a/db/migrations/5_add_enum_text_columns.sql b/db/migrations/5_add_enum_text_columns.sql deleted file mode 100644 index 30d09fa8..00000000 --- a/db/migrations/5_add_enum_text_columns.sql +++ /dev/null @@ -1,31 +0,0 @@ --- add unknown to CallIdEnum -alter type "0bf3c7d4ef" add value if not exists 'unknown' after 'set_proposal_duration'; - --- add unknown to EventIdEnum -alter type "8f5a39c8ee" add value if not exists 'Unknown' after 'VoteRejectReferendum'; - --- add unknown to ModuleIdEnum -alter type "7a0b4cc03e" add value if not exists 'unknown' after 'stocapped'; - -alter table events - add column if not exists module_id_text text, - add column if not exists event_id_text text; - -alter table extrinsics - add column if not exists module_id_text text, - add column if not exists call_id_text text; - -alter table polyx_transactions - add column if not exists event_id_text text, - add column if not exists call_id_text text, - add column if not exists module_id_text text; - -update events set event_id_text = event_id where event_id_text is null; -update events set module_id_text = module_id where module_id_text is null; - -update extrinsics set call_id_text = call_id where call_id_text is null; -update extrinsics set module_id_text = module_id where module_id_text is null; - -update polyx_transactions set event_id_text = event_id where event_id_text is null; -update polyx_transactions set call_id_text = call_id where call_id_text is null; -update polyx_transactions set module_id_text = module_id where module_id_text is null; diff --git a/db/migrations/6_new_events_for_7300000_spec.sql b/db/migrations/6_new_events_for_7300000_spec.sql deleted file mode 100644 index 3b190aef..00000000 --- a/db/migrations/6_new_events_for_7300000_spec.sql +++ /dev/null @@ -1,25 +0,0 @@ -alter type "0bf3c7d4ef" add value if not exists 'lock_instruction' after 'add_and_affirm_with_mediators'; -alter type "0bf3c7d4ef" add value if not exists 'enable_offchain_funding' after 'stop'; - -alter type "8f5a39c8ee" add value if not exists 'InstructionLocked' after 'InstructionMediators'; -alter type "8f5a39c8ee" add value if not exists 'FundraiserOffchainFundingEnabled' after 'FundraiserClosed'; - -alter table "public"."stos" add column if not exists "off_chain_funding_enabled" boolean not null default false; -alter table "public"."stos" add column if not exists "off_chain_funding_token" text; -alter table "public"."stos" alter column "raising_ticker" drop not null; - -alter type "b861be9158" add value if not exists 'Locked' after 'Failed'; - -alter type "7f3c7bae24" add value if not exists 'SettleAfterLock' after 'SettleManual'; - -DO $$ -BEGIN - IF NOT EXISTS (select 1 from pg_type where typname = '867b307be0') then - create type "867b307be0" AS ENUM ('OnChain', 'OffChain'); - END IF; -END -$$; - -alter table "public"."investments" add column if not exists "raising_asset_type" "867b307be0" not null default 'OnChain'; - -alter type "3e29b3f361" add value if not exists 'InstructionLocked' after 'InstructionFailed'; diff --git a/db/migrations/7_drop_ticker_null_constraints_on_investments.sql b/db/migrations/7_drop_ticker_null_constraints_on_investments.sql deleted file mode 100644 index f2816c91..00000000 --- a/db/migrations/7_drop_ticker_null_constraints_on_investments.sql +++ /dev/null @@ -1,2 +0,0 @@ -alter table investments alter column offering_token drop not null; -alter table investments alter column raise_token drop not null; \ No newline at end of file diff --git a/db/migrations/8_events_for_8000000_spec.sql b/db/migrations/8_events_for_8000000_spec.sql deleted file mode 100644 index 391462e2..00000000 --- a/db/migrations/8_events_for_8000000_spec.sql +++ /dev/null @@ -1,54 +0,0 @@ -alter type "7a0b4cc03e" add value if not exists 'validators' after 'electionprovidermultiphase'; - -alter type "0bf3c7d4ef" add value if not exists 'apply_authorized_upgrade' after 'placeholder_fill_block'; -alter type "0bf3c7d4ef" add value if not exists 'authorize_upgrade' after 'apply_authorized_upgrade'; -alter type "0bf3c7d4ef" add value if not exists 'authorize_upgrade_without_checks' after 'authorize_upgrade'; -alter type "0bf3c7d4ef" add value if not exists 'poke_deposit' after 'freeze'; -alter type "0bf3c7d4ef" add value if not exists 'force_set_balance' after 'burn_account_balance'; -alter type "0bf3c7d4ef" add value if not exists 'force_adjust_total_issuance' after 'force_set_balance'; -alter type "0bf3c7d4ef" add value if not exists 'force_unreserve' after 'force_adjust_total_issuance'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_all' after 'force_unreserve'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_allow_death' after 'transfer_all'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_keep_alive' after 'transfer_allow_death'; -alter type "0bf3c7d4ef" add value if not exists 'upgrade_accounts' after 'transfer_keep_alive'; -alter type "0bf3c7d4ef" add value if not exists 'update_payee' after 'set_staking_configs'; -alter type "0bf3c7d4ef" add value if not exists 'payout_stakers_by_page' after 'update_payee'; -alter type "0bf3c7d4ef" add value if not exists 'deprecate_controller_batch' after 'payout_stakers_by_page'; -alter type "0bf3c7d4ef" add value if not exists 'manual_slash' after 'deprecate_controller_batch'; -alter type "0bf3c7d4ef" add value if not exists 'migrate_currency' after 'manual_slash'; -alter type "0bf3c7d4ef" add value if not exists 'restore_ledger' after 'migrate_currency'; -alter type "0bf3c7d4ef" add value if not exists 'cancel_retry' after 'schedule_named_after'; -alter type "0bf3c7d4ef" add value if not exists 'cancel_retry_named' after 'cancel_retry'; -alter type "0bf3c7d4ef" add value if not exists 'set_retry' after 'cancel_retry_named'; -alter type "0bf3c7d4ef" add value if not exists 'set_retry_named' after 'set_retry'; -alter type "0bf3c7d4ef" add value if not exists 'migrate' after 'instantiate_old_weight'; -alter type "0bf3c7d4ef" add value if not exists 'ensure_updated' after 'unrequest_preimage'; - -alter type "8f5a39c8ee" add value if not exists 'RejectedInvalidAuthorizedUpgrade' after 'PlaceholderFillBlock'; -alter type "8f5a39c8ee" add value if not exists 'UpgradeAuthorized' after 'RejectedInvalidAuthorizedUpgrade'; -alter type "8f5a39c8ee" add value if not exists 'DepositPoked' after 'IndexFrozen'; -alter type "8f5a39c8ee" add value if not exists 'Deposit' after 'TransactionFeePaid'; -alter type "8f5a39c8ee" add value if not exists 'DustLost' after 'Deposit'; -alter type "8f5a39c8ee" add value if not exists 'Locked' after 'DustLost'; -alter type "8f5a39c8ee" add value if not exists 'Minted' after 'Locked'; -alter type "8f5a39c8ee" add value if not exists 'Rescinded' after 'Minted'; -alter type "8f5a39c8ee" add value if not exists 'Suspended' after 'Rescinded'; -alter type "8f5a39c8ee" add value if not exists 'Thawed' after 'Suspended'; -alter type "8f5a39c8ee" add value if not exists 'TotalIssuanceForced' after 'Thawed'; -alter type "8f5a39c8ee" add value if not exists 'TransferWithMemo' after 'TotalIssuanceForced'; -alter type "8f5a39c8ee" add value if not exists 'Unlocked' after 'TransferWithMemo'; -alter type "8f5a39c8ee" add value if not exists 'Upgraded' after 'Unlocked'; -alter type "8f5a39c8ee" add value if not exists 'Withdraw' after 'Upgraded'; -alter type "8f5a39c8ee" add value if not exists 'ControllerBatchDeprecated' after 'StakingElectionFailed'; -alter type "8f5a39c8ee" add value if not exists 'CurrencyMigrated' after 'ControllerBatchDeprecated'; -alter type "8f5a39c8ee" add value if not exists 'SnapshotTargetsSizeExceeded' after 'CurrencyMigrated'; -alter type "8f5a39c8ee" add value if not exists 'SnapshotVotersSizeExceeded' after 'SnapshotTargetsSizeExceeded'; -alter type "8f5a39c8ee" add value if not exists 'ValidatorDisabled' after 'NewSession'; -alter type "8f5a39c8ee" add value if not exists 'ValidatorReenabled' after 'ValidatorDisabled'; -alter type "8f5a39c8ee" add value if not exists 'AgendaIncomplete' after 'PermanentlyOverweight'; -alter type "8f5a39c8ee" add value if not exists 'RetryCancelled' after 'AgendaIncomplete'; -alter type "8f5a39c8ee" add value if not exists 'RetryFailed' after 'RetryCancelled'; -alter type "8f5a39c8ee" add value if not exists 'RetrySet' after 'RetryFailed'; -alter type "8f5a39c8ee" add value if not exists 'StorageDepositTransferredAndHeld' after 'DelegateCalled'; -alter type "8f5a39c8ee" add value if not exists 'StorageDepositTransferredAndReleased' after 'StorageDepositTransferredAndHeld'; - diff --git a/db/migrations/9_events_for_7400000_spec.sql b/db/migrations/9_events_for_7400000_spec.sql deleted file mode 100644 index 9615ac03..00000000 --- a/db/migrations/9_events_for_7400000_spec.sql +++ /dev/null @@ -1,5 +0,0 @@ -alter type "0bf3c7d4ef" add value if not exists 'receiver_affirm_asset_transfer' after 'update_global_metadata_spec'; -alter type "0bf3c7d4ef" add value if not exists 'reject_asset_transfer' after 'receiver_affirm_asset_transfer'; -alter type "0bf3c7d4ef" add value if not exists 'transfer_asset' after 'reject_asset_transfer'; - -alter type "8f5a39c8ee" add value if not exists 'CreatedAssetTransfer' after 'GlobalMetadataSpecUpdated'; From a0cf4ffe252a25f1888d0b50f7a1e9bcae475347 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:23:53 +0530 Subject: [PATCH 03/22] =?UTF-8?q?fix:=20=F0=9F=90=9B=20resolve=20accounts?= =?UTF-8?q?=20correctly=20when=20reindexing=20from=20the=20start?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When resolving which identity an address belongs to, the code always called a storage item that only exists on newer runtimes, so it threw on early blocks. It also created an identity without its default portfolio, which made a later identity-created event fail because the portfolio was missing. Both are now handled: the older storage item is used when the new one is absent, and the default portfolio is created alongside the identity. --- src/utils/accounts.ts | 72 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/src/utils/accounts.ts b/src/utils/accounts.ts index fbe377a5..be98638b 100644 --- a/src/utils/accounts.ts +++ b/src/utils/accounts.ts @@ -3,10 +3,18 @@ import { Codec } from '@polkadot/types/types'; import { u8aToHex } from '@polkadot/util'; import { getAccountCache } from '../mappings/blockContext'; import { createIdentity, createPermissions } from '../mappings/entities/identities/mapIdentities'; +import { createPortfolio } from '../mappings/entities/identities/mapPortfolio'; import { Attributes } from '../mappings/entities/common'; import { Account, EventIdEnum, Identity } from '../types'; -import { getFirstKeyFromJson, getFirstValueFromJson } from './common'; +import { + extractString, + getFirstKeyFromJson, + getFirstValueFromJson, + getTextValue, + padId, +} from './common'; import { evmAddressFromSs58, isEthDerivedAddress } from './eth'; +import { legacyQuery } from './legacyQuery'; export const serializeAccount = (item: Codec): string | undefined => { const s = item.toString(); @@ -36,6 +44,47 @@ export const getAccountKeyType = ( }; }; +/** + * The DID an address is a key of, and whether it is the primary key. + * + * `identity.keyRecords` is a 5.x rename of `identity.keyToIdentityIds`; a genesis resync sees the + * older name on early blocks. The legacy storage is `Option` on both public chains + * (Polymesh launched at v3, so there is no `LinkedKeyInfo` enum to unwrap), and primary vs + * secondary is read from `identity.didRecords`. + */ +const resolveKeyIdentity = async ( + address: string +): Promise<{ did: string; type: 'primaryKey' | 'secondaryKey' } | undefined> => { + if (typeof api.query.identity.keyRecords === 'function') { + const raw = (await api.query.identity.keyRecords(address)) as unknown as Codec; + + if (raw.isEmpty) { + return undefined; + } + + return { + did: getFirstValueFromJson(raw), + type: getFirstKeyFromJson(raw) === 'primaryKey' ? 'primaryKey' : 'secondaryKey', + }; + } + + const raw = (await legacyQuery( + 'identity', + 'keyToIdentityIds', + [3000, 5_000_002] + )(address)) as unknown as Codec; + + if (raw.isEmpty) { + return undefined; + } + + const did = getTextValue(raw); + const record = (await api.query.identity.didRecords(did)).toJSON() as Record; + const primaryKey = extractString(record, 'primary_key'); + + return { did, type: primaryKey === address ? 'primaryKey' : 'secondaryKey' }; +}; + /** * The `Account` an address belongs to, creating it and its identity when the chain knows of one. * @@ -65,22 +114,33 @@ export const getOrCreateAccount = async ( return account; } - const rawKeyRecord = (await api.query.identity.keyRecords(address)) as unknown as Codec; + const keyIdentity = await resolveKeyIdentity(address); - if (rawKeyRecord.isEmpty) { + if (!keyIdentity) { cache.set(address, undefined); return; } - const did = getFirstValueFromJson(rawKeyRecord); - const type = getFirstKeyFromJson(rawKeyRecord); + const { did, type } = keyIdentity; const eventId = EventIdEnum.AccountCreated; const identity = await Identity.get(did); - if (!identity || (type === 'primaryKey' && identity.primaryAccount !== address)) { + if (!identity) { + await createIdentity( + { did, eventId, datetime, primaryAccount: address, secondaryKeysFrozen: false }, + blockId + ); + + // The default portfolio, so a later `identity.DidCreated` for this DID finds it — its handler + // only creates portfolio 0 when it creates the identity, and this path got there first. + await createPortfolio( + { identityId: did, number: 0, eventIdx: 0, createdEventId: `${blockId}/${padId('0')}` }, + blockId + ); + } else if (type === 'primaryKey' && identity.primaryAccount !== address) { await createIdentity( { did, eventId, datetime, primaryAccount: address, secondaryKeysFrozen: false }, blockId From 9724123462757f5ead557a4ab63cc0d4f43d6cde Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:23:55 +0530 Subject: [PATCH 04/22] =?UTF-8?q?feat!:=20=F0=9F=8E=B8=20rebuild=20the=20P?= =?UTF-8?q?OLYX=20record=20as=20a=20double-entry=20ledger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the single PolyxTransaction table with two: PolyxEntry, one append-only row per account per side of a movement, each carrying the account's balance right after it; and AccountBalance, the running balance that can be read at any past block. Every balances event is decoded, turned into a move between the free and reserved pools, and written as entries plus an updated balance. This fixes a set of long-standing gaps: - the events that create and destroy reserved balance on the current runtime were never indexed, so reserved balance and staking bonds were invisible; - account reaping (dust) was not recorded; - a transfer-with-memo produced a spurious second row instead of just adding the memo to the real transfer; - an absolute "set balance" event was recorded as if it were a movement, corrupting every later total; it is now a checkpoint that records only the difference; - frozen balance is now the largest active lock rather than a sum, which the old model could not represent; - pre-upgrade staking bonds moved no balance (they were a lock) and no longer produce a movement row; on the current runtime the bond's balance effect comes from the paired hold/release events, so the staking events themselves are recorded as state only and not double-counted; - for rewards paid before the upgrade the event only named the stash, so the actual payee is now read from chain storage at the reward block; - opening balances are seeded from chain state at genesis, without which every derived balance would be off by the genesis allocation. BREAKING CHANGE: polyxTransactions is replaced by polyxEntries. Queries that matched an account across the from/to columns now filter on a single accountId. BalanceTypeEnum is gone; use PolyxEntry.pool, .kind and .direction. The same staking event name now means different things before and after the v8 upgrade (a lock versus a hold). --- db/compat.sql | 17 + project.ts | 31 +- schema.graphql | 170 ++- scripts/sync-metadata.ts | 9 +- src/decode/shapes/balances.ts | 73 + src/decode/shapes/index.ts | 2 + src/decode/shapes/staking.ts | 20 + .../entities/events/mapStakingEvent.ts | 14 +- .../entities/identities/mapPolyxLedger.ts | 1284 +++++++++++++++++ .../identities/mapPolyxTransaction.ts | 464 ------ .../entities/identities/reconcilePolyx.ts | 107 ++ src/mappings/entities/index.ts | 2 +- src/mappings/migrations/genesisHandler.ts | 5 + src/seed/accountBalance.ts | 67 + src/utils/common.ts | 12 - src/utils/index.ts | 1 + src/utils/staking.ts | 50 + tests/unit/extract8xStakingAmount.test.ts | 44 + tests/unit/mapPolyxLedger.test.ts | 498 +++++++ tests/unit/mapPolyxTransaction.test.ts | 192 --- tests/unit/rewardDestinationA15.test.ts | 139 ++ tests/unit/seedAccountBalances.test.ts | 98 ++ tests/unit/stakingEventHandlers.test.ts | 5 +- 23 files changed, 2591 insertions(+), 713 deletions(-) create mode 100644 src/decode/shapes/balances.ts create mode 100644 src/decode/shapes/staking.ts create mode 100644 src/mappings/entities/identities/mapPolyxLedger.ts delete mode 100644 src/mappings/entities/identities/mapPolyxTransaction.ts create mode 100644 src/mappings/entities/identities/reconcilePolyx.ts create mode 100644 src/seed/accountBalance.ts create mode 100644 src/utils/staking.ts create mode 100644 tests/unit/extract8xStakingAmount.test.ts create mode 100644 tests/unit/mapPolyxLedger.test.ts delete mode 100644 tests/unit/mapPolyxTransaction.test.ts create mode 100644 tests/unit/rewardDestinationA15.test.ts create mode 100644 tests/unit/seedAccountBalances.test.ts diff --git a/db/compat.sql b/db/compat.sql index cdf1abc6..92e00d6e 100644 --- a/db/compat.sql +++ b/db/compat.sql @@ -35,6 +35,23 @@ CREATE INDEX IF NOT EXISTS data_event_module_id_event_id_event_arg_2 ON events ( -- it reads exists in `schema.graphql`. CREATE INDEX IF NOT EXISTS data_event_transfer_from ON events (trim( '"' from attributes #>> '{2,value,did}')); +-- Denormalised filter columns on `events`. These would be `@index` in schema.graphql, but +-- `@subql/node` caps an entity at 10 indexes (`indexCountLimit`, not configurable) and Event is +-- already at the cap. Kept here, as `master` had them. +CREATE INDEX IF NOT EXISTS data_event_claim_type ON events (claim_type); +CREATE INDEX IF NOT EXISTS data_event_claim_scope ON events (claim_scope); +CREATE INDEX IF NOT EXISTS data_event_claim_issuer ON events (claim_issuer); +CREATE INDEX IF NOT EXISTS data_event_corporate_action_ticker ON events (corporate_action_ticker); +CREATE INDEX IF NOT EXISTS data_event_fundraiser_offering_asset ON events (fundraiser_offering_asset); + +-- Plain indexes that would otherwise be `@index` in schema.graphql but cannot be: `@subql/node` +-- caps an entity at 10 indexes (`indexCountLimit`, not configurable), and PolyxEntry is already +-- at the cap with its foreign keys and the three `@compositeIndexes`. These three back the +-- counterparty ("movements touching X"), era (reward/slash-per-era) and day-bucket queries. +CREATE INDEX IF NOT EXISTS data_polyx_entry_counterparty_address ON polyx_entries (counterparty_address); +CREATE INDEX IF NOT EXISTS data_polyx_entry_era_index ON polyx_entries (era_index); +CREATE INDEX IF NOT EXISTS data_polyx_entry_date ON polyx_entries (date); + -- Legacy views, dropped if an older deployment left them behind. DROP VIEW IF EXISTS data_block; DROP VIEW IF EXISTS data_event; diff --git a/project.ts b/project.ts index c2e33050..984baeae 100644 --- a/project.ts +++ b/project.ts @@ -54,28 +54,36 @@ const filters: Record> = { TransferWithData: [], }, balances: { + // POLYX ledger (docs/implementation/02-polyx-ledger.md). Every balances movement writes a + // PolyxEntry per account-side plus a running AccountBalance, from mapPolyxLedger.ts. AccountBalanceBurned: ['handleBalanceBurned'], BalanceSet: ['handleBalanceSet'], Burned: ['handleBalanceBurned'], - Deposit: ['handleBalanceDeposit'], - DustLost: [], + BurnedDebt: [], // issuance only — no account side + BurnedHeld: ['handleBalanceBurnedHeld'], + Deposit: ['handleBalanceMinted'], + DustLost: ['handleDustLost'], Endowed: ['handleBalanceEndowed'], Frozen: ['handleBalanceFrozen'], + Held: ['handleBalanceHeld'], Issued: [], Locked: ['handleBalanceLocked'], Minted: ['handleBalanceMinted'], + MintedCredit: [], // issuance only — no account side Rescinded: [], + Released: ['handleBalanceReleased'], Reserved: ['handleBalanceReserved'], ReserveRepatriated: ['handleReserveRepatriated'], Restored: ['handleBalanceMinted'], Slashed: ['handleBalanceBurned'], - // handleBalanceSuspended does not exist yet — it arrives with the POLYX ledger work - // (docs/implementation/02-polyx-ledger.md), which rewrites mapPolyxTransaction.ts wholesale - Suspended: [], - Thawed: [], + Suspended: ['handleBalanceSuspended'], // A3 — the handler did not exist before + Thawed: ['handleBalanceThawed'], TotalIssuanceForced: [], Transfer: ['handleBalanceTransfer'], - TransferWithMemo: ['handleBalanceTransfer'], + TransferAndHold: ['handleTransferAndHold'], + TransferOnHold: ['handleTransferOnHold'], + TransferWithMemo: ['handleBalanceTransferWithMemo'], // A2 — memo enrichment only, never its own entry + Unexpected: [], // anomaly marker; consider IndexerAnomaly Unlocked: ['handleBalanceUnlocked'], Unreserved: ['handleBalanceUnreserved'], Upgraded: [], @@ -238,7 +246,7 @@ const filters: Record> = { UserPortfolios: [], }, protocolFee: { - FeeCharged: ['handleFeeCharged'], + FeeCharged: ['handleTransactionFeeCharged'], }, settlement: { AffirmationWithdrawn: ['handleAffirmationWithdrawn'], @@ -285,14 +293,15 @@ const filters: Record> = { MinimumBondThresholdUpdated: [], Nominated: ['handleStakingEvent'], OldSlashingReportDiscarded: [], - PayoutStarted: [], + // supplies the `eraIndex` that `Rewarded` lacks; consumed by the POLYX ledger's handleReward + PayoutStarted: ['handlePayoutStarted'], PermissionedIdentityAdded: [], PermissionedIdentityRemoved: [], Reward: ['handleStakingEvent', 'handleReward'], Rewarded: ['handleStakingEvent', 'handleReward'], RewardPaymentSchedulingInterrupted: [], - Slash: ['handleStakingEvent'], - Slashed: ['handleStakingEvent'], + Slash: ['handleStakingEvent', 'handleStakingSlash'], + Slashed: ['handleStakingEvent', 'handleStakingSlash'], SlashReported: [], SlashingAllowedForChanged: [], SnapshotTargetsSizeExceeded: [], diff --git a/schema.graphql b/schema.graphql index 300bc6f7..71feee98 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1935,12 +1935,15 @@ type Event @entity @compositeIndexes(fields: [["moduleId", "eventId"]]) { eventArg_1: String eventArg_2: String eventArg_3: String - claimType: String @index(unique: false) - claimScope: String @index(unique: false) - claimIssuer: String @index(unique: false) + # The five denormalised filter columns below are indexed in db/compat.sql, not with `@index`: + # `@subql/node` caps an entity at 10 indexes and Event is at that cap with its foreign keys, + # eventIdx/extrinsicIdx/specVersionId, moduleId/eventId and the `[moduleId, eventId]` composite. + claimType: String + claimScope: String + claimIssuer: String claimExpiry: String - corporateActionTicker: String @index(unique: false) - fundraiserOfferingAsset: String @index(unique: false) + corporateActionTicker: String + fundraiserOfferingAsset: String transferTo: String extrinsic: Extrinsic } @@ -3041,39 +3044,156 @@ type Migration @entity { processedBlock: Int! } -""" -Represents possible all possible balance types -""" -enum BalanceTypeEnum { +# --------------------------------------------------------------------------- +# POLYX ledger (plan 02). PolyxEntry + AccountBalance replace PolyxTransaction +# and BalanceTypeEnum, which are removed once the reconciliation harness is +# green. Until then both models exist side by side. +# --------------------------------------------------------------------------- + +"The two balance pools an account holds on chain. `Bonded`/`Unbonded`/`Locked` are not pools and are not represented here." +enum PolyxPool { Free Reserved - Bonded - Unbonded - Locked +} + +"Which side of a movement an entry records." +enum EntryDirection { + Debit + Credit +} + +"Why POLYX is held in `reserved` under the v8 fungible-Holds API." +enum HoldReason { + Staking + Session + Preimage + Revive + Unknown +} + +"What an entry's movement represents. Denormalised onto the entry so aggregation does not need to traverse relations." +enum MovementKind { + Transfer + Endowment + Fee + Tip + TreasuryDisbursement + TreasuryReimbursement + StakingReward + Slash + Mint + Burn + DustLost + ReserveRepatriation + Hold + Release + BalanceSetAdjustment +} + +type KindTotal @jsonField { + "a `MovementKind` value — jsonField members cannot be enum-typed, same as `Scope.type` etc." + kind: String! + totalAbs: BigInt! + net: BigInt! + count: Int! +} + +type LockEntry @jsonField { + lockId: String! + amount: BigInt! + reasons: String +} + +type HoldEntry @jsonField { + "a `HoldReason` value — jsonField members cannot be enum-typed" + reason: String! + amount: BigInt! } """ -Represents transactions involving POLYX +One row per (account, movement side). Append-only. +Sibling entries of one on-chain movement share `movementId`. """ -type PolyxTransaction @entity { +type PolyxEntry @entity @compositeIndexes(fields: [["account", "date"], ["account", "kind"], ["kind", "date"]]) { + "padId(block)/padId(eventIdx)/side — D4" id: ID! - identityId: String - address: String @index(unique: false) - toId: String - toAddress: String @index(unique: false) + movementId: String! @index + + account: Account! + identity: Identity + "indexed in db/compat.sql — the schema is at the platform's 10-index-per-entity cap" + counterpartyAddress: String + counterpartyIdentity: Identity + + pool: PolyxPool! + "signed: negative = debit, positive = credit. SUM = net delta" amount: BigInt! - type: BalanceTypeEnum! + "unsigned. SUM = gross volume" + amountAbs: BigInt! + + kind: MovementKind! + "materialised — sign(amount) is not groupable (§8b)" + direction: EntryDirection! + holdReason: HoldReason + memo: String + + "balance after this entry — powers charts with no aggregation" + freeAfter: BigInt! + reservedAfter: BigInt! + frozenAfter: BigInt! + + "denormalised — aggregation cannot traverse relations" moduleId: ModuleIdEnum - moduleIdText: String callId: CallIdEnum - callIdText: String - eventId: EventIdEnum - eventIdText: String - memo: String + eventId: EventIdEnum! + specVersionId: Int! + "materialised time buckets — no date_trunc grouping exists (§8b). Indexed via the `[kind, date]` / `[account, date]` composites and a standalone index in db/compat.sql" + date: Date! + "indexed in db/compat.sql — the schema is at the platform's 10-index-per-entity cap" + eraIndex: Int + + "relation is `createdEvent` to avoid colliding with the denormalised `eventId` enum column, same as `AssetTransaction`" + createdEvent: Event! extrinsic: Extrinsic - datetime: Date! eventIdx: Int! + datetime: Date! createdBlock: Block! +} + +""" +Materialised POLYX balance for one account. Versioned on `historical: 'height'`, +so `accountBalance(id: X, blockHeight: N)` gives the balance at block N. +""" +type AccountBalance @entity { + "address" + id: ID! + account: Account! + identity: Identity @index + + free: BigInt! + reserved: BigInt! + "MAX over active locks — maintained from `locks`, never summed" + frozen: BigInt! + total: BigInt! + "free - frozen, floored at 0" + transferable: BigInt! + bonded: BigInt! + otherReserved: BigInt! + + "lifetime totals — free, this row is already versioned on these blocks" + totalReceived: BigInt! + totalSent: BigInt! + totalFeesPaid: BigInt! + totalRewards: BigInt! + totalSlashed: BigInt! + movementCount: Int! + lifetimeByKind: [KindTotal] + + "per-lock detail — required because frozen is a MAX" + locks: [LockEntry] + "v8 only; SUM = reserved" + holds: [HoldEntry] + updatedBlock: Block! } diff --git a/scripts/sync-metadata.ts b/scripts/sync-metadata.ts index def77cc6..3be8c431 100644 --- a/scripts/sync-metadata.ts +++ b/scripts/sync-metadata.ts @@ -33,7 +33,14 @@ const SCHEMA_PATH = join(ROOT, 'schema.graphql'); const ARITY_DIR = join(ROOT, 'tests', 'fixtures', 'event-arity'); /** The pallets whose event shapes the decode layer registers, and so the ones worth capturing */ -const CAPTURED_MODULES = ['asset', 'externalAgents', 'identity', 'settlement']; +const CAPTURED_MODULES = [ + 'asset', + 'balances', + 'externalAgents', + 'identity', + 'settlement', + 'staking', +]; export interface RuntimeSnapshot { specName: string; diff --git a/src/decode/shapes/balances.ts b/src/decode/shapes/balances.ts new file mode 100644 index 00000000..8fc447cc --- /dev/null +++ b/src/decode/shapes/balances.ts @@ -0,0 +1,73 @@ +import { LAST_V7 } from './consts'; +import { discontinuedAt, registerShape } from './registry'; + +/** + * `balances` pallet parameter shapes. + * + * Polymesh ran a custom `balances` pallet with tuple-style events through v7.4. v8.0.0 deleted it + * and moved to the upstream Substrate pallet, whose events are struct-style and decode straight + * from the block metadata (`namedFields`) with no entry here. So every shape below closes at the + * last 7.x spec version. + * + * Names come from the Rust event definitions (`pallets/balances/src/lib.rs` @ v7.4.0), + * cross-checked against `docs/reference/event-shape-verification.md` and the positional reads the + * pre-ledger `mapPolyxTransaction` handlers relied on. + */ + +// Endowed(IdentityId, AccountId, Balance) +registerShape('balances', 'Endowed', discontinuedAt(LAST_V7, ['identityId', 'account', 'balance'])); + +// Transfer(Option, AccountId, Option, AccountId, Balance, Option) +// The memo tail is absent on the transfers that did not carry one, so arity is 5 or 6. +registerShape('balances', 'Transfer', [ + { + from: 0, + to: LAST_V7, + fields: ['fromIdentityId', 'from', 'toIdentityId', 'to', 'amount', 'memo'], + optionalFrom: 5, + }, +]); + +// TransferWithMemo(from, to, amount, memo) — introduced v7.4.0 only, emitted alongside `Transfer` +registerShape( + 'balances', + 'TransferWithMemo', + discontinuedAt(LAST_V7, ['from', 'to', 'amount', 'memo']) +); + +// Reserved(AccountId, Balance) / Unreserved(AccountId, Balance) +registerShape('balances', 'Reserved', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Unreserved', discontinuedAt(LAST_V7, ['account', 'amount'])); + +// ReserveRepatriated(AccountId, AccountId, Balance, BalanceStatus) +registerShape( + 'balances', + 'ReserveRepatriated', + discontinuedAt(LAST_V7, ['from', 'to', 'amount', 'destinationStatus']) +); + +// BalanceSet(IdentityId, AccountId, free, reserved) — reserved is index 3 (defect A1) +registerShape( + 'balances', + 'BalanceSet', + discontinuedAt(LAST_V7, ['identityId', 'account', 'free', 'reserved']) +); + +// AccountBalanceBurned(IdentityId, AccountId, Balance) — the Polymesh burn event; no v8 equivalent +registerShape( + 'balances', + 'AccountBalanceBurned', + discontinuedAt(LAST_V7, ['identityId', 'account', 'amount']) +); + +// Deposit(AccountId, Balance) +registerShape('balances', 'Deposit', discontinuedAt(LAST_V7, ['account', 'amount'])); + +// 2-arg (AccountId, Balance) events that also existed pre-v8. Defensive: if a runtime emitted +// these as tuples the decoder covers them, and at v8 the struct-style metadata is used instead. +registerShape('balances', 'Burned', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Slashed', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Withdraw', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Minted', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'Restored', discontinuedAt(LAST_V7, ['account', 'amount'])); +registerShape('balances', 'DustLost', discontinuedAt(LAST_V7, ['account', 'amount'])); diff --git a/src/decode/shapes/index.ts b/src/decode/shapes/index.ts index b40d3a87..348337aa 100644 --- a/src/decode/shapes/index.ts +++ b/src/decode/shapes/index.ts @@ -13,9 +13,11 @@ * Importing this module registers every shape, so it is imported for its side effects. */ import './asset'; +import './balances'; import './externalAgents'; import './identity'; import './settlement'; +import './staking'; export * from './consts'; export * from './registry'; diff --git a/src/decode/shapes/staking.ts b/src/decode/shapes/staking.ts new file mode 100644 index 00000000..82a0d98c --- /dev/null +++ b/src/decode/shapes/staking.ts @@ -0,0 +1,20 @@ +import { LAST_V7 } from './consts'; +import { discontinuedAt, registerShape } from './registry'; + +/** + * `staking` pallet parameter shapes for the pre-v8 (Polymesh custom staking pallet) tuple events. + * + * v8.0.0 deleted the custom pallet and moved to upstream Substrate staking, whose events are + * struct-style and decode from block metadata directly. So every shape here closes at the last + * 7.x spec version. + * + * `Bonded` / `Unbonded` / `Reward` / `Rewarded` carried the `IdentityId` as their first parameter + * through v7.4.0; `Withdrawn` and `Slash` / `Slashed` never did (verified — defect log §C). + */ +registerShape('staking', 'Bonded', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Unbonded', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Reward', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Rewarded', discontinuedAt(LAST_V7, ['identityId', 'stash', 'amount'])); +registerShape('staking', 'Withdrawn', discontinuedAt(LAST_V7, ['stash', 'amount'])); +registerShape('staking', 'Slash', discontinuedAt(LAST_V7, ['stash', 'amount'])); +registerShape('staking', 'Slashed', discontinuedAt(LAST_V7, ['stash', 'amount'])); diff --git a/src/mappings/entities/events/mapStakingEvent.ts b/src/mappings/entities/events/mapStakingEvent.ts index 413c6666..1f86681d 100644 --- a/src/mappings/entities/events/mapStakingEvent.ts +++ b/src/mappings/entities/events/mapStakingEvent.ts @@ -4,6 +4,7 @@ import { SubstrateBlock, SubstrateEvent } from '@subql/types'; import { Account, EventIdEnum, StakingEvent } from '../../../types'; import { getBigIntValue, getTextValue } from '../../../utils'; import { is8xChain } from '../../../utils/common'; +import { resolveLegacyRewardDestination } from '../../../utils/staking'; import { extractArgs } from '../common'; const bondedUnbondedOrReward = new Set([ @@ -107,10 +108,10 @@ const get8xStakingEventDetails = (eventId: EventIdEnum, params: Codec[]): Stakin return { stashAccount }; }; -const getLegacyStakingEventDetails = ( +const getLegacyStakingEventDetails = async ( eventId: EventIdEnum, params: Codec[] -): StakingEventDetails => { +): Promise => { const [rawDid, rawAccount] = params; const stashAccount = getTextValue(rawAccount); const details: StakingEventDetails = { @@ -121,8 +122,11 @@ const getLegacyStakingEventDetails = ( if (bondedUnbondedOrReward.has(eventId)) { details.amount = getBigIntValue(params[2]); - if (eventId === EventIdEnum.Reward || eventId === EventIdEnum.Rewarded) { - details.rewardDestination = 'LegacyUnknown'; + if ((eventId === EventIdEnum.Reward || eventId === EventIdEnum.Rewarded) && stashAccount) { + // A15 — the pre-v8 event names only the stash; read the payee from chain storage. + const resolved = await resolveLegacyRewardDestination(stashAccount); + details.rewardDestination = resolved.rewardDestination; + details.rewardDestinationAccount = resolved.rewardDestinationAccount; } } @@ -143,7 +147,7 @@ const getStakingEventDetails = async ( } else if (is8xChain(block)) { details = get8xStakingEventDetails(eventId, params); } else { - details = getLegacyStakingEventDetails(eventId, params); + details = await getLegacyStakingEventDetails(eventId, params); } if (details.stashAccount && !details.identityId) { diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts new file mode 100644 index 00000000..9bda51d6 --- /dev/null +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -0,0 +1,1284 @@ +import { Codec } from '@polkadot/types/types'; +import { SubstrateEvent } from '@subql/types'; +import { decodeEvent } from '../../../decode'; +import { + Account, + AccountBalance, + EntryDirection, + EventIdEnum, + HoldReason, + Identity, + MovementKind, + PolyxEntry, + PolyxPool, +} from '../../../types'; +import { bytesToString, getBigIntValue, getTextValue, padId } from '../../../utils'; +import { camelToSnakeCase, is8xChain, snakeToCamelCase } from '../../../utils/common'; +import { resolveLegacyRewardDestination } from '../../../utils/staking'; +import { getAccountKeyType, getOrCreateAccount } from '../../../utils/accounts'; +import { getEventParams } from '../../../utils/events'; +import { extractArgs, HandlerArgs } from '../common'; +import { reconcileAccount } from './reconcilePolyx'; + +/** + * POLYX ledger — entry-centric replacement for `mapPolyxTransaction`. + * + * Every balances-pallet movement decodes to a pool transition (the "Event → pool transition" + * table in docs/implementation/02-polyx-ledger.md), which this module turns into one `PolyxEntry` + * per account-side plus a running `AccountBalance`. `BalanceSet` (a checkpoint), locks and staking + * are layered on in the later commits of the phase. + */ + +// --------------------------------------------------------------------------------------------- +// Decoded-field helpers +// --------------------------------------------------------------------------------------------- + +/** + * A decoded field, tolerating the snake_case ⇄ camelCase difference between an upstream struct + * event (`free_balance`) and a Polymesh shape-table entry (`freeBalance`). Returns `undefined` + * rather than letting the decode proxy throw when the field is genuinely absent. + */ +const optionalField = (decoded: Record, name: string): Codec | undefined => { + for (const candidate of new Set([name, camelToSnakeCase(name), snakeToCamelCase(name)])) { + if (candidate in decoded) { + return decoded[candidate]; + } + } + + return undefined; +}; + +const firstText = (decoded: Record, names: string[]): string | undefined => { + for (const name of names) { + const value = optionalField(decoded, name); + + if (value !== undefined) { + return getTextValue(value); + } + } + + return undefined; +}; + +const holder = (decoded: Record): string | undefined => + firstText(decoded, ['who', 'account', 'stash']); + +const amountOf = (decoded: Record): bigint => { + for (const name of ['amount', 'balance', 'freeBalance', 'free', 'value', 'actualFee']) { + const value = optionalField(decoded, name); + + if (value !== undefined) { + return getBigIntValue(value); + } + } + + return BigInt(0); +}; + +const holdReasonOf = (decoded: Record): HoldReason | undefined => { + const raw = optionalField(decoded, 'reason'); + + if (raw === undefined) { + return undefined; + } + + const key = getTextValue(raw)?.toLowerCase(); + const match = Object.values(HoldReason).find(member => member.toLowerCase() === key); + + return match ?? HoldReason.Unknown; +}; + +const memoOf = (decoded: Record): string | undefined => { + const raw = optionalField(decoded, 'memo'); + + return raw !== undefined ? bytesToString(raw) : undefined; +}; + +const startOfUtcDay = (datetime: Date): Date => + new Date(Date.UTC(datetime.getUTCFullYear(), datetime.getUTCMonth(), datetime.getUTCDate())); + +const floorZero = (value: bigint): bigint => (value > BigInt(0) ? value : BigInt(0)); + +/** + * `frozen` from an on-chain `AccountData`: `{ free, reserved, frozen, flags }` on v8, + * `{ free, reserved, miscFrozen, feeFrozen }` (frozen is the max of the two) on ≤ v7.4. + */ +export const accountDataFrozen = (data: Record): bigint => { + if (data.frozen !== undefined) { + return getBigIntValue(data.frozen); + } + + const misc = getBigIntValue(data.miscFrozen); + const fee = getBigIntValue(data.feeFrozen); + + return misc > fee ? misc : fee; +}; + +// --------------------------------------------------------------------------------------------- +// Account / balance state +// --------------------------------------------------------------------------------------------- + +/** + * The `Account` a POLYX-holding address belongs to. + * + * `getOrCreateAccount` covers every address the chain has a key record for. A pallet or system + * address (the treasury pot, the block-reward pot, …) holds POLYX without being a key, so a bare + * `Account` is created for it — `PolyxEntry.account` and `AccountBalance.account` are non-null + * relations and the account page query is keyed on them. + */ +export const ledgerAccount = async ( + address: string, + blockId: string, + datetime: Date +): Promise => { + const resolved = await getOrCreateAccount(address, blockId, datetime); + + if (resolved) { + return resolved; + } + + const account = Account.create({ + id: address, + address, + eventId: EventIdEnum.AccountCreated, + datetime, + ...getAccountKeyType(address), + createdBlockId: blockId, + updatedBlockId: blockId, + }); + + await account.save(); + + return account; +}; + +export const emptyBalance = ( + address: string, + identityId: string | undefined, + blockId: string +): AccountBalance => + AccountBalance.create({ + id: address, + accountId: address, + identityId, + free: BigInt(0), + reserved: BigInt(0), + frozen: BigInt(0), + total: BigInt(0), + transferable: BigInt(0), + bonded: BigInt(0), + otherReserved: BigInt(0), + totalReceived: BigInt(0), + totalSent: BigInt(0), + totalFeesPaid: BigInt(0), + totalRewards: BigInt(0), + totalSlashed: BigInt(0), + movementCount: 0, + lifetimeByKind: [], + locks: [], + holds: [], + updatedBlockId: blockId, + }); + +export const loadBalance = async ( + address: string, + identityId: string | undefined, + blockId: string +): Promise => { + const existing = await AccountBalance.get(address); + + if (existing) { + if (!existing.identityId && identityId) { + existing.identityId = identityId; + } + + return existing; + } + + return emptyBalance(address, identityId, blockId); +}; + +/** Pre-v8 staking bonds via a lock with this identifier; v8 bonds via a `Staking` hold. */ +export const STAKING_LOCK_ID = 'staking '; + +/** + * Recomputes every field that is a pure function of the pools, `locks` and `holds`: + * + * - `frozen` is the **MAX** over active locks, never a sum — the property the old model could not + * represent. + * - `bonded` is the staking lock (≤ v7.4) or the `Staking` hold (v8). + * - `total = free + reserved`; `transferable = free - frozen`, floored at 0. + */ +export const recomputeDerived = (balance: AccountBalance): void => { + const locks = balance.locks ?? []; + const holds = balance.holds ?? []; + + const stakingHold = holds.find(hold => hold.reason === HoldReason.Staking)?.amount ?? BigInt(0); + const stakingLock = locks.find(lock => lock.lockId === STAKING_LOCK_ID)?.amount ?? BigInt(0); + + balance.frozen = locks.reduce((max, lock) => (lock.amount > max ? lock.amount : max), BigInt(0)); + balance.bonded = stakingHold > stakingLock ? stakingHold : stakingLock; + balance.otherReserved = floorZero(balance.reserved - stakingHold); + balance.total = balance.free + balance.reserved; + balance.transferable = floorZero(balance.free - balance.frozen); +}; + +const bumpLifetimeByKind = ( + balance: AccountBalance, + kind: MovementKind, + direction: EntryDirection, + amountAbs: bigint +): void => { + const totals = balance.lifetimeByKind ?? []; + const signed = direction === EntryDirection.Credit ? amountAbs : -amountAbs; + const entry = totals.find(total => total.kind === kind); + + if (entry) { + entry.totalAbs += amountAbs; + entry.net += signed; + entry.count += 1; + } else { + totals.push({ kind, totalAbs: amountAbs, net: signed, count: 1 }); + } + + balance.lifetimeByKind = totals; +}; + +// --------------------------------------------------------------------------------------------- +// Pool transition → entries + balance mutation +// --------------------------------------------------------------------------------------------- + +interface Endpoint { + address: string; + pool: PolyxPool; +} + +interface Transition { + /** debit side; absent when value entered the system (mint, endow, reward) */ + from?: Endpoint; + /** credit side; absent when value left the system (burn, slash, fee, dust) */ + to?: Endpoint; + amount: bigint; + kind: MovementKind; + holdReason?: HoldReason; + memo?: string; +} + +const poolTag = (pool: PolyxPool): string => (pool === PolyxPool.Free ? 'f' : 'r'); + +/** + * Writes the entries for one pool transition and advances every touched `AccountBalance`. + * + * Sibling entries of one movement share `movementId` (the block/event id). Each entry carries the + * balance-after snapshot, so Balance History is a pure index scan. + */ +export const postTransition = async ( + args: HandlerArgs, + transition: Transition, + options: { eraIndex?: number } = {} +): Promise => { + if (!transition.from && !transition.to) { + return; + } + + const { blockId, block, eventIdx, blockEventId } = args; + const datetime = block.timestamp; + const params = getEventParams(args); + const date = startOfUtcDay(datetime); + + const isInternal = + transition.from !== undefined && + transition.to !== undefined && + transition.from.address === transition.to.address; + + const sides: Array<{ endpoint: Endpoint; direction: EntryDirection; counterparty?: string }> = []; + + if (transition.from?.address) { + sides.push({ + endpoint: transition.from, + direction: EntryDirection.Debit, + counterparty: transition.to?.address, + }); + } + + if (transition.to?.address) { + sides.push({ + endpoint: transition.to, + direction: EntryDirection.Credit, + counterparty: transition.from?.address, + }); + } + + for (const side of sides) { + const { address, pool } = side.endpoint; + const signed = + side.direction === EntryDirection.Credit ? transition.amount : -transition.amount; + + const account = await ledgerAccount(address, blockId, datetime); + const balance = await loadBalance(address, account.identityId, blockId); + + if (pool === PolyxPool.Free) { + balance.free += signed; + } else { + balance.reserved += signed; + } + + if (!isInternal) { + if (side.direction === EntryDirection.Credit) { + balance.totalReceived += transition.amount; + } else { + balance.totalSent += transition.amount; + } + } + + if (transition.kind === MovementKind.Fee) { + balance.totalFeesPaid += transition.amount; + } else if (transition.kind === MovementKind.StakingReward) { + balance.totalRewards += transition.amount; + } else if (transition.kind === MovementKind.Slash) { + balance.totalSlashed += transition.amount; + } + + bumpLifetimeByKind(balance, transition.kind, side.direction, transition.amount); + balance.movementCount += 1; + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); + + const counterpartyAccount = side.counterparty + ? await Account.get(side.counterparty) + : undefined; + + await PolyxEntry.create({ + id: `${blockId}/${padId(eventIdx.toString())}/${poolTag(pool)}${ + side.direction === EntryDirection.Debit ? 'd' : 'c' + }`, + movementId: blockEventId, + accountId: address, + identityId: account.identityId, + counterpartyAddress: side.counterparty, + counterpartyIdentityId: counterpartyAccount?.identityId, + pool, + amount: signed, + amountAbs: transition.amount, + kind: transition.kind, + direction: side.direction, + holdReason: transition.holdReason, + memo: transition.memo, + freeAfter: balance.free, + reservedAfter: balance.reserved, + frozenAfter: balance.frozen, + moduleId: params.moduleId, + callId: params.callId, + eventId: params.eventId, + specVersionId: block.specVersion, + date, + eraIndex: options.eraIndex, + createdEventId: blockEventId, + extrinsicId: params.extrinsicId, + eventIdx, + datetime, + createdBlockId: blockId, + }).save(); + + await reconcileAccount(address, blockId, block, { eventIdx }); + } +}; + +/** + * v8-only hold tracking. `Held`/`Released`/`BurnedHeld` move the balance through `postTransition`; + * this keeps the per-reason breakdown in `AccountBalance.holds` so `bonded`/`otherReserved` stay + * derivable without a scan. On a v8 chain `SUM(holds) == reserved`. + */ +const adjustHold = async ( + address: string, + reason: HoldReason, + delta: bigint, + blockId: string +): Promise => { + const balance = await AccountBalance.get(address); + + if (!balance) { + return; + } + + const holds = balance.holds ?? []; + const entry = holds.find(hold => hold.reason === reason); + + if (entry) { + entry.amount = floorZero(entry.amount + delta); + } else if (delta > BigInt(0)) { + holds.push({ reason, amount: delta }); + } + + balance.holds = holds.filter(hold => hold.amount > BigInt(0)); + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); +}; + +// --------------------------------------------------------------------------------------------- +// Locks (Locked / Unlocked / Frozen / Thawed) — a floor on `free`, not a pool. No PolyxEntry. +// --------------------------------------------------------------------------------------------- + +/** + * Adjusts one lock on `address` by `delta`, then recomputes `frozen = MAX(active locks)`. + * + * Locks aggregate by maximum, not by sum: two overlapping locks of 100 and 150 leave + * `frozen = 150`. This is why each lock is tracked individually in `AccountBalance.locks` rather + * than folded into a single number. + * + * PIPs vote locks (`PIPS_LOCK_ID`) are not wired here — the pips pallet emits no lock/unlock + * event, only `Voted`, and attributing the deposit needs the proposal-deposit model. Follow-up. + */ +export const adjustLock = async ( + address: string, + lockId: string, + delta: bigint, + blockId: string, + reasons?: string +): Promise => { + const balance = await AccountBalance.get(address); + + if (!balance) { + return; + } + + const locks = balance.locks ?? []; + const entry = locks.find(lock => lock.lockId === lockId); + + if (entry) { + entry.amount = floorZero(entry.amount + delta); + if (reasons !== undefined) { + entry.reasons = reasons; + } + } else if (delta > BigInt(0)) { + locks.push({ lockId, amount: delta, reasons }); + } + + balance.locks = locks.filter(lock => lock.amount > BigInt(0)); + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); +}; + +/** Sets one lock on `address` to an absolute amount (0 clears it). */ +export const setLock = async ( + address: string, + lockId: string, + amount: bigint, + blockId: string, + reasons?: string +): Promise => { + const balance = await AccountBalance.get(address); + const current = balance?.locks?.find(lock => lock.lockId === lockId)?.amount ?? BigInt(0); + + await adjustLock(address, lockId, amount - current, blockId, reasons); +}; + +const lockHandler = + (lockId: string, sign: bigint) => + async (event: SubstrateEvent): Promise => { + const { blockId, block } = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + + if (!who) { + return; + } + + // Ensure the balance row exists so the lock has somewhere to live. + await ledgerAccount(who, blockId, block.timestamp); + const balance = await loadBalance(who, undefined, blockId); + await balance.save(); + + await adjustLock(who, lockId, sign * amountOf(decoded), blockId); + }; + +/** `balances.Locked` / `Unlocked` — the `LockableCurrency` floor on `free`. */ +export const handleBalanceLocked = lockHandler('balances', BigInt(1)); +export const handleBalanceUnlocked = lockHandler('balances', BigInt(-1)); +/** `balances.Frozen` / `Thawed` — the upstream `fungible` freeze, also a floor on `free`. */ +export const handleBalanceFrozen = lockHandler('freeze', BigInt(1)); +export const handleBalanceThawed = lockHandler('freeze', BigInt(-1)); + +// --------------------------------------------------------------------------------------------- +// Cross-event pairing helpers +// --------------------------------------------------------------------------------------------- + +/** Memos seen before their paired `Transfer`, keyed by extrinsic + endpoints + amount, per block. */ +let pendingMemoBlock: string | undefined; +let pendingMemos = new Map(); + +const memoKey = ( + extrinsicId: string | undefined, + from: string, + to: string, + amount: bigint +): string => `${extrinsicId ?? '-'}/${from}/${to}/${amount.toString()}`; + +const stashPendingMemo = ( + args: HandlerArgs, + from: string, + to: string, + amount: bigint, + memo: string +): void => { + if (pendingMemoBlock !== args.blockId) { + pendingMemoBlock = args.blockId; + pendingMemos = new Map(); + } + + pendingMemos.set(memoKey(args.extrinsicId, from, to, amount), memo); +}; + +const takePendingMemo = ( + args: HandlerArgs, + from: string, + to: string, + amount: bigint +): string | undefined => { + if (pendingMemoBlock !== args.blockId) { + return undefined; + } + + const key = memoKey(args.extrinsicId, from, to, amount); + const memo = pendingMemos.get(key); + + if (memo !== undefined) { + pendingMemos.delete(key); + } + + return memo; +}; + +/** + * Entries already written in this extrinsic for `(kind, account)`, narrowed to `amountAbs` here. + * + * `store.getByFields` reads the write cache before the database, so a row saved earlier in this + * block is visible. Only indexed fields can go in the filter (`amountAbs` is not one — the schema + * is at the 10-index cap), so the amount match is applied in memory. + */ +const findExtrinsicEntries = async ( + args: HandlerArgs, + kind: MovementKind, + account: string | undefined, + amountAbs: bigint +): Promise => { + if (!args.extrinsicId || !account) { + return []; + } + + const rows = await PolyxEntry.getByFields( + [ + ['extrinsicId', '=', args.extrinsicId], + ['kind', '=', kind], + ['accountId', '=', account], + ], + { limit: 50 } + ); + + return rows.filter(row => row.amountAbs === amountAbs); +}; + +/** + * Entries written earlier in this block for `account` of one of `kinds`, narrowed to `amountAbs`. + * + * Staking rewards arrive from `on_initialize`, not an extrinsic, so the reward event and any + * paired `balances` deposit can only be matched on the block. Used to keep a v8 reward from being + * counted twice — once as `Mint`, once as `StakingReward`. + */ +const findBlockEntries = async ( + blockId: string, + account: string | undefined, + amountAbs: bigint, + kinds: MovementKind[] +): Promise => { + if (!account) { + return []; + } + + const rows = await PolyxEntry.getByFields( + [ + ['createdBlockId', '=', blockId], + ['accountId', '=', account], + ], + { limit: 100 } + ); + + return rows.filter(row => kinds.includes(row.kind) && row.amountAbs === amountAbs); +}; + +// --------------------------------------------------------------------------------------------- +// Balances-pallet handlers +// --------------------------------------------------------------------------------------------- + +export const handleBalanceEndowed = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition(args, { + to: { address: holder(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Endowment, + }); +}; + +export const handleBalanceTransfer = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['from']); + const to = firstText(decoded, ['to']); + const amount = amountOf(decoded); + const memo = memoOf(decoded) ?? takePendingMemo(args, from, to, amount); + + const [endowment] = await findExtrinsicEntries(args, MovementKind.Endowment, to, amount); + + if (endowment) { + // `balances.transfer` to a fresh account emits `Endowed` (already crediting `to/Free`) and + // `Transfer`. Enrich the endowment with the sender and post only the debit side. + endowment.counterpartyAddress = from; + endowment.counterpartyIdentityId = (await Account.get(from))?.identityId; + + if (memo) { + endowment.memo = memo; + } + + await endowment.save(); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Transfer, + memo, + }); + + return; + } + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Free }, + to: { address: to, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Transfer, + memo, + }); +}; + +/** + * A2: `TransferWithMemo` is emitted alongside the classic `Transfer` for one `transfer_with_memo` + * call. It is never its own movement — it only supplies the memo. If the `Transfer` was already + * indexed this enriches it; otherwise the memo is stashed for the `Transfer` still to come. + */ +export const handleBalanceTransferWithMemo = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['from']); + const to = firstText(decoded, ['to']); + const amount = amountOf(decoded); + const memo = memoOf(decoded); + + if (!memo) { + return; + } + + const existing = await findExtrinsicEntries(args, MovementKind.Transfer, to, amount); + + if (existing.length > 0) { + for (const entry of existing) { + entry.memo = memo; + await entry.save(); + } + + return; + } + + stashPendingMemo(args, from, to, amount, memo); +}; + +export const handleBalanceReserved = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Free }, + to: { address: who, pool: PolyxPool.Reserved }, + amount: amountOf(decoded), + kind: MovementKind.Hold, + }); +}; + +export const handleBalanceUnreserved = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Reserved }, + to: { address: who, pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Release, + }); +}; + +export const handleReserveRepatriated = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['from']); + const to = firstText(decoded, ['to']); + const status = firstText(decoded, ['destinationStatus'])?.toLowerCase(); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Reserved }, + to: { + address: to, + pool: status?.includes('reserved') ? PolyxPool.Reserved : PolyxPool.Free, + }, + amount: amountOf(decoded), + kind: MovementKind.ReserveRepatriation, + }); +}; + +export const handleBalanceHeld = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const who = holder(decoded); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded) ?? HoldReason.Unknown; + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Free }, + to: { address: who, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.Hold, + holdReason: reason, + }); + + await adjustHold(who, reason, amount, args.blockId); +}; + +export const handleBalanceReleased = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const who = holder(decoded); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded) ?? HoldReason.Unknown; + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Reserved }, + to: { address: who, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Release, + holdReason: reason, + }); + + await adjustHold(who, reason, -amount, args.blockId); +}; + +export const handleBalanceBurnedHeld = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const who = holder(decoded); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded) ?? HoldReason.Unknown; + + await postTransition(args, { + from: { address: who, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.Slash, + holdReason: reason, + }); + + await adjustHold(who, reason, -amount, args.blockId); +}; + +export const handleTransferOnHold = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['source', 'from']); + const to = firstText(decoded, ['dest', 'to']); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Reserved }, + to: { address: to, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.ReserveRepatriation, + holdReason: reason, + }); + + if (reason) { + await adjustHold(from, reason, -amount, args.blockId); + await adjustHold(to, reason, amount, args.blockId); + } +}; + +export const handleTransferAndHold = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const from = firstText(decoded, ['source', 'from']); + const to = firstText(decoded, ['dest', 'to']); + const amount = amountOf(decoded); + const reason = holdReasonOf(decoded); + + await postTransition(args, { + from: { address: from, pool: PolyxPool.Free }, + to: { address: to, pool: PolyxPool.Reserved }, + amount, + kind: MovementKind.ReserveRepatriation, + holdReason: reason, + }); + + if (reason) { + await adjustHold(to, reason, amount, args.blockId); + } +}; + +/** `Burned` / `Slashed` / `Withdraw` / `AccountBalanceBurned` — value leaves `who/Free`. */ +export const handleBalanceBurned = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition(args, { + from: { address: holder(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: args.eventId === EventIdEnum.Slashed ? MovementKind.Slash : MovementKind.Burn, + }); +}; + +/** + * A3: `balances.Suspended` — an upstream (v8-only) event that reaps `who`'s free balance. The + * registration pointed at a `handleBalanceSuspended` that never existed. + */ +export const handleBalanceSuspended = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition(args, { + from: { address: holder(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Burn, + }); +}; + +/** `Minted` / `Deposit` / `Restored` — value enters `who/Free`. */ +export const handleBalanceMinted = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const who = holder(decoded); + const amount = amountOf(decoded); + + // A v8 staking payout can emit both `staking.Rewarded` and a `balances` deposit for the same + // POLYX. If the reward side already recorded it, this is not a second movement. + const reward = await findBlockEntries(args.blockId, who, amount, [MovementKind.StakingReward]); + + if (reward.length > 0) { + return; + } + + await postTransition(args, { + to: { address: who, pool: PolyxPool.Free }, + amount, + kind: MovementKind.Mint, + }); +}; + +/** + * A9: `balances.DustLost` — account reaping. The remaining free balance is destroyed; the row was + * never written (`DustLost: []`). + */ +export const handleDustLost = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const account = firstText(decoded, ['account', 'who']); + + await postTransition(args, { + from: { address: account, pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.DustLost, + }); + + // account reaping — always reconcile the reaped account against chain state + await reconcileAccount(account, args.blockId, args.block, { + force: true, + eventIdx: args.eventIdx, + }); +}; + +// --------------------------------------------------------------------------------------------- +// BalanceSet — a checkpoint, not a movement (resolves A1 structurally) +// --------------------------------------------------------------------------------------------- + +/** + * `BalanceSet` *sets* `free` (and, pre-v8, `reserved`) to an absolute value — it is not a + * movement of that size. The old model recorded the set value as a delta, corrupting every + * running total after it. Here the pools are set directly and one `BalanceSetAdjustment` entry + * per changed pool records the delta so `SUM(entries)` still reconciles to the balance. + */ +export const handleBalanceSet = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const { blockId, block, eventIdx, blockEventId } = args; + const datetime = block.timestamp; + const decoded = decodeEvent(event); + + const who = holder(decoded) ?? firstText(decoded, ['account']); + const newFree = getBigIntValue(optionalField(decoded, 'free')); + const rawReserved = optionalField(decoded, 'reserved'); + const newReserved = rawReserved !== undefined ? getBigIntValue(rawReserved) : undefined; + + const account = await ledgerAccount(who, blockId, datetime); + const balance = await loadBalance(who, account.identityId, blockId); + + const deltas: Array<{ pool: PolyxPool; delta: bigint }> = []; + + const freeDelta = newFree - balance.free; + balance.free = newFree; + if (freeDelta !== BigInt(0)) { + deltas.push({ pool: PolyxPool.Free, delta: freeDelta }); + } + + if (newReserved !== undefined) { + const reservedDelta = newReserved - balance.reserved; + balance.reserved = newReserved; + if (reservedDelta !== BigInt(0)) { + deltas.push({ pool: PolyxPool.Reserved, delta: reservedDelta }); + } + } + + if (deltas.length > 0) { + balance.movementCount += 1; + for (const { delta } of deltas) { + bumpLifetimeByKind( + balance, + MovementKind.BalanceSetAdjustment, + delta > BigInt(0) ? EntryDirection.Credit : EntryDirection.Debit, + delta > BigInt(0) ? delta : -delta + ); + } + } + + recomputeDerived(balance); + balance.updatedBlockId = blockId; + await balance.save(); + + const params = getEventParams(args); + const date = startOfUtcDay(datetime); + + for (const { pool, delta } of deltas) { + await PolyxEntry.create({ + id: `${blockId}/${padId(eventIdx.toString())}/${poolTag(pool)}s`, + movementId: blockEventId, + accountId: who, + identityId: account.identityId, + counterpartyAddress: undefined, + counterpartyIdentityId: undefined, + pool, + amount: delta, + amountAbs: delta > BigInt(0) ? delta : -delta, + kind: MovementKind.BalanceSetAdjustment, + direction: delta > BigInt(0) ? EntryDirection.Credit : EntryDirection.Debit, + holdReason: undefined, + memo: undefined, + freeAfter: balance.free, + reservedAfter: balance.reserved, + frozenAfter: balance.frozen, + moduleId: params.moduleId, + callId: params.callId, + eventId: params.eventId, + specVersionId: block.specVersion, + date, + eraIndex: undefined, + createdEventId: blockEventId, + extrinsicId: params.extrinsicId, + eventIdx, + datetime, + createdBlockId: blockId, + }).save(); + } + + // a checkpoint should equal chain state — always reconcile right after it + await reconcileAccount(who, blockId, block, { force: true, eventIdx }); +}; + +// --------------------------------------------------------------------------------------------- +// Treasury and fees +// --------------------------------------------------------------------------------------------- + +const identityPrimaryAccount = async (did: string | undefined): Promise => + did ? (await Identity.get(did))?.primaryAccount : undefined; + +export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const [rawFromDid, rawToDid, rawTo, rawBalance] = args.params; + + // (IdentityId, IdentityId, AccountId, Balance) from 5.0.0; (IdentityId, IdentityId, Balance) before + const hasToAddress = args.params.length >= 4; + const amount = getBigIntValue(hasToAddress ? rawBalance : rawTo); + const fromAddress = await identityPrimaryAccount(getTextValue(rawFromDid)); + const toAddress = + (hasToAddress ? getTextValue(rawTo) : undefined) ?? + (await identityPrimaryAccount(getTextValue(rawToDid))); + + const [existingTransfer] = await findExtrinsicEntries( + args, + MovementKind.Transfer, + toAddress, + amount + ); + + if (existingTransfer) { + // `treasury.disbursement` to an identity emits both `Transfer` and `TreasuryDisbursement`; + // relabel the transfer rather than double-counting. + existingTransfer.kind = MovementKind.TreasuryDisbursement; + await existingTransfer.save(); + + return; + } + + await postTransition(args, { + from: fromAddress ? { address: fromAddress, pool: PolyxPool.Free } : undefined, + to: toAddress ? { address: toAddress, pool: PolyxPool.Free } : undefined, + amount, + kind: MovementKind.TreasuryDisbursement, + }); +}; + +export const handleTreasuryReimbursement = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const [rawIdentity, rawBalance] = args.params; + const { specVersion } = args.block; + + // Until 5.4.1 the event reported 80% of the amount actually taken from the payer. + const reported = getBigIntValue(rawBalance); + const amount = specVersion < 5004001 ? (reported * BigInt(125)) / BigInt(100) : reported; + + const treasury = await identityPrimaryAccount(getTextValue(rawIdentity)); + + await postTransition(args, { + to: treasury ? { address: treasury, pool: PolyxPool.Free } : undefined, + amount, + kind: MovementKind.TreasuryReimbursement, + }); +}; + +/** + * `protocolFee.FeeCharged` and `transactionPayment.TransactionFeePaid` — `who/Free → ∅`. + * + * Both carry `(AccountId, Balance)` as their first two parameters at every spec version they + * exist for (`TransactionFeePaid` adds a trailing `tip` that is not a POLYX movement), so these + * are read positionally rather than through the shape table. + */ +export const handleTransactionFeeCharged = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const [rawWho, rawAmount] = args.params; + + await postTransition(args, { + from: { address: getTextValue(rawWho), pool: PolyxPool.Free }, + amount: getBigIntValue(rawAmount), + kind: MovementKind.Fee, + }); +}; + +// --------------------------------------------------------------------------------------------- +// Staking — era-dependent, and inverted at v8 (defect A10, resolves A6) +// --------------------------------------------------------------------------------------------- + +/** + * ≤ v7.4: bonding is `set_lock(STAKING_ID, …)` — **no balance moves**. `Bonded`/`Unbonded`/ + * `Withdrawn` maintain the staking lock only, so `frozen` reflects it; they write **no + * `PolyxEntry`**. This is the A6 correction — the old `type: Bonded` rows asserted movements + * that never happened. + * + * v8: bonding is a Hold. The balance-side movement is the paired `balances.Held{reason:Staking}` + * / `Released` (written by the balances handlers). The `staking.*` events here become ledger + * state only, so they must **not** write a second entry or bonds/rewards double-count. + * + * NOT YET VERIFIED against a real v8 block: that `staking.Bonded` and `balances.Held{Staking}` + * are emitted within one extrinsic. If that pairing does not hold, v8 bonded POLYX is unindexed + * and this assumption must change — the reconciliation harness is designed to catch it. + */ + +/** `staking.PayoutStarted { eraIndex, validatorStash, … }` precedes the payout's `Rewarded` events. */ +let payoutEraBlock: string | undefined; +let payoutEraIndex: number | undefined; + +export const handlePayoutStarted = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + const raw = optionalField(decoded, 'eraIndex'); + + payoutEraBlock = args.blockId; + payoutEraIndex = raw !== undefined ? Number(getTextValue(raw)) : undefined; +}; + +const currentPayoutEra = (blockId: string): number | undefined => + payoutEraBlock === blockId ? payoutEraIndex : undefined; + +const stakingStash = (decoded: Record): string | undefined => + firstText(decoded, ['stash', 'account', 'staker', 'who']); + +/** + * The account a reward was actually paid to. + * + * v8: the `Rewarded` event carries the `RewardDestination`. Pre-v8 it carries only the stash, so + * `staking.payee(stash)` is read from chain storage (A15) — measured on mainnet to matter for a + * large share of pre-v8 rewards. + */ +const rewardRecipient = async ( + decoded: Record, + stash: string | undefined, + is8x: boolean +): Promise => { + if (!stash) { + return undefined; + } + + if (is8x) { + const dest = optionalField(decoded, 'dest'); + const json = dest?.toJSON() as string | Record | undefined; + + if (json && typeof json === 'object') { + return ((json.account ?? json.Account) as string | undefined) ?? stash; + } + + return stash; + } + + const { rewardDestinationAccount } = await resolveLegacyRewardDestination(stash); + + return rewardDestinationAccount ?? stash; +}; + +/** + * `staking.Reward` / `Rewarded` — `∅ → recipient/Free`, a real movement at both eras. + * + * On v8 the reward is deposited to the recipient's free balance (and, for `dest: Staked`, + * immediately held) by paired `balances` events — this relabels that `Mint` rather than writing a + * second movement. Pre-v8 there is no paired balances event, so the credit is written here. + */ +export const handleReward = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + const stash = stakingStash(decoded); + const amount = amountOf(decoded); + const eraIndex = currentPayoutEra(args.blockId); + const recipient = await rewardRecipient(decoded, stash, is8xChain(args.block)); + + if (!recipient) { + return; + } + + // If a `balances` deposit for this reward was already recorded as a plain mint, relabel it. + const mints = await findBlockEntries(args.blockId, recipient, amount, [MovementKind.Mint]); + + if (mints.length > 0) { + for (const mint of mints) { + mint.kind = MovementKind.StakingReward; + mint.eraIndex = eraIndex; + await mint.save(); + } + + const balance = await AccountBalance.get(recipient); + if (balance) { + balance.totalRewards += amount; + await balance.save(); + } + + return; + } + + await postTransition( + args, + { + to: { address: recipient, pool: PolyxPool.Free }, + amount, + kind: MovementKind.StakingReward, + }, + { eraIndex } + ); +}; + +/** `staking.Slash` / `Slashed` — `staker/Free → ∅`, a real movement at both eras. */ +export const handleStakingSlash = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + const decoded = decodeEvent(event); + + await postTransition( + args, + { + from: { address: stakingStash(decoded), pool: PolyxPool.Free }, + amount: amountOf(decoded), + kind: MovementKind.Slash, + }, + { eraIndex: currentPayoutEra(args.blockId) } + ); +}; + +const ensureBalanceRow = async ( + address: string, + blockId: string, + datetime: Date +): Promise => { + await ledgerAccount(address, blockId, datetime); + const balance = await loadBalance(address, undefined, blockId); + await balance.save(); +}; + +/** `staking.Bonded` — pre-v8 raises the staking lock; v8 is ledger state only (see `Held`). */ +export const handleBonded = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + + if (is8xChain(args.block)) { + return; + } + + const decoded = decodeEvent(event); + const stash = stakingStash(decoded); + + if (!stash) { + return; + } + + await ensureBalanceRow(stash, args.blockId, args.block.timestamp); + await adjustLock(stash, STAKING_LOCK_ID, amountOf(decoded), args.blockId, 'staking'); +}; + +/** + * `staking.Unbonded` — the unbonding queue keeps the balance locked (`ledger.total` is unchanged + * until `withdraw_unbonded`), so the lock does not move here on either era. + */ +export const handleUnbonded = async (): Promise => { + // intentionally a no-op for the balance ledger +}; + +/** `staking.Withdrawn` — pre-v8 lowers the staking lock as matured chunks leave; v8 is state only. */ +export const handleWithdrawn = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + + if (is8xChain(args.block)) { + return; + } + + const decoded = decodeEvent(event); + const stash = stakingStash(decoded); + + if (!stash) { + return; + } + + await adjustLock(stash, STAKING_LOCK_ID, -amountOf(decoded), args.blockId); +}; diff --git a/src/mappings/entities/identities/mapPolyxTransaction.ts b/src/mappings/entities/identities/mapPolyxTransaction.ts deleted file mode 100644 index ce0d71dd..00000000 --- a/src/mappings/entities/identities/mapPolyxTransaction.ts +++ /dev/null @@ -1,464 +0,0 @@ -import { SubstrateEvent } from '@subql/types'; -import BigNumber from 'bignumber.js'; -import { - Account, - BalanceTypeEnum, - EventIdEnum, - Identity, - ModuleIdEnum, - PolyxTransaction, -} from '../../../types'; -import { bytesToString, getBigIntValue, getEventParams, getTextValue } from '../../../utils'; -import { extract8xStakingAmount, getFirstKeyFromJson, is8xChain } from '../../../utils/common'; -import { HandlerArgs, extractArgs } from '../common'; -import { getPaginatedData } from './../../../utils/common'; - -const getBasicDetails = async ( - args: HandlerArgs -): Promise<{ address: string; amount: bigint; identityId: string | undefined }> => { - let address: string; - let amount: bigint; - let identityId: string | undefined; - if (is8xChain(args.block)) { - // On 8.x chain, staking events don't have DID as first param - // Bonded/Unbonded: [stash, amount] - 2 params - // Rewarded: [stash, dest, amount] - 3 params (dest is RewardDestination enum) - const [rawAddress, rawDest, rawAmount] = args.params; - address = getTextValue(rawAddress); - amount = extract8xStakingAmount(rawDest, rawAmount); - identityId = (await Account.get(address))?.identityId; - } else { - const [rawDid, rawAddress, rawBalance] = args.params; - identityId = getTextValue(rawDid); - address = getTextValue(rawAddress); - amount = getBigIntValue(rawBalance); - } - return { address, amount, identityId }; -}; - -export const handleTreasuryReimbursement = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - const [rawIdentity, rawBalance] = args.params; - const did = getTextValue(rawIdentity); - const balance = getTextValue(rawBalance); - const { specVersion } = args.block; - - const identity = await Identity.get(did); - - /** - * Till chain 5.4.1, treasury reimbursement was only 80% of the actual amount deducted - * Post that the split between author/treasury was removed - */ - let amount: bigint; - if (specVersion < 5004001) { - amount = BigInt( - new BigNumber(balance || 0).multipliedBy(1.25).integerValue(BigNumber.ROUND_FLOOR).toString() - ); - } else { - amount = BigInt(balance || 0); - } - const details = getEventParams(args); - - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const protocolFeePolyxTransaction: PolyxTransaction | undefined = transactions.find( - ({ eventId }) => eventId === EventIdEnum.FeeCharged - ); - - if (amount === protocolFeePolyxTransaction?.amount) { - // this is the case where treasury reimbursement is showing that 80% of protocol fee charged - // We ignore this case to insert in PolyxTransaction - return; - } - } - - await PolyxTransaction.create({ - ...details, - identityId: did, - address: identity?.primaryAccount, - toId: null, - toAddress: null, - amount, - type: BalanceTypeEnum.Free, - }).save(); -}; - -const processTreasuryDisbursementArgs = async (args: HandlerArgs) => { - let rawFromIdentity, rawToDid, rawTo, rawBalance; - - const specName = api.runtimeVersion.specName.toString(); - if (args.block.specVersion < 5000000 && specName !== 'polymesh_private_dev') { - [rawFromIdentity, rawToDid, rawBalance] = args.params; - } else { - [rawFromIdentity, rawToDid, rawTo, rawBalance] = args.params; - } - const identityId = getTextValue(rawFromIdentity); - const toId = getTextValue(rawToDid); - const amount = getBigIntValue(rawBalance); - - let toAddress = getTextValue(rawTo); - - if (!toAddress) { - ({ primaryAccount: toAddress } = await Identity.get(toId)); - } - - return { identityId, toId, toAddress, amount }; -}; - -export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const { identityId, toId, toAddress, amount } = await processTreasuryDisbursementArgs(args); - - const details = getEventParams(args); - - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const transferPolyxTransaction: PolyxTransaction | undefined = transactions.find( - ({ eventId }) => eventId === EventIdEnum.Transfer - ); - /** - * in case when `treasury.disbursement` extrinsic is used to disburse some amount to an identity, - * both `Transfer` and `TreasuryDisbursement` events are triggered, - * in this case we update the `Transfer` entry to reflect `Disbursement` - * and skip adding a separate `TreasuryDisbursement` entry - */ - if (amount === transferPolyxTransaction?.amount) { - // this is the case where treasury reimbursement is showing that 80% of protocol fee charged - // We ignore this case to insert in PolyxTransaction - transferPolyxTransaction.eventId = EventIdEnum.TreasuryDisbursement; - await PolyxTransaction.create(transferPolyxTransaction).save(); - return; - } - } - - const fromIdentity = await Identity.get(identityId); - - await PolyxTransaction.create({ - ...details, - identityId, - address: fromIdentity?.primaryAccount, - toId, - toAddress, - amount, - type: BalanceTypeEnum.Free, - }).save(); -}; - -export const handleBalanceTransfer = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - let address: string; - let toAddress: string; - let amount: bigint; - let memo: string | undefined; - let identityId: string | undefined; - let toId: string | undefined; - - if (is8xChain(args.block)) { - const [rawFromAddress, rawToAddress, rawBalance] = args.params; - address = getTextValue(rawFromAddress); - toAddress = getTextValue(rawToAddress); - amount = getBigIntValue(rawBalance); - if (args.params.length > 3) { - const rawMemo = args.params[3]; - memo = bytesToString(rawMemo); - } - identityId = (await Account.get(address))?.identityId; - toId = (await Account.get(toAddress))?.identityId; - } else { - const [rawFromDid, rawFrom, rawToDid, rawTo, rawBalance] = args.params; - - amount = getBigIntValue(rawBalance); - identityId = getTextValue(rawFromDid); - address = getTextValue(rawFrom); - toId = getTextValue(rawToDid); - toAddress = getTextValue(rawTo); - if (args.params.length > 5) { - const rawMemo = args.params[5]; - memo = bytesToString(rawMemo); - } - } - - const details = getEventParams(args); - - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const endowedPolyxTransaction: PolyxTransaction | undefined = transactions.find( - ({ eventId }) => eventId === EventIdEnum.Endowed - ); - /** - * in case when `balances.transfer` extrinsic is used to transfer some balance - * to an account for the first time, both `Endowed` and `Transfer` events are triggered, - * in this case we update the `Endowed` entry to reflect details of the account from which - * transfer was initiated and skip adding a separate `Transfer` entry - */ - if (amount === endowedPolyxTransaction?.amount) { - // this is the case where treasury reimbursement is showing that 80% of protocol fee charged - // We ignore this case to insert in PolyxTransaction - endowedPolyxTransaction.identityId = identityId; - endowedPolyxTransaction.address = address; - endowedPolyxTransaction.memo = memo; - - await PolyxTransaction.create(endowedPolyxTransaction).save(); - return; - } - } - - await PolyxTransaction.create({ - ...details, - identityId, - address, - toId, - toAddress, - amount, - memo, - type: BalanceTypeEnum.Free, - }).save(); -}; - -export const handleReserveRepatriated = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const [rawFromAddress, rawToAddress, rawAmount, rawType] = args.params; - - const fromAddress = getTextValue(rawFromAddress); - const toAddress = getTextValue(rawToAddress); - const amount = getBigIntValue(rawAmount); - const type = - getFirstKeyFromJson(rawType) === 'free' ? BalanceTypeEnum.Free : BalanceTypeEnum.Reserved; - - const details = getEventParams(args); - - await PolyxTransaction.create({ - ...details, - address: fromAddress, - identityId: (await Account.get(fromAddress))?.identityId, - toAddress, - toId: (await Account.get(toAddress))?.identityId, - amount, - type, - }).save(); -}; - -export const handleTransactionFeeCharged = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const [rawAddress, rawActualFee] = args.params; - const address = getTextValue(rawAddress); - const amount = getBigIntValue(rawActualFee); - - const details = getEventParams(args); - if (details.extrinsicId) { - const transactions: PolyxTransaction[] = await getPaginatedData< - PolyxTransaction, - 'extrinsicId' - >('PolyxTransaction', 'extrinsicId', details.extrinsicId); - - const reimbursementTransaction: PolyxTransaction | undefined = transactions - .slice() - .reverse() - .find(({ eventId }) => eventId === EventIdEnum.TreasuryReimbursement); - /** - * From chain 5.4, with `TreasuryReimbursement` there is a `TransactionFeePaid` event as well. - * In this case, we will update the already inserted `TreasuryReimbursement` to point that it was indeed for done for `TransactionFeePaid` - * We also update the amount to get the exact value (since in treasury reimbursement, we calculate the amount as amount * 1.25 which can be off by some balance amount) - */ - if (reimbursementTransaction) { - reimbursementTransaction.address = address; - reimbursementTransaction.amount = amount; - reimbursementTransaction.moduleId = ModuleIdEnum.transactionpayment; - reimbursementTransaction.eventId = EventIdEnum.TransactionFeePaid; - await PolyxTransaction.create(reimbursementTransaction).save(); - return; - } - } - - const account = await Account.get(address); - - await PolyxTransaction.create({ - ...details, - identityId: account?.identityId, - address, - amount, - type: BalanceTypeEnum.Free, - }).save(); -}; - -// this is not affected by 8x chain -const handleBalanceAdded = async (event: SubstrateEvent, type: BalanceTypeEnum): Promise => { - const args = extractArgs(event); - - const [rawAddress, rawBalance] = args.params; - const toAddress = getTextValue(rawAddress); - const amount = getBigIntValue(rawBalance); - - const details = getEventParams(args); - const account = await Account.get(toAddress); - - await PolyxTransaction.create({ - ...details, - toAddress: toAddress, - toId: account?.identityId, - amount, - type, - }).save(); -}; - -const handleBalanceCharged = async ( - event: SubstrateEvent, - type: BalanceTypeEnum -): Promise => { - const args = extractArgs(event); - - const [rawAddress, rawBalance] = args.params; - const address = getTextValue(rawAddress); - const amount = getBigIntValue(rawBalance); - - const account = await Account.get(address); - - await PolyxTransaction.create({ - ...getEventParams(args), - address, - identityId: account?.identityId, - amount, - type, - }).save(); -}; - -const handleBalanceReceived = async ( - event: SubstrateEvent, - type: BalanceTypeEnum -): Promise => { - const args = extractArgs(event); - - const { address: toAddress, amount, identityId: toId } = await getBasicDetails(args); - - await PolyxTransaction.create({ - ...getEventParams(args), - toId, - toAddress, - amount, - type, - }).save(); -}; - -const handleBalanceSpent = async (event: SubstrateEvent, type: BalanceTypeEnum): Promise => { - const args = extractArgs(event); - - const { address, amount, identityId } = await getBasicDetails(args); - - await PolyxTransaction.create({ - ...getEventParams(args), - identityId, - address, - amount, - type, - }).save(); -}; - -export const handleBalanceSet = async (event: SubstrateEvent): Promise => { - const args = extractArgs(event); - - const { address: toAddress, amount, identityId: toId } = await getBasicDetails(args); - let reservedAmount: bigint; - - if (!is8xChain(args.block)) { - // BalanceSet(IdentityId, AccountId, free, reserved) — reserved is params[3], not params[4] - reservedAmount = getBigIntValue(args.params[3]); - } - - const details = getEventParams(args); - - // add the newly set free balance - await PolyxTransaction.create({ - ...details, - toId, - toAddress, - amount, - type: BalanceTypeEnum.Free, - }).save(); - - if (reservedAmount) { - // add the newly set reserve balance - await PolyxTransaction.create({ - ...details, - toId, - toAddress, - amount: reservedAmount, - type: BalanceTypeEnum.Reserved, - }).save(); - } -}; - -export const handleBalanceEndowed = async (event: SubstrateEvent): Promise => { - await handleBalanceReceived(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceFrozen = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Locked); -}; - -export const handleBalanceLocked = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Locked); -}; - -export const handleBalanceUnlocked = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceReserved = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Reserved); -}; - -export const handleBalanceMinted = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceSlashed = async (event: SubstrateEvent): Promise => { - await handleBalanceSpent(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceUnreserved = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceBurned = async (event: SubstrateEvent): Promise => { - await handleBalanceSpent(event, BalanceTypeEnum.Free); -}; - -export const handleBonded = async (event: SubstrateEvent): Promise => { - await handleBalanceSpent(event, BalanceTypeEnum.Bonded); -}; - -export const handleUnbonded = async (event: SubstrateEvent): Promise => { - await handleBalanceReceived(event, BalanceTypeEnum.Unbonded); -}; - -export const handleReward = async (event: SubstrateEvent): Promise => { - await handleBalanceReceived(event, BalanceTypeEnum.Free); -}; - -export const handleWithdrawn = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Unbonded); -}; - -export const handleFeeCharged = async (event: SubstrateEvent): Promise => { - await handleBalanceCharged(event, BalanceTypeEnum.Free); -}; - -export const handleBalanceDeposit = async (event: SubstrateEvent): Promise => { - await handleBalanceAdded(event, BalanceTypeEnum.Free); -}; diff --git a/src/mappings/entities/identities/reconcilePolyx.ts b/src/mappings/entities/identities/reconcilePolyx.ts new file mode 100644 index 00000000..9e32499e --- /dev/null +++ b/src/mappings/entities/identities/reconcilePolyx.ts @@ -0,0 +1,107 @@ +import { Codec } from '@polkadot/types/types'; +import { SubstrateBlock } from '@subql/types'; +import { AccountBalance, AnomalyKind } from '../../../types'; +import { getBigIntValue } from '../../../utils'; +import { recordAnomaly } from '../../../utils/anomaly'; +import { accountDataFrozen, recomputeDerived } from './mapPolyxLedger'; + +/** + * In-flight reconciliation (D11). + * + * `api.query` targets the block being indexed, and `.at` is unsupported, so authoritative state + * can only be read for the current block. This compares the derived `AccountBalance` against + * `system.account` there — every Nth block for accounts touched in that block, and always after + * a `BalanceSet` or `DustLost`. + * + * On a mismatch it records a `BalanceReconciliationDrift` anomaly **and corrects** the derived + * value, so drift from one missed or mis-signed event cannot compound into every later balance. + * The offline harness (`scripts/reconcile-polyx.ts`) is what answers "is the history right"; this + * is the going-forward safety net. + */ + +/** Sample rate for the routine check. `BalanceSet`/`DustLost` always reconcile regardless. */ +const RECONCILE_EVERY_N_BLOCKS = 500; + +const blockNumber = (block: SubstrateBlock): number => Number(block.block.header.number.toString()); + +const shouldSample = (block: SubstrateBlock, force: boolean): boolean => + force || blockNumber(block) % RECONCILE_EVERY_N_BLOCKS === 0; + +interface OnChain { + free: bigint; + reserved: bigint; + frozen: bigint; +} + +const readOnChain = async (address: string): Promise => { + const info = (await api.query.system.account(address)) as unknown as { + data: Record; + }; + + return { + free: getBigIntValue(info.data.free), + reserved: getBigIntValue(info.data.reserved), + frozen: accountDataFrozen(info.data), + }; +}; + +export const reconcileAccount = async ( + address: string, + blockId: string, + block: SubstrateBlock, + { force = false, eventIdx }: { force?: boolean; eventIdx?: number } = {} +): Promise => { + if (!address || !shouldSample(block, force)) { + return; + } + + const balance = await AccountBalance.get(address); + + if (!balance) { + return; + } + + let onChain: OnChain; + + try { + onChain = await readOnChain(address); + } catch { + // A pruned node or a transient RPC error is not a ledger defect. + return; + } + + const drifts: string[] = []; + + if (balance.free !== onChain.free) { + drifts.push(`free ${balance.free} vs ${onChain.free}`); + } + if (balance.reserved !== onChain.reserved) { + drifts.push(`reserved ${balance.reserved} vs ${onChain.reserved}`); + } + if (balance.frozen !== onChain.frozen) { + drifts.push(`frozen ${balance.frozen} vs ${onChain.frozen}`); + } + + if (drifts.length === 0) { + return; + } + + await recordAnomaly({ + kind: AnomalyKind.BalanceReconciliationDrift, + detail: `${address}: ${drifts.join('; ')}`, + block, + eventIdx, + }); + + // Correct the derived value so the drift cannot compound. + balance.free = onChain.free; + balance.reserved = onChain.reserved; + balance.locks = + onChain.frozen > BigInt(0) + ? [{ lockId: 'reconciled', amount: onChain.frozen, reasons: undefined }] + : (balance.locks ?? []).filter(lock => lock.lockId !== 'reconciled'); + recomputeDerived(balance); + balance.updatedBlockId = blockId; + + await balance.save(); +}; diff --git a/src/mappings/entities/index.ts b/src/mappings/entities/index.ts index 6136eeab..21bbe99b 100644 --- a/src/mappings/entities/index.ts +++ b/src/mappings/entities/index.ts @@ -18,7 +18,7 @@ export * from './identities/mapIdentities'; export * from './multiSig/mapMultiSig'; export * from './multiSig/mapMultiSigProposal'; export * from './assets/mapNfts'; -export * from './identities/mapPolyxTransaction'; +export * from './identities/mapPolyxLedger'; export * from './identities/mapPortfolio'; export * from './pips/mapProposal'; export * from './settlements/mapSettlement'; diff --git a/src/mappings/migrations/genesisHandler.ts b/src/mappings/migrations/genesisHandler.ts index ae6c44a2..e03fa75b 100644 --- a/src/mappings/migrations/genesisHandler.ts +++ b/src/mappings/migrations/genesisHandler.ts @@ -25,6 +25,7 @@ import { createMultiSigSigner, } from '../entities/multiSig/mapMultiSig'; import { upsertEvmAccountMapping } from '../entities/revive/mapEvmAccountMapping'; +import { seedAccountBalances } from '../../seed/accountBalance'; const genesisBlock = padId('0'); type DidWithAccount = { did: string; accountId: string }; @@ -284,5 +285,9 @@ export default async (): Promise => { // runs last so that it can link to the Accounts created above await handleEvmAccountMappings(datetime); + // opening balance snapshot for the POLYX ledger — without it every derived balance is wrong by + // the genesis allocation (docs/implementation/02-polyx-ledger.md) + await seedAccountBalances({ blockId: genesisBlock, datetime }); + logger.info('Applied genesis migrations'); }; diff --git a/src/seed/accountBalance.ts b/src/seed/accountBalance.ts new file mode 100644 index 00000000..90051655 --- /dev/null +++ b/src/seed/accountBalance.ts @@ -0,0 +1,67 @@ +import { Codec } from '@polkadot/types/types'; +import { AccountBalance } from '../types'; +import { getBigIntValue } from '../utils'; +import { + accountDataFrozen, + emptyBalance, + ledgerAccount, + recomputeDerived, +} from '../mappings/entities/identities/mapPolyxLedger'; + +/** + * Snapshots `system.account` into `AccountBalance` rows. + * + * The POLYX ledger derives every balance from events, so without an opening snapshot every + * derived balance is wrong by the genesis allocation. `genesisHandler` seeds Accounts, + * Identities and Portfolios but **no balances** — this fills that gap. + * + * Written as a domain seeder rather than inline in `genesisHandler` because plan + * [10](../../docs/implementation/10-partial-index.md) needs the identical read at an arbitrary + * `START_BLOCK`: `api.query` always targets the block being indexed, so the same call seeds + * genesis when run from the genesis handler and block B when run from the partial-index seeder. + */ + +export interface SeedContext { + blockId: string; + datetime: Date; +} + +export const seedAccountBalances = async ({ + blockId, + datetime, +}: SeedContext): Promise<{ seeded: number }> => { + const entries = await api.query.system.account.entries(); + + const rows: AccountBalance[] = []; + + for (const [key, accountInfo] of entries) { + const address = key.args[0].toString(); + const data = (accountInfo as unknown as { data: Record }).data; + + const free = getBigIntValue(data.free); + const reserved = getBigIntValue(data.reserved); + const frozen = accountDataFrozen(data); + + if (free === BigInt(0) && reserved === BigInt(0) && frozen === BigInt(0)) { + continue; + } + + const account = await ledgerAccount(address, blockId, datetime); + const balance = emptyBalance(address, account.identityId, blockId); + + balance.free = free; + balance.reserved = reserved; + // A genesis freeze is recorded as a single lock so `frozen` stays a MAX going forward. + balance.locks = + frozen > BigInt(0) ? [{ lockId: 'genesis', amount: frozen, reasons: undefined }] : []; + recomputeDerived(balance); + + rows.push(balance); + } + + await Promise.all(rows.map(row => row.save())); + + logger.info(`Seeded ${rows.length} AccountBalance rows from system.account at ${blockId}`); + + return { seeded: rows.length }; +}; diff --git a/src/utils/common.ts b/src/utils/common.ts index c4553ef0..12b5a65b 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -377,15 +377,3 @@ export const getAllByFields = async ( offset += page.length; } }; - -/** - * @deprecated Single-field adapter for `getAllByFields`. - * - * Only `mapPolyxTransaction` still calls this, and the POLYX ledger phase deletes that file - * along with the `PolyxTransaction` entity. It goes with it. - */ -export const getPaginatedData = async ( - entityName: string, - field: F, - param: T[F] -): Promise => getAllByFields(entityName, [[field, '=', param]]); diff --git a/src/utils/index.ts b/src/utils/index.ts index 769042af..653de68b 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -13,5 +13,6 @@ export * from './multisigs'; export * from './portfolios'; export * from './proposals'; export * from './settlements'; +export * from './staking'; export * from './stos'; export * from './transferManagers'; diff --git a/src/utils/staking.ts b/src/utils/staking.ts new file mode 100644 index 00000000..cbde173b --- /dev/null +++ b/src/utils/staking.ts @@ -0,0 +1,50 @@ +export interface LegacyRewardDestination { + /** `Staked` | `Stash` | `Controller` | `Account` | `None` | `LegacyUnknown` (read failed) */ + rewardDestination: string; + /** The account the reward was actually paid to, where it can be resolved */ + rewardDestinationAccount?: string; +} + +/** + * Resolves where a pre-v8 staking reward for `stash` was actually paid. + * + * Defect A15: the pre-8.x `Reward`/`Rewarded` event carries only the stash and the amount. A + * staker who set a payee of `Controller` or an explicit `Account` received the POLYX somewhere + * the event does not name. Measured across a spread of eras on mainnet, a large share of pre-v8 + * rewards went somewhere other than the stash, so the destination is read from + * `staking.payee(stash)` — chain storage, at the block being indexed (`api.query` targets the + * current block). Cheap during the D5 genesis replay; needs an archive node afterwards, which is + * why it is done now rather than deferred. + */ +export const resolveLegacyRewardDestination = async ( + stash: string +): Promise => { + try { + const payee = await api.query.staking.payee(stash); + const json = payee.toJSON() as string | Record | null; + + if (json === 'Staked' || json === 'Stash') { + return { rewardDestination: json, rewardDestinationAccount: stash }; + } + + if (json === 'Controller') { + const controller = (await api.query.staking.bonded(stash)).toJSON() as string | null; + + return { + rewardDestination: 'Controller', + rewardDestinationAccount: controller ?? undefined, + }; + } + + if (json && typeof json === 'object') { + const account = (json.account ?? json.Account) as string | undefined; + + return { rewardDestination: 'Account', rewardDestinationAccount: account }; + } + + return { rewardDestination: typeof json === 'string' ? json : 'None' }; + } catch { + // A pruned node, or a runtime with no `staking.payee` storage — fall back to the placeholder. + return { rewardDestination: 'LegacyUnknown' }; + } +}; diff --git a/tests/unit/extract8xStakingAmount.test.ts b/tests/unit/extract8xStakingAmount.test.ts new file mode 100644 index 00000000..2fd462ab --- /dev/null +++ b/tests/unit/extract8xStakingAmount.test.ts @@ -0,0 +1,44 @@ +/** + * `extract8xStakingAmount` — on a v8 chain a staking event is `[stash, amount]` (Bonded/Unbonded) + * or `[stash, dest, amount]` (Rewarded, where `dest` is a `RewardDestination` enum). The amount's + * position is decided by whether the second parameter renders as a bare number. + * + * (These tests moved here from the deleted `mapPolyxTransaction.test.ts` when the POLYX ledger + * replaced that module; the helper itself still lives in `src/utils/common.ts`.) + */ + +import { extract8xStakingAmount } from '../../src/utils/common'; + +describe('extract8xStakingAmount', () => { + it('returns the amount from the second param when it is numeric (Bonded/Unbonded)', () => { + expect(extract8xStakingAmount(createMockCodec('1000000000000'))).toBe(BigInt('1000000000000')); + }); + + it('returns the amount from the third param when the second is a RewardDestination', () => { + expect( + extract8xStakingAmount(createMockCodec('Staked'), createMockCodec('2000000000000')) + ).toBe(BigInt('2000000000000')); + }); + + it('returns 0 when the second param is non-numeric and there is no third param', () => { + expect(extract8xStakingAmount(createMockCodec('Controller'), undefined)).toBe(BigInt(0)); + }); + + it('handles every RewardDestination enum value', () => { + REWARD_DESTINATIONS.forEach(dest => { + expect(extract8xStakingAmount(createMockCodec(dest), createMockCodec('5000000000000'))).toBe( + BigInt('5000000000000') + ); + }); + }); + + it('handles a zero amount', () => { + expect(extract8xStakingAmount(createMockCodec('0'))).toBe(BigInt(0)); + }); + + it('does not treat a numeric-prefixed string as the amount', () => { + expect( + extract8xStakingAmount(createMockCodec('123abc'), createMockCodec('8000000000000')) + ).toBe(BigInt('8000000000000')); + }); +}); diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts new file mode 100644 index 00000000..08b18b4a --- /dev/null +++ b/tests/unit/mapPolyxLedger.test.ts @@ -0,0 +1,498 @@ +/** + * The POLYX ledger — one fixture per row of the "Event → pool transition" table in + * docs/implementation/02-polyx-ledger.md, plus the properties the old `PolyxTransaction` model + * could not satisfy: a `Reserved`/`Unreserved` round-trip returns the pools to their starting + * values, and every movement writes a signed entry per account-side sharing one `movementId`. + * + * Events are built struct-style (v8), which is the surface A9 left entirely unindexed. + */ + +import { SubstrateEvent } from '@subql/types'; +import { EntryDirection, HoldReason, MovementKind, PolyxPool } from '../../src/types'; +import { + adjustLock, + handleBalanceBurned, + handleBalanceEndowed, + handleBalanceFrozen, + handleBalanceHeld, + handleBalanceReleased, + handleBalanceMinted, + handleBalanceReserved, + handleBalanceSet, + handleBalanceSuspended, + handleBalanceThawed, + handleBalanceTransfer, + handleBalanceUnreserved, + handleBonded, + handlePayoutStarted, + handleReward, + handleWithdrawn, + handleDustLost, + handleReserveRepatriated, +} from '../../src/mappings/entities/identities/mapPolyxLedger'; + +const ALICE = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; +const BOB = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; + +const storeGet = (): jest.Mock => (globalThis as any).store.get as jest.Mock; +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; +const storeGetByFields = (): jest.Mock => (globalThis as any).store.getByFields as jest.Mock; + +const mockCodec = (value: string) => ({ + toString: () => value, + toJSON: () => value, + toU8a: () => Buffer.from(value), +}); + +let blockHeight = 1_000_000; + +/** A struct-style event: the block metadata names every field. */ +const structEvent = ( + section: string, + method: string, + fields: Record, + { specVersion = 8_000_000, atHeight }: { specVersion?: number; atHeight?: number } = {} +): SubstrateEvent => { + if (atHeight === undefined) { + blockHeight += 1; + } + const height = atHeight ?? blockHeight; + + return { + idx: 4, + block: { + block: { header: { number: { toString: () => String(height) } } }, + timestamp: new Date('2024-06-01T12:00:00.000Z'), + specVersion, + }, + event: { + section, + method, + data: Object.values(fields).map(mockCodec), + meta: { + fields: Object.keys(fields).map(name => ({ + name: { isSome: true, unwrap: () => mockCodec(name) }, + typeName: { isSome: true, unwrap: () => mockCodec('Dummy') }, + })), + }, + }, + } as unknown as SubstrateEvent; +}; + +const balancesEvent = ( + method: string, + fields: Record, + opts: { specVersion?: number; atHeight?: number } = {} +): SubstrateEvent => structEvent('balances', method, fields, opts); + +/** A tuple-style event (pre-v8 Polymesh pallet): the block metadata carries no field names. */ +const tupleEvent = ( + section: string, + method: string, + values: string[], + specVersion: number +): SubstrateEvent => { + blockHeight += 1; + + return { + idx: 4, + block: { + block: { header: { number: { toString: () => String(blockHeight) } } }, + timestamp: new Date('2024-06-01T12:00:00.000Z'), + specVersion, + }, + event: { + section, + method, + data: values.map(mockCodec), + meta: { + fields: values.map(() => ({ + name: { isSome: false }, + typeName: { isSome: true, unwrap: () => mockCodec('Dummy') }, + })), + }, + }, + } as unknown as SubstrateEvent; +}; + +type Row = Record; + +let db: Record>; + +const clone = (value: Row): Row => { + const copy: Row = {}; + for (const [k, v] of Object.entries(value)) { + copy[k] = Array.isArray(v) + ? v.map(item => (item && typeof item === 'object' ? { ...item } : item)) + : v; + } + return copy; +}; + +beforeEach(() => { + db = {}; + blockHeight = 1_000_000; + + storeGet().mockImplementation((entity: string, id: string) => { + if (entity === 'Account') { + // Every test address is a known key, so `getOrCreateAccount` never reaches the chain. + return Promise.resolve({ id, address: id, identityId: undefined }); + } + const row = db[entity]?.[id]; + return Promise.resolve(row ? clone(row) : undefined); + }); + + storeSet().mockImplementation((entity: string, id: string, data: Row) => { + (db[entity] ??= {})[id] = clone(data); + return Promise.resolve(); + }); + + storeGetByFields().mockResolvedValue([]); +}); + +const entries = (): Row[] => Object.values(db['PolyxEntry'] ?? {}); +const balance = (address: string): Row | undefined => db['AccountBalance']?.[address]; + +describe('Event → pool transition', () => { + it('Transfer: from/Free → to/Free, one debit + one credit sharing a movementId', async () => { + await handleBalanceTransfer( + balancesEvent('Transfer', { from: ALICE, to: BOB, amount: '1000' }) + ); + + const rows = entries(); + expect(rows).toHaveLength(2); + expect(new Set(rows.map(r => r.movementId)).size).toBe(1); + + const debit = rows.find(r => r.direction === EntryDirection.Debit); + const credit = rows.find(r => r.direction === EntryDirection.Credit); + + expect(debit).toMatchObject({ + accountId: ALICE, + counterpartyAddress: BOB, + pool: PolyxPool.Free, + kind: MovementKind.Transfer, + amount: BigInt(-1000), + amountAbs: BigInt(1000), + }); + expect(credit).toMatchObject({ + accountId: BOB, + counterpartyAddress: ALICE, + amount: BigInt(1000), + }); + + expect(balance(ALICE)?.free).toBe(BigInt(-1000)); + expect(balance(BOB)?.free).toBe(BigInt(1000)); + }); + + it('Endowed: ∅ → who/Free', async () => { + await handleBalanceEndowed(balancesEvent('Endowed', { account: BOB, freeBalance: '500' })); + + expect(entries()).toHaveLength(1); + expect(entries()[0]).toMatchObject({ + accountId: BOB, + direction: EntryDirection.Credit, + kind: MovementKind.Endowment, + pool: PolyxPool.Free, + }); + expect(balance(BOB)?.free).toBe(BigInt(500)); + }); + + it('Reserved: who/Free → who/Reserved (Hold)', async () => { + await handleBalanceReserved(balancesEvent('Reserved', { who: ALICE, amount: '300' })); + + const rows = entries(); + expect(rows.map(r => r.pool).sort()).toEqual([PolyxPool.Free, PolyxPool.Reserved].sort()); + expect(rows.every(r => r.kind === MovementKind.Hold)).toBe(true); + expect(balance(ALICE)).toMatchObject({ free: BigInt(-300), reserved: BigInt(300) }); + }); + + it('Unreserved: who/Reserved → who/Free (Release)', async () => { + await handleBalanceUnreserved(balancesEvent('Unreserved', { who: ALICE, amount: '300' })); + + expect(balance(ALICE)).toMatchObject({ free: BigInt(300), reserved: BigInt(-300) }); + expect(entries().every(r => r.kind === MovementKind.Release)).toBe(true); + }); + + it('ReserveRepatriated to Free: from/Reserved → to/Free', async () => { + await handleReserveRepatriated( + balancesEvent('ReserveRepatriated', { + from: ALICE, + to: BOB, + amount: '250', + destinationStatus: 'Free', + }) + ); + + const credit = entries().find(r => r.direction === EntryDirection.Credit); + expect(credit).toMatchObject({ accountId: BOB, pool: PolyxPool.Free }); + expect(balance(ALICE)?.reserved).toBe(BigInt(-250)); + expect(balance(BOB)?.free).toBe(BigInt(250)); + }); + + it('Held{Staking}: who/Free → who/Reserved, and bonded tracks the hold', async () => { + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '900' }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(-900), + reserved: BigInt(900), + bonded: BigInt(900), + }); + expect(entries().every(r => r.holdReason === HoldReason.Staking)).toBe(true); + }); + + it('Released{Staking}: who/Reserved → who/Free, unwinding the hold', async () => { + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '900' }) + ); + await handleBalanceReleased( + balancesEvent('Released', { reason: 'Staking', who: ALICE, amount: '900' }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(0), + reserved: BigInt(0), + bonded: BigInt(0), + }); + }); + + it('Burned: who/Free → ∅', async () => { + await handleBalanceBurned(balancesEvent('Burned', { who: ALICE, amount: '120' })); + + expect(entries()).toHaveLength(1); + expect(entries()[0]).toMatchObject({ + direction: EntryDirection.Debit, + kind: MovementKind.Burn, + }); + expect(balance(ALICE)?.free).toBe(BigInt(-120)); + }); + + it('Slashed: who/Free → ∅ with kind Slash and totalSlashed', async () => { + await handleBalanceBurned(balancesEvent('Slashed', { who: ALICE, amount: '75' })); + + expect(entries()[0].kind).toBe(MovementKind.Slash); + expect(balance(ALICE)).toMatchObject({ free: BigInt(-75), totalSlashed: BigInt(75) }); + }); + + it('Minted: ∅ → who/Free', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: BOB, amount: '4000' })); + + expect(entries()[0]).toMatchObject({ + direction: EntryDirection.Credit, + kind: MovementKind.Mint, + }); + expect(balance(BOB)?.free).toBe(BigInt(4000)); + }); + + it('DustLost: account/Free → ∅', async () => { + await handleDustLost(balancesEvent('DustLost', { account: ALICE, amount: '7' })); + + expect(entries()[0].kind).toBe(MovementKind.DustLost); + expect(balance(ALICE)?.free).toBe(BigInt(-7)); + }); + + it('Suspended: who/Free → ∅ (A3 — handler now exists)', async () => { + await handleBalanceSuspended(balancesEvent('Suspended', { who: ALICE, amount: '9' })); + + expect(entries()).toHaveLength(1); + expect(balance(ALICE)?.free).toBe(BigInt(-9)); + }); +}); + +describe('properties the one-column model could not satisfy', () => { + it('Reserved then Unreserved returns free and reserved to their starting values', async () => { + await handleBalanceReserved(balancesEvent('Reserved', { who: ALICE, amount: '600' })); + await handleBalanceUnreserved(balancesEvent('Unreserved', { who: ALICE, amount: '600' })); + + expect(balance(ALICE)).toMatchObject({ free: BigInt(0), reserved: BigInt(0) }); + }); + + it('every movement is recorded as a signed entry: SUM(amount) is the net delta', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + await handleBalanceBurned(balancesEvent('Burned', { who: ALICE, amount: '250' })); + + const net = entries() + .filter(r => r.accountId === ALICE) + .reduce((sum, r) => sum + r.amount, BigInt(0)); + + expect(net).toBe(BigInt(750)); + expect(balance(ALICE)?.free).toBe(BigInt(750)); + }); + + it('BalanceSet sets the pool absolutely and does not corrupt subsequent totals', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + + await handleBalanceSet(balancesEvent('BalanceSet', { who: ALICE, free: '5000' })); + + // 5000, not 1000 + 5000 + expect(balance(ALICE)?.free).toBe(BigInt(5000)); + + const adjustment = entries().find(r => r.kind === MovementKind.BalanceSetAdjustment); + expect(adjustment).toMatchObject({ amount: BigInt(4000), freeAfter: BigInt(5000) }); + + await handleBalanceBurned(balancesEvent('Burned', { who: ALICE, amount: '500' })); + expect(balance(ALICE)?.free).toBe(BigInt(4500)); + }); + + it('BalanceSet writes a debit adjustment when it lowers the balance, and a per-pool entry pre-v8', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: BOB, amount: '9000' })); + + await handleBalanceSet( + balancesEvent('BalanceSet', { + identityId: '0x00', + account: BOB, + free: '8000', + reserved: '250', + }) + ); + + expect(balance(BOB)).toMatchObject({ free: BigInt(8000), reserved: BigInt(250) }); + + const adjustments = entries().filter(r => r.kind === MovementKind.BalanceSetAdjustment); + expect(adjustments.map(r => [r.pool, r.amount]).sort()).toEqual( + [ + [PolyxPool.Free, BigInt(-1000)], + [PolyxPool.Reserved, BigInt(250)], + ].sort() + ); + }); + + it('frozen is the MAX over active locks, not their sum', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + + await adjustLock(ALICE, 'staking ', BigInt(100), '0000009999'); + await adjustLock(ALICE, 'pips ', BigInt(150), '0000009999'); + + expect(balance(ALICE)?.frozen).toBe(BigInt(150)); // not 250 + expect(balance(ALICE)?.transferable).toBe(BigInt(850)); // free 1000 - frozen 150 + }); + + it('a lock writes no PolyxEntry and does not move the pools', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + const entriesAfterMint = entries().length; + + await handleBalanceFrozen(balancesEvent('Frozen', { who: ALICE, amount: '400' })); + + expect(entries().length).toBe(entriesAfterMint); // no new entry + expect(balance(ALICE)).toMatchObject({ + free: BigInt(1000), + reserved: BigInt(0), + frozen: BigInt(400), + }); + + await handleBalanceThawed(balancesEvent('Thawed', { who: ALICE, amount: '400' })); + expect(balance(ALICE)?.frozen).toBe(BigInt(0)); + }); + + it('carries the balance-after snapshot on each entry', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '1000' })); + await handleBalanceReserved(balancesEvent('Reserved', { who: ALICE, amount: '400' })); + + const last = entries() + .filter(r => r.accountId === ALICE) + .sort((a, b) => a.id.localeCompare(b.id)) + .at(-1); + + expect(last).toMatchObject({ freeAfter: BigInt(600), reservedAfter: BigInt(400) }); + }); +}); + +describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { + it('v7 Bonded produces no PolyxEntry and raises frozen via the staking lock', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + const beforeEntries = entries().length; + + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + expect(entries().length).toBe(beforeEntries); // no movement row + expect(balance(ALICE)).toMatchObject({ + free: BigInt(10000), // unchanged — pre-v8 bonding moves nothing + frozen: BigInt(4000), + bonded: BigInt(4000), + transferable: BigInt(6000), + }); + }); + + it('v7 Withdrawn lowers the staking lock', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + await handleWithdrawn(tupleEvent('staking', 'Withdrawn', [ALICE, '1500'], 7_004_001)); + + expect(balance(ALICE)?.frozen).toBe(BigInt(2500)); + }); + + it('v8 Bonded is ledger state only — no entry, no lock (the move is the paired Held)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + const beforeEntries = entries().length; + + await handleBonded( + balancesEvent('Bonded', { stash: ALICE, amount: '4000' }, { specVersion: 8_000_000 }) + ); + + expect(entries().length).toBe(beforeEntries); + expect(balance(ALICE)).toMatchObject({ + free: BigInt(10000), + frozen: BigInt(0), + bonded: BigInt(0), + }); + + // the actual v8 bonding movement: + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '4000' }) + ); + expect(balance(ALICE)).toMatchObject({ + free: BigInt(6000), + reserved: BigInt(4000), + bonded: BigInt(4000), + }); + }); + + it('pre-v8 Reward resolves an explicit Account payee — credited to that account, not the stash', async () => { + const PAYEE = '5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy'; + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => ({ account: PAYEE }) }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => null }), + }, + }; + + await handleReward(tupleEvent('staking', 'Reward', ['0xdid', ALICE, '900'], 7_004_001)); + + const reward = entries().find(r => r.kind === MovementKind.StakingReward); + expect(reward?.accountId).toBe(PAYEE); + expect(reward?.accountId).not.toBe(ALICE); + expect(balance(PAYEE)?.free).toBe(BigInt(900)); + + (globalThis as any).api.query = {}; + }); + + it('staking Reward is ∅ → stash/Free with the era from the preceding PayoutStarted', async () => { + await handlePayoutStarted( + structEvent( + 'staking', + 'PayoutStarted', + { eraIndex: '742', validatorStash: BOB, page: '0', next: '0' }, + { atHeight: 5_000_000 } + ) + ); + await handleReward( + structEvent( + 'staking', + 'Rewarded', + { stash: ALICE, dest: 'Staked', amount: '333' }, + { atHeight: 5_000_000 } + ) + ); + + const reward = entries().find(r => r.kind === MovementKind.StakingReward); + expect(reward).toMatchObject({ + accountId: ALICE, + direction: EntryDirection.Credit, + amount: BigInt(333), + eraIndex: 742, + }); + expect(balance(ALICE)).toMatchObject({ free: BigInt(333), totalRewards: BigInt(333) }); + }); +}); diff --git a/tests/unit/mapPolyxTransaction.test.ts b/tests/unit/mapPolyxTransaction.test.ts deleted file mode 100644 index fc8f36a6..00000000 --- a/tests/unit/mapPolyxTransaction.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Unit tests for mapPolyxTransaction parameter extraction logic. - * - * These tests verify the core logic for handling 8.x chain staking event parameters - * using the extract8xStakingAmount utility function from common.ts. - * - * The actual parameter extraction logic tested here: - * - On 8.x chain, Rewarded event has [stash, dest, amount] where dest is RewardDestination enum - * - On 8.x chain, Bonded/Unbonded have [stash, amount] - * - The code detects if second param is numeric to determine which param holds the amount - */ - -import { Codec } from '@polkadot/types/types'; -import { extract8xStakingAmount } from '../../src/utils/common'; - -/** - * Helper that wraps extract8xStakingAmount to return address too (for test compatibility) - */ -const extractAmountFrom8xParams = (params: Codec[]): { address: string; amount: bigint } => { - const [rawAddress, rawSecondParam, rawThirdParam] = params; - return { - address: getTextValue(rawAddress), - amount: extract8xStakingAmount(rawSecondParam, rawThirdParam), - }; -}; - -/** - * Pre-8.x parameter extraction logic - */ -const extractAmountFromPre8xParams = ( - params: Codec[] -): { identityId: string; address: string; amount: bigint } => { - const [rawDid, rawAddress, rawBalance] = params; - return { - identityId: getTextValue(rawDid), - address: getTextValue(rawAddress), - amount: getBigIntValue(rawBalance), - }; -}; - -describe('mapPolyxTransaction parameter extraction logic', () => { - describe('8.x chain - Bonded/Unbonded events (2 params)', () => { - it('should extract amount from second param when it is numeric', () => { - const params = [createMockCodec(TEST_ADDRESS), createMockCodec('1000000000000')]; - const result = extractAmountFrom8xParams(params); - - expect(result.address).toBe(TEST_ADDRESS); - expect(result.amount).toBe(BigInt('1000000000000')); - }); - - it.each([ - ['zero', '0', BigInt(0)], - ['small', '1', BigInt(1)], - ['large', '999999999999999999999999', BigInt('999999999999999999999999')], - ])('should handle %s amount', (_name, amountStr, expected) => { - const params = [createMockCodec(TEST_ADDRESS), createMockCodec(amountStr)]; - const result = extractAmountFrom8xParams(params); - expect(result.amount).toBe(expected); - }); - }); - - describe('8.x chain - Rewarded event with RewardDestination (3 params)', () => { - it.each(REWARD_DESTINATIONS)( - 'should extract amount from third param when second param is "%s"', - rewardDest => { - const params = [ - createMockCodec(TEST_ADDRESS), - createMockCodec(rewardDest), - createMockCodec('5000000000000'), - ]; - const result = extractAmountFrom8xParams(params); - - expect(result.address).toBe(TEST_ADDRESS); - expect(result.amount).toBe(BigInt('5000000000000')); - } - ); - }); - - describe('8.x chain - edge cases', () => { - it('should return 0 when second param is non-numeric and third param is missing', () => { - const params = [createMockCodec(TEST_ADDRESS), createMockCodec('UnknownEnum')]; - const result = extractAmountFrom8xParams(params); - expect(result.amount).toBe(BigInt(0)); - }); - - it('should not match strings with numeric prefix but non-numeric suffix', () => { - const params = [ - createMockCodec(TEST_ADDRESS), - createMockCodec('123abc'), - createMockCodec('8000000000000'), - ]; - const result = extractAmountFrom8xParams(params); - expect(result.amount).toBe(BigInt('8000000000000')); - }); - }); - - describe('pre-8.x chain parameter extraction', () => { - it('should extract identityId from first param and amount from third param', () => { - const params = [ - createMockCodec(TEST_DID), - createMockCodec(TEST_ADDRESS), - createMockCodec('9000000000000'), - ]; - const result = extractAmountFromPre8xParams(params); - - expect(result.identityId).toBe(TEST_DID); - expect(result.address).toBe(TEST_ADDRESS); - expect(result.amount).toBe(BigInt('9000000000000')); - }); - }); - - describe('pre-8.x BalanceSet layout (defect A1)', () => { - /** - * The chain emits `BalanceSet(IdentityId, AccountId, free, reserved)` — four params, with - * reserved at index 3 — identically at v5.4.3 / v6.3.5 / v7.0.0 / v7.4.0. `handleBalanceSet` - * read `params[4]`, which is out of bounds, so `getBigIntValue` returned `BigInt(0)` and the - * `if (reservedAmount)` guard silently skipped the Reserved row. It now reads `params[3]`. - */ - const extractBalanceSetReserved = (params: Codec[]): bigint => getBigIntValue(params[3]); - - it('reads the reserved balance from index 3', () => { - const params = [ - createMockCodec(TEST_DID), - createMockCodec(TEST_ADDRESS), - createMockCodec('1000'), // free - createMockCodec('250'), // reserved - ]; - - expect(extractBalanceSetReserved(params)).toBe(BigInt('250')); - }); - - it('index 4 is out of bounds for this event and would yield 0', () => { - const params = [ - createMockCodec(TEST_DID), - createMockCodec(TEST_ADDRESS), - createMockCodec('1000'), - createMockCodec('250'), - ]; - - expect(getBigIntValue(params[4])).toBe(BigInt(0)); - }); - }); - - describe('Numeric detection regex', () => { - it.each([ - ['0', true], - ['1', true], - ['123456789', true], - ['999999999999999999999999', true], - ...REWARD_DESTINATIONS.map(v => [v, false] as [string, boolean]), - ['123abc', false], - ['abc123', false], - ['', false], - [' 123', false], - ['123 ', false], - ])('isNumericString("%s") should be %s', (value, expected) => { - expect(isNumericString(value)).toBe(expected); - }); - }); - - describe('extract8xStakingAmount utility function', () => { - it('should return amount from second param when numeric (Bonded/Unbonded)', () => { - const rawSecondParam = createMockCodec('1000000000000'); - expect(extract8xStakingAmount(rawSecondParam)).toBe(BigInt('1000000000000')); - }); - - it('should return amount from third param when second is non-numeric (Rewarded)', () => { - const rawSecondParam = createMockCodec('Staked'); - const rawThirdParam = createMockCodec('2000000000000'); - expect(extract8xStakingAmount(rawSecondParam, rawThirdParam)).toBe(BigInt('2000000000000')); - }); - - it('should return 0 when second is non-numeric and third is undefined', () => { - const rawSecondParam = createMockCodec('Controller'); - expect(extract8xStakingAmount(rawSecondParam, undefined)).toBe(BigInt(0)); - }); - - it('should handle all RewardDestination enum values', () => { - const expectedAmount = BigInt('5000000000000'); - REWARD_DESTINATIONS.forEach(enumValue => { - const rawSecondParam = createMockCodec(enumValue); - const rawThirdParam = createMockCodec('5000000000000'); - expect(extract8xStakingAmount(rawSecondParam, rawThirdParam)).toBe(expectedAmount); - }); - }); - - it('should handle zero balance', () => { - const rawSecondParam = createMockCodec('0'); - expect(extract8xStakingAmount(rawSecondParam)).toBe(BigInt(0)); - }); - }); -}); diff --git a/tests/unit/rewardDestinationA15.test.ts b/tests/unit/rewardDestinationA15.test.ts new file mode 100644 index 00000000..06bfeb77 --- /dev/null +++ b/tests/unit/rewardDestinationA15.test.ts @@ -0,0 +1,139 @@ +/** + * Defect A15 — pre-v8 staking rewards and the account that received them. + * + * Measured across a spread of eras (`scripts/measure-a15-payees.ts`): a large share of pre-v8 + * mainnet rewards were paid to a `Controller` or an explicit `Account`, not the stash. So + * `handleStakingEvent` now reads `staking.payee(stash)` from chain storage at the reward block + * (`resolveLegacyRewardDestination`) rather than recording `LegacyUnknown`. + * + * These tests pin: the storage read resolves `Controller`/`Account`/`Staked`; and the + * `LegacyUnknown` placeholder still stands when the read is not possible (a pruned node), so it + * is never silently resolved to something wrong. + */ + +import { SubstrateEvent } from '@subql/types'; +import { handleStakingEvent } from '../../src/mappings/entities/events/mapStakingEvent'; + +const STASH = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; +const PAYEE = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; +const CONTROLLER = '5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy'; + +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; + +const mockPayee = (payee: any, bonded?: string) => { + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => payee }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => bonded ?? null }), + }, + }; +}; + +const codec = (value: string) => ({ toString: () => value, toJSON: () => value }); + +const rewardEvent = ( + method: 'Reward' | 'Rewarded', + data: any[], + specVersion: number +): SubstrateEvent => + ({ + idx: 1, + block: { + block: { header: { number: { toString: () => '5000' } } }, + timestamp: new Date('2022-01-01T00:00:00Z'), + specVersion, + }, + event: { + section: 'staking', + method, + data, + meta: { fields: data.map(() => ({ name: { isSome: false }, typeName: { isSome: false } })) }, + }, + extrinsic: undefined, + } as unknown as SubstrateEvent); + +const savedStakingEvent = () => + storeSet() + .mock.calls.filter(([entity]) => entity === 'StakingEvent') + .map(([, , row]) => row) + .at(-1); + +beforeEach(() => { + (globalThis as any).api.runtimeVersion.specName = { toString: () => 'polymesh' }; + (globalThis as any).api.query = {}; +}); + +describe('A15 — pre-v8 reward destination', () => { + it('reads staking.payee and resolves a Controller payee to the controller account', async () => { + mockPayee('Controller', CONTROLLER); + + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1000')], 7_004_001) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Controller'); + expect(row.rewardDestinationAccount).toBe(CONTROLLER); + expect(row.stashAccount).toBe(STASH); + }); + + it('resolves an explicit Account payee', async () => { + mockPayee({ account: PAYEE }); + + await handleStakingEvent( + rewardEvent('Rewarded', [codec('0x00'), codec(STASH), codec('2000')], 7_000_000) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Account'); + expect(row.rewardDestinationAccount).toBe(PAYEE); + }); + + it('resolves Staked/Stash to the stash itself', async () => { + mockPayee('Staked'); + + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1500')], 7_004_001) + ); + + expect(savedStakingEvent()).toMatchObject({ + rewardDestination: 'Staked', + rewardDestinationAccount: STASH, + }); + }); + + it('falls back to LegacyUnknown when the payee read is not possible, never to the stash', async () => { + // api.query.staking absent — a pruned node or a runtime with no such storage + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1000')], 7_004_001) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('LegacyUnknown'); + expect(row.rewardDestinationAccount).toBeUndefined(); + }); + + it('v8 resolves the destination account for an explicit Account payee', async () => { + await handleStakingEvent( + rewardEvent( + 'Rewarded', + [codec(STASH), { toJSON: () => ({ Account: PAYEE }) }, codec('3000')], + 8_000_000 + ) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Account'); + expect(row.rewardDestinationAccount).toBe(PAYEE); + }); + + it('v8 resolves Staked/Stash to the stash itself', async () => { + await handleStakingEvent( + rewardEvent('Rewarded', [codec(STASH), { toJSON: () => 'Staked' }, codec('4000')], 8_000_000) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Staked'); + expect(row.rewardDestinationAccount).toBe(STASH); + }); +}); diff --git a/tests/unit/seedAccountBalances.test.ts b/tests/unit/seedAccountBalances.test.ts new file mode 100644 index 00000000..4c6677aa --- /dev/null +++ b/tests/unit/seedAccountBalances.test.ts @@ -0,0 +1,98 @@ +/** + * The genesis balance seeder (`src/seed/accountBalance.ts`). Without an opening snapshot every + * balance the POLYX ledger derives is wrong by the genesis allocation, so this is a hard + * prerequisite for the ledger, not an optimisation. + */ + +import { seedAccountBalances } from '../../src/seed/accountBalance'; + +const A = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; +const B = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; + +const storeGet = (): jest.Mock => (globalThis as any).store.get as jest.Mock; +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; + +const codec = (value: string) => ({ toString: () => value, toJSON: () => value }); + +/** `[storageKey, accountInfo]` pairs as `api.query.system.account.entries()` yields them. */ +const accountEntry = ( + address: string, + data: { + free: string; + reserved?: string; + miscFrozen?: string; + feeFrozen?: string; + frozen?: string; + } +) => [ + { args: [codec(address)] }, + { + data: Object.fromEntries(Object.entries(data).map(([k, v]) => [k, codec(v as string)])), + }, +]; + +describe('seedAccountBalances', () => { + let db: Record>; + + beforeEach(() => { + db = {}; + + storeGet().mockImplementation((entity: string, id: string) => { + if (entity === 'Account') { + return Promise.resolve({ id, address: id, identityId: undefined }); + } + return Promise.resolve(db[entity]?.[id]); + }); + storeSet().mockImplementation((entity: string, id: string, data: any) => { + (db[entity] ??= {})[id] = { ...data }; + return Promise.resolve(); + }); + + (globalThis as any).api.query = { + system: { + account: { + entries: jest + .fn() + .mockResolvedValue([ + accountEntry(A, { + free: '1000000', + reserved: '250', + miscFrozen: '400', + feeFrozen: '100', + }), + accountEntry(B, { free: '5000', reserved: '0' }), + accountEntry('5zeroBalance', { free: '0', reserved: '0' }), + ]), + }, + }, + }; + }); + + it('creates one AccountBalance per funded account, skipping empty ones', async () => { + const { seeded } = await seedAccountBalances({ blockId: '0000000000', datetime: new Date(0) }); + + expect(seeded).toBe(2); + expect(Object.keys(db['AccountBalance'])).toEqual([A, B]); + }); + + it('takes free/reserved verbatim and frozen as MAX(miscFrozen, feeFrozen) pre-v8', async () => { + await seedAccountBalances({ blockId: '0000000000', datetime: new Date(0) }); + + expect(db['AccountBalance'][A]).toMatchObject({ + free: BigInt(1000000), + reserved: BigInt(250), + frozen: BigInt(400), // max(400, 100), not the sum + total: BigInt(1000250), + transferable: BigInt(999600), // free - frozen + }); + }); + + it('records a genesis freeze as a single lock so frozen stays a MAX going forward', async () => { + await seedAccountBalances({ blockId: '0000000000', datetime: new Date(0) }); + + expect(db['AccountBalance'][A].locks).toEqual([ + { lockId: 'genesis', amount: BigInt(400), reasons: undefined }, + ]); + expect(db['AccountBalance'][B].locks).toEqual([]); + }); +}); diff --git a/tests/unit/stakingEventHandlers.test.ts b/tests/unit/stakingEventHandlers.test.ts index b4512df9..0873a918 100644 --- a/tests/unit/stakingEventHandlers.test.ts +++ b/tests/unit/stakingEventHandlers.test.ts @@ -8,8 +8,9 @@ * Pre-8.x chain: * - Bonded/Unbonded/Rewarded: [did, account, amount] (3 params with DID) * - * Note: The extract8xStakingAmount utility function is comprehensively tested in - * mapPolyxTransaction.test.ts. This file focuses on is8xChain detection logic. + * Note: `extract8xStakingAmount` is covered in `extract8xStakingAmount.test.ts` and the pre-v8 + * reward-destination (A15) resolution in `rewardDestinationA15.test.ts`. This file focuses on + * `is8xChain` detection logic. */ import { is8xChain } from '../../src/utils/common'; From 45400780533e10b78d0788a4b1113171ee45dd35 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:23:58 +0530 Subject: [PATCH 05/22] =?UTF-8?q?test:=20=F0=9F=92=8D=20add=20a=20harness?= =?UTF-8?q?=20that=20checks=20derived=20balances=20against=20chain=20state?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An offline script samples accounts, oversampling the ones involved in the riskier cases, and compares the ledger's free, reserved and frozen balances against the chain at blocks on either side of each runtime upgrade, then prints a breakdown of any differences by cause. A second script measures how often pre-upgrade rewards were paid somewhere other than the stash. The running indexer also checks touched accounts against chain state periodically and after a set-balance or dust event, recording and correcting any drift. --- docs/reference/polyx-reconciliation.md | 90 ++++++ scripts/measure-a15-payees.ts | 184 +++++++++++++ scripts/reconcile-polyx.ts | 368 +++++++++++++++++++++++++ tests/unit/reconcilePolyx.test.ts | 130 +++++++++ 4 files changed, 772 insertions(+) create mode 100644 docs/reference/polyx-reconciliation.md create mode 100644 scripts/measure-a15-payees.ts create mode 100644 scripts/reconcile-polyx.ts create mode 100644 tests/unit/reconcilePolyx.test.ts diff --git a/docs/reference/polyx-reconciliation.md b/docs/reference/polyx-reconciliation.md new file mode 100644 index 00000000..0bbce924 --- /dev/null +++ b/docs/reference/polyx-reconciliation.md @@ -0,0 +1,90 @@ +# POLYX ledger — reconciliation and the accounting-fidelity findings + +Companion to [`../implementation/02-polyx-ledger.md`](../implementation/02-polyx-ledger.md). +Records what was verified against chain state while building the ledger, and how to run the +acceptance gate (D11). + +--- + +## Verified against real v8 blocks — the `staking.Bonded` ↔ `balances.Held{Staking}` pairing + +Plan 02 §"Staking" made the v8 handling conditional on confirming that bonding's balance movement +arrives as `balances.Held{reason:Staking}` in the same extrinsic as `staking.Bonded`. Checked on +Polymesh **testnet** (spec 8001000): + +| Block | Call | Events in the extrinsic | +|---|---|---| +| 25163012 | `staking.bondExtra` | `Withdraw`(fee), **`Held{Staking, 9640951}`**, **`Bonded{9640951}`**, `Deposit`(fee refund), `TransactionFeePaid` | +| 25107475 | `staking.withdrawUnbonded` | `Withdraw`(fee), **`Released{Staking, 5000000000}`**, **`Withdrawn{5000000000}`**, `Deposit`, `TransactionFeePaid` | +| 25721280 | `staking.rebond` | `Withdraw`(fee), `Bonded{5874231757}` — **no `Held`** | +| 25774711 | payout (`Initialization`) | per nominator: `Deposit{amount}`, `Held{Staking, amount}` (for `dest:Staked`), `Rewarded{stash, dest, amount}` | + +**Conclusion — the pairing holds.** The balance movement is always the `Held`/`Released` event +where there is one; `staking.Bonded`/`Unbonded`/`Withdrawn` are ledger-state events. `rebond` +correctly emits `Bonded` with no `Held` because no balance moves (funds move between the ledger's +`unlocking` and `active` within an existing hold). So on v8 the ledger writes **no `PolyxEntry`** +for the `staking.*` events — the entry comes from the paired `balances` event — and this is +correct, not an under-count. + +Pre-v8 (≤ v7.4) bonding is `set_lock(STAKING_ID, …)` and moves no balance: `Bonded`/`Withdrawn` +maintain `AccountBalance.locks` only, `frozen = MAX(active locks)`, still no `PolyxEntry`. + +--- + +## Measured — defect A15, the pre-v8 reward-destination gap + +Pre-8.x `staking.Reward`/`Rewarded` carries only the stash. `scripts/measure-a15-payees.ts` +sampled `staking.payee(stash)` at the reward block for pre-v8 reward stashes across a spread of +eras on both networks: + +- **Near the v8 boundary** (last pre-v8 payout era): ~190 distinct stashes, **100% `Staked`/`Stash`**. +- **Across earlier eras** (mainnet spec 3010 → 7004001): a **large share** paid to `Controller` + or an explicit `Account` — e.g. every sampled stash at spec 3010 used `Controller`; `Account` + payees common at spec 5003001 / 6001031 / 7003003 / 7004001. + +**Decision: not near-zero — the storage read was added.** `resolveLegacyRewardDestination` +(`src/utils/staking.ts`) reads `staking.payee(stash)` (and `staking.bonded(stash)` for the +`Controller` case) at the reward block, resolving the real recipient. It is wired into both +`mapStakingEvent` (`StakingEvent.rewardDestination` / `rewardDestinationAccount`) and the ledger +(`handleReward` credits the resolved account). `LegacyUnknown` remains only as the fallback when +the read is not possible (a pruned node), and `rewardDestinationA15.test.ts` pins that it is +never silently resolved to the stash instead. + +Cheap now — the read happens during the D5 genesis replay, which is running anyway. Awkward +later — it would need an archive node holding the pre-v8 state. That asymmetry is why it is done +in this phase rather than deferred. + +--- + +## The reconciliation harness (D11) — the acceptance gate + +### In-flight — `src/mappings/entities/identities/reconcilePolyx.ts` + +Wired into the ledger handlers. Every 500th block for accounts touched in that block, and always +after `BalanceSet` / `DustLost`, the derived `AccountBalance` is compared against `system.account` +read at the block being indexed (`api.query` targets the current block; `.at` is unsupported). +On a mismatch it writes a `BalanceReconciliationDrift` anomaly **and corrects** the derived +value, so drift from one missed or mis-signed event cannot compound into every later balance. + +### Offline — `scripts/reconcile-polyx.ts` + +Run separately from block indexing, against a **synced local database** and a public archive RPC: + +``` +DB_HOST=… DB_PORT=… DB_USER=… DB_PASS=… DB_DATABASE=… \ + yarn ts-node scripts/reconcile-polyx.ts --rpc wss://mainnet-rpc.polymesh.network +``` + +- Samples accounts stratified by activity, oversampling everyone in a `BalanceSetAdjustment`, + `DustLost`, `Slash` or pre-v8 `StakingReward` entry. +- Compares at one block before and one after each of 5_000_000, 6_000_000, 7_000_000, + 7_003_000, 7_004_001, 8_000_000. +- Compares `free`, `reserved`, `frozen` **independently**, and cross-checks `SUM(PolyxEntry)` + against `AccountBalance` (the two time-travel mechanisms in §7.4 of plan 02 must agree). +- Output is a **mismatch taxonomy**: constant drift from a block = one missed event; growing + drift = a systematically mis-signed one; drift confined to `reserved` = a pool-mapping error. +- Resumable via `.reconcile-polyx.checkpoint.json`. + +**Acceptance for the phase:** the harness reports zero unexplained mismatches across the sample, +with every explained one written up here or in the PR body. This — not "the resync completed" — +is the gate. diff --git a/scripts/measure-a15-payees.ts b/scripts/measure-a15-payees.ts new file mode 100644 index 00000000..0a7b76f7 --- /dev/null +++ b/scripts/measure-a15-payees.ts @@ -0,0 +1,184 @@ +/** + * Defect A15 — measures the share of pre-v8 staking rewards paid somewhere other than the stash. + * + * Pre-8.x `staking.Reward`/`Rewarded` carries only the stash, so where a staker set a payee of + * `Controller` or an explicit `Account` the index cannot say which account received the POLYX + * (`rewardDestination: 'LegacyUnknown'`). Whether that gap is worth closing with a + * `staking.payee(stash)` read during the genesis replay is a measurement question, not a + * principle one — this is the measurement. + * + * Method: for a spread of pre-v8 blocks (anchored across the chain's history), take the stashes + * that received a reward in that block and read `staking.payee(stash)` **at that block** (chain + * storage, historical). Classify `Staked`/`Stash` as "went to the stash" and `Controller`/ + * `Account` as "went elsewhere". + * + * Usage (from the repo root, no database needed): + * + * yarn ts-node scripts/measure-a15-payees.ts \ + * --rpc wss://mainnet-rpc.polymesh.network \ + * --dictionary https://mainnet-subql-dictionary.polymesh.network/ \ + * [--anchors 4000000,8000000,12000000,16000000,20000000,23000000] [--per-anchor 5] + */ +import { ApiPromise, WsProvider } from '@polkadot/api'; +import chainTypes from '../src/chainTypes'; + +const { types, typesBundle } = chainTypes; + +const V8 = 8_000_000; + +const argOf = (name: string, fallback?: string): string | undefined => { + const i = process.argv.indexOf(`--${name}`); + + return i >= 0 ? process.argv[i + 1] : fallback; +}; + +interface DictEvent { + blockHeight: string; +} + +const dictionaryQuery = async (url: string, query: string): Promise<{ nodes: DictEvent[] }> => { + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query }), + }); + + return ((await res.json()) as { data: { events: { nodes: DictEvent[] } } }).data.events; +}; + +const rewardBlocksAfter = async ( + dictionary: string, + anchor: number, + limit: number +): Promise => { + const { nodes } = await dictionaryQuery( + dictionary, + `{ events(filter: { + module: { equalTo: "staking" }, + event: { in: ["Reward", "Rewarded"] }, + blockHeight: { greaterThan: "${anchor}" } + }, orderBy: BLOCK_HEIGHT_ASC, first: ${limit * 6}) { nodes { blockHeight } } }` + ); + + return [...new Set(nodes.map(n => Number(n.blockHeight)))].slice(0, limit); +}; + +const main = async (): Promise => { + const rpc = argOf('rpc'); + const dictionary = argOf('dictionary'); + + if (!rpc || !dictionary) { + console.error('Pass --rpc and --dictionary '); + process.exit(1); + } + + const anchors = ( + argOf('anchors', '4000000,8000000,12000000,16000000,20000000,23000000') as string + ) + .split(',') + .map(Number); + const perAnchor = Number(argOf('per-anchor', '5')); + + const api = await ApiPromise.create({ + provider: new WsProvider(rpc), + noInitWarn: true, + types: types as never, + typesBundle: typesBundle as never, + }); + + const seen = new Set(); + let toStash = 0; + let elsewhere = 0; + let none = 0; + let errors = 0; + const nonStash: string[] = []; + + for (const anchor of anchors) { + for (const block of await rewardBlocksAfter(dictionary, anchor, perAnchor)) { + const hash = await api.rpc.chain.getBlockHash(block); + const specVersion = (await api.rpc.state.getRuntimeVersion(hash)).specVersion.toNumber(); + + if (specVersion >= V8) { + continue; + } + + let events; + try { + events = await api.query.system.events.at(hash); + } catch { + errors += 1; + continue; + } + + const at = await api.at(hash); + + const stashes = events + .filter( + record => + record.event.section === 'staking' && + ['Reward', 'Rewarded'].includes(record.event.method) + ) + .map(record => { + const data = record.event.data; + return (data.length >= 3 ? data[1] : data[0]).toString(); + }); + + for (const stash of stashes) { + if (!stash || seen.has(stash)) { + continue; + } + seen.add(stash); + + try { + const payee = (await at.query.staking.payee(stash)).toString(); + + if (payee === 'Staked' || payee === 'Stash') { + toStash += 1; + } else if (payee === 'None') { + none += 1; + } else { + elsewhere += 1; + nonStash.push(` block ${block} (spec ${specVersion}) ${stash} -> ${payee}`); + } + } catch { + errors += 1; + } + } + } + } + + await api.disconnect(); + + const total = toStash + elsewhere + none; + const pct = (n: number) => (total ? ((100 * n) / total).toFixed(1) : '0.0'); + + console.log(`\n=== A15 payee measurement (${rpc}) ===`); + console.log( + `distinct pre-v8 reward stashes sampled: ${total} across ${anchors.length} anchors (errors ${errors})` + ); + console.log( + ` payee = Stash/Staked (reward went to the stash): ${toStash} (${pct(toStash)}%)` + ); + console.log( + ` payee = Controller/Account (went elsewhere): ${elsewhere} (${pct(elsewhere)}%)` + ); + console.log(` payee = None: ${none}`); + + if (nonStash.length) { + console.log('\nnon-stash payees:'); + nonStash.forEach(line => console.log(line)); + } + + console.log( + `\nverdict: ${ + total > 0 && elsewhere / total < 0.01 + ? 'near-zero — LegacyUnknown documented in the schema is defensible; no storage-read backfill needed' + : 'material — add a staking.payee(stash) read at the reward block during the genesis replay' + }` + ); +}; + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/scripts/reconcile-polyx.ts b/scripts/reconcile-polyx.ts new file mode 100644 index 00000000..14b13c58 --- /dev/null +++ b/scripts/reconcile-polyx.ts @@ -0,0 +1,368 @@ +/** + * The POLYX reconciliation harness (decision D11) — the acceptance gate for the POLYX ledger. + * + * The in-flight reconciliation in `src/mappings/entities/identities/reconcilePolyx.ts` only ever + * compares at the block being indexed, so it cannot answer "is the history right". This script + * does, offline, against a synced local database and a public archive RPC. + * + * Method (docs/implementation/02-polyx-ledger.md §"The reconciliation harness"): + * + * 1. Sample accounts, stratified by activity, oversampling every account that appears in a + * `BalanceSet`, `DustLost`, `Slashed` or pre-v8 `StakingReward` entry — that is where the + * known accounting gaps live. + * 2. Compare at spec-version boundary blocks: one before and one after each of 5_000_000, + * 6_000_000, 7_000_000, 7_003_000, 7_004_001 and 8_000_000. A drift that appears on only one + * side of a boundary names the runtime that caused it. + * 3. Compare `free`, `reserved` and `frozen` INDEPENDENTLY — comparing only the total lets a + * pair of offsetting pool-mapping errors cancel out and pass. + * 4. Classify every mismatch. A drift constant from a block is one missed event; a drift that + * grows is a systematically mis-signed one; a drift confined to `reserved` is a pool-mapping + * error. The taxonomy is the output — a count is not actionable. + * + * Resumable: progress is checkpointed to `.reconcile-polyx.checkpoint.json`, so a rate-limited + * public endpoint can be worked through across several runs. + * + * Usage (from the repo root, against a synced DB): + * + * DB_HOST=h DB_PORT=p DB_USER=u DB_PASS=p DB_DATABASE=d \ + * yarn ts-node scripts/reconcile-polyx.ts \ + * --rpc wss://mainnet-rpc.polymesh.network \ + * [--sample 80] [--high 30] [--typical 30] [--reset] + */ +import { ApiPromise, WsProvider } from '@polkadot/api'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { DataSource } from 'typeorm'; +import { getPostgresDataSource } from '../db/utils'; + +const CHECKPOINT = join(__dirname, '..', '.reconcile-polyx.checkpoint.json'); +const PAD = 10; +const padId = (n: number | string): string => String(n).padStart(PAD, '0'); + +/** Spec-version boundaries on the public chain's scale, one comparison block on each side. */ +const BOUNDARIES = [5_000_000, 6_000_000, 7_000_000, 7_003_000, 7_004_001, 8_000_000]; + +const argOf = (name: string, fallback?: string): string | undefined => { + const i = process.argv.indexOf(`--${name}`); + + return i >= 0 ? process.argv[i + 1] : fallback; +}; + +const hasFlag = (name: string): boolean => process.argv.includes(`--${name}`); + +// --------------------------------------------------------------------------------------------- +// Sampling +// --------------------------------------------------------------------------------------------- + +interface Sample { + address: string; + reason: 'high-traffic' | 'typical' | 'balance-set' | 'dust-lost' | 'slashed' | 'legacy-reward'; +} + +const oversampledKinds: Array<[Sample['reason'], string]> = [ + ['balance-set', 'BalanceSetAdjustment'], + ['dust-lost', 'DustLost'], + ['slashed', 'Slash'], +]; + +const sampleAccounts = async ( + db: DataSource, + { high, typical }: { high: number; typical: number } +): Promise => { + const byReason = new Map(); + const add = (address: string, reason: Sample['reason']) => { + if (!byReason.has(address)) { + byReason.set(address, { address, reason }); + } + }; + + const ranked: Array<{ account_id: string; n: string }> = await db.query( + `SELECT account_id, count(*) AS n FROM polyx_entries GROUP BY account_id ORDER BY n DESC LIMIT $1`, + [high + typical * 4] + ); + + ranked.slice(0, high).forEach(r => add(r.account_id, 'high-traffic')); + ranked + .slice(high) + .filter((_, i) => i % 4 === 0) + .slice(0, typical) + .forEach(r => add(r.account_id, 'typical')); + + for (const [reason, kind] of oversampledKinds) { + const rows: Array<{ account_id: string }> = await db.query( + `SELECT DISTINCT account_id FROM polyx_entries WHERE kind = $1 LIMIT 200`, + [kind] + ); + rows.forEach(r => add(r.account_id, reason)); + } + + const legacyRewards: Array<{ account_id: string }> = await db.query( + `SELECT DISTINCT account_id FROM polyx_entries + WHERE kind = 'StakingReward' AND spec_version_id < 8000000 LIMIT 200` + ); + legacyRewards.forEach(r => add(r.account_id, 'legacy-reward')); + + return [...byReason.values()]; +}; + +// --------------------------------------------------------------------------------------------- +// Derived vs on-chain +// --------------------------------------------------------------------------------------------- + +interface Triple { + free: bigint; + reserved: bigint; + frozen: bigint; +} + +const zero: Triple = { free: BigInt(0), reserved: BigInt(0), frozen: BigInt(0) }; + +/** Derived balance at block N: the historical `account_balance` row whose range covers N. */ +const derivedAt = async (db: DataSource, address: string, block: number): Promise => { + const [row]: Array<{ free: string; reserved: string; frozen: string }> = await db.query( + `SELECT free, reserved, frozen FROM account_balance + WHERE id = $1 AND _block_range @> $2::int8 LIMIT 1`, + [address, block] + ); + + if (!row) { + return zero; + } + + return { free: BigInt(row.free), reserved: BigInt(row.reserved), frozen: BigInt(row.frozen) }; +}; + +/** The independent second mechanism: SUM(PolyxEntry.amount) by pool up to block N. */ +const summedAt = async ( + db: DataSource, + address: string, + block: number +): Promise<{ free: bigint; reserved: bigint }> => { + const rows: Array<{ pool: string; s: string }> = await db.query( + `SELECT pool, COALESCE(sum(amount), 0) AS s FROM polyx_entries + WHERE account_id = $1 AND created_block_id <= $2 GROUP BY pool`, + [address, padId(block)] + ); + + const map = new Map(rows.map(r => [r.pool, BigInt(r.s)])); + + return { free: map.get('Free') ?? BigInt(0), reserved: map.get('Reserved') ?? BigInt(0) }; +}; + +const chainAt = async (api: ApiPromise, address: string, block: number): Promise => { + const hash = await api.rpc.chain.getBlockHash(block); + const at = await api.at(hash); + const info = (await at.query.system.account(address)) as unknown as { + data: Record; + }; + const d = info.data; + const big = (v?: { toString(): string }) => BigInt(v?.toString() ?? '0'); + const misc = big(d.miscFrozen); + const fee = big(d.feeFrozen); + + return { + free: big(d.free), + reserved: big(d.reserved), + frozen: d.frozen !== undefined ? big(d.frozen) : misc > fee ? misc : fee, + }; +}; + +// --------------------------------------------------------------------------------------------- +// Comparison and taxonomy +// --------------------------------------------------------------------------------------------- + +type Field = 'free' | 'reserved' | 'frozen'; + +interface Mismatch { + address: string; + reason: Sample['reason']; + block: number; + boundary: number; + side: 'before' | 'after'; + field: Field; + derived: bigint; + onChain: bigint; + delta: bigint; + summedDisagreesWithBalance: boolean; +} + +const compare = ( + sample: Sample, + boundary: number, + side: 'before' | 'after', + block: number, + derived: Triple, + onChain: Triple, + summed: { free: bigint; reserved: bigint } +): Mismatch[] => { + const out: Mismatch[] = []; + + (['free', 'reserved', 'frozen'] as Field[]).forEach(field => { + if (derived[field] === onChain[field]) { + return; + } + + out.push({ + address: sample.address, + reason: sample.reason, + block, + boundary, + side, + field, + derived: derived[field], + onChain: onChain[field], + delta: derived[field] - onChain[field], + summedDisagreesWithBalance: + field !== 'frozen' && summed[field as 'free' | 'reserved'] !== derived[field], + }); + }); + + return out; +}; + +const classify = (mismatches: Mismatch[]): string => { + if (mismatches.length === 0) { + return 'no unexplained mismatches across the sample'; + } + + const byAccountField = new Map(); + mismatches.forEach(m => { + const key = `${m.address}|${m.field}`; + const group = byAccountField.get(key) ?? []; + group.push(m); + byAccountField.set(key, group); + }); + + const lines: string[] = []; + + for (const [key, group] of byAccountField) { + const sorted = [...group].sort((a, b) => a.block - b.block); + const deltas = sorted.map(m => m.delta); + const constant = deltas.every(d => d === deltas[0]); + const growing = + deltas.length > 1 && + deltas.every((d, i) => i === 0 || d > deltas[i - 1] === deltas[1] > deltas[0]); + const [address, field] = key.split('|'); + const firstDrift = sorted[0]; + + let verdict: string; + if (constant) { + verdict = `constant drift of ${deltas[0]} from block ${firstDrift.block} — one missed event`; + } else if (growing) { + verdict = `growing drift (${deltas[0]} → ${ + deltas[deltas.length - 1] + }) — a systematically mis-signed event`; + } else { + verdict = `irregular drift — needs manual inspection`; + } + + if (field === 'reserved' && !sorted.some(m => m.field !== 'reserved')) { + verdict += '; confined to `reserved` — a pool-mapping error'; + } + if (sorted.some(m => m.summedDisagreesWithBalance)) { + verdict += '; SUM(entries) also disagrees with AccountBalance — the two mechanisms diverged'; + } + + lines.push(` ${address} [${field}] (${firstDrift.reason}): ${verdict}`); + } + + return `${mismatches.length} field-mismatches:\n${lines.join('\n')}`; +}; + +// --------------------------------------------------------------------------------------------- +// Runner +// --------------------------------------------------------------------------------------------- + +interface Checkpoint { + rpc: string; + done: string[]; + mismatches: Mismatch[]; +} + +const loadCheckpoint = (rpc: string): Checkpoint => { + if (hasFlag('reset') || !existsSync(CHECKPOINT)) { + return { rpc, done: [], mismatches: [] }; + } + + const cp = JSON.parse(readFileSync(CHECKPOINT, 'utf-8'), (_k, v) => + typeof v === 'string' && /^-?\d+n$/.test(v) ? BigInt(v.slice(0, -1)) : v + ) as Checkpoint; + + return cp.rpc === rpc ? cp : { rpc, done: [], mismatches: [] }; +}; + +const saveCheckpoint = (cp: Checkpoint): void => + writeFileSync( + CHECKPOINT, + JSON.stringify(cp, (_k, v) => (typeof v === 'bigint' ? `${v}n` : v), 2) + ); + +const main = async (): Promise => { + const rpc = argOf('rpc'); + if (!rpc) { + console.error('Pass --rpc (public archive endpoint)'); + process.exit(1); + } + + const high = Number(argOf('high', '30')); + const typical = Number(argOf('typical', '30')); + + const db = await getPostgresDataSource(); + const api = await ApiPromise.create({ provider: new WsProvider(rpc), noInitWarn: true }); + const head = (await api.rpc.chain.getHeader()).number.toNumber(); + + const checkpoint = loadCheckpoint(rpc); + const done = new Set(checkpoint.done); + + const samples = await sampleAccounts(db, { high, typical }); + console.log( + `Sampled ${samples.length} accounts; comparing at ${BOUNDARIES.length * 2} boundary blocks each` + ); + + for (const sample of samples) { + for (const boundary of BOUNDARIES) { + for (const [side, block] of [ + ['before', boundary - 1], + ['after', boundary + 1], + ] as const) { + if (block > head) { + continue; + } + + const key = `${sample.address}|${boundary}|${side}`; + if (done.has(key)) { + continue; + } + + try { + const [derived, summed, onChain] = await Promise.all([ + derivedAt(db, sample.address, block), + summedAt(db, sample.address, block), + chainAt(api, sample.address, block), + ]); + + checkpoint.mismatches.push( + ...compare(sample, boundary, side, block, derived, onChain, summed) + ); + } catch (e) { + console.warn(`skip ${key}: ${(e as Error).message}`); + } + + done.add(key); + checkpoint.done = [...done]; + saveCheckpoint(checkpoint); + } + } + } + + await api.disconnect(); + await db.destroy(); + + console.log('\n=== POLYX reconciliation taxonomy ==='); + console.log(classify(checkpoint.mismatches)); +}; + +main().catch(e => { + console.error(e); + process.exit(1); +}); diff --git a/tests/unit/reconcilePolyx.test.ts b/tests/unit/reconcilePolyx.test.ts new file mode 100644 index 00000000..a7abd857 --- /dev/null +++ b/tests/unit/reconcilePolyx.test.ts @@ -0,0 +1,130 @@ +/** + * In-flight POLYX reconciliation (D11). Every Nth block for touched accounts, and always after a + * `BalanceSet` / `DustLost`, the derived `AccountBalance` is checked against `system.account` at + * the block being indexed. On a mismatch it records a `BalanceReconciliationDrift` anomaly and + * corrects the derived value so the drift cannot compound. + */ + +import { SubstrateBlock } from '@subql/types'; +import { reconcileAccount } from '../../src/mappings/entities/identities/reconcilePolyx'; + +const ADDR = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; + +const storeGet = (): jest.Mock => (globalThis as any).store.get as jest.Mock; +const storeSet = (): jest.Mock => (globalThis as any).store.set as jest.Mock; + +const codec = (v: string) => ({ toString: () => v }); + +const block = (height: number): SubstrateBlock => + ({ + block: { header: { number: { toString: () => String(height) } } }, + timestamp: new Date('2024-01-01T00:00:00Z'), + specVersion: 8_000_000, + } as unknown as SubstrateBlock); + +let db: Record>; + +const setDerived = (row: Partial>) => { + db['AccountBalance'] = { + [ADDR]: { + id: ADDR, + accountId: ADDR, + free: BigInt(0), + reserved: BigInt(0), + frozen: BigInt(0), + total: BigInt(0), + transferable: BigInt(0), + bonded: BigInt(0), + otherReserved: BigInt(0), + totalReceived: BigInt(0), + totalSent: BigInt(0), + totalFeesPaid: BigInt(0), + totalRewards: BigInt(0), + totalSlashed: BigInt(0), + movementCount: 0, + locks: [], + holds: [], + updatedBlockId: '0', + ...row, + }, + }; +}; + +const setChain = (free: string, reserved: string, frozen: string) => { + (globalThis as any).api.query = { + system: { + account: jest + .fn() + .mockResolvedValue({ + data: { free: codec(free), reserved: codec(reserved), frozen: codec(frozen) }, + }), + }, + }; +}; + +const anomalies = () => + storeSet() + .mock.calls.filter(([e]) => e === 'IndexerAnomaly') + .map(([, , row]) => row); + +beforeEach(() => { + db = {}; + storeGet().mockImplementation((entity: string, id: string) => Promise.resolve(db[entity]?.[id])); + storeSet().mockImplementation((entity: string, id: string, data: any) => { + (db[entity] ??= {})[id] = { ...data }; + return Promise.resolve(); + }); +}); + +describe('reconcileAccount', () => { + it('does nothing when the derived balance agrees with chain state', async () => { + setDerived({ free: BigInt(1000), total: BigInt(1000), transferable: BigInt(1000) }); + setChain('1000', '0', '0'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); + + expect(anomalies()).toHaveLength(0); + }); + + it('records a drift anomaly and corrects each pool independently', async () => { + setDerived({ free: BigInt(900), reserved: BigInt(100), total: BigInt(1000) }); + setChain('1000', '50', '0'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true, eventIdx: 3 }); + + expect(anomalies()).toHaveLength(1); + expect(anomalies()[0]).toMatchObject({ kind: 'BalanceReconciliationDrift' }); + expect(anomalies()[0].detail).toContain('free 900 vs 1000'); + expect(anomalies()[0].detail).toContain('reserved 100 vs 50'); + + expect(db['AccountBalance'][ADDR]).toMatchObject({ + free: BigInt(1000), + reserved: BigInt(50), + total: BigInt(1050), + }); + }); + + it('corrects frozen via a reconciled lock so it stays a MAX going forward', async () => { + setDerived({ free: BigInt(1000), frozen: BigInt(0), transferable: BigInt(1000) }); + setChain('1000', '0', '400'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); + + expect(db['AccountBalance'][ADDR]).toMatchObject({ + frozen: BigInt(400), + transferable: BigInt(600), + locks: [{ lockId: 'reconciled', amount: BigInt(400), reasons: undefined }], + }); + }); + + it('only samples every Nth block unless forced', async () => { + setDerived({ free: BigInt(1) }); + setChain('999', '0', '0'); + + await reconcileAccount(ADDR, '0000009001', block(9001)); // 9001 % 500 != 0 + expect(anomalies()).toHaveLength(0); + + await reconcileAccount(ADDR, '0000009000', block(9000)); // 9000 % 500 == 0 + expect(anomalies()).toHaveLength(1); + }); +}); From 133febc9a6864407ec0f0f5c38eb639742cf5740 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:52:36 +0530 Subject: [PATCH 06/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20accept=20the=20pr?= =?UTF-8?q?e-memo=20arity=20of=20settlement.InstructionCreated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `memo` is the trailing parameter of `InstructionCreated` and was added after the event first shipped. Register the shape with `optionalFrom: 7` so a 7-parameter event decodes with `memo` undefined rather than failing an arity check. `handleInstructionCreated` already tolerates an absent memo. --- src/decode/shapes/settlement.ts | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/decode/shapes/settlement.ts b/src/decode/shapes/settlement.ts index 3c8fae00..a327c301 100644 --- a/src/decode/shapes/settlement.ts +++ b/src/decode/shapes/settlement.ts @@ -17,20 +17,23 @@ registerShape( stable(['did', 'venueId', 'signers', 'updateType']) ); -registerShape( - 'settlement', - 'InstructionCreated', - stable([ - 'did', - 'venueId', - 'instructionId', - 'settlementType', - 'tradeDate', - 'valueDate', - 'legs', - 'memo', - ]) -); +registerShape('settlement', 'InstructionCreated', [ + { + from: 0, + fields: [ + 'did', + 'venueId', + 'instructionId', + 'settlementType', + 'tradeDate', + 'valueDate', + 'legs', + 'memo', + ], + // `memo` was added after the v3.x era; early testnet/mainnet blocks emit 7 params. + optionalFrom: 7, + }, +]); const portfolioAffirmation = ['did', 'portfolio', 'instructionId']; From 7979ce70cab3d965298e66a2ef51b6b5420c7ac7 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:52:42 +0530 Subject: [PATCH 07/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20skip=20identity.C?= =?UTF-8?q?laimRevoked=20events=20with=20no=20issuer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stripped `ClaimRevoked` carrying a zero issuer DID and a `NoData` claim matches no indexed claim and is not an attributable revocation. Return early instead of recording a MissingReferencedEntity anomaly for it. --- src/mappings/entities/identities/mapClaim.ts | 8 ++++++++ tests/unit/mapClaim.test.ts | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/mappings/entities/identities/mapClaim.ts b/src/mappings/entities/identities/mapClaim.ts index 7f01a1f9..8f409523 100644 --- a/src/mappings/entities/identities/mapClaim.ts +++ b/src/mappings/entities/identities/mapClaim.ts @@ -11,6 +11,7 @@ import { } from '../../../types'; import { END_OF_TIME, + emptyDid, extractClaimInfo, getAssetIdWithTicker, getTextValue, @@ -173,6 +174,13 @@ export const handleClaimRevoked = async (event: SubstrateEvent): Promise = const target = getTextValue(decodeEvent(event).did); + // Some early-chain revocations emit a stripped `ClaimRevoked` with a zero issuer and a `NoData` + // claim — there is no indexed claim these could match, and it is not a real attributable + // revocation, so it is skipped rather than recorded as a missing-entity anomaly. + if (!claimIssuer || claimIssuer === emptyDid) { + return; + } + const id = getId(target, claimIssuer, claimType, scope, jurisdiction, cddId, customClaimTypeId); const claim = await Claim.get(id); diff --git a/tests/unit/mapClaim.test.ts b/tests/unit/mapClaim.test.ts index 0e04fb34..219fbeda 100644 --- a/tests/unit/mapClaim.test.ts +++ b/tests/unit/mapClaim.test.ts @@ -260,4 +260,21 @@ describe('handleClaimAdded / handleClaimRevoked', () => { expect(anomalies[0].detail).toContain(ISSUER_A); expect(Object.keys(claims)).toHaveLength(0); }); + + it('silently skips a stripped ClaimRevoked with a zero issuer (no anomaly)', async () => { + await handleClaimRevoked( + mockClaimEvent('ClaimRevoked', { + issuer: '0x0000000000000000000000000000000000000000000000000000000000000000', + cddId: 'cdd-1', + dateValue: '0', + }) + ); + + const anomalies = storeSet() + .mock.calls.filter(([entity]) => entity === 'IndexerAnomaly') + .map(([, , row]) => row); + + expect(anomalies).toHaveLength(0); + expect(Object.keys(claims)).toHaveLength(0); + }); }); From b1a71d1634b554a0a9c9533e846cf7774456a872 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:52:52 +0530 Subject: [PATCH 08/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20refine=20POLYX=20?= =?UTF-8?q?ledger=20balance=20attribution=20and=20derivation=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Balance attribution: - TreasuryReimbursement(payerDid, amount) is the treasury's share of a fee, not a refund to the payer. Credit the treasury pallet account; the payer's fee is already debited by protocolFee.FeeCharged / transactionPayment.TransactionFeePaid. - A pre-v8 RewardDestination::Staked payee auto-restakes: the reward is credited to free and immediately locked, with no staking.Bonded event. Raise the staking lock in handleReward, and have the reconciler pin the staking lock to the on-chain frozen amount so later staking events adjust a real base. - staking.payee(stash).toJSON() renders a unit variant as a lower-cased single-key object ({ staked: null }), not the bare string 'Staked'. Normalise both forms so a Staked payee is classified correctly. - The in-flight reconciler ignores drift below 100 POLYX: the pre-v5.4 weight fee is applied with no event, and a sample taken mid-block on a multi-touch account compares partial derived state against the block's final on-chain state. Neither is a handler defect. Derivation cost: - Cache staking.payee per stash; it rarely changes and the reconciler corrects any stale-entry drift. - Reconcile every 2000 blocks rather than 500. - Memoise system.account per block, so an account touched N times in a sampled block costs one read rather than N. --- .../entities/identities/mapPolyxLedger.ts | 73 ++++++++++++------- .../entities/identities/reconcilePolyx.ts | 73 ++++++++++++++++--- src/utils/staking.ts | 54 +++++++++++--- tests/unit/mapPolyxLedger.test.ts | 44 +++++++++++ tests/unit/reconcilePolyx.test.ts | 61 ++++++++++------ tests/unit/rewardDestinationA15.test.ts | 17 +++++ 6 files changed, 247 insertions(+), 75 deletions(-) diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts index 9bda51d6..31ce87bf 100644 --- a/src/mappings/entities/identities/mapPolyxLedger.ts +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -18,6 +18,7 @@ import { resolveLegacyRewardDestination } from '../../../utils/staking'; import { getAccountKeyType, getOrCreateAccount } from '../../../utils/accounts'; import { getEventParams } from '../../../utils/events'; import { extractArgs, HandlerArgs } from '../common'; +import { getAccountId, systematicIssuers } from '../../consts'; import { reconcileAccount } from './reconcilePolyx'; /** @@ -1056,20 +1057,20 @@ export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise }); }; +const treasuryPalletAccount = (): string => + getAccountId(systematicIssuers.treasury.accountId, api.registry.chainSS58); + export const handleTreasuryReimbursement = async (event: SubstrateEvent): Promise => { const args = extractArgs(event); - const [rawIdentity, rawBalance] = args.params; - const { specVersion } = args.block; - - // Until 5.4.1 the event reported 80% of the amount actually taken from the payer. - const reported = getBigIntValue(rawBalance); - const amount = specVersion < 5004001 ? (reported * BigInt(125)) / BigInt(100) : reported; - - const treasury = await identityPrimaryAccount(getTextValue(rawIdentity)); + const [, rawBalance] = args.params; + // `TreasuryReimbursement(payerDid, amount)` — the amount routed to the treasury out of a fee. + // The payer's full fee is already debited by `protocolFee.FeeCharged` / + // `transactionPayment.TransactionFeePaid`, so this is the treasury's credit, not a refund to + // the payer. (The pre-5.4.1 author split is not emitted and stays outside the ledger.) await postTransition(args, { - to: treasury ? { address: treasury, pool: PolyxPool.Free } : undefined, - amount, + to: { address: treasuryPalletAccount(), pool: PolyxPool.Free }, + amount: getBigIntValue(rawBalance), kind: MovementKind.TreasuryReimbursement, }); }; @@ -1141,25 +1142,36 @@ const rewardRecipient = async ( decoded: Record, stash: string | undefined, is8x: boolean -): Promise => { +): Promise<{ recipient: string; restaked: boolean } | undefined> => { if (!stash) { return undefined; } if (is8x) { + // v8: `dest: Staked` restakes via the paired `balances.Held{Staking}`, so no lock work here. const dest = optionalField(decoded, 'dest'); const json = dest?.toJSON() as string | Record | undefined; if (json && typeof json === 'object') { - return ((json.account ?? json.Account) as string | undefined) ?? stash; + return { + recipient: ((json.account ?? json.Account) as string | undefined) ?? stash, + restaked: false, + }; } - return stash; + return { recipient: stash, restaked: false }; } - const { rewardDestinationAccount } = await resolveLegacyRewardDestination(stash); + const { rewardDestination, rewardDestinationAccount } = await resolveLegacyRewardDestination( + stash + ); - return rewardDestinationAccount ?? stash; + // Pre-v8 `Staked` auto-restakes: the reward lands in `free` and is immediately locked, and no + // `staking.Bonded` is emitted for it — so the lock has to be raised here. + return { + recipient: rewardDestinationAccount ?? stash, + restaked: rewardDestination === 'Staked', + }; }; /** @@ -1176,12 +1188,14 @@ export const handleReward = async (event: SubstrateEvent): Promise => { const stash = stakingStash(decoded); const amount = amountOf(decoded); const eraIndex = currentPayoutEra(args.blockId); - const recipient = await rewardRecipient(decoded, stash, is8xChain(args.block)); + const resolved = await rewardRecipient(decoded, stash, is8xChain(args.block)); - if (!recipient) { + if (!resolved) { return; } + const { recipient, restaked } = resolved; + // If a `balances` deposit for this reward was already recorded as a plain mint, relabel it. const mints = await findBlockEntries(args.blockId, recipient, amount, [MovementKind.Mint]); @@ -1197,19 +1211,22 @@ export const handleReward = async (event: SubstrateEvent): Promise => { balance.totalRewards += amount; await balance.save(); } - - return; + } else { + await postTransition( + args, + { + to: { address: recipient, pool: PolyxPool.Free }, + amount, + kind: MovementKind.StakingReward, + }, + { eraIndex } + ); } - await postTransition( - args, - { - to: { address: recipient, pool: PolyxPool.Free }, - amount, - kind: MovementKind.StakingReward, - }, - { eraIndex } - ); + // Pre-v8 `Staked` payee: the reward is added to the staking lock in the same step. + if (restaked) { + await adjustLock(recipient, STAKING_LOCK_ID, amount, args.blockId, 'staking'); + } }; /** `staking.Slash` / `Slashed` — `staker/Free → ∅`, a real movement at both eras. */ diff --git a/src/mappings/entities/identities/reconcilePolyx.ts b/src/mappings/entities/identities/reconcilePolyx.ts index 9e32499e..e9fc7152 100644 --- a/src/mappings/entities/identities/reconcilePolyx.ts +++ b/src/mappings/entities/identities/reconcilePolyx.ts @@ -3,7 +3,7 @@ import { SubstrateBlock } from '@subql/types'; import { AccountBalance, AnomalyKind } from '../../../types'; import { getBigIntValue } from '../../../utils'; import { recordAnomaly } from '../../../utils/anomaly'; -import { accountDataFrozen, recomputeDerived } from './mapPolyxLedger'; +import { accountDataFrozen, recomputeDerived, STAKING_LOCK_ID } from './mapPolyxLedger'; /** * In-flight reconciliation (D11). @@ -19,8 +19,26 @@ import { accountDataFrozen, recomputeDerived } from './mapPolyxLedger'; * is the going-forward safety net. */ -/** Sample rate for the routine check. `BalanceSet`/`DustLost` always reconcile regardless. */ -const RECONCILE_EVERY_N_BLOCKS = 500; +/** + * Sample rate for the routine check. `BalanceSet`/`DustLost` always reconcile regardless. + * Each sampled account costs one `system.account` RPC read; against a remote node this is the + * dominant cost of the genesis sweep, so the interval is coarse. The safety net still catches a + * mis-mapped event well before it can compound — a real defect drifts by thousands of POLYX and + * shows up at the next sample; the offline harness is what proves the history exact. + */ +const RECONCILE_EVERY_N_BLOCKS = 2000; + +/** + * Ignore drift below this (100 POLYX, 6 decimals). Two things produce sub-POLYX noise that is not + * a handler defect: the pre-v5.4 weight fee, which older Substrate charges with no event at all + * (so the ledger cannot see it and the balance runs a little high until this corrects it); and a + * sample landing mid-block on an account touched more than once, where the partial derived state + * is compared against the block's final on-chain state. A real mis-mapped or missed event drifts + * by thousands of POLYX. The offline harness sums with no threshold and catches slow accumulation. + */ +const MIN_DRIFT = BigInt(100_000_000); + +const abs = (value: bigint): bigint => (value < BigInt(0) ? -value : value); const blockNumber = (block: SubstrateBlock): number => Number(block.block.header.number.toString()); @@ -33,16 +51,45 @@ interface OnChain { frozen: bigint; } -const readOnChain = async (address: string): Promise => { +/** + * `system.account` is end-of-block state, so within one block it is the same no matter how many + * times or how late it is read. An account touched N times in a sampled block would otherwise + * cost N identical RPC reads; this memoises the read for the current block (the compare and + * correct still run every call, so the last one — with the most complete derived state — wins). + */ +let onChainCacheBlock = -1; +const onChainCache = new Map(); + +/** Test hook — the cache is keyed only by block height, so a suite reusing one height must clear it. */ +export const __resetOnChainCache = (): void => { + onChainCacheBlock = -1; + onChainCache.clear(); +}; + +const readOnChain = async (address: string, blockHeight: number): Promise => { + if (blockHeight !== onChainCacheBlock) { + onChainCacheBlock = blockHeight; + onChainCache.clear(); + } + + const hit = onChainCache.get(address); + if (hit) { + return hit; + } + const info = (await api.query.system.account(address)) as unknown as { data: Record; }; - return { + const onChain: OnChain = { free: getBigIntValue(info.data.free), reserved: getBigIntValue(info.data.reserved), frozen: accountDataFrozen(info.data), }; + + onChainCache.set(address, onChain); + + return onChain; }; export const reconcileAccount = async ( @@ -64,7 +111,7 @@ export const reconcileAccount = async ( let onChain: OnChain; try { - onChain = await readOnChain(address); + onChain = await readOnChain(address, blockNumber(block)); } catch { // A pruned node or a transient RPC error is not a ledger defect. return; @@ -72,13 +119,13 @@ export const reconcileAccount = async ( const drifts: string[] = []; - if (balance.free !== onChain.free) { + if (abs(balance.free - onChain.free) >= MIN_DRIFT) { drifts.push(`free ${balance.free} vs ${onChain.free}`); } - if (balance.reserved !== onChain.reserved) { + if (abs(balance.reserved - onChain.reserved) >= MIN_DRIFT) { drifts.push(`reserved ${balance.reserved} vs ${onChain.reserved}`); } - if (balance.frozen !== onChain.frozen) { + if (abs(balance.frozen - onChain.frozen) >= MIN_DRIFT) { drifts.push(`frozen ${balance.frozen} vs ${onChain.frozen}`); } @@ -93,13 +140,15 @@ export const reconcileAccount = async ( eventIdx, }); - // Correct the derived value so the drift cannot compound. + // Correct the derived value so the drift cannot compound. `frozen` is corrected by pinning the + // staking lock (which is nearly all of any pre-v8 account's frozen amount) to the on-chain + // value, so later `staking.*` events keep adjusting a realistic base rather than starting over. balance.free = onChain.free; balance.reserved = onChain.reserved; balance.locks = onChain.frozen > BigInt(0) - ? [{ lockId: 'reconciled', amount: onChain.frozen, reasons: undefined }] - : (balance.locks ?? []).filter(lock => lock.lockId !== 'reconciled'); + ? [{ lockId: STAKING_LOCK_ID, amount: onChain.frozen, reasons: 'staking' }] + : []; recomputeDerived(balance); balance.updatedBlockId = blockId; diff --git a/src/utils/staking.ts b/src/utils/staking.ts index cbde173b..ba9f01c6 100644 --- a/src/utils/staking.ts +++ b/src/utils/staking.ts @@ -5,6 +5,19 @@ export interface LegacyRewardDestination { rewardDestinationAccount?: string; } +/** + * Per-stash cache of a resolved payee. `staking.payee(stash)` is a chain-storage read on every + * reward, and a validator's set of ~20 stashes is re-queried every era for the whole genesis + * replay — the dominant RPC cost of the sweep against a remote node. A payee changes very rarely + * (an explicit `staking.setPayee`), and the in-flight reconciler corrects any `frozen` drift a + * stale entry could cause within ~1 era, so a plain process-lifetime cache is the right trade. + * Only successful resolutions are cached — a transient read failure must be retried, not pinned. + */ +const payeeCache = new Map(); + +/** Test hook — the cache is process-lifetime, so a suite that re-mocks `staking.payee` must clear it. */ +export const __resetPayeeCache = (): void => payeeCache.clear(); + /** * Resolves where a pre-v8 staking reward for `stash` was actually paid. * @@ -19,30 +32,49 @@ export interface LegacyRewardDestination { export const resolveLegacyRewardDestination = async ( stash: string ): Promise => { + const cached = payeeCache.get(stash); + if (cached) { + return cached; + } + try { const payee = await api.query.staking.payee(stash); const json = payee.toJSON() as string | Record | null; - if (json === 'Staked' || json === 'Stash') { - return { rewardDestination: json, rewardDestinationAccount: stash }; - } + // `RewardDestination` renders either as a bare string (`"Staked"`) or, via `.toJSON()`, as a + // single-key object with the variant name lower-cased (`{ staked: null }`, `{ account: "0x…" }`). + const variant = ( + typeof json === 'string' ? json : Object.keys(json ?? {})[0] ?? '' + ).toLowerCase(); + const value = + json && typeof json === 'object' ? (json as Record)[variant] : undefined; + + let result: LegacyRewardDestination; - if (json === 'Controller') { + if (variant === 'staked' || variant === 'stash') { + result = { + rewardDestination: variant === 'staked' ? 'Staked' : 'Stash', + rewardDestinationAccount: stash, + }; + } else if (variant === 'controller') { const controller = (await api.query.staking.bonded(stash)).toJSON() as string | null; - return { + result = { rewardDestination: 'Controller', rewardDestinationAccount: controller ?? undefined, }; + } else if (variant === 'account') { + result = { + rewardDestination: 'Account', + rewardDestinationAccount: typeof value === 'string' ? value : undefined, + }; + } else { + result = { rewardDestination: 'None' }; } - if (json && typeof json === 'object') { - const account = (json.account ?? json.Account) as string | undefined; - - return { rewardDestination: 'Account', rewardDestinationAccount: account }; - } + payeeCache.set(stash, result); - return { rewardDestination: typeof json === 'string' ? json : 'None' }; + return result; } catch { // A pruned node, or a runtime with no `staking.payee` storage — fall back to the placeholder. return { rewardDestination: 'LegacyUnknown' }; diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts index 08b18b4a..6f46ae91 100644 --- a/tests/unit/mapPolyxLedger.test.ts +++ b/tests/unit/mapPolyxLedger.test.ts @@ -29,7 +29,10 @@ import { handleWithdrawn, handleDustLost, handleReserveRepatriated, + handleTreasuryReimbursement, } from '../../src/mappings/entities/identities/mapPolyxLedger'; +import { getAccountId, systematicIssuers } from '../../src/mappings/consts'; +import { __resetPayeeCache } from '../../src/utils/staking'; const ALICE = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; const BOB = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; @@ -130,8 +133,10 @@ const clone = (value: Row): Row => { }; beforeEach(() => { + __resetPayeeCache(); db = {}; blockHeight = 1_000_000; + (globalThis as any).api.registry = { chainSS58: 42 }; storeGet().mockImplementation((entity: string, id: string) => { if (entity === 'Account') { @@ -298,6 +303,25 @@ describe('Event → pool transition', () => { expect(entries()).toHaveLength(1); expect(balance(ALICE)?.free).toBe(BigInt(-9)); }); + + it('TreasuryReimbursement credits the treasury pallet account, not the fee payer', async () => { + const payerDid = '0x8015a1702789fedf8474a042af07ba6a37f94e8d24b4eed89414e6eb79df084e'; + const treasury = getAccountId(systematicIssuers.treasury.accountId, 42); + + await handleTreasuryReimbursement( + tupleEvent('treasury', 'TreasuryReimbursement', [payerDid, '400'], 4_000_000) + ); + + expect(entries()).toHaveLength(1); + expect(entries()[0]).toMatchObject({ + accountId: treasury, + direction: EntryDirection.Credit, + kind: MovementKind.TreasuryReimbursement, + amount: BigInt(400), + }); + expect(entries()[0].accountId).not.toBe(payerDid); + expect(balance(treasury)?.free).toBe(BigInt(400)); + }); }); describe('properties the one-column model could not satisfy', () => { @@ -468,6 +492,26 @@ describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { (globalThis as any).api.query = {}; }); + it('pre-v8 Reward with a Staked payee credits free AND raises the staking lock', async () => { + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => 'Staked' }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => null }), + }, + }; + + await handleReward(tupleEvent('staking', 'Reward', ['0xdid', ALICE, '500'], 7_004_001)); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(500), + frozen: BigInt(500), // restaked — locked in the same step + bonded: BigInt(500), + transferable: BigInt(0), + }); + + (globalThis as any).api.query = {}; + }); + it('staking Reward is ∅ → stash/Free with the era from the preceding PayoutStarted', async () => { await handlePayoutStarted( structEvent( diff --git a/tests/unit/reconcilePolyx.test.ts b/tests/unit/reconcilePolyx.test.ts index a7abd857..58db7756 100644 --- a/tests/unit/reconcilePolyx.test.ts +++ b/tests/unit/reconcilePolyx.test.ts @@ -6,7 +6,10 @@ */ import { SubstrateBlock } from '@subql/types'; -import { reconcileAccount } from '../../src/mappings/entities/identities/reconcilePolyx'; +import { + __resetOnChainCache, + reconcileAccount, +} from '../../src/mappings/entities/identities/reconcilePolyx'; const ADDR = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; @@ -53,11 +56,9 @@ const setDerived = (row: Partial>) => { const setChain = (free: string, reserved: string, frozen: string) => { (globalThis as any).api.query = { system: { - account: jest - .fn() - .mockResolvedValue({ - data: { free: codec(free), reserved: codec(reserved), frozen: codec(frozen) }, - }), + account: jest.fn().mockResolvedValue({ + data: { free: codec(free), reserved: codec(reserved), frozen: codec(frozen) }, + }), }, }; }; @@ -68,6 +69,7 @@ const anomalies = () => .map(([, , row]) => row); beforeEach(() => { + __resetOnChainCache(); db = {}; storeGet().mockImplementation((entity: string, id: string) => Promise.resolve(db[entity]?.[id])); storeSet().mockImplementation((entity: string, id: string, data: any) => { @@ -76,10 +78,22 @@ beforeEach(() => { }); }); +// Values are in base units (6 decimals); drifts here are far above the MIN_DRIFT (100 POLYX) floor. +const P = (polyx: number): bigint => BigInt(polyx) * BigInt(1_000_000); + describe('reconcileAccount', () => { it('does nothing when the derived balance agrees with chain state', async () => { - setDerived({ free: BigInt(1000), total: BigInt(1000), transferable: BigInt(1000) }); - setChain('1000', '0', '0'); + setDerived({ free: P(1000), total: P(1000), transferable: P(1000) }); + setChain(P(1000).toString(), '0', '0'); + + await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); + + expect(anomalies()).toHaveLength(0); + }); + + it('ignores sub-100-POLYX drift (weight-fee gap / mid-block sample noise)', async () => { + setDerived({ free: P(1000) + BigInt(50_000_000), total: P(1000) }); + setChain(P(1000).toString(), '0', '0'); await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); @@ -87,44 +101,43 @@ describe('reconcileAccount', () => { }); it('records a drift anomaly and corrects each pool independently', async () => { - setDerived({ free: BigInt(900), reserved: BigInt(100), total: BigInt(1000) }); - setChain('1000', '50', '0'); + setDerived({ free: P(900), reserved: P(100), total: P(1000) }); + setChain(P(1000).toString(), P(50).toString(), '0'); await reconcileAccount(ADDR, '0000009000', block(9000), { force: true, eventIdx: 3 }); expect(anomalies()).toHaveLength(1); expect(anomalies()[0]).toMatchObject({ kind: 'BalanceReconciliationDrift' }); - expect(anomalies()[0].detail).toContain('free 900 vs 1000'); - expect(anomalies()[0].detail).toContain('reserved 100 vs 50'); + expect(anomalies()[0].detail).toContain(`free ${P(900)} vs ${P(1000)}`); expect(db['AccountBalance'][ADDR]).toMatchObject({ - free: BigInt(1000), - reserved: BigInt(50), - total: BigInt(1050), + free: P(1000), + reserved: P(50), + total: P(1050), }); }); - it('corrects frozen via a reconciled lock so it stays a MAX going forward', async () => { - setDerived({ free: BigInt(1000), frozen: BigInt(0), transferable: BigInt(1000) }); - setChain('1000', '0', '400'); + it('corrects frozen by pinning the staking lock, so later staking events adjust a real base', async () => { + setDerived({ free: P(1000), frozen: BigInt(0), transferable: P(1000) }); + setChain(P(1000).toString(), '0', P(400).toString()); await reconcileAccount(ADDR, '0000009000', block(9000), { force: true }); expect(db['AccountBalance'][ADDR]).toMatchObject({ - frozen: BigInt(400), - transferable: BigInt(600), - locks: [{ lockId: 'reconciled', amount: BigInt(400), reasons: undefined }], + frozen: P(400), + transferable: P(600), + locks: [{ lockId: 'staking ', amount: P(400), reasons: 'staking' }], }); }); it('only samples every Nth block unless forced', async () => { setDerived({ free: BigInt(1) }); - setChain('999', '0', '0'); + setChain(P(999).toString(), '0', '0'); - await reconcileAccount(ADDR, '0000009001', block(9001)); // 9001 % 500 != 0 + await reconcileAccount(ADDR, '0000009001', block(9001)); // 9001 % 2000 != 0 expect(anomalies()).toHaveLength(0); - await reconcileAccount(ADDR, '0000009000', block(9000)); // 9000 % 500 == 0 + await reconcileAccount(ADDR, '0000008000', block(8000)); // 8000 % 2000 == 0 expect(anomalies()).toHaveLength(1); }); }); diff --git a/tests/unit/rewardDestinationA15.test.ts b/tests/unit/rewardDestinationA15.test.ts index 06bfeb77..8e47a144 100644 --- a/tests/unit/rewardDestinationA15.test.ts +++ b/tests/unit/rewardDestinationA15.test.ts @@ -13,6 +13,7 @@ import { SubstrateEvent } from '@subql/types'; import { handleStakingEvent } from '../../src/mappings/entities/events/mapStakingEvent'; +import { __resetPayeeCache } from '../../src/utils/staking'; const STASH = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; const PAYEE = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; @@ -59,6 +60,7 @@ const savedStakingEvent = () => .at(-1); beforeEach(() => { + __resetPayeeCache(); (globalThis as any).api.runtimeVersion.specName = { toString: () => 'polymesh' }; (globalThis as any).api.query = {}; }); @@ -102,6 +104,21 @@ describe('A15 — pre-v8 reward destination', () => { }); }); + it('resolves the object form staking.payee returns via .toJSON() ({ staked: null }) to the stash', async () => { + // `RewardDestination::Staked` renders as the lower-cased single-key object `{ staked: null }` + // through `.toJSON()`, not the bare string `"Staked"`. + mockPayee({ staked: null }); + + await handleStakingEvent( + rewardEvent('Reward', [codec('0x00'), codec(STASH), codec('1500')], 7_004_001) + ); + + expect(savedStakingEvent()).toMatchObject({ + rewardDestination: 'Staked', + rewardDestinationAccount: STASH, + }); + }); + it('falls back to LegacyUnknown when the payee read is not possible, never to the stash', async () => { // api.query.staking absent — a pruned node or a runtime with no such storage await handleStakingEvent( From 14eaa508dbe155e2b04d4a279581d20691975d16 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:59:43 +0530 Subject: [PATCH 09/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20key=20RuntimeSnap?= =?UTF-8?q?shot=20sections=20the=20way=20every=20consumer=20spells=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapshotFromMetadata keyed events and calls by the metadata pallet spelling (Asset, ExternalAgents) while eventDrift, arityFixtureFor, the arity fixtures, CAPTURED_MODULES and project.ts all use the event.section spelling (asset, externalAgents). Exact-key lookups therefore missed and fell back to {} - eventDrift reported every fixture event as removed (81 against mainnet 8000020) and arityFixtureFor wrote "modules": {} under --write. Normalise the pallet name once, on the way in, via sectionId(); the metadata spelling never leaves snapshotFromMetadata. snapshot.modules is unaffected - section.toLowerCase() is the same value either way. The unit tests keyed both fixture and snapshot "Balances", so the suite stayed green over the broken script; they now use the real convention, plus a snapshotFromMetadata block against real v15 metadata from @polkadot/types-support. Co-authored-by: Francis --- scripts/sync-metadata.ts | 15 +++++-- tests/unit/syncMetadata.test.ts | 77 +++++++++++++++++++++++++++++---- 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/scripts/sync-metadata.ts b/scripts/sync-metadata.ts index 3be8c431..cec6e0fd 100644 --- a/scripts/sync-metadata.ts +++ b/scripts/sync-metadata.ts @@ -47,9 +47,9 @@ export interface RuntimeSnapshot { specVersion: number; /** Lowercased pallet names, as `event.section.toLowerCase()` reports them */ modules: string[]; - /** Pallet name (as the chain spells it) to event name to parameter count */ + /** Section id (`ExternalAgents` -> `externalAgents`) to event name to parameter count */ events: Record>; - /** Pallet name to snake_cased call names */ + /** Section id to snake_cased call names */ calls: Record; } @@ -63,6 +63,15 @@ export interface ArityFixture { const snakeCase = (value: string): string => value[0].toLowerCase() + value.slice(1).replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`); +/** + * A pallet name as `@polkadot/api` reports `event.section`: `ExternalAgents` -> `externalAgents`. + * + * Metadata spells pallet names in PascalCase. Every other producer and consumer of a section here + * - the arity fixtures, `CAPTURED_MODULES`, `project.ts` - uses the api spelling, so the metadata + * spelling is normalised once, on the way in, and never leaks past `snapshotFromMetadata`. + */ +export const sectionId = (name: string): string => name[0].toLowerCase() + name.slice(1); + // --------------------------------------------------------------------------------------------- // Reading metadata // --------------------------------------------------------------------------------------------- @@ -88,7 +97,7 @@ export const snapshotFromMetadata = ( }; for (const pallet of metadata.asLatest.pallets) { - const section = pallet.name.toString(); + const section = sectionId(pallet.name.toString()); snapshot.modules.push(section.toLowerCase()); diff --git a/tests/unit/syncMetadata.test.ts b/tests/unit/syncMetadata.test.ts index 79a671a8..e529383d 100644 --- a/tests/unit/syncMetadata.test.ts +++ b/tests/unit/syncMetadata.test.ts @@ -1,11 +1,16 @@ +import { Metadata, TypeRegistry } from '@polkadot/types'; +import metadataHex from '@polkadot/types-support/metadata/static-substrate'; import { applyEnumUpdates, + arityFixtureFor, ArityFixture, enumMembers, eventDrift, findEnumBlock, planEnumUpdates, RuntimeSnapshot, + sectionId, + snapshotFromMetadata, unhandledEvents, withAddedMembers, } from '../../scripts/sync-metadata'; @@ -36,11 +41,67 @@ const snapshot = (overrides: Partial = {}): RuntimeSnapshot => specName: 'polymesh', specVersion: 8_000_000, modules: ['system', 'balances'], - events: { Balances: { BalanceSet: 2, TransferWithMemo: 4 } }, - calls: { Balances: ['set_balance'] }, + events: { balances: { BalanceSet: 2, TransferWithMemo: 4 } }, + calls: { balances: ['set_balance'] }, ...overrides, }); +describe('snapshotFromMetadata', () => { + const registry = new TypeRegistry(); + const metadata = new Metadata(registry, metadataHex); + + registry.setMetadata(metadata); + + const real = snapshotFromMetadata(registry, metadata, 'substrate', 1); + + it('keys a section the way the arity fixtures spell it, not the way metadata spells it', () => { + expect(real.events.balances).toBeDefined(); + expect(real.events.Balances).toBeUndefined(); + }); + + it('lowercases the first letter only, so a multi-word pallet keeps its camelCase', () => { + expect(Object.keys(real.events)).toContain('transactionPayment'); + expect(Object.keys(real.calls)).toContain('electionProviderMultiPhase'); + }); + + it('still spells modules the way ModuleIdEnum does, fully lowercased', () => { + expect(real.modules).toContain('transactionpayment'); + }); + + /** + * The regression this file exists for: a fixture captured from a runtime has to read back + * against that same runtime as no drift at all. Keyed by the metadata spelling instead, every + * event in the fixture reads as removed - 81 of them, against mainnet. + */ + it('produces a fixture that reads back against its own runtime as no drift', () => { + const fixture: ArityFixture = { + specVersion: 1, + source: 'substrate static metadata', + modules: { balances: real.events.balances }, + }; + const drift = eventDrift(fixture, real); + + expect([drift.added, drift.removed, drift.reshaped]).toEqual([[], [], []]); + }); +}); + +describe('sectionId', () => { + it('maps a metadata pallet name onto the api section name', () => { + expect(sectionId('ExternalAgents')).toBe('externalAgents'); + expect(sectionId('Asset')).toBe('asset'); + }); +}); + +describe('arityFixtureFor', () => { + it('captures a pallet it was asked for, rather than writing an empty fixture', () => { + const captured = arityFixtureFor( + snapshot({ events: { asset: { AssetCreated: 8 }, notCaptured: { Whatever: 1 } } }) + ); + + expect(captured.modules).toEqual({ asset: { AssetCreated: 8 } }); + }); +}); + describe('enum parsing', () => { it('reads member names past comments, docstrings and directives', () => { expect(enumMembers(findEnumBlock(SCHEMA, 'ModuleIdEnum').body)).toEqual([ @@ -72,7 +133,7 @@ describe('planEnumUpdates', () => { it('sorts additions so two runs over the same runtime produce the same file', () => { const updates = planEnumUpdates( SCHEMA, - snapshot({ events: { Balances: { Zebra: 1, Apple: 1, BalanceSet: 2 } } }) + snapshot({ events: { balances: { Zebra: 1, Apple: 1, BalanceSet: 2 } } }) ); expect(updates.EventIdEnum.added).toEqual(['Apple', 'Zebra']); @@ -118,23 +179,23 @@ describe('eventDrift', () => { const fixture: ArityFixture = { specVersion: 7_004_001, source: 'test', - modules: { Balances: { BalanceSet: 4, Gone: 1 } }, + modules: { balances: { BalanceSet: 4, Gone: 1 } }, }; it('names an event whose parameter count changed, which positional decoding cannot see', () => { - expect(eventDrift(fixture, snapshot()).reshaped).toEqual(['Balances.BalanceSet: 4 -> 2']); + expect(eventDrift(fixture, snapshot()).reshaped).toEqual(['balances.BalanceSet: 4 -> 2']); }); it('names an event the runtime added since the fixture was captured', () => { - expect(eventDrift(fixture, snapshot()).added).toEqual(['Balances.TransferWithMemo']); + expect(eventDrift(fixture, snapshot()).added).toEqual(['balances.TransferWithMemo']); }); it('names an event the runtime no longer has', () => { - expect(eventDrift(fixture, snapshot()).removed).toEqual(['Balances.Gone']); + expect(eventDrift(fixture, snapshot()).removed).toEqual(['balances.Gone']); }); it('reports nothing when the runtime matches the fixture', () => { - const same = snapshot({ events: { Balances: { BalanceSet: 4, Gone: 1 } } }); + const same = snapshot({ events: { balances: { BalanceSet: 4, Gone: 1 } } }); const drift = eventDrift(fixture, same); expect([drift.added, drift.removed, drift.reshaped]).toEqual([[], [], []]); From efffcfa86e38c5840a5a09887977d81818a013a0 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:59:51 +0530 Subject: [PATCH 10/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20record=20stateTri?= =?UTF-8?q?eMigration=20events=20in=20EventIdEnum?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stateTrieMigration.Migrated / AutoMigrationFinished / Halted were not in EventIdEnum, so the generic event recorder logged an UnknownEnumValue anomaly and stored them as "Unknown". They carry no balance effect - just recorded like any other event now. --- schema.graphql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schema.graphql b/schema.graphql index 71feee98..dc15e230 100644 --- a/schema.graphql +++ b/schema.graphql @@ -778,6 +778,11 @@ enum EventIdEnum { UpgradeFailed UpgradeStarted + ## stateTrieMigration ## + Migrated + AutoMigrationFinished + Halted + "`Approval` is now active for assets pallet" Approval From fd0092ee60da67a128969ece07add71a11aac23b Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:59:57 +0530 Subject: [PATCH 11/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20fall=20back=20to?= =?UTF-8?q?=20api.runtimeVersion=20when=20a=20block=20reports=20a=20stale?= =?UTF-8?q?=20spec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @subql/node can hand a mapping the previous runtime's spec version for the one block a runtime upgrade takes effect on. At the v5->v6 boundary that made asset.AssetBalanceUpdated resolve no tuple shape and the throwing decode proxy killed the worker. resolveShapeTolerant retries once with api.runtimeVersion.specVersion (read from the block's own runtime) when the reported version resolves nothing - but only when it is newer and within one release line, so neither a correct reported version nor a stale/HEAD api.runtimeVersion pulls in a wildly wrong shape. --- src/decode/decode.ts | 48 ++++++++++++++++++++++++++++++-- tests/unit/decode.test.ts | 58 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/src/decode/decode.ts b/src/decode/decode.ts index 5b2c5ea7..fa27f385 100644 --- a/src/decode/decode.ts +++ b/src/decode/decode.ts @@ -2,7 +2,7 @@ import { Codec } from '@polkadot/types/types'; import { SubstrateEvent } from '@subql/types'; import { EventIdEnum, ModuleIdEnum } from '../types'; import { recordAnomaly } from '../utils/anomaly'; -import { DecodeError, FieldNotFound } from './errors'; +import { DecodeError, FieldNotFound, NoDecoderForSpecVersion } from './errors'; import { DecodedEvent, namedFields } from './field'; import { resolveShape } from './shapes'; import { normaliseSpecVersion } from './specVersion'; @@ -78,6 +78,50 @@ const guard = (event: SubstrateEvent, decoded: Record): DecodedEv }, }); +/** One Polymesh release line on the public spec-version scale (`v6.x` is `6_000_000..6_999_999`). */ +const ONE_RELEASE_LINE = 1_000_000; + +/** + * The shape for a tuple-style event, tolerating a stale block spec version. + * + * `@subql/node` occasionally reports the *previous* runtime's spec version for the one block a + * runtime upgrade takes effect on (seen at the v5→v6 boundary: `AssetBalanceUpdated` decoded as + * spec `5004003` and crashed the worker, though the block ran `6000001`). `api.runtimeVersion` is + * read from the block's own runtime, so when the reported version resolves no decoder, retry with + * it once — but only when it is newer and within one release line, so neither a correct reported + * version nor a stale/HEAD `api.runtimeVersion` can pull in a wildly wrong shape. + */ +const resolveShapeTolerant = ( + section: string, + method: string, + reportedSpecVersion: number, + arity: number +): ReturnType => { + try { + return resolveShape(section, method, reportedSpecVersion, arity); + } catch (error) { + if (!(error instanceof NoDecoderForSpecVersion)) { + throw error; + } + + let fromRuntime: number; + try { + fromRuntime = normaliseSpecVersion(api.runtimeVersion.specVersion.toNumber()); + } catch { + throw error; + } + + if ( + fromRuntime <= reportedSpecVersion || + fromRuntime - reportedSpecVersion > ONE_RELEASE_LINE + ) { + throw error; + } + + return resolveShape(section, method, fromRuntime, arity); + } +}; + /** * An event's parameters keyed by name. * @@ -103,7 +147,7 @@ export const decodeEvent = (event: SubstrateEvent): DecodedEvent => { const { section, method, data } = event.event; try { - const shape = resolveShape( + const shape = resolveShapeTolerant( section, method, normaliseSpecVersion(event.block.specVersion), diff --git a/tests/unit/decode.test.ts b/tests/unit/decode.test.ts index 8e58ae81..bf477ead 100644 --- a/tests/unit/decode.test.ts +++ b/tests/unit/decode.test.ts @@ -219,6 +219,64 @@ describe('decodeEvent, tuple events', () => { NoDecoderForSpecVersion ); }); + + describe('stale block spec version at a runtime-upgrade boundary', () => { + const setRuntimeSpec = (value: number): void => { + (api.runtimeVersion.specVersion as any).toNumber = () => value; + }; + + afterEach(() => setRuntimeSpec(8_000_000)); + + it('falls back to api.runtimeVersion when the reported version resolves no decoder', () => { + // `AssetBalanceUpdated` is registered from v6.0.0; the block reports a stale v5 spec but + // the block actually ran v6.0.1. + setRuntimeSpec(6_000_001); + + const decoded = decodeEvent( + tupleEvent( + 'asset', + 'AssetBalanceUpdated', + ['0xdid', '0xasset', '100', '0xfrom', '0xto', '{"transferred":null}'], + 5_999_999 + ) + ); + + expect(decoded.assetId.toString()).toBe('0xasset'); + expect(decoded.updateReason.toString()).toBe('{"transferred":null}'); + }); + + it('still throws when api.runtimeVersion is not newer than the reported version', () => { + setRuntimeSpec(5_999_999); + + expect(() => + decodeEvent( + tupleEvent( + 'asset', + 'AssetBalanceUpdated', + ['0xdid', '0xasset', '100', '0xfrom', '0xto', '{"transferred":null}'], + 5_999_999 + ) + ) + ).toThrow(NoDecoderForSpecVersion); + }); + + it('does not reach for a shape more than one release line ahead of the reported version', () => { + // A stale boundary spec is at most one release line off; a HEAD `api.runtimeVersion` + // against an old block must not decode it with a much later shape. + setRuntimeSpec(8_000_000); + + expect(() => + decodeEvent( + tupleEvent( + 'asset', + 'AssetBalanceUpdated', + ['0xdid', '0xasset', '100', '0xfrom', '0xto', '{"transferred":null}'], + 5_999_999 + ) + ) + ).toThrow(NoDecoderForSpecVersion); + }); + }); }); /** From 4622acb9cbfdd1e2d8fbbbb440a00a9248fbbbc0 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:39:50 +0530 Subject: [PATCH 12/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20address=20SonarCl?= =?UTF-8?q?oud=20code=20smell9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cognitive complexity: postTransition and handleBalanceSet split into named helpers; the reconcile-polyx and measure-a15-payees scripts' main loops each extract their inner body. Plus a bigint maxBig helper (Math.max cannot take bigint), toHaveLength over .length assertions, one extracted ternary, an optional chain, and .at(-1). --- scripts/measure-a15-payees.ts | 131 ++++---- scripts/reconcile-polyx.ts | 88 +++--- .../entities/identities/mapPolyxLedger.ts | 282 +++++++++++------- tests/unit/mapPolyxLedger.test.ts | 6 +- 4 files changed, 303 insertions(+), 204 deletions(-) diff --git a/scripts/measure-a15-payees.ts b/scripts/measure-a15-payees.ts index 0a7b76f7..7ded7b35 100644 --- a/scripts/measure-a15-payees.ts +++ b/scripts/measure-a15-payees.ts @@ -63,6 +63,71 @@ const rewardBlocksAfter = async ( return [...new Set(nodes.map(n => Number(n.blockHeight)))].slice(0, limit); }; +interface Tally { + toStash: number; + elsewhere: number; + none: number; + errors: number; + nonStash: string[]; + seen: Set; +} + +/** Stashes named by the `staking.Reward`/`Rewarded` events in a block. */ +const rewardedStashes = ( + events: { event: { section: string; method: string; data: unknown[] } }[] +) => + events + .filter( + record => + record.event.section === 'staking' && ['Reward', 'Rewarded'].includes(record.event.method) + ) + .map(record => { + const data = record.event.data; + return (data.length >= 3 ? data[1] : data[0])?.toString(); + }); + +/** Reads one block's reward payees and folds them into the running tally. */ +const tallyBlock = async (api: ApiPromise, block: number, tally: Tally): Promise => { + const hash = await api.rpc.chain.getBlockHash(block); + const specVersion = (await api.rpc.state.getRuntimeVersion(hash)).specVersion.toNumber(); + + if (specVersion >= V8) { + return; + } + + let events; + try { + events = await api.query.system.events.at(hash); + } catch { + tally.errors += 1; + return; + } + + const at = await api.at(hash); + + for (const stash of rewardedStashes(events as never)) { + if (!stash || tally.seen.has(stash)) { + continue; + } + tally.seen.add(stash); + + try { + const payee = (await at.query.staking.payee(stash)).toString(); + + if (payee === 'Staked' || payee === 'Stash') { + tally.toStash += 1; + } else if (payee === 'None') { + tally.none += 1; + } else { + tally.elsewhere += 1; + tally.nonStash.push(` block ${block} (spec ${specVersion}) ${stash} -> ${payee}`); + } + } catch { + tally.errors += 1; + } + } +}; + const main = async (): Promise => { const rpc = argOf('rpc'); const dictionary = argOf('dictionary'); @@ -86,67 +151,23 @@ const main = async (): Promise => { typesBundle: typesBundle as never, }); - const seen = new Set(); - let toStash = 0; - let elsewhere = 0; - let none = 0; - let errors = 0; - const nonStash: string[] = []; + const tally: Tally = { + toStash: 0, + elsewhere: 0, + none: 0, + errors: 0, + nonStash: [], + seen: new Set(), + }; for (const anchor of anchors) { for (const block of await rewardBlocksAfter(dictionary, anchor, perAnchor)) { - const hash = await api.rpc.chain.getBlockHash(block); - const specVersion = (await api.rpc.state.getRuntimeVersion(hash)).specVersion.toNumber(); - - if (specVersion >= V8) { - continue; - } - - let events; - try { - events = await api.query.system.events.at(hash); - } catch { - errors += 1; - continue; - } - - const at = await api.at(hash); - - const stashes = events - .filter( - record => - record.event.section === 'staking' && - ['Reward', 'Rewarded'].includes(record.event.method) - ) - .map(record => { - const data = record.event.data; - return (data.length >= 3 ? data[1] : data[0]).toString(); - }); - - for (const stash of stashes) { - if (!stash || seen.has(stash)) { - continue; - } - seen.add(stash); - - try { - const payee = (await at.query.staking.payee(stash)).toString(); - - if (payee === 'Staked' || payee === 'Stash') { - toStash += 1; - } else if (payee === 'None') { - none += 1; - } else { - elsewhere += 1; - nonStash.push(` block ${block} (spec ${specVersion}) ${stash} -> ${payee}`); - } - } catch { - errors += 1; - } - } + await tallyBlock(api, block, tally); } } + const { toStash, elsewhere, none, errors, nonStash } = tally; + await api.disconnect(); const total = toStash + elsewhere + none; diff --git a/scripts/reconcile-polyx.ts b/scripts/reconcile-polyx.ts index 14b13c58..52f042fa 100644 --- a/scripts/reconcile-polyx.ts +++ b/scripts/reconcile-polyx.ts @@ -157,13 +157,12 @@ const chainAt = async (api: ApiPromise, address: string, block: number): Promise }; const d = info.data; const big = (v?: { toString(): string }) => BigInt(v?.toString() ?? '0'); - const misc = big(d.miscFrozen); - const fee = big(d.feeFrozen); + const legacyFrozen = big(d.miscFrozen) > big(d.feeFrozen) ? big(d.miscFrozen) : big(d.feeFrozen); return { free: big(d.free), reserved: big(d.reserved), - frozen: d.frozen !== undefined ? big(d.frozen) : misc > fee ? misc : fee, + frozen: d.frozen !== undefined ? big(d.frozen) : legacyFrozen, }; }; @@ -249,9 +248,9 @@ const classify = (mismatches: Mismatch[]): string => { if (constant) { verdict = `constant drift of ${deltas[0]} from block ${firstDrift.block} — one missed event`; } else if (growing) { - verdict = `growing drift (${deltas[0]} → ${ - deltas[deltas.length - 1] - }) — a systematically mis-signed event`; + verdict = `growing drift (${deltas[0]} → ${deltas.at( + -1 + )}) — a systematically mis-signed event`; } else { verdict = `irregular drift — needs manual inspection`; } @@ -297,6 +296,48 @@ const saveCheckpoint = (cp: Checkpoint): void => JSON.stringify(cp, (_k, v) => (typeof v === 'bigint' ? `${v}n` : v), 2) ); +interface ProbeContext { + db: DataSource; + api: ApiPromise; + head: number; + checkpoint: Checkpoint; + done: Set; +} + +/** Compares one account against chain state at one side of one boundary, checkpointing the result. */ +const probeOne = async ( + { db, api, head, checkpoint, done }: ProbeContext, + sample: Sample, + boundary: number, + side: 'before' | 'after', + block: number +): Promise => { + if (block > head) { + return; + } + + const key = `${sample.address}|${boundary}|${side}`; + if (done.has(key)) { + return; + } + + try { + const [derived, summed, onChain] = await Promise.all([ + derivedAt(db, sample.address, block), + summedAt(db, sample.address, block), + chainAt(api, sample.address, block), + ]); + + checkpoint.mismatches.push(...compare(sample, boundary, side, block, derived, onChain, summed)); + } catch (e) { + console.warn(`skip ${key}: ${(e as Error).message}`); + } + + done.add(key); + checkpoint.done = [...done]; + saveCheckpoint(checkpoint); +}; + const main = async (): Promise => { const rpc = argOf('rpc'); if (!rpc) { @@ -312,7 +353,7 @@ const main = async (): Promise => { const head = (await api.rpc.chain.getHeader()).number.toNumber(); const checkpoint = loadCheckpoint(rpc); - const done = new Set(checkpoint.done); + const context: ProbeContext = { db, api, head, checkpoint, done: new Set(checkpoint.done) }; const samples = await sampleAccounts(db, { high, typical }); console.log( @@ -321,37 +362,8 @@ const main = async (): Promise => { for (const sample of samples) { for (const boundary of BOUNDARIES) { - for (const [side, block] of [ - ['before', boundary - 1], - ['after', boundary + 1], - ] as const) { - if (block > head) { - continue; - } - - const key = `${sample.address}|${boundary}|${side}`; - if (done.has(key)) { - continue; - } - - try { - const [derived, summed, onChain] = await Promise.all([ - derivedAt(db, sample.address, block), - summedAt(db, sample.address, block), - chainAt(api, sample.address, block), - ]); - - checkpoint.mismatches.push( - ...compare(sample, boundary, side, block, derived, onChain, summed) - ); - } catch (e) { - console.warn(`skip ${key}: ${(e as Error).message}`); - } - - done.add(key); - checkpoint.done = [...done]; - saveCheckpoint(checkpoint); - } + await probeOne(context, sample, boundary, 'before', boundary - 1); + await probeOne(context, sample, boundary, 'after', boundary + 1); } } diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts index 31ce87bf..087fb862 100644 --- a/src/mappings/entities/identities/mapPolyxLedger.ts +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -100,6 +100,9 @@ const startOfUtcDay = (datetime: Date): Date => const floorZero = (value: bigint): bigint => (value > BigInt(0) ? value : BigInt(0)); +/** `Math.max` for `bigint`, which `Math.max` itself cannot take. */ +const maxBig = (a: bigint, b: bigint): bigint => (a > b ? a : b); + /** * `frozen` from an on-chain `AccountData`: `{ free, reserved, frozen, flags }` on v8, * `{ free, reserved, miscFrozen, feeFrozen }` (frozen is the max of the two) on ≤ v7.4. @@ -112,7 +115,7 @@ export const accountDataFrozen = (data: Record): bigint => { const misc = getBigIntValue(data.miscFrozen); const fee = getBigIntValue(data.feeFrozen); - return misc > fee ? misc : fee; + return maxBig(misc, fee); }; // --------------------------------------------------------------------------------------------- @@ -217,8 +220,8 @@ export const recomputeDerived = (balance: AccountBalance): void => { const stakingHold = holds.find(hold => hold.reason === HoldReason.Staking)?.amount ?? BigInt(0); const stakingLock = locks.find(lock => lock.lockId === STAKING_LOCK_ID)?.amount ?? BigInt(0); - balance.frozen = locks.reduce((max, lock) => (lock.amount > max ? lock.amount : max), BigInt(0)); - balance.bonded = stakingHold > stakingLock ? stakingHold : stakingLock; + balance.frozen = locks.reduce((max, lock) => maxBig(max, lock.amount), BigInt(0)); + balance.bonded = maxBig(stakingHold, stakingLock); balance.otherReserved = floorZero(balance.reserved - stakingHold); balance.total = balance.free + balance.reserved; balance.transferable = floorZero(balance.free - balance.frozen); @@ -267,32 +270,15 @@ interface Transition { const poolTag = (pool: PolyxPool): string => (pool === PolyxPool.Free ? 'f' : 'r'); -/** - * Writes the entries for one pool transition and advances every touched `AccountBalance`. - * - * Sibling entries of one movement share `movementId` (the block/event id). Each entry carries the - * balance-after snapshot, so Balance History is a pure index scan. - */ -export const postTransition = async ( - args: HandlerArgs, - transition: Transition, - options: { eraIndex?: number } = {} -): Promise => { - if (!transition.from && !transition.to) { - return; - } - - const { blockId, block, eventIdx, blockEventId } = args; - const datetime = block.timestamp; - const params = getEventParams(args); - const date = startOfUtcDay(datetime); - - const isInternal = - transition.from !== undefined && - transition.to !== undefined && - transition.from.address === transition.to.address; +interface MovementSide { + endpoint: Endpoint; + direction: EntryDirection; + counterparty?: string; +} - const sides: Array<{ endpoint: Endpoint; direction: EntryDirection; counterparty?: string }> = []; +/** The one or two account-sides a transition touches, each paired with its counterparty. */ +const movementSides = (transition: Transition): MovementSide[] => { + const sides: MovementSide[] = []; if (transition.from?.address) { sides.push({ @@ -310,80 +296,141 @@ export const postTransition = async ( }); } - for (const side of sides) { - const { address, pool } = side.endpoint; - const signed = - side.direction === EntryDirection.Credit ? transition.amount : -transition.amount; + return sides; +}; - const account = await ledgerAccount(address, blockId, datetime); - const balance = await loadBalance(address, account.identityId, blockId); +/** The lifetime aggregate a movement kind feeds, if any. */ +const LIFETIME_TOTAL: Partial< + Record +> = { + [MovementKind.Fee]: 'totalFeesPaid', + [MovementKind.StakingReward]: 'totalRewards', + [MovementKind.Slash]: 'totalSlashed', +}; + +/** Advances the running `AccountBalance` for one side of a movement (pools + aggregates). */ +const advanceBalance = ( + balance: AccountBalance, + side: MovementSide, + transition: Transition, + signed: bigint, + isInternal: boolean +): void => { + if (side.endpoint.pool === PolyxPool.Free) { + balance.free += signed; + } else { + balance.reserved += signed; + } - if (pool === PolyxPool.Free) { - balance.free += signed; + if (!isInternal) { + if (side.direction === EntryDirection.Credit) { + balance.totalReceived += transition.amount; } else { - balance.reserved += signed; + balance.totalSent += transition.amount; } + } - if (!isInternal) { - if (side.direction === EntryDirection.Credit) { - balance.totalReceived += transition.amount; - } else { - balance.totalSent += transition.amount; - } - } + const lifetimeTotal = LIFETIME_TOTAL[transition.kind]; + if (lifetimeTotal) { + balance[lifetimeTotal] += transition.amount; + } - if (transition.kind === MovementKind.Fee) { - balance.totalFeesPaid += transition.amount; - } else if (transition.kind === MovementKind.StakingReward) { - balance.totalRewards += transition.amount; - } else if (transition.kind === MovementKind.Slash) { - balance.totalSlashed += transition.amount; - } + bumpLifetimeByKind(balance, transition.kind, side.direction, transition.amount); + balance.movementCount += 1; + recomputeDerived(balance); +}; - bumpLifetimeByKind(balance, transition.kind, side.direction, transition.amount); - balance.movementCount += 1; - recomputeDerived(balance); - balance.updatedBlockId = blockId; +interface TransitionContext { + args: HandlerArgs; + transition: Transition; + options: { eraIndex?: number }; + isInternal: boolean; + date: Date; + params: ReturnType; +} - await balance.save(); +/** Advances one account, writes its `PolyxEntry`, and reconciles it. */ +const writeMovementSide = async ( + side: MovementSide, + { args, transition, options, isInternal, date, params }: TransitionContext +): Promise => { + const { blockId, block, eventIdx, blockEventId } = args; + const { address, pool } = side.endpoint; + const signed = side.direction === EntryDirection.Credit ? transition.amount : -transition.amount; - const counterpartyAccount = side.counterparty - ? await Account.get(side.counterparty) - : undefined; + const account = await ledgerAccount(address, blockId, block.timestamp); + const balance = await loadBalance(address, account.identityId, blockId); - await PolyxEntry.create({ - id: `${blockId}/${padId(eventIdx.toString())}/${poolTag(pool)}${ - side.direction === EntryDirection.Debit ? 'd' : 'c' - }`, - movementId: blockEventId, - accountId: address, - identityId: account.identityId, - counterpartyAddress: side.counterparty, - counterpartyIdentityId: counterpartyAccount?.identityId, - pool, - amount: signed, - amountAbs: transition.amount, - kind: transition.kind, - direction: side.direction, - holdReason: transition.holdReason, - memo: transition.memo, - freeAfter: balance.free, - reservedAfter: balance.reserved, - frozenAfter: balance.frozen, - moduleId: params.moduleId, - callId: params.callId, - eventId: params.eventId, - specVersionId: block.specVersion, - date, - eraIndex: options.eraIndex, - createdEventId: blockEventId, - extrinsicId: params.extrinsicId, - eventIdx, - datetime, - createdBlockId: blockId, - }).save(); + advanceBalance(balance, side, transition, signed, isInternal); + balance.updatedBlockId = blockId; + await balance.save(); - await reconcileAccount(address, blockId, block, { eventIdx }); + const counterpartyAccount = side.counterparty ? await Account.get(side.counterparty) : undefined; + + await PolyxEntry.create({ + id: `${blockId}/${padId(eventIdx.toString())}/${poolTag(pool)}${ + side.direction === EntryDirection.Debit ? 'd' : 'c' + }`, + movementId: blockEventId, + accountId: address, + identityId: account.identityId, + counterpartyAddress: side.counterparty, + counterpartyIdentityId: counterpartyAccount?.identityId, + pool, + amount: signed, + amountAbs: transition.amount, + kind: transition.kind, + direction: side.direction, + holdReason: transition.holdReason, + memo: transition.memo, + freeAfter: balance.free, + reservedAfter: balance.reserved, + frozenAfter: balance.frozen, + moduleId: params.moduleId, + callId: params.callId, + eventId: params.eventId, + specVersionId: block.specVersion, + date, + eraIndex: options.eraIndex, + createdEventId: blockEventId, + extrinsicId: params.extrinsicId, + eventIdx, + datetime: block.timestamp, + createdBlockId: blockId, + }).save(); + + await reconcileAccount(address, blockId, block, { eventIdx }); +}; + +/** + * Writes the entries for one pool transition and advances every touched `AccountBalance`. + * + * Sibling entries of one movement share `movementId` (the block/event id). Each entry carries the + * balance-after snapshot, so Balance History is a pure index scan. + */ +export const postTransition = async ( + args: HandlerArgs, + transition: Transition, + options: { eraIndex?: number } = {} +): Promise => { + if (!transition.from && !transition.to) { + return; + } + + const isInternal = + transition.from?.address !== undefined && transition.from.address === transition.to?.address; + + const context: TransitionContext = { + args, + transition, + options, + isInternal, + date: startOfUtcDay(args.block.timestamp), + params: getEventParams(args), + }; + + for (const side of movementSides(transition)) { + await writeMovementSide(side, context); } }; @@ -923,6 +970,39 @@ export const handleDustLost = async (event: SubstrateEvent): Promise => { // BalanceSet — a checkpoint, not a movement (resolves A1 structurally) // --------------------------------------------------------------------------------------------- +interface PoolDelta { + pool: PolyxPool; + delta: bigint; +} + +const signOf = (delta: bigint): EntryDirection => + delta > BigInt(0) ? EntryDirection.Credit : EntryDirection.Debit; + +/** Sets each pool to its new absolute value, returning the non-zero deltas the set produced. */ +const applyBalanceSet = ( + balance: AccountBalance, + newFree: bigint, + newReserved: bigint | undefined +): PoolDelta[] => { + const deltas: PoolDelta[] = []; + + const freeDelta = newFree - balance.free; + balance.free = newFree; + if (freeDelta !== BigInt(0)) { + deltas.push({ pool: PolyxPool.Free, delta: freeDelta }); + } + + if (newReserved !== undefined) { + const reservedDelta = newReserved - balance.reserved; + balance.reserved = newReserved; + if (reservedDelta !== BigInt(0)) { + deltas.push({ pool: PolyxPool.Reserved, delta: reservedDelta }); + } + } + + return deltas; +}; + /** * `BalanceSet` *sets* `free` (and, pre-v8, `reserved`) to an absolute value — it is not a * movement of that size. The old model recorded the set value as a delta, corrupting every @@ -943,21 +1023,7 @@ export const handleBalanceSet = async (event: SubstrateEvent): Promise => const account = await ledgerAccount(who, blockId, datetime); const balance = await loadBalance(who, account.identityId, blockId); - const deltas: Array<{ pool: PolyxPool; delta: bigint }> = []; - - const freeDelta = newFree - balance.free; - balance.free = newFree; - if (freeDelta !== BigInt(0)) { - deltas.push({ pool: PolyxPool.Free, delta: freeDelta }); - } - - if (newReserved !== undefined) { - const reservedDelta = newReserved - balance.reserved; - balance.reserved = newReserved; - if (reservedDelta !== BigInt(0)) { - deltas.push({ pool: PolyxPool.Reserved, delta: reservedDelta }); - } - } + const deltas = applyBalanceSet(balance, newFree, newReserved); if (deltas.length > 0) { balance.movementCount += 1; @@ -965,7 +1031,7 @@ export const handleBalanceSet = async (event: SubstrateEvent): Promise => bumpLifetimeByKind( balance, MovementKind.BalanceSetAdjustment, - delta > BigInt(0) ? EntryDirection.Credit : EntryDirection.Debit, + signOf(delta), delta > BigInt(0) ? delta : -delta ); } @@ -990,7 +1056,7 @@ export const handleBalanceSet = async (event: SubstrateEvent): Promise => amount: delta, amountAbs: delta > BigInt(0) ? delta : -delta, kind: MovementKind.BalanceSetAdjustment, - direction: delta > BigInt(0) ? EntryDirection.Credit : EntryDirection.Debit, + direction: signOf(delta), holdReason: undefined, memo: undefined, freeAfter: balance.free, diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts index 6f46ae91..2030eb0f 100644 --- a/tests/unit/mapPolyxLedger.test.ts +++ b/tests/unit/mapPolyxLedger.test.ts @@ -398,7 +398,7 @@ describe('properties the one-column model could not satisfy', () => { await handleBalanceFrozen(balancesEvent('Frozen', { who: ALICE, amount: '400' })); - expect(entries().length).toBe(entriesAfterMint); // no new entry + expect(entries()).toHaveLength(entriesAfterMint); // no new entry expect(balance(ALICE)).toMatchObject({ free: BigInt(1000), reserved: BigInt(0), @@ -429,7 +429,7 @@ describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); - expect(entries().length).toBe(beforeEntries); // no movement row + expect(entries()).toHaveLength(beforeEntries); // no movement row expect(balance(ALICE)).toMatchObject({ free: BigInt(10000), // unchanged — pre-v8 bonding moves nothing frozen: BigInt(4000), @@ -455,7 +455,7 @@ describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { balancesEvent('Bonded', { stash: ALICE, amount: '4000' }, { specVersion: 8_000_000 }) ); - expect(entries().length).toBe(beforeEntries); + expect(entries()).toHaveLength(beforeEntries); expect(balance(ALICE)).toMatchObject({ free: BigInt(10000), frozen: BigInt(0), From 2f952ff195cfb29acbc54726ebd32afa98942519 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:40:07 +0530 Subject: [PATCH 13/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20declare=20the=20v?= =?UTF-8?q?alidators=20and=20mmr=20members=20sync-metadata=20--write=20fla?= =?UTF-8?q?gs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validators.AutomaticPayoutFinished / ValidatorPayoutFailed reach the generic event recorder and would log an UnknownEnumValue anomaly; mmr / mmrleaf are runtime pallets absent from ModuleIdEnum. sync-metadata.ts now reports zero missing members across all three enums. --- schema.graphql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schema.graphql b/schema.graphql index dc15e230..85db9c2a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -26,6 +26,8 @@ enum ModuleIdEnum { authoritydiscovery grandpa historical + mmr + mmrleaf imonline randomnesscollectiveflip sudo @@ -353,6 +355,10 @@ enum EventIdEnum { SnapshotTargetsSizeExceeded SnapshotVotersSizeExceeded + ## validators ## + AutomaticPayoutFinished + ValidatorPayoutFailed + ## offences ## Offence From 2e690b693abe9d91ac83798d795e47719229989b Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 00:39:46 +0530 Subject: [PATCH 14/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20type=20the=20rewa?= =?UTF-8?q?rd-event=20data=20as=20stringable,=20not=20unknown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rewardedStashes helper took data: unknown[], so data[1].toString() looked to Sonar (S6551) like it could stringify a plain object. The values are codecs; type the param as { toString(): string }[]. --- scripts/measure-a15-payees.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/measure-a15-payees.ts b/scripts/measure-a15-payees.ts index 7ded7b35..9c2c82ac 100644 --- a/scripts/measure-a15-payees.ts +++ b/scripts/measure-a15-payees.ts @@ -72,10 +72,16 @@ interface Tally { seen: Set; } -/** Stashes named by the `staking.Reward`/`Rewarded` events in a block. */ +/** A codec-ish value: only its `.toString()` (SS58 address / hex) is read here. */ +type Stringable = { toString(): string }; + +/** + * Stashes named by the `staking.Reward`/`Rewarded` events in a block. The stash is the first + * parameter on ≤7.x (`(stash, amount)`) and the second from 7.x (`(identityId, stash, amount)`). + */ const rewardedStashes = ( - events: { event: { section: string; method: string; data: unknown[] } }[] -) => + events: { event: { section: string; method: string; data: Stringable[] } }[] +): string[] => events .filter( record => @@ -83,7 +89,7 @@ const rewardedStashes = ( ) .map(record => { const data = record.event.data; - return (data.length >= 3 ? data[1] : data[0])?.toString(); + return (data.length >= 3 ? data[1] : data[0])?.toString() ?? ''; }); /** Reads one block's reward payees and folds them into the running tally. */ From e27ebd8ddbd9a7dd5e00b013f1b1d848cbcbac74 Mon Sep 17 00:00:00 2001 From: Francis O'Brien <42175565+F-OBrien@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:10:32 +0100 Subject: [PATCH 15/22] =?UTF-8?q?fix:=20=F0=9F=90=9B=20resolve=20RewardDes?= =?UTF-8?q?tination=20from=20its=20camel-cased=20toJSON=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Enum#toJSON()` camel-cases the variant name, so an 8.x `Rewarded` with an `Account` payee arrives as `{ account: "5..." }` and never matched the `'Account'` comparison: the row stored `rewardDestination: 'account'` with no destination account. Every object form was affected, `{ staked: null }` included, so no destination account was resolved for any of them. The parse now lives once in `utils/staking.ts`, shared with the pre-v8 `staking.payee` read. It takes `AnyJson` and narrows rather than casting `toJSON()` to a hand-written shape, and `rewardDestination` is a union of the variant names instead of `string`. `.toJSON()` stays in place of the generated `Option` accessors: those describe the current metadata only, while both callers read older runtimes, and the reason is now recorded at the helper. --- .../entities/events/mapStakingEvent.ts | 49 +++------- src/utils/staking.ts | 92 +++++++++++++++---- tests/unit/rewardDestinationA15.test.ts | 16 +++- 3 files changed, 103 insertions(+), 54 deletions(-) diff --git a/src/mappings/entities/events/mapStakingEvent.ts b/src/mappings/entities/events/mapStakingEvent.ts index 1f86681d..b09ad2bc 100644 --- a/src/mappings/entities/events/mapStakingEvent.ts +++ b/src/mappings/entities/events/mapStakingEvent.ts @@ -4,7 +4,11 @@ import { SubstrateBlock, SubstrateEvent } from '@subql/types'; import { Account, EventIdEnum, StakingEvent } from '../../../types'; import { getBigIntValue, getTextValue } from '../../../utils'; import { is8xChain } from '../../../utils/common'; -import { resolveLegacyRewardDestination } from '../../../utils/staking'; +import { + readRewardDestination, + resolveLegacyRewardDestination, + RewardDestinationName, +} from '../../../utils/staking'; import { extractArgs } from '../common'; const bondedUnbondedOrReward = new Set([ @@ -14,50 +18,25 @@ const bondedUnbondedOrReward = new Set([ EventIdEnum.Rewarded, // from 7.x Reward was renamed to Rewarded ]); -type RewardDestinationDetails = { - type: string; - account?: string; -}; - type StakingEventDetails = { amount?: bigint; stashAccount?: string; nominatedValidators?: string[]; identityId?: string; - rewardDestination?: string; + rewardDestination?: RewardDestinationName; rewardDestinationAccount?: string; }; -const getRewardDestinationDetails = (destParam: Codec): RewardDestinationDetails => { - const json = destParam.toJSON() as string | Record; - - if (typeof json === 'string') { - return { type: json }; - } - - const variant = Object.keys(json)[0] ?? 'Unknown'; - - const value = json[variant]; - - if (variant === 'Account') { - return { - type: variant, - account: typeof value === 'string' ? value : undefined, - }; - } - - return { type: variant }; -}; - const getRewardDestinationAccount = ( - destinationDetails: RewardDestinationDetails, + destination: RewardDestinationName, + account?: string, stashAccount?: string ): string | undefined => { - if (destinationDetails.type === 'Account') { - return destinationDetails.account; + if (destination === 'Account') { + return account; } - if (destinationDetails.type === 'Staked' || destinationDetails.type === 'Stash') { + if (destination === 'Staked' || destination === 'Stash') { return stashAccount; } @@ -88,13 +67,13 @@ const get8xStakingEventDetails = (eventId: EventIdEnum, params: Codec[]): Stakin const stashAccount = getTextValue(rawAccount); if (eventId === EventIdEnum.Rewarded) { - const destinationDetails = getRewardDestinationDetails(rawSecondParam); + const { destination, account } = readRewardDestination(rawSecondParam.toJSON()); return { stashAccount, amount: getBigIntValue(rawThirdParam), - rewardDestination: destinationDetails.type, - rewardDestinationAccount: getRewardDestinationAccount(destinationDetails, stashAccount), + rewardDestination: destination, + rewardDestinationAccount: getRewardDestinationAccount(destination, account, stashAccount), }; } diff --git a/src/utils/staking.ts b/src/utils/staking.ts index ba9f01c6..a1a5fa5b 100644 --- a/src/utils/staking.ts +++ b/src/utils/staking.ts @@ -1,10 +1,74 @@ +import type { AnyJson } from '@polkadot/types/types'; + +/** + * The chain's own `RewardDestination` variants, plus the placeholder recorded when the payee + * cannot be read at all. + * + * Spelled out rather than derived from `PalletStakingRewardDestination['type']`: `src/index.ts` + * loads both `polymesh-types` and `@polkadot/types-augment`, and each declares that interface + * into `@polkadot/types/lookup`. The duplicate declaration collapses `type` to a bare `string` + * (the conflict is in `node_modules`, so `skipLibCheck` hides it) — `is*`/`as*` survive it, + * `type` does not — so deriving the union would silently widen it back to `string`. + */ +export type RewardDestinationName = + | 'Staked' + | 'Stash' + | 'Controller' + | 'Account' + | 'None' + | 'LegacyUnknown'; + export interface LegacyRewardDestination { - /** `Staked` | `Stash` | `Controller` | `Account` | `None` | `LegacyUnknown` (read failed) */ - rewardDestination: string; + rewardDestination: RewardDestinationName; /** The account the reward was actually paid to, where it can be resolved */ rewardDestinationAccount?: string; } +/** `.toJSON()` camel-cases the variant name; this maps it back to the name the index records. */ +const rewardDestinationByVariant: Record = { + staked: 'Staked', + stash: 'Stash', + controller: 'Controller', + account: 'Account', + none: 'None', +}; + +/** + * Reads a decoded `RewardDestination` — from storage or from an event parameter — out of its + * `.toJSON()` form. + * + * `.toJSON()` rather than the generated `Option` accessors, + * deliberately: the generated types describe one metadata snapshot — the current one — while + * both callers read blocks from runtimes that predate it, and `api` decodes against the block's + * own registry. `.unwrap()` written against today's `OptionQuery` throws on a block where the + * entry decodes as a bare `RewardDestination`, and that throw lands in a `catch` that would turn + * every reward into `LegacyUnknown`. `.toJSON()` is the one accessor whose output is the same + * either way: `null`, a bare string (`"Staked"` — when every variant of that runtime's type is a + * unit variant), or a single-key object with the variant camel-cased (`{ staked: null }`, + * `{ account: "0x…" }`). + * + * An unrecognised variant resolves to `None` — no destination account is claimed for it. + */ +export const readRewardDestination = ( + json: AnyJson +): { destination: RewardDestinationName; account?: string } => { + let variant = ''; + let value: AnyJson = null; + + if (typeof json === 'string') { + variant = json; + } else if (json && typeof json === 'object' && !Array.isArray(json)) { + [variant = ''] = Object.keys(json); + value = json[variant] ?? null; + } + + const destination = rewardDestinationByVariant[variant.toLowerCase()] ?? 'None'; + + return destination === 'Account' && typeof value === 'string' + ? { destination, account: value } + : { destination }; +}; + /** * Per-stash cache of a resolved payee. `staking.payee(stash)` is a chain-storage read on every * reward, and a validator's set of ~20 stashes is re-queried every era for the whole genesis @@ -39,34 +103,26 @@ export const resolveLegacyRewardDestination = async ( try { const payee = await api.query.staking.payee(stash); - const json = payee.toJSON() as string | Record | null; - - // `RewardDestination` renders either as a bare string (`"Staked"`) or, via `.toJSON()`, as a - // single-key object with the variant name lower-cased (`{ staked: null }`, `{ account: "0x…" }`). - const variant = ( - typeof json === 'string' ? json : Object.keys(json ?? {})[0] ?? '' - ).toLowerCase(); - const value = - json && typeof json === 'object' ? (json as Record)[variant] : undefined; + const { destination, account } = readRewardDestination(payee.toJSON()); let result: LegacyRewardDestination; - if (variant === 'staked' || variant === 'stash') { + if (destination === 'Staked' || destination === 'Stash') { result = { - rewardDestination: variant === 'staked' ? 'Staked' : 'Stash', + rewardDestination: destination, rewardDestinationAccount: stash, }; - } else if (variant === 'controller') { - const controller = (await api.query.staking.bonded(stash)).toJSON() as string | null; + } else if (destination === 'Controller') { + const controller = (await api.query.staking.bonded(stash)).toJSON(); result = { rewardDestination: 'Controller', - rewardDestinationAccount: controller ?? undefined, + rewardDestinationAccount: typeof controller === 'string' ? controller : undefined, }; - } else if (variant === 'account') { + } else if (destination === 'Account') { result = { rewardDestination: 'Account', - rewardDestinationAccount: typeof value === 'string' ? value : undefined, + rewardDestinationAccount: account, }; } else { result = { rewardDestination: 'None' }; diff --git a/tests/unit/rewardDestinationA15.test.ts b/tests/unit/rewardDestinationA15.test.ts index 8e47a144..a2fda21b 100644 --- a/tests/unit/rewardDestinationA15.test.ts +++ b/tests/unit/rewardDestinationA15.test.ts @@ -134,7 +134,7 @@ describe('A15 — pre-v8 reward destination', () => { await handleStakingEvent( rewardEvent( 'Rewarded', - [codec(STASH), { toJSON: () => ({ Account: PAYEE }) }, codec('3000')], + [codec(STASH), { toJSON: () => ({ account: PAYEE }) }, codec('3000')], 8_000_000 ) ); @@ -153,4 +153,18 @@ describe('A15 — pre-v8 reward destination', () => { expect(row.rewardDestination).toBe('Staked'); expect(row.rewardDestinationAccount).toBe(STASH); }); + + it('v8 resolves the object form of a Staked payee to the stash', async () => { + await handleStakingEvent( + rewardEvent( + 'Rewarded', + [codec(STASH), { toJSON: () => ({ staked: null }) }, codec('4500')], + 8_000_000 + ) + ); + + const row = savedStakingEvent(); + expect(row.rewardDestination).toBe('Staked'); + expect(row.rewardDestinationAccount).toBe(STASH); + }); }); From 55e22c191f2c4cca7572e964257f56fc21fa63a6 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:15:53 +0530 Subject: [PATCH 16/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20derive=20the=20pr?= =?UTF-8?q?e-v8=20staking=20lock=20from=20staking.ledger.total?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The '\''staking '\'' lock was accumulated from Bonded / Withdrawn / restaked-Reward deltas, which never see the max-bond cap, the rounding of a compounded RewardDestination::Staked reward, unbonding chunks, or a slash - so it drifted from the real lock across the whole validator set. readStakingLock(stash) reads staking.ledger(controller).total from chain - the exact value pallet-staking passes to Currency::set_lock, hence what miscFrozen reports. syncStakingLock is wired into handleBonded / handleWithdrawn / handleReward (restaked) / handleStakingSlash, and falls back to the delta accumulator when the ledger cannot be read. Controller is resolved via staking.bonded(stash) and cached. --- .../entities/identities/mapPolyxLedger.ts | 39 ++++++++++-- src/utils/staking.ts | 44 +++++++++++++ tests/unit/mapPolyxLedger.test.ts | 61 ++++++++++++++++++- 3 files changed, 138 insertions(+), 6 deletions(-) diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts index 087fb862..ee3b8c73 100644 --- a/src/mappings/entities/identities/mapPolyxLedger.ts +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -14,7 +14,7 @@ import { } from '../../../types'; import { bytesToString, getBigIntValue, getTextValue, padId } from '../../../utils'; import { camelToSnakeCase, is8xChain, snakeToCamelCase } from '../../../utils/common'; -import { resolveLegacyRewardDestination } from '../../../utils/staking'; +import { readStakingLock, resolveLegacyRewardDestination } from '../../../utils/staking'; import { getAccountKeyType, getOrCreateAccount } from '../../../utils/accounts'; import { getEventParams } from '../../../utils/events'; import { extractArgs, HandlerArgs } from '../common'; @@ -527,6 +527,29 @@ export const setLock = async ( await adjustLock(address, lockId, amount - current, blockId, reasons); }; +/** + * Sets the pre-v8 `'staking '` lock on `stash` to `staking.ledger.total` read from chain. + * + * The chain read is authoritative — it already accounts for the max-bond cap, the rounding of a + * compounded `Staked` reward, unbonding chunks, and slashes. When the ledger cannot be read the + * `fallbackDelta` keeps the old accumulator behaviour so the reconciler still has a base to work + * from. + */ +const syncStakingLock = async ( + stash: string, + fallbackDelta: bigint, + blockId: string +): Promise => { + const total = await readStakingLock(stash); + + if (total === undefined) { + await adjustLock(stash, STAKING_LOCK_ID, fallbackDelta, blockId, 'staking'); + return; + } + + await setLock(stash, STAKING_LOCK_ID, total, blockId, 'staking'); +}; + const lockHandler = (lockId: string, sign: bigint) => async (event: SubstrateEvent): Promise => { @@ -1291,7 +1314,7 @@ export const handleReward = async (event: SubstrateEvent): Promise => { // Pre-v8 `Staked` payee: the reward is added to the staking lock in the same step. if (restaked) { - await adjustLock(recipient, STAKING_LOCK_ID, amount, args.blockId, 'staking'); + await syncStakingLock(recipient, amount, args.blockId); } }; @@ -1299,16 +1322,22 @@ export const handleReward = async (event: SubstrateEvent): Promise => { export const handleStakingSlash = async (event: SubstrateEvent): Promise => { const args = extractArgs(event); const decoded = decodeEvent(event); + const stash = stakingStash(decoded); await postTransition( args, { - from: { address: stakingStash(decoded), pool: PolyxPool.Free }, + from: { address: stash, pool: PolyxPool.Free }, amount: amountOf(decoded), kind: MovementKind.Slash, }, { eraIndex: currentPayoutEra(args.blockId) } ); + + // A slash reduces `ledger.total`, and pre-v8 nothing else re-reads the lock — resync it. + if (stash && !is8xChain(args.block)) { + await syncStakingLock(stash, -amountOf(decoded), args.blockId); + } }; const ensureBalanceRow = async ( @@ -1337,7 +1366,7 @@ export const handleBonded = async (event: SubstrateEvent): Promise => { } await ensureBalanceRow(stash, args.blockId, args.block.timestamp); - await adjustLock(stash, STAKING_LOCK_ID, amountOf(decoded), args.blockId, 'staking'); + await syncStakingLock(stash, amountOf(decoded), args.blockId); }; /** @@ -1363,5 +1392,5 @@ export const handleWithdrawn = async (event: SubstrateEvent): Promise => { return; } - await adjustLock(stash, STAKING_LOCK_ID, -amountOf(decoded), args.blockId); + await syncStakingLock(stash, -amountOf(decoded), args.blockId); }; diff --git a/src/utils/staking.ts b/src/utils/staking.ts index a1a5fa5b..dc34c2da 100644 --- a/src/utils/staking.ts +++ b/src/utils/staking.ts @@ -136,3 +136,47 @@ export const resolveLegacyRewardDestination = async ( return { rewardDestination: 'LegacyUnknown' }; } }; + +/** + * Cache of `stash -> controller`. `staking.ledger` is keyed by the controller, not the stash, so + * a `bonded(stash)` read is needed before every ledger read. `set_controller` is very rare, so a + * process-lifetime cache is safe; a stale entry only matters if the controller changed, and the + * in-flight reconciler corrects the resulting `frozen` drift. + */ +const controllerCache = new Map(); + +/** Test hook — the cache is process-lifetime, so a suite re-mocking `staking.bonded` must clear it. */ +export const __resetControllerCache = (): void => controllerCache.clear(); + +/** + * The pre-v8 staking lock on `stash`, read from chain: `staking.ledger(controller).total` + * (bonded active + everything still unlocking), which is exactly the value `pallet-staking` + * passes to `Currency::set_lock`, so it is what `miscFrozen` reports. + * + * Read rather than accumulated from `Bonded` / `Withdrawn` / restaked-`Reward` deltas: the deltas + * do not see the max-bond cap, the rounding of a compounded `RewardDestination::Staked` reward, + * or a slash — each of which leaves the accumulator drifting from the real lock. + * + * `undefined` when the ledger cannot be read (a runtime with a different shape, a pruned node) — + * the caller keeps its delta accumulator as the fallback. A killed ledger (fully withdrawn) + * reads back as `0`. + */ +export const readStakingLock = async (stash: string): Promise => { + try { + let controller = controllerCache.get(stash); + + if (!controller) { + const bonded = (await api.query.staking.bonded(stash)).toJSON(); + controller = typeof bonded === 'string' ? bonded : stash; + controllerCache.set(stash, controller); + } + + const ledger = (await api.query.staking.ledger(controller)).toJSON() as { + total?: string | number; + } | null; + + return ledger ? BigInt(ledger.total ?? 0) : BigInt(0); + } catch { + return undefined; + } +}; diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts index 2030eb0f..6c13fe71 100644 --- a/tests/unit/mapPolyxLedger.test.ts +++ b/tests/unit/mapPolyxLedger.test.ts @@ -32,7 +32,7 @@ import { handleTreasuryReimbursement, } from '../../src/mappings/entities/identities/mapPolyxLedger'; import { getAccountId, systematicIssuers } from '../../src/mappings/consts'; -import { __resetPayeeCache } from '../../src/utils/staking'; +import { __resetControllerCache, __resetPayeeCache } from '../../src/utils/staking'; const ALICE = '5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'; const BOB = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty'; @@ -134,6 +134,7 @@ const clone = (value: Row): Row => { beforeEach(() => { __resetPayeeCache(); + __resetControllerCache(); db = {}; blockHeight = 1_000_000; (globalThis as any).api.registry = { chainSS58: 42 }; @@ -512,6 +513,64 @@ describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { (globalThis as any).api.query = {}; }); + describe('the staking lock is read from staking.ledger.total, not accumulated', () => { + const mockLedger = (total: string, bonded: string | null = null) => { + (globalThis as any).api.query = { + staking: { + payee: jest.fn().mockResolvedValue({ toJSON: () => 'Staked' }), + bonded: jest.fn().mockResolvedValue({ toJSON: () => bonded }), + ledger: jest.fn().mockResolvedValue({ toJSON: () => ({ total, active: total }) }), + }, + }; + }; + + afterEach(() => { + (globalThis as any).api.query = {}; + }); + + it('v7 Bonded pins the lock to ledger.total, ignoring the event amount', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + mockLedger('4200'); // chain: 4200 bonded (4000 + a compounded 200 the event never carried) + + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + expect(balance(ALICE)).toMatchObject({ frozen: BigInt(4200), bonded: BigInt(4200) }); + }); + + it('a restaked Staked reward pins the lock to ledger.total (capped below the gross reward)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + mockLedger('4000000000000'); // at the max-bond cap + + await handleReward(tupleEvent('staking', 'Reward', ['0xdid', ALICE, '900'], 7_004_001)); + + // free still takes the whole reward; the lock is whatever the ledger says, not += 900 + expect(balance(ALICE)).toMatchObject({ + free: BigInt(10900), + frozen: BigInt('4000000000000'), + }); + }); + + it('v7 Withdrawn pins the lock to the reduced ledger.total', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + mockLedger('4000'); + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + mockLedger('2500'); + await handleWithdrawn(tupleEvent('staking', 'Withdrawn', [ALICE, '1500'], 7_004_001)); + + expect(balance(ALICE)?.frozen).toBe(BigInt(2500)); + }); + + it('falls back to the delta accumulator when the ledger cannot be read', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + (globalThis as any).api.query = {}; // no staking.ledger + + await handleBonded(tupleEvent('staking', 'Bonded', ['0xdid', ALICE, '4000'], 7_004_001)); + + expect(balance(ALICE)?.frozen).toBe(BigInt(4000)); + }); + }); + it('staking Reward is ∅ → stash/Free with the era from the preceding PayoutStarted', async () => { await handlePayoutStarted( structEvent( From 312c696d860b34517d1b8a4097704b8746455d4e Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:16:04 +0530 Subject: [PATCH 17/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20buffer=20NftHolde?= =?UTF-8?q?r=20writes=20per=20block?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NftHolder.nftIds is a JSON array and historical mode versions the whole array on every save. A bulk mint - hundreds of NFTPortfolioUpdated for one holder in one block - made that Sigma(1..n) array serialisations and poisoned the store cache with 30k-element arrays, degrading the indexer to a crawl on later empty blocks. Holder mutations are now buffered per block and each holder is saved once, on block change or from a new handleBlock (Block handler, filter modulo 100). getNftHolder reads the buffer first. Nothing inside the indexer reads NftHolder - it is written for external queries only - so a holder being at most one block-handler interval stale is acceptable. --- project.ts | 16 ++++++++ src/mappings/entities/assets/mapNfts.ts | 53 +++++++++++++++++++++++-- src/mappings/mappingHandlers.ts | 13 +++++- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/project.ts b/project.ts index 984baeae..a6b4cfc6 100644 --- a/project.ts +++ b/project.ts @@ -501,6 +501,22 @@ const project: SubstrateProject = { handlers, }, }, + { + kind: SubstrateDatasourceKind.Runtime, + startBlock, + mapping: { + file: './dist/index.js', + handlers: [ + { + kind: SubstrateHandlerKind.Block, + handler: 'handleBlock', + // Only to flush the NftHolder write buffer; a coarse cadence keeps the per-block + // overhead negligible while bounding how stale a buffered holder can get. + filter: { modulo: 100 }, + }, + ], + }, + }, ], }; diff --git a/src/mappings/entities/assets/mapNfts.ts b/src/mappings/entities/assets/mapNfts.ts index 44d25627..b80b9e4e 100644 --- a/src/mappings/entities/assets/mapNfts.ts +++ b/src/mappings/entities/assets/mapNfts.ts @@ -14,6 +14,43 @@ import { import { extractArgs, getAsset } from './../common'; import { createAssetTransaction } from './mapAsset'; +/** + * `NftHolder.nftIds` is a JSON array, and under historical mode every `.save()` writes a new + * versioned row carrying the whole array. A bulk mint — hundreds of `NFTPortfolioUpdated` for one + * holder in one block — turned that into Σ(1..n) array serialisations and poisoned the store + * cache with 30k-element arrays. So holder mutations are buffered per block and each holder is + * saved once, when the block changes (or `flushNftBuffer` is called from the block handler). + * Nothing inside the indexer reads `NftHolder` — it is written for external queries only — so a + * holder being at most one block-handler interval stale is acceptable. + */ +let bufferedBlock: string | undefined; +const bufferedHolders = new Map(); + +export const flushNftBuffer = async (): Promise => { + if (bufferedHolders.size === 0) { + return; + } + + await Promise.all([...bufferedHolders.values()].map(holder => holder.save())); + bufferedHolders.clear(); + bufferedBlock = undefined; +}; + +/** Test hook. */ +export const __resetNftBuffer = (): void => { + bufferedHolders.clear(); + bufferedBlock = undefined; +}; + +const bufferHolder = async (blockId: string, holder: NftHolder): Promise => { + if (blockId !== bufferedBlock) { + await flushNftBuffer(); + bufferedBlock = blockId; + } + + bufferedHolders.set(holder.id, holder); +}; + export const getNftHolder = async ( assetId: string, did: string, @@ -21,6 +58,12 @@ export const getNftHolder = async ( ): Promise => { const id = `${assetId}/${did}`; + // A holder mutated earlier in this same block lives in the buffer, not yet in the store. + const buffered = bufferedBlock === blockId ? bufferedHolders.get(id) : undefined; + if (buffered) { + return buffered; + } + let nftHolder = await NftHolder.get(id); if (!nftHolder) { @@ -88,7 +131,8 @@ export const handleNftHoldingsUpdates = async (event: SubstrateEvent): Promise !ids.includes(heldId)); nftHolder.updatedBlockId = blockId; - promises.push(nftHolder.save()); + await bufferHolder(blockId, nftHolder); } else if (reason === 'transferred' || reason === 'controllerTransfer') { const [fromHolder, toHolder] = await Promise.all([ getNftHolder(assetId, fromDid, blockId), @@ -104,8 +148,11 @@ export const handleNftHoldingsUpdates = async (event: SubstrateEvent): Promise !ids.includes(id)); toHolder.nftIds.push(...ids); + fromHolder.updatedBlockId = blockId; + toHolder.updatedBlockId = blockId; - promises.push(fromHolder.save(), toHolder.save()); + await bufferHolder(blockId, fromHolder); + await bufferHolder(blockId, toHolder); asset.totalTransfers += BigInt(1); diff --git a/src/mappings/mappingHandlers.ts b/src/mappings/mappingHandlers.ts index 90ecfae0..2f9f7e7b 100644 --- a/src/mappings/mappingHandlers.ts +++ b/src/mappings/mappingHandlers.ts @@ -1,4 +1,4 @@ -import { SubstrateEvent } from '@subql/types'; +import { SubstrateBlock, SubstrateEvent } from '@subql/types'; import { logError } from '../utils'; import { getBlockContext } from './blockContext'; import { mapExternalAgentAction } from './entities'; @@ -7,6 +7,7 @@ import mapChainUpgrade from './entities/block/mapChainUpgrade'; import { handleExtrinsic } from './entities/block/mapExtrinsic'; import mapSubqueryVersion from './entities/block/mapSubqueryVersion'; import { handleToolingEvent } from './entities/events/mapEvent'; +import { flushNftBuffer } from './entities/assets/mapNfts'; import genesisHandler from './migrations/genesisHandler'; export async function handleGenesis(): Promise { @@ -22,6 +23,16 @@ export async function handleMigration(substrateEvent: SubstrateEvent): Promise logError(e)); } +/** + * Runs on a coarse block cadence (see `project.ts`). Its only job is to flush the per-block + * `NftHolder` write buffer so a bulk mint's last block is not left pending until the next NFT + * event — which, in a quiet period, could be a long way off. + */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export async function handleBlock(_block: SubstrateBlock): Promise { + await flushNftBuffer().catch(e => logError(e)); +} + export async function handleStartup(): Promise { /** * This handles the insertion of new SQ version on every restart. From abf9ec9d4f446ca44256498f833276e709714094 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:16:11 +0530 Subject: [PATCH 18/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20tolerate=20a=20mi?= =?UTF-8?q?ssing=20asset=20in=20handleAssetBalanceUpdated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getAsset threw a plain Error on a missing asset, which crashed the worker for a block whose AssetCreated was skipped upstream. getAssetOrAnomaly records a MissingReferencedEntity and returns undefined instead; handleAssetBalanceUpdated uses it and returns early. --- src/mappings/entities/assets/mapAsset.ts | 13 ++++++++++-- src/mappings/entities/common.ts | 26 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/mappings/entities/assets/mapAsset.ts b/src/mappings/entities/assets/mapAsset.ts index bbf5d80c..3ef83ce7 100644 --- a/src/mappings/entities/assets/mapAsset.ts +++ b/src/mappings/entities/assets/mapAsset.ts @@ -38,7 +38,7 @@ import { specVersionOf, } from '../../../utils'; import { processInstructionId } from '../settlements/mapSettlement'; -import { extractArgs, getAsset } from './../common'; +import { extractArgs, getAsset, getAssetOrAnomaly } from './../common'; export const createFunding = ( blockId: string, @@ -622,7 +622,16 @@ export const handleAssetBalanceUpdated = async (event: SubstrateEvent): Promise< } = decodeEvent(event); const assetId = await getAssetId(rawAssetId, block); - const asset = await getAsset(assetId); + const asset = await getAssetOrAnomaly(assetId, { + block, + eventIdx, + eventId: EventIdEnum.Transfer, + }); + + if (!asset) { + return; + } + const transferAmount = getBigIntValue(rawAmount); const promises: Promise[] = []; diff --git a/src/mappings/entities/common.ts b/src/mappings/entities/common.ts index 2b53b128..a667fba7 100644 --- a/src/mappings/entities/common.ts +++ b/src/mappings/entities/common.ts @@ -35,6 +35,32 @@ export const getAsset = async (assetId: string): Promise => { return asset; }; +/** + * `getAsset`, but records a `MissingReferencedEntity` anomaly and returns `undefined` instead of + * throwing. For handlers where a missing asset is a data gap to note, not a reason to fail the + * whole block — e.g. an `AssetBalanceUpdated` whose `AssetCreated` was skipped upstream. + */ +export const getAssetOrAnomaly = async ( + assetId: string, + context: { block: SubstrateBlock; eventIdx?: number; eventId?: EventIdEnum } +): Promise => { + const asset = await Asset.get(assetId); + + if (asset) { + return asset; + } + + await recordAnomaly({ + kind: AnomalyKind.MissingReferencedEntity, + detail: `Asset ${assetId} was not found — its creation event was not indexed`, + block: context.block, + eventIdx: context.eventIdx, + eventId: context.eventId, + }); + + return undefined; +}; + /** * Context that lets an unmapped chain value be recorded as an `IndexerAnomaly` instead of * silently becoming `Unknown`. Optional so a caller with no block in hand still type checks, From 8fd8b39058d0e78ea7479a11ef9d2afd34acd66e Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:16:17 +0530 Subject: [PATCH 19/22] docs: correct the verified claim that the v8 reward-destination path was right CHANGES.md, entity-review.md and 02-polyx-ledger.md each asserted [V] that get8xStakingEventDetails resolved the RewardDestination correctly. PR #350 showed it did not - the object form never matched. Corrected in the repo's usual style. --- docs/CHANGES.md | 4 +++- docs/entity-review.md | 2 +- docs/implementation/02-polyx-ledger.md | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/CHANGES.md b/docs/CHANGES.md index ce12febd..15222685 100644 --- a/docs/CHANGES.md +++ b/docs/CHANGES.md @@ -70,7 +70,9 @@ The chain's numeric instruction sequence is stored as a `String`, so `orderBy: [ ### 2.3 A15 — pre-v8 staking rewards are unattributable **[V]** -`mapStakingEvent.ts` records `rewardDestination: 'LegacyUnknown'` for every pre-8.x reward, because the event carried only the stash. Where a staker set a payee other than their stash, the index cannot say which account received the POLYX. The v8 path is correct. +`mapStakingEvent.ts` records `rewardDestination: 'LegacyUnknown'` for every pre-8.x reward, because the event carried only the stash. Where a staker set a payee other than their stash, the index cannot say which account received the POLYX. + +> **Correction (PR #350).** "The v8 path is correct" **[V]** was wrong — the v8 `getRewardDestinationDetails` matched the variant against `'Account'` while `Enum#toJSON()` camel-cases it, so the object form (`{ account: … }`, `{ staked: null }`) never resolved. Fixed and consolidated into `readRewardDestination`. `LegacyUnknown` is an honest placeholder rather than a wrong value — this is a **coverage** gap, not a correctness one. It matters because these rows are used for accounting, where "looks complete and is not" is the expensive failure. diff --git a/docs/entity-review.md b/docs/entity-review.md index cd4e184b..aabfaf9d 100644 --- a/docs/entity-review.md +++ b/docs/entity-review.md @@ -234,7 +234,7 @@ Three entities for one concept (agent membership + history) — merge candidate | Entity | Verdict | Notes | |---|---|---| | `PolyxTransaction` | ❌ | `BalanceTypeEnum` conflates pools, lock-floors and staking-ledger states; v8 `reserved` entirely unindexed. See `reference/polyx-balance-model.md`. | -| `StakingEvent` | ❌ | A log, not a position — and for pre-v8 rewards, an *incomplete* log. `rewardDestination` is `'LegacyUnknown'` for every pre-8.x `Reward`/`Rewarded` **[V]**, because the event carried only the stash. The v8 path correctly decodes the `RewardDestination` variant and resolves the account. **No `StakingPosition` / nomination entity** — current bonded amount, nominations, and validator prefs are not queryable. `staking` is **8/32 handled**; `Nominated` is handled but `Chilled`, `Kicked`, `PayoutStarted`, `EraPaid`, `ValidatorPrefsSet`, `StakersElected` are not. | +| `StakingEvent` | ❌ | A log, not a position — and for pre-v8 rewards, an *incomplete* log. `rewardDestination` is `'LegacyUnknown'` for every pre-8.x `Reward`/`Rewarded` **[V]**, because the event carried only the stash. The v8 path decodes the `RewardDestination` variant and resolves the account (a camel-case bug in it — the object form never matched — was found and fixed in PR #350). **No `StakingPosition` / nomination entity** — current bonded amount, nominations, and validator prefs are not queryable. `staking` is **8/32 handled**; `Nominated` is handled but `Chilled`, `Kicked`, `PayoutStarted`, `EraPaid`, `ValidatorPrefsSet`, `StakersElected` are not. | | `BridgeEvent` | ⚠️ | Only `Bridged` handled of 17 events. `BridgeTxScheduled`, `BridgeTxFailed`, `BridgeLimitUpdated`, `ControllerChanged`, `AdminChanged` unhandled — so bridge failures and configuration changes are invisible. Also hardcodes `/ 1_000_000` with integer division. | 🚫 **Missing: `StakingPosition`, `Nomination`, `Validator`, `Era`.** Staking is recorded purely as an event stream, so *"how much is this account staking right now, and with whom"* requires replaying all events. diff --git a/docs/implementation/02-polyx-ledger.md b/docs/implementation/02-polyx-ledger.md index 660c098e..09f12922 100644 --- a/docs/implementation/02-polyx-ledger.md +++ b/docs/implementation/02-polyx-ledger.md @@ -192,7 +192,9 @@ Because `api.query` targets the block being indexed **[V]** (and `.at` is unsupp [`mapStakingEvent.ts:110-130`](../../src/mappings/entities/events/mapStakingEvent.ts#L110) records `rewardDestination: 'LegacyUnknown'` for every pre-8.x `Reward`/`Rewarded`, because the event carries only the **stash** and the amount. Where a staker set a payee other than their stash — `Controller`, or an explicit `Account` — the index cannot say which account received the POLYX. Defect A15. -The v8 path is correct: `get8xStakingEventDetails` decodes the `RewardDestination` variant and resolves `rewardDestinationAccount` for `Account`, `Staked` and `Stash` **[V]**. +The v8 path decodes the `RewardDestination` variant and resolves `rewardDestinationAccount` for `Account`, `Staked` and `Stash`. + +> **Correction (PR #350).** This paragraph previously said the v8 path was correct **[V]**. It was not: `getRewardDestinationDetails` compared the variant key against `'Account'`, but `Enum#toJSON()` camel-cases it (`{ account: … }`), so the object form never matched — a v8 `Rewarded` with an explicit `Account` or an object-form `Staked` payee stored `rewardDestination` lower-cased with no account. Both paths now share `readRewardDestination` in `utils/staking.ts`, which normalises the bare-string and camel-cased-object forms. `LegacyUnknown` is an honest placeholder, not a wrong value — but it means a pre-v8 reward cannot be reconciled against the receiving account's balance, and it is invisible to anyone who does not know what the string means. From cda7239502f5cebe76508e82bb3ea246f8ba561f Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:23:46 +0530 Subject: [PATCH 20/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20clear=20the=20sta?= =?UTF-8?q?king=20lock=20at=20the=20v8=20lock=E2=86=92hold=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v5-v7 `Currency::set_lock("staking ", ...)` bond becomes a `RuntimeHoldReason::Staking` hold on v8. `pallet_balances` runs the conversion as a no-extrinsic two-pass storage migration: pass 1 emits `Upgraded` + `Held{Staking}`, pass 2 emits `Unlocked` for the lingering currency lock. `handleBalanceHeld` already turns pass 1 into the `free -> reserved` movement; the lock side was unhandled, so `frozen` over-reported for the ~420k blocks between the two passes. `handleBalanceUnlocked` now clears the account's `"staking "` lock on a v8 `Unlocked` when one is present - post-v8 nothing re-creates a currency staking lock, so it can only be the migration removing it. Pre-v8 and non-staking unlocks keep the generic `"balances"` path. Confirmed against real testnet v8 blocks that `staking.Bonded` / `Unbonded` / `Withdrawn` pair with `balances.Held` / `Released` in the same extrinsic (and `rebond` correctly emits neither), so the docs' [I] on that pairing becomes [V]. --- docs/implementation/02-polyx-ledger.md | 8 +-- docs/reference/polyx-balance-model.md | 6 +-- docs/reference/polyx-reconciliation.md | 27 ++++++++++ .../entities/identities/mapPolyxLedger.ts | 38 ++++++++++++- tests/unit/mapPolyxLedger.test.ts | 54 +++++++++++++++++++ 5 files changed, 125 insertions(+), 8 deletions(-) diff --git a/docs/implementation/02-polyx-ledger.md b/docs/implementation/02-polyx-ledger.md index 09f12922..e9fe4247 100644 --- a/docs/implementation/02-polyx-ledger.md +++ b/docs/implementation/02-polyx-ledger.md @@ -158,7 +158,7 @@ Verified emissions **[V]** (`pallets/balances/src/lib.rs` @ v7.4.0, `types-looku - **`BalanceSet`** — a **checkpoint**. Set `AccountBalance.free`/`reserved` absolutely; write one `BalanceSetAdjustment` entry recording the delta so the ledger still reconciles. Resolves A1 structurally. - **`Locked`/`Unlocked`/`Frozen`/`Thawed`** — update `AccountBalance.locks` only; recompute `frozen = MAX(active locks)`. No entry. - **`Issued`/`Rescinded`/`TotalIssuanceForced`/`MintedCredit`/`BurnedDebt`** — total-issuance only, no account side. **[I]** Consider a `TotalIssuance` entity; out of scope here. -- **`Upgraded`** — account flag migration, no amount. +- **`Upgraded`** — account flag / lock→hold migration marker, no amount of its own (the paired `Held` / `Unlocked` carry it). - **`TransferWithMemo`** — memo enrichment of the paired `Transfer`, never its own entry (resolves A2 option (b)). ### Staking — era-dependent, and inverted at v8 **[V]** @@ -166,7 +166,9 @@ Verified emissions **[V]** (`pallets/balances/src/lib.rs` @ v7.4.0, `types-looku - **≤ v7.4**: bonding is `set_lock(STAKING_ID, …)` — **no balance moves**. `Bonded`/`Unbonded`/`Withdrawn` update `locks` only. This is the correction to A6: the current `type: Bonded` rows assert movements that never happened. - **v8**: bonding is a Hold. The balance effect arrives via `balances.Held{reason:Staking}` / `Released`; the `staking.*` events become ledger state only. Recording both would double count. -**[I]** Confirm the `staking.Bonded` ↔ `balances.Held{reason:Staking}` pairing within one extrinsic against a real v8 block before relying on it. +**[V]** Pairing confirmed on testnet — `bondExtra` / `withdrawUnbonded` / `dest:Staked` payouts all emit the `Held` / `Released` in the same extrinsic (or `Initialization` phase) as the `staking.*` event; `rebond` correctly emits `Bonded` with no `Held`. See [`../reference/polyx-reconciliation.md`](../reference/polyx-reconciliation.md). + +- **v5–v7 → v8 lock→hold migration**: a no-extrinsic `pallet_balances` storage migration in two passes. Pass 1 emits `Upgraded` + `Held{Staking}` (→ `handleBalanceHeld` moves `free → reserved`); pass 2 emits `Unlocked` for the lingering `"staking "` lock. `handleBalanceUnlocked` clears that lock on a v8 `Unlocked` that covers it, so `frozen` tracks the chain across the ~400k-block window where the account carries both lock and hold. ### Genesis @@ -285,7 +287,7 @@ Register the eight missing v8 events (A9) and give the two empty ones handlers: - **Unit, per row of the transition table:** fixture → expected `(fromPool, toPool, kind, amount)`. - **Unit:** `Reserved` then `Unreserved` returns `free`/`reserved` to their starting values — the property the current model cannot satisfy. -- **Unit:** v7 `Bonded` produces **no** entry and raises `frozen`; v8 `Held{Staking}` produces `Free → Reserved`. +- **Unit:** v7 `Bonded` produces **no** entry and raises `frozen`; v8 `Held{Staking}` produces `Free → Reserved`; a v8 `Unlocked` covering the `"staking "` lock clears it (the lock→hold migration) while a smaller `Unlocked` does not. - **Unit:** `BalanceSet` sets absolutely and does not corrupt subsequent totals. - **Unit:** two overlapping locks of 100 and 150 give `frozen = 150`, not 250. - **Integration:** after resync, `SUM(amount) WHERE toPool=X` minus `SUM WHERE fromPool=X` equals `AccountBalance` for a sample of accounts. diff --git a/docs/reference/polyx-balance-model.md b/docs/reference/polyx-balance-model.md index fb9218f8..3c0da479 100644 --- a/docs/reference/polyx-balance-model.md +++ b/docs/reference/polyx-balance-model.md @@ -305,8 +305,8 @@ This is the correction to audit A6: pre-v8 these must not produce `PolyxMovement | `DustLost{account,amount}` | `account/Free` | ∅ | DustLost | | `BalanceSet{who,free}` | — | — | checkpoint | | `Issued` / `Rescinded` / `TotalIssuanceForced` / `MintedCredit` / `BurnedDebt` | — | — | **total-issuance only, no account side** — track on a separate `TotalIssuance` entity, not the account ledger | -| `Locked` / `Unlocked` / `Frozen` / `Thawed` | — | — | `BalanceLock` only, no movement | -| `Upgraded{who}` | — | — | no amount; account flag migration only | +| `Locked` / `Unlocked` / `Frozen` / `Thawed` | — | — | `BalanceLock` only, no movement. A v8 `Unlocked` covering the `"staking "` lock is the lock→hold migration — it clears that lock, not the generic one | +| `Upgraded{who}` | — | — | no amount; account flag / lock→hold migration marker | ### 4.4 v8 staking @@ -314,7 +314,7 @@ This is the correction to audit A6: pre-v8 these must not produce `PolyxMovement This resolves the A6 `Withdrawn` bug structurally: it is no longer a credit to a fictional `Unbonded` pool, it is a staking-ledger transition whose balance effect is the paired `Released`. -**[I]** The exact pairing of `staking.Bonded` with `balances.Held{reason:Staking}` within one extrinsic should be confirmed against a real v8 block before relying on it for backfill. +**[V]** Pairing confirmed against real testnet v8 blocks (`bondExtra`, `withdrawUnbonded`, `dest:Staked` payouts all carry the paired `Held`/`Released`; `rebond` carries none, correctly). The v5–v7 `set_lock("staking ")` → v8 `Staking` hold conversion is a no-extrinsic two-pass `pallet_balances` storage migration: pass 1 emits `Upgraded` + `Held{Staking}`, pass 2 emits `Unlocked` for the lock. `handleBalanceHeld` covers the hold side unchanged; `handleBalanceUnlocked` clears the `"staking "` lock on the migration `Unlocked`. See [`polyx-reconciliation.md`](./polyx-reconciliation.md). --- diff --git a/docs/reference/polyx-reconciliation.md b/docs/reference/polyx-reconciliation.md index 0bbce924..cea9cbb6 100644 --- a/docs/reference/polyx-reconciliation.md +++ b/docs/reference/polyx-reconciliation.md @@ -29,6 +29,33 @@ correct, not an under-count. Pre-v8 (≤ v7.4) bonding is `set_lock(STAKING_ID, …)` and moves no balance: `Bonded`/`Withdrawn` maintain `AccountBalance.locks` only, `frozen = MAX(active locks)`, still no `PolyxEntry`. +### The v5–v7 lock → v8 hold storage migration + +`pallet_balances` converts every pre-v8 `set_lock("staking ", …)` into a `RuntimeHoldReason::Staking` +hold. It runs in **two passes**, each emitting events per account but in no extrinsic of the +staker's own: + +| Pass | Block (testnet) | Phase | Events per migrated account | +|---|---|---|---| +| 1 | 24,733,771 | `Initialization` | `Upgraded{who}`, `Held{Staking, ledger.total}` | +| 2 | 25,152,593 | a permissionless `balances` extrinsic | `Upgraded{who}` (if new), `Unlocked{who, lock}`, `Held{Staking, …}` (if new) | + +Between the two passes the account carries **both** the old `"staking "` lock and the new hold +on-chain — `frozen` stays at the lock amount until pass 2's `Unlocked` drops it. Probed on +`5C7kNpSv…`: + +``` +b24730485 spec7004001 free 5938e9 reserved 0 frozen 5103e9 lock 5103e9 hold — +b24733771 spec8000000 free 835e9 reserved 5103e9 frozen 5103e9 lock 5103e9 hold 5103e9 (pass 1: Held, lock kept) +b25152593 spec8000020 free 835e9 reserved 5207e9 frozen 0 lock — hold 5207e9 (pass 2: Unlocked) +``` + +The indexer needs no special handling for the hold side — `handleBalanceHeld` turns pass 1's +`Held` into the `free → reserved` movement exactly as for any other hold. The lock side is the +one addition: `handleBalanceUnlocked` treats a v8 `Unlocked` that covers the account's `"staking "` +lock as this migration and clears that lock (rather than the generic `"balances"` one), so +`frozen` tracks the chain across the two-pass window. + --- ## Measured — defect A15, the pre-v8 reward-destination gap diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts index ee3b8c73..4089c5dd 100644 --- a/src/mappings/entities/identities/mapPolyxLedger.ts +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -569,9 +569,43 @@ const lockHandler = await adjustLock(who, lockId, sign * amountOf(decoded), blockId); }; -/** `balances.Locked` / `Unlocked` — the `LockableCurrency` floor on `free`. */ +/** + * `balances.Locked` / `Unlocked` — the `LockableCurrency` floor on `free`. + * + * v8 adds one more source of `Unlocked`: the lock → hold storage migration. Polymesh holds a + * pre-v8 bond as `Currency::set_lock("staking ", …)`, and v8 converts it to a `Staking` hold, + * emitting per account — across two migration passes, in no extrinsic of the staker's own — + * `balances.Upgraded`, the paired `balances.Held{Staking}` (which `handleBalanceHeld` already + * turns into the `free → reserved` movement), and `balances.Unlocked{who, amount}` for the lock + * removal. Only the lock side is left to do here. Post-v8 nothing re-creates a `'staking '` + * `Currency` lock (new bonds are holds), so an account that still carries one on a v8 block has + * an un-migrated pre-v8 lock and the `Unlocked` it sees is that migration removing it — clear the + * `'staking '` lock rather than the generic `'balances'` one. Without this the pre-v8 staking lock + * lingers and `frozen` over-reports for the ~420k blocks between the hold appearing (pass 1) and + * the chain dropping the lock (pass 2). + */ export const handleBalanceLocked = lockHandler('balances', BigInt(1)); -export const handleBalanceUnlocked = lockHandler('balances', BigInt(-1)); + +const unlockGeneric = lockHandler('balances', BigInt(-1)); + +export const handleBalanceUnlocked = async (event: SubstrateEvent): Promise => { + const args = extractArgs(event); + + if (is8xChain(args.block)) { + const who = holder(decodeEvent(event)); + const hasStakingLock = + !!who && + ((await AccountBalance.get(who))?.locks ?? []).some(lock => lock.lockId === STAKING_LOCK_ID); + + if (who && hasStakingLock) { + await setLock(who, STAKING_LOCK_ID, BigInt(0), args.blockId, 'staking'); + return; + } + } + + await unlockGeneric(event); +}; + /** `balances.Frozen` / `Thawed` — the upstream `fungible` freeze, also a floor on `free`. */ export const handleBalanceFrozen = lockHandler('freeze', BigInt(1)); export const handleBalanceThawed = lockHandler('freeze', BigInt(-1)); diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts index 6c13fe71..4eab037d 100644 --- a/tests/unit/mapPolyxLedger.test.ts +++ b/tests/unit/mapPolyxLedger.test.ts @@ -22,6 +22,7 @@ import { handleBalanceSuspended, handleBalanceThawed, handleBalanceTransfer, + handleBalanceUnlocked, handleBalanceUnreserved, handleBonded, handlePayoutStarted, @@ -474,6 +475,59 @@ describe('staking — era-dependent, inverted at v8 (A10 / A6)', () => { }); }); + describe('the v5–v7 lock → v8 hold storage migration', () => { + it('a v8 Held{Staking} moves the bonded amount free → reserved, lock still standing', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await adjustLock(ALICE, 'staking ', BigInt(4000), '0000001000000', 'staking'); + + // pass 1 of the migration — Held with no paired Deposit + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '4000' }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(6000), + reserved: BigInt(4000), + frozen: BigInt(4000), // the "staking " lock is deliberately kept — chain frozen is still 4000 + bonded: BigInt(4000), + }); + }); + + it('a v8 Unlocked that covers the staking lock clears it (pass 2)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await adjustLock(ALICE, 'staking ', BigInt(4000), '0000001000000', 'staking'); + await handleBalanceHeld( + balancesEvent('Held', { reason: 'Staking', who: ALICE, amount: '4000' }) + ); + + await handleBalanceUnlocked( + balancesEvent('Unlocked', { who: ALICE, amount: '4000' }, { specVersion: 8_000_000 }) + ); + + expect(balance(ALICE)).toMatchObject({ + free: BigInt(6000), + reserved: BigInt(4000), + frozen: BigInt(0), // lock gone + bonded: BigInt(4000), // still bonded, now via the hold + transferable: BigInt(6000), + }); + expect(balance(ALICE)?.locks ?? []).toHaveLength(0); + }); + + it('a pre-v8 Unlocked leaves the staking lock alone (generic "balances" lock only)', async () => { + await handleBalanceMinted(balancesEvent('Minted', { who: ALICE, amount: '10000' })); + await adjustLock(ALICE, 'staking ', BigInt(4000), '0000001000000', 'staking'); + + await handleBalanceUnlocked( + balancesEvent('Unlocked', { who: ALICE, amount: '10' }, { specVersion: 7_004_001 }) + ); + + expect(balance(ALICE)?.locks?.find((l: any) => l.lockId === 'staking ')?.amount).toBe( + BigInt(4000) + ); + }); + }); + it('pre-v8 Reward resolves an explicit Account payee — credited to that account, not the stash', async () => { const PAYEE = '5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy'; (globalThis as any).api.query = { From 1ebd957e63cb1444310f018ac6b72a5860c0feb5 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:26:17 +0530 Subject: [PATCH 21/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20debit=20the=20tre?= =?UTF-8?q?asury=20pallet=20account=20on=20a=20disbursement,=20not=20the?= =?UTF-8?q?=20committee?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TreasuryDisbursement`'s first parameter is the identity that authorised the spend, not the source of funds — `treasury.disbursement` always moves POLYX out of the treasury pallet account, the mirror of the reimbursement handler's credit. Debiting the authorising committee's primary key instead left the treasury drifting high forever and the committee account low, on every spec version that emits no paired `balances.Transfer` (pre-5.0.0). The paired-transfer relabel now covers both sides of the movement too. --- .../entities/identities/mapPolyxLedger.ts | 34 +++++++++++++------ tests/unit/mapPolyxLedger.test.ts | 30 ++++++++++++++++ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/mappings/entities/identities/mapPolyxLedger.ts b/src/mappings/entities/identities/mapPolyxLedger.ts index 4089c5dd..99ea186d 100644 --- a/src/mappings/entities/identities/mapPolyxLedger.ts +++ b/src/mappings/entities/identities/mapPolyxLedger.ts @@ -1144,14 +1144,22 @@ export const handleBalanceSet = async (event: SubstrateEvent): Promise => const identityPrimaryAccount = async (did: string | undefined): Promise => did ? (await Identity.get(did))?.primaryAccount : undefined; +const treasuryPalletAccount = (): string => + getAccountId(systematicIssuers.treasury.accountId, api.registry.chainSS58); + export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise => { const args = extractArgs(event); - const [rawFromDid, rawToDid, rawTo, rawBalance] = args.params; - + const [, rawToDid, rawTo, rawBalance] = args.params; + + // `TreasuryDisbursement(authorizingDid, targetDid, targetAccount?, amount)`. The first param is + // the committee that authorised the spend, **not** the source of funds — `treasury.disbursement` + // always moves POLYX out of the treasury pallet account (the mirror of the reimbursement + // handler's credit). Debiting `authorizingDid`'s primary key instead left the treasury drifting + // high forever and the committee account low (defect: the block-352,843 disbursements). + // // (IdentityId, IdentityId, AccountId, Balance) from 5.0.0; (IdentityId, IdentityId, Balance) before const hasToAddress = args.params.length >= 4; const amount = getBigIntValue(hasToAddress ? rawBalance : rawTo); - const fromAddress = await identityPrimaryAccount(getTextValue(rawFromDid)); const toAddress = (hasToAddress ? getTextValue(rawTo) : undefined) ?? (await identityPrimaryAccount(getTextValue(rawToDid))); @@ -1164,25 +1172,29 @@ export const handleTreasuryDisbursement = async (event: SubstrateEvent): Promise ); if (existingTransfer) { - // `treasury.disbursement` to an identity emits both `Transfer` and `TreasuryDisbursement`; - // relabel the transfer rather than double-counting. - existingTransfer.kind = MovementKind.TreasuryDisbursement; - await existingTransfer.save(); + // From 5.0.0 `treasury.disbursement` also emits `balances.Transfer{treasury → recipient}`; + // relabel both sides of it rather than writing a second movement. + const siblings = await PolyxEntry.getByFields( + [['movementId', '=', existingTransfer.movementId]], + { limit: 10 } + ); + + for (const sibling of siblings) { + sibling.kind = MovementKind.TreasuryDisbursement; + await sibling.save(); + } return; } await postTransition(args, { - from: fromAddress ? { address: fromAddress, pool: PolyxPool.Free } : undefined, + from: { address: treasuryPalletAccount(), pool: PolyxPool.Free }, to: toAddress ? { address: toAddress, pool: PolyxPool.Free } : undefined, amount, kind: MovementKind.TreasuryDisbursement, }); }; -const treasuryPalletAccount = (): string => - getAccountId(systematicIssuers.treasury.accountId, api.registry.chainSS58); - export const handleTreasuryReimbursement = async (event: SubstrateEvent): Promise => { const args = extractArgs(event); const [, rawBalance] = args.params; diff --git a/tests/unit/mapPolyxLedger.test.ts b/tests/unit/mapPolyxLedger.test.ts index 4eab037d..eeefe25c 100644 --- a/tests/unit/mapPolyxLedger.test.ts +++ b/tests/unit/mapPolyxLedger.test.ts @@ -30,6 +30,7 @@ import { handleWithdrawn, handleDustLost, handleReserveRepatriated, + handleTreasuryDisbursement, handleTreasuryReimbursement, } from '../../src/mappings/entities/identities/mapPolyxLedger'; import { getAccountId, systematicIssuers } from '../../src/mappings/consts'; @@ -324,6 +325,35 @@ describe('Event → pool transition', () => { expect(entries()[0].accountId).not.toBe(payerDid); expect(balance(treasury)?.free).toBe(BigInt(400)); }); + + it('TreasuryDisbursement debits the treasury pallet account, not the authorising committee', async () => { + const committeeDid = '0x73797374656d3a676f7665726e616e63655f636f6d6d69747465650000000000'; + const recipientDid = '0x8015a1702789fedf8474a042af07ba6a37f94e8d24b4eed89414e6eb79df084e'; + const treasury = getAccountId(systematicIssuers.treasury.accountId, 42); + + // pre-5.0.0 shape carries no recipient account and no paired balances.Transfer + await handleTreasuryDisbursement( + tupleEvent( + 'treasury', + 'TreasuryDisbursement', + [committeeDid, recipientDid, BOB, '4014'], + 3010 + ) + ); + + const debit = entries().find(r => r.direction === EntryDirection.Debit); + const credit = entries().find(r => r.direction === EntryDirection.Credit); + + expect(debit).toMatchObject({ + accountId: treasury, + kind: MovementKind.TreasuryDisbursement, + amount: BigInt(-4014), + }); + expect(credit?.accountId).toBe(BOB); + expect(entries().some(r => r.accountId === committeeDid)).toBe(false); + expect(balance(treasury)?.free).toBe(BigInt(-4014)); + expect(balance(BOB)?.free).toBe(BigInt(4014)); + }); }); describe('properties the one-column model could not satisfy', () => { From d9f7b0d12f47b27b05c36c2e6337d3e41e9fcdc4 Mon Sep 17 00:00:00 2001 From: Prashant Bajpai <34747455+prashantasdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:26:26 +0530 Subject: [PATCH 22/22] =?UTF-8?q?chore:=20=F0=9F=A4=96=20identify=20a=20mi?= =?UTF-8?q?grated=20asset=20by=20byte=20length,=20not=20the=20block's=20sp?= =?UTF-8?q?ec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `getAssetId` chose between a 12-byte ticker and a 16-byte asset ID on `is7xChain(block)`. That is wrong whenever the block misreports its runtime: `@subql/node` has been seen serving the pre-upgrade spec for a long run of blocks after v7.0.0 actually activated (~169k blocks reported as spec 6003050 instead of 7000003), and an already-migrated 16-byte asset ID in one of those blocks was then run through `getAssetIdForLegacyTicker` and blake2-hashed into a bogus id, so its `Asset` lookup missed. A `Ticker` is `[u8; 12]`, so a 16-byte value is a migrated asset ID whatever spec the block claims. `isMigratedAssetId` tests that, and `getAssetId` / `getAssetIdWithTicker` / the `handleAssetCreated` ticker derivation / `getAssetIdForStatisticsEvent` all check it before falling back to the spec gate. --- src/mappings/entities/assets/mapAsset.ts | 4 +- src/mappings/entities/assets/mapStatistics.ts | 3 +- src/utils/assets.ts | 22 ++++++- tests/unit/assetId.test.ts | 66 +++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 tests/unit/assetId.test.ts diff --git a/src/mappings/entities/assets/mapAsset.ts b/src/mappings/entities/assets/mapAsset.ts index 3ef83ce7..3de0c71e 100644 --- a/src/mappings/entities/assets/mapAsset.ts +++ b/src/mappings/entities/assets/mapAsset.ts @@ -33,6 +33,7 @@ import { getStringArrayValue, getTextValue, is7xChain, + isMigratedAssetId, rawAssetHolderToAssetHolder, serializeTicker, specVersionOf, @@ -181,7 +182,8 @@ export const handleAssetCreated = async (event: SubstrateEvent): Promise = const ownerId = getTextValue(rawOwnerDid); - const ticker = is7xChain(block) ? undefined : serializeTicker(rawAssetId); + const ticker = + is7xChain(block) || isMigratedAssetId(rawAssetId) ? undefined : serializeTicker(rawAssetId); /** * Name isn't present on the old events so we need to query storage. diff --git a/src/mappings/entities/assets/mapStatistics.ts b/src/mappings/entities/assets/mapStatistics.ts index f2f2613f..ff0ecb19 100644 --- a/src/mappings/entities/assets/mapStatistics.ts +++ b/src/mappings/entities/assets/mapStatistics.ts @@ -19,6 +19,7 @@ import { getAllByFields, getTransferManagerValue, is7xChain, + isMigratedAssetId, } from '../../../utils'; import { Attributes, extractArgs } from '../common'; @@ -28,7 +29,7 @@ export const getAssetIdForStatisticsEvent = ( ): Promise => { let assetId: string; - if (is7xChain(block)) { + if (isMigratedAssetId(item) || is7xChain(block)) { assetId = item.toString(); } else { const scope = JSON.parse(item.toString()); diff --git a/src/utils/assets.ts b/src/utils/assets.ts index fff78dd3..0048a62a 100644 --- a/src/utils/assets.ts +++ b/src/utils/assets.ts @@ -112,11 +112,29 @@ export const getAssetIdForLegacyTicker = async (ticker: Codec | string): Promise return u8aToHex(rawBytes); }; +/** + * Whether a raw asset identifier is already a migrated 16-byte asset ID, rather than a legacy + * ticker. + * + * The public chain switched `asset` events from carrying a 12-byte `Ticker` to a 16-byte + * `PolymeshPrimitivesAssetAssetId` at v7.0.0. The spec-version gate below would be enough if the + * block always reported its true runtime — but `@subql/node` has been seen serving the + * *pre-upgrade* spec for a long run of blocks after v7.0.0 actually activated (testnet: ~169k + * blocks reported as spec 6003050 instead of 7000003, block 15,978,579 onward). A byte-length + * test is immune to that: a `Ticker` is `[u8; 12]`, so any `0x`-prefixed value that decodes to + * 16 bytes is unambiguously a migrated asset ID whatever spec the block claims. + */ +export const isMigratedAssetId = (value: string | Codec): boolean => { + const hex = typeof value === 'string' ? value : value.toString(); + + return hexHasPrefix(hex) && hexStripPrefix(hex).length === 32; +}; + export const getAssetId = async ( assetId: string | Codec, block: SubstrateBlock ): Promise => { - if (is7xChain(block)) { + if (isMigratedAssetId(assetId) || is7xChain(block)) { return typeof assetId === 'string' ? assetId : assetId.toString(); } @@ -138,7 +156,7 @@ export const getAssetIdWithTicker = async ( ): Promise => { let assetId: string; let ticker: string; - if (is7xChain(block)) { + if (isMigratedAssetId(assetIdOrTicker) || is7xChain(block)) { assetId = typeof assetIdOrTicker === 'string' ? assetIdOrTicker : assetIdOrTicker.toString(); const asset = await Asset.get(assetId); diff --git a/tests/unit/assetId.test.ts b/tests/unit/assetId.test.ts new file mode 100644 index 00000000..8da9b132 --- /dev/null +++ b/tests/unit/assetId.test.ts @@ -0,0 +1,66 @@ +/** + * `getAssetId` maps a raw `asset` event identifier to the id the `Asset` row is keyed on. + * + * The public chain switched `asset` events from a 12-byte `Ticker` to a 16-byte + * `PolymeshPrimitivesAssetAssetId` at v7.0.0. The switch was gated on the block's spec version — + * but `@subql/node` was seen serving the pre-upgrade spec (6003050) for ~169k blocks after + * v7.0.0 actually activated on testnet (block 15,978,579 onward), so an already-migrated + * 16-byte asset id in one of those blocks was run through `getAssetIdForLegacyTicker` and + * blake2-hashed into a bogus id — its `Asset` lookup then missed (`MissingReferencedEntity`). + * + * The fix disambiguates by byte length: a `Ticker` is `[u8; 12]`, so a 16-byte value is a + * migrated asset id whatever spec the block claims. + */ + +import { SubstrateBlock } from '@subql/types'; +import { getAssetId, getAssetIdForLegacyTicker, isMigratedAssetId } from '../../src/utils/assets'; + +const globalAny = globalThis as any; + +const block = (specVersion: number): SubstrateBlock => + ({ specVersion } as unknown as SubstrateBlock); + +/** A 16-byte migrated asset id (ticker "PSRF" as it exists on testnet from v7.0.0). */ +const ASSET_ID = '0x8f68f310c5ea8f27a189154812efd457'; +/** "PSRF" as a 12-byte ticker, hex (0x + 24 chars). */ +const TICKER_HEX = '0x505352460000000000000000'; + +beforeEach(() => { + globalAny.chainId = '0xnotstaging'; + globalAny.api.runtimeVersion.specName = { toString: () => 'polymesh' }; +}); + +describe('isMigratedAssetId', () => { + it('is true only for a 16-byte hex value', () => { + expect(isMigratedAssetId(ASSET_ID)).toBe(true); + expect(isMigratedAssetId({ toString: () => ASSET_ID } as any)).toBe(true); + }); + + it('is false for a 12-byte ticker, a short hex, or a plain string', () => { + expect(isMigratedAssetId(TICKER_HEX)).toBe(false); + expect(isMigratedAssetId('0xdead')).toBe(false); + expect(isMigratedAssetId('PSRF')).toBe(false); + }); +}); + +describe('getAssetId', () => { + it('returns a 16-byte asset id unchanged on a 7.x block', async () => { + expect(await getAssetId(ASSET_ID, block(7_000_003))).toBe(ASSET_ID); + }); + + it('hashes a legacy ticker on a pre-7.x block', async () => { + // "PSRF" was created pre-v7 as a ticker; the chain migration derives its asset id + // deterministically, and `getAssetIdForLegacyTicker` reproduces that — it is ASSET_ID. + expect(await getAssetId('PSRF', block(6_003_040))).toBe( + await getAssetIdForLegacyTicker('PSRF') + ); + expect(await getAssetId('PSRF', block(6_003_040))).toBe(ASSET_ID); + }); + + it('returns a 16-byte asset id unchanged even when the block reports a stale pre-7.x spec', async () => { + // the regression: v7.0.0 was live and the event already carried the 16-byte asset id, but + // @subql/node still reported spec 6003050 — the old code re-hashed ASSET_ID into a bogus id + expect(await getAssetId(ASSET_ID, block(6_003_050))).toBe(ASSET_ID); + expect(await getAssetId({ toString: () => ASSET_ID } as any, block(6_003_050))).toBe(ASSET_ID); + }); +});