diff --git a/crates/execution-e2e/src/setup.rs b/crates/execution-e2e/src/setup.rs index d3c9bd1..109b675 100644 --- a/crates/execution-e2e/src/setup.rs +++ b/crates/execution-e2e/src/setup.rs @@ -40,6 +40,7 @@ pub struct ArcSetup { addresses_denylist_config: Option, invalid_tx_list_config: Option, rpc_gas_cap: Option, + legacy_state_root_task_enabled: bool, } impl Default for ArcSetup { @@ -58,6 +59,7 @@ impl ArcSetup { addresses_denylist_config: None, invalid_tx_list_config: None, rpc_gas_cap: None, + legacy_state_root_task_enabled: false, } } @@ -93,6 +95,16 @@ impl ArcSetup { self } + /// Uses Reth's synchronous state-root path. + /// + /// In-process tests that repeatedly request state proofs can use this to + /// avoid nested proof-worker pools. The default remains the production + /// asynchronous path. + pub fn with_legacy_state_root_task_enabled(mut self) -> Self { + self.legacy_state_root_task_enabled = true; + self + } + /// Applies the setup to create the test environment. /// /// This creates a single Arc node and initializes the environment with @@ -106,8 +118,13 @@ impl ArcSetup { arc_node.invalid_tx_list_cfg = cfg; } - let (node, wallet, genesis_block) = - Self::launch_node(self.chain_spec, arc_node, self.rpc_gas_cap).await?; + let (node, wallet, genesis_block) = Self::launch_node( + self.chain_spec, + arc_node, + self.rpc_gas_cap, + self.legacy_state_root_task_enabled, + ) + .await?; env.set_node(node); env.set_wallet(wallet); @@ -120,6 +137,7 @@ impl ArcSetup { chain_spec: Arc, arc_node: ArcNode, rpc_gas_cap: Option, + legacy_state_root_task_enabled: bool, ) -> eyre::Result<( NodeHelperType, reth_e2e_test_utils::wallet::Wallet, @@ -137,8 +155,9 @@ impl ArcSetup { }, ..NetworkArgs::default() }; - let tree_config = - reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024); + let tree_config = reth_node_api::TreeConfig::default() + .with_cross_block_cache_size(1024 * 1024) + .with_legacy_state_root(legacy_state_root_task_enabled); let mut rpc_args = RpcServerArgs::default() .with_unused_ports() .with_http() @@ -146,10 +165,11 @@ impl ArcSetup { if let Some(cap) = rpc_gas_cap { rpc_args.rpc_gas_cap = cap; } - let node_config = NodeConfig::new(chain_spec.clone()) + let mut node_config = NodeConfig::new(chain_spec.clone()) .with_network(network_config) .with_unused_ports() .with_rpc(rpc_args); + node_config.engine.legacy_state_root_task_enabled = legacy_state_root_task_enabled; let NodeHandle { node, diff --git a/crates/execution-e2e/tests/export_leafage_a1b.rs b/crates/execution-e2e/tests/export_leafage_a1b.rs new file mode 100644 index 0000000..363ec5d --- /dev/null +++ b/crates/execution-e2e/tests/export_leafage_a1b.rs @@ -0,0 +1,716 @@ +// Copyright 2026 Circle Internet Group, Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The ignored test launches the Arc v0.7.3 writer and captures its real +//! `trace_debankBlock` payloads. It deliberately writes only producer output; +//! the companion Python script applies the background-tracer object encoding. + +mod helpers; + +use alloy_consensus::constants::{EMPTY_ROOT_HASH, KECCAK_EMPTY}; +use alloy_eips::{eip2935::HISTORY_STORAGE_ADDRESS, eip7685::RequestsOrHash, BlockId}; +use alloy_primitives::{address, keccak256, Address, B256, U256}; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; +use arc_execution_e2e::{ + actions::{assert_valid_or_syncing, SendTransaction}, + Action, ArcEnvironment, ArcSetup, ArcTestBuilder, BlockInfo, +}; +use arc_precompiles::system_accounting::SYSTEM_ACCOUNTING_ADDRESS; +use debank_rpc::debank_trace::{BlockStorageDiff, DebankOutPut}; +use eyre::{ensure, Context, Result}; +use futures_util::future::BoxFuture; +use helpers::{ + constants::{SYSTEM_ADDRESS, WALLET_FIRST_ADDRESS}, + contracts::{ + create2_balance_probe_calldata, create2_with_balance_probe, reverting_constructor_code, + }, + debank::{assert_eip7708_transfer, current_mainnet_test_chain_spec, transaction_view}, +}; +use jsonrpsee::{core::client::ClientT, rpc_params}; +use reth_chainspec::EthChainSpec; +use reth_ethereum::node::EthEngineTypes; +use reth_rpc_api::clients::EngineApiClient; +use std::{ + fs::{self, OpenOptions}, + io::Write, + path::{Path, PathBuf}, +}; + +const OUTPUT_ENV: &str = "ARC_A1B_CAPTURE_DIR"; +const FEE_RECIPIENT: Address = address!("0x65E0a200006D4FF91bD59F9694220dafc49dbBC1"); + +fn assert_storage_contracts( + output: &DebankOutPut, + state_diff: &BlockStorageDiff, + expected_contracts: &[Address], +) -> Result<()> { + ensure!( + output.block_file.storage_contracts.as_slice() == expected_contracts, + "block {}: unexpected storage contracts", + output.header.number + ); + + let mut expected_hashes: Vec<_> = expected_contracts + .iter() + .map(|address| keccak256(address.as_slice())) + .collect(); + expected_hashes.sort_unstable(); + let storage_hashes: Vec<_> = state_diff + .storage_diffs + .iter() + .map(|diff| diff.address) + .collect(); + ensure!( + storage_hashes == expected_hashes, + "block {}: StateDiff storage account set mismatch", + output.header.number + ); + Ok(()) +} + +fn assert_system_hook_state(output: &DebankOutPut, state_diff: &BlockStorageDiff) -> Result<()> { + assert_storage_contracts( + output, + state_diff, + &[HISTORY_STORAGE_ADDRESS, SYSTEM_ACCOUNTING_ADDRESS], + )?; + ensure!( + state_diff + .storage_diffs + .iter() + .all(|diff| diff.diffs.len() == 1), + "block {}: each Arc hook must write exactly one storage slot", + output.header.number + ); + Ok(()) +} + +fn assert_genesis(output: &DebankOutPut, state_diff: &BlockStorageDiff) -> Result<()> { + ensure!( + !output.block_file.transactions.is_empty() + && output.block_file.transactions.len() == output.block_file.traces.len(), + "genesis alloc was not represented as one root trace per synthetic transaction" + ); + ensure!( + output.block_file.error_traces.is_empty() + && output.block_file.events.is_empty() + && output.block_file.error_events.is_empty(), + "genesis alloc unexpectedly contains events or failed traces" + ); + ensure!( + !state_diff.new_accounts.is_empty() + && !state_diff.new_codes.is_empty() + && !state_diff.storage_diffs.is_empty(), + "genesis StateDiff is not a full alloc capture" + ); + Ok(()) +} + +fn assert_empty_hooks(output: &DebankOutPut, state_diff: &BlockStorageDiff) -> Result<()> { + ensure!( + output.block_file.transactions.is_empty() + && output.block_file.traces.is_empty() + && output.block_file.error_traces.is_empty() + && output.block_file.events.is_empty() + && output.block_file.error_events.is_empty(), + "empty-hooks scenario contains transaction data" + ); + ensure!( + state_diff.new_codes.is_empty(), + "empty-hooks scenario contains code" + ); + let mut expected_accounts = vec![ + keccak256(HISTORY_STORAGE_ADDRESS.as_slice()), + keccak256(SYSTEM_ACCOUNTING_ADDRESS.as_slice()), + ]; + expected_accounts.sort_unstable(); + let actual_accounts: Vec<_> = state_diff + .new_accounts + .iter() + .map(|account| account.address) + .collect(); + ensure!( + actual_accounts == expected_accounts, + "empty-hooks scenario contains non-hook account changes" + ); + assert_system_hook_state(output, state_diff) +} + +fn assert_native_transfer(output: &DebankOutPut, state_diff: &BlockStorageDiff) -> Result<()> { + let recipient = Address::repeat_byte(0xa5); + ensure!( + output.block_file.transactions.len() == 1, + "native-transfer scenario must contain one transaction" + ); + let transaction = &output.block_file.transactions[0]; + ensure!( + transaction.status + && transaction.nonce == 0 + && transaction.from == WALLET_FIRST_ADDRESS + && transaction.to == recipient + && transaction.value == U256::from(1), + "native-transfer transaction does not match the intended scenario" + ); + let view = transaction_view(output, transaction); + ensure!( + view.traces.len() == 1 + && view.error_traces.is_empty() + && view.events.len() == 1 + && view.error_events.is_empty(), + "native-transfer scenario must contain one successful root trace and event" + ); + let root = view.traces[0]; + ensure!( + root.parent_trace_id.is_empty() + && root.trace_address.is_empty() + && root.call_create_type == "call" + && root.call_type == "call" + && root.from_addr == WALLET_FIRST_ADDRESS + && root.to_addr == recipient + && root.value == U256::from(1) + && root.subtraces == 0 + && root.error.is_empty(), + "native-transfer root CALL trace mismatch" + ); + assert_eip7708_transfer( + view.events[0], + WALLET_FIRST_ADDRESS, + recipient, + U256::from(1), + &root.id, + 0, + 0, + )?; + ensure!( + view.events[0].contract_id == SYSTEM_ADDRESS, + "native-transfer event is not emitted by SYSTEM_ADDRESS" + ); + let recipient_account = state_diff + .new_accounts + .iter() + .find(|account| account.address == keccak256(recipient.as_slice())) + .ok_or_else(|| eyre::eyre!("native-transfer recipient is missing from StateDiff"))?; + ensure!( + recipient_account.balance == U256::from(1) + && recipient_account.nonce == 0 + && recipient_account.code_hash == KECCAK_EMPTY, + "native-transfer recipient state mismatch" + ); + assert_system_hook_state(output, state_diff) +} + +fn assert_create2(output: &DebankOutPut, state_diff: &BlockStorageDiff) -> Result<()> { + let probe = WALLET_FIRST_ADDRESS.create(1); + let child = probe.create2([0_u8; 32], keccak256([0_u8])); + ensure!( + output.block_file.transactions.len() == 2, + "create2 scenario must contain deploy and run transactions" + ); + let deploy = output + .block_file + .transactions + .iter() + .find(|transaction| transaction.nonce == 1) + .ok_or_else(|| eyre::eyre!("create2 deploy transaction is missing"))?; + let run = output + .block_file + .transactions + .iter() + .find(|transaction| transaction.nonce == 2) + .ok_or_else(|| eyre::eyre!("create2 run transaction is missing"))?; + ensure!( + deploy.status + && deploy.from == WALLET_FIRST_ADDRESS + && deploy.to == probe + && deploy.value == U256::from(1), + "create2 deploy transaction mismatch" + ); + let deploy_view = transaction_view(output, deploy); + ensure!( + deploy_view.traces.len() == 1 + && deploy_view.error_traces.is_empty() + && deploy_view.events.len() == 1 + && deploy_view.error_events.is_empty(), + "create2 deploy must contain one successful CREATE root and event" + ); + let deploy_root = deploy_view.traces[0]; + ensure!( + deploy_root.parent_trace_id.is_empty() + && deploy_root.trace_address.is_empty() + && deploy_root.call_create_type == "create" + && deploy_root.call_type.is_empty() + && deploy_root.from_addr == WALLET_FIRST_ADDRESS + && deploy_root.to_addr == probe + && deploy_root.value == U256::from(1) + && deploy_root.error.is_empty(), + "create2 deploy root CREATE trace mismatch" + ); + assert_eip7708_transfer( + deploy_view.events[0], + WALLET_FIRST_ADDRESS, + probe, + U256::from(1), + &deploy_root.id, + 0, + 0, + )?; + + ensure!( + run.status + && run.from == WALLET_FIRST_ADDRESS + && run.to == probe + && run.value == U256::ZERO, + "create2 run transaction mismatch" + ); + let run_view = transaction_view(output, run); + ensure!( + run_view.traces.len() == 2 + && run_view.error_traces.is_empty() + && run_view.events.len() == 1 + && run_view.error_events.is_empty(), + "create2 run must contain successful CALL/CREATE2 traces and one event" + ); + let call_root = run_view + .traces + .iter() + .copied() + .find(|trace| trace.parent_trace_id.is_empty()) + .ok_or_else(|| eyre::eyre!("create2 root CALL trace is missing"))?; + let create_child = run_view + .traces + .iter() + .copied() + .find(|trace| trace.call_create_type == "create") + .ok_or_else(|| eyre::eyre!("internal CREATE2 trace is missing"))?; + ensure!( + call_root.call_create_type == "call" + && call_root.call_type == "call" + && call_root.from_addr == WALLET_FIRST_ADDRESS + && call_root.to_addr == probe + && call_root.subtraces == 1 + && call_root.self_storage_change + && call_root.storage_change + && call_root.error.is_empty(), + "create2 run root CALL trace mismatch" + ); + ensure!( + create_child.parent_trace_id == call_root.id + && create_child.trace_address == [0] + && create_child.pos_in_parent_trace == 0 + && create_child.call_type.is_empty() + && create_child.from_addr == probe + && create_child.to_addr == child + && create_child.value == U256::from(1) + && create_child.error.is_empty(), + "internal CREATE2 child trace mismatch" + ); + assert_eip7708_transfer( + run_view.events[0], + probe, + child, + U256::from(1), + &create_child.id, + 0, + 1, + )?; + assert_storage_contracts( + output, + state_diff, + &[HISTORY_STORAGE_ADDRESS, SYSTEM_ACCOUNTING_ADDRESS, probe], + )?; + + let probe_account = state_diff + .new_accounts + .iter() + .find(|account| account.address == keccak256(probe.as_slice())) + .ok_or_else(|| eyre::eyre!("create2 probe account is missing from StateDiff"))?; + ensure!( + probe_account.code_hash != KECCAK_EMPTY + && state_diff + .new_codes + .iter() + .any(|code| { code.code_hash == probe_account.code_hash && !code.code.is_empty() }), + "create2 probe runtime code is missing from StateDiff" + ); + let child_account = state_diff + .new_accounts + .iter() + .find(|account| account.address == keccak256(child.as_slice())) + .ok_or_else(|| eyre::eyre!("CREATE2 child account is missing from StateDiff"))?; + ensure!( + child_account.balance == U256::from(1) + && child_account.nonce == 1 + && child_account.code_hash == KECCAK_EMPTY, + "CREATE2 child state mismatch" + ); + let probe_storage = state_diff + .storage_diffs + .iter() + .find(|account| account.address == keccak256(probe.as_slice())) + .ok_or_else(|| eyre::eyre!("create2 probe storage is missing from StateDiff"))?; + let slot_zero = keccak256([0_u8; 32]); + ensure!( + probe_storage.diffs.iter().any(|diff| { + diff.index == slot_zero && diff.value == U256::from_be_slice(child.as_slice()) + }), + "create2 probe did not persist the child address" + ); + Ok(()) +} + +fn assert_failed_create(output: &DebankOutPut, state_diff: &BlockStorageDiff) -> Result<()> { + let failed_create = WALLET_FIRST_ADDRESS.create(3); + ensure!( + output.block_file.transactions.len() == 1, + "failed-create scenario must contain one transaction" + ); + let transaction = &output.block_file.transactions[0]; + ensure!( + !transaction.status + && transaction.nonce == 3 + && transaction.from == WALLET_FIRST_ADDRESS + && transaction.to == failed_create + && transaction.value == U256::from(1_000_000), + "failed CREATE transaction or receipt-derived status mismatch" + ); + let view = transaction_view(output, transaction); + ensure!( + view.traces.is_empty() + && view.events.is_empty() + && view.error_traces.len() == 1 + && view.error_events.len() == 1, + "failed CREATE must appear only in the error trace/event buckets" + ); + let root = view.error_traces[0]; + ensure!( + root.parent_trace_id.is_empty() + && root.trace_address.is_empty() + && root.call_create_type == "create" + && root.call_type.is_empty() + && root.from_addr == WALLET_FIRST_ADDRESS + && root.to_addr == failed_create + && root.value == U256::from(1_000_000) + && root.subtraces == 0 + && root.error == "Reverted", + "failed CREATE root trace mismatch" + ); + assert_eip7708_transfer( + view.error_events[0], + WALLET_FIRST_ADDRESS, + failed_create, + U256::from(1_000_000), + &root.id, + 0, + 0, + )?; + let failed_hash = keccak256(failed_create.as_slice()); + ensure!( + state_diff + .new_accounts + .iter() + .all(|account| account.address != failed_hash) + && state_diff + .storage_diffs + .iter() + .all(|account| account.address != failed_hash) + && state_diff.new_codes.is_empty(), + "failed CREATE leaked account, code, or storage into StateDiff" + ); + assert_system_hook_state(output, state_diff) +} + +fn assert_fixture_scenario( + label: &str, + output: &DebankOutPut, + state_diff: &BlockStorageDiff, +) -> Result<()> { + ensure!( + state_diff.deleted_accounts.is_empty(), + "{label} scenario unexpectedly deletes accounts" + ); + match label { + "genesis" => assert_genesis(output, state_diff), + "empty-hooks" => assert_empty_hooks(output, state_diff), + "native-transfer" => assert_native_transfer(output, state_diff), + "create2" => assert_create2(output, state_diff), + "failed-create" => assert_failed_create(output, state_diff), + _ => Err(eyre::eyre!("unknown fixture scenario {label}")), + } +} + +fn output_dir() -> Result { + std::env::var_os(OUTPUT_ENV) + .map(PathBuf::from) + .ok_or_else(|| eyre::eyre!("{OUTPUT_ENV} is required")) +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .wrap_err_with(|| format!("create {}", path.display()))?; + file.write_all(bytes) + .wrap_err_with(|| format!("write {}", path.display())) +} + +#[derive(Debug)] +struct CaptureDebankBlock { + label: &'static str, +} + +impl CaptureDebankBlock { + const fn new(label: &'static str) -> Self { + Self { label } + } +} + +impl Action for CaptureDebankBlock { + fn execute<'a>(&'a mut self, env: &'a mut ArcEnvironment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let block_number = env.block_number(); + let client = env + .node() + .rpc_client() + .ok_or_else(|| eyre::eyre!("RPC client not available"))?; + let output: DebankOutPut = client + .request( + "trace_debankBlock", + rpc_params![BlockId::number(block_number)], + ) + .await?; + + ensure!( + output.header.number == block_number, + "producer returned block {} for requested block {block_number}", + output.header.number + ); + let state_diff = alloy_rlp::decode_exact::(&output.state_diff)?; + ensure!( + state_diff.hash == output.header.state_root, + "block {block_number}: StateDiff root does not match Header" + ); + ensure!( + output.validation_hash == output.block_file.validation().validation_hash, + "block {block_number}: BlockFile validation hash mismatch" + ); + + let canonical: Option = client + .request( + "eth_getBlockByNumber", + rpc_params![format!("{block_number:#x}"), false], + ) + .await?; + let canonical = canonical + .ok_or_else(|| eyre::eyre!("block {block_number}: canonical block unavailable"))?; + ensure!( + canonical.header.hash == output.header.hash + && canonical.header.state_root == output.header.state_root + && canonical.header.parent_hash == output.header.parent_hash, + "block {block_number}: trace Header does not match canonical block" + ); + ensure!( + output.block_file.block.id == canonical.header.hash + && output.block_file.block.parent_id == canonical.header.parent_hash, + "block {block_number}: BlockFile identity does not match canonical block" + ); + + if block_number == 0 { + ensure!( + state_diff.parent_hash == EMPTY_ROOT_HASH, + "genesis StateDiff parent root is not the empty trie root" + ); + } else { + let parent: Option = client + .request( + "eth_getBlockByHash", + rpc_params![canonical.header.parent_hash, false], + ) + .await?; + let parent = parent.ok_or_else(|| { + eyre::eyre!("block {block_number}: canonical parent unavailable") + })?; + ensure!( + state_diff.parent_hash == parent.header.state_root, + "block {block_number}: StateDiff parent root mismatch" + ); + } + + assert_fixture_scenario(self.label, &output, &state_diff)?; + + let path = output_dir()?.join(format!("{block_number}-{}.json", self.label)); + write_new(&path, &serde_json::to_vec_pretty(&output)?)?; + Ok(()) + }) + } +} + +/// The standard e2e helper randomizes `prev_randao`; fixed fixtures need the +/// same real payload-builder path with deterministic payload attributes. +#[derive(Debug)] +struct ProduceDeterministicBlocks(u64); + +impl ProduceDeterministicBlocks { + const fn new(count: u64) -> Self { + Self(count) + } +} + +impl Action for ProduceDeterministicBlocks { + fn execute<'a>(&'a mut self, env: &'a mut ArcEnvironment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + for _ in 0..self.0 { + let current = env.current_block().clone(); + let next_number = current.number + 1; + let engine_client = env.node().inner.auth_server_handle().http_client(); + let forkchoice = ForkchoiceState { + head_block_hash: current.hash, + safe_block_hash: current.hash, + finalized_block_hash: current.hash, + }; + let attributes = PayloadAttributes { + timestamp: current.timestamp + 1, + prev_randao: B256::repeat_byte(u8::try_from(next_number)?), + suggested_fee_recipient: FEE_RECIPIENT, + withdrawals: Some(Vec::new()), + parent_beacon_block_root: Some(B256::ZERO), + }; + let fcu = EngineApiClient::::fork_choice_updated_v3( + &engine_client, + forkchoice, + Some(attributes), + ) + .await?; + assert_valid_or_syncing(&fcu.payload_status.status, "fixture build FCU")?; + let payload_id = fcu + .payload_id + .ok_or_else(|| eyre::eyre!("fixture build FCU returned no payload ID"))?; + + // The fixture chain activates Osaka at genesis, so V5 is the + // required and deterministic getPayload version. + let envelope = + EngineApiClient::::get_payload_v5(&engine_client, payload_id) + .await?; + let payload = envelope.execution_payload.clone(); + let requests = envelope.execution_requests.clone(); + let block_hash = payload.payload_inner.payload_inner.block_hash; + let block_number = payload.payload_inner.payload_inner.block_number; + let timestamp = payload.payload_inner.payload_inner.timestamp; + ensure!(block_number == next_number, "unexpected payload height"); + + let submitted = EngineApiClient::::new_payload_v4( + &engine_client, + payload, + Vec::new(), + B256::ZERO, + RequestsOrHash::Requests(requests), + ) + .await?; + assert_valid_or_syncing(&submitted.status, "fixture newPayload")?; + let finalized = EngineApiClient::::fork_choice_updated_v3( + &engine_client, + ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash: block_hash, + finalized_block_hash: block_hash, + }, + None, + ) + .await?; + assert_valid_or_syncing(&finalized.payload_status.status, "fixture finalize FCU")?; + env.set_current_block(BlockInfo::new(block_hash, block_number, timestamp)); + } + Ok(()) + }) + } +} + +#[tokio::test] +#[ignore = "writes Arc producer fixtures to ARC_A1B_CAPTURE_DIR"] +async fn export_arc_a1b_fixtures() -> Result<()> { + reth_tracing::init_test_tracing(); + + let capture_dir = output_dir()?; + fs::create_dir_all(&capture_dir)?; + ensure!( + fs::read_dir(&capture_dir)?.next().is_none(), + "capture directory must be empty: {}", + capture_dir.display() + ); + + let chain_spec = current_mainnet_test_chain_spec(); + let chain_id = chain_spec.chain().id(); + let native_recipient = Address::repeat_byte(0xa5); + // The native transfer consumes wallet nonce 0 before the deployment. + let probe = WALLET_FIRST_ADDRESS.create(1); + let child = probe.create2([0_u8; 32], alloy_primitives::keccak256([0_u8])); + // deploy_probe and run_probe consume nonces 1 and 2. + let failed_create = WALLET_FIRST_ADDRESS.create(3); + let context = serde_json::json!({ + "chain": "arc-localdev-current-mainnet-hardforks", + "chain_id": chain_id, + "wallet": WALLET_FIRST_ADDRESS, + "native_recipient": native_recipient, + "create2_probe": probe, + "create2_child": child, + "failed_create_address": failed_create, + "comparison_normalization": { "process_start_timestamp": 0 }, + }); + write_new( + &capture_dir.join("capture-context.json"), + &serde_json::to_vec_pretty(&context)?, + )?; + + ArcTestBuilder::new() + .with_setup( + ArcSetup::new() + .with_chain_spec(chain_spec) + .with_legacy_state_root_task_enabled(), + ) + .with_action(CaptureDebankBlock::new("genesis")) + .with_action(ProduceDeterministicBlocks::new(1)) + .with_action(CaptureDebankBlock::new("empty-hooks")) + .with_action(SendTransaction::new("native_transfer").with_to(native_recipient)) + .with_action(ProduceDeterministicBlocks::new(1)) + .with_action(CaptureDebankBlock::new("native-transfer")) + .with_action( + SendTransaction::new("deploy_probe") + .with_create() + .with_data(create2_with_balance_probe()) + .with_value(U256::from(1)) + .with_gas_limit(200_000), + ) + .with_action( + SendTransaction::new("run_probe") + .with_to(probe) + .with_data(create2_balance_probe_calldata(probe)) + .with_value(U256::ZERO) + .with_gas_limit(200_000), + ) + .with_action(ProduceDeterministicBlocks::new(1)) + .with_action(CaptureDebankBlock::new("create2")) + .with_action( + SendTransaction::new("failed_create") + .with_create() + .with_data(reverting_constructor_code()) + .with_value(U256::from(1_000_000)) + .with_gas_limit(100_000), + ) + .with_action(ProduceDeterministicBlocks::new(1)) + .with_action(CaptureDebankBlock::new("failed-create")) + .run() + .await +} diff --git a/scripts/fixtures/build_leafage_a1b_fixtures.py b/scripts/fixtures/build_leafage_a1b_fixtures.py new file mode 100755 index 0000000..393f195 --- /dev/null +++ b/scripts/fixtures/build_leafage_a1b_fixtures.py @@ -0,0 +1,762 @@ +#!/usr/bin/env python3 +"""Convert Arc writer captures into pipeline-compatible semantic fixtures.""" + +from __future__ import annotations + +import argparse +import copy +import gzip +import hashlib +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, Callable + + +STATE_DIFF_BITMAP_BYTES = 125 +STATE_DIFF_OFFSET_COUNT = 1_001 +STATE_DIFF_INDEX_BYTES = STATE_DIFF_BITMAP_BYTES + STATE_DIFF_OFFSET_COUNT * 8 +WRITER_PRODUCER_COMMIT = "1ff564adb44a54d5ddd7a31e60a4f64498650801" +FORMAT_REFERENCE_RELEASE = "v0.1.43" +FORMAT_REFERENCE_COMMIT = "7c4e096bfbc132dcb79312e2371c80919b966a52" +FORMAT_REFERENCE_SOURCES = { + "bin/background-tracer/src/utils/codec.rs": "0fb07bbacddc59cbec352634472895a622fbec65", + "bin/background-tracer/src/upload/s3.rs": "597fa9ee6de41183f46c1e8b88bb50407094f0a7", + "types/src/debank.rs": "2926b245378d4b3f54b5584ec36ac7204bca112f", +} +EXPECTED_LABELS = { + 0: "genesis", + 1: "empty-hooks", + 2: "native-transfer", + 3: "create2", + 4: "failed-create", +} +CAPTURE_CONTEXT = "capture-context.json" +EXPECTED_CAPTURE_ENTRIES = frozenset( + {CAPTURE_CONTEXT} + | {f"{number}-{label}.json" for number, label in EXPECTED_LABELS.items()} +) +EXPORTER_ENTRYPOINT = Path("scripts/fixtures/build_leafage_a1b_fixtures.py") +ALLOWED_EXPORTER_CHANGES = frozenset( + { + "crates/execution-e2e/src/setup.rs", + "crates/execution-e2e/tests/export_leafage_a1b.rs", + EXPORTER_ENTRYPOINT.as_posix(), + "scripts/generate-leafage-a1b-fixtures.sh", + } +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--capture-dir", type=Path, required=True) + parser.add_argument("--verification-capture-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--exporter-worktree", type=Path, required=True) + parser.add_argument("--expected-exporter-commit", required=True) + parser.add_argument("--format-reference-worktree", type=Path, required=True) + return parser.parse_args() + + +def output_preflight_cli(arguments: list[str]) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--exporter-worktree", type=Path, required=True) + args = parser.parse_args(arguments) + resolve_final_output_path( + args.output_dir, args.exporter_worktree.resolve(strict=True) + ) + + +def git(worktree: Path, *args: str) -> str: + return subprocess.check_output( + ["git", "-C", str(worktree), *args], text=True + ).strip() + + +def write_new(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as output: + output.write(data) + + +def json_bytes( + value: Any, *, pretty: bool = False, trailing_newline: bool = True +) -> bytes: + if pretty: + encoded = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + else: + encoded = json.dumps(value, separators=(",", ":"), ensure_ascii=False) + if trailing_newline: + encoded += "\n" + return encoded.encode() + + +def gzip_json(value: Any) -> bytes: + # background-tracer compresses serde_json::to_vec(value), which has no + # trailing newline. mtime=0 makes the fixture container reproducible; its + # decoded JSON semantics, not a particular gzip header, are the contract. + return gzip.compress( + json_bytes(value, trailing_newline=False), compresslevel=6, mtime=0 + ) + + +def decode_quantity(value: str | int) -> int: + return value if isinstance(value, int) else int(value, 0) + + +def relative(path: Path, root: Path) -> str: + return path.relative_to(root).as_posix() + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def normalized_capture(output: dict[str, Any]) -> bytes: + normalized = copy.deepcopy(output) + normalized["block_file"]["block"]["process_start_timestamp"] = 0 + return ( + json.dumps( + normalized, separators=(",", ":"), sort_keys=True, ensure_ascii=False + ) + + "\n" + ).encode() + + +def load_captures(capture_dir: Path) -> list[tuple[str, dict[str, Any]]]: + captures = [] + for number, label in EXPECTED_LABELS.items(): + path = capture_dir / f"{number}-{label}.json" + captures.append((label, json.loads(path.read_text()))) + return captures + + +def validate_capture_directory(capture_dir: Path) -> None: + if capture_dir.is_symlink() or not capture_dir.is_dir(): + raise ValueError(f"capture path is not a directory: {capture_dir}") + + actual_entries = {entry.name for entry in capture_dir.iterdir()} + missing = sorted(EXPECTED_CAPTURE_ENTRIES - actual_entries) + unexpected = sorted(actual_entries - EXPECTED_CAPTURE_ENTRIES) + if missing or unexpected: + raise ValueError( + f"capture directory has missing entries {missing} " + f"and unexpected entries {unexpected}: {capture_dir}" + ) + + for name in sorted(EXPECTED_CAPTURE_ENTRIES): + entry = capture_dir / name + if entry.is_symlink() or not entry.is_file(): + raise ValueError(f"capture entry must be a regular file: {entry}") + + +def ensure_entrypoint_binding(script_path: Path, exporter_root: Path) -> None: + expected_path = (exporter_root / EXPORTER_ENTRYPOINT).resolve() + if script_path.resolve() != expected_path: + raise ValueError( + "executed converter does not belong to --exporter-worktree: " + f"{script_path.resolve()} != {expected_path}" + ) + + +def ensure_only_test_harness_changes(changed_paths: set[str]) -> None: + unexpected = sorted(changed_paths - ALLOWED_EXPORTER_CHANGES) + if unexpected: + raise ValueError( + "exporter commit changes paths outside the test-only allowlist: " + + ", ".join(unexpected) + ) + + +def ensure_expected_exporter_commit(actual: str, expected: str) -> None: + if actual != expected: + raise ValueError( + f"exporter HEAD changed: expected {expected}, found {actual}" + ) + + +def ensure_exporter_worktree_state( + exporter_root: Path, expected_exporter_commit: str +) -> None: + ensure_expected_exporter_commit( + git(exporter_root, "rev-parse", "HEAD"), expected_exporter_commit + ) + if git( + exporter_root, + "status", + "--porcelain", + "--untracked-files=all", + ): + raise ValueError("exporter worktree must be clean") + + +def resolve_final_output_path(output_dir: Path, exporter_root: Path) -> Path: + if output_dir.exists() or output_dir.is_symlink(): + raise ValueError(f"output directory already exists: {output_dir}") + + resolved_output = output_dir.resolve() + try: + resolved_output.relative_to(exporter_root) + except ValueError: + return resolved_output + raise ValueError( + f"output directory must be outside the exporter worktree: {output_dir}" + ) + + +def create_staging_output(final_output: Path) -> tempfile.TemporaryDirectory: + # TemporaryDirectory forces 0700; match Path.mkdir's mode after the current umask. + current_umask = os.umask(0) + os.umask(current_umask) + staging = tempfile.TemporaryDirectory( + prefix=f".{final_output.name}.staging.", dir=final_output.parent + ) + Path(staging.name).chmod(0o777 & ~current_umask) + return staging + + +def finalize_staged_output( + staging: tempfile.TemporaryDirectory, + final_output: Path, + exporter_root: Path, + expected_exporter_commit: str, +) -> None: + try: + ensure_exporter_worktree_state(exporter_root, expected_exporter_commit) + if final_output.exists() or final_output.is_symlink(): + raise ValueError(f"output directory already exists: {final_output}") + Path(staging.name).rename(final_output) + except BaseException: + staging.cleanup() + raise + staging.cleanup() + + +def expect_value_error(action: Callable[[], Any], message: str) -> None: + try: + action() + except ValueError: + return + raise AssertionError(message) + + +def validate_exporter_provenance( + exporter_worktree: Path, + expected_exporter_commit: str, +) -> tuple[str, str, str, list[str]]: + exporter_root = exporter_worktree.resolve(strict=True) + git_root = Path(git(exporter_root, "rev-parse", "--show-toplevel")).resolve( + strict=True + ) + if exporter_root != git_root: + raise ValueError("--exporter-worktree must be the git worktree root") + + script_path = Path(__file__).resolve(strict=True) + ensure_entrypoint_binding(script_path, exporter_root) + resolved_expected_commit = git( + exporter_root, "rev-parse", f"{expected_exporter_commit}^{{commit}}" + ) + ensure_expected_exporter_commit( + resolved_expected_commit, expected_exporter_commit + ) + ensure_exporter_worktree_state(exporter_root, expected_exporter_commit) + + release_commit = git(exporter_root, "rev-parse", "v0.7.3^{}") + subprocess.run( + [ + "git", + "-C", + str(exporter_root), + "merge-base", + "--is-ancestor", + release_commit, + WRITER_PRODUCER_COMMIT, + ], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(exporter_root), + "merge-base", + "--is-ancestor", + WRITER_PRODUCER_COMMIT, + expected_exporter_commit, + ], + check=True, + ) + changed_paths = sorted( + filter( + None, + git( + exporter_root, + "diff", + "--name-only", + "--no-renames", + f"{WRITER_PRODUCER_COMMIT}..{expected_exporter_commit}", + ).splitlines(), + ) + ) + ensure_only_test_harness_changes(set(changed_paths)) + + entrypoint_blob = git( + exporter_root, + "rev-parse", + f"{expected_exporter_commit}:{EXPORTER_ENTRYPOINT.as_posix()}", + ) + working_entrypoint_blob = git(exporter_root, "hash-object", str(script_path)) + if working_entrypoint_blob != entrypoint_blob: + raise ValueError( + "executed converter blob does not match the recorded exporter commit" + ) + return expected_exporter_commit, release_commit, entrypoint_blob, changed_paths + + +def provenance_self_test() -> None: + exporter_root = Path("/arc-a1b-exporter-self-test") + ensure_entrypoint_binding(exporter_root / EXPORTER_ENTRYPOINT, exporter_root) + ensure_only_test_harness_changes(set(ALLOWED_EXPORTER_CHANGES)) + ensure_expected_exporter_commit("a" * 40, "a" * 40) + + expect_value_error( + lambda: ensure_entrypoint_binding( + exporter_root / "copied-transformer.py", exporter_root + ), + "provenance self-test accepted a copied entrypoint", + ) + expect_value_error( + lambda: ensure_only_test_harness_changes( + set(ALLOWED_EXPORTER_CHANGES) | {"crates/node/src/lib.rs"} + ), + "provenance self-test accepted a source path outside the allowlist", + ) + expect_value_error( + lambda: ensure_expected_exporter_commit("a" * 40, "b" * 40), + "provenance self-test accepted a mismatched exporter commit", + ) + output_staging_self_test() + print("provenance self-test passed") + + +def output_staging_self_test() -> None: + with tempfile.TemporaryDirectory(prefix="arc-a1b-output-test.") as temp: + test_root = Path(temp) + exporter_root = test_root / "exporter" + exporter_root.mkdir() + exporter_root = exporter_root.resolve(strict=True) + inside_output = exporter_root / "fixtures" + expect_value_error( + lambda: resolve_final_output_path(inside_output, exporter_root), + "output self-test accepted a path inside the exporter", + ) + if inside_output.exists(): + raise AssertionError("output preflight created a rejected final path") + + exporter_alias = test_root / "exporter-alias" + exporter_alias.symlink_to(exporter_root) + expect_value_error( + lambda: resolve_final_output_path( + exporter_alias / "fixtures", exporter_root + ), + "output self-test accepted a symlink into the exporter", + ) + + subprocess.run( + ["git", "-C", str(exporter_root), "init", "--quiet"], check=True + ) + subprocess.run( + [ + "git", + "-C", + str(exporter_root), + "-c", + "user.name=Arc fixture self-test", + "-c", + "user.email=arc-fixture-self-test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "--no-verify", + "--allow-empty", + "-m", + "self-test", + ], + check=True, + ) + expected_commit = git(exporter_root, "rev-parse", "HEAD") + output_parent = test_root / "outputs" + output_parent.mkdir() + final_output = output_parent / "fixtures" + staging = create_staging_output(final_output) + staging_path = Path(staging.name) + (staging_path / "manifest.json").write_text("{}") + (exporter_root / "dirty").write_text("dirty") + expect_value_error( + lambda: finalize_staged_output( + staging, final_output, exporter_root, expected_commit + ), + "output self-test accepted a dirty exporter", + ) + if final_output.exists() or staging_path.exists(): + raise AssertionError("failed output publication left files behind") + + (exporter_root / "dirty").unlink() + staging = create_staging_output(final_output) + staging_path = Path(staging.name) + (staging_path / "manifest.json").write_text("{}") + control_output = test_root / "control-output" + control_output.mkdir() + finalize_staged_output(staging, final_output, exporter_root, expected_commit) + if not (final_output / "manifest.json").is_file() or staging_path.exists(): + raise AssertionError("output self-test failed to publish staged output") + if (final_output.stat().st_mode & 0o777) != ( + control_output.stat().st_mode & 0o777 + ): + raise AssertionError("published output root does not honor the current umask") + + +def capture_set_self_test() -> None: + with tempfile.TemporaryDirectory(prefix="arc-a1b-capture-set-test.") as temp: + capture_dir = Path(temp) + for name in EXPECTED_CAPTURE_ENTRIES: + (capture_dir / name).write_text("{}") + validate_capture_directory(capture_dir) + + missing = capture_dir / "4-failed-create.json" + missing.unlink() + expect_value_error( + lambda: validate_capture_directory(capture_dir), + "capture-set self-test accepted a missing file", + ) + missing.write_text("{}") + + unexpected = capture_dir / "5-unexpected.json" + unexpected.write_text("{}") + expect_value_error( + lambda: validate_capture_directory(capture_dir), + "capture-set self-test accepted an extra file", + ) + unexpected.unlink() + + unexpected_directory = capture_dir / "unexpected-directory" + unexpected_directory.mkdir() + expect_value_error( + lambda: validate_capture_directory(capture_dir), + "capture-set self-test accepted an extra directory", + ) + unexpected_directory.rmdir() + + expected = capture_dir / "0-genesis.json" + expected.unlink() + expected.symlink_to(capture_dir / "capture-context.json") + expect_value_error( + lambda: validate_capture_directory(capture_dir), + "capture-set self-test accepted a symlink", + ) + expected.unlink() + expected.mkdir() + expect_value_error( + lambda: validate_capture_directory(capture_dir), + "capture-set self-test accepted a directory", + ) + print("capture-set self-test passed") + + +def generate(staging_holder: list[tempfile.TemporaryDirectory]) -> None: + args = parse_args() + exporter_root = args.exporter_worktree.resolve(strict=True) + exporter_commit, release_commit, entrypoint_blob, exporter_changed_paths = ( + validate_exporter_provenance( + exporter_root, args.expected_exporter_commit + ) + ) + final_output = resolve_final_output_path(args.output_dir, exporter_root) + if not args.capture_dir.is_dir(): + raise SystemExit(f"capture directory does not exist: {args.capture_dir}") + if not args.verification_capture_dir.is_dir(): + raise SystemExit( + "verification capture directory does not exist: " + f"{args.verification_capture_dir}" + ) + validate_capture_directory(args.capture_dir) + validate_capture_directory(args.verification_capture_dir) + + format_reference_head = git(args.format_reference_worktree, "rev-parse", "HEAD") + format_reference_tag = git( + args.format_reference_worktree, + "rev-parse", + f"{FORMAT_REFERENCE_RELEASE}^{{}}", + ) + if git(args.format_reference_worktree, "status", "--porcelain"): + raise ValueError("format-reference worktree must be clean") + if ( + format_reference_head != FORMAT_REFERENCE_COMMIT + or format_reference_tag != FORMAT_REFERENCE_COMMIT + ): + raise ValueError( + "format-reference worktree must be checked out at " + f"{FORMAT_REFERENCE_RELEASE}@{FORMAT_REFERENCE_COMMIT}" + ) + for source, expected_blob in FORMAT_REFERENCE_SOURCES.items(): + actual_blob = git( + args.format_reference_worktree, + "rev-parse", + f"{FORMAT_REFERENCE_COMMIT}:{source}", + ) + if actual_blob != expected_blob: + raise ValueError( + f"format-reference source {source} is {actual_blob}, expected {expected_blob}" + ) + + context = json.loads((args.capture_dir / CAPTURE_CONTEXT).read_text()) + verification_context = json.loads( + (args.verification_capture_dir / CAPTURE_CONTEXT).read_text() + ) + if context != verification_context: + raise ValueError("independent captures used different chain context") + captures = load_captures(args.capture_dir) + verification_captures = load_captures(args.verification_capture_dir) + normalized_capture_sha256 = [] + for (label, output), (verification_label, verification_output) in zip( + captures, verification_captures, strict=True + ): + if label != verification_label: + raise ValueError("independent capture labels do not match") + normalized = normalized_capture(output) + if normalized != normalized_capture(verification_output): + raise ValueError( + f"{label}: independent captures differ beyond process_start_timestamp" + ) + normalized_capture_sha256.append( + { + "label": label, + "sha256": hashlib.sha256(normalized).hexdigest(), + } + ) + + ensure_exporter_worktree_state( + exporter_root, args.expected_exporter_commit + ) + final_output.parent.mkdir(parents=True, exist_ok=True) + staging = create_staging_output(final_output) + staging_holder.append(staging) + args.output_dir = Path(staging.name) + files: list[dict[str, Any]] = [] + blocks: list[dict[str, Any]] = [] + previous_hash: str | None = None + genesis_header: dict[str, Any] | None = None + genesis_state_diff: bytes | None = None + + def record(path: Path, kind: str, number: int | None = None) -> None: + entry: dict[str, Any] = { + "path": relative(path, args.output_dir), + "kind": kind, + "bytes": path.stat().st_size, + "sha256": sha256(path), + } + if number is not None: + entry["block_number"] = number + files.append(entry) + + for expected_number, (label, output) in enumerate(captures): + block_file = output["block_file"] + header = output["header"] + state_diff_hex = output["state_diff"] + number = decode_quantity(header["number"]) + block_number = decode_quantity(block_file["block"]["height"]) + if number != expected_number or block_number != expected_number: + raise ValueError( + f"{label}: expected block {expected_number}, got {number}/{block_number}" + ) + if block_file["block"]["id"].lower() != header["hash"].lower(): + raise ValueError(f"{label}: BlockFile/Header block hash mismatch") + if previous_hash is not None and header["parentHash"].lower() != previous_hash: + raise ValueError(f"{label}: Header parent hash is not linear") + previous_hash = header["hash"].lower() + + try: + state_diff = bytes.fromhex(state_diff_hex.removeprefix("0x")) + except ValueError as error: + raise ValueError(f"{label}: invalid StateDiff hex") from error + if not state_diff: + raise ValueError(f"{label}: producer returned an empty StateDiff") + + rpc_path = args.output_dir / "rpc" / f"{number}-{label}.json" + header_path = ( + args.output_dir / "per-block" / header["hash"].lower() / "block" + ) + state_diff_path = ( + args.output_dir + / "per-block" + / header["stateRoot"].lower() + / "stateDiff" + ) + block_file_path = ( + args.output_dir / "outer" / block_file["block"]["id"].lower() + ) + write_new(rpc_path, json_bytes(output, pretty=True)) + write_new(header_path, gzip_json(header)) + write_new(state_diff_path, state_diff) + write_new(block_file_path, gzip_json(block_file)) + record(rpc_path, "rpc_debank_output_json", number) + record(header_path, "per_block_header_gzip_json", number) + record(state_diff_path, "per_block_state_diff_raw_rlp", number) + record(block_file_path, "outer_block_file_gzip_json", number) + + blocks.append( + { + "label": label, + "number": number, + "hash": header["hash"].lower(), + "parent_hash": header["parentHash"].lower(), + "state_root": header["stateRoot"].lower(), + "extra_data": header["extraData"].lower(), + "validation_hash": output["validation_hash"], + "process_start_timestamp": block_file["block"][ + "process_start_timestamp" + ], + "rpc": relative(rpc_path, args.output_dir), + "header": relative(header_path, args.output_dir), + "state_diff": relative(state_diff_path, args.output_dir), + "block_file": relative(block_file_path, args.output_dir), + } + ) + if number == 0: + genesis_header = header + genesis_state_diff = state_diff + + if genesis_header is None or genesis_state_diff is None: + raise ValueError("genesis capture is missing") + + bundle_header_path = args.output_dir / "bundle" / "0" / "block" + bundle_state_diff_path = args.output_dir / "bundle" / "0" / "stateDiff" + write_new(bundle_header_path, gzip_json([genesis_header])) + index = bytearray(STATE_DIFF_INDEX_BYTES) + index[0] = 1 # position zero is a real producer StateDiff + offset_one = STATE_DIFF_BITMAP_BYTES + 8 + index[offset_one : offset_one + 8] = len(genesis_state_diff).to_bytes(8, "big") + write_new(bundle_state_diff_path, bytes(index) + genesis_state_diff) + record(bundle_header_path, "bundle0_header_gzip_json") + record(bundle_state_diff_path, "bundle0_index_and_state_diff_rlp") + + files.sort(key=lambda item: item["path"]) + ensure_exporter_worktree_state( + exporter_root, args.expected_exporter_commit + ) + manifest = { + "schema_version": 1, + "producer_baseline": { + "repository": "Chaintable/arc-node", + "commit": WRITER_PRODUCER_COMMIT, + "release": "v0.7.3", + "release_commit": release_commit, + }, + "producer": { + "repository": "Chaintable/arc-node", + "commit": exporter_commit, + "baseline_commit": WRITER_PRODUCER_COMMIT, + "changes_from_baseline": exporter_changed_paths, + "source_policy": "only test harness and fixture scripts may differ", + }, + "exporter": { + "repository": "Chaintable/arc-node", + "commit": exporter_commit, + }, + "fixture_transformer": { + "repository": "Chaintable/arc-node", + "commit": exporter_commit, + "entrypoint": "scripts/fixtures/build_leafage_a1b_fixtures.py", + "entrypoint_blob": entrypoint_blob, + "encoding": { + "rpc": "DebankOutPut JSON", + "per_block_header": "gzip JSON Header", + "per_block_state_diff": "raw RLP BlockStorageDiff", + "outer_block_file": "gzip JSON BlockFile", + "bundle0": "8133-byte index followed by genesis RLP entry", + }, + "compatibility_contract": { + "gzip_json_payload": ( + "compact JSON without a trailing newline, matching " + "background-tracer serde_json::to_vec semantics" + ), + "gzip_container": ( + "deterministic fixture encoding; gzip header and compressed bytes " + "are not a pipeline contract" + ), + }, + }, + "format_reference": { + "repository": "Chaintable/background-tracer", + "release": FORMAT_REFERENCE_RELEASE, + "commit": FORMAT_REFERENCE_COMMIT, + "sources": [ + {"path": source, "blob": blob} + for source, blob in FORMAT_REFERENCE_SOURCES.items() + ], + "scope": [ + "DebankOutPut JSON input schema", + "gzip JSON Header and BlockFile objects", + "raw RLP BlockStorageDiff object", + ], + "executed_by_fixture_generation": False, + }, + "generation": { + "command": ( + "BACKGROUND_TRACER_WORKTREE=/path/to/background-tracer " + "LEAFAGE_A1B_FIXTURE_OUT=/tmp/leafage-arc-a1b-fixtures " + "./scripts/generate-leafage-a1b-fixtures.sh" + ), + "comparison_normalization": {"process_start_timestamp": 0}, + "independent_capture_verification": normalized_capture_sha256, + "deterministic_payload": { + "timestamp": "parent timestamp + 1", + "prev_randao": "block number repeated as one byte", + }, + }, + "chain": context, + "coverage": [ + "genesis full alloc", + "empty block EIP-2935 and SystemAccounting hooks", + "native transfer", + "successful CREATE root", + "CALL root with internal CREATE2", + "failed CREATE root", + "new account, code, and storage", + ], + "excluded": ["StorageCleared"], + "blocks": blocks, + "files": files, + } + manifest_path = args.output_dir / "manifest.json" + write_new(manifest_path, json_bytes(manifest, pretty=True)) + finalize_staged_output( + staging, final_output, exporter_root, args.expected_exporter_commit + ) + + +def main() -> None: + staging_holder: list[tempfile.TemporaryDirectory] = [] + try: + generate(staging_holder) + finally: + for staging in staging_holder: + staging.cleanup() + + +if __name__ == "__main__": + if sys.argv[1:] == ["--self-test-provenance"]: + provenance_self_test() + elif sys.argv[1:] == ["--self-test-capture-set"]: + capture_set_self_test() + elif sys.argv[1:2] == ["--preflight-output"]: + output_preflight_cli(sys.argv[2:]) + else: + main() diff --git a/scripts/generate-leafage-a1b-fixtures.sh b/scripts/generate-leafage-a1b-fixtures.sh new file mode 100755 index 0000000..ccc461b --- /dev/null +++ b/scripts/generate-leafage-a1b-fixtures.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +writer_producer="1ff564adb44a54d5ddd7a31e60a4f64498650801" +format_reference_release="v0.1.43" +format_reference_commit="7c4e096bfbc132dcb79312e2371c80919b966a52" + +panic_log_detected() { + local log_file="$1" + LC_ALL=C grep -Eq "thread .* panicked at|panicked at .*deferred_trie\.rs(:[0-9]+)?" "$log_file" +} + +validate_capture_result() { + local cargo_status="$1" + local tee_status="$2" + local log_file="$3" + + if (( cargo_status != 0 )); then + echo "Arc fixture exporter cargo test failed with status $cargo_status" >&2 + return "$cargo_status" + fi + if (( tee_status != 0 )); then + echo "failed to capture Arc fixture exporter log with status $tee_status" >&2 + return "$tee_status" + fi + if panic_log_detected "$log_file"; then + echo "Arc fixture exporter emitted a Rust panic despite cargo status 0" >&2 + return 1 + fi +} + +panic_detection_self_test() ( + local test_dir safe_log panic_log + test_dir="$(mktemp -d "${TMPDIR:-/tmp}/arc-a1b-panic-test.XXXXXX")" + safe_log="$test_dir/safe.log" + panic_log="$test_dir/exit-zero-with-panic.log" + trap 'rm -rf -- "$test_dir"' EXIT + + printf '%s\n' 'test result: ok. 1 passed; 0 failed' >"$safe_log" + printf '%s\n' \ + 'test result: ok. 1 passed; 0 failed' \ + "thread 'proof-acct-00' panicked at /checkout/reth/crates/chain-state/src/deferred_trie.rs:316:9:" \ + 'wait_cloned must not be called from a rayon worker thread' >"$panic_log" + + if ! validate_capture_result 0 0 "$safe_log"; then + echo "capture validator rejected a clean cargo log" >&2 + return 1 + fi + if validate_capture_result 0 0 "$panic_log" >/dev/null 2>&1; then + echo "capture validator accepted an exit-zero deferred_trie panic log" >&2 + return 1 + fi + echo "capture validator self-test passed" +) + +if [[ "${1:-}" == "--self-test-panic-detection" ]]; then + panic_detection_self_test + exit 0 +fi + +if [[ "${1:-}" == "--self-test-provenance" ]]; then + python3 "$repo_root/scripts/fixtures/build_leafage_a1b_fixtures.py" \ + --self-test-provenance + exit 0 +fi + +if [[ "${1:-}" == "--self-test-capture-set" ]]; then + python3 "$repo_root/scripts/fixtures/build_leafage_a1b_fixtures.py" \ + --self-test-capture-set + exit 0 +fi + +if [[ $# -ne 0 ]]; then + echo "usage: $0 [--self-test-panic-detection|--self-test-provenance|--self-test-capture-set]" >&2 + exit 2 +fi + +panic_detection_self_test +python3 "$repo_root/scripts/fixtures/build_leafage_a1b_fixtures.py" \ + --self-test-provenance +python3 "$repo_root/scripts/fixtures/build_leafage_a1b_fixtures.py" \ + --self-test-capture-set + +format_reference_worktree="${BACKGROUND_TRACER_WORKTREE:?set BACKGROUND_TRACER_WORKTREE}" +output_dir="${LEAFAGE_A1B_FIXTURE_OUT:?set LEAFAGE_A1B_FIXTURE_OUT to a new output directory}" +python3 "$repo_root/scripts/fixtures/build_leafage_a1b_fixtures.py" \ + --preflight-output \ + --output-dir "$output_dir" \ + --exporter-worktree "$repo_root" + +assert_exporter_source_unchanged() { + local current_head + current_head="$(git -C "$repo_root" rev-parse HEAD)" + if [[ "$current_head" != "$exporter_commit" ]]; then + echo "Arc exporter HEAD changed: expected $exporter_commit, found $current_head" >&2 + return 1 + fi + if [[ -n "$(git -C "$repo_root" status --porcelain --untracked-files=all)" ]]; then + echo "Arc exporter worktree must remain clean" >&2 + return 1 + fi +} + +exporter_commit="$(git -C "$repo_root" rev-parse HEAD)" +if ! git -C "$repo_root" merge-base --is-ancestor "$writer_producer" "$exporter_commit"; then + echo "Arc exporter must descend from writer producer $writer_producer" >&2 + exit 1 +fi +assert_exporter_source_unchanged +format_reference_head="$(git -C "$format_reference_worktree" rev-parse HEAD)" +format_reference_tag_commit="$(git -C "$format_reference_worktree" rev-parse "${format_reference_release}^{}")" +if [[ "$format_reference_head" != "$format_reference_commit" ]] || \ + [[ "$format_reference_tag_commit" != "$format_reference_commit" ]]; then + echo "background-tracer format reference must be $format_reference_release@$format_reference_commit" >&2 + exit 1 +fi +if [[ -n "$(git -C "$format_reference_worktree" status --porcelain --untracked-files=all)" ]]; then + echo "background-tracer format-reference worktree must be clean" >&2 + exit 1 +fi +capture_dir="$(mktemp -d "${TMPDIR:-/tmp}/arc-a1b-capture.XXXXXX")" +verification_capture_dir="$(mktemp -d "${TMPDIR:-/tmp}/arc-a1b-verify.XXXXXX")" +capture_log_dir="$(mktemp -d "${TMPDIR:-/tmp}/arc-a1b-logs.XXXXXX")" +capture_log="$capture_log_dir/capture.log" +verification_capture_log="$capture_log_dir/verification.log" +cleanup() { + rm -rf -- "$capture_dir" "$verification_capture_dir" "$capture_log_dir" +} +trap cleanup EXIT + +run_capture() { + local target_dir="$1" + local log_file="$2" + local -a pipeline_status + + assert_exporter_source_unchanged + set +e + ARC_A1B_CAPTURE_DIR="$target_dir" \ + rustup run 1.91.1 cargo test \ + --manifest-path "$repo_root/Cargo.toml" \ + --locked \ + -p arc-execution-e2e \ + --test export_leafage_a1b \ + export_arc_a1b_fixtures \ + -- --ignored --exact --nocapture 2>&1 | tee "$log_file" + pipeline_status=("${PIPESTATUS[@]}") + set -e + + assert_exporter_source_unchanged + validate_capture_result "${pipeline_status[0]}" "${pipeline_status[1]}" "$log_file" +} + +run_capture "$capture_dir" "$capture_log" +run_capture "$verification_capture_dir" "$verification_capture_log" + +rm -- "$capture_log" "$verification_capture_log" + +assert_exporter_source_unchanged +python3 "$repo_root/scripts/fixtures/build_leafage_a1b_fixtures.py" \ + --capture-dir "$capture_dir" \ + --verification-capture-dir "$verification_capture_dir" \ + --output-dir "$output_dir" \ + --exporter-worktree "$repo_root" \ + --expected-exporter-commit "$exporter_commit" \ + --format-reference-worktree "$format_reference_worktree" + +echo "Arc A1b fixtures written to $output_dir"