Portal: mv accumulator based block proof code out of history_legacy (#3599)

kdeme · web-flow · commit 91ee0279ead8 · 2025-08-28T14:15:48.000+02:00
diff --git a/Makefile b/Makefile
@@ -289,12 +289,16 @@ all_history_network_custom_chain_tests: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -d:mergeBlockNumber:38130 -o:build/$@ "portal/tests/legacy_history_network_tests/$@.nim"
 
+all_eth_history_custom_chain_tests: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -d:mergeBlockNumber:38130 -o:build/$@ "portal/tests/eth_history_tests/$@.nim"
+
 all_portal_tests: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -o:build/$@ "portal/tests/$@.nim"
 
 # builds and runs the Portal test suite
-portal-test: | all_portal_tests all_history_network_custom_chain_tests
+portal-test: | all_portal_tests all_history_network_custom_chain_tests all_eth_history_custom_chain_tests
 
 # builds the Portal tools, wherever they are
 $(PORTAL_TOOLS): | build deps rocksdb
diff --git a/execution_chain/db/era1_db/db_desc.nim b/execution_chain/db/era1_db/db_desc.nim
@@ -15,7 +15,7 @@ import
   std/[os, parseutils, strutils, tables],
   results,
   eth/common/blocks,
-  ../../../portal/eth_data/era1
+  ../../../portal/eth_history/era1
 
 export results, blocks
 
diff --git a/portal/bridge/history/portal_history_bridge.nim b/portal/bridge/history/portal_history_bridge.nim
@@ -23,7 +23,7 @@ import
   ../../network/legacy_history/
     [history_content, history_type_conversions, history_validation],
   ../../network/network_metadata,
-  ../../eth_data/[era1, history_data_ssz_e2s, history_data_seeding],
+  ../../eth_history/[era1, history_data_ssz_e2s, history_data_seeding],
   ../../database/era1_db,
   ../common/rpc_helpers,
   ../nimbus_portal_bridge_conf
diff --git a/portal/bridge/nimbus_portal_bridge_conf.nim b/portal/bridge/nimbus_portal_bridge_conf.nim
@@ -13,7 +13,7 @@ import
   confutils/std/net,
   nimcrypto/hash,
   ../network/network_metadata,
-  ../eth_data/era1,
+  ../eth_history/era1,
   ../logging
 
 export net
diff --git a/portal/database/era1_db.nim b/portal/database/era1_db.nim
@@ -12,8 +12,8 @@ import
   stew/io2,
   results,
   eth/common/blocks,
-  ../network/legacy_history/validation/historical_hashes_accumulator,
-  ../eth_data/era1
+  ../eth_history/block_proofs/historical_hashes_accumulator,
+  ../eth_history/era1
 
 type Era1DB* = ref object
   ## The Era1 database manages a collection of era files that together make up
diff --git a/portal/eth_history/block_proofs/block_proof_common.nim b/portal/eth_history/block_proofs/block_proof_common.nim
diff --git a/portal/eth_history/block_proofs/block_proof_historical_hashes_accumulator.nim b/portal/eth_history/block_proofs/block_proof_historical_hashes_accumulator.nim
@@ -11,8 +11,8 @@ import
   eth/common/[headers_rlp],
   ssz_serialization,
   ssz_serialization/[proofs, merkleization],
-  ../../../common/common_types,
-  ../history_content,
+  ../../common/common_types,
+  ../../network/legacy_history/history_content,
   ./historical_hashes_accumulator
 
 export
diff --git a/portal/eth_history/block_proofs/block_proof_historical_roots.nim b/portal/eth_history/block_proofs/block_proof_historical_roots.nim
diff --git a/portal/eth_history/block_proofs/block_proof_historical_summaries.nim b/portal/eth_history/block_proofs/block_proof_historical_summaries.nim
diff --git a/portal/eth_history/block_proofs/historical_hashes_accumulator.nim b/portal/eth_history/block_proofs/historical_hashes_accumulator.nim
@@ -11,15 +11,16 @@ import
   eth/common/[headers_rlp],
   ssz_serialization,
   ssz_serialization/[proofs, merkleization],
-  ../../../common/common_types,
-  ../history_content
+  ../../common/common_types
 
 from eth/common/eth_types_rlp import rlpHash
 
 export ssz_serialization, merkleization, proofs, common_types
 
 # HistoricalHashesAccumulator, as per specification:
 # https://github.com/ethereum/portal-network-specs/blob/31bc7e58e2e8acfba895d5a12a9ae3472894d398/history/history-network.md#the-historical-hashes-accumulator
+# EIP-7643::
+# https://github.com/ethereum/EIPs/blob/b2bce97a0a6898faf5a37c5948b7bc58a1cd1bc5/EIPS/eip-7643.md
 
 const
   EPOCH_SIZE* = 8192 # block roots per epoch record
diff --git a/portal/eth_history/era1.nim b/portal/eth_history/era1.nim
@@ -17,7 +17,7 @@ import
   beacon_chain/spec/beacon_time,
   ssz_serialization,
   ncli/e2store,
-  ../network/legacy_history/validation/historical_hashes_accumulator
+  ./block_proofs/historical_hashes_accumulator
 
 from eth/common/eth_types_rlp import computeRlpHash
 from nimcrypto/hash import fromHex
diff --git a/portal/eth_history/history_data_json_store.nim b/portal/eth_history/history_data_json_store.nim
@@ -15,7 +15,8 @@ import
   chronicles,
   eth/common/[hashes, blocks, receipts, headers_rlp],
   ../../execution_chain/common/[chain_config, genesis],
-  ../network/legacy_history/[history_content, validation/historical_hashes_accumulator]
+  ../network/legacy_history/history_content,
+  ./block_proofs/historical_hashes_accumulator
 
 export results, tables
 
diff --git a/portal/eth_history/history_data_seeding.nim b/portal/eth_history/history_data_seeding.nim
@@ -12,12 +12,9 @@ import
   chronos,
   chronicles,
   ../network/wire/portal_protocol,
-  ../network/legacy_history/[
-    history_content,
-    history_network,
-    validation/block_proof_historical_hashes_accumulator,
-  ],
-  "."/[era1, history_data_ssz_e2s]
+  ../network/legacy_history/[history_content, history_network],
+  ./[era1, history_data_ssz_e2s],
+  ./block_proofs/block_proof_historical_hashes_accumulator
 
 from eth/rlp import computeRlpHash
 
diff --git a/portal/eth_history/history_data_ssz_e2s.nim b/portal/eth_history/history_data_ssz_e2s.nim
@@ -13,7 +13,8 @@ import
   results,
   eth/common/headers_rlp,
   ncli/e2store,
-  ../network/legacy_history/[history_content, validation/historical_hashes_accumulator]
+  ../network/legacy_history/history_content,
+  ./block_proofs/historical_hashes_accumulator
 
 export results
 
diff --git a/portal/eth_history/yaml_eth_types.nim b/portal/eth_history/yaml_eth_types.nim
diff --git a/portal/eth_history/yaml_utils.nim b/portal/eth_history/yaml_utils.nim
diff --git a/portal/network/legacy_history/history_validation.nim b/portal/network/legacy_history/history_validation.nim
@@ -14,7 +14,7 @@ import
   ../network_metadata,
   ../beacon/beacon_db,
   ./history_type_conversions,
-  ./validation/[
+  ../../eth_history/block_proofs/[
     block_proof_historical_hashes_accumulator, block_proof_historical_roots,
     block_proof_historical_summaries,
   ]
diff --git a/portal/network/network_metadata.nim b/portal/network/network_metadata.nim
@@ -13,7 +13,7 @@ import
   stew/io2,
   chronos/timer,
   beacon_chain/spec/forks,
-  ./legacy_history/validation/historical_hashes_accumulator
+  ../eth_history/block_proofs/historical_hashes_accumulator
 
 proc loadBootstrapNodes(path: string): seq[string] {.raises: [IOError].} =
   # Read a list of ENR URIs from a file containing a flat list of entries.
diff --git a/portal/network/portal_node.nim b/portal/network/portal_node.nim
@@ -13,7 +13,7 @@ import
   eth/p2p/discoveryv5/protocol,
   beacon_chain/spec/forks,
   stew/byteutils,
-  ../eth_data/history_data_ssz_e2s,
+  ../eth_history/history_data_ssz_e2s,
   ../database/content_db,
   ./network_metadata,
   ./wire/[portal_stream, portal_protocol_config],
diff --git a/portal/rpc/rpc_portal_debug_history_api.nim b/portal/rpc/rpc_portal_debug_history_api.nim
@@ -10,7 +10,7 @@
 import
   json_rpc/rpcserver,
   ../network/wire/portal_protocol,
-  ../eth_data/history_data_seeding,
+  ../eth_history/history_data_seeding,
   ../database/content_db,
   ./rpc_types
 
diff --git a/portal/scripts/test_portal_testnet.nim b/portal/scripts/test_portal_testnet.nim
@@ -18,9 +18,9 @@ import
   ../common/common_types,
   ../rpc/portal_rpc_client,
   ../rpc/eth_rpc_client,
-  ../eth_data/[history_data_seeding, history_data_json_store, history_data_ssz_e2s],
-  ../network/legacy_history/
-    [history_content, validation/block_proof_historical_hashes_accumulator],
+  ../eth_history/[history_data_seeding, history_data_json_store, history_data_ssz_e2s],
+  ../eth_history/block_proofs/block_proof_historical_hashes_accumulator,
+  ../network/legacy_history/history_content,
   ../tests/legacy_history_network_tests/test_history_util
 
 type
diff --git a/portal/tests/beacon_network_tests/test_beacon_content.nim b/portal/tests/beacon_network_tests/test_beacon_content.nim
@@ -17,7 +17,7 @@ import
   beacon_chain/spec/forks,
   ../../network/beacon/beacon_content,
   ../../network/beacon/beacon_init_loader,
-  ../../eth_data/yaml_utils,
+  ../../eth_history/yaml_utils,
   "."/light_client_test_data
 
 suite "Beacon Content Keys and Values - Test Vectors":
diff --git a/portal/tests/beacon_network_tests/test_beacon_historical_summaries_vectors.nim b/portal/tests/beacon_network_tests/test_beacon_historical_summaries_vectors.nim
@@ -14,7 +14,7 @@ import
   beacon_chain/networking/network_metadata,
   beacon_chain/spec/forks,
   ../../network/beacon/[beacon_chain_historical_summaries, beacon_content],
-  ../../eth_data/yaml_utils
+  ../../eth_history/yaml_utils
 
 from stew/bitops2 import log2trunc
 
diff --git a/portal/tests/eth_history_tests/all_eth_history_custom_chain_tests.nim b/portal/tests/eth_history_tests/all_eth_history_custom_chain_tests.nim
@@ -0,0 +1,10 @@
+# Nimbus
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.warning[UnusedImport]: off.}
+
+import ./test_historical_hashes_accumulator
diff --git a/portal/tests/eth_history_tests/all_eth_history_tests.nim b/portal/tests/eth_history_tests/all_eth_history_tests.nim
@@ -0,0 +1,16 @@
+# Nimbus
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.warning[UnusedImport]: off.}
+
+import
+  ./test_historical_hashes_accumulator_root,
+  ./test_block_proof_historical_roots,
+  ./test_block_proof_historical_roots_vectors,
+  ./test_block_proof_historical_summaries,
+  ./test_block_proof_historical_summaries_deneb,
+  ./test_block_proof_historical_summaries_vectors
diff --git a/portal/tests/eth_history_tests/test_block_proof_historical_roots.nim b/portal/tests/eth_history_tests/test_block_proof_historical_roots.nim
@@ -16,7 +16,7 @@ import
   beacon_chain /../ tests/testblockutil,
   # Mock helpers
   beacon_chain /../ tests/mocking/mock_genesis,
-  ../../network/legacy_history/validation/block_proof_historical_roots
+  ../../eth_history/block_proofs/block_proof_historical_roots
 
 # Test suite for the proofs:
 # - HistoricalRootsProof
diff --git a/portal/tests/eth_history_tests/test_block_proof_historical_roots_vectors.nim b/portal/tests/eth_history_tests/test_block_proof_historical_roots_vectors.nim
@@ -15,8 +15,8 @@ import
   yaml,
   beacon_chain/spec/datatypes/bellatrix,
   ../../network/network_metadata,
-  ../../network/legacy_history/validation/block_proof_historical_roots,
-  ../../eth_data/[yaml_utils, yaml_eth_types]
+  ../../eth_history/block_proofs/block_proof_historical_roots,
+  ../../eth_history/[yaml_utils, yaml_eth_types]
 
 suite "History Block Proofs - Historical Roots - Test Vectors":
   test "BlockProofHistoricalRoots for Execution BlockHeader":
diff --git a/portal/tests/eth_history_tests/test_block_proof_historical_summaries.nim b/portal/tests/eth_history_tests/test_block_proof_historical_summaries.nim
@@ -16,7 +16,7 @@ import
   beacon_chain /../ tests/testblockutil,
   # Mock helpers
   beacon_chain /../ tests/mocking/mock_genesis,
-  ../../network/legacy_history/validation/block_proof_historical_summaries
+  ../../eth_history/block_proofs/block_proof_historical_summaries
 
 # Test suite for the proofs:
 # - historicalSummariesProof
diff --git a/portal/tests/eth_history_tests/test_block_proof_historical_summaries_deneb.nim b/portal/tests/eth_history_tests/test_block_proof_historical_summaries_deneb.nim
@@ -16,7 +16,7 @@ import
   beacon_chain /../ tests/testblockutil,
   # Mock helpers
   beacon_chain /../ tests/mocking/mock_genesis,
-  ../../network/legacy_history/validation/block_proof_historical_summaries
+  ../../eth_history/block_proofs/block_proof_historical_summaries
 
 suite "History Block Proofs - Historical Summaries - Deneb":
   let
diff --git a/portal/tests/eth_history_tests/test_block_proof_historical_summaries_vectors.nim b/portal/tests/eth_history_tests/test_block_proof_historical_summaries_vectors.nim
@@ -16,10 +16,10 @@ import
   yaml,
   ssz_serialization,
   beacon_chain/spec/datatypes/capella,
-  ../../network/legacy_history/validation/block_proof_historical_summaries,
+  ../../eth_history/block_proofs/block_proof_historical_summaries,
   ../../network/beacon/beacon_init_loader,
-  ../../eth_data/[yaml_utils, yaml_eth_types],
-  ./test_history_util
+  ../../eth_history/[yaml_utils, yaml_eth_types],
+  ../legacy_history_network_tests/test_history_util
 
 suite "History Block Proofs - Historical Summaries - Test Vectors":
   test "BlockProofHistoricalSummaries for Execution BlockHeader":
diff --git a/portal/tests/eth_history_tests/test_historical_hashes_accumulator.nim b/portal/tests/eth_history_tests/test_historical_hashes_accumulator.nim
@@ -13,10 +13,9 @@ import
   unittest2,
   stint,
   eth/common/headers,
-  ../../eth_data/history_data_json_store,
-  ../../network/legacy_history/
-    [history_content, validation/block_proof_historical_hashes_accumulator],
-  ./test_history_util
+  ../../eth_history/history_data_json_store,
+  ../../eth_history/block_proofs/block_proof_historical_hashes_accumulator,
+  ../legacy_history_network_tests/test_history_util
 
 suite "Historical Hashes Accumulator":
   test "Historical Hashes Accumulator Canonical Verification":
diff --git a/portal/tests/eth_history_tests/test_historical_hashes_accumulator_root.nim b/portal/tests/eth_history_tests/test_historical_hashes_accumulator_root.nim
@@ -14,9 +14,8 @@ import
   stint,
   stew/byteutils,
   eth/common/headers,
-  ../../eth_data/history_data_json_store,
-  ../../network/legacy_history/
-    [history_content, validation/historical_hashes_accumulator]
+  ../../eth_history/history_data_json_store,
+  ../../eth_history/block_proofs/historical_hashes_accumulator
 
 suite "Historical Hashes Accumulator Root":
   test "Header Accumulator Update":
diff --git a/portal/tests/history_network_tests/test_history_content_values.nim b/portal/tests/history_network_tests/test_history_content_values.nim
@@ -12,7 +12,7 @@ import
   results,
   stew/byteutils,
   ../../common/common_types,
-  ../../eth_data/yaml_utils,
+  ../../eth_history/yaml_utils,
   ../../tools/eth_data_exporter/el_data_exporter,
   ../../network/history/history_validation
 
diff --git a/portal/tests/history_network_tests/test_history_endpoints.nim b/portal/tests/history_network_tests/test_history_endpoints.nim
@@ -11,7 +11,7 @@ import
   unittest2,
   stew/byteutils,
   chronos/unittest2/asynctests,
-  ../../eth_data/yaml_utils,
+  ../../eth_history/yaml_utils,
   ../../tools/eth_data_exporter/el_data_exporter,
   ../../network/wire/portal_protocol,
   ../../network/history/history_endpoints,
diff --git a/portal/tests/history_network_tests/test_history_network.nim b/portal/tests/history_network_tests/test_history_network.nim
@@ -11,7 +11,7 @@ import
   unittest2,
   stew/byteutils,
   chronos/unittest2/asynctests,
-  ../../eth_data/yaml_utils,
+  ../../eth_history/yaml_utils,
   ../../tools/eth_data_exporter/el_data_exporter,
   ../../network/wire/portal_protocol,
   ../../network/history/history_network,
diff --git a/portal/tests/legacy_history_network_tests/all_history_network_custom_chain_tests.nim b/portal/tests/legacy_history_network_tests/all_history_network_custom_chain_tests.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2024 Status Research & Development GmbH
+# Copyright (c) 2024-2025 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)
@@ -10,4 +10,4 @@
 # Note: These tests are separated because they require a custom merge block
 # number defined at compile time. Once runtime chain config gets added these
 # tests can be compiled together with all the other portal tests.
-import ./test_historical_hashes_accumulator, ./test_history_network
+import ./test_history_network
diff --git a/portal/tests/legacy_history_network_tests/all_history_network_tests.nim b/portal/tests/legacy_history_network_tests/all_history_network_tests.nim
@@ -8,12 +8,4 @@
 {.warning[UnusedImport]: off.}
 
 import
-  ./test_history_content_keys,
-  ./test_history_content,
-  ./test_history_content_validation,
-  ./test_historical_hashes_accumulator_root,
-  ./test_block_proof_historical_roots,
-  ./test_block_proof_historical_roots_vectors,
-  ./test_block_proof_historical_summaries,
-  ./test_block_proof_historical_summaries_deneb,
-  ./test_block_proof_historical_summaries_vectors
+  ./test_history_content_keys, ./test_history_content, ./test_history_content_validation
diff --git a/portal/tests/legacy_history_network_tests/test_history_content.nim b/portal/tests/legacy_history_network_tests/test_history_content.nim
@@ -15,10 +15,10 @@ import
   eth/common/headers_rlp,
   ../../network/network_metadata,
   ../../network/beacon/beacon_init_loader,
-  ../../eth_data/[history_data_json_store, history_data_ssz_e2s],
+  ../../eth_history/[history_data_json_store, history_data_ssz_e2s],
   ../../network/legacy_history/
     [history_content, history_type_conversions, history_validation],
-  ../../eth_data/yaml_utils,
+  ../../eth_history/yaml_utils,
   ./test_history_util
 
 from std/os import walkDir, splitFile, PathComponent
diff --git a/portal/tests/legacy_history_network_tests/test_history_content_validation.nim b/portal/tests/legacy_history_network_tests/test_history_content_validation.nim
@@ -1,5 +1,5 @@
 # Nimbus - Portal Network
-# Copyright (c) 2022-2024 Status Research & Development GmbH
+# Copyright (c) 2022-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -16,7 +16,7 @@ import
   results,
   eth/common/headers_rlp,
   ../../common/common_types,
-  ../../eth_data/history_data_json_store,
+  ../../eth_history/history_data_json_store,
   ../../network/legacy_history/[history_type_conversions, history_validation]
 
 const
diff --git a/portal/tests/legacy_history_network_tests/test_history_network.nim b/portal/tests/legacy_history_network_tests/test_history_network.nim
@@ -12,11 +12,7 @@ import
   eth/p2p/discoveryv5/routing_table,
   eth/common/[hashes, headers_rlp, blocks],
   ../../network/wire/[portal_protocol, portal_stream, portal_protocol_config],
-  ../../network/legacy_history/[
-    history_network,
-    history_content,
-    validation/block_proof_historical_hashes_accumulator,
-  ],
+  ../../network/legacy_history/[history_network, history_content],
   ../../database/content_db,
   ../../network/beacon/beacon_init_loader,
   ../test_helpers,
diff --git a/portal/tests/legacy_history_network_tests/test_history_util.nim b/portal/tests/legacy_history_network_tests/test_history_util.nim
diff --git a/portal/tools/eth_data_exporter.nim b/portal/tools/eth_data_exporter.nim
diff --git a/portal/tools/eth_data_exporter/cl_data_exporter.nim b/portal/tools/eth_data_exporter/cl_data_exporter.nim
diff --git a/portal/tools/eth_data_exporter/el_data_exporter.nim b/portal/tools/eth_data_exporter/el_data_exporter.nim
diff --git a/portal/tools/eth_data_exporter/exporter_common.nim b/portal/tools/eth_data_exporter/exporter_common.nim
