From 7c63c2bd5874df15e0102ed25bd46a1dcf54a9c5 Mon Sep 17 00:00:00 2001 From: arvidn Date: Thu, 23 Oct 2025 00:30:00 +0200 Subject: [PATCH 1/3] fix serialization of partial-proofs (list[uint64]) to bytes, for use as key in dictionary --- chia/_tests/farmer_harvester/test_farmer_harvester.py | 11 ++++++----- chia/farmer/farmer_api.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/chia/_tests/farmer_harvester/test_farmer_harvester.py b/chia/_tests/farmer_harvester/test_farmer_harvester.py index 481d16ea8b1d..5f9a0b4fe7c8 100644 --- a/chia/_tests/farmer_harvester/test_farmer_harvester.py +++ b/chia/_tests/farmer_harvester/test_farmer_harvester.py @@ -17,6 +17,7 @@ from chia._tests.util.time_out_assert import time_out_assert from chia.cmds.cmds_util import get_any_service_client from chia.farmer.farmer import Farmer +from chia.farmer.farmer_api import serialize from chia.farmer.farmer_service import FarmerService from chia.harvester.harvester_rpc_client import HarvesterRpcClient from chia.harvester.harvester_service import HarvesterService @@ -460,7 +461,7 @@ async def test_solution_response_handler( harvester_peer = await get_harvester_peer(farmer) # manually add pending request - key = bytes(partial_proofs.partial_proofs[0]) + key = serialize(partial_proofs.partial_proofs[0]) farmer.pending_solver_requests[key] = { "proof_data": partial_proofs, "peer": harvester_peer, @@ -486,7 +487,7 @@ async def test_solution_response_handler( assert original_peer == harvester_peer # verify pending request was removed - key = bytes(partial_proofs.partial_proofs[0]) + key = serialize(partial_proofs.partial_proofs[0]) assert key not in farmer.pending_solver_requests @@ -544,7 +545,7 @@ async def test_solution_response_empty_proof( harvester_peer.peer_node_id = "harvester_peer" # manually add pending request - key = bytes(partial_proofs.partial_proofs[0]) + key = serialize(partial_proofs.partial_proofs[0]) farmer.pending_solver_requests[key] = { "proof_data": partial_proofs, "peer": harvester_peer, @@ -563,7 +564,7 @@ async def test_solution_response_empty_proof( mock_new_proof.assert_not_called() # verify pending request was removed (cleanup still happens) - key = bytes(partial_proofs.partial_proofs[0]) + key = serialize(partial_proofs.partial_proofs[0]) assert key not in farmer.pending_solver_requests @@ -612,5 +613,5 @@ async def test_v2_partial_proofs_solver_exception( await farmer_api.partial_proofs(partial_proofs, harvester_peer) # verify pending request was cleaned up after exception - key = bytes(partial_proofs.partial_proofs[0]) + key = serialize(partial_proofs.partial_proofs[0]) assert key not in farmer.pending_solver_requests diff --git a/chia/farmer/farmer_api.py b/chia/farmer/farmer_api.py index 25b39f3ed6a3..db026304d47e 100644 --- a/chia/farmer/farmer_api.py +++ b/chia/farmer/farmer_api.py @@ -49,6 +49,13 @@ ) +def serialize(partial_proof: list[uint64]) -> bytes: + key = bytearray() + for val in partial_proof: + key += val.stream_to_bytes() + return bytes(key) + + class FarmerAPI: if TYPE_CHECKING: from chia.server.api_protocol import ApiProtocol @@ -514,7 +521,7 @@ async def partial_proofs(self, partial_proof_data: PartialProofsData, peer: WSCh size=partial_proof_data.plot_size, ) - key = bytes(partial_proof) + key = serialize(partial_proof) try: # store pending request data for matching with response self.farmer.pending_solver_requests[key] = { @@ -543,7 +550,7 @@ async def solution_response(self, response: SolverResponse, peer: WSChiaConnecti # find the matching pending request using partial_proof - key = bytes(response.partial_proof) + key = serialize(response.partial_proof) if key not in self.farmer.pending_solver_requests: self.farmer.log.warning(f"Received solver response for unknown partial proof {response.partial_proof[:5]}") return From 3d3a28d6039f7665b9fe73c309fff9035e321f8e Mon Sep 17 00:00:00 2001 From: arvidn Date: Thu, 16 Oct 2025 10:45:35 +0200 Subject: [PATCH 2/3] improve v2-plot support in plot-sync --- chia/_tests/plot_sync/test_delta.py | 1 + chia/_tests/plot_sync/test_plot_sync.py | 1 + chia/_tests/plot_sync/test_receiver.py | 1 + chia/_tests/util/network_protocol_data.py | 1 + chia/_tests/util/protocol_messages_bytes-v1.0 | Bin 51163 -> 51165 bytes chia/_tests/util/protocol_messages_json.py | 2 ++ chia/harvester/harvester.py | 4 ++- chia/harvester/harvester_api.py | 1 + chia/plot_sync/receiver.py | 24 ++++++++++-------- chia/plot_sync/sender.py | 20 +++++++++++---- chia/protocols/harvester_protocol.py | 1 + 11 files changed, 40 insertions(+), 16 deletions(-) diff --git a/chia/_tests/plot_sync/test_delta.py b/chia/_tests/plot_sync/test_delta.py index c5b4b5b1ff97..11c7433b771e 100644 --- a/chia/_tests/plot_sync/test_delta.py +++ b/chia/_tests/plot_sync/test_delta.py @@ -17,6 +17,7 @@ def dummy_plot(path: str) -> Plot: return Plot( filename=path, size=uint8(32), + strength=uint8(0), plot_id=bytes32(b"\00" * 32), pool_public_key=G1Element(), pool_contract_puzzle_hash=None, diff --git a/chia/_tests/plot_sync/test_plot_sync.py b/chia/_tests/plot_sync/test_plot_sync.py index 58d79295600d..e0993f11bbe8 100644 --- a/chia/_tests/plot_sync/test_plot_sync.py +++ b/chia/_tests/plot_sync/test_plot_sync.py @@ -68,6 +68,7 @@ def create_mock_plot(info: MockPlotInfo) -> Plot: return Plot( info.prover.get_filename(), uint8(0), + uint8(0), bytes32.zeros, None, None, diff --git a/chia/_tests/plot_sync/test_receiver.py b/chia/_tests/plot_sync/test_receiver.py index 0d155b579435..1074f2e0c3cc 100644 --- a/chia/_tests/plot_sync/test_receiver.py +++ b/chia/_tests/plot_sync/test_receiver.py @@ -171,6 +171,7 @@ def plot_sync_setup(seeded_random: random.Random) -> tuple[Receiver, list[SyncSt Plot( filename=str(x), size=uint8(0), + strength=uint8(0), plot_id=bytes32.random(seeded_random), pool_contract_puzzle_hash=None, pool_public_key=None, diff --git a/chia/_tests/util/network_protocol_data.py b/chia/_tests/util/network_protocol_data.py index fbf90ecda509..6b0d3a85385c 100644 --- a/chia/_tests/util/network_protocol_data.py +++ b/chia/_tests/util/network_protocol_data.py @@ -918,6 +918,7 @@ plot = harvester_protocol.Plot( "plot_1", uint8(124), + uint8(0), bytes32(bytes.fromhex("b2eb7e5c5239e8610a9dd0e137e185966ebb430faf31ae4a0e55d86251065b98")), G1Element.from_bytes( bytes.fromhex( diff --git a/chia/_tests/util/protocol_messages_bytes-v1.0 b/chia/_tests/util/protocol_messages_bytes-v1.0 index b080d9b669d9c34bc277f2986708084e5d1572f9..0ab939bc652455e965bbf92bd3ec5dad01e6f5ce 100644 GIT binary patch delta 54 ycmccJ&wRI^d4t#r-UAE_3~U8C`6cm&H4KyAuGlUL6kxao#Ei%?o8PW@CkFuY1QFc; delta 52 xcmccH&wRU|d4t#rp8X6A3~U8C`6cm&HIv`3*e(L(Gu#AXMkI;N?^nE&0|4As5#0a) diff --git a/chia/_tests/util/protocol_messages_json.py b/chia/_tests/util/protocol_messages_json.py index 5ea336bad695..b3d8eb2f8388 100644 --- a/chia/_tests/util/protocol_messages_json.py +++ b/chia/_tests/util/protocol_messages_json.py @@ -2297,6 +2297,7 @@ plot_json: dict[str, Any] = { "filename": "plot_1", "size": 124, + "strength": 0, "plot_id": "0xb2eb7e5c5239e8610a9dd0e137e185966ebb430faf31ae4a0e55d86251065b98", "pool_public_key": "0xa04c6b5ac7dfb935f6feecfdd72348ccf1d4be4fe7e26acf271ea3b7d308da61e0a308f7a62495328a81f5147b66634c", "pool_contract_puzzle_hash": "0x1c96d26def7be696f12e7ebb91d50211e6217ce5d9087c9cd1b84782d5d4b237", @@ -2313,6 +2314,7 @@ { "filename": "plot_1", "size": 124, + "strength": 0, "plot_id": "0xb2eb7e5c5239e8610a9dd0e137e185966ebb430faf31ae4a0e55d86251065b98", "pool_public_key": "0xa04c6b5ac7dfb935f6feecfdd72348ccf1d4be4fe7e26acf271ea3b7d308da61e0a308f7a62495328a81f5147b66634c", "pool_contract_puzzle_hash": "0x1c96d26def7be696f12e7ebb91d50211e6217ce5d9087c9cd1b84782d5d4b237", diff --git a/chia/harvester/harvester.py b/chia/harvester/harvester.py index 04fb200e2e9a..412d6ba5c5d5 100644 --- a/chia/harvester/harvester.py +++ b/chia/harvester/harvester.py @@ -193,14 +193,16 @@ def get_plots(self) -> tuple[list[dict[str, Any]], list[str], list[str]]: size = prover.get_size() if size.size_v1 is not None: k = size.size_v1 + strength = 0 else: assert size.size_v2 is not None k = size.size_v2 - # TODO: todo_v2_plots support v2 plots in RPC response + strength = prover.get_strength() response_plots.append( { "filename": str(path), "size": k, + "strength": strength, "plot_id": prover.get_id(), "pool_public_key": plot_info.pool_public_key, "pool_contract_puzzle_hash": plot_info.pool_contract_puzzle_hash, diff --git a/chia/harvester/harvester_api.py b/chia/harvester/harvester_api.py index 2e8199d25521..790ad833606b 100644 --- a/chia/harvester/harvester_api.py +++ b/chia/harvester/harvester_api.py @@ -516,6 +516,7 @@ async def request_plots(self, _: harvester_protocol.RequestPlots) -> Message: Plot( plot["filename"], plot["size"], + plot["strength"], plot["plot_id"], plot["pool_public_key"], plot["pool_contract_puzzle_hash"], diff --git a/chia/plot_sync/receiver.py b/chia/plot_sync/receiver.py index bec063b03d5e..ff7e8a01efe6 100644 --- a/chia/plot_sync/receiver.py +++ b/chia/plot_sync/receiver.py @@ -113,8 +113,8 @@ def __init__( async def trigger_callback(self, update: Optional[Delta] = None) -> None: try: await self._update_callback(self._connection.peer_node_id, update) - except Exception as e: - log.error(f"_update_callback: node_id {self.connection().peer_node_id}, raised {e}") + except Exception: + log.exception(f"_update_callback: node_id {self.connection().peer_node_id}") def reset(self) -> None: log.info(f"reset: node_id {self.connection().peer_node_id}, current_sync: {self._current_sync}") @@ -181,13 +181,13 @@ async def send_response(plot_sync_error: Optional[PlotSyncError] = None) -> None await method(message) await send_response() except InvalidIdentifierError as e: - log.warning(f"_process: node_id {self.connection().peer_node_id}, InvalidIdentifierError {e}") + log.exception(f"_process: node_id {self.connection().peer_node_id}") await send_response(PlotSyncError(int16(e.error_code), f"{e}", e.expected_identifier)) except PlotSyncException as e: - log.warning(f"_process: node_id {self.connection().peer_node_id}, Error {e}") + log.exception(f"_process: node_id {self.connection().peer_node_id}") await send_response(PlotSyncError(int16(e.error_code), f"{e}", None)) except Exception as e: - log.warning(f"_process: node_id {self.connection().peer_node_id}, Exception {e}") + log.exception(f"_process: node_id {self.connection().peer_node_id}") await send_response(PlotSyncError(int16(ErrorCodes.unknown), f"{e}", None)) def _validate_identifier(self, identifier: PlotSyncIdentifier, start: bool = False) -> None: @@ -349,12 +349,16 @@ async def _sync_done(self, data: PlotSyncDone) -> None: self._keys_missing = self._current_sync.delta.keys_missing.additions.copy() self._duplicates = self._current_sync.delta.duplicates.additions.copy() self._total_plot_size = sum(plot.file_size for plot in self._plots.values()) + + def expected_plot_size(pi: Plot) -> int: + if pi.strength == 0: + plot_size = PlotSize.make_v1(pi.size) + else: + plot_size = PlotSize.make_v2(pi.size) + return int(_expected_plot_size(plot_size)) + self._total_effective_plot_size = int( - # TODO: todo_v2_plots support v2 plots - sum( - UI_ACTUAL_SPACE_CONSTANT_FACTOR * int(_expected_plot_size(PlotSize.make_v1(plot.size))) - for plot in self._plots.values() - ) + sum(UI_ACTUAL_SPACE_CONSTANT_FACTOR * expected_plot_size(plot) for plot in self._plots.values()) ) # Save current sync as last sync and create a new current sync self._last_sync = self._current_sync diff --git a/chia/plot_sync/sender.py b/chia/plot_sync/sender.py index 46189c2c53a0..c8dd448ebef8 100644 --- a/chia/plot_sync/sender.py +++ b/chia/plot_sync/sender.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any, Generic, Optional, TypeVar -from chia_rs.sized_ints import int16, uint32, uint64 +from chia_rs.sized_ints import int16, uint8, uint32, uint64 from typing_extensions import Protocol from chia.plot_sync.exceptions import AlreadyStartedError, InvalidConnectionTypeError @@ -37,13 +37,23 @@ def _convert_plot_info_list(plot_infos: list[PlotInfo]) -> list[Plot]: converted: list[Plot] = [] for plot_info in plot_infos: - # TODO: todo_v2_plots support v2 plots - k = plot_info.prover.get_size().size_v1 - assert k is not None + plot_size = plot_info.prover.get_size() + if plot_size.size_v1 is not None: + k = plot_size.size_v1 + strength = 0 + else: + # todo_v2_plots the k-size should probably be set to the constant, + # fixed k-size for v2 plots. Then we would need access to + # ConsensusConstants in here. + assert plot_size.size_v2 is not None + k = plot_size.size_v2 + strength = plot_info.prover.get_strength() + converted.append( Plot( filename=plot_info.prover.get_filename(), - size=k, + size=uint8(k), + strength=uint8(strength), plot_id=plot_info.prover.get_id(), pool_public_key=plot_info.pool_public_key, pool_contract_puzzle_hash=plot_info.pool_contract_puzzle_hash, diff --git a/chia/protocols/harvester_protocol.py b/chia/protocols/harvester_protocol.py index 920ade68f4c6..e2ea78153e72 100644 --- a/chia/protocols/harvester_protocol.py +++ b/chia/protocols/harvester_protocol.py @@ -143,6 +143,7 @@ class RespondSignatures(Streamable): class Plot(Streamable): filename: str size: uint8 + strength: uint8 # This is 0 for v1 plots plot_id: bytes32 pool_public_key: Optional[G1Element] pool_contract_puzzle_hash: Optional[bytes32] From 45bc771f9ccac862671025747be40b360f96d6a8 Mon Sep 17 00:00:00 2001 From: arvidn Date: Thu, 16 Oct 2025 10:50:55 +0200 Subject: [PATCH 3/3] add support for creating v2 plots in block_tools --- chia/_tests/conftest.py | 1 + .../core/custom_types/test_proof_of_space.py | 21 ++--- chia/_tests/core/test_farmer_harvester_rpc.py | 22 ++++- .../test_filter_prefix_bits.py | 2 +- chia/plotting/create_plots.py | 64 ++++++++++++- chia/plotting/manager.py | 4 +- chia/simulator/block_tools.py | 89 ++++++++++++++++++- chia/simulator/start_simulator.py | 4 +- .../types/blockchain_format/proof_of_space.py | 12 +++ 9 files changed, 197 insertions(+), 22 deletions(-) diff --git a/chia/_tests/conftest.py b/chia/_tests/conftest.py index 9305887a267b..d3facf98d5fd 100644 --- a/chia/_tests/conftest.py +++ b/chia/_tests/conftest.py @@ -1288,6 +1288,7 @@ async def farmer_harvester_2_simulators_zero_bits_plot_filter( num_og_plots=0, num_pool_plots=0, num_non_keychain_plots=0, + num_v2_plots=0, config_overrides=config_overrides, ) for _ in range(2) diff --git a/chia/_tests/core/custom_types/test_proof_of_space.py b/chia/_tests/core/custom_types/test_proof_of_space.py index 1c09031f35db..162be96983ee 100644 --- a/chia/_tests/core/custom_types/test_proof_of_space.py +++ b/chia/_tests/core/custom_types/test_proof_of_space.py @@ -112,25 +112,23 @@ def b32(key: str) -> bytes32: id="v2 plot size 0", pos_challenge=bytes32(b"1" * 32), plot_size=PlotSize.make_v2(0), + pool_contract_puzzle_hash=bytes32(b"1" * 32), plot_public_key=G1Element(), - pool_public_key=G1Element(), expected_error="Plot size (0) is lower than the minimum (28)", ), ProofOfSpaceCase( id="v2 plot size 34", pos_challenge=bytes32(b"1" * 32), plot_size=PlotSize.make_v2(34), + pool_contract_puzzle_hash=bytes32(b"1" * 32), plot_public_key=G1Element(), - pool_public_key=G1Element(), expected_error="Plot size (34) is higher than the maximum (32)", ), ProofOfSpaceCase( id="Not passing the plot filter v2", - pos_challenge=b32("3d29ea79d19b3f7e99ebf764ae53697cbe143603909873946af6ab1ece606861"), + pos_challenge=b32("4cfaacbd2782db64d07cf490ca938534adb07dfbd2f92b0e479e2e5b196178db"), plot_size=PlotSize.make_v2(32), - pool_public_key=g1( - "b6449c2c68df97c19e884427e42ee7350982d4020571ead08732615ff39bd216bfd630b6460784982bec98b49fea79d0" - ), + pool_contract_puzzle_hash=bytes32(b"1" * 32), plot_public_key=g1( "879526b4e7b616cfd64984d8ad140d0798b048392a6f11e2faf09054ef467ea44dc0dab5e5edb2afdfa850c5c8b629cc" ), @@ -160,16 +158,15 @@ def test_verify_and_get_quality_string(caplog: pytest.LogCaptureFixture, case: P @datacases( ProofOfSpaceCase( id="v2 plot are not implemented", - plot_size=PlotSize.make_v2(30), - pos_challenge=b32("47deb938e145d25d7b3b3c85ca9e3972b76c01aeeb78a02fe5d3b040d282317e"), + plot_size=PlotSize.make_v2(28), + pos_challenge=b32("6a85c4c5f21e5728c6668b01c0758e33bb7f3ae20d38703d4863ad151f983d9c"), plot_public_key=g1( "afa3aaf09c03885154be49216ee7fb2e4581b9c4a4d7e9cc402e27280bf0cfdbdf1b9ba674e301fd1d1450234b3b1868" ), - pool_public_key=g1( - "b6449c2c68df97c19e884427e42ee7350982d4020571ead08732615ff39bd216bfd630b6460784982bec98b49fea79d0" - ), - expected_error="NotImplementedError", + pool_contract_puzzle_hash=bytes32(b"1" * 32), + expected_error="Did not pass the plot filter", ), + # TODO: todo_v2_plots add test case that passes the plot filter ) def test_verify_and_get_quality_string_v2(caplog: pytest.LogCaptureFixture, case: ProofOfSpaceCase) -> None: pos = make_pos( diff --git a/chia/_tests/core/test_farmer_harvester_rpc.py b/chia/_tests/core/test_farmer_harvester_rpc.py index b90f5821ebee..fe089fcecf1c 100644 --- a/chia/_tests/core/test_farmer_harvester_rpc.py +++ b/chia/_tests/core/test_farmer_harvester_rpc.py @@ -87,7 +87,7 @@ async def non_zero_plots() -> bool: res = await harvester_rpc_client.get_plots() nonlocal harvester_plots harvester_plots = res["plots"] - return len(harvester_plots) > 0 + return len(harvester_plots) == 340 await time_out_assert(10, non_zero_plots) @@ -293,7 +293,7 @@ async def wait_for_plot_sync() -> bool: await farmer_api.farmer.update_pool_state() pool_plot_count: int = (await farmer_rpc_client.get_pool_state())["pool_state"][0]["plot_count"] - assert pool_plot_count == 5 + assert pool_plot_count == 325 # TODO: Maybe improve this to not remove from Receiver directly but instead from the harvester and then wait for # plot sync event. @@ -333,8 +333,8 @@ def test_plot_matches_filter(filter_item: FilterItem, match: bool) -> None: @pytest.mark.parametrize( "endpoint, filtering, sort_key, reverse, expected_plot_count", [ - (FarmerRpcClient.get_harvester_plots_valid, [], "filename", False, 20), - (FarmerRpcClient.get_harvester_plots_valid, [], "size", True, 20), + (FarmerRpcClient.get_harvester_plots_valid, [], "filename", False, 340), + (FarmerRpcClient.get_harvester_plots_valid, [], "size", True, 340), ( FarmerRpcClient.get_harvester_plots_valid, [FilterItem("pool_contract_puzzle_hash", None)], @@ -349,6 +349,20 @@ def test_plot_matches_filter(filter_item: FilterItem, match: bool) -> None: False, 4, ), + ( + FarmerRpcClient.get_harvester_plots_valid, + [FilterItem("strength", "2")], + "size", + True, + 320, + ), + ( + FarmerRpcClient.get_harvester_plots_valid, + [FilterItem("strength", "0")], + "size", + True, + 20, + ), (FarmerRpcClient.get_harvester_plots_invalid, [], None, True, 13), (FarmerRpcClient.get_harvester_plots_invalid, ["invalid_0"], None, False, 6), (FarmerRpcClient.get_harvester_plots_invalid, ["inval", "lid_1"], None, False, 2), diff --git a/chia/_tests/farmer_harvester/test_filter_prefix_bits.py b/chia/_tests/farmer_harvester/test_filter_prefix_bits.py index 78941d9535dd..0a52ea47e63a 100644 --- a/chia/_tests/farmer_harvester/test_filter_prefix_bits.py +++ b/chia/_tests/farmer_harvester/test_filter_prefix_bits.py @@ -84,7 +84,7 @@ async def have_connections() -> bool: await harvester_rpc_cl.await_closed() -@pytest.mark.parametrize(argnames=["peak_height", "eligible_plots"], argvalues=[(5495999, 0), (5496000, 1)]) +@pytest.mark.parametrize(argnames=["peak_height", "eligible_plots"], argvalues=[(5495999, 9), (5496000, 10)]) @pytest.mark.anyio async def test_filter_prefix_bits_with_farmer_harvester( farmer_harvester_with_filter_size_9: tuple[HarvesterService, FarmerAPI], diff --git a/chia/plotting/create_plots.py b/chia/plotting/create_plots.py index 14b056c88046..e395135cce62 100644 --- a/chia/plotting/create_plots.py +++ b/chia/plotting/create_plots.py @@ -5,8 +5,9 @@ from pathlib import Path from typing import Optional -from chia_rs import AugSchemeMPL, G1Element, PrivateKey +from chia_rs import AugSchemeMPL, G1Element, PrivateKey, create_v2_plot from chia_rs.sized_bytes import bytes32 +from chia_rs.sized_ints import uint8 from chiapos import DiskPlotter from chia.daemon.keychain_proxy import KeychainProxy, connect_to_keychain_and_validate, wrap_local_keychain @@ -14,6 +15,7 @@ from chia.types.blockchain_format.proof_of_space import ( calculate_plot_id_ph, calculate_plot_id_pk, + calculate_plot_id_v2, generate_plot_public_key, ) from chia.util.bech32m import decode_puzzle_hash @@ -275,3 +277,63 @@ async def create_plots( log.info(created_path.name) return created_plots, existing_plots + + +async def create_v2_plots( + final_dir: Path, + *, + pool_ph: bytes32, + farmer_pk: G1Element, + size: int = 28, + strength: int = 2, + num: int = 1, + use_datetime: bool = True, + test_private_keys: Optional[list[PrivateKey]] = None, +) -> tuple[dict[bytes32, Path], dict[bytes32, Path]]: + log.info( + f"Creating {num} plots of size {size}, pool contract address: " + f"{pool_ph} farmer public key: {bytes(farmer_pk).hex()}" + ) + + final_dir.mkdir(parents=True, exist_ok=True) + + created_plots: dict[bytes32, Path] = {} + existing_plots: dict[bytes32, Path] = {} + for i in range(num): + # Generate a random master secret key + if test_private_keys is not None: + assert len(test_private_keys) == num + sk: PrivateKey = test_private_keys[i] + else: + sk = AugSchemeMPL.key_gen(bytes32.secret()) + + # The plot public key is the combination of the harvester and farmer keys + # New plots will also include a taproot of the keys, for extensibility + plot_public_key = generate_plot_public_key(master_sk_to_local_sk(sk).get_g1(), farmer_pk, include_taproot=True) + + # The plot id is based on the harvester, farmer, pool contract puzzle + # hash and strength + plot_id = calculate_plot_id_v2(pool_ph, plot_public_key, uint8(strength)) + plot_memo = stream_plot_info_ph(pool_ph, farmer_pk, sk) + + dt_string = datetime.now().strftime("%Y-%m-%d-%H-%M") + + if use_datetime: + filename: str = f"plot-k{size}-{dt_string}-{plot_id}.plot2" + else: + filename = f"plot-k{size}-{plot_id}.plot2" + full_path: Path = final_dir / filename + + if not full_path.exists(): + log.info(f"Starting plot {i + 1}/{num}") + create_v2_plot(str(full_path.absolute()), size, strength, plot_id, plot_memo) + created_plots[plot_id] = full_path + else: + log.info(f"Plot {filename} already exists") + existing_plots[plot_id] = full_path + + log.info(f"Created a total of {len(created_plots)} new plots") + for created_path in created_plots.values(): + log.info(created_path.name) + + return created_plots, existing_plots diff --git a/chia/plotting/manager.py b/chia/plotting/manager.py index 6ba06c8fa005..8996b28259f1 100644 --- a/chia/plotting/manager.py +++ b/chia/plotting/manager.py @@ -278,8 +278,8 @@ def _refresh_task(self, sleep_interval_ms: int): f"total_result.removed {len(total_result.removed)}, " f"total_duration {total_result.duration:.2f} seconds" ) - except Exception as e: - log.error(f"_refresh_callback raised: {e} with the traceback: {traceback.format_exc()}") + except Exception: + log.exception("_refresh_callback raised") self.reset() def refresh_batch(self, plot_paths: list[Path], plot_directories: set[Path]) -> PlotRefreshResult: diff --git a/chia/simulator/block_tools.py b/chia/simulator/block_tools.py index c97ecd7a6306..c404c49e9a99 100644 --- a/chia/simulator/block_tools.py +++ b/chia/simulator/block_tools.py @@ -63,7 +63,7 @@ from chia.consensus.vdf_info_computation import get_signage_point_vdf_info from chia.daemon.keychain_proxy import KeychainProxy, connect_to_keychain_and_validate, wrap_local_keychain from chia.full_node.bundle_tools import simple_solution_generator, simple_solution_generator_backrefs -from chia.plotting.create_plots import PlotKeys, create_plots +from chia.plotting.create_plots import PlotKeys, create_plots, create_v2_plots from chia.plotting.manager import PlotManager from chia.plotting.prover import PlotVersion, QualityProtocol, V1Prover, V2Prover, V2Quality from chia.plotting.util import ( @@ -319,6 +319,7 @@ def __init__( self.temp_dir.mkdir(parents=True, exist_ok=True) self.expected_plots: dict[bytes32, Path] = {} self.created_plots: int = 0 + self.created_plots2: int = 0 self.total_result = PlotRefreshResult() def test_callback(event: PlotRefreshEvents, update_result: PlotRefreshResult) -> None: @@ -338,6 +339,29 @@ def test_callback(event: PlotRefreshEvents, update_result: PlotRefreshResult) -> assert self.total_result.processed == update_result.processed assert self.total_result.duration == update_result.duration assert update_result.remaining == 0 + + expected_plots: set[str] = set() + found_plots: set[str] = set() + if len(self.plot_manager.plots) != len(self.expected_plots): + for pid, filename in self.expected_plots.items(): + expected_plots.add(filename.name) + for filename, _ in self.plot_manager.plots.items(): + found_plots.add(filename.name) + print(f"directory: {self.plot_dir}") + print(f"expected: {len(expected_plots)}") + for f in expected_plots: + print(f) + print(f"plot manager: {len(found_plots)}") + for f in found_plots: + print(f) + diff = found_plots.difference(expected_plots) + print(f"found unexpected: {len(diff)}") + for f in diff: + print(f) + diff = expected_plots.difference(found_plots) + print(f"not found: {len(diff)}") + for f in diff: + print(f) assert len(self.plot_manager.plots) == len(self.expected_plots) self.plot_manager: PlotManager = PlotManager( @@ -498,11 +522,25 @@ async def setup_plots( num_og_plots: int = 15, num_pool_plots: int = 5, num_non_keychain_plots: int = 3, + num_v2_plots: int = 320, plot_size: int = 20, bitfield: bool = True, ) -> bool: + # we have 20 v1 plots, each about 16.6 MB + # in order to balance that out with v2 plots, we need approximately the + # same size on disk. A v2 k-18 plot is expected to be about 1 MB, so we + # need about 16 times more v2 plots, i.e. 320. + # Additionally, the current reference plots are quite a bit larger than + # the expected final plots. So until we have the optimized plot format + # completed, the actual disk space for the v2 plots will be larger. + print( + f"setup_plots({num_og_plots}, {num_pool_plots}, " + f"{num_non_keychain_plots}, {num_v2_plots}) " + f"plot-dir: {self.plot_dir}" + ) self.add_plot_directory(self.plot_dir) assert self.created_plots == 0 + assert self.created_plots2 == 0 existing_plots: bool = True # OG Plots for i in range(num_og_plots): @@ -525,6 +563,11 @@ async def setup_plots( ) if plot.new_plot: existing_plots = False + # v2 plots + for i in range(num_v2_plots): + plot = await self.new_plot2(plot_size=18) + if plot.new_plot: + existing_plots = False await self.refresh_plots() assert len(self.plot_manager.plots) == len(self.expected_plots) return existing_plots @@ -604,6 +647,48 @@ async def new_plot( shutil.rmtree(self.temp_dir, ignore_errors=True) sys.exit(1) + async def new_plot2( + self, + path: Optional[Path] = None, + exclude_plots: bool = False, + plot_size: int = 18, + ) -> BlockToolsNewPlotResult: + final_dir = self.plot_dir + if path is not None: + final_dir = path + final_dir.mkdir(parents=True, exist_ok=True) + + # No datetime in the filename, to get deterministic filenames and not re-plot + created, existed = await create_v2_plots( + final_dir=Path(final_dir), + size=plot_size, + pool_ph=self.pool_ph, + farmer_pk=self.farmer_pk, + use_datetime=False, + test_private_keys=[AugSchemeMPL.key_gen(std_hash(self.created_plots2.to_bytes(2, "big")))], + ) + self.created_plots2 += 1 + + plot_id_new: Optional[bytes32] = None + path_new: Optional[Path] = None + new_plot: bool = True + + if len(created): + assert len(existed) == 0 + plot_id_new, path_new = next(iter(created.items())) + + if len(existed): + assert len(created) == 0 + plot_id_new, path_new = next(iter(existed.items())) + new_plot = False + assert plot_id_new is not None + assert path_new is not None + + if not exclude_plots: + self.expected_plots[plot_id_new] = path_new + + return BlockToolsNewPlotResult(plot_id_new, new_plot) + async def refresh_plots(self) -> None: self.plot_manager.refresh_parameter = replace( self.plot_manager.refresh_parameter, batch_size=uint32(4 if len(self.expected_plots) % 3 == 0 else 3) @@ -2080,6 +2165,7 @@ async def create_block_tools_async( num_og_plots: int = 15, num_pool_plots: int = 5, num_non_keychain_plots: int = 3, + num_v2_plots: int = 320, ) -> BlockTools: global create_block_tools_async_count create_block_tools_async_count += 1 @@ -2090,6 +2176,7 @@ async def create_block_tools_async( num_og_plots=num_og_plots, num_pool_plots=num_pool_plots, num_non_keychain_plots=num_non_keychain_plots, + num_v2_plots=num_v2_plots, ) return bt diff --git a/chia/simulator/start_simulator.py b/chia/simulator/start_simulator.py index 0e2967f340bd..8899993ec6e1 100644 --- a/chia/simulator/start_simulator.py +++ b/chia/simulator/start_simulator.py @@ -117,7 +117,9 @@ async def async_main( plot_dir=plot_dir, ) await bt.setup_keys(fingerprint=fingerprint, reward_ph=farming_puzzle_hash) - await bt.setup_plots(num_og_plots=PLOTS, num_pool_plots=0, num_non_keychain_plots=0, plot_size=PLOT_SIZE) + await bt.setup_plots( + num_og_plots=PLOTS, num_pool_plots=0, num_non_keychain_plots=0, plot_size=PLOT_SIZE, num_v2_plots=PLOTS + ) # Everything after this is not simulator specific, excluding the if test_mode. initialize_logging( service_name=SERVICE_NAME, diff --git a/chia/types/blockchain_format/proof_of_space.py b/chia/types/blockchain_format/proof_of_space.py index 4b949a930b90..755b1fbc093d 100644 --- a/chia/types/blockchain_format/proof_of_space.py +++ b/chia/types/blockchain_format/proof_of_space.py @@ -59,6 +59,14 @@ def make_pos( def get_plot_id(pos: ProofOfSpace) -> bytes32: assert pos.pool_public_key is None or pos.pool_contract_puzzle_hash is None + + plot_size = pos.size() + if plot_size.size_v2 is not None: + assert pos.pool_contract_puzzle_hash + # v2 plots have a fixed k-size, so we store the *strength* in this field + # instead + return calculate_plot_id_v2(pos.pool_contract_puzzle_hash, pos.plot_public_key, uint8(plot_size.size_v2)) + if pos.pool_public_key is None: assert pos.pool_contract_puzzle_hash is not None return calculate_plot_id_ph(pos.pool_contract_puzzle_hash, pos.plot_public_key) @@ -203,6 +211,10 @@ def calculate_plot_id_ph( return std_hash(bytes(pool_contract_puzzle_hash) + bytes(plot_public_key)) +def calculate_plot_id_v2(pool_contract_puzzle_hash: bytes32, plot_public_key: G1Element, strength: uint8) -> bytes32: + return std_hash(bytes(pool_contract_puzzle_hash) + bytes(plot_public_key) + strength.stream_to_bytes()) + + def generate_taproot_sk(local_pk: G1Element, farmer_pk: G1Element) -> PrivateKey: taproot_message: bytes = bytes(local_pk + farmer_pk) + bytes(local_pk) + bytes(farmer_pk) taproot_hash: bytes32 = std_hash(taproot_message)