From 16ec330a79af93518ac3b82b2ffa462424191e16 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 28 Aug 2019 02:05:19 +1000 Subject: [PATCH 01/47] Started aligning API spec with implementation. - Adding some missing fields to structs - Rearranged the endpoints in the rest_api router, and renamed, using an 'implementation_pending' function - Added 'content-type' headers, to distinguish difference with /node/metrics - Updated OpenAPI spec to v0.2.0 - Split /node/fork into /node/chain_id and /beacon/fork - Moved /metrics to /node/metrics - Added example to /node/metrics, since it's text/plain - Moved /node/network to just /network - Added lots of stubs for endpoints which exist in the router - Reordered large parts of the OpenAPI spec - Moved /chain/beacon/... to just /beacon/... --- beacon_node/rest_api/src/beacon.rs | 5 + beacon_node/rest_api/src/lib.rs | 79 ++-- beacon_node/rest_api/src/metrics.rs | 10 +- beacon_node/rest_api/src/network.rs | 2 +- docs/rest_oapi.yaml | 584 ++++++++++++++++------------ 5 files changed, 404 insertions(+), 276 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 1c66a2819..fb8386661 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -12,6 +12,11 @@ pub struct HeadResponse { pub slot: Slot, pub block_root: Hash256, pub state_root: Hash256, + /* Not implemented: + pub finalized_slot: Slot, + pub finalized_block_root: Hash256, + pub justified_slot: Hash256, + */ } /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index b943a1d45..770793491 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -124,27 +124,17 @@ pub fn start_server( // Route the request to the correct handler. let result = match (req.method(), path.as_ref()) { - // Methods for Beacon Node - //TODO: Remove? - //(&Method::GET, "/beacon/best_slot") => beacon::get_best_slot::(req), - (&Method::GET, "/beacon/head") => beacon::get_head::(req), - (&Method::GET, "/beacon/block") => beacon::get_block::(req), - (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), - //TODO Is the below replaced by finalized_checkpoint? - (&Method::GET, "/beacon/chainhead") => { + // Methods for Client + (&Method::GET, "/node/version") => node::get_version(req), + (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), + (&Method::GET, "/node/deposit_contract") => { helpers::implementation_pending_response(req) } - (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), - (&Method::GET, "/beacon/latest_finalized_checkpoint") => { - beacon::get_latest_finalized_checkpoint::(req) - } - (&Method::GET, "/beacon/state") => beacon::get_state::(req), - (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), + (&Method::GET, "/node/chain_id") => helpers::implementation_pending_response(req), + (&Method::GET, "/node/metrics") => metrics::get_prometheus::(req), - //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances - - // Methods for Client - (&Method::GET, "/metrics") => metrics::get_prometheus::(req), + // Methods for Network (&Method::GET, "/network/enr") => network::get_enr::(req), (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), @@ -153,36 +143,54 @@ pub fn start_server( (&Method::GET, "/network/listen_addresses") => { network::get_listen_addresses::(req) } - (&Method::GET, "/node/version") => node::get_version(req), - (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), - (&Method::GET, "/node/deposit_contract") => { + (&Method::GET, "/network/stats") => helpers::implementation_pending_response(req), + (&Method::GET, "/network/block_discovery") => { helpers::implementation_pending_response(req) } - (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), - (&Method::GET, "/node/fork") => helpers::implementation_pending_response(req), - // Methods for Network - (&Method::GET, "/network/enr") => network::get_enr::(req), - (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), - (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), - (&Method::GET, "/network/peers") => network::get_peer_list::(req), - (&Method::GET, "/network/listen_addresses") => { - network::get_listen_addresses::(req) + // Methods for Beacon Node + //TODO: Remove? + //(&Method::GET, "/beacon/best_slot") => beacon::get_best_slot::(req), + (&Method::GET, "/beacon/head") => beacon::get_head::(req), + (&Method::GET, "/beacon/block") => beacon::get_block::(req), + (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), + (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), + (&Method::GET, "/beacon/fork") => helpers::implementation_pending_response(req), + (&Method::GET, "/beacon/latest_finalized_checkpoint") => { + beacon::get_latest_finalized_checkpoint::(req) + } + (&Method::GET, "/beacon/attestations") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/beacon/attestations/pending") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/beacon/attestations") => { + helpers::implementation_pending_response(req) } // Methods for Validator - (&Method::GET, "/validator/duties") => validator::get_validator_duties::(req), - (&Method::GET, "/validator/block") => helpers::implementation_pending_response(req), - (&Method::POST, "/validator/block") => { + (&Method::GET, "/beacon/validator/duties") => { + validator::get_validator_duties::(req) + } + (&Method::GET, "/beacon/validator/block") => { helpers::implementation_pending_response(req) } - (&Method::GET, "/validator/attestation") => { + (&Method::POST, "/beacon/validator/block") => { helpers::implementation_pending_response(req) } - (&Method::POST, "/validator/attestation") => { + (&Method::GET, "/beacon/validator/attestation") => { + helpers::implementation_pending_response(req) + } + (&Method::POST, "/beacon/validator/attestation") => { helpers::implementation_pending_response(req) } + (&Method::GET, "/beacon/state") => beacon::get_state::(req), + (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances + + // Methods for bootstrap and checking configuration (&Method::GET, "/spec") => spec::get_spec::(req), (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), @@ -237,6 +245,7 @@ pub fn start_server( fn success_response(body: Body) -> Response { Response::builder() .status(StatusCode::OK) + .header("content-type", "application/json") .body(body) .expect("We should always be able to make response from the success body.") } diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 064359337..1a7ca886e 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -64,6 +64,14 @@ pub fn get_prometheus(req: Request) -> ApiR .unwrap(); String::from_utf8(buffer) - .map(|string| success_response(Body::from(string))) + .map(|string| { + let mut response = success_response(Body::from(string)); + // Need to change the header to text/plain for prometheius + response + .headers_mut() + .insert("content-type", "text/plain; charset=utf-8".parse().unwrap()) + .unwrap(); + response + }) .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e))) } diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs index a3e4c5ee7..dffa949c9 100644 --- a/beacon_node/rest_api/src/network.rs +++ b/beacon_node/rest_api/src/network.rs @@ -21,7 +21,7 @@ pub fn get_listen_addresses(req: Request) -> ApiResul ))) } -/// HTTP handle to return the list of libp2p multiaddr the client is listening on. +/// HTTP handle to return network port the client is listening on. /// /// Returns a list of `Multiaddr`, serialized according to their `serde` impl. pub fn get_listen_port(req: Request) -> ApiResult { diff --git a/docs/rest_oapi.yaml b/docs/rest_oapi.yaml index dea892c18..0c2f8616d 100644 --- a/docs/rest_oapi.yaml +++ b/docs/rest_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Lighthouse REST API" description: "" - version: "0.1.0" + version: "0.2.0" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" @@ -85,7 +85,7 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/fork: + /node/chain_id: get: tags: - Phase0 @@ -99,8 +99,6 @@ paths: schema: type: object properties: - fork: - $ref: '#/components/schemas/Fork' chain_id: type: integer format: uint64 @@ -108,32 +106,74 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/stats: + /node/metrics: get: tags: - - Future - summary: "Get operational information about the node." - description: "Fetches some operational information about the node's process, such as memory usage, database size, etc." + - Phase0 + summary: "Get Promethius metrics for the node" + description: "Fetches a range of metrics for measuring nodes health. It is intended for this endpoint to be consumed by Promethius." + responses: + 200: + description: Request successful + content: + text/plain: + example: + summary: 'Promethius metrics' + value: "# HELP beacon_head_state_active_validators_total Count of active validators at the head of the chain + # TYPE beacon_head_state_active_validators_total gauge + beacon_head_state_active_validators_total 16 + # HELP beacon_head_state_current_justified_epoch Current justified epoch at the head of the chain + # TYPE beacon_head_state_current_justified_epoch gauge + beacon_head_state_current_justified_epoch 0 + # HELP beacon_head_state_current_justified_root Current justified root at the head of the chain + # TYPE beacon_head_state_current_justified_root gauge + beacon_head_state_current_justified_root 0 + # HELP beacon_head_state_eth1_deposit_index Eth1 deposit index at the head of the chain + # TYPE beacon_head_state_eth1_deposit_index gauge + beacon_head_state_eth1_deposit_index 16 + # HELP beacon_head_state_finalized_epoch Finalized epoch at the head of the chain + # TYPE beacon_head_state_finalized_epoch gauge + beacon_head_state_finalized_epoch 0 + # HELP beacon_head_state_finalized_root Finalized root at the head of the chain + # TYPE beacon_head_state_finalized_root gauge + beacon_head_state_finalized_root 0 + # HELP beacon_head_state_latest_block_slot Latest block slot at the head of the chain + # TYPE beacon_head_state_latest_block_slot gauge + beacon_head_state_latest_block_slot 0 + # HELP beacon_head_state_previous_justified_epoch Previous justified epoch at the head of the chain + # TYPE beacon_head_state_previous_justified_epoch gauge + beacon_head_state_previous_justified_epoch 0 + # HELP beacon_head_state_previous_justified_root Previous justified root at the head of the chain + # TYPE beacon_head_state_previous_justified_root gauge + beacon_head_state_previous_justified_root 0 + # HELP beacon_head_state_root Root of the block at the head of the chain + # TYPE beacon_head_state_root gauge + beacon_head_state_root -7566315470565629000 + # HELP beacon_head_state_shard_total Count of shards in the beacon chain + # TYPE beacon_head_state_shard_total gauge + beacon_head_state_shard_total 8 + # HELP beacon_head_state_slashed_validators_total Count of all slashed validators at the head of the chain + # TYPE beacon_head_state_slashed_validators_total gauge + beacon_head_state_slashed_validators_total 0" + + #TODO: Complete the /network/enr request + /network/enr: + get: + tags: + - Phase0 + summary: "" + description: "" responses: 200: description: Request successful content: application/json: schema: - type: object - properties: - memory_usage: - type: integer - format: uint64 - description: "The amount of memory used by the currently running beacon node process, expressed in bytes." - uptime: - type: integer - format: uint64 - description: "The number of seconds that have elapsed since beacon node process was started." - #TODO: what other useful process information could be expressed here? + type: integer + format: uint16 + example: 2468 - - /node/network/peer_count: + /network/peer_count: get: tags: - Phase0 @@ -149,7 +189,10 @@ paths: format: uint64 example: 25 - /node/network/peers: + #TODO: Complete our peer ID + /network/peer_id: + + /network/peers: get: tags: - Phase0 @@ -165,7 +208,24 @@ paths: items: $ref: '#/components/schemas/Peer' - /node/network/listening: + #TODO: Complete the /network/listen_port endpoint + /network/listen_port: + get: + tags: + - Phase0 + summary: "" + description: "" + responses: + 200: + description: Request successful + content: + application/json: + schema: + type: integer + format: uint16 + example: 2468 + + /network/listen_addresses: get: tags: - Phase0 @@ -183,10 +243,12 @@ paths: type: boolean nullable: false description: "True if the node is listening for incoming network connections. False if networking has been disabled or if the node has been configured to only connect with a static set of peers." - listen_address: - $ref: '#/components/schemas/multiaddr' + addresses: + type: array + items: + $ref: '#/components/schemas/multiaddr' - /node/network/stats: + /network/stats: get: tags: - Future @@ -215,7 +277,7 @@ paths: description: "The total number of unique peers (by multiaddr) that have been discovered since the beacon node instance was started." #TODO: This might be too difficult to collect - /node/network/block_discovery: + /network/block_discovery: get: tags: - Future @@ -254,177 +316,72 @@ paths: - #TODO: Add the endpoints that enable a validator to join, exit, withdraw, etc. - /validator/duties: + + /beacon/head: get: tags: - Phase0 - summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganization (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganizations are recognized, and to ensure that the beacon node is properly synchronized. If no epoch parameter is provided, then the current epoch is assumed." - parameters: - - name: validator_pubkeys - in: query - required: true - description: "An array of hex-encoded BLS public keys" - schema: - type: array - items: - $ref: '#/components/schemas/pubkey' - minItems: 1 - - name: epoch - in: query - required: false - schema: - type: integer - format: uint64 + summary: "Detail the current perspective of the beacon node." + description: "Request the beacon node to identify the most up-to-date information about the beacon chain from its perspective. This includes the latest block, which slots have been finalized, etc." responses: 200: description: Success response content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/ValidatorDuty' - 400: - $ref: '#/components/responses/InvalidRequest' - 406: - description: "Duties cannot be provided for the requested epoch." - 500: - $ref: '#/components/responses/InternalError' - 503: - $ref: '#/components/responses/CurrentlySyncing' + type: object + description: "The latest information about the head of the beacon chain." + properties: + slot: + type: integer + format: uint64 + description: "The slot of the head block." + block_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The merkle tree root of the canonical head block in the beacon node." + state_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The merkle tree root of the current beacon state." + finalized_slot: + type: integer + format: uint64 + description: "The slot number of the most recent finalized slot." + finalized_block_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The block root for the most recent finalized block." + justified_slot: + type: integer + format: uint64 + description: "The slot number of the most recent justified slot." + justified_block_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The block root of the most recent justified block." + previous_justified_slot: + type: integer + format: uint64 + description: "The slot number of the second most recent justified slot." + previous_justified_block_root: + type: integer + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The block root of the second most recent justified block." - /validator/block: - get: - tags: - - Phase0 - summary: "Produce a new block, without signature." - description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." - parameters: - - name: slot - in: query - required: true - description: "The slot for which the block should be proposed." - schema: - type: integer - format: uint64 - - name: randao_reveal - in: query - required: true - description: "The validator's randao reveal value." - schema: - type: string - format: byte - responses: - 200: - description: Success response - content: - application/json: - schema: - $ref: '#/components/schemas/BeaconBlock' - 400: - $ref: '#/components/responses/InvalidRequest' - 500: - $ref: '#/components/responses/InternalError' - 503: - $ref: '#/components/responses/CurrentlySyncing' - post: - tags: - - Phase0 - summary: "Publish a signed block." - description: "Instructs the beacon node to broadcast a newly signed beacon block to the beacon network, to be included in the beacon chain. The beacon node is not required to validate the signed `BeaconBlock`, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new block into its state, and therefore validate the block internally, however blocks which fail the validation are still broadcast but a different status code is returned (202)" - parameters: - - name: beacon_block - in: query - required: true - description: "The `BeaconBlock` object, as sent from the beacon node originally, but now with the signature field completed." - schema: - $ref: '#/components/schemas/BeaconBlock' - responses: - 200: - description: "The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." - 202: - description: "The block failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." - 400: - $ref: '#/components/responses/InvalidRequest' - 500: - $ref: '#/components/responses/InternalError' - 503: - $ref: '#/components/responses/CurrentlySyncing' - /validator/attestation: - get: - tags: - - Phase0 - summary: "Produce an attestation, without signature." - description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." - parameters: - - name: validator_pubkey - in: query - required: true - description: "Uniquely identifying which validator this attestation is to be produced for." - schema: - $ref: '#/components/schemas/pubkey' - - name: poc_bit - in: query - required: true - description: "The proof-of-custody bit that is to be reported by the requesting validator. This bit will be inserted into the appropriate location in the returned `IndexedAttestation`." - schema: - type: integer - format: uint32 - minimum: 0 - maximum: 1 - - name: slot - in: query - required: true - description: "The slot for which the attestation should be proposed." - schema: - type: integer - - name: shard - in: query - required: true - description: "The shard number for which the attestation is to be proposed." - schema: - type: integer - responses: - 200: - description: Success response - content: - application/json: - schema: - $ref: '#/components/schemas/IndexedAttestation' - 400: - $ref: '#/components/responses/InvalidRequest' - 500: - $ref: '#/components/responses/InternalError' - 503: - $ref: '#/components/responses/CurrentlySyncing' - post: - tags: - - Phase0 - summary: "Publish a signed attestation." - description: "Instructs the beacon node to broadcast a newly signed IndexedAttestation object to the intended shard subnet. The beacon node is not required to validate the signed IndexedAttestation, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new attestation into its state, and therefore validate the attestation internally, however attestations which fail the validation are still broadcast but a different status code is returned (202)" - parameters: - - name: attestation - in: query - required: true - description: "An `IndexedAttestation` structure, as originally provided by the beacon node, but now with the signature field completed." - schema: - $ref: '#/components/schemas/IndexedAttestation' - responses: - 200: - description: "The attestation was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." - 202: - description: "The attestation failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." - 400: - $ref: '#/components/responses/InvalidRequest' - 500: - $ref: '#/components/responses/InternalError' - 503: - $ref: '#/components/responses/CurrentlySyncing' + #TODO Fill out block endpoint + /beacon/block: - /chain/beacon/blocks: + #TODO Fill out block_root endpoint + /beacon/block_root: + + /beacon/blocks: get: tags: - Phase0 @@ -468,59 +425,25 @@ paths: $ref: '#/components/responses/InvalidRequest' #TODO: Make this request error more specific if one of the parameters is not provided correctly. - /chain/beacon/chainhead: + + /beacon/fork: get: tags: - Phase0 - summary: "Detail the current perspective of the beacon node." - description: "Request the beacon node to identify the most up-to-date information about the beacon chain from its perspective. This includes the latest block, which slots have been finalized, etc." + summary: 'Retrieve the current Fork information.' + description: 'Request the beacon node identify the fork it is currently on, from the beacon state.' responses: 200: - description: Success response + description: Success response. content: application/json: schema: - type: object - description: "The latest information about the head of the beacon chain." - properties: - block_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The merkle tree root of the canonical head block in the beacon node." - block_slot: - type: integer - format: uint64 - description: "The slot of the head block." - finalized_slot: - type: integer - format: uint64 - description: "The slot number of the most recent finalized slot." - finalized_block_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The block root for the most recent finalized block." - justified_slot: - type: integer - format: uint64 - description: "The slot number of the most recent justified slot." - justified_block_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The block root of the most recent justified block." - previous_justified_slot: - type: integer - format: uint64 - description: "The slot number of the second most recent justified slot." - previous_justified_block_root: - type: integer - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The block root of the second most recent justified block." + $ref: '#/components/schemas/Fork' - /chain/beacon/attestations: + #TODO fill out latest_finalized_checkpoint + /beacon/latest_finalized_checkpoint: + + /beacon/attestations: get: tags: - Phase0 @@ -564,7 +487,7 @@ paths: $ref: '#/components/responses/InvalidRequest' #TODO: Make this request error more specific if one of the parameters is not provided correctly. - /chain/beacon/attestations/pending: + /beacon/attestations/pending: get: tags: - Phase0 @@ -583,7 +506,8 @@ paths: $ref: '#/components/responses/InvalidRequest' #TODO: Make this request error more specific if one of the parameters is not provided correctly. - /chain/beacon/validators: + #TODO: do all these '/beacon/validators' endpoints come under '/beacon/state' subqueries? + /beacon/validators: get: tags: - Phase0 @@ -614,7 +538,7 @@ paths: items: $ref: '#/components/schemas/ValidatorInfo' - /chain/beacon/validators/activesetchanges: + /beacon/validators/activesetchanges: get: tags: - Phase0 @@ -656,7 +580,7 @@ paths: items: $ref: '#/components/schemas/pubkey' - /chain/beacon/validators/assignments: + /beacon/validators/assignments: get: tags: - Phase0 @@ -688,7 +612,7 @@ paths: $ref: '#/components/schemas/ValidatorDuty' #TODO: This does not include the crosslink committee value, which must be included for Phase1? - /chain/beacon/validators/indices: + /beacon/validators/indices: get: tags: - Phase0 @@ -714,7 +638,7 @@ paths: items: $ref: '#/components/schemas/ValidatorIndexMapping' - /chain/beacon/validators/pubkeys: + /beacon/validators/pubkeys: get: tags: - Phase0 @@ -742,7 +666,7 @@ paths: items: $ref: '#/components/schemas/ValidatorIndexMapping' - /chain/beacon/validators/balances: + /beacon/validators/balances: get: tags: - Phase0 @@ -803,7 +727,7 @@ paths: format: uint64 description: "The balance of the validator at the specified epoch, expressed in Gwei" - /chain/beacon/validators/participation: + /beacon/validators/participation: get: tags: - Phase0 @@ -848,7 +772,7 @@ paths: format: uint64 description: "The total amount of ether, expressed in Gwei, that is eligible for voting in the specified epoch." - /chain/beacon/validators/queue: + /beacon/validators/queue: get: tags: - Phase0 @@ -889,6 +813,188 @@ paths: items: $ref: '#/components/schemas/pubkey' + #TODO: Add the endpoints that enable a validator to join, exit, withdraw, etc. + /beacon/validator/duties: + get: + tags: + - Phase0 + summary: "Get validator duties for the requested validators." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganization (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganizations are recognized, and to ensure that the beacon node is properly synchronized. If no epoch parameter is provided, then the current epoch is assumed." + parameters: + - name: validator_pubkeys + in: query + required: true + description: "An array of hex-encoded BLS public keys" + schema: + type: array + items: + $ref: '#/components/schemas/pubkey' + minItems: 1 + - name: epoch + in: query + required: false + schema: + type: integer + format: uint64 + responses: + 200: + description: Success response + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/ValidatorDuty' + 400: + $ref: '#/components/responses/InvalidRequest' + 406: + description: "Duties cannot be provided for the requested epoch." + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + + /beacon/validator/block: + get: + tags: + - Phase0 + summary: "Produce a new block, without signature." + description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." + parameters: + - name: slot + in: query + required: true + description: "The slot for which the block should be proposed." + schema: + type: integer + format: uint64 + - name: randao_reveal + in: query + required: true + description: "The validator's randao reveal value." + schema: + type: string + format: byte + responses: + 200: + description: Success response + content: + application/json: + schema: + $ref: '#/components/schemas/BeaconBlock' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + post: + tags: + - Phase0 + summary: "Publish a signed block." + description: "Instructs the beacon node to broadcast a newly signed beacon block to the beacon network, to be included in the beacon chain. The beacon node is not required to validate the signed `BeaconBlock`, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new block into its state, and therefore validate the block internally, however blocks which fail the validation are still broadcast but a different status code is returned (202)" + parameters: + - name: beacon_block + in: query + required: true + description: "The `BeaconBlock` object, as sent from the beacon node originally, but now with the signature field completed." + schema: + $ref: '#/components/schemas/BeaconBlock' + responses: + 200: + description: "The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." + 202: + description: "The block failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + + /beacon/validator/attestation: + get: + tags: + - Phase0 + summary: "Produce an attestation, without signature." + description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." + parameters: + - name: validator_pubkey + in: query + required: true + description: "Uniquely identifying which validator this attestation is to be produced for." + schema: + $ref: '#/components/schemas/pubkey' + - name: poc_bit + in: query + required: true + description: "The proof-of-custody bit that is to be reported by the requesting validator. This bit will be inserted into the appropriate location in the returned `IndexedAttestation`." + schema: + type: integer + format: uint32 + minimum: 0 + maximum: 1 + - name: slot + in: query + required: true + description: "The slot for which the attestation should be proposed." + schema: + type: integer + - name: shard + in: query + required: true + description: "The shard number for which the attestation is to be proposed." + schema: + type: integer + responses: + 200: + description: Success response + content: + application/json: + schema: + $ref: '#/components/schemas/IndexedAttestation' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + post: + tags: + - Phase0 + summary: "Publish a signed attestation." + description: "Instructs the beacon node to broadcast a newly signed IndexedAttestation object to the intended shard subnet. The beacon node is not required to validate the signed IndexedAttestation, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new attestation into its state, and therefore validate the attestation internally, however attestations which fail the validation are still broadcast but a different status code is returned (202)" + parameters: + - name: attestation + in: query + required: true + description: "An `IndexedAttestation` structure, as originally provided by the beacon node, but now with the signature field completed." + schema: + $ref: '#/components/schemas/IndexedAttestation' + responses: + 200: + description: "The attestation was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." + 202: + description: "The attestation failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + + #TODO fill out /beacon/state + /beacon/state: + + #TODO fill out /beacon/state_root + /beacon/state_root: + + #TODO fill spec + /spec: + + #TODO fill spec/slots_per_epoch + /spec/slots_per_epoch: + components: schemas: pubkey: From 77e2f576af49c191425f85d285cae39fbb4a4344 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 28 Aug 2019 21:25:38 +1000 Subject: [PATCH 02/47] Further aligning the API & implementation. - Completed implementation of /beacon/head - renamed 'latest_finalized_checkpoint' to 'current_finalized_checkpoint' for consistency - Reorganised list of endpoints in both spec & router so that they match - Fixed the content-type modifications for /metrics - Added a new 'RFC' tag to the spec, to tag things that we have not implemented and aren't sure if it's useful. - Moved 'deposit_contract' under /spec --- beacon_node/rest_api/src/beacon.rs | 35 +++++-- beacon_node/rest_api/src/lib.rs | 27 +++-- beacon_node/rest_api/src/metrics.rs | 9 +- docs/rest_oapi.yaml | 154 ++++++++++++++-------------- 4 files changed, 128 insertions(+), 97 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index fb8386661..b489f1fe7 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -12,11 +12,12 @@ pub struct HeadResponse { pub slot: Slot, pub block_root: Hash256, pub state_root: Hash256, - /* Not implemented: pub finalized_slot: Slot, pub finalized_block_root: Hash256, - pub justified_slot: Hash256, - */ + pub justified_slot: Slot, + pub justified_block_root: Hash256, + pub previous_justified_slot: Slot, + pub previous_justified_block_root: Hash256, } /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. @@ -26,10 +27,30 @@ pub fn get_head(req: Request) -> ApiResult .get::>>() .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let chain_head = beacon_chain.head(); + let head = HeadResponse { - slot: beacon_chain.head().beacon_state.slot, - block_root: beacon_chain.head().beacon_block_root, - state_root: beacon_chain.head().beacon_state_root, + slot: chain_head.beacon_state.slot, + block_root: chain_head.beacon_block_root, + state_root: chain_head.beacon_state_root, + finalized_slot: chain_head + .beacon_state + .finalized_checkpoint + .epoch + .start_slot(T::EthSpec::slots_per_epoch()), + finalized_block_root: chain_head.beacon_state.finalized_checkpoint.root, + justified_slot: chain_head + .beacon_state + .current_justified_checkpoint + .epoch + .start_slot(T::EthSpec::slots_per_epoch()), + justified_block_root: chain_head.beacon_state.current_justified_checkpoint.root, + previous_justified_slot: chain_head + .beacon_state + .previous_justified_checkpoint + .epoch + .start_slot(T::EthSpec::slots_per_epoch()), + previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root, }; let json: String = serde_json::to_string(&head) @@ -178,7 +199,7 @@ pub fn get_state_root(req: Request) -> ApiR } /// HTTP handler to return the highest finalized slot. -pub fn get_latest_finalized_checkpoint( +pub fn get_current_finalized_checkpoint( req: Request, ) -> ApiResult { let beacon_chain = req diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 770793491..99a8c6343 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -127,12 +127,8 @@ pub fn start_server( // Methods for Client (&Method::GET, "/node/version") => node::get_version(req), (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), - (&Method::GET, "/node/deposit_contract") => { - helpers::implementation_pending_response(req) - } (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), (&Method::GET, "/node/chain_id") => helpers::implementation_pending_response(req), - (&Method::GET, "/node/metrics") => metrics::get_prometheus::(req), // Methods for Network (&Method::GET, "/network/enr") => network::get_enr::(req), @@ -143,29 +139,30 @@ pub fn start_server( (&Method::GET, "/network/listen_addresses") => { network::get_listen_addresses::(req) } - (&Method::GET, "/network/stats") => helpers::implementation_pending_response(req), (&Method::GET, "/network/block_discovery") => { helpers::implementation_pending_response(req) } // Methods for Beacon Node - //TODO: Remove? - //(&Method::GET, "/beacon/best_slot") => beacon::get_best_slot::(req), (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), (&Method::GET, "/beacon/fork") => helpers::implementation_pending_response(req), - (&Method::GET, "/beacon/latest_finalized_checkpoint") => { - beacon::get_latest_finalized_checkpoint::(req) - } (&Method::GET, "/beacon/attestations") => { helpers::implementation_pending_response(req) } (&Method::GET, "/beacon/attestations/pending") => { helpers::implementation_pending_response(req) } - (&Method::GET, "/beacon/attestations") => { + + (&Method::GET, "/beacon/validators") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/beacon/validators/indicies") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/beacon/validators/pubkeys") => { helpers::implementation_pending_response(req) } @@ -188,11 +185,19 @@ pub fn start_server( (&Method::GET, "/beacon/state") => beacon::get_state::(req), (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + (&Method::GET, "/beacon/state/current_finalized_checkpoint") => { + beacon::get_current_finalized_checkpoint::(req) + } //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances // Methods for bootstrap and checking configuration (&Method::GET, "/spec") => spec::get_spec::(req), (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), + (&Method::GET, "/spec/deposit_contract") => { + helpers::implementation_pending_response(req) + } + + (&Method::GET, "/metrics") => metrics::get_prometheus::(req), _ => Err(ApiError::NotFound( "Request path and/or method not found.".to_owned(), diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 1a7ca886e..9d2ecc343 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -1,5 +1,6 @@ use crate::{success_response, ApiError, ApiResult, DBPath}; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use http::HeaderValue; use hyper::{Body, Request}; use prometheus::{Encoder, TextEncoder}; use std::sync::Arc; @@ -67,10 +68,10 @@ pub fn get_prometheus(req: Request) -> ApiR .map(|string| { let mut response = success_response(Body::from(string)); // Need to change the header to text/plain for prometheius - response - .headers_mut() - .insert("content-type", "text/plain; charset=utf-8".parse().unwrap()) - .unwrap(); + response.headers_mut().insert( + "content-type", + HeaderValue::from_static("text/plain; charset=utf-8"), + ); response }) .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e))) diff --git a/docs/rest_oapi.yaml b/docs/rest_oapi.yaml index 0c2f8616d..be8991956 100644 --- a/docs/rest_oapi.yaml +++ b/docs/rest_oapi.yaml @@ -13,6 +13,8 @@ tags: description: Endpoints which will be implemented for phase 1 of Ethereum Serenity - name: Future description: Potential future endpoints or optional nice-to-haves + - name: RFC + description: Do we need these endpoints at all? This is a request for comments if you think they're useful. paths: /node/version: @@ -47,21 +49,6 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/deposit_contract: - get: - tags: - - Phase0 - summary: "Get the address of the Ethereum 1 deposit contract." - description: "Requests the address of the deposit contract on the Ethereum 1 chain, which was used to start the current beacon chain." - responses: - 200: - description: Request successful - content: - application/json: - schema: - $ref: '#/components/schemas/ethereum_address' - 500: - $ref: '#/components/responses/InternalError' /node/syncing: get: @@ -106,55 +93,6 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/metrics: - get: - tags: - - Phase0 - summary: "Get Promethius metrics for the node" - description: "Fetches a range of metrics for measuring nodes health. It is intended for this endpoint to be consumed by Promethius." - responses: - 200: - description: Request successful - content: - text/plain: - example: - summary: 'Promethius metrics' - value: "# HELP beacon_head_state_active_validators_total Count of active validators at the head of the chain - # TYPE beacon_head_state_active_validators_total gauge - beacon_head_state_active_validators_total 16 - # HELP beacon_head_state_current_justified_epoch Current justified epoch at the head of the chain - # TYPE beacon_head_state_current_justified_epoch gauge - beacon_head_state_current_justified_epoch 0 - # HELP beacon_head_state_current_justified_root Current justified root at the head of the chain - # TYPE beacon_head_state_current_justified_root gauge - beacon_head_state_current_justified_root 0 - # HELP beacon_head_state_eth1_deposit_index Eth1 deposit index at the head of the chain - # TYPE beacon_head_state_eth1_deposit_index gauge - beacon_head_state_eth1_deposit_index 16 - # HELP beacon_head_state_finalized_epoch Finalized epoch at the head of the chain - # TYPE beacon_head_state_finalized_epoch gauge - beacon_head_state_finalized_epoch 0 - # HELP beacon_head_state_finalized_root Finalized root at the head of the chain - # TYPE beacon_head_state_finalized_root gauge - beacon_head_state_finalized_root 0 - # HELP beacon_head_state_latest_block_slot Latest block slot at the head of the chain - # TYPE beacon_head_state_latest_block_slot gauge - beacon_head_state_latest_block_slot 0 - # HELP beacon_head_state_previous_justified_epoch Previous justified epoch at the head of the chain - # TYPE beacon_head_state_previous_justified_epoch gauge - beacon_head_state_previous_justified_epoch 0 - # HELP beacon_head_state_previous_justified_root Previous justified root at the head of the chain - # TYPE beacon_head_state_previous_justified_root gauge - beacon_head_state_previous_justified_root 0 - # HELP beacon_head_state_root Root of the block at the head of the chain - # TYPE beacon_head_state_root gauge - beacon_head_state_root -7566315470565629000 - # HELP beacon_head_state_shard_total Count of shards in the beacon chain - # TYPE beacon_head_state_shard_total gauge - beacon_head_state_shard_total 8 - # HELP beacon_head_state_slashed_validators_total Count of all slashed validators at the head of the chain - # TYPE beacon_head_state_slashed_validators_total gauge - beacon_head_state_slashed_validators_total 0" #TODO: Complete the /network/enr request /network/enr: @@ -251,7 +189,7 @@ paths: /network/stats: get: tags: - - Future + - RFC summary: "Get some simple network statistics from the node." description: "Request that the beacon node provide some historical summary information about its networking interface." #TODO: Do we actually collect these stats? Should we? @@ -280,7 +218,7 @@ paths: /network/block_discovery: get: tags: - - Future + - RFC summary: "Identify the time at which particular blocks were first seen." description: "Request the node to provide the time at which particular blocks were first seen on the network." parameters: @@ -369,7 +307,7 @@ paths: format: uint64 description: "The slot number of the second most recent justified slot." previous_justified_block_root: - type: integer + type: string format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The block root of the second most recent justified block." @@ -440,8 +378,6 @@ paths: schema: $ref: '#/components/schemas/Fork' - #TODO fill out latest_finalized_checkpoint - /beacon/latest_finalized_checkpoint: /beacon/attestations: get: @@ -506,7 +442,6 @@ paths: $ref: '#/components/responses/InvalidRequest' #TODO: Make this request error more specific if one of the parameters is not provided correctly. - #TODO: do all these '/beacon/validators' endpoints come under '/beacon/state' subqueries? /beacon/validators: get: tags: @@ -541,7 +476,7 @@ paths: /beacon/validators/activesetchanges: get: tags: - - Phase0 + - RFC summary: "Retrieve the changes in active validator set." description: "Request that the beacon node describe the changes that occurred at the specified epoch, as compared with the prior epoch." parameters: @@ -583,7 +518,7 @@ paths: /beacon/validators/assignments: get: tags: - - Phase0 + - RFC summary: "Retrieve the assigned responsibilities for validators in a particular epoch." description: "Request that the beacon node list the duties which have been assigned to the active validator set in a particular epoch." parameters: @@ -669,7 +604,7 @@ paths: /beacon/validators/balances: get: tags: - - Phase0 + - RFC summary: "Retrieve the balances of validators at a specified epoch." description: "Retrieve the balances of validators at a specified epoch (or the current epoch if none specified). The list of balances can be filtered by providing a list of validator public keys or indices." parameters: @@ -730,7 +665,7 @@ paths: /beacon/validators/participation: get: tags: - - Phase0 + - RFC summary: "Retrieve aggregate information about validator participation in an epoch." description: "Retrieve some aggregate information about the participation of validators in a specified epoch (or the current epoch if none specified)." parameters: @@ -775,7 +710,7 @@ paths: /beacon/validators/queue: get: tags: - - Phase0 + - RFC summary: "Retrieve information about the validator queue at the specified epoch." description: "Retrieve information about the queue of validators for the specified epoch (or the current epoch if none specified)." parameters: @@ -989,12 +924,81 @@ paths: #TODO fill out /beacon/state_root /beacon/state_root: + #TODO fill out current_finalized_checkpoint + /beacon/current_finalized_checkpoint: + #TODO fill spec /spec: #TODO fill spec/slots_per_epoch /spec/slots_per_epoch: + /spec/deposit_contract: + get: + tags: + - Phase0 + summary: "Get the address of the Ethereum 1 deposit contract." + description: "Requests the address of the deposit contract on the Ethereum 1 chain, which was used to start the current beacon chain." + responses: + 200: + description: Request successful + content: + application/json: + schema: + $ref: '#/components/schemas/ethereum_address' + 500: + $ref: '#/components/responses/InternalError' + + /metrics: + get: + tags: + - Phase0 + summary: "Get Promethius metrics for the node" + description: "Fetches a range of metrics for measuring nodes health. It is intended for this endpoint to be consumed by Promethius." + responses: + 200: + description: Request successful + content: + text/plain: + example: + summary: 'Promethius metrics' + value: "# HELP beacon_head_state_active_validators_total Count of active validators at the head of the chain + # TYPE beacon_head_state_active_validators_total gauge + beacon_head_state_active_validators_total 16 + # HELP beacon_head_state_current_justified_epoch Current justified epoch at the head of the chain + # TYPE beacon_head_state_current_justified_epoch gauge + beacon_head_state_current_justified_epoch 0 + # HELP beacon_head_state_current_justified_root Current justified root at the head of the chain + # TYPE beacon_head_state_current_justified_root gauge + beacon_head_state_current_justified_root 0 + # HELP beacon_head_state_eth1_deposit_index Eth1 deposit index at the head of the chain + # TYPE beacon_head_state_eth1_deposit_index gauge + beacon_head_state_eth1_deposit_index 16 + # HELP beacon_head_state_finalized_epoch Finalized epoch at the head of the chain + # TYPE beacon_head_state_finalized_epoch gauge + beacon_head_state_finalized_epoch 0 + # HELP beacon_head_state_finalized_root Finalized root at the head of the chain + # TYPE beacon_head_state_finalized_root gauge + beacon_head_state_finalized_root 0 + # HELP beacon_head_state_latest_block_slot Latest block slot at the head of the chain + # TYPE beacon_head_state_latest_block_slot gauge + beacon_head_state_latest_block_slot 0 + # HELP beacon_head_state_previous_justified_epoch Previous justified epoch at the head of the chain + # TYPE beacon_head_state_previous_justified_epoch gauge + beacon_head_state_previous_justified_epoch 0 + # HELP beacon_head_state_previous_justified_root Previous justified root at the head of the chain + # TYPE beacon_head_state_previous_justified_root gauge + beacon_head_state_previous_justified_root 0 + # HELP beacon_head_state_root Root of the block at the head of the chain + # TYPE beacon_head_state_root gauge + beacon_head_state_root -7566315470565629000 + # HELP beacon_head_state_shard_total Count of shards in the beacon chain + # TYPE beacon_head_state_shard_total gauge + beacon_head_state_shard_total 8 + # HELP beacon_head_state_slashed_validators_total Count of all slashed validators at the head of the chain + # TYPE beacon_head_state_slashed_validators_total gauge + beacon_head_state_slashed_validators_total 0" + components: schemas: pubkey: From 0bd5ce65f418cc77f1f237b694292aeeba0c1b34 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 28 Aug 2019 21:26:18 +1000 Subject: [PATCH 03/47] Renamed the YAML spec document inside the 'docs' folder. --- docs/{rest_oapi.yaml => api_spec.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{rest_oapi.yaml => api_spec.yaml} (100%) diff --git a/docs/rest_oapi.yaml b/docs/api_spec.yaml similarity index 100% rename from docs/rest_oapi.yaml rename to docs/api_spec.yaml From faef347d181aea4c485859c5de201f1a99932132 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 28 Aug 2019 23:33:34 +1000 Subject: [PATCH 04/47] Fleshed out some API endpoints. - Added the /beacon/validator/block endpoint for GET (untested) - Added the /beacon/fork endpoint for GET - Cleaned up a bunch of unused imports & variables - Removed '/network/block_discovery' endpoint --- beacon_node/beacon_chain/src/beacon_chain.rs | 18 ++++- beacon_node/rest_api/src/beacon.rs | 16 +++++ beacon_node/rest_api/src/helpers.rs | 4 +- beacon_node/rest_api/src/lib.rs | 7 +- beacon_node/rest_api/src/url_query.rs | 2 +- beacon_node/rest_api/src/validator.rs | 70 +++++++++++++++++--- beacon_node/rpc/src/beacon_block.rs | 2 +- 7 files changed, 99 insertions(+), 20 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5feefd841..500f6411f 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -983,20 +983,34 @@ impl BeaconChain { Ok(BlockProcessingOutcome::Processed { block_root }) } - /// Produce a new block at the present slot. + /// Produce a new block at the specified slot. /// /// The produced block will not be inherently valid, it must be signed by a block producer. /// Block signing is out of the scope of this function and should be done by a separate program. pub fn produce_block( &self, randao_reveal: Signature, + slot: Slot, ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { let state = self.state.read().clone(); + + self.produce_block_on_state(state, slot, randao_reveal) + } + + /// Produce a new block at the current slot + /// + /// Calls `produce_block`, with the slot parameter set as the current. + /// + /// ** This function is probably obsolete (was for previous RPC), and can probably be removed ** + pub fn produce_current_block( + &self, + randao_reveal: Signature, + ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { let slot = self .read_slot_clock() .ok_or_else(|| BlockProductionError::UnableToReadSlot)?; - self.produce_block_on_state(state, slot, randao_reveal) + self.produce_block(randao_reveal, slot) } /// Produce a block for some `slot` upon the given `state`. diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index b489f1fe7..5dcbc728a 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -130,6 +130,22 @@ pub fn get_block_root(req: Request) -> ApiR Ok(success_response(Body::from(json))) } +/// HTTP handler to return the `Fork` of the current head. +pub fn get_fork(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let chain_head = beacon_chain.head(); + + let json: String = serde_json::to_string(&chain_head.beacon_state.fork).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) + })?; + + Ok(success_response(Body::from(json))) +} + #[derive(Serialize)] #[serde(bound = "T: EthSpec")] pub struct StateResponse { diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 88755fcde..2477884c4 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -2,9 +2,7 @@ use crate::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use hex; -use hyper::{Body, Request, StatusCode}; -use serde::de::value::StringDeserializer; -use serde_json::Deserializer; +use hyper::{Body, Request}; use store::{iter::AncestorIter, Store}; use types::{BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 99a8c6343..a6ee948ae 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -139,16 +139,13 @@ pub fn start_server( (&Method::GET, "/network/listen_addresses") => { network::get_listen_addresses::(req) } - (&Method::GET, "/network/block_discovery") => { - helpers::implementation_pending_response(req) - } // Methods for Beacon Node (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), - (&Method::GET, "/beacon/fork") => helpers::implementation_pending_response(req), + (&Method::GET, "/beacon/fork") => beacon::get_fork::(req), (&Method::GET, "/beacon/attestations") => { helpers::implementation_pending_response(req) } @@ -171,7 +168,7 @@ pub fn start_server( validator::get_validator_duties::(req) } (&Method::GET, "/beacon/validator/block") => { - helpers::implementation_pending_response(req) + validator::get_new_beacon_block::(req) } (&Method::POST, "/beacon/validator/block") => { helpers::implementation_pending_response(req) diff --git a/beacon_node/rest_api/src/url_query.rs b/beacon_node/rest_api/src/url_query.rs index e39a9a449..3802ff831 100644 --- a/beacon_node/rest_api/src/url_query.rs +++ b/beacon_node/rest_api/src/url_query.rs @@ -64,7 +64,7 @@ impl<'a> UrlQuery<'a> { /// Returns a vector of all values present where `key` is in `keys /// /// If no match is found, an `InvalidQueryParams` error is returned. - pub fn all_of(mut self, key: &str) -> Result, ApiError> { + pub fn all_of(self, key: &str) -> Result, ApiError> { let queries: Vec<_> = self .0 .filter_map(|(k, v)| { diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 4294f9c20..645a35837 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,13 +1,12 @@ use super::{success_response, ApiResult}; use crate::{helpers::*, ApiError, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use bls::PublicKey; +use bls::{PublicKey, Signature}; use hyper::{Body, Request}; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use store::Store; use types::beacon_state::EthSpec; -use types::{BeaconBlock, BeaconState, Epoch, RelativeEpoch, Shard, Slot}; +use types::{Epoch, RelativeEpoch, Shard, Slot}; #[derive(Debug, Serialize, Deserialize)] pub struct ValidatorDuty { @@ -39,16 +38,14 @@ pub fn get_validator_duties(req: Request) - .extensions() .get::>>() .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + //TODO Surely this state_cache thing is not necessary? let _ = beacon_chain .ensure_state_caches_are_built() .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; - let head_state = beacon_chain - .speculative_state() - .expect("This is legacy code and should be removed."); + let head_state = &beacon_chain.head().beacon_state; // Parse and check query parameters let query = UrlQuery::from_request(&req)?; - let current_epoch = head_state.current_epoch(); let epoch = match query.first_of(&["epoch"]) { Ok((_, v)) => Epoch::new(v.parse::().map_err(|e| { @@ -66,7 +63,7 @@ pub fn get_validator_duties(req: Request) - )) })?; //TODO: Handle an array of validators, currently only takes one - let mut validators: Vec = match query.all_of("validator_pubkeys") { + let validators: Vec = match query.all_of("validator_pubkeys") { Ok(v) => v .iter() .map(|pk| parse_pubkey(pk)) @@ -147,3 +144,60 @@ pub fn get_validator_duties(req: Request) - ); Ok(success_response(body)) } + +/// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. +pub fn get_new_beacon_block(req: Request) -> ApiResult { + // Get beacon state + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + //TODO Surely this state_cache thing is not necessary? + let _ = beacon_chain + .ensure_state_caches_are_built() + .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + + let query = UrlQuery::from_request(&req)?; + let slot = match query.first_of(&["slot"]) { + Ok((_, v)) => Slot::new(v.parse::().map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + })?), + Err(e) => { + return Err(e); + } + }; + let randao_reveal = match query.first_of(&["randao_reveal"]) { + Ok((_, v)) => Signature::from_bytes( + hex::decode(&v) + .map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Invalid hex string for randao_reveal: {:?}", + e + )) + })? + .as_slice(), + ) + .map_err(|e| { + ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) + })?, + Err(e) => { + return Err(e); + } + }; + + let new_block = match beacon_chain.produce_block(randao_reveal, slot) { + Ok((block, _state)) => block, + Err(e) => { + return Err(ApiError::ServerError(format!( + "Beacon node is not able to produce a block: {:?}", + e + ))); + } + }; + + let body = Body::from( + serde_json::to_string(&new_block) + .expect("We should always be able to serialize a new block that we created."), + ); + Ok(success_response(body)) +} diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index b1a67399e..012fcb678 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -51,7 +51,7 @@ impl BeaconBlockService for BeaconBlockServiceInstance { } }; - let produced_block = match self.chain.produce_block(randao_reveal) { + let produced_block = match self.chain.produce_current_block(randao_reveal) { Ok((block, _state)) => block, Err(e) => { // could not produce a block From ca07d7245397d7c7efac729326b0c356fc6c471d Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 29 Aug 2019 12:42:45 +1000 Subject: [PATCH 05/47] Removed methods for 'chain_id', since this is no longer applicable to ETH2. --- beacon_node/rest_api/src/lib.rs | 1 - docs/api_spec.yaml | 22 ---------------------- 2 files changed, 23 deletions(-) diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index a6ee948ae..b7fd3f581 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -128,7 +128,6 @@ pub fn start_server( (&Method::GET, "/node/version") => node::get_version(req), (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), - (&Method::GET, "/node/chain_id") => helpers::implementation_pending_response(req), // Methods for Network (&Method::GET, "/network/enr") => network::get_enr::(req), diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index be8991956..901df3179 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -72,28 +72,6 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/chain_id: - get: - tags: - - Phase0 - summary: "Get fork information from running beacon node." - description: "Requests the beacon node to provide which fork version it is currently on." - responses: - 200: - description: Request successful - content: - application/json: - schema: - type: object - properties: - chain_id: - type: integer - format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analogous to Eth1.0 JSON-RPC net_version." - 500: - $ref: '#/components/responses/InternalError' - - #TODO: Complete the /network/enr request /network/enr: get: From 5b5e458938b29a56fef1631a209d78640cf30d0a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 29 Aug 2019 13:12:56 +1000 Subject: [PATCH 06/47] Flesh out the API spec for the /network endpoints. --- docs/api_spec.yaml | 64 +++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index 901df3179..5053c5dd2 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -72,22 +72,24 @@ paths: 500: $ref: '#/components/responses/InternalError' - #TODO: Complete the /network/enr request /network/enr: get: tags: - Phase0 - summary: "" - description: "" + summary: "Get the node's Ethereum Node Record (ENR)." + description: "The Ethereum Node Record (ENR) contains a compressed public key, an IPv4 address, a TCP port and a UDP port, which is all encoded using base64. This endpoint fetches the base64 encoded version of the ENR for the running beacon node." responses: 200: description: Request successful content: application/json: schema: - type: integer - format: uint16 - example: 2468 + type: string + format: byte + example: "-IW4QHzEZbIB0YN47bVlsUrGbcL9vl21n7xF5gRKjMNkJ4MxfcwiqrsE7Ows8EnzOvC8P4ZyAjfOhr2ffk0bWAxDGq8BgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjzKzqo5c33ydUUHrWJ4FWwIXJa2MN9BBsgZkj6mhthp" + pattern: "^[^-A-Za-z0-9+/=]+$" + 500: + $ref: '#/components/responses/InternalError' /network/peer_count: get: @@ -104,9 +106,27 @@ paths: type: integer format: uint64 example: 25 + 500: + $ref: '#/components/responses/InternalError' - #TODO: Complete our peer ID /network/peer_id: + get: + tags: + - Phase0 + summary: "Get the node's libp2p peer ID." + description: "Requests the node to provide it's libp2p ['peer ID'](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md), which is a base58 encoded SHA2-256 'multihash' of the node's public key struct." + responses: + 200: + description: Request successful + content: + application/json: + schema: + type: string + format: byte + example: "QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N" + pattern: "^[1-9A-HJ-NP-Za-km-z]{46}$" + 500: + $ref: '#/components/responses/InternalError' /network/peers: get: @@ -123,14 +143,15 @@ paths: type: array items: $ref: '#/components/schemas/Peer' + 500: + $ref: '#/components/responses/InternalError' - #TODO: Complete the /network/listen_port endpoint /network/listen_port: get: tags: - Phase0 - summary: "" - description: "" + summary: "Get the TCP port number for the libp2p listener." + description: "Libp2p is configured to listen to a particular TCP port upon startup of the beacon node. This endpoint returns the port number that the beacon node is listening on. Please note, this is for the libp2p communications, not for discovery." responses: 200: description: Request successful @@ -139,30 +160,27 @@ paths: schema: type: integer format: uint16 - example: 2468 + example: 9000 + 500: + $ref: '#/components/responses/InternalError' /network/listen_addresses: get: tags: - Phase0 - summary: "Identify if the beacon node is listening for networking connections, and on what address." - description: "Requests that the beacon node identify whether it is listening for incoming networking connections, and if so, what network address(es) are being used." + summary: "Identify the port and addresses listened to by the beacon node." + description: "Libp2p is configured to listen to a particular address, on a particular port. This address is represented the [`multiaddr`](https://multiformats.io/multiaddr/) format, and this endpoint requests the beacon node to list all listening addresses in this format." responses: 200: description: Request successful content: application/json: schema: - type: object - properties: - listening: - type: boolean - nullable: false - description: "True if the node is listening for incoming network connections. False if networking has been disabled or if the node has been configured to only connect with a static set of peers." - addresses: - type: array - items: - $ref: '#/components/schemas/multiaddr' + type: array + items: + $ref: '#/components/schemas/multiaddr' + 500: + $ref: '#/components/responses/InternalError' /network/stats: get: From b9276da9db8dedced012b995f354994015b70a28 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 29 Aug 2019 13:36:51 +1000 Subject: [PATCH 07/47] Flesh spec. & update display bugs. - Add correct string formatting when incorrect parameters provided. - Fill /beacon/block and /beacon/block_root endpoints - Add 500 error responses to endpoints as appropriate --- beacon_node/rest_api/src/beacon.rs | 8 ++-- docs/api_spec.yaml | 69 ++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 5dcbc728a..66d0b2673 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -81,7 +81,7 @@ pub fn get_block(req: Request) -> ApiResult let target = parse_slot(&value)?; block_root_at_slot(&beacon_chain, target).ok_or_else(|| { - ApiError::NotFound(format!("Unable to find BeaconBlock for slot {}", target)) + ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) })? } ("root", value) => parse_root(&value)?, @@ -93,7 +93,7 @@ pub fn get_block(req: Request) -> ApiResult .get::>(&block_root)? .ok_or_else(|| { ApiError::NotFound(format!( - "Unable to find BeaconBlock for root {}", + "Unable to find BeaconBlock for root {:?}", block_root )) })?; @@ -121,7 +121,7 @@ pub fn get_block_root(req: Request) -> ApiR let target = parse_slot(&slot_string)?; let root = block_root_at_slot(&beacon_chain, target).ok_or_else(|| { - ApiError::NotFound(format!("Unable to find BeaconBlock for slot {}", target)) + ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) })?; let json: String = serde_json::to_string(&root) @@ -174,7 +174,7 @@ pub fn get_state(req: Request) -> ApiResult let state = beacon_chain .store .get(root)? - .ok_or_else(|| ApiError::NotFound(format!("No state for root: {}", root)))?; + .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))?; (*root, state) } diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index 5053c5dd2..b43d99b8a 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -248,9 +248,6 @@ paths: format: uint64 description: "UNIX time in milliseconds that the block was first discovered, either from a network peer or the validator client." - - - /beacon/head: get: tags: @@ -307,13 +304,75 @@ paths: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The block root of the second most recent justified block." + 500: + $ref: '#/components/responses/InternalError' - #TODO Fill out block endpoint /beacon/block: + get: + tags: + - Phase0 + summary: 'Retrieve blocks by root or slot.' + description: "Request that the beacon node return beacon chain blocks that match the provided criteria (a block root or beacon chain slot). Only one of the parameters can be be provided at a time." + parameters: + - name: root + description: "Filter by block root." + in: query + required: false + schema: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + - name: slot + description: "Filter blocks by slot number. Only one block which has been finalized, or is believed to be the canonical block for that slot, is returned." + in: query + required: false + schema: + type: integer + format: uint64 + responses: + 200: + description: Success response. + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/BeaconBlock' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' #TODO Fill out block_root endpoint /beacon/block_root: + get: + tags: + - Phase0 + summary: "Retrieve the canonical block root, given a particular slot." + description: "Request that the beacon node return the root of the canonical beacon chain block, which matches the provided slot number." + parameters: + - name: slot + description: "Filter blocks by slot number. Only one block which has been finalized, or is believed to be the canonical block for that slot, is returned." + in: query + required: true + schema: + type: integer + format: uint64 + responses: + 200: + description: Success response. + content: + application/json: + schema: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The 0x prefixed block root." + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' /beacon/blocks: get: @@ -373,6 +432,8 @@ paths: application/json: schema: $ref: '#/components/schemas/Fork' + 500: + $ref: '#/components/responses/InternalError' /beacon/attestations: From eaec5e7b69b6ab2bf484f4203c448be84582a1be Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 29 Aug 2019 14:58:49 +1000 Subject: [PATCH 08/47] Start implementation of 'get attstation' validator function. - Created new /beacon/validator/attestation endpoint - Updated some small issues with the API spec. --- beacon_node/rest_api/src/lib.rs | 2 +- beacon_node/rest_api/src/validator.rs | 66 ++++++++++++++++++++++++++- docs/api_spec.yaml | 4 +- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index b7fd3f581..2c7b90e3f 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -173,7 +173,7 @@ pub fn start_server( helpers::implementation_pending_response(req) } (&Method::GET, "/beacon/validator/attestation") => { - helpers::implementation_pending_response(req) + validator::get_new_attestation::(req) } (&Method::POST, "/beacon/validator/attestation") => { helpers::implementation_pending_response(req) diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 645a35837..450ef5e5f 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -62,7 +62,6 @@ pub fn get_validator_duties(req: Request) - e )) })?; - //TODO: Handle an array of validators, currently only takes one let validators: Vec = match query.all_of("validator_pubkeys") { Ok(v) => v .iter() @@ -197,7 +196,70 @@ pub fn get_new_beacon_block(req: Request) - let body = Body::from( serde_json::to_string(&new_block) - .expect("We should always be able to serialize a new block that we created."), + .expect("We should always be able to serialize a new block that we produced."), + ); + Ok(success_response(body)) +} + +/// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. +pub fn get_new_attestation(req: Request) -> ApiResult { + // Get beacon state + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + //TODO Surely this state_cache thing is not necessary? + let _ = beacon_chain + .ensure_state_caches_are_built() + .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + + let query = UrlQuery::from_request(&req)?; + let validator: PublicKey = match query.first_of(&["validator_pubkey"]) { + Ok((_, v)) => parse_pubkey(v.as_str())?, + Err(e) => { + return Err(e); + } + }; + let poc_bit: bool = match query.first_of(&["poc_bit"]) { + Ok((_, v)) => v.parse::().map_err(|e| ApiError::InvalidQueryParams(format!("poc_bit is not a valid boolean value: {:?}", e)))?, + Err(e) => { + return Err(e); + } + }; + //TODO: this is probably unnecessary if we're always doing it by current slot. + let _slot = match query.first_of(&["slot"]) { + Ok((_, v)) => { + let requested_slot = v.parse::().map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + })?; + let current_slot = beacon_chain.head().beacon_state.slot.as_u64(); + if requested_slot != current_slot { + return Err(ApiError::InvalidQueryParams(format!("Attestation data can only be requested for the current slot ({:?}), not your requested slot ({:?})", current_slot, requested_slot))); + } + Slot::new(requested_slot) + }, + Err(e) => { + return Err(e); + } + }; + let shard: Shard = match query.first_of(&["shard"]) { + Ok((_, v)) => v.parse::().map_err(|e| ApiError::InvalidQueryParams(format!("Shard is not a valid u64 value: {:?}", e)))?, + Err(e) => { + return Err(e); + } + }; + + let attestation_data = match beacon_chain.produce_attestation_data(shard) { + Ok(v) => v, + Err(e) => { + return Err(ApiError::ServerError(format!("Could not produce an attestation: {:?}", e))); + } + }; + + //TODO: This is currently AttestationData, but should be IndexedAttestation? + let body = Body::from( + serde_json::to_string(&attestation_data) + .expect("We should always be able to serialize a new attestation that we produced."), ); Ok(success_response(body)) } diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index b43d99b8a..42b394e69 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -839,8 +839,6 @@ paths: $ref: '#/components/schemas/ValidatorDuty' 400: $ref: '#/components/responses/InvalidRequest' - 406: - description: "Duties cannot be provided for the requested epoch." 500: $ref: '#/components/responses/InternalError' 503: @@ -867,6 +865,8 @@ paths: schema: type: string format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "A valid BLS signature." responses: 200: description: Success response From 5ee1bb20b74a5938e1b34dbd4c561e2525419b31 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sat, 31 Aug 2019 23:56:35 +1000 Subject: [PATCH 09/47] WIP: Furthered attestation production for validator. --- beacon_node/rest_api/src/validator.rs | 105 ++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 450ef5e5f..1c72874f2 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,12 +1,12 @@ use super::{success_response, ApiResult}; use crate::{helpers::*, ApiError, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes}; -use bls::{PublicKey, Signature}; +use bls::{AggregateSignature, PublicKey, Signature}; use hyper::{Body, Request}; use serde::{Deserialize, Serialize}; use std::sync::Arc; use types::beacon_state::EthSpec; -use types::{Epoch, RelativeEpoch, Shard, Slot}; +use types::{Attestation, BitList, Epoch, RelativeEpoch, Shard, Slot}; #[derive(Debug, Serialize, Deserialize)] pub struct ValidatorDuty { @@ -212,53 +212,130 @@ pub fn get_new_attestation(req: Request) -> let _ = beacon_chain .ensure_state_caches_are_built() .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; - let validator: PublicKey = match query.first_of(&["validator_pubkey"]) { + let val_pk: PublicKey = match query.first_of(&["validator_pubkey"]) { Ok((_, v)) => parse_pubkey(v.as_str())?, Err(e) => { return Err(e); } }; + // Get the validator index from the supplied public key + // If it does not exist in the index, we cannot continue. + let val_index: usize = match head_state.get_validator_index(&val_pk) { + Ok(Some(i)) => i, + Ok(None) => { + return Err(ApiError::InvalidQueryParams( + "The provided validator public key does not correspond to a validator index." + .into(), + )); + } + Err(e) => { + return Err(ApiError::ServerError(format!( + "Unable to read validator index cache. {:?}", + e + ))); + } + }; + // Get the duties of the validator, to make sure they match up. + // If they don't have duties this epoch, then return an error + let val_duty = match head_state.get_attestation_duties(val_index, RelativeEpoch::Current) { + Ok(Some(d)) => d, + Ok(None) => { + return Err(ApiError::InvalidQueryParams("No validator duties could be found for the requested validator. Cannot provide valid attestation.".into())); + } + Err(e) => { + return Err(ApiError::ServerError(format!( + "unable to read cache for attestation duties: {:?}", + e + ))) + } + }; + + // Check that we are requesting an attestation during the slot where it is relevant. + let present_slot = match beacon_chain.read_slot_clock() { + Some(s) => s, + None => { + return Err(ApiError::ServerError( + "Beacon node is unable to determine present slot, either the state isn't generated or the chain hasn't begun.".into() + )); + } + }; + if val_duty.slot != present_slot { + return Err(ApiError::InvalidQueryParams(format!("Validator is only able to request an attestation during the slot they are allocated. Current slot: {:?}, allocated slot: {:?}", head_state.slot, val_duty.slot))); + } + + // Parse the POC bit and insert it into the aggregation bits let poc_bit: bool = match query.first_of(&["poc_bit"]) { - Ok((_, v)) => v.parse::().map_err(|e| ApiError::InvalidQueryParams(format!("poc_bit is not a valid boolean value: {:?}", e)))?, + Ok((_, v)) => v.parse::().map_err(|e| { + ApiError::InvalidQueryParams(format!("poc_bit is not a valid boolean value: {:?}", e)) + })?, Err(e) => { return Err(e); } }; - //TODO: this is probably unnecessary if we're always doing it by current slot. + let mut aggregation_bits: BitList = BitList::with_capacity(val_duty.committee_len) + .expect("An empty BitList should always be created, or we have bigger problems.") + .into(); + aggregation_bits.set(val_duty.committee_index, poc_bit); + + // Allow a provided slot parameter to check against the expected slot as a sanity check. + // Presently, we don't support attestations at future or past slots. let _slot = match query.first_of(&["slot"]) { Ok((_, v)) => { let requested_slot = v.parse::().map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + ApiError::InvalidQueryParams(format!( + "Invalid slot parameter, must be a u64. {:?}", + e + )) })?; let current_slot = beacon_chain.head().beacon_state.slot.as_u64(); if requested_slot != current_slot { return Err(ApiError::InvalidQueryParams(format!("Attestation data can only be requested for the current slot ({:?}), not your requested slot ({:?})", current_slot, requested_slot))); } Slot::new(requested_slot) - }, - Err(e) => { - return Err(e); } - }; - let shard: Shard = match query.first_of(&["shard"]) { - Ok((_, v)) => v.parse::().map_err(|e| ApiError::InvalidQueryParams(format!("Shard is not a valid u64 value: {:?}", e)))?, + Err(ApiError::InvalidQueryParams(_)) => { + // Just fill _slot with a dummy value for now, making the slot parameter optional + // We'll get the real slot from the ValidatorDuty + Slot::new(0) + } Err(e) => { return Err(e); } }; + let shard: Shard = match query.first_of(&["shard"]) { + Ok((_, v)) => v.parse::().map_err(|e| { + ApiError::InvalidQueryParams(format!("Shard is not a valid u64 value: {:?}", e)) + })?, + Err(e) => { + // This is a mandatory parameter, return the error + return Err(e); + } + }; let attestation_data = match beacon_chain.produce_attestation_data(shard) { Ok(v) => v, Err(e) => { - return Err(ApiError::ServerError(format!("Could not produce an attestation: {:?}", e))); + return Err(ApiError::ServerError(format!( + "Could not produce an attestation: {:?}", + e + ))); } }; + let attestation = Attestation { + aggregation_bits, + data: attestation_data, + custody_bits: BitList::with_capacity(val_duty.committee_len) + .expect("Should be able to create an empty BitList for the custody bits."), + signature: AggregateSignature::new(), + }; + //TODO: This is currently AttestationData, but should be IndexedAttestation? let body = Body::from( - serde_json::to_string(&attestation_data) + serde_json::to_string(&attestation) .expect("We should always be able to serialize a new attestation that we produced."), ); Ok(success_response(body)) From 8ea11675632b7190f344f8eb752d152d9f9b9d35 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sun, 1 Sep 2019 15:09:01 +1000 Subject: [PATCH 10/47] Factored out getting beacon_chain from request into it's own function. --- beacon_node/rest_api/src/helpers.rs | 13 +++++ beacon_node/rest_api/src/lib.rs | 4 +- beacon_node/rest_api/src/validator.rs | 84 ++++++++++++++++++--------- 3 files changed, 72 insertions(+), 29 deletions(-) diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 2477884c4..98293e75c 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -5,6 +5,7 @@ use hex; use hyper::{Body, Request}; use store::{iter::AncestorIter, Store}; use types::{BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; +use std::sync::Arc; /// Parse a slot from a `0x` preixed string. /// @@ -169,6 +170,18 @@ pub fn implementation_pending_response(_req: Request) -> ApiResult { )) } +pub fn get_beacon_chain_from_request(req: &Request) -> Result>, ApiError> { + // Get beacon state + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Request is missing the beacon chain extension".into()))?; + let _ = beacon_chain + .ensure_state_caches_are_built() + .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + Ok(beacon_chain.clone()) +} + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 2c7b90e3f..2c9c4011a 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -172,9 +172,9 @@ pub fn start_server( (&Method::POST, "/beacon/validator/block") => { helpers::implementation_pending_response(req) } - (&Method::GET, "/beacon/validator/attestation") => { + /*(&Method::GET, "/beacon/validator/attestation") => { validator::get_new_attestation::(req) - } + }*/ (&Method::POST, "/beacon/validator/attestation") => { helpers::implementation_pending_response(req) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 1c72874f2..f60acbad8 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -5,6 +5,7 @@ use bls::{AggregateSignature, PublicKey, Signature}; use hyper::{Body, Request}; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use std::borrow::Borrow; use types::beacon_state::EthSpec; use types::{Attestation, BitList, Epoch, RelativeEpoch, Shard, Slot}; @@ -33,15 +34,7 @@ impl ValidatorDuty { /// HTTP Handler to retrieve a the duties for a set of validators during a particular epoch pub fn get_validator_duties(req: Request) -> ApiResult { - // Get beacon state - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; - //TODO Surely this state_cache thing is not necessary? - let _ = beacon_chain - .ensure_state_caches_are_built() - .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let head_state = &beacon_chain.head().beacon_state; // Parse and check query parameters @@ -146,15 +139,8 @@ pub fn get_validator_duties(req: Request) - /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. pub fn get_new_beacon_block(req: Request) -> ApiResult { - // Get beacon state - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; - //TODO Surely this state_cache thing is not necessary? - let _ = beacon_chain - .ensure_state_caches_are_built() - .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; let slot = match query.first_of(&["slot"]) { @@ -201,17 +187,60 @@ pub fn get_new_beacon_block(req: Request) - Ok(success_response(body)) } +/// HTTP Handler to accept a validator-signed BeaconBlock, and publish it to the network. +pub fn publish_beacon_block(req: Request) -> ApiResult { + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = &beacon_chain.head().beacon_state; + + let query = UrlQuery::from_request(&req)?; + let slot = match query.first_of(&["slot"]) { + Ok((_, v)) => Slot::new(v.parse::().map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + })?), + Err(e) => { + return Err(e); + } + }; + let randao_reveal = match query.first_of(&["randao_reveal"]) { + Ok((_, v)) => Signature::from_bytes( + hex::decode(&v) + .map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Invalid hex string for randao_reveal: {:?}", + e + )) + })? + .as_slice(), + ) + .map_err(|e| { + ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) + })?, + Err(e) => { + return Err(e); + } + }; + + let new_block = match beacon_chain.produce_block(randao_reveal, slot) { + Ok((block, _state)) => block, + Err(e) => { + return Err(ApiError::ServerError(format!( + "Beacon node is not able to produce a block: {:?}", + e + ))); + } + }; + + let body = Body::from( + serde_json::to_string(&new_block) + .expect("We should always be able to serialize a new block that we produced."), + ); + Ok(success_response(body)) +} + +/* /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { - // Get beacon state - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; - //TODO Surely this state_cache thing is not necessary? - let _ = beacon_chain - .ensure_state_caches_are_built() - .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + let beacon_chain = get_beacon_chain_from_request(req)?; let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; @@ -340,3 +369,4 @@ pub fn get_new_attestation(req: Request) -> ); Ok(success_response(body)) } +*/ From 632c13a9ec18606ca0a97c86bb2e52a6b1643221 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sun, 1 Sep 2019 15:41:03 +1000 Subject: [PATCH 11/47] Fixing some API bits - Adding the validator routes into the main function. - Fixing the setting of the aggregation bits, and handling errors correctly. - Rust format fixes, and addressing compiler warnings. --- beacon_node/rest_api/src/helpers.rs | 10 +++++--- beacon_node/rest_api/src/lib.rs | 6 ++--- beacon_node/rest_api/src/validator.rs | 33 +++++++++++++-------------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 98293e75c..d47afc02c 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -3,9 +3,9 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use hex; use hyper::{Body, Request}; +use std::sync::Arc; use store::{iter::AncestorIter, Store}; use types::{BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; -use std::sync::Arc; /// Parse a slot from a `0x` preixed string. /// @@ -170,12 +170,16 @@ pub fn implementation_pending_response(_req: Request) -> ApiResult { )) } -pub fn get_beacon_chain_from_request(req: &Request) -> Result>, ApiError> { +pub fn get_beacon_chain_from_request( + req: &Request, +) -> Result>, ApiError> { // Get beacon state let beacon_chain = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("Request is missing the beacon chain extension".into()))?; + .ok_or_else(|| { + ApiError::ServerError("Request is missing the beacon chain extension".into()) + })?; let _ = beacon_chain .ensure_state_caches_are_built() .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 2c9c4011a..b269bd476 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -170,11 +170,11 @@ pub fn start_server( validator::get_new_beacon_block::(req) } (&Method::POST, "/beacon/validator/block") => { - helpers::implementation_pending_response(req) + validator::publish_beacon_block::(req) } - /*(&Method::GET, "/beacon/validator/attestation") => { + (&Method::GET, "/beacon/validator/attestation") => { validator::get_new_attestation::(req) - }*/ + } (&Method::POST, "/beacon/validator/attestation") => { helpers::implementation_pending_response(req) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index f60acbad8..427f6a514 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,11 +1,9 @@ use super::{success_response, ApiResult}; use crate::{helpers::*, ApiError, UrlQuery}; -use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::BeaconChainTypes; use bls::{AggregateSignature, PublicKey, Signature}; use hyper::{Body, Request}; use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use std::borrow::Borrow; use types::beacon_state::EthSpec; use types::{Attestation, BitList, Epoch, RelativeEpoch, Shard, Slot}; @@ -140,7 +138,6 @@ pub fn get_validator_duties(req: Request) - /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. pub fn get_new_beacon_block(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; let slot = match query.first_of(&["slot"]) { @@ -190,7 +187,6 @@ pub fn get_new_beacon_block(req: Request) - /// HTTP Handler to accept a validator-signed BeaconBlock, and publish it to the network. pub fn publish_beacon_block(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; let slot = match query.first_of(&["slot"]) { @@ -212,9 +208,9 @@ pub fn publish_beacon_block(req: Request) - })? .as_slice(), ) - .map_err(|e| { - ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) - })?, + .map_err(|e| { + ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) + })?, Err(e) => { return Err(e); } @@ -237,10 +233,9 @@ pub fn publish_beacon_block(req: Request) - Ok(success_response(body)) } -/* /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request(req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; @@ -304,10 +299,16 @@ pub fn get_new_attestation(req: Request) -> return Err(e); } }; - let mut aggregation_bits: BitList = BitList::with_capacity(val_duty.committee_len) - .expect("An empty BitList should always be created, or we have bigger problems.") - .into(); - aggregation_bits.set(val_duty.committee_index, poc_bit); + let mut aggregation_bits = BitList::with_capacity(val_duty.committee_len) + .expect("An empty BitList should always be created, or we have bigger problems."); + aggregation_bits + .set(val_duty.committee_index, poc_bit) + .map_err(|e| { + ApiError::ServerError(format!( + "Unable to set aggregation bits for the attestation: {:?}", + e + )) + })?; // Allow a provided slot parameter to check against the expected slot as a sanity check. // Presently, we don't support attestations at future or past slots. @@ -354,7 +355,7 @@ pub fn get_new_attestation(req: Request) -> } }; - let attestation = Attestation { + let attestation: Attestation = Attestation { aggregation_bits, data: attestation_data, custody_bits: BitList::with_capacity(val_duty.committee_len) @@ -362,11 +363,9 @@ pub fn get_new_attestation(req: Request) -> signature: AggregateSignature::new(), }; - //TODO: This is currently AttestationData, but should be IndexedAttestation? let body = Body::from( serde_json::to_string(&attestation) .expect("We should always be able to serialize a new attestation that we produced."), ); Ok(success_response(body)) } -*/ From c13f27e24558f4b8f396a3b3b0bff89e8f651ae3 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 3 Sep 2019 16:30:04 +1000 Subject: [PATCH 12/47] Updating REST API. - Made /beacon/state return the current 'head' state when no parameters are provided. - Added some of the YAML api spec stuff to the /beacon/state endpoint in the rest_api spec. --- beacon_node/rest_api/src/beacon.rs | 22 ++++++++++++++-- docs/api_spec.yaml | 41 +++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 66d0b2673..36e7f6c57 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -163,8 +163,26 @@ pub fn get_state(req: Request) -> ApiResult .get::>>() .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; - let query_params = ["root", "slot"]; - let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; + let (key, value) = match UrlQuery::from_request(&req) { + Ok(query) => { + // We have *some* parameters, check them. + let query_params = ["root", "slot"]; + match query.first_of(&query_params) { + Ok((k, v)) => (k, v), + Err(e) => { + // Wrong parameters provided, or another error, return the error. + return Err(e); + } + } + }, + Err(ApiError::InvalidQueryParams(_)) => { + // No parameters provided at all, use current slot. + (String::from("slot"), beacon_chain.head().beacon_state.slot.to_string()) + } + Err(e) => { + return Err(e); + } + }; let (root, state): (Hash256, BeaconState) = match (key.as_ref(), value) { ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?, diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index 42b394e69..892ce7a68 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -344,7 +344,6 @@ paths: 500: $ref: '#/components/responses/InternalError' - #TODO Fill out block_root endpoint /beacon/block_root: get: tags: @@ -977,6 +976,46 @@ paths: #TODO fill out /beacon/state /beacon/state: + get: + tags: + - Phase0 + summary: "Get the full beacon state, at a particular slot or block root." + description: "Requests the beacon node to provide the full beacon state object, and the state root, given a particular slot number or block root. If no parameters are provided, the latest slot of the beacon node (the 'head' slot) is used." + parameters: + - name: root + description: "The block root at which the state should be provided." + in: query + required: false + schema: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + - name: slot + description: "The slot number at which the state should be provided." + in: query + required: false + schema: + type: integer + format: uint64 + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + properties: + root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + beacon_state: + #TODO: Need to add BeaconState Schema + $ref: '#/components/schemas/BeaconState' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' #TODO fill out /beacon/state_root /beacon/state_root: From 777987a49ecdf18e9bea7a3914f345e7452ff980 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 3 Sep 2019 20:13:26 +1000 Subject: [PATCH 13/47] Updating the spec to align with what's implemented. - Filled the OpenAPI spec for some major functions: - /beacon/state - /beacon/get_finalized_checkpoint - /beacon/state_root - Created some new schemas in the spec, such as Shard, Checkpoint, Validator, Eth1Data, BeaconState, PendingAttestation, Crosslink --- docs/api_spec.yaml | 368 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 311 insertions(+), 57 deletions(-) diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index 892ce7a68..2356d1f66 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -527,7 +527,7 @@ paths: validators: type: array items: - $ref: '#/components/schemas/ValidatorInfo' + $ref: '#/components/schemas/Validator' /beacon/validators/activesetchanges: get: @@ -974,7 +974,6 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - #TODO fill out /beacon/state /beacon/state: get: tags: @@ -1010,18 +1009,63 @@ paths: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" beacon_state: - #TODO: Need to add BeaconState Schema $ref: '#/components/schemas/BeaconState' 400: $ref: '#/components/responses/InvalidRequest' 500: $ref: '#/components/responses/InternalError' - #TODO fill out /beacon/state_root /beacon/state_root: + get: + tags: + - Phase0 + summary: "Get the beacon state root, at a particular slot." + description: "Requests the beacon node to provide the root of the beacon state object, given a particular slot number." + parameters: + - name: slot + description: "The slot number at which the state should be provided." + in: query + required: true + schema: + type: integer + format: uint64 + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + properties: + root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The state root" + + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' - #TODO fill out current_finalized_checkpoint /beacon/current_finalized_checkpoint: + get: + tags: + - Phase0 + summary: "Get the current finalized checkpoint." + #TODO: is this description correct? + description: "Requests the beacon node to provide the checkpoint for the current finalized epoch." + responses: + 200: + description: Success response + content: + application/json: + schema: + $ref: '#/components/schemas/Checkpoint' + + + 500: + $ref: '#/components/responses/InternalError' #TODO fill spec /spec: @@ -1134,6 +1178,28 @@ components: pattern: "^0x[a-fA-F0-9]{64}$" description: "A hex encoded ethereum address." + Shard: + type: integer + format: uint64 + description: "A shard number." + example: 5 + maximum: 1023 + minimum: 0 + + Checkpoint: + type: object + description: "A checkpoint." + properties: + epoch: + type: integer + format: uint64 + description: "The epoch to which the checkpoint applies." + root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A block root, which is being checkpointed." + Peer: type: object properties: @@ -1163,7 +1229,7 @@ components: format: uint64 description: "The global ValidatorIndex value." - ValidatorInfo: + Validator: type: object properties: public_key: @@ -1173,6 +1239,13 @@ components: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The 32 byte hash of the public key which the validator uses for withdrawing their rewards." + effective_balance: + type: integer + format: uint64 + description: "The effective balance of the validator, measured in Gwei." + slashed: + type: boolean + description: "Whether the validator has or has not been slashed." activation_eligiblity_epoch: type: integer format: uint64 @@ -1191,13 +1264,6 @@ components: format: uint64 nullable: true description: "Epoch when the validator is eligible to withdraw their funds, or null if the validator has not exited." - slashed: - type: boolean - description: "Whether the validator has or has not been slashed." - effective_balance: - type: integer - format: uint64 - description: "The effective balance of the validator, measured in Gwei." ValidatorDuty: type: object @@ -1235,6 +1301,25 @@ components: format: uint64 description: "Globally, the estimated most recent slot number, or current target slot number." + Eth1Data: + type: object + description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." + properties: + deposit_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the deposit tree." + deposit_count: + type: integer + format: uint64 + description: "Total number of deposits." + block_hash: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Ethereum 1.x block hash." + BeaconBlock: description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." allOf: @@ -1291,24 +1376,7 @@ components: pattern: "^0x[a-fA-F0-9]{192}$" description: "The RanDAO reveal value provided by the validator." eth1_data: - title: Eth1Data - type: object - description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." - properties: - deposit_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "Root of the deposit tree." - deposit_count: - type: integer - format: uint64 - description: "Total number of deposits." - block_hash: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "Ethereum 1.x block hash." + $ref: '#/components/schemas/Eth1Data' graffiti: type: string format: byte @@ -1442,6 +1510,161 @@ components: pattern: "^0x[a-fA-F0-9]{192}$" description: "Sender signature." + BeaconState: + type: object + description: "The [`BeaconState`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconstate) object from the Eth2.0 spec." + properties: + genesis_time: + $ref: '#/components/schemas/genesis_time' + slot: + type: integer + format: uint64 + description: "The latest slot, which the state represents." + fork: + $ref: '#/components/schemas/Fork' + latest_block_header: + $ref: '#/components/schemas/BeaconBlockHeader' + #TODO: Are these descriptions correct? + block_roots: + type: array + description: "The historical block roots." + minLength: 8192 + maxLength: 8192 #The SLOTS_PER_HISTORICAL_ROOT value from the Eth2.0 Spec. + items: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A block root" + state_roots: + type: array + description: "The historical state roots." + minLength: 8192 + maxLength: 8192 #The SLOTS_PER_HISTORICAL_ROOT value from the Eth2.0 Spec. + items: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A state root" + historical_roots: + type: array + #TODO: are these historical *state* roots? + description: "The historical state roots." + maxLength: 16777216 #The HISTORICAL_ROOTS_LIMIT value from the Eth2.0 Spec. + items: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A state root" + eth1_data: + $ref: '#/components/schemas/Eth1Data' + eth1_data_votes: + type: array + description: "The validator votes for the Eth1Data." + maxLength: 1024 #The SLOTS_PER_ETH1_VOTING_PERIOD value from the Eth2.0 spec. + items: + $ref: '#/components/schemas/Eth1Data' + eth1_deposit_index: + type: integer + format: uint64 + #TODO: Clarify this description + description: "The index of the Eth1 deposit." + validators: + type: array + description: "A list of the current validators." + maxLength: 1099511627776 + items: + $ref: '#/components/schemas/Validator' + balances: + type: array + description: "An array of the validator balances." + maxLength: 1099511627776 + items: + type: integer + format: uint64 + description: "The validator balance in GWei." + start_shard: + $ref: '#/components/schemas/Shard' + randao_mixes: + type: array + description: "The hashes for the randao mix." + minLength: 65536 + maxLength: 65536 #The EPOCHS_PER_HISTORICAL_VECTOR value from the Eth2.0 spec. + items: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A randao mix hash." + active_index_roots: + type: array + description: "Active index digests for light clients." + minLength: 65536 + maxLength: 65536 #The EPOCHS_PER_HISTORICAL_VECTOR value from the Eth2.0 spec. + items: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Active index digest" + compact_committees_roots: + type: array + description: "Committee digests for light clients." + minLength: 65536 + maxLength: 65536 #The EPOCHS_PER_HISTORICAL_VECTOR value from the Eth2.0 spec. + items: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Committee digest." + slashings: + type: array + description: "Per-epoch sums of slashed effective balances." + minLength: 8192 + maxLength: 8192 #The EPOCHS_PER_SLASHINGS_VECTOR value from the Eth2.0 spec. + items: + type: integer + format: uint64 + description: "Sum of slashed balance for an epoch." + previous_epoch_attestations: + type: array + description: "A list of attestations in the previous epoch." + maxLength: 8192 # MAX_ATTESTATIONS * SLOTS_PER_EPOCH from the Eth2.0 spec. + items: + $ref: '#/components/schemas/PendingAttestation' + current_epoch_attestations: + type: array + description: "A list of attestations in the current epoch." + maxLength: 8192 # MAX_ATTESTATIONS * SLOTS_PER_EPOCH from the Eth2.0 spec. + items: + $ref: '#/components/schemas/PendingAttestation' + previous_crosslinks: + type: array + description: "The shard crosslinks from the previous epoch." + minLength: 1024 + maxLength: 1024 #The SHARD_COUNT value from the Eth2.0 spec + items: + $ref: '#/components/schemas/Crosslink' + current_crosslinks: + type: array + description: "The shard crosslinks for the current epoch." + minLength: 1024 + maxLength: 1024 #The SHARD_COUNT value from the Eth2.0 spec + items: + $ref: '#/components/schemas/Crosslink' + justification_bits: + type: array + description: "Bit set for every recent justified epoch." + minLength: 4 + maxLength: 4 #The JUSTIFICATION_BITS_LENGTH from the Eth2.0 spec. + items: + type: boolean + #TODO: Check this description + description: "Whethere the recent epochs have been finalized." + previous_justified_checkpoint: + $ref: '#/components/schemas/Checkpoint' + current_justified_checkpoint: + $ref: '#/components/schemas/Checkpoint' + finalized_checkpoint: + $ref: '#/components/schemas/Checkpoint' + Fork: type: object description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." @@ -1508,6 +1731,35 @@ components: data: $ref: '#/components/schemas/AttestationData' + PendingAttestation: + type: object + description: "The [`PendingAttestation`](https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/specs/core/0_beacon-chain.md#pendingattestation) object from the Eth2.0 spec." + properties: + aggregation_bits: + type: array + description: "The bits representing aggregation of validator signatures and attestations." + maxLength: 4096 #The MAX_VALIDATORS_PER_COMMITTEE value from the Eth2.0 spec. + items: + type: boolean + description: "Whether the validator has been aggregated or not" + data: + $ref: '#/components/schemas/AttestationData' + inclusion_delay: + type: integer + format: uint64 + description: "The Slot at which it should be included." + proposer_index: + type: integer + format: uint64 + #TODO: This is the block proposer index, not the attestaion right? + description: "The ValidatorIndex of the block proposer" + + + + + + + AttestationData: type: object description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." @@ -1536,34 +1788,36 @@ components: pattern: "^0x[a-fA-F0-9]{64}$" description: "Target root from FFG vote." crosslink: - title: CrossLink - type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." - properties: - shard: - type: integer - format: uint64 - description: "The shard number." - start_epoch: - type: integer - format: uint64 - description: "The first epoch which the crosslinking data references." - end_epoch: - type: integer - format: uint64 - description: "The 'end' epoch referred to by the crosslinking data; no data in this Crosslink should refer to the `end_epoch` since it is not included in the crosslinking data interval." - parent_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "Root of the previous crosslink." - data_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "Root of the crosslinked shard data since the previous crosslink." + $ref: '#/components/schemas/Crosslink' + Crosslink: + type: object + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." + properties: + shard: + type: integer + format: uint64 + description: "The shard number." + start_epoch: + type: integer + format: uint64 + description: "The first epoch which the crosslinking data references." + end_epoch: + type: integer + format: uint64 + description: "The 'end' epoch referred to by the crosslinking data; no data in this Crosslink should refer to the `end_epoch` since it is not included in the crosslinking data interval." + parent_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the previous crosslink." + data_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the crosslinked shard data since the previous crosslink." + responses: Success: description: "Request successful." From dcd074877b7f432fbd8c7ad2224cd6b173f0ff6d Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 4 Sep 2019 10:57:09 +1000 Subject: [PATCH 14/47] Removed block publish feature, since it's incomplete currently. --- beacon_node/rest_api/src/lib.rs | 2 +- beacon_node/rest_api/src/validator.rs | 49 --------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index b269bd476..2c7b90e3f 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -170,7 +170,7 @@ pub fn start_server( validator::get_new_beacon_block::(req) } (&Method::POST, "/beacon/validator/block") => { - validator::publish_beacon_block::(req) + helpers::implementation_pending_response(req) } (&Method::GET, "/beacon/validator/attestation") => { validator::get_new_attestation::(req) diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 427f6a514..bbc976175 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -184,55 +184,6 @@ pub fn get_new_beacon_block(req: Request) - Ok(success_response(body)) } -/// HTTP Handler to accept a validator-signed BeaconBlock, and publish it to the network. -pub fn publish_beacon_block(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; - - let query = UrlQuery::from_request(&req)?; - let slot = match query.first_of(&["slot"]) { - Ok((_, v)) => Slot::new(v.parse::().map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) - })?), - Err(e) => { - return Err(e); - } - }; - let randao_reveal = match query.first_of(&["randao_reveal"]) { - Ok((_, v)) => Signature::from_bytes( - hex::decode(&v) - .map_err(|e| { - ApiError::InvalidQueryParams(format!( - "Invalid hex string for randao_reveal: {:?}", - e - )) - })? - .as_slice(), - ) - .map_err(|e| { - ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) - })?, - Err(e) => { - return Err(e); - } - }; - - let new_block = match beacon_chain.produce_block(randao_reveal, slot) { - Ok((block, _state)) => block, - Err(e) => { - return Err(ApiError::ServerError(format!( - "Beacon node is not able to produce a block: {:?}", - e - ))); - } - }; - - let body = Body::from( - serde_json::to_string(&new_block) - .expect("We should always be able to serialize a new block that we produced."), - ); - Ok(success_response(body)) -} - /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; From b432c8c58c356c8a0c8de96176b091781adc60c6 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 4 Sep 2019 11:18:29 +1000 Subject: [PATCH 15/47] Replaced unnecessary match statements with map_err and ok_or --- beacon_node/rest_api/src/validator.rs | 50 +++++++++------------------ 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index bbc976175..84ea485b5 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -53,15 +53,11 @@ pub fn get_validator_duties(req: Request) - e )) })?; - let validators: Vec = match query.all_of("validator_pubkeys") { - Ok(v) => v - .iter() - .map(|pk| parse_pubkey(pk)) - .collect::, _>>()?, - Err(e) => { - return Err(e); - } - }; + let validators: Vec = query + .all_of("validator_pubkeys")? + .iter() + .map(|pk| parse_pubkey(pk)) + .collect::, _>>()?; let mut duties: Vec = Vec::new(); // Get a list of all validators for this epoch @@ -167,15 +163,14 @@ pub fn get_new_beacon_block(req: Request) - } }; - let new_block = match beacon_chain.produce_block(randao_reveal, slot) { - Ok((block, _state)) => block, - Err(e) => { - return Err(ApiError::ServerError(format!( + let (new_block, _state) = beacon_chain + .produce_block(randao_reveal, slot) + .map_err(|e| { + ApiError::ServerError(format!( "Beacon node is not able to produce a block: {:?}", e - ))); - } - }; + )) + })?; let body = Body::from( serde_json::to_string(&new_block) @@ -229,14 +224,9 @@ pub fn get_new_attestation(req: Request) -> }; // Check that we are requesting an attestation during the slot where it is relevant. - let present_slot = match beacon_chain.read_slot_clock() { - Some(s) => s, - None => { - return Err(ApiError::ServerError( - "Beacon node is unable to determine present slot, either the state isn't generated or the chain hasn't begun.".into() - )); - } - }; + let present_slot = beacon_chain.read_slot_clock().ok_or(ApiError::ServerError( + "Beacon node is unable to determine present slot, either the state isn't generated or the chain hasn't begun.".into() + ))?; if val_duty.slot != present_slot { return Err(ApiError::InvalidQueryParams(format!("Validator is only able to request an attestation during the slot they are allocated. Current slot: {:?}, allocated slot: {:?}", head_state.slot, val_duty.slot))); } @@ -296,15 +286,9 @@ pub fn get_new_attestation(req: Request) -> return Err(e); } }; - let attestation_data = match beacon_chain.produce_attestation_data(shard) { - Ok(v) => v, - Err(e) => { - return Err(ApiError::ServerError(format!( - "Could not produce an attestation: {:?}", - e - ))); - } - }; + let attestation_data = beacon_chain + .produce_attestation_data(shard) + .map_err(|e| ApiError::ServerError(format!("Could not produce an attestation: {:?}", e)))?; let attestation: Attestation = Attestation { aggregation_bits, From 0c1ceab5276d97b18a21163fcb6199d2c283b593 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 4 Sep 2019 13:43:45 +1000 Subject: [PATCH 16/47] Addressed Paul's suggestions. - Updated some comments. - Replaced match statements with map functions. --- beacon_node/rest_api/src/metrics.rs | 2 +- beacon_node/rest_api/src/network.rs | 14 +-- beacon_node/rest_api/src/validator.rs | 162 +++++++++++--------------- 3 files changed, 74 insertions(+), 104 deletions(-) diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 9d2ecc343..2239249b6 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -67,7 +67,7 @@ pub fn get_prometheus(req: Request) -> ApiR String::from_utf8(buffer) .map(|string| { let mut response = success_response(Body::from(string)); - // Need to change the header to text/plain for prometheius + // Need to change the header to text/plain for prometheus response.headers_mut().insert( "content-type", HeaderValue::from_static("text/plain; charset=utf-8"), diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs index dffa949c9..4f1f53bb9 100644 --- a/beacon_node/rest_api/src/network.rs +++ b/beacon_node/rest_api/src/network.rs @@ -4,7 +4,7 @@ use eth2_libp2p::{Enr, Multiaddr, PeerId}; use hyper::{Body, Request}; use std::sync::Arc; -/// HTTP handle to return the list of libp2p multiaddr the client is listening on. +/// HTTP handler to return the list of libp2p multiaddr the client is listening on. /// /// Returns a list of `Multiaddr`, serialized according to their `serde` impl. pub fn get_listen_addresses(req: Request) -> ApiResult { @@ -21,9 +21,9 @@ pub fn get_listen_addresses(req: Request) -> ApiResul ))) } -/// HTTP handle to return network port the client is listening on. +/// HTTP handler to return the network port the client is listening on. /// -/// Returns a list of `Multiaddr`, serialized according to their `serde` impl. +/// Returns the TCP port number in its plain form (which is also valid JSON serialization) pub fn get_listen_port(req: Request) -> ApiResult { let network = req .extensions() @@ -36,7 +36,7 @@ pub fn get_listen_port(req: Request) -> ApiResult { ))) } -/// HTTP handle to return the Discv5 ENR from the client's libp2p service. +/// HTTP handler to return the Discv5 ENR from the client's libp2p service. /// /// ENR is encoded as base64 string. pub fn get_enr(req: Request) -> ApiResult { @@ -53,7 +53,7 @@ pub fn get_enr(req: Request) -> ApiResult { ))) } -/// HTTP handle to return the `PeerId` from the client's libp2p service. +/// HTTP handler to return the `PeerId` from the client's libp2p service. /// /// PeerId is encoded as base58 string. pub fn get_peer_id(req: Request) -> ApiResult { @@ -70,7 +70,7 @@ pub fn get_peer_id(req: Request) -> ApiResult { ))) } -/// HTTP handle to return the number of peers connected in the client's libp2p service. +/// HTTP handler to return the number of peers connected in the client's libp2p service. pub fn get_peer_count(req: Request) -> ApiResult { let network = req .extensions() @@ -85,7 +85,7 @@ pub fn get_peer_count(req: Request) -> ApiResult { ))) } -/// HTTP handle to return the list of peers connected to the client's libp2p service. +/// HTTP handler to return the list of peers connected to the client's libp2p service. /// /// Peers are presented as a list of `PeerId::to_string()`. pub fn get_peer_list(req: Request) -> ApiResult { diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 84ea485b5..229d84674 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -136,32 +136,24 @@ pub fn get_new_beacon_block(req: Request) - let beacon_chain = get_beacon_chain_from_request::(&req)?; let query = UrlQuery::from_request(&req)?; - let slot = match query.first_of(&["slot"]) { - Ok((_, v)) => Slot::new(v.parse::().map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) - })?), - Err(e) => { - return Err(e); - } - }; - let randao_reveal = match query.first_of(&["randao_reveal"]) { - Ok((_, v)) => Signature::from_bytes( - hex::decode(&v) - .map_err(|e| { - ApiError::InvalidQueryParams(format!( - "Invalid hex string for randao_reveal: {:?}", - e - )) - })? - .as_slice(), - ) + let slot = query + .first_of(&["slot"]) + .map(|(_key, value)| value)? + .parse::() + .map(Slot::from) .map_err(|e| { - ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) - })?, - Err(e) => { - return Err(e); - } - }; + ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + })?; + let randao_bytes = query + .first_of(&["randao_reveal"]) + .map(|(_key, value)| value) + .map(hex::decode)? + .map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid hex string for randao_reveal: {:?}", e)) + })?; + let randao_reveal = Signature::from_bytes(randao_bytes.as_slice()).map_err(|e| { + ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) + })?; let (new_block, _state) = beacon_chain .produce_block(randao_reveal, slot) @@ -185,43 +177,33 @@ pub fn get_new_attestation(req: Request) -> let head_state = &beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; - let val_pk: PublicKey = match query.first_of(&["validator_pubkey"]) { - Ok((_, v)) => parse_pubkey(v.as_str())?, - Err(e) => { - return Err(e); - } - }; + let val_pk_str = query + .first_of(&["validator_pubkey"]) + .map(|(_key, value)| value)?; + let val_pk = parse_pubkey(val_pk_str.as_str())?; + // Get the validator index from the supplied public key // If it does not exist in the index, we cannot continue. - let val_index: usize = match head_state.get_validator_index(&val_pk) { - Ok(Some(i)) => i, - Ok(None) => { - return Err(ApiError::InvalidQueryParams( - "The provided validator public key does not correspond to a validator index." - .into(), - )); - } - Err(e) => { - return Err(ApiError::ServerError(format!( - "Unable to read validator index cache. {:?}", - e - ))); - } - }; + let val_index = head_state + .get_validator_index(&val_pk) + .map_err(|e| { + ApiError::ServerError(format!("Unable to read validator index cache. {:?}", e)) + })? + .ok_or(ApiError::InvalidQueryParams( + "The provided validator public key does not correspond to a validator index.".into(), + ))?; + // Get the duties of the validator, to make sure they match up. // If they don't have duties this epoch, then return an error - let val_duty = match head_state.get_attestation_duties(val_index, RelativeEpoch::Current) { - Ok(Some(d)) => d, - Ok(None) => { - return Err(ApiError::InvalidQueryParams("No validator duties could be found for the requested validator. Cannot provide valid attestation.".into())); - } - Err(e) => { - return Err(ApiError::ServerError(format!( + let val_duty = head_state + .get_attestation_duties(val_index, RelativeEpoch::Current) + .map_err(|e| { + ApiError::ServerError(format!( "unable to read cache for attestation duties: {:?}", e - ))) - } - }; + )) + })? + .ok_or(ApiError::InvalidQueryParams("No validator duties could be found for the requested validator. Cannot provide valid attestation.".into()))?; // Check that we are requesting an attestation during the slot where it is relevant. let present_slot = beacon_chain.read_slot_clock().ok_or(ApiError::ServerError( @@ -232,14 +214,14 @@ pub fn get_new_attestation(req: Request) -> } // Parse the POC bit and insert it into the aggregation bits - let poc_bit: bool = match query.first_of(&["poc_bit"]) { - Ok((_, v)) => v.parse::().map_err(|e| { - ApiError::InvalidQueryParams(format!("poc_bit is not a valid boolean value: {:?}", e)) - })?, - Err(e) => { - return Err(e); - } - }; + let poc_bit = query + .first_of(&["poc_bit"]) + .map(|(_key, value)| value)? + .parse::() + .map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + })?; + let mut aggregation_bits = BitList::with_capacity(val_duty.committee_len) .expect("An empty BitList should always be created, or we have bigger problems."); aggregation_bits @@ -251,41 +233,29 @@ pub fn get_new_attestation(req: Request) -> )) })?; - // Allow a provided slot parameter to check against the expected slot as a sanity check. + // Allow a provided slot parameter to check against the expected slot as a sanity check only. // Presently, we don't support attestations at future or past slots. - let _slot = match query.first_of(&["slot"]) { - Ok((_, v)) => { - let requested_slot = v.parse::().map_err(|e| { - ApiError::InvalidQueryParams(format!( - "Invalid slot parameter, must be a u64. {:?}", - e - )) - })?; - let current_slot = beacon_chain.head().beacon_state.slot.as_u64(); - if requested_slot != current_slot { - return Err(ApiError::InvalidQueryParams(format!("Attestation data can only be requested for the current slot ({:?}), not your requested slot ({:?})", current_slot, requested_slot))); - } - Slot::new(requested_slot) - } - Err(ApiError::InvalidQueryParams(_)) => { - // Just fill _slot with a dummy value for now, making the slot parameter optional - // We'll get the real slot from the ValidatorDuty - Slot::new(0) - } - Err(e) => { - return Err(e); - } - }; + let requested_slot = query + .first_of(&["slot"]) + .map(|(_key, value)| value)? + .parse::() + .map(Slot::from) + .map_err(|e| { + ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + })?; + let current_slot = beacon_chain.head().beacon_state.slot.as_u64(); + if requested_slot != current_slot { + return Err(ApiError::InvalidQueryParams(format!("Attestation data can only be requested for the current slot ({:?}), not your requested slot ({:?})", current_slot, requested_slot))); + } - let shard: Shard = match query.first_of(&["shard"]) { - Ok((_, v)) => v.parse::().map_err(|e| { + let shard = query + .first_of(&["shard"]) + .map(|(_key, value)| value)? + .parse::() + .map_err(|e| { ApiError::InvalidQueryParams(format!("Shard is not a valid u64 value: {:?}", e)) - })?, - Err(e) => { - // This is a mandatory parameter, return the error - return Err(e); - } - }; + })?; + let attestation_data = beacon_chain .produce_attestation_data(shard) .map_err(|e| ApiError::ServerError(format!("Could not produce an attestation: {:?}", e)))?; From 28a2ce2bdc9f994255011a11896cee28cf0900ed Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 4 Sep 2019 14:19:48 +1000 Subject: [PATCH 17/47] Fix formatting with rustfmt. --- beacon_node/rest_api/src/beacon.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 36e7f6c57..11e1446d9 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -174,10 +174,13 @@ pub fn get_state(req: Request) -> ApiResult return Err(e); } } - }, + } Err(ApiError::InvalidQueryParams(_)) => { // No parameters provided at all, use current slot. - (String::from("slot"), beacon_chain.head().beacon_state.slot.to_string()) + ( + String::from("slot"), + beacon_chain.head().beacon_state.slot.to_string(), + ) } Err(e) => { return Err(e); From eeba69cd0f41e0cdc7cd07a5765142fe8b6c9527 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 4 Sep 2019 22:03:55 +1000 Subject: [PATCH 18/47] Moved beacon chain from request functionality into its own function. --- beacon_node/rest_api/src/beacon.rs | 20 ++++---------------- beacon_node/rest_api/src/metrics.rs | 10 +++------- beacon_node/rest_api/src/spec.rs | 8 +++----- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 1da640baf..935368892 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -109,10 +109,7 @@ pub fn get_block(req: Request) -> ApiResult /// HTTP handler to return a `BeaconBlock` root at a given `slot`. pub fn get_block_root(req: Request) -> ApiResult { - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; let target = parse_slot(&slot_string)?; @@ -163,10 +160,7 @@ pub struct StateResponse { /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn get_state(req: Request) -> ApiResult { - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let (key, value) = match UrlQuery::from_request(&req) { Ok(query) => { @@ -220,10 +214,7 @@ pub fn get_state(req: Request) -> ApiResult /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn get_state_root(req: Request) -> ApiResult { - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; let slot = parse_slot(&slot_string)?; @@ -240,10 +231,7 @@ pub fn get_state_root(req: Request) -> ApiR pub fn get_current_finalized_checkpoint( req: Request, ) -> ApiResult { - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let checkpoint = beacon_chain .head() diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 2239249b6..01dc4d22d 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -1,9 +1,8 @@ -use crate::{success_response, ApiError, ApiResult, DBPath}; -use beacon_chain::{BeaconChain, BeaconChainTypes}; +use crate::{helpers::*, success_response, ApiError, ApiResult, DBPath}; +use beacon_chain::BeaconChainTypes; use http::HeaderValue; use hyper::{Body, Request}; use prometheus::{Encoder, TextEncoder}; -use std::sync::Arc; pub use lighthouse_metrics::*; @@ -31,10 +30,7 @@ pub fn get_prometheus(req: Request) -> ApiR let mut buffer = vec![]; let encoder = TextEncoder::new(); - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let db_path = req .extensions() .get::() diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index 86d1c227d..a353b3833 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -1,6 +1,7 @@ use super::{success_response, ApiResult}; +use crate::helpers::*; use crate::ApiError; -use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::BeaconChainTypes; use eth2_config::Eth2Config; use hyper::{Body, Request}; use std::sync::Arc; @@ -8,10 +9,7 @@ use types::EthSpec; /// HTTP handler to return the full spec object. pub fn get_spec(req: Request) -> ApiResult { - let beacon_chain = req - .extensions() - .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let json: String = serde_json::to_string(&beacon_chain.spec) .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; From bf2f4597738c987d4b1454b8d9adfbdf011ee12a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 4 Sep 2019 23:03:05 +1000 Subject: [PATCH 19/47] Extended API - Added a /beacon/validators function, to list all validators active in a particular epoch - Moved 'get_genesis_state' function, to align with router. - Added content-type for error responses - Tried adding a cache update call to fix issue getting validator duties (this is WIP) --- beacon_node/rest_api/src/beacon.rs | 57 +++++++++++++++++++++++---- beacon_node/rest_api/src/lib.rs | 5 +-- beacon_node/rest_api/src/validator.rs | 13 ++++++ 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 935368892..ae112883e 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -6,7 +6,7 @@ use serde::Serialize; use ssz_derive::Encode; use std::sync::Arc; use store::Store; -use types::{BeaconBlock, BeaconState, EthSpec, Hash256, Slot}; +use types::{BeaconBlock, BeaconState, Epoch, EthSpec, Hash256, Slot}; #[derive(Serialize)] pub struct HeadResponse { @@ -136,16 +136,47 @@ pub fn get_fork(req: Request) -> ApiResult Ok(success_response(Body::from(json))) } -/// HTTP handler to return a `BeaconState` at a given `root` or `slot`. +/// HTTP handler to return the set of validators for an `Epoch` /// -/// Will not return a state if the request slot is in the future. Will return states higher than -/// the current head by skipping slots. -pub fn get_genesis_state(req: Request) -> ApiResult { +/// The `Epoch` parameter can be any epoch number. If it is not specified, +/// the current epoch is assumed. +pub fn get_validators(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; + let epoch = match UrlQuery::from_request(&req) { + // We have some parameters, so make sure it's the epoch one and parse it + Ok(query) => query + .only_one("epoch")? + .parse::() + .map(Epoch::from) + .map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Invalid epoch parameter, must be a u64. {:?}", + e + )) + })?, + // In this case, our url query did not contain any parameters, so we take the default + Err(_) => beacon_chain.epoch().map_err(|e| { + ApiError::ServerError(format!("Unable to determine current epoch: {:?}", e)) + })?, + }; - ResponseBuilder::new(&req).body(&state) + let all_validators = &beacon_chain.head().beacon_state.validators; + let mut active_validators = Vec::with_capacity(all_validators.len()); + for (_index, validator) in all_validators.iter().enumerate() { + if validator.is_active_at(epoch) { + active_validators.push(validator) + } + } + active_validators.shrink_to_fit(); + let json: String = serde_json::to_string(&active_validators).map_err(|e| { + ApiError::ServerError(format!( + "Unable to serialize list of active validators: {:?}", + e + )) + })?; + + Ok(success_response(Body::from(json))) } #[derive(Serialize, Encode)] @@ -244,3 +275,15 @@ pub fn get_current_finalized_checkpoint( Ok(success_response(Body::from(json))) } + +/// HTTP handler to return a `BeaconState` at a given `root` or `slot`. +/// +/// Will not return a state if the request slot is in the future. Will return states higher than +/// the current head by skipping slots. +pub fn get_genesis_state(req: Request) -> ApiResult { + let beacon_chain = get_beacon_chain_from_request::(&req)?; + + let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; + + ResponseBuilder::new(&req).body(&state) +} diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 4dbbdda51..02c68c639 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -54,6 +54,7 @@ impl Into> for ApiError { }; Response::builder() .status(status_code.0) + .header("content-type", "text/plain") .body(Body::from(status_code.1)) .expect("Response should always be created.") } @@ -160,9 +161,7 @@ pub fn start_server( helpers::implementation_pending_response(req) } - (&Method::GET, "/beacon/validators") => { - helpers::implementation_pending_response(req) - } + (&Method::GET, "/beacon/validators") => beacon::get_validators::(req), (&Method::GET, "/beacon/validators/indicies") => { helpers::implementation_pending_response(req) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 010e49305..2374373bd 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -60,6 +60,18 @@ pub fn get_validator_duties(req: Request) - .collect::, _>>()?; let mut duties: Vec = Vec::new(); + // Update the committee cache + // TODO: Do we need to update the cache on the state, for the epoch which has been specified? + beacon_chain + .state_now() + .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))? + .maybe_as_mut_ref() + .ok_or(ApiError::ServerError( + "Unable to get mutable BeaconState".into(), + ))? + .build_committee_cache(relative_epoch, &beacon_chain.spec) + .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + // Get a list of all validators for this epoch let validator_proposers: Vec = epoch .slot_iter(T::EthSpec::slots_per_epoch()) @@ -67,6 +79,7 @@ pub fn get_validator_duties(req: Request) - head_state .get_beacon_proposer_index(slot, relative_epoch, &beacon_chain.spec) .map_err(|e| { + // TODO: why are we getting an uninitialized state error here??? ApiError::ServerError(format!( "Unable to get proposer index for validator: {:?}", e From 32ca8e951d7e5b712e20f1ff37f4d0a6eb868e7c Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 5 Sep 2019 00:36:06 +1000 Subject: [PATCH 20/47] Updated content-type acceptance and returning, mainly for /spec/eth2_config --- beacon_node/rest_api/src/beacon.rs | 5 +-- beacon_node/rest_api/src/response_builder.rs | 35 ++++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index ae112883e..70b3f3ee9 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -276,10 +276,7 @@ pub fn get_current_finalized_checkpoint( Ok(success_response(Body::from(json))) } -/// HTTP handler to return a `BeaconState` at a given `root` or `slot`. -/// -/// Will not return a state if the request slot is in the future. Will return states higher than -/// the current head by skipping slots. +/// HTTP handler to return a `BeaconState` at the genesis block. pub fn get_genesis_state(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; diff --git a/beacon_node/rest_api/src/response_builder.rs b/beacon_node/rest_api/src/response_builder.rs index 9b8819996..c1df4892c 100644 --- a/beacon_node/rest_api/src/response_builder.rs +++ b/beacon_node/rest_api/src/response_builder.rs @@ -26,24 +26,31 @@ impl ResponseBuilder { } pub fn body(self, item: &T) -> ApiResult { - let body: Body = match self.encoding { - Encoding::JSON => Body::from(serde_json::to_string(&item).map_err(|e| { - ApiError::ServerError(format!( - "Unable to serialize response body as JSON: {:?}", - e - )) - })?), - Encoding::SSZ => Body::from(item.as_ssz_bytes()), - Encoding::YAML => Body::from(serde_yaml::to_string(&item).map_err(|e| { - ApiError::ServerError(format!( - "Unable to serialize response body as YAML: {:?}", - e - )) - })?), + let (body, content_type) = match self.encoding { + Encoding::JSON => ( + Body::from(serde_json::to_string(&item).map_err(|e| { + ApiError::ServerError(format!( + "Unable to serialize response body as JSON: {:?}", + e + )) + })?), + "application/json", + ), + Encoding::SSZ => (Body::from(item.as_ssz_bytes()), "application/ssz"), + Encoding::YAML => ( + Body::from(serde_yaml::to_string(&item).map_err(|e| { + ApiError::ServerError(format!( + "Unable to serialize response body as YAML: {:?}", + e + )) + })?), + "application/ssz", + ), }; Response::builder() .status(StatusCode::OK) + .header("content-type", content_type) .body(Body::from(body)) .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) } From 4339e372c7f859ac3d0135d8caaa9e73899f7ded Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 5 Sep 2019 00:39:54 +1000 Subject: [PATCH 21/47] Updated the API spec. - Moved ENR into it's own object - Moved the POST request parameters into the requestBody attribute (instead of query) - Updated IndexedAttestation to Attestation - Updated path for current_finalized_checkpoint - Completed the /spec and /spec and /spec/slots_per_epoch endpoints - Completed the /beacon/state/genesis endpoint - Completed the /spec/eth2_config endpoint - Fixed the prometheus example value - Added various example values to reflect real world values - Fixed incorrect indenting of Eth1Data - Added the whole ChainSpec schema --- docs/api_spec.yaml | 333 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 281 insertions(+), 52 deletions(-) diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index 2356d1f66..ced07e96d 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -84,10 +84,7 @@ paths: content: application/json: schema: - type: string - format: byte - example: "-IW4QHzEZbIB0YN47bVlsUrGbcL9vl21n7xF5gRKjMNkJ4MxfcwiqrsE7Ows8EnzOvC8P4ZyAjfOhr2ffk0bWAxDGq8BgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjzKzqo5c33ydUUHrWJ4FWwIXJa2MN9BBsgZkj6mhthp" - pattern: "^[^-A-Za-z0-9+/=]+$" + $ref: '#/components/schemas/ENR' 500: $ref: '#/components/responses/InternalError' @@ -884,13 +881,13 @@ paths: - Phase0 summary: "Publish a signed block." description: "Instructs the beacon node to broadcast a newly signed beacon block to the beacon network, to be included in the beacon chain. The beacon node is not required to validate the signed `BeaconBlock`, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new block into its state, and therefore validate the block internally, however blocks which fail the validation are still broadcast but a different status code is returned (202)" - parameters: - - name: beacon_block - in: query - required: true - description: "The `BeaconBlock` object, as sent from the beacon node originally, but now with the signature field completed." - schema: - $ref: '#/components/schemas/BeaconBlock' + requestBody: + description: "The `BeaconBlock` object, as sent from the beacon node originally, but now with the signature field completed. Must be sent in JSON format in the body of the request." + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BeaconBlock' responses: 200: description: "The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." @@ -908,7 +905,7 @@ paths: tags: - Phase0 summary: "Produce an attestation, without signature." - description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." + description: "Requests that the beacon node produce an Attestation, with a blank signature field, which the validator will then sign." parameters: - name: validator_pubkey in: query @@ -919,7 +916,7 @@ paths: - name: poc_bit in: query required: true - description: "The proof-of-custody bit that is to be reported by the requesting validator. This bit will be inserted into the appropriate location in the returned `IndexedAttestation`." + description: "The proof-of-custody bit that is to be reported by the requesting validator. This bit will be inserted into the appropriate location in the returned `Attestation`." schema: type: integer format: uint32 @@ -943,7 +940,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/IndexedAttestation' + $ref: '#/components/schemas/Attestation' 400: $ref: '#/components/responses/InvalidRequest' 500: @@ -954,14 +951,14 @@ paths: tags: - Phase0 summary: "Publish a signed attestation." - description: "Instructs the beacon node to broadcast a newly signed IndexedAttestation object to the intended shard subnet. The beacon node is not required to validate the signed IndexedAttestation, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new attestation into its state, and therefore validate the attestation internally, however attestations which fail the validation are still broadcast but a different status code is returned (202)" - parameters: - - name: attestation - in: query - required: true - description: "An `IndexedAttestation` structure, as originally provided by the beacon node, but now with the signature field completed." - schema: - $ref: '#/components/schemas/IndexedAttestation' + description: "Instructs the beacon node to broadcast a newly signed Attestation object to the intended shard subnet. The beacon node is not required to validate the signed Attestation, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new attestation into its state, and therefore validate the attestation internally, however attestations which fail the validation are still broadcast but a different status code is returned (202)" + requestBody: + description: "An `Attestation` structure, as originally provided by the beacon node, but now with the signature field completed. Must be sent in JSON format in the body of the request." + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/Attestation' responses: 200: description: "The attestation was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." @@ -1042,13 +1039,12 @@ paths: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The state root" - 400: $ref: '#/components/responses/InvalidRequest' 500: $ref: '#/components/responses/InternalError' - /beacon/current_finalized_checkpoint: + /beacon/state/current_finalized_checkpoint: get: tags: - Phase0 @@ -1062,16 +1058,63 @@ paths: application/json: schema: $ref: '#/components/schemas/Checkpoint' - - 500: $ref: '#/components/responses/InternalError' - #TODO fill spec - /spec: + /beacon/state/genesis: + get: + tags: + - Phase0 + summary: "Get the full beacon state, as it was at genesis." + description: "Requests the beacon node to provide the full beacon state object and the state root, as it was for the genesis block." + responses: + 200: + description: Success response + content: + application/json: + schema: + $ref: '#/components/schemas/BeaconState' + application/yaml: + schema: + $ref: '#/components/schemas/BeaconState' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + + /spec: + get: + tags: + - Phase0 + summary: "Get the current ChainSpec configuration." + description: "Requests the beacon node to provide the configuration that it has used to start the beacon chain." + responses: + 200: + description: Success response + content: + application/json: + schema: + $ref: '#/components/schemas/ChainSpec' + 500: + $ref: '#/components/responses/InternalError' - #TODO fill spec/slots_per_epoch /spec/slots_per_epoch: + get: + tags: + - Phase0 + summary: "Get the configured number of slots per epoch." + description: "The number of slots in each epoch is part of the Eth2.0 spec. This function simply returns an integer representing this value." + responses: + 200: + description: Success response + content: + application/json: + schema: + type: integer + format: uint64 + example: 64 + 500: + $ref: '#/components/responses/InternalError' /spec/deposit_contract: get: @@ -1089,6 +1132,28 @@ paths: 500: $ref: '#/components/responses/InternalError' + /spec/eth2_config: + get: + tags: + - Phase0 + summary: "Gets the Eth2.0 spec, including the identifying string." + description: "" + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + properties: + spec_constants: + type: string + example: "mainnet" + spec: + $ref: '#/components/schemas/ChainSpec' + 500: + $ref: '#/components/responses/InternalError' + /metrics: get: tags: @@ -1100,9 +1165,7 @@ paths: description: Request successful content: text/plain: - example: - summary: 'Promethius metrics' - value: "# HELP beacon_head_state_active_validators_total Count of active validators at the head of the chain + example: "# HELP beacon_head_state_active_validators_total Count of active validators at the head of the chain # TYPE beacon_head_state_active_validators_total gauge beacon_head_state_active_validators_total 16 # HELP beacon_head_state_current_justified_epoch Current justified epoch at the head of the chain @@ -1178,6 +1241,12 @@ components: pattern: "^0x[a-fA-F0-9]{64}$" description: "A hex encoded ethereum address." + ENR: + type: string + format: byte + example: "-IW4QHzEZbIB0YN47bVlsUrGbcL9vl21n7xF5gRKjMNkJ4MxfcwiqrsE7Ows8EnzOvC8P4ZyAjfOhr2ffk0bWAxDGq8BgmlwhH8AAAGDdGNwgiMog3VkcIIjKIlzZWNwMjU2azGhAjzKzqo5c33ydUUHrWJ4FWwIXJa2MN9BBsgZkj6mhthp" + pattern: "^[^-A-Za-z0-9+/=]+$" + Shard: type: integer format: uint64 @@ -1239,13 +1308,16 @@ components: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The 32 byte hash of the public key which the validator uses for withdrawing their rewards." + example: "0x00ec7ef7780c9d151597924036262dd28dc60e1228f4da6fecf9d402cb3f3594" effective_balance: type: integer format: uint64 description: "The effective balance of the validator, measured in Gwei." + example: 32000000000 slashed: type: boolean description: "Whether the validator has or has not been slashed." + example: false activation_eligiblity_epoch: type: integer format: uint64 @@ -1259,11 +1331,13 @@ components: format: uint64 nullable: true description: "Epoch when the validator was exited, or null if the validator has not exited." + example: 18446744073709551615 withdrawable_epoch: type: integer format: uint64 nullable: true description: "Epoch when the validator is eligible to withdraw their funds, or null if the validator has not exited." + example: 18446744073709551615 ValidatorDuty: type: object @@ -1301,24 +1375,24 @@ components: format: uint64 description: "Globally, the estimated most recent slot number, or current target slot number." - Eth1Data: - type: object - description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." - properties: - deposit_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "Root of the deposit tree." - deposit_count: - type: integer - format: uint64 - description: "Total number of deposits." - block_hash: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "Ethereum 1.x block hash." + Eth1Data: + type: object + description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." + properties: + deposit_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the deposit tree." + deposit_count: + type: integer + format: uint64 + description: "Total number of deposits." + block_hash: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Ethereum 1.x block hash." BeaconBlock: description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." @@ -1404,9 +1478,9 @@ components: description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." properties: attestation_1: - $ref: '#/components/schemas/IndexedAttestation' + $ref: '#/components/schemas/Attestation' attestation_2: - $ref: '#/components/schemas/IndexedAttestation' + $ref: '#/components/schemas/Attestation' attestations: type: array items: @@ -1818,6 +1892,161 @@ components: pattern: "^0x[a-fA-F0-9]{64}$" description: "Root of the crosslinked shard data since the previous crosslink." + ChainSpec: + type: object + description: "Stores all of the values which specify a particular chain. The `ChainSpec` object in Lighthouse" + properties: + far_future_epoch: + type: integer + format: uint64 + example: 18446744073709551615 + base_rewards_per_epoch: + type: integer + format: uint64 + example: 5 + deposit_contract_tree_depth: + type: integer + format: uint64 + example: 32 + seconds_per_day: + type: integer + format: uint64 + example: 86400 + target_committee_size: + type: integer + format: uint64 + example: 128 + min_per_epoch_churn_limit: + type: integer + format: uint64 + example: 4 + churn_limit_quotient: + type: integer + format: uint64 + example: 65536 + shuffle_round_count: + type: integer + format: uint8 + example: 90 + min_genesis_active_validator_count: + type: integer + format: uint64 + example: 65536 + min_genesis_time: + type: integer + format: uint64 + example: 1578009600 + min_deposit_amount: + type: integer + format: uint64 + example: 1000000000 + max_effective_balance: + type: integer + format: uint64 + example: 32000000000 + ejection_balance: + type: integer + format: uint64 + example: 16000000000 + effective_balance_increment: + type: integer + format: uint64 + example: 1000000000 + genesis_slot: + type: integer + format: uint64 + example: 0 + bls_withdrawal_prefix_byte: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{2}$" + example: "0x00" + milliseconds_per_slot: + type: integer + format: uint64 + example: 6000 + min_attestation_inclusion_delay: + type: integer + format: uint64 + example: 1 + min_seed_lookahead: + type: integer + format: uint64 + example: 1 + activation_exit_delay: + type: integer + format: uint64 + example: 4 + min_validator_withdrawability_delay: + type: integer + format: uint64 + example: 256 + persistent_committee_period: + type: integer + format: uint64 + example: 2048 + max_epochs_per_crosslink: + type: integer + format: uint64 + example: 64 + min_epochs_to_inactivity_penalty: + type: integer + format: uint64 + example: 4 + base_reward_factor: + type: integer + format: uint64 + example: 64 + whistleblower_reward_quotient: + type: integer + format: uint64 + example: 512 + proposer_reward_quotient: + type: integer + format: uint64 + example: 8 + inactivity_penalty_quotient: + type: integer + format: uint64 + example: 33554432 + min_slashing_penalty_quotient: + type: integer + format: uint64 + example: 32 + domain_beacon_proposer: + type: integer + format: uint32 + example: 0 + domain_randao: + type: integer + format: uint32 + example: 1 + domain_attestation: + type: integer + format: uint32 + example: 2 + domain_deposit: + type: integer + format: uint32 + example: 3 + domain_voluntary_exit: + type: integer + format: uint32 + example: 4 + domain_transfer: + type: integer + format: uint32 + example: 5 + boot_nodes: + type: array + items: + $ref: '#/components/schemas/ENR' + network_id: + type: integer + format: uint8 + example: 2 + + responses: Success: description: "Request successful." From 50fca17308914e2bd9c52fd4a583556b28ca6faf Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 5 Sep 2019 00:40:23 +1000 Subject: [PATCH 22/47] Updated ChainSpec serialization and added some comments about potentially missing components. --- eth2/types/src/chain_spec.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index d59e0db0a..17e9dba49 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -24,11 +24,13 @@ pub struct ChainSpec { /* * Constants */ - #[serde(skip_serializing)] // skipped because Serde TOML has trouble with u64::max pub far_future_epoch: Epoch, + // The above may need to be skipped because Serde TOML has trouble with u64::max. + // Use: #[serde(skip_serializing)] pub base_rewards_per_epoch: u64, pub deposit_contract_tree_depth: u64, pub seconds_per_day: u64, + //TODO missing JUSTIFICATION_BITS_LENGTH and ENDIANNESS /* * Misc @@ -52,6 +54,7 @@ pub struct ChainSpec { * Initial Values */ pub genesis_slot: Slot, + //TODO Missing genesis_epoch #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")] pub bls_withdrawal_prefix_byte: u8, @@ -59,6 +62,7 @@ pub struct ChainSpec { * Time parameters */ pub milliseconds_per_slot: u64, + //TODO should we also have SECONDS_PER_SLOT? pub min_attestation_inclusion_delay: u64, pub min_seed_lookahead: Epoch, pub activation_exit_delay: u64, From 6cbef7b58bc84c43c1dcbf29d5e9eef63fd7c7ad Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 5 Sep 2019 20:06:46 +1000 Subject: [PATCH 23/47] Undoing changes to ChainSpec. The discrepancies from the Eth2.0 spec are necessary in our case. --- docs/api_spec.yaml | 4 ---- eth2/types/src/chain_spec.rs | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/api_spec.yaml b/docs/api_spec.yaml index ced07e96d..23608807e 100644 --- a/docs/api_spec.yaml +++ b/docs/api_spec.yaml @@ -1896,10 +1896,6 @@ components: type: object description: "Stores all of the values which specify a particular chain. The `ChainSpec` object in Lighthouse" properties: - far_future_epoch: - type: integer - format: uint64 - example: 18446744073709551615 base_rewards_per_epoch: type: integer format: uint64 diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 17e9dba49..d59e0db0a 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -24,13 +24,11 @@ pub struct ChainSpec { /* * Constants */ + #[serde(skip_serializing)] // skipped because Serde TOML has trouble with u64::max pub far_future_epoch: Epoch, - // The above may need to be skipped because Serde TOML has trouble with u64::max. - // Use: #[serde(skip_serializing)] pub base_rewards_per_epoch: u64, pub deposit_contract_tree_depth: u64, pub seconds_per_day: u64, - //TODO missing JUSTIFICATION_BITS_LENGTH and ENDIANNESS /* * Misc @@ -54,7 +52,6 @@ pub struct ChainSpec { * Initial Values */ pub genesis_slot: Slot, - //TODO Missing genesis_epoch #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")] pub bls_withdrawal_prefix_byte: u8, @@ -62,7 +59,6 @@ pub struct ChainSpec { * Time parameters */ pub milliseconds_per_slot: u64, - //TODO should we also have SECONDS_PER_SLOT? pub min_attestation_inclusion_delay: u64, pub min_seed_lookahead: Epoch, pub activation_exit_delay: u64, From e6a0d038e93fd97f6f14673bf32e53bdf76accd8 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 6 Sep 2019 14:10:49 +1000 Subject: [PATCH 24/47] Added YAML types for list of validators and added some logging to duties function. --- beacon_node/rest_api/src/beacon.rs | 22 +++++++--------------- beacon_node/rest_api/src/helpers.rs | 10 ++++++++++ beacon_node/rest_api/src/validator.rs | 25 +++++++++++++++++++++---- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 70b3f3ee9..f9b2d0383 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -6,7 +6,7 @@ use serde::Serialize; use ssz_derive::Encode; use std::sync::Arc; use store::Store; -use types::{BeaconBlock, BeaconState, Epoch, EthSpec, Hash256, Slot}; +use types::{BeaconBlock, BeaconState, Epoch, EthSpec, Hash256, Slot, Validator}; #[derive(Serialize)] pub struct HeadResponse { @@ -162,21 +162,13 @@ pub fn get_validators(req: Request) -> ApiR }; let all_validators = &beacon_chain.head().beacon_state.validators; - let mut active_validators = Vec::with_capacity(all_validators.len()); - for (_index, validator) in all_validators.iter().enumerate() { - if validator.is_active_at(epoch) { - active_validators.push(validator) - } - } - active_validators.shrink_to_fit(); - let json: String = serde_json::to_string(&active_validators).map_err(|e| { - ApiError::ServerError(format!( - "Unable to serialize list of active validators: {:?}", - e - )) - })?; + let active_vals: Vec = all_validators + .iter() + .filter(|v| v.is_active_at(epoch)) + .cloned() + .collect(); - Ok(success_response(Body::from(json))) + ResponseBuilder::new(&req).body(&active_vals) } #[derive(Serialize, Encode)] diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index e15c27df5..08ccbb6c9 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -179,6 +179,7 @@ pub fn get_beacon_chain_from_request( .get::>>() .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".into()))?; + /* let _state_now = beacon_chain .state_now() .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))? @@ -188,10 +189,19 @@ pub fn get_beacon_chain_from_request( ))? .build_all_caches(&beacon_chain.spec) .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; + */ Ok(beacon_chain.clone()) } +pub fn get_logger_from_request(req: &Request) -> slog::Logger { + let log = req + .extensions() + .get::() + .expect("Should always get the logger from the request, since we put it in there."); + log.to_owned() +} + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 2374373bd..c559777c0 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -32,27 +32,44 @@ impl ValidatorDuty { /// HTTP Handler to retrieve a the duties for a set of validators during a particular epoch pub fn get_validator_duties(req: Request) -> ApiResult { + let log = get_logger_from_request(&req); + slog::trace!(log, "Validator duties requested of API: {:?}", &req); let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = &beacon_chain.head().beacon_state; + let mut head_state = beacon_chain + .state_now() + .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))?; + slog::trace!(log, "Got head state from request."); // Parse and check query parameters let query = UrlQuery::from_request(&req)?; let current_epoch = head_state.current_epoch(); let epoch = match query.first_of(&["epoch"]) { - Ok((_, v)) => Epoch::new(v.parse::().map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid epoch parameter, must be a u64. {:?}", e)) - })?), + Ok((_, v)) => { + slog::trace!(log, "Requested epoch {:?}", v); + Epoch::new(v.parse::().map_err(|e| { + slog::info!(log, "Invalid epoch {:?}", e); + ApiError::InvalidQueryParams(format!( + "Invalid epoch parameter, must be a u64. {:?}", + e + )) + })?) + } Err(_) => { // epoch not supplied, use the current epoch + slog::info!(log, "Using default epoch {:?}", current_epoch); current_epoch } }; let relative_epoch = RelativeEpoch::from_epoch(current_epoch, epoch).map_err(|e| { + slog::info!(log, "Requested epoch out of range."); ApiError::InvalidQueryParams(format!( "Cannot get RelativeEpoch, epoch out of range: {:?}", e )) })?; + if let Some(s) = head_state.maybe_as_mut_ref() { + s.build_all_caches(&beacon_chain.spec).ok(); + } let validators: Vec = query .all_of("validator_pubkeys")? .iter() From 99c673045c8be6422a6ffde12da95cffedffc11d Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 9 Sep 2019 12:10:41 +1000 Subject: [PATCH 25/47] Moved chain/cache building into separate function, and made sure that all REST API endpoints are using this function to get the state. --- beacon_node/rest_api/src/beacon.rs | 22 +++++++++------------- beacon_node/rest_api/src/helpers.rs | 21 ++++++++------------- beacon_node/rest_api/src/metrics.rs | 2 +- beacon_node/rest_api/src/node.rs | 8 ++++---- beacon_node/rest_api/src/spec.rs | 2 +- beacon_node/rest_api/src/validator.rs | 13 +++---------- 6 files changed, 26 insertions(+), 42 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index f9b2d0383..66f5e7731 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -109,7 +109,7 @@ pub fn get_block(req: Request) -> ApiResult /// HTTP handler to return a `BeaconBlock` root at a given `slot`. pub fn get_block_root(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; let target = parse_slot(&slot_string)?; @@ -126,10 +126,9 @@ pub fn get_block_root(req: Request) -> ApiR /// HTTP handler to return the `Fork` of the current head. pub fn get_fork(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; - let chain_head = beacon_chain.head(); + let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; - let json: String = serde_json::to_string(&chain_head.beacon_state.fork).map_err(|e| { + let json: String = serde_json::to_string(&head_state.fork).map_err(|e| { ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) })?; @@ -141,7 +140,7 @@ pub fn get_fork(req: Request) -> ApiResult /// The `Epoch` parameter can be any epoch number. If it is not specified, /// the current epoch is assumed. pub fn get_validators(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let epoch = match UrlQuery::from_request(&req) { // We have some parameters, so make sure it's the epoch one and parse it @@ -183,7 +182,7 @@ pub struct StateResponse { /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn get_state(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; let (key, value) = match UrlQuery::from_request(&req) { Ok(query) => { @@ -199,10 +198,7 @@ pub fn get_state(req: Request) -> ApiResult } Err(ApiError::InvalidQueryParams(_)) => { // No parameters provided at all, use current slot. - ( - String::from("slot"), - beacon_chain.head().beacon_state.slot.to_string(), - ) + (String::from("slot"), head_state.slot.to_string()) } Err(e) => { return Err(e); @@ -237,7 +233,7 @@ pub fn get_state(req: Request) -> ApiResult /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn get_state_root(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; let slot = parse_slot(&slot_string)?; @@ -254,7 +250,7 @@ pub fn get_state_root(req: Request) -> ApiR pub fn get_current_finalized_checkpoint( req: Request, ) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let checkpoint = beacon_chain .head() @@ -270,7 +266,7 @@ pub fn get_current_finalized_checkpoint( /// HTTP handler to return a `BeaconState` at the genesis block. pub fn get_genesis_state(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 08ccbb6c9..76fc78750 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -172,26 +172,21 @@ pub fn implementation_pending_response(_req: Request) -> ApiResult { pub fn get_beacon_chain_from_request( req: &Request, -) -> Result>, ApiError> { +) -> Result<(Arc>, BeaconState), ApiError> { // Get beacon state let beacon_chain = req .extensions() .get::>>() .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".into()))?; - - /* - let _state_now = beacon_chain + let mut head_state = beacon_chain .state_now() - .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))? - .maybe_as_mut_ref() - .ok_or(ApiError::ServerError( - "Unable to get mutable BeaconState".into(), - ))? - .build_all_caches(&beacon_chain.spec) - .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; - */ + .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))?; - Ok(beacon_chain.clone()) + if let Some(s) = head_state.maybe_as_mut_ref() { + s.build_all_caches(&beacon_chain.spec).ok(); + } + + Ok((beacon_chain.clone(), head_state.clone())) } pub fn get_logger_from_request(req: &Request) -> slog::Logger { diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 01dc4d22d..62a769de1 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -30,7 +30,7 @@ pub fn get_prometheus(req: Request) -> ApiR let mut buffer = vec![]; let encoder = TextEncoder::new(); - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let db_path = req .extensions() .get::() diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index 4dbd41229..c75d3ba20 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -1,7 +1,7 @@ +use crate::helpers::get_beacon_chain_from_request; use crate::{success_response, ApiResult}; -use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::BeaconChainTypes; use hyper::{Body, Request}; -use std::sync::Arc; use version; /// Read the version string from the current Lighthouse build. @@ -15,8 +15,8 @@ pub fn get_version(_req: Request) -> ApiResult { /// Read the genesis time from the current beacon chain state. pub fn get_genesis_time(req: Request) -> ApiResult { - let beacon_chain = req.extensions().get::>>().unwrap(); - let gen_time: u64 = beacon_chain.head().beacon_state.genesis_time; + let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; + let gen_time: u64 = head_state.genesis_time; let body = Body::from( serde_json::to_string(&gen_time) .expect("Genesis should time always have a valid JSON serialization."), diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index a353b3833..ad168faf1 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -9,7 +9,7 @@ use types::EthSpec; /// HTTP handler to return the full spec object. pub fn get_spec(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let json: String = serde_json::to_string(&beacon_chain.spec) .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index c559777c0..49b4c0441 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -34,10 +34,7 @@ impl ValidatorDuty { pub fn get_validator_duties(req: Request) -> ApiResult { let log = get_logger_from_request(&req); slog::trace!(log, "Validator duties requested of API: {:?}", &req); - let beacon_chain = get_beacon_chain_from_request::(&req)?; - let mut head_state = beacon_chain - .state_now() - .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))?; + let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; slog::trace!(log, "Got head state from request."); // Parse and check query parameters @@ -67,9 +64,6 @@ pub fn get_validator_duties(req: Request) - e )) })?; - if let Some(s) = head_state.maybe_as_mut_ref() { - s.build_all_caches(&beacon_chain.spec).ok(); - } let validators: Vec = query .all_of("validator_pubkeys")? .iter() @@ -163,7 +157,7 @@ pub fn get_validator_duties(req: Request) - /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. pub fn get_new_beacon_block(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let query = UrlQuery::from_request(&req)?; let slot = query @@ -203,8 +197,7 @@ pub fn get_new_beacon_block(req: Request) - /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { - let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = &beacon_chain.head().beacon_state; + let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; let query = UrlQuery::from_request(&req)?; let val_pk_str = query From 0136eb33b092a8ab474c5ddf3db666a542dfda0b Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 9 Sep 2019 12:54:14 +1000 Subject: [PATCH 26/47] WIP: Added POST functionality for pusblish_beacon_block. Currently doesn't compile, struggling with the borrow checker. --- beacon_node/client/src/lib.rs | 1 + beacon_node/rest_api/src/helpers.rs | 42 +++++++++++++++++++- beacon_node/rest_api/src/lib.rs | 8 +++- beacon_node/rest_api/src/validator.rs | 57 +++++++++++++++++++++++++-- 4 files changed, 103 insertions(+), 5 deletions(-) diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index afcd538b5..f26a5503c 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -215,6 +215,7 @@ where executor, beacon_chain.clone(), network.clone(), + network_send.clone(), client_config.db_path().expect("unable to read datadir"), eth2_config.clone(), &log, diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 76fc78750..d6ea0397f 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -1,11 +1,17 @@ use crate::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; +use eth2_libp2p::{PubsubMessage, Topic}; +use eth2_libp2p::{BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX}; use hex; use hyper::{Body, Request}; +use network::NetworkMessage; +use ssz::Encode; +use std::borrow::BorrowMut; use std::sync::Arc; use store::{iter::AncestorIter, Store}; -use types::{BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; +use tokio::sync::mpsc; +use types::{BeaconBlock, BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; /// Parse a slot from a `0x` preixed string. /// @@ -197,6 +203,40 @@ pub fn get_logger_from_request(req: &Request) -> slog::Logger { log.to_owned() } +pub fn publish_beacon_block_to_network( + req: &Request, + block: BeaconBlock, +) -> Result<(), ApiError> { + // Get the network service from the request + let mut network_chan = req + .extensions() + .get::>() + .expect( + "Should always get the network channel from the request, since we put it in there.", + ); + + // create the network topic to send on + let topic_string = format!( + "/{}/{}/{}", + TOPIC_PREFIX, BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX + ); + let topic = Topic::new(topic_string); + let message = PubsubMessage::Block(block.as_ssz_bytes()); + + // Publish the block to the p2p network via gossipsub. + if let Err(e) = &network_chan.try_send(NetworkMessage::Publish { + topics: vec![topic], + message: message, + }) { + return Err(ApiError::ServerError(format!( + "Unable to send new block to network: {:?}", + e + ))); + } + + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 02c68c639..c0927dde3 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -14,6 +14,7 @@ mod url_query; mod validator; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use client_network::NetworkMessage; use client_network::Service as NetworkService; use eth2_config::Eth2Config; use hyper::rt::Future; @@ -25,6 +26,7 @@ use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::TaskExecutor; +use tokio::sync::mpsc; use url_query::UrlQuery; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; @@ -83,6 +85,7 @@ pub fn start_server( executor: &TaskExecutor, beacon_chain: Arc>, network_service: Arc>, + network_chan: mpsc::UnboundedSender, db_path: PathBuf, eth2_config: Eth2Config, log: &slog::Logger, @@ -113,6 +116,7 @@ pub fn start_server( let beacon_chain = server_bc.clone(); let db_path = db_path.clone(); let network_service = network_service.clone(); + let network_chan = network_chan.clone(); let eth2_config = eth2_config.clone(); // Create a simple handler for the router, inject our stateful objects into the request. @@ -126,6 +130,8 @@ pub fn start_server( req.extensions_mut().insert::(db_path.clone()); req.extensions_mut() .insert::>>(network_service.clone()); + req.extensions_mut() + .insert::>(network_chan.clone()); req.extensions_mut() .insert::>(eth2_config.clone()); @@ -177,7 +183,7 @@ pub fn start_server( validator::get_new_beacon_block::(req) } (&Method::POST, "/beacon/validator/block") => { - helpers::implementation_pending_response(req) + validator::publish_beacon_block::(req) } (&Method::GET, "/beacon/validator/attestation") => { validator::get_new_attestation::(req) diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 49b4c0441..632aee0ac 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,11 +1,14 @@ use super::{success_response, ApiResult}; use crate::{helpers::*, ApiError, UrlQuery}; -use beacon_chain::BeaconChainTypes; +use beacon_chain::{BeaconChainTypes, BlockProcessingOutcome}; use bls::{AggregateSignature, PublicKey, Signature}; -use hyper::{Body, Request}; +use futures::future::Future; +use futures::stream::Stream; +use hyper::{Body, Error, Request}; use serde::{Deserialize, Serialize}; +use slog::info; use types::beacon_state::EthSpec; -use types::{Attestation, BitList, Epoch, RelativeEpoch, Shard, Slot}; +use types::{Attestation, BeaconBlock, BitList, Epoch, RelativeEpoch, Shard, Slot}; #[derive(Debug, Serialize, Deserialize)] pub struct ValidatorDuty { @@ -195,6 +198,54 @@ pub fn get_new_beacon_block(req: Request) - Ok(success_response(body)) } +/// HTTP Handler to publish a BeaconBlock, which has been signed by a validator. +pub fn publish_beacon_block(req: Request) -> ApiResult { + let log = get_logger_from_request(&req); + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + + let (_head, body) = req.into_parts(); + let block_future = body + .fold(Vec::new(), |mut acc, chunk| { + acc.extend_from_slice(&*chunk); + futures::future::ok::<_, Error>(acc) + }) + .map_err(|e| ApiError::ServerError(format!("Unable parse request body: {:?}", e))) + .and_then(|body| { + let block_result: Result, ApiError> = + serde_json::from_slice(&body.as_slice()).map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Unable to deserialize JSON into a BeaconBlock: {:?}", + e + )) + }); + block_result + }); + let block = block_future.wait()?; + match beacon_chain.process_block(block.clone()) { + Ok(BlockProcessingOutcome::Processed { + block_root: block_root, + }) => { + // Block was processed, publish via gossipsub + info!(log, "Processed valid block from API"; "block_slot" => block.slot, "block_root" => format!("{}", block_root)); + publish_beacon_block_to_network::(&req, block)?; + } + Ok(outcome) => { + return Err(ApiError::InvalidQueryParams(format!( + "The BeaconBlock could not be processed: {:?}", + outcome + ))); + } + Err(e) => { + return Err(ApiError::ServerError(format!( + "Unable to process block: {:?}", + e + ))); + } + } + + Ok(success_response(Body::empty())) +} + /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; From ca9094e79a4bf62a34b8349e5868782e5a1a557f Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 10 Sep 2019 10:54:37 +1000 Subject: [PATCH 27/47] WIP: Made block publishing validator function, which sends to a network channel. Untested. --- beacon_node/rest_api/Cargo.toml | 4 +- beacon_node/rest_api/src/helpers.rs | 30 +++++++++------ beacon_node/rest_api/src/lib.rs | 5 ++- beacon_node/rest_api/src/validator.rs | 54 +++++++++++++++++++++------ 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index 863ea04da..a3d31e410 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -25,7 +25,7 @@ types = { path = "../../eth2/types" } clap = "2.32.0" http = "^0.1.17" prometheus = { version = "^0.6", features = ["process"] } -hyper = "0.12.32" +hyper = "0.12.34" futures = "0.1" exit-future = "0.1.3" tokio = "0.1.17" @@ -35,3 +35,5 @@ eth2_config = { path = "../../eth2/utils/eth2_config" } lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } slot_clock = { path = "../../eth2/utils/slot_clock" } hex = "0.3.2" +parking_lot = "0.9" + diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index d6ea0397f..bff7d9ece 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -4,10 +4,11 @@ use bls::PublicKey; use eth2_libp2p::{PubsubMessage, Topic}; use eth2_libp2p::{BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX}; use hex; +use http::header; use hyper::{Body, Request}; use network::NetworkMessage; +use parking_lot::RwLock; use ssz::Encode; -use std::borrow::BorrowMut; use std::sync::Arc; use store::{iter::AncestorIter, Store}; use tokio::sync::mpsc; @@ -41,6 +42,21 @@ pub fn parse_root(string: &str) -> Result { } } +/// Checks the provided request to ensure that the `content-type` header. +/// +/// The content-type header should either be omitted, in which case JSON is assumed, or it should +/// explicity specify `application/json`. If anything else is provided, an error is returned. +pub fn check_content_type_for_json(req: &Request) -> Result<(), ApiError> { + match req.headers().get(header::CONTENT_TYPE) { + Some(h) if h == "application/json" => Ok(()), + Some(h) => Err(ApiError::InvalidQueryParams(format!( + "The provided content-type {:?} is not available, it must be JSON.", + h + ))), + _ => Ok(()), + } +} + /// Parse a PublicKey from a `0x` prefixed hex string pub fn parse_pubkey(string: &str) -> Result { const PREFIX: &str = "0x"; @@ -204,17 +220,9 @@ pub fn get_logger_from_request(req: &Request) -> slog::Logger { } pub fn publish_beacon_block_to_network( - req: &Request, + chan: Arc>>, block: BeaconBlock, ) -> Result<(), ApiError> { - // Get the network service from the request - let mut network_chan = req - .extensions() - .get::>() - .expect( - "Should always get the network channel from the request, since we put it in there.", - ); - // create the network topic to send on let topic_string = format!( "/{}/{}/{}", @@ -224,7 +232,7 @@ pub fn publish_beacon_block_to_network( let message = PubsubMessage::Block(block.as_ssz_bytes()); // Publish the block to the p2p network via gossipsub. - if let Err(e) = &network_chan.try_send(NetworkMessage::Publish { + if let Err(e) = chan.write().try_send(NetworkMessage::Publish { topics: vec![topic], message: message, }) { diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index c0927dde3..adab0c3bb 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -18,8 +18,9 @@ use client_network::NetworkMessage; use client_network::Service as NetworkService; use eth2_config::Eth2Config; use hyper::rt::Future; -use hyper::service::service_fn_ok; -use hyper::{Body, Method, Response, Server, StatusCode}; +use hyper::service::Service; +use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use parking_lot::RwLock; use response_builder::ResponseBuilder; use slog::{info, o, warn}; use std::ops::Deref; diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 632aee0ac..2ead55d14 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -5,8 +5,13 @@ use bls::{AggregateSignature, PublicKey, Signature}; use futures::future::Future; use futures::stream::Stream; use hyper::{Body, Error, Request}; +use network::NetworkMessage; +use parking_lot::RwLock; use serde::{Deserialize, Serialize}; -use slog::info; +use slog::{info, trace, warn}; +use std::sync::Arc; +use tokio; +use tokio::sync::mpsc; use types::beacon_state::EthSpec; use types::{Attestation, BeaconBlock, BitList, Epoch, RelativeEpoch, Shard, Slot}; @@ -200,17 +205,41 @@ pub fn get_new_beacon_block(req: Request) - /// HTTP Handler to publish a BeaconBlock, which has been signed by a validator. pub fn publish_beacon_block(req: Request) -> ApiResult { + let _ = check_content_type_for_json(&req)?; let log = get_logger_from_request(&req); let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + // Get the network sending channel from the request, for later transmission + let network_chan = req + .extensions() + .get::>>>() + .expect("Should always get the network channel from the request, since we put it in there.") + .clone(); - let (_head, body) = req.into_parts(); - let block_future = body - .fold(Vec::new(), |mut acc, chunk| { - acc.extend_from_slice(&*chunk); - futures::future::ok::<_, Error>(acc) + let body = req.into_body(); + trace!( + log, + "Got the request body, now going to parse it into a block." + ); + let block = body + .concat2() + .map(move |chunk| chunk.iter().cloned().collect::>()) + .map(|chunks| { + let block_result: Result, ApiError> = + serde_json::from_slice(&chunks.as_slice()).map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Unable to deserialize JSON into a BeaconBlock: {:?}", + e + )) + }); + block_result }) + .unwrap() + .unwrap(); + + /* .map_err(|e| ApiError::ServerError(format!("Unable parse request body: {:?}", e))) .and_then(|body| { + trace!(log, "parsing json"); let block_result: Result, ApiError> = serde_json::from_slice(&body.as_slice()).map_err(|e| { ApiError::InvalidQueryParams(format!( @@ -220,16 +249,19 @@ pub fn publish_beacon_block(req: Request) - }); block_result }); + tokio::run(block_future); let block = block_future.wait()?; + */ + trace!(log, "BeaconBlock successfully parsed from JSON"; "block" => serde_json::to_string(&block).expect("We should always be able to serialize a block that we just created.")); match beacon_chain.process_block(block.clone()) { - Ok(BlockProcessingOutcome::Processed { - block_root: block_root, - }) => { + Ok(BlockProcessingOutcome::Processed { block_root }) => { // Block was processed, publish via gossipsub - info!(log, "Processed valid block from API"; "block_slot" => block.slot, "block_root" => format!("{}", block_root)); - publish_beacon_block_to_network::(&req, block)?; + info!(log, "Processed valid block from API, transmitting to network."; "block_slot" => block.slot, "block_root" => format!("{}", block_root)); + publish_beacon_block_to_network::(network_chan, block)?; } Ok(outcome) => { + warn!(log, "Block could not be processed, but is being sent to the network anyway."; "block_slot" => block.slot, "outcome" => format!("{:?}", outcome)); + //TODO need to send to network and return http 202 return Err(ApiError::InvalidQueryParams(format!( "The BeaconBlock could not be processed: {:?}", outcome From 476cbae57746edd5e2067b6535e346d6880ff135 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 10 Sep 2019 10:55:46 +1000 Subject: [PATCH 28/47] Updated validator client to do better logging, including of JSON serialised signatures and such, for debugging purposes. --- validator_client/Cargo.toml | 1 + validator_client/src/block_producer/mod.rs | 30 ++++++++++++++-------- validator_client/src/main.rs | 10 +++++++- validator_client/src/service.rs | 3 ++- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 2000f5409..706b28f86 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -25,6 +25,7 @@ slot_clock = { path = "../eth2/utils/slot_clock" } types = { path = "../eth2/types" } serde = "1.0" serde_derive = "1.0" +serde_json = "^1.0" slog = "^2.2.3" slog-async = "^2.3.0" slog-json = "^2.3" diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index ca1e3a1d8..f61cde146 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -6,7 +6,8 @@ pub use self::beacon_node_block::{BeaconNodeError, PublishOutcome}; pub use self::grpc::BeaconBlockGrpcClient; use crate::signer::Signer; use core::marker::PhantomData; -use slog::{error, info, warn}; +use serde_json; +use slog::{error, info, trace, warn}; use std::sync::Arc; use tree_hash::{SignedRoot, TreeHash}; use types::{BeaconBlock, ChainSpec, Domain, EthSpec, Fork, Slot}; @@ -53,27 +54,29 @@ pub struct BlockProducer<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> { pub slots_per_epoch: u64, /// Mere vessel for E. pub _phantom: PhantomData, + /// The logger, for logging + pub log: slog::Logger, } impl<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> BlockProducer<'a, B, S, E> { /// Handle outputs and results from block production. - pub fn handle_produce_block(&mut self, log: slog::Logger) { + pub fn handle_produce_block(&mut self) { match self.produce_block() { Ok(ValidatorEvent::BlockProduced(_slot)) => { - info!(log, "Block produced"; "Validator" => format!("{}", self.signer)) + info!(self.log, "Block produced"; "Validator" => format!("{}", self.signer)) } - Err(e) => error!(log, "Block production error"; "Error" => format!("{:?}", e)), + Err(e) => error!(self.log, "Block production error"; "Error" => format!("{:?}", e)), Ok(ValidatorEvent::SignerRejection(_slot)) => { - error!(log, "Block production error"; "Error" => "Signer Could not sign the block".to_string()) + error!(self.log, "Block production error"; "Error" => "Signer Could not sign the block".to_string()) } Ok(ValidatorEvent::SlashableBlockNotProduced(_slot)) => { - error!(log, "Block production error"; "Error" => "Rejected the block as it could have been slashed".to_string()) + error!(self.log, "Block production error"; "Error" => "Rejected the block as it could have been slashed".to_string()) } Ok(ValidatorEvent::BeaconNodeUnableToProduceBlock(_slot)) => { - error!(log, "Block production error"; "Error" => "Beacon node was unable to produce a block".to_string()) + error!(self.log, "Block production error"; "Error" => "Beacon node was unable to produce a block".to_string()) } Ok(v) => { - warn!(log, "Unknown result for block production"; "Error" => format!("{:?}",v)) + warn!(self.log, "Unknown result for block production"; "Error" => format!("{:?}",v)) } } } @@ -90,14 +93,21 @@ impl<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> BlockProducer<'a, B, S, E> { /// slashing. pub fn produce_block(&mut self) -> Result { let epoch = self.slot.epoch(self.slots_per_epoch); + trace!(self.log, "Producing block"; "epoch" => epoch); let message = epoch.tree_hash_root(); let randao_reveal = match self.signer.sign_message( &message, self.spec.get_domain(epoch, Domain::Randao, &self.fork), ) { - None => return Ok(ValidatorEvent::SignerRejection(self.slot)), - Some(signature) => signature, + None => { + warn!(self.log, "Signing rejected"; "message" => format!("{:?}", message)); + return Ok(ValidatorEvent::SignerRejection(self.slot)); + } + Some(signature) => { + info!(self.log, "Signed tree_hash_root for randao_reveal"; "message" => format!("{:?}", message), "signature" => serde_json::to_string(&signature).expect("We should always be able to serialize a signature as JSON.")); + signature + } }; if let Some(block) = self diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 39b2e3eae..bb791aa20 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -54,7 +54,6 @@ fn main() { ) .arg( Arg::with_name("spec") - .short("s") .long("spec") .value_name("TITLE") .help("Specifies the default eth2 spec type.") @@ -132,6 +131,15 @@ fn main() { .help("The number of validators.")) ) ) + .subcommand(SubCommand::with_name("sign_block") + .about("Connects to the beacon server, requests a new block (after providing reveal),\ + and prints the signed block to standard out") + .arg(Arg::with_name("validator") + .value_name("VALIDATOR") + .required(true) + .help("The pubkey of the validator that should sign the block.") + ) + ) .get_matches(); let drain = match matches.value_of("debug-level") { diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 5169f67f8..4fe744ea2 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -369,8 +369,9 @@ impl Service, + log, }; - block_producer.handle_produce_block(log); + block_producer.handle_produce_block(); }); } if work_type.attestation_duty.is_some() { From 405a59e8b9fa28b2e56c8148affa733f57b95cb5 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 10 Sep 2019 10:56:50 +1000 Subject: [PATCH 29/47] WIP: Trying to restructure ApiService to be async. --- beacon_node/rest_api/src/error.rs | 61 ++++++ beacon_node/rest_api/src/lib.rs | 309 ++++++++++++++---------------- 2 files changed, 204 insertions(+), 166 deletions(-) create mode 100644 beacon_node/rest_api/src/error.rs diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs new file mode 100644 index 000000000..bae07ea0b --- /dev/null +++ b/beacon_node/rest_api/src/error.rs @@ -0,0 +1,61 @@ +use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use std::error::Error as StdError; + +type Cause = Box; + +pub struct ApiError { + kind: ApiErrorKind, + cause: Option, +} + +#[derive(PartialEq, Debug)] +pub enum ApiErrorKind { + MethodNotAllowed(String), + ServerError(String), + NotImplemented(String), + InvalidQueryParams(String), + NotFound(String), + ImATeapot(String), // Just in case. +} + +pub type ApiResult = Result, ApiError>; + +impl Into> for ApiError { + fn into(self) -> Response { + let status_code: (StatusCode, String) = match self { + ApiError::MethodNotAllowed(desc) => (StatusCode::METHOD_NOT_ALLOWED, desc), + ApiError::ServerError(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc), + ApiError::NotImplemented(desc) => (StatusCode::NOT_IMPLEMENTED, desc), + ApiError::InvalidQueryParams(desc) => (StatusCode::BAD_REQUEST, desc), + ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc), + ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc), + }; + Response::builder() + .status(status_code.0) + .header("content-type", "text/plain") + .body(Body::from(status_code.1)) + .expect("Response should always be created.") + } +} + +impl From for ApiError { + fn from(e: store::Error) -> ApiError { + ApiError::ServerError(format!("Database error: {:?}", e)) + } +} + +impl From for ApiError { + fn from(e: types::BeaconStateError) -> ApiError { + ApiError::ServerError(format!("BeaconState error: {:?}", e)) + } +} + +impl From for ApiError { + fn from(e: state_processing::per_slot_processing::Error) -> ApiError { + ApiError::ServerError(format!("PerSlotProcessing error: {:?}", e)) + } +} + +impl std::error::Error for ApiError { + fn cause(&self) -> Option<&Error> {} +} diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index adab0c3bb..56ed8c7bb 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -4,6 +4,7 @@ extern crate network as client_network; mod beacon; mod config; +mod error; mod helpers; mod metrics; mod network; @@ -32,52 +33,143 @@ use url_query::UrlQuery; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; +use eth2_libp2p::rpc::RequestId; +use serde::ser::StdError; -#[derive(PartialEq, Debug)] -pub enum ApiError { - MethodNotAllowed(String), - ServerError(String), - NotImplemented(String), - InvalidQueryParams(String), - NotFound(String), - ImATeapot(String), // Just in case. +type BoxFut = Box, Error = ApiError> + Send>; + +pub struct ApiService { + log: slog::Logger, + beacon_chain: Arc>, + db_path: DBPath, + network_service: Arc>, + network_channel: Arc>>, + eth2_config: Arc, } -pub type ApiResult = Result, ApiError>; +impl Service for ApiService { + type ReqBody = Body; + type ResBody = Body; + type Error = ApiError; + type Future = BoxFut; -impl Into> for ApiError { - fn into(self) -> Response { - let status_code: (StatusCode, String) = match self { - ApiError::MethodNotAllowed(desc) => (StatusCode::METHOD_NOT_ALLOWED, desc), - ApiError::ServerError(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc), - ApiError::NotImplemented(desc) => (StatusCode::NOT_IMPLEMENTED, desc), - ApiError::InvalidQueryParams(desc) => (StatusCode::BAD_REQUEST, desc), - ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc), - ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc), + fn call(&mut self, mut req: Request) -> Self::Future { + metrics::inc_counter(&metrics::REQUEST_COUNT); + let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); + + req.extensions_mut() + .insert::(self.log.clone()); + req.extensions_mut() + .insert::>>(self.beacon_chain.clone()); + req.extensions_mut().insert::(self.db_path.clone()); + req.extensions_mut() + .insert::>>(self.network_service.clone()); + req.extensions_mut() + .insert::>>>( + self.network_channel.clone(), + ); + req.extensions_mut() + .insert::>(self.eth2_config.clone()); + + let path = req.uri().path().to_string(); + + // Route the request to the correct handler. + let result = match (req.method(), path.as_ref()) { + // Methods for Client + (&Method::GET, "/node/version") => node::get_version(req), + /* + (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), + (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), + + // Methods for Network + (&Method::GET, "/network/enr") => network::get_enr::(req), + (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), + (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), + (&Method::GET, "/network/peers") => network::get_peer_list::(req), + (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), + (&Method::GET, "/network/listen_addresses") => { + network::get_listen_addresses::(req) + } + + // Methods for Beacon Node + (&Method::GET, "/beacon/head") => beacon::get_head::(req), + (&Method::GET, "/beacon/block") => beacon::get_block::(req), + (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), + (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), + (&Method::GET, "/beacon/fork") => beacon::get_fork::(req), + (&Method::GET, "/beacon/attestations") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/beacon/attestations/pending") => { + helpers::implementation_pending_response(req) + } + + (&Method::GET, "/beacon/validators") => beacon::get_validators::(req), + (&Method::GET, "/beacon/validators/indicies") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/beacon/validators/pubkeys") => { + helpers::implementation_pending_response(req) + } + + // Methods for Validator + (&Method::GET, "/beacon/validator/duties") => { + validator::get_validator_duties::(req) + } + (&Method::GET, "/beacon/validator/block") => { + validator::get_new_beacon_block::(req) + } + (&Method::POST, "/beacon/validator/block") => { + validator::publish_beacon_block::(req) + } + (&Method::GET, "/beacon/validator/attestation") => { + validator::get_new_attestation::(req) + } + (&Method::POST, "/beacon/validator/attestation") => { + helpers::implementation_pending_response(req) + } + + (&Method::GET, "/beacon/state") => beacon::get_state::(req), + (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + (&Method::GET, "/beacon/state/current_finalized_checkpoint") => { + beacon::get_current_finalized_checkpoint::(req) + } + (&Method::GET, "/beacon/state/genesis") => beacon::get_genesis_state::(req), + //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances + + // Methods for bootstrap and checking configuration + (&Method::GET, "/spec") => spec::get_spec::(req), + (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), + (&Method::GET, "/spec/deposit_contract") => { + helpers::implementation_pending_response(req) + } + (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::(req), + + (&Method::GET, "/metrics") => metrics::get_prometheus::(req), + + */ + _ => Err(ApiError::NotFound( + "Request path and/or method not found.".to_owned(), + )), }; - Response::builder() - .status(status_code.0) - .header("content-type", "text/plain") - .body(Body::from(status_code.1)) - .expect("Response should always be created.") - } -} -impl From for ApiError { - fn from(e: store::Error) -> ApiError { - ApiError::ServerError(format!("Database error: {:?}", e)) - } -} + let response = match result { + // Return the `hyper::Response`. + Ok(response) => { + metrics::inc_counter(&metrics::SUCCESS_COUNT); + slog::debug!(self.log, "Request successful: {:?}", path); + Box::new(response) + } + // Map the `ApiError` into `hyper::Response`. + Err(e) => { + slog::debug!(self.log, "Request failure: {:?}", path); + Box::new(e.into()) + } + }; -impl From for ApiError { - fn from(e: types::BeaconStateError) -> ApiError { - ApiError::ServerError(format!("BeaconState error: {:?}", e)) - } -} + metrics::stop_timer(timer); -impl From for ApiError { - fn from(e: state_processing::per_slot_processing::Error) -> ApiError { - ApiError::ServerError(format!("PerSlotProcessing error: {:?}", e)) + Box::new(futures::future::ok(response)) } } @@ -112,128 +204,13 @@ pub fn start_server( let server_bc = beacon_chain.clone(); let eth2_config = Arc::new(eth2_config); - let service = move || { - let log = server_log.clone(); - let beacon_chain = server_bc.clone(); - let db_path = db_path.clone(); - let network_service = network_service.clone(); - let network_chan = network_chan.clone(); - let eth2_config = eth2_config.clone(); - - // Create a simple handler for the router, inject our stateful objects into the request. - service_fn_ok(move |mut req| { - metrics::inc_counter(&metrics::REQUEST_COUNT); - let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); - - req.extensions_mut().insert::(log.clone()); - req.extensions_mut() - .insert::>>(beacon_chain.clone()); - req.extensions_mut().insert::(db_path.clone()); - req.extensions_mut() - .insert::>>(network_service.clone()); - req.extensions_mut() - .insert::>(network_chan.clone()); - req.extensions_mut() - .insert::>(eth2_config.clone()); - - let path = req.uri().path().to_string(); - - // Route the request to the correct handler. - let result = match (req.method(), path.as_ref()) { - // Methods for Client - (&Method::GET, "/node/version") => node::get_version(req), - (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), - (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), - - // Methods for Network - (&Method::GET, "/network/enr") => network::get_enr::(req), - (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), - (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), - (&Method::GET, "/network/peers") => network::get_peer_list::(req), - (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), - (&Method::GET, "/network/listen_addresses") => { - network::get_listen_addresses::(req) - } - - // Methods for Beacon Node - (&Method::GET, "/beacon/head") => beacon::get_head::(req), - (&Method::GET, "/beacon/block") => beacon::get_block::(req), - (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), - (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), - (&Method::GET, "/beacon/fork") => beacon::get_fork::(req), - (&Method::GET, "/beacon/attestations") => { - helpers::implementation_pending_response(req) - } - (&Method::GET, "/beacon/attestations/pending") => { - helpers::implementation_pending_response(req) - } - - (&Method::GET, "/beacon/validators") => beacon::get_validators::(req), - (&Method::GET, "/beacon/validators/indicies") => { - helpers::implementation_pending_response(req) - } - (&Method::GET, "/beacon/validators/pubkeys") => { - helpers::implementation_pending_response(req) - } - - // Methods for Validator - (&Method::GET, "/beacon/validator/duties") => { - validator::get_validator_duties::(req) - } - (&Method::GET, "/beacon/validator/block") => { - validator::get_new_beacon_block::(req) - } - (&Method::POST, "/beacon/validator/block") => { - validator::publish_beacon_block::(req) - } - (&Method::GET, "/beacon/validator/attestation") => { - validator::get_new_attestation::(req) - } - (&Method::POST, "/beacon/validator/attestation") => { - helpers::implementation_pending_response(req) - } - - (&Method::GET, "/beacon/state") => beacon::get_state::(req), - (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), - (&Method::GET, "/beacon/state/current_finalized_checkpoint") => { - beacon::get_current_finalized_checkpoint::(req) - } - (&Method::GET, "/beacon/state/genesis") => beacon::get_genesis_state::(req), - //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances - - // Methods for bootstrap and checking configuration - (&Method::GET, "/spec") => spec::get_spec::(req), - (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), - (&Method::GET, "/spec/deposit_contract") => { - helpers::implementation_pending_response(req) - } - (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::(req), - - (&Method::GET, "/metrics") => metrics::get_prometheus::(req), - - _ => Err(ApiError::NotFound( - "Request path and/or method not found.".to_owned(), - )), - }; - - let response = match result { - // Return the `hyper::Response`. - Ok(response) => { - metrics::inc_counter(&metrics::SUCCESS_COUNT); - slog::debug!(log, "Request successful: {:?}", path); - response - } - // Map the `ApiError` into `hyper::Response`. - Err(e) => { - slog::debug!(log, "Request failure: {:?}", path); - e.into() - } - }; - - metrics::stop_timer(timer); - - response - }) + let service = move || ApiService { + log: server_log.clone(), + beacon_chain: server_bc.clone(), + db_path: db_path.clone(), + network_service: network_service.clone(), + network_channel: Arc::new(RwLock::new(network_chan.clone())), + eth2_config: eth2_config.clone(), }; let log_clone = log.clone(); @@ -242,16 +219,16 @@ pub fn start_server( .with_graceful_shutdown(server_exit) .map_err(move |e| { warn!( - log_clone, - "API failed to start, Unable to bind"; "address" => format!("{:?}", e) + log_clone, + "API failed to start, Unable to bind"; "address" => format!("{:?}", e) ) }); info!( - log, - "REST API started"; - "address" => format!("{}", config.listen_address), - "port" => config.port, + log, + "REST API started"; + "address" => format!("{}", config.listen_address), + "port" => config.port, ); executor.spawn(server); From 965d6f1df9683e2e53f65bac48190c25b60f6c81 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 10 Sep 2019 15:35:54 +1000 Subject: [PATCH 30/47] WIP: More restructuring to have ApiService be a future. --- beacon_node/rest_api/Cargo.toml | 1 + beacon_node/rest_api/src/error.rs | 44 ++++++++++++++++++------------- beacon_node/rest_api/src/lib.rs | 16 ++++++++--- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index a3d31e410..ac762ebb7 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -36,4 +36,5 @@ lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } slot_clock = { path = "../../eth2/utils/slot_clock" } hex = "0.3.2" parking_lot = "0.9" +futures = "0.1.25" diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index bae07ea0b..f3eb597a0 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -1,15 +1,8 @@ use hyper::{Body, Method, Request, Response, Server, StatusCode}; use std::error::Error as StdError; -type Cause = Box; - -pub struct ApiError { - kind: ApiErrorKind, - cause: Option, -} - #[derive(PartialEq, Debug)] -pub enum ApiErrorKind { +pub enum ApiError { MethodNotAllowed(String), ServerError(String), NotImplemented(String), @@ -20,21 +13,27 @@ pub enum ApiErrorKind { pub type ApiResult = Result, ApiError>; -impl Into> for ApiError { - fn into(self) -> Response { - let status_code: (StatusCode, String) = match self { +impl ApiError { + pub fn status_code(&self) -> (StatusCode, &String) { + match self { ApiError::MethodNotAllowed(desc) => (StatusCode::METHOD_NOT_ALLOWED, desc), ApiError::ServerError(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc), ApiError::NotImplemented(desc) => (StatusCode::NOT_IMPLEMENTED, desc), ApiError::InvalidQueryParams(desc) => (StatusCode::BAD_REQUEST, desc), ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc), ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc), - }; + } + } +} + +impl Into> for ApiError { + fn into(self) -> Response { + let status_code = self.status_code(); Response::builder() - .status(status_code.0) - .header("content-type", "text/plain") - .body(Body::from(status_code.1)) - .expect("Response should always be created.") + .status(status_code.0) + .header("content-type", "text/plain") + .body(Body::from(*status_code.1)) + .expect("Response should always be created.") } } @@ -56,6 +55,15 @@ impl From for ApiError { } } -impl std::error::Error for ApiError { - fn cause(&self) -> Option<&Error> {} +impl StdError for ApiError { + fn cause(&self) -> Option<&StdError> { + None + } +} + +impl std::fmt::Display for ApiError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let status = self.status_code(); + write!(f, "{:?}: {:?}", status.0, status.1) + } } diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 56ed8c7bb..d7ea72cc5 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -14,6 +14,7 @@ mod spec; mod url_query; mod validator; +use error::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use client_network::NetworkMessage; use client_network::Service as NetworkService; @@ -34,7 +35,6 @@ use url_query::UrlQuery; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; use eth2_libp2p::rpc::RequestId; -use serde::ser::StdError; type BoxFut = Box, Error = ApiError> + Send>; @@ -158,18 +158,19 @@ impl Service for ApiService { Ok(response) => { metrics::inc_counter(&metrics::SUCCESS_COUNT); slog::debug!(self.log, "Request successful: {:?}", path); - Box::new(response) + response } // Map the `ApiError` into `hyper::Response`. Err(e) => { slog::debug!(self.log, "Request failure: {:?}", path); - Box::new(e.into()) + e.into() } }; metrics::stop_timer(timer); Box::new(futures::future::ok(response)) + } } @@ -236,6 +237,15 @@ pub fn start_server( Ok(exit_signal) } +impl Future for ApiService { + type Item = Result, ApiError>; + type Error = ApiError; + + fn poll(&mut self) -> Result, Self::Error> { + unimplemented!() + } +} + fn success_response(body: Body) -> Response { Response::builder() .status(StatusCode::OK) From 576712cefeb3016b811eac0a6ad20c2dd63d9b3a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 10 Sep 2019 17:16:41 +1000 Subject: [PATCH 31/47] WIP: Trying to get futures to work... --- beacon_node/rest_api/Cargo.toml | 1 - beacon_node/rest_api/src/lib.rs | 49 ++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index ac762ebb7..7ea21eeba 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -26,7 +26,6 @@ clap = "2.32.0" http = "^0.1.17" prometheus = { version = "^0.6", features = ["process"] } hyper = "0.12.34" -futures = "0.1" exit-future = "0.1.3" tokio = "0.1.17" url = "2.0" diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index d7ea72cc5..2062d4e03 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -20,7 +20,7 @@ use client_network::NetworkMessage; use client_network::Service as NetworkService; use eth2_config::Eth2Config; use hyper::rt::Future; -use hyper::service::Service; +use hyper::service::{Service, MakeService}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; use parking_lot::RwLock; use response_builder::ResponseBuilder; @@ -31,13 +31,44 @@ use std::sync::Arc; use tokio::runtime::TaskExecutor; use tokio::sync::mpsc; use url_query::UrlQuery; +use hyper::server::conn::AddrStream; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; use eth2_libp2p::rpc::RequestId; +use serde::export::PhantomData; type BoxFut = Box, Error = ApiError> + Send>; +pub struct ApiMaker { + log: slog::Logger, + beacon_chain: Arc>, + db_path: DBPath, + network_service: Arc>, + network_channel: Arc>>, + eth2_config: Arc, +} + +impl MakeService for ApiMaker { + type ReqBody = Body; + type ResBody = Body; + type Error = ApiError; + type Service = ApiService; + type Future = futures::future::FutureResult; + type MakeError = String; + + fn make_service(&mut self, _ctx: AddrStream) -> Self::Future { + futures::future::ok(ApiService { + log: self.log.clone(), + beacon_chain: self.beacon_chain.clone(), + db_path: self.db_path.clone(), + network_service: self.network_service.clone(), + network_channel: self.network_channel.clone(), + eth2_config: self.eth2_config.clone(), + }) + } +} + pub struct ApiService { log: slog::Logger, beacon_chain: Arc>, @@ -205,15 +236,16 @@ pub fn start_server( let server_bc = beacon_chain.clone(); let eth2_config = Arc::new(eth2_config); - let service = move || ApiService { - log: server_log.clone(), - beacon_chain: server_bc.clone(), + let service = move || ApiMaker { + log: log.clone(), + beacon_chain: beacon_chain.clone(), db_path: db_path.clone(), network_service: network_service.clone(), network_channel: Arc::new(RwLock::new(network_chan.clone())), eth2_config: eth2_config.clone(), }; + let log_clone = log.clone(); let server = Server::bind(&bind_addr) .serve(service) @@ -237,15 +269,6 @@ pub fn start_server( Ok(exit_signal) } -impl Future for ApiService { - type Item = Result, ApiError>; - type Error = ApiError; - - fn poll(&mut self) -> Result, Self::Error> { - unimplemented!() - } -} - fn success_response(body: Body) -> Response { Response::builder() .status(StatusCode::OK) From 4dcad27381595fce55811cd34e12118a6a2421c9 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 10 Sep 2019 17:27:28 +1000 Subject: [PATCH 32/47] Fix ApiService woes (hopefully) --- beacon_node/rest_api/src/lib.rs | 53 ++++++++------------------------- 1 file changed, 12 insertions(+), 41 deletions(-) diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 2062d4e03..a5f62360d 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -14,13 +14,14 @@ mod spec; mod url_query; mod validator; -use error::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use client_network::NetworkMessage; use client_network::Service as NetworkService; +use error::{ApiError, ApiResult}; use eth2_config::Eth2Config; use hyper::rt::Future; -use hyper::service::{Service, MakeService}; +use hyper::server::conn::AddrStream; +use hyper::service::{MakeService, Service}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; use parking_lot::RwLock; use response_builder::ResponseBuilder; @@ -31,7 +32,6 @@ use std::sync::Arc; use tokio::runtime::TaskExecutor; use tokio::sync::mpsc; use url_query::UrlQuery; -use hyper::server::conn::AddrStream; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; @@ -40,35 +40,6 @@ use serde::export::PhantomData; type BoxFut = Box, Error = ApiError> + Send>; -pub struct ApiMaker { - log: slog::Logger, - beacon_chain: Arc>, - db_path: DBPath, - network_service: Arc>, - network_channel: Arc>>, - eth2_config: Arc, -} - -impl MakeService for ApiMaker { - type ReqBody = Body; - type ResBody = Body; - type Error = ApiError; - type Service = ApiService; - type Future = futures::future::FutureResult; - type MakeError = String; - - fn make_service(&mut self, _ctx: AddrStream) -> Self::Future { - futures::future::ok(ApiService { - log: self.log.clone(), - beacon_chain: self.beacon_chain.clone(), - db_path: self.db_path.clone(), - network_service: self.network_service.clone(), - network_channel: self.network_channel.clone(), - eth2_config: self.eth2_config.clone(), - }) - } -} - pub struct ApiService { log: slog::Logger, beacon_chain: Arc>, @@ -201,7 +172,6 @@ impl Service for ApiService { metrics::stop_timer(timer); Box::new(futures::future::ok(response)) - } } @@ -236,16 +206,17 @@ pub fn start_server( let server_bc = beacon_chain.clone(); let eth2_config = Arc::new(eth2_config); - let service = move || ApiMaker { - log: log.clone(), - beacon_chain: beacon_chain.clone(), - db_path: db_path.clone(), - network_service: network_service.clone(), - network_channel: Arc::new(RwLock::new(network_chan.clone())), - eth2_config: eth2_config.clone(), + let service = move || -> futures::future::FutureResult, String> { + futures::future::ok(ApiService { + log: log.clone(), + beacon_chain: beacon_chain.clone(), + db_path: db_path.clone(), + network_service: network_service.clone(), + network_channel: Arc::new(RwLock::new(network_chan.clone())), + eth2_config: eth2_config.clone(), + }) }; - let log_clone = log.clone(); let server = Server::bind(&bind_addr) .serve(service) From b0090df5432bfa702e290afca1f7476f6167da53 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 10 Sep 2019 19:30:36 +1000 Subject: [PATCH 33/47] Getting the restructured ApiService to work. --- beacon_node/rest_api/src/error.rs | 14 +++++++------- beacon_node/rest_api/src/lib.rs | 26 +++++++------------------- beacon_node/rest_api/src/validator.rs | 11 +++++------ 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index f3eb597a0..b6b1bbfb5 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -1,7 +1,7 @@ use hyper::{Body, Method, Request, Response, Server, StatusCode}; use std::error::Error as StdError; -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] pub enum ApiError { MethodNotAllowed(String), ServerError(String), @@ -14,7 +14,7 @@ pub enum ApiError { pub type ApiResult = Result, ApiError>; impl ApiError { - pub fn status_code(&self) -> (StatusCode, &String) { + pub fn status_code(self) -> (StatusCode, String) { match self { ApiError::MethodNotAllowed(desc) => (StatusCode::METHOD_NOT_ALLOWED, desc), ApiError::ServerError(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc), @@ -30,10 +30,10 @@ impl Into> for ApiError { fn into(self) -> Response { let status_code = self.status_code(); Response::builder() - .status(status_code.0) - .header("content-type", "text/plain") - .body(Body::from(*status_code.1)) - .expect("Response should always be created.") + .status(status_code.0) + .header("content-type", "text/plain") + .body(Body::from(status_code.1)) + .expect("Response should always be created.") } } @@ -63,7 +63,7 @@ impl StdError for ApiError { impl std::fmt::Display for ApiError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let status = self.status_code(); + let status = self.clone().status_code(); write!(f, "{:?}: {:?}", status.0, status.1) } } diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index a5f62360d..0852dd1a3 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -79,7 +79,6 @@ impl Service for ApiService { let result = match (req.method(), path.as_ref()) { // Methods for Client (&Method::GET, "/node/version") => node::get_version(req), - /* (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), @@ -89,9 +88,7 @@ impl Service for ApiService { (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), (&Method::GET, "/network/peers") => network::get_peer_list::(req), (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), - (&Method::GET, "/network/listen_addresses") => { - network::get_listen_addresses::(req) - } + (&Method::GET, "/network/listen_addresses") => network::get_listen_addresses::(req), // Methods for Beacon Node (&Method::GET, "/beacon/head") => beacon::get_head::(req), @@ -99,9 +96,7 @@ impl Service for ApiService { (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), (&Method::GET, "/beacon/fork") => beacon::get_fork::(req), - (&Method::GET, "/beacon/attestations") => { - helpers::implementation_pending_response(req) - } + (&Method::GET, "/beacon/attestations") => helpers::implementation_pending_response(req), (&Method::GET, "/beacon/attestations/pending") => { helpers::implementation_pending_response(req) } @@ -115,15 +110,9 @@ impl Service for ApiService { } // Methods for Validator - (&Method::GET, "/beacon/validator/duties") => { - validator::get_validator_duties::(req) - } - (&Method::GET, "/beacon/validator/block") => { - validator::get_new_beacon_block::(req) - } - (&Method::POST, "/beacon/validator/block") => { - validator::publish_beacon_block::(req) - } + (&Method::GET, "/beacon/validator/duties") => validator::get_validator_duties::(req), + (&Method::GET, "/beacon/validator/block") => validator::get_new_beacon_block::(req), + //(&Method::POST, "/beacon/validator/block") => validator::publish_beacon_block::(req), (&Method::GET, "/beacon/validator/attestation") => { validator::get_new_attestation::(req) } @@ -149,7 +138,6 @@ impl Service for ApiService { (&Method::GET, "/metrics") => metrics::get_prometheus::(req), - */ _ => Err(ApiError::NotFound( "Request path and/or method not found.".to_owned(), )), @@ -208,8 +196,8 @@ pub fn start_server( let service = move || -> futures::future::FutureResult, String> { futures::future::ok(ApiService { - log: log.clone(), - beacon_chain: beacon_chain.clone(), + log: server_log.clone(), + beacon_chain: server_bc.clone(), db_path: db_path.clone(), network_service: network_service.clone(), network_channel: Arc::new(RwLock::new(network_chan.clone())), diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 2ead55d14..8b2bbd2ac 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -203,7 +203,9 @@ pub fn get_new_beacon_block(req: Request) - Ok(success_response(body)) } -/// HTTP Handler to publish a BeaconBlock, which has been signed by a validator. +/* + + HTTP Handler to publish a BeaconBlock, which has been signed by a validator. pub fn publish_beacon_block(req: Request) -> ApiResult { let _ = check_content_type_for_json(&req)?; let log = get_logger_from_request(&req); @@ -232,11 +234,8 @@ pub fn publish_beacon_block(req: Request) - )) }); block_result - }) - .unwrap() - .unwrap(); + }); - /* .map_err(|e| ApiError::ServerError(format!("Unable parse request body: {:?}", e))) .and_then(|body| { trace!(log, "parsing json"); @@ -251,7 +250,6 @@ pub fn publish_beacon_block(req: Request) - }); tokio::run(block_future); let block = block_future.wait()?; - */ trace!(log, "BeaconBlock successfully parsed from JSON"; "block" => serde_json::to_string(&block).expect("We should always be able to serialize a block that we just created.")); match beacon_chain.process_block(block.clone()) { Ok(BlockProcessingOutcome::Processed { block_root }) => { @@ -277,6 +275,7 @@ pub fn publish_beacon_block(req: Request) - Ok(success_response(Body::empty())) } + */ /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { From b8667217f0da801d471aac51409cd3107d2b0656 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 11 Sep 2019 00:40:22 +1000 Subject: [PATCH 34/47] Made async functions work! - Cleaned up imports - Moved ApiError and such to it's own error.rs - Obsoleted 'success_response' in favour of new async regular and json only flavours - Made ApiError work async and be derived from hyper errors - Added a check to ensure an error is thrown if a non-json encoding is requested on a json-only function - Made all the individual service functions return futures (only node and network for now) --- beacon_node/rest_api/src/beacon.rs | 15 +++-- beacon_node/rest_api/src/error.rs | 13 ++++ beacon_node/rest_api/src/helpers.rs | 68 ++++++++++++++------ beacon_node/rest_api/src/lib.rs | 20 ++---- beacon_node/rest_api/src/metrics.rs | 5 +- beacon_node/rest_api/src/network.rs | 62 ++++++------------ beacon_node/rest_api/src/node.rs | 28 ++++---- beacon_node/rest_api/src/response_builder.rs | 25 ++++--- beacon_node/rest_api/src/spec.rs | 8 +-- beacon_node/rest_api/src/validator.rs | 37 +++-------- 10 files changed, 143 insertions(+), 138 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 66f5e7731..4c57e4770 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -1,5 +1,6 @@ -use super::{success_response, ApiResult, ResponseBuilder}; -use crate::{helpers::*, ApiError, UrlQuery}; +use crate::helpers::*; +use crate::response_builder::ResponseBuilder; +use crate::{ApiError, ApiResult, BoxFut, NetworkService, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use hyper::{Body, Request}; use serde::Serialize; @@ -57,7 +58,7 @@ pub fn get_head(req: Request) -> ApiResult let json: String = serde_json::to_string(&head) .map_err(|e| ApiError::ServerError(format!("Unable to serialize HeadResponse: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } #[derive(Serialize, Encode)] @@ -121,7 +122,7 @@ pub fn get_block_root(req: Request) -> ApiR let json: String = serde_json::to_string(&root) .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the `Fork` of the current head. @@ -132,7 +133,7 @@ pub fn get_fork(req: Request) -> ApiResult ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) })?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the set of validators for an `Epoch` @@ -243,7 +244,7 @@ pub fn get_state_root(req: Request) -> ApiR let json: String = serde_json::to_string(&root) .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the highest finalized slot. @@ -261,7 +262,7 @@ pub fn get_current_finalized_checkpoint( let json: String = serde_json::to_string(&checkpoint) .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return a `BeaconState` at the genesis block. diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index b6b1bbfb5..138affae4 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -1,3 +1,4 @@ +use crate::BoxFut; use hyper::{Body, Method, Request, Response, Server, StatusCode}; use std::error::Error as StdError; @@ -37,6 +38,12 @@ impl Into> for ApiError { } } +impl Into for ApiError { + fn into(self) -> BoxFut { + Box::new(futures::future::err(self)) + } +} + impl From for ApiError { fn from(e: store::Error) -> ApiError { ApiError::ServerError(format!("Database error: {:?}", e)) @@ -55,6 +62,12 @@ impl From for ApiError { } } +impl From for ApiError { + fn from(e: hyper::error::Error) -> ApiError { + ApiError::ServerError(format!("Networking error: {:?}", e)) + } +} + impl StdError for ApiError { fn cause(&self) -> Option<&StdError> { None diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index bff7d9ece..006deb268 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -1,19 +1,46 @@ -use crate::{ApiError, ApiResult}; +use crate::response_builder::ResponseBuilder; +use crate::{ApiError, ApiResult, BoxFut}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use eth2_libp2p::{PubsubMessage, Topic}; use eth2_libp2p::{BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX}; use hex; use http::header; -use hyper::{Body, Request}; +use hyper::{Body, Request, Response, StatusCode}; use network::NetworkMessage; use parking_lot::RwLock; +use serde::Serialize; use ssz::Encode; use std::sync::Arc; use store::{iter::AncestorIter, Store}; use tokio::sync::mpsc; use types::{BeaconBlock, BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; +pub fn success_response(req: Request, item: &T) -> BoxFut { + Box::new(match ResponseBuilder::new(&req).body(item) { + Ok(resp) => futures::future::ok(resp), + Err(e) => futures::future::err(e), + }) +} + +pub fn success_response_json(req: Request, item: &T) -> BoxFut { + if let Err(e) = check_content_type_for_json(&req) { + return Box::new(futures::future::err(e)); + } + Box::new(match ResponseBuilder::new(&req).body_json(item) { + Ok(resp) => futures::future::ok(resp), + Err(e) => futures::future::err(e), + }) +} + +pub fn success_response_old(body: Body) -> Response { + Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/json") + .body(body) + .expect("We should always be able to make response from the success body.") +} + /// Parse a slot from a `0x` preixed string. /// /// E.g., `"1234"` @@ -24,6 +51,21 @@ pub fn parse_slot(string: &str) -> Result { .map_err(|e| ApiError::InvalidQueryParams(format!("Unable to parse slot: {:?}", e))) } +/// Checks the provided request to ensure that the `content-type` header. +/// +/// The content-type header should either be omitted, in which case JSON is assumed, or it should +/// explicity specify `application/json`. If anything else is provided, an error is returned. +pub fn check_content_type_for_json(req: &Request) -> Result<(), ApiError> { + match req.headers().get(header::CONTENT_TYPE) { + Some(h) if h == "application/json" => Ok(()), + Some(h) => Err(ApiError::InvalidQueryParams(format!( + "The provided content-type {:?} is not available, this endpoint only supports json.", + h + ))), + _ => Ok(()), + } +} + /// Parse a root from a `0x` preixed string. /// /// E.g., `"0x0000000000000000000000000000000000000000000000000000000000000000"` @@ -42,21 +84,6 @@ pub fn parse_root(string: &str) -> Result { } } -/// Checks the provided request to ensure that the `content-type` header. -/// -/// The content-type header should either be omitted, in which case JSON is assumed, or it should -/// explicity specify `application/json`. If anything else is provided, an error is returned. -pub fn check_content_type_for_json(req: &Request) -> Result<(), ApiError> { - match req.headers().get(header::CONTENT_TYPE) { - Some(h) if h == "application/json" => Ok(()), - Some(h) => Err(ApiError::InvalidQueryParams(format!( - "The provided content-type {:?} is not available, it must be JSON.", - h - ))), - _ => Ok(()), - } -} - /// Parse a PublicKey from a `0x` prefixed hex string pub fn parse_pubkey(string: &str) -> Result { const PREFIX: &str = "0x"; @@ -186,10 +213,11 @@ pub fn state_root_at_slot( } } -pub fn implementation_pending_response(_req: Request) -> ApiResult { - Err(ApiError::NotImplemented( +pub fn implementation_pending_response(_req: Request) -> BoxFut { + ApiError::NotImplemented( "API endpoint has not yet been implemented, but is planned to be soon.".to_owned(), - )) + ) + .into() } pub fn get_beacon_chain_from_request( diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 0852dd1a3..dc4abc2bf 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -24,7 +24,6 @@ use hyper::server::conn::AddrStream; use hyper::service::{MakeService, Service}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; use parking_lot::RwLock; -use response_builder::ResponseBuilder; use slog::{info, o, warn}; use std::ops::Deref; use std::path::PathBuf; @@ -59,6 +58,8 @@ impl Service for ApiService { metrics::inc_counter(&metrics::REQUEST_COUNT); let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); + // Add all the useful bits into the request, so that we can pull them out in the individual + // functions. req.extensions_mut() .insert::(self.log.clone()); req.extensions_mut() @@ -90,6 +91,7 @@ impl Service for ApiService { (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), (&Method::GET, "/network/listen_addresses") => network::get_listen_addresses::(req), + /* // Methods for Beacon Node (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), @@ -137,13 +139,13 @@ impl Service for ApiService { (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::(req), (&Method::GET, "/metrics") => metrics::get_prometheus::(req), - - _ => Err(ApiError::NotFound( + */ + _ => Box::new(futures::future::err(ApiError::NotFound( "Request path and/or method not found.".to_owned(), - )), + ))), }; - let response = match result { + let response = match result.wait() { // Return the `hyper::Response`. Ok(response) => { metrics::inc_counter(&metrics::SUCCESS_COUNT); @@ -228,14 +230,6 @@ pub fn start_server( Ok(exit_signal) } -fn success_response(body: Body) -> Response { - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(body) - .expect("We should always be able to make response from the success body.") -} - #[derive(Clone)] pub struct DBPath(PathBuf); diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 62a769de1..d430db3f5 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -1,4 +1,5 @@ -use crate::{helpers::*, success_response, ApiError, ApiResult, DBPath}; +use crate::helpers::*; +use crate::{ApiError, ApiResult, DBPath}; use beacon_chain::BeaconChainTypes; use http::HeaderValue; use hyper::{Body, Request}; @@ -62,7 +63,7 @@ pub fn get_prometheus(req: Request) -> ApiR String::from_utf8(buffer) .map(|string| { - let mut response = success_response(Body::from(string)); + let mut response = success_response_old(Body::from(string)); // Need to change the header to text/plain for prometheus response.headers_mut().insert( "content-type", diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs index 4f1f53bb9..e037d43f0 100644 --- a/beacon_node/rest_api/src/network.rs +++ b/beacon_node/rest_api/src/network.rs @@ -1,4 +1,5 @@ -use crate::{success_response, ApiError, ApiResult, NetworkService}; +use crate::helpers::*; +use crate::{ApiError, BoxFut, NetworkService}; use beacon_chain::BeaconChainTypes; use eth2_libp2p::{Enr, Multiaddr, PeerId}; use hyper::{Body, Request}; @@ -7,92 +8,75 @@ use std::sync::Arc; /// HTTP handler to return the list of libp2p multiaddr the client is listening on. /// /// Returns a list of `Multiaddr`, serialized according to their `serde` impl. -pub fn get_listen_addresses(req: Request) -> ApiResult { +pub fn get_listen_addresses(req: Request) -> BoxFut { let network = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; - + .expect("The network service should always be there, we put it there"); let multiaddresses: Vec = network.listen_multiaddrs(); - - Ok(success_response(Body::from( - serde_json::to_string(&multiaddresses) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, - ))) + success_response_json(req, &multiaddresses) } /// HTTP handler to return the network port the client is listening on. /// /// Returns the TCP port number in its plain form (which is also valid JSON serialization) -pub fn get_listen_port(req: Request) -> ApiResult { +pub fn get_listen_port(req: Request) -> BoxFut { let network = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + .expect("The network service should always be there, we put it there") + .clone(); - Ok(success_response(Body::from( - serde_json::to_string(&network.listen_port()) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize port: {:?}", e)))?, - ))) + success_response(req, &network.listen_port()) } /// HTTP handler to return the Discv5 ENR from the client's libp2p service. /// /// ENR is encoded as base64 string. -pub fn get_enr(req: Request) -> ApiResult { +pub fn get_enr(req: Request) -> BoxFut { let network = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + .expect("The network service should always be there, we put it there"); let enr: Enr = network.local_enr(); - - Ok(success_response(Body::from( - serde_json::to_string(&enr.to_base64()) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, - ))) + success_response_json(req, &enr.to_base64()) } /// HTTP handler to return the `PeerId` from the client's libp2p service. /// /// PeerId is encoded as base58 string. -pub fn get_peer_id(req: Request) -> ApiResult { +pub fn get_peer_id(req: Request) -> BoxFut { let network = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + .expect("The network service should always be there, we put it there"); let peer_id: PeerId = network.local_peer_id(); - Ok(success_response(Body::from( - serde_json::to_string(&peer_id.to_base58()) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, - ))) + success_response_json(req, &peer_id.to_base58()) } /// HTTP handler to return the number of peers connected in the client's libp2p service. -pub fn get_peer_count(req: Request) -> ApiResult { +pub fn get_peer_count(req: Request) -> BoxFut { let network = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + .expect("The network service should always be there, we put it there"); let connected_peers: usize = network.connected_peers(); - Ok(success_response(Body::from( - serde_json::to_string(&connected_peers) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, - ))) + success_response(req, &connected_peers) } /// HTTP handler to return the list of peers connected to the client's libp2p service. /// /// Peers are presented as a list of `PeerId::to_string()`. -pub fn get_peer_list(req: Request) -> ApiResult { +pub fn get_peer_list(req: Request) -> BoxFut { let network = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + .expect("The network service should always be there, we put it there"); let connected_peers: Vec = network .connected_peer_set() @@ -100,9 +84,5 @@ pub fn get_peer_list(req: Request) -> ApiResult { .map(PeerId::to_string) .collect(); - Ok(success_response(Body::from( - serde_json::to_string(&connected_peers).map_err(|e| { - ApiError::ServerError(format!("Unable to serialize Vec: {:?}", e)) - })?, - ))) + success_response_json(req, &connected_peers) } diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index c75d3ba20..8ca7fb48a 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -1,25 +1,23 @@ -use crate::helpers::get_beacon_chain_from_request; -use crate::{success_response, ApiResult}; +use crate::helpers::*; +use crate::{ApiResult, BoxFut}; use beacon_chain::BeaconChainTypes; use hyper::{Body, Request}; use version; /// Read the version string from the current Lighthouse build. -pub fn get_version(_req: Request) -> ApiResult { - let body = Body::from( - serde_json::to_string(&version::version()) - .expect("Version should always be serialializable as JSON."), - ); - Ok(success_response(body)) +pub fn get_version(req: Request) -> BoxFut { + success_response_json(req, &version::version()) } /// Read the genesis time from the current beacon chain state. -pub fn get_genesis_time(req: Request) -> ApiResult { - let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_genesis_time(req: Request) -> BoxFut { + let bc = get_beacon_chain_from_request::(&req); + let (_beacon_chain, head_state) = match bc { + Ok((bc, hs)) => (bc, hs), + Err(e) => { + return e.into(); + } + }; let gen_time: u64 = head_state.genesis_time; - let body = Body::from( - serde_json::to_string(&gen_time) - .expect("Genesis should time always have a valid JSON serialization."), - ); - Ok(success_response(body)) + success_response(req, &gen_time) } diff --git a/beacon_node/rest_api/src/response_builder.rs b/beacon_node/rest_api/src/response_builder.rs index c1df4892c..b48b9e41a 100644 --- a/beacon_node/rest_api/src/response_builder.rs +++ b/beacon_node/rest_api/src/response_builder.rs @@ -27,15 +27,9 @@ impl ResponseBuilder { pub fn body(self, item: &T) -> ApiResult { let (body, content_type) = match self.encoding { - Encoding::JSON => ( - Body::from(serde_json::to_string(&item).map_err(|e| { - ApiError::ServerError(format!( - "Unable to serialize response body as JSON: {:?}", - e - )) - })?), - "application/json", - ), + Encoding::JSON => { + return self.body_json(item); + } Encoding::SSZ => (Body::from(item.as_ssz_bytes()), "application/ssz"), Encoding::YAML => ( Body::from(serde_yaml::to_string(&item).map_err(|e| { @@ -54,4 +48,17 @@ impl ResponseBuilder { .body(Body::from(body)) .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) } + + pub fn body_json(self, item: &T) -> ApiResult { + Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_string(&item).map_err(|e| { + ApiError::ServerError(format!( + "Unable to serialize response body as JSON: {:?}", + e + )) + })?)) + .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) + } } diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index ad168faf1..3132f3cd4 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -1,4 +1,4 @@ -use super::{success_response, ApiResult}; +use super::ApiResult; use crate::helpers::*; use crate::ApiError; use beacon_chain::BeaconChainTypes; @@ -14,7 +14,7 @@ pub fn get_spec(req: Request) -> ApiResult let json: String = serde_json::to_string(&beacon_chain.spec) .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the full Eth2Config object. @@ -27,7 +27,7 @@ pub fn get_eth2_config(req: Request) -> Api let json: String = serde_json::to_string(eth2_config.as_ref()) .map_err(|e| ApiError::ServerError(format!("Unable to serialize Eth2Config: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the full spec object. @@ -35,5 +35,5 @@ pub fn get_slots_per_epoch(_req: Request) - let json: String = serde_json::to_string(&T::EthSpec::slots_per_epoch()) .map_err(|e| ApiError::ServerError(format!("Unable to serialize epoch: {:?}", e)))?; - Ok(success_response(Body::from(json))) + Ok(success_response_old(Body::from(json))) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 8b2bbd2ac..7d236e0cf 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,5 +1,5 @@ -use super::{success_response, ApiResult}; -use crate::{helpers::*, ApiError, UrlQuery}; +use crate::helpers::*; +use crate::{ApiError, ApiResult, UrlQuery}; use beacon_chain::{BeaconChainTypes, BlockProcessingOutcome}; use bls::{AggregateSignature, PublicKey, Signature}; use futures::future::Future; @@ -160,7 +160,7 @@ pub fn get_validator_duties(req: Request) - serde_json::to_string(&duties) .expect("We should always be able to serialize the duties we created."), ); - Ok(success_response(body)) + Ok(success_response_old(body)) } /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. @@ -200,12 +200,10 @@ pub fn get_new_beacon_block(req: Request) - serde_json::to_string(&new_block) .expect("We should always be able to serialize a new block that we produced."), ); - Ok(success_response(body)) + Ok(success_response_old(body)) } -/* - - HTTP Handler to publish a BeaconBlock, which has been signed by a validator. +/// HTTP Handler to publish a BeaconBlock, which has been signed by a validator. pub fn publish_beacon_block(req: Request) -> ApiResult { let _ = check_content_type_for_json(&req)?; let log = get_logger_from_request(&req); @@ -222,7 +220,7 @@ pub fn publish_beacon_block(req: Request) - log, "Got the request body, now going to parse it into a block." ); - let block = body + let block_future = body .concat2() .map(move |chunk| chunk.iter().cloned().collect::>()) .map(|chunks| { @@ -235,22 +233,8 @@ pub fn publish_beacon_block(req: Request) - }); block_result }); - - .map_err(|e| ApiError::ServerError(format!("Unable parse request body: {:?}", e))) - .and_then(|body| { - trace!(log, "parsing json"); - let block_result: Result, ApiError> = - serde_json::from_slice(&body.as_slice()).map_err(|e| { - ApiError::InvalidQueryParams(format!( - "Unable to deserialize JSON into a BeaconBlock: {:?}", - e - )) - }); - block_result - }); - tokio::run(block_future); - let block = block_future.wait()?; - trace!(log, "BeaconBlock successfully parsed from JSON"; "block" => serde_json::to_string(&block).expect("We should always be able to serialize a block that we just created.")); + let block = block_future.wait()??; + trace!(log, "BeaconBlock successfully parsed from JSON"); match beacon_chain.process_block(block.clone()) { Ok(BlockProcessingOutcome::Processed { block_root }) => { // Block was processed, publish via gossipsub @@ -273,9 +257,8 @@ pub fn publish_beacon_block(req: Request) - } } - Ok(success_response(Body::empty())) + Ok(success_response_old(Body::empty())) } - */ /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { @@ -377,5 +360,5 @@ pub fn get_new_attestation(req: Request) -> serde_json::to_string(&attestation) .expect("We should always be able to serialize a new attestation that we produced."), ); - Ok(success_response(body)) + Ok(success_response_old(body)) } From ebd97730d572a80343f1cd8779d19e1b56e8a5eb Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 11 Sep 2019 01:43:49 +1000 Subject: [PATCH 35/47] Converted the Beacon API service to Futures - Added SSZ encode for HeadResponse - Converted all of the /beacon/ endpoints to return BoxFut instead of ApiResult - Wrapped all of the '?'s in a new macro try_future!() - Copied the try macro to try_future, so that a boxed future can easily be returned. - Replaced all of the response serializations to use the new success_response --- beacon_node/rest_api/src/beacon.rs | 164 +++++++++++++--------------- beacon_node/rest_api/src/helpers.rs | 2 +- beacon_node/rest_api/src/lib.rs | 4 +- beacon_node/rest_api/src/macros.rs | 13 +++ 4 files changed, 94 insertions(+), 89 deletions(-) create mode 100644 beacon_node/rest_api/src/macros.rs diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 4c57e4770..f5abc51af 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use store::Store; use types::{BeaconBlock, BeaconState, Epoch, EthSpec, Hash256, Slot, Validator}; -#[derive(Serialize)] +#[derive(Serialize, Encode)] pub struct HeadResponse { pub slot: Slot, pub block_root: Hash256, @@ -23,11 +23,12 @@ pub struct HeadResponse { } /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. -pub fn get_head(req: Request) -> ApiResult { +pub fn get_head(req: Request) -> BoxFut { let beacon_chain = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + .expect("BeaconChain extension must be there, because we put it there.") + .clone(); let chain_head = beacon_chain.head(); @@ -55,10 +56,7 @@ pub fn get_head(req: Request) -> ApiResult previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root, }; - let json: String = serde_json::to_string(&head) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize HeadResponse: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + success_response(req, &head) } #[derive(Serialize, Encode)] @@ -69,84 +67,83 @@ pub struct BlockResponse { } /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. -pub fn get_block(req: Request) -> ApiResult { +pub fn get_block(req: Request) -> BoxFut { let beacon_chain = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + .expect("BeaconChain extension must be there, because we put it there.") + .clone(); let query_params = ["root", "slot"]; - let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; + let query = try_future!(UrlQuery::from_request(&req)); + let (key, value) = try_future!(query.first_of(&query_params)); let block_root = match (key.as_ref(), value) { ("slot", value) => { - let target = parse_slot(&value)?; + let target = try_future!(parse_slot(&value)); - block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + try_future!(block_root_at_slot(&beacon_chain, target).ok_or_else(|| { ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) - })? + })) + } + ("root", value) => try_future!(parse_root(&value)), + _ => { + return Box::new(futures::future::err(ApiError::ServerError( + "Unexpected query parameter".into(), + ))) } - ("root", value) => parse_root(&value)?, - _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), }; - let block = beacon_chain + let block = try_future!(try_future!(beacon_chain .store - .get::>(&block_root)? - .ok_or_else(|| { - ApiError::NotFound(format!( - "Unable to find BeaconBlock for root {:?}", - block_root - )) - })?; + .get::>(&block_root)) + .ok_or_else(|| { + ApiError::NotFound(format!( + "Unable to find BeaconBlock for root {:?}", + block_root + )) + })); let response = BlockResponse { root: block_root, beacon_block: block, }; - ResponseBuilder::new(&req).body(&response) + success_response(req, &response) } /// HTTP handler to return a `BeaconBlock` root at a given `slot`. -pub fn get_block_root(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_block_root(req: Request) -> BoxFut { + let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); - let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; - let target = parse_slot(&slot_string)?; + let slot_string: String = + try_future!(try_future!(UrlQuery::from_request(&req)).only_one("slot")); + let target: Slot = try_future!(parse_slot(&slot_string)); - let root = block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + let root = try_future!(block_root_at_slot(&beacon_chain, target).ok_or_else(|| { ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) - })?; + })); - let json: String = serde_json::to_string(&root) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + success_response(req, &root) } /// HTTP handler to return the `Fork` of the current head. -pub fn get_fork(req: Request) -> ApiResult { - let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_fork(req: Request) -> BoxFut { + let (_beacon_chain, head_state) = try_future!(get_beacon_chain_from_request::(&req)); - let json: String = serde_json::to_string(&head_state.fork).map_err(|e| { - ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) - })?; - - Ok(success_response_old(Body::from(json))) + success_response(req, &head_state.fork) } /// HTTP handler to return the set of validators for an `Epoch` /// /// The `Epoch` parameter can be any epoch number. If it is not specified, /// the current epoch is assumed. -pub fn get_validators(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_validators(req: Request) -> BoxFut { + let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); let epoch = match UrlQuery::from_request(&req) { // We have some parameters, so make sure it's the epoch one and parse it - Ok(query) => query - .only_one("epoch")? + Ok(query) => try_future!(try_future!(query.only_one("epoch")) .parse::() .map(Epoch::from) .map_err(|e| { @@ -154,11 +151,11 @@ pub fn get_validators(req: Request) -> ApiR "Invalid epoch parameter, must be a u64. {:?}", e )) - })?, + })), // In this case, our url query did not contain any parameters, so we take the default - Err(_) => beacon_chain.epoch().map_err(|e| { + Err(_) => try_future!(beacon_chain.epoch().map_err(|e| { ApiError::ServerError(format!("Unable to determine current epoch: {:?}", e)) - })?, + })), }; let all_validators = &beacon_chain.head().beacon_state.validators; @@ -168,7 +165,7 @@ pub fn get_validators(req: Request) -> ApiR .cloned() .collect(); - ResponseBuilder::new(&req).body(&active_vals) + success_response(req, &active_vals) } #[derive(Serialize, Encode)] @@ -182,43 +179,42 @@ pub struct StateResponse { /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. -pub fn get_state(req: Request) -> ApiResult { - let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_state(req: Request) -> BoxFut { + let (beacon_chain, head_state) = try_future!(get_beacon_chain_from_request::(&req)); let (key, value) = match UrlQuery::from_request(&req) { Ok(query) => { - // We have *some* parameters, check them. + // We have *some* parameters, just check them. let query_params = ["root", "slot"]; - match query.first_of(&query_params) { - Ok((k, v)) => (k, v), - Err(e) => { - // Wrong parameters provided, or another error, return the error. - return Err(e); - } - } + try_future!(query.first_of(&query_params)) } Err(ApiError::InvalidQueryParams(_)) => { // No parameters provided at all, use current slot. (String::from("slot"), head_state.slot.to_string()) } Err(e) => { - return Err(e); + return Box::new(futures::future::err(e)); } }; let (root, state): (Hash256, BeaconState) = match (key.as_ref(), value) { - ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?, + ("slot", value) => try_future!(state_at_slot( + &beacon_chain, + try_future!(parse_slot(&value)) + )), ("root", value) => { - let root = &parse_root(&value)?; + let root = &try_future!(parse_root(&value)); - let state = beacon_chain - .store - .get(root)? - .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))?; + let state = try_future!(try_future!(beacon_chain.store.get(root)) + .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))); (*root, state) } - _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), + _ => { + return Box::new(futures::future::err(ApiError::ServerError( + "Unexpected query parameter".into(), + ))) + } }; let response = StateResponse { @@ -226,32 +222,29 @@ pub fn get_state(req: Request) -> ApiResult beacon_state: state, }; - ResponseBuilder::new(&req).body(&response) + success_response(req, &response) } /// HTTP handler to return a `BeaconState` root at a given `slot`. /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. -pub fn get_state_root(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_state_root(req: Request) -> BoxFut { + let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); - let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; - let slot = parse_slot(&slot_string)?; + let slot_string = try_future!(try_future!(UrlQuery::from_request(&req)).only_one("slot")); + let slot = try_future!(parse_slot(&slot_string)); - let root = state_root_at_slot(&beacon_chain, slot)?; + let root = try_future!(state_root_at_slot(&beacon_chain, slot)); - let json: String = serde_json::to_string(&root) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + success_response(req, &root) } /// HTTP handler to return the highest finalized slot. pub fn get_current_finalized_checkpoint( req: Request, -) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; +) -> BoxFut { + let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); let checkpoint = beacon_chain .head() @@ -259,17 +252,14 @@ pub fn get_current_finalized_checkpoint( .finalized_checkpoint .clone(); - let json: String = serde_json::to_string(&checkpoint) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + success_response(req, &checkpoint) } /// HTTP handler to return a `BeaconState` at the genesis block. -pub fn get_genesis_state(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; +pub fn get_genesis_state(req: Request) -> BoxFut { + let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); - let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; + let (_root, state) = try_future!(state_at_slot(&beacon_chain, Slot::new(0))); - ResponseBuilder::new(&req).body(&state) + success_response(req, &state) } diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 006deb268..9eae4a00a 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -227,7 +227,7 @@ pub fn get_beacon_chain_from_request( let beacon_chain = req .extensions() .get::>>() - .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".into()))?; + .expect("BeaconChain extension must be there, because we put it there."); let mut head_state = beacon_chain .state_now() .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))?; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index dc4abc2bf..e4f88caf3 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -1,4 +1,6 @@ #[macro_use] +mod macros; +#[macro_use] extern crate lazy_static; extern crate network as client_network; @@ -91,7 +93,6 @@ impl Service for ApiService { (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), (&Method::GET, "/network/listen_addresses") => network::get_listen_addresses::(req), - /* // Methods for Beacon Node (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), @@ -111,6 +112,7 @@ impl Service for ApiService { helpers::implementation_pending_response(req) } + /* // Methods for Validator (&Method::GET, "/beacon/validator/duties") => validator::get_validator_duties::(req), (&Method::GET, "/beacon/validator/block") => validator::get_new_beacon_block::(req), diff --git a/beacon_node/rest_api/src/macros.rs b/beacon_node/rest_api/src/macros.rs new file mode 100644 index 000000000..e95cfb8ae --- /dev/null +++ b/beacon_node/rest_api/src/macros.rs @@ -0,0 +1,13 @@ +macro_rules! try_future { + ($expr:expr) => { + match $expr { + core::result::Result::Ok(val) => val, + core::result::Result::Err(err) => { + return Box::new(futures::future::err(std::convert::From::from(err))) + } + } + }; + ($expr:expr,) => { + $crate::try_future!($expr) + }; +} From 2739ee83f9f97508b2236f5648550ee8f4ff343f Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 11 Sep 2019 18:02:00 +1000 Subject: [PATCH 36/47] Further restructuring futures API. - Adding try_future! macros where necessary - Returning ApiResult and mapping it to future instead - Upgrading POST publish block to return a future directly --- beacon_node/rest_api/src/beacon.rs | 2 +- beacon_node/rest_api/src/helpers.rs | 14 +++-- beacon_node/rest_api/src/lib.rs | 13 +++-- beacon_node/rest_api/src/node.rs | 16 ++---- beacon_node/rest_api/src/validator.rs | 83 ++++++++++++++------------- 5 files changed, 68 insertions(+), 60 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index f5abc51af..cf540c65a 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -203,7 +203,7 @@ pub fn get_state(req: Request) -> BoxFut { try_future!(parse_slot(&value)) )), ("root", value) => { - let root = &try_future!(parse_root(&value)); + let root: &Hash256 = &try_future!(parse_root(&value)); let state = try_future!(try_future!(beacon_chain.store.get(root)) .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))); diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 9eae4a00a..a3633cfec 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -23,6 +23,13 @@ pub fn success_response(req: Request, item: &T) -> }) } +pub fn success_response_2(req: Request, item: &T) -> ApiResult { + ResponseBuilder::new(&req).body(item) +} +pub fn success_response_2_json(req: Request, item: &T) -> ApiResult { + ResponseBuilder::new(&req).body_json(item) +} + pub fn success_response_json(req: Request, item: &T) -> BoxFut { if let Err(e) = check_content_type_for_json(&req) { return Box::new(futures::future::err(e)); @@ -213,11 +220,10 @@ pub fn state_root_at_slot( } } -pub fn implementation_pending_response(_req: Request) -> BoxFut { - ApiError::NotImplemented( +pub fn implementation_pending_response(_req: Request) -> ApiResult { + Err(ApiError::NotImplemented( "API endpoint has not yet been implemented, but is planned to be soon.".to_owned(), - ) - .into() + )) } pub fn get_beacon_chain_from_request( diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index e4f88caf3..bc3d3d159 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -33,6 +33,7 @@ use std::sync::Arc; use tokio::runtime::TaskExecutor; use tokio::sync::mpsc; use url_query::UrlQuery; +use futures::future::IntoFuture; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; @@ -81,9 +82,10 @@ impl Service for ApiService { // Route the request to the correct handler. let result = match (req.method(), path.as_ref()) { // Methods for Client - (&Method::GET, "/node/version") => node::get_version(req), - (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), - (&Method::GET, "/node/syncing") => helpers::implementation_pending_response(req), + (&Method::GET, "/node/version") => Box::new(node::get_version(req).into_future()), + (&Method::GET, "/node/genesis_time") => Box::new(node::get_genesis_time::(req).into_future()), + (&Method::GET, "/node/syncing") => Box::new(helpers::implementation_pending_response(req).into_future()), + /* // Methods for Network (&Method::GET, "/network/enr") => network::get_enr::(req), @@ -112,11 +114,12 @@ impl Service for ApiService { helpers::implementation_pending_response(req) } - /* // Methods for Validator (&Method::GET, "/beacon/validator/duties") => validator::get_validator_duties::(req), (&Method::GET, "/beacon/validator/block") => validator::get_new_beacon_block::(req), - //(&Method::POST, "/beacon/validator/block") => validator::publish_beacon_block::(req), + */ + (&Method::POST, "/beacon/validator/block") => validator::publish_beacon_block::(req), + /* (&Method::GET, "/beacon/validator/attestation") => { validator::get_new_attestation::(req) } diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index 8ca7fb48a..33b8e055c 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -5,19 +5,13 @@ use hyper::{Body, Request}; use version; /// Read the version string from the current Lighthouse build. -pub fn get_version(req: Request) -> BoxFut { - success_response_json(req, &version::version()) +pub fn get_version(req: Request) -> ApiResult { + success_response_2_json(req, &version::version()) } /// Read the genesis time from the current beacon chain state. -pub fn get_genesis_time(req: Request) -> BoxFut { - let bc = get_beacon_chain_from_request::(&req); - let (_beacon_chain, head_state) = match bc { - Ok((bc, hs)) => (bc, hs), - Err(e) => { - return e.into(); - } - }; +pub fn get_genesis_time(req: Request) -> ApiResult { + let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; let gen_time: u64 = head_state.genesis_time; - success_response(req, &gen_time) + success_response_2(req, &gen_time) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 7d236e0cf..1de164704 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,5 +1,6 @@ use crate::helpers::*; -use crate::{ApiError, ApiResult, UrlQuery}; +use crate::response_builder::ResponseBuilder; +use crate::{ApiError, ApiResult, UrlQuery, BoxFut}; use beacon_chain::{BeaconChainTypes, BlockProcessingOutcome}; use bls::{AggregateSignature, PublicKey, Signature}; use futures::future::Future; @@ -204,10 +205,10 @@ pub fn get_new_beacon_block(req: Request) - } /// HTTP Handler to publish a BeaconBlock, which has been signed by a validator. -pub fn publish_beacon_block(req: Request) -> ApiResult { - let _ = check_content_type_for_json(&req)?; +pub fn publish_beacon_block(req: Request) -> BoxFut { + let _ = try_future!(check_content_type_for_json(&req)); let log = get_logger_from_request(&req); - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); // Get the network sending channel from the request, for later transmission let network_chan = req .extensions() @@ -215,49 +216,53 @@ pub fn publish_beacon_block(req: Request) - .expect("Should always get the network channel from the request, since we put it in there.") .clone(); + let response_builder = ResponseBuilder::new(&req); + let body = req.into_body(); trace!( log, "Got the request body, now going to parse it into a block." ); - let block_future = body + Box::new(body .concat2() - .map(move |chunk| chunk.iter().cloned().collect::>()) - .map(|chunks| { - let block_result: Result, ApiError> = - serde_json::from_slice(&chunks.as_slice()).map_err(|e| { - ApiError::InvalidQueryParams(format!( - "Unable to deserialize JSON into a BeaconBlock: {:?}", + .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}",e))) + .map(|chunk| chunk.iter().cloned().collect::>()) + .and_then(|chunks| { + serde_json::from_slice(&chunks.as_slice()).map_err(|e| { + ApiError::InvalidQueryParams(format!( + "Unable to deserialize JSON into a BeaconBlock: {:?}", + e + )) + }) + }) + .and_then(move |block: BeaconBlock| { + let slot = block.slot; + match beacon_chain.process_block(block.clone()) { + Ok(BlockProcessingOutcome::Processed { block_root }) => { + // Block was processed, publish via gossipsub + info!(log, "Processed valid block from API, transmitting to network."; "block_slot" => slot, "block_root" => format!("{}", block_root)); + publish_beacon_block_to_network::(network_chan, block) + } + Ok(outcome) => { + warn!(log, "Block could not be processed, but is being sent to the network anyway."; "block_slot" => slot, "outcome" => format!("{:?}", outcome)); + //TODO need to send to network and return http 202 + Err(ApiError::InvalidQueryParams(format!( + "The BeaconBlock could not be processed: {:?}", + outcome + ))) + } + Err(e) => { + Err(ApiError::ServerError(format!( + "Unable to process block: {:?}", e - )) - }); - block_result - }); - let block = block_future.wait()??; - trace!(log, "BeaconBlock successfully parsed from JSON"); - match beacon_chain.process_block(block.clone()) { - Ok(BlockProcessingOutcome::Processed { block_root }) => { - // Block was processed, publish via gossipsub - info!(log, "Processed valid block from API, transmitting to network."; "block_slot" => block.slot, "block_root" => format!("{}", block_root)); - publish_beacon_block_to_network::(network_chan, block)?; - } - Ok(outcome) => { - warn!(log, "Block could not be processed, but is being sent to the network anyway."; "block_slot" => block.slot, "outcome" => format!("{:?}", outcome)); - //TODO need to send to network and return http 202 - return Err(ApiError::InvalidQueryParams(format!( - "The BeaconBlock could not be processed: {:?}", - outcome - ))); - } - Err(e) => { - return Err(ApiError::ServerError(format!( - "Unable to process block: {:?}", - e - ))); - } - } + ))) + } + } + }).and_then(|_| { + response_builder.body_json(&()) + })) + - Ok(success_response_old(Body::empty())) } /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. From 0bd8187ff648e1eca0224a1f9d27792ce612db24 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 11 Sep 2019 18:21:51 +1000 Subject: [PATCH 37/47] Revert most of commit ebd97730d572a80343f1cd8779d19e1b56e8a5eb. - Undoing making of the beacon chain read stuff return futures, instead dealing with it elsewhere. --- beacon_node/rest_api/src/beacon.rs | 160 +++++++++++++++------------- beacon_node/rest_api/src/helpers.rs | 2 +- beacon_node/rest_api/src/lib.rs | 1 + 3 files changed, 87 insertions(+), 76 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index cf540c65a..850c77dc1 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -23,12 +23,11 @@ pub struct HeadResponse { } /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. -pub fn get_head(req: Request) -> BoxFut { +pub fn get_head(req: Request) -> ApiResult { let beacon_chain = req .extensions() .get::>>() - .expect("BeaconChain extension must be there, because we put it there.") - .clone(); + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; let chain_head = beacon_chain.head(); @@ -56,7 +55,10 @@ pub fn get_head(req: Request) -> BoxFut { previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root, }; - success_response(req, &head) + let json: String = serde_json::to_string(&head) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize HeadResponse: {:?}", e)))?; + + Ok(success_response_old(Body::from(json))) } #[derive(Serialize, Encode)] @@ -67,83 +69,84 @@ pub struct BlockResponse { } /// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. -pub fn get_block(req: Request) -> BoxFut { +pub fn get_block(req: Request) -> ApiResult { let beacon_chain = req .extensions() .get::>>() - .expect("BeaconChain extension must be there, because we put it there.") - .clone(); + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; let query_params = ["root", "slot"]; - let query = try_future!(UrlQuery::from_request(&req)); - let (key, value) = try_future!(query.first_of(&query_params)); + let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; let block_root = match (key.as_ref(), value) { ("slot", value) => { - let target = try_future!(parse_slot(&value)); + let target = parse_slot(&value)?; - try_future!(block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + block_root_at_slot(&beacon_chain, target).ok_or_else(|| { ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) - })) - } - ("root", value) => try_future!(parse_root(&value)), - _ => { - return Box::new(futures::future::err(ApiError::ServerError( - "Unexpected query parameter".into(), - ))) + })? } + ("root", value) => parse_root(&value)?, + _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), }; - let block = try_future!(try_future!(beacon_chain + let block = beacon_chain .store - .get::>(&block_root)) - .ok_or_else(|| { - ApiError::NotFound(format!( - "Unable to find BeaconBlock for root {:?}", - block_root - )) - })); + .get::>(&block_root)? + .ok_or_else(|| { + ApiError::NotFound(format!( + "Unable to find BeaconBlock for root {:?}", + block_root + )) + })?; let response = BlockResponse { root: block_root, beacon_block: block, }; - success_response(req, &response) + ResponseBuilder::new(&req).body(&response) } /// HTTP handler to return a `BeaconBlock` root at a given `slot`. -pub fn get_block_root(req: Request) -> BoxFut { - let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); +pub fn get_block_root(req: Request) -> ApiResult { + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; - let slot_string: String = - try_future!(try_future!(UrlQuery::from_request(&req)).only_one("slot")); - let target: Slot = try_future!(parse_slot(&slot_string)); + let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; + let target = parse_slot(&slot_string)?; - let root = try_future!(block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + let root = block_root_at_slot(&beacon_chain, target).ok_or_else(|| { ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) - })); + })?; - success_response(req, &root) + let json: String = serde_json::to_string(&root) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; + + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the `Fork` of the current head. -pub fn get_fork(req: Request) -> BoxFut { - let (_beacon_chain, head_state) = try_future!(get_beacon_chain_from_request::(&req)); +pub fn get_fork(req: Request) -> ApiResult { + let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; - success_response(req, &head_state.fork) + let json: String = serde_json::to_string(&head_state.fork).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) + })?; + + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the set of validators for an `Epoch` /// /// The `Epoch` parameter can be any epoch number. If it is not specified, /// the current epoch is assumed. -pub fn get_validators(req: Request) -> BoxFut { - let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); +pub fn get_validators(req: Request) -> ApiResult { + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let epoch = match UrlQuery::from_request(&req) { // We have some parameters, so make sure it's the epoch one and parse it - Ok(query) => try_future!(try_future!(query.only_one("epoch")) + Ok(query) => query + .only_one("epoch")? .parse::() .map(Epoch::from) .map_err(|e| { @@ -151,11 +154,11 @@ pub fn get_validators(req: Request) -> BoxF "Invalid epoch parameter, must be a u64. {:?}", e )) - })), + })?, // In this case, our url query did not contain any parameters, so we take the default - Err(_) => try_future!(beacon_chain.epoch().map_err(|e| { + Err(_) => beacon_chain.epoch().map_err(|e| { ApiError::ServerError(format!("Unable to determine current epoch: {:?}", e)) - })), + })?, }; let all_validators = &beacon_chain.head().beacon_state.validators; @@ -165,7 +168,7 @@ pub fn get_validators(req: Request) -> BoxF .cloned() .collect(); - success_response(req, &active_vals) + ResponseBuilder::new(&req).body(&active_vals) } #[derive(Serialize, Encode)] @@ -179,42 +182,43 @@ pub struct StateResponse { /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. -pub fn get_state(req: Request) -> BoxFut { - let (beacon_chain, head_state) = try_future!(get_beacon_chain_from_request::(&req)); +pub fn get_state(req: Request) -> ApiResult { + let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; let (key, value) = match UrlQuery::from_request(&req) { Ok(query) => { // We have *some* parameters, just check them. let query_params = ["root", "slot"]; - try_future!(query.first_of(&query_params)) + match query.first_of(&query_params) { + Ok((k, v)) => (k, v), + Err(e) => { + // Wrong parameters provided, or another error, return the error. + return Err(e); + } + } } Err(ApiError::InvalidQueryParams(_)) => { // No parameters provided at all, use current slot. (String::from("slot"), head_state.slot.to_string()) } Err(e) => { - return Box::new(futures::future::err(e)); + return Err(e); } }; let (root, state): (Hash256, BeaconState) = match (key.as_ref(), value) { - ("slot", value) => try_future!(state_at_slot( - &beacon_chain, - try_future!(parse_slot(&value)) - )), + ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?, ("root", value) => { - let root: &Hash256 = &try_future!(parse_root(&value)); + let root = &parse_root(&value)?; - let state = try_future!(try_future!(beacon_chain.store.get(root)) - .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))); + let state = beacon_chain + .store + .get(root)? + .ok_or_else(|| ApiError::NotFound(format!("No state for root: {:?}", root)))?; (*root, state) } - _ => { - return Box::new(futures::future::err(ApiError::ServerError( - "Unexpected query parameter".into(), - ))) - } + _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), }; let response = StateResponse { @@ -222,29 +226,32 @@ pub fn get_state(req: Request) -> BoxFut { beacon_state: state, }; - success_response(req, &response) + ResponseBuilder::new(&req).body(&response) } /// HTTP handler to return a `BeaconState` root at a given `slot`. /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. -pub fn get_state_root(req: Request) -> BoxFut { - let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); +pub fn get_state_root(req: Request) -> ApiResult { + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; - let slot_string = try_future!(try_future!(UrlQuery::from_request(&req)).only_one("slot")); - let slot = try_future!(parse_slot(&slot_string)); + let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; + let slot = parse_slot(&slot_string)?; - let root = try_future!(state_root_at_slot(&beacon_chain, slot)); + let root = state_root_at_slot(&beacon_chain, slot)?; - success_response(req, &root) + let json: String = serde_json::to_string(&root) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; + + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return the highest finalized slot. pub fn get_current_finalized_checkpoint( req: Request, -) -> BoxFut { - let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); +) -> ApiResult { + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; let checkpoint = beacon_chain .head() @@ -252,14 +259,17 @@ pub fn get_current_finalized_checkpoint( .finalized_checkpoint .clone(); - success_response(req, &checkpoint) + let json: String = serde_json::to_string(&checkpoint) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; + + Ok(success_response_old(Body::from(json))) } /// HTTP handler to return a `BeaconState` at the genesis block. -pub fn get_genesis_state(req: Request) -> BoxFut { - let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); +pub fn get_genesis_state(req: Request) -> ApiResult { + let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; - let (_root, state) = try_future!(state_at_slot(&beacon_chain, Slot::new(0))); + let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; - success_response(req, &state) + ResponseBuilder::new(&req).body(&state) } diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index a3633cfec..41915138f 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -233,7 +233,7 @@ pub fn get_beacon_chain_from_request( let beacon_chain = req .extensions() .get::>>() - .expect("BeaconChain extension must be there, because we put it there."); + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".into()))?; let mut head_state = beacon_chain .state_now() .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))?; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index bc3d3d159..97ed971c2 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -95,6 +95,7 @@ impl Service for ApiService { (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), (&Method::GET, "/network/listen_addresses") => network::get_listen_addresses::(req), + /* // Methods for Beacon Node (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), From c254ac8c2ecd2816959d23a829dc628f4a713a0a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 12 Sep 2019 01:44:45 +1000 Subject: [PATCH 38/47] Separated acquisition of BeaconChain and BeaconState. --- beacon_node/rest_api/src/beacon.rs | 23 +++++++++++------------ beacon_node/rest_api/src/helpers.rs | 19 +++++++++++-------- beacon_node/rest_api/src/metrics.rs | 2 +- beacon_node/rest_api/src/node.rs | 5 +++-- beacon_node/rest_api/src/spec.rs | 2 +- beacon_node/rest_api/src/validator.rs | 20 +++++--------------- 6 files changed, 32 insertions(+), 39 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 66f5e7731..3b9b2a008 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -109,7 +109,7 @@ pub fn get_block(req: Request) -> ApiResult /// HTTP handler to return a `BeaconBlock` root at a given `slot`. pub fn get_block_root(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; let target = parse_slot(&slot_string)?; @@ -126,7 +126,8 @@ pub fn get_block_root(req: Request) -> ApiR /// HTTP handler to return the `Fork` of the current head. pub fn get_fork(req: Request) -> ApiResult { - let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = get_head_state(beacon_chain)?; let json: String = serde_json::to_string(&head_state.fork).map_err(|e| { ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) @@ -140,7 +141,7 @@ pub fn get_fork(req: Request) -> ApiResult /// The `Epoch` parameter can be any epoch number. If it is not specified, /// the current epoch is assumed. pub fn get_validators(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let epoch = match UrlQuery::from_request(&req) { // We have some parameters, so make sure it's the epoch one and parse it @@ -182,7 +183,8 @@ pub struct StateResponse { /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn get_state(req: Request) -> ApiResult { - let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = get_head_state(beacon_chain.clone())?; let (key, value) = match UrlQuery::from_request(&req) { Ok(query) => { @@ -233,7 +235,7 @@ pub fn get_state(req: Request) -> ApiResult /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn get_state_root(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; let slot = parse_slot(&slot_string)?; @@ -250,13 +252,10 @@ pub fn get_state_root(req: Request) -> ApiR pub fn get_current_finalized_checkpoint( req: Request, ) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = get_head_state(beacon_chain)?; - let checkpoint = beacon_chain - .head() - .beacon_state - .finalized_checkpoint - .clone(); + let checkpoint = head_state.finalized_checkpoint.clone(); let json: String = serde_json::to_string(&checkpoint) .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; @@ -266,7 +265,7 @@ pub fn get_current_finalized_checkpoint( /// HTTP handler to return a `BeaconState` at the genesis block. pub fn get_genesis_state(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 76fc78750..4dd8a475d 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -172,21 +172,24 @@ pub fn implementation_pending_response(_req: Request) -> ApiResult { pub fn get_beacon_chain_from_request( req: &Request, -) -> Result<(Arc>, BeaconState), ApiError> { +) -> Result<(Arc>), ApiError> { // Get beacon state let beacon_chain = req .extensions() .get::>>() .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".into()))?; - let mut head_state = beacon_chain - .state_now() - .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))?; - if let Some(s) = head_state.maybe_as_mut_ref() { - s.build_all_caches(&beacon_chain.spec).ok(); - } + Ok(beacon_chain.clone()) +} - Ok((beacon_chain.clone(), head_state.clone())) +pub fn get_head_state( + bc: Arc>, +) -> Result, ApiError> { + let mut head_state = bc.head().beacon_state; + head_state + .build_all_caches(&bc.spec) + .map_err(|e| ApiError::ServerError(format!("Unable to build state cache: {:?}", e)))?; + Ok(head_state) } pub fn get_logger_from_request(req: &Request) -> slog::Logger { diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 62a769de1..01dc4d22d 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -30,7 +30,7 @@ pub fn get_prometheus(req: Request) -> ApiR let mut buffer = vec![]; let encoder = TextEncoder::new(); - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let db_path = req .extensions() .get::() diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index c75d3ba20..4a9f11be0 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -1,4 +1,4 @@ -use crate::helpers::get_beacon_chain_from_request; +use crate::helpers::*; use crate::{success_response, ApiResult}; use beacon_chain::BeaconChainTypes; use hyper::{Body, Request}; @@ -15,7 +15,8 @@ pub fn get_version(_req: Request) -> ApiResult { /// Read the genesis time from the current beacon chain state. pub fn get_genesis_time(req: Request) -> ApiResult { - let (_beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = get_head_state(beacon_chain)?; let gen_time: u64 = head_state.genesis_time; let body = Body::from( serde_json::to_string(&gen_time) diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index ad168faf1..a353b3833 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -9,7 +9,7 @@ use types::EthSpec; /// HTTP handler to return the full spec object. pub fn get_spec(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let json: String = serde_json::to_string(&beacon_chain.spec) .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 49b4c0441..84cb7a308 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -34,7 +34,8 @@ impl ValidatorDuty { pub fn get_validator_duties(req: Request) -> ApiResult { let log = get_logger_from_request(&req); slog::trace!(log, "Validator duties requested of API: {:?}", &req); - let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = get_head_state(beacon_chain.clone())?; slog::trace!(log, "Got head state from request."); // Parse and check query parameters @@ -71,18 +72,6 @@ pub fn get_validator_duties(req: Request) - .collect::, _>>()?; let mut duties: Vec = Vec::new(); - // Update the committee cache - // TODO: Do we need to update the cache on the state, for the epoch which has been specified? - beacon_chain - .state_now() - .map_err(|e| ApiError::ServerError(format!("Unable to get current BeaconState {:?}", e)))? - .maybe_as_mut_ref() - .ok_or(ApiError::ServerError( - "Unable to get mutable BeaconState".into(), - ))? - .build_committee_cache(relative_epoch, &beacon_chain.spec) - .map_err(|e| ApiError::ServerError(format!("Unable to build state caches: {:?}", e)))?; - // Get a list of all validators for this epoch let validator_proposers: Vec = epoch .slot_iter(T::EthSpec::slots_per_epoch()) @@ -157,7 +146,7 @@ pub fn get_validator_duties(req: Request) - /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. pub fn get_new_beacon_block(req: Request) -> ApiResult { - let (beacon_chain, _head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; let query = UrlQuery::from_request(&req)?; let slot = query @@ -197,7 +186,8 @@ pub fn get_new_beacon_block(req: Request) - /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { - let (beacon_chain, head_state) = get_beacon_chain_from_request::(&req)?; + let beacon_chain = get_beacon_chain_from_request::(&req)?; + let head_state = get_head_state(beacon_chain.clone())?; let query = UrlQuery::from_request(&req)?; let val_pk_str = query From cd8f40b4b70d6b52d250e14fda8fcd6063c4f518 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 12 Sep 2019 15:20:31 +1000 Subject: [PATCH 39/47] Getting regular endpoint functions to return futures. - Wrapped endpoint functions in new into_boxfut function - Undid changes to Network API service, now returning ApiResult again. - Cleaning up of functions, and removal of success_response functions in updated endpoints. - A bunch of other clean-ups. --- beacon_node/rest_api/src/beacon.rs | 8 +---- beacon_node/rest_api/src/error.rs | 2 ++ beacon_node/rest_api/src/lib.rs | 37 +++++++++++++------- beacon_node/rest_api/src/network.rs | 39 +++++++++------------- beacon_node/rest_api/src/node.rs | 6 ++-- beacon_node/rest_api/src/validator.rs | 6 ++-- validator_client/src/block_producer/mod.rs | 4 +-- 7 files changed, 50 insertions(+), 52 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 23ef12aa8..d74ab2ed9 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -191,13 +191,7 @@ pub fn get_state(req: Request) -> ApiResult Ok(query) => { // We have *some* parameters, just check them. let query_params = ["root", "slot"]; - match query.first_of(&query_params) { - Ok((k, v)) => (k, v), - Err(e) => { - // Wrong parameters provided, or another error, return the error. - return Err(e); - } - } + query.first_of(&query_params)? } Err(ApiError::InvalidQueryParams(_)) => { // No parameters provided at all, use current slot. diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index 138affae4..82dc73da0 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -1,4 +1,6 @@ use crate::BoxFut; +use futures::future::IntoFuture; +use futures::Future; use hyper::{Body, Method, Request, Response, Server, StatusCode}; use std::error::Error as StdError; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 97ed971c2..3903d0ea1 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -21,6 +21,7 @@ use client_network::NetworkMessage; use client_network::Service as NetworkService; use error::{ApiError, ApiResult}; use eth2_config::Eth2Config; +use futures::future::IntoFuture; use hyper::rt::Future; use hyper::server::conn::AddrStream; use hyper::service::{MakeService, Service}; @@ -33,7 +34,6 @@ use std::sync::Arc; use tokio::runtime::TaskExecutor; use tokio::sync::mpsc; use url_query::UrlQuery; -use futures::future::IntoFuture; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; @@ -51,6 +51,14 @@ pub struct ApiService { eth2_config: Arc, } +fn into_boxfut(item: F) -> BoxFut +where + F: IntoFuture, Error = ApiError>, + F::Future: Send, +{ + Box::new(item.into_future()) +} + impl Service for ApiService { type ReqBody = Body; type ResBody = Body; @@ -82,20 +90,25 @@ impl Service for ApiService { // Route the request to the correct handler. let result = match (req.method(), path.as_ref()) { // Methods for Client - (&Method::GET, "/node/version") => Box::new(node::get_version(req).into_future()), - (&Method::GET, "/node/genesis_time") => Box::new(node::get_genesis_time::(req).into_future()), - (&Method::GET, "/node/syncing") => Box::new(helpers::implementation_pending_response(req).into_future()), - /* + (&Method::GET, "/node/version") => into_boxfut(node::get_version(req)), + (&Method::GET, "/node/genesis_time") => into_boxfut(node::get_genesis_time::(req)), + (&Method::GET, "/node/syncing") => { + into_boxfut(helpers::implementation_pending_response(req)) + } // Methods for Network - (&Method::GET, "/network/enr") => network::get_enr::(req), - (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), - (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), - (&Method::GET, "/network/peers") => network::get_peer_list::(req), - (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), - (&Method::GET, "/network/listen_addresses") => network::get_listen_addresses::(req), - + (&Method::GET, "/network/enr") => into_boxfut(network::get_enr::(req)), + (&Method::GET, "/network/peer_count") => into_boxfut(network::get_peer_count::(req)), + (&Method::GET, "/network/peer_id") => into_boxfut(network::get_peer_id::(req)), + (&Method::GET, "/network/peers") => into_boxfut(network::get_peer_list::(req)), + (&Method::GET, "/network/listen_port") => { + into_boxfut(network::get_listen_port::(req)) + } + (&Method::GET, "/network/listen_addresses") => { + into_boxfut(network::get_listen_addresses::(req)) + } /* + // Methods for Beacon Node (&Method::GET, "/beacon/head") => beacon::get_head::(req), (&Method::GET, "/beacon/block") => beacon::get_block::(req), diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs index e037d43f0..26e5623c2 100644 --- a/beacon_node/rest_api/src/network.rs +++ b/beacon_node/rest_api/src/network.rs @@ -1,5 +1,7 @@ +use crate::error::{ApiError, ApiResult}; use crate::helpers::*; -use crate::{ApiError, BoxFut, NetworkService}; +use crate::response_builder::ResponseBuilder; +use crate::NetworkService; use beacon_chain::BeaconChainTypes; use eth2_libp2p::{Enr, Multiaddr, PeerId}; use hyper::{Body, Request}; @@ -8,81 +10,70 @@ use std::sync::Arc; /// HTTP handler to return the list of libp2p multiaddr the client is listening on. /// /// Returns a list of `Multiaddr`, serialized according to their `serde` impl. -pub fn get_listen_addresses(req: Request) -> BoxFut { +pub fn get_listen_addresses(req: Request) -> ApiResult { let network = req .extensions() .get::>>() .expect("The network service should always be there, we put it there"); let multiaddresses: Vec = network.listen_multiaddrs(); - success_response_json(req, &multiaddresses) + ResponseBuilder::new(&req).body_json(&multiaddresses) } /// HTTP handler to return the network port the client is listening on. /// /// Returns the TCP port number in its plain form (which is also valid JSON serialization) -pub fn get_listen_port(req: Request) -> BoxFut { +pub fn get_listen_port(req: Request) -> ApiResult { let network = req .extensions() .get::>>() .expect("The network service should always be there, we put it there") .clone(); - - success_response(req, &network.listen_port()) + ResponseBuilder::new(&req).body(&network.listen_port()) } /// HTTP handler to return the Discv5 ENR from the client's libp2p service. /// /// ENR is encoded as base64 string. -pub fn get_enr(req: Request) -> BoxFut { +pub fn get_enr(req: Request) -> ApiResult { let network = req .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - - let enr: Enr = network.local_enr(); - success_response_json(req, &enr.to_base64()) + ResponseBuilder::new(&req).body_json(&network.local_enr().to_base64()) } /// HTTP handler to return the `PeerId` from the client's libp2p service. /// /// PeerId is encoded as base58 string. -pub fn get_peer_id(req: Request) -> BoxFut { +pub fn get_peer_id(req: Request) -> ApiResult { let network = req .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - - let peer_id: PeerId = network.local_peer_id(); - - success_response_json(req, &peer_id.to_base58()) + ResponseBuilder::new(&req).body_json(&network.local_peer_id().to_base58()) } /// HTTP handler to return the number of peers connected in the client's libp2p service. -pub fn get_peer_count(req: Request) -> BoxFut { +pub fn get_peer_count(req: Request) -> ApiResult { let network = req .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - - let connected_peers: usize = network.connected_peers(); - - success_response(req, &connected_peers) + ResponseBuilder::new(&req).body(&network.connected_peers()) } /// HTTP handler to return the list of peers connected to the client's libp2p service. /// /// Peers are presented as a list of `PeerId::to_string()`. -pub fn get_peer_list(req: Request) -> BoxFut { +pub fn get_peer_list(req: Request) -> ApiResult { let network = req .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - let connected_peers: Vec = network .connected_peer_set() .iter() .map(PeerId::to_string) .collect(); - - success_response_json(req, &connected_peers) + ResponseBuilder::new(&req).body_json(&connected_peers) } diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index 433aae6cf..5ef35f9a0 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -1,4 +1,5 @@ use crate::helpers::*; +use crate::response_builder::ResponseBuilder; use crate::{ApiResult, BoxFut}; use beacon_chain::BeaconChainTypes; use hyper::{Body, Request}; @@ -6,13 +7,12 @@ use version; /// Read the version string from the current Lighthouse build. pub fn get_version(req: Request) -> ApiResult { - success_response_2_json(req, &version::version()) + ResponseBuilder::new(&req).body_json(&version::version()) } /// Read the genesis time from the current beacon chain state. pub fn get_genesis_time(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; let head_state = get_head_state(beacon_chain)?; - let gen_time: u64 = head_state.genesis_time; - success_response_2(req, &gen_time) + ResponseBuilder::new(&req).body(&head_state.genesis_time) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index dfa35e298..7d0fbdabd 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,6 +1,6 @@ use crate::helpers::*; use crate::response_builder::ResponseBuilder; -use crate::{ApiError, ApiResult, UrlQuery, BoxFut}; +use crate::{ApiError, ApiResult, BoxFut, UrlQuery}; use beacon_chain::{BeaconChainTypes, BlockProcessingOutcome}; use bls::{AggregateSignature, PublicKey, Signature}; use futures::future::Future; @@ -197,7 +197,7 @@ pub fn get_new_beacon_block(req: Request) - pub fn publish_beacon_block(req: Request) -> BoxFut { let _ = try_future!(check_content_type_for_json(&req)); let log = get_logger_from_request(&req); - let (beacon_chain, _head_state) = try_future!(get_beacon_chain_from_request::(&req)); + let beacon_chain = try_future!(get_beacon_chain_from_request::(&req)); // Get the network sending channel from the request, for later transmission let network_chan = req .extensions() @@ -250,8 +250,6 @@ pub fn publish_beacon_block(req: Request) - }).and_then(|_| { response_builder.body_json(&()) })) - - } /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index 0716d740c..bb9c5741d 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -63,12 +63,12 @@ impl<'a, B: BeaconNodeBlock, S: Signer, E: EthSpec> BlockProducer<'a, B, S, E> { pub fn handle_produce_block(&mut self) { match self.produce_block() { Ok(ValidatorEvent::BlockProduced(slot)) => info!( - log, + self.log, "Block produced"; "validator" => format!("{}", self.signer), "slot" => slot, ), - Err(e) => error!(log, "Block production error"; "Error" => format!("{:?}", e)), + Err(e) => error!(self.log, "Block production error"; "Error" => format!("{:?}", e)), Ok(ValidatorEvent::SignerRejection(_slot)) => { error!(self.log, "Block production error"; "Error" => "Signer Could not sign the block".to_string()) } From b0e3ce78855c8823f1929a4205cc2e1577006a86 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 18:58:08 +1000 Subject: [PATCH 40/47] Addressed Paul's comments regarding head_state. --- beacon_node/rest_api/src/beacon.rs | 6 +++--- beacon_node/rest_api/src/helpers.rs | 10 ---------- beacon_node/rest_api/src/metrics.rs | 1 + beacon_node/rest_api/src/node.rs | 9 ++------- beacon_node/rest_api/src/validator.rs | 19 +++++++++++++++---- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 3b9b2a008..c1f49c1fc 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use store::Store; use types::{BeaconBlock, BeaconState, Epoch, EthSpec, Hash256, Slot, Validator}; -#[derive(Serialize)] +#[derive(Serialize, Encode)] pub struct HeadResponse { pub slot: Slot, pub block_root: Hash256, @@ -184,7 +184,7 @@ pub struct StateResponse { /// the current head by skipping slots. pub fn get_state(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = get_head_state(beacon_chain.clone())?; + let head_state = beacon_chain.head().beacon_state; let (key, value) = match UrlQuery::from_request(&req) { Ok(query) => { @@ -253,7 +253,7 @@ pub fn get_current_finalized_checkpoint( req: Request, ) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = get_head_state(beacon_chain)?; + let head_state = beacon_chain.head().beacon_state; let checkpoint = head_state.finalized_checkpoint.clone(); diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 4dd8a475d..c58bf1038 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -182,16 +182,6 @@ pub fn get_beacon_chain_from_request( Ok(beacon_chain.clone()) } -pub fn get_head_state( - bc: Arc>, -) -> Result, ApiError> { - let mut head_state = bc.head().beacon_state; - head_state - .build_all_caches(&bc.spec) - .map_err(|e| ApiError::ServerError(format!("Unable to build state cache: {:?}", e)))?; - Ok(head_state) -} - pub fn get_logger_from_request(req: &Request) -> slog::Logger { let log = req .extensions() diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 01dc4d22d..8ce9cb6af 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -1,3 +1,4 @@ +use crate::response_builder::ResponseBuilder; use crate::{helpers::*, success_response, ApiError, ApiResult, DBPath}; use beacon_chain::BeaconChainTypes; use http::HeaderValue; diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index 4a9f11be0..9048cb0f7 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -1,4 +1,5 @@ use crate::helpers::*; +use crate::response_builder::ResponseBuilder; use crate::{success_response, ApiResult}; use beacon_chain::BeaconChainTypes; use hyper::{Body, Request}; @@ -16,11 +17,5 @@ pub fn get_version(_req: Request) -> ApiResult { /// Read the genesis time from the current beacon chain state. pub fn get_genesis_time(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = get_head_state(beacon_chain)?; - let gen_time: u64 = head_state.genesis_time; - let body = Body::from( - serde_json::to_string(&gen_time) - .expect("Genesis should time always have a valid JSON serialization."), - ); - Ok(success_response(body)) + ResponseBuilder::new(&req).body(&beacon_chain.head().beacon_state.genesis_time) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 84cb7a308..e3075e623 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,4 +1,5 @@ use super::{success_response, ApiResult}; +use crate::response_builder::ResponseBuilder; use crate::{helpers::*, ApiError, UrlQuery}; use beacon_chain::BeaconChainTypes; use bls::{AggregateSignature, PublicKey, Signature}; @@ -35,7 +36,7 @@ pub fn get_validator_duties(req: Request) - let log = get_logger_from_request(&req); slog::trace!(log, "Validator duties requested of API: {:?}", &req); let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = get_head_state(beacon_chain.clone())?; + let mut head_state = beacon_chain.head().beacon_state; slog::trace!(log, "Got head state from request."); // Parse and check query parameters @@ -72,6 +73,10 @@ pub fn get_validator_duties(req: Request) - .collect::, _>>()?; let mut duties: Vec = Vec::new(); + // Build cache for the requested epoch + head_state + .build_committee_cache(relative_epoch, &beacon_chain.spec) + .map_err(|e| ApiError::ServerError(format!("Unable to build committee cache: {:?}", e)))?; // Get a list of all validators for this epoch let validator_proposers: Vec = epoch .slot_iter(T::EthSpec::slots_per_epoch()) @@ -79,7 +84,6 @@ pub fn get_validator_duties(req: Request) - head_state .get_beacon_proposer_index(slot, relative_epoch, &beacon_chain.spec) .map_err(|e| { - // TODO: why are we getting an uninitialized state error here??? ApiError::ServerError(format!( "Unable to get proposer index for validator: {:?}", e @@ -181,13 +185,13 @@ pub fn get_new_beacon_block(req: Request) - serde_json::to_string(&new_block) .expect("We should always be able to serialize a new block that we produced."), ); - Ok(success_response(body)) + ResponseBuilder::new(&req).body(&new_block) } /// HTTP Handler to produce a new Attestation from the current state, ready to be signed by a validator. pub fn get_new_attestation(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = get_head_state(beacon_chain.clone())?; + let mut head_state = beacon_chain.head().beacon_state; let query = UrlQuery::from_request(&req)?; let val_pk_str = query @@ -195,6 +199,9 @@ pub fn get_new_attestation(req: Request) -> .map(|(_key, value)| value)?; let val_pk = parse_pubkey(val_pk_str.as_str())?; + head_state + .update_pubkey_cache() + .map_err(|e| ApiError::ServerError(format!("Unable to build pubkey cache: {:?}", e)))?; // Get the validator index from the supplied public key // If it does not exist in the index, we cannot continue. let val_index = head_state @@ -206,6 +213,10 @@ pub fn get_new_attestation(req: Request) -> "The provided validator public key does not correspond to a validator index.".into(), ))?; + // Build cache for the requested epoch + head_state + .build_committee_cache(RelativeEpoch::Current, &beacon_chain.spec) + .map_err(|e| ApiError::ServerError(format!("Unable to build committee cache: {:?}", e)))?; // Get the duties of the validator, to make sure they match up. // If they don't have duties this epoch, then return an error let val_duty = head_state From 0b2f3bbf00d74cc59b17ae84fb5aee3514ec7b3d Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 18:59:01 +1000 Subject: [PATCH 41/47] Updated another function for head_state (missing from previous commit). --- beacon_node/rest_api/src/beacon.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index c1f49c1fc..8f4c730f9 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -127,13 +127,7 @@ pub fn get_block_root(req: Request) -> ApiR /// HTTP handler to return the `Fork` of the current head. pub fn get_fork(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - let head_state = get_head_state(beacon_chain)?; - - let json: String = serde_json::to_string(&head_state.fork).map_err(|e| { - ApiError::ServerError(format!("Unable to serialize BeaconState::Fork: {:?}", e)) - })?; - - Ok(success_response(Body::from(json))) + ResponseBuilder::new(&req).body(&beacon_chain.head().beacon_state) } /// HTTP handler to return the set of validators for an `Epoch` From 91f5f17566596e1c87c2042f1a3fec0a60a9fb92 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 19:10:11 +1000 Subject: [PATCH 42/47] Removing 'success_respons' in favour of 'ResponseBuilder' --- beacon_node/rest_api/src/beacon.rs | 20 ++++---------------- beacon_node/rest_api/src/metrics.rs | 2 +- beacon_node/rest_api/src/validator.rs | 16 ++-------------- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 302061395..7828898e8 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -55,10 +55,7 @@ pub fn get_head(req: Request) -> ApiResult previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root, }; - let json: String = serde_json::to_string(&head) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize HeadResponse: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + ResponseBuilder::new(&req).body(&head) } #[derive(Serialize, Encode)] @@ -119,10 +116,7 @@ pub fn get_block_root(req: Request) -> ApiR ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) })?; - let json: String = serde_json::to_string(&root) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + ResponseBuilder::new(&req).body(&root) } /// HTTP handler to return the `Fork` of the current head. @@ -231,10 +225,7 @@ pub fn get_state_root(req: Request) -> ApiR let root = state_root_at_slot(&beacon_chain, slot)?; - let json: String = serde_json::to_string(&root) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + ResponseBuilder::new(&req).body(&root) } /// HTTP handler to return the highest finalized slot. @@ -246,10 +237,7 @@ pub fn get_current_finalized_checkpoint( let checkpoint = head_state.finalized_checkpoint.clone(); - let json: String = serde_json::to_string(&checkpoint) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + ResponseBuilder::new(&req).body(&checkpoint) } /// HTTP handler to return a `BeaconState` at the genesis block. diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 09d361b8a..d50e4a98c 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -1,5 +1,5 @@ use crate::response_builder::ResponseBuilder; -use crate::{helpers::*, success_response, ApiError, ApiResult, DBPath}; +use crate::{helpers::*, ApiError, ApiResult, DBPath}; use beacon_chain::BeaconChainTypes; use http::HeaderValue; use hyper::{Body, Request}; diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index c665a0b1f..05ae6b8c9 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -149,11 +149,7 @@ pub fn get_validator_duties(req: Request) - duties.append(&mut vec![duty]); } - let body = Body::from( - serde_json::to_string(&duties) - .expect("We should always be able to serialize the duties we created."), - ); - Ok(success_response_old(body)) + ResponseBuilder::new(&req).body_json(&duties) } /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. @@ -189,10 +185,6 @@ pub fn get_new_beacon_block(req: Request) - )) })?; - let body = Body::from( - serde_json::to_string(&new_block) - .expect("We should always be able to serialize a new block that we produced."), - ); ResponseBuilder::new(&req).body(&new_block) } @@ -359,9 +351,5 @@ pub fn get_new_attestation(req: Request) -> signature: AggregateSignature::new(), }; - let body = Body::from( - serde_json::to_string(&attestation) - .expect("We should always be able to serialize a new attestation that we produced."), - ); - Ok(success_response_old(body)) + ResponseBuilder::new(&req).body(&attestation) } From 006350c0cdaf3032404c8c07bb299cef22bb4ceb Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 19:14:09 +1000 Subject: [PATCH 43/47] Fixed small bug with get_fork function --- beacon_node/rest_api/src/beacon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 7828898e8..fef3cbdf1 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -122,7 +122,7 @@ pub fn get_block_root(req: Request) -> ApiR /// HTTP handler to return the `Fork` of the current head. pub fn get_fork(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - ResponseBuilder::new(&req).body(&beacon_chain.head().beacon_state) + ResponseBuilder::new(&req).body(&beacon_chain.head().beacon_state.fork) } /// HTTP handler to return the set of validators for an `Epoch` From 1dd86baf1abe0f15893124a9be9cc714841d7834 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 19:38:40 +1000 Subject: [PATCH 44/47] Cleaning up the rest of the API functions. - Removed all unused imports - Fixed random compiler errors - Removed all of the 'sucess_response' helpers. - Enabled all of the API endpoints again, wrapping in 'into_boxfut' - Tidied up /metrics endpoint - Added a 'body_text' part to ResponseBuilder, mainly for the Prometheus /metrics endpoint - Cleaned up the unnecessary helpers::* imports, to be more explicit. --- beacon_node/rest_api/src/beacon.rs | 2 +- beacon_node/rest_api/src/error.rs | 6 +- beacon_node/rest_api/src/helpers.rs | 38 +---------- beacon_node/rest_api/src/lib.rs | 71 +++++++++++--------- beacon_node/rest_api/src/metrics.rs | 16 ++--- beacon_node/rest_api/src/network.rs | 5 +- beacon_node/rest_api/src/node.rs | 4 +- beacon_node/rest_api/src/response_builder.rs | 8 +++ beacon_node/rest_api/src/spec.rs | 21 ++---- beacon_node/rest_api/src/validator.rs | 7 +- 10 files changed, 70 insertions(+), 108 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index fef3cbdf1..159337de4 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -1,6 +1,6 @@ use crate::helpers::*; use crate::response_builder::ResponseBuilder; -use crate::{ApiError, ApiResult, BoxFut, NetworkService, UrlQuery}; +use crate::{ApiError, ApiResult, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use hyper::{Body, Request}; use serde::Serialize; diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index 82dc73da0..26cf7ba1f 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -1,7 +1,5 @@ use crate::BoxFut; -use futures::future::IntoFuture; -use futures::Future; -use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use hyper::{Body, Response, StatusCode}; use std::error::Error as StdError; #[derive(PartialEq, Debug, Clone)] @@ -71,7 +69,7 @@ impl From for ApiError { } impl StdError for ApiError { - fn cause(&self) -> Option<&StdError> { + fn cause(&self) -> Option<&dyn StdError> { None } } diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 99a8f1dd4..3385e2017 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -1,53 +1,19 @@ -use crate::response_builder::ResponseBuilder; -use crate::{ApiError, ApiResult, BoxFut}; +use crate::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use eth2_libp2p::{PubsubMessage, Topic}; use eth2_libp2p::{BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX}; use hex; use http::header; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Body, Request}; use network::NetworkMessage; use parking_lot::RwLock; -use serde::Serialize; use ssz::Encode; use std::sync::Arc; use store::{iter::AncestorIter, Store}; use tokio::sync::mpsc; use types::{BeaconBlock, BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; -pub fn success_response(req: Request, item: &T) -> BoxFut { - Box::new(match ResponseBuilder::new(&req).body(item) { - Ok(resp) => futures::future::ok(resp), - Err(e) => futures::future::err(e), - }) -} - -pub fn success_response_2(req: Request, item: &T) -> ApiResult { - ResponseBuilder::new(&req).body(item) -} -pub fn success_response_2_json(req: Request, item: &T) -> ApiResult { - ResponseBuilder::new(&req).body_json(item) -} - -pub fn success_response_json(req: Request, item: &T) -> BoxFut { - if let Err(e) = check_content_type_for_json(&req) { - return Box::new(futures::future::err(e)); - } - Box::new(match ResponseBuilder::new(&req).body_json(item) { - Ok(resp) => futures::future::ok(resp), - Err(e) => futures::future::err(e), - }) -} - -pub fn success_response_old(body: Body) -> Response { - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(body) - .expect("We should always be able to make response from the success body.") -} - /// Parse a slot from a `0x` preixed string. /// /// E.g., `"1234"` diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 3903d0ea1..35678391a 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -23,9 +23,8 @@ use error::{ApiError, ApiResult}; use eth2_config::Eth2Config; use futures::future::IntoFuture; use hyper::rt::Future; -use hyper::server::conn::AddrStream; -use hyper::service::{MakeService, Service}; -use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use hyper::service::Service; +use hyper::{Body, Method, Request, Response, Server}; use parking_lot::RwLock; use slog::{info, o, warn}; use std::ops::Deref; @@ -37,8 +36,6 @@ use url_query::UrlQuery; pub use beacon::{BlockResponse, HeadResponse, StateResponse}; pub use config::Config as ApiConfig; -use eth2_libp2p::rpc::RequestId; -use serde::export::PhantomData; type BoxFut = Box, Error = ApiError> + Send>; @@ -107,58 +104,66 @@ impl Service for ApiService { (&Method::GET, "/network/listen_addresses") => { into_boxfut(network::get_listen_addresses::(req)) } - /* // Methods for Beacon Node - (&Method::GET, "/beacon/head") => beacon::get_head::(req), - (&Method::GET, "/beacon/block") => beacon::get_block::(req), - (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), - (&Method::GET, "/beacon/blocks") => helpers::implementation_pending_response(req), - (&Method::GET, "/beacon/fork") => beacon::get_fork::(req), - (&Method::GET, "/beacon/attestations") => helpers::implementation_pending_response(req), + (&Method::GET, "/beacon/head") => into_boxfut(beacon::get_head::(req)), + (&Method::GET, "/beacon/block") => into_boxfut(beacon::get_block::(req)), + (&Method::GET, "/beacon/block_root") => into_boxfut(beacon::get_block_root::(req)), + (&Method::GET, "/beacon/blocks") => { + into_boxfut(helpers::implementation_pending_response(req)) + } + (&Method::GET, "/beacon/fork") => into_boxfut(beacon::get_fork::(req)), + (&Method::GET, "/beacon/attestations") => { + into_boxfut(helpers::implementation_pending_response(req)) + } (&Method::GET, "/beacon/attestations/pending") => { - helpers::implementation_pending_response(req) + into_boxfut(helpers::implementation_pending_response(req)) } - (&Method::GET, "/beacon/validators") => beacon::get_validators::(req), + (&Method::GET, "/beacon/validators") => into_boxfut(beacon::get_validators::(req)), (&Method::GET, "/beacon/validators/indicies") => { - helpers::implementation_pending_response(req) + into_boxfut(helpers::implementation_pending_response(req)) } (&Method::GET, "/beacon/validators/pubkeys") => { - helpers::implementation_pending_response(req) + into_boxfut(helpers::implementation_pending_response(req)) } // Methods for Validator - (&Method::GET, "/beacon/validator/duties") => validator::get_validator_duties::(req), - (&Method::GET, "/beacon/validator/block") => validator::get_new_beacon_block::(req), - */ + (&Method::GET, "/beacon/validator/duties") => { + into_boxfut(validator::get_validator_duties::(req)) + } + (&Method::GET, "/beacon/validator/block") => { + into_boxfut(validator::get_new_beacon_block::(req)) + } (&Method::POST, "/beacon/validator/block") => validator::publish_beacon_block::(req), - /* (&Method::GET, "/beacon/validator/attestation") => { - validator::get_new_attestation::(req) + into_boxfut(validator::get_new_attestation::(req)) } (&Method::POST, "/beacon/validator/attestation") => { - helpers::implementation_pending_response(req) + into_boxfut(helpers::implementation_pending_response(req)) } - (&Method::GET, "/beacon/state") => beacon::get_state::(req), - (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + (&Method::GET, "/beacon/state") => into_boxfut(beacon::get_state::(req)), + (&Method::GET, "/beacon/state_root") => into_boxfut(beacon::get_state_root::(req)), (&Method::GET, "/beacon/state/current_finalized_checkpoint") => { - beacon::get_current_finalized_checkpoint::(req) + into_boxfut(beacon::get_current_finalized_checkpoint::(req)) + } + (&Method::GET, "/beacon/state/genesis") => { + into_boxfut(beacon::get_genesis_state::(req)) } - (&Method::GET, "/beacon/state/genesis") => beacon::get_genesis_state::(req), //TODO: Add aggreggate/filtered state lookups here, e.g. /beacon/validators/balances // Methods for bootstrap and checking configuration - (&Method::GET, "/spec") => spec::get_spec::(req), - (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), - (&Method::GET, "/spec/deposit_contract") => { - helpers::implementation_pending_response(req) + (&Method::GET, "/spec") => into_boxfut(spec::get_spec::(req)), + (&Method::GET, "/spec/slots_per_epoch") => { + into_boxfut(spec::get_slots_per_epoch::(req)) } - (&Method::GET, "/spec/eth2_config") => spec::get_eth2_config::(req), + (&Method::GET, "/spec/deposit_contract") => { + into_boxfut(helpers::implementation_pending_response(req)) + } + (&Method::GET, "/spec/eth2_config") => into_boxfut(spec::get_eth2_config::(req)), - (&Method::GET, "/metrics") => metrics::get_prometheus::(req), - */ + (&Method::GET, "/metrics") => into_boxfut(metrics::get_prometheus::(req)), _ => Box::new(futures::future::err(ApiError::NotFound( "Request path and/or method not found.".to_owned(), ))), diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index d50e4a98c..33437a534 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -1,7 +1,7 @@ +use crate::helpers::get_beacon_chain_from_request; use crate::response_builder::ResponseBuilder; -use crate::{helpers::*, ApiError, ApiResult, DBPath}; +use crate::{ApiError, ApiResult, DBPath}; use beacon_chain::BeaconChainTypes; -use http::HeaderValue; use hyper::{Body, Request}; use prometheus::{Encoder, TextEncoder}; @@ -62,14 +62,6 @@ pub fn get_prometheus(req: Request) -> ApiR .unwrap(); String::from_utf8(buffer) - .map(|string| { - let mut response = success_response_old(Body::from(string)); - // Need to change the header to text/plain for prometheus - response.headers_mut().insert( - "content-type", - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response - }) - .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e))) + .map(|string| ResponseBuilder::new(&req).body_text(string)) + .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))? } diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs index 26e5623c2..afbddde84 100644 --- a/beacon_node/rest_api/src/network.rs +++ b/beacon_node/rest_api/src/network.rs @@ -1,9 +1,8 @@ -use crate::error::{ApiError, ApiResult}; -use crate::helpers::*; +use crate::error::ApiResult; use crate::response_builder::ResponseBuilder; use crate::NetworkService; use beacon_chain::BeaconChainTypes; -use eth2_libp2p::{Enr, Multiaddr, PeerId}; +use eth2_libp2p::{Multiaddr, PeerId}; use hyper::{Body, Request}; use std::sync::Arc; diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index 3eb8e5594..cb1b28df7 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -1,6 +1,6 @@ -use crate::helpers::*; +use crate::helpers::get_beacon_chain_from_request; use crate::response_builder::ResponseBuilder; -use crate::{ApiResult, BoxFut}; +use crate::ApiResult; use beacon_chain::BeaconChainTypes; use hyper::{Body, Request}; use version; diff --git a/beacon_node/rest_api/src/response_builder.rs b/beacon_node/rest_api/src/response_builder.rs index b48b9e41a..31f717697 100644 --- a/beacon_node/rest_api/src/response_builder.rs +++ b/beacon_node/rest_api/src/response_builder.rs @@ -61,4 +61,12 @@ impl ResponseBuilder { })?)) .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) } + + pub fn body_text(self, text: String) -> ApiResult { + Response::builder() + .status(StatusCode::OK) + .header("content-type", "text/plain; charset=utf-8") + .body(Body::from(text)) + .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) + } } diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index 5ab518636..55a139f16 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -1,5 +1,6 @@ use super::ApiResult; -use crate::helpers::*; +use crate::helpers::get_beacon_chain_from_request; +use crate::response_builder::ResponseBuilder; use crate::ApiError; use beacon_chain::BeaconChainTypes; use eth2_config::Eth2Config; @@ -10,11 +11,7 @@ use types::EthSpec; /// HTTP handler to return the full spec object. pub fn get_spec(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - - let json: String = serde_json::to_string(&beacon_chain.spec) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + ResponseBuilder::new(&req).body_json(&beacon_chain.spec) } /// HTTP handler to return the full Eth2Config object. @@ -24,16 +21,10 @@ pub fn get_eth2_config(req: Request) -> Api .get::>() .ok_or_else(|| ApiError::ServerError("Eth2Config extension missing".to_string()))?; - let json: String = serde_json::to_string(eth2_config.as_ref()) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize Eth2Config: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) + ResponseBuilder::new(&req).body_json(eth2_config.as_ref()) } /// HTTP handler to return the full spec object. -pub fn get_slots_per_epoch(_req: Request) -> ApiResult { - let json: String = serde_json::to_string(&T::EthSpec::slots_per_epoch()) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize epoch: {:?}", e)))?; - - Ok(success_response_old(Body::from(json))) +pub fn get_slots_per_epoch(req: Request) -> ApiResult { + ResponseBuilder::new(&req).body(&T::EthSpec::slots_per_epoch()) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 05ae6b8c9..d9d55bad4 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,11 +1,14 @@ -use crate::helpers::*; +use crate::helpers::{ + check_content_type_for_json, get_beacon_chain_from_request, get_logger_from_request, + parse_pubkey, publish_beacon_block_to_network, +}; use crate::response_builder::ResponseBuilder; use crate::{ApiError, ApiResult, BoxFut, UrlQuery}; use beacon_chain::{BeaconChainTypes, BlockProcessingOutcome}; use bls::{AggregateSignature, PublicKey, Signature}; use futures::future::Future; use futures::stream::Stream; -use hyper::{Body, Error, Request}; +use hyper::{Body, Request}; use network::NetworkMessage; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; From f48311900ead67638ccddf09cb4b198c15589f7f Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 20:42:56 +1000 Subject: [PATCH 45/47] Restructured response builder to give YAML or JSON when SSZ is not available, not just JSON. --- beacon_node/rest_api/src/beacon.rs | 18 ++--- beacon_node/rest_api/src/error.rs | 2 + beacon_node/rest_api/src/metrics.rs | 2 +- beacon_node/rest_api/src/network.rs | 12 ++-- beacon_node/rest_api/src/node.rs | 4 +- beacon_node/rest_api/src/response_builder.rs | 72 +++++++++++++------- beacon_node/rest_api/src/spec.rs | 6 +- beacon_node/rest_api/src/validator.rs | 8 +-- 8 files changed, 76 insertions(+), 48 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 159337de4..13f52dc9a 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -55,7 +55,7 @@ pub fn get_head(req: Request) -> ApiResult previous_justified_block_root: chain_head.beacon_state.previous_justified_checkpoint.root, }; - ResponseBuilder::new(&req).body(&head) + ResponseBuilder::new(&req)?.body(&head) } #[derive(Serialize, Encode)] @@ -102,7 +102,7 @@ pub fn get_block(req: Request) -> ApiResult beacon_block: block, }; - ResponseBuilder::new(&req).body(&response) + ResponseBuilder::new(&req)?.body(&response) } /// HTTP handler to return a `BeaconBlock` root at a given `slot`. @@ -116,13 +116,13 @@ pub fn get_block_root(req: Request) -> ApiR ApiError::NotFound(format!("Unable to find BeaconBlock for slot {:?}", target)) })?; - ResponseBuilder::new(&req).body(&root) + ResponseBuilder::new(&req)?.body(&root) } /// HTTP handler to return the `Fork` of the current head. pub fn get_fork(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - ResponseBuilder::new(&req).body(&beacon_chain.head().beacon_state.fork) + ResponseBuilder::new(&req)?.body(&beacon_chain.head().beacon_state.fork) } /// HTTP handler to return the set of validators for an `Epoch` @@ -157,7 +157,7 @@ pub fn get_validators(req: Request) -> ApiR .cloned() .collect(); - ResponseBuilder::new(&req).body(&active_vals) + ResponseBuilder::new(&req)?.body(&active_vals) } #[derive(Serialize, Encode)] @@ -210,7 +210,7 @@ pub fn get_state(req: Request) -> ApiResult beacon_state: state, }; - ResponseBuilder::new(&req).body(&response) + ResponseBuilder::new(&req)?.body(&response) } /// HTTP handler to return a `BeaconState` root at a given `slot`. @@ -225,7 +225,7 @@ pub fn get_state_root(req: Request) -> ApiR let root = state_root_at_slot(&beacon_chain, slot)?; - ResponseBuilder::new(&req).body(&root) + ResponseBuilder::new(&req)?.body(&root) } /// HTTP handler to return the highest finalized slot. @@ -237,7 +237,7 @@ pub fn get_current_finalized_checkpoint( let checkpoint = head_state.finalized_checkpoint.clone(); - ResponseBuilder::new(&req).body(&checkpoint) + ResponseBuilder::new(&req)?.body(&checkpoint) } /// HTTP handler to return a `BeaconState` at the genesis block. @@ -246,5 +246,5 @@ pub fn get_genesis_state(req: Request) -> A let (_root, state) = state_at_slot(&beacon_chain, Slot::new(0))?; - ResponseBuilder::new(&req).body(&state) + ResponseBuilder::new(&req)?.body(&state) } diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index 26cf7ba1f..e52ba4af6 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -9,6 +9,7 @@ pub enum ApiError { NotImplemented(String), InvalidQueryParams(String), NotFound(String), + UnsupportedType(String), ImATeapot(String), // Just in case. } @@ -22,6 +23,7 @@ impl ApiError { ApiError::NotImplemented(desc) => (StatusCode::NOT_IMPLEMENTED, desc), ApiError::InvalidQueryParams(desc) => (StatusCode::BAD_REQUEST, desc), ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc), + ApiError::UnsupportedType(desc) => (StatusCode::UNSUPPORTED_MEDIA_TYPE, desc), ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc), } } diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 33437a534..e9d98434e 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -62,6 +62,6 @@ pub fn get_prometheus(req: Request) -> ApiR .unwrap(); String::from_utf8(buffer) - .map(|string| ResponseBuilder::new(&req).body_text(string)) + .map(|string| ResponseBuilder::new(&req)?.body_text(string)) .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))? } diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs index afbddde84..f193ef8ea 100644 --- a/beacon_node/rest_api/src/network.rs +++ b/beacon_node/rest_api/src/network.rs @@ -15,7 +15,7 @@ pub fn get_listen_addresses(req: Request) -> ApiResul .get::>>() .expect("The network service should always be there, we put it there"); let multiaddresses: Vec = network.listen_multiaddrs(); - ResponseBuilder::new(&req).body_json(&multiaddresses) + ResponseBuilder::new(&req)?.body_no_ssz(&multiaddresses) } /// HTTP handler to return the network port the client is listening on. @@ -27,7 +27,7 @@ pub fn get_listen_port(req: Request) -> ApiResult { .get::>>() .expect("The network service should always be there, we put it there") .clone(); - ResponseBuilder::new(&req).body(&network.listen_port()) + ResponseBuilder::new(&req)?.body(&network.listen_port()) } /// HTTP handler to return the Discv5 ENR from the client's libp2p service. @@ -38,7 +38,7 @@ pub fn get_enr(req: Request) -> ApiResult { .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - ResponseBuilder::new(&req).body_json(&network.local_enr().to_base64()) + ResponseBuilder::new(&req)?.body_no_ssz(&network.local_enr().to_base64()) } /// HTTP handler to return the `PeerId` from the client's libp2p service. @@ -49,7 +49,7 @@ pub fn get_peer_id(req: Request) -> ApiResult { .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - ResponseBuilder::new(&req).body_json(&network.local_peer_id().to_base58()) + ResponseBuilder::new(&req)?.body_no_ssz(&network.local_peer_id().to_base58()) } /// HTTP handler to return the number of peers connected in the client's libp2p service. @@ -58,7 +58,7 @@ pub fn get_peer_count(req: Request) -> ApiResult { .extensions() .get::>>() .expect("The network service should always be there, we put it there"); - ResponseBuilder::new(&req).body(&network.connected_peers()) + ResponseBuilder::new(&req)?.body(&network.connected_peers()) } /// HTTP handler to return the list of peers connected to the client's libp2p service. @@ -74,5 +74,5 @@ pub fn get_peer_list(req: Request) -> ApiResult { .iter() .map(PeerId::to_string) .collect(); - ResponseBuilder::new(&req).body_json(&connected_peers) + ResponseBuilder::new(&req)?.body_no_ssz(&connected_peers) } diff --git a/beacon_node/rest_api/src/node.rs b/beacon_node/rest_api/src/node.rs index cb1b28df7..882edcfd5 100644 --- a/beacon_node/rest_api/src/node.rs +++ b/beacon_node/rest_api/src/node.rs @@ -7,11 +7,11 @@ use version; /// Read the version string from the current Lighthouse build. pub fn get_version(req: Request) -> ApiResult { - ResponseBuilder::new(&req).body_json(&version::version()) + ResponseBuilder::new(&req)?.body_no_ssz(&version::version()) } /// Read the genesis time from the current beacon chain state. pub fn get_genesis_time(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - ResponseBuilder::new(&req).body(&beacon_chain.head().beacon_state.genesis_time) + ResponseBuilder::new(&req)?.body(&beacon_chain.head().beacon_state.genesis_time) } diff --git a/beacon_node/rest_api/src/response_builder.rs b/beacon_node/rest_api/src/response_builder.rs index 31f717697..793360cd2 100644 --- a/beacon_node/rest_api/src/response_builder.rs +++ b/beacon_node/rest_api/src/response_builder.rs @@ -8,6 +8,7 @@ pub enum Encoding { JSON, SSZ, YAML, + TEXT, } pub struct ResponseBuilder { @@ -15,22 +16,55 @@ pub struct ResponseBuilder { } impl ResponseBuilder { - pub fn new(req: &Request) -> Self { - let encoding = match req.headers().get(header::CONTENT_TYPE) { - Some(h) if h == "application/ssz" => Encoding::SSZ, - Some(h) if h == "application/yaml" => Encoding::YAML, + pub fn new(req: &Request) -> Result { + let content_header = req + .headers() + .get(header::CONTENT_TYPE) + .map_or(Ok(""), |h| h.to_str()) + .map_err(|e| { + ApiError::InvalidQueryParams(format!( + "The content-type header contains invalid characters: {:?}", + e + )) + }) + .map(|h| String::from(h))?; + + let encoding = match content_header { + ref h if h.starts_with("application/ssz") => Encoding::SSZ, + ref h if h.starts_with("application/yaml") => Encoding::YAML, + ref h if h.starts_with("text/plain") => Encoding::TEXT, _ => Encoding::JSON, }; - - Self { encoding } + Ok(Self { encoding }) } pub fn body(self, item: &T) -> ApiResult { + match self.encoding { + Encoding::SSZ => Response::builder() + .status(StatusCode::OK) + .header("content-type", "application/ssz") + .body(Body::from(item.as_ssz_bytes())) + .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))), + _ => self.body_no_ssz(item), + } + } + + pub fn body_no_ssz(self, item: &T) -> ApiResult { let (body, content_type) = match self.encoding { - Encoding::JSON => { - return self.body_json(item); + Encoding::JSON => ( + Body::from(serde_json::to_string(&item).map_err(|e| { + ApiError::ServerError(format!( + "Unable to serialize response body as JSON: {:?}", + e + )) + })?), + "application/json", + ), + Encoding::SSZ => { + return Err(ApiError::UnsupportedType( + "Response cannot be encoded as SSZ.".into(), + )); } - Encoding::SSZ => (Body::from(item.as_ssz_bytes()), "application/ssz"), Encoding::YAML => ( Body::from(serde_yaml::to_string(&item).map_err(|e| { ApiError::ServerError(format!( @@ -38,8 +72,13 @@ impl ResponseBuilder { e )) })?), - "application/ssz", + "application/yaml", ), + Encoding::TEXT => { + return Err(ApiError::UnsupportedType( + "Response cannot be encoded as plain text.".into(), + )); + } }; Response::builder() @@ -49,19 +88,6 @@ impl ResponseBuilder { .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) } - pub fn body_json(self, item: &T) -> ApiResult { - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(serde_json::to_string(&item).map_err(|e| { - ApiError::ServerError(format!( - "Unable to serialize response body as JSON: {:?}", - e - )) - })?)) - .map_err(|e| ApiError::ServerError(format!("Failed to build response: {:?}", e))) - } - pub fn body_text(self, text: String) -> ApiResult { Response::builder() .status(StatusCode::OK) diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs index 55a139f16..083ff5ad4 100644 --- a/beacon_node/rest_api/src/spec.rs +++ b/beacon_node/rest_api/src/spec.rs @@ -11,7 +11,7 @@ use types::EthSpec; /// HTTP handler to return the full spec object. pub fn get_spec(req: Request) -> ApiResult { let beacon_chain = get_beacon_chain_from_request::(&req)?; - ResponseBuilder::new(&req).body_json(&beacon_chain.spec) + ResponseBuilder::new(&req)?.body_no_ssz(&beacon_chain.spec) } /// HTTP handler to return the full Eth2Config object. @@ -21,10 +21,10 @@ pub fn get_eth2_config(req: Request) -> Api .get::>() .ok_or_else(|| ApiError::ServerError("Eth2Config extension missing".to_string()))?; - ResponseBuilder::new(&req).body_json(eth2_config.as_ref()) + ResponseBuilder::new(&req)?.body_no_ssz(eth2_config.as_ref()) } /// HTTP handler to return the full spec object. pub fn get_slots_per_epoch(req: Request) -> ApiResult { - ResponseBuilder::new(&req).body(&T::EthSpec::slots_per_epoch()) + ResponseBuilder::new(&req)?.body(&T::EthSpec::slots_per_epoch()) } diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index d9d55bad4..b79466b4d 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -152,7 +152,7 @@ pub fn get_validator_duties(req: Request) - duties.append(&mut vec![duty]); } - ResponseBuilder::new(&req).body_json(&duties) + ResponseBuilder::new(&req)?.body_no_ssz(&duties) } /// HTTP Handler to produce a new BeaconBlock from the current state, ready to be signed by a validator. @@ -188,7 +188,7 @@ pub fn get_new_beacon_block(req: Request) - )) })?; - ResponseBuilder::new(&req).body(&new_block) + ResponseBuilder::new(&req)?.body(&new_block) } /// HTTP Handler to publish a BeaconBlock, which has been signed by a validator. @@ -246,7 +246,7 @@ pub fn publish_beacon_block(req: Request) - } } }).and_then(|_| { - response_builder.body_json(&()) + response_builder?.body_no_ssz(&()) })) } @@ -354,5 +354,5 @@ pub fn get_new_attestation(req: Request) -> signature: AggregateSignature::new(), }; - ResponseBuilder::new(&req).body(&attestation) + ResponseBuilder::new(&req)?.body(&attestation) } From d3ce182ddc80417fa091d3d8818c6a098cd00f15 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 20:52:12 +1000 Subject: [PATCH 46/47] Renamed 'InvalidQueryParams' to 'BadRequest', since it is a more general error that is returned in a number of cases. --- beacon_node/rest_api/src/beacon.rs | 7 ++--- beacon_node/rest_api/src/error.rs | 4 +-- beacon_node/rest_api/src/helpers.rs | 16 +++++----- beacon_node/rest_api/src/response_builder.rs | 7 +++-- beacon_node/rest_api/src/url_query.rs | 8 ++--- beacon_node/rest_api/src/validator.rs | 33 +++++++++----------- 6 files changed, 34 insertions(+), 41 deletions(-) diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index 13f52dc9a..c1a9da6ee 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -139,10 +139,7 @@ pub fn get_validators(req: Request) -> ApiR .parse::() .map(Epoch::from) .map_err(|e| { - ApiError::InvalidQueryParams(format!( - "Invalid epoch parameter, must be a u64. {:?}", - e - )) + ApiError::BadRequest(format!("Invalid epoch parameter, must be a u64. {:?}", e)) })?, // In this case, our url query did not contain any parameters, so we take the default Err(_) => beacon_chain.epoch().map_err(|e| { @@ -181,7 +178,7 @@ pub fn get_state(req: Request) -> ApiResult let query_params = ["root", "slot"]; query.first_of(&query_params)? } - Err(ApiError::InvalidQueryParams(_)) => { + Err(ApiError::BadRequest(_)) => { // No parameters provided at all, use current slot. (String::from("slot"), head_state.slot.to_string()) } diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index e52ba4af6..70384dce9 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -7,7 +7,7 @@ pub enum ApiError { MethodNotAllowed(String), ServerError(String), NotImplemented(String), - InvalidQueryParams(String), + BadRequest(String), NotFound(String), UnsupportedType(String), ImATeapot(String), // Just in case. @@ -21,7 +21,7 @@ impl ApiError { ApiError::MethodNotAllowed(desc) => (StatusCode::METHOD_NOT_ALLOWED, desc), ApiError::ServerError(desc) => (StatusCode::INTERNAL_SERVER_ERROR, desc), ApiError::NotImplemented(desc) => (StatusCode::NOT_IMPLEMENTED, desc), - ApiError::InvalidQueryParams(desc) => (StatusCode::BAD_REQUEST, desc), + ApiError::BadRequest(desc) => (StatusCode::BAD_REQUEST, desc), ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc), ApiError::UnsupportedType(desc) => (StatusCode::UNSUPPORTED_MEDIA_TYPE, desc), ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc), diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 3385e2017..3f76f4e25 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -21,7 +21,7 @@ pub fn parse_slot(string: &str) -> Result { string .parse::() .map(Slot::from) - .map_err(|e| ApiError::InvalidQueryParams(format!("Unable to parse slot: {:?}", e))) + .map_err(|e| ApiError::BadRequest(format!("Unable to parse slot: {:?}", e))) } /// Checks the provided request to ensure that the `content-type` header. @@ -31,7 +31,7 @@ pub fn parse_slot(string: &str) -> Result { pub fn check_content_type_for_json(req: &Request) -> Result<(), ApiError> { match req.headers().get(header::CONTENT_TYPE) { Some(h) if h == "application/json" => Ok(()), - Some(h) => Err(ApiError::InvalidQueryParams(format!( + Some(h) => Err(ApiError::BadRequest(format!( "The provided content-type {:?} is not available, this endpoint only supports json.", h ))), @@ -49,9 +49,9 @@ pub fn parse_root(string: &str) -> Result { let trimmed = string.trim_start_matches(PREFIX); trimmed .parse() - .map_err(|e| ApiError::InvalidQueryParams(format!("Unable to parse root: {:?}", e))) + .map_err(|e| ApiError::BadRequest(format!("Unable to parse root: {:?}", e))) } else { - Err(ApiError::InvalidQueryParams( + Err(ApiError::BadRequest( "Root must have a '0x' prefix".to_string(), )) } @@ -62,13 +62,13 @@ pub fn parse_pubkey(string: &str) -> Result { const PREFIX: &str = "0x"; if string.starts_with(PREFIX) { let pubkey_bytes = hex::decode(string.trim_start_matches(PREFIX)) - .map_err(|e| ApiError::InvalidQueryParams(format!("Invalid hex string: {:?}", e)))?; + .map_err(|e| ApiError::BadRequest(format!("Invalid hex string: {:?}", e)))?; let pubkey = PublicKey::from_bytes(pubkey_bytes.as_slice()).map_err(|e| { - ApiError::InvalidQueryParams(format!("Unable to deserialize public key: {:?}.", e)) + ApiError::BadRequest(format!("Unable to deserialize public key: {:?}.", e)) })?; return Ok(pubkey); } else { - return Err(ApiError::InvalidQueryParams( + return Err(ApiError::BadRequest( "Public key must have a '0x' prefix".to_string(), )); } @@ -145,7 +145,7 @@ pub fn state_root_at_slot( // // We could actually speculate about future state roots by skipping slots, however that's // likely to cause confusion for API users. - Err(ApiError::InvalidQueryParams(format!( + Err(ApiError::BadRequest(format!( "Requested slot {} is past the current slot {}", slot, current_slot ))) diff --git a/beacon_node/rest_api/src/response_builder.rs b/beacon_node/rest_api/src/response_builder.rs index 793360cd2..d5b530f8a 100644 --- a/beacon_node/rest_api/src/response_builder.rs +++ b/beacon_node/rest_api/src/response_builder.rs @@ -17,22 +17,23 @@ pub struct ResponseBuilder { impl ResponseBuilder { pub fn new(req: &Request) -> Result { - let content_header = req + let content_header: String = req .headers() .get(header::CONTENT_TYPE) .map_or(Ok(""), |h| h.to_str()) .map_err(|e| { - ApiError::InvalidQueryParams(format!( + ApiError::BadRequest(format!( "The content-type header contains invalid characters: {:?}", e )) }) .map(|h| String::from(h))?; + // JSON is our default encoding, unless something else is requested. let encoding = match content_header { ref h if h.starts_with("application/ssz") => Encoding::SSZ, ref h if h.starts_with("application/yaml") => Encoding::YAML, - ref h if h.starts_with("text/plain") => Encoding::TEXT, + ref h if h.starts_with("text/") => Encoding::TEXT, _ => Encoding::JSON, }; Ok(Self { encoding }) diff --git a/beacon_node/rest_api/src/url_query.rs b/beacon_node/rest_api/src/url_query.rs index 3802ff831..f0c587a32 100644 --- a/beacon_node/rest_api/src/url_query.rs +++ b/beacon_node/rest_api/src/url_query.rs @@ -12,7 +12,7 @@ impl<'a> UrlQuery<'a> { /// Returns `Err` if `req` does not contain any query parameters. pub fn from_request(req: &'a Request) -> Result { let query_str = req.uri().query().ok_or_else(|| { - ApiError::InvalidQueryParams( + ApiError::BadRequest( "URL query must be valid and contain at least one key.".to_string(), ) })?; @@ -28,7 +28,7 @@ impl<'a> UrlQuery<'a> { .find(|(key, _value)| keys.contains(&&**key)) .map(|(key, value)| (key.into_owned(), value.into_owned())) .ok_or_else(|| { - ApiError::InvalidQueryParams(format!( + ApiError::BadRequest(format!( "URL query must contain at least one of the following keys: {:?}", keys )) @@ -48,13 +48,13 @@ impl<'a> UrlQuery<'a> { if first_key == key { Ok(first_value.to_string()) } else { - Err(ApiError::InvalidQueryParams(format!( + Err(ApiError::BadRequest(format!( "Only the {} query parameter is supported", key ))) } } else { - Err(ApiError::InvalidQueryParams(format!( + Err(ApiError::BadRequest(format!( "Only one query parameter is allowed, {} supplied", queries.len() ))) diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index b79466b4d..53d9e8b8b 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -58,10 +58,7 @@ pub fn get_validator_duties(req: Request) - slog::trace!(log, "Requested epoch {:?}", v); Epoch::new(v.parse::().map_err(|e| { slog::info!(log, "Invalid epoch {:?}", e); - ApiError::InvalidQueryParams(format!( - "Invalid epoch parameter, must be a u64. {:?}", - e - )) + ApiError::BadRequest(format!("Invalid epoch parameter, must be a u64. {:?}", e)) })?) } Err(_) => { @@ -72,7 +69,7 @@ pub fn get_validator_duties(req: Request) - }; let relative_epoch = RelativeEpoch::from_epoch(current_epoch, epoch).map_err(|e| { slog::info!(log, "Requested epoch out of range."); - ApiError::InvalidQueryParams(format!( + ApiError::BadRequest(format!( "Cannot get RelativeEpoch, epoch out of range: {:?}", e )) @@ -166,17 +163,17 @@ pub fn get_new_beacon_block(req: Request) - .parse::() .map(Slot::from) .map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + ApiError::BadRequest(format!("Invalid slot parameter, must be a u64. {:?}", e)) })?; let randao_bytes = query .first_of(&["randao_reveal"]) .map(|(_key, value)| value) .map(hex::decode)? .map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid hex string for randao_reveal: {:?}", e)) + ApiError::BadRequest(format!("Invalid hex string for randao_reveal: {:?}", e)) })?; let randao_reveal = Signature::from_bytes(randao_bytes.as_slice()).map_err(|e| { - ApiError::InvalidQueryParams(format!("randao_reveal is not a valid signature: {:?}", e)) + ApiError::BadRequest(format!("randao_reveal is not a valid signature: {:?}", e)) })?; let (new_block, _state) = beacon_chain @@ -216,7 +213,7 @@ pub fn publish_beacon_block(req: Request) - .map(|chunk| chunk.iter().cloned().collect::>()) .and_then(|chunks| { serde_json::from_slice(&chunks.as_slice()).map_err(|e| { - ApiError::InvalidQueryParams(format!( + ApiError::BadRequest(format!( "Unable to deserialize JSON into a BeaconBlock: {:?}", e )) @@ -233,7 +230,7 @@ pub fn publish_beacon_block(req: Request) - Ok(outcome) => { warn!(log, "Block could not be processed, but is being sent to the network anyway."; "block_slot" => slot, "outcome" => format!("{:?}", outcome)); //TODO need to send to network and return http 202 - Err(ApiError::InvalidQueryParams(format!( + Err(ApiError::BadRequest(format!( "The BeaconBlock could not be processed: {:?}", outcome ))) @@ -271,7 +268,7 @@ pub fn get_new_attestation(req: Request) -> .map_err(|e| { ApiError::ServerError(format!("Unable to read validator index cache. {:?}", e)) })? - .ok_or(ApiError::InvalidQueryParams( + .ok_or(ApiError::BadRequest( "The provided validator public key does not correspond to a validator index.".into(), ))?; @@ -289,14 +286,14 @@ pub fn get_new_attestation(req: Request) -> e )) })? - .ok_or(ApiError::InvalidQueryParams("No validator duties could be found for the requested validator. Cannot provide valid attestation.".into()))?; + .ok_or(ApiError::BadRequest("No validator duties could be found for the requested validator. Cannot provide valid attestation.".into()))?; // Check that we are requesting an attestation during the slot where it is relevant. let present_slot = beacon_chain.slot().map_err(|e| ApiError::ServerError( format!("Beacon node is unable to determine present slot, either the state isn't generated or the chain hasn't begun. {:?}", e) ))?; if val_duty.slot != present_slot { - return Err(ApiError::InvalidQueryParams(format!("Validator is only able to request an attestation during the slot they are allocated. Current slot: {:?}, allocated slot: {:?}", head_state.slot, val_duty.slot))); + return Err(ApiError::BadRequest(format!("Validator is only able to request an attestation during the slot they are allocated. Current slot: {:?}, allocated slot: {:?}", head_state.slot, val_duty.slot))); } // Parse the POC bit and insert it into the aggregation bits @@ -305,7 +302,7 @@ pub fn get_new_attestation(req: Request) -> .map(|(_key, value)| value)? .parse::() .map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + ApiError::BadRequest(format!("Invalid slot parameter, must be a u64. {:?}", e)) })?; let mut aggregation_bits = BitList::with_capacity(val_duty.committee_len) @@ -327,20 +324,18 @@ pub fn get_new_attestation(req: Request) -> .parse::() .map(Slot::from) .map_err(|e| { - ApiError::InvalidQueryParams(format!("Invalid slot parameter, must be a u64. {:?}", e)) + ApiError::BadRequest(format!("Invalid slot parameter, must be a u64. {:?}", e)) })?; let current_slot = beacon_chain.head().beacon_state.slot.as_u64(); if requested_slot != current_slot { - return Err(ApiError::InvalidQueryParams(format!("Attestation data can only be requested for the current slot ({:?}), not your requested slot ({:?})", current_slot, requested_slot))); + return Err(ApiError::BadRequest(format!("Attestation data can only be requested for the current slot ({:?}), not your requested slot ({:?})", current_slot, requested_slot))); } let shard = query .first_of(&["shard"]) .map(|(_key, value)| value)? .parse::() - .map_err(|e| { - ApiError::InvalidQueryParams(format!("Shard is not a valid u64 value: {:?}", e)) - })?; + .map_err(|e| ApiError::BadRequest(format!("Shard is not a valid u64 value: {:?}", e)))?; let attestation_data = beacon_chain .produce_attestation_data(shard, current_slot.into()) From 23ce271b5fb06527c5b2f6e90c3d245ca1f75bdd Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 13 Sep 2019 21:22:32 +1000 Subject: [PATCH 47/47] Return HTTP 202 to indicate processing error. - A processing error of a validator's block or attestation should not prevent publishing. Now a 202 error is returned, to indicate that it has not been processed, but has still been published. - Added a publish_attestation function to the API, handling POST requests for /beacon/validator/attestation. --- beacon_node/rest_api/src/error.rs | 6 ++- beacon_node/rest_api/src/helpers.rs | 34 +++++++++++-- beacon_node/rest_api/src/lib.rs | 3 +- beacon_node/rest_api/src/validator.rs | 72 ++++++++++++++++++++++++--- 4 files changed, 102 insertions(+), 13 deletions(-) diff --git a/beacon_node/rest_api/src/error.rs b/beacon_node/rest_api/src/error.rs index 70384dce9..9f815a7d3 100644 --- a/beacon_node/rest_api/src/error.rs +++ b/beacon_node/rest_api/src/error.rs @@ -10,7 +10,8 @@ pub enum ApiError { BadRequest(String), NotFound(String), UnsupportedType(String), - ImATeapot(String), // Just in case. + ImATeapot(String), // Just in case. + ProcessingError(String), // A 202 error, for when a block/attestation cannot be processed, but still transmitted. } pub type ApiResult = Result, ApiError>; @@ -25,6 +26,7 @@ impl ApiError { ApiError::NotFound(desc) => (StatusCode::NOT_FOUND, desc), ApiError::UnsupportedType(desc) => (StatusCode::UNSUPPORTED_MEDIA_TYPE, desc), ApiError::ImATeapot(desc) => (StatusCode::IM_A_TEAPOT, desc), + ApiError::ProcessingError(desc) => (StatusCode::ACCEPTED, desc), } } } @@ -34,7 +36,7 @@ impl Into> for ApiError { let status_code = self.status_code(); Response::builder() .status(status_code.0) - .header("content-type", "text/plain") + .header("content-type", "text/plain; charset=utf-8") .body(Body::from(status_code.1)) .expect("Response should always be created.") } diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 3f76f4e25..a711246b0 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -2,7 +2,9 @@ use crate::{ApiError, ApiResult}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use eth2_libp2p::{PubsubMessage, Topic}; -use eth2_libp2p::{BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX}; +use eth2_libp2p::{ + BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC, TOPIC_ENCODING_POSTFIX, TOPIC_PREFIX, +}; use hex; use http::header; use hyper::{Body, Request}; @@ -12,7 +14,7 @@ use ssz::Encode; use std::sync::Arc; use store::{iter::AncestorIter, Store}; use tokio::sync::mpsc; -use types::{BeaconBlock, BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; +use types::{Attestation, BeaconBlock, BeaconState, EthSpec, Hash256, RelativeEpoch, Slot}; /// Parse a slot from a `0x` preixed string. /// @@ -227,7 +229,7 @@ pub fn publish_beacon_block_to_network( // Publish the block to the p2p network via gossipsub. if let Err(e) = chan.write().try_send(NetworkMessage::Publish { topics: vec![topic], - message: message, + message, }) { return Err(ApiError::ServerError(format!( "Unable to send new block to network: {:?}", @@ -238,6 +240,32 @@ pub fn publish_beacon_block_to_network( Ok(()) } +pub fn publish_attestation_to_network( + chan: Arc>>, + attestation: Attestation, +) -> Result<(), ApiError> { + // create the network topic to send on + let topic_string = format!( + "/{}/{}/{}", + TOPIC_PREFIX, BEACON_ATTESTATION_TOPIC, TOPIC_ENCODING_POSTFIX + ); + let topic = Topic::new(topic_string); + let message = PubsubMessage::Attestation(attestation.as_ssz_bytes()); + + // Publish the attestation to the p2p network via gossipsub. + if let Err(e) = chan.write().try_send(NetworkMessage::Publish { + topics: vec![topic], + message, + }) { + return Err(ApiError::ServerError(format!( + "Unable to send new attestation to network: {:?}", + e + ))); + } + + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 35678391a..133fc3a26 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -140,7 +140,7 @@ impl Service for ApiService { into_boxfut(validator::get_new_attestation::(req)) } (&Method::POST, "/beacon/validator/attestation") => { - into_boxfut(helpers::implementation_pending_response(req)) + validator::publish_attestation::(req) } (&Method::GET, "/beacon/state") => into_boxfut(beacon::get_state::(req)), @@ -164,6 +164,7 @@ impl Service for ApiService { (&Method::GET, "/spec/eth2_config") => into_boxfut(spec::get_eth2_config::(req)), (&Method::GET, "/metrics") => into_boxfut(metrics::get_prometheus::(req)), + _ => Box::new(futures::future::err(ApiError::NotFound( "Request path and/or method not found.".to_owned(), ))), diff --git a/beacon_node/rest_api/src/validator.rs b/beacon_node/rest_api/src/validator.rs index 53d9e8b8b..60c0eed06 100644 --- a/beacon_node/rest_api/src/validator.rs +++ b/beacon_node/rest_api/src/validator.rs @@ -1,10 +1,10 @@ use crate::helpers::{ check_content_type_for_json, get_beacon_chain_from_request, get_logger_from_request, - parse_pubkey, publish_beacon_block_to_network, + parse_pubkey, publish_attestation_to_network, publish_beacon_block_to_network, }; use crate::response_builder::ResponseBuilder; use crate::{ApiError, ApiResult, BoxFut, UrlQuery}; -use beacon_chain::{BeaconChainTypes, BlockProcessingOutcome}; +use beacon_chain::{AttestationProcessingOutcome, BeaconChainTypes, BlockProcessingOutcome}; use bls::{AggregateSignature, PublicKey, Signature}; use futures::future::Future; use futures::stream::Stream; @@ -228,16 +228,16 @@ pub fn publish_beacon_block(req: Request) - publish_beacon_block_to_network::(network_chan, block) } Ok(outcome) => { - warn!(log, "Block could not be processed, but is being sent to the network anyway."; "block_slot" => slot, "outcome" => format!("{:?}", outcome)); - //TODO need to send to network and return http 202 - Err(ApiError::BadRequest(format!( - "The BeaconBlock could not be processed: {:?}", + warn!(log, "BeaconBlock could not be processed, but is being sent to the network anyway."; "outcome" => format!("{:?}", outcome)); + publish_beacon_block_to_network::(network_chan, block)?; + Err(ApiError::ProcessingError(format!( + "The BeaconBlock could not be processed, but has still been published: {:?}", outcome ))) } Err(e) => { Err(ApiError::ServerError(format!( - "Unable to process block: {:?}", + "Error while processing block: {:?}", e ))) } @@ -351,3 +351,61 @@ pub fn get_new_attestation(req: Request) -> ResponseBuilder::new(&req)?.body(&attestation) } + +/// HTTP Handler to publish an Attestation, which has been signed by a validator. +pub fn publish_attestation(req: Request) -> BoxFut { + let _ = try_future!(check_content_type_for_json(&req)); + let log = get_logger_from_request(&req); + let beacon_chain = try_future!(get_beacon_chain_from_request::(&req)); + // Get the network sending channel from the request, for later transmission + let network_chan = req + .extensions() + .get::>>>() + .expect("Should always get the network channel from the request, since we put it in there.") + .clone(); + + let response_builder = ResponseBuilder::new(&req); + + let body = req.into_body(); + trace!( + log, + "Got the request body, now going to parse it into an attesation." + ); + Box::new(body + .concat2() + .map_err(|e| ApiError::ServerError(format!("Unable to get request body: {:?}",e))) + .map(|chunk| chunk.iter().cloned().collect::>()) + .and_then(|chunks| { + serde_json::from_slice(&chunks.as_slice()).map_err(|e| { + ApiError::BadRequest(format!( + "Unable to deserialize JSON into a BeaconBlock: {:?}", + e + )) + }) + }) + .and_then(move |attestation: Attestation| { + match beacon_chain.process_attestation(attestation.clone()) { + Ok(AttestationProcessingOutcome::Processed) => { + // Block was processed, publish via gossipsub + info!(log, "Processed valid attestation from API, transmitting to network."); + publish_attestation_to_network::(network_chan, attestation) + } + Ok(outcome) => { + warn!(log, "Attestation could not be processed, but is being sent to the network anyway."; "outcome" => format!("{:?}", outcome)); + publish_attestation_to_network::(network_chan, attestation)?; + Err(ApiError::ProcessingError(format!( + "The Attestation could not be processed, but has still been published: {:?}", + outcome + ))) + } + Err(e) => { + Err(ApiError::ServerError(format!( + "Error while processing attestation: {:?}", + e + ))) + } + } + }).and_then(|_| { + response_builder?.body_no_ssz(&()) + })) +}