diff --git a/src/app/fdctl/run/tiles/fd_bank.c b/src/app/fdctl/run/tiles/fd_bank.c index 987c93d1fe..3971f06874 100644 --- a/src/app/fdctl/run/tiles/fd_bank.c +++ b/src/app/fdctl/run/tiles/fd_bank.c @@ -42,6 +42,7 @@ typedef struct { ulong txn_load_address_lookup_tables[ 6 ]; ulong transaction_result[ 40 ]; ulong processing_failed; + ulong precompile_verify_failure; ulong fee_only; ulong exec_failed; ulong success; @@ -68,14 +69,18 @@ scratch_footprint( fd_topo_tile_t const * tile ) { static inline void metrics_write( fd_bank_ctx_t * ctx ) { FD_MCNT_ENUM_COPY( BANK, SLOT_ACQUIRE, ctx->metrics.slot_acquire ); + FD_MCNT_ENUM_COPY( BANK, TRANSACTION_RESULT, ctx->metrics.transaction_result ); +} +static inline void +metrics_write_fixed_interval( fd_bank_ctx_t * ctx ) { FD_MCNT_ENUM_COPY( BANK, TRANSACTION_LOAD_ADDRESS_TABLES, ctx->metrics.txn_load_address_lookup_tables ); - FD_MCNT_ENUM_COPY( BANK, TRANSACTION_RESULT, ctx->metrics.transaction_result ); - FD_MCNT_SET( BANK, PROCESSING_FAILED, ctx->metrics.processing_failed ); - FD_MCNT_SET( BANK, FEE_ONLY_TRANSACTIONS, ctx->metrics.fee_only ); - FD_MCNT_SET( BANK, EXECUTED_FAILED_TRANSACTIONS, ctx->metrics.exec_failed ); - FD_MCNT_SET( BANK, SUCCESSFUL_TRANSACTIONS, ctx->metrics.success ); + FD_MCNT_SET( BANK, PROCESSING_FAILED, ctx->metrics.processing_failed ); + FD_MCNT_SET( BANK, PRECOMPILE_VERIFY_FAILURE, ctx->metrics.precompile_verify_failure ); + FD_MCNT_SET( BANK, FEE_ONLY_TRANSACTIONS, ctx->metrics.fee_only ); + FD_MCNT_SET( BANK, EXECUTED_FAILED_TRANSACTIONS, ctx->metrics.exec_failed ); + FD_MCNT_SET( BANK, SUCCESSFUL_TRANSACTIONS, ctx->metrics.success ); } static int @@ -172,7 +177,7 @@ handle_microblock( fd_bank_ctx_t * ctx, int precompile_result = fd_ext_bank_verify_precompiles( ctx->_bank, abi_txn ); if( FD_UNLIKELY( precompile_result ) ) { - FD_MCNT_INC( BANK, PRECOMPILE_VERIFY_FAILURE, 1 ); + ctx->metrics.precompile_verify_failure++; continue; } @@ -299,12 +304,6 @@ handle_microblock( fd_bank_ctx_t * ctx, FD_STATIC_ASSERT( MAX_MICROBLOCK_SZ-(MAX_TXN_PER_MICROBLOCK*sizeof(fd_txn_p_t))>=sizeof(fd_microblock_trailer_t), poh_shred_mtu ); FD_STATIC_ASSERT( MAX_MICROBLOCK_SZ-(MAX_TXN_PER_MICROBLOCK*sizeof(fd_txn_p_t))>=sizeof(fd_microblock_bank_trailer_t), poh_shred_mtu ); - /* We have a race window with the GUI, where if the slot is ending it - will snap these metrics to draw the waterfall, but see them outdated - because housekeeping hasn't run. For now just update them here, but - PoH should eventually flush the pipeline before ending the slot. */ - metrics_write( ctx ); - ulong bank_sig = fd_disco_bank_sig( slot, ctx->_microblock_idx ); /* We always need to publish, even if there are no successfully executed @@ -351,7 +350,7 @@ handle_bundle( fd_bank_ctx_t * ctx, int precompile_result = fd_ext_bank_verify_precompiles( ctx->_bank, abi_txn ); if( FD_UNLIKELY( precompile_result ) ) { execution_success = 0; - FD_MCNT_INC( BANK, PRECOMPILE_VERIFY_FAILURE, 1 ); + ctx->metrics.precompile_verify_failure++; continue; } @@ -452,8 +451,6 @@ handle_bundle( fd_bank_ctx_t * ctx, fd_stem_publish( stem, 0UL, bank_sig, ctx->out_chunk, new_sz, 0UL, 0UL, tspub ); ctx->out_chunk = fd_dcache_compact_next( ctx->out_chunk, new_sz, ctx->out_chunk0, ctx->out_wmark ); } - - metrics_write( ctx ); } static inline void @@ -520,10 +517,11 @@ unprivileged_init( fd_topo_t * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_bank_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_bank_ctx_t) -#define STEM_CALLBACK_METRICS_WRITE metrics_write -#define STEM_CALLBACK_BEFORE_FRAG before_frag -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_BEFORE_FRAG before_frag +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../../../disco/stem/fd_stem.c" diff --git a/src/app/fdctl/run/tiles/fd_bundle.c b/src/app/fdctl/run/tiles/fd_bundle.c index 63da7fbc4c..3ddf4331a1 100644 --- a/src/app/fdctl/run/tiles/fd_bundle.c +++ b/src/app/fdctl/run/tiles/fd_bundle.c @@ -86,11 +86,15 @@ during_housekeeping( fd_bundle_ctx_t * ctx ) { static inline void metrics_write( fd_bundle_ctx_t * ctx ) { - FD_MCNT_SET( BUNDLE, TRANSACTION_RECEIVED, ctx->metrics.txn_received ); FD_MCNT_SET( BUNDLE, BUNDLE_RECEIVED, ctx->metrics.bundle_received ); FD_MCNT_SET( BUNDLE, PACKET_RECEIVED, ctx->metrics.packet_received ); } +static inline void +metrics_write_fixed_interval( fd_bundle_ctx_t * ctx ) { + FD_MCNT_SET( BUNDLE, TRANSACTION_RECEIVED, ctx->metrics.txn_received ); +} + extern void plugin_bundle_poll( void * plugin, int reload_identity, @@ -423,9 +427,10 @@ populate_allowed_fds( fd_topo_t const * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_bundle_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_bundle_ctx_t) -#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping -#define STEM_CALLBACK_METRICS_WRITE metrics_write -#define STEM_CALLBACK_AFTER_CREDIT after_credit +#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_AFTER_CREDIT after_credit #include "../../../../disco/stem/fd_stem.c" diff --git a/src/app/fdctl/run/tiles/fd_dedup.c b/src/app/fdctl/run/tiles/fd_dedup.c index 0ef2e3e9ac..ea98af5719 100644 --- a/src/app/fdctl/run/tiles/fd_dedup.c +++ b/src/app/fdctl/run/tiles/fd_dedup.c @@ -57,6 +57,7 @@ typedef struct { struct { ulong bundle_peer_failure_cnt; ulong dedup_fail_cnt; + ulong gossiped_votes_received; } metrics; } fd_dedup_ctx_t; @@ -75,8 +76,16 @@ scratch_footprint( fd_topo_tile_t const * tile ) { static inline void metrics_write( fd_dedup_ctx_t * ctx ) { + (void)ctx; + return; +} + +static inline void +metrics_write_fixed_interval( fd_dedup_ctx_t * ctx ) { FD_MCNT_SET( DEDUP, TRANSACTION_BUNDLE_PEER_FAILURE, ctx->metrics.bundle_peer_failure_cnt ); - FD_MCNT_SET( DEDUP, TRANSACTION_DEDUP_FAILURE, ctx->metrics.dedup_fail_cnt ); + FD_MCNT_SET( DEDUP, TRANSACTION_DEDUP_FAILURE, ctx->metrics.dedup_fail_cnt ); + FD_MCNT_SET( DEDUP, GOSSIPED_VOTES_RECEIVED, ctx->metrics.gossiped_votes_received ); + } /* during_frag is called between pairs for sequence number checks, as @@ -168,7 +177,7 @@ after_frag( fd_dedup_ctx_t * ctx, txnm->txn_t_sz = (ushort)fd_txn_parse( fd_txn_m_payload( txnm ), txnm->payload_sz, txn, NULL ); if( FD_UNLIKELY( !txnm->txn_t_sz ) ) FD_LOG_ERR(( "fd_txn_parse failed for vote transactions that should have been sigverified" )); - if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL ); + if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) ctx->metrics.gossiped_votes_received++; } int is_dup = 0; @@ -301,9 +310,10 @@ populate_allowed_fds( fd_topo_t const * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_dedup_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_dedup_ctx_t) -#define STEM_CALLBACK_METRICS_WRITE metrics_write -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../../../disco/stem/fd_stem.c" diff --git a/src/app/fdctl/run/tiles/fd_pack.c b/src/app/fdctl/run/tiles/fd_pack.c index 5e4623c4ea..684e2f02d3 100644 --- a/src/app/fdctl/run/tiles/fd_pack.c +++ b/src/app/fdctl/run/tiles/fd_pack.c @@ -192,6 +192,8 @@ typedef struct { increases, we expire old transactions. */ ulong highest_observed_slot; + ulong transaction_expired; + /* microblock_duration_ns, and wait_duration respectively scaled to be in ticks instead of nanoseconds */ ulong microblock_duration_ticks; @@ -203,6 +205,9 @@ typedef struct { full. This is an fd_deque. */ fd_txn_e_t * extra_txn_deq; int insert_to_extra; /* whether the last insert was into pack or the extra deq */ + ulong transaction_dropped_from_extra;, + ulong transaction_inserted_from_extra;, + ulong transaction_inserted_to_extra; #endif fd_pack_in_ctx_t in[ 32 ]; @@ -250,6 +255,8 @@ typedef struct { fd_txn_e_t * const * bundle; /* points to _txn when non-NULL */ } current_bundle[1]; + ulong partial_bundle_dropped_txn_cnt; + block_builder_info_t blk_engine_cfg[1]; struct { @@ -358,9 +365,7 @@ log_end_block_metrics( fd_pack_ctx_t * ctx, static inline void metrics_write( fd_pack_ctx_t * ctx ) { - FD_MCNT_ENUM_COPY( PACK, TRANSACTION_INSERTED, ctx->insert_result ); FD_MCNT_ENUM_COPY( PACK, METRIC_TIMING, ((ulong*)ctx->metric_timing) ); - FD_MCNT_ENUM_COPY( PACK, BUNDLE_CRANK_STATUS, ctx->crank->metrics ); FD_MHIST_COPY( PACK, SCHEDULE_MICROBLOCK_DURATION_SECONDS, ctx->schedule_duration ); FD_MHIST_COPY( PACK, NO_SCHED_MICROBLOCK_DURATION_SECONDS, ctx->no_sched_duration ); FD_MHIST_COPY( PACK, INSERT_TRANSACTION_DURATION_SECONDS, ctx->insert_duration ); @@ -369,6 +374,22 @@ metrics_write( fd_pack_ctx_t * ctx ) { fd_pack_metrics_write( ctx->pack ); } +static inline void +metrics_write_fixed_interval( fd_pack_ctx_t * ctx ) { + FD_MCNT_ENUM_COPY( PACK, TRANSACTION_INSERTED, ctx->insert_result ); + FD_MCNT_SET( PACK, TRANSACTION_EXPIRED, ctx->transaction_expired ); + FD_MCNT_ENUM_COPY( PACK, BUNDLE_CRANK_STATUS, ctx->crank->metrics ); + FD_MCNT_SET( PACK, TRANSACTION_DROPPED_PARTIAL_BUNDLE, ctx->partial_bundle_dropped_txn_cnt ); + +#if FD_PACK_USE_EXTRA_STORAGE + FD_MCNT_SET( PACK, TRANSACTION_DROPPED_FROM_EXTRA, ctx->transaction_dropped_from_extra ); + FD_MCNT_SET( PACK, TRANSACTION_INSERTED_FROM_EXTRA, ctx->transaction_inserted_from_extra ); + FD_MCNT_SET( PACK, TRANSACTION_INSERTED_TO_EXTRA, ctx->transaction_inserted_to_extra ); +#endif + + fd_pack_metrics_fixed_int_write(ctx->pack); +} + static inline void during_housekeeping( fd_pack_ctx_t * ctx ) { ctx->approx_wallclock_ns = fd_log_wallclock(); @@ -428,7 +449,7 @@ insert_from_extra( fd_pack_ctx_t * ctx ) { insert_duration += fd_tickcount(); ctx->insert_result[ result + FD_PACK_INSERT_RETVAL_OFF ]++; fd_histf_sample( ctx->insert_duration, (ulong)insert_duration ); - FD_MCNT_INC( PACK, TRANSACTION_INSERTED_FROM_EXTRA, 1UL ); + ctx->transaction_inserted_from_extra++; return result; } #endif @@ -747,8 +768,7 @@ during_frag( fd_pack_ctx_t * ctx, } ctx->leader_slot = fd_disco_poh_sig_slot( sig ); - ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->leader_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS ); - FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt ); + ctx->transaction_expired += fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->leader_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS ); fd_became_leader_t * became_leader = (fd_became_leader_t *)dcache_entry; ctx->leader_bank = became_leader->bank; @@ -817,8 +837,7 @@ during_frag( fd_pack_ctx_t * ctx, with expired but high-fee-paying transactions. That can only happen if we are getting transactions. */ ctx->highest_observed_slot = sig; - ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->highest_observed_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS ); - FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt ); + ctx->transaction_expired += fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->highest_observed_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS ); } @@ -826,7 +845,7 @@ during_frag( fd_pack_ctx_t * ctx, ctx->is_bundle = 1; if( FD_LIKELY( txnm->block_engine.bundle_id!=ctx->current_bundle->id ) ) { if( FD_UNLIKELY( ctx->current_bundle->bundle ) ) { - FD_MCNT_INC( PACK, TRANSACTION_DROPPED_PARTIAL_BUNDLE, ctx->current_bundle->txn_received ); + ctx->partial_bundle_dropped_txn_cnt += ctx->current_bundle->txn_received; fd_pack_insert_bundle_cancel( ctx->pack, ctx->current_bundle->bundle, ctx->current_bundle->txn_cnt ); } ctx->current_bundle->id = txnm->block_engine.bundle_id; @@ -835,7 +854,7 @@ during_frag( fd_pack_ctx_t * ctx, ctx->current_bundle->txn_received = 0UL; if( FD_UNLIKELY( ctx->current_bundle->txn_cnt==0UL ) ) { - FD_MCNT_INC( PACK, TRANSACTION_DROPPED_PARTIAL_BUNDLE, 1UL ); + ctx->partial_bundle_dropped_txn_cnt++; ctx->current_bundle->id = 0UL; return; } @@ -855,7 +874,7 @@ during_frag( fd_pack_ctx_t * ctx, } else { if( FD_UNLIKELY( extra_txn_deq_full( ctx->extra_txn_deq ) ) ) { extra_txn_deq_remove_head( ctx->extra_txn_deq ); - FD_MCNT_INC( PACK, TRANSACTION_DROPPED_FROM_EXTRA, 1UL ); + ctx->transaction_dropped_from_extra++; } ctx->cur_spot = extra_txn_deq_peek_tail( extra_txn_deq_insert_tail( ctx->extra_txn_deq ) ); /* We want to store the current time in cur_spot so that we can @@ -863,7 +882,7 @@ during_frag( fd_pack_ctx_t * ctx, fields, since those aren't important right now. */ ctx->cur_spot->txnp->blockhash_slot = sig; ctx->insert_to_extra = 1; - FD_MCNT_INC( PACK, TRANSACTION_INSERTED_TO_EXTRA, 1UL ); + ctx->transaction_inserted_to_extra++; } #else ctx->cur_spot = fd_pack_insert_txn_init( ctx->pack ); @@ -1092,9 +1111,12 @@ unprivileged_init( fd_topo_t * topo, ctx->ticks_per_ns = fd_tempo_tick_per_ns( NULL ); ctx->last_successful_insert = 0L; ctx->highest_observed_slot = 0UL; + ctx->transaction_expired = 0UL; ctx->microblock_duration_ticks = (ulong)(fd_tempo_tick_per_ns( NULL )*(double)MICROBLOCK_DURATION_NS + 0.5); #if FD_PACK_USE_EXTRA_STORAGE ctx->insert_to_extra = 0; + ctx->transaction_dropped_from_extra= 0UL; + ctx->transaction_inserted_to_extra = 0UL; #endif ctx->use_consumed_cus = tile->pack.use_consumed_cus; ctx->crank->enabled = tile->pack.bundle.enabled; @@ -1202,12 +1224,13 @@ populate_allowed_fds( fd_topo_t const * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_pack_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_pack_ctx_t) -#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping -#define STEM_CALLBACK_BEFORE_CREDIT before_credit -#define STEM_CALLBACK_AFTER_CREDIT after_credit -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag -#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping +#define STEM_CALLBACK_BEFORE_CREDIT before_credit +#define STEM_CALLBACK_AFTER_CREDIT after_credit +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_METRICS_WRITE metrics_write #include "../../../../disco/stem/fd_stem.c" diff --git a/src/app/fdctl/run/tiles/fd_resolv.c b/src/app/fdctl/run/tiles/fd_resolv.c index c0a5620f5b..8a44b481f3 100644 --- a/src/app/fdctl/run/tiles/fd_resolv.c +++ b/src/app/fdctl/run/tiles/fd_resolv.c @@ -136,6 +136,7 @@ typedef struct { struct { ulong lut[ FD_METRICS_COUNTER_RESOLV_LUT_RESOLVED_CNT ]; + ulong no_bank_drop; ulong blockhash_expired; ulong blockhash_unknown; ulong bundle_peer_failure_cnt; @@ -177,8 +178,9 @@ fd_ext_resolv_tile_cnt( void ) { } static inline void -metrics_write( fd_resolv_ctx_t * ctx ) { +metrics_write_fixed_interval( fd_resolv_ctx_t * ctx ) { FD_MCNT_SET( RESOLV, BLOCKHASH_EXPIRED, ctx->metrics.blockhash_expired ); + FD_MCNT_SET( RESOLV, NO_BANK_DROP, ctx->metrics.no_bank_drop ); FD_MCNT_ENUM_COPY( RESOLV, LUT_RESOLVED, ctx->metrics.lut ); FD_MCNT_ENUM_COPY( RESOLV, STASH_OPERATION, ctx->metrics.stash ); FD_MCNT_SET( RESOLV, TRANSACTION_BUNDLE_PEER_FAILURE, ctx->metrics.bundle_peer_failure_cnt ); @@ -237,7 +239,7 @@ publish_txn( fd_resolv_ctx_t * ctx, if( FD_UNLIKELY( txnt->addr_table_adtl_cnt ) ) { if( FD_UNLIKELY( !ctx->root_bank ) ) { - FD_MCNT_INC( RESOLV, NO_BANK_DROP, 1 ); + ctx->metrics.no_bank_drop++; return 0; } else { int result = fd_bank_abi_resolve_address_lookup_tables( ctx->root_bank, 0, ctx->root_slot, txnt, fd_txn_m_payload( txnm ), fd_txn_m_alut( txnm ) ); @@ -430,7 +432,7 @@ after_frag( fd_resolv_ctx_t * ctx, if( FD_UNLIKELY( txnt->addr_table_adtl_cnt ) ) { if( FD_UNLIKELY( !ctx->root_bank ) ) { - FD_MCNT_INC( RESOLV, NO_BANK_DROP, 1 ); + ctx->metrics.no_bank_drop++; if( FD_UNLIKELY( txnm->block_engine.bundle_id ) ) ctx->bundle_failed = 1; return; } @@ -517,11 +519,11 @@ unprivileged_init( fd_topo_t * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_resolv_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_resolv_ctx_t) -#define STEM_CALLBACK_METRICS_WRITE metrics_write -#define STEM_CALLBACK_AFTER_CREDIT after_credit -#define STEM_CALLBACK_BEFORE_FRAG before_frag -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_AFTER_CREDIT after_credit +#define STEM_CALLBACK_BEFORE_FRAG before_frag +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../../../disco/stem/fd_stem.c" diff --git a/src/app/fdctl/run/tiles/fd_verify.c b/src/app/fdctl/run/tiles/fd_verify.c index bf5b71219d..b5ee950087 100644 --- a/src/app/fdctl/run/tiles/fd_verify.c +++ b/src/app/fdctl/run/tiles/fd_verify.c @@ -30,6 +30,12 @@ scratch_footprint( fd_topo_tile_t const * tile ) { static inline void metrics_write( fd_verify_ctx_t * ctx ) { + (void)ctx; + return; +} + +static inline void +metrics_write_fixed_interval( fd_verify_ctx_t * ctx ) { FD_MCNT_SET( VERIFY, TRANSACTION_BUNDLE_PEER_FAILURE, ctx->metrics.bundle_peer_fail_cnt ); FD_MCNT_SET( VERIFY, TRANSACTION_PARSE_FAILURE, ctx->metrics.parse_fail_cnt ); FD_MCNT_SET( VERIFY, TRANSACTION_DEDUP_FAILURE, ctx->metrics.dedup_fail_cnt ); @@ -245,10 +251,11 @@ populate_allowed_fds( fd_topo_t const * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_verify_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_verify_ctx_t) -#define STEM_CALLBACK_METRICS_WRITE metrics_write -#define STEM_CALLBACK_BEFORE_FRAG before_frag -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_BEFORE_FRAG before_frag +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../../../disco/stem/fd_stem.c" diff --git a/src/disco/gui/sankey_debug.py b/src/disco/gui/sankey_debug.py index d0497ab748..82e38643e2 100644 --- a/src/disco/gui/sankey_debug.py +++ b/src/disco/gui/sankey_debug.py @@ -105,6 +105,7 @@ def print_sankey(summed: Dict[Tuple[str, Optional[str]], int]): gossip: {in_gossip:10,} udp: {in_udp:10,} quic: {in_quic:10,} +pack_cranked: {pack_cranked:10,} ---------------------------------------- IN TOTAL: {in_block_engine + in_gossip + in_udp + in_quic:10,} @@ -133,7 +134,6 @@ def print_sankey(summed: Dict[Tuple[str, Optional[str]], int]): RECONCILE RESOLV IN: {recon_resolv_in:10,} RECONCILE RESOLV OUT: {recon_resolv_out:10,} -pack_cranked: {pack_cranked:10,} pack_retained: {pack_retained:10,} pack_leader_slot: {pack_leader_slot:10,} pack_expired: {pack_expired:10,} diff --git a/src/disco/pack/fd_pack.c b/src/disco/pack/fd_pack.c index a14c563fa3..53f0ca6d87 100644 --- a/src/disco/pack/fd_pack.c +++ b/src/disco/pack/fd_pack.c @@ -1583,7 +1583,6 @@ fd_pack_metrics_write( fd_pack_t const * pack ) { ulong pending_votes = treap_ele_cnt( pack->pending_votes ); ulong pending_bundle = treap_ele_cnt( pack->pending_bundles ); ulong conflicting = pack->pending_txn_cnt - pending_votes - pending_bundle - treap_ele_cnt( pack->pending ); - FD_MGAUGE_SET( PACK, AVAILABLE_TRANSACTIONS_ALL, pack->pending_txn_cnt ); FD_MGAUGE_SET( PACK, AVAILABLE_TRANSACTIONS_REGULAR, pending_regular ); FD_MGAUGE_SET( PACK, AVAILABLE_TRANSACTIONS_VOTES, pending_votes ); FD_MGAUGE_SET( PACK, AVAILABLE_TRANSACTIONS_CONFLICTING, conflicting ); @@ -1591,6 +1590,11 @@ fd_pack_metrics_write( fd_pack_t const * pack ) { FD_MGAUGE_SET( PACK, SMALLEST_PENDING_TRANSACTION, pack->pending_smallest->cus ); } +void +fd_pack_metrics_fixed_int_write( fd_pack_t const * pack ) { + FD_MGAUGE_SET(PACK, AVAILABLE_TRANSACTIONS_ALL, pack->pending_txn_cnt ); +} + typedef struct { ushort clear_rw_bit; ushort clear_w_bit; diff --git a/src/disco/pack/fd_pack.h b/src/disco/pack/fd_pack.h index 19de9ecce0..4dba713b7b 100644 --- a/src/disco/pack/fd_pack.h +++ b/src/disco/pack/fd_pack.h @@ -617,6 +617,11 @@ void fd_pack_metrics_write( fd_pack_t const * pack ); +/* fd_pack_metrics_fixed_int_write writes period metric values to the metrics + system which are intended to be written . pack must be a valid local join. */ +void +fd_pack_metrics_fixed_int_write( fd_pack_t const * pack ); + /* fd_pack_leave leaves a local join of a pack object. Returns pack. */ void * fd_pack_leave( fd_pack_t * pack ); /* fd_pack_delete unformats a memory region used to store a pack object diff --git a/src/disco/quic/fd_quic_tile.c b/src/disco/quic/fd_quic_tile.c index 24c8ac959d..e1aab2dbc6 100644 --- a/src/disco/quic/fd_quic_tile.c +++ b/src/disco/quic/fd_quic_tile.c @@ -116,22 +116,12 @@ before_credit( fd_quic_ctx_t * ctx, static inline void metrics_write( fd_quic_ctx_t * ctx ) { - FD_MCNT_SET ( QUIC, TXNS_RECEIVED_UDP, ctx->metrics.txns_received_udp ); - FD_MCNT_SET ( QUIC, TXNS_RECEIVED_QUIC_FAST, ctx->metrics.txns_received_quic_fast ); - FD_MCNT_SET ( QUIC, TXNS_RECEIVED_QUIC_FRAG, ctx->metrics.txns_received_quic_frag ); FD_MCNT_SET ( QUIC, FRAGS_OK, ctx->metrics.frag_ok_cnt ); FD_MCNT_SET ( QUIC, FRAGS_GAP, ctx->metrics.frag_gap_cnt ); FD_MCNT_SET ( QUIC, FRAGS_DUP, ctx->metrics.frag_dup_cnt ); - FD_MCNT_SET ( QUIC, TXNS_OVERRUN, ctx->metrics.reasm_overrun ); - FD_MCNT_SET ( QUIC, TXNS_ABANDONED, ctx->metrics.reasm_abandoned ); FD_MCNT_SET ( QUIC, TXN_REASMS_STARTED, ctx->metrics.reasm_started ); FD_MGAUGE_SET( QUIC, TXN_REASMS_ACTIVE, (ulong)fd_long_max( ctx->metrics.reasm_active, 0L ) ); - FD_MCNT_SET( QUIC, LEGACY_TXN_UNDERSZ, ctx->metrics.udp_pkt_too_small ); - FD_MCNT_SET( QUIC, LEGACY_TXN_OVERSZ, ctx->metrics.udp_pkt_too_large ); - FD_MCNT_SET( QUIC, TXN_UNDERSZ, ctx->metrics.quic_txn_too_small ); - FD_MCNT_SET( QUIC, TXN_OVERSZ, ctx->metrics.quic_txn_too_large ); - FD_MCNT_SET( QUIC, RECEIVED_PACKETS, ctx->quic->metrics.net_rx_pkt_cnt ); FD_MCNT_SET( QUIC, RECEIVED_BYTES, ctx->quic->metrics.net_rx_byte_cnt ); FD_MCNT_SET( QUIC, SENT_PACKETS, ctx->quic->metrics.net_tx_pkt_cnt ); @@ -148,16 +138,6 @@ metrics_write( fd_quic_ctx_t * ctx ) { FD_MCNT_SET( QUIC, CONNECTION_ERROR_NO_SLOTS, ctx->quic->metrics.conn_err_no_slots_cnt ); FD_MCNT_SET( QUIC, CONNECTION_ERROR_RETRY_FAIL, ctx->quic->metrics.conn_err_retry_fail_cnt ); - FD_MCNT_ENUM_COPY( QUIC, PKT_CRYPTO_FAILED, ctx->quic->metrics.pkt_decrypt_fail_cnt ); - FD_MCNT_ENUM_COPY( QUIC, PKT_NO_KEY, ctx->quic->metrics.pkt_no_key_cnt ); - FD_MCNT_SET( QUIC, PKT_NO_CONN, ctx->quic->metrics.pkt_no_conn_cnt ); - FD_MCNT_SET( QUIC, PKT_TX_ALLOC_FAIL, ctx->quic->metrics.pkt_tx_alloc_fail_cnt ); - FD_MCNT_SET( QUIC, PKT_NET_HEADER_INVALID, ctx->quic->metrics.pkt_net_hdr_err_cnt ); - FD_MCNT_SET( QUIC, PKT_QUIC_HEADER_INVALID, ctx->quic->metrics.pkt_quic_hdr_err_cnt ); - FD_MCNT_SET( QUIC, PKT_UNDERSZ, ctx->quic->metrics.pkt_undersz_cnt ); - FD_MCNT_SET( QUIC, PKT_OVERSZ, ctx->quic->metrics.pkt_oversz_cnt ); - FD_MCNT_SET( QUIC, PKT_VERNEG, ctx->quic->metrics.pkt_verneg_cnt ); - FD_MCNT_SET( QUIC, HANDSHAKES_CREATED, ctx->quic->metrics.hs_created_cnt ); FD_MCNT_SET( QUIC, HANDSHAKE_ERROR_ALLOC_FAIL, ctx->quic->metrics.hs_err_alloc_fail_cnt ); @@ -173,6 +153,30 @@ metrics_write( fd_quic_ctx_t * ctx ) { FD_MHIST_COPY( QUIC, RECEIVE_DURATION_SECONDS, ctx->quic->metrics.receive_duration ); } +static inline void +metrics_write_fixed_interval( fd_quic_ctx_t * ctx ) { + FD_MCNT_SET ( QUIC, TXNS_RECEIVED_UDP, ctx->metrics.txns_received_udp ); + FD_MCNT_SET ( QUIC, TXNS_RECEIVED_QUIC_FAST, ctx->metrics.txns_received_quic_fast ); + FD_MCNT_SET ( QUIC, TXNS_RECEIVED_QUIC_FRAG, ctx->metrics.txns_received_quic_frag ); + FD_MCNT_SET ( QUIC, TXNS_OVERRUN, ctx->metrics.reasm_overrun ); + FD_MCNT_SET ( QUIC, TXNS_ABANDONED, ctx->metrics.reasm_abandoned ); + + FD_MCNT_SET( QUIC, LEGACY_TXN_UNDERSZ, ctx->metrics.udp_pkt_too_small ); + FD_MCNT_SET( QUIC, LEGACY_TXN_OVERSZ, ctx->metrics.udp_pkt_too_large ); + FD_MCNT_SET( QUIC, TXN_UNDERSZ, ctx->metrics.quic_txn_too_small ); + FD_MCNT_SET( QUIC, TXN_OVERSZ, ctx->metrics.quic_txn_too_large ); + + FD_MCNT_ENUM_COPY( QUIC, PKT_CRYPTO_FAILED, ctx->quic->metrics.pkt_decrypt_fail_cnt ); + FD_MCNT_ENUM_COPY( QUIC, PKT_NO_KEY, ctx->quic->metrics.pkt_no_key_cnt ); + FD_MCNT_SET( QUIC, PKT_NO_CONN, ctx->quic->metrics.pkt_no_conn_cnt ); + FD_MCNT_SET( QUIC, PKT_TX_ALLOC_FAIL, ctx->quic->metrics.pkt_tx_alloc_fail_cnt ); + FD_MCNT_SET( QUIC, PKT_NET_HEADER_INVALID, ctx->quic->metrics.pkt_net_hdr_err_cnt ); + FD_MCNT_SET( QUIC, PKT_QUIC_HEADER_INVALID, ctx->quic->metrics.pkt_quic_hdr_err_cnt ); + FD_MCNT_SET( QUIC, PKT_UNDERSZ, ctx->quic->metrics.pkt_undersz_cnt ); + FD_MCNT_SET( QUIC, PKT_OVERSZ, ctx->quic->metrics.pkt_oversz_cnt ); + FD_MCNT_SET( QUIC, PKT_VERNEG, ctx->quic->metrics.pkt_verneg_cnt ); +} + static int before_frag( fd_quic_ctx_t * ctx, ulong in_idx, @@ -613,12 +617,13 @@ populate_allowed_fds( fd_topo_t const * topo, #define STEM_CALLBACK_CONTEXT_TYPE fd_quic_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_quic_ctx_t) -#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping -#define STEM_CALLBACK_METRICS_WRITE metrics_write -#define STEM_CALLBACK_BEFORE_CREDIT before_credit -#define STEM_CALLBACK_BEFORE_FRAG before_frag -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping +#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval +#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_BEFORE_CREDIT before_credit +#define STEM_CALLBACK_BEFORE_FRAG before_frag +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../stem/fd_stem.c" diff --git a/src/disco/stem/fd_stem.c b/src/disco/stem/fd_stem.c index 95058ba378..e4ba867686 100644 --- a/src/disco/stem/fd_stem.c +++ b/src/disco/stem/fd_stem.c @@ -31,6 +31,17 @@ or monitoring tools. The ctx is a user-provided context object from when the stem tile was initialized. + FIXED_METRICS_WRITE_INTERVAL + Tiles that accumulate high frequency metrics may need to periodically + publish them synchronously with other tiles. This allows observers + to sample metrics across multiple tiles and use them in the same + calculation coherently. This callback supports this use case by + running at a fixed interval syncronized across all tiles. This is + done on a best-effort basis; invocations of this callback will occur + at roughly the same time. This callback should be used minimally + to avoid overcrowding shared memory. + + BEFORE_CREDIT Is called every iteration of the stem run loop, whether there is a new frag ready to receive or not. This callback is also still @@ -341,12 +352,29 @@ STEM_(run1)( ulong in_cnt, FD_LOG_INFO(( "Running stem" )); FD_MGAUGE_SET( TILE, STATUS, 1UL ); long then = fd_tickcount(); + long sync_min = (long)((double)lazy * fd_tempo_tick_per_ns( NULL )); /* # of ticks between fixed-interval metrics runs */ + long then_fixed_interval = (then + sync_min) - ( (then) % sync_min ); /* Align counter with sync_min */ long now = then; for(;;) { + ulong housekeeping_ticks = 0UL; + + /* Run metrics callback on a fixed interval */ + if ( FD_UNLIKELY( (now-then_fixed_interval)>=0L ) ) { + FD_COMPILER_MFENCE(); +#ifdef STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL + STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL( ctx ); +#endif + FD_COMPILER_MFENCE(); + + /* Reload housekeeping timer */ + then_fixed_interval += sync_min; + long next = fd_tickcount(); + housekeeping_ticks += (ulong)(next - now); + now = next; + } /* Do housekeeping at a low rate in the background */ - ulong housekeeping_ticks = 0UL; if( FD_UNLIKELY( (now-then)>=0L ) ) { ulong event_idx = (ulong)event_map[ event_seq ]; @@ -443,7 +471,7 @@ STEM_(run1)( ulong in_cnt, /* Reload housekeeping timer */ then = now + (long)fd_tempo_async_reload( rng, async_min ); long next = fd_tickcount(); - housekeeping_ticks = (ulong)(next - now); + housekeeping_ticks += (ulong)(next - now); now = next; } @@ -727,6 +755,7 @@ STEM_(run)( fd_topo_t * topo, #undef STEM_LAZY #undef STEM_CALLBACK_DURING_HOUSEKEEPING #undef STEM_CALLBACK_METRICS_WRITE +#undef STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL #undef STEM_CALLBACK_BEFORE_CREDIT #undef STEM_CALLBACK_AFTER_CREDIT #undef STEM_CALLBACK_BEFORE_FRAG