Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gui: sync metrics sampling #4318

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

gui: sync metrics sampling #4318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 17 additions & 19 deletions src/app/fdctl/run/tiles/fd_bank.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ typedef struct {
ulong txn_load_address_lookup_tables[ 6 ];
ulong transaction_result[ 40 ];
ulong processing_failed;
ulong precompile_verify_failure;
ulong fee_only;
ulong exec_failed;
ulong success;
Expand All @@ -68,14 +69,18 @@ scratch_footprint( fd_topo_tile_t const * tile ) {
static inline void
metrics_write( fd_bank_ctx_t * ctx ) {
FD_MCNT_ENUM_COPY( BANK, SLOT_ACQUIRE, ctx->metrics.slot_acquire );
FD_MCNT_ENUM_COPY( BANK, TRANSACTION_RESULT, ctx->metrics.transaction_result );
}

static inline void
metrics_write_fixed_interval( fd_bank_ctx_t * ctx ) {
FD_MCNT_ENUM_COPY( BANK, TRANSACTION_LOAD_ADDRESS_TABLES, ctx->metrics.txn_load_address_lookup_tables );
FD_MCNT_ENUM_COPY( BANK, TRANSACTION_RESULT, ctx->metrics.transaction_result );

FD_MCNT_SET( BANK, PROCESSING_FAILED, ctx->metrics.processing_failed );
FD_MCNT_SET( BANK, FEE_ONLY_TRANSACTIONS, ctx->metrics.fee_only );
FD_MCNT_SET( BANK, EXECUTED_FAILED_TRANSACTIONS, ctx->metrics.exec_failed );
FD_MCNT_SET( BANK, SUCCESSFUL_TRANSACTIONS, ctx->metrics.success );
FD_MCNT_SET( BANK, PROCESSING_FAILED, ctx->metrics.processing_failed );
FD_MCNT_SET( BANK, PRECOMPILE_VERIFY_FAILURE, ctx->metrics.precompile_verify_failure );
FD_MCNT_SET( BANK, FEE_ONLY_TRANSACTIONS, ctx->metrics.fee_only );
FD_MCNT_SET( BANK, EXECUTED_FAILED_TRANSACTIONS, ctx->metrics.exec_failed );
FD_MCNT_SET( BANK, SUCCESSFUL_TRANSACTIONS, ctx->metrics.success );
}

static int
Expand Down Expand Up @@ -172,7 +177,7 @@ handle_microblock( fd_bank_ctx_t * ctx,

int precompile_result = fd_ext_bank_verify_precompiles( ctx->_bank, abi_txn );
if( FD_UNLIKELY( precompile_result ) ) {
FD_MCNT_INC( BANK, PRECOMPILE_VERIFY_FAILURE, 1 );
ctx->metrics.precompile_verify_failure++;
continue;
}

Expand Down Expand Up @@ -299,12 +304,6 @@ handle_microblock( fd_bank_ctx_t * ctx,
FD_STATIC_ASSERT( MAX_MICROBLOCK_SZ-(MAX_TXN_PER_MICROBLOCK*sizeof(fd_txn_p_t))>=sizeof(fd_microblock_trailer_t), poh_shred_mtu );
FD_STATIC_ASSERT( MAX_MICROBLOCK_SZ-(MAX_TXN_PER_MICROBLOCK*sizeof(fd_txn_p_t))>=sizeof(fd_microblock_bank_trailer_t), poh_shred_mtu );

/* We have a race window with the GUI, where if the slot is ending it
will snap these metrics to draw the waterfall, but see them outdated
because housekeeping hasn't run. For now just update them here, but
PoH should eventually flush the pipeline before ending the slot. */
metrics_write( ctx );

ulong bank_sig = fd_disco_bank_sig( slot, ctx->_microblock_idx );

/* We always need to publish, even if there are no successfully executed
Expand Down Expand Up @@ -351,7 +350,7 @@ handle_bundle( fd_bank_ctx_t * ctx,
int precompile_result = fd_ext_bank_verify_precompiles( ctx->_bank, abi_txn );
if( FD_UNLIKELY( precompile_result ) ) {
execution_success = 0;
FD_MCNT_INC( BANK, PRECOMPILE_VERIFY_FAILURE, 1 );
ctx->metrics.precompile_verify_failure++;
continue;
}

Expand Down Expand Up @@ -452,8 +451,6 @@ handle_bundle( fd_bank_ctx_t * ctx,
fd_stem_publish( stem, 0UL, bank_sig, ctx->out_chunk, new_sz, 0UL, 0UL, tspub );
ctx->out_chunk = fd_dcache_compact_next( ctx->out_chunk, new_sz, ctx->out_chunk0, ctx->out_wmark );
}

metrics_write( ctx );
}

static inline void
Expand Down Expand Up @@ -520,10 +517,11 @@ unprivileged_init( fd_topo_t * topo,
#define STEM_CALLBACK_CONTEXT_TYPE fd_bank_ctx_t
#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_bank_ctx_t)

#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_BEFORE_FRAG before_frag
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag
#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval
#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_BEFORE_FRAG before_frag
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag

#include "../../../../disco/stem/fd_stem.c"

Expand Down
13 changes: 9 additions & 4 deletions src/app/fdctl/run/tiles/fd_bundle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,15 @@ during_housekeeping( fd_bundle_ctx_t * ctx ) {

static inline void
metrics_write( fd_bundle_ctx_t * ctx ) {
FD_MCNT_SET( BUNDLE, TRANSACTION_RECEIVED, ctx->metrics.txn_received );
FD_MCNT_SET( BUNDLE, BUNDLE_RECEIVED, ctx->metrics.bundle_received );
FD_MCNT_SET( BUNDLE, PACKET_RECEIVED, ctx->metrics.packet_received );
}

static inline void
metrics_write_fixed_interval( fd_bundle_ctx_t * ctx ) {
FD_MCNT_SET( BUNDLE, TRANSACTION_RECEIVED, ctx->metrics.txn_received );
}

extern void
plugin_bundle_poll( void * plugin,
int reload_identity,
Expand Down Expand Up @@ -423,9 +427,10 @@ populate_allowed_fds( fd_topo_t const * topo,
#define STEM_CALLBACK_CONTEXT_TYPE fd_bundle_ctx_t
#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_bundle_ctx_t)

#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_AFTER_CREDIT after_credit
#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval
#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_AFTER_CREDIT after_credit

#include "../../../../disco/stem/fd_stem.c"

Expand Down
20 changes: 15 additions & 5 deletions src/app/fdctl/run/tiles/fd_dedup.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ typedef struct {
struct {
ulong bundle_peer_failure_cnt;
ulong dedup_fail_cnt;
ulong gossiped_votes_received;
} metrics;
} fd_dedup_ctx_t;

Expand All @@ -75,8 +76,16 @@ scratch_footprint( fd_topo_tile_t const * tile ) {

static inline void
metrics_write( fd_dedup_ctx_t * ctx ) {
(void)ctx;
return;
}

static inline void
metrics_write_fixed_interval( fd_dedup_ctx_t * ctx ) {
FD_MCNT_SET( DEDUP, TRANSACTION_BUNDLE_PEER_FAILURE, ctx->metrics.bundle_peer_failure_cnt );
FD_MCNT_SET( DEDUP, TRANSACTION_DEDUP_FAILURE, ctx->metrics.dedup_fail_cnt );
FD_MCNT_SET( DEDUP, TRANSACTION_DEDUP_FAILURE, ctx->metrics.dedup_fail_cnt );
FD_MCNT_SET( DEDUP, GOSSIPED_VOTES_RECEIVED, ctx->metrics.gossiped_votes_received );

}

/* during_frag is called between pairs for sequence number checks, as
Expand Down Expand Up @@ -168,7 +177,7 @@ after_frag( fd_dedup_ctx_t * ctx,
txnm->txn_t_sz = (ushort)fd_txn_parse( fd_txn_m_payload( txnm ), txnm->payload_sz, txn, NULL );
if( FD_UNLIKELY( !txnm->txn_t_sz ) ) FD_LOG_ERR(( "fd_txn_parse failed for vote transactions that should have been sigverified" ));

if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL );
if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) ctx->metrics.gossiped_votes_received++;
}

int is_dup = 0;
Expand Down Expand Up @@ -301,9 +310,10 @@ populate_allowed_fds( fd_topo_t const * topo,
#define STEM_CALLBACK_CONTEXT_TYPE fd_dedup_ctx_t
#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_dedup_ctx_t)

#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag
#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval
#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag

#include "../../../../disco/stem/fd_stem.c"

Expand Down
57 changes: 40 additions & 17 deletions src/app/fdctl/run/tiles/fd_pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ typedef struct {
increases, we expire old transactions. */
ulong highest_observed_slot;

ulong transaction_expired;

/* microblock_duration_ns, and wait_duration
respectively scaled to be in ticks instead of nanoseconds */
ulong microblock_duration_ticks;
Expand All @@ -203,6 +205,9 @@ typedef struct {
full. This is an fd_deque. */
fd_txn_e_t * extra_txn_deq;
int insert_to_extra; /* whether the last insert was into pack or the extra deq */
ulong transaction_dropped_from_extra;,
ulong transaction_inserted_from_extra;,
ulong transaction_inserted_to_extra;
#endif

fd_pack_in_ctx_t in[ 32 ];
Expand Down Expand Up @@ -250,6 +255,8 @@ typedef struct {
fd_txn_e_t * const * bundle; /* points to _txn when non-NULL */
} current_bundle[1];

ulong partial_bundle_dropped_txn_cnt;

block_builder_info_t blk_engine_cfg[1];

struct {
Expand Down Expand Up @@ -358,9 +365,7 @@ log_end_block_metrics( fd_pack_ctx_t * ctx,

static inline void
metrics_write( fd_pack_ctx_t * ctx ) {
FD_MCNT_ENUM_COPY( PACK, TRANSACTION_INSERTED, ctx->insert_result );
FD_MCNT_ENUM_COPY( PACK, METRIC_TIMING, ((ulong*)ctx->metric_timing) );
FD_MCNT_ENUM_COPY( PACK, BUNDLE_CRANK_STATUS, ctx->crank->metrics );
FD_MHIST_COPY( PACK, SCHEDULE_MICROBLOCK_DURATION_SECONDS, ctx->schedule_duration );
FD_MHIST_COPY( PACK, NO_SCHED_MICROBLOCK_DURATION_SECONDS, ctx->no_sched_duration );
FD_MHIST_COPY( PACK, INSERT_TRANSACTION_DURATION_SECONDS, ctx->insert_duration );
Expand All @@ -369,6 +374,22 @@ metrics_write( fd_pack_ctx_t * ctx ) {
fd_pack_metrics_write( ctx->pack );
}

static inline void
metrics_write_fixed_interval( fd_pack_ctx_t * ctx ) {
FD_MCNT_ENUM_COPY( PACK, TRANSACTION_INSERTED, ctx->insert_result );
FD_MCNT_SET( PACK, TRANSACTION_EXPIRED, ctx->transaction_expired );
FD_MCNT_ENUM_COPY( PACK, BUNDLE_CRANK_STATUS, ctx->crank->metrics );
FD_MCNT_SET( PACK, TRANSACTION_DROPPED_PARTIAL_BUNDLE, ctx->partial_bundle_dropped_txn_cnt );

#if FD_PACK_USE_EXTRA_STORAGE
FD_MCNT_SET( PACK, TRANSACTION_DROPPED_FROM_EXTRA, ctx->transaction_dropped_from_extra );
FD_MCNT_SET( PACK, TRANSACTION_INSERTED_FROM_EXTRA, ctx->transaction_inserted_from_extra );
FD_MCNT_SET( PACK, TRANSACTION_INSERTED_TO_EXTRA, ctx->transaction_inserted_to_extra );
#endif

fd_pack_metrics_fixed_int_write(ctx->pack);
}

static inline void
during_housekeeping( fd_pack_ctx_t * ctx ) {
ctx->approx_wallclock_ns = fd_log_wallclock();
Expand Down Expand Up @@ -428,7 +449,7 @@ insert_from_extra( fd_pack_ctx_t * ctx ) {
insert_duration += fd_tickcount();
ctx->insert_result[ result + FD_PACK_INSERT_RETVAL_OFF ]++;
fd_histf_sample( ctx->insert_duration, (ulong)insert_duration );
FD_MCNT_INC( PACK, TRANSACTION_INSERTED_FROM_EXTRA, 1UL );
ctx->transaction_inserted_from_extra++;
return result;
}
#endif
Expand Down Expand Up @@ -747,8 +768,7 @@ during_frag( fd_pack_ctx_t * ctx,
}
ctx->leader_slot = fd_disco_poh_sig_slot( sig );

ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->leader_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt );
ctx->transaction_expired += fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->leader_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );

fd_became_leader_t * became_leader = (fd_became_leader_t *)dcache_entry;
ctx->leader_bank = became_leader->bank;
Expand Down Expand Up @@ -817,16 +837,15 @@ during_frag( fd_pack_ctx_t * ctx,
with expired but high-fee-paying transactions. That can only
happen if we are getting transactions. */
ctx->highest_observed_slot = sig;
ulong exp_cnt = fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->highest_observed_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
FD_MCNT_INC( PACK, TRANSACTION_EXPIRED, exp_cnt );
ctx->transaction_expired += fd_pack_expire_before( ctx->pack, fd_ulong_max( ctx->highest_observed_slot, TRANSACTION_LIFETIME_SLOTS )-TRANSACTION_LIFETIME_SLOTS );
}


if( FD_UNLIKELY( txnm->block_engine.bundle_id ) ) {
ctx->is_bundle = 1;
if( FD_LIKELY( txnm->block_engine.bundle_id!=ctx->current_bundle->id ) ) {
if( FD_UNLIKELY( ctx->current_bundle->bundle ) ) {
FD_MCNT_INC( PACK, TRANSACTION_DROPPED_PARTIAL_BUNDLE, ctx->current_bundle->txn_received );
ctx->partial_bundle_dropped_txn_cnt += ctx->current_bundle->txn_received;
fd_pack_insert_bundle_cancel( ctx->pack, ctx->current_bundle->bundle, ctx->current_bundle->txn_cnt );
}
ctx->current_bundle->id = txnm->block_engine.bundle_id;
Expand All @@ -835,7 +854,7 @@ during_frag( fd_pack_ctx_t * ctx,
ctx->current_bundle->txn_received = 0UL;

if( FD_UNLIKELY( ctx->current_bundle->txn_cnt==0UL ) ) {
FD_MCNT_INC( PACK, TRANSACTION_DROPPED_PARTIAL_BUNDLE, 1UL );
ctx->partial_bundle_dropped_txn_cnt++;
ctx->current_bundle->id = 0UL;
return;
}
Expand All @@ -855,15 +874,15 @@ during_frag( fd_pack_ctx_t * ctx,
} else {
if( FD_UNLIKELY( extra_txn_deq_full( ctx->extra_txn_deq ) ) ) {
extra_txn_deq_remove_head( ctx->extra_txn_deq );
FD_MCNT_INC( PACK, TRANSACTION_DROPPED_FROM_EXTRA, 1UL );
ctx->transaction_dropped_from_extra++;
}
ctx->cur_spot = extra_txn_deq_peek_tail( extra_txn_deq_insert_tail( ctx->extra_txn_deq ) );
/* We want to store the current time in cur_spot so that we can
track its expiration better. We just stash it in the CU
fields, since those aren't important right now. */
ctx->cur_spot->txnp->blockhash_slot = sig;
ctx->insert_to_extra = 1;
FD_MCNT_INC( PACK, TRANSACTION_INSERTED_TO_EXTRA, 1UL );
ctx->transaction_inserted_to_extra++;
}
#else
ctx->cur_spot = fd_pack_insert_txn_init( ctx->pack );
Expand Down Expand Up @@ -1092,9 +1111,12 @@ unprivileged_init( fd_topo_t * topo,
ctx->ticks_per_ns = fd_tempo_tick_per_ns( NULL );
ctx->last_successful_insert = 0L;
ctx->highest_observed_slot = 0UL;
ctx->transaction_expired = 0UL;
ctx->microblock_duration_ticks = (ulong)(fd_tempo_tick_per_ns( NULL )*(double)MICROBLOCK_DURATION_NS + 0.5);
#if FD_PACK_USE_EXTRA_STORAGE
ctx->insert_to_extra = 0;
ctx->transaction_dropped_from_extra= 0UL;
ctx->transaction_inserted_to_extra = 0UL;
#endif
ctx->use_consumed_cus = tile->pack.use_consumed_cus;
ctx->crank->enabled = tile->pack.bundle.enabled;
Expand Down Expand Up @@ -1202,12 +1224,13 @@ populate_allowed_fds( fd_topo_t const * topo,
#define STEM_CALLBACK_CONTEXT_TYPE fd_pack_ctx_t
#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_pack_ctx_t)

#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
#define STEM_CALLBACK_BEFORE_CREDIT before_credit
#define STEM_CALLBACK_AFTER_CREDIT after_credit
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag
#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
#define STEM_CALLBACK_BEFORE_CREDIT before_credit
#define STEM_CALLBACK_AFTER_CREDIT after_credit
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag
#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval
#define STEM_CALLBACK_METRICS_WRITE metrics_write

#include "../../../../disco/stem/fd_stem.c"

Expand Down
18 changes: 10 additions & 8 deletions src/app/fdctl/run/tiles/fd_resolv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ typedef struct {

struct {
ulong lut[ FD_METRICS_COUNTER_RESOLV_LUT_RESOLVED_CNT ];
ulong no_bank_drop;
ulong blockhash_expired;
ulong blockhash_unknown;
ulong bundle_peer_failure_cnt;
Expand Down Expand Up @@ -177,8 +178,9 @@ fd_ext_resolv_tile_cnt( void ) {
}

static inline void
metrics_write( fd_resolv_ctx_t * ctx ) {
metrics_write_fixed_interval( fd_resolv_ctx_t * ctx ) {
FD_MCNT_SET( RESOLV, BLOCKHASH_EXPIRED, ctx->metrics.blockhash_expired );
FD_MCNT_SET( RESOLV, NO_BANK_DROP, ctx->metrics.no_bank_drop );
FD_MCNT_ENUM_COPY( RESOLV, LUT_RESOLVED, ctx->metrics.lut );
FD_MCNT_ENUM_COPY( RESOLV, STASH_OPERATION, ctx->metrics.stash );
FD_MCNT_SET( RESOLV, TRANSACTION_BUNDLE_PEER_FAILURE, ctx->metrics.bundle_peer_failure_cnt );
Expand Down Expand Up @@ -237,7 +239,7 @@ publish_txn( fd_resolv_ctx_t * ctx,

if( FD_UNLIKELY( txnt->addr_table_adtl_cnt ) ) {
if( FD_UNLIKELY( !ctx->root_bank ) ) {
FD_MCNT_INC( RESOLV, NO_BANK_DROP, 1 );
ctx->metrics.no_bank_drop++;
return 0;
} else {
int result = fd_bank_abi_resolve_address_lookup_tables( ctx->root_bank, 0, ctx->root_slot, txnt, fd_txn_m_payload( txnm ), fd_txn_m_alut( txnm ) );
Expand Down Expand Up @@ -430,7 +432,7 @@ after_frag( fd_resolv_ctx_t * ctx,

if( FD_UNLIKELY( txnt->addr_table_adtl_cnt ) ) {
if( FD_UNLIKELY( !ctx->root_bank ) ) {
FD_MCNT_INC( RESOLV, NO_BANK_DROP, 1 );
ctx->metrics.no_bank_drop++;
if( FD_UNLIKELY( txnm->block_engine.bundle_id ) ) ctx->bundle_failed = 1;
return;
}
Expand Down Expand Up @@ -517,11 +519,11 @@ unprivileged_init( fd_topo_t * topo,
#define STEM_CALLBACK_CONTEXT_TYPE fd_resolv_ctx_t
#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_resolv_ctx_t)

#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_AFTER_CREDIT after_credit
#define STEM_CALLBACK_BEFORE_FRAG before_frag
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag
#define STEM_CALLBACK_FIXED_METRICS_WRITE_INTERVAL metrics_write_fixed_interval
#define STEM_CALLBACK_AFTER_CREDIT after_credit
#define STEM_CALLBACK_BEFORE_FRAG before_frag
#define STEM_CALLBACK_DURING_FRAG during_frag
#define STEM_CALLBACK_AFTER_FRAG after_frag

#include "../../../../disco/stem/fd_stem.c"

Expand Down
Loading
Loading