From bcd0baea3ceeed747b59ffa23d74d44d8b802461 Mon Sep 17 00:00:00 2001 From: Daniel Pezely Date: Thu, 30 Jun 2022 13:38:07 -0600 Subject: [PATCH] HIP-46: facilitate routing by netid-to-oui --- include/blockchain_vars.hrl | 4 + rebar.config | 1 + rebar.lock | 4 + .../blockchain_state_channels_client.erl | 143 ++++++++++++++++-- .../v1/blockchain_txn_vars_v1.erl | 41 +++++ test/blockchain_ct_utils.erl | 5 +- test/blockchain_simple_SUITE.erl | 107 +++++++++++++ test/blockchain_state_channel_SUITE.erl | 59 ++++++-- 8 files changed, 339 insertions(+), 25 deletions(-) diff --git a/include/blockchain_vars.hrl b/include/blockchain_vars.hrl index e599bf24ee..c19530a36f 100644 --- a/include/blockchain_vars.hrl +++ b/include/blockchain_vars.hrl @@ -618,6 +618,9 @@ %% Block size limit variable (in bytes). Set to 25 * 1024 * 1024. -define(block_size_limit, block_size_limit). +%% List of peer routers associating NetID to allocated OUI +-define(routers_by_netid_to_oui, routers_by_netid_to_oui). + %% ------------------------------------------------------------------ %% Token version (aka support multiple tokens). Set to 2 (pos_integer). -define(token_version, token_version). @@ -629,3 +632,4 @@ -define(deprecate_security_exchange_v1, deprecate_security_exchange_v1). %% How many reward server keys to allow -define(allowed_num_reward_server_keys, allowed_num_reward_server_keys). + diff --git a/rebar.config b/rebar.config index 4894b8190d..1805972cbb 100644 --- a/rebar.config +++ b/rebar.config @@ -41,6 +41,7 @@ {e2qc, ".*", {git, "https://github.com/helium/e2qc", {branch, "master"}}}, {vincenty, ".*", {git, "https://github.com/helium/vincenty", {branch, "master"}}}, {helium_proto, {git, "https://github.com/helium/proto.git", {branch, "master"}}}, + {lorawan, {git, "https://github.com/helium/erlang-lorawan.git", {branch, "master"}}}, {merkerl, ".*", {git, "https://github.com/helium/merkerl.git", {branch, "master"}}}, {xxhash, {git, "https://github.com/pierreis/erlang-xxhash", {branch, "master"}}}, {exor_filter, ".*", {git, "https://github.com/mpope9/exor_filter", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index 8eb4cd222b..fcaf27bc11 100644 --- a/rebar.lock +++ b/rebar.lock @@ -98,6 +98,10 @@ {git,"https://github.com/helium/libp2p-crypto.git", {ref,"a969bb7affc7d3fa1472aef80d736c467d3f6d45"}}, 1}, + {<<"lorawan">>, + {git,"https://github.com/helium/erlang-lorawan.git", + {ref,"49925a083347e91e74daee3516a2105e82ef6c07"}}, + 0}, {<<"merkerl">>, {git,"https://github.com/helium/merkerl.git", {ref,"26ddcaf7f3c2c76eebf6f9258822f923ce69cb75"}}, diff --git a/src/state_channel/blockchain_state_channels_client.erl b/src/state_channel/blockchain_state_channels_client.erl index a0c0a573e6..1bd78bfaa5 100644 --- a/src/state_channel/blockchain_state_channels_client.erl +++ b/src/state_channel/blockchain_state_channels_client.erl @@ -32,6 +32,7 @@ -include("blockchain.hrl"). -include("blockchain_rocks.hrl"). -include("blockchain_vars.hrl"). +-include_lib("grpc/autogen/server/state_channel_pb.hrl"). -define(SERVER, ?MODULE). -define(ROUTING_CACHE, sc_client_routing). @@ -49,7 +50,8 @@ packets = #{} :: #{pid() => queue:queue(blockchain_helium_packet_v1:packet())}, waiting = #{} :: waiting(), pending_closes = [] :: list(), %% TODO GC these - sc_client_transport_handler :: atom() + sc_client_transport_handler :: atom(), + routers = [] :: list(netid_to_oui()) }). -type state() :: #state{}. @@ -59,6 +61,27 @@ -type waiting_packet() :: {Packet :: blockchain_helium_packet_v1:packet(), Region :: atom(), ReceivedTime :: non_neg_integer()}. -type waiting_key() :: non_neg_integer() | string(). -type waiting() :: #{waiting_key() => [waiting_packet()]}. +-type netid_to_oui() :: {pos_integer(), pos_integer()}. + +-ifdef(TEST). + +-export([ + set_routers/2, + get_routers/1, + get_waiting/1, + handle_route_by_netid/6 +]). + +-spec set_routers(list(string()), blockchain:blockchain()) -> state(). +set_routers(Routers, Chain) -> #state{chain=Chain, routers=Routers}. + +-spec get_routers(state()) -> list(netid_to_oui()). +get_routers(State) -> State#state.routers. + +-spec get_waiting(state()) -> waiting(). +get_waiting(State) -> State#state.waiting. + +-endif. %% ------------------------------------------------------------------ %% API Function Definitions @@ -154,18 +177,19 @@ handle_cast({banner, Banner, HandlerPid}, #state{sc_client_transport_handler = H {noreply, maybe_send_packets(AddressOrOUI, HandlerPid, State)} end end; +%% Handle Uplink packets +handle_cast({packet, + #packet_pb{routing = #routing_information_pb{data = {devaddr, DevAddr}}} = Packet, + DefaultRouters, Region, ReceivedTime}, + State) + when is_integer(DevAddr) andalso DevAddr > 0 -> + State2 = + handle_route_by_netid(Packet, DevAddr, DefaultRouters, Region, ReceivedTime, State), + {noreply, State2}; +%% Handle Join packets handle_cast({packet, Packet, DefaultRouters, Region, ReceivedTime}, #state{chain=Chain}=State) -> State2 = - case find_routing(Packet, Chain) of - {error, _Reason} -> - lager:notice( - "failed to find router for join packet with routing information ~p:~p, trying default routers", - [blockchain_helium_packet_v1:routing_info(Packet), _Reason] - ), - handle_packet(Packet, DefaultRouters, Region, ReceivedTime, State); - {ok, Routes} -> - handle_packet(Packet, Routes, Region, ReceivedTime, State) - end, + handle_packet_routing(Packet, Chain, DefaultRouters, Region, ReceivedTime, State), {noreply, State2}; handle_cast({reject, Rejection, HandlerPid}, State) -> lager:warning("Got rejection: ~p for: ~p, dropping packet", [Rejection, HandlerPid]), @@ -197,10 +221,13 @@ handle_info(post_init, #state{chain=undefined}=State) -> erlang:send_after(500, self(), post_init), {noreply, State}; Chain -> - {noreply, State#state{chain=Chain}} + %% Also scan incoming blocks for updates; see add_block event. + State1 = chain_var_routers_by_netid_to_oui(Chain, State), + {noreply, State1#state{chain=Chain}} end; handle_info({blockchain_event, {new_chain, NC}}, State) -> - {noreply, State#state{chain=NC}}; + State1 = chain_var_routers_by_netid_to_oui(NC, State), + {noreply, State1#state{chain=NC}}; handle_info({dial_fail, AddressOrOUI, _Reason}, State0) -> Packets = get_waiting_packet(AddressOrOUI, State0), lager:error("failed to dial ~p: ~p dropping ~p packets", [AddressOrOUI, _Reason, erlang:length(Packets)+1]), @@ -229,7 +256,8 @@ handle_info({dial_success, OUIOrAddress, Stream}, State0) -> _ -> {noreply, maybe_send_packets(OUIOrAddress, Stream, State1)} end; -handle_info({blockchain_event, {add_block, _BlockHash, true, Ledger}}, State) -> +handle_info({blockchain_event, {add_block, _BlockHash, true, Ledger}}, State) + when Ledger =/= undefined -> SCs = state_channels(State), {ok, LedgerHeight} = blockchain_ledger_v1:current_height(Ledger), SCGrace = case blockchain:config(?sc_grace_blocks, Ledger) of @@ -247,7 +275,8 @@ handle_info({blockchain_event, {add_block, _BlockHash, true, Ledger}}, State) -> ok end end, SCs), - {noreply, State}; + State1 = chain_var_ledger_routers_by_netid_to_oui(Ledger, State), + {noreply, State1}; handle_info({blockchain_event, {add_block, BlockHash, false, Ledger}}, #state{chain=Chain, pubkey_bin=PubkeyBin, sig_fun=SigFun, pending_closes=PendingCloses}=State) when Chain /= undefined -> Block = @@ -1068,6 +1097,90 @@ debug_multiple_scs(SC, KnownSCs) -> ok end. +-spec chain_var_routers_by_netid_to_oui( + Chain :: undefined | blockchain:blockchain(), + State :: state() + ) -> State1 :: state(). +chain_var_routers_by_netid_to_oui(undefined, State) -> + Ledger = blockchain:ledger(), + chain_var_ledger_routers_by_netid_to_oui(Ledger, State); +chain_var_routers_by_netid_to_oui(Chain, State) -> + Ledger = blockchain:ledger(Chain), + chain_var_ledger_routers_by_netid_to_oui(Ledger, State). + +-spec chain_var_ledger_routers_by_netid_to_oui( + Ledger :: blockchain:ledger(), + State :: state() + ) -> State1 :: state(). +chain_var_ledger_routers_by_netid_to_oui(Ledger, State) -> + Routers = + case blockchain_ledger_v1:config(?routers_by_netid_to_oui, Ledger) of + {ok, Bin} -> binary_to_term(Bin); + _ -> [] + end, + State#state{routers=Routers}. + +-spec handle_route_by_netid( + Packet :: blockchain_helium_packet_v1:packet(), + DevAddr :: number() | binary(), + DefaultRouters :: [blockchain_ledger_routing_v1:routing()], + Region :: atom(), + ReceivedTime :: non_neg_integer(), + State :: state() + ) -> State1 :: state(). +handle_route_by_netid(Packet, DevAddr, DefaultRouters, Region, ReceivedTime, State) -> + #state{chain=Chain, routers=RoamingRouters} = State, + OurNetID = application:get_env(blockchain, devaddr_prefix, $H), + case lora_subnet:parse_netid(DevAddr) of + {ok, OurNetID} -> + handle_packet_routing(Packet, Chain, DefaultRouters, Region, ReceivedTime, State); + {ok, ExtractedNetID} -> + FoldFn = + fun({NetID, OUI}, Acc) when NetID == ExtractedNetID -> + Ledger = blockchain:ledger(Chain), + case blockchain_ledger_v1:find_routing(OUI, Ledger) of + {ok, Route} -> [Route|Acc]; + _ -> Acc + end; + ({_OtherNetID, _}, Acc) -> + Acc + end, + RoutesOrAddresses = + case lists:foldl(FoldFn, [], RoamingRouters) of + [] -> + lager:debug("no routes found for netid ~p", [ExtractedNetID]), + DefaultRouters; + Routes -> + lager:debug("found ~p for netid ~p", [[blockchain_ledger_routing_v1:oui(R) || R <- Routes], ExtractedNetID]), + Routes + end, + handle_packet(Packet, RoutesOrAddresses, Region, ReceivedTime, State); + _Error -> + %% Drop undeliverable packet + lager:warning("failed to route ~p with devaddr=~p", [_Error, DevAddr]), + State + end. + +-spec handle_packet_routing( + Packet :: blockchain_helium_packet_v1:packet(), + Chain :: blockchain:blockchain(), + DefaultRouters :: [string()] | [blockchain_ledger_routing_v1:routing()], + Region :: atom(), + ReceivedTime :: non_neg_integer(), + State :: state() + ) -> state(). +handle_packet_routing(Packet, Chain, DefaultRouters, Region, ReceivedTime, State) -> + case find_routing(Packet, Chain) of + {error, _Reason} -> + lager:warning( + "failed to find router for join packet with routing information ~p:~p, trying default routers", + [blockchain_helium_packet_v1:routing_info(Packet), _Reason] + ), + handle_packet(Packet, DefaultRouters, Region, ReceivedTime, State); + {ok, Routes} -> + lager:debug("found routes ~p", [Routes]), + handle_packet(Packet, Routes, Region, ReceivedTime, State) + end. print_routes(RoutesOrAddresses) -> lists:map(fun(RouteOrAddress) -> diff --git a/src/transactions/v1/blockchain_txn_vars_v1.erl b/src/transactions/v1/blockchain_txn_vars_v1.erl index ff40742e8a..8c3aab76b9 100644 --- a/src/transactions/v1/blockchain_txn_vars_v1.erl +++ b/src/transactions/v1/blockchain_txn_vars_v1.erl @@ -1553,6 +1553,14 @@ validate_var(?discard_zero_freq_witness, Value) -> validate_var(?block_size_limit, Value) -> validate_int(Value, "block_size_limit", 1*1024*1024, 512*1024*1024, false); +validate_var(?routers_by_netid_to_oui, Bin) when is_binary(Bin) -> + %% Even though Rust will need to process the structure too, + %% gateway-rs fetches this via Validator, which in turn converts + %% to ProtoBuf. + validate_routers_by_netid_to_oui(binary_to_term(Bin)); +validate_var(?routers_by_netid_to_oui, _NotBinary) -> + throw({error, {invalid_routers_by_netid_to_oui, expect_binary_list_of_pairs}}); + validate_var(?token_version, Value) -> case Value of 2 -> ok; %% Add support for multiple tokens @@ -1685,6 +1693,26 @@ validate_region_params(Var, Value) -> purge_pocs(Ledger) -> blockchain_ledger_v1:purge_pocs(Ledger). +validate_routers_by_netid_to_oui(List) when is_list(List) -> + validate_routers_by_netid_to_oui(List, 1); +validate_routers_by_netid_to_oui(BadValue) -> + throw({error, {invalid_routers_by_netid_to_oui, {badvalue, BadValue}, + expect_list_of_pairs}}). + +validate_routers_by_netid_to_oui([{NetID, OUI} | T], Index) + when is_integer(NetID) andalso is_integer(OUI) andalso + NetID >= 0 andalso NetID =< 16#FFFFFF andalso + OUI >= 0 andalso OUI =< 16#FFFFFFFFFFFFFFFF-> + validate_routers_by_netid_to_oui(T, Index + 1); +validate_routers_by_netid_to_oui([], _) -> + true; +validate_routers_by_netid_to_oui([H | _T], Index) -> + throw({error, {invalid_routers_by_netid_to_oui, {{index, Index}, {entry, H}}, + expect_pair}}); +validate_routers_by_netid_to_oui(BadValue, Index) -> + throw({error, {invalid_routers_by_netid_to_oui, {{index, Index}, {value, BadValue}}, + expect_list_of_pairs}}). + %% ------------------------------------------------------------------ %% EUNIT Tests %% ------------------------------------------------------------------ @@ -1744,5 +1772,18 @@ to_json_test() -> [type, hash, vars, version_predicate, proof, master_key, key_proof, cancels, unsets, nonce])), ?assertEqual(<<"f is for ffffff\0">>, base64:decode(maps:get(f, maps:get(vars, Json)))). +routers_test() -> + ?assertEqual(true, validate_routers_by_netid_to_oui([{0, 0}])), + ?assertEqual(true, validate_routers_by_netid_to_oui([{0, 0}, {1, 1}])), + ?assertException(throw, {error, {invalid_routers_by_netid_to_oui, {{index, 2}, _}, _}}, + validate_routers_by_netid_to_oui([{0, 0}, {}])), + ?assertException(throw, {error, {invalid_routers_by_netid_to_oui, {{index, 1}, _}, _}}, + validate_routers_by_netid_to_oui([{non_integer, 0}])), + ?assertException(throw, {error, {invalid_routers_by_netid_to_oui, {{index, 1}, _}, _}}, + validate_routers_by_netid_to_oui([{0, non_integer}])), + ?assertException(throw, {error, {invalid_routers_by_netid_to_oui, _, _}}, + validate_routers_by_netid_to_oui({{0, 0}, {1, 1}})), + ?assertException(throw, {error, {invalid_routers_by_netid_to_oui, {{index, 2}, _}, _}}, + validate_routers_by_netid_to_oui([{0, 0}, {1, 2, 3}])). -endif. diff --git a/test/blockchain_ct_utils.erl b/test/blockchain_ct_utils.erl index 838039be1b..c5c1494b14 100644 --- a/test/blockchain_ct_utils.erl +++ b/test/blockchain_ct_utils.erl @@ -440,7 +440,10 @@ raw_vars(Vars) -> ?max_subnet_size => 65536, ?min_subnet_size => 8, ?max_subnet_num => 20, - ?dc_payload_size => 24 + ?dc_payload_size => 24, + ?routers_by_netid_to_oui => + term_to_binary([{16#000009, 1}, {16#600025, 1}, + {16#000037, 1}, {16#60003A, 1}]) }, maps:merge(DefVars, Vars). diff --git a/test/blockchain_simple_SUITE.erl b/test/blockchain_simple_SUITE.erl index 60448f4562..21a2f16402 100644 --- a/test/blockchain_simple_SUITE.erl +++ b/test/blockchain_simple_SUITE.erl @@ -4,6 +4,7 @@ -include_lib("eunit/include/eunit.hrl"). -include("blockchain_vars.hrl"). -include("blockchain.hrl"). +-include("blockchain_ct_utils.hrl"). -include_lib("helium_proto/include/blockchain_txn_token_burn_v1_pb.hrl"). -include_lib("helium_proto/include/blockchain_txn_payment_v1_pb.hrl"). @@ -25,6 +26,7 @@ fees_since_test/1, security_token_test/1, routing_test/1, + routing_netid_to_oui_test/1, block_save_failed_test/1, absorb_failed_test/1, missing_last_block_test/1, @@ -1344,6 +1346,111 @@ routing_test(Config) -> meck:unload(blockchain_ledger_v1), ok. +routing_netid_to_oui_test(Config) -> + %% Mostly same as routing_test()... + ConsensusMembers = ?config(consensus_members, Config), + Chain = ?config(chain, Config), + Swarm = ?config(swarm, Config), + Ledger = blockchain:ledger(Chain), + + [_, {Payer, {_, PayerPrivKey, _}}, {Router1, {_, RouterPrivKey, _}}|_] = ConsensusMembers, + SigFun = libp2p_crypto:mk_sig_fun(PayerPrivKey), + _RouterSigFun = libp2p_crypto:mk_sig_fun(RouterPrivKey), + + meck:new(blockchain_txn_oui_v1, [no_link, passthrough]), + %% See comment in routing_test() for caveats. + meck:expect(blockchain_ledger_v1, check_dc_or_hnt_balance, fun(_, _, _, _) -> ok end), + meck:expect(blockchain_ledger_v1, debit_fee, fun(_, _, _, _, _, _) -> ok end), + + OUI1 = 1, + Addresses0 = [libp2p_swarm:pubkey_bin(Swarm), Router1], + {Filter, _} = xor16:to_bin(xor16:new([0], fun xxhash:hash64/1)), + OUITxn0 = blockchain_txn_oui_v1:new(OUI1, Payer, Addresses0, Filter, 8), + SignedOUITxn0 = blockchain_txn_oui_v1:sign(OUITxn0, SigFun), + + ?assertEqual({error, not_found}, blockchain_ledger_v1:find_routing(OUI1, Ledger)), + + {ok, Block0} = test_utils:create_block(ConsensusMembers, [SignedOUITxn0]), + _ = blockchain_gossip_handler:add_block(Block0, Chain, self(), blockchain_swarm:tid()), + + ok = test_utils:wait_until(fun() -> {ok, 2} == blockchain:height(Chain) end), + + Routing0 = blockchain_ledger_routing_v1:new(OUI1, Payer, Addresses0, Filter, + <<0:25/integer-unsigned-big, (blockchain_ledger_routing_v1:subnet_size_to_mask(8)):23/integer-unsigned-big>>, 0), + ?assertEqual({ok, Routing0}, blockchain_ledger_v1:find_routing(OUI1, Ledger)), + + %% Diverging from routing_test()... + + RoamingRouters = + case blockchain_ledger_v1:config(?routers_by_netid_to_oui, Ledger) of + {ok, Bin} -> binary_to_term(Bin); + _ -> [] + end, + ?assertNotEqual([], RoamingRouters), + State = blockchain_state_channels_client:set_routers(RoamingRouters, Chain), + Region = 'US915', + Now = erlang:system_time(seconds), + DefaultRouters = + [libp2p_crypto:pubkey_bin_to_p2p(binary:list_to_bin(blockchain_ct_utils:randname(16))) + || _X <- lists:seq(1, 3)], + + %% Send packet using devaddr outside any of our test netids. + + DevAddr0 = lora_subnet:devaddr_from_netid($Z, 16#F0F0F0), + Payload0 = crypto:strong_rand_bytes(120), + Packet0 = blockchain_helium_packet_v1:new({devaddr, DevAddr0}, Payload0), + NewState0 = + blockchain_state_channels_client:handle_route_by_netid( + Packet0, DevAddr0, DefaultRouters, Region, Now, State + ), + + %% Checking packets ready to be sent; it should route them to default Routers + %% as the net id $Z is unknown to us. + Waiting0 = blockchain_state_channels_client:get_waiting(NewState0), + lists:foreach( + fun(K) -> + ?assert(lists:member(K, DefaultRouters)) + end, + maps:keys(Waiting0) + ), + + %% Send packet using devaddr within each of our test netids, + %% and confirm results of route_by_netid(). + lists:foreach( + fun({NetID, OUI}) -> + DevAddr = lora_subnet:devaddr_from_subnet(1, [NetID]), + Payload = crypto:strong_rand_bytes(120), + Packet = blockchain_helium_packet_v1:new({devaddr, DevAddr}, Payload), + NewState = + blockchain_state_channels_client:handle_route_by_netid( + Packet, DevAddr, DefaultRouters, Region, Now, State + ), + Waiting = blockchain_state_channels_client:get_waiting(NewState), + {ok, Route} = blockchain_ledger_v1:find_routing(OUI, Ledger), + ?assertEqual([blockchain_ledger_routing_v1:oui(Route)], maps:keys(Waiting)) + end, + RoamingRouters + ), + + %% Use devaddr within default router slab. + %% Official NetID assigned to Nova Labs by LoRa Alliance: + NetID = 16#60002D, + DevAddr1 = lora_subnet:devaddr_from_netid(NetID, 16#F0F0), + Payload1 = crypto:strong_rand_bytes(120), + Packet1 = blockchain_helium_packet_v1:new({devaddr, DevAddr1}, Payload1), + NewState1 = + blockchain_state_channels_client:handle_route_by_netid( + Packet1, DevAddr1, DefaultRouters, Region, Now, State + ), + Waiting1 = blockchain_state_channels_client:get_waiting(NewState1), + lists:foreach( + fun(K) -> + ?assert(lists:member(K, DefaultRouters)) + end, + maps:keys(Waiting1) + ), + ok. + max_subnet_test(Config) -> ConsensusMembers = ?config(consensus_members, Config), Chain = ?config(chain, Config), diff --git a/test/blockchain_state_channel_SUITE.erl b/test/blockchain_state_channel_SUITE.erl index 5d523988a5..2f46418dcf 100644 --- a/test/blockchain_state_channel_SUITE.erl +++ b/test/blockchain_state_channel_SUITE.erl @@ -128,11 +128,11 @@ init_per_testcase(Test, Config) -> ConsensusAddrs = lists:sublist(lists:sort(Addrs), NumConsensusMembers), - %% the SC tests use the first two nodes as the gateway and router - %% for the GRPC group to work we need to ensure these two nodes are - %% connected to each other in blockchain_ct_utils:init_per_testcase the - %% nodes are connected to a majority of the group but that does not - %% guarantee these two nodes are connected + %% The SC tests use the first two nodes as the gateway and router. + %% For the GRPC group to work we need to ensure these two nodes are + %% connected to each other in blockchain_ct_utils:init_per_testcase(). + %% The nodes are connected to a majority of the group, but that does not + %% guarantee these two nodes are connected. [RouterNode, GatewayNode] = blockchain_ct_utils:find_connected_node_pair(NodeAddrList), @@ -149,16 +149,57 @@ init_per_testcase(Test, Config) -> blockchain_state_channels_handler, blockchain_state_channels_server, blockchain_state_channels_worker, - blockchain_txn_state_channel_close_v1] + blockchain_txn_state_channel_close_v1, + blockchain_state_channel_sup] ), debug_modules_for_node( GatewayNode, Dir ++ "sc_client_1.log", [blockchain_state_channel_v1, blockchain_state_channels_client, - blockchain_state_channels_handler] + blockchain_state_channels_handler, + blockchain_state_channel_sup] ), + %% accumulate the address of each node + Addrs = lists:foldl(fun(Node, Acc) -> + Addr = ct_rpc:call(Node, blockchain_swarm, pubkey_bin, []), + [Addr | Acc] + end, [], Nodes), + + ConsensusAddrs = lists:sublist(lists:sort(Addrs), NumConsensusMembers), + + %% the SC tests use the first two nodes as the gateway and router + %% for the GRPC group to work we need to ensure these two nodes are connected to each other + %% in blockchain_ct_utils:init_per_testcase the nodes are connected to a majority of the group + %% but that does not guarantee these two nodes will be connected + [RouterNode, GatewayNode|_] = Nodes, + [RouterNodeAddr, GatewayNodeAddr|_] = Addrs, + ok = blockchain_ct_utils:wait_until( + fun() -> + lists:all( + fun({Node, AddrToConnectToo}) -> + try + GossipPeers = ct_rpc:call(Node, blockchain_swarm, gossip_peers, [], 500), + ct:pal("~p connected to peers ~p", [Node, GossipPeers]), + case lists:member(libp2p_crypto:pubkey_bin_to_p2p(AddrToConnectToo), GossipPeers) of + true -> true; + false -> + ct:pal("~p is not connected to desired peer ~p", [Node, AddrToConnectToo]), + Swarm = ct_rpc:call(Node, blockchain_swarm, swarm, [], 500), + CRes = ct_rpc:call(Node, libp2p_swarm, connect, [Swarm, libp2p_crypto:pubkey_bin_to_p2p(AddrToConnectToo)], 500), + ct:pal("Connecting ~p to ~p: ~p", [Node, AddrToConnectToo, CRes]), + case CRes of + {ok, _} -> true; + _ -> false + end + end + catch _C:_E -> + false + end + end, [{RouterNode, GatewayNodeAddr}, {GatewayNode, RouterNodeAddr}]) + end, 200, 150), + SCDisputeStrat = case Test == sc_dispute_prevention_test of false -> 0; true -> 1 @@ -313,6 +354,7 @@ full_test(Config) -> %% Sending 1 packet DevNonce0 = crypto:strong_rand_bytes(2), Packet0 = blockchain_ct_utils:join_packet(?APPKEY, DevNonce0, 0.0), + ct:pal("Gateway node1 ~p sending ~p", [GatewayNode1, Packet0]), ok = ct_rpc:call(GatewayNode1, blockchain_state_channels_client, packet, [Packet0, [], 'US915']), %% Checking state channel on server/client @@ -321,7 +363,7 @@ full_test(Config) -> %% Sending another packet DevNonce1 = crypto:strong_rand_bytes(2), Packet1 = blockchain_ct_utils:join_packet(?APPKEY, DevNonce1, 0.0), - ct:pal("Gateway node1 ~p", [GatewayNode1]), + ct:pal("Gateway node1 ~p sending ~p", [GatewayNode1, Packet1]), ok = ct_rpc:call(GatewayNode1, blockchain_state_channels_client, packet, [Packet1, [], 'US915']), %% Checking state channel on server/client @@ -2630,7 +2672,6 @@ default_routers_test(Config) -> ok. - %% ------------------------------------------------------------------ %% Helper functions %% ------------------------------------------------------------------