feature/multiprovider

src/cart/crt_group.c

@@ -1853,28 +1853,19 @@ crt_group_config_path_set(const char *path)
 int
 crt_nr_secondary_remote_tags_set(int idx, int num_tags)
 {
-	struct crt_prov_gdata *prov_data;
-
 	D_DEBUG(DB_ALL, "secondary_idx=%d num_tags=%d\n", idx, num_tags);
 
 	if (idx != 0) {
 		D_ERROR("Only idx=0 is currently supported\n");
 		return -DER_NONEXIST;
 	}
 
-	if ((crt_gdata.cg_prov_gdata_secondary == NULL) ||
-	    (idx >= crt_gdata.cg_num_secondary_provs)) {
-		D_ERROR("Secondary providers not initialized\n");
-		return -DER_NONEXIST;
-	}
-
 	if (num_tags <= 0) {
 		D_ERROR("Invalid number of tags: %d\n", num_tags);
 		return -DER_INVAL;
 	}
 
-	prov_data = &crt_gdata.cg_prov_gdata_secondary[idx];
-	prov_data->cpg_num_remote_tags = num_tags;
+	crt_gdata.cg_num_remote_tags = num_tags;
 
 	return DER_SUCCESS;
 }

src/cart/crt_hg_proc.c

@@ -583,7 +583,8 @@ crt_proc_in_common(crt_proc_t proc, crt_rpc_input_t *data)
 						);
 			hdr->cch_dst_tag = rpc_priv->crp_pub.cr_ep.ep_tag;
 
-			hdr->cch_src_timeout = rpc_priv->crp_timeout_sec;
+			hdr->cch_src_timeout    = rpc_priv->crp_timeout_sec;
+			hdr->cch_src_is_primary = rpc_priv->crp_src_is_primary;
 			if (crt_is_service()) {
 				hdr->cch_src_rank =
 					crt_grp_priv_get_primary_rank(

src/cart/crt_init.c

@@ -44,6 +44,10 @@ crt_lib_init(void)
 	crt_gdata.cg_inited = 0;
 	crt_gdata.cg_primary_prov = CRT_PROV_OFI_TCP_RXM;
 
+	/* By default set number of secondary remote tags to 1 */
+	crt_gdata.cg_num_remote_tags = 1;
+	crt_gdata.cg_last_remote_tag = 0;
+
 	d_srand(d_timeus_secdiff(0) + getpid());
 	start_rpcid = ((uint64_t)d_rand()) << 32;
 

src/cart/crt_internal_types.h

@@ -129,6 +129,12 @@ struct crt_gdata {
 
 	ATOMIC uint64_t		cg_rpcid; /* rpc id */
 
+	/** Last remote tag sent */
+	uint32_t		cg_last_remote_tag;
+
+	/** Number of remote tags */
+	uint32_t		cg_num_remote_tags;
+
 	/* protects crt_gdata (see the lock order comment on crp_mutex) */
 	pthread_rwlock_t	cg_rwlock;
 

src/cart/crt_rpc.c

@@ -1201,7 +1201,6 @@ crt_req_ep_lc_lookup(struct crt_rpc_priv *rpc_priv, bool *uri_exists)
 	crt_phy_addr_t		 uri = NULL;
 	int			 rc = 0;
 	crt_phy_addr_t		 base_addr = NULL;
-	struct crt_prov_gdata	*prov_data;
 	int			 dst_tag;
 
 	req = &rpc_priv->crp_pub;
@@ -1216,13 +1215,12 @@ crt_req_ep_lc_lookup(struct crt_rpc_priv *rpc_priv, bool *uri_exists)
 	/* For a secondary provider round-robin between all available remote contexts */
 	if (!crt_gdata.cg_provider_is_primary) {
 
-		prov_data = &crt_gdata.cg_prov_gdata_secondary[0];
+		D_RWLOCK_WRLOCK(&crt_gdata.cg_rwlock);
+		crt_gdata.cg_last_remote_tag++;
+		crt_gdata.cg_last_remote_tag %= crt_gdata.cg_num_remote_tags;
 
-		D_MUTEX_LOCK(&prov_data->cpg_mutex);
-		prov_data->cpg_last_remote_tag++;
-		prov_data->cpg_last_remote_tag %= prov_data->cpg_num_remote_tags;
-		dst_tag = prov_data->cpg_last_remote_tag;
-		D_MUTEX_UNLOCK(&prov_data->cpg_mutex);
+		dst_tag = crt_gdata.cg_last_remote_tag;
+		D_RWLOCK_UNLOCK(&crt_gdata.cg_rwlock);
 	}
 
 	crt_grp_lc_lookup(grp_priv, ctx->cc_idx,
@@ -1805,6 +1803,10 @@ crt_rpc_common_hdlr(struct crt_rpc_priv *rpc_priv)
 			skip_check = true;
 	}
 
+	/* Skip check for secondary provider clients */
+	if (!rpc_priv->crp_req_hdr.cch_src_is_primary)
+		skip_check = true;
+
 	if ((self_rank != rpc_priv->crp_req_hdr.cch_dst_rank) ||
 	    (crt_ctx->cc_idx != rpc_priv->crp_req_hdr.cch_dst_tag)) {
 		if (!skip_check) {
@@ -1926,6 +1928,28 @@ crt_req_src_rank_get(crt_rpc_t *rpc, d_rank_t *rank)
 	return rc;
 }
 
+int
+crt_req_src_provider_is_primary(crt_rpc_t *req, bool *result)
+{
+	struct crt_rpc_priv	*rpc_priv = NULL;
+	int			rc = 0;
+
+	if (req == NULL) {
+		D_ERROR("req is NULL\n");
+		D_GOTO(out, rc = -DER_INVAL);
+	}
+
+	if (result == NULL) {
+		D_ERROR("result is NULL\n");
+		D_GOTO(out, rc = -DER_INVAL);
+	}
+
+	rpc_priv = container_of(req, struct crt_rpc_priv, crp_pub);
+	*result = rpc_priv->crp_req_hdr.cch_src_is_primary;
+out:
+	return rc;
+}
+
 int
 crt_req_dst_rank_get(crt_rpc_t *rpc, d_rank_t *rank)
 {

src/cart/crt_rpc.h

@@ -66,8 +66,9 @@ struct crt_common_hdr {
 	/* originator rank in default primary group */
 	d_rank_t	cch_src_rank;
 	/* destination tag */
-	uint32_t	cch_dst_tag;
-
+	uint16_t	cch_dst_tag;
+	/* indicates whether rpc originator intended to send on a primary ctx */
+	uint32_t	cch_src_is_primary : 1;
 
 	/* used in crp_reply_hdr to propagate rpc failure back to sender */
 	/* TODO: workaround for DAOS-13973 */

src/container/srv_cli.c

@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2017-2021 Intel Corporation.
+ * (C) Copyright 2017-2022 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */

src/container/srv_container.c

@@ -1487,12 +1487,10 @@ find_hdls_by_cont_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg)
 }
 
 static int cont_close_hdls(struct cont_svc *svc,
-			   struct cont_tgt_close_rec *recs, int nrecs,
-			   crt_context_t ctx);
+			   struct cont_tgt_close_rec *recs, int nrecs);
 
 static int
-evict_hdls(struct rdb_tx *tx, struct cont *cont, bool force, struct ds_pool_hdl *pool_hdl,
-	   crt_context_t ctx)
+evict_hdls(struct rdb_tx *tx, struct cont *cont, bool force, struct ds_pool_hdl *pool_hdl)
 {
 	struct find_hdls_by_cont_arg	arg;
 	int				rc;
@@ -1518,7 +1516,7 @@ evict_hdls(struct rdb_tx *tx, struct cont *cont, bool force, struct ds_pool_hdl
 		goto out;
 	}
 
-	rc = cont_close_hdls(cont->c_svc, arg.fha_buf.rb_recs, arg.fha_buf.rb_nrecs, ctx);
+	rc = cont_close_hdls(cont->c_svc, arg.fha_buf.rb_recs, arg.fha_buf.rb_nrecs);
 
 out:
 	recs_buf_fini(&arg.fha_buf);
@@ -1570,11 +1568,11 @@ cont_destroy(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont,
 		D_GOTO(out_prop, rc = -DER_NO_PERM);
 	}
 
-	rc = evict_hdls(tx, cont, force, NULL /* pool_hdl */, rpc->cr_ctx);
+	rc = evict_hdls(tx, cont, force, NULL /* pool_hdl */);
 	if (rc != 0)
 		goto out_prop;
 
-	rc = cont_destroy_bcast(rpc->cr_ctx, cont->c_svc, cont->c_uuid);
+	rc = cont_destroy_bcast(dss_get_module_info()->dmi_ctx, cont->c_svc, cont->c_uuid);
 	if (rc != 0)
 		goto out_prop;
 
@@ -2279,7 +2277,7 @@ cont_open(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont, cr
 
 	if (flags & (DAOS_COO_EVICT | DAOS_COO_EVICT_ALL)) {
 		rc = evict_hdls(tx, cont, true /* force */,
-				(flags & DAOS_COO_EVICT_ALL) ? NULL : pool_hdl, rpc->cr_ctx);
+				(flags & DAOS_COO_EVICT_ALL) ? NULL : pool_hdl);
 		if (rc != 0) {
 			daos_prop_free(prop);
 			goto out;
@@ -2451,8 +2449,7 @@ cont_open(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont, cr
 }
 
 static int
-cont_close_recs(crt_context_t ctx, struct cont_svc *svc,
-		struct cont_tgt_close_rec recs[], int nrecs)
+cont_close_recs(struct cont_svc *svc, struct cont_tgt_close_rec recs[], int nrecs)
 {
 	int	i;
 	int	rc = 0;
@@ -2492,7 +2489,7 @@ cont_close_recs(crt_context_t ctx, struct cont_svc *svc,
 
 static int
 cont_close_one_hdl(struct rdb_tx *tx, struct d_hash_table *nhc, struct cont_svc *svc,
-		   crt_context_t ctx, const uuid_t uuid)
+		   const uuid_t uuid)
 {
 	d_iov_t			key;
 	d_iov_t			value;
@@ -2529,8 +2526,7 @@ cont_close_one_hdl(struct rdb_tx *tx, struct d_hash_table *nhc, struct cont_svc
 
 /* Close an array of handles, possibly belonging to different containers. */
 static int
-cont_close_hdls(struct cont_svc *svc, struct cont_tgt_close_rec *recs,
-		int nrecs, crt_context_t ctx)
+cont_close_hdls(struct cont_svc *svc, struct cont_tgt_close_rec *recs, int nrecs)
 {
 	struct rdb_tx		tx;
 	struct d_hash_table	txs_nhc;	/* TX per-container number of handles cache (HT). */
@@ -2544,7 +2540,7 @@ cont_close_hdls(struct cont_svc *svc, struct cont_tgt_close_rec *recs,
 		" recs[0].hce="DF_U64"\n", DP_CONT(svc->cs_pool_uuid, NULL),
 		nrecs, DP_UUID(recs[0].tcr_hdl), recs[0].tcr_hce);
 
-	rc = cont_close_recs(ctx, svc, recs, nrecs);
+	rc = cont_close_recs(svc, recs, nrecs);
 	if (rc != 0) {
 		D_ERROR(DF_CONT": failed to close %d recs: "DF_RC"\n",
 			DP_CONT(svc->cs_pool_uuid, NULL), nrecs, DP_RC(rc));
@@ -2566,7 +2562,7 @@ cont_close_hdls(struct cont_svc *svc, struct cont_tgt_close_rec *recs,
 		DP_CONT(svc->cs_pool_uuid, NULL), num_tx);
 
 	for (i = 0; i < nrecs; i++) {
-		rc = cont_close_one_hdl(&tx, &txs_nhc, svc, ctx, recs[i].tcr_hdl);
+		rc = cont_close_one_hdl(&tx, &txs_nhc, svc, recs[i].tcr_hdl);
 		if (rc != 0) {
 			D_ERROR(DF_CONT": failed to close handle: "DF_UUID", "DF_RC"\n",
 				DP_CONT(svc->cs_pool_uuid, NULL), DP_UUID(recs[i].tcr_hdl),
@@ -2669,11 +2665,11 @@ cont_close(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont,
 		DP_CONT(cont->c_svc->cs_pool_uuid, in->cci_op.ci_uuid),
 		DP_UUID(rec.tcr_hdl), rec.tcr_hce);
 
-	rc = cont_close_recs(rpc->cr_ctx, cont->c_svc, &rec, 1 /* nrecs */);
+	rc = cont_close_recs(cont->c_svc, &rec, 1 /* nrecs */);
 	if (rc != 0)
 		D_GOTO(out, rc);
 
-	rc = cont_close_one_hdl(tx, NULL /* nhc */, cont->c_svc, rpc->cr_ctx, rec.tcr_hdl);
+	rc = cont_close_one_hdl(tx, NULL /* nhc */, cont->c_svc, rec.tcr_hdl);
 
 	/* On success update modify time (except if open specified read-only metadata stats) */
 	if (rc == 0 && !(chdl.ch_flags & DAOS_COO_RO_MDSTATS))
@@ -3330,7 +3326,7 @@ cont_query(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont *cont,
 	if (qbits & DAOS_CO_QUERY_TGT) {
 		/* need RF if user query cont_info */
 		qbits |= (DAOS_CO_QUERY_PROP_REDUN_FAC | DAOS_CO_QUERY_PROP_REDUN_LVL);
-		rc = cont_query_bcast(rpc->cr_ctx, cont, in->cqi_op.ci_pool_hdl,
+		rc = cont_query_bcast(dss_get_module_info()->dmi_ctx, cont, in->cqi_op.ci_pool_hdl,
 				      in->cqi_op.ci_hdl, out);
 		if (rc)
 			return rc;
@@ -4001,8 +3997,7 @@ close_iter_cb(daos_handle_t ih, d_iov_t *key, d_iov_t *val, void *varg)
  * and managed by local container services.
  */
 int
-ds_cont_close_by_pool_hdls(uuid_t pool_uuid, uuid_t *pool_hdls, int n_pool_hdls,
-			   crt_context_t ctx)
+ds_cont_close_by_pool_hdls(uuid_t pool_uuid, uuid_t *pool_hdls, int n_pool_hdls)
 {
 	struct cont_svc		       *svc;
 	struct rdb_tx			tx;
@@ -4040,7 +4035,7 @@ ds_cont_close_by_pool_hdls(uuid_t pool_uuid, uuid_t *pool_hdls, int n_pool_hdls,
 
 	if (arg.cia_buf.rb_nrecs > 0)
 		rc = cont_close_hdls(svc, arg.cia_buf.rb_recs,
-				     arg.cia_buf.rb_nrecs, ctx);
+				     arg.cia_buf.rb_nrecs);
 
 out_buf:
 	recs_buf_fini(&arg.cia_buf);

src/container/srv_epoch.c

@@ -310,7 +310,8 @@ ds_cont_snap_create(struct rdb_tx *tx, struct ds_pool_hdl *pool_hdl, struct cont
 
 	cont_epoch_op_in_get_data(rpc, CONT_SNAP_CREATE, cont_proto_ver, &snap_eph, &opts);
 
-	rc = snap_create_bcast(tx, cont, in->cei_op.ci_hdl, opts, rpc->cr_ctx, &snap_eph);
+	rc = snap_create_bcast(tx, cont, in->cei_op.ci_hdl, opts, dss_get_module_info()->dmi_ctx,
+			       &snap_eph);
 	if (rc == 0) {
 		out->ceo_epoch = snap_eph;
 		*(daos_epoch_t *)op_val->ov_resvd = snap_eph;

src/control/cmd/daos_agent/attachinfo.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2020-2021 Intel Corporation.
+// (C) Copyright 2020-2022 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -22,7 +22,8 @@ type dumpAttachInfoCmd struct {
 	configCmd
 	ctlInvokerCmd
 	cmdutil.JSONOutputCmd
-	Output string `short:"o" long:"output" default:"stdout" description:"Dump output to this location"`
+	Output      string `short:"o" long:"output" default:"stdout" description:"Dump output to this location"`
+	ProviderIdx *uint  `short:"n" long:"provider_idx" description:"Index of provider to fetch (if multiple)"`
 }
 
 func (cmd *dumpAttachInfoCmd) Execute(_ []string) error {
@@ -50,10 +51,21 @@ func (cmd *dumpAttachInfoCmd) Execute(_ []string) error {
 		return cmd.OutputJSON(resp, err)
 	}
 
+	providerIdx := cmd.cfg.ProviderIdx
+	if cmd.ProviderIdx != nil {
+		providerIdx = *cmd.ProviderIdx
+	}
+
+	ranks, err := getServiceRanksForProviderIdx(resp, int(providerIdx))
+	if err != nil {
+		return err
+	}
+
 	system := cmd.cfg.SystemName
 	if resp.System != "" {
 		system = resp.System
 	}
+
 	/**
 	 * cart/crt_group.c:crt_group_config_save()
 	 *
@@ -77,11 +89,37 @@ func (cmd *dumpAttachInfoCmd) Execute(_ []string) error {
 	 */
 	ew := txtfmt.NewErrWriter(out)
 	fmt.Fprintf(ew, "name %s\n", system)
-	fmt.Fprintf(ew, "size %d\n", len(resp.ServiceRanks))
+	fmt.Fprintf(ew, "size %d\n", len(ranks))
 	fmt.Fprintln(ew, "all")
-	for _, psr := range resp.ServiceRanks {
+	for _, psr := range ranks {
 		fmt.Fprintf(ew, "%d %s\n", psr.Rank, psr.Uri)
 	}
 
 	return ew.Err
 }
+
+func getServiceRanksForProviderIdx(inResp *control.GetAttachInfoResp, idx int) ([]*control.PrimaryServiceRank, error) {
+	if idx == 0 {
+		// Primary provider
+		return inResp.ServiceRanks, nil
+	}
+
+	secIdx := idx - 1
+	if secIdx < 0 || secIdx >= len(inResp.AlternateClientNetHints) {
+		return nil, errors.Errorf("provider index must be in range 0 <= idx <= %d", len(inResp.AlternateClientNetHints))
+	}
+
+	hint := inResp.AlternateClientNetHints[secIdx]
+	ranks := make([]*control.PrimaryServiceRank, 0)
+	for _, r := range inResp.AlternateServiceRanks {
+		if r.ProviderIdx == hint.ProviderIdx {
+			ranks = append(ranks, r)
+		}
+	}
+
+	if len(ranks) == 0 {
+		return nil, errors.Errorf("no ranks for provider %q (idx %d)", hint.Provider, idx)
+	}
+
+	return ranks, nil
+}

src/control/cmd/daos_agent/config.go

@@ -55,6 +55,7 @@ type Config struct {
 	DisableAutoEvict    bool                      `yaml:"disable_auto_evict,omitempty"`
 	ExcludeFabricIfaces common.StringSet          `yaml:"exclude_fabric_ifaces,omitempty"`
 	FabricInterfaces    []*NUMAFabricConfig       `yaml:"fabric_ifaces,omitempty"`
+	ProviderIdx         uint                      `yaml:"provider_idx"`
 }
 
 // NUMAFabricConfig defines a list of fabric interfaces that belong to a NUMA

src/control/cmd/daos_agent/config_test.go

@@ -68,6 +68,7 @@ fabric_ifaces:
   -
      iface: ib3
      domain: mlx5_3
+provider_idx: 1
 `)
 
 	badLogMaskCfg := test.CreateTestFile(t, dir, `
@@ -165,6 +166,7 @@ transport_config:
 						},
 					},
 				},
+				ProviderIdx: 1,
 			},
 		},
 	} {