feature/multiprovider

src/control/cmd/daos_agent/config.go

@@ -34,6 +34,7 @@ type Config struct {
 	LogLevel         common.ControlLogLevel    `yaml:"control_log_mask,omitempty"`
 	TransportConfig  *security.TransportConfig `yaml:"transport_config"`
 	FabricInterfaces []*NUMAFabricConfig       `yaml:"fabric_ifaces,omitempty"`
+	Provider         string                    `yaml:"provider,omitempty"`
 }
 
 // NUMAFabricConfig defines a list of fabric interfaces that belong to a NUMA

src/control/cmd/daos_agent/config_test.go

@@ -62,6 +62,7 @@ fabric_ifaces:
   -
      iface: ib3
      domain: mlx5_3
+provider: ofi+tcp
 `)
 
 	badLogMaskCfg := common.CreateTestFile(t, dir, `
@@ -155,6 +156,7 @@ transport_config:
 						},
 					},
 				},
+				Provider: "ofi+tcp",
 			},
 		},
 	} {

src/control/cmd/daos_agent/mgmt_rpc.go

@@ -35,6 +35,7 @@ type mgmtModule struct {
 	monitor        *procMon
 	useDefaultNUMA bool
 	numaGetter     hardware.ProcessNUMAProvider
+	provider       string
 }
 
 func (mod *mgmtModule) HandleCall(ctx context.Context, session *drpc.Session, method drpc.Method, req []byte) ([]byte, error) {
@@ -143,13 +144,19 @@ func (mod *mgmtModule) getNUMANode(ctx context.Context, pid int32) (uint, error)
 }
 
 func (mod *mgmtModule) getAttachInfo(ctx context.Context, numaNode int, sys string) (*mgmtpb.GetAttachInfoResp, error) {
-	resp, err := mod.getAttachInfoResp(ctx, numaNode, sys)
+	rawResp, err := mod.getAttachInfoResp(ctx, numaNode, sys)
 	if err != nil {
 		mod.log.Errorf("failed to fetch remote AttachInfo: %s", err.Error())
 		return nil, err
 	}
 
-	fabricIF, err := mod.getFabricInterface(ctx, numaNode, hardware.NetDevClass(resp.ClientNetHint.NetDevClass), resp.ClientNetHint.Provider)
+	resp, err := mod.getProviderAttachInfo(rawResp)
+	if err != nil {
+		return nil, err
+	}
+
+	fabricIF, err := mod.getFabricInterface(ctx, numaNode, hardware.NetDevClass(resp.ClientNetHint.NetDevClass),
+		resp.ClientNetHint.Provider)
 	if err != nil {
 		mod.log.Errorf("failed to fetch fabric interface of type %s: %s",
 			hardware.NetDevClass(resp.ClientNetHint.NetDevClass), err.Error())
@@ -171,6 +178,37 @@ func (mod *mgmtModule) getAttachInfoResp(ctx context.Context, numaNode int, sys
 	return mod.attachInfo.Get(ctx, numaNode, sys, mod.getAttachInfoRemote)
 }
 
+func (mod *mgmtModule) getProviderAttachInfo(srvResp *mgmtpb.GetAttachInfoResp) (*mgmtpb.GetAttachInfoResp, error) {
+	if mod.provider == "" || mod.provider == srvResp.ClientNetHint.Provider {
+		return srvResp, nil
+	}
+
+	uris := []*mgmtpb.GetAttachInfoResp_RankUri{}
+	for _, uri := range srvResp.SecondaryRankUris {
+		if uri.Provider == mod.provider {
+			uris = append(uris, uri)
+		}
+	}
+
+	if len(uris) == 0 {
+		return nil, errors.Errorf("no rank URIs for provider %q", mod.provider)
+	}
+
+	for _, hint := range srvResp.SecondaryClientNetHints {
+		if hint.Provider == mod.provider {
+
+			return &mgmtpb.GetAttachInfoResp{
+				Status:        srvResp.Status,
+				RankUris:      uris,
+				MsRanks:       srvResp.MsRanks,
+				ClientNetHint: hint,
+			}, nil
+		}
+	}
+
+	return nil, errors.Errorf("no ClientNetHint for provider %q", mod.provider)
+}
+
 func (mod *mgmtModule) getAttachInfoRemote(ctx context.Context, numaNode int, sys string) (*mgmtpb.GetAttachInfoResp, error) {
 	// Ask the MS for _all_ info, regardless of pbReq.AllRanks, so that the
 	// cache can serve future "pbReq.AllRanks == true" requests.