From ae889d1d2773bbecb82096b4b40170779f3aa492 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 24 Jan 2024 11:22:29 +0100 Subject: [PATCH 1/4] azure/options: add resource-group option can be used to deploy resources in an existing resource group Signed-off-by: Mathieu Tortuyaux --- cmd/kola/options.go | 1 + platform/api/azure/options.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index e68b94a55..39941c752 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -124,6 +124,7 @@ func init() { bv(&kola.AzureOptions.UsePrivateIPs, "azure-use-private-ips", false, "Assume nodes are reachable using private IP addresses") bv(&kola.AzureOptions.UseIdentity, "azure-identity", false, "Use VM managed identity for authentication (default false)") sv(&kola.AzureOptions.DiskController, "azure-disk-controller", "default", "Use a specific disk-controller for storage (default \"default\", also \"nvme\" and \"scsi\")") + sv(&kola.AzureOptions.ResourceGroup, "azure-resource-group", "", "Deploy resources in an existing resource group") // do-specific options sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")") diff --git a/platform/api/azure/options.go b/platform/api/azure/options.go index b8ae767f4..f94843def 100644 --- a/platform/api/azure/options.go +++ b/platform/api/azure/options.go @@ -51,6 +51,8 @@ type Options struct { // Azure Storage API endpoint suffix. If unset, the Azure SDK default will be used. StorageEndpointSuffix string - // UseUserData can be use to enable custom data only or user-data only. + // UseUserData can be used to enable custom data only or user-data only. UseUserData bool + // ResourceGroup is an existing resource group to deploy resources in. + ResourceGroup string } From a26c698f3797427458acd3c37555b4e83017e466 Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 24 Jan 2024 11:22:53 +0100 Subject: [PATCH 2/4] azure/options: add availability set option can be used to deploy instances in. Signed-off-by: Mathieu Tortuyaux --- cmd/kola/options.go | 1 + platform/api/azure/api.go | 4 ++++ platform/api/azure/options.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/cmd/kola/options.go b/cmd/kola/options.go index 39941c752..c904e3a3a 100644 --- a/cmd/kola/options.go +++ b/cmd/kola/options.go @@ -125,6 +125,7 @@ func init() { bv(&kola.AzureOptions.UseIdentity, "azure-identity", false, "Use VM managed identity for authentication (default false)") sv(&kola.AzureOptions.DiskController, "azure-disk-controller", "default", "Use a specific disk-controller for storage (default \"default\", also \"nvme\" and \"scsi\")") sv(&kola.AzureOptions.ResourceGroup, "azure-resource-group", "", "Deploy resources in an existing resource group") + sv(&kola.AzureOptions.AvailabilitySet, "azure-availability-set", "", "Deploy instances with an existing availibity set") // do-specific options sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")") diff --git a/platform/api/azure/api.go b/platform/api/azure/api.go index ff5be7c5e..e2ecf4328 100644 --- a/platform/api/azure/api.go +++ b/platform/api/azure/api.go @@ -154,6 +154,10 @@ func New(opts *Options) (*API, error) { client = management.NewAnonymousClient() } + if opts.AvailabilitySet != "" && opts.ResourceGroup == "" { + return nil, fmt.Errorf("ResourceGroup must match AvailabilitySet") + } + api := &API{ client: client, Opts: opts, diff --git a/platform/api/azure/options.go b/platform/api/azure/options.go index f94843def..f53f13595 100644 --- a/platform/api/azure/options.go +++ b/platform/api/azure/options.go @@ -55,4 +55,6 @@ type Options struct { UseUserData bool // ResourceGroup is an existing resource group to deploy resources in. ResourceGroup string + // AvailabilitySet is an existing availability set to deploy the instance in. + AvailabilitySet string } From fa39c84efd027fd3a3bb674954f26fb4419296ec Mon Sep 17 00:00:00 2001 From: Mathieu Tortuyaux Date: Wed, 24 Jan 2024 11:51:39 +0100 Subject: [PATCH 3/4] api/azure: inject availability set Signed-off-by: Mathieu Tortuyaux Signed-off-by: Jeremi Piotrowski --- platform/api/azure/instance.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/platform/api/azure/instance.go b/platform/api/azure/instance.go index b2b2339e2..0a620e086 100644 --- a/platform/api/azure/instance.go +++ b/platform/api/azure/instance.go @@ -39,6 +39,13 @@ type Machine struct { PublicIPName string } +func (a *API) getAvset() string { + if a.Opts.AvailabilitySet == "" { + return "" + } + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s", a.Opts.SubscriptionID, a.Opts.ResourceGroup, a.Opts.AvailabilitySet) +} + func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, ip *network.PublicIPAddress, nic *network.Interface) compute.VirtualMachine { osProfile := compute.OSProfile{ AdminUsername: util.StrToPtr("core"), @@ -156,6 +163,11 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, } } + availabilitySetID := a.getAvset() + if availabilitySetID != "" { + vm.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: &availabilitySetID} + } + return vm } From 7f3319e1f6a776f1d2a3fcaf424d92b962119073 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Wed, 7 Feb 2024 11:48:08 +0100 Subject: [PATCH 4/4] api/azure: Reuse resource group only for VMs The remaining objects stay in a garbage collected kola managed group. This allows us to use an availability set for the vms, which requires that VMs be in the same rg as the avset. Kola correctly removes the VMs through TerminateInstance() separately from any RG that it creates. Signed-off-by: Jeremi Piotrowski --- platform/api/azure/instance.go | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/platform/api/azure/instance.go b/platform/api/azure/instance.go index 0a620e086..acfb62cbf 100644 --- a/platform/api/azure/instance.go +++ b/platform/api/azure/instance.go @@ -46,6 +46,14 @@ func (a *API) getAvset() string { return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s", a.Opts.SubscriptionID, a.Opts.ResourceGroup, a.Opts.AvailabilitySet) } +func (a *API) getVMRG(rg string) string { + vmrg := rg + if a.Opts.ResourceGroup != "" { + vmrg = a.Opts.ResourceGroup + } + return vmrg +} + func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, ip *network.PublicIPAddress, nic *network.Interface) compute.VirtualMachine { osProfile := compute.OSProfile{ AdminUsername: util.StrToPtr("core"), @@ -172,6 +180,9 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string, } func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccount string, network Network) (*Machine, error) { + // only VMs are created in the user supplied resource group, kola still manages a resource group + // for the gallery and storage account. + vmResourceGroup := a.getVMRG(resourceGroup) subnet := network.subnet ip, err := a.createPublicIP(resourceGroup) @@ -193,22 +204,31 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou vmParams := a.getVMParameters(name, userdata, sshkey, fmt.Sprintf("https://%s.blob.core.windows.net/", storageAccount), ip, nic) plog.Infof("Creating Instance %s", name) - future, err := a.compClient.CreateOrUpdate(context.TODO(), resourceGroup, name, vmParams) + clean := func() { + _, _ = a.compClient.Delete(context.TODO(), vmResourceGroup, name, &forceDelete) + _, _ = a.intClient.Delete(context.TODO(), resourceGroup, *nic.Name) + _, _ = a.ipClient.Delete(context.TODO(), resourceGroup, *ip.Name) + } + + future, err := a.compClient.CreateOrUpdate(context.TODO(), vmResourceGroup, name, vmParams) if err != nil { + clean() return nil, err } err = future.WaitForCompletionRef(context.TODO(), a.compClient.Client) if err != nil { + clean() return nil, err } _, err = future.Result(a.compClient) if err != nil { + clean() return nil, err } plog.Infof("Instance %s created", name) err = util.WaitUntilReady(5*time.Minute, 10*time.Second, func() (bool, error) { - vm, err := a.compClient.Get(context.TODO(), resourceGroup, name, "") + vm, err := a.compClient.Get(context.TODO(), vmResourceGroup, name, "") if err != nil { return false, err } @@ -221,13 +241,11 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou }) plog.Infof("Instance %s ready", name) if err != nil { - _, _ = a.compClient.Delete(context.TODO(), resourceGroup, name, &forceDelete) - _, _ = a.intClient.Delete(context.TODO(), resourceGroup, *nic.Name) - _, _ = a.ipClient.Delete(context.TODO(), resourceGroup, *ip.Name) + clean() return nil, fmt.Errorf("waiting for machine to become active: %v", err) } - vm, err := a.compClient.Get(context.TODO(), resourceGroup, name, "") + vm, err := a.compClient.Get(context.TODO(), vmResourceGroup, name, "") if err != nil { return nil, err } @@ -257,6 +275,7 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou // TerminateInstance deletes a VM created by CreateInstance. Public IP, NIC and // OS disk are deleted automatically together with the VM. func (a *API) TerminateInstance(machine *Machine, resourceGroup string) error { + resourceGroup = a.getVMRG(resourceGroup) future, err := a.compClient.Delete(context.TODO(), resourceGroup, machine.ID, &forceDelete) if err != nil { return err @@ -284,7 +303,8 @@ func (a *API) GetConsoleOutput(name, resourceGroup, storageAccount string) ([]by k := *kr.Keys key := *k[0].Value - vm, err := a.compClient.Get(context.TODO(), resourceGroup, name, compute.InstanceViewTypesInstanceView) + vmResourceGroup := a.getVMRG(resourceGroup) + vm, err := a.compClient.Get(context.TODO(), vmResourceGroup, name, compute.InstanceViewTypesInstanceView) if err != nil { return nil, fmt.Errorf("could not get VM: %v", err) }