Attempt to reduce deployment failures due to ingress configuration delays (#739)

Author: BMurri

src/deploy-tes-on-azure/Deployer.cs

@@ -597,9 +597,13 @@ await PerformHelmDeploymentAsync(resourceGroup,
                             await ExecuteQueriesOnAzurePostgreSQLDbFromK8(kubernetesClient, deploymentName, deploymentNamespace);
                             await kubernetesClient.AppsV1.DeleteNamespacedDeploymentAsync(deploymentName, deploymentNamespace, cancellationToken: cts.Token);
 
-
                             if (configuration.EnableIngress.GetValueOrDefault())
                             {
+                                var tmpValues = await kubernetesManager.ConfigureAltLocalValuesYamlAsync("no-ingress.yml", values => values.Service["enableIngress"] = $"{false}");
+                                var backupValues = kubernetesManager.SwapLocalValuesYaml(tmpValues);
+                                await kubernetesManager.DeployHelmChartToClusterAsync(kubernetesClient);
+                                kubernetesManager.RestoreLocalValuesYaml(backupValues);
+
                                 await Execute(
                                     $"Enabling Ingress {kubernetesManager.TesHostname}",
                                     async () =>

src/deploy-tes-on-azure/KubernetesManager.cs

@@ -231,6 +231,7 @@ await ExecHelmProcessAsync($"upgrade --install tesonazure ./helm --kubeconfig \"
                 workingDirectory: workingDirectoryTemp);
             await WaitForWorkloadAsync(kubernetesClient, "tes", configuration.AksCoANamespace, cancellationToken);
         }
+
         public async Task RemovePodAadChart()
         {
             await ExecHelmProcessAsync($"uninstall aad-pod-identity", throwOnNonZeroExitCode: false);
@@ -300,6 +301,27 @@ public async Task UpgradeValuesYamlAsync(IStorageAccount storageAccount, Diction
             await Deployer.UploadTextToStorageAccountAsync(storageAccount, Deployer.ConfigurationContainerName, "aksValues.yaml", valuesString, cancellationToken);
         }
 
+        public async Task<FileInfo> ConfigureAltLocalValuesYamlAsync(string altName, Action<HelmValues> configure)
+        {
+            FileInfo altValues = new(Path.Combine(Path.GetDirectoryName(TempHelmValuesYamlPath), altName));
+            var values = KubernetesYaml.Deserialize<HelmValues>(await File.ReadAllTextAsync(TempHelmValuesYamlPath, cancellationToken));
+            configure(values);
+            await File.WriteAllTextAsync(altValues.FullName, KubernetesYaml.Serialize(values), cancellationToken);
+            return altValues;
+        }
+
+        public FileInfo SwapLocalValuesYaml(FileInfo file)
+        {
+            FileInfo backup = new(Path.Combine(file.DirectoryName, "backup.bak"));
+            File.Replace(file.FullName, TempHelmValuesYamlPath, backup.FullName);
+            return backup;
+        }
+
+        public void RestoreLocalValuesYaml(FileInfo backup)
+        {
+            File.Replace(backup.FullName, TempHelmValuesYamlPath, default);
+        }
+
         public async Task<Dictionary<string, string>> GetAKSSettingsAsync(IStorageAccount storageAccount)
         {
             var values = KubernetesYaml.Deserialize<HelmValues>(await Deployer.DownloadTextFromStorageAccountAsync(storageAccount, Deployer.ConfigurationContainerName, "aksValues.yaml", cancellationToken));