Merge pull request #50 from ArchetypicalSoftware/hotfix/dockerhub-local

This pull request introduces improvements to the Kubernetes cluster creation process and enhances the robustness of the Flux deployment workflow. The main changes include updating the default Kubernetes version, ensuring proper configuration file handling for containerd, and adding a mechanism to wait for all Flux kustomizations to be ready before proceeding.

Author: ewassef

.claude/settings.local.json

@@ -2,7 +2,17 @@
   "permissions": {
     "allow": [
       "Bash(docker stop:*)",
-      "Bash(docker rm:*)"
+      "Bash(docker rm:*)",
+      "Bash(docker run:*)",
+      "Bash(MSYS_NO_PATHCONV=1 docker run:*)",
+      "Bash(MSYS_NO_PATHCONV=1 docker exec:*)",
+      "Bash(for node in idp-worker idp-worker2)",
+      "Bash(do:*)",
+      "Bash(echo:*)",
+      "Bash(done)",
+      "Bash(for node in idp-control-plane idp-worker idp-worker2)",
+      "Bash(kubectl --context kind-idp get pods:*)",
+      "Bash(kubectl --context kind-idp delete pod:*)"
     ]
   }
 }

cli/src/Vdk/Commands/CreateClusterCommand.cs

@@ -51,7 +51,7 @@ public CreateClusterCommand(
         controlNodes.Aliases.Add("-c");
         var workers = new Option<int>("--Workers") { DefaultValueFactory = _ => Defaults.WorkerNodes, Description = "The number of worker nodes in the cluster." };
         workers.Aliases.Add("-w");
-        var kubeVersion = new Option<string>("--KubeVersion") { DefaultValueFactory = _ => "1.29", Description = "The kubernetes api version." };
+        var kubeVersion = new Option<string>("--KubeVersion") { DefaultValueFactory = _ => "1.32", Description = "The kubernetes api version." };
         kubeVersion.Aliases.Add("-k");
 
         Options.Add(nameOption);
@@ -111,6 +111,16 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla
             return;
         }
 
+        // Write hosts.toml to a temp file for containerd registry config
+        // This ensures the file is accessible to Docker regardless of working directory
+        var hostsTomlContent = """
+            server = "http://host.docker.internal:5000"
+            [host."http://host.docker.internal:5000"]
+              capabilities = ["pull", "resolve"]
+            """;
+        var hostsTomlPath = _fileSystem.Path.Combine(_fileSystem.Path.GetTempPath(), $"hosts-{Guid.NewGuid()}.toml");
+        await _fileSystem.File.WriteAllTextAsync(hostsTomlPath, hostsTomlContent);
+
         var cluster = new KindCluster();
         for (int index = 0; index < controlPlaneNodes; index++)
         {
@@ -127,7 +137,7 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla
             {
                 new()
                 {
-                    HostPath = _fileSystem.FileInfo.New("ConfigMounts/hosts.toml").FullName,
+                    HostPath = hostsTomlPath,
                     ContainerPath = "/etc/containerd/certs.d/hub.dev-k8s.cloud/hosts.toml"
                 }
             };
@@ -144,7 +154,7 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla
                 {
                     new()
                     {
-                        HostPath = _fileSystem.FileInfo.New("ConfigMounts/hosts.toml").FullName,
+                        HostPath = hostsTomlPath,
                         ContainerPath = "/etc/containerd/certs.d/hub.dev-k8s.cloud/hosts.toml"
                     }
                 }
@@ -194,6 +204,10 @@ public async Task InvokeAsync(string name = Defaults.ClusterName, int controlPla
         }
 
         _flux.Bootstrap(name.ToLower(), "./clusters/default", branch: "main");
+
+        // Wait for all Flux kustomizations to reconcile before configuring the reverse proxy
+        _flux.WaitForKustomizations(name.ToLower());
+
         try
         {
             _reverseProxy.UpsertCluster(name.ToLower(), masterNode.ExtraPortMappings.First().HostPort,

cli/src/Vdk/Services/FluxClient.cs

@@ -1,4 +1,5 @@
 using System.Net;
+using System.Text.Json;
 using k8s;
 using k8s.Autorest;
 using k8s.Models;
@@ -114,6 +115,91 @@ public void Bootstrap(string clusterName, string path, string branch = DefaultBr
         }
 
         _console.WriteLine("Flux bootstrap complete.");
-        
+
+    }
+
+    public bool WaitForKustomizations(string clusterName, int maxAttempts = 60, int delaySeconds = 5)
+    {
+        _console.WriteLine("Waiting for Flux kustomizations to reconcile...");
+
+        for (int attempt = 0; attempt < maxAttempts; attempt++)
+        {
+            try
+            {
+                var result = _client(clusterName).ApiClient.CustomObjects
+                    .ListNamespacedCustomObject(
+                        "kustomize.toolkit.fluxcd.io", "v1",
+                        "flux-system", "kustomizations");
+
+                var json = JsonSerializer.Serialize(result);
+                using var doc = JsonDocument.Parse(json);
+                var items = doc.RootElement.GetProperty("items");
+
+                if (items.GetArrayLength() == 0)
+                {
+                    if (attempt % 5 == 0)
+                        _console.WriteLine("  No kustomizations found yet. Waiting...");
+                    Thread.Sleep(delaySeconds * 1000);
+                    continue;
+                }
+
+                int total = items.GetArrayLength();
+                int readyCount = 0;
+
+                foreach (var item in items.EnumerateArray())
+                {
+                    var name = item.GetProperty("metadata").GetProperty("name").GetString();
+                    bool isReady = false;
+
+                    if (item.TryGetProperty("status", out var status) &&
+                        status.TryGetProperty("conditions", out var conditions))
+                    {
+                        foreach (var condition in conditions.EnumerateArray())
+                        {
+                            if (condition.GetProperty("type").GetString() == "Ready")
+                            {
+                                var condStatus = condition.GetProperty("status").GetString();
+                                if (condStatus == "True")
+                                {
+                                    isReady = true;
+                                }
+                                else if (attempt % 5 == 0)
+                                {
+                                    var reason = condition.TryGetProperty("reason", out var r)
+                                        ? r.GetString() : "Unknown";
+                                    var message = condition.TryGetProperty("message", out var m)
+                                        ? m.GetString() : "";
+                                    _console.WriteLine(
+                                        $"  Kustomization '{name}' not ready: {reason} - {message}");
+                                }
+                                break;
+                            }
+                        }
+                    }
+
+                    if (isReady) readyCount++;
+                }
+
+                if (attempt % 5 == 0 || readyCount == total)
+                    _console.WriteLine($"  Kustomizations ready: {readyCount}/{total}");
+
+                if (readyCount == total)
+                {
+                    _console.WriteLine("All Flux kustomizations are ready.");
+                    return true;
+                }
+            }
+            catch (Exception ex)
+            {
+                if (attempt % 5 == 0)
+                    _console.WriteLine($"  Error checking kustomizations: {ex.Message}. Retrying...");
+            }
+
+            Thread.Sleep(delaySeconds * 1000);
+        }
+
+        _console.WriteWarning(
+            "Timed out waiting for Flux kustomizations to reconcile. Proceeding anyway...");
+        return false;
     }
 }

cli/src/Vdk/Services/IFluxClient.cs

@@ -3,4 +3,5 @@ namespace Vdk.Services;
 public interface IFluxClient
 {
     void Bootstrap(string clusterName, string path, string branch = FluxClient.DefaultBranch);
+    bool WaitForKustomizations(string clusterName, int maxAttempts = 60, int delaySeconds = 5);
 }