Skip to content

Commit fb2576a

Browse files
NautiluXNautiluX
committed
wip refactor command execution
1 parent 34e9117 commit fb2576a

File tree

6 files changed

+2019
-3178
lines changed

6 files changed

+2019
-3178
lines changed

‎cli/cmd/bootstrap_gcp.go‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func (c *BootstrapGcpCmd) RunE(_ *cobra.Command, args []string) error {
3838
return nil
3939
}
4040

41-
func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) {
41+
func AddBootstrapGcpCmd(parent *cobra.Command, opts *GlobalOptions) {
4242
bootstrapGcpCmd := BootstrapGcpCmd{
4343
cmd: &cobra.Command{
4444
Use: "bootstrap-gcp",
@@ -53,6 +53,7 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) {
5353
Env: env.NewEnv(),
5454
CodesphereEnv: &gcp.CodesphereEnvironment{},
5555
}
56+
bootstrapGcpCmd.cmd.RunE = bootstrapGcpCmd.RunE
5657

5758
flags := bootstrapGcpCmd.cmd.Flags()
5859
flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.ProjectName, "project-name", "", "Unique GCP Project Name (required)")
@@ -82,8 +83,8 @@ func AddBootstrapGcpCmd(root *cobra.Command, opts *GlobalOptions) {
8283
util.MarkFlagRequired(bootstrapGcpCmd.cmd, "billing-account")
8384
util.MarkFlagRequired(bootstrapGcpCmd.cmd, "base-domain")
8485

85-
bootstrapGcpCmd.cmd.RunE = bootstrapGcpCmd.RunE
86-
root.AddCommand(bootstrapGcpCmd.cmd)
86+
parent.AddCommand(bootstrapGcpCmd.cmd)
87+
AddBootstrapGcpPostconfigCmd(bootstrapGcpCmd.cmd, opts)
8788
}
8889

8990
func (c *BootstrapGcpCmd) BootstrapGcp() error {
@@ -92,9 +93,8 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error {
9293
icg := installer.NewInstallConfigManager()
9394
gcpClient := gcp.NewGCPClient(ctx, stlog, os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
9495
fw := util.NewFilesystemWriter()
95-
nm := node.NewNode(fw, c.CodesphereEnv.SSHPrivateKeyPath, c.SSHQuiet)
9696

97-
bs, err := gcp.NewGCPBootstrapper(ctx, c.Env, stlog, c.CodesphereEnv, icg, gcpClient, nm, fw)
97+
bs, err := gcp.NewGCPBootstrapper(ctx, c.Env, stlog, c.CodesphereEnv, icg, gcpClient, fw, node.NewSSHNodeClient(c.SSHQuiet))
9898
if err != nil {
9999
return err
100100
}
@@ -103,21 +103,34 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error {
103103

104104
err = bs.Bootstrap()
105105
envBytes, err2 := json.MarshalIndent(bs.Env, "", " ")
106+
106107
envString := string(envBytes)
107108
if err2 != nil {
108109
envString = ""
109110
}
111+
110112
if err != nil {
111-
if bs.Env.Jumpbox != nil && bs.Env.Jumpbox.GetExternalIP() != "" {
113+
if bs.Env.Jumpbox.GetExternalIP() != "" {
112114
log.Printf("To debug on the jumpbox host:\nssh-add $SSH_KEY_PATH; ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o SendEnv=OMS_PORTAL_API_KEY root@%s", bs.Env.Jumpbox.GetExternalIP())
113115
}
114116
return fmt.Errorf("failed to bootstrap GCP: %w, env: %s", err, envString)
115117
}
116118

119+
workdir := env.NewEnv().GetOmsWorkdir()
120+
err = fw.MkdirAll(workdir, 0755)
121+
if err != nil {
122+
return fmt.Errorf("failed to create workdir: %w", err)
123+
}
124+
infraFilePath := gcp.GetInfraFilePath()
125+
err = fw.WriteFile(infraFilePath, envBytes, 0644)
126+
if err != nil {
127+
return fmt.Errorf("failed to write gcp bootstrap env file: %w", err)
128+
}
129+
117130
log.Println("\n🎉🎉🎉 GCP infrastructure bootstrapped successfully!")
118131
log.Println(envString)
132+
log.Printf("Infrastructure details written to %s", infraFilePath)
119133
log.Printf("Start the Codesphere installation using OMS from the jumpbox host:\nssh-add $SSH_KEY_PATH; ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o SendEnv=OMS_PORTAL_API_KEY root@%s", bs.Env.Jumpbox.GetExternalIP())
120-
log.Printf("When the installation is done, run the k0s configuration script generated at the k0s-1 host %s /root/configure-k0s.sh.", bs.Env.ControlPlaneNodes[0].GetInternalIP())
121134

122-
return err
135+
return nil
123136
}

‎cli/cmd/bootstrap_gcp_postconfig.go‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,22 @@
11
package cmd
22

33
import (
4+
"encoding/json"
5+
"fmt"
46
"log"
57

68
"github.com/codesphere-cloud/cs-go/pkg/io"
9+
"github.com/codesphere-cloud/oms/internal/bootstrap/gcp"
10+
"github.com/codesphere-cloud/oms/internal/installer"
11+
"github.com/codesphere-cloud/oms/internal/util"
712
"github.com/spf13/cobra"
813
)
914

1015
type BootstrapGcpPostconfigCmd struct {
1116
cmd *cobra.Command
1217

13-
Opts *BootstrapGcpPostconfigOpts
18+
Opts *BootstrapGcpPostconfigOpts
19+
CodesphereEnv gcp.CodesphereEnvironment
1420
}
1521

1622
type BootstrapGcpPostconfigOpts struct {
@@ -22,7 +28,26 @@ type BootstrapGcpPostconfigOpts struct {
2228
func (c *BootstrapGcpPostconfigCmd) RunE(_ *cobra.Command, args []string) error {
2329
log.Printf("running post-configuration steps...")
2430

25-
return nil
31+
icg := installer.NewInstallConfigManager()
32+
33+
fw := util.NewFilesystemWriter()
34+
35+
envFileContent, err := fw.ReadFile(gcp.GetInfraFilePath())
36+
if err != nil {
37+
return fmt.Errorf("failed to read gcp infra file: %w", err)
38+
}
39+
40+
err = json.Unmarshal(envFileContent, &c.CodesphereEnv)
41+
if err != nil {
42+
return fmt.Errorf("failed to unmarshal gcp infra file: %w", err)
43+
}
44+
45+
err = icg.LoadInstallConfigFromFile(c.Opts.InstallConfigPath)
46+
if err != nil {
47+
return fmt.Errorf("failed to load config file: %w", err)
48+
}
49+
50+
return fmt.Errorf("not implemented: run config script on k0s-1 node to install GCP CCM")
2651
}
2752

2853
func AddBootstrapGcpPostconfigCmd(bootstrapGcp *cobra.Command, opts *GlobalOptions) {

‎internal/bootstrap/gcp/gcp.go‎

Lines changed: 68 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -54,34 +54,33 @@ var vmDefs = []VMDef{
5454
}
5555

5656
type GCPBootstrapper struct {
57-
ctx context.Context
58-
stlog *bootstrap.StepLogger
59-
fw util.FileIO
60-
icg installer.InstallConfigManager
61-
NodeManager node.NodeManager
62-
GCPClient GCPClientManager
57+
ctx context.Context
58+
stlog *bootstrap.StepLogger
59+
fw util.FileIO
60+
icg installer.InstallConfigManager
61+
GCPClient GCPClientManager
6362
// Environment
6463
Env *CodesphereEnvironment
65-
// SSH options
66-
sshQuiet bool
64+
// SSH command runner
65+
CommandRunner node.NodeClient
6766
}
6867

6968
type CodesphereEnvironment struct {
70-
ProjectID string `json:"project_id"`
71-
ProjectName string `json:"project_name"`
72-
DNSProjectID string `json:"dns_project_id"`
73-
Jumpbox node.NodeManager `json:"jumpbox"`
74-
PostgreSQLNode node.NodeManager `json:"postgresql_node"`
75-
ControlPlaneNodes []node.NodeManager `json:"control_plane_nodes"`
76-
CephNodes []node.NodeManager `json:"ceph_nodes"`
77-
ContainerRegistryURL string `json:"-"`
78-
ExistingConfigUsed bool `json:"-"`
79-
InstallCodesphereVersion string `json:"install_codesphere_version"`
80-
Preemptible bool `json:"preemptible"`
81-
WriteConfig bool `json:"-"`
82-
GatewayIP string `json:"gateway_ip"`
83-
PublicGatewayIP string `json:"public_gateway_ip"`
84-
RegistryType RegistryType `json:"registry_type"`
69+
ProjectID string `json:"project_id"`
70+
ProjectName string `json:"project_name"`
71+
DNSProjectID string `json:"dns_project_id"`
72+
Jumpbox *node.Node `json:"jumpbox"`
73+
PostgreSQLNode *node.Node `json:"postgres_node"`
74+
ControlPlaneNodes []*node.Node `json:"control_plane_nodes"`
75+
CephNodes []*node.Node `json:"ceph_nodes"`
76+
ContainerRegistryURL string `json:"-"`
77+
ExistingConfigUsed bool `json:"-"`
78+
InstallCodesphereVersion string `json:"install_codesphere_version"`
79+
Preemptible bool `json:"preemptible"`
80+
WriteConfig bool `json:"-"`
81+
GatewayIP string `json:"gateway_ip"`
82+
PublicGatewayIP string `json:"public_gateway_ip"`
83+
RegistryType RegistryType `json:"registry_type"`
8584

8685
// Config
8786
InstallConfigPath string `json:"-"`
@@ -106,19 +105,31 @@ type CodesphereEnvironment struct {
106105
DNSZoneName string `json:"dns_zone_name"`
107106
}
108107

109-
func NewGCPBootstrapper(ctx context.Context, env env.Env, stlog *bootstrap.StepLogger, CodesphereEnv *CodesphereEnvironment, icg installer.InstallConfigManager, gcpClient GCPClientManager, nm node.NodeManager, fw util.FileIO) (*GCPBootstrapper, error) {
108+
func NewGCPBootstrapper(
109+
ctx context.Context,
110+
env env.Env,
111+
stlog *bootstrap.StepLogger,
112+
CodesphereEnv *CodesphereEnvironment,
113+
icg installer.InstallConfigManager,
114+
gcpClient GCPClientManager,
115+
fw util.FileIO,
116+
sshRunner node.NodeClient) (*GCPBootstrapper, error) {
110117
return &GCPBootstrapper{
111-
ctx: ctx,
112-
stlog: stlog,
113-
fw: fw,
114-
icg: icg,
115-
GCPClient: gcpClient,
116-
NodeManager: nm,
117-
Env: CodesphereEnv,
118-
sshQuiet: true,
118+
ctx: ctx,
119+
stlog: stlog,
120+
fw: fw,
121+
icg: icg,
122+
GCPClient: gcpClient,
123+
Env: CodesphereEnv,
124+
CommandRunner: sshRunner,
119125
}, nil
120126
}
121127

128+
func GetInfraFilePath() string {
129+
workdir := env.NewEnv().GetOmsWorkdir()
130+
return fmt.Sprintf("%s/gcp-infra.json", workdir)
131+
}
132+
122133
func (b *GCPBootstrapper) Bootstrap() error {
123134
err := b.stlog.Step("Ensure install config", b.EnsureInstallConfig)
124135
if err != nil {
@@ -659,18 +670,20 @@ func (b *GCPBootstrapper) EnsureComputeInstances() error {
659670
}
660671

661672
// Create nodes from results (in main goroutine, not in spawned goroutines)
673+
b.Env.Jumpbox = &node.Node{
674+
NodeClient: b.CommandRunner,
675+
}
662676
for result := range resultCh {
663677
switch result.vmType {
664678
case "jumpbox":
665-
b.NodeManager.UpdateNode(result.name, result.externalIP, result.internalIP)
666-
b.Env.Jumpbox = b.NodeManager
679+
b.Env.Jumpbox.UpdateNode(result.name, result.externalIP, result.internalIP)
667680
case "postgres":
668-
b.Env.PostgreSQLNode = b.NodeManager.CreateSubNode(result.name, result.externalIP, result.internalIP)
681+
b.Env.PostgreSQLNode = b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)
669682
case "ceph":
670-
node := b.NodeManager.CreateSubNode(result.name, result.externalIP, result.internalIP)
683+
node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)
671684
b.Env.CephNodes = append(b.Env.CephNodes, node)
672685
case "k0s":
673-
node := b.NodeManager.CreateSubNode(result.name, result.externalIP, result.internalIP)
686+
node := b.Env.Jumpbox.CreateSubNode(result.name, result.externalIP, result.internalIP)
674687
b.Env.ControlPlaneNodes = append(b.Env.ControlPlaneNodes, node)
675688
}
676689
}
@@ -734,7 +747,7 @@ func (b *GCPBootstrapper) EnsureExternalIP(name string) (string, error) {
734747
}
735748

736749
func (b *GCPBootstrapper) EnsureRootLoginEnabled() error {
737-
allNodes := []node.NodeManager{
750+
allNodes := []*node.Node{
738751
b.Env.Jumpbox,
739752
}
740753
allNodes = append(allNodes, b.Env.ControlPlaneNodes...)
@@ -753,8 +766,8 @@ func (b *GCPBootstrapper) EnsureRootLoginEnabled() error {
753766
return nil
754767
}
755768

756-
func (b *GCPBootstrapper) ensureRootLoginEnabledInNode(node node.NodeManager) error {
757-
err := node.WaitForSSH(30 * time.Second)
769+
func (b *GCPBootstrapper) ensureRootLoginEnabledInNode(node *node.Node) error {
770+
err := node.NodeClient.WaitReady(node, 30*time.Second)
758771
if err != nil {
759772
return fmt.Errorf("timed out waiting for SSH service to start on %s: %w", node.GetName(), err)
760773
}
@@ -829,7 +842,7 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
829842
// Figure out if registry is already running
830843
b.stlog.Logf("Checking if local container registry is already running on postgres node")
831844
checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"`
832-
err := b.Env.PostgreSQLNode.RunSSHCommand("root", checkCommand, b.sshQuiet)
845+
err := b.Env.PostgreSQLNode.RunSSHCommand("root", checkCommand)
833846
if err == nil && b.Env.InstallConfig.Registry != nil && b.Env.InstallConfig.Registry.Server == localRegistryServer &&
834847
b.Env.InstallConfig.Registry.Username != "" && b.Env.InstallConfig.Registry.Password != "" {
835848
b.stlog.Logf("Local container registry already running on postgres node")
@@ -863,7 +876,7 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
863876
}
864877
for _, cmd := range commands {
865878
b.stlog.Logf("Running command on postgres node: %s", util.Truncate(cmd, 12))
866-
err := b.Env.PostgreSQLNode.RunSSHCommand("root", cmd, b.sshQuiet)
879+
err := b.Env.PostgreSQLNode.RunSSHCommand("root", cmd)
867880
if err != nil {
868881
return fmt.Errorf("failed to run command on postgres node: %w", err)
869882
}
@@ -872,15 +885,15 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error {
872885
allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...)
873886
for _, node := range allNodes {
874887
b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName())
875-
err := b.Env.PostgreSQLNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt", b.sshQuiet)
888+
err := b.Env.PostgreSQLNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt")
876889
if err != nil {
877890
return fmt.Errorf("failed to copy registry certificate to node %s: %w", node.GetInternalIP(), err)
878891
}
879-
err = node.RunSSHCommand("root", "update-ca-certificates", b.sshQuiet)
892+
err = node.RunSSHCommand("root", "update-ca-certificates")
880893
if err != nil {
881894
return fmt.Errorf("failed to update CA certificates on node %s: %w", node.GetInternalIP(), err)
882895
}
883-
err = node.RunSSHCommand("root", "systemctl restart docker.service || true", true) // docker is probably not yet installed
896+
err = node.RunSSHCommand("root", "systemctl restart docker.service || true") // docker is probably not yet installed
884897
if err != nil {
885898
return fmt.Errorf("failed to restart docker service on node %s: %w", node.GetInternalIP(), err)
886899
}
@@ -1128,25 +1141,25 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error {
11281141
return fmt.Errorf("failed to write vault file: %w", err)
11291142
}
11301143

1131-
err := b.Env.Jumpbox.CopyFile(b.Env.InstallConfigPath, "/etc/codesphere/config.yaml")
1144+
err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallConfigPath, "/etc/codesphere/config.yaml")
11321145
if err != nil {
11331146
return fmt.Errorf("failed to copy install config to jumpbox: %w", err)
11341147
}
11351148

1136-
err = b.Env.Jumpbox.CopyFile(b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml")
1149+
err = b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.SecretsFilePath, b.Env.SecretsDir+"/prod.vault.yaml")
11371150
if err != nil {
11381151
return fmt.Errorf("failed to copy secrets file to jumpbox: %w", err)
11391152
}
11401153
return nil
11411154
}
11421155

11431156
func (b *GCPBootstrapper) EnsureAgeKey() error {
1144-
hasKey := b.Env.Jumpbox.HasFile(b.Env.SecretsDir + "/age_key.txt")
1157+
hasKey := b.Env.Jumpbox.NodeClient.HasFile(b.Env.Jumpbox, b.Env.SecretsDir+"/age_key.txt")
11451158
if hasKey {
11461159
return nil
11471160
}
11481161

1149-
err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s/age_key.txt", b.Env.SecretsDir, b.Env.SecretsDir), b.sshQuiet)
1162+
err := b.Env.Jumpbox.RunSSHCommand("root", fmt.Sprintf("mkdir -p %s; age-keygen -o %s/age_key.txt", b.Env.SecretsDir, b.Env.SecretsDir))
11501163
if err != nil {
11511164
return fmt.Errorf("failed to generate age key on jumpbox: %w", err)
11521165
}
@@ -1155,12 +1168,12 @@ func (b *GCPBootstrapper) EnsureAgeKey() error {
11551168
}
11561169

11571170
func (b *GCPBootstrapper) EncryptVault() error {
1158-
err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml{,.bak}", b.sshQuiet)
1171+
err := b.Env.Jumpbox.RunSSHCommand("root", "cp "+b.Env.SecretsDir+"/prod.vault.yaml{,.bak}")
11591172
if err != nil {
11601173
return fmt.Errorf("failed backup vault on jumpbox: %w", err)
11611174
}
11621175

1163-
err = b.Env.Jumpbox.RunSSHCommand("root", "sops --encrypt --in-place --age $(age-keygen -y "+b.Env.SecretsDir+"/age_key.txt) "+b.Env.SecretsDir+"/prod.vault.yaml", b.sshQuiet)
1176+
err = b.Env.Jumpbox.RunSSHCommand("root", "sops --encrypt --in-place --age $(age-keygen -y "+b.Env.SecretsDir+"/age_key.txt) "+b.Env.SecretsDir+"/prod.vault.yaml")
11641177
if err != nil {
11651178
return fmt.Errorf("failed to encrypt vault on jumpbox: %w", err)
11661179
}
@@ -1216,12 +1229,12 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error {
12161229
}
12171230

12181231
func (b *GCPBootstrapper) InstallCodesphere() error {
1219-
err := b.Env.Jumpbox.RunSSHCommand("root", "oms-cli download package "+b.Env.InstallCodesphereVersion, b.sshQuiet)
1232+
err := b.Env.Jumpbox.RunSSHCommand("root", "oms-cli download package "+b.Env.InstallCodesphereVersion)
12201233
if err != nil {
12211234
return fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
12221235
}
12231236

1224-
err = b.Env.Jumpbox.RunSSHCommand("root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k "+b.Env.SecretsDir+"/age_key.txt -p "+b.Env.InstallCodesphereVersion+".tar.gz", b.sshQuiet)
1237+
err = b.Env.Jumpbox.RunSSHCommand("root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k "+b.Env.SecretsDir+"/age_key.txt -p "+b.Env.InstallCodesphereVersion+".tar.gz")
12251238
if err != nil {
12261239
return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)
12271240
}
@@ -1317,11 +1330,11 @@ systemctl restart k0scontroller
13171330
if err != nil {
13181331
return fmt.Errorf("failed to write configure-k0s.sh: %w", err)
13191332
}
1320-
err = b.Env.ControlPlaneNodes[0].CopyFile("configure-k0s.sh", "/root/configure-k0s.sh")
1333+
err = b.Env.ControlPlaneNodes[0].NodeClient.CopyFile(b.Env.ControlPlaneNodes[0], "configure-k0s.sh", "/root/configure-k0s.sh")
13211334
if err != nil {
13221335
return fmt.Errorf("failed to copy configure-k0s.sh to control plane node: %w", err)
13231336
}
1324-
err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh", b.sshQuiet)
1337+
err = b.Env.ControlPlaneNodes[0].RunSSHCommand("root", "chmod +x /root/configure-k0s.sh")
13251338
if err != nil {
13261339
return fmt.Errorf("failed to make configure-k0s.sh executable on control plane node: %w", err)
13271340
}

0 commit comments

Comments
 (0)