feat: add package checksum tracking and re-extraction logic (#193)

Add the ability to download/install new builds of the same branch
without cleaning up the oms-workdir first.

[Clickup](https://app.clickup.com/t/869buu10w)

---------

Signed-off-by: OliverTrautvetter <66372584+OliverTrautvetter@users.noreply.github.com>

Author: OliverTrautvetter

NOTICE

@@ -401,15 +401,15 @@ License URL: https://cs.opensource.google/go/x/time/+/v0.14.0:LICENSE
 
 ----------
 Module: google.golang.org/api
-Version: v0.266.0
+Version: v0.267.0
 License: BSD-3-Clause
-License URL: https://github.com/googleapis/google-api-go-client/blob/v0.266.0/LICENSE
+License URL: https://github.com/googleapis/google-api-go-client/blob/v0.267.0/LICENSE
 
 ----------
 Module: google.golang.org/api/internal/third_party/uritemplates
-Version: v0.266.0
+Version: v0.267.0
 License: BSD-3-Clause
-License URL: https://github.com/googleapis/google-api-go-client/blob/v0.266.0/internal/third_party/uritemplates/LICENSE
+License URL: https://github.com/googleapis/google-api-go-client/blob/v0.267.0/internal/third_party/uritemplates/LICENSE
 
 ----------
 Module: google.golang.org/genproto/googleapis

cli/cmd/download_package.go

@@ -6,7 +6,6 @@ package cmd
 import (
 	"fmt"
 	"log"
-	"strings"
 
 	"github.com/codesphere-cloud/cs-go/pkg/io"
 	"github.com/spf13/cobra"
@@ -100,7 +99,7 @@ func (c *DownloadPackageCmd) DownloadBuild(p portal.Portal, build portal.Build,
 		return fmt.Errorf("failed to find artifact in package: %w", err)
 	}
 
-	fullFilename := strings.ReplaceAll(build.Version, "/", "-") + "-" + filename
+	fullFilename := build.BuildPackageFilename(filename)
 	out, err := c.FileWriter.OpenAppend(fullFilename)
 	if err != nil {
 		out, err = c.FileWriter.Create(fullFilename)

cli/cmd/download_package_test.go

@@ -22,6 +22,7 @@ var _ = Describe("DownloadPackages", func() {
 		c              cmd.DownloadPackageCmd
 		filename       string
 		version        string
+		hash           string
 		build          portal.Build
 		mockPortal     *portal.MockPortal
 		mockFileWriter *util.MockFileIO
@@ -30,6 +31,7 @@ var _ = Describe("DownloadPackages", func() {
 	BeforeEach(func() {
 		filename = "installer.tar.gz"
 		version = "codesphere-1.42.0"
+		hash = "abc1234567"
 		mockPortal = portal.NewMockPortal(GinkgoT())
 		mockFileWriter = util.NewMockFileIO(GinkgoT())
 	})
@@ -44,6 +46,7 @@ var _ = Describe("DownloadPackages", func() {
 		}
 		build = portal.Build{
 			Version: version,
+			Hash:    hash,
 			Artifacts: []portal.Artifact{
 				{Filename: filename},
 				{Filename: "otherFilename.tar.gz"},
@@ -169,35 +172,64 @@ var _ = Describe("DownloadPackages", func() {
 		It("Downloads the correct artifact to the correct output file", func() {
 			expectedBuildToDownload := portal.Build{
 				Version: version,
+				Hash:    hash,
 				Artifacts: []portal.Artifact{
 					{Filename: filename},
 				},
 			}
 
 			fakeFile := os.NewFile(uintptr(0), filename)
-			mockFileWriter.EXPECT().OpenAppend(version+"-"+filename).Return(fakeFile, nil)
-			mockFileWriter.EXPECT().Open(version+"-"+filename).Return(fakeFile, nil)
+			mockFileWriter.EXPECT().OpenAppend(version+"-"+hash+"-"+filename).Return(fakeFile, nil)
+			mockFileWriter.EXPECT().Open(version+"-"+hash+"-"+filename).Return(fakeFile, nil)
 			mockPortal.EXPECT().DownloadBuildArtifact(portal.CodesphereProduct, expectedBuildToDownload, mock.Anything, 0, false).Return(nil)
 			mockPortal.EXPECT().VerifyBuildArtifactDownload(mock.Anything, expectedBuildToDownload).Return(nil)
 			err := c.DownloadBuild(mockPortal, build, filename)
 			Expect(err).NotTo(HaveOccurred())
 		})
 
+		It("Uses long hash in filename", func() {
+			longHash := "abc1234567890defghij"
+			buildWithLongHash := portal.Build{
+				Version: version,
+				Hash:    longHash,
+				Artifacts: []portal.Artifact{
+					{Filename: filename},
+					{Filename: "otherFilename.tar.gz"},
+				},
+			}
+			expectedBuildToDownload := portal.Build{
+				Version: version,
+				Hash:    longHash,
+				Artifacts: []portal.Artifact{
+					{Filename: filename},
+				},
+			}
+
+			fakeFile := os.NewFile(uintptr(0), filename)
+			mockFileWriter.EXPECT().OpenAppend(version+"-"+longHash+"-"+filename).Return(fakeFile, nil)
+			mockFileWriter.EXPECT().Open(version+"-"+longHash+"-"+filename).Return(fakeFile, nil)
+			mockPortal.EXPECT().DownloadBuildArtifact(portal.CodesphereProduct, expectedBuildToDownload, mock.Anything, 0, false).Return(nil)
+			mockPortal.EXPECT().VerifyBuildArtifactDownload(mock.Anything, expectedBuildToDownload).Return(nil)
+			err := c.DownloadBuild(mockPortal, buildWithLongHash, filename)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
 		Context("Version contains a slash", func() {
 			BeforeEach(func() {
 				version = "other/version/v1.42.0"
 			})
 			It("Downloads the correct artifact to the correct output file", func() {
 				expectedBuildToDownload := portal.Build{
 					Version: version,
+					Hash:    hash,
 					Artifacts: []portal.Artifact{
 						{Filename: filename},
 					},
 				}
 
 				fakeFile := os.NewFile(uintptr(0), filename)
-				mockFileWriter.EXPECT().OpenAppend("other-version-v1.42.0-"+filename).Return(fakeFile, nil)
-				mockFileWriter.EXPECT().Open("other-version-v1.42.0-"+filename).Return(fakeFile, nil)
+				mockFileWriter.EXPECT().OpenAppend("other-version-v1.42.0-"+hash+"-"+filename).Return(fakeFile, nil)
+				mockFileWriter.EXPECT().Open("other-version-v1.42.0-"+hash+"-"+filename).Return(fakeFile, nil)
 				mockPortal.EXPECT().DownloadBuildArtifact(portal.CodesphereProduct, expectedBuildToDownload, mock.Anything, 0, false).Return(nil)
 				mockPortal.EXPECT().VerifyBuildArtifactDownload(mock.Anything, expectedBuildToDownload).Return(nil)
 				err := c.DownloadBuild(mockPortal, build, filename)

internal/bootstrap/gcp/gcp.go

@@ -306,6 +306,10 @@ func (b *GCPBootstrapper) ValidateInput() error {
 		return fmt.Errorf("failed to get codesphere package: %w", err)
 	}
 
+	if b.Env.InstallHash == "" {
+		b.Env.InstallHash = build.Hash
+	}
+
 	requiredFilename := "installer.tar.gz"
 	if b.Env.RegistryType == RegistryTypeGitHub {
 		requiredFilename = "installer-lite.tar.gz"
@@ -1408,14 +1412,19 @@ func (b *GCPBootstrapper) InstallCodesphere() error {
 		skipStepsArg = " -s " + strings.Join(skipSteps, ",")
 	}
 
-	downloadCmd := "oms-cli download package -f " + packageFile + " " + b.Env.InstallVersion
+	downloadCmd := "oms-cli download package -f " + packageFile
+	if b.Env.InstallHash != "" {
+		downloadCmd += " -H " + b.Env.InstallHash
+	}
+	downloadCmd += " " + b.Env.InstallVersion
 	err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd)
 	if err != nil {
 		return fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
 	}
 
-	installCmd := fmt.Sprintf("oms-cli install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt -p %s-%s%s",
-		b.Env.SecretsDir, b.Env.InstallVersion, packageFile, skipStepsArg)
+	fullPackageFilename := portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFile)
+	installCmd := fmt.Sprintf("oms-cli install codesphere -c /etc/codesphere/config.yaml -k %s/age_key.txt -p %s%s",
+		b.Env.SecretsDir, fullPackageFilename, skipStepsArg)
 	err = b.Env.Jumpbox.RunSSHCommand("root", installCmd)
 	if err != nil {
 		return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)

internal/bootstrap/gcp/gcp_test.go

@@ -1385,6 +1385,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 	Describe("InstallCodesphere", func() {
 		BeforeEach(func() {
 			csEnv.InstallVersion = "v1.2.3"
+			csEnv.InstallHash = "abc1234567890"
 		})
 		Describe("Valid InstallCodesphere", func() {
 			Context("Direct GitHub access", func() {
@@ -1396,23 +1397,37 @@ var _ = Describe("GCP Bootstrapper", func() {
 				})
 				It("downloads and installs lite package", func() {
 					// Expect download package
-					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer-lite.tar.gz v1.2.3").Return(nil)
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer-lite.tar.gz -H abc1234567890 v1.2.3").Return(nil)
 
 					// Expect install codesphere
 					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root",
-						"oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-installer-lite.tar.gz -s load-container-images").Return(nil)
+						"oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-abc1234567890-installer-lite.tar.gz -s load-container-images").Return(nil)
 
 					err := bs.InstallCodesphere()
 					Expect(err).NotTo(HaveOccurred())
 				})
 			})
 
-			It("downloads and installs codesphere", func() {
-				// Expect download package
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz v1.2.3").Return(nil)
+			Context("without explicit hash", func() {
+				BeforeEach(func() {
+					// Simulate that ValidateInput has populated the hash
+					csEnv.InstallHash = "def9876543210"
+				})
+				It("downloads and installs codesphere", func() {
+					// Expect download package
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz -H def9876543210 v1.2.3").Return(nil)
+
+					// Expect install codesphere
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-def9876543210-installer.tar.gz").Return(nil)
+
+					err := bs.InstallCodesphere()
+					Expect(err).NotTo(HaveOccurred())
+				})
+			})
 
-				// Expect install codesphere
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-installer.tar.gz").Return(nil)
+			It("downloads and installs codesphere with hash", func() {
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil)
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-abc1234567890-installer.tar.gz").Return(nil)
 
 				err := bs.InstallCodesphere()
 				Expect(err).NotTo(HaveOccurred())
@@ -1421,16 +1436,16 @@ var _ = Describe("GCP Bootstrapper", func() {
 
 		Describe("Invalid cases", func() {
 			It("fails when download package fails", func() {
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz v1.2.3").Return(fmt.Errorf("download error"))
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(fmt.Errorf("download error"))
 
 				err := bs.InstallCodesphere()
 				Expect(err).To(HaveOccurred())
 				Expect(err.Error()).To(ContainSubstring("failed to download Codesphere package from jumpbox"))
 			})
 
 			It("fails when install codesphere fails", func() {
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz v1.2.3").Return(nil).Once()
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-installer.tar.gz").Return(fmt.Errorf("install error")).Once()
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil).Once()
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpbboxMatcher), "root", "oms-cli install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt -p v1.2.3-abc1234567890-installer.tar.gz").Return(fmt.Errorf("install error")).Once()
 
 				err := bs.InstallCodesphere()
 				Expect(err).To(HaveOccurred())

internal/installer/package.go

@@ -86,6 +86,7 @@ func (p *Package) Extract(force bool) error {
 	if err != nil {
 		return fmt.Errorf("failed to figure out if package %s is already extracted in %s: %w", p.Filename, workDir, err)
 	}
+
 	if alreadyExtracted && !force {
 		log.Println("Skipping extraction, package already unpacked. Use force option to overwrite.")
 		return nil

internal/portal/package.go

@@ -5,6 +5,7 @@ package portal
 
 import (
 	"fmt"
+	"strings"
 	"time"
 )
 
@@ -42,3 +43,18 @@ func (b *Build) GetBuildForDownload(filename string) (Build, error) {
 
 	return Build{}, fmt.Errorf("artifact not found: %s", filename)
 }
+
+// BuildPackageFilename generates the standard package filename for a given build
+// Format: {version}-{hash}-{filename}
+// Version slashes are replaced with dashes.
+func (b *Build) BuildPackageFilename(filename string) string {
+	return BuildPackageFilenameFromParts(b.Version, b.Hash, filename)
+}
+
+// BuildPackageFilenameFromParts generates the standard package filename from individual parts
+// Format: {version}-{hash}-{filename}
+// Version slashes are replaced with dashes.
+func BuildPackageFilenameFromParts(version, hash, filename string) string {
+	sanitizedVersion := strings.ReplaceAll(version, "/", "-")
+	return sanitizedVersion + "-" + hash + "-" + filename
+}

internal/portal/package_test.go

@@ -33,3 +33,66 @@ var _ = Describe("GetBuildForDownload", func() {
 	})
 
 })
+
+var _ = Describe("BuildPackageFilename", func() {
+	Describe("BuildPackageFilename method", func() {
+		It("generates filename with long hash", func() {
+			build := portal.Build{
+				Version: "v1.2.3",
+				Hash:    "abc1234567890def",
+			}
+			filename := build.BuildPackageFilename("installer.tar.gz")
+			Expect(filename).To(Equal("v1.2.3-abc1234567890def-installer.tar.gz"))
+		})
+
+		It("handles short hash", func() {
+			build := portal.Build{
+				Version: "v1.2.3",
+				Hash:    "abc123",
+			}
+			filename := build.BuildPackageFilename("installer.tar.gz")
+			Expect(filename).To(Equal("v1.2.3-abc123-installer.tar.gz"))
+		})
+
+		It("replaces slashes in version with dashes", func() {
+			build := portal.Build{
+				Version: "feature/my-branch",
+				Hash:    "abc1234567890",
+			}
+			filename := build.BuildPackageFilename("installer-lite.tar.gz")
+			Expect(filename).To(Equal("feature-my-branch-abc1234567890-installer-lite.tar.gz"))
+		})
+	})
+
+	Describe("BuildPackageFilenameFromParts function", func() {
+		It("generates filename from parts with long hash", func() {
+			filename := portal.BuildPackageFilenameFromParts("v1.2.3", "abc1234567890def", "installer.tar.gz")
+			Expect(filename).To(Equal("v1.2.3-abc1234567890def-installer.tar.gz"))
+		})
+
+		It("handles branch versions with slashes", func() {
+			filename := portal.BuildPackageFilenameFromParts("feature/test", "abc1234567890", "installer.tar.gz")
+			Expect(filename).To(Equal("feature-test-abc1234567890-installer.tar.gz"))
+		})
+
+		It("handles exact 10 character hash", func() {
+			filename := portal.BuildPackageFilenameFromParts("v1.0.0", "1234567890", "installer.tar.gz")
+			Expect(filename).To(Equal("v1.0.0-1234567890-installer.tar.gz"))
+		})
+
+		It("handles empty hash", func() {
+			filename := portal.BuildPackageFilenameFromParts("v1.0.0", "", "installer.tar.gz")
+			Expect(filename).To(Equal("v1.0.0--installer.tar.gz"))
+		})
+
+		It("handles empty version", func() {
+			filename := portal.BuildPackageFilenameFromParts("", "abc1234567", "installer.tar.gz")
+			Expect(filename).To(Equal("-abc1234567-installer.tar.gz"))
+		})
+
+		It("handles multiple slashes in version", func() {
+			filename := portal.BuildPackageFilenameFromParts("feature/sub/branch/v1", "abc1234567", "installer.tar.gz")
+			Expect(filename).To(Equal("feature-sub-branch-v1-abc1234567-installer.tar.gz"))
+		})
+	})
+})

internal/tmpl/NOTICE

@@ -401,15 +401,15 @@ License URL: https://cs.opensource.google/go/x/time/+/v0.14.0:LICENSE
 
 ----------
 Module: google.golang.org/api
-Version: v0.266.0
+Version: v0.267.0
 License: BSD-3-Clause
-License URL: https://github.com/googleapis/google-api-go-client/blob/v0.266.0/LICENSE
+License URL: https://github.com/googleapis/google-api-go-client/blob/v0.267.0/LICENSE
 
 ----------
 Module: google.golang.org/api/internal/third_party/uritemplates
-Version: v0.266.0
+Version: v0.267.0
 License: BSD-3-Clause
-License URL: https://github.com/googleapis/google-api-go-client/blob/v0.266.0/internal/third_party/uritemplates/LICENSE
+License URL: https://github.com/googleapis/google-api-go-client/blob/v0.267.0/internal/third_party/uritemplates/LICENSE
 
 ----------
 Module: google.golang.org/genproto/googleapis