fix: fix download problems

cli/cmd/download_package.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"strings"
+	"time"
 
 	"github.com/codesphere-cloud/cs-go/pkg/io"
 	"github.com/spf13/cobra"
@@ -24,10 +25,11 @@ type DownloadPackageCmd struct {
 
 type DownloadPackageOpts struct {
 	*GlobalOptions
-	Version  string
-	Hash     string
-	Filename string
-	Quiet    bool
+	Version    string
+	Hash       string
+	Filename   string
+	Quiet      bool
+	MaxRetries int
 }
 
 func (c *DownloadPackageCmd) RunE(_ *cobra.Command, args []string) error {
@@ -89,6 +91,7 @@ func AddDownloadPackageCmd(download *cobra.Command, opts *GlobalOptions) {
 	pkg.cmd.Flags().StringVarP(&pkg.Opts.Hash, "hash", "H", "", "Hash of the version to download if multiple builds exist for the same version")
 	pkg.cmd.Flags().StringVarP(&pkg.Opts.Filename, "file", "f", "installer.tar.gz", "Specify artifact to download")
 	pkg.cmd.Flags().BoolVarP(&pkg.Opts.Quiet, "quiet", "q", false, "Suppress progress output during download")
+	pkg.cmd.Flags().IntVarP(&pkg.Opts.MaxRetries, "max-retries", "r", 5, "Maximum number of download retry attempts")
 	download.AddCommand(pkg.cmd)
 
 	pkg.cmd.RunE = pkg.RunE
@@ -101,25 +104,57 @@ func (c *DownloadPackageCmd) DownloadBuild(p portal.Portal, build portal.Build,
 	}
 
 	fullFilename := strings.ReplaceAll(build.Version, "/", "-") + "-" + filename
-	out, err := c.FileWriter.OpenAppend(fullFilename)
-	if err != nil {
-		out, err = c.FileWriter.Create(fullFilename)
+
+	maxRetries := max(c.Opts.MaxRetries, 1)
+	retryDelay := 5 * time.Second
+
+	var downloadErr error
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		out, err := c.FileWriter.OpenAppend(fullFilename)
 		if err != nil {
-			return fmt.Errorf("failed to create file %s: %w", fullFilename, err)
+			out, err = c.FileWriter.Create(fullFilename)
+			if err != nil {
+				return fmt.Errorf("failed to create file %s: %w", fullFilename, err)
+			}
+		}
+
+		// Get current file size for resume functionality
+		fileSize := 0
+		fileInfo, err := out.Stat()
+		if err == nil {
+			fileSize = int(fileInfo.Size())
 		}
-	}
-	defer util.CloseFileIgnoreError(out)
 
-	// get already downloaded file size of fullFilename
-	fileSize := 0
-	fileInfo, err := out.Stat()
-	if err == nil {
-		fileSize = int(fileInfo.Size())
+		downloadErr = p.DownloadBuildArtifact("codesphere", download, out, fileSize, c.Opts.Quiet)
+		util.CloseFileIgnoreError(out)
+
+		if downloadErr == nil {
+			break
+		}
+
+		// Determine if error is retryable
+		errMsg := strings.ToLower(downloadErr.Error())
+		shouldRetry := strings.Contains(errMsg, "timeout") ||
+			strings.Contains(errMsg, "connection") ||
+			strings.Contains(errMsg, "eof") ||
+			strings.Contains(errMsg, "reset by peer") ||
+			strings.Contains(errMsg, "temporary failure") ||
+			strings.Contains(errMsg, "no such host") ||
+			strings.Contains(errMsg, "dial tcp") ||
+			strings.Contains(errMsg, "network is unreachable") ||
+			strings.Contains(errMsg, "i/o timeout")
+
+		if !shouldRetry || attempt == maxRetries {
+			return fmt.Errorf("failed to download build after %d attempts: %w", attempt, downloadErr)
+		}
+
+		log.Printf("Download interrupted (attempt %d/%d). Retrying in %v...", attempt, maxRetries, retryDelay)
+		time.Sleep(retryDelay)
+		retryDelay = time.Duration(float64(retryDelay) * 1.5) // Exponential backoff
 	}
 
-	err = p.DownloadBuildArtifact("codesphere", download, out, fileSize, c.Opts.Quiet)
-	if err != nil {
-		return fmt.Errorf("failed to download build: %w", err)
+	if downloadErr != nil {
+		return fmt.Errorf("failed to download build: %w", downloadErr)
 	}
 
 	verifyFile, err := c.FileWriter.Open(fullFilename)

docs/oms-cli_download_package.md

@@ -28,11 +28,12 @@ $ oms-cli download package --version codesphere-v1.55.0 --file installer-lite.ta
 ### Options
 
 ```
-  -f, --file string      Specify artifact to download (default "installer.tar.gz")
-  -H, --hash string      Hash of the version to download if multiple builds exist for the same version
-  -h, --help             help for package
-  -q, --quiet            Suppress progress output during download
-  -V, --version string   Codesphere version to download
+  -f, --file string       Specify artifact to download (default "installer.tar.gz")
+  -H, --hash string       Hash of the version to download if multiple builds exist for the same version
+  -h, --help              help for package
+  -r, --max-retries int   Maximum number of download retry attempts (default 5)
+  -q, --quiet             Suppress progress output during download
+  -V, --version string    Codesphere version to download
 ```
 
 ### SEE ALSO

docs/oms-cli_update_package.md

@@ -28,11 +28,12 @@ $ oms-cli download package --version codesphere-v1.55.0 --file installer-lite.ta
 ### Options
 
 ```
-  -f, --file string      Specify artifact to download (default "installer.tar.gz")
-  -H, --hash string      Hash of the version to download if multiple builds exist for the same version
-  -h, --help             help for package
-  -q, --quiet            Suppress progress output during download
-  -V, --version string   Codesphere version to download
+  -f, --file string       Specify artifact to download (default "installer.tar.gz")
+  -H, --hash string       Hash of the version to download if multiple builds exist for the same version
+  -h, --help              help for package
+  -r, --max-retries int   Maximum number of download retry attempts (default 5)
+  -q, --quiet             Suppress progress output during download
+  -V, --version string    Codesphere version to download
 ```
 
 ### SEE ALSO

internal/portal/http.go

@@ -22,7 +22,7 @@ type HttpWrapper struct {
 
 func NewHttpWrapper() *HttpWrapper {
 	return &HttpWrapper{
-		HttpClient: http.DefaultClient,
+		HttpClient: NewConfiguredHttpClient(),
 	}
 }
 
@@ -77,7 +77,7 @@ func (c *HttpWrapper) Download(url string, file io.Writer, quiet bool) error {
 
 	counter := file
 	if !quiet {
-		counter = NewWriteCounter(file)
+		counter = NewWriteCounterWithTotal(file, resp.ContentLength, 0)
 	}
 
 	_, err = io.Copy(counter, resp.Body)

internal/portal/portal.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/url"
 	"slices"
@@ -45,7 +46,26 @@ type HttpClient interface {
 func NewPortalClient() *PortalClient {
 	return &PortalClient{
 		Env:        env.NewEnv(),
-		HttpClient: http.DefaultClient,
+		HttpClient: NewConfiguredHttpClient(),
+	}
+}
+
+// NewConfiguredHttpClient creates an HTTP client with proper timeouts
+func NewConfiguredHttpClient() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout:   30 * time.Second,
+			ResponseHeaderTimeout: 2 * time.Minute,
+			ExpectContinueTimeout: 1 * time.Second,
+			IdleConnTimeout:       90 * time.Second,
+			MaxIdleConns:          100,
+			MaxIdleConnsPerHost:   10,
+		},
 	}
 }
 
@@ -56,6 +76,29 @@ const (
 	OmsProduct        Product = "oms"
 )
 
+// TruncateHTMLResponse detects HTML responses and truncates them to avoid verbose error messages.
+func TruncateHTMLResponse(body string) string {
+	// Check if response looks like HTML (case-insensitive) using a single trimmed value
+	trimmed := strings.TrimSpace(body)
+	normalized := strings.ToLower(trimmed)
+	if strings.HasPrefix(normalized, "<!doctype") || strings.HasPrefix(normalized, "<html") {
+		// Extract title if present
+		if idx := strings.Index(body, "<title>"); idx != -1 {
+			endIdx := strings.Index(body[idx:], "</title>")
+			if endIdx != -1 {
+				title := body[idx+7 : idx+endIdx]
+				return fmt.Sprintf("Server says: %s", title)
+			}
+		}
+		return "Received HTML response instead of JSON"
+	}
+
+	if len(body) <= 500 {
+		return body
+	}
+	return body[:500] + "... (truncated)"
+}
+
 // AuthorizedHttpRequest sends a HTTP request with the necessary authorization headers.
 func (c *PortalClient) AuthorizedHttpRequest(req *http.Request) (resp *http.Response, err error) {
 	apiKey, err := c.Env.GetOmsPortalApiKey()
@@ -65,6 +108,9 @@ func (c *PortalClient) AuthorizedHttpRequest(req *http.Request) (resp *http.Resp
 	}
 
 	req.Header.Set("X-API-Key", apiKey)
+	if req.Header.Get("Accept") == "" {
+		req.Header.Set("Accept", "application/json")
+	}
 
 	resp, err = c.HttpClient.Do(req)
 	if err != nil {
@@ -80,9 +126,12 @@ func (c *PortalClient) AuthorizedHttpRequest(req *http.Request) (resp *http.Resp
 	if resp.StatusCode >= 300 {
 		if resp.Body != nil {
 			respBody, _ = io.ReadAll(resp.Body)
+			_ = resp.Body.Close()
 		}
-		log.Printf("Non-2xx response received - Status: %d, Body: %s", resp.StatusCode, string(respBody))
-		err = fmt.Errorf("unexpected response status: %d - %s, %s", resp.StatusCode, http.StatusText(resp.StatusCode), string(respBody))
+		truncatedBody := TruncateHTMLResponse(string(respBody))
+		log.Printf("Non-2xx response received - Status: %d", resp.StatusCode)
+		log.Printf("%s", truncatedBody)
+		err = fmt.Errorf("unexpected response status: %d - %s (%s)", resp.StatusCode, http.StatusText(resp.StatusCode), truncatedBody)
 		return
 	}
 
@@ -106,6 +155,7 @@ func (c *PortalClient) HttpRequest(method string, path string, body []byte) (res
 	if len(body) > 0 {
 		req.Header.Set("Content-Type", "application/json")
 	}
+	req.Header.Set("Accept", "application/json")
 	return c.AuthorizedHttpRequest(req)
 }
 
@@ -212,17 +262,26 @@ func (c *PortalClient) DownloadBuildArtifact(product Product, build Build, file
 
 	// Download the file from startByte to allow resuming
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
 	resp, err := c.AuthorizedHttpRequest(req)
 	if err != nil {
 		return fmt.Errorf("GET request to download build failed: %w", err)
 	}
 	defer func() { _ = resp.Body.Close() }()
 
+	if !quiet && resp.ContentLength > 0 {
+		log.Printf("Starting download of %s...", ByteCountToHumanReadable(resp.ContentLength))
+	}
+
 	// Create a WriteCounter to wrap the output file and report progress, unless quiet is requested.
 	// Default behavior: report progress. Quiet callers should pass true for quiet.
 	counter := file
 	if !quiet {
-		counter = NewWriteCounter(file)
+		totalSize := resp.ContentLength
+		if startByte > 0 && totalSize > 0 {
+			totalSize = totalSize + int64(startByte)
+		}
+		counter = NewWriteCounterWithTotal(file, totalSize, int64(startByte))
 	}
 
 	_, err = io.Copy(counter, resp.Body)
@@ -376,6 +435,7 @@ func (c *PortalClient) GetApiKeyId(oldKey string) (string, error) {
 	}
 
 	req.Header.Set("X-API-Key", oldKey)
+	req.Header.Set("Accept", "application/json")
 
 	resp, err := c.HttpClient.Do(req)
 	if err != nil {
@@ -385,7 +445,8 @@ func (c *PortalClient) GetApiKeyId(oldKey string) (string, error) {
 
 	if resp.StatusCode >= 300 {
 		respBody, _ := io.ReadAll(resp.Body)
-		return "", fmt.Errorf("unexpected response status: %d - %s, %s", resp.StatusCode, http.StatusText(resp.StatusCode), string(respBody))
+		truncatedBody := TruncateHTMLResponse(string(respBody))
+		return "", fmt.Errorf("unexpected response status: %d - %s, %s", resp.StatusCode, http.StatusText(resp.StatusCode), truncatedBody)
 	}
 
 	var result struct {

internal/portal/portal_test.go

@@ -16,6 +16,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/codesphere-cloud/oms/internal/env"
@@ -532,3 +533,61 @@ var _ = Describe("PortalClient", func() {
 		})
 	})
 })
+
+var _ = Describe("truncateHTMLResponse", func() {
+	It("returns short non-HTML response unchanged", func() {
+		body := `{"error": "not found"}`
+		result := portal.TruncateHTMLResponse(body)
+		Expect(result).To(Equal(body))
+	})
+
+	It("truncates long non-HTML responses", func() {
+		body := strings.Repeat("a", 600)
+		result := portal.TruncateHTMLResponse(body)
+		Expect(result).To(HaveLen(500 + len("... (truncated)")))
+		Expect(result).To(HaveSuffix("... (truncated)"))
+	})
+
+	It("extracts title from HTML response with DOCTYPE", func() {
+		body := `<!DOCTYPE html><html><head><title>502 Bad Gateway</title></head><body>...</body></html>`
+		result := portal.TruncateHTMLResponse(body)
+		Expect(result).To(Equal("Server says: 502 Bad Gateway"))
+	})
+
+	It("extracts title from HTML response starting with html tag", func() {
+		body := `<html><head><title>Service Unavailable</title></head><body>...</body></html>`
+		result := portal.TruncateHTMLResponse(body)
+		Expect(result).To(Equal("Server says: Service Unavailable"))
+	})
+
+	It("handles HTML without title tag", func() {
+		body := `<!DOCTYPE html><html><body>Error page</body></html>`
+		result := portal.TruncateHTMLResponse(body)
+		Expect(result).To(Equal("Received HTML response instead of JSON"))
+	})
+
+	It("handles HTML with whitespace before DOCTYPE", func() {
+		body := `   <!DOCTYPE html><html><head><title>Error</title></head></html>`
+		result := portal.TruncateHTMLResponse(body)
+		Expect(result).To(Equal("Server says: Error"))
+	})
+})
+
+var _ = Describe("newConfiguredHttpClient", func() {
+	It("creates an HTTP client with configured timeouts", func() {
+		client := portal.NewConfiguredHttpClient()
+
+		Expect(client).NotTo(BeNil())
+		Expect(client.Timeout).To(Equal(time.Duration(0)))
+
+		transport, ok := client.Transport.(*http.Transport)
+		Expect(ok).To(BeTrue())
+		Expect(transport.Proxy).NotTo(BeNil())
+		Expect(transport.TLSHandshakeTimeout).To(Equal(30 * time.Second))
+		Expect(transport.ResponseHeaderTimeout).To(Equal(2 * time.Minute))
+		Expect(transport.ExpectContinueTimeout).To(Equal(1 * time.Second))
+		Expect(transport.IdleConnTimeout).To(Equal(90 * time.Second))
+		Expect(transport.MaxIdleConns).To(Equal(100))
+		Expect(transport.MaxIdleConnsPerHost).To(Equal(10))
+	})
+})