feat: auto-convert Go library deps to weak dependencies

Go packages with packaging: library are now automatically treated as
weak dependencies when referenced. This means:

- Source files are copied to _deps/ (not built artifacts)
- go.mod replace directives are added
- Builds run in parallel (don't block dependent package)
- Version tracking ensures cache invalidation when libraries change

This improves build parallelism for Go monorepos where libraries
are used for source code via go.mod replace, not for built artifacts.

Co-authored-by: Ona <no-reply@ona.com>

Author: iQQBot

README.md

@@ -108,6 +108,8 @@ srcs:
 - "glob/**/path"
 # Deps list dependencies to other packages which must be built prior to building this package. How these dependencies are made
 # available during build depends on the package type.
+# NOTE: Go packages with `packaging: library` are automatically treated as "weak dependencies" - their source files
+# are copied (not built artifacts), and they don't block the build. See "Go Library Dependencies" section below.
 deps:
 - some/other:package
 # Argdeps makes build arguments version relevant. I.e. if the value of a build arg listed here changes, so does the package version.
@@ -125,6 +127,8 @@ config:
 ```YAML
 config:
   # Packaging method. See https://godoc.org/github.com/gitpod-io/leeway/pkg/leeway#GoPackaging for details. Defaults to library.
+  # IMPORTANT: Packages with `packaging: library` are treated as "weak dependencies" when referenced by other packages.
+  # This means their source files are copied (not built artifacts), and they build in parallel rather than blocking.
   packaging: library
   # If true leeway runs `go generate -v ./...` prior to testing/building. Defaults to false.
   generate: false
@@ -144,6 +148,43 @@ config:
   goMod: "../go.mod"
 ```
 
+#### Go Library Dependencies (Weak Dependencies)
+
+When a Go package with `packaging: library` is listed as a dependency, leeway automatically treats it as a "weak dependency". This behavior is optimized for Go's module system:
+
+| Aspect | Regular Dependency | Go Library (Weak) Dependency |
+|--------|-------------------|------------------------------|
+| Affects package version | ✅ | ✅ |
+| Must be built first | ✅ | ❌ |
+| What's copied to `_deps/` | Built artifact | Source files |
+| `go.mod replace` added | ✅ | ✅ |
+| Added to build queue | ✅ | ✅ |
+
+**Why this matters:**
+- Go libraries are typically used for their source code via `go.mod replace` directives
+- The library's tests can run in parallel with the dependent package's build
+- Changes to the library still trigger rebuilds of dependent packages (version tracking)
+- Build times improve because packages don't wait for library builds to complete
+
+**Example:**
+```yaml
+# my-lib/BUILD.yaml
+packages:
+- name: lib
+  type: go
+  config:
+    packaging: library  # This makes it a weak dependency when referenced
+
+# my-app/BUILD.yaml  
+packages:
+- name: app
+  type: go
+  deps:
+    - my-lib:lib  # Automatically treated as weak dep - sources copied, builds in parallel
+  config:
+    packaging: app
+```
+
 ### Yarn packages
 ```YAML
 config:

pkg/leeway/build.go

@@ -687,6 +687,21 @@ func Build(pkg *Package, opts ...BuildOption) (err error) {
 	requirements := pkg.GetTransitiveDependencies()
 	allpkg := append(requirements, pkg)
 
+	weakDeps := collectWeakDependencies(pkg)
+	for _, wd := range weakDeps {
+		// Only add if not already in allpkg (avoid duplicates)
+		found := false
+		for _, p := range allpkg {
+			if p.FullName() == wd.FullName() {
+				found = true
+				break
+			}
+		}
+		if !found {
+			allpkg = append(allpkg, wd)
+		}
+	}
+
 	pkgsInLocalCache := make(map[*Package]struct{})
 	var pkgsToCheckRemoteCache []*Package
 	for _, p := range allpkg {
@@ -872,7 +887,38 @@ func Build(pkg *Package, opts ...BuildOption) (err error) {
 		return nil
 	}
 
-	buildErr := pkg.build(ctx)
+	// Build hard deps of weak deps first - they need to be available for extraction
+	// when the main package builds. Use GetTransitiveWeakDependencies to get only
+	// the actual weak deps (Go libraries), not the mixed result from collectWeakDependencies.
+	hardDepsOfWeakDeps := collectHardDepsOfWeakDepsForBuild(pkg.GetTransitiveWeakDependencies())
+	if len(hardDepsOfWeakDeps) > 0 {
+		var hdGroup errgroup.Group
+		for _, hd := range hardDepsOfWeakDeps {
+			hardDep := hd
+			hdGroup.Go(func() error {
+				return hardDep.build(ctx)
+			})
+		}
+		if err := hdGroup.Wait(); err != nil {
+			return xerrors.Errorf("build failed")
+		}
+	}
+
+	// Now build weak deps and main package in parallel
+	var buildGroup errgroup.Group
+
+	for _, wd := range weakDeps {
+		weakDep := wd
+		buildGroup.Go(func() error {
+			return weakDep.build(ctx)
+		})
+	}
+
+	buildGroup.Go(func() error {
+		return pkg.build(ctx)
+	})
+
+	buildErr := buildGroup.Wait()
 
 	// Check for build errors immediately and return if there are any
 	if buildErr != nil {
@@ -1452,6 +1498,10 @@ func (p *Package) packagesToDownload(inLocalCache map[*Package]struct{}, inRemot
 	// 			as we also need components/gitpod-protocol:gitpod-schema to be available on disk to perform the build.
 	case YarnPackage, GoPackage:
 		deps = p.GetTransitiveDependencies()
+		// Also include hard deps of weak deps (e.g., generic packages that Go libraries depend on)
+		for _, wd := range p.GetTransitiveWeakDependencies() {
+			deps = append(deps, wd.GetTransitiveDependencies()...)
+		}
 	// For Generic and Docker packages it is sufficient to have the direct dependencies.
 	case GenericPackage, DockerPackage:
 		deps = p.GetDependencies()
@@ -1462,6 +1512,91 @@ func (p *Package) packagesToDownload(inLocalCache map[*Package]struct{}, inRemot
 	}
 }
 
+// collectHardDepsOfWeakDeps returns hard dependencies of weak deps that aren't already in transdep.
+// These need to be extracted as built artifacts for the weak dep sources to compile.
+func collectHardDepsOfWeakDeps(weakdeps []*Package, transdep []*Package) []*Package {
+	existing := make(map[string]struct{})
+	for _, dep := range transdep {
+		existing[dep.FullName()] = struct{}{}
+	}
+
+	seen := make(map[string]struct{})
+	var result []*Package
+
+	for _, wd := range weakdeps {
+		for _, hd := range wd.GetTransitiveDependencies() {
+			if _, ok := existing[hd.FullName()]; ok {
+				continue
+			}
+			if _, ok := seen[hd.FullName()]; ok {
+				continue
+			}
+			seen[hd.FullName()] = struct{}{}
+			result = append(result, hd)
+		}
+	}
+
+	return result
+}
+
+// collectHardDepsOfWeakDepsForBuild returns hard deps of weak deps that need to be built
+// before the main package can build (they need to be extracted to _deps/).
+func collectHardDepsOfWeakDepsForBuild(weakDeps []*Package) []*Package {
+	seen := make(map[string]struct{})
+	var result []*Package
+
+	for _, wd := range weakDeps {
+		for _, hd := range wd.GetTransitiveDependencies() {
+			if _, ok := seen[hd.FullName()]; ok {
+				continue
+			}
+			seen[hd.FullName()] = struct{}{}
+			result = append(result, hd)
+		}
+	}
+
+	return result
+}
+
+// collectWeakDependencies collects all weak dependencies from a package and its dependency tree,
+// including hard deps of weak deps (they need to be built for the weak dep to work).
+func collectWeakDependencies(pkg *Package) []*Package {
+	seen := make(map[string]struct{})
+	visited := make(map[string]struct{})
+	var result []*Package
+
+	var collectFromPackage func(p *Package)
+	collectFromPackage = func(p *Package) {
+		if _, ok := visited[p.FullName()]; ok {
+			return
+		}
+		visited[p.FullName()] = struct{}{}
+
+		for _, wd := range p.GetWeakDependencies() {
+			if _, ok := seen[wd.FullName()]; !ok {
+				seen[wd.FullName()] = struct{}{}
+				result = append(result, wd)
+			}
+
+			collectFromPackage(wd)
+
+			for _, td := range wd.GetTransitiveDependencies() {
+				if _, ok := seen[td.FullName()]; !ok {
+					seen[td.FullName()] = struct{}{}
+					result = append(result, td)
+				}
+			}
+		}
+
+		for _, dep := range p.GetDependencies() {
+			collectFromPackage(dep)
+		}
+	}
+
+	collectFromPackage(pkg)
+	return result
+}
+
 // validateDependenciesAvailable checks if all required dependencies of a package are available.
 // A dependency is considered available if it's in the local cache OR will be built (PackageNotBuiltYet).
 // Returns true if all dependencies are available, false otherwise.
@@ -1475,6 +1610,10 @@ func validateDependenciesAvailable(p *Package, localCache cache.LocalCache, pkgs
 	case YarnPackage, GoPackage:
 		// Go and Yarn packages need all transitive dependencies
 		deps = p.GetTransitiveDependencies()
+		// Also include hard deps of weak deps (e.g., generic packages that Go libraries depend on)
+		for _, wd := range p.GetTransitiveWeakDependencies() {
+			deps = append(deps, wd.GetTransitiveDependencies()...)
+		}
 	case GenericPackage, DockerPackage:
 		// Generic and Docker packages only need direct dependencies
 		deps = p.GetDependencies()
@@ -2068,43 +2207,81 @@ func (p *Package) buildGo(buildctx *buildContext, wd, result string) (res *packa
 	}
 
 	transdep := p.GetTransitiveDependencies()
-	if len(transdep) > 0 {
+	weakdeps := p.GetTransitiveWeakDependencies()
+
+	// Collect hard deps of weak deps - they need to be extracted as built artifacts
+	hardDepsOfWeakDeps := collectHardDepsOfWeakDeps(weakdeps, transdep)
+
+	needsDepsDir := len(transdep) > 0 || len(weakdeps) > 0 || len(hardDepsOfWeakDeps) > 0
+
+	if needsDepsDir {
 		commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"mkdir", "_deps"})
+	}
 
-		for _, dep := range transdep {
-			if dep.Ephemeral {
-				continue
-			}
+	// Combine transdep and hardDepsOfWeakDeps for extraction
+	allHardDeps := make(map[string]*Package)
+	for _, dep := range transdep {
+		allHardDeps[dep.FullName()] = dep
+	}
+	for _, dep := range hardDepsOfWeakDeps {
+		allHardDeps[dep.FullName()] = dep
+	}
 
-			builtpkg, ok := buildctx.LocalCache.Location(dep)
-			if !ok {
-				return nil, PkgNotBuiltErr{dep}
-			}
+	for _, dep := range allHardDeps {
+		if dep.Ephemeral {
+			continue
+		}
 
-			tgt := filepath.Join("_deps", p.BuildLayoutLocation(dep))
-			untarCmd, err := BuildUnTarCommand(
-				WithInputFile(builtpkg),
-				WithTargetDir(tgt),
-				WithAutoDetectCompression(true),
-			)
-			if err != nil {
-				return nil, err
-			}
+		builtpkg, ok := buildctx.LocalCache.Location(dep)
+		if !ok {
+			return nil, PkgNotBuiltErr{dep}
+		}
 
-			commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], [][]string{
-				{"mkdir", tgt},
-				untarCmd,
-			}...)
+		tgt := filepath.Join("_deps", p.BuildLayoutLocation(dep))
+		untarCmd, err := BuildUnTarCommand(
+			WithInputFile(builtpkg),
+			WithTargetDir(tgt),
+			WithAutoDetectCompression(true),
+		)
+		if err != nil {
+			return nil, err
+		}
 
-			if dep.Type != GoPackage {
-				continue
-			}
+		commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], [][]string{
+			{"mkdir", tgt},
+			untarCmd,
+		}...)
 
-			if isGoWorkspace {
-				commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"go", "work", "use", tgt})
-			} else {
-				commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"sh", "-c", fmt.Sprintf("%s mod edit -replace $(cd %s; grep module go.mod | cut -d ' ' -f 2 | head -n1)=./%s", goCommand, tgt, tgt)})
-			}
+		if dep.Type != GoPackage {
+			continue
+		}
+
+		if isGoWorkspace {
+			commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"go", "work", "use", tgt})
+		} else {
+			commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"sh", "-c", fmt.Sprintf("%s mod edit -replace $(cd %s; grep module go.mod | cut -d ' ' -f 2 | head -n1)=./%s", goCommand, tgt, tgt)})
+		}
+	}
+
+	for _, dep := range weakdeps {
+		if dep.Type != GoPackage {
+			log.WithField("package", p.FullName()).WithField("weakdep", dep.FullName()).
+				Warn("weak dependencies are only supported for Go packages, skipping")
+			continue
+		}
+
+		tgt := filepath.Join("_deps", p.BuildLayoutLocation(dep))
+		srcDir := dep.C.Origin
+
+		commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], [][]string{
+			{"mkdir", "-p", tgt},
+			{"sh", "-c", fmt.Sprintf("cp -r %s/* %s/", srcDir, tgt)},
+		}...)
+
+		if isGoWorkspace {
+			commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"go", "work", "use", tgt})
+		} else {
+			commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"sh", "-c", fmt.Sprintf("%s mod edit -replace $(cd %s; grep module go.mod | cut -d ' ' -f 2 | head -n1)=./%s", goCommand, tgt, tgt)})
 		}
 	}