Fixed and Improved Gitlab Project Metadata in-memory cache

Earlier, we used a map to temporarily store GitLab project metadata. While maps work well for small datasets, they don’t scale efficiently for larger ones.
There was also a bug in the caching logic: when storing entries, we used the GitLab HTTPURLToRepo field as the cache key, but when retrieving entries,
we used the normalized URL. As a result, cache lookups almost never succeeded, and the cache kept growing without being effectively used.

With this fix, we’ve replaced the map with an LRU cache, which is better suited for this use case.
The cache now stores up to 15,000 entries for one hour, after which the LRU mechanism automatically evicts old items, keeping memory usage under control.
We also consistently use the normalized URL for both setting and fetching cache entries.

Author: kashifkhan0771

pkg/sources/gitlab/gitlab.go

@@ -10,6 +10,14 @@ import (
 	"sync"
 	"time"
 
+	gogit "github.com/go-git/go-git/v5"
+	"github.com/gobwas/glob"
+	gitlab "gitlab.com/gitlab-org/api/client-go"
+	"golang.org/x/oauth2"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
@@ -20,20 +28,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
-
-	gogit "github.com/go-git/go-git/v5"
-	"github.com/gobwas/glob"
-	gitlab "gitlab.com/gitlab-org/api/client-go"
-	"golang.org/x/oauth2"
-	"golang.org/x/sync/errgroup"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/anypb"
 )
 
 const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB
 
-// This is the URL for gitlab hosted at gitlab.com
-const gitlabBaseURL = "https://gitlab.com/"
+// Base URL for GitLab Cloud (hosted at gitlab.com)
+const gitlabCloudBaseUrl = "https://gitlab.com/"
 
 type Source struct {
 	name     string
@@ -72,16 +72,15 @@ type Source struct {
 	sources.CommonSourceUnitUnmarshaller
 
 	useAuthInUrl bool
-
-	clonePath string
-	noCleanup bool
+	clonePath    string
+	noCleanup    bool
 
 	printLegacyJSON bool
 
 	projectsPerPage int64
 
 	// cache of repo URL to project info, used when generating metadata for chunks
-	repoToProjCache repoToProjectCache
+	*projectMetadataCache
 }
 
 // WithCustomContentWriter sets the useCustomContentWriter flag on the source.
@@ -242,11 +241,12 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 				Timestamp:           sanitizer.UTF8(timestamp),
 				Line:                line,
 			}
-			proj, ok := s.repoToProjCache.get(repository)
+			// check for project metadata in the cache
+			project, ok := s.projectMetadataCache.get(repository)
 			if ok {
-				gitlabMetadata.ProjectId = int64(proj.id)
-				gitlabMetadata.ProjectName = proj.name
-				gitlabMetadata.ProjectOwner = proj.owner
+				gitlabMetadata.ProjectId = project.id
+				gitlabMetadata.ProjectName = project.name
+				gitlabMetadata.ProjectOwner = project.owner
 			}
 
 			return &source_metadatapb.MetaData{
@@ -260,9 +260,7 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
 	}
 	s.git = git.NewGit(cfg)
 
-	s.repoToProjCache = repoToProjectCache{
-		cache: make(map[string]*project),
-	}
+	s.projectMetadataCache = NewProjectMetadataCache()
 
 	return nil
 }
@@ -314,8 +312,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar
 
 	} else {
 		gitlabReposEnumerated.WithLabelValues(s.name).Set(float64(len(repos)))
-		// ensure project details for specified repos are cached
-		// this is required to populate metadata during chunking
+		// Ensure project details for the specified repositories are cached.
+		// This is required to populate metadata during chunking.
+		// Note: Repository URLs are already normalized, so the cache check
+		// uses the normalized repo URL directly.
 		for _, repo := range repos {
 			s.ensureProjectInCache(ctx, repo)
 		}
@@ -586,7 +586,10 @@ func (s *Source) getAllProjectRepos(
 			}
 			// Report the unit.
 			ctx.Logger().V(3).Info("accepting project")
-			s.cacheGitlabProject(proj)
+
+			// Cache the GitLab project metadata.
+			s.cacheGitlabProjectMetadata(ctx, proj)
+
 			unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo}
 			gitlabReposEnumerated.WithLabelValues(s.name).Inc()
 			projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)
@@ -630,7 +633,7 @@ func (s *Source) getAllProjectRepos(
 		Owned:        gitlab.Ptr(false),
 	}
 
-	if s.url != gitlabBaseURL {
+	if s.url != gitlabCloudBaseUrl {
 		listGroupsOptions.AllAvailable = gitlab.Ptr(true)
 	}
 
@@ -722,7 +725,7 @@ func (s *Source) getAllProjectReposV2(
 	}
 
 	// for gitlab.com instance, include only projects where the user is a member.
-	if s.url == gitlabBaseURL {
+	if s.url == gitlabCloudBaseUrl {
 		projectQueryOptions.Membership = gitlab.Ptr(true)
 	}
 
@@ -783,7 +786,9 @@ func (s *Source) getAllProjectReposV2(
 			// report the unit.
 			projCtx.Logger().V(3).Info("accepting project")
 
-			s.cacheGitlabProject(project)
+			// Cache the GitLab project metadata.
+			s.cacheGitlabProjectMetadata(projCtx, project)
+
 			unit := git.SourceUnit{Kind: git.UnitRepo, ID: project.HTTPURLToRepo}
 			gitlabReposEnumerated.WithLabelValues(s.name).Inc()
 
@@ -837,7 +842,7 @@ func (s *Source) getAllProjectReposInGroups(
 	}
 
 	// For non gitlab.com instances, you might want to adjust access levels
-	if s.url != gitlabBaseURL {
+	if s.url != gitlabCloudBaseUrl {
 		projectOpts.MinAccessLevel = gitlab.Ptr(gitlab.GuestPermissions)
 	}
 
@@ -899,7 +904,9 @@ func (s *Source) getAllProjectReposInGroups(
 				// report the unit.
 				projCtx.Logger().V(3).Info("accepting project")
 
-				s.cacheGitlabProject(proj)
+				// Cache the GitLab project metadata.
+				s.cacheGitlabProjectMetadata(projCtx, proj)
+
 				unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo}
 				gitlabReposEnumerated.WithLabelValues(s.name).Inc()
 				projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)
@@ -1022,77 +1029,6 @@ func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
 	s.scanOptions = scanOptions
 }
 
-func buildIgnorer(include, exclude []string, onCompile func(err error, pattern string)) func(repo string) bool {
-
-	// compile and load globRepoFilter
-	globRepoFilter := newGlobRepoFilter(include, exclude, onCompile)
-
-	f := func(repo string) bool {
-		if !globRepoFilter.includeRepo(repo) || globRepoFilter.ignoreRepo(repo) {
-			return true
-		}
-		return false
-	}
-
-	return f
-}
-
-func normalizeRepos(repos []string) ([]string, []error) {
-	// Optimistically allocate space for all valid repositories.
-	validRepos := make([]string, 0, len(repos))
-	var errs []error
-	for _, prj := range repos {
-		repo, err := giturl.NormalizeGitlabRepo(prj)
-		if err != nil {
-			errs = append(errs, fmt.Errorf("unable to normalize gitlab repo url %q: %w", prj, err))
-			continue
-		}
-
-		validRepos = append(validRepos, repo)
-	}
-	return validRepos, errs
-}
-
-// normalizeGitlabEndpoint ensures that if an endpoint is going to gitlab.com, we use https://gitlab.com/ as the endpoint.
-// If we see the protocol is http, we error, because this shouldn't be used.
-// Otherwise, it ensures we are using https as our protocol, if none was provided.
-func normalizeGitlabEndpoint(gitlabEndpoint string) (string, error) {
-	if gitlabEndpoint == "" {
-		return gitlabBaseURL, nil
-	}
-
-	gitlabURL, err := url.Parse(gitlabEndpoint)
-	if err != nil {
-		return "", err
-	}
-
-	// We probably didn't receive a URL with a scheme, which messed up the parsing.
-	if gitlabURL.Host == "" {
-		gitlabURL, err = url.Parse("https://" + gitlabEndpoint)
-		if err != nil {
-			return "", err
-		}
-	}
-
-	// If the host is gitlab.com, this is the cloud version, which has only one valid endpoint.
-	if gitlabURL.Host == "gitlab.com" {
-		return gitlabBaseURL, nil
-	}
-
-	// Beyond here, on-prem gitlab is being used, so we have to mostly leave things as-is.
-
-	if gitlabURL.Scheme != "https" {
-		return "", fmt.Errorf("https was not used as URL scheme, but is required. Please use https")
-	}
-
-	// The gitlab library wants trailing slashes.
-	if !strings.HasSuffix(gitlabURL.Path, "/") {
-		gitlabURL.Path = gitlabURL.Path + "/"
-	}
-
-	return gitlabURL.String(), nil
-}
-
 // Enumerate reports all GitLab repositories to be scanned to the reporter. If
 // none are configured, it will find all repositories within all projects that
 // the configured user has access to, while respecting the configured ignore
@@ -1191,23 +1127,25 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte
 	return s.git.ScanRepo(ctx, repo, path, s.scanOptions, reporter)
 }
 
-// ensureProjectInCache checks if the project for the given repo URL is in the cache,
-// and if not, queries the GitLab API to fetch the project and adds it to the cache.
-func (s *Source) ensureProjectInCache(ctx context.Context, repoUrl string) {
-	// check if project is already in cache
-	if _, ok := s.repoToProjCache.get(repoUrl); ok {
+// ensureProjectInCache ensures that the project for the given repository URL
+// exists in the cache. If not, it fetches the project from the GitLab API
+// and stores it in the cache.
+func (s *Source) ensureProjectInCache(ctx context.Context, repoURL string) {
+	// Check if the project is already cached.
+	if _, ok := s.projectMetadataCache.get(repoURL); ok {
+		ctx.Logger().V(5).Info("cache hit: found project metadata in the cache", "cache_key", repoURL)
 		return
 	}
 
-	// query project
-	proj, err := s.getGitlabProject(ctx, repoUrl)
+	// Fetch the project from GitLab.
+	project, err := s.getGitlabProject(ctx, repoURL)
 	if err != nil {
-		ctx.Logger().Error(err, "could not fetch project for repo", "repo", repoUrl)
+		ctx.Logger().Error(err, "failed to fetch GitLab project", "repo", repoURL)
 		return
 	}
 
-	// add to cache
-	s.cacheGitlabProject(proj)
+	// Cache the project metadata.
+	s.cacheGitlabProjectMetadata(ctx, project)
 }
 
 func (s *Source) getGitlabProject(ctx context.Context, repoUrl string) (*gitlab.Project, error) {
@@ -1230,16 +1168,100 @@ func (s *Source) getGitlabProject(ctx context.Context, repoUrl string) (*gitlab.
 	return proj, nil
 }
 
-func (s *Source) cacheGitlabProject(gitlabProj *gitlab.Project) {
+// cacheGitlabProjectMetadata caches GitLab project metadata keyed by the
+// normalized GitLab repository URL.
+func (s *Source) cacheGitlabProjectMetadata(ctx context.Context, glProject *gitlab.Project) {
 	proj := &project{
-		id:   gitlabProj.ID,
-		name: gitlabProj.NameWithNamespace,
+		id:   int64(glProject.ID),
+		name: glProject.NameWithNamespace,
+	}
+
+	if glProject.Owner != nil {
+		if email := glProject.Owner.Email; email != "" {
+			proj.owner = email
+		} else {
+			proj.owner = glProject.Owner.Username
+		}
 	}
-	if gitlabProj.Owner != nil {
-		proj.owner = gitlabProj.Owner.Email
-		if proj.owner == "" {
-			proj.owner = gitlabProj.Owner.Username
+
+	repoURL, err := giturl.NormalizeGitlabRepo(glProject.HTTPURLToRepo)
+	if err != nil {
+		ctx.Logger().Error(err, "failed to normalize GitLab Repo", "repo", glProject.HTTPURLToRepo)
+		return
+	}
+
+	ctx.Logger().V(5).Info("cache set: added project metadata in the cache", "cache_key", repoURL)
+	s.projectMetadataCache.set(repoURL, proj)
+}
+
+func buildIgnorer(include, exclude []string, onCompile func(err error, pattern string)) func(repo string) bool {
+
+	// compile and load globRepoFilter
+	globRepoFilter := newGlobRepoFilter(include, exclude, onCompile)
+
+	f := func(repo string) bool {
+		if !globRepoFilter.includeRepo(repo) || globRepoFilter.ignoreRepo(repo) {
+			return true
 		}
+		return false
 	}
-	s.repoToProjCache.set(gitlabProj.HTTPURLToRepo, proj)
+
+	return f
+}
+
+// normalizeRepos convert the repo urls from https://gitlab.com/org/repo -> https://gitlab.com/org/repo.git
+func normalizeRepos(repos []string) ([]string, []error) {
+	// Optimistically allocate space for all valid repositories.
+	validRepos := make([]string, 0, len(repos))
+	var errs []error
+	for _, prj := range repos {
+		repo, err := giturl.NormalizeGitlabRepo(prj)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("unable to normalize gitlab repo url %q: %w", prj, err))
+			continue
+		}
+
+		validRepos = append(validRepos, repo)
+	}
+	return validRepos, errs
+}
+
+// normalizeGitlabEndpoint ensures that if an endpoint is going to gitlab.com, we use https://gitlab.com/ as the endpoint.
+// If we see the protocol is http, we error, because this shouldn't be used.
+// Otherwise, it ensures we are using https as our protocol, if none was provided.
+func normalizeGitlabEndpoint(gitlabEndpoint string) (string, error) {
+	if gitlabEndpoint == "" {
+		return gitlabCloudBaseUrl, nil
+	}
+
+	gitlabURL, err := url.Parse(gitlabEndpoint)
+	if err != nil {
+		return "", err
+	}
+
+	// We probably didn't receive a URL with a scheme, which messed up the parsing.
+	if gitlabURL.Host == "" {
+		gitlabURL, err = url.Parse("https://" + gitlabEndpoint)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	// If the host is gitlab.com, this is the cloud version, which has only one valid endpoint.
+	if gitlabURL.Host == "gitlab.com" {
+		return gitlabCloudBaseUrl, nil
+	}
+
+	// Beyond here, on-prem gitlab is being used, so we have to mostly leave things as-is.
+
+	if gitlabURL.Scheme != "https" {
+		return "", fmt.Errorf("https was not used as URL scheme, but is required. Please use https")
+	}
+
+	// The gitlab library wants trailing slashes.
+	if !strings.HasSuffix(gitlabURL.Path, "/") {
+		gitlabURL.Path = gitlabURL.Path + "/"
+	}
+
+	return gitlabURL.String(), nil
 }