feat: query with empty filter

Author: zulkhair

pkg/cardinal/internal/ecs/search.go

@@ -13,36 +13,11 @@ import (
 // We use expr lang for the where clause to filter the entities, please refer to its documentation
 // for more details: https://expr-lang.org/docs/getting-started.
 type SearchParam struct {
-	Find  []string    // List of component names to search for
-	Match SearchMatch // A match type to use for the search
-	Where string      // Optional expr language string to filter the results.
-}
-
-// validateAndGetFilter validates the search parameters and returns an expr VM program compiled
-// from the where clause.
-func (s *SearchParam) validateAndGetFilter() (*vm.Program, error) {
-	if len(s.Find) == 0 {
-		return nil, eris.New("component list cannot be empty")
-	}
-
-	if s.Match != MatchExact && s.Match != MatchContains {
-		return nil, eris.Errorf("invalid `match` value: must be either '%s' or '%s'", MatchExact, MatchContains)
-	}
-
-	var filter *vm.Program
-
-	// If no expression is provided, return a nil program
-	if len(s.Where) == 0 {
-		return filter, nil
-	}
-
-	// Compile the expression and check that the return type is boolean.
-	filter, err := expr.Compile(s.Where, expr.AsBool())
-	if err != nil {
-		return nil, eris.Wrap(err, "failed to parse where clause")
-	}
-
-	return filter, nil
+	Find   []string    // List of component names to search for. Must be empty when Match is MatchAll.
+	Match  SearchMatch // A match type to use for the search.
+	Where  string      // Optional expr language string to filter the results.
+	Limit  uint32      // Maximum number of results to return (default: unlimited, 0 = unlimited)
+	Offset uint32      // Number of results to skip before returning (default: 0)
 }
 
 // SearchMatch is the type of match to use for the search.
@@ -54,9 +29,16 @@ const (
 	// MatchContains matches entities that contains the specified components, but may have other
 	// components as well.
 	MatchContains SearchMatch = "contains"
+	// MatchAll matches all entities regardless of components. Find must be empty when using this.
+	MatchAll SearchMatch = "all"
 )
 
+// DefaultQueryLimit is the default maximum number of results returned when Limit is 0 (unlimited).
+const DefaultQueryLimit = ^uint32(0) // Max uint32 (4294967295)
+
 // NewSearch returns a map of entities that match the given search parameters.
+//
+//nolint:gocognit // Complexity from sequential filtering (where, offset, limit) kept together for readability.
 func (w *World) NewSearch(params SearchParam) ([]map[string]any, error) {
 	filter, err := params.validateAndGetFilter()
 	if err != nil {
@@ -68,60 +50,99 @@ func (w *World) NewSearch(params SearchParam) ([]map[string]any, error) {
 		return nil, eris.Wrap(err, "failed to get archetypes from components")
 	}
 
+	limit := params.Limit
+	if limit == 0 {
+		limit = DefaultQueryLimit
+	}
+
 	results := make([]map[string]any, 0)
+	skipped := uint32(0)
+	collected := uint32(0)
+
 	for _, id := range archetypeIDs {
-		// Makes a copy of the arch.
 		arch := w.state.archetypes[id]
 
 		for eid, components := range archIter(arch) {
-			// Create the payload map.
-			result := make(map[string]any)
-			// We have to cast id from EntityID to int here or else we can't query the data because for some
-			// reason expr can't compare EntityID with integers in the expression.
-			result["_id"] = uint32(eid)
-
-			for _, component := range components {
-				result[component.Name()] = component
+			result := buildEntityResult(eid, components)
+
+			// Apply Where filter if present
+			if filter != nil {
+				matches, err := matchesFilter(filter, result)
+				if err != nil {
+					return nil, err
+				}
+				if !matches {
+					continue // Skip this entity
+				}
 			}
 
-			// If there's no filter, include all entities.
-			if filter == nil {
-				results = append(results, result)
+			// Apply offset: skip first N matching entities
+			if skipped < params.Offset {
+				skipped++
 				continue
 			}
 
-			// Run the filter expression. We set the entity map as the environment for `Run` so the vm
-			// program has access to the entity data to filter.
-			output, innerErr := expr.Run(filter, result)
-			if innerErr != nil {
-				return nil, eris.Wrap(innerErr, "failed to run filter expression")
-			}
-
-			isMatchFilter, ok := output.(bool)
-			// Because we compile the expr once without passing in the environment, as it's only available
-			// while iterating, expr.Compile can't fully check if the expression returns a bool,x
-			// especially when we filter for a struct field e.g. health.hp > 200, expr can't determine the
-			// type of health.hp during compilation.
-			if !ok {
-				return nil, eris.New("invalid where clause")
-			}
+			// Add to results
+			results = append(results, result)
+			collected++
 
-			if isMatchFilter {
-				results = append(results, result)
+			// Early termination: stop when limit is reached
+			if collected >= limit {
+				return results, nil
 			}
 		}
 	}
 
 	return results, nil
 }
 
+// validateAndGetFilter validates the search parameters and returns an expr VM program compiled
+// from the where clause.
+func (s *SearchParam) validateAndGetFilter() (*vm.Program, error) {
+	// Validate Match and Find relationship
+	if s.Match == MatchAll {
+		if len(s.Find) > 0 {
+			return nil, eris.New("find must be empty when match is 'all'")
+		}
+	} else {
+		if len(s.Find) == 0 {
+			return nil, eris.New("find must not be empty when match is not 'all'")
+		}
+		if s.Match != MatchExact && s.Match != MatchContains {
+			return nil, eris.Errorf("invalid `match` value: must be either '%s' or '%s'", MatchExact, MatchContains)
+		}
+	}
+
+	var filter *vm.Program
+
+	// If no expression is provided, return a nil program
+	if len(s.Where) == 0 {
+		return filter, nil
+	}
+
+	// Compile the expression and check that the return type is boolean.
+	filter, err := expr.Compile(s.Where, expr.AsBool())
+	if err != nil {
+		return nil, eris.Wrap(err, "failed to parse where clause")
+	}
+
+	return filter, nil
+}
+
 // findMatchingArchetypes returns the archetypes that match the given components and match type.
 func findMatchingArchetypes(w *World, compNames []string, match SearchMatch) ([]archetypeID, error) {
-	if len(compNames) == 0 {
-		return nil, eris.New("component list cannot be empty")
+	ws := w.state
+
+	// If match is MatchAll, return all archetype IDs
+	if match == MatchAll {
+		archIDs := make([]archetypeID, 0, len(ws.archetypes))
+		for id := range ws.archetypes {
+			archIDs = append(archIDs, id)
+		}
+		return archIDs, nil
 	}
 
-	ws := w.state
+	// Build component bitmap from names
 	component := bitmap.Bitmap{}
 	for _, name := range compNames {
 		id, exists := ws.components.catalog[name]
@@ -131,6 +152,7 @@ func findMatchingArchetypes(w *World, compNames []string, match SearchMatch) ([]
 		component.Set(id)
 	}
 
+	// Find matching archetypes based on match type
 	var archIDs []int
 	switch match {
 	case MatchExact:
@@ -140,10 +162,49 @@ func findMatchingArchetypes(w *World, compNames []string, match SearchMatch) ([]
 		}
 	case MatchContains:
 		archIDs = ws.archContains(component)
+	case MatchAll:
+		// This case should never be reached as MatchAll is handled earlier in the function
+		// but included for exhaustive switch coverage
+		return nil, eris.New("MatchAll should be handled before this switch")
 	}
 	return archIDs, nil
 }
 
+// buildEntityResult creates a result map from an entity ID and its components.
+func buildEntityResult(eid EntityID, components []Component) map[string]any {
+	result := make(map[string]any)
+	// We have to cast id from EntityID to int here or else we can't query the data because for some
+	// reason expr can't compare EntityID with integers in the expression.
+	result["_id"] = uint32(eid)
+
+	for _, component := range components {
+		result[component.Name()] = component
+	}
+
+	return result
+}
+
+// matchesFilter checks if an entity matches the filter expression.
+func matchesFilter(filter *vm.Program, result map[string]any) (bool, error) {
+	// Run the filter expression. We set the entity map as the environment for `Run` so the vm
+	// program has access to the entity data to filter.
+	output, err := expr.Run(filter, result)
+	if err != nil {
+		return false, eris.Wrap(err, "failed to run filter expression")
+	}
+
+	isMatchFilter, ok := output.(bool)
+	// Because we compile the expr once without passing in the environment, as it's only available
+	// while iterating, expr.Compile can't fully check if the expression returns a bool,
+	// especially when we filter for a struct field e.g. health.hp > 200, expr can't determine the
+	// type of health.hp during compilation.
+	if !ok {
+		return false, eris.New("invalid where clause")
+	}
+
+	return isMatchFilter, nil
+}
+
 // archIter returns an iterator of the archetypes entities and its components.
 func archIter(a *archetype) iter.Seq2[EntityID, []Component] {
 	return func(yield func(EntityID, []Component) bool) {

pkg/cardinal/service/query.go

@@ -19,20 +19,26 @@ import (
 // For now it has the same structure as ecs.SearchParam, but we might add more fields in the future
 // so it's better to keep these as separate types.
 type Query struct {
-	// List of component names to search for.
+	// List of component names to search for. Must be empty when Match is MatchAll.
 	Find []string `json:"find"`
-	// Match type: "exact" or "contains".
+	// Match type: "exact", "contains", or "all".
 	Match ecs.SearchMatch `json:"match"`
 	// Optional expr language string to filter the results.
 	// See https://expr-lang.org/ for documentation.
 	Where string `json:"where,omitempty"`
+	// Maximum number of results to return (default: unlimited, 0 = unlimited).
+	Limit uint32 `json:"limit,omitempty"`
+	// Number of results to skip before returning (default: 0).
+	Offset uint32 `json:"offset,omitempty"`
 }
 
 // reset resets the Query object for reuse.
 func (q *Query) reset() {
 	q.Find = q.Find[:0] // Reuse the underlying array
 	q.Match = ""
 	q.Where = ""
+	q.Limit = 0
+	q.Offset = 0
 }
 
 // handleQuery creates a new query handler for the world.
@@ -52,9 +58,11 @@ func (s *ShardService) handleQuery(ctx context.Context, req *micro.Request) *mic
 	defer s.queryPool.Put(query)
 
 	results, err := s.world.NewSearch(ecs.SearchParam{
-		Find:  query.Find,
-		Match: query.Match,
-		Where: query.Where,
+		Find:   query.Find,
+		Match:  query.Match,
+		Where:  query.Where,
+		Limit:  query.Limit,
+		Offset: query.Offset,
 	})
 	if err != nil {
 		return micro.NewErrorResponse(req, eris.Wrap(err, "failed to search entities"), codes.Internal)
@@ -86,6 +94,11 @@ func parseQuery(pool *sync.Pool, req *micro.Request) (*Query, error) {
 	query.Match = ecs.SearchMatch(iscv1MatchToString(payload.GetMatch()))
 	query.Where = payload.GetWhere()
 
+	// Parse Limit and Offset
+	// In proto3, unset uint32 fields return 0, which means unlimited for limit
+	query.Limit = payload.GetLimit()
+	query.Offset = payload.GetOffset()
+
 	return query, nil
 }
 
@@ -123,6 +136,8 @@ func iscv1MatchToString(m iscv1.Query_Match) string {
 		return "exact"
 	case iscv1.Query_MATCH_CONTAINS:
 		return "contains"
+	case iscv1.Query_MATCH_ALL:
+		return "all"
 	case iscv1.Query_MATCH_UNSPECIFIED:
 		fallthrough
 	default: