Allow ARRAY JOIN to be interleaved with other JOIN statements (#230)

Author: hermanschaaf

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.18.x]
+        go-version: [1.21.x]
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/AfterShip/clickhouse-sql-parser
 
-go 1.18
+go 1.21.0
 
 require (
 	github.com/sebdah/goldie/v2 v2.5.3

parser/ast.go

@@ -6422,6 +6422,13 @@ func (j *JoinExpr) Pos() Pos {
 }
 
 func (j *JoinExpr) End() Pos {
+	// Return the rightmost position
+	if j.Right != nil {
+		return j.Right.End()
+	}
+	if j.Constraints != nil {
+		return j.Constraints.End()
+	}
 	return j.Left.End()
 }
 
@@ -7237,36 +7244,6 @@ func (f *WindowFrameParam) Accept(visitor ASTVisitor) error {
 	return visitor.VisitWindowFrameParam(f)
 }
 
-type ArrayJoinClause struct {
-	ArrayPos Pos
-	Type     string
-	Expr     Expr
-}
-
-func (a *ArrayJoinClause) Pos() Pos {
-	return a.ArrayPos
-}
-
-func (a *ArrayJoinClause) End() Pos {
-	return a.Expr.End()
-}
-
-func (a *ArrayJoinClause) String() string {
-	if a.Type != "" {
-		return a.Type + " ARRAY JOIN " + a.Expr.String()
-	}
-	return "ARRAY JOIN " + a.Expr.String()
-}
-
-func (a *ArrayJoinClause) Accept(visitor ASTVisitor) error {
-	visitor.Enter(a)
-	defer visitor.Leave(a)
-	if err := a.Expr.Accept(visitor); err != nil {
-		return err
-	}
-	return visitor.VisitArrayJoinExpr(a)
-}
-
 type SelectQuery struct {
 	SelectPos     Pos
 	StatementEnd  Pos
@@ -7276,7 +7253,6 @@ type SelectQuery struct {
 	DistinctOn    *DistinctOn
 	SelectItems   []*SelectItem
 	From          *FromClause
-	ArrayJoin     []*ArrayJoinClause
 	Window        *WindowClause
 	Prewhere      *PrewhereClause
 	Where         *WhereClause
@@ -7337,10 +7313,6 @@ func (s *SelectQuery) String() string { // nolint: funlen
 		builder.WriteString(" ")
 		builder.WriteString(s.From.String())
 	}
-	for _, arrayJoin := range s.ArrayJoin {
-		builder.WriteString(" ")
-		builder.WriteString(arrayJoin.String())
-	}
 	if s.Window != nil {
 		builder.WriteString(" ")
 		builder.WriteString(s.Window.String())
@@ -7419,11 +7391,6 @@ func (s *SelectQuery) Accept(visitor ASTVisitor) error {
 			return err
 		}
 	}
-	for _, arrayJoin := range s.ArrayJoin {
-		if err := arrayJoin.Accept(visitor); err != nil {
-			return err
-		}
-	}
 	if s.Window != nil {
 		if err := s.Window.Accept(visitor); err != nil {
 			return err

parser/ast_visitor.go

@@ -153,7 +153,6 @@ type ASTVisitor interface {
 	VisitWindowFrameUnbounded(expr *WindowFrameUnbounded) error
 	VisitWindowFrameNumber(expr *WindowFrameNumber) error
 	VisitWindowFrameParam(expr *WindowFrameParam) error
-	VisitArrayJoinExpr(expr *ArrayJoinClause) error
 	VisitSelectQuery(expr *SelectQuery) error
 	VisitSubQueryExpr(expr *SubQuery) error
 	VisitNotExpr(expr *NotExpr) error
@@ -1270,13 +1269,6 @@ func (v *DefaultASTVisitor) VisitWindowFrameParam(expr *WindowFrameParam) error
 	return nil
 }
 
-func (v *DefaultASTVisitor) VisitArrayJoinExpr(expr *ArrayJoinClause) error {
-	if v.Visit != nil {
-		return v.Visit(expr)
-	}
-	return nil
-}
-
 func (v *DefaultASTVisitor) VisitSelectQuery(expr *SelectQuery) error {
 	if v.Visit != nil {
 		return v.Visit(expr)

parser/parser_query.go

@@ -3,6 +3,8 @@ package parser
 import (
 	"errors"
 	"fmt"
+
+	"slices"
 )
 
 func (p *Parser) tryParseWithClause(pos Pos) (*WithClause, error) {
@@ -196,11 +198,11 @@ func (p *Parser) parseJoinOp(_ Pos) []string {
 	case p.matchKeyword(KeywordInner):
 		modifiers = append(modifiers, p.last().String)
 		_ = p.lexer.consumeToken()
-		if p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAsof) {
+		if p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAsof) || p.matchKeyword(KeywordArray) {
 			modifiers = append(modifiers, p.last().String)
 			_ = p.lexer.consumeToken()
 		}
-	case p.matchKeyword(KeywordLeft), p.matchKeyword(KeywordRight):
+	case p.matchKeyword(KeywordLeft):
 		modifiers = append(modifiers, p.last().String)
 		_ = p.lexer.consumeToken()
 		if p.matchKeyword(KeywordOuter) {
@@ -213,6 +215,19 @@ func (p *Parser) parseJoinOp(_ Pos) []string {
 			modifiers = append(modifiers, p.last().String)
 			_ = p.lexer.consumeToken()
 		}
+	case p.matchKeyword(KeywordRight):
+		modifiers = append(modifiers, p.last().String)
+		_ = p.lexer.consumeToken()
+		if p.matchKeyword(KeywordOuter) {
+			modifiers = append(modifiers, p.last().String)
+			_ = p.lexer.consumeToken()
+		}
+		if p.matchKeyword(KeywordSemi) || p.matchKeyword(KeywordAnti) ||
+			p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAll) ||
+			p.matchKeyword(KeywordAsof) {
+			modifiers = append(modifiers, p.last().String)
+			_ = p.lexer.consumeToken()
+		}
 	case p.matchKeyword(KeywordFull):
 		modifiers = append(modifiers, p.last().String)
 		_ = p.lexer.consumeToken()
@@ -224,6 +239,9 @@ func (p *Parser) parseJoinOp(_ Pos) []string {
 			modifiers = append(modifiers, p.last().String)
 			_ = p.lexer.consumeToken()
 		}
+	case p.matchKeyword(KeywordArray):
+		modifiers = append(modifiers, p.last().String)
+		_ = p.lexer.consumeToken()
 	}
 	return modifiers
 }
@@ -281,6 +299,30 @@ func (p *Parser) parseJoinRightExpr(pos Pos) (expr Expr, err error) {
 	}
 
 	modifiers = append(modifiers, KeywordJoin)
+
+	// Check if this is an ARRAY JOIN
+	if slices.Contains(modifiers, KeywordArray) {
+		// For ARRAY JOIN, parse column expression list instead of table expression
+		expr, err = p.parseColumnExprList(p.Pos())
+		if err != nil {
+			return nil, err
+		}
+
+		// ARRAY JOIN doesn't have constraints (ON/USING)
+		// try parse next join
+		rightExpr, err = p.parseJoinRightExpr(p.Pos())
+		if err != nil {
+			return nil, err
+		}
+		return &JoinExpr{
+			JoinPos:     pos,
+			Left:        expr,
+			Right:       rightExpr,
+			Modifiers:   modifiers,
+			Constraints: nil,
+		}, nil
+	}
+
 	expr, err = p.parseJoinTableExpr(p.Pos())
 	if err != nil {
 		return nil, err
@@ -852,41 +894,6 @@ func (p *Parser) parseWindowClause(pos Pos) (*WindowClause, error) {
 	}, nil
 }
 
-func (p *Parser) tryParseArrayJoinClause(pos Pos) (*ArrayJoinClause, error) {
-	if !p.matchKeyword(KeywordLeft) && !p.matchKeyword(KeywordInner) && !p.matchKeyword(KeywordArray) {
-		return nil, nil
-	}
-	return p.parseArrayJoinClause(pos)
-}
-
-func (p *Parser) parseArrayJoinClause(_ Pos) (*ArrayJoinClause, error) {
-	var typ string
-	switch {
-	case p.matchKeyword(KeywordLeft), p.matchKeyword(KeywordInner):
-		typ = p.last().String
-		_ = p.lexer.consumeToken()
-	}
-	arrayPos := p.Pos()
-	if err := p.expectKeyword(KeywordArray); err != nil {
-		return nil, err
-	}
-
-	if err := p.expectKeyword(KeywordJoin); err != nil {
-		return nil, err
-	}
-
-	expr, err := p.parseColumnExprList(p.Pos())
-	if err != nil {
-		return nil, err
-	}
-
-	return &ArrayJoinClause{
-		ArrayPos: arrayPos,
-		Type:     typ,
-		Expr:     expr,
-	}, nil
-}
-
 func (p *Parser) tryParseHavingClause(pos Pos) (*HavingClause, error) {
 	if !p.matchKeyword(KeywordHaving) {
 		return nil, nil
@@ -1009,18 +1016,6 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun
 	if from != nil {
 		statementEnd = from.End()
 	}
-	var arrayJoins []*ArrayJoinClause
-	for {
-		arrayJoin, err := p.tryParseArrayJoinClause(p.Pos())
-		if err != nil {
-			return nil, err
-		}
-		if arrayJoin == nil {
-			break
-		}
-		arrayJoins = append(arrayJoins, arrayJoin)
-		statementEnd = arrayJoin.End()
-	}
 	prewhere, err := p.tryParsePrewhereClause(p.Pos())
 	if err != nil {
 		return nil, err
@@ -1129,7 +1124,6 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun
 		DistinctOn:   distinctOn,
 		SelectItems:  selectItems,
 		From:         from,
-		ArrayJoin:    arrayJoins,
 		Window:       window,
 		Prewhere:     prewhere,
 		Where:        where,

parser/parser_test.go

@@ -141,6 +141,9 @@ func TestParser_InvalidSyntax(t *testing.T) {
 		"SELECT n FROM t ORDER BY n WITH FILL INTERPOLATE (x",         // Missing closing paren
 		"SELECT n FROM t ORDER BY n WITH FILL INTERPOLATE x AS x + 1", // Missing parens around column list
 		"ALTER TABLE foo_mv MODIFY QUERY AS SELECT * FROM baz",        // MODIFY QUERY followed by an invalid query
+		// Invalid ARRAY JOIN types (only ARRAY JOIN, LEFT ARRAY JOIN, and INNER ARRAY JOIN are valid)
+		"SELECT * FROM t RIGHT ARRAY JOIN arr AS a", // RIGHT ARRAY JOIN not supported
+		"SELECT * FROM t FULL ARRAY JOIN arr AS a",  // FULL ARRAY JOIN not supported
 	}
 	for _, sql := range invalidSQLs {
 		parser := NewParser(sql)

parser/testdata/basic/output/quantile_functions.sql.golden.json

@@ -108,7 +108,6 @@
       }
     ],
     "From": null,
-    "ArrayJoin": null,
     "Window": null,
     "Prewhere": null,
     "Where": null,

parser/testdata/ddl/output/bug_001.sql.golden.json

@@ -523,7 +523,6 @@
             "HasFinal": false
           }
         },
-        "ArrayJoin": null,
         "Window": null,
         "Prewhere": null,
         "Where": {

parser/testdata/ddl/output/create_live_view_basic.sql.golden.json

@@ -119,7 +119,6 @@
             "HasFinal": false
           }
         },
-        "ArrayJoin": null,
         "Window": null,
         "Prewhere": null,
         "Where": null,

parser/testdata/ddl/output/create_materialized_view_basic.sql.golden.json

@@ -490,7 +490,6 @@
             "HasFinal": false
           }
         },
-        "ArrayJoin": null,
         "Window": null,
         "Prewhere": null,
         "Where": {