From 6e4d12594874aa3c1cd6f847295ef08a5fd4eb83 Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:49:31 -0500 Subject: [PATCH 01/14] move filters to function --- .gitignore | 2 + .../macros/bigquery/aggregate_metrics.sql | 2 - .../macros/distinct_values.sql | 2 - .../macros/expression_metrics.sql | 1 - .../rasgotransforms/macros/filter.sql | 59 ------------ .../macros/snowflake/aggregate_metrics.sql | 2 - .../rasgotransforms/render/environment.py | 95 +++++++++++++++---- .../tests/transforms/plot/plot_test.py | 7 +- .../transforms/clean/bigquery/clean.sql | 4 +- .../transforms/clean/clean.sql | 4 +- .../transforms/filter/filter.sql | 1 - .../rasgotransforms/transforms/plot/plot.sql | 1 - 12 files changed, 91 insertions(+), 89 deletions(-) diff --git a/.gitignore b/.gitignore index e9402e1b..f403247a 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,5 @@ cython_debug/ .idea test_artifacts/ artifacts/ + +pylintrc diff --git a/rasgotransforms/rasgotransforms/macros/bigquery/aggregate_metrics.sql b/rasgotransforms/rasgotransforms/macros/bigquery/aggregate_metrics.sql index f0e27517..66de8272 100644 --- a/rasgotransforms/rasgotransforms/macros/bigquery/aggregate_metrics.sql +++ b/rasgotransforms/rasgotransforms/macros/bigquery/aggregate_metrics.sql @@ -1,5 +1,3 @@ -{% from 'filter.sql' import get_filter_statement %} - {% macro calculate_timeseries_metric_values( aggregations, time_dimension, diff --git a/rasgotransforms/rasgotransforms/macros/distinct_values.sql b/rasgotransforms/rasgotransforms/macros/distinct_values.sql index 87df3433..0f278fed 100644 --- a/rasgotransforms/rasgotransforms/macros/distinct_values.sql +++ b/rasgotransforms/rasgotransforms/macros/distinct_values.sql @@ -1,5 +1,3 @@ -{% from 'filter.sql' import get_filter_statement %} - {% macro get_distinct_vals( columns, target_metric, diff --git a/rasgotransforms/rasgotransforms/macros/expression_metrics.sql b/rasgotransforms/rasgotransforms/macros/expression_metrics.sql index 15146a5b..777a4b83 100644 --- a/rasgotransforms/rasgotransforms/macros/expression_metrics.sql +++ b/rasgotransforms/rasgotransforms/macros/expression_metrics.sql @@ -1,5 +1,4 @@ {% from 'aggregate_metrics.sql' import calculate_timeseries_metric_values %} -{% from 'filter.sql' import combine_filters %} {% macro calculate_expression_metric_values( name, diff --git a/rasgotransforms/rasgotransforms/macros/filter.sql b/rasgotransforms/rasgotransforms/macros/filter.sql index 1b47d8c2..e69de29b 100644 --- a/rasgotransforms/rasgotransforms/macros/filter.sql +++ b/rasgotransforms/rasgotransforms/macros/filter.sql @@ -1,59 +0,0 @@ -{% macro get_filter_statement(filters) %} -{% if filters %} -{% if filters is string %} -{{ filters }} -{% else %} -{% set logical_operator = namespace(value='AND') %} -{% for filter in filters %} - {% if filter is not string %} - {% if filter is not mapping %} - {% set filter = dict(filter) %} - {% endif %} - {% if filter is mapping and 'compoundBoolean' in filter and filter['compoundBoolean'] %} - {% set logical_operator.value = filter['compoundBoolean'] %} - {% endif %} - {% endif %} -{% endfor %} -( - {% for filter in filters %} - {% if filter is not string and filter is not mapping %} - {% set filter = dict(filter) %} - {% endif %} - {% if 'columnName' in filter %} - {% do filter.__setitem__('column_name', filter.columnName) %} - {% endif %} - {% if 'comparisonValue' in filter %} - {% do filter.__setitem__('comparison_value', filter.comparisonValue) %} - {% endif %} - {% if filter is not mapping %} - {{ logical_operator.value + ' ' if not loop.first }}{{ filter }} - {% elif filter.operator|upper == 'CONTAINS' %} - {{ logical_operator.value + ' ' if not loop.first }}{{ filter.column_name }} like '%{{ filter.comparison_value }}%' - {% else %} - {{ logical_operator.value + ' ' if not loop.first }}{{ filter.column_name }} {{ filter.operator }} {{ filter.comparison_value }} - {% endif %} - {% endfor %} -) -{% endif %} -{% else %} -true -{% endif %} -{% endmacro %} - - -{% macro combine_filters(filters_a, filters_b, condition) %} -{% set condition = condition if condition is defined else 'AND' %} -{% if filters_a and not filters_b %} -{{ get_filter_statement(filters_a) }} -{% elif filters_b and not filters_a %} -{{ get_filter_statement(filters_b) }} -{% elif not filters_a and not filters_b %} -true -{% else %} -( - {{ get_filter_statement(filters_a)|indent }} - {{ condition }} - {{ get_filter_statement(filters_b)|indent }} -) -{% endif %} -{% endmacro %} diff --git a/rasgotransforms/rasgotransforms/macros/snowflake/aggregate_metrics.sql b/rasgotransforms/rasgotransforms/macros/snowflake/aggregate_metrics.sql index b4a63ec7..2d4f3d54 100644 --- a/rasgotransforms/rasgotransforms/macros/snowflake/aggregate_metrics.sql +++ b/rasgotransforms/rasgotransforms/macros/snowflake/aggregate_metrics.sql @@ -1,5 +1,3 @@ -{% from 'filter.sql' import get_filter_statement %} - {% macro calculate_timeseries_metric_values( aggregations, time_dimension, diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index 8d0c07b0..01eb0ed0 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -1,9 +1,9 @@ import re from datetime import datetime from itertools import combinations, permutations, product -from typing import Callable, Optional, Dict -from pathlib import Path from os.path import getmtime +from pathlib import Path +from typing import Callable, Dict, List, Optional, Union from jinja2 import Environment, BaseLoader from jinja2.exceptions import TemplateNotFound @@ -12,19 +12,33 @@ from rasgotransforms.exceptions import RenderException from rasgotransforms.main import DataWarehouse +ALLOWED_OPERATORS = ( + ">", + "<", + "=", + "<>", + ">=", + "<=", + "CONTAINS", + "IS NULL", + "IS NOT NULL", + "NOT CONTAINS", + "IN", + "NOT IN", +) class RasgoEnvironment(Environment): def __init__(self, run_query: Optional[Callable] = None, dw_type: Optional[str] = None, *args, **kwargs): super().__init__(*args, extensions=self.rasgo_extensions, loader=RasgoLoader(), **kwargs) if not dw_type: - dw_type = 'snowflake' + dw_type = "snowflake" self._dw_type = DataWarehouse(dw_type) for filter_name, method in self.rasgo_filters.items(): self.filters[filter_name] = method for name, value in self.rasgo_globals.items(): self.globals[name] = value self._run_query = run_query - self.globals['run_query'] = self._run_query + self.globals["run_query"] = self._run_query @property def dw_type(self) -> DataWarehouse: @@ -36,12 +50,14 @@ def dw_type(self, dw_type: str): @property def rasgo_extensions(self): - return ['jinja2.ext.do', 'jinja2.ext.loopcontrols'] + return ["jinja2.ext.do", "jinja2.ext.loopcontrols"] @property def rasgo_globals(self): return { "cleanse_name": cleanse_template_symbol, + "get_filter_statement": get_filter_statement, + "combine_filters": combine_filters, "raise_exception": raise_exception, "itertools": {"combinations": combinations, "permutations": permutations, "product": product}, } @@ -69,46 +85,93 @@ def render( Source columns can is a mapping of table names to columns (e.g. {table_name: {column_name: column_type}}) """ - arguments['source_table'] = source_table + arguments["source_table"] = source_table if not override_globals: override_globals = {} - if source_columns and 'get_columns' not in override_globals: + if source_columns and "get_columns" not in override_globals: def get_columns(fqtn): return source_columns[fqtn] - override_globals['get_columns'] = get_columns - if 'get_columns' in override_globals: - self.globals['get_columns'] = override_globals['get_columns'] + override_globals["get_columns"] = get_columns + if "get_columns" in override_globals: + self.globals["get_columns"] = override_globals["get_columns"] try: template = self.from_string(source_code) rendered = template.render(**arguments, **override_globals) except Exception as e: - raise RenderException(e) + raise RenderException(e) from e return trim_blank_lines(rendered) def cleanse_template_symbol(symbol: str) -> str: - symbol = str(symbol).strip().replace(' ', '_').replace('-', '_') - symbol = re.sub('[^A-Za-z0-9_]+', '', symbol) - symbol = '_' + symbol if not symbol or symbol[0].isdecimal() else symbol + symbol = str(symbol).strip().replace(" ", "_").replace("-", "_") + symbol = re.sub("[^A-Za-z0-9_]+", "", symbol) + symbol = "_" + symbol if not symbol or symbol[0].isdecimal() else symbol return symbol +def combine_filters( + filters_a: Union[List, str], + filters_b: Union[List, str], + condition: str +) -> str: + """ + Parse & combine multiple filters, return a single SQL statement + """ + condition = condition or "AND" + if filters_a and not filters_b: + return get_filter_statement(filters_a) + elif filters_b and not filters_a: + return get_filter_statement(filters_b) + elif not filters_a and not filters_b: + return "true" + return f"({get_filter_statement(filters_a)} {condition} {get_filter_statement(filters_b)})" + + +def get_filter_statement(filters: Union[List, str]) -> str: + """ + Parse a list of string or dict filters to a simple SQL string + """ + if isinstance(filters, str): + return filters + + filter_string = "" + for fil in filters: + fil: dict + + # Handle variable casing from past versions: only support snake evnetually + column_name = fil.get("column_name", fil.get("columnName")) + operator = fil.get("operator") + comparison_value = fil.get("comparison_value", fil.get("comparisonValue")) + compound_boolean = fil.get("compound_boolean", fil.get("compoundBoolean"), "AND") + + # Handle override operators + if operator not in ALLOWED_OPERATORS: + raise_exception(f"operator {operator} is not supported") + if operator == "CONTAINS": + operator = "LIKE" + + compound_boolean = "" if filter_string == "" else compound_boolean + filter_string += f"{compound_boolean} {column_name} {operator} {comparison_value} " + + return filter_string + + def raise_exception(message: str) -> None: raise Exception(message) def trim_blank_lines(sql: str) -> str: - return re.sub(r'[\n][\s]*\n', '\n', sql) + return re.sub(r"[\n][\s]*\n", "\n", sql) class RasgoLoader(BaseLoader): def __init__(self, root_path=None): if not root_path: - root_path = Path(__file__).parent.parent / 'macros' + root_path = Path(__file__).parent.parent / "macros" self.root_path = root_path def get_source(self, environment: RasgoEnvironment, template): diff --git a/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py b/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py index cbc12afa..66e27b1e 100644 --- a/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py +++ b/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py @@ -33,7 +33,12 @@ def test_datetime_no_dimensions(self, dw_type): 'operator': '>=', 'comparison_value': 1, }, - {'column_name': 'COLOR', 'operator': '=', 'comparison_value': "'Black'", 'compoundBoolean': 'OR'}, + { + 'column_name': 'COLOR', + 'operator': '=', + 'comparison_value': "'Black'", + 'compoundBoolean': 'OR' + }, ], } environment = RasgoEnvironment(dw_type=dw_type, run_query=partial(run_query, dw_type=dw_type)) diff --git a/rasgotransforms/rasgotransforms/transforms/clean/bigquery/clean.sql b/rasgotransforms/rasgotransforms/transforms/clean/bigquery/clean.sql index 613ecacc..afb80533 100644 --- a/rasgotransforms/rasgotransforms/transforms/clean/bigquery/clean.sql +++ b/rasgotransforms/rasgotransforms/transforms/clean/bigquery/clean.sql @@ -56,7 +56,7 @@ {{ output }} as {{ output_col_name }} {%- endmacro -%} -{%- macro get_filter_statement(columns) -%} +{%- macro get_where_clause(columns) -%} {%- set filter_statements = [] -%} {%- for column in columns.keys() %} {%- set output_col_name = column if columns[column].name is not defined else cleanse_name(columns[column].name) -%} @@ -68,7 +68,7 @@ {{ "where " if loop.first else "" }}{{ filter_statement }}{{ " \n and " if not loop.last else "" }} {%- endfor %} {%- endmacro -%} -{%- set filter_statement = get_filter_statement(columns) -%} +{%- set filter_statement = get_where_clause(columns) -%} {%- if filter_statement == "" -%} select diff --git a/rasgotransforms/rasgotransforms/transforms/clean/clean.sql b/rasgotransforms/rasgotransforms/transforms/clean/clean.sql index 346f971a..b6bb959f 100644 --- a/rasgotransforms/rasgotransforms/transforms/clean/clean.sql +++ b/rasgotransforms/rasgotransforms/transforms/clean/clean.sql @@ -34,7 +34,7 @@ {{ output }} as {{ output_col_name }} {%- endmacro -%} -{%- macro get_filter_statement(columns) -%} +{%- macro get_where_clause(columns) -%} {%- set filter_statements = [] -%} {%- for column in columns.keys() %} {%- set output_col_name = column if columns[column].name is not defined else cleanse_name(columns[column].name) -%} @@ -46,7 +46,7 @@ {{ "where " if loop.first else "" }}{{ filter_statement }}{{ " \n and " if not loop.last else "" }} {%- endfor %} {%- endmacro -%} -{%- set filter_statement = get_filter_statement(columns) -%} +{%- set filter_statement = get_where_clause(columns) -%} {%- if filter_statement == "" -%} select diff --git a/rasgotransforms/rasgotransforms/transforms/filter/filter.sql b/rasgotransforms/rasgotransforms/transforms/filter/filter.sql index 73450e4a..b1099711 100644 --- a/rasgotransforms/rasgotransforms/transforms/filter/filter.sql +++ b/rasgotransforms/rasgotransforms/transforms/filter/filter.sql @@ -1,4 +1,3 @@ -{% from 'filter.sql' import get_filter_statement %} {% if items is not defined %} {% if filter_statements is not defined %} {{ raise_exception('items is empty: there are no filters to apply') }} diff --git a/rasgotransforms/rasgotransforms/transforms/plot/plot.sql b/rasgotransforms/rasgotransforms/transforms/plot/plot.sql index bec2d990..b335b822 100644 --- a/rasgotransforms/rasgotransforms/transforms/plot/plot.sql +++ b/rasgotransforms/rasgotransforms/transforms/plot/plot.sql @@ -2,7 +2,6 @@ {% from 'expression_metrics.sql' import calculate_expression_metric_values %} {% from 'distinct_values.sql' import get_distinct_vals %} {% from 'pivot.sql' import pivot_plot_values %} -{% from 'filter.sql' import get_filter_statement, combine_filters %} {% set dimensions = group_by if group_by is defined else [] %} {% set flatten = flatten if flatten is defined else true %} {% set max_num_groups = max_num_groups if max_num_groups is defined else 10 %} From 30219b0d07e85b6a8f4b217513dfb53ddf0704c1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 20:53:14 +0000 Subject: [PATCH 02/14] [pre-commit.ci] auto fixes from pre-commit.com hooks --- rasgotransforms/rasgotransforms/render/environment.py | 7 ++----- .../rasgotransforms/tests/transforms/plot/plot_test.py | 7 +------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index d685fe5a..035c657c 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -27,6 +27,7 @@ "NOT IN", ) + class RasgoEnvironment(Environment): def __init__(self, run_query: Optional[Callable] = None, dw_type: Optional[str] = None, *args, **kwargs): super().__init__(*args, extensions=self.rasgo_extensions, loader=RasgoLoader(), **kwargs) @@ -117,11 +118,7 @@ def cleanse_template_symbol(symbol: str) -> str: return symbol -def combine_filters( - filters_a: Union[List, str], - filters_b: Union[List, str], - condition: str -) -> str: +def combine_filters(filters_a: Union[List, str], filters_b: Union[List, str], condition: str) -> str: """ Parse & combine multiple filters, return a single SQL statement """ diff --git a/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py b/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py index 74a534e2..a68320ae 100644 --- a/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py +++ b/rasgotransforms/rasgotransforms/tests/transforms/plot/plot_test.py @@ -33,12 +33,7 @@ def test_datetime_no_dimensions(self, dw_type): 'operator': '>=', 'comparison_value': 1, }, - { - 'column_name': 'COLOR', - 'operator': '=', - 'comparison_value': "'Black'", - 'compoundBoolean': 'OR' - }, + {'column_name': 'COLOR', 'operator': '=', 'comparison_value': "'Black'", 'compoundBoolean': 'OR'}, ], } environment = RasgoEnvironment(dw_type=dw_type, run_query=partial(run_query, dw_type=dw_type)) From 91613810558b1afc8852966903ea7a0f767cad9e Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Fri, 9 Dec 2022 00:13:07 -0500 Subject: [PATCH 03/14] apply PR feedback --- .../rasgotransforms/render/environment.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index 035c657c..5594f49b 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -139,24 +139,25 @@ def get_filter_statement(filters: Union[List, str]) -> str: if isinstance(filters, str): return filters - filter_string = "" + filter_string = "TRUE" + compound_boolean = "AND" for fil in filters: - fil: dict - - # Handle variable casing from past versions: only support snake evnetually - column_name = fil.get("column_name", fil.get("columnName")) - operator = fil.get("operator") - comparison_value = fil.get("comparison_value", fil.get("comparisonValue")) - compound_boolean = fil.get("compound_boolean", fil.get("compoundBoolean"), "AND") - - # Handle override operators - if operator not in ALLOWED_OPERATORS: - raise_exception(f"operator {operator} is not supported") - if operator == "CONTAINS": - operator = "LIKE" - - compound_boolean = "" if filter_string == "" else compound_boolean - filter_string += f"{compound_boolean} {column_name} {operator} {comparison_value} " + if isinstance(fil, dict): + # Handle variable casing from past versions: only support snake eventually + column_name = fil.get("column_name", fil.get("columnName")) + operator = fil.get("operator") + comparison_value = fil.get("comparison_value", fil.get("comparisonValue")) + compound_boolean = fil.get("compound_boolean", fil.get("compoundBoolean", compound_boolean)) + + # Handle override operators + if operator not in ALLOWED_OPERATORS: + raise_exception(f"operator {operator} is not supported") + if operator == "CONTAINS": + operator = "LIKE" + + filter_string += f"{compound_boolean} {column_name} {operator} {comparison_value} \n" + elif isinstance(fil, str) and fil != "": + filter_string += f"{compound_boolean} {fil}" return filter_string From 3ebcd78ea1e70eb7491901565139ce13b447fcfa Mon Sep 17 00:00:00 2001 From: Kimball Hill Date: Fri, 9 Dec 2022 10:43:51 -0700 Subject: [PATCH 04/14] added whitespace --- rasgotransforms/rasgotransforms/render/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index 5594f49b..daff7ac0 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -155,9 +155,9 @@ def get_filter_statement(filters: Union[List, str]) -> str: if operator == "CONTAINS": operator = "LIKE" - filter_string += f"{compound_boolean} {column_name} {operator} {comparison_value} \n" + filter_string += f" {compound_boolean} {column_name} {operator} {comparison_value} \n" elif isinstance(fil, str) and fil != "": - filter_string += f"{compound_boolean} {fil}" + filter_string += f" {compound_boolean} {fil} \n" return filter_string From 088e58c764995b9bdb00e10f2c6c7c2a2720dcdc Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Sat, 17 Dec 2022 14:38:29 -0500 Subject: [PATCH 05/14] support relative date filters --- .../rasgotransforms/render/environment.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index daff7ac0..bcf482ae 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -1,3 +1,4 @@ +import functools import re from datetime import datetime, timedelta from itertools import combinations, permutations, product @@ -59,8 +60,8 @@ def rasgo_globals(self): "min": min, "max": max, "cleanse_name": cleanse_template_symbol, - "get_filter_statement": get_filter_statement, - "combine_filters": combine_filters, + "get_filter_statement": functools.partial(get_filter_statement, dw_type=self.dw_type), + "combine_filters": functools.partial(combine_filters, dw_type=self.dw_type), "raise_exception": raise_exception, "itertools": {"combinations": combinations, "permutations": permutations, "product": product}, "dw_type": lambda: self.dw_type.value, @@ -118,21 +119,29 @@ def cleanse_template_symbol(symbol: str) -> str: return symbol -def combine_filters(filters_a: Union[List, str], filters_b: Union[List, str], condition: str) -> str: +def combine_filters( + filters_a: Union[List, str], + filters_b: Union[List, str], + condition: str, + dw_type: DataWarehouse, +) -> str: """ Parse & combine multiple filters, return a single SQL statement """ condition = condition or "AND" if filters_a and not filters_b: - return get_filter_statement(filters_a) + return get_filter_statement(filters_a, dw_type) elif filters_b and not filters_a: - return get_filter_statement(filters_b) + return get_filter_statement(filters_b, dw_type) elif not filters_a and not filters_b: - return "true" - return f"({get_filter_statement(filters_a)} {condition} {get_filter_statement(filters_b)})" + return "TRUE" + return f"({get_filter_statement(filters_a, dw_type)} {condition} {get_filter_statement(filters_b, dw_type)})" -def get_filter_statement(filters: Union[List, str]) -> str: +def get_filter_statement( + filters: Union[List, str], + dw_type: DataWarehouse, +) -> str: """ Parse a list of string or dict filters to a simple SQL string """ @@ -155,6 +164,19 @@ def get_filter_statement(filters: Union[List, str]) -> str: if operator == "CONTAINS": operator = "LIKE" + # Parse comparison value + if isinstance(comparison_value, dict): + # Relative Date filter + if "date_part" in comparison_value: + if dw_type == "BIGQUERY": + comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {comparison_value['date_part']})" + elif dw_type == "SNOWFLAKE": + comparison_value = f"DATEADD({comparison_value['date_part']}, {comparison_value['offset']}, CURRENT_DATE)" + else: + comparison_value = f"CURRENT_DATE + INTERVAL {comparison_value['offset']} {comparison_value['date_part']}" + + # Other filter types here... + filter_string += f" {compound_boolean} {column_name} {operator} {comparison_value} \n" elif isinstance(fil, str) and fil != "": filter_string += f" {compound_boolean} {fil} \n" From ebde58b246336a1ea6dca5adcd581b36b15b4b60 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 Dec 2022 19:38:40 +0000 Subject: [PATCH 06/14] [pre-commit.ci] auto fixes from pre-commit.com hooks --- rasgotransforms/rasgotransforms/render/environment.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index bcf482ae..e9de4a79 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -171,9 +171,13 @@ def get_filter_statement( if dw_type == "BIGQUERY": comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {comparison_value['date_part']})" elif dw_type == "SNOWFLAKE": - comparison_value = f"DATEADD({comparison_value['date_part']}, {comparison_value['offset']}, CURRENT_DATE)" + comparison_value = ( + f"DATEADD({comparison_value['date_part']}, {comparison_value['offset']}, CURRENT_DATE)" + ) else: - comparison_value = f"CURRENT_DATE + INTERVAL {comparison_value['offset']} {comparison_value['date_part']}" + comparison_value = ( + f"CURRENT_DATE + INTERVAL {comparison_value['offset']} {comparison_value['date_part']}" + ) # Other filter types here... From 232c60508479bf84c6c47960e8e7bbfc346d587c Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Sat, 17 Dec 2022 14:41:40 -0500 Subject: [PATCH 07/14] fix --- .../rasgotransforms/transforms/metric_plot/metric_plot.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/rasgotransforms/rasgotransforms/transforms/metric_plot/metric_plot.sql b/rasgotransforms/rasgotransforms/transforms/metric_plot/metric_plot.sql index d13cce20..15efdaf8 100644 --- a/rasgotransforms/rasgotransforms/transforms/metric_plot/metric_plot.sql +++ b/rasgotransforms/rasgotransforms/transforms/metric_plot/metric_plot.sql @@ -2,7 +2,6 @@ {% from 'expression_metrics.sql' import calculate_expression_metric_values %} {% from 'distinct_values.sql' import get_distinct_vals %} {% from 'pivot.sql' import pivot_plot_values %} -{% from 'filter.sql' import get_filter_statement, combine_filters %} {% from 'secondary_calculation.sql' import render_secondary_calculations, adjust_start_date %} {% set dimensions = group_by_dimensions if group_by_dimensions is defined else [] %} {% set max_num_groups = max_num_groups if max_num_groups is defined else 10 %} From 1e46b56803720b778e9a73175d0e4c38d8269e87 Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Sat, 17 Dec 2022 14:46:30 -0500 Subject: [PATCH 08/14] fix caps --- .../rasgotransforms/render/environment.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index e9de4a79..de68d0ca 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -168,9 +168,9 @@ def get_filter_statement( if isinstance(comparison_value, dict): # Relative Date filter if "date_part" in comparison_value: - if dw_type == "BIGQUERY": + if dw_type == "bigquery": comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {comparison_value['date_part']})" - elif dw_type == "SNOWFLAKE": + elif dw_type == "snowflake": comparison_value = ( f"DATEADD({comparison_value['date_part']}, {comparison_value['offset']}, CURRENT_DATE)" ) @@ -198,19 +198,19 @@ def trim_blank_lines(sql: str) -> str: def get_timedelta(time_grain: str, interval: int) -> timedelta: time_grain = time_grain.lower() - if time_grain == 'hour': + if time_grain == "hour": return timedelta(hours=interval) - elif time_grain == 'day': + elif time_grain == "day": return timedelta(days=interval) - elif time_grain == 'week': + elif time_grain == "week": return timedelta(weeks=interval) - elif time_grain == 'month': + elif time_grain == "month": interval *= 31 return timedelta(days=interval) - elif time_grain == 'quarter': + elif time_grain == "quarter": interval *= 122 return timedelta(days=interval) - elif time_grain == 'year': + elif time_grain == "year": interval *= 365 return timedelta(days=interval) else: From 397f97dbfd2102cc57b1e570310736435fbdd9da Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Sat, 17 Dec 2022 14:57:09 -0500 Subject: [PATCH 09/14] handle casing --- .../rasgotransforms/render/environment.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index de68d0ca..b0a9c3f2 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -164,27 +164,19 @@ def get_filter_statement( if operator == "CONTAINS": operator = "LIKE" - # Parse comparison value + # Parse complex comparison values if isinstance(comparison_value, dict): # Relative Date filter - if "date_part" in comparison_value: + if date_part := comparison_value.get("date_part", comparison_value.get("datePart")): if dw_type == "bigquery": - comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {comparison_value['date_part']})" + comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {date_part})" elif dw_type == "snowflake": - comparison_value = ( - f"DATEADD({comparison_value['date_part']}, {comparison_value['offset']}, CURRENT_DATE)" - ) + comparison_value = f"DATEADD({date_part}, {comparison_value['offset']}, CURRENT_DATE)" else: - comparison_value = ( - f"CURRENT_DATE + INTERVAL {comparison_value['offset']} {comparison_value['date_part']}" - ) - - # Other filter types here... - + comparison_value = f"(CURRENT_DATE + INTERVAL {comparison_value['offset']} {date_part})" filter_string += f" {compound_boolean} {column_name} {operator} {comparison_value} \n" elif isinstance(fil, str) and fil != "": filter_string += f" {compound_boolean} {fil} \n" - return filter_string From 7883b36cad70cb972fd67cdb24cf5773b33003b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 Dec 2022 19:57:20 +0000 Subject: [PATCH 10/14] [pre-commit.ci] auto fixes from pre-commit.com hooks --- rasgotransforms/rasgotransforms/render/environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index b0a9c3f2..ac5e0b46 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -169,7 +169,9 @@ def get_filter_statement( # Relative Date filter if date_part := comparison_value.get("date_part", comparison_value.get("datePart")): if dw_type == "bigquery": - comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {date_part})" + comparison_value = ( + f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {date_part})" + ) elif dw_type == "snowflake": comparison_value = f"DATEADD({date_part}, {comparison_value['offset']}, CURRENT_DATE)" else: From 7a4ffbd4eb2365a88c4e2e40be4c4689627be076 Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Sat, 17 Dec 2022 15:11:59 -0500 Subject: [PATCH 11/14] introduce comparison value type --- rasgotransforms/rasgotransforms/render/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index b0a9c3f2..6b954086 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -167,7 +167,8 @@ def get_filter_statement( # Parse complex comparison values if isinstance(comparison_value, dict): # Relative Date filter - if date_part := comparison_value.get("date_part", comparison_value.get("datePart")): + if comparison_value.get("type", "") == "RELATIVEDATE": + date_part = comparison_value.get("date_part", comparison_value.get("datePart")) if dw_type == "bigquery": comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {date_part})" elif dw_type == "snowflake": From dd2eebc22c211d63882e77fb6c9ce439613f0d7c Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Tue, 20 Dec 2022 10:56:54 -0500 Subject: [PATCH 12/14] fix it --- .../rasgotransforms/render/environment.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index 2531d266..4667a3bb 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -154,7 +154,7 @@ def get_filter_statement( if isinstance(fil, dict): # Handle variable casing from past versions: only support snake eventually column_name = fil.get("column_name", fil.get("columnName")) - operator = fil.get("operator") + operator = fil.get("operator", "").upper() comparison_value = fil.get("comparison_value", fil.get("comparisonValue")) compound_boolean = fil.get("compound_boolean", fil.get("compoundBoolean", compound_boolean)) @@ -167,16 +167,20 @@ def get_filter_statement( # Parse complex comparison values if isinstance(comparison_value, dict): # Relative Date filter - if comparison_value.get("type", "") == "RELATIVEDATE": + if comparison_value.get("type", "").upper() == "RELATIVEDATE": date_part = comparison_value.get("date_part", comparison_value.get("datePart")) - if dw_type == "bigquery": + interval = f"{'-' if comparison_value['direction'] == 'past' else ''}{comparison_value['offset']}" + if not (date_part and interval): + raise_exception("relativedate comparisons must pass arguments: date_part, offset, direction") + print(dw_type) + if dw_type is DataWarehouse.BIGQUERY: comparison_value = ( - f"DATE_ADD(CURRENT_DATE(), INTERVAL {comparison_value['offset']} {date_part})" + f"DATE_ADD(CURRENT_DATE(), INTERVAL {interval} {date_part})" ) - elif dw_type == "snowflake": - comparison_value = f"DATEADD({date_part}, {comparison_value['offset']}, CURRENT_DATE)" + elif dw_type is DataWarehouse.SNOWFLAKE: + comparison_value = f"DATEADD({date_part}, {interval}, CURRENT_DATE)" else: - comparison_value = f"(CURRENT_DATE + INTERVAL {comparison_value['offset']} {date_part})" + comparison_value = f"(CURRENT_DATE + INTERVAL '{interval} {date_part}')" filter_string += f" {compound_boolean} {column_name} {operator} {comparison_value} \n" elif isinstance(fil, str) and fil != "": filter_string += f" {compound_boolean} {fil} \n" From 43ef483e4c2d3c4f167d079caf4af87e7521506a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Dec 2022 15:57:14 +0000 Subject: [PATCH 13/14] [pre-commit.ci] auto fixes from pre-commit.com hooks --- rasgotransforms/rasgotransforms/render/environment.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rasgotransforms/rasgotransforms/render/environment.py b/rasgotransforms/rasgotransforms/render/environment.py index 4667a3bb..276bdc3b 100644 --- a/rasgotransforms/rasgotransforms/render/environment.py +++ b/rasgotransforms/rasgotransforms/render/environment.py @@ -174,9 +174,7 @@ def get_filter_statement( raise_exception("relativedate comparisons must pass arguments: date_part, offset, direction") print(dw_type) if dw_type is DataWarehouse.BIGQUERY: - comparison_value = ( - f"DATE_ADD(CURRENT_DATE(), INTERVAL {interval} {date_part})" - ) + comparison_value = f"DATE_ADD(CURRENT_DATE(), INTERVAL {interval} {date_part})" elif dw_type is DataWarehouse.SNOWFLAKE: comparison_value = f"DATEADD({date_part}, {interval}, CURRENT_DATE)" else: From fee6647f5599a55061bcb31188705c6602aa5e07 Mon Sep 17 00:00:00 2001 From: griff <70294474+griffatrasgo@users.noreply.github.com> Date: Tue, 20 Dec 2022 21:45:03 -0500 Subject: [PATCH 14/14] bump version --- rasgotransforms/rasgotransforms/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasgotransforms/rasgotransforms/version.py b/rasgotransforms/rasgotransforms/version.py index 9f4470e1..2d67729f 100644 --- a/rasgotransforms/rasgotransforms/version.py +++ b/rasgotransforms/rasgotransforms/version.py @@ -1,4 +1,4 @@ """ Package version for pypi """ -__version__ = "2.4.1" +__version__ = "2.5.0a1"