Skip to content

Commit f9b5cc6

Browse files
flarcoflarco
committed
feat(mysql): add tests for LOAD DATA LOCAL INFILE and NULL handling; enhance MySQL/MariaDB connection for local infile support
1 parent f32e80a commit f9b5cc6

File tree

6 files changed

+391
-60
lines changed

6 files changed

+391
-60
lines changed

‎cmd/sling/tests/replications/r.93.mysql_load_data_local.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Test MySQL LoadDataLocal using RegisterReaderHandler pattern
2+
# This validates that LOAD DATA LOCAL INFILE works with the go-sql-driver's
3+
# native Reader:: handler pattern without requiring external mysql binary.
4+
source: local
5+
target: mysql
6+
7+
defaults:
8+
mode: full-refresh
9+
10+
hooks:
11+
end:
12+
# Check execution succeeded
13+
- type: check
14+
check: execution.status.error == 0
15+
on_failure: break
16+
17+
# Verify data was loaded
18+
- type: query
19+
connection: '{target.name}'
20+
query: SELECT COUNT(*) as cnt FROM mysql.test_load_local
21+
into: result
22+
23+
- type: log
24+
message: "Row count: {store.result[0].cnt}"
25+
26+
# Verify row count matches source file (18 rows in test1.1.csv)
27+
- type: check
28+
check: int_parse(store.result[0].cnt) == 18
29+
failure_message: "Expected 18 rows but found {store.result[0].cnt}"
30+
31+
# Sample some data to verify correctness
32+
- type: query
33+
connection: '{target.name}'
34+
query: SELECT id, first_name, last_name, email FROM mysql.test_load_local WHERE id = 1
35+
into: sample_row
36+
37+
- type: log
38+
message: "Sample row: {store.sample_row}"
39+
40+
- type: check
41+
check: int_parse(store.sample_row[0].id) == 1
42+
failure_message: "Expected id=1 but found {store.sample_row[0].id}"
43+
44+
- type: log
45+
message: "SUCCESS: MySQL LoadDataLocal test passed"
46+
47+
# Cleanup target table
48+
- type: query
49+
connection: '{target.name}'
50+
query: DROP TABLE IF EXISTS mysql.test_load_local
51+
52+
streams:
53+
file://cmd/sling/tests/files/test1.1.csv:
54+
object: mysql.test_load_local

‎cmd/sling/tests/replications/r.94.mysql_load_data_local_nulls.yaml‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
# Test MySQL LoadDataLocal NULL handling
2+
# Validates that NULL values are correctly transmitted via LOAD DATA LOCAL INFILE
3+
# Uses PostgreSQL as source with generate_data and manual NULL insertions
4+
source: postgres
5+
target: mysql
6+
7+
defaults:
8+
mode: full-refresh
9+
10+
hooks:
11+
start:
12+
# Generate test data in PostgreSQL
13+
- type: query
14+
connection: '{source.name}'
15+
operation: generate_data
16+
params:
17+
table: public.mysql_null_test
18+
rows: 50
19+
columns:
20+
col_bigint: bigint
21+
col_bool: bool
22+
col_date: date
23+
col_datetime: datetime
24+
col_decimal: decimal
25+
col_integer: integer
26+
col_smallint: smallint
27+
col_string: string
28+
col_text: text
29+
col_float: float
30+
31+
# Insert 3 rows with NULL values and special characters
32+
- type: query
33+
connection: '{source.name}'
34+
query: |
35+
INSERT INTO public.mysql_null_test
36+
(col_bigint, col_bool, col_date, col_datetime, col_decimal, col_integer, col_smallint, col_string, col_text, col_float)
37+
VALUES
38+
(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
39+
(999999, NULL, NULL, '2024-01-15 10:30:00', NULL, 12345, NULL, NULL, 'text with value', NULL),
40+
(888888, true, '2024-06-20', '2024-06-20 15:45:30', 123.456, 54321, 100, 'string with "double quotes" and ''single quotes''', 'text with tabs and
41+
newlines and "quotes" here', 999.99);
42+
43+
end:
44+
# Check execution succeeded
45+
- type: check
46+
check: execution.status.error == 0
47+
on_failure: break
48+
49+
# Verify row count matches (50 generated + 3 manual = 53)
50+
- type: query
51+
connection: '{source.name}'
52+
query: SELECT COUNT(*) as cnt FROM public.mysql_null_test
53+
into: source_count
54+
55+
- type: query
56+
connection: '{target.name}'
57+
query: SELECT COUNT(*) as cnt FROM mysql.mysql_null_test
58+
into: target_count
59+
60+
- type: log
61+
message: "Source rows: {store.source_count[0].cnt}, Target rows: {store.target_count[0].cnt}"
62+
63+
- type: check
64+
check: store.source_count[0].cnt == store.target_count[0].cnt
65+
failure_message: "Row count mismatch: source={store.source_count[0].cnt}, target={store.target_count[0].cnt}"
66+
67+
# Count NULL values in source
68+
- type: query
69+
connection: '{source.name}'
70+
query: |
71+
SELECT
72+
SUM(CASE WHEN col_string IS NULL THEN 1 ELSE 0 END) as null_string_cnt,
73+
SUM(CASE WHEN col_integer IS NULL THEN 1 ELSE 0 END) as null_integer_cnt,
74+
SUM(CASE WHEN col_decimal IS NULL THEN 1 ELSE 0 END) as null_decimal_cnt,
75+
SUM(CASE WHEN col_bool IS NULL THEN 1 ELSE 0 END) as null_bool_cnt,
76+
SUM(CASE WHEN col_text IS NULL THEN 1 ELSE 0 END) as null_text_cnt
77+
FROM public.mysql_null_test
78+
into: source_nulls
79+
80+
# Count NULL values in target
81+
- type: query
82+
connection: '{target.name}'
83+
query: |
84+
SELECT
85+
SUM(CASE WHEN col_string IS NULL THEN 1 ELSE 0 END) as null_string_cnt,
86+
SUM(CASE WHEN col_integer IS NULL THEN 1 ELSE 0 END) as null_integer_cnt,
87+
SUM(CASE WHEN col_decimal IS NULL THEN 1 ELSE 0 END) as null_decimal_cnt,
88+
SUM(CASE WHEN col_bool IS NULL THEN 1 ELSE 0 END) as null_bool_cnt,
89+
SUM(CASE WHEN col_text IS NULL THEN 1 ELSE 0 END) as null_text_cnt
90+
FROM mysql.mysql_null_test
91+
into: target_nulls
92+
93+
- type: log
94+
message: |
95+
Source NULLs: string={store.source_nulls[0].null_string_cnt}, integer={store.source_nulls[0].null_integer_cnt}, decimal={store.source_nulls[0].null_decimal_cnt}
96+
Target NULLs: string={store.target_nulls[0].null_string_cnt}, integer={store.target_nulls[0].null_integer_cnt}, decimal={store.target_nulls[0].null_decimal_cnt}
97+
98+
# Verify NULL counts match for each column type
99+
- type: check
100+
check: int_parse(store.source_nulls[0].null_string_cnt) == int_parse(store.target_nulls[0].null_string_cnt)
101+
failure_message: "NULL string count mismatch: source={store.source_nulls[0].null_string_cnt}, target={store.target_nulls[0].null_string_cnt}"
102+
103+
- type: check
104+
check: int_parse(store.source_nulls[0].null_integer_cnt) == int_parse(store.target_nulls[0].null_integer_cnt)
105+
failure_message: "NULL integer count mismatch: source={store.source_nulls[0].null_integer_cnt}, target={store.target_nulls[0].null_integer_cnt}"
106+
107+
- type: check
108+
check: int_parse(store.source_nulls[0].null_decimal_cnt) == int_parse(store.target_nulls[0].null_decimal_cnt)
109+
failure_message: "NULL decimal count mismatch: source={store.source_nulls[0].null_decimal_cnt}, target={store.target_nulls[0].null_decimal_cnt}"
110+
111+
- type: check
112+
check: int_parse(store.source_nulls[0].null_text_cnt) == int_parse(store.target_nulls[0].null_text_cnt)
113+
failure_message: "NULL text count mismatch: source={store.source_nulls[0].null_text_cnt}, target={store.target_nulls[0].null_text_cnt}"
114+
115+
# Calculate checksums on source (PostgreSQL)
116+
- type: query
117+
connection: '{source.name}'
118+
query: |
119+
SELECT
120+
SUM(COALESCE(col_bigint, 0)) as sum_bigint,
121+
SUM(COALESCE(col_integer, 0)) as sum_integer,
122+
SUM(COALESCE(col_smallint, 0)) as sum_smallint,
123+
SUM(COALESCE(CAST(col_decimal AS NUMERIC), 0)) as sum_decimal,
124+
SUM(COALESCE(LENGTH(col_string), 0)) as sum_string_len,
125+
SUM(COALESCE(LENGTH(col_text), 0)) as sum_text_len
126+
FROM public.mysql_null_test
127+
into: source_checksum
128+
129+
# Calculate checksums on target (MySQL)
130+
- type: query
131+
connection: '{target.name}'
132+
query: |
133+
SELECT
134+
SUM(COALESCE(col_bigint, 0)) as sum_bigint,
135+
SUM(COALESCE(col_integer, 0)) as sum_integer,
136+
SUM(COALESCE(col_smallint, 0)) as sum_smallint,
137+
SUM(COALESCE(CAST(col_decimal AS DECIMAL(30,10)), 0)) as sum_decimal,
138+
SUM(COALESCE(LENGTH(col_string), 0)) as sum_string_len,
139+
SUM(COALESCE(LENGTH(col_text), 0)) as sum_text_len
140+
FROM mysql.mysql_null_test
141+
into: target_checksum
142+
143+
- type: log
144+
message: |
145+
Checksums:
146+
Source: bigint={store.source_checksum[0].sum_bigint}, integer={store.source_checksum[0].sum_integer}, string_len={store.source_checksum[0].sum_string_len}
147+
Target: bigint={store.target_checksum[0].sum_bigint}, integer={store.target_checksum[0].sum_integer}, string_len={store.target_checksum[0].sum_string_len}
148+
149+
# Verify checksums match
150+
- type: check
151+
check: int_parse(store.source_checksum[0].sum_bigint) == int_parse(store.target_checksum[0].sum_bigint)
152+
failure_message: "Bigint checksum mismatch"
153+
154+
- type: check
155+
check: int_parse(store.source_checksum[0].sum_integer) == int_parse(store.target_checksum[0].sum_integer)
156+
failure_message: "Integer checksum mismatch"
157+
158+
- type: check
159+
check: int_parse(store.source_checksum[0].sum_string_len) == int_parse(store.target_checksum[0].sum_string_len)
160+
failure_message: "String length checksum mismatch"
161+
162+
- type: check
163+
check: int_parse(store.source_checksum[0].sum_text_len) == int_parse(store.target_checksum[0].sum_text_len)
164+
failure_message: "Text length checksum mismatch"
165+
166+
# Verify special characters row (col_bigint = 888888) transferred correctly
167+
- type: query
168+
connection: '{target.name}'
169+
query: |
170+
SELECT col_string, col_text
171+
FROM mysql.mysql_null_test
172+
WHERE col_bigint = 888888
173+
into: special_row
174+
175+
- type: log
176+
message: |
177+
Special characters row:
178+
col_string: {store.special_row[0].col_string}
179+
col_text: {store.special_row[0].col_text}
180+
181+
- type: check
182+
check: contains(store.special_row[0].col_string, "\"double quotes\"") && contains(store.special_row[0].col_string, "'single quotes'")
183+
failure_message: "col_string should contain both double and single quotes"
184+
185+
- type: check
186+
check: contains(store.special_row[0].col_text, "tabs and")
187+
failure_message: "col_text should contain 'tabs and'"
188+
189+
- type: check
190+
check: contains(store.special_row[0].col_text, "newlines and")
191+
failure_message: "col_text should contain 'newlines and'"
192+
193+
- type: check
194+
check: contains(store.special_row[0].col_text, "quotes")
195+
failure_message: "col_text should contain quotes"
196+
197+
- type: log
198+
message: "SUCCESS: MySQL LoadDataLocal NULL handling test passed"
199+
200+
# Cleanup
201+
- type: query
202+
connection: '{source.name}'
203+
query: DROP TABLE IF EXISTS public.mysql_null_test
204+
205+
- type: query
206+
connection: '{target.name}'
207+
query: DROP TABLE IF EXISTS mysql.mysql_null_test
208+
209+
streams:
210+
public.mysql_null_test:
211+
object: mysql.mysql_null_test

‎cmd/sling/tests/suite.cli.yaml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1493,4 +1493,24 @@
14931493
output_contains:
14941494
- "using text since type 'xmltype' not mapped"
14951495
- 'execution succeeded'
1496-
- 'SUCCESS: Oracle XMLTYPE to BigQuery transfer completed without hanging!'
1496+
- 'SUCCESS: Oracle XMLTYPE to BigQuery transfer completed without hanging!'
1497+
1498+
# Test MySQL LoadDataLocal using RegisterReaderHandler pattern
1499+
- id: 164
1500+
name: 'Test MySQL LoadDataLocal with native Go driver'
1501+
run: 'sling run -d -r cmd/sling/tests/replications/r.93.mysql_load_data_local.yaml'
1502+
streams: 1
1503+
rows: 18
1504+
output_contains:
1505+
- 'execution succeeded'
1506+
- 'SUCCESS: MySQL LoadDataLocal test passed'
1507+
1508+
# Test MySQL LoadDataLocal NULL handling with checksums
1509+
- id: 165
1510+
name: 'Test MySQL LoadDataLocal NULL handling'
1511+
run: 'sling run -d -r cmd/sling/tests/replications/r.94.mysql_load_data_local_nulls.yaml'
1512+
streams: 1
1513+
rows: 53
1514+
output_contains:
1515+
- 'execution succeeded'
1516+
- 'SUCCESS: MySQL LoadDataLocal NULL handling test passed'

0 commit comments

Comments
 (0)