fix: 1341 mdb 2126 doesnt download even though its a zip file

functions-python/batch_process_dataset/README.md

@@ -49,6 +49,23 @@ The function expects a Pub/Sub message with the following format:
     }
   }
 }
+
+{
+  "message": {
+    "data": {
+      "execution_id": "batch-trace-e5eaa516bd884c0a39861d08de301d97/2153210919778512803;o=1",
+      "producer_url": "https://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip",
+      "feed_stable_id": "mdb-2126",
+      "feed_id": "9f1748c5-b482-4577-819e-ce78c75980b3",
+      "dataset_stable_id": "mdb-2126-202504170018",
+      "dataset_hash": "7d019543ee12b2a44d580d7780d71546108a2cb1c4f3bfcc5cf3ee97b847fafd",
+      "authentication_type": "0",
+      "authentication_info_url": "",
+      "api_key_parameter_name": ""
+    }
+  }
+}
+
 ```
 
 # Function configuration

functions-python/batch_process_dataset/src/scripts/HTML_headers_verifier.sh

@@ -0,0 +1,14 @@
+# Download while capturing headers and using a browser-like client identity
+#- Valid ZIP should begin with “PK..” (hex 50 4b 03 04 or 50 4b 05 06).
+#- If you see “<!DOCTYPE” or “<html…”, you got an HTML page (redirect, consent, 403, etc.).
+curl -sSL --fail \
+  -D /tmp/headers.txt \
+  -A "Mozilla/5.0" \
+  -H "Referer: https://www.stm.info/" \
+  -H "Accept: */*" \
+  -o /tmp/stm.zip \
+  "https://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip"
+
+echo "== Headers ==" && cat /tmp/headers.txt
+echo "== Size ==" && wc -c /tmp/stm.zip
+echo "== First 64 bytes ==" && hexdump -C /tmp/stm.zip | head -n 4

functions-python/batch_process_dataset/src/scripts/headers_verifier.sh

@@ -0,0 +1,11 @@
+# This is to inspect what the server actually returned: Save headers to see final status and Content-Type
+# If Content-Type is text/html or status is 30x/403, you’re not getting the ZIP.
+curl -sSL --fail \
+  -D /tmp/headers.txt \
+  -A "Mozilla/5.0" \
+  -o /tmp/stm.zip \
+  "https://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip"
+
+echo "== Headers ==" && cat /tmp/headers.txt
+file /tmp/stm.zip
+hexdump -C /tmp/stm.zip | head

functions-python/batch_process_dataset/src/scripts/zip_verifier.sh

@@ -0,0 +1,14 @@
+# Inspect headers and redirect chain (no file saved)
+curl -ILv "https://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip"
+
+# Download the body (ZIP) — no -I, follow redirects, set UA
+curl -L --fail -A "Mozilla/5.0" \
+  -o /tmp/stm.zip \
+  "https://www.stm.info/sites/default/files/gtfs/gtfs_stm.zip"
+
+# Sanity checks
+file /tmp/stm.zip
+python - <<'PY'
+import zipfile
+print("is_zip:", zipfile.is_zipfile("/tmp/stm.zip"))
+PY