Merge pull request #259 from pre-commit/golang-module

Support Go module fallback for nested package paths

Author: asottile

pre_commit_mirror_maker/languages.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import json
+import os
 import re
 import subprocess
+import urllib.error
 import urllib.request
 
 from packaging import requirements
@@ -41,11 +43,26 @@ def golang_get_package_versions(package_name: str) -> list[str]:
         r'[A-Z]',
         lambda m: f'!{m.group(0).lower()}', package_name,
     )
-    url = f'https://proxy.golang.org/{escaped}/@v/list'
-    resp = urllib.request.urlopen(url).read().decode()
-    return sorted(
-        (v.removeprefix('v') for v in resp.splitlines()),
-        key=version.parse,
+
+    # Greedily choose the longest non-404 path
+    # (based on https://go.dev/ref/mod#resolve-pkg-mod)
+    while escaped:
+        url = f'https://proxy.golang.org/{escaped}/@v/list'
+        try:
+            resp = urllib.request.urlopen(url).read().decode()
+        except urllib.error.HTTPError as exc:
+            if exc.code == 404:
+                escaped = os.path.dirname(escaped)
+                continue
+            raise
+
+        return sorted(
+            (v.removeprefix('v') for v in resp.splitlines()),
+            key=version.parse,
+        )
+
+    raise ValueError(
+        f'Cannot find package name {package_name} on proxy.golang.org',
     )
 
 

tests/languages_test.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import pytest
+
 from pre_commit_mirror_maker.languages import golang_get_package_versions
 from pre_commit_mirror_maker.languages import node_get_package_versions
 from pre_commit_mirror_maker.languages import python_get_package_versions
@@ -42,8 +44,20 @@ def test_rust_get_package_version_output():
     assert_all_text(ret)
 
 
-def test_golang_get_package_version_output():
-    ret = golang_get_package_versions('mvdan.cc/gofumpt')
+@pytest.mark.parametrize(
+    'package_name',
+    (
+        'mvdan.cc/gofumpt',
+        'mvdan.cc/sh/v3/cmd/shfmt',
+    ),
+)
+def test_golang_get_package_version_output(package_name):
+    ret = golang_get_package_versions(package_name)
     assert ret
     assert not any(v.startswith('v') for v in ret)
     assert_all_text(ret)
+
+
+def test_golang_get_package_version_invalid_package():
+    with pytest.raises(ValueError):
+        golang_get_package_versions('mvdan.cc/foo')