feat: Allow customization of delay and rate limit

soccerdata/_common.py

@@ -215,6 +215,10 @@ class BaseReader(ABC):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    max_delay: float
+        Maximum random delay added between requests in seconds.
+    rate_limit: float
+        Minimum delay between requests in seconds.
     """
 
     def __init__(
@@ -224,6 +228,8 @@ def __init__(
         no_cache: bool = False,
         no_store: bool = False,
         data_dir: Path = DATA_DIR,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Create a new data reader."""
         if isinstance(proxy, str) and proxy.lower() == "tor":
@@ -241,8 +247,8 @@ def __init__(
         self.no_cache = no_cache
         self.no_store = no_store
         self.data_dir = data_dir
-        self.rate_limit = 0
-        self.max_delay = 0
+        self.rate_limit = rate_limit
+        self.max_delay = max_delay
         if self.no_store:
             logger.info("Caching is disabled")
         else:
@@ -477,6 +483,8 @@ def __init__(
         no_store: bool = False,
         data_dir: Path = DATA_DIR,
         headers: Optional[dict[str, str]] = None,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Initialize the reader."""
         super().__init__(
@@ -485,6 +493,8 @@ def __init__(
             leagues=leagues,
             proxy=proxy,
             data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
 
         self._session = self._init_session(headers)
@@ -546,6 +556,8 @@ def __init__(
         data_dir: Path = DATA_DIR,
         path_to_browser: Optional[Path] = None,
         headless: bool = True,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Initialize the reader."""
         super().__init__(
@@ -554,6 +566,8 @@ def __init__(
             leagues=leagues,
             proxy=proxy,
             data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
         self.path_to_browser = path_to_browser
         self.headless = headless

soccerdata/clubelo.py

@@ -48,6 +48,10 @@ class ClubElo(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -56,9 +60,18 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = CLUB_ELO_DATADIR,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Initialize a new ClubElo reader."""
-        super().__init__(proxy=proxy, no_cache=no_cache, no_store=no_store, data_dir=data_dir)
+        super().__init__(
+            proxy=proxy,
+            no_cache=no_cache,
+            no_store=no_store,
+            data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
+        )
 
     def read_by_date(self, date: Optional[Union[str, datetime]] = None) -> pd.DataFrame:
         """Retrieve ELO scores for all teams at specified date.

soccerdata/espn.py

@@ -50,6 +50,10 @@ class ESPN(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -60,6 +64,8 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = ESPN_DATADIR,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Initialize a new ESPN reader."""
         super().__init__(
@@ -68,6 +74,8 @@ def __init__(
             no_cache=no_cache,
             no_store=no_store,
             data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
         self.seasons = seasons
 

soccerdata/fbref.py

@@ -63,6 +63,10 @@ class FBref(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -73,6 +77,8 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = FBREF_DATADIR,
+        rate_limit: float = 7,
+        max_delay: float = 0,
     ):
         """Initialize FBref reader."""
         super().__init__(
@@ -82,8 +88,9 @@ def __init__(
             no_store=no_store,
             data_dir=data_dir,
             headers=FBREF_HEADERS,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
-        self.rate_limit = 7
         self.seasons = seasons
         # check if all top 5 leagues are selected
         if (

soccerdata/match_history.py

@@ -60,6 +60,10 @@ class MatchHistory(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path, optional
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -70,9 +74,17 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = MATCH_HISTORY_DATA_DIR,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         super().__init__(
-            leagues=leagues, proxy=proxy, no_cache=no_cache, no_store=no_store, data_dir=data_dir
+            leagues=leagues,
+            proxy=proxy,
+            no_cache=no_cache,
+            no_store=no_store,
+            data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
         self.seasons = seasons
 

soccerdata/sofascore.py

@@ -45,6 +45,10 @@ class Sofascore(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -55,6 +59,8 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = SOFASCORE_DATADIR,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Initialize the Sofascore reader."""
         super().__init__(
@@ -63,6 +69,8 @@ def __init__(
             no_cache=no_cache,
             no_store=no_store,
             data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
         self.seasons = seasons
         if not self.no_store:

soccerdata/sofifa.py

@@ -53,6 +53,10 @@ class SoFIFA(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -63,6 +67,8 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = SO_FIFA_DATADIR,
+        rate_limit: float = 1,
+        max_delay: float = 0,
     ):
         """Initialize SoFIFA reader."""
         super().__init__(
@@ -71,8 +77,9 @@ def __init__(
             no_cache=no_cache,
             no_store=no_store,
             data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
-        self.rate_limit = 1
         if versions == "latest":
             self.versions = self.read_versions().tail(n=1)
         elif versions == "all":

soccerdata/understat.py

@@ -65,6 +65,10 @@ class Understat(BaseRequestsReader):
         If True, will not store downloaded data.
     data_dir : Path
         Path to directory where data will be cached.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -75,6 +79,8 @@ def __init__(
         no_cache: bool = NOCACHE,
         no_store: bool = NOSTORE,
         data_dir: Path = UNDERSTAT_DATADIR,
+        rate_limit: float = 0,
+        max_delay: float = 0,
     ):
         """Initialize a new Understat reader."""
         super().__init__(
@@ -83,6 +89,8 @@ def __init__(
             no_cache=no_cache,
             no_store=no_store,
             data_dir=data_dir,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
         self.seasons = seasons
         self._cookies_initialized = False

soccerdata/whoscored.py

@@ -143,6 +143,10 @@ class WhoScored(BaseSeleniumReader):
     headless : bool, default: True
         If True, will run Chrome in headless mode. Setting this to False might
         help to avoid getting blocked. Only supported for Selenium <4.13.
+    rate_limit : float
+        Minimum delay between requests in seconds.
+    max_delay : float
+        Maximum random delay added between requests in seconds.
     """
 
     def __init__(
@@ -155,6 +159,8 @@ def __init__(
         data_dir: Path = WHOSCORED_DATADIR,
         path_to_browser: Optional[Path] = None,
         headless: bool = False,
+        rate_limit: float = 5,
+        max_delay: float = 5,
     ):
         """Initialize the WhoScored reader."""
         super().__init__(
@@ -165,10 +171,10 @@ def __init__(
             data_dir=data_dir,
             path_to_browser=path_to_browser,
             headless=headless,
+            rate_limit=rate_limit,
+            max_delay=max_delay,
         )
         self.seasons = seasons
-        self.rate_limit = 5
-        self.max_delay = 5
         if not self.no_store:
             (self.data_dir / "seasons").mkdir(parents=True, exist_ok=True)
             (self.data_dir / "matches").mkdir(parents=True, exist_ok=True)

tests/test_common.py

@@ -1,5 +1,6 @@
 """Unittests for soccerdata._common."""
 
+import time
 import json
 from datetime import datetime, timezone
 from unittest.mock import MagicMock, patch
@@ -11,6 +12,7 @@
 import soccerdata
 from soccerdata._common import (
     BaseRequestsReader,
+    BaseSeleniumReader,
     SeasonCode,
     add_alt_team_names,
     add_standardized_team_name,
@@ -150,6 +152,34 @@ def test_download_and_save_variable_no_store_no_filepath(mock_tls_client):
     assert stats["statData"]["player"] == "Messi"
 
 
+def test_requests_rate_limit(mock_tls_client):
+    # Setup mock
+    mock_tls_client.return_csv()
+
+    reader = BaseRequestsReader(no_store=True, no_cache=True, rate_limit=2.0)
+    url = "http://api.clubelo.com/Barcelona"
+    init_time = time.time()
+    data = reader.get(url, filepath=None)
+    data = reader.get(url, filepath=None)
+    end_time = time.time()
+    elapsed = end_time - init_time
+    assert elapsed >= 4.0
+
+
+def test_selenium_rate_limit(mock_tls_client):
+    # Setup mock
+    mock_tls_client.return_csv()
+
+    reader = BaseSeleniumReader(no_store=True, no_cache=True, rate_limit=2.0, headless=True)
+    url = "http://api.clubelo.com/Barcelona"
+    init_time = time.time()
+    data = reader.get(url, filepath=None)
+    data = reader.get(url, filepath=None)
+    end_time = time.time()
+    elapsed = end_time - init_time
+    assert elapsed >= 4.0
+
+
 # def test_download_and_save_requests_tor(tmp_path):
 #     url = "https://check.torproject.org/api/ip"
 #     reader = BaseRequestsReader(proxy=None)