diff --git a/src/search.py b/src/search.py index 50558a7..830453e 100644 --- a/src/search.py +++ b/src/search.py @@ -5,6 +5,7 @@ from time import sleep from botasaurus import * from .utils import default_request_options +from botasaurus.request_decorator import request import requests FAILED_DUE_TO_CREDITS_EXHAUSTED = "FAILED_DUE_TO_CREDITS_EXHAUSTED" @@ -12,6 +13,12 @@ FAILED_DUE_TO_NO_KEY = "FAILED_DUE_TO_NO_KEY" FAILED_DUE_TO_UNKNOWN_ERROR = "FAILED_DUE_TO_UNKNOWN_ERROR" +PROTOCOL = "https" + +HOST = "google-scraper.p.rapidapi.com" +API_PATH = "search/" +HOST_URL = f"{PROTOCOL}://{HOST}/{API_PATH}" + def update_credits(): credits_used = bt.LocalStorage.get_item("credits_used", 0) bt.LocalStorage.set_item("credits_used", credits_used + 1) @@ -36,7 +43,7 @@ def do_request(data, retry_count=3): headers = { "X-RapidAPI-Key": key, - "X-RapidAPI-Host": "google-scraper.p.rapidapi.com" + "X-RapidAPI-Host": HOST } @@ -99,7 +106,7 @@ def search(_, data, metadata): "error":FAILED_DUE_TO_NO_KEY }) max_items = data['max'] - url = "https://google-scraper.p.rapidapi.com/search/" + url = HOST_URL qp = {"query": data['query']} params = {**qp, 'link':cl.join_link(url, query_params=qp)} diff --git a/src/write_output.py b/src/write_output.py index 6546bba..2cf95e6 100644 --- a/src/write_output.py +++ b/src/write_output.py @@ -1,5 +1,5 @@ from botasaurus import bt -from botasaurus.decorators import print_filenames +from botasaurus.decorators_common import print_filenames from .write_output_utils import kebab_case, make_folders